diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index 457cfd108..000000000 --- a/.claude/settings.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(aspire deploy:*)", - "Bash(aspire mcp:*)", - "Bash(az account:*)", - "Bash(az containerapp * list:*)", - "Bash(az containerapp * logs:*)", - "Bash(az containerapp * show:*)", - "Bash(az containerapp list:*)", - "Bash(az containerapp logs:*)", - "Bash(az containerapp show:*)", - "Bash(az group list:*)", - "Bash(az group show:*)", - "Bash(az monitor * list:*)", - "Bash(az monitor * show:*)", - "Bash(az monitor activity-log:*)", - "Bash(az monitor list:*)", - "Bash(az monitor log-analytics query:*)", - "Bash(az monitor metrics:*)", - "Bash(az monitor show:*)", - "Bash(az postgres * list:*)", - "Bash(az postgres * show:*)", - "Bash(az postgres list:*)", - "Bash(az postgres show:*)", - "Bash(az resource list:*)", - "Bash(az resource show:*)", - "Bash(cat:*)", - "Bash(claude:*)", - "Bash(cmp:*)", - "Bash(cp:*)", - "Bash(curl:*)", - "Bash(diff:*)", - "Bash(docker cp:*)", - "Bash(docker exec:*)", - "Bash(docker inspect:*)", - "Bash(docker ps:*)", - "Bash(docker run:*)", - "Bash(dotnet-dump analyze:*)", - "Bash(dotnet-ildasm:*)", - "Bash(dotnet:*)", - "Bash(echo:*)", - "Bash(exit 0)", - "Bash(file:*)", - "Bash(find:*)", - "Bash(gh:*)", - "Bash(git add:*)", - "Bash(git blame:*)", - "Bash(git check-ignore:*)", - "Bash(git checkout:*)", - "Bash(git diff:*)", - "Bash(git fetch:*)", - "Bash(git grep:*)", - "Bash(git log:*)", - "Bash(git ls-remote:*)", - "Bash(git ls-tree:*)", - "Bash(git mv:*)", - "Bash(git pull:*)", - "Bash(git restore:*)", - "Bash(git rev-parse:*)", - "Bash(git show:*)", - "Bash(git stash:*)", - "Bash(git status:*)", - "Bash(grep:*)", - "Bash(head:*)", - "Bash(iconv:*)", - "Bash(ilspycmd:*)", - "Bash(kill:*)", - "Bash(ls:*)", - "Bash(lsof:*)", - "Bash(meshweaver-thumbnails:*)", - "Bash(mkdir:*)", - "Bash(mv:*)", - "Bash(netstat:*)", - "Bash(nm:*)", - "Bash(node --check:*)", - "Bash(pgrep:*)", - "Bash(pkill:*)", - "Bash(python3:*)", - "Bash(python:*)", - "Bash(rg:*)", - "Bash(sed:*)", - "Bash(sleep:*)", - "Bash(tail:*)", - "Bash(tee:*)", - "Bash(test:*)", - "Bash(timeout:*)", - "Bash(tr:*)", - "Bash(tree:*)", - "Bash(true)", - "Bash(unzip:*)", - "Bash(wait:*)", - "Bash(wc:*)", - "Bash(xargs:*)", - - "WebFetch(domain:aspire.dev)", - "WebFetch(domain:cdnjs.cloudflare.com)", - "WebFetch(domain:docs.anthropic.com)", - "WebFetch(domain:en.wikipedia.org)", - "WebFetch(domain:fluent2.microsoft.design)", - "WebFetch(domain:gist.github.com)", - "WebFetch(domain:github.com)", - "WebFetch(domain:localhost)", - "WebFetch(domain:raw.githubusercontent.com)", - "WebFetch(domain:support.claude.com)", - "WebFetch(domain:www.fluentui-blazor.net)", - "WebFetch(domain:www.nuget.org)", - "WebFetch(domain:xunit.net)", - - "WebSearch", - - "mcp__aspire__list_apphosts", - "mcp__aspire__list_console_logs", - "mcp__aspire__list_resources", - "mcp__aspire__list_structured_logs", - "mcp__aspire__list_traces", - "mcp__aspire__select_apphost" - ], - "deny": [ - "Bash(az group delete:*)", - "Bash(az resource delete:*)", - "Bash(git push --force:*)", - "Bash(git push -f:*)", - "Bash(git reset --hard:*)", - "Bash(rm -rf /:*)", - "Bash(rm -rf ~:*)", - "Bash(sudo:*)" - ] - }, - "enableAllProjectMcpServers": true -} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..6d9f4a881 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +# Keep the Docker build context small. The portal Dockerfile builds from the repo root, +# so it needs source, *.csproj, *.slnx, Directory.*.props AND embedded content +# (e.g. MeshWeaver.Documentation/Data/**/*.md, samples/Graph/Data) — those must NOT be +# excluded. Only build artifacts, VCS, IDE, and local runtime state are dropped. +**/bin/ +**/obj/ +**/.vs/ +**/.vscode/ +**/.idea/ +**/*.user +.git/ +.github/ +artifacts/ +Azurite/ +**/TestResults/ +**/node_modules/ +deploy/compose/data/ diff --git a/.github/workflows/dotnet-test.yml b/.github/workflows/dotnet-test.yml index de60b2c73..65f9e18c8 100644 --- a/.github/workflows/dotnet-test.yml +++ b/.github/workflows/dotnet-test.yml @@ -1,5 +1,17 @@ -# This workflow will build a .NET project -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net +# This workflow builds the solution ONCE, then runs the test suite across +# parallel shards that consume the prebuilt output, and finally consolidates +# every shard's results in a single collector job. +# +# build ──▶ test (matrix shard 0..3) ──▶ collect-results +# +# Why this shape (vs. the old "every shard restores+builds the whole solution"): +# * The 50+ project solution was compiled 4× — once per shard. Building once +# and shipping the output to the shards removes 3 redundant full builds. +# * Shards run `dotnet test --no-build --no-restore` against the extracted +# output — they START FROM ARTIFACTS, never recompile, so a shard can only +# fail for a real test reason (or genuine infra failure), not a flaky build. +# * Results are published from ONE collector job → a single "Test Results" +# check is the pass/fail gate, instead of 4 separate per-shard checks. name: MeshWeaver Build and Test @@ -10,14 +22,52 @@ on: branches: [ "main" ] workflow_dispatch: +# Pin the NuGet global-packages folder INSIDE the workspace so the build job's +# restore output travels in the same artifact as bin/obj. ${{ github.workspace }} +# is identical across jobs on ubuntu-latest (/home/runner/work//), +# so the absolute paths baked into project.assets.json / *.nuget.g.props stay +# valid after the shards extract the tarball — that's what makes +# `dotnet test --no-build --no-restore` resolve packages offline. +env: + NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages + +# publish-unit-test-result-action runs per shard AND in the collector — both need +# checks: write to create check-runs and pull-requests: write to comment. +permissions: + contents: read + checks: write + pull-requests: write + jobs: + # ───────────────────────────── BUILD ───────────────────────────── + # Compile the whole solution exactly once and package the compiled output + # (every project's bin + obj) together with the NuGet global-packages folder + # into a single self-contained tarball the shards consume. build: - name: Build and Run Unit tests - timeout-minutes: 30 + name: "Build solution (once)" + timeout-minutes: 25 runs-on: ubuntu-latest - steps: - uses: actions/checkout@v6 + # The build job packages bin+obj of all 50+ projects PLUS the whole + # .nuget/packages folder into a zstd tarball — that's a multi-GB copy on top + # of the originals, which overflowed the default ~14 GB ("No space left on + # device" in the runner Worker, mid-package). Reclaim the ~25 GB of + # preinstalled tooling a .NET build never touches (Android SDK, Haskell/GHC, + # preloaded Docker images, big apt packages) — identical to what the test + # shards already do. NOT a band-aid for a leak: the build output is a + # legitimate, bounded artifact; this just removes genuinely-unused runner + # bloat so it fits. Keep .NET + the hosted tool cache (setup-dotnet uses them). + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + dotnet: false + android: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true - name: Setup .NET uses: actions/setup-dotnet@v5 with: @@ -27,44 +77,371 @@ jobs: - name: Restore dependencies run: dotnet restore - name: Build - run: dotnet build --no-restore -p:CIRun=true + run: dotnet build --no-restore -c Release -p:CIRun=true + # Node Source can pull MeshWeaver libraries in via a version-LESS `#r "nuget:MeshWeaver.X"` + # (e.g. the BusinessRules scope generator the PensionFund sample uses). Tests resolve those + # from the mesh-local feed (nuget.config → dist/packages), which is git-ignored — so the + # build job must produce them. Pack the just-built RELEASE output at the dev-feed $(Version); + # dist/packages travels in the build artifact below, so the --no-build/--no-restore + # shards (and the runtime #r resolver inside them) can resolve them offline. These are the + # same Release nupkgs you'd publish directly. Self-contained: BusinessRules depends only on + # framework packages (already in the restored .nuget cache). + - name: Pack mesh-local #r packages + run: | + set -euo pipefail + # -c Release matches the Release build above. NO -p:PackageVersion override: pack at the + # repo's continuous dev-feed $(Version) from Directory.Build.props (= 3.0.0-rc1.ci.0 here), + # exactly as the portal image's BakeMeshLocalFeed target and the test's documented local + # repro do. + # + # 🚨 Do NOT pin -p:PackageVersion=3.0.0-preview1. nuget.org carries a STALE + # MeshWeaver.BusinessRules.Generator 3.0.0-preview1 (published 2026-04-16, before the + # generator shipped its lib/ asset) that has analyzers/dotnet/cs ONLY — no lib/. NuGet's + # global-packages cache is keyed solely by (id, version), so once that lib-less copy lands + # under .nuget/packages/meshweaver.businessrules.generator/3.0.0-preview1 the runtime #r + # resolver's install short-circuit reuses it and NEVER surfaces the lib/ assembly — scope + # nodes compile but generate nothing (PensionFund balance sheet renders empty; + # NuGetAssemblyResolverTest fails at the lib/ assertion). packageSourceMapping pins resolution + # to this feed, but it can't un-poison a colliding cache key. rc1.ci.N is never published on + # nuget.org, so the cache key can ONLY ever be this lib-containing local package. + dotnet pack src/MeshWeaver.BusinessRules/MeshWeaver.BusinessRules.csproj \ + -c Release --no-build --no-restore -o dist/packages --nologo + dotnet pack src/MeshWeaver.BusinessRules.Generator/MeshWeaver.BusinessRules.Generator.csproj \ + -c Release --no-build --no-restore -o dist/packages --nologo + ls -lh dist/packages + - name: Package build output + run: | + set -euo pipefail + # Collect every project's compiled output (bin) + restore graph (obj), + # plus the workspace-local NuGet packages folder. -prune stops find from + # descending once a bin/obj matches, so nested matches aren't tarred + # twice. .git and the packages tree are excluded from the bin/obj scan; + # the packages folder is appended explicitly (added whole below). + find . -type d \( -name bin -o -name obj \) \ + -not -path './.git/*' -not -path './.nuget/*' -prune -print > .build-paths.txt + echo "./.nuget/packages" >> .build-paths.txt + # The mesh-local #r feed packed above — the test shards' runtime #r resolver reads it. + echo "./dist/packages" >> .build-paths.txt + # One zstd tarball (-T0 = all cores) moves through upload/download-artifact + # FAR faster than the millions of loose bin/obj/packages files would. + tar -I 'zstd -T0' -cf build-output.tar.zst -T .build-paths.txt + echo "Packaged build output:" + ls -lh build-output.tar.zst + - name: "Upload artifact: build output" + uses: actions/upload-artifact@v6 + with: + name: build-output + path: build-output.tar.zst + # Intermediate, consumed within this run only. + retention-days: 1 + # Already zstd-compressed — skip the redundant zip pass. + compression-level: 0 + + # ───────────────────────────── TEST ────────────────────────────── + # Matrix-sharded: the ~45 test projects are split across SHARD_TOTAL parallel + # runners (each runs a disjoint subset). Each shard downloads the prebuilt + # output and runs `dotnet test --no-build --no-restore` — NO restore, NO + # build. It cuts the per-runner project count (and the leftover-state / memory + # accumulation that made late projects flake under the old single-runner + # sequential loop) by ~SHARD_TOTAL×. + # 30 min: a single shard's ~10 projects run in ~15-20 min; headroom for one + # hung project burning its 6m wall-clock cap. + test: + name: "Run tests (shard ${{ matrix.shard }})" + needs: build + timeout-minutes: 30 + runs-on: ubuntu-latest + strategy: + # Don't cancel sibling shards when one fails — we want every shard's + # results so the PR shows the full failure picture, not just the first. + fail-fast: false + matrix: + shard: [0, 1, 2, 3] + + steps: + - uses: actions/checkout@v6 + # The ~14 GB free on a GitHub-hosted runner does not fit the extracted 50-project + # build output + the pgvector Testcontainers image + per-project test outputs/blame + # dumps + runner _diag logs — shards were dying with "No space left on device" (the + # runner Worker itself, mid-run). Reclaim the ~25 GB of preinstalled tooling a + # .NET+Postgres test run never touches (Android SDK, Haskell/GHC, preloaded Docker + # images, big apt packages). NOT a band-aid for a code leak — it removes genuinely + # unused runner bloat so the legitimate test workload fits. Keep .NET + the hosted + # tool cache (setup-dotnet uses them). + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + dotnet: false + android: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.x + # Workloads (e.g. aspire) live in the SDK, not in the build artifact — they + # are needed for MSBuild to evaluate workload-referencing projects even under + # --no-build. This installs SDK packs only; it does NOT recompile the solution. + - name: Restore workloads + run: dotnet workload restore + - name: "Download artifact: build output" + uses: actions/download-artifact@v6 + with: + name: build-output + - name: Extract build output + run: | + set -euo pipefail + tar -I zstd -xf build-output.tar.zst + rm -f build-output.tar.zst - name: Run Tests - continue-on-error: true env: DOTNET_ENVIRONMENT: Development Logging__LogLevel__Default: Warning + SHARD_INDEX: ${{ matrix.shard }} + SHARD_TOTAL: 4 run: | - find test -name '*.csproj' \ - ! -path '*PostgreSql*' \ - ! -path '*Cosmos*' \ - ! -path '*Orleans*' \ - ! -path '*Acme*' \ - ! -path '*FutuRe*' \ - -exec dotnet test {} --no-build --verbosity normal -l:trx \ - --blame-hang-timeout 3m --blame-hang-dump-type mini \; 2>&1 | tee test/test-results.log + # Only Cosmos is excluded — needs the Cosmos emulator which is heavy to + # set up on a runner. Everything else runs (PostgreSql via Testcontainers + # using pre-installed Docker, Orleans via in-proc TestCluster, Acme/FutuRe + # via dynamic Code-piece compilation). + # + # set +e so a hung/aborted test host (non-zero exit) doesn't terminate + # the loop — we want EVERY csproj to attempt to run, then surface what + # broke after the loop completes. The collect-results job is the gate; + # this step always exits 0 so per-project failures don't mask sibling + # projects that still need to run. + # + # Disk hygiene: `--verbosity minimal`, no tee — trx files capture full + # per-test results; only the per-project exit markers are persisted to + # test/test-results.log. + set +e + : > test/test-results.log + echo "::group::Per-project test runs (shard $SHARD_INDEX/$SHARD_TOTAL)" + # `! -path '*/bin/*'`: the build copies sample csproj fixtures into test + # bin dirs (e.g. MeshWeaver.Samples.Graph.csproj under + # test/MeshWeaver.Acme.Test/bin/Debug/net10.0/SamplesGraph/). They match + # the *.csproj pattern but aren't test projects — invoking dotnet test on + # each wastes ~1 s × N entries per run. + # + # Shard selection (weighted): the ~7 dynamic-compilation / collectible-ALC + # heavyweights (Acme, AI, Content, FutuRe, Graph, Hosting.Monolith, + # Hosting.Orleans) dominate BOTH wall-clock AND the per-runner memory + # accumulation (leaked NodeType ALCs) that makes a late project GC-stall and + # time out. A plain `idx % SHARD_TOTAL` over the *alphabetical* list happened + # to drop the three worst — AI + FutuRe + Orleans — all onto shard 2, which + # then memory-pressured and timed out the Orleans Resubmit test at 45 s + + # crashed teardown. Fix: round-robin the heavies on their OWN counter so + # consecutive heavies always land on different shards (AI/FutuRe/Orleans → + # distinct shards), and round-robin the light projects on a separate counter. + # Net: no shard carries more than two heavies, never the three ALC-heaviest + # together. AI.Test.FakeCli is a helper exe, not a heavy test → stays light. + heavy_idx=0 + light_idx=0 + for csproj in $(find test -name '*.csproj' \ + ! -path '*Cosmos*' \ + ! -path '*/bin/*' \ + | sort); do + name=$(basename "$csproj" .csproj) + case "$name" in + MeshWeaver.Acme.Test|MeshWeaver.AI.Test|MeshWeaver.Content.Test|MeshWeaver.FutuRe.Test|MeshWeaver.Graph.Test|MeshWeaver.Hosting.Monolith.Test|MeshWeaver.Hosting.Orleans.Test) + assigned=$(( heavy_idx % SHARD_TOTAL )) + heavy_idx=$(( heavy_idx + 1 )) + ;; + *) + assigned=$(( light_idx % SHARD_TOTAL )) + light_idx=$(( light_idx + 1 )) + ;; + esac + if [ "$assigned" -ne "$SHARD_INDEX" ]; then + continue + fi + echo "::endgroup::" + echo "::group::$name" + # `timeout 6m`: per-project wall-clock cap. blame-hang-timeout 90s only + # fires when xUnit considers a *test* to have stalled — fixture-init + # and between-class hangs slip past it. 6 min covers the slowest + # legitimate project today (FutuRe ~5 min) with headroom; defense-in- + # depth backstop in case any of the skipped Linux-hang projects gets + # re-enabled or another regresses to the same pattern. + # blame-hang-timeout: 90 s of test-runner inactivity before blame + # collects a hang dump and aborts the test host. 90 s gives genuine + # teardown room (hosted-hub / static-cache dispose) without masking real + # hangs (the per-project 6 m wall-clock cap is the hard backstop). + # --no-build --no-restore: run STRICTLY against the extracted build + # output — never recompile or hit the network on a test runner. + timeout --signal=TERM --kill-after=30s 6m \ + dotnet test "$csproj" -c Release --no-build --no-restore --verbosity minimal -l:trx \ + --blame-hang-timeout 90s --blame-hang-dump-type mini + rc=$? + if [ "$rc" = "124" ] || [ "$rc" = "137" ]; then + marker="[CI] $name exit=$rc TIMEOUT (6m wall-clock cap hit — likely fixture/init hang)" + else + marker="[CI] $name exit=$rc" + fi + echo "$marker" + echo "$marker" >> test/test-results.log + done + echo "::endgroup::" + exit 0 - name: Collect test logs for artifact if: always() run: | mkdir -p collected-logs # Find all test-logs directories and collect their contents without renaming find . -path "*/bin/*/test-logs/*.log" -type f -exec cp {} collected-logs/ \; 2>/dev/null || true - - name: Publish Test Results -# uses: EnricoMi/publish-unit-test-result-action/composite@v2 - uses: EnricoMi/publish-unit-test-result-action@v2.12.0 - if: always() - with: - action_fail: true - # File patterns of test result files. Relative paths are known to work best, while the composite action also works with absolute paths. Supports "*", "**", "?", and "[]" character ranges. Use multiline string for multiple patterns. Patterns starting with "!" exclude the matching files. There have to be at least one pattern starting without a "!". - files: | - test/**/*.trx - - name: "Upload artifact: Test Results" + # The MonolithMeshTestBase phase trace + dispose trace live in the + # process tempdir (Path.GetTempPath() == /tmp on the runner). They + # carry the per-class INIT_MEM / DISPOSE_MEM lines that pinpoint + # which test class drives the AI.Test / Autocomplete.Test / Content.Test + # / Hosting.Monolith.Test OOM. Copy with rename so they don't collide. + if [ -f /tmp/meshweaver-test-trace.log ]; then + cp /tmp/meshweaver-test-trace.log collected-logs/_meshweaver-test-trace.log + fi + if [ -f /tmp/meshweaver-dispose-trace.log ]; then + cp /tmp/meshweaver-dispose-trace.log collected-logs/_meshweaver-dispose-trace.log + fi + # Per-class INIT → DISPOSE memory delta summary (one line per test instance): + # managed=… rss=… rssAnon=… unmanaged=… shared=0|1 + # rssAnon (Linux only) is where the Autofac Reflection.Emit factory pin lives; + # unmanaged = rss − managed is the portable approximation. Grep this file to + # find the worst leakers without wading through the full per-event trace. + if [ -f /tmp/meshweaver-memory-delta.log ]; then + cp /tmp/meshweaver-memory-delta.log collected-logs/_meshweaver-memory-delta.log + fi + - name: "Upload artifact: shard test results" uses: actions/upload-artifact@v6 if: always() with: - name: testResults + # Unique per shard so the 4 parallel matrix legs don't clobber each + # other; collect-results downloads them all by the testResults-shard* glob. + name: testResults-shard${{ matrix.shard }} path: | test/**/*.trx test/test-results.log + test/**/blame-*.dmp collected-logs/ compression-level: 9 retention-days: 15 + # ── Per-shard surfacing: publish THIS shard's results as soon as it + # finishes, so you don't wait for the slowest sibling + the collector to + # see shard 0's outcome. The collect-results job re-consolidates all + # shards into one check at the end. + - name: Summarize test failures (this shard) + if: always() + run: | + echo "### Failed tests — shard ${{ matrix.shard }}" >> "$GITHUB_STEP_SUMMARY" + any=0 + for trx in $(find test -name '*.trx' 2>/dev/null | sort); do + fails=$(grep -oE ']*outcome="Failed"[^>]*>' "$trx" \ + | grep -oE 'testName="[^"]*"' \ + | sed -E 's/testName="([^"]*)"/\1/' | sort -u) + if [ -n "$fails" ]; then + any=1 + while IFS= read -r t; do + [ -z "$t" ] && continue + echo "::error title=Test failed (shard ${{ matrix.shard }})::$t" + echo "- \`$t\`" >> "$GITHUB_STEP_SUMMARY" + done <<< "$fails" + fi + done + if [ "$any" = "0" ]; then + echo "_No failed tests in trx for shard ${{ matrix.shard }} (a hang/crash would show below)._" >> "$GITHUB_STEP_SUMMARY" + fi + - name: Publish Test Results (this shard) + uses: EnricoMi/publish-unit-test-result-action@v2.12.0 + if: always() + with: + # Per-shard check so the 4 matrix legs don't clobber each other's + # check-run; distinct from the collector's "Test Results" check. + check_name: "Test Results (shard ${{ matrix.shard }})" + action_fail: true + files: | + test/**/*.trx + - name: Fail on hang / aborted test run (this shard) + # A hang aborts the test host and writes blame-*.dmp; fail the shard loudly + # so it isn't masked behind a green "results published" check. Last step so + # the artifact upload + per-shard publish above already ran. + if: always() + run: | + hang_dumps=$(find test -name 'blame-*.dmp' 2>/dev/null | wc -l) + if [ "$hang_dumps" -gt 0 ]; then + echo "::error::Shard ${{ matrix.shard }}: a test process hung — blame-*.dmp written." + find test -name 'blame-*.dmp' 2>/dev/null + cat test/test-results.log + exit 1 + fi + + # ──────────────────────── COLLECT RESULTS ──────────────────────── + # Single fan-in: pull every shard's results and produce ONE consolidated + # "Test Results" check plus a combined step summary. This job is the run's + # pass/fail gate for tests + hangs. (A shard job that infra-fails reds the run + # on its own — GitHub's run conclusion is failure if ANY job fails, regardless + # of this always() collector's own conclusion.) + collect-results: + name: "Consolidate test results" + needs: test + if: always() + runs-on: ubuntu-latest + steps: + - name: Download all shard results + uses: actions/download-artifact@v6 + with: + # No merge-multiple: keep each shard under its own dir so same-named + # files (test-results.log) from different shards don't overwrite. + pattern: testResults-shard* + path: all-results + - name: Summarize test failures + # Parse every shard's trx and echo each failed test as an ::error:: + # annotation + into the job step summary, so failing tests are visible + # directly on THIS job without opening the published check. + if: always() + run: | + echo "### Failed tests (all shards)" >> "$GITHUB_STEP_SUMMARY" + any=0 + for trx in $(find all-results -name '*.trx' 2>/dev/null | sort); do + # testName on each UnitTestResult element whose outcome is Failed. + fails=$(grep -oE ']*outcome="Failed"[^>]*>' "$trx" \ + | grep -oE 'testName="[^"]*"' \ + | sed -E 's/testName="([^"]*)"/\1/' | sort -u) + if [ -n "$fails" ]; then + any=1 + while IFS= read -r t; do + [ -z "$t" ] && continue + echo "::error title=Test failed::$t" + echo "- \`$t\`" >> "$GITHUB_STEP_SUMMARY" + done <<< "$fails" + fi + done + if [ "$any" = "0" ]; then + echo "_No failed tests in trx (a hang/crash would show below)._" >> "$GITHUB_STEP_SUMMARY" + echo "No failed tests found in trx across all shards." + fi + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2.12.0 + if: always() + with: + # One consolidated check across all shards — the single pass/fail gate. + check_name: "Test Results" + action_fail: true + files: | + all-results/**/*.trx + - name: Fail on hang / aborted test run + # blame-hang aborts the test host and writes a minidump (blame-*.dmp). + # Test *failures* are surfaced by Publish Test Results above; this step's + # only job is to fail loudly on a HANG so it doesn't get masked behind a + # "test results published" green checkmark when a trx contains only what + # completed before the abort. Runs even if Publish failed (if: always). + if: always() + run: | + hang_dumps=$(find all-results -name 'blame-*.dmp' 2>/dev/null | wc -l) + if [ "$hang_dumps" -gt 0 ]; then + echo "::error::A test process hung — blame-*.dmp files were written." + echo "--- Hang dumps ---" + find all-results -name 'blame-*.dmp' 2>/dev/null + echo "--- Per-project exit codes (non-zero indicates the hung project) ---" + find all-results -name 'test-results.log' -exec cat {} \; + exit 1 + fi diff --git a/.github/workflows/main-cd.yml b/.github/workflows/main-cd.yml new file mode 100644 index 000000000..2f20deae0 --- /dev/null +++ b/.github/workflows/main-cd.yml @@ -0,0 +1,148 @@ +name: Continuous Delivery (main) + +# CONTINUOUS-channel delivery: every merge to main that PASSES the test suite ships a +# fresh deployment image and rolls it out to every environment. +# +# push main ─▶ "MeshWeaver Build and Test" (existing) ─▶ [THIS] images ─▶ deploy (memex, atioz, memex-cloud) +# +# Why workflow_run (and not push: branches:[main]): we want the image+deploy to fire ONLY +# after the full test suite is green on the merged commit — never ship an image that failed +# tests. workflow_run starts after the named workflow COMPLETES; the `if` below filters to a +# successful run that came from a push to main (not a PR run, not a failed run). +# +# This is the CONTINUOUS channel (build-numbered 3.0.0-ci. baked in, images tagged by +# commit SHA). The RELEASED channel — clean 3.0.0 NuGet packages + clean GHCR images — is a +# SEPARATE path fired by pushing a v*.*.* tag (release-packages.yml + release-images.yml). +# "Freeze a release" = push a tag; "merge to main" = this workflow. +on: + workflow_run: + workflows: ["MeshWeaver Build and Test"] + types: [completed] + +# id-token: write is required for OIDC federated login to Azure (no stored client secret). +permissions: + contents: read + id-token: write + +# Never let two main merges deploy concurrently — a later merge waits for the in-flight +# rollout rather than racing it. cancel-in-progress:false so we don't abort a half-done deploy. +concurrency: + group: main-cd + cancel-in-progress: false + +env: + ACR: meshweaver.azurecr.io + AKS_RG: memex-aks-rg + AKS_CLUSTER: memexaks-cluster + +jobs: + # ─────────────────────────── BUILD + PUSH IMAGES ─────────────────────────── + images: + name: "Build + push images to ACR" + # Gate: only a SUCCESSFUL test run, from a PUSH (not a PR), on main. + if: >- + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'main' + runs-on: ubuntu-latest + timeout-minutes: 30 + outputs: + tag: ${{ steps.vars.outputs.sha }} + steps: + - uses: actions/checkout@v6 + with: + # Build the EXACT commit the tests passed on — not whatever main is now. + ref: ${{ github.event.workflow_run.head_sha }} + # The SDK container build + the publish output overflow the runner's ~14 GB; reclaim the + # ~25 GB of preinstalled tooling a .NET build never touches (same as the test workflow). + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + dotnet: false + android: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.x + - name: Compute image tag (short SHA) + id: vars + run: echo "sha=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" + # OIDC federated login — no client secret stored. Requires a federated credential on the + # Azure AD app scoped to this repo's main branch (see the README block at the bottom). + - name: Azure login (OIDC) + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + - name: ACR login + run: az acr login --name meshweaver + - name: Restore workloads + run: dotnet workload restore + # portal-ai app image — the deployment image (aspnet + node + Claude Code/Copilot CLIs), + # layered on the base already in ACR. The base (memex-portal-ai-base) changes rarely and is + # rebuilt by the release path / runbook §1 (az acr build), NOT on every main merge. + # CIRun=true → continuous version 3.0.0-ci. baked in; image tagged + moving `main`. + - name: Build + push portal-ai + run: > + dotnet publish memex/aspire/Memex.Portal.Distributed/Memex.Portal.Distributed.csproj + -c Release -r linux-x64 --no-self-contained -t:PublishContainer -p:PublishProfile= + -p:CIRun=true + -p:ContainerRegistry=${{ env.ACR }} + -p:ContainerRepository=memex-portal-ai + -p:ContainerImageTags="${{ steps.vars.outputs.sha }};main" + -p:ContainerBaseImage=${{ env.ACR }}/memex-portal-ai-base:latest + - name: Build + push migration + run: > + dotnet publish memex/aspire/Memex.Database.Migration/Memex.Database.Migration.csproj + -c Release -r linux-x64 --no-self-contained -t:PublishContainer -p:PublishProfile= + -p:CIRun=true + -p:ContainerRegistry=${{ env.ACR }} + -p:ContainerRepository=memex-migration + -p:ContainerImageTags="${{ steps.vars.outputs.sha }};main" + + # ───────────────────────────────── DEPLOY ────────────────────────────────── + # One leg per environment (namespace) on the shared private cluster. Each leg points the + # deployment at the freshly-pushed SHA tag and restarts — the documented code-update flow + # (kubectl set image + rollout restart + rollout status), NOT deploy.sh (env setup). + # fail-fast:false so one environment failing doesn't abort the rollout to the others. + deploy: + name: "Deploy ${{ matrix.ns }}" + needs: images + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + ns: [memex, atioz, memex-cloud] + # Map each namespace to a GitHub Environment so prod (memex-cloud) CAN be given a required- + # reviewer / wait-timer protection rule later without touching this file. No rule = auto-deploy. + environment: ${{ matrix.ns }} + steps: + - name: Azure login (OIDC) + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + # Private cluster → kubectl ONLY via `az aks command invoke`. $NS/$TAG are expanded by the + # runner shell here, so the resolved namespace + tag are baked into the command string the + # cluster runs. migration + portal are set + restarted together (matches AGENTS.md); we then + # WAIT on the portal rollout so an unhealthy image fails this job loudly instead of going green. + - name: Roll out to ${{ matrix.ns }} + env: + NS: ${{ matrix.ns }} + TAG: ${{ needs.images.outputs.tag }} + run: | + set -euo pipefail + az aks command invoke -g "$AKS_RG" -n "$AKS_CLUSTER" --command " + kubectl -n $NS set image deployment/memex-portal-deployment memex-portal=$ACR/memex-portal-ai:$TAG; + kubectl -n $NS set image deployment/memex-migration-deployment memex-migration=$ACR/memex-migration:$TAG; + kubectl -n $NS rollout restart deployment/memex-migration-deployment deployment/memex-portal-deployment; + kubectl -n $NS rollout status deployment/memex-portal-deployment --timeout=300s + " diff --git a/.github/workflows/release-images.yml b/.github/workflows/release-images.yml new file mode 100644 index 000000000..b6815edef --- /dev/null +++ b/.github/workflows/release-images.yml @@ -0,0 +1,83 @@ +name: Publish Container Images + +# Prod release path for the Memex deployment images. Fires on the SAME v*.*.* tag as +# release-packages.yml (NuGet) so one tag ships the Aspire.Hosting.Memex package AND the +# images the generated compose/Helm/ACA artifacts reference. Local testing uses the local +# registry instead (see deploy/ + the repo's local-publish flow) — never this workflow. +on: + push: + tags: + - 'v*.*.*' + +permissions: + contents: read + packages: write + +jobs: + images: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup .NET + uses: actions/setup-dotnet@v5 + with: + dotnet-version: '10.x' + + - name: Extract version from tag + run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> "$GITHUB_ENV" + + - name: Lowercase image namespace (GHCR requires lowercase) + run: echo "IMAGE_NS=$(echo '${{ github.repository_owner }}' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_ENV" + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Restore workloads + run: dotnet workload restore + + # 1. portal-ai BASE image (the one hand-authored Dockerfile: aspnet + node20 + + # @anthropic-ai/claude-code + @github/copilot). On this Linux runner the host RID is + # linux-x64, so the SDK-built portal layers below also get the linux copilot binary. + - name: Build + push portal-ai base image + run: | + BASE=ghcr.io/$IMAGE_NS/memex-portal-ai-base + docker build -t "$BASE:$VERSION" -t "$BASE:latest" deploy/base-images/portal-ai + docker push "$BASE:$VERSION" + docker push "$BASE:latest" + + # 2. portal-ai app image (SDK container build, layered on the base; CLIs baked in). + - name: Build + push portal-ai + run: > + dotnet publish memex/aspire/Memex.Portal.Distributed/Memex.Portal.Distributed.csproj + -c Release -r linux-x64 --no-self-contained -t:PublishContainer -p:PublishProfile= + -p:Version=$VERSION + -p:ContainerRegistry=ghcr.io + -p:ContainerRepository=$IMAGE_NS/memex-portal-ai + -p:ContainerImageTags="$VERSION;latest" + -p:ContainerBaseImage=ghcr.io/$IMAGE_NS/memex-portal-ai-base:$VERSION + + # 3. lean portal app image (default SDK base, no co-hosted CLIs). + - name: Build + push portal (lean) + run: > + dotnet publish memex/aspire/Memex.Portal.Distributed/Memex.Portal.Distributed.csproj + -c Release -r linux-x64 --no-self-contained -t:PublishContainer -p:PublishProfile= + -p:Version=$VERSION + -p:ContainerRegistry=ghcr.io + -p:ContainerRepository=$IMAGE_NS/memex-portal + -p:ContainerImageTags="$VERSION;latest" + + # 4. db-migration image (SDK container build, default base). + - name: Build + push migration + run: > + dotnet publish memex/aspire/Memex.Database.Migration/Memex.Database.Migration.csproj + -c Release -r linux-x64 --no-self-contained -t:PublishContainer -p:PublishProfile= + -p:Version=$VERSION + -p:ContainerRegistry=ghcr.io + -p:ContainerRepository=$IMAGE_NS/memex-migration + -p:ContainerImageTags="$VERSION;latest" diff --git a/.github/workflows/release-packages.yml b/.github/workflows/release-packages.yml index f399b4209..5d6fdb9ee 100644 --- a/.github/workflows/release-packages.yml +++ b/.github/workflows/release-packages.yml @@ -20,7 +20,7 @@ jobs: - name: Setup .NET uses: actions/setup-dotnet@v5 with: - dotnet-version: '9.x' + dotnet-version: '10.x' - name: Extract version from tag id: extract_version diff --git a/.gitignore b/.gitignore index 2022398b1..8fa61d50d 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,21 @@ bld/ [Oo]utput/ [Dd]eploy/ +# MeshWeaver deployment artifacts (Aspire-generated compose/Helm, Marketplace ARM, the +# co-hosted-CLI base image, env templates) live under /deploy — re-include them from the +# Visual Studio "[Dd]eploy/" web-deploy-output rule above. +!/deploy/ +!/deploy/** +# ...but keep build outputs (the dedicated deploy AppHost compiles here) ignored. +/deploy/**/bin/ +/deploy/**/obj/ +# Per-tenant AKS deploy envs are PRIVATE — keep ONLY the memex-cloud reference example; +# every other tenant env directory is ignored in full (directory names are tenant +# identities and must not enter this public repo). Placed after the !/deploy/** re-include +# above so this re-exclusion wins. +/deploy/aks/envs/*/ +!/deploy/aks/envs/memex-cloud/ + # Visual Studio 2015/2017 cache/options directory .vs/ .vscode diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..772f74d9b --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,526 @@ +# AGENTS.md + +This file provides guidance to AI agents working with this repository. + +## Git Workflow + +**NEVER commit or push automatically.** Always wait for the user to explicitly ask. + +## 🚨🚨🚨 ABSOLUTE: No band-aids — root cause only, literally always + +**The user is LITERALLY NEVER interested in a band-aid, workaround, mitigation, or symptom-suppression.** When something hangs, deadlocks, flakes, or errors, find the EXACT defect and fix THAT — never paper over it. + +These are band-aids, and proposing one as "the fix" is forbidden: +- **Increasing a bound to make it pass**: pool size, timeout, retry count, buffer size, `maxParallelThreads`. The question is never "how do I get more headroom" — it's "why is the slot/thread/budget not being released, or why is it erroring." A capped pool that exhausts means a slot is leaked or blocked; a timeout that trips means something never completes — fix the leak/block/non-completion. +- **A watchdog / timer / poller that resubscribes or retries** to recover from a state that "shouldn't happen." If the initial state never arrives, find why it's dropped/erroring — don't add a timer to paper over it. (The 2026-06-08 prod outage was exactly this: an initial-state retry watchdog amplified a mishandled error into a resubscribe storm.) +- **`catch {}` / swallow-and-continue / `.Catch(Observable.Empty)`** that hides a fault instead of surfacing or fixing it. +- **Revert-and-move-on** when the revert just hides a defect that's still live underneath. +- **A `Clear()` for test isolation, a widened `.Timeout(...)`, a sleep** — each is the *tell* of an unfixed root cause. + +If active bleeding genuinely needs a stopgap before the real fix lands, say so EXPLICITLY: "this is a temporary stopgap; the root cause is X; I will fix X" — then fix X. Default to writing a **deterministic repro** (a concurrency/deadlock test if that's the failure mode) that pins the true cause before changing code, so the fix is proven, not guessed. Full reference: memory `feedback_no_bandaids`. + +## 🚨🚨🚨 ABSOLUTE: No hand-woven async/concurrency primitives — the actor model does NOT tolerate `SemaphoreSlim` + +**A `SemaphoreSlim` (or any hand-rolled async gate / lock-for-async / signal) anywhere in `src/` is FORBIDDEN — outside the one place sealed inside `IoPool`.** `SemaphoreSlim.WaitAsync()` blocks/parks a thread and its continuation captures the awaiting scheduler. On a hub it parks the single-threaded action block (or a grain turn) → the message you're waiting on can never be processed → **deadlock**. This is the same defect class as `async`/`await`/`Task` in hub code; a `SemaphoreSlim` is just a lock-shaped version of it. + +- **Serialization channels through the hub, never a semaphore.** "Only one at a time" / "wait your turn" is what the hub's single-threaded action block already gives you for free. When you need ordered, one-at-a-time processing, push items into a `Subject` and run them with `.Select(Run).Concat().Subscribe(...)` (Concat subscribes the next only after the previous completes — order without a lock; the canonical fix is `KernelExecutor`'s REPL queue, which **replaced** a hand-woven `SemaphoreSlim`), or route state changes through `GetMeshNodeStream(path).Update(...)` (the owning hub serialises every writer). +- **Concurrency bounding / one-shot init / "run once" channels through `IIoPool`.** A bounded I/O gate, a promise-cached one-shot (schema provisioning, blob-cache init, connect handshake), a "first caller does it, the rest wait" — that is `pool.Run(...)` cached in an **instance** `ConcurrentDictionary` (the promise-cache; ReplaySubject-backed: runs once, replays to all). NOT a `SemaphoreSlim(1,1)` `_initLock` / `_connectGate`. +- **`Task`-as-a-gate is the same sin.** `TaskCompletionSource` used to make callers "await a signal", a `Task.Delay` timeout race, `ManualResetEventSlim`, `lock`-around-`await` — all hand-woven async. Make the **source observable** (`AsyncSubject`/`Subject` + `Concat`) and `Subscribe`, or push it onto `IIoPool`. + +**The ONLY sanctioned `SemaphoreSlim` is the one sealed inside `IoPool` itself** (`MeshWeaver.Mesh.Threading`) — it IS the single boundary between the turn-based hub schedulers and genuinely-async I/O leaves, running work OFF the hub with `ConfigureAwait(false)`. Everywhere else, a `SemaphoreSlim` is a bug to delete. The litmus test: if your gate runs on (or is awaited from) a hub action block / grain turn / Blazor circuit, it deadlocks — channel it through a hub or `IIoPool` instead. Full reference: [ControlledIoPooling.md](src/MeshWeaver.Documentation/Data/Architecture/ControlledIoPooling.md), [AsynchronousCalls.md](src/MeshWeaver.Documentation/Data/Architecture/AsynchronousCalls.md), memory `feedback_no_semaphoreslim`. + +## 🚨🚨🚨 ABSOLUTE: Never hand-roll UI / data-binding / persistence / submit — use the framework + +**A "UI feature" means wiring up the framework's EXISTING pieces, never reinventing them.** The framework already does data binding, node-content editing, auto-persistence, picking, **rendering (tables/lists via `DataGrid` and the typed controls)**, and thread submission ONE standard way that every layout area uses. Hand-rolling a parallel version — including emitting raw HTML strings instead of using a control — is FORBIDDEN. + +- **Editing a mesh node's content, data-bound + auto-persisting** → bind the GUI client DIRECTLY to the node stream: declare `MeshNodeContentEditorControl.ForType(path, typeof(MyContent))` (simple scalar/bool fields) and the Blazor view reads from `Hub.GetMeshNodeStream(path)` and writes per-field via `GetMeshNodeStream(path).Update(...)`. ONE source of truth — the node stream. Rich content (markdown/picking) → already-node-bound controls (`MarkdownEditorControl.WithAutoSave`, `MeshNodePickerControl`, `CollaborativeMarkdownView`). +- **🚨 NEVER replicate the node into a layout-area `/data/{id}` copy + a server-side save subscription.** `host.UpdateData(id, node.Content)` + `GetDataStream(id).Debounce/Throttle.Subscribe(...GetMeshNodeStream(path).Update...)` — a.k.a. `OverviewLayoutArea.SetupAutoSave` / any `*AutoSave` helper / a "Save" button that reads `/data` and writes the node — is the FORBIDDEN replicate-then-save antipattern (two stores drift; the save loop clobbers unedited fields). The "standard EditNode / MeshNodePropertyEditor" editor IS this antipattern — do not copy it; migrate it. +- **Picking a mesh node** → `[MeshNode("query")]` → `MeshNodePickerControl` (stores the node PATH). **Form controls** → the `Edit` macro + `[UiControl]`/`[Description]`/`[Editable(false)]`. Don't hand-build selects/checkboxes/textareas + a data section. +- **🚨 Rendering tabular / structured data → a framework CONTROL, NEVER hand-built HTML.** Tables → `Controls.DataGrid(rows).WithColumn(new PropertyColumnControl { Property = nameof(Row.X).ToCamelCase() }.WithTitle("…").WithFormat("N0"))` bound to a plain row record (sorting / formatting / theming / virtualization for free; column API: `samples/Graph/Data/Cornerstone/Pricing/Source/PricingLayoutAreas.cs`). Compose `Controls.Stack` / `Controls.LayoutGrid` / `Controls.Title` / `Controls.Markdown`. **FORBIDDEN: building HTML strings** — `StringBuilder`/`$"…"`/`$"
…"`, any `RenderHtml`-shaped helper, or `Controls.Html(handBuiltMarkup)` for structured data (`Controls.Html` is ONLY for genuinely pre-rendered markdown/rich text). This is the exact hack the user banned 2026-06-19 (*"use just controls and layout areas … get rid of RenderHtml … I don't want to see such hacks any more"*) — and the hand-rolled `RenderHtml`'s string-interpolation + LINQ also caused the >10-min MeshWeaver.AI compile regression (e30e9b5f1). If you're reaching for a string of HTML, STOP and find the control. +- **Submitting a chat message** → existing `hub.StartThread(...)` / `hub.SubmitMessage(...)` (see the thread tests: `client.SubmitMessage(threadPath, text, …)`). No wrapper class, no path→id resolution, no create-or-submit logic beyond those APIs. Pass node PATHS through; downstream loads the node (e.g. `StartThread` takes a model PATH and loads the model from its mesh-node stream — don't pre-resolve an id). +- **Never** `.Take(1)` on a stream feeding a live data-bound view — it freezes the binding (GUI/DataBinding.md). + +Before writing ANY UI/binding/persistence code, FIND the existing framework area/control/macro/extension and use it. If you're reaching for `GetDataStream`/`Subscribe`/`Update`/`CombineLatest`/a new wrapper for a UI feature, STOP. Full reference: memory `feedback_no_handrolling`; GUI/DataBinding.md; the data-bind tests (`InlineEditingWorkflowTest`). + +## 🚨🚨🚨 ABSOLUTE: Never change log levels in code for debug reasons + +**Editing `LogInformation` ↔ `LogDebug` ↔ `LogTrace` (or `appsettings.json` under `src/`) to dial verbosity up or down for a debugging session is FORBIDDEN.** Log levels in code reflect the production cost model — `Information` lines ship to App Insights and we pay per ingest. Changing them temporarily silently bleeds budget the next CI run. + +To turn the volume up for debugging, **edit the appsettings.json in the test's `bin/Debug/net10.0/` (or the equivalent runtime config)** — `reloadOnChange: true` is wired so the level flips mid-run without a rebuild. The src-tree `appsettings.json` and every `Log*` call in `src/` is committed contract. + +If a Log call is genuinely too noisy or too quiet at the level it's written, fix it permanently with a real commit message explaining the cost/value trade-off — don't sneak it in alongside an unrelated change. + +## Test Triage + +When CI fails, **DO NOT run entire test projects** — iterate one test at a time: + +1. Read failed test names from CI logs (`gh run view --log`) +2. `dotnet test --filter "FullyQualifiedName~" --no-build --no-restore` +3. **No skipping** — CI-only failures catch real timing/state bugs + +Full guidance: [WritingTests.md](src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md) · [CqrsAndContentAccess.md](src/MeshWeaver.Documentation/Data/Architecture/CqrsAndContentAccess.md) · [TestStateIsolation.md](src/MeshWeaver.Documentation/Data/Architecture/TestStateIsolation.md) + +## GitHub PR Operations + +`gh` CLI has **read + push** only — cannot merge, resolve threads, or request reviewers. + +```bash +# Find unresolved review threads +gh api graphql -f query='query($owner:String!, $repo:String!, $pr:Int!) { repository(owner:$owner, name:$repo) { pullRequest(number:$pr) { reviewThreads(first:100) { nodes { id isResolved } } } } }' \ + -f owner=Systemorph -f repo=MeshWeaver -F pr=PR_NUMBER \ + --jq '.data.repository.pullRequest.reviewThreads.nodes[] | select(.isResolved==false) | .id' +# Resolve a thread +gh api graphql -f query='mutation($id:ID!){ resolveReviewThread(input:{threadId:$id}){ clientMutationId }}' -f id=THREAD_ID +gh pr merge PR_NUMBER --merge +``` + +**If `FORBIDDEN`**: re-authenticate with `! gh auth login`. + +## 🚨 Postgres: One Schema Per Partition + +**`public.mesh_nodes` is empty by design.** Data lives in per-partition schemas (`acme.mesh_nodes`, `rbuergi.mesh_nodes`, etc.). + +Satellite table routing by path segment: + +| Path segment | Table | +|---|---| +| `…/_Access/…` | `access` | +| `…/_Thread/…` | `threads` | +| `…/_Activity/…` | `activities` | +| `…/_Comment/…`, `_Approval`, `_Tracking` | `annotations` | +| `…/Source/…` or `…/Test/…` | `code` | +| (none) | `mesh_nodes` | + +**`namespace` keeps the partition prefix — never strip it.** `namespace = rbuergi/ApiToken`, not `ApiToken`. + +**Never run raw `psql UPDATE` on a live portal** — bypasses the workspace cache. Use `MoveNodeRequest` or add a Repair vN migration. If you must SQL-edit, restart `Memex.Portal.Distributed`. + +**🚨 Partition schema: provision + existence are REACTIVE + POOLED — never declare a `PartitionDefinition` node to force a schema, never lowercase by hand.** The standard surface is on `IPartitionStorageProvider`: +- `EnsurePartitionProvisioned(namespace) : IObservable` — the ONE entry point that creates a partition's schema + tables. Reactive, idempotent (promise-cached), and **pooled** on the `pg:{adapter}` IoPool (the PG impl lowercases the schema correctly). Subscribe it; compose with `.SelectMany(_ => write…)` before writing to a not-yet-provisioned partition. +- `PartitionExists(namespace) : IObservable` — reactive existence check (`null` = indeterminate; OR-fold across providers as `PartitionWriteGuardValidator` does). + +The router maps a path's first segment to `seg.ToLowerInvariant()`; a `PartitionDefinition` with `Schema` left null provisions the schema **verbatim** (`"Agent"` capital) while writes hit `"agent"` → 42P01. So the way to make code that writes a not-yet-provisioned partition work is `EnsurePartitionProvisioned(p).SelectMany(_ => write…)` — **not** a partition-def node. The async schema DDL runs inside the IoPool, never `Observable.FromAsync` (see [ControlledIoPooling.md](src/MeshWeaver.Documentation/Data/Architecture/ControlledIoPooling.md)). + +Full reference: [PostgresSchemaArchitecture.md](src/MeshWeaver.Documentation/Data/Architecture/PostgresSchemaArchitecture.md) + +## 🛡️ Global admin = admin on the Admin partition + +**"Global/platform admin" has ONE meaning: `Permission.All` at scope `Admin`** — an `AccessAssignment` granting the `Admin` role in the **`Admin/_Access`** namespace (`Admin` is a standard partition, schema `admin`, that holds platform-level data). This is a **platform admin, NOT a data superuser**: `Admin/_Access` is Admin-scoped — it does NOT grant access to spaces or user partitions. Standing access = platform management (invites, deletes, config); emergency cross-partition data change requires explicit **elevation (break-glass)**, never standing. A **root** `_Access` grant is the data-superuser shape and is deliberately NOT how platform admins are provisioned. + +- **The one predicate is `hub.IsGlobalAdmin()` / `hub.IsGlobalAdmin(userId)`** (`HubPermissionExtensions`). Every gate goes through it — NEVER an ad-hoc role-name (`Roles.Contains("PlatformAdmin")`) or root-scope (`GetEffectivePermissions("")`) check. +- **The grant lives in `Admin/_Access`, never root `_Access`.** Writers (`GlobalAdminSeed` from `Auth:GlobalAdmins`, `UserOnboardingService.GrantPlatformAdmin`) and readers (`AdminMenuGate`, `UserNodeType.GetGlobalAdminTabAsync`, `UserProfile`) must agree on the Admin partition — a writer/reader split (root vs Admin) silently locks admins out of every admin tab (2026-06-08). + +Full reference: [AccessControl.md](src/MeshWeaver.Documentation/Data/Architecture/AccessControl.md) → "The Admin partition". + +## Documentation + +All docs embedded in `src/MeshWeaver.Documentation/` and served under `Doc/` at runtime. + +| Topic area | Path | +|---|---| +| Architecture | `src/MeshWeaver.Documentation/Data/Architecture/` | +| DataMesh | `src/MeshWeaver.Documentation/Data/DataMesh/` | +| GUI | `src/MeshWeaver.Documentation/Data/GUI/` | +| AI Integration | `src/MeshWeaver.Documentation/Data/AI/` | +| Agent definitions | `src/MeshWeaver.AI/Data/Agent/` | + +**Writing/editing a doc page:** follow [AuthoringDocumentation.md](src/MeshWeaver.Documentation/Data/Architecture/AuthoringDocumentation.md). Links resolve against the page's FULL node path at render time — sibling links need `../Sibling`, absolute links start `/Doc/…`; `xref:` and `.md` suffixes never resolve. `DocumentationLinkIntegrityTest` (test/MeshWeaver.Documentation.Test) fails on any broken internal link — run it after doc edits. + +**Hub-handler test hangs or message disappears:** read [DebuggingMessageFlow.md](src/MeshWeaver.Documentation/Data/Architecture/DebuggingMessageFlow.md) first — it tells you which trace tags to grep and why you should never rerun a hung test "to see". + +**`type 'X' is not registered in this hub's TypeRegistry`:** Fix is `WithType(typeof(X), nameof(X))` on the receiving hub. See DebuggingMessageFlow.md → "Type-registry mismatch". + +**Use `hub.Observe(...)` not `RegisterCallback`/`AwaitResponse`** — those overloads are `[Obsolete]` and deadlock. Tests use `MonolithMeshTestBase.AwaitResponseAsync(...)`. + +## Deployment + +**Two deploy routes, different targets — neither deprecated.** Pick by target, don't mix: +- **AKS** — the shared cluster `memex` portal. Full ref: [DeploymentAKS.md](src/MeshWeaver.Documentation/Data/Architecture/DeploymentAKS.md). +- **Azure Container Apps** — the Aspire `test`/`prod` modes, via `tools/deploy.sh prod|test`. Full ref: [DeploymentContainerApps.md](src/MeshWeaver.Documentation/Data/Architecture/DeploymentContainerApps.md). + +**🚨 Before any AKS deploy, read [DeploymentAKS.md](src/MeshWeaver.Documentation/Data/Architecture/DeploymentAKS.md) end-to-end** — it is the source of truth for build → roll-out → verify AND for the **auto-baked mesh-local `#r` package feed** (the `BakeMeshLocalFeed` target packs `MeshWeaver.BusinessRules` + `.Generator` into the image so scope/`IScope` nodes compile offline in prod — Release publish only, no manual pack step). The commands inlined below are a quick reference, not a substitute for the doc. + +The `memex` portal runs on the shared **AKS cluster** `memexaks-cluster` (RG `memex-aks-rg`, swedencentral) — namespace `memex` — against the Postgres Flexible Server, images in ACR `meshweaver.azurecr.io`. **Private cluster: `kubectl` ONLY via `az aks command invoke -g memex-aks-rg -n memexaks-cluster --command "…"`.** + +**On AKS a code update = build image → set image → restart** (the AKS route does NOT use `tools/deploy.sh` or `aspire deploy` — those are the Container Apps route): + +```bash +az acr login -n meshweaver +# Portal (custom base) AND migration (the migration is what creates schema + the matview): +dotnet publish memex/aspire/Memex.Portal.Distributed/Memex.Portal.Distributed.csproj -c Release \ + -t:PublishContainer -p:ContainerRegistry=meshweaver.azurecr.io \ + -p:ContainerRepository=memex-portal-ai -p:ContainerImageTag= \ + -p:ContainerBaseImage=meshweaver.azurecr.io/memex-portal-ai-base:latest +dotnet publish memex/aspire/Memex.Database.Migration/Memex.Database.Migration.csproj -c Release \ + -t:PublishContainer -p:ContainerRegistry=meshweaver.azurecr.io \ + -p:ContainerRepository=memex-migration -p:ContainerImageTag= +# Roll out (NS = memex): +az aks command invoke -g memex-aks-rg -n memexaks-cluster --command "\ + kubectl -n set image deployment/memex-portal-deployment memex-portal=meshweaver.azurecr.io/memex-portal-ai:; \ + kubectl -n set image deployment/memex-migration-deployment memex-migration=meshweaver.azurecr.io/memex-migration:; \ + kubectl -n rollout restart deployment/memex-migration-deployment deployment/memex-portal-deployment; \ + kubectl -n rollout status deployment/memex-portal-deployment --timeout=300s" +``` + +- **`deploy/aks/envs//deploy.sh` is first-time ENV SETUP only** (helm install + PVCs + KV SecretProviderClass + ingress + connection-string patch). Do NOT use it for a code update — it re-runs the whole chart and can reset live config. +- **Don't run `tools/deploy.sh` or `aspire deploy` against the AKS cluster** — those are the *Container Apps* route (a different target), not a code-update path for AKS. +- `memex-migration` runs the migration then exits 0 and the Deployment restarts it (benign `CrashLoopBackOff`). Before declaring success, confirm its log shows `Database migration completed. Version: N` AND the portal serves (HTTP 200). + +Routes + full reference: [Deployment.md](src/MeshWeaver.Documentation/Data/Architecture/Deployment.md) (index) · [DeploymentAKS.md](src/MeshWeaver.Documentation/Data/Architecture/DeploymentAKS.md) · [DeploymentContainerApps.md](src/MeshWeaver.Documentation/Data/Architecture/DeploymentContainerApps.md) + +## Bash Command Guidelines + +**Stay in root** (`C:\dev\MeshWeaver`). Avoid chained commands (`&&`, `||`), `for` loops, and `cd` — they all require user confirmation. + +## Development Commands + +```bash +dotnet build # Build solution +dotnet test test/MeshWeaver.Data.Test --no-restore # Run one test project +dotnet run --project memex/Memex.Portal.Monolith # Monolith standalone (https://localhost:7122, http://localhost:5022) +dotnet run --project memex/aspire/Memex.AppHost # Aspire (requires Docker) — portal at https://localhost:7202, http://localhost:5202 +aspire run --project memex/aspire/Memex.AppHost # Aspire via CLI (registers with `aspire mcp`) — same URLs as above +aspire start --no-build --project memex/aspire/Memex.AppHost # Background + NO rebuild — fast bring-up; `aspire ps` / `aspire stop` to manage. --no-build reuses the last build (won't pick up source edits) +``` + +### Restarting just the Portal (no full Aspire restart) + +When you change code in `Memex.Portal.Distributed` or any project it references, you do NOT need to kill the whole AppHost. Three options, ordered by cost: + +1. **Hot reload (cheapest)** — start with `dotnet watch` instead of `dotnet run` / `aspire run`: + ```bash + dotnet watch --project memex/aspire/Memex.AppHost + ``` + File save → Aspire restarts the affected resource only. Preserves the dashboard, the Postgres container, and the SignalR endpoints. Most code changes apply within seconds. +2. **Aspire dashboard UI** — open `https://localhost:17200/` → Resources tab → click the ⋯ next to `memex-portal-distributed` → **Restart**. Runs `dotnet build` + restart in-place. +3. **Process kill (last resort, when watch missed a change)**: + ```powershell + Get-Process Memex.Portal.Distributed -ErrorAction SilentlyContinue | Stop-Process -Force + ``` + Aspire's resource watcher detects the exit and restarts the resource within ~5 s. Avoids a full `aspire run` restart (which would also rebuild every other resource and re-launch Postgres / blob-storage containers). + +**Don't** kill the whole `aspire` / `Memex.AppHost` process unless you changed AppHost wiring itself — full restart costs 30-60 s and loses the dashboard auth token. + +Full reference: [LocalDevWorkflow.md](src/MeshWeaver.Documentation/Data/Architecture/LocalDevWorkflow.md) + +## 🚨🚨🚨 ABSOLUTE: `GetMeshNodeStream().Update()` is the ONLY mutation API + +**Every mesh-node mutation goes through `workspace.GetMeshNodeStream(path).Update(current => modified)`. There is no other mutation surface — do NOT invent one: no `SubmitMessageRequest`-style wire messages, no completion callbacks via `hub.Set>`, no bespoke `IRequest`/`IResponse` pairs for state changes. Migrate any straggler you touch to `stream.Update`.** + +**Thread submissions** go through the canonical `IMessageHub` extension surface defined in `src/MeshWeaver.AI/HubThreadExtensions.cs`: + +```csharp +hub.StartThread(namespacePath, userText, agentName: ..., contextPath: ..., onCreated: ..., onError: ...); +hub.SubmitMessage(threadPath, userText, agentName: ..., contextPath: ...); +hub.ResubmitMessage(threadPath, userMessageId, newUserText: ...); +hub.DeleteFromMessage(threadPath, atMessageId); +hub.MarkThreadDone(threadPath, done); +hub.RecordSubmissionFailure(threadPath, userMessageId, userText, errorMessage); +``` + +Every method writes the thread node via `hub.GetWorkspace().GetMeshNodeStream(threadPath).Update(...)` (or `CreateNodeRequest` for new-thread lifecycle). The per-thread submission watcher reacts to the resulting state changes, drains `PendingUserMessages` into `Messages`, allocates cells, and invokes `ThreadExecution.ExecuteMessageAsync(execHub, RoundParams, AccessContext?)` **directly as a method** — no message dispatch. It returns `IObservable`; the watcher **subscribes** and treats completion (gated on the terminal `Status` write) as round-done. **Tests, GUI, and agents all call these extensions — this is the complete submission surface; there is no other entry point.** + +Full reference: [ThreadOperations.md](src/MeshWeaver.Documentation/Data/Architecture/ThreadOperations.md). + +**Activity operations** go through the matching `IMessageHub` extensions in `src/MeshWeaver.Mesh.Contract/HubActivityExtensions.cs`: + +```csharp +hub.CancelActivity(activityPath); // RequestedStatus = Cancelled +hub.RequestActivityStatus(activityPath, ActivityStatus.Running); // generic flip +``` + +Both write the activity node via `hub.GetWorkspace().GetMeshNodeStream(activityPath).Update(...)`; the activity hub's `WatchControlPlane` subscription reacts. Full reference: [ActivityOperations.md](src/MeshWeaver.Documentation/Data/Architecture/ActivityOperations.md). + +**Completion**: agent reaching terminal state writes `Status = Completed/Cancelled/Error` to the response cell via `PushToResponseMessage(...)` (stream.Update), AND creates a `Notification` MeshNode satellite at `{threadPath}/_Notification/{id}` via `EmitCompletionNotification`. The user's notification bell databinds to this — same source the tests assert on. Query shape: `path:{threadPath}/_Notification scope:children nodeType:Notification` (filter by nodeType for robustness when other satellite types live under the thread). + +**Observing completion**: subscribe to `workspace.GetMeshNodeStream(path)` and wait for the relevant state on the node's `Content` (e.g. `MeshThread.Messages.Count >= 2`, `RequestedStatus = Cancelled`, `Status = Completed`). The GUI databinds the same way; tests do too. + +**Tests**: any test that posts a verb-shaped request and waits for a response shape (`*Request → *Response`) is testing a deprecated API. Migrate to: write via `stream.Update`, observe via `GetMeshNodeStream(path).Where(node => predicate).FirstAsync().Timeout(...)`. + +**Application code uses only `stream.Update`.** Internal plumbing that `stream.Update` itself uses (`PatchDataRequest` for cross-hub writes, `DataChangedEvent` for stream fan-out) is fine where it already exists — but you never `hub.Post(PatchDataRequest, ...)` from application code. If you find yourself doing that, you're bypassing the API; use `workspace.GetMeshNodeStream(path).Update(...)` and the framework posts the patch for you. + +### Updating an external node — `GetMeshNodeStream(path).Update(...)` + +The same API works for nodes the caller does NOT own. `workspace.GetMeshNodeStream(path)` returns a handle that auto-dispatches: + +- `path == my-hub's-address`: writes go through the local data source (`UpdateOwn`). +- `path != my-hub's-address`: writes route to the owning per-node hub via the process-wide `IMeshNodeStreamCache`, which opens a sync subscription + posts a JSON-merge `PatchDataRequest` (RFC 7396) to that hub. The owner serialises every mirror's write through its single-threaded action block — no race, no clobber. + +```csharp +// Own node (this hub) — Update is COLD: the trailing Subscribe runs the write. +workspace.GetMeshNodeStream().Update(node => node with { Content = ... }) + .Subscribe(_ => { }, ex => logger.LogWarning(ex, "update failed")); + +// External node (anywhere in the mesh — same API): +workspace.GetMeshNodeStream(otherPath).Update(node => node with { Content = ... }) + .Subscribe(_ => { }, ex => logger.LogWarning(ex, "update failed")); +``` + +The remote variant returns the locally-computed updated snapshot optimistically; if you need the owner's reconciled state, take the next emission off the same `GetMeshNodeStream(path)` handle. + +**Eventual-consistency safe**: cross-hub `stream.Update` does NOT send the whole node back. It diffs `current` vs `update(current)` and sends only the RFC 7396 JSON-merge patch. The owner merges the patch against its CURRENT state, so concurrent writers from different mirrors don't clobber each other's fields (Mirror A's `{Content: {Field1: X}}` and Mirror B's `{Content: {Field2: Y}}` both land — never "last write wins on whole node"). Treat your `update` lambda accordingly: touch only the fields you intend to change. + +### The 3 rules (unchanged) + +This is the unification of three rules we used to write separately: + +1. **Writes**: `stream.Update(current => current with { Content = ... })`. The owning hub's action block serialises; no race. State-machine semantics? Set a `RequestedX` field — the owning hub's watcher reacts (see [ActivityControlPlane.md](src/MeshWeaver.Documentation/Data/Architecture/ActivityControlPlane.md)). +2. **Reads**: `workspace.GetMeshNodeStream(path)` / `Hub.GetMeshNodeStream(path)` — server-side AND Blazor, backed by the process-wide [IMeshNodeStreamCache](src/MeshWeaver.Hosting/MeshNodeStreamCache.cs) (one shared handle per path; see [GUI Data Binding](src/MeshWeaver.Documentation/Data/GUI/DataBinding.md)). `GetRemoteStream` is framework plumbing — never use it for a node by path. Never `meshService.QueryAsync(path:X)` for a single node's content (stale by design). +3. **Delete the request type.** If you find yourself writing `class XxxRequest` to mutate a thread / message / NodeType, stop. Add a `RequestedXxx` field to the node's content and watch it from the owning hub. + +Sanctioned exceptions (NOT for state mutations): +- `CreateNodeRequest` / `DeleteNodeRequest` / `MoveNodeRequest` — node-lifecycle on the mesh hub. These route, they don't mutate node content. +- Transient queries that don't belong on any node (e.g. autocomplete completions). + +Why this rule unblocks tests: every "hub becomes unresponsive after the second compile" failure (CodeEditRecompile, NodeTypeRelease, LinkedInPullActions, ThreadAgentIntegration in CI 26036857424) traces back to bespoke request/response patterns that race the watcher → two concurrent activities → leaked callbacks → wedged hub. + +Canonical references: +- [MeshNodeStreamCache.md](src/MeshWeaver.Documentation/Data/Architecture/MeshNodeStreamCache.md) — the handle contract: one cache per silo, one shared handle per path, serial write queue, storm breaker. +- [RequestViaStreamUpdate.md](src/MeshWeaver.Documentation/Data/Architecture/RequestViaStreamUpdate.md) — the canonical pattern + helpers (`hub.WatchControlPlane`, `hub.WatchSubmission`). +- [ActivityControlPlane.md](src/MeshWeaver.Documentation/Data/Architecture/ActivityControlPlane.md) — `Status`/`RequestedStatus` pair, operations-as-scripts. +- [CqrsAndContentAccess.md](src/MeshWeaver.Documentation/Data/Architecture/CqrsAndContentAccess.md) — read semantics + why `QueryAsync` lags. +- [DataBinding.md](src/MeshWeaver.Documentation/Data/GUI/DataBinding.md) — the Blazor-side mirror of the same pattern. + +## 🚨 Never write as hub — AccessContext propagation + +**Every framework write primitive (`meshService.CreateNode/UpdateNode/DeleteNode/CopyNode`, `MeshNodeStreamHandle.Update`, `IMeshNodeStreamCache.Update`) automatically carries the caller's `AccessContext` through `.Subscribe()` boundaries.** Callers keep writing the natural `.Subscribe(...)` shape; the framework guarantees the operation runs under the calling user's identity even when the emission lands on another thread. + +If a write must run as system/hub (legitimate infrastructure — cache hydration, SyncStream heartbeats), wrap explicitly: +- `using (accessService.ImpersonateAsSystem()) { … }` — well-known `"system-security"` identity; `Permission.All` granted unconditionally. +- `using (accessService.ImpersonateAsHub(hub)) { … }` or `o.ImpersonateAsHub(hub.Address)` on the post — stamps the hub's address as principal. + +PostPipeline fails closed when no context is set. The "silently stamp hub-self as principal" fallback was deleted 2026-05-21 — it masked the prod EventCalendar bug. Application code that needs to write MUST have a real user identity on `AccessService.Context` (set by MessageHub on every handler invocation from `delivery.AccessContext`). + +Canonical reference: [AccessContextPropagation.md](src/MeshWeaver.Documentation/Data/Architecture/AccessContextPropagation.md). + +## 🚨🚨🚨 ABSOLUTE: Nothing async, EVER — *NO* `async`, *NO* `await`, *NO* `Task` in hub/UI code + +**The user is LITERALLY NEVER OK with `async`/`await`/`Task`/`.ToTask()`/`TaskCompletionSource` in any hub-reachable OR Blazor-view/component code.** It runs continuations on the wrong scheduler, deadlocks the single-threaded action block, and (the 2026-06-10 chat regression) NotFound-storms a partition hub until the whole portal wedges. **Read [AsynchronousCalls.md](src/MeshWeaver.Documentation/Data/Architecture/AsynchronousCalls.md) BEFORE writing any call that touches the hub, a mesh node, or a stream.** + +Everything is `IObservable` end-to-end — compose and **`.Subscribe(...)`**, never `await`: +- **Create / read / update a node** → `meshService.CreateNode(node).Subscribe(...)` · `hub.Observe(req).Subscribe(...)` · `workspace.GetMeshNodeStream(path).Update(cur => …).Subscribe(...)`. NEVER `await …Async()`. (For create-or-update use the reactive `hub.Observe(new CreateOrUpdateNodeRequest(node)).Subscribe(...)` — see `StaticRepoImporter` / `NodeCopyHelper`.) +- Handlers, services, layout areas → return `IObservable` (or `void` for fire-and-forget). Never `Task`. +- Compose with `.SelectMany`, `.Select`, `.Where`, `.Timeout`. Chain dependent work in `.SelectMany`, not `await`. +- Click actions: `WithClickAction(ctx => { ...; return Task.CompletedTask; })` — never `async ctx =>`. +- `async`/`await`/`Task.Run`/`TaskCompletionSource`/`.ToTask()`/`.Result`/`.Wait()` in hub or Blazor-view code = red flag — delete it, return/compose `IObservable` and Subscribe. +- **Tests only**: `await .FirstAsync().ToTask()` is acceptable. Nowhere else. + +### 🚨🚨🚨 ABSOLUTE: `Observable.FromAsync` is NEVER tolerated + +**Writing `Observable.FromAsync(...)` anywhere in `src/` is FORBIDDEN — no exceptions, no "Postgres is special", no "storage is the hot path".** A bare `FromAsync` runs the function's synchronous prologue on the **subscribing thread** (the hub/grain scheduler when the subscribe happens mid-handler) and applies no concurrency bound — the exact deadlock-and-exhaustion bug class the I/O pool exists to kill. There is exactly **one** place `FromAsync` may appear: sealed *inside* `IoPool` itself. Everywhere else it is a defect. + +**Every async / blocking / IO edge goes through `IIoPool`** (`MeshWeaver.Mesh.Threading`), resolved from `IoPoolRegistry` (mesh-scoped singleton — never static): + +| You have | Use | +|---|---| +| A `Task`-returning leaf (DB round-trip, blob, HTTP, async file) | `pool.Invoke(ct => SomethingAsync(ct))` — or `pool.Run(...)` for the eager **promise-cache** (ReplaySubject-backed: runs once, replays to all) | +| A sync-blocking / CPU leaf (`File.ReadAllBytes`, Roslyn compile, `Process`) | `pool.InvokeBlocking(ct => Work(ct))` | +| An `IAsyncEnumerable` leaf | `pool.InvokeStream(...)` / `pool.RunStream(...)` | + +**The promise-cache pattern (idempotent one-shots like schema provisioning):** cache the `pool.Run(...)` observable in an *instance* `ConcurrentDictionary>` (never static) — the first caller kicks the work off on the pool, every later subscriber replays the cached completion. Canonical: `PostgreSqlPartitionStorageProvider.EnsurePartitionProvisioned` (`_provisioned.GetOrAdd(schema, _ => _ioPool.Run(ct => EnsureSchemaAsync(def, ct)))`). PG pools are named `pg:{adapter}` and capped at **1** so the gate *is* the single Npgsql connection. + +- **Public surface returns `IObservable`, never `Task`.** A `Task`-returning method that does IO is the smell; rewrite it to return `IObservable` and bridge the leaf through `IIoPool` internally. +- **MCP/SDK surface adapters**: one-line `public Task Patch(...) => ops.Patch(...).FirstAsync().ToTask();` is the only place `Task` appears at the boundary — and even there the body is reactive. +- Full reference: [ControlledIoPooling.md](src/MeshWeaver.Documentation/Data/Architecture/ControlledIoPooling.md). + +**🚨 Cold observables: Subscribe is mandatory.** Every method that performs a write returns a cold `IObservable` — the side effect runs on `Subscribe`, not on call. Forgetting to subscribe means the work silently doesn't happen. + +```csharp +// ❌ WRONG — fire-and-forget. UpdateMeshNode is cold; the dsStream.Update side +// effect only runs on Subscribe. This was the chat-doesn't-work root cause. +workspace.GetMeshNodeStream().Update(node => node with { … }); + +// ✅ RIGHT — subscribe with explicit error propagation. +workspace.GetMeshNodeStream().Update(node => node with { … }) + .Subscribe(_ => { }, ex => logger.LogWarning(ex, "Update failed for {Path}", path)); +``` + +`workspace.GetMeshNodeStream()` returns a `MeshNodeStreamHandle` that is both `IObservable` (read) AND has `.Update(update)` (write). Writes return `RequireSubscribeObservable` which **logs a warning at GC if Subscribe was never called** — search the `MeshWeaver.Mesh.RequireSubscribe` log channel after every CI run. Old API `workspace.UpdateMeshNode(...)` is `[Obsolete]`. + +**Auto-save pattern:** Form fields update the MeshNode via `stream.UpdateMeshNode` (debounced). The click action reads nothing — just flips a trigger field. No `Take(1)` on a hot stream. + +Full patterns + mistake ledger: [AsynchronousCalls.md](src/MeshWeaver.Documentation/Data/Architecture/AsynchronousCalls.md) + +## 🚨 CQRS — Never Query for a Single Node's Content + +`QueryAsync`/`ObserveQuery` are eventually consistent — **stale after writes**. To read a specific node: + +```csharp +// ❌ WRONG — lagged index, stale after writes +var node = await mesh.QueryAsync($"path:{path}").FirstOrDefaultAsync(); + +// ✅ CORRECT — authoritative, live (shared IMeshNodeStreamCache handle) +workspace.GetMeshNodeStream(path) + .Where(node => node is not null) + .Take(1).Timeout(TimeSpan.FromSeconds(10)); +``` + +**Valid query uses:** listing children (`path/*`), searching by predicate, existence checks, autocomplete. +**Wrong:** reading content by exact path, reading state before a write, polling for job completion. + +`GetMeshNodeStream(path)` + `Where(...).Take(1)` is also the right primitive for **waiting for work to finish**. + +**Free-floating words → vector search.** When a query contains bare text tokens (`laptop nodeType:Story`) AND PG is the backend AND an `IEmbeddingProvider` is registered, `PostgreSqlMeshQuery.QueryAsync` automatically routes through the HNSW cosine index instead of ILIKE substring scan. Structured-only queries (`nodeType:Story namespace:ACME`) stay on the regular SQL path. Full reference: [VectorSearch.md](src/MeshWeaver.Documentation/Data/Architecture/VectorSearch.md). + +Full treatment: [CqrsAndContentAccess.md](src/MeshWeaver.Documentation/Data/Architecture/CqrsAndContentAccess.md) + +## Mesh URL Shape + +`{baseUrl}/{meshpath}` — no `/node/` segment, no URL-encoding of separators. + +| Environment | Base URL | +|---|---| +| Prod | `https://memex.meshweaver.cloud` | +| Dev — Aspire (`memex/aspire/Memex.AppHost`) | `https://localhost:7202` (HTTP fallback `http://localhost:5202`) | +| Dev — Monolith standalone (`memex/Memex.Portal.Monolith`) | `https://localhost:7122` (HTTP fallback `http://localhost:5022`) | + +## `@/` is Local-Only + +`@/path` is a Unified Content Reference for markdown links (`[text](@/Path)`), autocomplete, and agent tool args — **never in `href=""` attributes or HTTP URLs**. Markdig strips `@` in native markdown syntax but NOT inside ``. + +## 🚨🚨🚨 ABSOLUTE: No static collections — ever + +**A `static` field that is a collection or cache is FORBIDDEN** anywhere in `src/` or `test/`: no `static ConcurrentDictionary`, `static Dictionary`, `static HashSet`, `static List`, `static ConcurrentBag`/`Queue`, `static MemoryCache`/`IMemoryCache`, `[ThreadStatic]`, or `static Lazy<…>` of mutable data. Process-wide static state survives mesh disposal, so it **bleeds across tests** — the moment you add a `Clear()` method "for test isolation", that method *is* the proof of the bug — and across users/partitions in prod. + +**Every cache and every repository is an instance owned by the mesh.** Register it in `MeshBuilder` (`ConfigureServices` / `WithServices`) as a **singleton** so its lifetime IS the mesh's: when the mesh hub is disposed (end of test / shutdown), the cache dies with it. Hold the backing store (`IMemoryCache`, an instance `ConcurrentDictionary`, …) as an **instance field** on that singleton; resolve via `hub.ServiceProvider.GetRequiredService()`. + +```csharp +// ❌ FORBIDDEN — process-wide, survives mesh disposal, bleeds across tests +public static class NodeTypeRegistry +{ + private static readonly ConcurrentDictionary Nodes = new(); + public static void Clear() => Nodes.Clear(); // ← "for test isolation" = the tell +} + +// ✅ REQUIRED — instance repo, registered in MeshBuilder, lifetime = mesh +public sealed class NodeTypeRepository +{ + private readonly ConcurrentDictionary nodes = new(); // instance, not static + public void Register(MeshNode node) => nodes[node.Path] = node; + public bool TryGet(string path, out MeshNode? node) => nodes.TryGetValue(path, out node); +} +builder.ConfigureServices(s => s.AddSingleton()); // dies with the mesh — no Clear() needed +``` + +**Allowed `static readonly`:** immutable, read-only constant lookups initialized once and never written at runtime (media-type maps, reserved-word sets, role tables). If it never takes a write after construction it's a *constant*, not a cache — fine. The instant something writes to it at runtime, it must become a mesh-scoped instance singleton. + +Full reference: [NoStaticState.md](src/MeshWeaver.Documentation/Data/Architecture/NoStaticState.md). + +## Collections Policy + +**NEVER use mutable collections.** Always `System.Collections.Immutable`: +`List` → `ImmutableList`, `Dictionary` → `ImmutableDictionary`, `HashSet` → `ImmutableHashSet`, `Queue` → `ImmutableQueue`. +Exception: `ConcurrentDictionary` for concurrent mutation — **as an instance field on a mesh-scoped singleton, never `static`** (see "No static collections" above). + +## Architecture Overview + +Actor-model message hub (`MeshWeaver.Messaging.Hub`) with address-based partitioning. UI is reactive Layout Areas rendered in Blazor Server. AI agents use plugins (MeshPlugin, LayoutAreaPlugin). + +| Directory | Contents | +|---|---| +| `src/` | Core framework (50+ projects) | +| `samples/Graph/Data/` | Sample data nodes (ACME, Northwind, Cornerstone, etc.) | +| `memex/Memex.Portal.Monolith/` | Dev portal with full Graph + Documentation support | +| `memex/aspire/` | Microservices with .NET Aspire orchestration | + +**Request-Response:** `hub.Observe(request, o => o.WithTarget(address)).Subscribe(resp => …, ex => …)` +Response sent as: `hub.Post(responseMessage, o => o.ResponseFor(request))` +**Fire-and-Forget:** `hub.Post(message, o => o.WithTarget(address))` +**Layout area route:** `@{address}/{areaName}/{areaId}` + +## Data Access Patterns + +Never use `IMeshStorage` or `IMeshCatalog` directly — internal infrastructure only. + +| Operation | API | +|---|---| +| Read (query) | `IMeshService.QueryAsync(...)` | +| Read (single node) | `workspace.GetMeshNodeStream(path)` | +| Create/Delete | `meshService.CreateNode(node).Subscribe(...)` / `meshService.DeleteNode(path).Subscribe(...)` | +| Update | `workspace.GetMeshNodeStream(path).Update(current => current with { … })` | +| Move | `hub.Observe(new MoveNodeRequest(src, dst)).Subscribe(...)` | + +Always `GetRequiredService()` — never `GetService()` + null check for required services. + +Full reference: [DataAccessPatterns.md](src/MeshWeaver.Documentation/Data/Architecture/DataAccessPatterns.md) + +## Memex is available through MCP + +The memex mesh is reachable through the **`meshweaver` MCP server** — for agents working on this repo (the `atioz` / `memex-systemorph` MCP tools you already have) AND for the co-hosted **Claude Code / GitHub Copilot** harnesses, which get a per-user `meshweaver` HTTP MCP server wired **automatically** (authenticated as the calling user). The mesh — NOT a local file tree — is the workspace: use the MCP tools to read/modify mesh content rather than guessing — `get` / `search` (read), `create` / `update` / `patch` / `move` / `copy` / `delete` (mutate), `execute_script`, `render_area`, `navigate_to`, `upload`. This file (`AGENTS.md`, read by both Claude Code and Copilot) is the canonical place that tells the co-hosted agents the mesh is MCP-accessible. + +## MCP Mutations — Always Show a Diff + +For every MCP mutation (`patch`, `update`, `create`, `delete`, `move`, `copy`): +1. `get @path` **before** — cache the JSON +2. Mutate +3. `get @path` **after** — cache the new JSON +4. Render a ` ```diff ` block showing the changed region in your response + +Read-only tools skip this: `get`, `search`, `recycle`, `get_diagnostics`, `navigate_to`, `execute_script`. + +## Development Patterns + +For detailed patterns with code examples, read: +- Layout areas + UI controls: [UserInterface.md](src/MeshWeaver.Documentation/Data/Architecture/UserInterface.md) and [GUI docs](src/MeshWeaver.Documentation/Data/GUI/) +- Message handling: [MessageBasedCommunication.md](src/MeshWeaver.Documentation/Data/Architecture/MessageBasedCommunication.md) +- AI plugins: [AI docs](src/MeshWeaver.Documentation/Data/AI/) +- Activity control plane / operations as scripts: [ActivityControlPlane.md](src/MeshWeaver.Documentation/Data/Architecture/ActivityControlPlane.md) +- Reactive click handlers + service patterns: [AsynchronousCalls.md](src/MeshWeaver.Documentation/Data/Architecture/AsynchronousCalls.md) + +**Static handlers for one-shot pipelines** — don't extract `IFooService` for DI cleanliness when there's no state. Resolve deps via `hub.ServiceProvider.GetRequiredService()` inside the static handler. + +**Operations with inputs + progress + output** (export, import, compile, mirror) → Code MeshNode template + form-bound inputs + `RequestedStatus = Running` trigger. Not a bespoke `XxxRequest/XxxResponse` handler. See [ActivityControlPlane.md](src/MeshWeaver.Documentation/Data/Architecture/ActivityControlPlane.md). + +## Key Dependencies + +.NET 10.0 · Orleans · Blazor Server · Microsoft.Extensions.AI · xUnit v3 · FluentAssertions · Markdig · Chart.js · Azure SDKs + +## Testing Guidelines + +Before building NodeTypes, data models, layout areas, or CSV loaders — read [Coder.md](src/MeshWeaver.AI/Data/Agent/Coder.md) first (canonical guide + non-negotiable testing standards). + +**No mocking.** Use `MonolithMeshTestBase` or `OrleansTestBase` — never mock `IMessageHub`, `IMeshService`, or core interfaces. +**Always `run_in_background: true`** for test runs (they take minutes). +**Never `--verbosity minimal`** when tests may fail — it hides stack traces. + +**Never `Task.Delay` to wait for propagation.** A fixed sleep races CI load: too short → flakes, too long → wastes minutes across the suite. Wait on the actual condition via `stream.Where(...).FirstAsync().Timeout(...)`. When the source is request/response (not an observable), wrap the re-query in `Observable.Interval(50.Milliseconds()).StartWith(0L).SelectMany(...).Where(predicate).FirstAsync().Timeout(...)`. Hand-rolled `while + Task.Delay(50)` poll loops are forbidden. Sanctioned `Task.Delay` uses: forcing distinct timestamps for sort assertions, and "wait to confirm nothing happened" negative tests where there's no positive signal to filter for. See WritingTests.md → "Polling loops around QueryAsync" for the full pattern. + +**Never assert "exactly N change events"** on a stream backed by pg_notify or any change feed that can race the initial-snapshot path. Filter on the emission shape (e.g. `.Where(c => c.ChangeType == QueryChangeType.Initial)`), not the count. + +xUnit v3 config (`xunit.runner.json`): `parallelizeAssembly: false`, `maxParallelThreads: 1`, `methodTimeout: 60000ms`. + +Full guidance: [WritingTests.md](src/MeshWeaver.Documentation/Data/Architecture/WritingTests.md) + +### Running Tests + +```bash +dotnet test test/MeshWeaver.Hosting.Monolith.Test --no-restore +dotnet test test/MeshWeaver.Graph.Test --filter "ClassName~AccessAssignment" --no-restore +``` + +Workflow: run once in background → read failures → fix → run once more. **🚨 NEVER re-run a test (single or suite) unless code under test has changed.** Re-running to "see if it was a flake" hides the bug — flakes are real races. Either fix the race or pin the failure with a smaller repro; do not retry. The only exceptions: (a) the test harness itself crashed (MSBuild MSB4166, infrastructure error — re-run is the same input), (b) the previous run was killed by the user before completion. + +### DevLogin and Access Control + +`MonolithMeshTestBase` auto-logs in `rbuergi@systemorph.com` as Admin. Available helpers: `TestUsers.Admin`, `TestUsers.SampleUsers()`, `builder.AddSampleUsers()`. + +For per-user access control tests, use `accessService.SetCircuitContext(new AccessContext { ObjectId = "...", Name = "..." })` before creating test data; set `null` after. + +### Node Types + +Standard types from `AddGraph()`: `Markdown`, `Code`, `Agent`, `Group`, `User`, `VUser`, `Role`, `Notification`, `Approval`, `AccessAssignment`, `GroupMembership`, `PartitionAccessPolicy`, `ActivityLog`, `UserActivity`, `Comment`, `Thread`, `ThreadMessage` + +Custom types: `builder.AddMeshNodes(new MeshNode("MyType") { Name = "My Type" })` in `ConfigureMesh`. + +### Test Base Classes + +- **`MonolithMeshTestBase`** (recommended) — full integration with persistence, messaging, DI; use `AwaitResponseAsync(request, ...)` for request/response in tests +- **`HubTestBase`** — message routing / layout tests; bridge to Task via `.FirstAsync().ToTask(ct)` + +For satellite entities (comments, threads, tracked changes): [SatelliteEntityPatterns.md](src/MeshWeaver.Documentation/Data/Architecture/SatelliteEntityPatterns.md) + +## Project Structure + +Framework code in `src/`, tests in `test/`, samples in `samples/`. +Main branch: `main`. Solution file: `MeshWeaver.slnx` (50+ projects). +Package management: `Directory.Packages.props` — update this, not individual `.csproj` files. diff --git a/CLAUDE.md b/CLAUDE.md index 3c3e8db27..f0348c1f3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,569 +1 @@ -# AGENTS.md - -This file provides guidance to AI agents when working with code in this repository. - -## Git Workflow - -**NEVER commit or push automatically.** Always wait for the user to explicitly ask for a commit or push. Present changes for review first. - -## GitHub PR Operations - -The `gh` CLI token has **read + push** permissions but **cannot** merge PRs, resolve review threads, or request reviewers. For these operations: - -### Resolve review threads + merge via GraphQL -```bash -# 1. Find unresolved threads -gh api graphql -f query=' -query($owner:String!, $repo:String!, $pr:Int!) { - repository(owner:$owner, name:$repo) { - pullRequest(number:$pr) { - reviewThreads(first:100) { - nodes { id isResolved } - } - } - } -}' -f owner=Systemorph -f repo=MeshWeaver -F pr=PR_NUMBER \ - --jq '.data.repository.pullRequest.reviewThreads.nodes[] | select(.isResolved==false) | .id' - -# 2. Resolve each thread -gh api graphql -f query='mutation($id:ID!){ resolveReviewThread(input:{threadId:$id}){ clientMutationId }}' -f id=THREAD_ID - -# 3. Merge -gh pr merge PR_NUMBER --merge -``` - -**If these fail with `FORBIDDEN`**, the token lacks write scope — do it from the GitHub UI or re-authenticate with `! gh auth login`. - -## Documentation - -Documentation is embedded in `src/MeshWeaver.Documentation/` and served under the `Doc/` namespace at runtime. - -### Architecture - -The documentation on the architecture is accessible via src/MeshWeaver.Documentation/Data/Architecture/ - -Topics: Message-based communication, Actor model, UI streaming, AI agents, Data versioning, Serialization, Access control, Partitioned persistence, Business rules & calculations - -### DataMesh - -The documentation on the data mesh is accessible via src/MeshWeaver.Documentation/Data/DataMesh/ - -Topics: Node type configuration, Query syntax, Unified Path references, Interactive markdown, Collaborative editing, CRUD operations, Data modeling - -### GUI - -The documentation on the GUI is accessible via src/MeshWeaver.Documentation/Data/GUI/ - -Topics: Container controls (Stack, Tabs, Toolbar, Splitter), Layout grid, DataGrid, Editor, Observables, Data binding, Attributes, Reactive dialogs - -### AI Integration - -The documentation on AI integration is accessible via src/MeshWeaver.Documentation/Data/AI/ - -Topics: Agentic AI, MCP authentication, MeshPlugin tools (Get, Search, Create, Update, Delete, NavigateTo) - -### Deployment - -The documentation on deployment is accessible via src/MeshWeaver.Documentation/Data/Architecture/Deployment.md - -Topics: Aspire CLI deployment, deployment modes (local/test/prod/monolith), secrets management, Azure Container Apps, PostgreSQL, Orleans clustering, infrastructure provisioning - -**Quick deploy commands** (run from repo root): -- **Prod**: `aspire deploy --project memex/aspire/Memex.AppHost/Memex.AppHost.csproj -- --mode prod` -- **Test**: `aspire deploy --project memex/aspire/Memex.AppHost/Memex.AppHost.csproj -- --mode test` - -Prerequisites: Azure CLI authenticated, Aspire CLI installed, Docker running. See the full deployment doc for details. - -### Agents - -Built-in agent definitions are embedded in src/MeshWeaver.AI/Data/Agent/ - -Agents: Executor, Navigator, Planner, Research - -## Bash Command Guidelines - -**Stay in the root directory** (`C:\dev\MeshWeaver`) and use simple, single commands. Chained commands (`&&`, `||`), `for` loops, and `cd` all require user confirmation — avoid them. -```bash -# CORRECT — simple single commands from root directory -dotnet build src/MeshWeaver.Graph/MeshWeaver.Graph.csproj -dotnet test test/MeshWeaver.Graph.Test --no-build - -# WRONG — these all require extra approval: -cd /c/dev/MeshWeaver && dotnet build # chained cd -for d in test/*; do dotnet test $d; done # for loop -dotnet build && dotnet test # chained commands -``` - -## Development Commands - -### Build and Test -```bash -# Build entire solution -dotnet build - -# Run tests (uses xUnit v3) -dotnet test - -# Run specific test project (example) -dotnet test test/MeshWeaver.Data.Test/MeshWeaver.Data.Test.csproj - -# Clean solution -dotnet clean - -# Restore packages -dotnet restore -``` - -### Running Applications - -#### Memex Portal (Recommended for Development) -```bash -dotnet run --project memex/Memex.Portal.Monolith -# Access at https://localhost:7122 -``` - -The Memex Portal uses `AddGraph()` to dynamically load Graph nodes from `samples/Graph/Data/`, and `AddDocumentation()` to serve embedded documentation under the `Doc/` namespace. This is the recommended portal for development. - -#### Microservices Portal (.NET Aspire) -```bash -dotnet run --project memex/aspire/Memex.AppHost -# Access Aspire dashboard for service management -# Requires Docker for dependencies -``` - -## Reactive Pattern — NO AWAIT IN UI / HUB FLOWS - -**Rule: `await` inside hub handlers, button click actions, and service layers that are called from those paths is FORBIDDEN. It deadlocks.** Every write/read to the mesh must be composed as an `IObservable` chain. - -This is the single most important pattern in MeshWeaver. Violating it is the cause of most "button does nothing", "popup doesn't show", and "freezes under load" bugs. - -### The three building blocks - -1. **`IMeshService.CreateNode / UpdateNode / DeleteNode` return `IObservable`** (NOT `Task`). They internally `hub.Post` + `hub.RegisterCallback`. Subscribe to drive them — never call `.ToTask()` / `.FirstAsync()` / `await` on them from a click action or hub handler. -2. **Click actions must be synchronous**: `WithClickAction(ctx => { ...; return Task.CompletedTask; })`. Never `async ctx => await ...`. -3. **Read form data via `Subscribe(...)` with `Take(1)`**, not `await FirstAsync()`. The data stream emits its current value synchronously on subscribe. - -### The canonical reactive click handler - -```csharp -.WithClickAction(ctx => -{ - // Immediate optimistic UI feedback — the click registered. - ctx.Host.UpdateData(resultId, "

Working…

"); - - // Read form data via Subscribe (sync emission for BehaviorSubject-style streams). - ctx.Host.Stream.GetDataStream>(formId) - .Take(1) - .Subscribe(data => - { - var label = data?.GetValueOrDefault("label")?.ToString() ?? ""; - if (string.IsNullOrEmpty(label)) - { - ctx.Host.UpdateData(resultId, "

Please enter a label.

"); - return; - } - - // Reactive service call — returns IObservable, no await. - // Service internally composes meshService.CreateNode/UpdateNode/DeleteNode chains. - myService.DoWork(label).Subscribe( - result => ctx.Host.UpdateData(resultId, $"

Done: {result}

"), - ex => ctx.Host.UpdateData(resultId, $"

Error: {ex.Message}

")); - }); - - return Task.CompletedTask; // ← click action itself is sync -}) -``` - -### Writing reactive services - -Compose `IObservable` chains with `SelectMany`, `Select`, `FirstOrDefaultAsync`. Return `IObservable` (not `Task`) from any method that will be called from a hub handler or click action. - -```csharp -public IObservable CreateToken(...) -{ - var userNode = new MeshNode(...); - return nodeFactory.CreateNode(userNode) // IObservable - .SelectMany(created => - { - var indexNode = new MeshNode(...) { ... }; - return nodeFactory.CreateNode(indexNode) // chain the second write - .Select(_ => new TokenCreationResult(raw, created)); - }); - // No await anywhere. The consumer calls .Subscribe(onNext, onError). -} - -// Wrap IAsyncEnumerable queries into observables: -public IObservable DeleteToken(string path) => - Observable.FromAsync(() => - meshQuery.QueryAsync(MeshQueryRequest.FromQuery($"path:{path}")) - .FirstOrDefaultAsync().AsTask()) - .SelectMany(node => - { - /* ... */ - return nodeFactory.DeleteNode(path); // IObservable - }); -``` - -### What NOT to do - -```csharp -// ❌ DEADLOCKS the hub under load. -.WithClickAction(async ctx => -{ - var data = await ctx.Host.Stream.GetDataStream(id).FirstAsync(); - var result = await myService.DoWorkAsync(data); // never awaiting hub-backed services - ctx.Host.UpdateData(resultId, result); -}) - -// ❌ Task.Run is a crutch, not a fix — identity doesn't flow, failures are invisible. -.WithClickAction(ctx => -{ - _ = Task.Run(async () => { await myService.DoWorkAsync(); }); - return Task.CompletedTask; -}) - -// ❌ Hub handlers must NOT await mesh writes either. -public async Task HandleFoo(IMessageDelivery req) -{ - await meshService.CreateNodeAsync(...); // deadlock risk - return req.Processed(); -} -``` - -### When `await` IS acceptable - -- Top-level app startup code (`Main`, `ConfigureServices`, `InitializeAsync` of test base classes). -- Pure CPU / file-I/O work that does NOT flow through the hub (e.g., `File.ReadAllTextAsync`). -- Test code that explicitly wants to block until a stream emits (use `.FirstAsync().ToTask()` then await, but only in tests). - -**Everywhere else, the shape is `Subscribe(onNext, onError)`.** If a service you need only exposes `…Async` / `Task`, add a reactive overload that returns `IObservable` and refactor. - -## Collections Policy - -**NEVER use mutable collections.** Always use `System.Collections.Immutable`: -- `List` → `ImmutableList.Empty` + `= list.Add(item)` -- `Dictionary` → `ImmutableDictionary.Empty` + `= dict.SetItem(key, val)` -- `HashSet` → `ImmutableHashSet.Empty` + `= set.Add(item)` -- `Queue` → `ImmutableQueue.Empty` + `= queue.Enqueue(item)` / `= queue.Dequeue(out var item)` -- `.ToList()` → `.ToImmutableList()`, `.ToHashSet()` → `.ToImmutableHashSet()` - -The codebase is distributed (Orleans, reactive streams). Mutable collections cause race conditions and unpredictable behavior. The only exception is `ConcurrentDictionary` for thread-safe concurrent mutation patterns. - -## Architecture Overview - -### Core Concepts - -**Message Hub Architecture**: MeshWeaver is built on an actor-model message hub system (`MeshWeaver.Messaging.Hub`). All application interactions flow through hierarchical message routing with address-based partitioning (e.g., `@app/Address/AreaName`). - -**Layout Areas**: The UI system uses reactive Layout Areas - framework-agnostic UI abstractions that render in Blazor Server. Layout areas are addressed by route and automatically update via reactive streams. - -**AI-First Design**: First-class AI integration using Microsoft.Extensions.AI with plugins (MeshPlugin, LayoutAreaPlugin) that provide agents access to application state and functionality. - -### Key Directory Structure - -- **`src/`** - Core framework libraries (50+ projects) - - `MeshWeaver.Messaging.Hub` - Actor-based message routing - - `MeshWeaver.Layout` - Framework-agnostic UI abstractions - - `MeshWeaver.AI` - Agent framework with plugin architecture - - `MeshWeaver.Blazor` - Blazor Server implementation - - `MeshWeaver.Data` - CRUD operations with activity tracking - - `MeshWeaver.Documentation` - Embedded documentation (served under Doc/) - - `MeshWeaver.Graph` - Graph node configuration and node type system - -- **`samples/`** - Sample business domain applications - - `Graph/Data/` - Sample data nodes (ACME, Northwind, Cornerstone, etc.) - - `Graph/content/` - Static content files (icons, images, attachments) - -- **`memex/`** - Memex Portal (recommended for development) - - `Memex.Portal.Monolith/` - Development portal with full Graph support - - `aspire/` - Microservices with .NET Aspire orchestration - -### Architectural Patterns - -**Request-Response**: Use `hub.AwaitResponse(request, o => o.WithTarget(address))` for operations requiring results. -The response is submitted as `hub.Post(responseMessage, o => o.ResponseFor(request))`. - -**Fire-and-Forget**: Use `hub.Post(message, o => o.WithTarget(address))` for notifications and events. - -**Address-Based Routing**: Services register at specific addresses (e.g., `bookings/q1_2025`, `app/northwind`, `pricing/id`). -Layout areas follow the pattern `@{address}/{areaName}/{areaId}`. The areaId is optional and depends on the view. -E.g. `{address}/Details/{itemId}` would render a details view for the item with `itemId`. - -Layout areas are typically kept on the same address as the underlying data. - -**Reactive UI**: All UI state changes flow through the message hub. Controls are immutable records that specify their current state. - -## Data Access Patterns - -**IMPORTANT:** Application code must never use `IMeshStorage` or `IMeshCatalog` directly — these are internal infrastructure interfaces. - -### Reads — Use IMeshService -```csharp -var query = hub.ServiceProvider.GetRequiredService(); -var node = await query.QueryAsync("path:org/Acme", maxResults: 1).FirstOrDefaultAsync(ct); -``` - -### Creates/Deletes — Use IMeshNodeFactory -```csharp -var factory = hub.ServiceProvider.GetRequiredService(); -await factory.CreateNodeAsync(node, createdBy: userId, ct); -await factory.DeleteNodeAsync(path, recursive: true, ct); -``` - -### Updates/Moves — Use message requests -```csharp -hub.Post(new UpdateNodeRequest(updatedNode)); -await hub.AwaitResponse(new MoveNodeRequest(sourcePath, targetPath), ct); -hub.Post(new DataChangeRequest { Updates = [entity] }); -``` - -### Service Resolution -Always use `GetRequiredService()` for core services (`IMeshNodeFactory`, `IMeshService`). Never use `GetService()` + null check for services that must be registered. - -For full documentation see `src/MeshWeaver.Documentation/Data/Architecture/DataAccessPatterns.md`. - -## Development Patterns - -### Adding New Layout Areas -```csharp -public static class MyLayoutArea -{ - public static void AddMyLayoutArea(this LayoutConfiguration config) => - config.AddLayoutArea(nameof(MyLayout), MyLayout); - - public static UiControl MyLayout(LayoutAreaHost host, RenderingContext ctx) => - Controls.Stack - .WithView(Controls.Html("Some text") - .WithView(Controls.Markdown("Some markdown view")) - ); - -} -``` -We support rich markdown with mermaid diagrams, code blocks, MathJax, -and live execution via dynamic markdown. Layout areas can be inserted by -using `@{address}/{areaName}/{areaId}` - -### Message Handling -Messages are registered in the configuration of the hub. Also DI is set up on the level of hub configuration: -```csharp -public static class NorthwindHubConfiguration -{ - public static MessageHubConfiguration AddNorthwindHub(this MessageHubConfiguration config) - { - return config.AddHandler(HandleMyRequestAsync) - .AddHandler(HandleMyRequest); - - } - - public static async Task HandleMyRequestAsync(MessageHub hub, IMessageDelivery request, CancellationToken ct) - { - // Process the request - var result = await SomeService.ProcessAsync(request.Message); - - // Send response - await hub.Post(new MyResponse(result), o => o.ResponseFor(request)); - return request.Processed(); - } - - public static IMessageDelivery HandleMyRequest(MessageHub hub, IMessageDelivery request) - { - // Process the request - var result = SomeService.Process(request.Input); - - // Send response - hub.Post(new MyResponse(result), o => o.ResponseFor(request)); - return request.Processed(); - } -} -``` - -### AI Plugin Development -```csharp -public class MyPlugin(IMessageHub hub, IAgentChat chat) -{ - [Description("Description on how to use")] - public async Task DoSomething([Description("Description for input")]string input) - { - var request = new MyRequest(input); // Create a request object - var address = GetAddress(request); // Get the address for the plugin, e.g., "app/northwind" - // Use the message hub to send a request and receive a response - var response = await hub.AwaitResponse(request, o => o.WithTarget(address)); - return JsonSerializer.Serialize(response.Message, hub.JsonSerializationOptions); - } - - public Address GetAddress(MyRequest request) - { - // Logic to determine the address based on the request - // the chat contains a context, which is usually good to use. - // can also contain agent specific mapping logic. - return chat.Context.Address; - } -} -``` - -## Key Dependencies - -- **.NET 10.0** - Target framework -- **Orleans** - Distributed deployment (distributed deployment, microservices) -- **Blazor Server** - Web UI framework -- **Microsoft.Extensions.AI** - AI integration -- **xUnit v3** - Testing framework -- **FluentAssertions** - Test assertions -- **Chart.js** - Data visualization -- **Azure SDKs** - Cloud integration -- **Markdig** - Markdown processing - - -## Testing Guidelines - -Tests use xUnit v3 with structured logging and test parallelization configured via `xunit.runner.json`: -- `parallelizeAssembly: false` -- `parallelizeTestCollections: false` -- `maxParallelThreads: 1` -- `methodTimeout: 60000ms` (1 minute per test method) - -**No mocking.** Tests that need infrastructure (persistence, messaging, DI) must use `MonolithMeshTestBase` or `OrleansTestBase` — never mock `IMessageHub`, `IMeshService`, or other core interfaces. - -### Satellite Entity Patterns - -For implementing and testing satellite entities (comments, threads, tracked changes), see `src/MeshWeaver.Documentation/Data/Architecture/SatelliteEntityPatterns.md`. - -**Key rules:** -- Handler must be synchronous (`IMessageDelivery`, not `async Task`) -- Use `meshService.CreateNode()` (Observable) + `.Subscribe(onNext, onError)` — never `await` -- Use `workspace.UpdateMeshNode()` for parent node content updates (in-memory, persisted via debounce) -- Post response inside the `Subscribe(onNext)` callback, not before -- Orleans tests: client configurator must call `AddGraph()` for type registry alignment -- Verify via `GetDataRequest` or `GetRemoteStream` — never `QueryAsync` in distributed tests - -### Running Tests - -Run tests from the root directory using sub-paths. Do NOT write output to `/tmp` or temp directories — test results (.trx) are automatically collected in the project's `bin/` directory. - -**CRITICAL: Always use `run_in_background: true`** for test runs. Tests can take minutes — never block the conversation waiting for them. Use `timeout: 180000` (3 min) max for Bash test commands. The xunit.runner.json `methodTimeout` is 60000ms (1 min) per test method. - -**Do NOT use `--verbosity minimal`** (or `-v m`) when tests are expected to fail. Minimal verbosity hides error details (stack traces, assertion messages), forcing you to re-run with normal verbosity — wasting time and frustrating the user. Use default verbosity or `--verbosity normal` so failures are visible on the first run. Only use `--verbosity minimal` when you are confident all tests will pass and just need a quick green/red check. - -```bash -# Run from root directory with sub-path -dotnet test test/MeshWeaver.Hosting.Monolith.Test --no-restore - -# Run a specific test project -dotnet test test/MeshWeaver.Graph.Test --no-restore - -# Filter to specific tests -dotnet test test/MeshWeaver.Graph.Test --filter "ClassName~AccessAssignment" --no-restore -``` - -**Workflow:** -1. Run tests **once** in background (`run_in_background: true`) -2. If failures: read the output to understand errors — do NOT re-run -3. Fix the code -4. Run tests **once** again to verify fixes -5. Repeat 2–4 until green - -### DevLogin and Access Control in Tests - -`MonolithMeshTestBase` automatically logs in `rbuergi@systemorph.com` as Admin via `TestUsers.DevLogin(Mesh)` in `InitializeAsync()`. This means all tests start with a logged-in admin user — no manual setup needed for basic CRUD. - -**TestUsers** (`MeshWeaver.Hosting.Monolith.TestBase.TestUsers`): -- `TestUsers.Admin` — default admin AccessContext -- `TestUsers.SampleUsers()` — MeshNode array of sample users from `samples/Graph/Data/User/` -- `TestUsers.DevLogin(mesh)` — logs in the admin user (called automatically by base class) -- `builder.AddSampleUsers()` — extension to pre-seed user MeshNodes in `ConfigureMesh` - -When tests with `AddRowLevelSecurity()` need **per-user** access control (e.g., testing that User1 can't see User2's data), use explicit admin setup for data creation: - -```csharp -// Before creating test data: set up admin context -var accessService = Mesh.ServiceProvider.GetRequiredService(); -var securityService = Mesh.ServiceProvider.GetRequiredService(); -await securityService.AddUserRoleAsync("setup-admin", "Admin", null, "system"); -accessService.SetCircuitContext(new AccessContext { ObjectId = "setup-admin", Name = "Setup Admin" }); - -// ... create test nodes ... - -// After setup: clear admin context so tests start clean -accessService.SetCircuitContext(null); -``` - -### Node Types - -Only use **registered** node types in tests. Standard types registered by `AddGraph()`: -`Markdown`, `Code`, `Agent`, `Group`, `User`, `VUser`, `Role`, `Notification`, `Approval`, `AccessAssignment`, `GroupMembership`, `PartitionAccessPolicy`, `ActivityLog`, `UserActivity`, `Comment`, `Thread`, `ThreadMessage` - -Custom types can be registered via `builder.AddMeshNodes(new MeshNode("MyType") { Name = "My Type" })` in `ConfigureMesh`. - -### MonolithMeshTestBase (recommended for most tests) - -Reference `MeshWeaver.Hosting.Monolith.TestBase` and inherit from `MonolithMeshTestBase`: - -```csharp -public class MyTest(ITestOutputHelper output) : MonolithMeshTestBase(output) -{ - // Override ConfigureMesh to add services and sample users - protected override MeshBuilder ConfigureMesh(MeshBuilder builder) - => base.ConfigureMesh(builder) - .AddGraph() - .AddSampleUsers() - .ConfigureHub(hub => hub.AddMyHub()); - - [Fact] - public async Task MyTestMethod() - { - var meshQuery = Mesh.ServiceProvider.GetRequiredService(); - var nodeFactory = Mesh.ServiceProvider.GetRequiredService(); - - // Create test data - await nodeFactory.CreateNodeAsync(new MeshNode("test", "Namespace") { Name = "Test" }, "testuser"); - - // Query - var result = await meshQuery.QueryAsync("path:Namespace/test").FirstOrDefaultAsync(); - result.Should().NotBeNull(); - } -} -``` - -### HubTestBase (for message routing / layout tests) - -```csharp -public class MyTest : HubTestBase, IAsyncLifetime -{ - protected override MessageHubConfiguration ConfigureHost(MessageHubConfiguration config) - => base.ConfigureHost(config).AddNorthwindHub(); - - protected override MessageHubConfiguration ConfigureClient(MessageHubConfiguration config) - => base.ConfigureClient(config).AddLayoutClient(); - - [Fact] - public async Task MyTestMethod() - { - var hub = GetClient(); - var response = await hub.AwaitResponse(request, o => o.WithTarget(new HostAddress())); - response.Should().NotBeNull(); - } -} -``` - -## Project Structure Guidelines - -- Framework code belongs in `src/` -- Test code belongs in `test/` -- Sample applications go in `samples/` -- Each module should have its own set of hubs and address spaces (e.g., `@app/northwind`) -- UI components should be framework-agnostic in the layout layer. The language are the controls inheriting from `UiControl`. -- AI agents should use plugins to access application functionality - -## Solution Management - -The solution uses centralized package management via `Directory.Packages.props`. When adding new dependencies, update the central package file rather than individual project files. - -### Key Configuration Files -- `Directory.Build.props` - Global MSBuild properties and versioning -- `Directory.Packages.props` - Centralized NuGet package version management -- `nuget.config` - NuGet package sources configuration -- `xunit.runner.json` - Test execution configuration - -### Branch and Development -- Main branch: `main` (use for PRs) -- Solution file: `MeshWeaver.slnx` contains 50+ projects +@AGENTS.md \ No newline at end of file diff --git a/Directory.Build.props b/Directory.Build.props index 825857cef..08727f16f 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -9,18 +9,85 @@ Systemorph MIT https://github.com/Systemorph/MeshWeaver - 649;CA2255;NU5104;$(NoWarn) + + 649;CA2255;NU5104;CS1591;CS1573;CS1712;$(NoWarn) + + true $(DefineConstants);CIRun true + + 3.0.0 + + + + + <_SecondsToday>$([System.DateTime]::UtcNow.TimeOfDay.TotalSeconds.ToString("F0")) + <_BuildNumber Condition="'$(CIRun)' == 'true'">$([MSBuild]::Divide($(_SecondsToday), 2).ToString("F0")) + <_BuildNumber Condition="'$(CIRun)' != 'true'">0 + + <_VersionDash>$(PlatformVersion.IndexOf('-')) + <_VersionNumeric Condition="'$(_VersionDash)' == '-1'">$(PlatformVersion) + <_VersionNumeric Condition="'$(_VersionDash)' != '-1'">$(PlatformVersion.Substring(0, $(_VersionDash))) + + <_CiSep Condition="'$(_VersionDash)' == '-1'">- + <_CiSep Condition="'$(_VersionDash)' != '-1'">. + $(PlatformVersion) + $(PlatformVersion)$(_CiSep)ci.$(_BuildNumber) + + $(Version)+build.$([System.DateTime]::UtcNow.Ticks) + $(Version) + + $(_VersionNumeric).$(_BuildNumber) + $(_VersionNumeric).$(_BuildNumber) - + opt out of CPM and don't need these packages. --> + + diff --git a/Directory.Packages.props b/Directory.Packages.props index 3d6884b08..7b4e840ca 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,40 +4,43 @@ $(NoWarn);NU1608 - + - - - - - - - - - - - - + + + + + + + + + + + + + + + - - + + - + - + - - + + - - - + + + - - + + @@ -45,141 +48,150 @@ - - + + - + - - + + + + - - - - + + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + - - + + + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + + - + - + - - + + - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - + + + - - - - - - + + + + + + + + + + + - + - - - - - - + + + + + + + + - - - - - - - + + + + + + + - - - - - - + + + + + + + - + - - - + + + - \ No newline at end of file + + diff --git a/Doc/Architecture/DebuggingPostgres.md b/Doc/Architecture/DebuggingPostgres.md new file mode 100644 index 000000000..8129b816f --- /dev/null +++ b/Doc/Architecture/DebuggingPostgres.md @@ -0,0 +1,137 @@ +--- +Name: Debugging Postgres in Prod / Test +Category: Documentation +Description: How to connect to the Aspire-deployed Azure Postgres Flexible Server using Azure AD auth, run ad-hoc queries, and inspect migration state. +Icon: +--- + +# Connecting to Aspire's Azure Postgres + +The deployed clusters (`prod`, `test`) provision **Azure Postgres Flexible Server with password auth disabled** — only Azure AD entra-id auth is accepted. Password from `dotnet user-secrets` is for legacy/break-glass and **does not work**. + +## Quick connect + +| Mode | FQDN | +|---|---| +| prod | `memexpostgres-d272wxvys4nvo.postgres.database.azure.com` | +| test | look up: `az postgres flexible-server list -g test-memex --query "[].fullyQualifiedDomainName" -o tsv` | + +```bash +# Verify which AAD identity you're signed in as — your user must be granted +# Postgres AAD admin (or be a member of an AAD group that is). +az account show --query "user.name" -o tsv + +# 1-shot token good for ~1 hour +PGPASSWORD=$(az account get-access-token \ + --resource-type oss-rdbms --query accessToken -o tsv) + +psql "host=memexpostgres-d272wxvys4nvo.postgres.database.azure.com \ + port=5432 dbname=memex user=$(az account show --query user.name -o tsv) \ + sslmode=require" +``` + +The `oss-rdbms` token resource maps to `https://ossrdbms-aad.database.windows.net/.default`. SSL is mandatory. + +If `psql` is not installed: `winget install PostgreSQL.PostgreSQL` (Windows) — picks up `psql.exe` on `PATH`. Or use the C# script below. + +## C# alternative (no psql install needed) + +Drop a script under `tools/`: + +```csharp +#r "nuget: Npgsql, 9.0.2" +#r "nuget: Azure.Identity, 1.13.1" +using Azure.Core; +using Azure.Identity; +using Npgsql; + +const string Host = "memexpostgres-d272wxvys4nvo.postgres.database.azure.com"; +const string Db = "memex"; +const string User = "rbuergi@systemorph.com"; // your AAD UPN + +var token = await new DefaultAzureCredential().GetTokenAsync( + new TokenRequestContext(new[] { "https://ossrdbms-aad.database.windows.net/.default" })); +await using var conn = new NpgsqlConnection( + $"Host={Host};Database={Db};Username={User};Password={token.Token};SSL Mode=Require"); +await conn.OpenAsync(); + +await using var cmd = new NpgsqlCommand("SELECT current_user, version()", conn); +await using var rdr = await cmd.ExecuteReaderAsync(); +while (await rdr.ReadAsync()) + Console.WriteLine($"{rdr[0]} {rdr[1]}"); +``` + +Run with `dotnet script tools/your-query.csx` (one-time `dotnet tool install -g dotnet-script` if missing). + +There's a worked example at `tools/check-prod-db.csx` — uses the same pattern to run a battery of diagnostic queries (migration version, per-user schemas, access assignments, thread distribution). + +## Cheat sheet for migration / partition state + +```sql +-- 1. What migration version did the runner reach? +SELECT id, content + FROM admin.mesh_nodes + WHERE id = 'db_version'; + +-- 2. Per-user / per-org content schemas (post-V10 layout) +SELECT schema_name FROM information_schema.schemata s + WHERE EXISTS (SELECT 1 FROM information_schema.tables t + WHERE t.table_schema = s.schema_name AND t.table_name='mesh_nodes') + AND s.schema_name NOT IN ('public','admin','information_schema','pg_catalog','pg_toast','user') + AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' + ORDER BY schema_name; + +-- 3. Where do AccessAssignments live for a given user? +SELECT 'user' AS schema, namespace, content + FROM "user".access WHERE content->>'accessObject' = 'rbuergi' +UNION ALL +SELECT 'partnerre' AS schema, namespace, content + FROM partnerre.access WHERE content->>'accessObject' = 'rbuergi'; + +-- 4. Cross-schema search for a node by id (use when "where does X live?" is the question) +DO $$ +DECLARE r RECORD; +BEGIN + FOR r IN SELECT schema_name FROM information_schema.schemata s + WHERE EXISTS (SELECT 1 FROM information_schema.tables t + WHERE t.table_schema = s.schema_name AND t.table_name='mesh_nodes') + AND s.schema_name NOT IN ('information_schema','pg_catalog','pg_toast','public') + LOOP + EXECUTE format( + 'SELECT %L AS schema, id, namespace, node_type FROM %I.mesh_nodes WHERE id = ''loss-model''', + r.schema_name, r.schema_name); + END LOOP; +END $$; +``` + +## Reading migration logs + +The migration runs as an Aspire `db-migration` resource that completes **before** the portal starts. Logs are in Container Apps: + +```bash +az containerapp logs show -n db-migration -g prod-memex --tail 200 +# follow live: +az containerapp logs show -n db-migration -g prod-memex --follow +``` + +If migration crashed mid-run, you'll see the `Unhandled exception` at the bottom and the partial schema state in the DB. The `db_version` row is only written **after** all migrations complete cleanly — so a missing `db_version` plus a non-empty schema set means the runner crashed mid-flight. + +## Common failure modes + +- **`28000: no pg_hba.conf entry for host … user "app"`** — the migration code built an `NpgsqlDataSource` from the raw connection string instead of going through the Aspire-configured Azure-AD password provider. Every per-schema datasource (e.g. `SchemaHelpers.BuildSchemaDataSource`) needs the same AAD token-acquisition hook the main runner uses. Fix the helper to wire `dsb.UsePeriodicPasswordProvider(...)` instead of `dsb.Build()` directly. +- **Migration aborts mid-run, `db_version` missing** — see above. The runner only persists `db_version` after the loop completes; a single failed `Vxx` leaves the version unchanged. After fixing the underlying issue, the runner re-runs every migration `> 0` (i.e., everything) on next deploy. +- **AAD token expired during long migration** — `az account get-access-token` issues a token with ~1h lifetime. Migrations that exceed that need `UsePeriodicPasswordProvider` (refreshes automatically) rather than a one-shot password. The Aspire `AddAzureNpgsqlDataSource` already does this for the main connection. +- **Wrong AAD identity** — token is for whoever `az login` was last run as. If your user isn't a Postgres AAD admin, `28000` again. Add via portal: *Azure Database for PostgreSQL → Authentication → Add Microsoft Entra admin*. + +## Where the prod DB lives + +| Resource | Value | +|---|---| +| Resource Group | `prod-memex` | +| Server | `memexpostgres-d272wxvys4nvo.postgres.database.azure.com` | +| Database | `memex` | +| Auth | Azure AD only (password disabled) | +| Tenant | `3a01d7ac-3330-444d-942d-975eb491b5d6` | +| App Insights | `appinsights-d272wxvys4nvo` (same RG; for portal logs) | + +For test cluster, swap `prod-memex` → `test-memex` and discover the FQDN with the `az postgres flexible-server list` command above. diff --git a/MeshWeaver.slnx b/MeshWeaver.slnx index 52e80bbad..045448c03 100644 --- a/MeshWeaver.slnx +++ b/MeshWeaver.slnx @@ -29,6 +29,7 @@ + @@ -36,7 +37,7 @@ - + @@ -45,7 +46,7 @@ - + @@ -60,6 +61,9 @@ + + + @@ -72,8 +76,10 @@ + + @@ -93,9 +99,12 @@ + + + @@ -104,9 +113,13 @@ + + + + @@ -114,29 +127,29 @@ - - + - + + - + @@ -145,13 +158,14 @@ + + - diff --git a/Readme.md b/Readme.md index 9a38703a9..15fd039ab 100644 --- a/Readme.md +++ b/Readme.md @@ -4,23 +4,21 @@ MeshWeaver is a modular framework for building data-driven applications with rea ## Getting Started -You can run MeshWeaver in two modes: +You can run the Memex portal in two modes: ### Monolithic Setup ```bash -cd portal/MeshWeaver.Portal -dotnet run +dotnet run --project memex/Memex.Portal.Monolith ``` -This setup is useful for smaller projects which are deployed as monoliths. If you are unsure which approach to pick, pick this one. +Runs at `https://localhost:7122` (HTTP fallback `http://localhost:5022`). This setup is useful for smaller projects deployed as monoliths. If you are unsure which approach to pick, pick this one. ### Microservices Setup (with .NET Aspire) ```bash -cd portal/aspire/MeshWeaver.Portal.AppHost -dotnet run +dotnet run --project memex/aspire/Memex.AppHost ``` -Please note that this approach requires running docker. Microservices are generally more complex to handle, but they provide big flexibility running in productive setups. +Runs the portal at `https://localhost:7202` (HTTP fallback `http://localhost:5202`). This approach requires Docker. Microservices are more complex to operate but provide the flexibility needed for production setups. ## Creating New Projects @@ -173,9 +171,9 @@ The backbone of MeshWeaver is its message hub system (`MeshWeaver.Messaging.Hub` - Built-in dependency injection ### Data Processing -- **Messaging**: Send and received messages between addresses. Route them inside the mesh. +- **Messaging**: Send and receive messages between addresses. Route them inside the mesh. - **Concurrency**: Fully asynchronous concurrency using the actor model. -- **Data Synchronization**: Full-fledged data replication for Create Read Update Delete. +- **Data Synchronization**: Full-fledged data replication for Create, Read, Update, Delete. - **Business Rules**: Rule engine with scope-based state management - **Import**: Flexible data import system with activity tracking @@ -185,11 +183,11 @@ The backbone of MeshWeaver is its message hub system (`MeshWeaver.Messaging.Hub` ### UI and Visualization - **Layout**: Framework-agnostic UI control abstractions -- **Reporting**: Fexible and interactive reporting +- **Reporting**: Flexible and interactive reporting ### Flexible deployment options - **Elasticity**: Create a fully elastic setup using Orleans -- **Integration**: Integrate with almost any available technology through Aspire. +- **Integration**: Integrate with almost any available technology through Aspire. ## Resources @@ -197,22 +195,25 @@ The backbone of MeshWeaver is its message hub system (`MeshWeaver.Messaging.Hub` - [Website](https://meshweaver.cloud) - Learn more about MeshWeaver - [Discord](https://discord.gg/ACSYBWPy) - Join our community -## Architecture +## Deployment Options - -### Deployment Options - -1. **Monolithic** (`portal/MeshWeaver.Portal`) - - Single process deployment +1. **Monolithic** (`memex/Memex.Portal.Monolith`) + - Single-process deployment - Simplified setup - Suitable for development and smaller deployments -2. **Microservices** (`portal/aspire/MeshWeaver.Portal.AppHost`) +2. **Microservices** (`memex/aspire/Memex.AppHost`) - .NET Aspire-based orchestration - Service discovery - - Azure integration - PostgreSQL for persistence - - Azure Blob Storage for articles + - Filesystem or Azure Blob Storage backends + +Production deployment recipes live under [`deploy/`](deploy/): + +- [`deploy/helm`](deploy/helm) — generic Kubernetes/Helm chart (Azure-free self-host) +- [`deploy/aks`](deploy/aks) — production-grade AKS sample (private cluster, P2S VPN, ACR, pgBackRest PITR) +- [`deploy/aca`](deploy/aca) — Azure Container Apps (Bicep) +- [`deploy/compose`](deploy/compose) · [`deploy/compose-ha`](deploy/compose-ha) — Docker Compose (single-node / HA) ## Contributing diff --git a/deploy/.env.example b/deploy/.env.example new file mode 100644 index 000000000..983686787 --- /dev/null +++ b/deploy/.env.example @@ -0,0 +1,53 @@ +# Memex self-host environment surface (Docker Compose / any non-Azure host). +# Copy to `.env` next to your compose file and fill in. DO NOT commit your real `.env`. +# Config keys use the ASP.NET double-underscore form (Section__Key) so they map 1:1 to +# appsettings sections. The same keys flow through ACA env and ARM container env. + +# ---- Postgres (pgvector) — mesh data lives here in EVERY topology ---- +ConnectionStrings__memex=Host=postgres;Port=5432;Database=memex;Username=memex;Password=change-me + +# ---- Deployment backend: Azure-free self-host ---- +# Backend=Filesystem moves object storage / NodeType compile cache / NuGet cache / +# DataProtection keys onto DataRoot (a local volume single-node; a shared NFS/CIFS +# volume in HA). Mesh data stays in Postgres regardless. +Deployment__Backend=Filesystem +Deployment__DataRoot=/data +# Orleans clustering: Localhost (single-node) | AdoNet (HA, Postgres) | AzureTables (ACA). +Deployment__Orleans__Clustering=Localhost + +# ---- Content storage on the filesystem ---- +Storage__Name=content +Storage__SourceType=FileSystem +Storage__BasePath=/data/content +Graph__Storage__Type=PostgreSql +Graph__Storage__BasePath=/data/graph + +# ---- Encryption master key for provider credentials (envelope encryption) ---- +# REQUIRED for production. Generate a base64 32-byte key, inject as a secret — never +# commit it. Leave blank only for throwaway/dev (then keys are stored as plaintext). +Ai__KeyProtection__MasterKey= + +# ---- Feature flags: declare which capabilities this deployment ships ---- +# All default to true (absent = current behaviour). Turn capabilities OFF explicitly; +# a disabled flag wins even if a key below is set. Startup warns if everything is off. +Features__Ai__Providers__Anthropic=true +Features__Ai__Providers__AzureFoundry=false +Features__Ai__Providers__AzureOpenAI=true +Features__Ai__Providers__OpenAI=false +# Co-hosted CLIs require the portal-ai image (binaries baked in) to actually run. +Features__Ai__Clis__ClaudeCode=false +Features__Ai__Clis__Copilot=false + +# ---- AI provider keys (bring your own; only flags that are ON consume these) ---- +Anthropic__ApiKey= +AzureOpenAI__ApiKey= +AzureOpenAI__Endpoint= + +# ---- Authentication ---- +# Dev login is fine for a local demo; for any real deployment set EnableDevLogin=false +# and configure an external provider (Microsoft/Google/Apple/LinkedIn) instead. +Authentication__EnableDevLogin=true + +# ---- MCP back-connection: the portal's own externally reachable base URL ---- +# Used to compose {Mcp__BaseUrl}/mcp for the co-hosted CLIs' per-user MCP connection. +Mcp__BaseUrl=http://localhost:8080 diff --git a/deploy/Memex.Deploy.slnx b/deploy/Memex.Deploy.slnx new file mode 100644 index 000000000..f54f17a39 --- /dev/null +++ b/deploy/Memex.Deploy.slnx @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/deploy/aca/main.bicep b/deploy/aca/main.bicep new file mode 100644 index 000000000..357b16bc3 --- /dev/null +++ b/deploy/aca/main.bicep @@ -0,0 +1,40 @@ +targetScope = 'subscription' + +param resourceGroupName string + +param location string + +param principalId string + +resource rg 'Microsoft.Resources/resourceGroups@2023-07-01' = { + name: resourceGroupName + location: location +} + +module memex_aca_acr 'memex-aca-acr/memex-aca-acr.bicep' = { + name: 'memex-aca-acr' + scope: rg + params: { + location: location + } +} + +module memex_aca 'memex-aca/memex-aca.bicep' = { + name: 'memex-aca' + scope: rg + params: { + location: location + memex_aca_acr_outputs_name: memex_aca_acr.outputs.name + userPrincipalId: principalId + } +} + +output memex_aca_AZURE_CONTAINER_APPS_ENVIRONMENT_DEFAULT_DOMAIN string = memex_aca.outputs.AZURE_CONTAINER_APPS_ENVIRONMENT_DEFAULT_DOMAIN + +output memex_aca_AZURE_CONTAINER_APPS_ENVIRONMENT_ID string = memex_aca.outputs.AZURE_CONTAINER_APPS_ENVIRONMENT_ID + +output memex_aca_volumes_memex_postgres_0 string = memex_aca.outputs.volumes_memex_postgres_0 + +output memex_aca_volumes_memex_portal_0 string = memex_aca.outputs.volumes_memex_portal_0 + +output memex_aca_volumes_memex_portal_1 string = memex_aca.outputs.volumes_memex_portal_1 \ No newline at end of file diff --git a/deploy/aca/memex-aca-acr/memex-aca-acr.bicep b/deploy/aca/memex-aca-acr/memex-aca-acr.bicep new file mode 100644 index 000000000..706f49b43 --- /dev/null +++ b/deploy/aca/memex-aca-acr/memex-aca-acr.bicep @@ -0,0 +1,19 @@ +@description('The location for the resource(s) to be deployed.') +param location string = resourceGroup().location + +resource memex_aca_acr 'Microsoft.ContainerRegistry/registries@2025-04-01' = { + name: take('memexacaacr${uniqueString(resourceGroup().id)}', 50) + location: location + sku: { + name: 'Basic' + } + tags: { + 'aspire-resource-name': 'memex-aca-acr' + } +} + +output name string = memex_aca_acr.name + +output loginServer string = memex_aca_acr.properties.loginServer + +output id string = memex_aca_acr.id \ No newline at end of file diff --git a/deploy/aca/memex-aca/memex-aca.bicep b/deploy/aca/memex-aca/memex-aca.bicep new file mode 100644 index 000000000..26be33bed --- /dev/null +++ b/deploy/aca/memex-aca/memex-aca.bicep @@ -0,0 +1,175 @@ +@description('The location for the resource(s) to be deployed.') +param location string = resourceGroup().location + +param userPrincipalId string = '' + +param tags object = { } + +param memex_aca_acr_outputs_name string + +resource memex_aca_mi 'Microsoft.ManagedIdentity/userAssignedIdentities@2024-11-30' = { + name: take('memex_aca_mi-${uniqueString(resourceGroup().id)}', 128) + location: location + tags: tags +} + +resource memex_aca_acr 'Microsoft.ContainerRegistry/registries@2025-04-01' existing = { + name: memex_aca_acr_outputs_name +} + +resource memex_aca_acr_memex_aca_mi_AcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(memex_aca_acr.id, memex_aca_mi.id, subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d')) + properties: { + principalId: memex_aca_mi.properties.principalId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d') + principalType: 'ServicePrincipal' + } + scope: memex_aca_acr +} + +resource memex_aca_law 'Microsoft.OperationalInsights/workspaces@2025-02-01' = { + name: take('memexacalaw-${uniqueString(resourceGroup().id)}', 63) + location: location + properties: { + sku: { + name: 'PerGB2018' + } + } + tags: tags +} + +resource memex_aca 'Microsoft.App/managedEnvironments@2025-07-01' = { + name: take('memexaca${uniqueString(resourceGroup().id)}', 24) + location: location + properties: { + appLogsConfiguration: { + destination: 'log-analytics' + logAnalyticsConfiguration: { + customerId: memex_aca_law.properties.customerId + sharedKey: memex_aca_law.listKeys().primarySharedKey + } + } + workloadProfiles: [ + { + name: 'consumption' + workloadProfileType: 'Consumption' + } + ] + } + tags: tags +} + +resource aspireDashboard 'Microsoft.App/managedEnvironments/dotNetComponents@2025-10-02-preview' = { + name: 'aspire-dashboard' + properties: { + componentType: 'AspireDashboard' + } + parent: memex_aca +} + +resource memex_aca_storageVolume 'Microsoft.Storage/storageAccounts@2024-01-01' = { + name: take('memexacastoragevolume${uniqueString(resourceGroup().id)}', 24) + kind: 'StorageV2' + location: location + sku: { + name: 'Standard_LRS' + } + properties: { + largeFileSharesState: 'Enabled' + minimumTlsVersion: 'TLS1_2' + } + tags: tags +} + +resource storageVolumeFileService 'Microsoft.Storage/storageAccounts/fileServices@2024-01-01' = { + name: 'default' + parent: memex_aca_storageVolume +} + +resource shares_volumes_memex_postgres_0 'Microsoft.Storage/storageAccounts/fileServices/shares@2024-01-01' = { + name: take('sharesvolumesmemexpostgres0-${uniqueString(resourceGroup().id)}', 63) + properties: { + enabledProtocols: 'SMB' + shareQuota: 1024 + } + parent: storageVolumeFileService +} + +resource managedStorage_volumes_memex_postgres_0 'Microsoft.App/managedEnvironments/storages@2025-07-01' = { + name: take('managedstoragevolumesmemexpostgres${uniqueString(resourceGroup().id)}', 24) + properties: { + azureFile: { + accountName: memex_aca_storageVolume.name + accountKey: memex_aca_storageVolume.listKeys().keys[0].value + accessMode: 'ReadWrite' + shareName: shares_volumes_memex_postgres_0.name + } + } + parent: memex_aca +} + +resource shares_volumes_memex_portal_0 'Microsoft.Storage/storageAccounts/fileServices/shares@2024-01-01' = { + name: take('sharesvolumesmemexportal0-${uniqueString(resourceGroup().id)}', 63) + properties: { + enabledProtocols: 'SMB' + shareQuota: 1024 + } + parent: storageVolumeFileService +} + +resource managedStorage_volumes_memex_portal_0 'Microsoft.App/managedEnvironments/storages@2025-07-01' = { + name: take('managedstoragevolumesmemexportal${uniqueString(resourceGroup().id)}', 24) + properties: { + azureFile: { + accountName: memex_aca_storageVolume.name + accountKey: memex_aca_storageVolume.listKeys().keys[0].value + accessMode: 'ReadWrite' + shareName: shares_volumes_memex_portal_0.name + } + } + parent: memex_aca +} + +resource shares_volumes_memex_portal_1 'Microsoft.Storage/storageAccounts/fileServices/shares@2024-01-01' = { + name: take('sharesvolumesmemexportal1-${uniqueString(resourceGroup().id)}', 63) + properties: { + enabledProtocols: 'SMB' + shareQuota: 1024 + } + parent: storageVolumeFileService +} + +resource managedStorage_volumes_memex_portal_1 'Microsoft.App/managedEnvironments/storages@2025-07-01' = { + name: take('managedstoragevolumesmemexportal${uniqueString(resourceGroup().id)}', 24) + properties: { + azureFile: { + accountName: memex_aca_storageVolume.name + accountKey: memex_aca_storageVolume.listKeys().keys[0].value + accessMode: 'ReadWrite' + shareName: shares_volumes_memex_portal_1.name + } + } + parent: memex_aca +} + +output volumes_memex_postgres_0 string = managedStorage_volumes_memex_postgres_0.name + +output volumes_memex_portal_0 string = managedStorage_volumes_memex_portal_0.name + +output volumes_memex_portal_1 string = managedStorage_volumes_memex_portal_1.name + +output AZURE_LOG_ANALYTICS_WORKSPACE_NAME string = memex_aca_law.name + +output AZURE_LOG_ANALYTICS_WORKSPACE_ID string = memex_aca_law.id + +output AZURE_CONTAINER_REGISTRY_NAME string = memex_aca_acr.name + +output AZURE_CONTAINER_REGISTRY_ENDPOINT string = memex_aca_acr.properties.loginServer + +output AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID string = memex_aca_mi.id + +output AZURE_CONTAINER_APPS_ENVIRONMENT_NAME string = memex_aca.name + +output AZURE_CONTAINER_APPS_ENVIRONMENT_ID string = memex_aca.id + +output AZURE_CONTAINER_APPS_ENVIRONMENT_DEFAULT_DOMAIN string = memex_aca.properties.defaultDomain \ No newline at end of file diff --git a/deploy/aca/memex-migration/memex-migration.bicep b/deploy/aca/memex-migration/memex-migration.bicep new file mode 100644 index 000000000..a919cb768 --- /dev/null +++ b/deploy/aca/memex-migration/memex-migration.bicep @@ -0,0 +1,79 @@ +@description('The location for the resource(s) to be deployed.') +param location string = resourceGroup().location + +param memex_aca_outputs_azure_container_apps_environment_default_domain string + +param memex_aca_outputs_azure_container_apps_environment_id string + +@secure() +param memex_postgres_password_value string + +resource memex_migration 'Microsoft.App/containerApps@2025-07-01' = { + name: 'memex-migration' + location: location + properties: { + configuration: { + secrets: [ + { + name: 'connectionstrings--memex' + value: 'Host=memex-postgres;Port=5432;Username=postgres;Password=${memex_postgres_password_value};Database=memex' + } + { + name: 'memex-password' + value: memex_postgres_password_value + } + { + name: 'memex-uri' + value: 'postgresql://postgres:${uriComponent(memex_postgres_password_value)}@memex-postgres:5432/memex' + } + ] + activeRevisionsMode: 'Single' + } + environmentId: memex_aca_outputs_azure_container_apps_environment_id + template: { + containers: [ + { + image: 'ghcr.io/systemorph/memex-migration:latest' + name: 'memex-migration' + env: [ + { + name: 'ConnectionStrings__memex' + secretRef: 'connectionstrings--memex' + } + { + name: 'MEMEX_HOST' + value: 'memex-postgres' + } + { + name: 'MEMEX_PORT' + value: '5432' + } + { + name: 'MEMEX_USERNAME' + value: 'postgres' + } + { + name: 'MEMEX_PASSWORD' + secretRef: 'memex-password' + } + { + name: 'MEMEX_URI' + secretRef: 'memex-uri' + } + { + name: 'MEMEX_JDBCCONNECTIONSTRING' + value: 'jdbc:postgresql://memex-postgres:5432/memex' + } + { + name: 'MEMEX_DATABASENAME' + value: 'memex' + } + ] + } + ] + scale: { + minReplicas: 1 + } + } + } +} \ No newline at end of file diff --git a/deploy/aca/memex-portal/memex-portal.bicep b/deploy/aca/memex-portal/memex-portal.bicep new file mode 100644 index 000000000..4f99ae11e --- /dev/null +++ b/deploy/aca/memex-portal/memex-portal.bicep @@ -0,0 +1,150 @@ +@description('The location for the resource(s) to be deployed.') +param location string = resourceGroup().location + +param memex_aca_outputs_azure_container_apps_environment_default_domain string + +param memex_aca_outputs_azure_container_apps_environment_id string + +@secure() +param memex_postgres_password_value string + +param memex_aca_outputs_volumes_memex_portal_0 string + +param memex_aca_outputs_volumes_memex_portal_1 string + +resource memex_portal 'Microsoft.App/containerApps@2025-07-01' = { + name: 'memex-portal' + location: location + properties: { + configuration: { + secrets: [ + { + name: 'connectionstrings--memex' + value: 'Host=memex-postgres;Port=5432;Username=postgres;Password=${memex_postgres_password_value};Database=memex' + } + { + name: 'memex-password' + value: memex_postgres_password_value + } + { + name: 'memex-uri' + value: 'postgresql://postgres:${uriComponent(memex_postgres_password_value)}@memex-postgres:5432/memex' + } + ] + activeRevisionsMode: 'Single' + ingress: { + external: true + targetPort: 8080 + transport: 'http' + } + } + environmentId: memex_aca_outputs_azure_container_apps_environment_id + template: { + containers: [ + { + image: 'ghcr.io/systemorph/memex-portal-ai:latest' + name: 'memex-portal' + env: [ + { + name: 'ConnectionStrings__memex' + secretRef: 'connectionstrings--memex' + } + { + name: 'MEMEX_HOST' + value: 'memex-postgres' + } + { + name: 'MEMEX_PORT' + value: '5432' + } + { + name: 'MEMEX_USERNAME' + value: 'postgres' + } + { + name: 'MEMEX_PASSWORD' + secretRef: 'memex-password' + } + { + name: 'MEMEX_URI' + secretRef: 'memex-uri' + } + { + name: 'MEMEX_JDBCCONNECTIONSTRING' + value: 'jdbc:postgresql://memex-postgres:5432/memex' + } + { + name: 'MEMEX_DATABASENAME' + value: 'memex' + } + { + name: 'ASPNETCORE_HTTP_PORTS' + value: '8080' + } + { + name: 'Deployment__Backend' + value: 'Filesystem' + } + { + name: 'Deployment__DataRoot' + value: '/data' + } + { + name: 'Deployment__Orleans__Clustering' + value: 'Localhost' + } + { + name: 'Storage__Name' + value: 'content' + } + { + name: 'Storage__SourceType' + value: 'FileSystem' + } + { + name: 'Storage__BasePath' + value: '/data/content' + } + { + name: 'Graph__Storage__Type' + value: 'PostgreSql' + } + { + name: 'Graph__Storage__BasePath' + value: '/data/graph' + } + { + name: 'Mcp__BaseUrl' + value: 'https://memex-portal.${memex_aca_outputs_azure_container_apps_environment_default_domain}' + } + ] + volumeMounts: [ + { + volumeName: 'v0' + mountPath: '/data' + } + { + volumeName: 'v1' + mountPath: '/mnt/users' + } + ] + } + ] + scale: { + minReplicas: 1 + } + volumes: [ + { + name: 'v0' + storageType: 'AzureFile' + storageName: memex_aca_outputs_volumes_memex_portal_0 + } + { + name: 'v1' + storageType: 'AzureFile' + storageName: memex_aca_outputs_volumes_memex_portal_1 + } + ] + } + } +} \ No newline at end of file diff --git a/deploy/aca/memex-postgres/memex-postgres.bicep b/deploy/aca/memex-postgres/memex-postgres.bicep new file mode 100644 index 000000000..92188d50c --- /dev/null +++ b/deploy/aca/memex-postgres/memex-postgres.bicep @@ -0,0 +1,75 @@ +@description('The location for the resource(s) to be deployed.') +param location string = resourceGroup().location + +param memex_aca_outputs_azure_container_apps_environment_default_domain string + +param memex_aca_outputs_azure_container_apps_environment_id string + +@secure() +param memex_postgres_password_value string + +param memex_aca_outputs_volumes_memex_postgres_0 string + +resource memex_postgres 'Microsoft.App/containerApps@2025-07-01' = { + name: 'memex-postgres' + location: location + properties: { + configuration: { + secrets: [ + { + name: 'postgres-password' + value: memex_postgres_password_value + } + ] + activeRevisionsMode: 'Single' + ingress: { + external: false + targetPort: 5432 + transport: 'tcp' + } + } + environmentId: memex_aca_outputs_azure_container_apps_environment_id + template: { + containers: [ + { + image: 'docker.io/pgvector/pgvector:pg17' + name: 'memex-postgres' + env: [ + { + name: 'POSTGRES_HOST_AUTH_METHOD' + value: 'scram-sha-256' + } + { + name: 'POSTGRES_INITDB_ARGS' + value: '--auth-host=scram-sha-256 --auth-local=scram-sha-256' + } + { + name: 'POSTGRES_USER' + value: 'postgres' + } + { + name: 'POSTGRES_PASSWORD' + secretRef: 'postgres-password' + } + ] + volumeMounts: [ + { + volumeName: 'v0' + mountPath: '/var/lib/postgresql/data' + } + ] + } + ] + scale: { + minReplicas: 1 + } + volumes: [ + { + name: 'v0' + storageType: 'AzureFile' + storageName: memex_aca_outputs_volumes_memex_postgres_0 + } + ] + } + } +} \ No newline at end of file diff --git a/deploy/aks/DEPLOY-RUNBOOK.md b/deploy/aks/DEPLOY-RUNBOOK.md new file mode 100644 index 000000000..df2743bd3 --- /dev/null +++ b/deploy/aks/DEPLOY-RUNBOOK.md @@ -0,0 +1,153 @@ +# Deploy Memex to AKS — the `memex.systemorph.com` runbook + +This is the **exact, verified** sequence used to bring up `https://memex.systemorph.com` on a +private AKS cluster in **swedencentral**. It is the reproducible template behind the sample in +this folder (`infra/` Bicep, `values.aks.yaml`, `manifests/`) and the image-based Aspire +AppHost at [`../aspire/Memex.Deploy.AppHost`](../aspire/Memex.Deploy.AppHost). + +> **Model:** one Aspire AppHost (`Memex.Deploy.AppHost`) models the workload from published +> images; the **Kubernetes publisher** generates the Helm chart (`../helm`); this folder adds the +> AKS *platform* (Bicep) + overlay. AKS is the deploy target. All config flows from deploy +> parameters → env. See "Why a runbook and not pure `aspire up`" at the bottom. + +Architecture decisions baked in (see the AGENTS memory + `../helm`): +- **Private** AKS API server + **private** Postgres Flexible Server; **only** the portal is public (`:443`). +- **One shared ACR** `meshweaver.azurecr.io` (RG `meshweaver-shared`) across all solutions. +- **Filesystem backend** with content on RWX **Azure Files** (`/mnt/content`); mesh data in Postgres. +- **Blazor sticky sessions** (cookie affinity = ACA's "bind tab to server"); **1 replica** today + (multi-replica needs Orleans `AzureTables` clustering — a follow-up). +- TLS via **cert-manager + Let's Encrypt** (HTTP-01). + +--- + +## 0. Prerequisites +- `az` ≥ 2.84 (logged in to the target subscription/tenant), `az bicep`, `docker`, .NET 10 SDK. +- A globally-unique shared ACR (here `meshweaver`). Create once: + `az group create -n meshweaver-shared -l swedencentral` ; + `az acr create -g meshweaver-shared -n meshweaver --sku Premium`. +- DNS zone for your domain in Azure DNS (here `systemorph.com`, RG `dns`). + +## 1. Build + push images to the shared ACR +```bash +# Base image (node + Claude Code + Copilot CLIs) — Linux builder => correct copilot-linux-x64 +az acr build --registry meshweaver --image memex-portal-ai-base:latest deploy/base-images/portal-ai +# App image — MUST pass -r linux-x64 (the Copilot SDK keys the CLI binary off the RID) +az acr login --name meshweaver +dotnet publish memex/aspire/Memex.Portal.Distributed/Memex.Portal.Distributed.csproj \ + -c Release -r linux-x64 --no-self-contained -t:PublishContainer -p:PublishProfile= \ + -p:ContainerRegistry=meshweaver.azurecr.io -p:ContainerRepository=memex-portal-ai \ + -p:ContainerImageTag=latest -p:ContainerBaseImage=meshweaver.azurecr.io/memex-portal-ai-base:latest +dotnet publish memex/aspire/Memex.Database.Migration/Memex.Database.Migration.csproj \ + -c Release -r linux-x64 --no-self-contained -t:PublishContainer -p:PublishProfile= \ + -p:ContainerRegistry=meshweaver.azurecr.io -p:ContainerRepository=memex-migration -p:ContainerImageTag=latest +``` + +## 2. Provision the AKS platform (Bicep) +Edit `infra/main.parameters.json` (region, node size/count within your vCPU quota — swedencentral +defaulted to 2× `Standard_D4s_v3` under a 10-vCPU cap). Then: +```bash +PG_PW="$(openssl rand -base64 24 | tr -dc 'A-Za-z0-9' | head -c 28)Aa1" # or your own +az deployment sub create --name memex-aks-infra-sc --location swedencentral \ + --template-file deploy/aks/infra/main.bicep \ + --parameters @deploy/aks/infra/main.parameters.json \ + --parameters postgresAdminPassword="$PG_PW" +``` +Outputs: cluster name, the Postgres FQDN, the shared-ACR login server. Grant the cluster kubelet +**AcrPull** on the shared ACR (cross-RG, so done out-of-band): +```bash +KUBELET=$(az aks show -g memex-aks-rg -n memexaks-cluster --query identityProfile.kubeletidentity.objectId -o tsv) +az role assignment create --assignee-object-id $KUBELET --assignee-principal-type ServicePrincipal \ + --role AcrPull --scope $(az acr show -n meshweaver --query id -o tsv) +``` +> Postgres connection uses the **private IP + password + SSL** (the FQDN would trip the portal's +> `database.azure.com` → Entra-token branch, which doesn't match a password server). Get it with: +> `az network private-dns record-set a list -g memex-aks-rg -z -o table`. + +## 3. External sign-in (OAuth) apps +- **Microsoft/Entra** (single-tenant home): + ```bash + az ad app create --display-name "Memex Portal (memex.systemorph.com)" --sign-in-audience AzureADMyOrg \ + --web-redirect-uris "https://memex.systemorph.com/signin-microsoft" + az ad app credential reset --id --display-name aks --years 1 # => client secret + ``` +- **Google** (Cloud Console) + **LinkedIn** (Developer portal): create web OAuth clients with + redirect URIs `https://memex.systemorph.com/signin-google` and `/signin-linkedin`. + +## 4. Deploy the workload (private cluster → `az aks command invoke`) +Copy `scripts/values.deploy.example.yaml` → `scripts/values.deploy.yaml`, fill in the **real** +connection string, master key, and OAuth secrets (keep it OUT of git — `artifacts/`/Key Vault), then: +```bash +az aks approuting enable -g memex-aks-rg -n memexaks-cluster # managed nginx (public LB) +cd deploy/aks/scripts +export MEMEX_PG_CONN='Host=;Port=5432;Username=memexadmin;Password=;Database=memex;SslMode=Require;Trust Server Certificate=true' +az aks command invoke -g memex-aks-rg -n memexaks-cluster --command "bash deploy.sh" --file . +``` +`deploy.sh` does: namespace + RWX PVCs → `helm upgrade --install` (chart + `values.aks.yaml` + +`values.deploy.yaml`) → scale the chart's in-cluster pg to 0 (we use the Flexible Server) → +`kubectl set image` to the shared ACR → patch the portal to 1 replica + the Azure Files mounts → +**patch the connection-string secret** (the generated chart hardcodes the in-cluster pg — known +chart-gen gap). **Observability is folded in:** export `GRAFANA_PW=...` alongside `MEMEX_PG_CONN` +and `deploy.sh` also brings up Grafana + Loki + Prometheus (see §6); omit it to skip monitoring. +At the model level, `AddMemex`'s `OtlpEndpoint` option wires `OTEL_EXPORTER_OTLP_ENDPOINT` for +OTLP traces/metrics (not needed for log shipping — Promtail scrapes stdout). + +## 5. Public ingress + TLS + DNS +```bash +IP=$(az aks command invoke -g memex-aks-rg -n memexaks-cluster \ + --command "kubectl get svc -n app-routing-system nginx -o jsonpath='{.status.loadBalancer.ingress[0].ip}'") +az network dns record-set a add-record -g dns -z systemorph.com -n memex --ipv4-address $IP --ttl 300 +cd deploy/aks/scripts +az aks command invoke -g memex-aks-rg -n memexaks-cluster --command "bash tls.sh" --file tls.sh # cert-manager + Let's Encrypt + ingress +``` +HTTP→HTTPS redirect is automatic once the ingress has TLS. Verify (bypassing DNS cache): +```bash +curl -sS -o /dev/null -w "%{http_code} verify=%{ssl_verify_result}\n" \ + --resolve memex.systemorph.com:443:$IP https://memex.systemorph.com/ +``` + +--- + +## 6. Observability (Grafana + Loki + Prometheus) + admin access via VPN +Everything except the portal stays private, so admin tools (Grafana, kubectl) go through the +**P2S VPN**, not a public endpoint. + +**Install the stack** (`scripts/install-observability.sh` — grafana/loki-stack: Loki + Promtail + +Grafana + Prometheus, datasources auto-wired, Promtail ships every pod's logs to Loki): +```bash +export GRAFANA_PW='' +cd deploy/aks/scripts +az aks command invoke -g memex-aks-rg -n memexaks-cluster \ + --command "GRAFANA_PW=$GRAFANA_PW bash install-observability.sh" --file install-observability.sh +``` + +**Set up the P2S VPN client** (the gateway + a root cert are provisioned by the Bicep + step 2): +```bash +# 1. Generate a P2S root+client cert (Windows) and upload the ROOT public cert to the gateway: +# $root = New-SelfSignedCertificate -Type Custom -KeySpec Signature -Subject "CN=MemexP2SRootCert" -KeyUsage CertSign -KeyExportPolicy Exportable -CertStoreLocation Cert:\CurrentUser\My -HashAlgorithm sha256 -KeyLength 2048 +# $client = New-SelfSignedCertificate -Type Custom -DnsName MemexP2SChild -KeySpec Signature -Subject "CN=MemexP2SChildCert" -Signer $root -KeyExportPolicy Exportable -CertStoreLocation Cert:\CurrentUser\My -HashAlgorithm sha256 -KeyLength 2048 -TextExtension @("2.5.29.37={text}1.3.6.1.5.5.7.3.2") +# [IO.File]::WriteAllText("root.txt",[Convert]::ToBase64String($root.RawData)) +# NOTE: this az version reads --public-cert-data as a FILE PATH, so pass the path (NOT the inline string, NOT @file): +# az network vnet-gateway root-cert create -g memex-aks-rg --gateway-name memexaks-vpngw --name MemexP2SRootCert --public-cert-data root.txt +# 2. Download + install the VPN client, then connect: +az network vnet-gateway vpn-client generate -g memex-aks-rg -n memexaks-vpngw -o tsv # -> download URL (zip) +# 3. With the VPN connected: +az aks get-credentials -g memex-aks-rg -n memexaks-cluster +kubectl -n monitoring port-forward svc/loki-grafana 3000:80 # http://localhost:3000 (admin / $GRAFANA_PW) +``` +In Grafana → Explore → Loki, the portal logs are `{namespace="memex"}` (e.g. add +`|= "error"` or `|~ "signin-microsoft"`). + +## Known gaps / follow-ups +- **Multi-replica HA**: needs Orleans `AzureTables` clustering wired on the Filesystem backend + (the portal currently registers the clustering table client only in the Azure-backend branch). +- **Chart connection string**: `../helm/templates/memex-portal/secrets.yaml` hardcodes the + in-cluster pg host/user — hence the post-install secret patch in `deploy.sh`. Fix at the + chart-generator (AddMemex) so an external connection string flows from values. +- **Secrets → Key Vault**: move the PG password, master key, and OAuth secrets into + `meshweaverkeyvault` via the CSI Secrets Store add-on (enabled in `infra/modules/aks.bicep`). + +## Why a runbook and not pure `aspire up` +Aspire's Kubernetes publisher generates the **workload** chart, but it does not provision an AKS +**cluster**, a private Postgres Flexible Server, a VPN, or Let's Encrypt. Those platform pieces are +the Bicep + these steps. The AppHost (`../aspire/Memex.Deploy.AppHost`) owns the app model + the +deploy parameters (including the OAuth providers); this runbook stitches the platform around it. diff --git a/deploy/aks/README.md b/deploy/aks/README.md new file mode 100644 index 000000000..5d63a7a38 --- /dev/null +++ b/deploy/aks/README.md @@ -0,0 +1,864 @@ +# MeshWeaver Memex on AKS — production-grade deployment sample + +A reference, operator-facing deployment of the **Memex portal** on a **private +Azure Kubernetes Service** cluster. It layers AKS-specific Azure infrastructure +(Bicep) and a Helm values overlay on top of the generic Kubernetes chart that +already lives at [`../helm`](../helm). Everything here is **infra / YAML / +markdown only** — no application code changes. + +> This is a **sample**. Read it end-to-end, tune the parameters for your +> environment (regions, SKUs, CIDRs, DNS names, secrets), and treat the security +> defaults as a starting point, not a finished hardening. + +--- + +## What you get + +| Concern | This sample's choice | +|---|---| +| Cluster | **Private AKS** (`enablePrivateCluster=true`) — API server has a private IP only | +| kubectl reach | **Azure VPN Gateway (Point-to-Site)** + linked **private DNS zone** `privatelink..azmk8s.io` | +| Registry | **Azure Container Registry** (Premium) with **AcrPull** granted to the cluster's kubelet identity | +| Networking | VNet with `aks-nodes`, `GatewaySubnet`, `AzureBastionSubnet` subnets; Azure CNI overlay + Cilium | +| Portal | Blazor Server, **HA (3 replicas across 3 zones)** behind ingress with **cookie session affinity** | +| Shared storage | **Azure Files (RWX)** drives mounted at explicit paths: `/data` (caches), `/mnt/content` (content collection), `/mnt/attachments`, `/mnt/users` — via a custom `azurefile-memex` StorageClass tuned for the non-root portal (uid 1654) | +| Database | Self-managed **Postgres (pgvector) StatefulSet** on a Premium-SSD PVC | +| Backup / PITR | **pgBackRest** → **Azure Blob** (full + diff CronJobs + WAL archiving) → restore `--type=time` | +| Observability | **OpenTelemetry Collector DaemonSet** captures cluster-wide pod logs + portal OTLP → **Azure Files** log archive (`/mnt/otel-logs`) | +| Identity | **Workload Identity (OIDC)** so pgBackRest reaches Blob **keyless** | + +### Topology + +``` + ┌──────────────── operator laptop ───────────────┐ + │ azure-vpn / OpenVPN client (cert auth) │ + └───────────────────────┬─────────────────────────┘ + │ P2S tunnel (172.16.201.0/24) + ┌────────────────────────────▼──────────────────────────┐ + │ VNet 10.42.0.0/16 │ + │ ┌── GatewaySubnet ──┐ ┌── aks-nodes 10.42.0.0/20 ──┐ │ + │ │ VPN Gateway │ │ AKS node pool (3x, zonal) │ │ + │ └───────────────────┘ │ ├ memex-portal x3 (RWX) │ │ + │ privatelink.. │ ├ memex-postgres (PVC) │ │ + │ azmk8s.io ◄── private │ │ └ pgbackrest sidecar │ │ + │ API server A record │ └ pgbackrest CronJobs │ │ + │ └─────────────┬───────────────┘ │ + └────────────────────────────────────────┼─────────────────┘ + │ Workload Identity + ACR (AcrPull) Azure Blob ◄─┘ (WAL + backups, keyless) +``` + +--- + +## Repository layout + +``` +deploy/aks/ +├── README.md ← you are here +├── values.aks.yaml ← Helm overlay for ../helm (AKS overrides) +├── infra/ +│ ├── main.bicep ← subscription-scoped orchestrator (creates RG) +│ ├── main.parameters.json ← edit these +│ └── modules/ +│ ├── network.bicep ← VNet + subnets + private DNS zone + VNet link +│ ├── acr.bicep ← Azure Container Registry +│ ├── aks.bicep ← PRIVATE AKS + identities + AcrPull + CSI + OIDC +│ ├── vpn.bicep ← P2S VPN Gateway (cert auth, OpenVPN/IKEv2) +│ ├── storage.bicep ← Blob storage + Workload Identity for pgBackRest +│ └── files.bicep ← Azure Files account + named shares for STATIC PV binding +└── manifests/ ← applied alongside the Helm release + ├── storageclass-azurefile.yaml ← custom azurefile-memex SC (uid 1654, nobrl) + ├── portal-pvcs.yaml ← RWX drives: data/content/attachments/users + pg PVC + ├── portal-ha-patch.yaml ← replicas=3, zone spread, probes, RWX volume mounts + ├── postgres-pvc-patch.yaml← bind pg StatefulSet to its PVC + ├── portal-ingress.yaml ← ingress + cookie session affinity (Blazor) + ├── observability/ + │ ├── otel-collector-config.yaml ← collector pipeline (filelog+otlp → file+debug) + │ ├── otel-collector.yaml ← collector DaemonSet + SA/RBAC + Service + │ └── otel-pvc.yaml ← RWX Azure Files PVC for the log archive + └── pgbackrest/ + ├── serviceaccount.yaml← Workload-Identity SA for keyless Blob + ├── configmap.yaml ← pgbackrest.conf (Azure repo) + WAL archive conf + ├── sidecar-patch.yaml ← pgBackRest sidecar + WAL archiving wiring + └── cronjobs.yaml ← stanza-create Job + full/diff backup CronJobs +``` + +### Why a Helm overlay **and** extra manifests? + +The chart at `../helm` is generated from the Aspire model and is intentionally +**generic** (Azure-free, single replica, `emptyDir` volumes, no ingress). We do +**not** fork or regenerate it. Instead: + +- **`values.aks.yaml`** sets the keys the chart already consumes (`config.*`, + `secrets.*`) with AKS-correct values (e.g. Orleans `AdoNet` clustering for HA). +- **`manifests/`** supplies the things the generic chart does not template — + RWX PVCs, ingress with sticky sessions, the HA replica/zone patch, and the + pgBackRest sidecar/CronJobs — as `kubectl apply` / `kubectl patch` steps you + run right after `helm install`. + +The annotated `ingress:`, `persistence:`, `replicas:`, and `pgbackrest:` blocks +at the bottom of `values.aks.yaml` double as a forward-looking contract: if the +`../helm` chart is later extended to template these, the overlay already carries +the right values. + +--- + +## Prerequisites + +- `az` CLI ≥ 2.84 (`az version`) +- `az bicep` ≥ 0.41 (`az bicep version`) — `az bicep upgrade` if older +- `kubectl` and `helm` ≥ 3.12 +- A subscription where you can create resource groups + role assignments + (Owner or User Access Administrator — the deployment grants AcrPull, DNS, and + Blob roles) +- `openssl` (to mint the P2S VPN certificates), or Windows `New-SelfSignedCertificate` + +Validate the Bicep before deploying: + +```bash +az bicep build --file infra/main.bicep --stdout > /dev/null # compiles clean +``` + +--- + +## Step 1 — Provision the infrastructure + +Edit `infra/main.parameters.json` (region, `namePrefix`, node SKU/count, CIDRs, +toggles). Then deploy at **subscription** scope (the template creates the +resource group): + +```bash +az deployment sub create \ + --name memex-aks-infra \ + --location westeurope \ + --template-file infra/main.bicep \ + --parameters @infra/main.parameters.json \ + --parameters postgresAdminPassword="$PG_ADMIN_PW" # required: deployPostgresFlexible=true +``` + +> `postgresAdminPassword` is a `@secure()` parameter — it is NOT in +> `main.parameters.json` (never commit a DB password). Pass it at deploy time. +> If you set `deployPostgresFlexible: false` (use the in-cluster StatefulSet +> instead), you can omit it. + +This is **infra only** — it does not install the portal. Capture the outputs you +need for later steps: + +```bash +az deployment sub show --name memex-aks-infra \ + --query "properties.outputs.{rg:resourceGroupName.value, cluster:clusterName.value, acr:acrLoginServer.value, vpn:vpnGatewayName.value, pgFqdn:postgresFqdn.value, pgDb:postgresDatabaseName.value, pgUser:postgresAdminLogin.value, filesAccount:contentFilesAccount.value, oidc:oidcIssuerUrl.value}" -o jsonc +``` + +> The VPN Gateway takes **20–45 minutes** to provision — this dominates the +> deploy time. Set `deployVpnGateway: false` if you'll use +> `az aks command invoke` or a Bastion jumpbox instead (see Step 2 alternatives). + +### Key parameters + +| Parameter | Default | Notes | +|---|---|---| +| `location` | `westeurope` | drives the private DNS zone name | +| `namePrefix` | `memexaks` | ≤ 12 chars, prefixes every resource | +| `systemNodeVmSize` / `systemNodeCount` | `Standard_D8s_v3` / 3 | 8 vCPU / **32 GiB** nodes, autoscales 3→6. (Pick a family with quota in your region — DSv5 was 0 in this subscription's westeurope, DSv3 had 100 vCPU.) | +| `availabilityZones` | `["1","2","3"]` | zonal spread for HA | +| `vnetAddressSpace` | `10.42.0.0/16` | must not collide with peered nets | +| `deployVpnGateway` | `true` | the P2S kubectl path | +| `vpnClientAddressPool` | `172.16.201.0/24` | **must not overlap the VNet** | +| `vpnClientRootCertData` | `""` | base64 root public cert (can add later) | +| `deployBackupStorage` | `false` | self-managed pgBackRest blob; **off** because we use the managed private Flexible Server instead | +| `deployContentFileShares` | `true` | Azure Files account + named shares for **static** PV binding (dynamic provisioning needs no shares) | +| `deployPostgresFlexible` | `true` | **PRIVATE (VNet-injected) PostgreSQL Flexible Server** with pgvector + managed PITR | +| `postgresAdminPassword` | *(required, `@secure`)* | pass at deploy time: `--parameters postgresAdminPassword=...` — never commit | +| `postgresSkuName` | `Standard_D2ds_v5` | 2 vCPU / 8 GiB GeneralPurpose; bump for more DB headroom | +| `postgresHighAvailability` | `true` | zone-redundant hot standby in a 2nd AZ | + +--- + +## Step 2 — Reach the private API server (P2S VPN) + +Because the cluster is private, `kubectl` only works from inside the VNet. The +P2S VPN attaches your laptop to the VNet; the linked private DNS zone then +resolves the API server FQDN to its private IP. + +### 2a. Create the P2S certificates (cert-based auth) + +```bash +# Root CA +openssl genrsa -out p2sRoot.key 2048 +openssl req -x509 -new -nodes -key p2sRoot.key -subj "/CN=Memex-P2S-Root" -days 3650 -out p2sRoot.crt + +# Client cert signed by the root +openssl genrsa -out p2sClient.key 2048 +openssl req -new -key p2sClient.key -subj "/CN=Memex-P2S-Client" -out p2sClient.csr +openssl x509 -req -in p2sClient.csr -CA p2sRoot.crt -CAkey p2sRoot.key -CAcreateserial -days 365 -out p2sClient.crt + +# Base64 of the ROOT public cert (single line, no PEM headers) — feed to Bicep +openssl x509 -in p2sRoot.crt -outform der | base64 -w0 ; echo +``` + +You can either: + +- paste that base64 string into `vpnClientRootCertData` and redeploy infra, **or** +- upload it after the fact without redeploying: + +```bash +az network vnet-gateway root-cert create \ + --resource-group --gateway-name -vpngw \ + --name P2SRootCert --public-cert-data "" +``` + +### 2b. Download + connect the VPN client + +```bash +az network vnet-gateway vpn-client generate \ + --resource-group --name -vpngw --output tsv +``` + +Download the returned zip, install the profile (the **Azure VPN Client** or +OpenVPN; the bundle ships an OpenVPN `.ovpn` you augment with `p2sClient.crt` + +`p2sClient.key`), and connect. + +### 2c. Get credentials and run kubectl + +```bash +az aks get-credentials --resource-group --name +kubectl get nodes # resolves the PRIVATE API server over the tunnel +``` + +If `kubectl` times out: confirm the VPN is connected, that the private DNS zone +`privatelink..azmk8s.io` is linked to the VNet (it is, via +`network.bicep`), and that your client gets a `172.16.201.x` address. + +### Alternatives to the P2S VPN (not implemented here, but supported) + +- **`az aks command invoke`** — runs a command/`kubectl`/`helm` inside a + transient pod on the cluster; no network line-of-sight needed. Great for CI: + `az aks command invoke -g -n --command "kubectl get pods -A"`. +- **Jumpbox + Azure Bastion** — the `AzureBastionSubnet` is already carved out; + deploy Bastion + a small VM in the VNet and run kubectl from there. + +The P2S VPN is implemented because it gives operators a native local `kubectl` +experience without a standing VM. + +--- + +## Step 3 — Image strategy + +The chart references `ghcr.io/systemorph/memex-portal-ai` and +`ghcr.io/systemorph/memex-migration`. Two options: + +**Option A — pull from GHCR directly** (simplest; needs node egress to ghcr.io, +which the default `outboundType: loadBalancer` provides). Keep +`image.registry: ghcr.io/systemorph` in `values.aks.yaml`. + +**Option B — import into the private ACR** (recommended for a locked-down +cluster; AcrPull is already granted to the kubelet identity): + +```bash +ACR= # from outputs (without .azurecr.io) +az acr import --name $ACR --source ghcr.io/systemorph/memex-portal-ai:latest --image memex-portal-ai:latest +az acr import --name $ACR --source ghcr.io/systemorph/memex-migration:latest --image memex-migration:latest +# optional lean / base variants: +az acr import --name $ACR --source ghcr.io/systemorph/memex-portal:latest --image memex-portal:latest +az acr import --name $ACR --source ghcr.io/systemorph/memex-portal-ai-base:latest --image memex-portal-ai-base:latest +``` + +Then set `image.registry: .azurecr.io` in `values.aks.yaml` (and, since +the generic chart hardcodes the GHCR path in its templates, repoint the running +Deployments with `kubectl set image` or extend the chart to read +`.Values.image.registry`). + +--- + +## Step 4 — Install the portal (Helm + manifests) + +```bash +NS=memex +kubectl create namespace $NS + +# 0) Custom StorageClass for the non-root portal (uid 1654) — cluster-scoped, +# so no namespace. Must exist before the RWX PVCs that reference it. +kubectl apply -f manifests/storageclass-azurefile.yaml + +# 1) Real RWX + DB PVCs (must exist before the workloads mount them) +kubectl apply -n $NS -f manifests/portal-pvcs.yaml + +# 2) Install the chart with the AKS overlay (set a real PG password!) +helm install memex ../helm \ + -f ../helm/values.yaml \ + -f values.aks.yaml \ + --namespace $NS \ + --set secrets.memex_postgres.memex_postgres_password='' \ + --set secrets.memex_migration.memex_postgres_password='' \ + --set secrets.memex_portal.memex_postgres_password='' + +# 3) Bind Postgres to its PVC, then scale the portal out + go RWX +kubectl patch statefulset memex-postgres-statefulset -n $NS \ + --type strategic --patch-file manifests/postgres-pvc-patch.yaml +kubectl patch deployment memex-portal-deployment -n $NS \ + --type strategic --patch-file manifests/portal-ha-patch.yaml + +# 4) Ingress with cookie session affinity (enable a controller first) +az aks approuting enable -g -n # managed nginx +kubectl apply -n $NS -f manifests/portal-ingress.yaml +``` + +> **Secrets**: never commit a real password. The CSI Secrets Store add-on is +> enabled in `aks.bicep` — wire `secrets.memex_*` to Key Vault via a +> `SecretProviderClass` for production rather than `--set`. + +> **Blazor sticky sessions**: the ingress affinity cookie is mandatory. Without +> it, SignalR circuit reconnects can land on the wrong replica and users see +> "Attempting to reconnect…" loops. The annotations are in +> `manifests/portal-ingress.yaml` (nginx today, AGIC commented). + +--- + +## Storage drives — mountable Azure Files at explicit `/mnt` paths + +The portal's persistent data is split across **dedicated Azure Files (RWX) +drives**, one per concern, each mounted at an explicit path. This keeps user +content off the small/churny framework-cache volume and lets you size, expand, +and (optionally) back up each drive independently. + +| Drive (PVC) | Mount path | Holds | Repointed by | +|---|---|---|---| +| `memex-data` | `/data` | Framework caches only: DataProtection keys (`/data/dataprotection-keys`), NodeType assembly-cache, NuGet package-cache | `Deployment__DataRoot=/data` | +| `memex-content` | `/mnt/content` | **The content collection** — uploaded files / media / per-node-hub content (`{BasePath}/content/{nodePath}`) | `Storage__BasePath=/mnt/content` | +| `memex-attachments` | `/mnt/attachments` | Attachments drive (see note) | *(forward-looking — no env knob today)* | +| `memex-users` | `/mnt/users` | Co-hosted CLI configs | *(unchanged)* | +| `memex-pgdata` | Postgres data dir | Database files (RWO, managed-csi) | — | + +### Why the custom `azurefile-memex` StorageClass (uid 1654) + +The portal image runs as the .NET **`app` user — uid 1654 / gid 1654** (the +non-root uid baked into the chiseled `dotnet/aspnet` images). The default +`azurefile-csi` class mounts shares `uid=0,gid=0` (root-owned, mode 0777). That +*usually* works, but a non-root process on a root-owned share is brittle — it's +exactly the failure mode that produced +`UnauthorizedAccessException: Access to the path '/data/dataprotection-keys' is denied` +on the Docker-Compose deploy. `manifests/storageclass-azurefile.yaml` pins +`uid=1654,gid=1654` so **every inode on the share is owned by the portal user**, +plus `mfsymlinks` (DataProtection writes symlinks), `cache=strict`, `actimeo=30`, +and `nobrl` (Azure Files SMB rejects the POSIX byte-range locks that SQLite and +other file-lock-y libraries take; `nobrl` makes them no-ops). `reclaimPolicy: +Retain` keeps the share + its keys/content if a PVC is accidentally deleted. + +### How the portal reads these paths (no app code change) + +- **Content** — `MemexConfiguration.ConfigureMemexMesh` reads `Storage:BasePath` + as the FileSystem content-collection root and gives each node hub a + `content/{nodePath}` subdirectory under it. The overlay sets + `Storage__BasePath=/mnt/content`. **This is the real functional change.** +- **Attachments** — the portal also maps an `attachments` collection + (`MapContentCollection("attachments", "storage", "attachments/{nodePath}")`). + ⚠️ In the **Distributed / Filesystem** backend (the image this sample runs) the + `storage` *source* collection is **not** separately registered — only the + Monolith registers it — so **attachments has no independently env-repointable + base path today** (there is no `Storage__Attachments__BasePath` setting). We + mount `/mnt/attachments` anyway so the drive exists and is ready: if the app + later registers a filesystem `storage` source rooted there, no manifest change + is needed. (If you run the **Monolith** image instead, it *does* register the + `storage` source from the `Storage` section, so attachments follows + `Storage__BasePath` — but then content + attachments share one drive.) + +### Option A — dynamic provisioning (default, simplest) + +`manifests/portal-pvcs.yaml` requests the drives against the `azurefile-memex` +StorageClass; the CSI driver **creates a share per PVC automatically**. Nothing +else to do — this is what Step 4 applies. + +### Option B — static PV binding to pre-created named shares + +If you'd rather pre-create named shares in one account (to size/quota/firewall/ +back them up centrally), set `deployContentFileShares: true` (default) so +`infra/modules/files.bicep` provisions a `StorageV2 / Standard_ZRS / +largeFileSharesState=Enabled` account with shares `content`, `attachments`, +`data`, `users`, `otel-logs`. Then bind a **static PV** per share. Grab the +account name + key: + +```bash +SA=$(az deployment sub show --name memex-aks-infra \ + --query "properties.outputs.contentFilesAccount.value" -o tsv) +RG=$(az deployment sub show --name memex-aks-infra \ + --query "properties.outputs.resourceGroupName.value" -o tsv) +KEY=$(az storage account keys list -g $RG -n $SA --query "[0].value" -o tsv) +kubectl create secret generic azure-files-creds -n memex \ + --from-literal=azurestorageaccountname=$SA \ + --from-literal=azurestorageaccountkey=$KEY +``` + +Then a PV/PVC pair per share (content shown; repeat for attachments/data/users): + +```yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: memex-content-pv +spec: + capacity: { storage: 128Gi } + accessModes: [ReadWriteMany] + storageClassName: "" # static — no dynamic provisioner + persistentVolumeReclaimPolicy: Retain + mountOptions: [dir_mode=0777, file_mode=0777, uid=1654, gid=1654, mfsymlinks, cache=strict, actimeo=30, nobrl] + csi: + driver: file.csi.azure.com + volumeHandle: memex-content # any cluster-unique id + volumeAttributes: + resourceGroup: + storageAccount: + shareName: content # the pre-created share from files.bicep + nodeStageSecretRef: + name: azure-files-creds + namespace: memex +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: { name: memex-content, namespace: memex } +spec: + accessModes: [ReadWriteMany] + storageClassName: "" + volumeName: memex-content-pv + resources: { requests: { storage: 128Gi } } +``` + +> Keyless alternative: instead of the account-key secret, federate a Workload +> Identity with the portal's ServiceAccount and grant it *Storage File Data SMB +> Share Contributor* — same pattern as the pgBackRest identity in `storage.bicep`. + +Keep **dynamic (Option A) as the default**; reach for static binding only when +you need named, centrally-managed shares. + +--- + +## Observability — OpenTelemetry across the cluster → Azure Files archive + +A single **OpenTelemetry Collector DaemonSet** captures telemetry for the whole +cluster and archives it to a mounted **Azure Files** share — no per-GB App +Insights ingest. (This repo bills App Insights per ingest, so an in-cluster file +archive is the cost-conscious default for a self-hosted deployment.) + +``` + every node: Azure Files (RWX) + ┌─ pod stdout/stderr ─┐ filelog (hostPath /var/log/pods) /mnt/otel-logs + │ (ALL namespaces) ├────────────────┐ ├ logs-.json + └─────────────────────┘ ▼ ├ traces-.json + ┌─ otel-collector (DaemonSet) ─┐ └ metrics-.json + memex-portal x3 ─ OTLP ────►│ k8sattributes + resourcedetect│──► file exporter (rotated) + (:4317 grpc via Service) │ + batch │──► debug exporter (kubectl logs) + └───────────────────────────────┘ +``` + +- **Sources**: `filelog` tails `/var/log/pods/**/*.log` on every node (so **all** + pod logs cluster-wide are captured, not just the portal), and `otlp` + (gRPC :4317 / HTTP :4318) receives the portal's traces/logs/metrics. +- **Enrichment**: `k8sattributes` (pod/namespace/node/deployment) + + `resourcedetection` + `batch`. +- **Sink**: the `file` exporter writes rotated JSON to `/mnt/otel-logs` + (`max_megabytes: 100`, `max_backups: 10`). Each DaemonSet pod namespaces its + output by node name (`logs-.json`) via the downward-API `NODE_NAME` env, + so replicas don't clobber each other on the shared share. A `debug` exporter + (verbosity `basic`) mirrors a summary to the collector's own stdout for + `kubectl logs ds/otel-collector`. + +### How the portal emits OTLP (verified wiring — no code change) + +`Memex.Portal.ServiceDefaults/ServiceDefaults.cs` → +`AddOpenTelemetryExporters()` does: + +```csharp +var useOtlp = !string.IsNullOrWhiteSpace(builder.Configuration["OTEL_EXPORTER_OTLP_ENDPOINT"]); +var useAzureMonitor = !string.IsNullOrWhiteSpace(builder.Configuration["APPLICATIONINSIGHTS_CONNECTION_STRING"]); +if (useAzureMonitor) builder.Services.AddOpenTelemetry().UseAzureMonitor(); +else if (useOtlp) builder.Services.AddOpenTelemetry().UseOtlpExporter(); +``` + +So setting **`OTEL_EXPORTER_OTLP_ENDPOINT`** turns on the OTLP exporter — **but +only if `APPLICATIONINSIGHTS_CONNECTION_STRING` is unset** (Azure Monitor wins). +`values.aks.yaml` sets `OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317` +and `OTEL_EXPORTER_OTLP_PROTOCOL=grpc` and does **not** set the App Insights +connection string, so the portal exports to the in-cluster collector. (To go back +to App Insights, set the connection string and the OTLP env is ignored.) + +### Apply it + +```bash +NS=memex +kubectl apply -f manifests/storageclass-azurefile.yaml # if not already applied +kubectl apply -n $NS -f manifests/observability/otel-pvc.yaml +kubectl apply -n $NS -f manifests/observability/otel-collector-config.yaml +kubectl apply -n $NS -f manifests/observability/otel-collector.yaml +# portal already points at the collector via values.aks.yaml — restart to pick up env if needed +kubectl rollout restart deployment memex-portal-deployment -n $NS +``` + +### Read / download the archived logs + +The archive lives on the `otel-logs` Azure Files share. Inspect from a pod: + +```bash +kubectl exec -n memex ds/otel-collector -- ls -lh /mnt/otel-logs +kubectl exec -n memex ds/otel-collector -- tail -n 50 /mnt/otel-logs/logs-.json +``` + +…or download straight from the Files share with the account key (Option B account, +or the dynamically-created share — find it under the cluster's node resource group): + +```bash +az storage file download-batch \ + --account-name --account-key \ + --source otel-logs --destination ./otel-archive +``` + +### Retention / rotation, cost, and scale-up + +- **Rotation** is per-node-file: 100 MB × 10 backups ⇒ ~1 GB/node retained, then + oldest rolls off. Bump `max_backups` / the `otel-logs` PVC size for longer + retention; add an Azure Files lifecycle/cleanup CronJob for time-based pruning. +- **Cost**: a flat Azure Files share (≈€0.06/GB-month Standard) vs. per-GB App + Insights ingest — the archive is the cheap default for self-hosting. +- **Azure Table storage has NO native OTel Collector exporter** — Azure **Files** + (the `file` exporter over a mounted SMB share) is the chosen sink. For richer + query/alerting at scale, swap the `file` exporter for either: + - **Grafana Loki backed by Azure Blob** (the `loki` exporter → Loki → Blob + object store) for label-indexed log search, or + - the **`azuremonitor` exporter** to ship into Azure Monitor / Log Analytics + (KQL, alerts) — accepting the per-GB ingest cost. + + Both are drop-in exporter swaps in `otel-collector-config.yaml`; neither is + implemented here to keep the default zero-PaaS and cheap. + +--- + +## Database — PRIVATE PostgreSQL Flexible Server (default) + +This sample defaults to a **managed, private** database: +`infra/modules/postgres.bicep` provisions an **Azure Database for PostgreSQL +Flexible Server** injected into the delegated `postgres` subnet — no public +endpoint. It resolves only inside the VNet (and over the P2S VPN) via the +`*.private.postgres.database.azure.com` private DNS zone that `network.bicep` +links. This matches the private-everything posture: private API server, private +drives (Azure Files), private DB. + +- **pgvector** is allowlisted (`azure.extensions = VECTOR,UUID-OSSP`) for the + portal's embeddings + HNSW vector search; the `memex` database is created. +- **Managed PITR** — automatic backups + WAL; restore to any second in the + retention window with `az postgres flexible-server restore`. No in-cluster + backup machinery, so `deployBackupStorage: false` and the + `postgres-pvc-patch` / `pgbackrest` manifests are **not** applied. +- **HA** — `postgresHighAvailability: true` runs a zone-redundant hot standby. + +Point the portal at the server's private FQDN (from the `postgresFqdn` output) +in `values.aks.yaml` — set `MEMEX_HOST` and the connection-string secret to the +Flexible Server endpoint, user `memexadmin`, db `memex`. The portal's +Postgres path auto-detects Azure-managed-identity vs basic auth from the +connection string (see `Memex.Portal.Distributed/Program.cs`); basic auth with +the admin password works out of the box. + +> **In-cluster alternative**: set `deployPostgresFlexible: false` (+ revert +> `deployBackupStorage: true`) to use the self-managed Postgres StatefulSet + +> pgBackRest PITR instead — see Step 5 below. The two are mutually exclusive; +> pick one. + +--- + +## Documentation search — full-text + optional vector + +The built-in MeshWeaver platform documentation ships **inside the images** (embedded +resources) and is served from memory at runtime. So that it also shows up in the portal's +**main search bar**, the one-shot **migration mirrors every doc page into a Postgres `doc` +schema** on each deploy: + +- **Full-text search** (always on, no external dependency). Each doc's title + one-line + description is indexed; the search bar finds docs by keyword out of the box. +- **Semantic / vector search** (opt-in). When an embeddings endpoint is configured, the + migration also computes an embedding per doc (title + description + body) and stores it in + the pgvector **HNSW** index, so natural-language queries (“how do I cancel a running job”) + rank the right page. The portal embeds the search query the same way, so both sides must use + the same model. (`pgvector` is already allowlisted on the Flexible Server — see the Database + section.) + +The mirror is a **full replace + incremental embed**: every deploy upserts the current doc set +and prunes rows whose source page no longer ships, and the (paid) embedding call only fires for +pages whose content actually changed since the last run. Reads/navigation still come from the +in-memory copy — the `doc` schema is purely a search index. + +### Configure it + +The embeddings provider is **optional**. Leave it unset and docs are full-text-searchable with +no external AI dependency; set it to enable vector ranking. The deploy AppHost +(`deploy/aspire/Memex.Deploy.AppHost`) reads three parameters and flows them to **both** the +migration and the portal: + +| Deploy parameter | Container env (migration **and** portal) | Notes | +|---|---|---| +| `Parameters:embedding-endpoint` | `Embedding__Endpoint` | Azure AI Foundry embeddings endpoint (Cohere embed-v4). Empty ⇒ full-text only. | +| `Parameters:embedding-key` | `Embedding__ApiKey` | Secret — only emitted when set (ACA/compose reject empty secrets). | +| `Parameters:embedding-model` | `Embedding__Model` | Defaults to `embed-v-4-0` (the Cohere embed-v4 Azure AI Foundry deployment name). Migration + portal must agree (sizes the vector column). | + +Set them via `dotnet user-secrets` / env / GitHub secrets at publish time, e.g.: + +```bash +aspire publish --apphost deploy/aspire/Memex.Deploy.AppHost/Memex.Deploy.AppHost.csproj \ + -o deploy/helm -- --mode kubernetes \ + -- --Parameters:embedding-endpoint=https://.services.ai.azure.com/... \ + --Parameters:embedding-key= +``` + +For the **AKS / Helm** path these surface as `config.Embedding__*` (and the key as a +`secrets.*` entry) on the regenerated chart's migration Job and portal Deployment — set them in +`values.aks.yaml` (or `--set`) alongside the other secrets, and wire the key through the CSI +Secrets Store add-on for production rather than committing it. + +--- + +## Orleans clustering — Postgres-backed (never Localhost in prod) + +HA runs the portal as **multiple silos**, which must form one cluster via a shared membership +store. This deployment uses **Postgres-backed ADO.NET clustering** on the **same Postgres server +in a separate `orleans` database** (so silo membership never shares tables or locks with mesh +data). It works for a single silo too, so the self-host AppHosts use it in every mode — Localhost +clustering is never used in a deployment. + +How it's wired (all DB config flows through Aspire): + +- The `AddMemex` integration declares the `orleans` database on the same Postgres server and + references it on both the portal and the migration, so Aspire injects `ConnectionStrings:orleans`. +- The portal silo selects the provider from the **feature flag `Features:Orleans:Clustering`** + (set to `AdoNet` by the self-host AppHosts; legacy `Deployment:Orleans:Clustering` still works) + and calls `UseAdoNetClustering(Invariant=Npgsql)` against that injected connection string. +- The **db-migration creates the Orleans membership tables** (`OrleansQuery`, + `OrleansMembershipTable`, …) in the `orleans` database from the verbatim Orleans 10 PostgreSQL + scripts — idempotent, and it auto-creates the database on self-managed Postgres. The Orleans + provider does *not* self-create these tables, so the migration must run before the silos start + (the portal already `WaitForCompletion(migration)`). + +> Aspire's Orleans integration only wires Redis / Azure-Table clustering — not ADO.NET — so the +> `orleans` database lives in Aspire while the silo wiring and the membership DDL live in the +> portal and the migration. (The Azure/ACA path instead uses Azure Table Storage clustering via +> the Aspire Orleans integration and doesn't need any of this.) + +**AKS / Flexible Server note:** on the managed-Postgres path, ensure the `orleans` database exists +on the server (the chart's migration Job creates the tables but the managed server must allow the +DB; `azure.extensions` already includes pgvector for the mesh DB). The regenerated chart carries +`Features__Orleans__Clustering=AdoNet` and the `orleans` connection string from the Aspire model; +set the connection-string secret in `values.aks.yaml` alongside the `memex` one. HA needs **≥2 +replicas** (the `portal-ha-patch.yaml` already sets 3). + +--- + +## Authentication — Systemorph AAD (home) + Google + LinkedIn + +`values.aks.yaml` wires the login providers the portal's auth pipeline +(`AuthenticationBuilderExtensions`) reads from `Authentication:*`: + +| Provider | Config keys (env: `Authentication__

__*`) | Redirect URI to register | +|---|---|---| +| **Microsoft / Entra (HOME)** | `TenantId` (Systemorph tenant GUID), `ClientId`, `ClientSecret` | `https://memex.systemorph.com/signin-microsoft` | +| **Google** | `ClientId`, `ClientSecret` | `https://memex.systemorph.com/signin-google` | +| **LinkedIn** | `ClientId`, `ClientSecret` | `https://memex.systemorph.com/signin-linkedin` | + +- Setting `Authentication__Microsoft__TenantId` to a **real tenant GUID** (not + `common`) makes that AAD the **home** directory. This subscription's tenant is + `3a01d7ac-3330-444d-942d-975eb491b5d6` (Systemorph) and is pre-filled. +- Any provider with a `ClientId` set is offered on the login page; the presence + of external providers flips the portal into multi-provider mode and dev login + is off in the Distributed image. +- **You still must create the app registrations / OAuth clients** and fill the + `CHANGE_ME_*` `ClientId`s (config) + `ClientSecret`s (secrets) — those are real + credentials this repo does not contain. Register each redirect URI above. +- Host is `memex.systemorph.com` (ingress + TLS). Point a DNS A/CNAME at the + ingress controller's IP and issue the `memex-tls` cert (cert-manager or a + pre-created secret). + +--- + +## Step 5 — PITR backups with pgBackRest → Azure Blob (in-cluster alternative) + +> **Recommended for turnkey prod: use Azure Database for PostgreSQL Flexible +> Server instead** (managed PITR, automatic WAL, restore to any point in the +> retention window with one CLI call, no in-cluster moving parts). See the +> [Flexible Server](#alternative-azure-database-for-postgresql-flexible-server) +> section. pgBackRest is the **all-in-cluster, self-managed** option for when you +> want the database to live next to the workload. + +### Wire it up + +```bash +NS=memex +# Workload-Identity SA — put the pgBackRestIdentityClientId output in the SA +kubectl apply -n $NS -f manifests/pgbackrest/serviceaccount.yaml # edit the client-id first +kubectl apply -n $NS -f manifests/pgbackrest/configmap.yaml + +# Add the WAL-archive wiring + sidecar to the Postgres StatefulSet +kubectl patch statefulset memex-postgres-statefulset -n $NS \ + --type strategic --patch-file manifests/pgbackrest/sidecar-patch.yaml + +# Wait for the DB pod to roll, then create the stanza + scheduled backups +kubectl apply -n $NS -f manifests/pgbackrest/cronjobs.yaml +``` + +Before applying, substitute your storage account + container into the manifests +(they carry `__AZURE_ACCOUNT__` / `pgbackrest` placeholders): the +`backupStorageAccount` and `backupContainerName` infra outputs, and the +`pgBackRestIdentityClientId` into the service account annotation. + +How it works: + +- **WAL archiving**: the init container appends `archive_command = pgbackrest … + archive-push` to `postgresql.conf`, so every completed WAL segment is pushed to + Blob continuously. This is what makes PITR (replay to an arbitrary timestamp) + possible. +- **Scheduled backups**: `pgbackrest-full` (weekly) + `pgbackrest-diff` (daily) + CronJobs write full/differential base backups to the same Blob repo. +- **Keyless auth**: the SA is federated (Workload Identity) with the managed + identity that holds *Storage Blob Data Contributor* — no account key on disk. + (To use a key instead, set `repo1-azure-key` in the ConfigMap and drop the + workload-identity annotation.) + +### Backup runbook + +```bash +NS=memex; POD=memex-postgres-statefulset-0 +# Ad-hoc full backup (zero contention — runs in the sidecar): +kubectl exec -n $NS $POD -c pgbackrest -- \ + pgbackrest --config=/etc/pgbackrest/pgbackrest.conf --stanza=memex --type=full backup + +# Verify repo health + list backups: +kubectl exec -n $NS $POD -c pgbackrest -- \ + pgbackrest --config=/etc/pgbackrest/pgbackrest.conf --stanza=memex check +kubectl exec -n $NS $POD -c pgbackrest -- \ + pgbackrest --config=/etc/pgbackrest/pgbackrest.conf --stanza=memex info +``` + +### Restore runbook (Point-In-Time) + +Restore is **destructive** to the live data dir — it replaces cluster files, +so the database must be stopped during the restore. + +```bash +NS=memex +# 1) Scale the portal down (no writers) and stop Postgres. +kubectl scale deployment memex-portal-deployment -n $NS --replicas=0 +kubectl scale statefulset memex-postgres-statefulset -n $NS --replicas=0 + +# 2) Run pgBackRest restore against the PVC from a one-off pod that mounts +# memex-pgdata + the pgbackrest config. PITR to a timestamp: +kubectl run pgbackrest-restore -n $NS --rm -i --restart=Never \ + --image=docker.io/woblerr/pgbackrest:2.54.2 \ + --overrides='{ + "spec": { + "serviceAccountName": "pgbackrest-sa", + "containers": [{ + "name": "restore", + "image": "docker.io/woblerr/pgbackrest:2.54.2", + "command": ["pgbackrest","--config=/etc/pgbackrest/pgbackrest.conf","--stanza=memex", + "--type=time","--target=2026-05-30 14:30:00+00","--delta","restore"], + "volumeMounts": [ + {"name":"memex-pgdata","mountPath":"/var/lib/postgresql/data"}, + {"name":"pgbackrest-conf","mountPath":"/etc/pgbackrest"} + ] + }], + "volumes": [ + {"name":"memex-pgdata","persistentVolumeClaim":{"claimName":"memex-pgdata"}}, + {"name":"pgbackrest-conf","configMap":{"name":"pgbackrest-config","items":[{"key":"pgbackrest.conf","path":"pgbackrest.conf"}]}} + ], + "metadata": {"labels": {"azure.workload.identity/use": "true"}} + } + }' + +# 3) Bring Postgres back; it replays WAL up to the target time, then promotes. +kubectl scale statefulset memex-postgres-statefulset -n $NS --replicas=1 +kubectl logs -n $NS memex-postgres-statefulset-0 -f # watch recovery complete +kubectl scale deployment memex-portal-deployment -n $NS --replicas=3 +``` + +> Replace the `--target` timestamp (and remember the ConfigMap's account/container +> placeholders must be rendered). For "latest possible" recovery drop +> `--type=time --target=…` and pgBackRest replays all archived WAL. + +### Alternative: Azure Database for PostgreSQL Flexible Server + +For most production deployments, prefer the managed database: + +- **Built-in PITR** — automatic backups + WAL; restore to any second in the + retention window (7–35 days) via `az postgres flexible-server restore`. +- No StatefulSet, no PVC, no pgBackRest sidecar/CronJobs to operate. +- pgvector is supported (`azure.extensions`). + +To switch: set `deployBackupStorage: false`, do **not** apply the +`postgres-pvc-patch` / `pgbackrest` manifests, scale the chart's Postgres +StatefulSet to 0, provision a Flexible Server (private-access / VNet-injected +into a delegated subnet), and point the portal's `MEMEX_HOST` / +`MEMEX_JDBCCONNECTIONSTRING` / connection-string secret at it in +`values.aks.yaml`. The portal is unchanged — it just talks to a different +Postgres endpoint. + +--- + +## Generating this from Aspire + +The repo already models the deployment in +[`deploy/aspire/Memex.Deploy.AppHost`](../aspire). Running +`aspire publish` (or `azd`) against that model is what produced the generic +[`../helm`](../helm) chart and the [`../aca`](../aca) Container Apps Bicep. + +This AKS sample is **complementary**, not generated: the Aspire publishers emit a +portable Helm chart and an ACA topology, but they do **not** emit a private-AKS + +P2S-VPN + pgBackRest-PITR stack. So the relationship is: + +- **Aspire owns the app model** → it generates `../helm` (Deployments, Service, + StatefulSet, migration Job, config/secret templates). Keep regenerating that + from Aspire when the app composition changes. +- **This sample owns the AKS platform** → `infra/*.bicep` (private cluster, VPN, + ACR, backup storage) + `values.aks.yaml` overlay + `manifests/` for the pieces + the generic chart doesn't template. These are hand-authored Azure platform + concerns that don't belong in the app model. + +If you want the Aspire AppHost to drive the **infra** too, you can call this +Bicep from a `Memex.Deploy.AppHost` publisher: add it as an +`AddBicepTemplate("aks-infra", "infra/main.bicep")` resource (or invoke +`az deployment sub create` from a publish hook) and pass the cluster/ACR/storage +outputs into the Helm release step. That keeps a single `aspire`/`azd`-driven +entry point while this directory remains the source of truth for the AKS-specific +Azure resources. The chart stays Aspire-generated; only the platform Bicep and +the overlay are added here. + +To keep the chart in sync after an app-model change, regenerate `../helm` from +Aspire and re-run Step 4 — `values.aks.yaml` and `manifests/` continue to apply +on top unchanged (they only reference stable resource names like +`memex-portal-deployment` / `memex-postgres-statefulset`). + +--- + +## Teardown + +```bash +helm uninstall memex -n memex +kubectl delete namespace memex # also deletes the PVCs (Azure Files/Disk) +az group delete --name --yes --no-wait # cluster, VPN, ACR, storage, VNet +``` + +> Deleting the namespace deletes the PVCs and their backing Azure Files shares / +> managed disks. The pgBackRest **Blob** repo lives in the separate backup +> storage account and survives the cluster — delete the resource group (or just +> the storage account) to remove backups, mindful of the 30-day soft-delete. + +--- + +## Security notes (read before prod) + +- **Secrets**: move `memex_postgres_password` out of `--set`/values into Key + Vault via the CSI Secrets Store add-on (already enabled). +- **API server**: private-only; `enablePrivateClusterPublicFQDN=false`. Local + accounts stay enabled so the VPN kubeconfig works — consider Entra-only + (`disableLocalAccounts=true` + AKS-managed Entra RBAC) for prod. +- **VPN auth**: this sample uses certificate auth for simplicity. Entra ID + authentication on the P2S is stronger (revocation, conditional access). +- **ACR**: `publicNetworkAccess` is Enabled for first-run `az acr import`. For a + fully private cluster, switch ACR to Premium + Private Endpoint and disable + public access once images are imported. +- **Egress**: default `outboundType: loadBalancer` allows node egress to GHCR / + Azure. For a locked-down network use `userDefinedRouting` + Azure Firewall and + Option B (ACR import) so no pull traffic leaves the VNet. +``` diff --git a/deploy/aks/dashboards/atioz-logs-errors.json b/deploy/aks/dashboards/atioz-logs-errors.json new file mode 100644 index 000000000..e735b79e2 --- /dev/null +++ b/deploy/aks/dashboards/atioz-logs-errors.json @@ -0,0 +1,72 @@ +{ + "dashboard": { + "uid": "atioz-logs-errors", + "title": "ATIOZ Logs & Errors", + "tags": ["atioz", "logs"], + "schemaVersion": 39, + "refresh": "30s", + "time": { "from": "now-3h", "to": "now" }, + "templating": { + "list": [ + { "type": "textbox", "name": "filter", "label": "Text filter (regex)", "query": "", "current": { "text": "", "value": "" } } + ] + }, + "panels": [ + { + "type": "timeseries", + "title": "Log volume by level", + "gridPos": { "h": 8, "w": 16, "x": 0, "y": 0 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { "defaults": { "custom": { "drawStyle": "bars", "fillOpacity": 70, "stacking": { "mode": "normal" } }, "unit": "short" }, "overrides": [{ "matcher": { "id": "byName", "options": "errors" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] }, { "matcher": { "id": "byName", "options": "warnings" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } }] }, { "matcher": { "id": "byName", "options": "info" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] }] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"atioz\"} |~ `^info:` [$__auto]))", "legendFormat": "info", "queryType": "range" }, + { "refId": "B", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"atioz\"} |~ `^warn:` [$__auto]))", "legendFormat": "warnings", "queryType": "range" }, + { "refId": "C", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"atioz\"} |~ `^(fail|crit):` [$__auto]))", "legendFormat": "errors", "queryType": "range" } + ] + }, + { + "type": "stat", + "title": "Total log events (range)", + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [{ "color": "blue", "value": null }] }, "unit": "short" }, "overrides": [] }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto" }, + "targets": [{ "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"atioz\"} |~ `^(info|warn|fail|crit|dbug|trce):` [$__range]))", "queryType": "instant" }] + }, + { + "type": "stat", + "title": "Errors (range)", + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 4 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, "unit": "short" }, "overrides": [] }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "textMode": "auto" }, + "targets": [{ "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"atioz\"} |~ `^(fail|crit):` [$__range]))", "queryType": "instant" }] + }, + { + "type": "table", + "title": "Top warning / error categories (range)", + "gridPos": { "h": 9, "w": 24, "x": 0, "y": 8 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { "defaults": { "custom": { "align": "auto" } }, "overrides": [] }, + "options": { "showHeader": true, "sortBy": [{ "displayName": "Count", "desc": true }] }, + "targets": [{ "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "topk(20, sum by (category) (count_over_time({namespace=\"atioz\"} |~ `^(warn|fail|crit):` | regexp `^\\w{4}: (?P[A-Za-z0-9_.]+)` [$__range])))", "queryType": "instant" }], + "transformations": [ + { "id": "labelsToFields", "options": {} }, + { "id": "organize", "options": { "excludeByName": { "Time": true }, "renameByName": { "category": "Category", "Value": "Count" } } } + ] + }, + { + "type": "logs", + "title": "Live logs (use the Text filter variable above)", + "gridPos": { "h": 11, "w": 24, "x": 0, "y": 17 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true, "dedupStrategy": "none", "prettifyLogMessage": false }, + "targets": [{ "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "{namespace=\"atioz\"} |~ `(?i)${filter:raw}`", "queryType": "range" }] + } + ] + }, + "overwrite": true, + "folderId": 0, + "message": "Provisioned by Claude Code" +} diff --git a/deploy/aks/dashboards/atioz-overview.json b/deploy/aks/dashboards/atioz-overview.json new file mode 100644 index 000000000..6f55c881a --- /dev/null +++ b/deploy/aks/dashboards/atioz-overview.json @@ -0,0 +1,91 @@ +{ + "dashboard": { + "uid": "atioz-overview", + "title": "ATIOZ Overview", + "tags": ["atioz"], + "schemaVersion": 39, + "refresh": "30s", + "time": { "from": "now-6h", "to": "now" }, + "templating": { "list": [] }, + "panels": [ + { + "type": "stat", + "title": "Portal replicas ready", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, "unit": "short" }, "overrides": [] }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "background", "graphMode": "none", "textMode": "auto" }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(kube_deployment_status_replicas_available{namespace=\"atioz\",deployment=~\"memex-portal.*\"})" }] + }, + { + "type": "stat", + "title": "Pods running (atioz)", + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [{ "color": "green", "value": null }] }, "unit": "short" }, "overrides": [] }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "value", "graphMode": "none", "textMode": "auto" }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "count(kube_pod_status_phase{namespace=\"atioz\",phase=\"Running\"} == 1)" }] + }, + { + "type": "stat", + "title": "Error log events (range)", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, "unit": "short" }, "overrides": [] }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "textMode": "auto" }, + "targets": [{ "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"atioz\"} |~ `^(fail|crit):` [$__range]))", "queryType": "instant" }] + }, + { + "type": "stat", + "title": "Warning log events (range)", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }] }, "unit": "short" }, "overrides": [] }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "textMode": "auto" }, + "targets": [{ "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"atioz\"} |~ `^warn:` [$__range]))", "queryType": "instant" }] + }, + { + "type": "timeseries", + "title": "Log volume by level", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { "defaults": { "custom": { "drawStyle": "bars", "fillOpacity": 60, "stacking": { "mode": "normal" } }, "unit": "short" }, "overrides": [{ "matcher": { "id": "byName", "options": "errors" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] }, { "matcher": { "id": "byName", "options": "warnings" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } }] }, { "matcher": { "id": "byName", "options": "info" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] }] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"atioz\"} |~ `^info:` [$__auto]))", "legendFormat": "info", "queryType": "range" }, + { "refId": "B", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"atioz\"} |~ `^warn:` [$__auto]))", "legendFormat": "warnings", "queryType": "range" }, + { "refId": "C", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"atioz\"} |~ `^(fail|crit):` [$__auto]))", "legendFormat": "errors", "queryType": "range" } + ] + }, + { + "type": "timeseries", + "title": "Portal CPU (cores)", + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15 }, "unit": "short" }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"atioz\",pod=~\"memex-portal.*\",container!=\"\",container!=\"POD\"}[5m]))", "legendFormat": "cpu cores" }] + }, + { + "type": "timeseries", + "title": "Portal memory (MiB)", + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15 }, "unit": "decmbytes" }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(container_memory_working_set_bytes{namespace=\"atioz\",pod=~\"memex-portal.*\",container!=\"\",container!=\"POD\"}) / 1024 / 1024", "legendFormat": "working set" }] + }, + { + "type": "logs", + "title": "Recent errors & warnings", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 12 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true, "dedupStrategy": "none", "prettifyLogMessage": false }, + "targets": [{ "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "{namespace=\"atioz\"} |~ `(?i)(error|exception|fail|warn|crit)`", "queryType": "range" }] + } + ] + }, + "overwrite": true, + "folderId": 0, + "message": "Provisioned by Claude Code" +} diff --git a/deploy/aks/dashboards/memex-cluster-pods.json b/deploy/aks/dashboards/memex-cluster-pods.json new file mode 100644 index 000000000..df1e0f917 --- /dev/null +++ b/deploy/aks/dashboards/memex-cluster-pods.json @@ -0,0 +1,73 @@ +{ + "dashboard": { + "uid": "memex-cluster-pods", + "title": "Memex Cluster & Pods", + "tags": ["memex", "infra"], + "schemaVersion": 39, + "refresh": "30s", + "time": { "from": "now-6h", "to": "now" }, + "templating": { "list": [] }, + "panels": [ + { + "type": "timeseries", + "title": "Node CPU used (%)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15 }, "unit": "percent", "min": 0, "max": 100 }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "100 * (1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])))", "legendFormat": "{{instance}}" }] + }, + { + "type": "timeseries", + "title": "Node memory available (%)", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15 }, "unit": "percent", "min": 0, "max": 100 }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "100 * node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes", "legendFormat": "{{instance}}" }] + }, + { + "type": "table", + "title": "Pods (memex namespace)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { "defaults": { "custom": { "align": "auto" } }, "overrides": [] }, + "options": { "showHeader": true }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "kube_pod_status_phase{namespace=\"memex\"} == 1", "format": "table", "instant": true }], + "transformations": [ + { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": true, "__name__": true, "container": true, "endpoint": true, "job": true, "instance": true, "service": true, "uid": true, "namespace": true }, "renameByName": { "pod": "Pod", "phase": "Phase" } } } + ] + }, + { + "type": "timeseries", + "title": "Pod container restarts (memex)", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 10 }, "unit": "short" }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum by (pod) (kube_pod_container_status_restarts_total{namespace=\"memex\"})", "legendFormat": "{{pod}}" }] + }, + { + "type": "timeseries", + "title": "Container CPU by pod (cores)", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15 }, "unit": "short" }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"memex\",container!=\"\",container!=\"POD\"}[5m]))", "legendFormat": "{{pod}}" }] + }, + { + "type": "timeseries", + "title": "Container memory by pod (MiB)", + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15 }, "unit": "decmbytes" }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"memex\",container!=\"\",container!=\"POD\"}) / 1024 / 1024", "legendFormat": "{{pod}}" }] + } + ] + }, + "overwrite": true, + "folderId": 0, + "message": "Provisioned by Claude Code" +} diff --git a/deploy/aks/dashboards/memex-logs-errors.json b/deploy/aks/dashboards/memex-logs-errors.json new file mode 100644 index 000000000..c787001a8 --- /dev/null +++ b/deploy/aks/dashboards/memex-logs-errors.json @@ -0,0 +1,72 @@ +{ + "dashboard": { + "uid": "memex-logs-errors", + "title": "Memex Logs & Errors", + "tags": ["memex", "logs"], + "schemaVersion": 39, + "refresh": "30s", + "time": { "from": "now-3h", "to": "now" }, + "templating": { + "list": [ + { "type": "textbox", "name": "filter", "label": "Text filter (regex)", "query": "", "current": { "text": "", "value": "" } } + ] + }, + "panels": [ + { + "type": "timeseries", + "title": "Log volume by level", + "gridPos": { "h": 8, "w": 16, "x": 0, "y": 0 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { "defaults": { "custom": { "drawStyle": "bars", "fillOpacity": 70, "stacking": { "mode": "normal" } }, "unit": "short" }, "overrides": [{ "matcher": { "id": "byName", "options": "errors" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] }, { "matcher": { "id": "byName", "options": "warnings" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } }] }, { "matcher": { "id": "byName", "options": "info" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] }] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"memex\"} |~ `^info:` [$__auto]))", "legendFormat": "info", "queryType": "range" }, + { "refId": "B", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"memex\"} |~ `^warn:` [$__auto]))", "legendFormat": "warnings", "queryType": "range" }, + { "refId": "C", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"memex\"} |~ `^(fail|crit):` [$__auto]))", "legendFormat": "errors", "queryType": "range" } + ] + }, + { + "type": "stat", + "title": "Total log events (range)", + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [{ "color": "blue", "value": null }] }, "unit": "short" }, "overrides": [] }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "value", "graphMode": "area", "textMode": "auto" }, + "targets": [{ "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"memex\"} |~ `^(info|warn|fail|crit|dbug|trce):` [$__range]))", "queryType": "instant" }] + }, + { + "type": "stat", + "title": "Errors (range)", + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 4 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, "unit": "short" }, "overrides": [] }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "textMode": "auto" }, + "targets": [{ "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"memex\"} |~ `^(fail|crit):` [$__range]))", "queryType": "instant" }] + }, + { + "type": "table", + "title": "Top warning / error categories (range)", + "gridPos": { "h": 9, "w": 24, "x": 0, "y": 8 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { "defaults": { "custom": { "align": "auto" } }, "overrides": [] }, + "options": { "showHeader": true, "sortBy": [{ "displayName": "Count", "desc": true }] }, + "targets": [{ "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "topk(20, sum by (category) (count_over_time({namespace=\"memex\"} |~ `^(warn|fail|crit):` | regexp `^\\w{4}: (?P[A-Za-z0-9_.]+)` [$__range])))", "queryType": "instant" }], + "transformations": [ + { "id": "labelsToFields", "options": {} }, + { "id": "organize", "options": { "excludeByName": { "Time": true }, "renameByName": { "category": "Category", "Value": "Count" } } } + ] + }, + { + "type": "logs", + "title": "Live logs (use the Text filter variable above)", + "gridPos": { "h": 11, "w": 24, "x": 0, "y": 17 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true, "dedupStrategy": "none", "prettifyLogMessage": false }, + "targets": [{ "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "{namespace=\"memex\"} |~ `(?i)${filter:raw}`", "queryType": "range" }] + } + ] + }, + "overwrite": true, + "folderId": 0, + "message": "Provisioned by Claude Code" +} diff --git a/deploy/aks/dashboards/memex-overview.json b/deploy/aks/dashboards/memex-overview.json new file mode 100644 index 000000000..39560c235 --- /dev/null +++ b/deploy/aks/dashboards/memex-overview.json @@ -0,0 +1,91 @@ +{ + "dashboard": { + "uid": "memex-overview", + "title": "Memex Overview", + "tags": ["memex"], + "schemaVersion": 39, + "refresh": "30s", + "time": { "from": "now-6h", "to": "now" }, + "templating": { "list": [] }, + "panels": [ + { + "type": "stat", + "title": "Portal replicas ready", + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] }, "unit": "short" }, "overrides": [] }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "background", "graphMode": "none", "textMode": "auto" }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(kube_deployment_status_replicas_available{namespace=\"memex\",deployment=~\"memex-portal.*\"})" }] + }, + { + "type": "stat", + "title": "Pods running (memex)", + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [{ "color": "green", "value": null }] }, "unit": "short" }, "overrides": [] }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "value", "graphMode": "none", "textMode": "auto" }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "count(kube_pod_status_phase{namespace=\"memex\",phase=\"Running\"} == 1)" }] + }, + { + "type": "stat", + "title": "Error log events (range)", + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 1 }] }, "unit": "short" }, "overrides": [] }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "textMode": "auto" }, + "targets": [{ "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"memex\"} |~ `^(fail|crit):` [$__range]))", "queryType": "instant" }] + }, + { + "type": "stat", + "title": "Warning log events (range)", + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "thresholds": { "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }] }, "unit": "short" }, "overrides": [] }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "colorMode": "background", "graphMode": "area", "textMode": "auto" }, + "targets": [{ "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"memex\"} |~ `^warn:` [$__range]))", "queryType": "instant" }] + }, + { + "type": "timeseries", + "title": "Log volume by level", + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "fieldConfig": { "defaults": { "custom": { "drawStyle": "bars", "fillOpacity": 60, "stacking": { "mode": "normal" } }, "unit": "short" }, "overrides": [{ "matcher": { "id": "byName", "options": "errors" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] }, { "matcher": { "id": "byName", "options": "warnings" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "yellow" } }] }, { "matcher": { "id": "byName", "options": "info" }, "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] }] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "multi" } }, + "targets": [ + { "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"memex\"} |~ `^info:` [$__auto]))", "legendFormat": "info", "queryType": "range" }, + { "refId": "B", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"memex\"} |~ `^warn:` [$__auto]))", "legendFormat": "warnings", "queryType": "range" }, + { "refId": "C", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "sum(count_over_time({namespace=\"memex\"} |~ `^(fail|crit):` [$__auto]))", "legendFormat": "errors", "queryType": "range" } + ] + }, + { + "type": "timeseries", + "title": "Portal CPU (cores)", + "gridPos": { "h": 8, "w": 6, "x": 12, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15 }, "unit": "short" }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"memex\",pod=~\"memex-portal.*\",container!=\"\",container!=\"POD\"}[5m]))", "legendFormat": "cpu cores" }] + }, + { + "type": "timeseries", + "title": "Portal memory (MiB)", + "gridPos": { "h": 8, "w": 6, "x": 18, "y": 4 }, + "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", "fillOpacity": 15 }, "unit": "decmbytes" }, "overrides": [] }, + "options": { "legend": { "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, + "targets": [{ "refId": "A", "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "expr": "sum(container_memory_working_set_bytes{namespace=\"memex\",pod=~\"memex-portal.*\",container!=\"\",container!=\"POD\"}) / 1024 / 1024", "legendFormat": "working set" }] + }, + { + "type": "logs", + "title": "Recent errors & warnings", + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 12 }, + "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, + "options": { "showTime": true, "wrapLogMessage": true, "sortOrder": "Descending", "enableLogDetails": true, "dedupStrategy": "none", "prettifyLogMessage": false }, + "targets": [{ "refId": "A", "datasource": { "type": "loki", "uid": "P8E80F9AEF21F6940" }, "expr": "{namespace=\"memex\"} |~ `(?i)(error|exception|fail|warn|crit)`", "queryType": "range" }] + } + ] + }, + "overwrite": true, + "folderId": 0, + "message": "Provisioned by Claude Code" +} diff --git a/deploy/aks/envs/.gitignore b/deploy/aks/envs/.gitignore new file mode 100644 index 000000000..7fc1e9769 --- /dev/null +++ b/deploy/aks/envs/.gitignore @@ -0,0 +1,9 @@ +# Per-environment deployment config — NOT version-controlled. +# These carry deployment-specific values (Entra client/tenant ids, OAuth client +# ids, sender mailbox, Key Vault secret names, host/db wiring). The deploy +# scripts read them from disk; they are managed out-of-band, not in git. +# (The *.deploy.example.yaml templates and the reusable scripts/manifests/README +# stay tracked.) +*/values.*.yaml +!*/values.deploy.example.yaml +*/secretproviderclass.yaml diff --git a/deploy/aks/envs/memex-cloud/deploy.sh b/deploy/aks/envs/memex-cloud/deploy.sh new file mode 100644 index 000000000..1d3f1bd41 --- /dev/null +++ b/deploy/aks/envs/memex-cloud/deploy.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Deploy the migrated memex.meshweaver.cloud portal onto the shared AKS cluster, +# namespace `memex-cloud`, on the D16s_v5 `silos` pool, against the `memexcloud` +# database (already loaded with the prod data — see migrate-db.sh). +# +# The db-migration is NOT run here: the data was restored from prod at its +# current schema version, so we deploy against it as-is (run the migration +# deliberately only if a schema delta is needed). Stage like the atioz env: +# STAGE with this script + values.memexcloud.yaml + portal-pvcs.yaml + +# portal-ingress.yaml + secretproviderclass.yaml + portal-patch.json + ./helm +# export MEMEX_PG_CONN='Host=10.42.18.4;...;Database=memexcloud;SslMode=Require;Trust Server Certificate=true' +# export IMAGE_TAG= +# az aks command invoke -g memex-aks-rg -n memexaks-cluster \ +# --command "MEMEX_PG_CONN='$MEMEX_PG_CONN' IMAGE_TAG='$IMAGE_TAG' bash deploy.sh" --file . +set -uo pipefail +NS=memex-cloud +RELEASE=memexcloud +ACR=meshweaver.azurecr.io +IMAGE_TAG="${IMAGE_TAG:-latest}" +: "${MEMEX_PG_CONN:?set MEMEX_PG_CONN to the Flexible Server connection string for the memexcloud database}" + +kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - +kubectl apply -f ./portal-pvcs.yaml + +helm upgrade --install "$RELEASE" ./helm \ + -f ./helm/values.yaml -f ./values.memexcloud.yaml -n "$NS" + +# Use the shared Flexible Server (`memexcloud`) — don't run the chart's in-cluster +# pg, and DON'T run the migration (data already restored from prod). +kubectl -n "$NS" scale statefulset memex-postgres-statefulset --replicas=0 || true +kubectl -n "$NS" scale deployment memex-migration-deployment --replicas=0 || true + +kubectl -n "$NS" set image deployment/memex-portal-deployment memex-portal="$ACR/memex-portal-ai:$IMAGE_TAG" + +kubectl apply -f ./secretproviderclass.yaml +kubectl -n "$NS" patch deployment memex-portal-deployment --type=json --patch-file ./portal-patch.json + +kubectl -n "$NS" patch secret memex-portal-secrets --type merge \ + -p "{\"stringData\":{\"ConnectionStrings__memex\":\"${MEMEX_PG_CONN}\"}}" || true + +kubectl apply -f ./portal-ingress.yaml +kubectl -n "$NS" rollout restart deployment/memex-portal-deployment || true +echo "=== memex-cloud deployed ==="; kubectl -n "$NS" get deploy,pvc,svc,ingress -o wide diff --git a/deploy/aks/envs/memex-cloud/migrate-db.sh b/deploy/aks/envs/memex-cloud/migrate-db.sh new file mode 100644 index 000000000..00a13ae4f --- /dev/null +++ b/deploy/aks/envs/memex-cloud/migrate-db.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# One-shot prod DB migration: dump memex.meshweaver.cloud's prod Postgres +# (memexpostgres-d272wxvys4nvo, Entra-auth) and restore into the `memexcloud` +# database on the shared memexaks-pg Flexible Server. +# +# Runs a postgres:16 pod inside the cluster (the only place with line-of-sight to +# BOTH the prod PG public endpoint — via the AKS egress IP, firewall-allowed — and +# the private memexaks-pg at 10.42.18.4). Reads: +# TOKEN = an AAD access token for an Entra admin on the prod PG (rbuergi@systemorph.com) +# PW = the memexaks-pg `memexadmin` password +# both provided as env on the `az aks command invoke` that runs this file. +set -uo pipefail +: "${TOKEN:?set TOKEN to a prod-PG AAD access token}" +: "${PW:?set PW to the memexaks-pg memexadmin password}" + +kubectl -n default delete pod pgmig --ignore-not-found >/dev/null 2>&1 +kubectl -n default delete secret pgmig-creds --ignore-not-found >/dev/null 2>&1 +kubectl -n default create secret generic pgmig-creds \ + --from-literal=TOKEN="$TOKEN" --from-literal=PW="$PW" >/dev/null + +cat > /tmp/pgmig.yaml <<'YAML' +apiVersion: v1 +kind: Pod +metadata: { name: pgmig, namespace: default } +spec: + restartPolicy: Never + containers: + - name: pgmig + image: postgres:16 + command: ["bash","-c"] + args: + - | + set -uo pipefail + export PGSSLMODE=require + echo "== dump prod (memex) ==" + PGPASSWORD="$TOKEN" pg_dump --no-owner --no-acl --verbose \ + -h memexpostgres-d272wxvys4nvo.postgres.database.azure.com \ + -U 'rbuergi@systemorph.com' -d memex -f /tmp/d.sql 2>/tmp/dump.err + echo "DUMP_EXIT=$? bytes=$(wc -c /dev/null)" + tail -3 /tmp/dump.err + echo "== restore -> memexcloud ==" + PGPASSWORD="$PW" psql -v ON_ERROR_STOP=0 \ + -h 10.42.18.4 -U memexadmin -d memexcloud -f /tmp/d.sql >/tmp/r.log 2>&1 + echo "RESTORE_PSQL_EXIT=$?" + echo "-- restore errors (if any) --" + grep -iE 'ERROR|FATAL' /tmp/r.log | grep -viE 'already exists|does not exist, skipping' | head -20 || true + echo "-- table counts in memexcloud --" + PGPASSWORD="$PW" psql -h 10.42.18.4 -U memexadmin -d memexcloud -tAc \ + "select count(*) || ' tables' from information_schema.tables where table_schema not in ('pg_catalog','information_schema')" 2>&1 + echo "DONE" + env: + - { name: TOKEN, valueFrom: { secretKeyRef: { name: pgmig-creds, key: TOKEN } } } + - { name: PW, valueFrom: { secretKeyRef: { name: pgmig-creds, key: PW } } } +YAML + +kubectl apply -f /tmp/pgmig.yaml >/dev/null +echo "pgmig pod created; waiting for completion (up to 10m)..." +kubectl -n default wait --for=jsonpath='{.status.phase}'=Succeeded pod/pgmig --timeout=600s 2>&1 || true +echo "===== pgmig logs =====" +kubectl -n default logs pgmig --tail=60 +echo "===== phase =====" +kubectl -n default get pod pgmig -o jsonpath='{.status.phase}'; echo +kubectl -n default delete secret pgmig-creds --ignore-not-found >/dev/null 2>&1 diff --git a/deploy/aks/envs/memex-cloud/portal-ingress.yaml b/deploy/aks/envs/memex-cloud/portal-ingress.yaml new file mode 100644 index 000000000..d0c5c4109 --- /dev/null +++ b/deploy/aks/envs/memex-cloud/portal-ingress.yaml @@ -0,0 +1,31 @@ +# Ingress for memex.meshweaver.cloud (ns memex-cloud), cookie session affinity. +# kubectl apply -n memex-cloud -f portal-ingress.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: memex-portal + namespace: memex-cloud + annotations: + nginx.ingress.kubernetes.io/affinity: "cookie" + nginx.ingress.kubernetes.io/affinity-mode: "persistent" + nginx.ingress.kubernetes.io/session-cookie-name: "MEMEXCLOUD_AFFINITY" + nginx.ingress.kubernetes.io/session-cookie-max-age: "172800" + nginx.ingress.kubernetes.io/session-cookie-samesite: "Lax" + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-body-size: "100m" +spec: + ingressClassName: webapprouting.kubernetes.azure.com + tls: + - hosts: [memex.meshweaver.cloud] + secretName: memexcloud-tls + rules: + - host: memex.meshweaver.cloud + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: memex-portal-service + port: { number: 8080 } diff --git a/deploy/aks/envs/memex-cloud/portal-patch.json b/deploy/aks/envs/memex-cloud/portal-patch.json new file mode 100644 index 000000000..486cb3711 --- /dev/null +++ b/deploy/aks/envs/memex-cloud/portal-patch.json @@ -0,0 +1,38 @@ +[ + { "op": "replace", "path": "/spec/replicas", "value": 1 }, + { "op": "add", "path": "/spec/template/spec/nodeSelector", "value": { "workload": "silos" } }, + + { "op": "replace", "path": "/spec/template/spec/volumes/0", + "value": { "name": "memex-data", "persistentVolumeClaim": { "claimName": "memex-data" } } }, + { "op": "replace", "path": "/spec/template/spec/volumes/1", + "value": { "name": "memex-users", "persistentVolumeClaim": { "claimName": "memex-users" } } }, + { "op": "add", "path": "/spec/template/spec/volumes/-", + "value": { "name": "memex-content", "persistentVolumeClaim": { "claimName": "memex-content" } } }, + { "op": "add", "path": "/spec/template/spec/volumes/-", + "value": { "name": "memex-attachments", "persistentVolumeClaim": { "claimName": "memex-attachments" } } }, + { "op": "add", "path": "/spec/template/spec/volumes/-", + "value": { "name": "memex-workspace", "persistentVolumeClaim": { "claimName": "memex-workspace" } } }, + { "op": "add", "path": "/spec/template/spec/volumes/-", + "value": { "name": "kv-ai-secrets", + "csi": { "driver": "secrets-store.csi.k8s.io", "readOnly": true, + "volumeAttributes": { "secretProviderClass": "memexcloud-portal-ai-secrets" } } } }, + + { "op": "add", "path": "/spec/template/spec/containers/0/volumeMounts/-", + "value": { "name": "memex-content", "mountPath": "/mnt/content" } }, + { "op": "add", "path": "/spec/template/spec/containers/0/volumeMounts/-", + "value": { "name": "memex-attachments", "mountPath": "/mnt/attachments" } }, + { "op": "add", "path": "/spec/template/spec/containers/0/volumeMounts/-", + "value": { "name": "memex-workspace", "mountPath": "/workspace" } }, + { "op": "add", "path": "/spec/template/spec/containers/0/volumeMounts/-", + "value": { "name": "kv-ai-secrets", "mountPath": "/mnt/secrets-store", "readOnly": true } }, + + { "op": "add", "path": "/spec/template/spec/containers/0/envFrom/-", + "value": { "secretRef": { "name": "memexcloud-portal-ai-secrets" } } }, + + { "op": "add", "path": "/spec/template/spec/containers/0/resources", + "value": { "requests": { "cpu": "4", "memory": "8Gi" }, "limits": { "cpu": "6", "memory": "16Gi" } } }, + { "op": "add", "path": "/spec/template/spec/containers/0/readinessProbe", + "value": { "httpGet": { "path": "/", "port": 8080 }, "initialDelaySeconds": 20, "periodSeconds": 10, "failureThreshold": 6 } }, + { "op": "add", "path": "/spec/template/spec/containers/0/livenessProbe", + "value": { "httpGet": { "path": "/", "port": 8080 }, "initialDelaySeconds": 90, "periodSeconds": 20, "failureThreshold": 6 } } +] diff --git a/deploy/aks/envs/memex-cloud/portal-pvcs.yaml b/deploy/aks/envs/memex-cloud/portal-pvcs.yaml new file mode 100644 index 000000000..6947fc11d --- /dev/null +++ b/deploy/aks/envs/memex-cloud/portal-pvcs.yaml @@ -0,0 +1,29 @@ +# RWX Azure Files PVCs for the memex-cloud portal (ns memex-cloud). Uses the +# cluster-wide azurefile-memex StorageClass. No pg PVC (uses memexaks-pg). +# kubectl apply -n memex-cloud -f portal-pvcs.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: { name: memex-data, namespace: memex-cloud, labels: { app.kubernetes.io/component: memex-portal } } +spec: { accessModes: [ReadWriteMany], storageClassName: azurefile-memex, resources: { requests: { storage: 16Gi } } } +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: { name: memex-content, namespace: memex-cloud, labels: { app.kubernetes.io/component: memex-portal } } +spec: { accessModes: [ReadWriteMany], storageClassName: azurefile-memex, resources: { requests: { storage: 128Gi } } } +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: { name: memex-attachments, namespace: memex-cloud, labels: { app.kubernetes.io/component: memex-portal } } +spec: { accessModes: [ReadWriteMany], storageClassName: azurefile-memex, resources: { requests: { storage: 64Gi } } } +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: { name: memex-users, namespace: memex-cloud, labels: { app.kubernetes.io/component: memex-portal } } +spec: { accessModes: [ReadWriteMany], storageClassName: azurefile-memex, resources: { requests: { storage: 32Gi } } } +--- +# Per-user git working trees (/workspace/{userId}/{repoSlug}) for the in-portal checkout/edit/commit +# feature + the co-hosted AI CLIs. RWX so it survives pod restarts and is shared across replicas. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: { name: memex-workspace, namespace: memex-cloud, labels: { app.kubernetes.io/component: memex-portal } } +spec: { accessModes: [ReadWriteMany], storageClassName: azurefile-memex, resources: { requests: { storage: 64Gi } } } diff --git a/deploy/aks/envs/memex-cloud/tls.sh b/deploy/aks/envs/memex-cloud/tls.sh new file mode 100644 index 000000000..3e6ab7062 --- /dev/null +++ b/deploy/aks/envs/memex-cloud/tls.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Issue the Let's Encrypt cert for memex.meshweaver.cloud on the memex-cloud +# ingress (reuses the cluster-wide cert-manager + letsencrypt-prod issuer). +# Run ONLY AFTER memex.meshweaver.cloud's DNS A-record points to the AKS ingress +# IP (HTTP-01 validates over the internet) — i.e. at/after the DNS cutover. +# az aks command invoke -g memex-aks-rg -n memexaks-cluster --command "bash tls.sh" --file tls.sh +set -uo pipefail +NS=memex-cloud +HOST="${INGRESS_HOST:-memex.meshweaver.cloud}" +kubectl -n "$NS" annotate ingress memex-portal cert-manager.io/cluster-issuer=letsencrypt-prod --overwrite +kubectl -n "$NS" patch ingress memex-portal --type=json \ + -p "[{\"op\":\"add\",\"path\":\"/spec/tls\",\"value\":[{\"hosts\":[\"${HOST}\"],\"secretName\":\"memexcloud-tls\"}]}]" || true +echo "=== memexcloud cert issuing (HTTP-01); watch: kubectl -n $NS get certificate memexcloud-tls -w ===" diff --git a/deploy/aks/infra/main.bicep b/deploy/aks/infra/main.bicep new file mode 100644 index 000000000..d69d41ed2 --- /dev/null +++ b/deploy/aks/infra/main.bicep @@ -0,0 +1,273 @@ +// =========================================================================== +// main.bicep — Top-level infrastructure for the MeshWeaver Memex AKS SAMPLE. +// +// Scope: subscription. Creates the resource group, then wires: +// network.bicep -> VNet + subnets + private DNS zone (privatelink..azmk8s.io) +// acr.bicep -> Azure Container Registry (private image cache/mirror) +// aks.bicep -> PRIVATE AKS cluster (private API server) + AcrPull grant +// vpn.bicep -> P2S VPN Gateway so kubectl reaches the private API server +// storage.bicep -> Blob storage + Workload Identity for pgBackRest PITR +// +// Deploy: +// az deployment sub create \ +// --location \ +// --template-file main.bicep \ +// --parameters @main.parameters.json +// +// This provisions INFRA ONLY. The portal itself is installed afterwards with +// Helm using ../values.aks.yaml against the existing ../helm chart (see README). +// +// NOTE on Aspire: this sample is the operator-facing IaC. The repo's Aspire +// model (deploy/aspire/Memex.Deploy.AppHost) generated the ../helm chart and the +// ../aca bicep; it does not emit a private-AKS + VPN + pgBackRest topology. +// See ../README.md "Generating this from Aspire" for how this sample relates to +// the Aspire publish output and how to keep them in sync. +// =========================================================================== + +targetScope = 'subscription' + +@description('Resource group to create / deploy into.') +param resourceGroupName string = 'memex-aks-rg' + +@description('Azure region for all resources.') +param location string = 'westeurope' + +@description('Short name prefix used across resources (lowercase, <= 12 chars).') +@maxLength(12) +param namePrefix string = 'memexaks' + +// --- ACR ------------------------------------------------------------------- +@description('Login server of an EXISTING shared ACR (e.g. meshweaver.azurecr.io). When set, NO per-RG ACR is created — images are pulled from this shared registry, and the cluster kubelet must be granted AcrPull on it out-of-band (cross-RG, see README). Leave empty to create a dedicated per-RG ACR.') +param sharedAcrLoginServer string = 'meshweaver.azurecr.io' + +@description('ACR name (globally unique). Only used when sharedAcrLoginServer is empty. Defaults to a hashed name.') +param acrName string = '' + +@description('ACR SKU. Only used when sharedAcrLoginServer is empty.') +param acrSku string = 'Premium' + +// --- AKS ------------------------------------------------------------------- +@description('Kubernetes version. Empty = region default.') +param kubernetesVersion string = '' + +@description('System node pool VM size.') +param systemNodeVmSize string = 'Standard_D4s_v5' + +@description('Initial / desired node count.') +param systemNodeCount int = 3 + +@description('Enable cluster autoscaler.') +param enableAutoScaling bool = true + +@description('Autoscaler min nodes.') +param minNodeCount int = 3 + +@description('Autoscaler max nodes.') +param maxNodeCount int = 6 + +@description('Availability zones for nodes.') +param availabilityZones array = [ + '1' + '2' + '3' +] + +// --- Networking ------------------------------------------------------------ +@description('VNet address space.') +param vnetAddressSpace string = '10.42.0.0/16' + +// --- VPN ------------------------------------------------------------------- +@description('Deploy the P2S VPN Gateway (set false to use az aks command invoke / Bastion instead).') +param deployVpnGateway bool = true + +@description('P2S client address pool (must not overlap the VNet).') +param vpnClientAddressPool string = '172.16.201.0/24' + +@description('Base64 public cert data of the P2S root cert (single line, no PEM headers). Empty = add later.') +param vpnClientRootCertData string = '' + +@description('VPN Gateway SKU. Must be a zone-redundant *AZ SKU — Azure retired the non-AZ VpnGw1-5 SKUs (NonAzSkusNotAllowedForVPNGateway).') +param gatewaySku string = 'VpnGw1AZ' + +// --- Backup storage -------------------------------------------------------- +@description('Deploy the pgBackRest blob storage + workload identity (self-managed PITR). Set false if using Azure DB Flexible Server.') +param deployBackupStorage bool = true + +@description('Storage account name for pgBackRest (globally unique, 3-24 lowercase). Empty = hashed name.') +param backupStorageAccountName string = '' + +@description('Kubernetes namespace + service account the pgBackRest pods run as.') +param pgBackRestServiceAccountSubject string = 'system:serviceaccount:memex:pgbackrest-sa' + +// --- Content / log Azure Files shares -------------------------------------- +@description('Provision the dedicated Azure Files account with named shares (content, attachments, users, data, otel-logs) for STATIC PV binding. Dynamic azurefile provisioning is the default and does not require this.') +param deployContentFileShares bool = true + +@description('Azure Files account name for content + observability shares (globally unique, 3-24 lowercase). Empty = hashed name.') +param contentFilesAccountName string = '' + +// --- Private PostgreSQL Flexible Server ------------------------------------ +@description('Provision a PRIVATE (VNet-injected) Azure Database for PostgreSQL Flexible Server with pgvector. When true, point the portal at its FQDN and set deployBackupStorage=false (managed PITR replaces pgBackRest).') +param deployPostgresFlexible bool = true + +@description('Flexible Server name (globally unique). Empty = "-pg".') +param postgresServerName string = '' + +@description('PostgreSQL administrator password. REQUIRED when deployPostgresFlexible=true. Pass at deploy time (--parameters or env); never commit it.') +@secure() +param postgresAdminPassword string = '' + +@description('Flexible Server compute SKU (Standard_D2ds_v5 = 2 vCPU/8 GiB; Standard_D4ds_v5 = 4 vCPU/16 GiB).') +param postgresSkuName string = 'Standard_D2ds_v5' + +@description('Flexible Server zone-redundant HA (standby in another AZ). Off by default — not every region allows it (e.g. westeurope returns HADisabledForRegion); enable only where supported.') +param postgresHighAvailability bool = false + +@description('Tags applied to all resources.') +param tags object = { + project: 'meshweaver-memex' + sample: 'aks' +} + +// Deterministic unique names where the caller didn't supply one. +var effectiveAcrName = empty(acrName) ? take('${namePrefix}acr${uniqueString(subscription().id, resourceGroupName)}', 50) : acrName +var effectiveBackupSa = empty(backupStorageAccountName) ? take('${namePrefix}bkp${uniqueString(subscription().id, resourceGroupName)}', 24) : backupStorageAccountName +var effectiveContentFilesSa = empty(contentFilesAccountName) ? take('${namePrefix}files${uniqueString(subscription().id, resourceGroupName)}', 24) : contentFilesAccountName +var effectivePostgresServer = empty(postgresServerName) ? '${namePrefix}-pg' : postgresServerName +var clusterName = '${namePrefix}-cluster' + +// Shared-ACR axis: when sharedAcrLoginServer is set, no per-RG ACR is created and the +// kubelet's AcrPull on the shared registry is granted out-of-band (cross-RG — the role +// assignment can't be authored from this RG-scoped module). effectiveAcrId is then empty, +// which makes aks.bicep skip its (same-RG) AcrPull grant. +var useSharedAcr = !empty(sharedAcrLoginServer) +var effectiveAcrId = useSharedAcr ? '' : acr!.outputs.acrId +var effectiveAcrLoginServer = useSharedAcr ? sharedAcrLoginServer : acr!.outputs.acrLoginServer + +resource rg 'Microsoft.Resources/resourceGroups@2024-03-01' = { + name: resourceGroupName + location: location + tags: tags +} + +module network 'modules/network.bicep' = { + name: 'network' + scope: rg + params: { + location: location + namePrefix: namePrefix + vnetAddressSpace: vnetAddressSpace + tags: tags + } +} + +module acr 'modules/acr.bicep' = if (!useSharedAcr) { + name: 'acr' + scope: rg + params: { + location: location + acrName: effectiveAcrName + acrSku: acrSku + tags: tags + } +} + +module aks 'modules/aks.bicep' = { + name: 'aks' + scope: rg + params: { + location: location + clusterName: clusterName + kubernetesVersion: kubernetesVersion + aksSubnetId: network.outputs.aksSubnetId + privateDnsZoneId: network.outputs.privateDnsZoneId + acrId: effectiveAcrId + systemNodeVmSize: systemNodeVmSize + systemNodeCount: systemNodeCount + enableAutoScaling: enableAutoScaling + minNodeCount: minNodeCount + maxNodeCount: maxNodeCount + availabilityZones: availabilityZones + tags: tags + } +} + +module vpn 'modules/vpn.bicep' = if (deployVpnGateway) { + name: 'vpn' + scope: rg + params: { + location: location + namePrefix: namePrefix + gatewaySubnetId: network.outputs.gatewaySubnetId + vpnClientAddressPool: vpnClientAddressPool + vpnClientRootCertData: vpnClientRootCertData + gatewaySku: gatewaySku + tags: tags + } +} + +module storage 'modules/storage.bicep' = if (deployBackupStorage) { + name: 'backupStorage' + scope: rg + params: { + location: location + storageAccountName: effectiveBackupSa + oidcIssuerUrl: aks.outputs.oidcIssuerUrl + serviceAccountSubject: pgBackRestServiceAccountSubject + tags: tags + } +} + +module contentFiles 'modules/files.bicep' = if (deployContentFileShares) { + name: 'contentFiles' + scope: rg + params: { + location: location + storageAccountName: effectiveContentFilesSa + tags: tags + } +} + +module postgres 'modules/postgres.bicep' = if (deployPostgresFlexible) { + name: 'postgres' + scope: rg + params: { + location: location + serverName: effectivePostgresServer + delegatedSubnetId: network.outputs.postgresSubnetId + privateDnsZoneId: network.outputs.postgresPrivateDnsZoneId + administratorPassword: postgresAdminPassword + skuName: postgresSkuName + highAvailability: postgresHighAvailability + tags: tags + } +} + +// --- Outputs (consumed by the README's get-credentials / helm steps) ------- +output resourceGroupName string = rg.name +output clusterName string = aks.outputs.clusterName +output apiServerPrivateFqdn string = aks.outputs.apiServerPrivateFqdn +output oidcIssuerUrl string = aks.outputs.oidcIssuerUrl +output acrLoginServer string = effectiveAcrLoginServer +output acrName string = useSharedAcr ? split(sharedAcrLoginServer, '.')[0] : acr!.outputs.acrName +output privateDnsZoneName string = network.outputs.privateDnsZoneName +output vpnGatewayName string = vpn.?outputs.vpnGatewayName ?? '' +output backupStorageAccount string = storage.?outputs.storageAccountName ?? '' +output backupBlobEndpoint string = storage.?outputs.blobEndpoint ?? '' +output backupContainerName string = storage.?outputs.backupContainerName ?? '' +output pgBackRestIdentityClientId string = storage.?outputs.backupIdentityClientId ?? '' + +// --- Content / log Azure Files (static PV binding) ------------------------- +output contentFilesAccount string = contentFiles.?outputs.storageAccountName ?? '' +output contentFilesEndpoint string = contentFiles.?outputs.fileEndpoint ?? '' +output contentShareName string = contentFiles.?outputs.contentShareName ?? '' +output attachmentsShareName string = contentFiles.?outputs.attachmentsShareName ?? '' +output dataShareName string = contentFiles.?outputs.dataShareName ?? '' +output usersShareName string = contentFiles.?outputs.usersShareName ?? '' +output otelLogsShareName string = contentFiles.?outputs.otelLogsShareName ?? '' + +// --- Private PostgreSQL Flexible Server ------------------------------------ +output postgresServerName string = postgres.?outputs.serverName ?? '' +output postgresFqdn string = postgres.?outputs.fullyQualifiedDomainName ?? '' +output postgresDatabaseName string = postgres.?outputs.databaseName ?? '' +output postgresAdminLogin string = postgres.?outputs.administratorLogin ?? '' diff --git a/deploy/aks/infra/main.parameters.json b/deploy/aks/infra/main.parameters.json new file mode 100644 index 000000000..2f2e390db --- /dev/null +++ b/deploy/aks/infra/main.parameters.json @@ -0,0 +1,72 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "resourceGroupName": { + "value": "memex-aks-rg" + }, + "location": { + "value": "swedencentral" + }, + "namePrefix": { + "value": "memexaks" + }, + "acrSku": { + "value": "Premium" + }, + "kubernetesVersion": { + "value": "" + }, + "systemNodeVmSize": { + "value": "Standard_D4s_v3" + }, + "systemNodeCount": { + "value": 2 + }, + "enableAutoScaling": { + "value": false + }, + "minNodeCount": { + "value": 2 + }, + "maxNodeCount": { + "value": 2 + }, + "availabilityZones": { + "value": [] + }, + "vnetAddressSpace": { + "value": "10.42.0.0/16" + }, + "deployVpnGateway": { + "value": true + }, + "vpnClientAddressPool": { + "value": "172.16.201.0/24" + }, + "vpnClientRootCertData": { + "value": "" + }, + "gatewaySku": { + "value": "VpnGw1AZ" + }, + "deployBackupStorage": { + "value": false + }, + "pgBackRestServiceAccountSubject": { + "value": "system:serviceaccount:memex:pgbackrest-sa" + }, + "deployContentFileShares": { + "value": true + }, + "deployPostgresFlexible": { + "value": true + }, + "postgresSkuName": { + "value": "Standard_D2ds_v5" + }, + "postgresHighAvailability": { + "value": false + } + } +} diff --git a/deploy/aks/infra/modules/acr.bicep b/deploy/aks/infra/modules/acr.bicep new file mode 100644 index 000000000..5875b0427 --- /dev/null +++ b/deploy/aks/infra/modules/acr.bicep @@ -0,0 +1,51 @@ +// --------------------------------------------------------------------------- +// acr.bicep — Azure Container Registry for the Memex portal images. +// +// Two usage modes (see README "Image strategy"): +// 1. Pull straight from GHCR (ghcr.io/systemorph/...) — then this ACR is +// optional and used only as a private cache/mirror. +// 2. Import the GHCR images into this ACR (`az acr import ...`) and point the +// Helm overlay's image.registry at acr.loginServer — recommended for a +// private cluster with no public egress. +// +// AcrPull is granted to the AKS cluster's kubelet managed identity in aks.bicep +// (the kubelet identity is only known after the cluster exists), so this module +// just emits the registry. anonymousPullEnabled stays false. +// --------------------------------------------------------------------------- + +@description('Azure region for the registry.') +param location string + +@description('ACR name. Must be globally unique, alphanumeric, 5-50 chars.') +param acrName string + +@description('ACR SKU. Premium is required for private endpoints / geo-replication.') +@allowed([ + 'Basic' + 'Standard' + 'Premium' +]) +param acrSku string = 'Premium' + +@description('Tags applied to the registry.') +param tags object = {} + +resource acr 'Microsoft.ContainerRegistry/registries@2025-04-01' = { + name: acrName + location: location + tags: tags + sku: { + name: acrSku + } + properties: { + adminUserEnabled: false + anonymousPullEnabled: false + // Premium-only knobs; harmless defaults on lower SKUs. + publicNetworkAccess: 'Enabled' + zoneRedundancy: 'Disabled' + } +} + +output acrId string = acr.id +output acrName string = acr.name +output acrLoginServer string = acr.properties.loginServer diff --git a/deploy/aks/infra/modules/aks.bicep b/deploy/aks/infra/modules/aks.bicep new file mode 100644 index 000000000..cc063b81e --- /dev/null +++ b/deploy/aks/infra/modules/aks.bicep @@ -0,0 +1,242 @@ +// --------------------------------------------------------------------------- +// aks.bicep — PRIVATE AKS cluster for the Memex portal. +// +// Key properties for this sample: +// - apiServerAccessProfile.enablePrivateCluster = true +// The API server gets a PRIVATE IP only; there is no public control-plane +// endpoint. `kubectl` therefore only works from inside the VNet — that's +// why we ship the P2S VPN Gateway (vpn.bicep) and link the private DNS +// zone (network.bicep). +// - privateDNSZone = the region-specific zone we created, passed in by id so +// AKS writes its API server A record there (BYO private DNS zone mode). +// - userAssignedIdentity for the control plane; the auto-created kubelet +// identity gets AcrPull on the ACR so nodes can pull private images. +// - Azure CNI overlay networking inside the aks-nodes subnet. +// - Workload Identity + OIDC issuer enabled so the pgBackRest sidecar (and any +// future pod) can use a federated identity to reach Azure Blob without +// storing account keys (see README "PITR"). +// - Azure Files CSI driver enabled for ReadWriteMany PVCs (azurefile-csi +// storage class), required for HA portal replicas sharing /mnt/users. +// --------------------------------------------------------------------------- + +@description('Azure region for the cluster.') +param location string + +@description('AKS cluster name.') +param clusterName string + +@description('DNS prefix for the cluster.') +param dnsPrefix string = clusterName + +@description('Kubernetes version. Leave empty to use the AKS default for the region.') +param kubernetesVersion string = '' + +@description('Subnet resource id for the system node pool (the aks-nodes subnet).') +param aksSubnetId string + +@description('Resource id of the BYO private DNS zone (privatelink..azmk8s.io).') +param privateDnsZoneId string + +@description('Resource id of the ACR to grant AcrPull to the kubelet identity. Empty = a shared/cross-RG ACR is used; the AcrPull grant is then done out-of-band (this RG-scoped module cannot author a role assignment in another RG).') +param acrId string = '' + +@description('System node pool VM size.') +param systemNodeVmSize string = 'Standard_D4s_v5' + +@description('System node pool node count (per-zone if availabilityZones is set).') +@minValue(1) +@maxValue(100) +param systemNodeCount int = 3 + +@description('Availability zones for the node pool. Empty = no zonal spread.') +param availabilityZones array = [ + '1' + '2' + '3' +] + +@description('Enable the AKS-managed cluster autoscaler on the system pool.') +param enableAutoScaling bool = true + +@description('Autoscaler minimum node count.') +param minNodeCount int = 3 + +@description('Autoscaler maximum node count.') +param maxNodeCount int = 6 + +@description('Tags applied to the cluster.') +param tags object = {} + +// Control-plane user-assigned identity. Using a UAMI (rather than system- +// assigned) makes the private-DNS-zone role grant deterministic and reusable. +resource aksIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2024-11-30' = { + name: '${clusterName}-cp-mi' + location: location + tags: tags +} + +// "Private DNS Zone Contributor" — the control-plane identity must be able to +// write the API server A record into the BYO private DNS zone. +var privateDnsZoneContributorRoleId = 'b12aa53e-6015-4669-85d0-8515ebb3ae7f' + +resource dnsZoneRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(privateDnsZoneId, aksIdentity.id, privateDnsZoneContributorRoleId) + scope: privateDnsZone + properties: { + principalId: aksIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', privateDnsZoneContributorRoleId) + principalType: 'ServicePrincipal' + } +} + +// "Network Contributor" on the VNET (not just the node subnet). AKS must manage +// NICs/routes in the subnet AND create the private-DNS-zone -> VNet link during +// reconcile, and that link requires Microsoft.Network/virtualNetworks/join/action +// at the VNET scope. A subnet-scoped grant only covers subnet-level join, so the +// private-DNS reconcile fails with LinkedAuthorizationFailed. (Intermittent — it +// depends on RBAC propagation timing: westeurope happened to pass, swedencentral +// failed. VNet scope is the correct, deterministic fix.) +var networkContributorRoleId = '4d97b98b-1d4f-4787-a291-c67834d212e7' + +resource aksVnetRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(aksVnet.id, aksIdentity.id, networkContributorRoleId) + scope: aksVnet + properties: { + principalId: aksIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', networkContributorRoleId) + principalType: 'ServicePrincipal' + } +} + +// Existing references so role assignments can target them by scope. +resource privateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: last(split(privateDnsZoneId, '/')) +} + +// The subnet's parent vnet name and the subnet name are derived from the id. +resource aksVnet 'Microsoft.Network/virtualNetworks@2024-05-01' existing = { + name: split(aksSubnetId, '/')[8] +} + +resource aks 'Microsoft.ContainerService/managedClusters@2024-09-01' = { + name: clusterName + location: location + tags: tags + sku: { + name: 'Base' + tier: 'Standard' // Standard tier = SLA-backed control plane; use for prod. + } + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${aksIdentity.id}': {} + } + } + properties: { + dnsPrefix: dnsPrefix + kubernetesVersion: empty(kubernetesVersion) ? null : kubernetesVersion + enableRBAC: true + disableLocalAccounts: false // keep admin kubeconfig usable over the VPN + // --- PRIVATE CLUSTER: the whole point of this sample ------------------- + apiServerAccessProfile: { + enablePrivateCluster: true + privateDNSZone: privateDnsZoneId + enablePrivateClusterPublicFQDN: false + } + // --- Networking -------------------------------------------------------- + networkProfile: { + networkPlugin: 'azure' + networkPluginMode: 'overlay' + networkPolicy: 'cilium' + networkDataplane: 'cilium' + loadBalancerSku: 'standard' + outboundType: 'loadBalancer' + serviceCidr: '10.43.0.0/16' + dnsServiceIP: '10.43.0.10' + podCidr: '10.244.0.0/16' + } + // --- Workload Identity (for pgBackRest -> Blob, keyless) --------------- + oidcIssuerProfile: { + enabled: true + } + securityProfile: { + workloadIdentity: { + enabled: true + } + } + // --- Add-ons ----------------------------------------------------------- + addonProfiles: { + azureKeyvaultSecretsProvider: { + enabled: true + config: { + enableSecretRotation: 'true' + } + } + // NOTE: the Azure Files / Disk CSI drivers are enabled via + // storageProfile below (the modern location); the azurepolicy and + // ingress add-ons are intentionally left to the operator (see README: + // AGIC vs ingress-nginx). + } + storageProfile: { + fileCSIDriver: { + enabled: true // azurefile-csi storage class for ReadWriteMany PVCs + } + diskCSIDriver: { + enabled: true // managed-csi storage class for the Postgres PVC + } + snapshotController: { + enabled: true + } + } + agentPoolProfiles: [ + { + name: 'system' + mode: 'System' + osType: 'Linux' + osSKU: 'AzureLinux' + vmSize: systemNodeVmSize + count: systemNodeCount + enableAutoScaling: enableAutoScaling + minCount: enableAutoScaling ? minNodeCount : null + maxCount: enableAutoScaling ? maxNodeCount : null + vnetSubnetID: aksSubnetId + availabilityZones: empty(availabilityZones) ? null : availabilityZones + type: 'VirtualMachineScaleSets' + maxPods: 60 + } + ] + } + dependsOn: [ + dnsZoneRoleAssignment + aksVnetRoleAssignment + ] +} + +// Grant the auto-created kubelet identity AcrPull on the ACR so nodes can pull +// private images imported into the registry. +var acrPullRoleId = '7f951dda-4ed3-4680-a7ca-43fe172d538d' + +resource acr 'Microsoft.ContainerRegistry/registries@2025-04-01' existing = if (!empty(acrId)) { + name: last(split(acrId, '/')) +} + +resource acrPullRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = if (!empty(acrId)) { + name: guid(acrId, aks.id, acrPullRoleId) + scope: acr + properties: { + principalId: aks.properties.identityProfile.kubeletidentity.objectId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', acrPullRoleId) + principalType: 'ServicePrincipal' + } +} + +output clusterName string = aks.name +output clusterId string = aks.id +output controlPlaneIdentityId string = aksIdentity.id +output kubeletIdentityObjectId string = aks.properties.identityProfile.kubeletidentity.objectId +output oidcIssuerUrl string = aks.properties.oidcIssuerProfile.issuerURL +output nodeResourceGroup string = aks.properties.nodeResourceGroup +// The private API server FQDN is read back at deploy time from the cluster's +// fqdn property (the private endpoint variant). `az aks get-credentials` +// discovers it automatically, so this is informational only. +output apiServerPrivateFqdn string = aks.properties.privateFQDN diff --git a/deploy/aks/infra/modules/files.bicep b/deploy/aks/infra/modules/files.bicep new file mode 100644 index 000000000..74b2e0c58 --- /dev/null +++ b/deploy/aks/infra/modules/files.bicep @@ -0,0 +1,139 @@ +// --------------------------------------------------------------------------- +// files.bicep — A dedicated Azure Files storage account holding the named SMB +// shares ("drives") that the portal + observability stack mount. +// +// WHY A SEPARATE ACCOUNT (vs. dynamic azurefile provisioning)? +// ------------------------------------------------------------ +// The default path uses the `azurefile-memex` StorageClass to DYNAMICALLY create +// a share per PVC (simplest — see manifests/storageclass-azurefile.yaml). This +// module is the OPTIONAL static-binding path: operators who want pre-created, +// named, individually-sized/quota'd shares (and a single account to back up, +// firewall, or lifecycle-manage) provision this account, then bind STATIC PVs to +// the named shares via the file.csi.azure.com driver (see README "Static PV +// binding"). Dynamic stays the default; this is here for operators who prefer +// named drives. +// +// Shares (one "drive" per concern, matching the /data + /mnt/* mounts): +// data -> /data framework caches (DataProtection keys, caches) +// content -> /mnt/content content collection (Storage__BasePath) +// attachments -> /mnt/attachments attachments collection +// users -> /mnt/users co-hosted CLI configs +// otel-logs -> /mnt/otel-logs OpenTelemetry Collector file-exporter archive +// +// StorageV2 + Standard_ZRS (zone-redundant) + largeFileSharesState=Enabled so +// individual shares can exceed 5 TiB (up to 100 TiB). SMB multichannel left at +// default. This is a content/log account — NOT the pgBackRest BLOB account in +// storage.bicep; keeping them separate isolates blast radius and access policy. +// --------------------------------------------------------------------------- + +@description('Azure region for the Files storage account.') +param location string + +@description('Globally-unique storage account name (3-24 lowercase alphanumerics).') +param storageAccountName string + +@description('Quota (GiB) for the content-collection share.') +param contentShareQuotaGib int = 128 + +@description('Quota (GiB) for the attachments share.') +param attachmentsShareQuotaGib int = 64 + +@description('Quota (GiB) for the framework-cache /data share.') +param dataShareQuotaGib int = 16 + +@description('Quota (GiB) for the co-hosted CLI-config /mnt/users share.') +param usersShareQuotaGib int = 32 + +@description('Quota (GiB) for the OpenTelemetry Collector log-archive share.') +param otelLogsShareQuotaGib int = 64 + +@description('Tags applied to every resource.') +param tags object = {} + +resource files 'Microsoft.Storage/storageAccounts@2024-01-01' = { + name: storageAccountName + location: location + tags: tags + kind: 'StorageV2' + sku: { + // Zone-redundant: shares survive a single-AZ loss, matching the 3-zone + // node spread. Switch to Standard_LRS to cut cost where ZRS isn't needed, + // or Premium_ZRS (FileStorage kind) for IOPS-heavy content. + name: 'Standard_ZRS' + } + properties: { + // Allow >5 TiB shares (up to 100 TiB) so content can grow without re-homing. + largeFileSharesState: 'Enabled' + minimumTlsVersion: 'TLS1_2' + allowBlobPublicAccess: false + supportsHttpsTrafficOnly: true + accessTier: 'Hot' + } +} + +resource fileService 'Microsoft.Storage/storageAccounts/fileServices@2024-01-01' = { + parent: files + name: 'default' + properties: { + // 14-day share soft-delete: an accidental `kubectl delete pvc` (with a Delete + // reclaimPolicy) or share drop is recoverable. + shareDeleteRetentionPolicy: { + enabled: true + days: 14 + } + } +} + +resource contentShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2024-01-01' = { + parent: fileService + name: 'content' + properties: { + shareQuota: contentShareQuotaGib + enabledProtocols: 'SMB' + } +} + +resource attachmentsShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2024-01-01' = { + parent: fileService + name: 'attachments' + properties: { + shareQuota: attachmentsShareQuotaGib + enabledProtocols: 'SMB' + } +} + +resource dataShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2024-01-01' = { + parent: fileService + name: 'data' + properties: { + shareQuota: dataShareQuotaGib + enabledProtocols: 'SMB' + } +} + +resource usersShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2024-01-01' = { + parent: fileService + name: 'users' + properties: { + shareQuota: usersShareQuotaGib + enabledProtocols: 'SMB' + } +} + +resource otelLogsShare 'Microsoft.Storage/storageAccounts/fileServices/shares@2024-01-01' = { + parent: fileService + name: 'otel-logs' + properties: { + shareQuota: otelLogsShareQuotaGib + enabledProtocols: 'SMB' + } +} + +output storageAccountName string = files.name +output storageAccountId string = files.id +output fileEndpoint string = files.properties.primaryEndpoints.file +output contentShareName string = contentShare.name +output attachmentsShareName string = attachmentsShare.name +output dataShareName string = dataShare.name +output usersShareName string = usersShare.name +output otelLogsShareName string = otelLogsShare.name diff --git a/deploy/aks/infra/modules/network.bicep b/deploy/aks/infra/modules/network.bicep new file mode 100644 index 000000000..9ad9cb6d5 --- /dev/null +++ b/deploy/aks/infra/modules/network.bicep @@ -0,0 +1,155 @@ +// --------------------------------------------------------------------------- +// network.bicep — VNet + subnets + private DNS zone for the private AKS cluster. +// +// Subnet layout: +// - aks-nodes : the AKS node pool(s) live here (kubenet/Azure CNI overlay) +// - GatewaySubnet : RESERVED NAME (required by Azure) for the VPN Gateway +// - bastion-subnet : optional AzureBastionSubnet for a jumpbox (alternative +// to the P2S VPN — see README; left empty by default) +// +// The private DNS zone privatelink..azmk8s.io is what makes a private +// AKS reachable: AKS publishes the API server's private IP into this zone, and +// linking the zone to the VNet means anything *inside* the VNet (including a +// P2S VPN client, which is logically attached to the VNet) resolves the API +// server FQDN to that private IP. Without this link, `kubectl` cannot find the +// control plane at all. +// --------------------------------------------------------------------------- + +@description('Azure region for the network resources.') +param location string + +@description('Resource name prefix (e.g. memex-aks).') +param namePrefix string + +@description('Address space for the whole VNet.') +param vnetAddressSpace string = '10.42.0.0/16' + +@description('Subnet CIDR for the AKS node pool.') +param aksSubnetPrefix string = '10.42.0.0/20' + +@description('Subnet CIDR for the VPN GatewaySubnet (must be named GatewaySubnet).') +param gatewaySubnetPrefix string = '10.42.16.0/24' + +@description('Subnet CIDR for an optional AzureBastionSubnet (jumpbox alternative).') +param bastionSubnetPrefix string = '10.42.17.0/26' + +@description('Subnet CIDR for the delegated Azure Database for PostgreSQL Flexible Server subnet (private/VNet-injected PG).') +param postgresSubnetPrefix string = '10.42.18.0/24' + +@description('Tags applied to every resource.') +param tags object = {} + +// The private-link DNS zone name is region-specific. AKS expects exactly this +// shape; using anything else means the cluster cannot register its private IP. +var privateDnsZoneName = 'privatelink.${location}.azmk8s.io' + +resource vnet 'Microsoft.Network/virtualNetworks@2024-05-01' = { + name: '${namePrefix}-vnet' + location: location + tags: tags + properties: { + addressSpace: { + addressPrefixes: [vnetAddressSpace] + } + subnets: [ + { + name: 'aks-nodes' + properties: { + addressPrefix: aksSubnetPrefix + } + } + { + // RESERVED name — Azure VPN/ExpressRoute gateways MUST live in a subnet + // literally called "GatewaySubnet". Do not rename. + name: 'GatewaySubnet' + properties: { + addressPrefix: gatewaySubnetPrefix + } + } + { + // RESERVED name — Azure Bastion requires "AzureBastionSubnet". + // Provisioned but unused unless you deploy Bastion (see README). + name: 'AzureBastionSubnet' + properties: { + addressPrefix: bastionSubnetPrefix + } + } + { + // Delegated subnet for a PRIVATE (VNet-injected) PostgreSQL Flexible + // Server. The delegation is mandatory: Flexible Server injects its NIC + // here and the subnet can host nothing else. Used only when + // deployPostgresFlexible=true (see postgres.bicep); harmless otherwise. + name: 'postgres' + properties: { + addressPrefix: postgresSubnetPrefix + delegations: [ + { + name: 'fs-delegation' + properties: { + serviceName: 'Microsoft.DBforPostgreSQL/flexibleServers' + } + } + ] + } + } + ] + } +} + +// Private DNS zone for the VNet-injected Flexible Server. Flexible Server's +// private-access mode REQUIRES a zone named exactly *.private.postgres.database.azure.com +// linked to the VNet; the server's FQDN resolves to its private NIC IP only +// inside the VNet (and over the P2S VPN). Created here so the postgres module +// can attach to it; left unlinked-to-nothing-else if PG-flexible isn't deployed. +resource postgresPrivateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: '${namePrefix}.private.postgres.database.azure.com' + location: 'global' + tags: tags +} + +resource postgresPrivateDnsZoneVnetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { + parent: postgresPrivateDnsZone + name: '${namePrefix}-pg-dnslink' + location: 'global' + tags: tags + properties: { + registrationEnabled: false + virtualNetwork: { + id: vnet.id + } + } +} + +// Region-specific private DNS zone for the AKS private API server. +resource privateDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: privateDnsZoneName + location: 'global' + tags: tags +} + +// Link the zone to the VNet so in-VNet clients (incl. P2S VPN) resolve the +// API server's private IP. registrationEnabled stays false — AKS writes the +// A record itself via its control-plane managed identity. +resource privateDnsZoneVnetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { + parent: privateDnsZone + name: '${namePrefix}-dnslink' + location: 'global' + tags: tags + properties: { + registrationEnabled: false + virtualNetwork: { + id: vnet.id + } + } +} + +output vnetId string = vnet.id +output vnetName string = vnet.name +output aksSubnetId string = '${vnet.id}/subnets/aks-nodes' +output gatewaySubnetId string = '${vnet.id}/subnets/GatewaySubnet' +output bastionSubnetId string = '${vnet.id}/subnets/AzureBastionSubnet' +output postgresSubnetId string = '${vnet.id}/subnets/postgres' +output privateDnsZoneId string = privateDnsZone.id +output privateDnsZoneName string = privateDnsZone.name +output postgresPrivateDnsZoneId string = postgresPrivateDnsZone.id +output postgresPrivateDnsZoneName string = postgresPrivateDnsZone.name diff --git a/deploy/aks/infra/modules/postgres.bicep b/deploy/aks/infra/modules/postgres.bicep new file mode 100644 index 000000000..3e1526276 --- /dev/null +++ b/deploy/aks/infra/modules/postgres.bicep @@ -0,0 +1,133 @@ +// --------------------------------------------------------------------------- +// postgres.bicep — PRIVATE (VNet-injected) Azure Database for PostgreSQL +// Flexible Server with pgvector, matching the private-everything +// style of this sample. +// +// This is the MANAGED, PRIVATE database option: a Flexible Server injected into +// the delegated `postgres` subnet (network.bicep), reachable ONLY from inside the +// VNet (and over the P2S VPN) via the linked private DNS zone. No public endpoint. +// It replaces the in-cluster self-managed Postgres StatefulSet + pgBackRest when +// you want managed PITR (automatic backups + WAL, restore to any second in the +// retention window) and no in-cluster database moving parts. +// +// Toggle in main.bicep with `deployPostgresFlexible=true`; when on, also set +// `deployBackupStorage=false` and DON'T apply the postgres-pvc / pgbackrest +// manifests — point the portal's connection string at the output FQDN instead. +// +// pgvector: enabled via the `azure.extensions` server parameter (the portal's +// embeddings + HNSW vector search need it). The DB itself is created here. +// --------------------------------------------------------------------------- + +@description('Azure region for the Flexible Server.') +param location string + +@description('Flexible Server name (becomes .postgres.database.azure.com).') +param serverName string + +@description('Resource id of the DELEGATED subnet for VNet injection (network.outputs.postgresSubnetId).') +param delegatedSubnetId string + +@description('Resource id of the private DNS zone *.private.postgres.database.azure.com (network.outputs.postgresPrivateDnsZoneId).') +param privateDnsZoneId string + +@description('Administrator login name.') +param administratorLogin string = 'memexadmin' + +@description('Administrator password. Pass at deploy time; never commit a real one.') +@secure() +param administratorPassword string + +@description('Compute SKU (e.g. Standard_D2ds_v5 = 2 vCPU / 8 GiB; Standard_D4ds_v5 = 4 vCPU / 16 GiB).') +param skuName string = 'Standard_D2ds_v5' + +@description('Compute tier.') +@allowed([ + 'Burstable' + 'GeneralPurpose' + 'MemoryOptimized' +]) +param skuTier string = 'GeneralPurpose' + +@description('Storage size in GiB.') +param storageSizeGib int = 128 + +@description('PostgreSQL major version.') +param postgresVersion string = '16' + +@description('Backup retention in days (7-35) for managed PITR.') +@minValue(7) +@maxValue(35) +param backupRetentionDays int = 14 + +@description('Enable zone-redundant HA (a standby in a second zone). Costs ~2x compute.') +param highAvailability bool = true + +@description('Name of the application database to create.') +param databaseName string = 'memex' + +@description('Tags applied to every resource.') +param tags object = {} + +resource postgres 'Microsoft.DBforPostgreSQL/flexibleServers@2024-08-01' = { + name: serverName + location: location + tags: tags + sku: { + name: skuName + tier: skuTier + } + properties: { + version: postgresVersion + administratorLogin: administratorLogin + administratorLoginPassword: administratorPassword + storage: { + storageSizeGB: storageSizeGib + autoGrow: 'Enabled' + } + backup: { + backupRetentionDays: backupRetentionDays + geoRedundantBackup: 'Disabled' + } + highAvailability: { + // Zone-redundant HA pairs a hot standby in another AZ — matches the + // 3-zone node spread. Set false (Disabled) to halve compute cost. + mode: highAvailability ? 'ZoneRedundant' : 'Disabled' + } + // PRIVATE access: inject into the delegated subnet + private DNS zone. + // No publicNetworkAccess endpoint is created in this mode. + network: { + delegatedSubnetResourceId: delegatedSubnetId + privateDnsZoneArmResourceId: privateDnsZoneId + } + } +} + +// Enable pgvector (and uuid-ossp, commonly needed) via allowlist server param. +// azure.extensions must list every extension before CREATE EXTENSION works. +resource azureExtensions 'Microsoft.DBforPostgreSQL/flexibleServers/configurations@2024-08-01' = { + parent: postgres + name: 'azure.extensions' + properties: { + value: 'VECTOR,UUID-OSSP' + source: 'user-override' + } +} + +resource memexDatabase 'Microsoft.DBforPostgreSQL/flexibleServers/databases@2024-08-01' = { + parent: postgres + name: databaseName + properties: { + charset: 'UTF8' + collation: 'en_US.utf8' + } + dependsOn: [ + azureExtensions + ] +} + +output serverName string = postgres.name +output serverId string = postgres.id +// Private FQDN — resolves to the VNet NIC IP only inside the VNet / over the VPN. +output fullyQualifiedDomainName string = postgres.properties.fullyQualifiedDomainName +output databaseName string = memexDatabase.name +output administratorLogin string = administratorLogin diff --git a/deploy/aks/infra/modules/storage.bicep b/deploy/aks/infra/modules/storage.bicep new file mode 100644 index 000000000..bf5ac4c5e --- /dev/null +++ b/deploy/aks/infra/modules/storage.bicep @@ -0,0 +1,121 @@ +// --------------------------------------------------------------------------- +// storage.bicep — Azure Blob storage for pgBackRest PITR backups of the +// self-managed Postgres container, plus a keyless +// (Workload Identity / federated) path for the backup pods. +// +// pgBackRest writes full + differential backups AND archived WAL segments to a +// blob container. With WAL archiving on, `pgbackrest restore --type=time` can +// roll the database forward to any second between the last full backup and the +// last archived WAL — that's Point-In-Time Recovery. +// +// Auth model: AKS Workload Identity. We create a user-assigned managed identity, +// grant it "Storage Blob Data Contributor" on the storage account, and federate +// it with the pgBackRest Kubernetes service account (subject is supplied by the +// caller as serviceAccountSubject, e.g. +// system:serviceaccount:memex:pgbackrest-sa +// ). The pod then authenticates to Blob with no account key on disk. +// +// pgBackRest can also use a shared key / SAS token (set repo1-azure-key in the +// pgbackrest secret instead) — see README if you prefer keys over Workload +// Identity. Account-key auth is left enabled here so the simpler path works too. +// --------------------------------------------------------------------------- + +@description('Azure region for the storage account.') +param location string + +@description('Globally-unique storage account name (3-24 lowercase alphanumerics).') +param storageAccountName string + +@description('Blob container name for pgBackRest backups + WAL.') +param backupContainerName string = 'pgbackrest' + +@description('OIDC issuer URL of the AKS cluster (from aks.bicep output).') +param oidcIssuerUrl string + +@description('Federated subject for the pgBackRest pods, e.g. system:serviceaccount::.') +param serviceAccountSubject string = 'system:serviceaccount:memex:pgbackrest-sa' + +@description('Days to retain soft-deleted blobs / containers.') +param softDeleteRetentionDays int = 30 + +@description('Tags applied to every resource.') +param tags object = {} + +resource storage 'Microsoft.Storage/storageAccounts@2024-01-01' = { + name: storageAccountName + location: location + tags: tags + kind: 'StorageV2' + sku: { + name: 'Standard_ZRS' // zone-redundant: backups survive a single-AZ loss + } + properties: { + minimumTlsVersion: 'TLS1_2' + allowBlobPublicAccess: false + supportsHttpsTrafficOnly: true + accessTier: 'Hot' + } +} + +resource blobService 'Microsoft.Storage/storageAccounts/blobServices@2024-01-01' = { + parent: storage + name: 'default' + properties: { + deleteRetentionPolicy: { + enabled: true + days: softDeleteRetentionDays + } + containerDeleteRetentionPolicy: { + enabled: true + days: softDeleteRetentionDays + } + } +} + +resource backupContainer 'Microsoft.Storage/storageAccounts/blobServices/containers@2024-01-01' = { + parent: blobService + name: backupContainerName + properties: { + publicAccess: 'None' + } +} + +// --- Workload Identity: keyless access for the pgBackRest pods -------------- +resource backupIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2024-11-30' = { + name: '${storageAccountName}-pgbackrest-mi' + location: location + tags: tags +} + +// "Storage Blob Data Contributor" so pgBackRest can read+write blobs. +var blobDataContributorRoleId = 'ba92f5b4-2d11-453d-a403-e96b0029c9fe' + +resource backupBlobRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(storage.id, backupIdentity.id, blobDataContributorRoleId) + scope: storage + properties: { + principalId: backupIdentity.properties.principalId + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', blobDataContributorRoleId) + principalType: 'ServicePrincipal' + } +} + +// Federate the managed identity with the pgBackRest Kubernetes service account. +resource federatedCredential 'Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials@2024-11-30' = { + parent: backupIdentity + name: 'pgbackrest-federated' + properties: { + issuer: oidcIssuerUrl + subject: serviceAccountSubject + audiences: [ + 'api://AzureADTokenExchange' + ] + } +} + +output storageAccountName string = storage.name +output storageAccountId string = storage.id +output backupContainerName string = backupContainer.name +output blobEndpoint string = storage.properties.primaryEndpoints.blob +output backupIdentityClientId string = backupIdentity.properties.clientId +output backupIdentityId string = backupIdentity.id diff --git a/deploy/aks/infra/modules/vpn.bicep b/deploy/aks/infra/modules/vpn.bicep new file mode 100644 index 000000000..6ad701e34 --- /dev/null +++ b/deploy/aks/infra/modules/vpn.bicep @@ -0,0 +1,127 @@ +// --------------------------------------------------------------------------- +// vpn.bicep — Point-to-Site (P2S) VPN Gateway for reaching the PRIVATE AKS +// API server with kubectl. +// +// Why this exists: +// The cluster's API server has a private IP only (enablePrivateCluster=true). +// An operator's laptop cannot reach it over the internet. A P2S VPN attaches +// the laptop logically into the cluster VNet; combined with the private DNS +// zone link (network.bicep), the operator then resolves the API server FQDN +// to its private IP and `kubectl` works. +// +// Operator flow (full steps in README): +// 1. az deployment ... (this infra) +// 2. Generate root+client certs, upload root public cert (vpnClientRootCert) +// 3. Download the VPN client from the portal / `az network vnet-gateway vpn-client generate` +// 4. Connect the P2S VPN +// 5. az aks get-credentials --resource-group --name +// 6. kubectl get nodes # resolves the private API server over the tunnel +// +// Cert-based auth is used here (simplest, no Entra ID dependency). For prod you +// may prefer Azure AD (Entra) auth on the P2S — noted in README. +// --------------------------------------------------------------------------- + +@description('Azure region for the gateway.') +param location string + +@description('Resource name prefix (e.g. memex-aks).') +param namePrefix string + +@description('Resource id of the GatewaySubnet (must be a subnet named GatewaySubnet).') +param gatewaySubnetId string + +@description('Address pool handed out to P2S VPN clients. Must NOT overlap the VNet.') +param vpnClientAddressPool string = '172.16.201.0/24' + +@description('Base64 public cert data of the P2S root certificate (no PEM headers, single line). Leave empty to deploy the gateway and add the cert later.') +param vpnClientRootCertData string = '' + +@description('Friendly name for the uploaded root certificate.') +param vpnClientRootCertName string = 'P2SRootCert' + +@description('VPN Gateway SKU. VpnGw1AZ is the cheapest that supports P2S + OpenVPN. Azure retired the non-AZ VpnGw1-5 SKUs (error NonAzSkusNotAllowedForVPNGateway) — only the zone-redundant *AZ SKUs can be created now.') +@allowed([ + 'VpnGw1AZ' + 'VpnGw2AZ' + 'VpnGw3AZ' +]) +param gatewaySku string = 'VpnGw1AZ' + +@description('Tags applied to every resource.') +param tags object = {} + +// Public IP for the gateway's tunnel endpoint (the data path stays private; the +// IKE/OpenVPN control endpoint needs a public IP — this is normal for P2S). +resource vpnPublicIp 'Microsoft.Network/publicIPAddresses@2024-05-01' = { + name: '${namePrefix}-vpngw-pip' + location: location + tags: tags + sku: { + name: 'Standard' + } + zones: [ + '1' + '2' + '3' + ] + properties: { + publicIPAllocationMethod: 'Static' + } +} + +resource vpnGateway 'Microsoft.Network/virtualNetworkGateways@2024-05-01' = { + name: '${namePrefix}-vpngw' + location: location + tags: tags + properties: { + gatewayType: 'Vpn' + vpnType: 'RouteBased' + enableBgp: false + activeActive: false + sku: { + name: gatewaySku + tier: gatewaySku + } + ipConfigurations: [ + { + name: 'vnetGatewayConfig' + properties: { + privateIPAllocationMethod: 'Dynamic' + subnet: { + id: gatewaySubnetId + } + publicIPAddress: { + id: vpnPublicIp.id + } + } + } + ] + // Point-to-Site configuration. + vpnClientConfiguration: { + vpnClientAddressPool: { + addressPrefixes: [vpnClientAddressPool] + } + // OpenVPN supports the widest range of clients (incl. azure-vpn / OpenVPN + // on Linux/Mac/Windows). IkeV2 added for native Windows clients. + vpnClientProtocols: [ + 'OpenVPN' + 'IkeV2' + ] + // Cert-based auth: upload the root public cert. If empty we skip it so the + // gateway still deploys; add the cert afterwards with + // `az network vnet-gateway root-cert create`. + vpnClientRootCertificates: empty(vpnClientRootCertData) ? [] : [ + { + name: vpnClientRootCertName + properties: { + publicCertData: vpnClientRootCertData + } + } + ] + } + } +} + +output vpnGatewayId string = vpnGateway.id +output vpnGatewayName string = vpnGateway.name +output vpnPublicIp string = vpnPublicIp.properties.ipAddress diff --git a/deploy/aks/manifests/observability/otel-collector-config.yaml b/deploy/aks/manifests/observability/otel-collector-config.yaml new file mode 100644 index 000000000..aff4b8373 --- /dev/null +++ b/deploy/aks/manifests/observability/otel-collector-config.yaml @@ -0,0 +1,131 @@ +# --------------------------------------------------------------------------- +# otel-collector-config.yaml — ConfigMap with the OpenTelemetry Collector +# pipeline for the WHOLE cluster. +# +# Two log sources feed one logs pipeline: +# 1. filelog — tails /var/log/pods/**/*.log on every node (DaemonSet + hostPath), +# so ALL pod stdout/stderr across the cluster is captured ("otel for +# the entire AKS"), not just the portal. +# 2. otlp — the portal pushes its structured logs/traces/metrics here via +# OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 (gRPC). +# +# Sink: the `file` exporter writes rotated JSON to /mnt/otel-logs (a mounted +# Azure Files share — the otel-logs PVC). Operators download the archive from the +# Files share (see README "Observability"). A `debug` exporter mirrors a basic +# view to the collector's own stdout so `kubectl logs ds/otel-collector` works. +# +# Azure TABLE storage has NO native OTel Collector exporter — Azure FILES (the +# `file` exporter over a mounted SMB share) is the chosen archive sink. See README +# for scale-up alternatives (Loki/Blob, azuremonitor exporter). +# +# Pinned image: otel/opentelemetry-collector-contrib (contrib has filelog + +# k8sattributes + resourcedetection). Version pinned in otel-collector.yaml. +# --------------------------------------------------------------------------- +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector-config + namespace: memex + labels: + app.kubernetes.io/component: otel-collector +data: + collector.yaml: | + receivers: + # Portal (and any OTLP client) push endpoint. + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + # Node-local container logs. The container runtime writes one file per + # container under /var/log/pods/__//N.log. + filelog: + include: + - /var/log/pods/*/*/*.log + # Don't re-ingest our own collector output (it would loop). + exclude: + - /var/log/pods/*otel-collector*/*/*.log + start_at: end + include_file_path: true + include_file_name: false + # Standard containerd/CRI-O parsing: CRI log line -> timestamp/stream/log, + # then attach k8s metadata parsed from the file path. + operators: + - type: container + id: container-parser + # k8sattributes also enriches; container parser handles the CRI format and + # extracts namespace/pod/container from the path. + + processors: + batch: + timeout: 5s + send_batch_size: 1024 + # Detect node/cloud resource attributes (region, node, k8s.* from env). + resourcedetection: + detectors: [env, system] + timeout: 5s + override: false + # Enrich every record with pod/namespace/node/deployment metadata by + # correlating the source IP / pod UID against the k8s API. + k8sattributes: + auth_type: serviceAccount + passthrough: false + extract: + metadata: + - k8s.namespace.name + - k8s.pod.name + - k8s.pod.uid + - k8s.node.name + - k8s.deployment.name + - k8s.replicaset.name + pod_association: + - sources: + - from: resource_attribute + name: k8s.pod.uid + - sources: + - from: connection + + exporters: + # Primary archive sink: rotated JSON on the mounted Azure Files share. + # ${env:NODE_NAME} namespaces the file per node so DaemonSet replicas don't + # clobber one another's output on the shared share (downward API in the + # DaemonSet sets NODE_NAME from spec.nodeName). + file/logs: + path: /mnt/otel-logs/logs-${env:NODE_NAME}.json + rotation: + max_megabytes: 100 + max_backups: 10 + file/traces: + path: /mnt/otel-logs/traces-${env:NODE_NAME}.json + rotation: + max_megabytes: 100 + max_backups: 10 + file/metrics: + path: /mnt/otel-logs/metrics-${env:NODE_NAME}.json + rotation: + max_megabytes: 100 + max_backups: 10 + # Mirror to stdout for `kubectl logs` visibility (basic = one summary line + # per batch, NOT full payloads — keeps the collector's own log cheap). + debug: + verbosity: basic + + service: + extensions: [] + pipelines: + logs: + receivers: [filelog, otlp] + processors: [k8sattributes, resourcedetection, batch] + exporters: [file/logs, debug] + traces: + receivers: [otlp] + processors: [k8sattributes, resourcedetection, batch] + exporters: [file/traces, debug] + metrics: + receivers: [otlp] + processors: [resourcedetection, batch] + exporters: [file/metrics] + telemetry: + logs: + level: info diff --git a/deploy/aks/manifests/observability/otel-collector.yaml b/deploy/aks/manifests/observability/otel-collector.yaml new file mode 100644 index 000000000..145f2c1a2 --- /dev/null +++ b/deploy/aks/manifests/observability/otel-collector.yaml @@ -0,0 +1,162 @@ +# --------------------------------------------------------------------------- +# otel-collector.yaml — OpenTelemetry Collector as a DaemonSet (one pod per node) +# + ServiceAccount/RBAC + a ClusterIP Service. +# +# DaemonSet (not Deployment) because the `filelog` receiver tails the node-local +# /var/log/pods hostPath — every node needs a collector to read its own pods' +# stdout/stderr. The Service (otel-collector:4317/4318) front-ends the OTLP +# receivers so the portal can push to a stable name; OTLP traffic load-balances +# across the per-node collector pods (any of them can receive + archive). +# +# Apply (after the PVC + ConfigMap + StorageClass): +# kubectl apply -n memex -f ../storageclass-azurefile.yaml +# kubectl apply -n memex -f otel-pvc.yaml +# kubectl apply -n memex -f otel-collector-config.yaml +# kubectl apply -n memex -f otel-collector.yaml +# +# Image pinned to a recent contrib release (filelog + k8sattributes + file exporter +# live in -contrib, not the core distro). Bump deliberately; don't float :latest. +# --------------------------------------------------------------------------- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: otel-collector + namespace: memex + labels: + app.kubernetes.io/component: otel-collector +--- +# k8sattributes needs to read pod/namespace/node metadata cluster-wide. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: otel-collector + labels: + app.kubernetes.io/component: otel-collector +rules: + - apiGroups: [""] + resources: ["pods", "namespaces", "nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["replicasets", "deployments"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: otel-collector + labels: + app.kubernetes.io/component: otel-collector +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: otel-collector +subjects: + - kind: ServiceAccount + name: otel-collector + namespace: memex +--- +apiVersion: v1 +kind: Service +metadata: + name: otel-collector + namespace: memex + labels: + app.kubernetes.io/component: otel-collector +spec: + # Headless not required; a normal ClusterIP load-balances OTLP across node pods. + selector: + app.kubernetes.io/component: otel-collector + ports: + - name: otlp-grpc + port: 4317 + targetPort: 4317 + protocol: TCP + - name: otlp-http + port: 4318 + targetPort: 4318 + protocol: TCP +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: otel-collector + namespace: memex + labels: + app.kubernetes.io/component: otel-collector +spec: + selector: + matchLabels: + app.kubernetes.io/component: otel-collector + template: + metadata: + labels: + app.kubernetes.io/component: otel-collector + spec: + serviceAccountName: otel-collector + # Tolerate control-plane / tainted nodes so logs are captured everywhere. + tolerations: + - operator: Exists + securityContext: + # Container logs under /var/log/pods are root-owned; run the collector as + # root so filelog can read them. (The Azure Files archive share is mounted + # 0777 uid/gid 1654 via the StorageClass, so root can write it too.) + runAsUser: 0 + containers: + - name: otel-collector + image: otel/opentelemetry-collector-contrib:0.116.1 + args: + - --config=/conf/collector.yaml + env: + # Namespaces the archive file per node so DaemonSet replicas don't + # clobber each other on the shared Files share (see config exporters). + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ports: + - name: otlp-grpc + containerPort: 4317 + protocol: TCP + - name: otlp-http + containerPort: 4318 + protocol: TCP + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + memory: "512Mi" + volumeMounts: + - name: config + mountPath: /conf + readOnly: true + # Node-local container logs (read-only): the actual log files + the + # symlinked /var/log/containers and the /var/lib/docker|containerd + # backing dirs the symlinks point into. + - name: varlogpods + mountPath: /var/log/pods + readOnly: true + - name: varlogcontainers + mountPath: /var/log/containers + readOnly: true + # The Azure Files archive sink. + - name: otel-logs + mountPath: /mnt/otel-logs + volumes: + - name: config + configMap: + name: otel-collector-config + items: + - key: collector.yaml + path: collector.yaml + - name: varlogpods + hostPath: + path: /var/log/pods + type: Directory + - name: varlogcontainers + hostPath: + path: /var/log/containers + type: Directory + - name: otel-logs + persistentVolumeClaim: + claimName: otel-logs diff --git a/deploy/aks/manifests/observability/otel-pvc.yaml b/deploy/aks/manifests/observability/otel-pvc.yaml new file mode 100644 index 000000000..020eb5dc5 --- /dev/null +++ b/deploy/aks/manifests/observability/otel-pvc.yaml @@ -0,0 +1,30 @@ +# --------------------------------------------------------------------------- +# otel-pvc.yaml — RWX Azure Files PVC for the OpenTelemetry Collector log archive. +# +# The collector's `file` exporter writes rotated JSON here (/mnt/otel-logs). It is +# ReadWriteMany because the collector runs as a DaemonSet (one pod per node) and +# every pod writes to the same share — each namespaces its file by node name +# (logs-.json) so they don't collide. Uses the custom `azurefile-memex` +# StorageClass (uid/gid 1654, nobrl) — apply storageclass-azurefile.yaml first. +# +# kubectl apply -n memex -f ../storageclass-azurefile.yaml # if not already applied +# kubectl apply -n memex -f otel-pvc.yaml +# +# For STATIC binding to the pre-created `otel-logs` share in files.bicep, replace +# this with a PV referencing volumeAttributes { storageAccount, shareName: otel-logs } +# + a matching PVC (same pattern as README "Static PV binding"). +# --------------------------------------------------------------------------- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: otel-logs + namespace: memex + labels: + app.kubernetes.io/component: otel-collector +spec: + accessModes: + - ReadWriteMany + storageClassName: azurefile-memex + resources: + requests: + storage: 64Gi diff --git a/deploy/aks/manifests/pgbackrest/configmap.yaml b/deploy/aks/manifests/pgbackrest/configmap.yaml new file mode 100644 index 000000000..6ddf654bf --- /dev/null +++ b/deploy/aks/manifests/pgbackrest/configmap.yaml @@ -0,0 +1,66 @@ +# --------------------------------------------------------------------------- +# configmap.yaml — pgBackRest configuration for an Azure Blob repository. +# +# Defines a single stanza "memex" whose repo1 is the Azure blob container. With +# repo1-azure-key-type=auto and a Workload-Identity service account, pgBackRest +# uses the projected federated token — NO account key on disk. If you prefer a +# key, set repo1-azure-key (and repo1-azure-key-type=shared) from the +# pgbackrest secret instead, and drop the workload-identity annotation. +# +# WAL archiving (archive-async + the postgres archive_command) is what enables +# Point-In-Time Recovery: every completed WAL segment is pushed to blob, so a +# restore can replay to any timestamp. +# +# kubectl apply -n memex -f configmap.yaml +# --------------------------------------------------------------------------- +apiVersion: v1 +kind: ConfigMap +metadata: + name: pgbackrest-config + namespace: memex + labels: + app.kubernetes.io/component: pgbackrest +data: + pgbackrest.conf: | + [global] + # ---- Azure Blob repository ---- + repo1-type=azure + repo1-path=/memex + repo1-azure-account=__AZURE_ACCOUNT__ + repo1-azure-container=__AZURE_CONTAINER__ + # auto = use the AKS Workload Identity federated token (keyless). For a + # shared-key setup instead, set: repo1-azure-key-type=shared + # repo1-azure-key= + repo1-azure-key-type=auto + repo1-storage-verify-tls=y + + # ---- Retention ---- + repo1-retention-full=4 + repo1-retention-diff=6 + + # ---- Performance / safety ---- + process-max=4 + compress-type=zst + compress-level=3 + start-fast=y + delta=y + archive-async=y + spool-path=/var/spool/pgbackrest + log-level-console=info + log-level-file=detail + log-path=/var/log/pgbackrest + + [memex] + pg1-path=/var/lib/postgresql/data + pg1-port=5432 + pg1-user=postgres + + # Postgres parameters that must be appended for WAL archiving. The chart's + # pgvector image reads /var/lib/postgresql/data/postgresql.conf; the + # sidecar-patch initContainer appends these on first boot (idempotent). + postgres-archive.conf: | + archive_mode = on + archive_command = 'pgbackrest --config=/etc/pgbackrest/pgbackrest.conf --stanza=memex archive-push %p' + archive_timeout = 60 + wal_level = replica + max_wal_senders = 3 diff --git a/deploy/aks/manifests/pgbackrest/cronjobs.yaml b/deploy/aks/manifests/pgbackrest/cronjobs.yaml new file mode 100644 index 000000000..d42ab7e85 --- /dev/null +++ b/deploy/aks/manifests/pgbackrest/cronjobs.yaml @@ -0,0 +1,176 @@ +# --------------------------------------------------------------------------- +# cronjobs.yaml — Scheduled pgBackRest backups + one-time stanza bootstrap. +# +# Three resources: +# 1. Job pgbackrest-stanza-create — run ONCE after the DB is up + WAL +# archiving is enabled. Creates the stanza in the blob repo and runs an +# initial `check`. Re-runnable (idempotent). +# 2. CronJob pgbackrest-full — weekly full backup. +# 3. CronJob pgbackrest-diff — daily differential backup. +# +# All three exec pgBackRest against the SAME pg1-path/repo as the sidecar by +# mounting the same config + the data PVC, and run as pgbackrest-sa for keyless +# blob auth. They target the running postgres via the StatefulSet's headless +# pod (memex-postgres-statefulset-0); pgBackRest needs the data dir locally, +# hence they schedule onto the node holding the pgdata PVC by reusing it RWO — +# in practice run them as `kubectl exec` into the sidecar (see README runbook) +# OR rely on these Jobs which attach the same RWO PVC when the DB pod is +# briefly drained. The exec-into-sidecar path (README) is the recommended, +# zero-contention method; these CronJobs are the unattended fallback. +# +# kubectl apply -n memex -f cronjobs.yaml +# kubectl create job --from=cronjob/pgbackrest-full pgbackrest-bootstrap -n memex # optional manual full +# --------------------------------------------------------------------------- +apiVersion: batch/v1 +kind: Job +metadata: + name: pgbackrest-stanza-create + namespace: memex + labels: + app.kubernetes.io/component: pgbackrest +spec: + backoffLimit: 3 + ttlSecondsAfterFinished: 86400 + template: + metadata: + labels: + azure.workload.identity/use: "true" + spec: + serviceAccountName: pgbackrest-sa + restartPolicy: OnFailure + containers: + - name: stanza-create + image: docker.io/woblerr/pgbackrest:2.54.2 + command: + - /bin/sh + - -c + - | + set -e + sed -e "s/__AZURE_ACCOUNT__/${PGBACKREST_AZURE_ACCOUNT}/" \ + -e "s/__AZURE_CONTAINER__/${PGBACKREST_AZURE_CONTAINER}/" \ + /etc/pgbackrest-src/pgbackrest.conf > /etc/pgbackrest/pgbackrest.conf + pgbackrest --config=/etc/pgbackrest/pgbackrest.conf --stanza=memex stanza-create + pgbackrest --config=/etc/pgbackrest/pgbackrest.conf --stanza=memex check + env: + - name: PGBACKREST_AZURE_ACCOUNT + value: "__AZURE_ACCOUNT__" + - name: PGBACKREST_AZURE_CONTAINER + value: "pgbackrest" + volumeMounts: + - { name: memex-pgdata, mountPath: /var/lib/postgresql/data } + - { name: pgbackrest-src, mountPath: /etc/pgbackrest-src } + - { name: pgbackrest-conf, mountPath: /etc/pgbackrest } + volumes: + - name: memex-pgdata + persistentVolumeClaim: + claimName: memex-pgdata + - name: pgbackrest-src + configMap: + name: pgbackrest-config + items: [{ key: pgbackrest.conf, path: pgbackrest.conf }] + - name: pgbackrest-conf + emptyDir: {} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: pgbackrest-full + namespace: memex + labels: + app.kubernetes.io/component: pgbackrest +spec: + schedule: "0 2 * * 0" # weekly full, Sunday 02:00 UTC + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + template: + metadata: + labels: + azure.workload.identity/use: "true" + spec: + serviceAccountName: pgbackrest-sa + restartPolicy: OnFailure + containers: + - name: backup-full + image: docker.io/woblerr/pgbackrest:2.54.2 + command: + - /bin/sh + - -c + - | + set -e + sed -e "s/__AZURE_ACCOUNT__/${PGBACKREST_AZURE_ACCOUNT}/" \ + -e "s/__AZURE_CONTAINER__/${PGBACKREST_AZURE_CONTAINER}/" \ + /etc/pgbackrest-src/pgbackrest.conf > /etc/pgbackrest/pgbackrest.conf + pgbackrest --config=/etc/pgbackrest/pgbackrest.conf --stanza=memex --type=full backup + env: + - { name: PGBACKREST_AZURE_ACCOUNT, value: "__AZURE_ACCOUNT__" } + - { name: PGBACKREST_AZURE_CONTAINER, value: "pgbackrest" } + volumeMounts: + - { name: memex-pgdata, mountPath: /var/lib/postgresql/data } + - { name: pgbackrest-src, mountPath: /etc/pgbackrest-src } + - { name: pgbackrest-conf, mountPath: /etc/pgbackrest } + volumes: + - name: memex-pgdata + persistentVolumeClaim: + claimName: memex-pgdata + - name: pgbackrest-src + configMap: + name: pgbackrest-config + items: [{ key: pgbackrest.conf, path: pgbackrest.conf }] + - name: pgbackrest-conf + emptyDir: {} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: pgbackrest-diff + namespace: memex + labels: + app.kubernetes.io/component: pgbackrest +spec: + schedule: "0 2 * * 1-6" # daily differential, Mon-Sat 02:00 UTC + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + template: + metadata: + labels: + azure.workload.identity/use: "true" + spec: + serviceAccountName: pgbackrest-sa + restartPolicy: OnFailure + containers: + - name: backup-diff + image: docker.io/woblerr/pgbackrest:2.54.2 + command: + - /bin/sh + - -c + - | + set -e + sed -e "s/__AZURE_ACCOUNT__/${PGBACKREST_AZURE_ACCOUNT}/" \ + -e "s/__AZURE_CONTAINER__/${PGBACKREST_AZURE_CONTAINER}/" \ + /etc/pgbackrest-src/pgbackrest.conf > /etc/pgbackrest/pgbackrest.conf + pgbackrest --config=/etc/pgbackrest/pgbackrest.conf --stanza=memex --type=diff backup + env: + - { name: PGBACKREST_AZURE_ACCOUNT, value: "__AZURE_ACCOUNT__" } + - { name: PGBACKREST_AZURE_CONTAINER, value: "pgbackrest" } + volumeMounts: + - { name: memex-pgdata, mountPath: /var/lib/postgresql/data } + - { name: pgbackrest-src, mountPath: /etc/pgbackrest-src } + - { name: pgbackrest-conf, mountPath: /etc/pgbackrest } + volumes: + - name: memex-pgdata + persistentVolumeClaim: + claimName: memex-pgdata + - name: pgbackrest-src + configMap: + name: pgbackrest-config + items: [{ key: pgbackrest.conf, path: pgbackrest.conf }] + - name: pgbackrest-conf + emptyDir: {} diff --git a/deploy/aks/manifests/pgbackrest/serviceaccount.yaml b/deploy/aks/manifests/pgbackrest/serviceaccount.yaml new file mode 100644 index 000000000..0eec974fb --- /dev/null +++ b/deploy/aks/manifests/pgbackrest/serviceaccount.yaml @@ -0,0 +1,24 @@ +# --------------------------------------------------------------------------- +# serviceaccount.yaml — Workload-Identity service account for pgBackRest pods. +# +# The federated identity created in infra/modules/storage.bicep trusts the +# subject system:serviceaccount:memex:pgbackrest-sa . The +# azure.workload.identity/client-id annotation tells the AKS Workload Identity +# webhook which managed identity to project a token for, giving pgBackRest +# KEYLESS access to the backup blob container. +# +# Replace with the pgBackRestIdentityClientId +# output from the infra deployment (or set pgbackrest.azure.workloadIdentityClientId +# in values.aks.yaml and template it). +# +# kubectl apply -n memex -f serviceaccount.yaml +# --------------------------------------------------------------------------- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pgbackrest-sa + namespace: memex + labels: + azure.workload.identity/use: "true" + annotations: + azure.workload.identity/client-id: "" diff --git a/deploy/aks/manifests/pgbackrest/sidecar-patch.yaml b/deploy/aks/manifests/pgbackrest/sidecar-patch.yaml new file mode 100644 index 000000000..121fe91f8 --- /dev/null +++ b/deploy/aks/manifests/pgbackrest/sidecar-patch.yaml @@ -0,0 +1,102 @@ +# --------------------------------------------------------------------------- +# sidecar-patch.yaml — Add the pgBackRest sidecar + WAL-archive wiring to the +# chart's Postgres StatefulSet. +# +# Strategic-merge patch applied AFTER helm install + postgres-pvc-patch.yaml: +# +# kubectl patch statefulset memex-postgres-statefulset -n memex \ +# --type strategic --patch-file pgbackrest/sidecar-patch.yaml +# +# What it adds: +# - serviceAccountName: pgbackrest-sa (Workload Identity -> Blob, keyless) +# - an initContainer that appends postgres-archive.conf to postgresql.conf so +# the server runs archive_command = pgbackrest archive-push (idempotent) +# - shared volumes: the pgbackrest config, spool dir, log dir, and the data +# PVC (so the sidecar can read pg1-path for backups) +# - a long-running pgBackRest sidecar that hosts the async archive worker and +# is the exec target for ad-hoc backup/restore commands +# +# The scheduled full/diff backups themselves run as CronJobs (./cronjobs.yaml), +# not in the sidecar, so a backup never competes with the live archive worker. +# --------------------------------------------------------------------------- +spec: + template: + metadata: + labels: + azure.workload.identity/use: "true" + spec: + serviceAccountName: pgbackrest-sa + initContainers: + # Enable WAL archiving on the data dir before postgres starts. + - name: enable-wal-archiving + image: docker.io/pgvector/pgvector:pg17 + command: + - /bin/sh + - -c + - | + set -e + CONF=/var/lib/postgresql/data/postgresql.conf + MARK="# pgbackrest-archive (managed)" + if [ -f "$CONF" ] && ! grep -q "$MARK" "$CONF"; then + echo "" >> "$CONF" + echo "$MARK" >> "$CONF" + cat /etc/pgbackrest-archive/postgres-archive.conf >> "$CONF" + echo "appended WAL archiving config" + else + echo "WAL archiving config already present or data dir not initialized yet" + fi + volumeMounts: + - name: memex-pgdata + mountPath: /var/lib/postgresql/data + - name: pgbackrest-archive-conf + mountPath: /etc/pgbackrest-archive + containers: + # The sidecar: hosts the async WAL archive worker + exec target. + - name: pgbackrest + image: docker.io/woblerr/pgbackrest:2.54.2 + command: + - /bin/sh + - -c + - | + # Render account/container into the config from env, then idle so + # the async archive-push worker (spawned by postgres) and exec'd + # backup commands share this container's filesystem + identity. + sed -e "s/__AZURE_ACCOUNT__/${PGBACKREST_AZURE_ACCOUNT}/" \ + -e "s/__AZURE_CONTAINER__/${PGBACKREST_AZURE_CONTAINER}/" \ + /etc/pgbackrest-src/pgbackrest.conf > /etc/pgbackrest/pgbackrest.conf + echo "pgbackrest sidecar ready"; exec sleep infinity + env: + - name: PGBACKREST_AZURE_ACCOUNT + value: "__AZURE_ACCOUNT__" # replace or template from values + - name: PGBACKREST_AZURE_CONTAINER + value: "pgbackrest" + volumeMounts: + - name: memex-pgdata + mountPath: /var/lib/postgresql/data + - name: pgbackrest-src + mountPath: /etc/pgbackrest-src + - name: pgbackrest-conf + mountPath: /etc/pgbackrest + - name: pgbackrest-spool + mountPath: /var/spool/pgbackrest + - name: pgbackrest-log + mountPath: /var/log/pgbackrest + volumes: + - name: pgbackrest-src + configMap: + name: pgbackrest-config + items: + - key: pgbackrest.conf + path: pgbackrest.conf + - name: pgbackrest-archive-conf + configMap: + name: pgbackrest-config + items: + - key: postgres-archive.conf + path: postgres-archive.conf + - name: pgbackrest-conf + emptyDir: {} + - name: pgbackrest-spool + emptyDir: {} + - name: pgbackrest-log + emptyDir: {} diff --git a/deploy/aks/manifests/portal-ha-patch.yaml b/deploy/aks/manifests/portal-ha-patch.yaml new file mode 100644 index 000000000..0a438263f --- /dev/null +++ b/deploy/aks/manifests/portal-ha-patch.yaml @@ -0,0 +1,109 @@ +# --------------------------------------------------------------------------- +# portal-ha-patch.yaml — Strategic-merge patches to turn the single-replica +# chart Deployment/StatefulSet into an HA, persistent +# AKS workload. +# +# The ../helm chart ships replicas: 1 with emptyDir volumes. Rather than fork the +# chart, apply these patches AFTER `helm install`: +# +# kubectl patch deployment memex-portal-deployment -n memex \ +# --type strategic --patch-file portal-ha-patch.yaml +# kubectl patch statefulset memex-postgres-statefulset -n memex \ +# --type strategic --patch-file postgres-pvc-patch.yaml +# +# This document is the PORTAL patch. (Postgres patch is the next file.) +# +# Changes: +# - replicas 1 -> 2 (launch 2 by default; bump to 3 for full 3-zone spread) +# - emptyDir -> the RWX PVCs from portal-pvcs.yaml +# /data -> memex-data (framework caches) +# /mnt/users -> memex-users (CLI configs) +# /mnt/content -> memex-content (content collection — NEW) +# /mnt/attachments -> memex-attachments (attachments drive — NEW) +# - zone-spread topologySpreadConstraints so the 3 replicas land in 3 AZs +# - readiness/liveness probes on the portal HTTP port +# +# STRATEGIC-MERGE MERGE-KEY BEHAVIOUR (why this is safe / additive): +# * `spec.template.spec.containers` merges by key `name` +# * containers[].volumeMounts merges by key `mountPath` +# * spec.template.spec.volumes merges by key `name` +# The chart template already defines volumeMounts for /data and /mnt/users +# (volume names memex-data / memex-users) and declares those volumes as +# emptyDir. This patch: +# - re-declares the memex-data / memex-users volumeMounts (no-op merge on the +# same mountPath) and ADDS the two new ones (/mnt/content, /mnt/attachments); +# - re-declares the memex-data / memex-users volumes with a PVC source — same +# `name`, so the merge OVERRIDES the emptyDir source with the PVC (intended); +# - ADDS the memex-content / memex-attachments volumes. +# No list is wholesale-replaced: strategic merge augments by merge key. (A plain +# JSON merge / `--type merge` WOULD replace the whole list — use --type strategic.) +# --------------------------------------------------------------------------- +spec: + replicas: 2 + template: + spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/component: memex-portal + containers: + - name: memex-portal + readinessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 20 + periodSeconds: 10 + failureThreshold: 6 + livenessProbe: + httpGet: + path: / + port: 8080 + initialDelaySeconds: 60 + periodSeconds: 20 + failureThreshold: 6 + # Sizable AI image with GENEROUS limits per replica (kept in sync with + # resources.portal in values.aks.yaml). The AI image is memory-hungry + # (NodeType compile cache, agent runtimes); on the 32 GiB Standard_D8s_v5 + # nodes these generous limits avoid OOMKills with room to spare. + resources: + requests: + cpu: "4" + memory: "8Gi" + limits: + cpu: "6" + memory: "16Gi" + # volumeMounts merge by `mountPath`. The first two mirror the chart's + # existing mounts (no-op); the last two are the NEW /mnt drives. + volumeMounts: + - name: memex-data + mountPath: /data + - name: memex-users + mountPath: /mnt/users + - name: memex-content + mountPath: /mnt/content + - name: memex-attachments + mountPath: /mnt/attachments + - name: memex-workspace + mountPath: /workspace + # volumes merge by `name`. memex-data / memex-users override the chart's + # emptyDir with the real RWX PVCs; memex-content / memex-attachments are new. + volumes: + - name: memex-data + persistentVolumeClaim: + claimName: memex-data + - name: memex-users + persistentVolumeClaim: + claimName: memex-users + - name: memex-content + persistentVolumeClaim: + claimName: memex-content + - name: memex-attachments + persistentVolumeClaim: + claimName: memex-attachments + - name: memex-workspace + persistentVolumeClaim: + claimName: memex-workspace diff --git a/deploy/aks/manifests/portal-ingress.yaml b/deploy/aks/manifests/portal-ingress.yaml new file mode 100644 index 000000000..e06e71fb7 --- /dev/null +++ b/deploy/aks/manifests/portal-ingress.yaml @@ -0,0 +1,59 @@ +# --------------------------------------------------------------------------- +# portal-ingress.yaml — Ingress for the Memex portal with COOKIE SESSION +# AFFINITY (mandatory for Blazor Server). +# +# Blazor Server keeps a stateful SignalR circuit per browser tab. If a reconnect +# is routed to a different replica the circuit is lost and the user sees "Attempting +# to reconnect..." then a full reload. Cookie-based affinity pins each client to +# the replica that owns its circuit. +# +# This file shows the AKS application-routing (managed ingress-nginx) variant. +# For AGIC, swap the annotations for the appgw.* set (commented below) and set +# ingressClassName: azure-application-gateway. +# +# kubectl apply -n memex -f portal-ingress.yaml +# +# Prereq: enable an ingress controller. Either +# az aks approuting enable -g -n # managed nginx (used here) +# or install AGIC / ingress-nginx yourself (see README). +# --------------------------------------------------------------------------- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: memex-portal + namespace: memex + annotations: + # ---- ingress-nginx / AKS app-routing: cookie affinity ---- + nginx.ingress.kubernetes.io/affinity: "cookie" + nginx.ingress.kubernetes.io/affinity-mode: "persistent" + nginx.ingress.kubernetes.io/session-cookie-name: "MEMEX_AFFINITY" + nginx.ingress.kubernetes.io/session-cookie-max-age: "172800" + nginx.ingress.kubernetes.io/session-cookie-samesite: "Lax" + # Long-lived SignalR / WebSocket connections — bump the proxy timeouts. + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" + # Allow large markdown/asset uploads. + nginx.ingress.kubernetes.io/proxy-body-size: "100m" + # ---- AGIC alternative (uncomment, and set ingressClassName below) ---- + # appgw.ingress.kubernetes.io/cookie-based-affinity: "true" + # appgw.ingress.kubernetes.io/request-timeout: "3600" + # appgw.ingress.kubernetes.io/connection-draining: "true" + # appgw.ingress.kubernetes.io/connection-draining-timeout: "60" +spec: + ingressClassName: webapprouting.kubernetes.azure.com + # For AGIC use: ingressClassName: azure-application-gateway + tls: + - hosts: + - memex.systemorph.com + secretName: memex-tls + rules: + - host: memex.systemorph.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: memex-portal-service + port: + number: 8080 diff --git a/deploy/aks/manifests/portal-pvcs.yaml b/deploy/aks/manifests/portal-pvcs.yaml new file mode 100644 index 000000000..7dc7dcdc3 --- /dev/null +++ b/deploy/aks/manifests/portal-pvcs.yaml @@ -0,0 +1,129 @@ +# --------------------------------------------------------------------------- +# portal-pvcs.yaml — ReadWriteMany PVCs for the HA portal + the Postgres PVC. +# +# The ../helm chart mounts /data and /mnt/users as emptyDir (single-pod). For HA +# (replicas > 1) those must be shared, persistent ReadWriteMany volumes so every +# Blazor Server replica sees the same filesystem backend. Azure Files CSI +# provides RWX; Azure Disk cannot. +# +# STORAGE LAYOUT (one Azure Files "drive" per concern, mounted at explicit paths): +# memex-data -> /data framework caches ONLY (DataProtection keys, +# assembly-cache, nuget-cache). Small + churny. +# memex-content -> /mnt/content the CONTENT COLLECTION (uploaded files, media, +# attachments served per node hub). Storage__BasePath. +# memex-attachments -> /mnt/attachments dedicated drive for the "attachments" collection +# (forward-looking — see values.aks.yaml note). +# memex-users -> /mnt/users co-hosted CLI configs (unchanged). +# memex-pgdata -> Postgres data single-writer RWO on managed-csi (Premium SSD). +# +# All RWX shares use the custom `azurefile-memex` StorageClass (storageclass-azurefile.yaml) +# so the non-root portal (uid 1654) writes cleanly. Apply order: +# kubectl apply -f storageclass-azurefile.yaml +# kubectl apply -n memex -f portal-pvcs.yaml +# ...then patch the portal Deployment to use these claims (see ./portal-ha-patch.yaml). +# --------------------------------------------------------------------------- +# /data — framework caches only (DataProtection keys, assembly-cache, nuget-cache). +# Shrunk from 64Gi: content no longer lives here (it moved to memex-content), so +# /data just holds small, churny cache files. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: memex-data + namespace: memex + labels: + app.kubernetes.io/component: memex-portal +spec: + accessModes: + - ReadWriteMany + storageClassName: azurefile-memex + resources: + requests: + storage: 16Gi +--- +# /mnt/content — the content collection (Storage__BasePath). Uploaded files, media, +# per-node-hub content subdirectories. Sized for real user content; expand as needed +# (allowVolumeExpansion is on in the StorageClass). +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: memex-content + namespace: memex + labels: + app.kubernetes.io/component: memex-portal +spec: + accessModes: + - ReadWriteMany + storageClassName: azurefile-memex + resources: + requests: + storage: 128Gi +--- +# /mnt/attachments — dedicated drive for the "attachments" collection. The portal +# maps an "attachments" content collection; isolating it on its own share keeps +# large binary attachments off the content drive. See values.aks.yaml for the +# forward-looking note on repointing attachments by env. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: memex-attachments + namespace: memex + labels: + app.kubernetes.io/component: memex-portal +spec: + accessModes: + - ReadWriteMany + storageClassName: azurefile-memex + resources: + requests: + storage: 64Gi +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: memex-users + namespace: memex + labels: + app.kubernetes.io/component: memex-portal +spec: + accessModes: + - ReadWriteMany + storageClassName: azurefile-memex + resources: + requests: + storage: 32Gi +--- +# /workspace — per-user git working trees ({userId}/{repoSlug}) for the in-portal +# checkout/edit/commit feature + the co-hosted AI CLIs. RWX so both HA replicas share +# the same checkouts. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: memex-workspace + namespace: memex + labels: + app.kubernetes.io/component: memex-portal +spec: + accessModes: + - ReadWriteMany + storageClassName: azurefile-memex + resources: + requests: + storage: 64Gi +--- +# Postgres data volume — single-writer, so ReadWriteOnce on Premium SSD. +# (If you keep the chart's StatefulSet emptyDir you LOSE data on pod restart — +# always bind a real PVC for the database.) +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: memex-pgdata + namespace: memex + labels: + app.kubernetes.io/component: memex-postgres +spec: + accessModes: + - ReadWriteOnce + storageClassName: managed-csi + resources: + requests: + storage: 128Gi diff --git a/deploy/aks/manifests/postgres-pvc-patch.yaml b/deploy/aks/manifests/postgres-pvc-patch.yaml new file mode 100644 index 000000000..1ef413d85 --- /dev/null +++ b/deploy/aks/manifests/postgres-pvc-patch.yaml @@ -0,0 +1,34 @@ +# --------------------------------------------------------------------------- +# postgres-pvc-patch.yaml — Bind the Postgres StatefulSet to a real PVC and add +# the pgBackRest WAL-archive volume + sidecar. +# +# The chart's StatefulSet uses emptyDir for /var/lib/postgresql/data (DATA LOSS +# on restart). This patch: +# - replaces the data emptyDir with the memex-pgdata PVC (managed-csi, RWO) +# - mounts a shared "pgbackrest" emptyDir for the spool/socket so the sidecar +# and the postgres container share pgBackRest's stanza working dir +# - injects the pgbackrest config from the ConfigMap +# - adds the WAL archiving env so the chart's postgres container runs with +# archive_command pointing at pgbackrest (see configmap.yaml -> command) +# +# Apply with: +# kubectl patch statefulset memex-postgres-statefulset -n memex \ +# --type strategic --patch-file postgres-pvc-patch.yaml +# +# NOTE: the pgBackRest *sidecar* and the WAL-archive wiring are in +# ./pgbackrest/sidecar-patch.yaml — keep this file focused on persistence so you +# can apply just the PVC binding even if you disable pgBackRest. +# --------------------------------------------------------------------------- +spec: + template: + spec: + containers: + - name: memex-postgres + volumeMounts: + - name: memex-pgdata + mountPath: /var/lib/postgresql/data + volumes: + # Drop the emptyDir; bind the real PVC created by portal-pvcs.yaml. + - name: memex-pgdata + persistentVolumeClaim: + claimName: memex-pgdata diff --git a/deploy/aks/manifests/secretproviderclass.yaml b/deploy/aks/manifests/secretproviderclass.yaml new file mode 100644 index 000000000..805b9799b --- /dev/null +++ b/deploy/aks/manifests/secretproviderclass.yaml @@ -0,0 +1,55 @@ +# =========================================================================== +# SecretProviderClass — mounts the AzureFoundry API key from the `Systemorph` +# Key Vault into the portal pod via the AKS Key Vault Secrets Provider (CSI) +# add-on, and syncs it into a K8s Secret the portal reads via `envFrom`. +# +# Apply alongside the Helm release. The portal Deployment MUST mount this SPC as +# a CSI volume for the `secretObjects` sync to fire (the CSI driver only syncs to +# a K8s Secret while a pod mounts the volume). See the volume / volumeMount / +# envFrom patch in Doc/Architecture/DeploymentOptions.md. +# +# Prereqs: +# • azureKeyvaultSecretsProvider add-on enabled on memexaks-cluster (it is). +# • Its user-assigned identity (clientId below) has GET on Systemorph KV secrets +# — RBAC role "Key Vault Secrets User" on the vault, or an access policy with +# secret get/list. +# +# NOTE: the encryption master key (Ai:KeyProtection:MasterKey) is intentionally +# NOT managed here — it is provided out-of-band and overriding it would make every +# stored `enc:` provider key undecryptable. Add it to `objects`/`secretObjects` +# ONLY if you point it at the exact same value the running deployment already uses. +# =========================================================================== +apiVersion: secrets-store.csi.x-k8s.io/v1 +kind: SecretProviderClass +metadata: + name: memex-portal-ai-secrets + namespace: memex +spec: + provider: azure + parameters: + usePodIdentity: "false" + useVMManagedIdentity: "true" + # azureKeyvaultSecretsProvider add-on user-assigned identity (clientId) on memexaks-cluster. + userAssignedIdentityID: "6c9dcc8d-d5b3-4545-afa1-209b33e8a1ba" + keyvaultName: "Systemorph" + tenantId: "3a01d7ac-3330-444d-942d-975eb491b5d6" + objects: | + array: + - | + objectName: AzureFoundry-ApiKey + objectType: secret + - | + objectName: Ai-KeyProtection-MasterKey + objectType: secret + # Sync the mounted KV secrets into a K8s Secret the portal envFrom's. The secret KEY is the env + # var name → config key: AzureFoundry__ApiKey → AzureFoundry:ApiKey (the AzureFoundry factory key), + # Ai__KeyProtection__MasterKey → Ai:KeyProtection:MasterKey (the at-rest envelope key — MUST equal + # the value the deployment already uses, or stored enc: provider keys become undecryptable). + secretObjects: + - secretName: memex-portal-ai-secrets + type: Opaque + data: + - objectName: AzureFoundry-ApiKey + key: AzureFoundry__ApiKey + - objectName: Ai-KeyProtection-MasterKey + key: Ai__KeyProtection__MasterKey diff --git a/deploy/aks/manifests/storageclass-azurefile.yaml b/deploy/aks/manifests/storageclass-azurefile.yaml new file mode 100644 index 000000000..6b949e527 --- /dev/null +++ b/deploy/aks/manifests/storageclass-azurefile.yaml @@ -0,0 +1,66 @@ +# --------------------------------------------------------------------------- +# storageclass-azurefile.yaml — Custom Azure Files (SMB) StorageClass tuned for +# the NON-ROOT Memex portal container. +# +# WHY A CUSTOM CLASS (and not the built-in `azurefile-csi`)? +# ---------------------------------------------------------- +# The portal image runs as the .NET `app` user — uid 1654 / gid 1654 (the +# standard non-root uid baked into the `mcr.microsoft.com/dotnet/aspnet` +# chiseled images). On a freshly-provisioned Azure Files share the default CSI +# mountOptions are `file_mode=0777,dir_mode=0777,uid=0,gid=0` — world-writable +# but root-OWNED. That is *usually* enough (0777 lets anyone write), but it is +# brittle: any path the app `chmod`s, any tool that checks ownership, or a share +# whose CSI defaults drift to a tighter mode, and the non-root process hits +# UnauthorizedAccessException: Access to the path '/.../dataprotection-keys' is denied +# (the exact failure that bit the Docker-Compose deploy on a root-owned volume). +# +# Pinning `uid=1654,gid=1654` makes EVERY file/dir on the share owned by the +# portal user, so writes succeed regardless of mode drift — explicit and robust, +# as the deployment brief requires. +# +# mountOptions rationale: +# dir_mode=0777,file_mode=0777 — permissive mode (belt-and-braces with uid) +# uid=1654,gid=1654 — own every inode as the .NET `app` user +# mfsymlinks — emulate POSIX symlinks over SMB (DataProtection +# and some asset flows create symlinks) +# cache=strict — coherent caching; safe for the read-mostly +# content/cache workloads here +# actimeo=30 — 30 s attribute cache; cuts SMB round-trips on +# the many small-file stats the content service does +# nobrl — do NOT send byte-range-lock requests. Azure Files +# SMB rejects the POSIX advisory locks that SQLite +# and other file-lock-y libraries take, surfacing as +# "Operation not supported" / EIO. `nobrl` makes +# those locks no-ops so file-locking workloads run. +# +# Used by the memex-data / memex-content / memex-attachments / memex-users PVCs +# (portal-pvcs.yaml) and the otel-logs PVC (observability/). Apply BEFORE the PVCs: +# +# kubectl apply -f storageclass-azurefile.yaml +# +# NOTE: provisioner `file.csi.azure.com` is the Azure Files CSI driver enabled in +# aks.bicep (storageProfile.fileCSIDriver). `skuName: Standard_LRS` keeps cost low; +# switch to `Premium_LRS` (and mind the 100 GiB minimum share size) for IOPS-heavy +# content. `Standard_ZRS` gives zone-redundant durability where the region offers it. +# --------------------------------------------------------------------------- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azurefile-memex +provisioner: file.csi.azure.com +# Retain so an accidental PVC delete doesn't nuke the share + its content/keys. +# (The built-in azurefile-csi uses Delete; we deliberately diverge for safety.) +reclaimPolicy: Retain +volumeBindingMode: Immediate +allowVolumeExpansion: true +parameters: + skuName: Standard_LRS +mountOptions: + - dir_mode=0777 + - file_mode=0777 + - uid=1654 + - gid=1654 + - mfsymlinks + - cache=strict + - actimeo=30 + - nobrl diff --git a/deploy/aks/scripts/aks-extras.yaml b/deploy/aks/scripts/aks-extras.yaml new file mode 100644 index 000000000..101ee45e0 --- /dev/null +++ b/deploy/aks/scripts/aks-extras.yaml @@ -0,0 +1,29 @@ +# Non-root-writable Azure Files StorageClass + RWX PVCs for the portal's /data +# (framework caches) and /mnt/content (content collection). dir/file_mode=0777 lets the +# non-root `app` user write the SMB mount regardless of its uid (SMB ignores fsGroup). +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: azurefile-memex +provisioner: file.csi.azure.com +allowVolumeExpansion: true +parameters: + skuName: Standard_LRS +mountOptions: [dir_mode=0777, file_mode=0777, uid=1654, gid=1654, mfsymlinks, cache=strict, actimeo=30, nobrl] +reclaimPolicy: Retain +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: { name: memex-data, namespace: memex } +spec: + accessModes: [ReadWriteMany] + storageClassName: azurefile-memex + resources: { requests: { storage: 16Gi } } +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: { name: memex-content, namespace: memex } +spec: + accessModes: [ReadWriteMany] + storageClassName: azurefile-memex + resources: { requests: { storage: 64Gi } } diff --git a/deploy/aks/scripts/deploy.sh b/deploy/aks/scripts/deploy.sh new file mode 100644 index 000000000..a8f3ab7c5 --- /dev/null +++ b/deploy/aks/scripts/deploy.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Deploy the Memex portal + migration onto the (private) AKS cluster. +# +# Run from a STAGING dir that contains: this script, aks-extras.yaml, values.deploy.yaml +# (your secrets — see values.deploy.example.yaml), a copy of the Helm chart as ./helm, and a +# copy of ../values.aks.yaml. Then, from a machine with `az`: +# +# cd +# cp -r /deploy/helm ./helm +# cp /deploy/aks/values.aks.yaml . +# export MEMEX_PG_CONN='Host=;Port=5432;Username=memexadmin;Password=;Database=memex;SslMode=Require;Trust Server Certificate=true' +# az aks command invoke -g -n --command "MEMEX_PG_CONN='$MEMEX_PG_CONN' bash deploy.sh" --file . +set -uo pipefail +NS=memex +ACR=meshweaver.azurecr.io +: "${MEMEX_PG_CONN:?set MEMEX_PG_CONN to the Flexible Server connection string}" + +kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - +kubectl apply -f ./aks-extras.yaml # StorageClass + RWX PVCs +helm upgrade --install memex ./helm -f ./helm/values.yaml -f ./values.aks.yaml -f ./values.deploy.yaml -n "$NS" + +# External managed Postgres -> don't run the chart's in-cluster pg. +kubectl -n "$NS" scale statefulset memex-postgres-statefulset --replicas=0 || true +# The chart hardcodes the ghcr image path -> repoint to the shared ACR. +kubectl -n "$NS" set image deployment/memex-portal-deployment memex-portal="$ACR/memex-portal-ai:latest" +kubectl -n "$NS" set image deployment/memex-migration-deployment memex-migration="$ACR/memex-migration:latest" || true +# 1-replica baseline + mount the Azure Files PVCs (/data already mounted by the chart; add /mnt/content). +kubectl -n "$NS" patch deployment memex-portal-deployment --type=json -p '[{"op":"replace","path":"/spec/replicas","value":1},{"op":"replace","path":"/spec/template/spec/volumes/0","value":{"name":"memex-data","persistentVolumeClaim":{"claimName":"memex-data"}}},{"op":"add","path":"/spec/template/spec/volumes/-","value":{"name":"memex-content","persistentVolumeClaim":{"claimName":"memex-content"}}},{"op":"add","path":"/spec/template/spec/containers/0/volumeMounts/-","value":{"name":"memex-content","mountPath":"/mnt/content"}}]' +# Chart-gen gap: the secret template hardcodes the in-cluster pg connection string -> repoint +# both portal + migration at the external Flexible Server (private IP + password + SSL). +for s in memex-portal-secrets memex-migration-secrets; do + kubectl -n "$NS" patch secret "$s" --type merge -p "{\"stringData\":{\"ConnectionStrings__memex\":\"${MEMEX_PG_CONN}\"}}" +done +kubectl -n "$NS" rollout restart deployment/memex-portal-deployment deployment/memex-migration-deployment +echo "=== deployed ==="; kubectl -n "$NS" get deploy,pvc,svc -o wide + +# Observability (opt-in, folded into the standard deploy): set GRAFANA_PW to also bring up +# Grafana + Loki + Promtail + Prometheus (Promtail scrapes every pod's stdout into Loki, so the +# portal logs flow with no portal-side config). Stays private — reach it via the P2S VPN + +# `kubectl port-forward` (see DEPLOY-RUNBOOK.md §6). Non-fatal: a monitoring failure must not +# fail the app deploy. +if [ -n "${GRAFANA_PW:-}" ] && [ -f ./install-observability.sh ]; then + echo "=== observability (GRAFANA_PW set) ===" + GRAFANA_PW="$GRAFANA_PW" bash ./install-observability.sh || echo "WARN: observability install failed (non-fatal)" +fi diff --git a/deploy/aks/scripts/import-dashboards.sh b/deploy/aks/scripts/import-dashboards.sh new file mode 100644 index 000000000..e502681d5 --- /dev/null +++ b/deploy/aks/scripts/import-dashboards.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Import Grafana dashboard JSON files into the in-cluster Grafana (loki-stack chart, +# `monitoring` namespace). Each file under deploy/aks/dashboards/ is already in the +# Grafana `/api/dashboards/db` payload shape ({"dashboard":{...},"overwrite":true,...}), +# so this just POSTs every *.json in the working directory. Idempotent (overwrite:true). +# +# Grafana is private (ClusterIP), so run this INSIDE the cluster via command-invoke, +# uploading the script + the dashboards alongside it: +# +# az aks command invoke -g memex-aks-rg -n memexaks-cluster \ +# --command "bash import-dashboards.sh" \ +# --file deploy/aks/scripts/import-dashboards.sh \ +# --file deploy/aks/dashboards/atioz-overview.json \ +# --file deploy/aks/dashboards/atioz-logs-errors.json +# +# The admin password is read from the chart's `loki-grafana` secret; no creds to pass. +set -euo pipefail + +PW=$(kubectl -n monitoring get secret loki-grafana -o jsonpath='{.data.admin-password}' | base64 -d) +G=${GRAFANA_URL:-http://loki-grafana.monitoring.svc.cluster.local} + +shopt -s nullglob +files=(*.json) +if [ ${#files[@]} -eq 0 ]; then + echo "no dashboard *.json found in $(pwd)"; exit 1 +fi + +for f in "${files[@]}"; do + code=$(curl -s -o /tmp/grafana-resp -w '%{http_code}' -u "admin:$PW" \ + -H 'Content-Type: application/json' \ + -X POST "$G/api/dashboards/db" --data-binary @"$f") + echo "$f -> HTTP $code $(cat /tmp/grafana-resp)" +done diff --git a/deploy/aks/scripts/install-observability.sh b/deploy/aks/scripts/install-observability.sh new file mode 100644 index 000000000..bb05e0849 --- /dev/null +++ b/deploy/aks/scripts/install-observability.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Observability stack for the AKS deployment: Grafana + Loki + Promtail + Prometheus +# (the grafana/loki-stack chart — Promtail ships every pod's stdout into Loki, datasources +# auto-wired in Grafana). Run via az aks command invoke on the private cluster: +# +# export GRAFANA_PW='' +# az aks command invoke -g memex-aks-rg -n memexaks-cluster \ +# --command "GRAFANA_PW=$GRAFANA_PW bash install-observability.sh" --file install-observability.sh +set -uo pipefail +: "${GRAFANA_PW:?set GRAFANA_PW (Grafana admin password)}" +helm repo add grafana https://grafana.github.io/helm-charts >/dev/null 2>&1 || true +helm repo update >/dev/null 2>&1 +helm upgrade --install loki grafana/loki-stack -n monitoring --create-namespace \ + --set grafana.enabled=true --set prometheus.enabled=true \ + --set grafana.adminPassword="$GRAFANA_PW" --set grafana.service.type=ClusterIP \ + --wait --timeout 10m +kubectl -n monitoring get pods +echo +echo "Access (private cluster -> via the P2S VPN):" +echo " az aks get-credentials -g memex-aks-rg -n memexaks-cluster" +echo " kubectl -n monitoring port-forward svc/loki-grafana 3000:80" +echo " open http://localhost:3000 (user: admin / pass: \$GRAFANA_PW)" +echo "Loki datasource is pre-wired; query e.g. {namespace=\"memex\"} in Grafana Explore." diff --git a/deploy/aks/scripts/tls.sh b/deploy/aks/scripts/tls.sh new file mode 100644 index 000000000..6698524fe --- /dev/null +++ b/deploy/aks/scripts/tls.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Install cert-manager + a Let's Encrypt ClusterIssuer and enable TLS on the portal ingress +# (HTTP-01 via the app-routing nginx). The ingress 'memex-portal' must already exist and the +# host must resolve publicly to the ingress IP (Let's Encrypt validates over the internet). +# +# export LE_EMAIL=you@example.com INGRESS_HOST=memex.systemorph.com +# az aks command invoke -g -n --command "LE_EMAIL=$LE_EMAIL INGRESS_HOST=$INGRESS_HOST bash tls.sh" --file tls.sh +set -uo pipefail +NS=memex +: "${LE_EMAIL:?set LE_EMAIL for the Let's Encrypt account}" +HOST="${INGRESS_HOST:-memex.systemorph.com}" + +helm repo add jetstack https://charts.jetstack.io >/dev/null 2>&1 || true +helm repo update >/dev/null 2>&1 +helm upgrade --install cert-manager jetstack/cert-manager -n cert-manager --create-namespace \ + --set crds.enabled=true --wait --timeout 5m + +cat <" + # NOTE: the chart hardcodes the in-cluster pg in ConnectionStrings__memex; deploy.sh patches + # the rendered secret with $MEMEX_PG_CONN afterwards, so the value here is not authoritative. + memex_portal: + memex_postgres_password: "" + # base64 of 32 random bytes — envelope key for stored provider credentials at rest + Ai__KeyProtection__MasterKey: "" + # OAuth client SECRETS (the ClientIds are non-secret -> config below) + Authentication__Microsoft__ClientSecret: "" + Authentication__Google__ClientSecret: "" + Authentication__LinkedIn__ClientSecret: "" +config: + memex_portal: + # 1 replica => Localhost. For >1 replica use AzureTables clustering (follow-up). + Deployment__Orleans__Clustering: "Localhost" + Authentication__Provider: "Custom" + Authentication__EnableDevLogin: "false" + # Microsoft/Entra single-tenant home directory + Authentication__Microsoft__ClientId: "" + Authentication__Microsoft__TenantId: "" + # Leave a provider's ClientId empty to NOT offer it on the login page + Authentication__Google__ClientId: "" + Authentication__LinkedIn__ClientId: "" + # Observability: logs ship to Loki via Promtail (the cluster log agent) with NO portal config — + # run deploy.sh with GRAFANA_PW set to fold Grafana + Loki + Prometheus into the deploy. + # Only set the OTLP endpoint below if you run an OpenTelemetry collector (traces/metrics): + # OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector.monitoring:4317" diff --git a/deploy/aks/values.aks.yaml b/deploy/aks/values.aks.yaml new file mode 100644 index 000000000..470d306c2 --- /dev/null +++ b/deploy/aks/values.aks.yaml @@ -0,0 +1,307 @@ +# =========================================================================== +# values.aks.yaml — AKS overlay for the existing ../helm chart (deploy/helm). +# +# Apply with: +# helm install memex ../helm -f ../helm/values.yaml -f values.aks.yaml \ +# --namespace memex --create-namespace +# +# This file ONLY sets the keys that the existing chart templates consume +# (config.* / secrets.*). The chart's templates currently hardcode the image, +# replica count, volumes (emptyDir) and have no ingress/storageClass plumbing, +# so the genuinely AKS-specific pieces — RWX PVCs, ingress with cookie session +# affinity, HA replicas, and the pgBackRest PITR sidecar/CronJob — are shipped +# as standalone manifests under ./manifests and applied alongside the release. +# See ./README.md "Helm + manifests layering" for the exact order. +# +# The annotated blocks at the bottom (ingress:, persistence:, replicas:, +# pgbackrest:) are the values the manifests read AND a forward-looking contract: +# if/when the ../helm chart is extended to template these, this overlay already +# carries the AKS-correct values. +# =========================================================================== + +# --- Postgres password ------------------------------------------------------ +# Replace with your own secret BEFORE installing, or (recommended) pass via +# `--set-file` / an external secret store (CSI Secrets Store add-on is enabled +# in aks.bicep). Never commit a real password to git. +secrets: + memex_postgres: + memex_postgres_password: "CHANGE_ME_strong_password" + memex_migration: + ConnectionStrings__memex: "" # chart builds this from the password if blank + memex_postgres_password: "CHANGE_ME_strong_password" + MEMEX_URI: "" + memex_portal: + ConnectionStrings__memex: "" + memex_postgres_password: "CHANGE_ME_strong_password" + MEMEX_URI: "" + # ---- OAuth client SECRETS (memex.systemorph.com sign-in) ---------------- + # Client SECRETS belong in the secret, not the configmap. Pass via + # `--set secrets.memex_portal.Authentication__Microsoft__ClientSecret=...` + # or (recommended) Key Vault via the CSI Secrets Store add-on. NEVER commit + # real secrets. The matching non-secret ClientId / TenantId live under + # config.memex_portal below. + Authentication__Microsoft__ClientSecret: "CHANGE_ME_aad_client_secret" + Authentication__Google__ClientSecret: "CHANGE_ME_google_client_secret" + Authentication__LinkedIn__ClientSecret: "CHANGE_ME_linkedin_client_secret" + # ---- AI provider keys --------------------------------------------------- + # Never commit a real key. The chart emits each secret key ONLY when non-empty, so leaving + # these empty means "don't manage / preserve whatever is set out-of-band". Inject the real + # AzureFoundry key (the s-meshweaver key1) at deploy via Key Vault CSI + # (manifests/secretproviderclass.yaml) or `--set secrets.memex_portal.AzureFoundry__ApiKey=...`. + AzureFoundry__ApiKey: "" + # Ai__KeyProtection__MasterKey: intentionally UNSET — it is provided out-of-band, and overriding + # it makes stored enc: provider keys undecryptable. If you DO manage it via the chart, point it + # at the SAME Key Vault secret the deployment already uses (a different value breaks decryption). + Ai__KeyProtection__MasterKey: "" + +config: + memex_postgres: + POSTGRES_HOST_AUTH_METHOD: "scram-sha-256" + POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256 --auth-local=scram-sha-256" + POSTGRES_USER: "postgres" + memex_migration: + MEMEX_HOST: "memex-postgres-service" + MEMEX_PORT: "5432" + MEMEX_USERNAME: "postgres" + MEMEX_JDBCCONNECTIONSTRING: "jdbc:postgresql://memex-postgres-service:5432/memex" + MEMEX_DATABASENAME: "memex" + memex_portal: + MEMEX_HOST: "memex-postgres-service" + MEMEX_PORT: "5432" + MEMEX_USERNAME: "postgres" + MEMEX_JDBCCONNECTIONSTRING: "jdbc:postgresql://memex-postgres-service:5432/memex" + MEMEX_DATABASENAME: "memex" + ASPNETCORE_HTTP_PORTS: "8080" + Deployment__Backend: "Filesystem" + # /data holds ONLY framework caches now: DataProtection keys + # (/data/dataprotection-keys), the NodeType assembly-cache, and the NuGet + # package-cache. The content collection moved off /data onto its own Azure + # Files drive (/mnt/content) — see Storage__BasePath below. + Deployment__DataRoot: "/data" + # ---- HA: multiple portal replicas REQUIRE AdoNet Orleans clustering ----- + # With more than one portal replica the Orleans silos must form a cluster + # via the shared Postgres (AdoNet), NOT "Localhost" (single-process). This + # mirrors deploy/compose-ha. If you keep replicas: 1 you may leave this as + # "Localhost". + Deployment__Orleans__Clustering: "AdoNet" + # ---- Content collection -> its own mountable Azure Files drive ---------- + # The portal reads `Storage:BasePath` as the FileSystem content-collection + # root (MemexConfiguration.ConfigureMemexMesh -> contentStorageConfig). Each + # node hub gets a per-node subdirectory under it: {BasePath}/content/{nodePath}. + # We point it at the dedicated /mnt/content drive (memex-content PVC) so user + # content lives on its own share, separate from the framework caches on /data. + Storage__Name: "content" + Storage__SourceType: "FileSystem" + Storage__BasePath: "/mnt/content" + # ---- Attachments drive (forward-looking) -------------------------------- + # The portal also maps an "attachments" collection + # (MemexConfiguration: MapContentCollection("attachments","storage", + # "attachments/{nodePath}")). In THIS (Distributed / Filesystem) backend the + # "storage" SOURCE collection is not separately registered — only the Monolith + # registers it — so attachments has NO independently env-repointable base path + # today: there is no `Storage__Attachments__BasePath` knob in the app. We mount + # /mnt/attachments anyway (memex-attachments PVC) so the drive exists and is + # ready: if the app later registers a filesystem "storage" source collection + # rooted here (e.g. a Storage2/attachments section), no manifest change is + # needed. The REAL functional change in this overlay is Storage__BasePath above. + # + # If you ALSO run the Monolith image (which DOES register the "storage" source + # from the Storage section), set Storage__BasePath=/mnt/attachments to root the + # attachments mapping there — but then content + attachments share one drive. + # For the HA Distributed portal, keep Storage__BasePath=/mnt/content. + Graph__Storage__Type: "PostgreSql" + Graph__Storage__BasePath: "/data/graph" + Mcp__BaseUrl: "http://memex-portal-service:8080" + # ---- AI model providers (templated by the chart) ------------------------ + # These seed the system model catalog on deploy (BuiltInLanguageModelProvider + # scans {Section}:Models/:Endpoint → nodeType:LanguageModel nodes the picker shows). + # Only the AzureFoundry (open-weight /models) provider is configured via Helm. + # Section name "AzureFoundry" is what the code binds (NOT "AzureAIS" — dead section). + # Anthropic/Claude is intentionally NOT wired here — add it node-based later (a + # ModelProvider node with the /anthropic endpoint). Key is in secrets.memex_portal + # below (source from Key Vault via the CSI add-on). Full guide: Doc/AI/ModelProviderSetup. + AzureFoundry__Endpoint: "https://s-meshweaver.services.ai.azure.com/models" + AzureFoundry__Models__0: "DeepSeek-V4-Pro" + AzureFoundry__Models__1: "DeepSeek-V3-0324" + AzureFoundry__Models__2: "DeepSeek-V4-Flash" + # Tier → model (h/m/l + utility). Open-weight DeepSeek ladder, all deployed on + # s-meshweaver. Claude works very well but costs more — it's NOT wired via Helm + # here; add it node-based later and pin per-agent via PreferredModel when worth it. + ModelTier__Heavy: "DeepSeek-V4-Pro" + ModelTier__Standard: "DeepSeek-V3-0324" + ModelTier__Light: "DeepSeek-V4-Flash" + ModelTier__Utility: "DeepSeek-V4-Flash" + # Per-user Claude Code config root → the RWX /mnt/users share (mounted by the portal deployment). + # Lets each user connect Claude Code under their own dir; pairs with the PTY login fix. + ClaudeCode__ConfigDirRoot: "/mnt/users" + # ---- OpenTelemetry: export to the in-cluster OTel Collector ------------- + # ServiceDefaults.AddOpenTelemetryExporters() calls UseOtlpExporter() when + # OTEL_EXPORTER_OTLP_ENDPOINT is set — UNLESS APPLICATIONINSIGHTS_CONNECTION_STRING + # is also set (Azure Monitor wins). So to ship traces/logs/metrics to the + # collector instead of App Insights, set this AND leave the App Insights + # connection string unset. The collector Service (manifests/observability/ + # otel-collector.yaml) listens on 4317 (gRPC) / 4318 (HTTP). UseOtlpExporter + # defaults to gRPC, so the :4317 endpoint is correct; OTEL_EXPORTER_OTLP_PROTOCOL + # is set explicitly for clarity. + OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317" + OTEL_EXPORTER_OTLP_PROTOCOL: "grpc" + # ---- Sign-in: Systemorph AAD as HOME tenant + Google + LinkedIn --------- + # The portal's auth pipeline (AuthenticationBuilderExtensions) reads these + # Authentication:* keys. Any provider with a ClientId set is offered on the + # login page; "Microsoft" with a real TenantId pins the HOME (primary) AAD. + # + # Provider mode: setting any external-provider ClientId flips the portal into + # multi-provider "Custom" mode automatically (HasExternalProviders == true) — + # dev login is already disabled in the Distributed image (Program.cs). We set + # it explicitly for clarity. Public host = memex.systemorph.com. + Authentication__Provider: "Custom" + Authentication__EnableDevLogin: "false" + # -- Microsoft / Entra ID (HOME tenant = Systemorph) ---------------------- + # TenantId MUST be the Systemorph AAD tenant GUID (NOT "common"/"organizations") + # so this AAD is the home directory. ClientId = the app registration's + # Application (client) ID. Add redirect URI https://memex.systemorph.com/signin-microsoft + # to that app registration. ClientSecret is in secrets.memex_portal above. + # Systemorph AAD home tenant (the subscription's own tenant). + Authentication__Microsoft__TenantId: "3a01d7ac-3330-444d-942d-975eb491b5d6" + Authentication__Microsoft__ClientId: "CHANGE_ME_aad_application_client_id" + # -- Google (other accounts) ---------------------------------------------- + # Redirect URI: https://memex.systemorph.com/signin-google + Authentication__Google__ClientId: "CHANGE_ME_google_oauth_client_id" + # -- LinkedIn (other accounts) -------------------------------------------- + # Redirect URI: https://memex.systemorph.com/signin-linkedin + Authentication__LinkedIn__ClientId: "CHANGE_ME_linkedin_oauth_client_id" + +# =========================================================================== +# AKS-specific values consumed by ./manifests (and a chart-extension contract). +# =========================================================================== + +# --- Container images ------------------------------------------------------- +# Default = pull straight from GHCR (no ACR needed). To use the private ACR, +# `az acr import` the images (see README) and set registry to the ACR login +# server, e.g. memexaksacrxxxx.azurecr.io. The chart currently hardcodes the +# ghcr.io path; ./manifests/portal-ingress.yaml and the README's +# `helm upgrade --set` examples show how to repoint it. +image: + registry: "ghcr.io/systemorph" # or ".azurecr.io" + # Sizable image: memex-portal-ai is the FULL AI-enabled portal (co-hosted + # Claude Code / Copilot CLIs, all providers) — the heavyweight variant, as + # opposed to the lean `memex-portal`. Paired with the larger resources block + # below + portal-ha-patch.yaml so each replica gets a few CPUs and a bit of RAM. + portal: "memex-portal-ai" + migration: "memex-migration" + tag: "latest" + pullPolicy: "IfNotPresent" + +# --- Portal pod resources (GENEROUS — sizable AI image on 32 GiB nodes) ------ +# Applied to each replica. The actual requests/limits land on the container via +# manifests/portal-ha-patch.yaml (kept in sync with these). The AI image is +# memory-hungry (NodeType compile cache, agent runtimes), so on the Standard_D8s_v5 +# nodes (8 vCPU / 32 GiB) we give it generous headroom — 2 replicas + system pods +# still fit comfortably per node. +resources: + portal: + requests: + cpu: "4" # a few CPUs + memory: "8Gi" # a bit of RAM + limits: + cpu: "6" # generous CPU ceiling + memory: "16Gi" # generous memory ceiling (half a 32 GiB node) + +# --- HA / replicas ---------------------------------------------------------- +# Launch 2 portal replicas by default (still needs cookie session affinity + +# AdoNet Orleans clustering; both are configured). Bump to 3 for full 3-zone +# spread. portal-ha-patch.yaml sets the same count — keep the two in sync (or +# `kubectl scale deployment memex-portal-deployment --replicas=2`). +replicas: + portal: 2 # Blazor Server: 2 replicas (zone-spread is ScheduleAnyway) + postgres: 1 # self-managed pg StatefulSet stays single-writer + +# --- Persistence (Azure Files CSI for ReadWriteMany) ------------------------ +# Blazor Server portal replicas share every /data + /mnt/* path, so those PVCs +# MUST be ReadWriteMany. They use the custom `azurefile-memex` StorageClass +# (manifests/storageclass-azurefile.yaml) which pins uid/gid=1654 so the non-root +# portal user can write. The Postgres data volume is single-writer and uses +# managed-csi (Premium SSD) for IOPS. The chart ships these as emptyDir today; +# ./manifests/portal-pvcs.yaml creates the real PVCs. +# +# One Azure Files "drive" per concern, mounted at an explicit path: +# data -> /data framework caches only (DP keys, asm + nuget cache) +# content -> /mnt/content the content collection (Storage__BasePath) +# attachments -> /mnt/attachments dedicated attachments drive (forward-looking) +# users -> /mnt/users co-hosted CLI configs +persistence: + data: + storageClass: "azurefile-memex" # RWX, uid/gid 1654 + accessMode: "ReadWriteMany" + mountPath: "/data" + size: "16Gi" # shrunk: content moved off /data + content: + storageClass: "azurefile-memex" # RWX, uid/gid 1654 + accessMode: "ReadWriteMany" + mountPath: "/mnt/content" + size: "128Gi" + attachments: + storageClass: "azurefile-memex" # RWX, uid/gid 1654 + accessMode: "ReadWriteMany" + mountPath: "/mnt/attachments" + size: "64Gi" + users: + storageClass: "azurefile-memex" # RWX, uid/gid 1654 + accessMode: "ReadWriteMany" + mountPath: "/mnt/users" + size: "32Gi" + postgres: + storageClass: "managed-csi" # RWO, Premium SSD + accessMode: "ReadWriteOnce" + size: "128Gi" + +# --- Ingress with cookie session affinity (Blazor Server is sticky) --------- +# Blazor Server holds a per-circuit SignalR connection; without sticky sessions +# a reconnect can land on a different replica and drop the circuit. Pick ONE +# ingress controller (see README for AGIC vs ingress-nginx trade-offs); both +# annotation sets are provided in ./manifests/portal-ingress.yaml. +ingress: + enabled: true + className: "webapprouting.kubernetes.azure.com" # AKS app routing (managed nginx) + host: "memex.systemorph.com" + tls: + enabled: true + secretName: "memex-tls" + sessionAffinity: + enabled: true + cookieName: "MEMEX_AFFINITY" + # nginx ("webapprouting"/"ingress-nginx") cookie-affinity annotations: + nginxAnnotations: + nginx.ingress.kubernetes.io/affinity: "cookie" + nginx.ingress.kubernetes.io/affinity-mode: "persistent" + nginx.ingress.kubernetes.io/session-cookie-name: "MEMEX_AFFINITY" + nginx.ingress.kubernetes.io/session-cookie-max-age: "172800" + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" # long-lived SignalR + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" + # AGIC (Application Gateway Ingress Controller) cookie-affinity annotations: + agicAnnotations: + appgw.ingress.kubernetes.io/cookie-based-affinity: "true" + appgw.ingress.kubernetes.io/request-timeout: "3600" + +# --- pgBackRest PITR sidecar (self-managed Postgres backup to Azure Blob) --- +# Toggle for the ./manifests/pgbackrest resources. Disable if you migrate to +# Azure Database for PostgreSQL Flexible Server (managed PITR — recommended for +# turnkey prod; see README). +pgbackrest: + enabled: true + # Values from the storage.bicep outputs: + azure: + account: "memexaksbkpXXXX" # backupStorageAccount output + container: "pgbackrest" # backupContainerName output + # Workload Identity client id (backupIdentityClientId output). Keyless. + workloadIdentityClientId: "00000000-0000-0000-0000-000000000000" + # Alternatively set a storage account key here and switch repo auth in the + # pgbackrest config (see manifests/pgbackrest/configmap.yaml). + accountKey: "" + serviceAccount: "pgbackrest-sa" + retention: + full: 4 # keep 4 full backups + diff: 6 # keep 6 differentials + schedule: + full: "0 2 * * 0" # weekly full, Sun 02:00 UTC + diff: "0 2 * * 1-6" # daily diff, Mon-Sat 02:00 UTC diff --git a/deploy/aspire/Memex.Deploy.AppHost/Memex.Deploy.AppHost.csproj b/deploy/aspire/Memex.Deploy.AppHost/Memex.Deploy.AppHost.csproj new file mode 100644 index 000000000..57dbd48f9 --- /dev/null +++ b/deploy/aspire/Memex.Deploy.AppHost/Memex.Deploy.AppHost.csproj @@ -0,0 +1,26 @@ + + + + Exe + enable + enable + true + memex-deploy-apphost + $(NoWarn);ASPIRECOSMOSDB001;ASPIREACADOMAINS001 + + + + + + + + + + + + + + diff --git a/deploy/aspire/Memex.Deploy.AppHost/Program.cs b/deploy/aspire/Memex.Deploy.AppHost/Program.cs new file mode 100644 index 000000000..d3be76629 --- /dev/null +++ b/deploy/aspire/Memex.Deploy.AppHost/Program.cs @@ -0,0 +1,122 @@ +// Dedicated, image-based deployment AppHost for the MeshWeaver Memex portal. +// +// Mirrors the conventions of the main memex/aspire/Memex.AppHost, but deploys the PUBLISHED +// GHCR images via the Aspire.Hosting.Memex integration (builder.AddMemex → AddContainer) rather +// than building the portal from source. One model → many artifacts via the Aspire publishers: +// +// aspire publish --apphost deploy/aspire/Memex.Deploy.AppHost/Memex.Deploy.AppHost.csproj \ +// -o deploy/compose -- --mode compose # Docker Compose (single) +// aspire publish ... -o deploy/compose-ha -- --mode compose-ha # Docker Compose (HA) +// aspire publish ... -o deploy/helm -- --mode kubernetes # Kubernetes / Helm +// aspire publish ... -o deploy/aca -- --mode azure # Azure Container Apps (bicep) +// +// Tunables (dotnet user-secrets / env / GitHub secrets), all optional: +// Parameters:image-registry (default ghcr.io/systemorph) +// Parameters:image-tag (default latest) +// Parameters:include-ai-clis (default true → portal-ai image) +// Parameters:key-protection-master-key (REQUIRED for production) + +var builder = DistributedApplication.CreateBuilder(args); + +var mode = builder.Configuration["mode"]?.ToLowerInvariant() ?? "compose"; +var ha = mode.EndsWith("-ha", StringComparison.Ordinal); + +if (mode.StartsWith("kubernetes", StringComparison.Ordinal)) +{ + builder.AddKubernetesEnvironment("k8s") + .WithHelm(helm => helm + .WithChartName("memex") + .WithChartDescription("MeshWeaver Memex portal — Azure-free Kubernetes self-host.")); +} +else if (mode == "azure") +{ + builder.AddAzureContainerAppEnvironment("memex-aca"); +} +else +{ + builder.AddDockerComposeEnvironment("self-host"); +} + +var portal = builder.AddMemex("memex", o => o + .WithImage( + builder.Configuration["Parameters:image-registry"], + builder.Configuration["Parameters:image-tag"]) + .WithAiClis(!string.Equals(builder.Configuration["Parameters:include-ai-clis"], "false", + StringComparison.OrdinalIgnoreCase)) + .WithBackend("Filesystem") + // Real, Postgres-backed cluster membership in every deployment (never Localhost in prod). + // Works for a single silo or an HA replica set; the `ha` flag only drives replica count. + .WithOrleansClustering("AdoNet") + .WithMasterKey(builder.Configuration["Parameters:key-protection-master-key"]) + + // Embeddings (vector search). With the endpoint + key set, the one-shot migration + // vector-indexes the built-in documentation and the portal embeds search-bar queries. + // Leave unset to ship docs as full-text-searchable only (no external AI dependency). + .WithEmbeddings( + builder.Configuration["Parameters:embedding-endpoint"], + builder.Configuration["Parameters:embedding-key"], + builder.Configuration["Parameters:embedding-model"]) + + // External sign-in (OAuth) providers — deploy parameters. Provide via + // `dotnet user-secrets` / env / GitHub secrets locally, or the Marketplace + // createUiDefinition wizard for an Azure Application install. Each provider is + // offered only when its ClientId is set. Register the redirect URI on each app: + // {BaseUrl}/signin-{microsoft|google|linkedin}. + .WithMicrosoftSignIn( + builder.Configuration["Parameters:microsoft-client-id"], + builder.Configuration["Parameters:microsoft-client-secret"], + builder.Configuration["Parameters:microsoft-tenant-id"]) + .WithGoogleSignIn( + builder.Configuration["Parameters:google-client-id"], + builder.Configuration["Parameters:google-client-secret"]) + .WithLinkedIn( + builder.Configuration["Parameters:linkedin-client-id"], + builder.Configuration["Parameters:linkedin-client-secret"]) + + // Outbound email (Microsoft Graph /sendMail) — invitations + script-triggered notifications. + // On AKS the client secret comes from Key Vault (email-clientsecret → Email__ClientSecret via + // the SecretProviderClass), so it is NOT passed here; the rest are non-secret parameters. + .WithOutboundEmail( + enabled: ParseBool(builder.Configuration["Parameters:email-enabled"]), + mailboxAddress: builder.Configuration["Parameters:email-mailbox-address"], + tenantId: builder.Configuration["Parameters:email-tenant-id"], + clientId: builder.Configuration["Parameters:email-client-id"], + clientSecret: builder.Configuration["Parameters:email-client-secret"], + useManagedIdentity: ParseBool(builder.Configuration["Parameters:email-use-managed-identity"])) + + // Inbound email→agent channel (Graph subscription + webhook). Needs Mail.ReadWrite + a public URL. + .WithInboundEmail( + enabled: ParseBool(builder.Configuration["Parameters:email-inbound-enabled"]), + webhookBaseUrl: builder.Configuration["Parameters:email-webhook-base-url"], + clientState: builder.Configuration["Parameters:email-subscription-client-state"]) + + // Invitation-only onboarding (Features:Onboarding:InvitationOnly). + .WithInvitationOnly(ParseBool(builder.Configuration["Parameters:invitation-only"])) + + // Microsoft Teams bot (bidirectional). Needs an Azure Bot resource + a Teams app. On AKS the bot + // secret comes from Key Vault (teams-apppassword → Teams__AppPassword); the rest are non-secret. + .WithTeams( + enabled: ParseBool(builder.Configuration["Parameters:teams-enabled"]), + appId: builder.Configuration["Parameters:teams-app-id"], + appPassword: builder.Configuration["Parameters:teams-app-password"], + tenantId: builder.Configuration["Parameters:teams-tenant-id"])); + +// Self-host filesystem backend: the portal writes DataProtection keys, the NodeType +// assembly cache, and the NuGet cache under /data. The aspnet base image runs as the +// non-root `app` user, but a freshly-created Docker named volume is root-owned, so the +// app cannot create those directories — startup dies with +// `UnauthorizedAccessException: Access to the path '/data/dataprotection-keys' is denied`. +// Run the portal as root in the Compose targets so it owns its mounted data volume. +// (Kubernetes/AKS handles this via the platform overlay — Azure Files CSI mounts 0777 / +// uid-mapped — and ACA runs containers as root by default, so this is Compose-only.) +if (!mode.StartsWith("kubernetes", StringComparison.Ordinal) && mode != "azure") +{ + portal.PublishAsDockerComposeService((_, service) => service.User = "root"); +} + +builder.Build().Run(); + +// Parses an optional bool deploy parameter: null when unset (leave the portal default), +// otherwise true/false. Hoisted local function — usable from the AddMemex lambda above. +static bool? ParseBool(string? value) => + string.IsNullOrEmpty(value) ? null : string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); diff --git a/deploy/aspire/Memex.Deploy.AppHost/Properties/launchSettings.json b/deploy/aspire/Memex.Deploy.AppHost/Properties/launchSettings.json new file mode 100644 index 000000000..3f5111f0a --- /dev/null +++ b/deploy/aspire/Memex.Deploy.AppHost/Properties/launchSettings.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "Memex.Deploy.AppHost": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": false, + "environmentVariables": { + "DOTNET_ENVIRONMENT": "Development" + } + } + } +} diff --git a/deploy/base-images/portal-ai/Dockerfile b/deploy/base-images/portal-ai/Dockerfile new file mode 100644 index 000000000..62e959916 --- /dev/null +++ b/deploy/base-images/portal-ai/Dockerfile @@ -0,0 +1,39 @@ +# Base image for the portal-ai flavour: the ASP.NET runtime PLUS the two co-hosted CLIs +# (Node 20 + Claude Code + GitHub Copilot). This is the ONE hand-authored image artifact — +# Aspire's SDK container build can't apt-install Node, so it layers the published portal app +# ON TOP of this base via the portal's . The lean `portal` flavour skips +# this and uses the default mcr.microsoft.com/dotnet/aspnet base. +# +# Build + push once per release: +# docker build -t ghcr.io/systemorph/memex-portal-ai-base:$TAG deploy/base-images/portal-ai +# docker push ghcr.io/systemorph/memex-portal-ai-base:$TAG +# +# Both CLIs are bundled deliberately (locked decision) rather than relying on the publish-time +# copilot fetch, which keys off the build-host RID. npm packages verified: @github/copilot +# (the GitHub Copilot CLI) + @anthropic-ai/claude-code (Claude Code has no standalone binary). +ARG DOTNET_VERSION=10.0 +FROM mcr.microsoft.com/dotnet/aspnet:${DOTNET_VERSION} + +# git is required by BOTH the co-hosted CLIs (Claude Code / Copilot operate on a working tree) +# AND the in-portal working-tree feature (GitWorkingTreeService shells `git` for clone/commit/push). +# One system git, shared by all three — so a tree the harness edits and a tree the editor edits are +# byte-identical. +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl ca-certificates gnupg git \ + && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && npm install -g @anthropic-ai/claude-code @github/copilot \ + && npm cache clean --force \ + && apt-get purge -y gnupg \ + && apt-get autoremove -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Per-user CLI config / shared state mount point (a local volume single-node; an NFS / Azure +# Files share in HA so every replica sees the same per-user .claude / copilot dirs). +ENV ClaudeCode__ConfigDirRoot=/mnt/users + +# On-disk git working-tree root (the RWX workspace PVC mount). Binds to GitWorkingTreeOptions.Root +# via GitWorkspace:Root; per-user trees live at /workspace/{userId}/{repoSlug}. The deployment must +# mount the memex-workspace PVC here for checkout/commit to work. +ENV GitWorkspace__Root=/workspace diff --git a/deploy/compose-ha/.env b/deploy/compose-ha/.env new file mode 100644 index 000000000..d58abe4f9 --- /dev/null +++ b/deploy/compose-ha/.env @@ -0,0 +1,3 @@ +# Parameter memex-postgres-password +MEMEX_POSTGRES_PASSWORD= + diff --git a/deploy/compose-ha/docker-compose.yaml b/deploy/compose-ha/docker-compose.yaml new file mode 100644 index 000000000..0b99e6de1 --- /dev/null +++ b/deploy/compose-ha/docker-compose.yaml @@ -0,0 +1,93 @@ +services: + self-host-dashboard: + image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" + ports: + - "18888" + expose: + - "18889" + - "18890" + networks: + - "aspire" + restart: "always" + memex-postgres: + image: "docker.io/pgvector/pgvector:pg17" + environment: + POSTGRES_HOST_AUTH_METHOD: "scram-sha-256" + POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256 --auth-local=scram-sha-256" + POSTGRES_USER: "postgres" + POSTGRES_PASSWORD: "${MEMEX_POSTGRES_PASSWORD}" + expose: + - "5432" + volumes: + - type: "volume" + target: "/var/lib/postgresql/data" + source: "memex-pgdata" + read_only: false + networks: + - "aspire" + memex-migration: + image: "ghcr.io/systemorph/memex-migration:latest" + environment: + ConnectionStrings__memex: "Host=memex-postgres;Port=5432;Username=postgres;Password=${MEMEX_POSTGRES_PASSWORD};Database=memex" + MEMEX_HOST: "memex-postgres" + MEMEX_PORT: "5432" + MEMEX_USERNAME: "postgres" + MEMEX_PASSWORD: "${MEMEX_POSTGRES_PASSWORD}" + MEMEX_URI: "postgresql://postgres:${MEMEX_POSTGRES_PASSWORD}@memex-postgres:5432/memex" + MEMEX_JDBCCONNECTIONSTRING: "jdbc:postgresql://memex-postgres:5432/memex" + MEMEX_DATABASENAME: "memex" + depends_on: + memex-postgres: + condition: "service_started" + networks: + - "aspire" + memex-portal: + image: "ghcr.io/systemorph/memex-portal-ai:latest" + environment: + ConnectionStrings__memex: "Host=memex-postgres;Port=5432;Username=postgres;Password=${MEMEX_POSTGRES_PASSWORD};Database=memex" + MEMEX_HOST: "memex-postgres" + MEMEX_PORT: "5432" + MEMEX_USERNAME: "postgres" + MEMEX_PASSWORD: "${MEMEX_POSTGRES_PASSWORD}" + MEMEX_URI: "postgresql://postgres:${MEMEX_POSTGRES_PASSWORD}@memex-postgres:5432/memex" + MEMEX_JDBCCONNECTIONSTRING: "jdbc:postgresql://memex-postgres:5432/memex" + MEMEX_DATABASENAME: "memex" + ASPNETCORE_HTTP_PORTS: "8080" + Deployment__Backend: "Filesystem" + Deployment__DataRoot: "/data" + Deployment__Orleans__Clustering: "AdoNet" + Storage__Name: "content" + Storage__SourceType: "FileSystem" + Storage__BasePath: "/data/content" + Graph__Storage__Type: "PostgreSql" + Graph__Storage__BasePath: "/data/graph" + Mcp__BaseUrl: "http://memex-portal:8080" + ports: + - "8080" + volumes: + - type: "volume" + target: "/data" + source: "memex-data" + read_only: false + - type: "volume" + target: "/mnt/users" + source: "memex-users" + read_only: false + depends_on: + memex-postgres: + condition: "service_started" + memex-migration: + condition: "service_completed_successfully" + user: "root" + networks: + - "aspire" +networks: + aspire: + driver: "bridge" +volumes: + memex-pgdata: + driver: "local" + memex-data: + driver: "local" + memex-users: + driver: "local" diff --git a/deploy/compose/.env b/deploy/compose/.env new file mode 100644 index 000000000..d58abe4f9 --- /dev/null +++ b/deploy/compose/.env @@ -0,0 +1,3 @@ +# Parameter memex-postgres-password +MEMEX_POSTGRES_PASSWORD= + diff --git a/deploy/compose/docker-compose.yaml b/deploy/compose/docker-compose.yaml new file mode 100644 index 000000000..9c55eebe2 --- /dev/null +++ b/deploy/compose/docker-compose.yaml @@ -0,0 +1,93 @@ +services: + self-host-dashboard: + image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" + ports: + - "18888" + expose: + - "18889" + - "18890" + networks: + - "aspire" + restart: "always" + memex-postgres: + image: "docker.io/pgvector/pgvector:pg17" + environment: + POSTGRES_HOST_AUTH_METHOD: "scram-sha-256" + POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256 --auth-local=scram-sha-256" + POSTGRES_USER: "postgres" + POSTGRES_PASSWORD: "${MEMEX_POSTGRES_PASSWORD}" + expose: + - "5432" + volumes: + - type: "volume" + target: "/var/lib/postgresql/data" + source: "memex-pgdata" + read_only: false + networks: + - "aspire" + memex-migration: + image: "ghcr.io/systemorph/memex-migration:latest" + environment: + ConnectionStrings__memex: "Host=memex-postgres;Port=5432;Username=postgres;Password=${MEMEX_POSTGRES_PASSWORD};Database=memex" + MEMEX_HOST: "memex-postgres" + MEMEX_PORT: "5432" + MEMEX_USERNAME: "postgres" + MEMEX_PASSWORD: "${MEMEX_POSTGRES_PASSWORD}" + MEMEX_URI: "postgresql://postgres:${MEMEX_POSTGRES_PASSWORD}@memex-postgres:5432/memex" + MEMEX_JDBCCONNECTIONSTRING: "jdbc:postgresql://memex-postgres:5432/memex" + MEMEX_DATABASENAME: "memex" + depends_on: + memex-postgres: + condition: "service_started" + networks: + - "aspire" + memex-portal: + image: "ghcr.io/systemorph/memex-portal-ai:latest" + environment: + ConnectionStrings__memex: "Host=memex-postgres;Port=5432;Username=postgres;Password=${MEMEX_POSTGRES_PASSWORD};Database=memex" + MEMEX_HOST: "memex-postgres" + MEMEX_PORT: "5432" + MEMEX_USERNAME: "postgres" + MEMEX_PASSWORD: "${MEMEX_POSTGRES_PASSWORD}" + MEMEX_URI: "postgresql://postgres:${MEMEX_POSTGRES_PASSWORD}@memex-postgres:5432/memex" + MEMEX_JDBCCONNECTIONSTRING: "jdbc:postgresql://memex-postgres:5432/memex" + MEMEX_DATABASENAME: "memex" + ASPNETCORE_HTTP_PORTS: "8080" + Deployment__Backend: "Filesystem" + Deployment__DataRoot: "/data" + Deployment__Orleans__Clustering: "Localhost" + Storage__Name: "content" + Storage__SourceType: "FileSystem" + Storage__BasePath: "/data/content" + Graph__Storage__Type: "PostgreSql" + Graph__Storage__BasePath: "/data/graph" + Mcp__BaseUrl: "http://memex-portal:8080" + ports: + - "8080" + volumes: + - type: "volume" + target: "/data" + source: "memex-data" + read_only: false + - type: "volume" + target: "/mnt/users" + source: "memex-users" + read_only: false + depends_on: + memex-postgres: + condition: "service_started" + memex-migration: + condition: "service_completed_successfully" + user: "root" + networks: + - "aspire" +networks: + aspire: + driver: "bridge" +volumes: + memex-pgdata: + driver: "local" + memex-data: + driver: "local" + memex-users: + driver: "local" diff --git a/deploy/helm/Chart.yaml b/deploy/helm/Chart.yaml new file mode 100644 index 000000000..57c9a2ffe --- /dev/null +++ b/deploy/helm/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: "v2" +name: "memex" +version: "0.1.0" +kubeVersion: ">= 1.18.0-0" +description: "MeshWeaver Memex portal — Azure-free Kubernetes self-host." +type: "application" +keywords: + - "aspire" + - "kubernetes" +appVersion: "0.1.0" +deprecated: false diff --git a/deploy/helm/templates/k8s-dashboard/deployment.yaml b/deploy/helm/templates/k8s-dashboard/deployment.yaml new file mode 100644 index 000000000..5a6df6451 --- /dev/null +++ b/deploy/helm/templates/k8s-dashboard/deployment.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: "k8s-dashboard-deployment" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "k8s-dashboard" + app.kubernetes.io/instance: "{{ .Release.Name }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "k8s-dashboard" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "mcr.microsoft.com/dotnet/nightly/aspire-dashboard:latest" + name: "k8s-dashboard" + ports: + - name: "http" + protocol: "TCP" + containerPort: 18888 + - name: "otlp-grpc" + protocol: "TCP" + containerPort: 18889 + - name: "otlp-http" + protocol: "TCP" + containerPort: 18890 + imagePullPolicy: "IfNotPresent" + selector: + matchLabels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "k8s-dashboard" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 + revisionHistoryLimit: 3 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: "RollingUpdate" diff --git a/deploy/helm/templates/k8s-dashboard/service.yaml b/deploy/helm/templates/k8s-dashboard/service.yaml new file mode 100644 index 000000000..d0a283768 --- /dev/null +++ b/deploy/helm/templates/k8s-dashboard/service.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: "v1" +kind: "Service" +metadata: + name: "k8s-dashboard-service" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "k8s-dashboard" + app.kubernetes.io/instance: "{{ .Release.Name }}" +spec: + type: "ClusterIP" + selector: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "k8s-dashboard" + app.kubernetes.io/instance: "{{ .Release.Name }}" + ports: + - name: "http" + protocol: "TCP" + port: 18888 + targetPort: 18888 + - name: "otlp-grpc" + protocol: "TCP" + port: 18889 + targetPort: 18889 + - name: "otlp-http" + protocol: "TCP" + port: 18890 + targetPort: 18890 diff --git a/deploy/helm/templates/memex-migration/config.yaml b/deploy/helm/templates/memex-migration/config.yaml new file mode 100644 index 000000000..35b72321c --- /dev/null +++ b/deploy/helm/templates/memex-migration/config.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: "v1" +kind: "ConfigMap" +metadata: + name: "memex-migration-config" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-migration" + app.kubernetes.io/instance: "{{ .Release.Name }}" +data: + MEMEX_HOST: "{{ .Values.config.memex_migration.MEMEX_HOST }}" + MEMEX_PORT: "{{ .Values.config.memex_migration.MEMEX_PORT }}" + MEMEX_USERNAME: "{{ .Values.config.memex_migration.MEMEX_USERNAME }}" + MEMEX_JDBCCONNECTIONSTRING: "{{ .Values.config.memex_migration.MEMEX_JDBCCONNECTIONSTRING }}" + MEMEX_DATABASENAME: "{{ .Values.config.memex_migration.MEMEX_DATABASENAME }}" diff --git a/deploy/helm/templates/memex-migration/deployment.yaml b/deploy/helm/templates/memex-migration/deployment.yaml new file mode 100644 index 000000000..7af1152a6 --- /dev/null +++ b/deploy/helm/templates/memex-migration/deployment.yaml @@ -0,0 +1,38 @@ +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: "memex-migration-deployment" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-migration" + app.kubernetes.io/instance: "{{ .Release.Name }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-migration" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "ghcr.io/systemorph/memex-migration:latest" + name: "memex-migration" + envFrom: + - configMapRef: + name: "memex-migration-config" + - secretRef: + name: "memex-migration-secrets" + imagePullPolicy: "IfNotPresent" + selector: + matchLabels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-migration" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 + revisionHistoryLimit: 3 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: "RollingUpdate" diff --git a/deploy/helm/templates/memex-migration/secrets.yaml b/deploy/helm/templates/memex-migration/secrets.yaml new file mode 100644 index 000000000..67d34fd0d --- /dev/null +++ b/deploy/helm/templates/memex-migration/secrets.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: "v1" +kind: "Secret" +metadata: + name: "memex-migration-secrets" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-migration" + app.kubernetes.io/instance: "{{ .Release.Name }}" +stringData: + ConnectionStrings__memex: "Host=memex-postgres-service;Port=5432;Username=postgres;Password={{ .Values.secrets.memex_migration.memex_postgres_password }};Database=memex" + MEMEX_PASSWORD: "{{ .Values.secrets.memex_migration.memex_postgres_password }}" + MEMEX_URI: "postgresql://postgres:{{ .Values.secrets.memex_migration.memex_postgres_password }}@memex-postgres-service:5432/memex" +type: "Opaque" diff --git a/deploy/helm/templates/memex-portal/config.yaml b/deploy/helm/templates/memex-portal/config.yaml new file mode 100644 index 000000000..2f87dc3c9 --- /dev/null +++ b/deploy/helm/templates/memex-portal/config.yaml @@ -0,0 +1,116 @@ +--- +apiVersion: "v1" +kind: "ConfigMap" +metadata: + name: "memex-portal-config" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-portal" + app.kubernetes.io/instance: "{{ .Release.Name }}" +data: + MEMEX_HOST: "{{ .Values.config.memex_portal.MEMEX_HOST }}" + MEMEX_PORT: "{{ .Values.config.memex_portal.MEMEX_PORT }}" + MEMEX_USERNAME: "{{ .Values.config.memex_portal.MEMEX_USERNAME }}" + MEMEX_JDBCCONNECTIONSTRING: "{{ .Values.config.memex_portal.MEMEX_JDBCCONNECTIONSTRING }}" + MEMEX_DATABASENAME: "{{ .Values.config.memex_portal.MEMEX_DATABASENAME }}" + ASPNETCORE_HTTP_PORTS: "{{ .Values.config.memex_portal.ASPNETCORE_HTTP_PORTS }}" + Deployment__Backend: "{{ .Values.config.memex_portal.Deployment__Backend }}" + Deployment__DataRoot: "{{ .Values.config.memex_portal.Deployment__DataRoot }}" + Deployment__Orleans__Clustering: "{{ .Values.config.memex_portal.Deployment__Orleans__Clustering }}" + Storage__Name: "{{ .Values.config.memex_portal.Storage__Name }}" + Storage__SourceType: "{{ .Values.config.memex_portal.Storage__SourceType }}" + Storage__BasePath: "{{ .Values.config.memex_portal.Storage__BasePath }}" + Graph__Storage__Type: "{{ .Values.config.memex_portal.Graph__Storage__Type }}" + Graph__Storage__BasePath: "{{ .Values.config.memex_portal.Graph__Storage__BasePath }}" + Mcp__BaseUrl: "{{ .Values.config.memex_portal.Mcp__BaseUrl }}" + # ---- Embeddings (vector search + content indexing) — optional ------------- + # With Endpoint + ApiKey (secret, in secrets.yaml) set, the portal registers the embedding + # provider: search-bar queries hit the HNSW vector index AND the content-indexing pipeline + # activates (chunk→embed→store in the SEPARATE `contentindex` database on the same Postgres + # server). Empty = full-text search only, content indexing inert. See PostgreSqlExtensions. + Embedding__Endpoint: "{{ .Values.config.memex_portal.Embedding__Endpoint | default "" }}" + Embedding__Model: "{{ .Values.config.memex_portal.Embedding__Model | default "" }}" + # ---- AI model providers (non-secret: endpoints + model catalog) ---------- + # BuiltInLanguageModelProvider scans {Section}:Models / :Endpoint and emits the + # nodeType:LanguageModel mesh nodes the chat picker shows. Section names are the + # ones the catalog sources actually bind: "Anthropic" (claude) and "AzureFoundry" + # (open-weight multi-model /models endpoint). NOTE: NOT "AzureAIS" — nothing binds + # that section (the AppHost's AzureAIS__* env is dead config). Keys come from + # secrets.yaml. Leave a Models slot blank to drop that model. + Anthropic__Endpoint: "{{ .Values.config.memex_portal.Anthropic__Endpoint }}" + Anthropic__Models__0: "{{ .Values.config.memex_portal.Anthropic__Models__0 }}" + Anthropic__Models__1: "{{ .Values.config.memex_portal.Anthropic__Models__1 }}" + Anthropic__Models__2: "{{ .Values.config.memex_portal.Anthropic__Models__2 }}" + AzureFoundry__Endpoint: "{{ .Values.config.memex_portal.AzureFoundry__Endpoint }}" + AzureFoundry__Models__0: "{{ .Values.config.memex_portal.AzureFoundry__Models__0 }}" + AzureFoundry__Models__1: "{{ .Values.config.memex_portal.AzureFoundry__Models__1 }}" + AzureFoundry__Models__2: "{{ .Values.config.memex_portal.AzureFoundry__Models__2 }}" + # Tier → concrete model id (agents that declare ModelTier resolve through these). + ModelTier__Heavy: "{{ .Values.config.memex_portal.ModelTier__Heavy }}" + ModelTier__Standard: "{{ .Values.config.memex_portal.ModelTier__Standard }}" + ModelTier__Light: "{{ .Values.config.memex_portal.ModelTier__Light }}" + ModelTier__Utility: "{{ .Values.config.memex_portal.ModelTier__Utility }}" + # Per-user co-hosted Claude Code config root (the /mnt/users RWX share): each user's `claude` + # login + .credentials.json live under {root}/{userId}/.claude. Drives the Connect login dir. + ClaudeCode__ConfigDirRoot: "{{ .Values.config.memex_portal.ClaudeCode__ConfigDirRoot }}" + # ---- AI provider feature flags -------------------------------------------- + # Gate whether each provider's catalog source is registered at all + # (MemexConfiguration: `if (features.Ai.Providers.Anthropic) AddAnthropic()`). + # Default "true" (provider on) so existing envs are unchanged; set "false" + # per-env to drop the provider entirely — no catalog source → it vanishes from + # the model picker and its Model/ nodes never seed. atioz sets this false + # (Claude there is per-user Claude Code, NOT a wired API provider). + Features__Ai__Providers__Anthropic: "{{ .Values.config.memex_portal.Features__Ai__Providers__Anthropic | default "true" }}" + # Same gate for the AzureFoundry (open-weight /models, e.g. DeepSeek) chat provider. + # Default "true" so existing envs keep DeepSeek; set "false" per-env to drop it from + # the picker (catalog source not registered). Does NOT affect embeddings, which are + # wired separately from Embedding__* — see MemexConfiguration. + Features__Ai__Providers__AzureFoundry: "{{ .Values.config.memex_portal.Features__Ai__Providers__AzureFoundry | default "true" }}" + # ---- AI: extra non-secret knobs (optional; default empty) ----------------- + # Order is an int — empty string fails Int32 binding, so default to "0" (NOT ""). + Anthropic__Order: "{{ .Values.config.memex_portal.Anthropic__Order | default "0" }}" + AzureAIS__Endpoint: "{{ .Values.config.memex_portal.AzureAIS__Endpoint | default "" }}" + AzureAIS__Order: "{{ .Values.config.memex_portal.AzureAIS__Order | default "0" }}" + # ---- Observability (OTLP collector) — optional ---------------------------- + OTEL_EXPORTER_OTLP_ENDPOINT: "{{ .Values.config.memex_portal.OTEL_EXPORTER_OTLP_ENDPOINT | default "" }}" + OTEL_EXPORTER_OTLP_PROTOCOL: "{{ .Values.config.memex_portal.OTEL_EXPORTER_OTLP_PROTOCOL | default "" }}" + # ---- Sign-in (non-secret: provider client ids + mode) --------------------- + # A provider's button shows iff its ClientId is non-empty (AuthenticationBuilder + # no-ops on empty). Set a ClientId to "" to DISABLE that provider even when the + # image's appsettings.json ships a default (env overrides appsettings). Secrets + # (client secrets) come from the SecretProviderClass, not here. + Authentication__Provider: "{{ .Values.config.memex_portal.Authentication__Provider | default "" }}" + Authentication__EnableDevLogin: "{{ .Values.config.memex_portal.Authentication__EnableDevLogin | default "false" }}" + Authentication__Microsoft__ClientId: "{{ .Values.config.memex_portal.Authentication__Microsoft__ClientId | default "" }}" + Authentication__Microsoft__TenantId: "{{ .Values.config.memex_portal.Authentication__Microsoft__TenantId | default "" }}" + Authentication__Google__ClientId: "{{ .Values.config.memex_portal.Authentication__Google__ClientId | default "" }}" + Authentication__LinkedIn__ClientId: "{{ .Values.config.memex_portal.Authentication__LinkedIn__ClientId | default "" }}" + Social__LinkedIn__ClientId: "{{ .Values.config.memex_portal.Social__LinkedIn__ClientId | default "" }}" + # ---- Onboarding: invitation-only gate (first user always bootstraps) ------ + Features__Onboarding__InvitationOnly: "{{ .Values.config.memex_portal.Features__Onboarding__InvitationOnly | default "false" }}" + # ---- Static-repo → DB sync: which partitions are materialized into + served from the DB + # (instead of the in-memory static provider). Default = Doc, Agent, Model. Override per-env by + # setting these keys (empty string disables a slot). See Doc/Architecture/StaticRepoImport.md. + Features__StaticRepoSync__Partitions__0: "{{ .Values.config.memex_portal.Features__StaticRepoSync__Partitions__0 | default "Doc" }}" + Features__StaticRepoSync__Partitions__1: "{{ .Values.config.memex_portal.Features__StaticRepoSync__Partitions__1 | default "Agent" }}" + Features__StaticRepoSync__Partitions__2: "{{ .Values.config.memex_portal.Features__StaticRepoSync__Partitions__2 | default "Model" }}" + Features__StaticRepoSync__Partitions__3: "{{ .Values.config.memex_portal.Features__StaticRepoSync__Partitions__3 | default "Harness" }}" + Features__StaticRepoSync__Partitions__4: "{{ .Values.config.memex_portal.Features__StaticRepoSync__Partitions__4 | default "Skill" }}" + # ---- System email (Microsoft Graph) — BIDIRECTIONAL. Outbound invitations + notifications + # (/sendMail, Mail.Send app permission) AND optional inbound email→agent channel (inbox + # change-notification subscription, Mail.ReadWrite app permission). Non-secret here; + # Email__ClientSecret comes from the SecretProviderClass. When Email__Enabled is empty/false + # the portal uses the NoOp sender (no mail sent). + Email__Enabled: "{{ .Values.config.memex_portal.Email__Enabled | default "false" }}" + # The mailbox the portal sends AND receives AS (a real/shared mailbox the app may send-as). + # NOTE: binds EmailOptions.MailboxAddress — the old key Email__NoReplyAddress no longer binds. + Email__MailboxAddress: "{{ .Values.config.memex_portal.Email__MailboxAddress | default "" }}" + Email__TenantId: "{{ .Values.config.memex_portal.Email__TenantId | default "" }}" + Email__ClientId: "{{ .Values.config.memex_portal.Email__ClientId | default "" }}" + Email__UseManagedIdentity: "{{ .Values.config.memex_portal.Email__UseManagedIdentity | default "false" }}" + # Inbound email→agent channel (optional). Requires Mail.ReadWrite + a public WebhookBaseUrl; + # the webhook is {WebhookBaseUrl}/api/email. SubscriptionClientState is a per-deployment random + # value echoed by Graph on every notification (the webhook rejects mismatches). + Email__InboundEnabled: "{{ .Values.config.memex_portal.Email__InboundEnabled | default "false" }}" + Email__WebhookBaseUrl: "{{ .Values.config.memex_portal.Email__WebhookBaseUrl | default "" }}" + Email__SubscriptionClientState: "{{ .Values.config.memex_portal.Email__SubscriptionClientState | default "" }}" diff --git a/deploy/helm/templates/memex-portal/deployment.yaml b/deploy/helm/templates/memex-portal/deployment.yaml new file mode 100644 index 000000000..bb8e4ac4b --- /dev/null +++ b/deploy/helm/templates/memex-portal/deployment.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: "memex-portal-deployment" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-portal" + app.kubernetes.io/instance: "{{ .Release.Name }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-portal" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "ghcr.io/systemorph/memex-portal-ai:latest" + name: "memex-portal" + envFrom: + - configMapRef: + name: "memex-portal-config" + - secretRef: + name: "memex-portal-secrets" + ports: + - name: "http" + protocol: "TCP" + containerPort: 8080 + volumeMounts: + - name: "memex-data" + mountPath: "/data" + - name: "memex-users" + mountPath: "/mnt/users" + imagePullPolicy: "IfNotPresent" + volumes: + - name: "memex-data" + emptyDir: {} + - name: "memex-users" + emptyDir: {} + selector: + matchLabels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-portal" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 + revisionHistoryLimit: 3 + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + type: "RollingUpdate" diff --git a/deploy/helm/templates/memex-portal/secrets.yaml b/deploy/helm/templates/memex-portal/secrets.yaml new file mode 100644 index 000000000..def9c0a08 --- /dev/null +++ b/deploy/helm/templates/memex-portal/secrets.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: "v1" +kind: "Secret" +metadata: + name: "memex-portal-secrets" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-portal" + app.kubernetes.io/instance: "{{ .Release.Name }}" +stringData: + ConnectionStrings__memex: "Host=memex-postgres-service;Port=5432;Username=postgres;Password={{ .Values.secrets.memex_portal.memex_postgres_password }};Database=memex" + # Content indexing writes its tables into per-partition schemas in THIS same mesh database + # (content_chunks/content_files alongside each partition's mesh_nodes) — no separate connection. + MEMEX_PASSWORD: "{{ .Values.secrets.memex_portal.memex_postgres_password }}" + MEMEX_URI: "postgresql://postgres:{{ .Values.secrets.memex_portal.memex_postgres_password }}@memex-postgres-service:5432/memex" + # ---- AI provider keys (secret) ------------------------------------------- + # Conditionally emitted: a key is set ONLY when its value is non-empty, so the chart never + # overwrites a value injected out-of-band (Key Vault CSI / kubectl). This matters most for + # Ai__KeyProtection__MasterKey — overwriting it makes every stored enc: provider key + # undecryptable. Source real values from Key Vault via the CSI add-on; never commit them. + {{- if .Values.secrets.memex_portal.Anthropic__ApiKey }} + Anthropic__ApiKey: "{{ .Values.secrets.memex_portal.Anthropic__ApiKey }}" + {{- end }} + {{- if .Values.secrets.memex_portal.AzureFoundry__ApiKey }} + AzureFoundry__ApiKey: "{{ .Values.secrets.memex_portal.AzureFoundry__ApiKey }}" + {{- end }} + {{- if .Values.secrets.memex_portal.Embedding__ApiKey }} + Embedding__ApiKey: "{{ .Values.secrets.memex_portal.Embedding__ApiKey }}" + {{- end }} + {{- if .Values.secrets.memex_portal.Ai__KeyProtection__MasterKey }} + Ai__KeyProtection__MasterKey: "{{ .Values.secrets.memex_portal.Ai__KeyProtection__MasterKey }}" + {{- end }} +type: "Opaque" diff --git a/deploy/helm/templates/memex-portal/service.yaml b/deploy/helm/templates/memex-portal/service.yaml new file mode 100644 index 000000000..9813098be --- /dev/null +++ b/deploy/helm/templates/memex-portal/service.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: "v1" +kind: "Service" +metadata: + name: "memex-portal-service" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-portal" + app.kubernetes.io/instance: "{{ .Release.Name }}" +spec: + type: "ClusterIP" + selector: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-portal" + app.kubernetes.io/instance: "{{ .Release.Name }}" + ports: + - name: "http" + protocol: "TCP" + port: 8080 + targetPort: 8080 diff --git a/deploy/helm/templates/memex-postgres/config.yaml b/deploy/helm/templates/memex-postgres/config.yaml new file mode 100644 index 000000000..aa4a2ca68 --- /dev/null +++ b/deploy/helm/templates/memex-postgres/config.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: "v1" +kind: "ConfigMap" +metadata: + name: "memex-postgres-config" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-postgres" + app.kubernetes.io/instance: "{{ .Release.Name }}" +data: + POSTGRES_HOST_AUTH_METHOD: "{{ .Values.config.memex_postgres.POSTGRES_HOST_AUTH_METHOD }}" + POSTGRES_INITDB_ARGS: "{{ .Values.config.memex_postgres.POSTGRES_INITDB_ARGS }}" + POSTGRES_USER: "{{ .Values.config.memex_postgres.POSTGRES_USER }}" diff --git a/deploy/helm/templates/memex-postgres/secrets.yaml b/deploy/helm/templates/memex-postgres/secrets.yaml new file mode 100644 index 000000000..ceb0d3362 --- /dev/null +++ b/deploy/helm/templates/memex-postgres/secrets.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: "v1" +kind: "Secret" +metadata: + name: "memex-postgres-secrets" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-postgres" + app.kubernetes.io/instance: "{{ .Release.Name }}" +stringData: + POSTGRES_PASSWORD: "{{ .Values.secrets.memex_postgres.memex_postgres_password }}" +type: "Opaque" diff --git a/deploy/helm/templates/memex-postgres/service.yaml b/deploy/helm/templates/memex-postgres/service.yaml new file mode 100644 index 000000000..7993e7047 --- /dev/null +++ b/deploy/helm/templates/memex-postgres/service.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: "v1" +kind: "Service" +metadata: + name: "memex-postgres-service" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-postgres" + app.kubernetes.io/instance: "{{ .Release.Name }}" +spec: + type: "ClusterIP" + selector: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-postgres" + app.kubernetes.io/instance: "{{ .Release.Name }}" + ports: + - name: "tcp" + protocol: "TCP" + port: 5432 + targetPort: 5432 diff --git a/deploy/helm/templates/memex-postgres/statefulset.yaml b/deploy/helm/templates/memex-postgres/statefulset.yaml new file mode 100644 index 000000000..43428f5f1 --- /dev/null +++ b/deploy/helm/templates/memex-postgres/statefulset.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: "apps/v1" +kind: "StatefulSet" +metadata: + name: "memex-postgres-statefulset" + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-postgres" + app.kubernetes.io/instance: "{{ .Release.Name }}" +spec: + template: + metadata: + labels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-postgres" + app.kubernetes.io/instance: "{{ .Release.Name }}" + spec: + containers: + - image: "docker.io/pgvector/pgvector:pg17" + name: "memex-postgres" + envFrom: + - configMapRef: + name: "memex-postgres-config" + - secretRef: + name: "memex-postgres-secrets" + ports: + - name: "tcp" + protocol: "TCP" + containerPort: 5432 + volumeMounts: + - name: "memex-pgdata" + mountPath: "/var/lib/postgresql/data" + imagePullPolicy: "IfNotPresent" + volumes: + - name: "memex-pgdata" + emptyDir: {} + selector: + matchLabels: + app.kubernetes.io/name: "{{ .Chart.Name }}" + app.kubernetes.io/component: "memex-postgres" + app.kubernetes.io/instance: "{{ .Release.Name }}" + replicas: 1 + persistentVolumeClaimRetentionPolicy: {} + updateStrategy: + rollingUpdate: {} + type: "RollingUpdate" diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml new file mode 100644 index 000000000..abdd4e904 --- /dev/null +++ b/deploy/helm/values.yaml @@ -0,0 +1,75 @@ +parameters: {} +secrets: + memex_postgres: + memex_postgres_password: "" + memex_migration: + ConnectionStrings__memex: "" + memex_postgres_password: "" + MEMEX_URI: "" + memex_portal: + ConnectionStrings__memex: "" + memex_postgres_password: "" + MEMEX_URI: "" + # AI provider keys — empty by default (neutral chart). Set in an overlay + # (e.g. values.aks.yaml) or via --set / Key Vault CSI. Empty = no system + # catalog seeded for that provider. + Anthropic__ApiKey: "" + AzureFoundry__ApiKey: "" + Ai__KeyProtection__MasterKey: "" +config: + memex_postgres: + POSTGRES_HOST_AUTH_METHOD: "scram-sha-256" + POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256 --auth-local=scram-sha-256" + POSTGRES_USER: "postgres" + memex_migration: + MEMEX_HOST: "memex-postgres-service" + MEMEX_PORT: "5432" + MEMEX_USERNAME: "postgres" + MEMEX_JDBCCONNECTIONSTRING: "jdbc:postgresql://memex-postgres-service:5432/memex" + MEMEX_DATABASENAME: "memex" + memex_portal: + MEMEX_HOST: "memex-postgres-service" + MEMEX_PORT: "5432" + MEMEX_USERNAME: "postgres" + MEMEX_JDBCCONNECTIONSTRING: "jdbc:postgresql://memex-postgres-service:5432/memex" + MEMEX_DATABASENAME: "memex" + ASPNETCORE_HTTP_PORTS: "8080" + Deployment__Backend: "Filesystem" + Deployment__DataRoot: "/data" + Deployment__Orleans__Clustering: "Localhost" + Storage__Name: "content" + Storage__SourceType: "FileSystem" + Storage__BasePath: "/data/content" + Graph__Storage__Type: "PostgreSql" + Graph__Storage__BasePath: "/data/graph" + Mcp__BaseUrl: "http://memex-portal-service:8080" + # AI model catalog — empty by default. An overlay sets endpoints + model + # lists (+ keys in secrets above) to seed the system catalog on deploy. + Anthropic__Endpoint: "" + Anthropic__Models__0: "" + Anthropic__Models__1: "" + Anthropic__Models__2: "" + AzureFoundry__Endpoint: "" + AzureFoundry__Models__0: "" + AzureFoundry__Models__1: "" + AzureFoundry__Models__2: "" + ModelTier__Heavy: "" + ModelTier__Standard: "" + ModelTier__Light: "" + ModelTier__Utility: "" + ClaudeCode__ConfigDirRoot: "" + # ---- System email (Microsoft Graph) — BIDIRECTIONAL. Off by default; an + # overlay turns it on per environment by setting Email__Enabled + the env's + # own MailboxAddress / TenantId / ClientId (+ WebhookBaseUrl for inbound). + # The secret Email__ClientSecret comes from the SecretProviderClass. + Email__Enabled: "false" + Email__MailboxAddress: "" + Email__TenantId: "" + Email__ClientId: "" + Email__UseManagedIdentity: "false" + Email__InboundEnabled: "false" + Email__WebhookBaseUrl: "" + # Shared webhook-guard value Graph echoes on every inbound notification; the + # /api/email webhook rejects mismatches. Global (same across envs is fine — + # each deployment validates against its own configured value). + Email__SubscriptionClientState: "meshweaver-emailhook-7f3c9a21e84b46d2b1c05e9af6d3" diff --git a/deploy/marketplace/README.md b/deploy/marketplace/README.md new file mode 100644 index 000000000..f717a9189 --- /dev/null +++ b/deploy/marketplace/README.md @@ -0,0 +1,67 @@ +# Memex — Azure Marketplace (Azure Application) + +This folder packages the Memex portal as an **Azure Application** offer (customer deploys into +their own subscription; they own the infra + data; we ship updates as new offer versions). + +## What's here + +| File | What | +|---|---| +| `mainTemplate.json` | The ARM template, **generated** from the Aspire model — not hand-authored. | +| `createUiDefinition.json` | The deploy-wizard UI (region, sizing, AI providers + key, master key, self-onboarding). | + +## How `mainTemplate.json` is generated (single source of truth = Aspire) + +```bash +# 1. Aspire emits the ACA bicep from the dedicated image-based AppHost's `azure` mode: +aspire publish --apphost deploy/aspire/Memex.Deploy.AppHost/Memex.Deploy.AppHost.csproj \ + -o deploy/aca -- --mode azure +# 2. Convert bicep -> ARM JSON: +az bicep build --file deploy/aca/main.bicep --outfile deploy/marketplace/mainTemplate.json +``` + +The bicep (`deploy/aca/`) and this ARM both come from the same `AddMemex` model that produces the +Docker Compose (`deploy/compose/`) and Helm (`deploy/helm/`) artifacts — one model, three targets. + +## 🚧 Reconciliations required before this is a publishable offer + +The generated template proves the pipeline but is **not yet a turn-key Marketplace solution +template**. Four gaps, each tracked: + +1. **Deployment scope.** The generated `mainTemplate.json` is `subscriptionDeploymentTemplate` + (`targetScope = 'subscription'`, it *creates* the resource group — the azd/Aspire pattern). A + Marketplace **solution template** is **resource-group-scoped** (the RG is chosen in the wizard's + Basics step). Either adapt to RG-scope (drop the `Microsoft.Resources/resourceGroups` resource + + re-scope the nested deployments) or publish as a **Managed Application** (which accepts a + subscription-scoped appliance). The latter also lets us retain ops access if desired. + +2. **Parameterize customer inputs.** The generated ARM bakes the app config (image tag, the + `Ai:KeyProtection:MasterKey`, model-provider key, `Features:*`, self-onboarding) as **literals** + inside the module resources, so `createUiDefinition` has nothing to bind to yet. Surface them as + Aspire **`AddParameter(...)`** in `Memex.Deploy.AppHost` (Aspire renders parameters as ARM + `parameters`, secrets as `secureString`); then wire `createUiDefinition.json` `outputs` → + those params. The wizard here already collects the intended inputs. + +3. **Managed Postgres.** The image-based `AddMemex` runs a **pgvector container** (great for + Compose/Helm self-host). For a production Azure offer, switch the Azure target to **Azure + Database for PostgreSQL Flexible Server** (the `Backend=Azure` branch of `AddMemex` — the + Azure-unification work) so the customer's data lives on managed, backed-up infra. + +4. **Public images.** `mainTemplate.json` references `ghcr.io/systemorph/memex-portal-ai` + + `memex-migration`. These must be **publicly pullable** by arbitrary customer subscriptions — + built + pushed by the GHCR CI workflow. + +## Non-code, your action (long lead) + +- Microsoft **Partner Center** account enrolled in the Commercial Marketplace program. +- Offer listing assets (logo, screenshots, description, privacy/terms URLs, support contact). +- A test tenant for **Preview** validation, plus **ARM-TTK** (`Test-AzTemplate`) on the packaged + offer before go-live. + +## Validate locally + +```bash +az deployment group validate --resource-group --template-file mainTemplate.json # after RG-scope reconcile +# ARM-TTK: +Test-AzTemplate -TemplatePath deploy/marketplace +``` diff --git a/deploy/marketplace/createUiDefinition.json b/deploy/marketplace/createUiDefinition.json new file mode 100644 index 000000000..f59b81caf --- /dev/null +++ b/deploy/marketplace/createUiDefinition.json @@ -0,0 +1,119 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/0.1.2-preview/CreateUIDefinition.MultiVm.json#", + "handler": "Microsoft.Azure.CreateUIDef", + "version": "0.1.2-preview", + "parameters": { + "config": { + "isWizard": false, + "basics": { + "description": "Deploy the **MeshWeaver Memex** portal into your own Azure subscription. You own the infrastructure and the data; updates ship as new offer versions. The portal and its database run as Azure Container Apps from the public images at `ghcr.io/systemorph`.", + "location": { + "label": "Region", + "toolTip": "Azure region for the Container Apps environment, Postgres, and storage.", + "resourceTypes": [ "Microsoft.App/managedEnvironments" ] + } + } + }, + "basics": [], + "steps": [ + { + "name": "sizing", + "label": "Sizing", + "elements": [ + { + "name": "imageTag", + "type": "Microsoft.Common.TextBox", + "label": "Image tag", + "defaultValue": "latest", + "toolTip": "Tag of the ghcr.io/systemorph/memex-portal-ai + memex-migration images to deploy (e.g. a released version).", + "constraints": { "required": true } + }, + { + "name": "replicas", + "type": "Microsoft.Common.Slider", + "label": "Portal replicas", + "min": 1, + "max": 6, + "defaultValue": 2, + "toolTip": "Orleans needs >= 2 for resilience under load. Single-node uses localhost clustering; >1 uses ADO.NET-Postgres clustering (HA)." + } + ] + }, + { + "name": "ai", + "label": "AI capabilities", + "elements": [ + { + "name": "providers", + "type": "Microsoft.Common.OptionsGroup", + "label": "In-process model providers to enable", + "multiselect": true, + "defaultValue": [ "Azure OpenAI" ], + "toolTip": "Which API-key providers ship (Features:Ai:Providers). Bring your own key below.", + "constraints": { + "allowedValues": [ + { "label": "Anthropic", "value": "Anthropic" }, + { "label": "Azure OpenAI", "value": "AzureOpenAI" }, + { "label": "Azure AI Foundry", "value": "AzureFoundry" }, + { "label": "OpenAI", "value": "OpenAI" } + ] + } + }, + { + "name": "modelProviderKey", + "type": "Microsoft.Common.PasswordBox", + "label": { "password": "Model provider API key", "confirmPassword": "Confirm key" }, + "toolTip": "Your Azure OpenAI / Azure AI Foundry / Anthropic key. Stored as an ARM secureString; injected as a container secret.", + "options": { "hideConfirmation": true }, + "constraints": { "required": false } + }, + { + "name": "modelProviderEndpoint", + "type": "Microsoft.Common.TextBox", + "label": "Model provider endpoint", + "toolTip": "Azure OpenAI / Foundry endpoint URL (leave blank for direct OpenAI/Anthropic).", + "constraints": { "required": false } + }, + { + "name": "includeAiClis", + "type": "Microsoft.Common.CheckBox", + "label": "Bundle co-hosted Claude Code + GitHub Copilot CLIs (portal-ai image)", + "toolTip": "Deploys the larger portal-ai image so users can connect their own Claude/Copilot subscriptions. Uncheck for the lean portal." + } + ] + }, + { + "name": "security", + "label": "Security", + "elements": [ + { + "name": "masterKey", + "type": "Microsoft.Common.PasswordBox", + "label": { "password": "Provider-key encryption master key", "confirmPassword": "Confirm master key" }, + "toolTip": "Base64 32-byte key (Ai:KeyProtection:MasterKey) that envelope-encrypts stored provider credentials at rest. Keep it safe; rotating it invalidates stored keys.", + "options": { "hideConfirmation": true }, + "constraints": { "required": true } + }, + { + "name": "allowSelfOnboarding", + "type": "Microsoft.Common.CheckBox", + "label": "Allow new users to self-onboard (open registration)", + "defaultValue": true, + "toolTip": "When unchecked, only an administrator can provision users (the first user still bootstraps as admin). Maps to Features:Onboarding:AllowSelfOnboarding." + } + ] + } + ], + "outputs": { + "location": "[location()]", + "imageTag": "[steps('sizing').imageTag]", + "replicas": "[steps('sizing').replicas]", + "aiProviders": "[steps('ai').providers]", + "modelProviderKey": "[steps('ai').modelProviderKey]", + "modelProviderEndpoint": "[steps('ai').modelProviderEndpoint]", + "includeAiClis": "[steps('ai').includeAiClis]", + "keyProtectionMasterKey": "[steps('security').masterKey]", + "allowSelfOnboarding": "[steps('security').allowSelfOnboarding]" + } + } +} diff --git a/deploy/marketplace/mainTemplate.json b/deploy/marketplace/mainTemplate.json new file mode 100644 index 000000000..525dcd592 --- /dev/null +++ b/deploy/marketplace/mainTemplate.json @@ -0,0 +1,408 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.41.2.15936", + "templateHash": "8255626853934111120" + } + }, + "parameters": { + "resourceGroupName": { + "type": "string" + }, + "location": { + "type": "string" + }, + "principalId": { + "type": "string" + } + }, + "resources": [ + { + "type": "Microsoft.Resources/resourceGroups", + "apiVersion": "2023-07-01", + "name": "[parameters('resourceGroupName')]", + "location": "[parameters('location')]" + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2025-04-01", + "name": "memex-aca-acr", + "resourceGroup": "[parameters('resourceGroupName')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "location": { + "value": "[parameters('location')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.41.2.15936", + "templateHash": "17549538876489140601" + } + }, + "parameters": { + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The location for the resource(s) to be deployed." + } + } + }, + "resources": [ + { + "type": "Microsoft.ContainerRegistry/registries", + "apiVersion": "2025-04-01", + "name": "[take(format('memexacaacr{0}', uniqueString(resourceGroup().id)), 50)]", + "location": "[parameters('location')]", + "sku": { + "name": "Basic" + }, + "tags": { + "aspire-resource-name": "memex-aca-acr" + } + } + ], + "outputs": { + "name": { + "type": "string", + "value": "[take(format('memexacaacr{0}', uniqueString(resourceGroup().id)), 50)]" + }, + "loginServer": { + "type": "string", + "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries', take(format('memexacaacr{0}', uniqueString(resourceGroup().id)), 50)), '2025-04-01').loginServer]" + }, + "id": { + "type": "string", + "value": "[resourceId('Microsoft.ContainerRegistry/registries', take(format('memexacaacr{0}', uniqueString(resourceGroup().id)), 50))]" + } + } + } + }, + "dependsOn": [ + "[subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('resourceGroupName'))]" + ] + }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2025-04-01", + "name": "memex-aca", + "resourceGroup": "[parameters('resourceGroupName')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "location": { + "value": "[parameters('location')]" + }, + "memex_aca_acr_outputs_name": { + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'memex-aca-acr'), '2025-04-01').outputs.name.value]" + }, + "userPrincipalId": { + "value": "[parameters('principalId')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.41.2.15936", + "templateHash": "13259423154412693621" + } + }, + "parameters": { + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "The location for the resource(s) to be deployed." + } + }, + "userPrincipalId": { + "type": "string", + "defaultValue": "" + }, + "tags": { + "type": "object", + "defaultValue": {} + }, + "memex_aca_acr_outputs_name": { + "type": "string" + } + }, + "resources": [ + { + "type": "Microsoft.ManagedIdentity/userAssignedIdentities", + "apiVersion": "2024-11-30", + "name": "[take(format('memex_aca_mi-{0}', uniqueString(resourceGroup().id)), 128)]", + "location": "[parameters('location')]", + "tags": "[parameters('tags')]" + }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "scope": "[resourceId('Microsoft.ContainerRegistry/registries', parameters('memex_aca_acr_outputs_name'))]", + "name": "[guid(resourceId('Microsoft.ContainerRegistry/registries', parameters('memex_aca_acr_outputs_name')), resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', take(format('memex_aca_mi-{0}', uniqueString(resourceGroup().id)), 128)), subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d'))]", + "properties": { + "principalId": "[reference(resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', take(format('memex_aca_mi-{0}', uniqueString(resourceGroup().id)), 128)), '2024-11-30').principalId]", + "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '7f951dda-4ed3-4680-a7ca-43fe172d538d')]", + "principalType": "ServicePrincipal" + }, + "dependsOn": [ + "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', take(format('memex_aca_mi-{0}', uniqueString(resourceGroup().id)), 128))]" + ] + }, + { + "type": "Microsoft.OperationalInsights/workspaces", + "apiVersion": "2025-02-01", + "name": "[take(format('memexacalaw-{0}', uniqueString(resourceGroup().id)), 63)]", + "location": "[parameters('location')]", + "properties": { + "sku": { + "name": "PerGB2018" + } + }, + "tags": "[parameters('tags')]" + }, + { + "type": "Microsoft.App/managedEnvironments", + "apiVersion": "2025-07-01", + "name": "[take(format('memexaca{0}', uniqueString(resourceGroup().id)), 24)]", + "location": "[parameters('location')]", + "properties": { + "appLogsConfiguration": { + "destination": "log-analytics", + "logAnalyticsConfiguration": { + "customerId": "[reference(resourceId('Microsoft.OperationalInsights/workspaces', take(format('memexacalaw-{0}', uniqueString(resourceGroup().id)), 63)), '2025-02-01').customerId]", + "sharedKey": "[listKeys(resourceId('Microsoft.OperationalInsights/workspaces', take(format('memexacalaw-{0}', uniqueString(resourceGroup().id)), 63)), '2025-02-01').primarySharedKey]" + } + }, + "workloadProfiles": [ + { + "name": "consumption", + "workloadProfileType": "Consumption" + } + ] + }, + "tags": "[parameters('tags')]", + "dependsOn": [ + "[resourceId('Microsoft.OperationalInsights/workspaces', take(format('memexacalaw-{0}', uniqueString(resourceGroup().id)), 63))]" + ] + }, + { + "type": "Microsoft.App/managedEnvironments/dotNetComponents", + "apiVersion": "2025-10-02-preview", + "name": "[format('{0}/{1}', take(format('memexaca{0}', uniqueString(resourceGroup().id)), 24), 'aspire-dashboard')]", + "properties": { + "componentType": "AspireDashboard" + }, + "dependsOn": [ + "[resourceId('Microsoft.App/managedEnvironments', take(format('memexaca{0}', uniqueString(resourceGroup().id)), 24))]" + ] + }, + { + "type": "Microsoft.Storage/storageAccounts", + "apiVersion": "2024-01-01", + "name": "[take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24)]", + "kind": "StorageV2", + "location": "[parameters('location')]", + "sku": { + "name": "Standard_LRS" + }, + "properties": { + "largeFileSharesState": "Enabled", + "minimumTlsVersion": "TLS1_2" + }, + "tags": "[parameters('tags')]" + }, + { + "type": "Microsoft.Storage/storageAccounts/fileServices", + "apiVersion": "2024-01-01", + "name": "[format('{0}/{1}', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24), 'default')]", + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24))]" + ] + }, + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2024-01-01", + "name": "[format('{0}/{1}/{2}', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24), 'default', take(format('sharesvolumesmemexpostgres0-{0}', uniqueString(resourceGroup().id)), 63))]", + "properties": { + "enabledProtocols": "SMB", + "shareQuota": 1024 + }, + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24), 'default')]" + ] + }, + { + "type": "Microsoft.App/managedEnvironments/storages", + "apiVersion": "2025-07-01", + "name": "[format('{0}/{1}', take(format('memexaca{0}', uniqueString(resourceGroup().id)), 24), take(format('managedstoragevolumesmemexpostgres{0}', uniqueString(resourceGroup().id)), 24))]", + "properties": { + "azureFile": { + "accountName": "[take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24)]", + "accountKey": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24)), '2024-01-01').keys[0].value]", + "accessMode": "ReadWrite", + "shareName": "[take(format('sharesvolumesmemexpostgres0-{0}', uniqueString(resourceGroup().id)), 63)]" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.App/managedEnvironments', take(format('memexaca{0}', uniqueString(resourceGroup().id)), 24))]", + "[resourceId('Microsoft.Storage/storageAccounts', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24))]", + "[resourceId('Microsoft.Storage/storageAccounts/fileServices/shares', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24), 'default', take(format('sharesvolumesmemexpostgres0-{0}', uniqueString(resourceGroup().id)), 63))]" + ] + }, + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2024-01-01", + "name": "[format('{0}/{1}/{2}', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24), 'default', take(format('sharesvolumesmemexportal0-{0}', uniqueString(resourceGroup().id)), 63))]", + "properties": { + "enabledProtocols": "SMB", + "shareQuota": 1024 + }, + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24), 'default')]" + ] + }, + { + "type": "Microsoft.App/managedEnvironments/storages", + "apiVersion": "2025-07-01", + "name": "[format('{0}/{1}', take(format('memexaca{0}', uniqueString(resourceGroup().id)), 24), take(format('managedstoragevolumesmemexportal{0}', uniqueString(resourceGroup().id)), 24))]", + "properties": { + "azureFile": { + "accountName": "[take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24)]", + "accountKey": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24)), '2024-01-01').keys[0].value]", + "accessMode": "ReadWrite", + "shareName": "[take(format('sharesvolumesmemexportal0-{0}', uniqueString(resourceGroup().id)), 63)]" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.App/managedEnvironments', take(format('memexaca{0}', uniqueString(resourceGroup().id)), 24))]", + "[resourceId('Microsoft.Storage/storageAccounts', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24))]", + "[resourceId('Microsoft.Storage/storageAccounts/fileServices/shares', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24), 'default', take(format('sharesvolumesmemexportal0-{0}', uniqueString(resourceGroup().id)), 63))]" + ] + }, + { + "type": "Microsoft.Storage/storageAccounts/fileServices/shares", + "apiVersion": "2024-01-01", + "name": "[format('{0}/{1}/{2}', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24), 'default', take(format('sharesvolumesmemexportal1-{0}', uniqueString(resourceGroup().id)), 63))]", + "properties": { + "enabledProtocols": "SMB", + "shareQuota": 1024 + }, + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts/fileServices', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24), 'default')]" + ] + }, + { + "type": "Microsoft.App/managedEnvironments/storages", + "apiVersion": "2025-07-01", + "name": "[format('{0}/{1}', take(format('memexaca{0}', uniqueString(resourceGroup().id)), 24), take(format('managedstoragevolumesmemexportal{0}', uniqueString(resourceGroup().id)), 24))]", + "properties": { + "azureFile": { + "accountName": "[take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24)]", + "accountKey": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24)), '2024-01-01').keys[0].value]", + "accessMode": "ReadWrite", + "shareName": "[take(format('sharesvolumesmemexportal1-{0}', uniqueString(resourceGroup().id)), 63)]" + } + }, + "dependsOn": [ + "[resourceId('Microsoft.App/managedEnvironments', take(format('memexaca{0}', uniqueString(resourceGroup().id)), 24))]", + "[resourceId('Microsoft.Storage/storageAccounts', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24))]", + "[resourceId('Microsoft.Storage/storageAccounts/fileServices/shares', take(format('memexacastoragevolume{0}', uniqueString(resourceGroup().id)), 24), 'default', take(format('sharesvolumesmemexportal1-{0}', uniqueString(resourceGroup().id)), 63))]" + ] + } + ], + "outputs": { + "volumes_memex_postgres_0": { + "type": "string", + "value": "[take(format('managedstoragevolumesmemexpostgres{0}', uniqueString(resourceGroup().id)), 24)]" + }, + "volumes_memex_portal_0": { + "type": "string", + "value": "[take(format('managedstoragevolumesmemexportal{0}', uniqueString(resourceGroup().id)), 24)]" + }, + "volumes_memex_portal_1": { + "type": "string", + "value": "[take(format('managedstoragevolumesmemexportal{0}', uniqueString(resourceGroup().id)), 24)]" + }, + "AZURE_LOG_ANALYTICS_WORKSPACE_NAME": { + "type": "string", + "value": "[take(format('memexacalaw-{0}', uniqueString(resourceGroup().id)), 63)]" + }, + "AZURE_LOG_ANALYTICS_WORKSPACE_ID": { + "type": "string", + "value": "[resourceId('Microsoft.OperationalInsights/workspaces', take(format('memexacalaw-{0}', uniqueString(resourceGroup().id)), 63))]" + }, + "AZURE_CONTAINER_REGISTRY_NAME": { + "type": "string", + "value": "[parameters('memex_aca_acr_outputs_name')]" + }, + "AZURE_CONTAINER_REGISTRY_ENDPOINT": { + "type": "string", + "value": "[reference(resourceId('Microsoft.ContainerRegistry/registries', parameters('memex_aca_acr_outputs_name')), '2025-04-01').loginServer]" + }, + "AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID": { + "type": "string", + "value": "[resourceId('Microsoft.ManagedIdentity/userAssignedIdentities', take(format('memex_aca_mi-{0}', uniqueString(resourceGroup().id)), 128))]" + }, + "AZURE_CONTAINER_APPS_ENVIRONMENT_NAME": { + "type": "string", + "value": "[take(format('memexaca{0}', uniqueString(resourceGroup().id)), 24)]" + }, + "AZURE_CONTAINER_APPS_ENVIRONMENT_ID": { + "type": "string", + "value": "[resourceId('Microsoft.App/managedEnvironments', take(format('memexaca{0}', uniqueString(resourceGroup().id)), 24))]" + }, + "AZURE_CONTAINER_APPS_ENVIRONMENT_DEFAULT_DOMAIN": { + "type": "string", + "value": "[reference(resourceId('Microsoft.App/managedEnvironments', take(format('memexaca{0}', uniqueString(resourceGroup().id)), 24)), '2025-07-01').defaultDomain]" + } + } + } + }, + "dependsOn": [ + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'memex-aca-acr')]", + "[subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('resourceGroupName'))]" + ] + } + ], + "outputs": { + "memex_aca_AZURE_CONTAINER_APPS_ENVIRONMENT_DEFAULT_DOMAIN": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'memex-aca'), '2025-04-01').outputs.AZURE_CONTAINER_APPS_ENVIRONMENT_DEFAULT_DOMAIN.value]" + }, + "memex_aca_AZURE_CONTAINER_APPS_ENVIRONMENT_ID": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'memex-aca'), '2025-04-01').outputs.AZURE_CONTAINER_APPS_ENVIRONMENT_ID.value]" + }, + "memex_aca_volumes_memex_postgres_0": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'memex-aca'), '2025-04-01').outputs.volumes_memex_postgres_0.value]" + }, + "memex_aca_volumes_memex_portal_0": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'memex-aca'), '2025-04-01').outputs.volumes_memex_portal_0.value]" + }, + "memex_aca_volumes_memex_portal_1": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'memex-aca'), '2025-04-01').outputs.volumes_memex_portal_1.value]" + } + } +} \ No newline at end of file diff --git a/failing-tests.txt b/failing-tests.txt new file mode 100644 index 000000000..230d601dd --- /dev/null +++ b/failing-tests.txt @@ -0,0 +1,75 @@ +# Failing tests as of 2026-04-28 13:50 (after Acme cache-dir fix + ValidateToken IObservable conversion). +# 30 known failures across 13 projects (excluding FutuRe — owned by another agent). +# Format: | + +# ========== Markdown.Test (1) ========== +MeshWeaver.Markdown.Test|InteractiveMarkdownExecutionTest.MultipleBlocks_ShareKernelState_ViaSharedAddress + +# ========== AccessControl.Test (2) ========== +MeshWeaver.AccessControl.Test|AccessAssignmentThumbnailTest.Thumbnail_ClickRemoveRole_RemovesChip +MeshWeaver.AccessControl.Test|AccessAssignmentThumbnailTest.UpdateAccessObject_ChangesSubject_ViaDataChange + +# ========== Insurance.Test (1) ========== +MeshWeaver.Insurance.Test|PricingCatalogTests.GetPricingCatalog_UsingLayoutAreaReference_ShouldReturnPricingsControl + +# ========== Todo.Test (1) ========== +MeshWeaver.Todo.Test|TodoDataChangeTest.Step1_SetupDataContext_WithTodoItems + +# ========== Content.Test (1) ========== +MeshWeaver.Content.Test|VersionViewsTest.VersionsArea_SingleVersion_RendersWithoutError + +# ========== Persistence.Test (3) ========== +MeshWeaver.Persistence.Test|MonolithKernelTest.InteractiveShowcaseMd_FullPipeline_AllBlocksExecute +MeshWeaver.Persistence.Test|MonolithKernelTest.MultipleSubmissions_ShareKernelState +MeshWeaver.Persistence.Test|PageLoadingTest.MarkdownNode_LoadsWithoutHanging + +# ========== Auth.Test (1) — pending verification of ValidateToken IObservable refactor ========== +MeshWeaver.Auth.Test|ApiTokenServiceTests.ValidateToken_RevokedToken_ReturnsNull + +# ========== Security.Test (9) ========== +MeshWeaver.Security.Test|AccessControlPipelineTest.SubscribeRequest_WithReadPermission_Succeeds +MeshWeaver.Security.Test|AccessControlPipelineTest.SubscribeRequest_WithoutReadPermission_ReturnsDeliveryFailure +MeshWeaver.Security.Test|McpAccessControlTests.McpSearch_User1SeesOnlyPermittedNodes +MeshWeaver.Security.Test|McpAccessControlTests.McpUpdate_User1CannotUpdatePrivateOrg_User2Can +MeshWeaver.Security.Test|McpAccessControlTests.McpGet_User1CanReadPublicNode +MeshWeaver.Security.Test|McpAccessControlTests.McpSearch_User1CannotSearchPrivateOrg +MeshWeaver.Security.Test|McpAccessControlTests.McpUpdate_User1CannotUpdate_User2Can +MeshWeaver.Security.Test|McpAccessControlTests.McpGet_User1CannotReadPrivateOrg_User2Can +MeshWeaver.Security.Test|McpAccessControlTests.McpGet_User1CannotReadConfidentialNode_User2Can + +# ========== Autocomplete.Test (4) ========== +MeshWeaver.Autocomplete.Test|MeshNodeAutocompleteTest.CanCreateTypeAtPath_ReturnsTrueForValidType +MeshWeaver.Autocomplete.Test|MeshNodeAutocompleteTest.GetCreatableTypes_DifferentNodesDifferentTypes +MeshWeaver.Autocomplete.Test|MeshNodeAutocompleteTest.GetCreatableTypes_ReturnsTypesForNode +MeshWeaver.Autocomplete.Test|AutocompleteMultiSourceTest.LocalFirst_ChildrenOfContextScoreHigherThanDistant + +# ========== Hosting.PostgreSql.Test (1) — likely infra-related (no Docker?) ========== +MeshWeaver.Hosting.PostgreSql.Test|EffectivePermissionPostgresTest.CreateOrganization_HasPermission_ReturnsAdmin + +# ========== Query.Test (3) ========== +MeshWeaver.Query.Test|ChatCompletionOrchestratorTest.AtText_ReturnsCurrentNodeAndGlobal +MeshWeaver.Query.Test|RemoteStreamCacheTest.GetRemoteStream_AfterDispose_ReturnsFreshInstance +MeshWeaver.Query.Test|SyncedQueryTest.PropertyChange_NoLongerMatchesQuery_RemovesFromCollection + +# ========== Acme.Test (4) — was 12, fixed 8 via per-session cache dir ========== +MeshWeaver.Acme.Test|TodoDataChangeWorkflowTest.MultipleTodoHubs_CanBeAccessedIndependently +MeshWeaver.Acme.Test|AcmeSearchTest.DescendantsSearch_FindsOrganizationRootNode +MeshWeaver.Acme.Test|AcmeSearchTest.AcmeOrganization_IsAccessibleToAuthenticatedUser +MeshWeaver.Acme.Test|AcmeSearchTest.SubtreeSearch_FindsOrganizationRootNode + +# ========== Hosting.Orleans.Test (3) — sub-agent owned ========== +MeshWeaver.Hosting.Orleans.Test|OrleansReentrancyTest.ToolCall_DuringStreaming_DoesNotDeadlock +MeshWeaver.Hosting.Orleans.Test|OrleansMarkdownExportTest.SubHub_WithExportTypesRegistered_DeserializesPolymorphicExportDocumentControl +MeshWeaver.Hosting.Orleans.Test|OrleansMarkdownExportTest.ExportPdfArea_RendersExportDocumentControl_ClientDeserializes + +# ========== Fixed in this session (verified passing) ========== +# Hosting.Blazor.Test|NavigationServiceTest.* (2) — IMeshQueryCore + ObserveQuery + select projection +# Threading.Test (2) — pre-existing fix in dependency +# NodeOperations.Test (3) — pre-existing fix in dependency +# Persistence.Test|ResolvePathAsync_*, Move_LargeSubtree (3) — pre-existing fix +# Content.Test|VersionsMenu_AppearsInNodeMenu, VersionsArea_RendersVersionList (2) — pre-existing fix +# AccessControl.Test|Overview_RendersChangeSubjectButton (1) — pre-existing fix +# Insurance.Test|GetPricingCatalog_ShouldReturnPricings (1) — pre-existing fix +# Auth.Test|ValidateToken_ValidToken_ReturnsApiToken (1) — pre-existing fix +# Query.Test (6 of 9) — pre-existing fix +# Acme.Test (8 of 12) — per-session cache dir fix diff --git a/memex/Memex.Portal.Monolith/Program.cs b/memex/Memex.Portal.Monolith/Program.cs index 8815af949..51d97340b 100644 --- a/memex/Memex.Portal.Monolith/Program.cs +++ b/memex/Memex.Portal.Monolith/Program.cs @@ -7,6 +7,7 @@ using MeshWeaver.Hosting.Monolith; using MeshWeaver.Messaging; using Microsoft.AspNetCore.DataProtection; +using Microsoft.Extensions.DependencyInjection; var builder = WebApplication.CreateBuilder(args); @@ -21,6 +22,15 @@ builder.Services.AddDataProtection() .PersistKeysToFileSystem(new DirectoryInfo(keysPath)); +// NodeType compile cache: filesystem-backed in monolith (shared-blob isn't available +// without an Azure account). Versioned entries under {LocalAppData}/Memex/assembly-cache +// persist across restarts; cross-replica sharing isn't applicable here since the +// monolith runs in a single process. +var assemblyCachePath = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + "Memex", "assembly-cache"); +builder.Services.AddFileSystemAssemblyStore(assemblyCachePath); + // Add Aspire service defaults (health checks, OpenTelemetry, service discovery) builder.AddServiceDefaults(); @@ -44,7 +54,9 @@ // Register storage collection at mesh level for static file serving (monolith only) if (storageConfig != null) { - storageConfig = storageConfig with { IsEditable = false, IsStatic = true, ExposeInChildren = false }; + // Storage collection: read-only static backing store, hidden from children. + // IsEditable / ExposeInChildren default to false — leave unset. + storageConfig = storageConfig with { IsStatic = true }; config.ConfigureHub(hub => hub.AddContentCollection(_ => storageConfig)); } diff --git a/memex/Memex.Portal.Monolith/appsettings.Development.json b/memex/Memex.Portal.Monolith/appsettings.Development.json index 4b76908f2..0fd566969 100644 --- a/memex/Memex.Portal.Monolith/appsettings.Development.json +++ b/memex/Memex.Portal.Monolith/appsettings.Development.json @@ -4,13 +4,7 @@ "LogLevel": { "Default": "Warning", "Microsoft.AspNetCore": "Warning", - "MeshWeaver.Hosting": "Warning", - "MeshWeaver.Blazor": "Warning", - "MeshWeaver.Graph": "Debug", - "MeshWeaver.Mesh": "Debug", - "MeshWeaver.AccessContext": "Debug", - "MeshWeaver.AI": "Debug", - "MeshWeaver.AI.Threading": "Debug" + "MeshWeaver": "Warning" } }, "Styles": { diff --git a/memex/Memex.Portal.Monolith/appsettings.json b/memex/Memex.Portal.Monolith/appsettings.json index 6e5e3bcc2..4e238c76d 100644 --- a/memex/Memex.Portal.Monolith/appsettings.json +++ b/memex/Memex.Portal.Monolith/appsettings.json @@ -1,7 +1,7 @@ { "Logging": { "LogLevel": { - "Default": "Information", + "Default": "Warning", "Microsoft.AspNetCore": "Warning" } }, @@ -17,10 +17,5 @@ "EnableDevLogin": true, "Providers": [] }, - "ModelTier": { - "Heavy": "claude-opus-4-6", - "Standard": "claude-sonnet-4-6", - "Light": "claude-haiku-4-5" - }, "AllowedHosts": "*" } diff --git a/memex/Memex.Portal.Shared/Api/MeshApiEndpoints.cs b/memex/Memex.Portal.Shared/Api/MeshApiEndpoints.cs new file mode 100644 index 000000000..1e5083a0e --- /dev/null +++ b/memex/Memex.Portal.Shared/Api/MeshApiEndpoints.cs @@ -0,0 +1,230 @@ +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using System.Text.Json; +using MeshWeaver.AI; +using MeshWeaver.Blazor.AI; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Memex.Portal.Shared.Authentication; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.Routing; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Memex.Portal.Shared.Api; + +///

+/// REST surface for the mesh — a transport-mirror of McpMeshPlugin. +/// +/// +/// Every endpoint is a thin wrapper over (the same +/// shared core that backs the MCP tools), so REST and MCP cannot drift: a change +/// to a verb's semantics happens once, in MeshOperations, and both +/// transports inherit it. +/// +/// +/// +/// Auth: gated by the existing +/// policy — same Authorization: Bearer mw_… token format as /mcp, validated +/// by ApiTokenAuthenticationHandler. +/// +/// +/// +/// Session hub: each request resolves a per-caller hosted hub via +/// (shared with the MCP plugin), so REST callers +/// get the same routing semantics that MCP already has — kernel dispatch, workspace +/// isolation, response routing back to the caller's stream. +/// +/// +/// +/// Shape: RPC-mirror — POST /api/mesh/<verb> with JSON body, 1:1 +/// with MCP tool names. Multipart for binary upload. +/// +/// +public static class MeshApiEndpoints +{ + public const string RoutePrefix = "/api/mesh"; + + /// + /// Maps the /api/mesh/* endpoint group. Call after UseAuthentication / + /// UseAuthorization, alongside MapMeshMcp. + /// + public static IEndpointRouteBuilder MapMeshApi(this IEndpointRouteBuilder endpoints) + { + var group = endpoints.MapGroup(RoutePrefix) + .RequireAuthorization(Memex.Portal.Shared.Authentication.McpAuthenticationExtensions.PolicyName); + + group.MapPost("/get", (HttpContext http, IMessageHub rootHub, GetBody body, CancellationToken ct) => + RunString(http, rootHub, ct, ops => ops.Get(body.Path))); + + group.MapPost("/search", (HttpContext http, IMessageHub rootHub, SearchBody body, CancellationToken ct) => + RunString(http, rootHub, ct, ops => ops.Search(body.Query, body.BasePath))); + + group.MapPost("/create", (HttpContext http, IMessageHub rootHub, CreateBody body, CancellationToken ct) => + RunString(http, rootHub, ct, ops => ops.Create(body.Node))); + + group.MapPost("/update", (HttpContext http, IMessageHub rootHub, UpdateBody body, CancellationToken ct) => + RunString(http, rootHub, ct, ops => ops.Update(body.Nodes))); + + group.MapPost("/patch", (HttpContext http, IMessageHub rootHub, PatchBody body, CancellationToken ct) => + RunString(http, rootHub, ct, ops => ops.Patch(body.Path, body.Fields))); + + group.MapPost("/delete", (HttpContext http, IMessageHub rootHub, DeleteBody body, CancellationToken ct) => + RunString(http, rootHub, ct, ops => ops.Delete(body.Paths))); + + group.MapPost("/move", (HttpContext http, IMessageHub rootHub, MoveBody body, CancellationToken ct) => + RunString(http, rootHub, ct, ops => ops.Move(body.SourcePath, body.TargetPath))); + + group.MapPost("/copy", (HttpContext http, IMessageHub rootHub, CopyBody body, CancellationToken ct) => + RunString(http, rootHub, ct, ops => ops.Copy(body.SourcePath, body.TargetNamespace, body.Force))); + + group.MapPost("/recycle", (HttpContext http, IMessageHub rootHub, PathBody body, CancellationToken ct) => + RunString(http, rootHub, ct, ops => ops.Recycle(body.Path))); + + group.MapPost("/compile", (HttpContext http, IMessageHub rootHub, PathBody body, CancellationToken ct) => + RunString(http, rootHub, ct, ops => ops.Compile(body.Path))); + + group.MapPost("/diagnostics", (HttpContext http, IMessageHub rootHub, PathBody body, CancellationToken ct) => + RunString(http, rootHub, ct, ops => ops.GetDiagnostics(body.Path))); + + group.MapPost("/execute-script", (HttpContext http, IMessageHub rootHub, ExecuteScriptBody body, CancellationToken ct) => + RunString(http, rootHub, ct, ops => ops.ExecuteScript(body.Path, body.TimeoutSeconds ?? 120))); + + // Mirror Push/Pull — these talk to the mesh hub directly (same as MCP plugin's PostMirror). + group.MapPost("/mirror", HandleMirror); + + // Local helpers — same logic as the MCP plugin's NavigateTo / GetBaseUrl. + group.MapPost("/navigate-to", HandleNavigateTo); + group.MapPost("/base-url", HandleBaseUrl); + + // Binary upload — multipart so `curl -F file=@logo.png -F path=@Foo/content/logo.png` works. + // DisableAntiforgery: bearer-auth form posts can't carry an antiforgery token; the request + // is already authenticated by ApiTokenAuthenticationHandler, which is the protection here. + group.MapPost("/upload", HandleUpload).DisableAntiforgery(); + + return endpoints; + } + + private static async Task HandleMirror( + HttpContext http, IMessageHub rootHub, MirrorRequest body, CancellationToken ct) + { + var sessionHub = ResolveSession(http, rootHub); + var delivery = await sessionHub.Observe(body, o => o.WithTarget(new Address("mesh"))) + .Catch((Exception _) => Observable.Return((IMessageDelivery)null!)) + .FirstAsync().ToTask(ct); + var result = delivery?.Message ?? new MirrorResult + { + Status = "Error", + Direction = body.Direction, + SourcePath = body.SourcePath, + TargetPath = body.TargetPath ?? body.SourcePath, + Error = "No response from mirror handler — is the mesh hub reachable and AddPersistence configured?", + }; + return Results.Content(JsonSerializer.Serialize(result, sessionHub.JsonSerializerOptions), "application/json"); + } + + private static IResult HandleNavigateTo(HttpContext http, IOptions? mcp, NavigateBody body) + { + var baseUrl = ResolveBaseUrl(http, mcp); + var resolved = MeshOperations.ResolvePath(body.Path).TrimStart('/'); + return Results.Json(new { url = $"{baseUrl}/{resolved}" }); + } + + private static IResult HandleBaseUrl(HttpContext http, IOptions? mcp) => + Results.Json(new { url = ResolveBaseUrl(http, mcp) }); + + private static async Task HandleUpload(HttpContext http, IMessageHub rootHub, CancellationToken ct) + { + if (!http.Request.HasFormContentType) + return Results.BadRequest(new { error = "Content-Type must be multipart/form-data." }); + + var form = await http.Request.ReadFormAsync(ct); + var path = form["path"].FirstOrDefault(); + var file = form.Files.FirstOrDefault(); + if (string.IsNullOrWhiteSpace(path)) + return Results.BadRequest(new { error = "Form field 'path' is required." }); + if (file is null || file.Length == 0) + return Results.BadRequest(new { error = "Form file 'file' is required." }); + + using var ms = new MemoryStream(); + await using (var stream = file.OpenReadStream()) + await stream.CopyToAsync(ms, ct); + + var sessionHub = ResolveSession(http, rootHub); + var ops = new MeshOperations(sessionHub); + var result = await ops.Upload(path, ms.ToArray()).FirstAsync().ToTask(ct); + return Results.Content(result, "application/json"); + } + + /// + /// Registers the bits the REST module needs that aren't already in DI from the + /// MCP wiring: lift the multipart upload size cap (default 30 MB is too small + /// for typical document uploads) and ensure is + /// bound (shared with MCP — same Mcp__BaseUrl env var). + /// + public static IServiceCollection AddMeshApi(this IServiceCollection services) + { + services.Configure(o => + { + // 200 MB — generous but bounded. Matches the working assumption that + // document / image / spreadsheet uploads are the common case; binaries + // larger than this should go through a different ingest path. + o.MultipartBodyLengthLimit = 200L * 1024 * 1024; + o.ValueLengthLimit = int.MaxValue; + o.MultipartHeadersLengthLimit = int.MaxValue; + }); + + // McpConfiguration is already bound by AddMeshMcp(); BindConfiguration is + // idempotent so a second call is harmless if the MCP wiring is absent. + services.AddOptions().BindConfiguration("Mcp"); + + return services; + } + + private static IMessageHub ResolveSession(HttpContext http, IMessageHub rootHub) + { + var logger = http.RequestServices.GetRequiredService().CreateLogger(typeof(MeshApiEndpoints)); + return SessionHubResolver.ResolveSessionHub(rootHub, http, "api", logger); + } + + private static async Task RunString( + HttpContext http, + IMessageHub rootHub, + CancellationToken ct, + Func> work) + { + var sessionHub = ResolveSession(http, rootHub); + var ops = new MeshOperations(sessionHub); + var result = await work(ops).FirstAsync().ToTask(ct); + // MeshOperations returns either a JSON document or an "Error: …" sentinel string. + // Both are safe to ship as application/json — the error string is just a JSON-quoted + // value the client can branch on (mirrors the MCP-tool contract). + return Results.Content(result, "application/json"); + } + + private static string ResolveBaseUrl(HttpContext http, IOptions? mcp) + { + var configured = mcp?.Value.BaseUrl; + if (!string.IsNullOrEmpty(configured)) + return configured.TrimEnd('/'); + var req = http.Request; + return $"{req.Scheme}://{req.Host.Value}".TrimEnd('/'); + } + + // Request DTOs — the framework's System.Text.Json infrastructure binds JSON bodies + // by property name (case-insensitive). All optional fields default to null / false. + public record GetBody(string Path); + public record SearchBody(string Query, string? BasePath); + public record CreateBody(string Node); + public record UpdateBody(string Nodes); + public record PatchBody(string Path, string Fields); + public record DeleteBody(string Paths); + public record MoveBody(string SourcePath, string TargetPath); + public record CopyBody(string SourcePath, string TargetNamespace, bool Force = false); + public record PathBody(string Path); + public record ExecuteScriptBody(string Path, int? TimeoutSeconds); + public record NavigateBody(string Path); +} diff --git a/memex/Memex.Portal.Shared/App.razor b/memex/Memex.Portal.Shared/App.razor index ba7f6d51a..58898e0f0 100644 --- a/memex/Memex.Portal.Shared/App.razor +++ b/memex/Memex.Portal.Shared/App.razor @@ -21,6 +21,26 @@ + + @* + Load Monaco early, in parallel with HTML parsing, and expose a readiness + Promise that Blazor.start() awaits below. BlazorMonaco 3.4 ships Monaco + 0.54 whose `editor/editor.main.js` is a tiny AMD stub — the real 3.6MB + bundle (editor.api-*.js) is fetched asynchronously by the AMD loader. + If Blazor activates a circuit and renders a + before that async load finishes, BlazorMonaco's jsInterop calls + `monaco.editor.create(...)` while `monaco` is undefined, the circuit + crashes, and the user sees a broken page. Using require([...], cb) to + gate Blazor.start removes the race. + *@ + + + + Memex Portal @if (!string.IsNullOrEmpty(aiConnectionString)) @@ -36,11 +56,142 @@ - - - + + @* + Custom Blazor Server reconnect UI. Blazor adds one of these classes to + the container while handling a lost circuit: + - components-reconnect-show → attempting to reconnect + - components-reconnect-hide → hidden (reconnected) + - components-reconnect-retrying → retry in flight + - components-reconnect-failed → retries exhausted + - components-reconnect-rejected → server doesn't know this circuit (redeploy) + A deploy invalidates every circuit on the server, so stale clients hit + `rejected` (404 on reconnect). Instead of keeping the user stuck on a + generic "Reconnecting…" modal for minutes, auto-reload on the terminal + states so the user is back on a fresh circuit within a few seconds. + *@ +
+
+ +
Reconnecting…
+
The server was updated. Reloading the page to pick up the latest version.
+
+
+ + + - + + diff --git a/memex/Memex.Portal.Shared/Authentication/ApiTokenAuthenticationHandler.cs b/memex/Memex.Portal.Shared/Authentication/ApiTokenAuthenticationHandler.cs index 26510edd5..5cd18497f 100644 --- a/memex/Memex.Portal.Shared/Authentication/ApiTokenAuthenticationHandler.cs +++ b/memex/Memex.Portal.Shared/Authentication/ApiTokenAuthenticationHandler.cs @@ -1,5 +1,8 @@ +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; using System.Security.Claims; using System.Text.Encodings.Web; +using MeshWeaver.Messaging; using Microsoft.AspNetCore.Authentication; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -32,14 +35,63 @@ protected override async Task HandleAuthenticateAsync() return AuthenticateResult.NoResult(); var tokenService = serviceProvider.GetRequiredService(); - var apiToken = await tokenService.ValidateTokenAsync(rawToken); + // HTTP boundary — bridge IObservable to Task once. The service exposes + // IObservable per the "no async in hub-reachable code" rule. + var apiToken = await tokenService.ValidateToken(rawToken).FirstAsync().ToTask(); if (apiToken == null) return AuthenticateResult.Fail("Invalid or expired API token"); + var claims = BuildClaims(apiToken).ToList(); + + // Enrich with DB-resolved AccessAssignment roles so Bearer requests + // see the same role set as cookie/OAuth sessions. Without this, the + // principal only carries roles that were stamped on the API token at + // creation time — any AccessAssignment granted to the user later + // (e.g. an admin promotion after the token was minted) would silently + // not apply for MCP requests, even though the same user logging in + // through the browser would see them. Live mesh query, bounded so a + // wedged data source can't slow auth. + try + { + var dbRoles = await UserRoleResolver.LoadDbRolesAsync(serviceProvider, apiToken.UserId); + foreach (var role in dbRoles) + { + if (string.IsNullOrEmpty(role)) continue; + if (claims.Any(c => c.Type == ClaimTypes.Role && c.Value == role)) + continue; + claims.Add(new Claim(ClaimTypes.Role, role)); + } + } + catch + { + // Role enrichment is best-effort; the token's own Roles still apply. + } + + var identity = new ClaimsIdentity(claims, SchemeName); + var principal = new ClaimsPrincipal(identity); + var ticket = new AuthenticationTicket(principal, SchemeName); + + // Login tracking lives in UserContextMiddleware so it fires for both + // Bearer and cookie authentication on the same code path — see + // UserContextMiddleware.TrackLogin. + return AuthenticateResult.Success(ticket); + } + + /// + /// Builds the claim list for an authenticated API token. Public + static + /// so unit tests can assert the claim shape (in particular: that + /// become + /// claims) without needing an HTTP host. + /// Mirrors what UserContextMiddleware.ExtractUserContext() reads + /// back into . + /// + public static IReadOnlyList BuildClaims(MeshWeaver.Mesh.Security.ApiToken apiToken) + { // Build claims matching UserContextMiddleware.ExtractUserContext(): // ObjectId = preferred_username // Name = ClaimTypes.Name or "name" // Email = ClaimTypes.Email or "email" + // Roles = each ClaimTypes.Role claim → AccessContext.Roles var claims = new List { new("preferred_username", apiToken.UserId), @@ -51,10 +103,21 @@ protected override async Task HandleAuthenticateAsync() new("token_label", apiToken.Label), }; - var identity = new ClaimsIdentity(claims, SchemeName); - var principal = new ClaimsPrincipal(identity); - var ticket = new AuthenticationTicket(principal, SchemeName); + // Stamp the token's Roles list as ClaimTypes.Role claims. Without + // this, UserContextMiddleware sets AccessContext.Roles to an empty + // list and SecurityService.GetEffectivePermissions can't resolve + // claim-based Admin — every API-token request that depended on a + // role grant rather than a static AccessAssignment got denied. + // The token's Roles surface is exactly the right vehicle: the + // creator chose them at token creation; the validator preserves + // them through ValidateTokenResponse.Roles; the auth handler + // mints them onto the principal here. + foreach (var role in apiToken.Roles) + { + if (!string.IsNullOrEmpty(role)) + claims.Add(new Claim(ClaimTypes.Role, role)); + } - return AuthenticateResult.Success(ticket); + return claims; } } diff --git a/memex/Memex.Portal.Shared/Authentication/ApiTokenController.cs b/memex/Memex.Portal.Shared/Authentication/ApiTokenController.cs index fe49f8f7f..fa71fb738 100644 --- a/memex/Memex.Portal.Shared/Authentication/ApiTokenController.cs +++ b/memex/Memex.Portal.Shared/Authentication/ApiTokenController.cs @@ -1,4 +1,8 @@ -using System.Security.Claims; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using System.Threading; +using MeshWeaver.Blazor.Infrastructure; // PortalApplication +using MeshWeaver.Messaging; // AccessService / AccessContext using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.DependencyInjection; @@ -15,79 +19,110 @@ namespace Memex.Portal.Shared.Authentication; public class ApiTokenController(IServiceProvider serviceProvider) : ControllerBase { private ApiTokenService tokenService => serviceProvider.GetRequiredService(); + + /// + /// The mesh-resolved identity for this request, as stamped on the portal + /// hub's by UserContextMiddleware. + /// + /// 🚨 We deliberately do NOT read preferred_username off the claims + /// principal here. Entra/OIDC fill that claim with the UPN, which is the + /// user's email (e.g. rbuergi@systemorph.com). The mesh + /// partition key is the User node's Id (e.g. rbuergi), and the + /// middleware already does the email→User resolution + normalisation (and + /// refuses email-shaped ids). Passing the raw email through as the token's + /// userId routed the token node AND its _Access self-scope into a + /// non-existent {email} partition — which 401'd every freshly-minted + /// token once the router stopped lazy-creating schemas. Reading the + /// already-resolved context guarantees the token lands in exactly the + /// partition the user's other data lives in. + /// + /// + private AccessContext? CurrentUser => + serviceProvider.GetRequiredService() + .Hub.ServiceProvider.GetRequiredService() + .Context; + + /// + /// The mesh User.Id for the current request, or null if the request has no + /// resolved (non-email) mesh identity — in which case token operations must + /// be refused rather than routed to a parallel {email} partition. + /// + private static string? MeshUserId(AccessContext? user) + { + var id = user?.ObjectId; + return string.IsNullOrEmpty(id) || id.Contains('@') ? null : id; + } + /// /// Creates a new API token. Returns the raw token once — it cannot be retrieved again. /// [HttpPost] - public async Task CreateToken([FromBody] CreateTokenRequest request) + public Task CreateToken([FromBody] CreateTokenRequest request, CancellationToken ct) { - var userId = User.FindFirstValue("preferred_username") - ?? User.FindFirstValue(ClaimTypes.NameIdentifier) - ?? ""; - var userName = User.FindFirstValue(ClaimTypes.Name) - ?? User.FindFirstValue("name") - ?? ""; - var userEmail = User.FindFirstValue(ClaimTypes.Email) - ?? User.FindFirstValue("email") - ?? ""; + var user = CurrentUser; + var userId = MeshUserId(user); + if (userId is null) + return Task.FromResult(Unauthorized("No user identity found")); - if (string.IsNullOrEmpty(userId)) - return Unauthorized("No user identity found"); + var userName = user!.Name ?? ""; + var userEmail = user.Email ?? ""; DateTimeOffset? expiresAt = request.ExpiresInDays > 0 ? DateTimeOffset.UtcNow.AddDays(request.ExpiresInDays.Value) : null; - var (rawToken, node) = await tokenService.CreateTokenAsync( - userId, userName, userEmail, request.Label ?? "API Token", expiresAt); + var label = request.Label ?? "API Token"; - return Ok(new CreateTokenResponse - { - RawToken = rawToken, - NodePath = node.Path, - Label = request.Label ?? "API Token", - CreatedAt = DateTimeOffset.UtcNow, - ExpiresAt = expiresAt, - }); + // No await: pull IObservable up to the controller's return type. The + // single bridge to Task happens at .ToTask(ct) — passing the + // request's cancellation token so a client disconnect tears down the + // reactive subscription instead of orphaning it. + return tokenService.CreateToken(userId, userName, userEmail, label, expiresAt) + .Select(creation => (IActionResult)Ok(new CreateTokenResponse + { + RawToken = creation.RawToken, + NodePath = creation.Node.Path, + Label = label, + CreatedAt = DateTimeOffset.UtcNow, + ExpiresAt = expiresAt, + })) + .FirstAsync() + .ToTask(ct); } /// /// Lists all tokens for the current user. Never returns raw tokens. /// [HttpGet] - public async Task ListTokens() + public Task ListTokens(CancellationToken ct) { - var userId = User.FindFirstValue("preferred_username") - ?? User.FindFirstValue(ClaimTypes.NameIdentifier) - ?? ""; + var userId = MeshUserId(CurrentUser); + if (userId is null) + return Task.FromResult(Unauthorized("No user identity found")); - if (string.IsNullOrEmpty(userId)) - return Unauthorized("No user identity found"); - - var tokens = await tokenService.GetTokensForUserAsync(userId); - return Ok(tokens); + return tokenService.GetTokensForUser(userId) + .Select(tokens => (IActionResult)Ok(tokens)) + .FirstAsync() + .ToTask(ct); } /// /// Revokes a token by its node path. /// [HttpDelete("{*nodePath}")] - public async Task RevokeToken(string nodePath) + public Task RevokeToken(string nodePath, CancellationToken ct) { - // Verify the token belongs to the current user - var userId = User.FindFirstValue("preferred_username") - ?? User.FindFirstValue(ClaimTypes.NameIdentifier) - ?? ""; - - if (string.IsNullOrEmpty(userId)) - return Unauthorized("No user identity found"); - - var tokens = await tokenService.GetTokensForUserAsync(userId); - if (!tokens.Any(t => t.NodePath == nodePath)) - return NotFound("Token not found or does not belong to you"); + var userId = MeshUserId(CurrentUser); + if (userId is null) + return Task.FromResult(Unauthorized("No user identity found")); - var success = await tokenService.RevokeTokenAsync(nodePath); - return success ? Ok() : NotFound(); + return tokenService.GetTokensForUser(userId) + .SelectMany(tokens => tokens.Any(t => t.NodePath == nodePath) + ? tokenService.RevokeToken(nodePath) + .Select(success => success ? (IActionResult)Ok() : NotFound()) + : Observable.Return(NotFound("Token not found or does not belong to you"))) + .FirstAsync() + .ToTask(ct); } } diff --git a/memex/Memex.Portal.Shared/Authentication/ApiTokenService.cs b/memex/Memex.Portal.Shared/Authentication/ApiTokenService.cs index 43a1686ee..e12d71887 100644 --- a/memex/Memex.Portal.Shared/Authentication/ApiTokenService.cs +++ b/memex/Memex.Portal.Shared/Authentication/ApiTokenService.cs @@ -1,6 +1,9 @@ using System.Reactive.Linq; using System.Runtime.CompilerServices; using System.Security.Cryptography; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; using MeshWeaver.Mesh; using MeshWeaver.Mesh.Security; using MeshWeaver.Mesh.Services; @@ -14,10 +17,22 @@ namespace Memex.Portal.Shared.Authentication; /// /// Service for creating, validating, and revoking API tokens. -/// Tokens are stored as MeshNodes with nodeType "ApiToken". -/// Raw tokens are never persisted — only their SHA-256 hash. +/// Tokens are stored as MeshNodes with nodeType "ApiToken". Raw tokens +/// are never persisted — only their SHA-256 hash. +/// +/// +/// 🚨 No async / Task / FromAsync / await anywhere in this file. Every +/// reachable method returns and the chain +/// stays observable end-to-end. Reads of known paths go through +/// hub.GetMeshNode(path) (one-shot) or +/// workspace.GetMeshNodeStream(path) (live); listings go through +/// workspace.GetQuery(id, queries...) (synced + path-keyed dedup). +/// QueryAsync / iteration is forbidden +/// in this file per Doc/Architecture/AsynchronousCalls.md and +/// Doc/Architecture/SyncedMeshNodeQueries.md. +/// /// -internal class ApiTokenService(IMeshService nodeFactory, IMeshService meshQuery, IMessageHub hub, ILogger logger) +internal class ApiTokenService(IMeshService nodeFactory, IMessageHub hub, ILogger logger) { private const string TokenPrefix = "mw_"; private const int TokenByteLength = 32; @@ -39,375 +54,396 @@ public IObservable CreateToken( var hash = HashToken(rawToken); var hashPrefix = hash[..12]; - var apiToken = new ApiToken - { - TokenHash = hash, - UserId = userId, - UserName = userName, - UserEmail = userEmail, - Label = label, - CreatedAt = DateTimeOffset.UtcNow, - ExpiresAt = expiresAt, - }; - - var userTokenNamespace = $"User/{userId}/{ApiTokenNamespace}"; - var userNode = new MeshNode(hashPrefix, userTokenNamespace) - { - Name = $"API Token: {label}", - NodeType = NodeTypeApiToken, - State = MeshNodeState.Active, - Content = apiToken, - }; + // Per-user partition layout (Repair v10): tokens live at + // {userId}/ApiToken/{hashPrefix}, NOT under User/{userId}/ApiToken. + // The global ApiToken/{hashPrefix} index entry routes incoming + // bearer tokens to the right user-scoped node at validation time. + var userTokenNamespace = $"{userId}/{ApiTokenNamespace}"; + var assignmentPath = $"{userId}/_Access/{userId}_Access"; - var accessService = hub.ServiceProvider.GetService(); + var rolesObs = ResolveSelfScopeRoles(assignmentPath); - // Reactive chain: create user node (as current user), then create index node - // (promoted to System identity). Emits the raw token + created node once both - // writes commit, or errors on the first failure. - return nodeFactory.CreateNode(userNode) - .SelectMany(created => + return rolesObs.SelectMany(capturedRoles => + { + var apiToken = new ApiToken { - var indexNode = new MeshNode(hashPrefix, ApiTokenNamespace) + TokenHash = hash, + UserId = userId, + UserName = userName, + UserEmail = userEmail, + Label = label, + CreatedAt = DateTimeOffset.UtcNow, + ExpiresAt = expiresAt, + Roles = capturedRoles, + }; + + var userNode = new MeshNode(hashPrefix, userTokenNamespace) + { + Name = $"API Token: {label}", + NodeType = NodeTypeApiToken, + State = MeshNodeState.Active, + MainNode = userId, + Content = apiToken, + }; + + var accessService = hub.ServiceProvider.GetService(); + + return nodeFactory.CreateNode(userNode) + .SelectMany(created => { - Name = $"API Token: {label}", - NodeType = NodeTypeApiToken, - State = MeshNodeState.Active, - Content = new ApiTokenIndex + var indexNode = new MeshNode(hashPrefix, ApiTokenNamespace) { - TokenHash = hash, - TokenPath = created.Path, - }, - }; - - // Index writes require System identity (users don't have Create on ApiToken/). - IObservable indexObs; - if (accessService != null) - { - using (accessService.SwitchAccessContext( - new AccessContext { ObjectId = WellKnownUsers.System, Name = "system-security" })) + Name = $"API Token: {label}", + NodeType = NodeTypeApiToken, + State = MeshNodeState.Active, + Content = new ApiTokenIndex + { + TokenHash = hash, + TokenPath = created.Path, + }, + }; + + // Index writes require System identity — the global ApiToken/ + // namespace is a separately-gated partition for security + // infrastructure that ordinary users don't have Create on. + // See git history on this file for the SwitchAccessContext- + // outside-Defer bug that this lambda layout fixes (System + // context must be active during CaptureContext at Subscribe + // time, not when the outer using-block returned). + IObservable indexObs; + if (accessService != null) + { + indexObs = Observable.Defer(() => + { + var disp = accessService.SwitchAccessContext( + new AccessContext { ObjectId = WellKnownUsers.System, Name = "system-security" }); + return nodeFactory.CreateNode(indexNode).Finally(() => disp.Dispose()); + }); + } + else { indexObs = nodeFactory.CreateNode(indexNode); } - } - else - { - indexObs = nodeFactory.CreateNode(indexNode); - } - logger.LogInformation("Creating API token {Label} for user {UserId} (hash prefix {HashPrefix})", - label, userId, hashPrefix); + logger.LogInformation("Creating API token {Label} for user {UserId} (hash prefix {HashPrefix})", + label, userId, hashPrefix); - return indexObs.Select(_ => new TokenCreationResult(rawToken, created)); - }); + return indexObs.Select(_ => new TokenCreationResult(rawToken, created)); + }); + }); } - public async Task<(string RawToken, MeshNode Node)> CreateTokenAsync( - string userId, string userName, string userEmail, string label, DateTimeOffset? expiresAt = null) + /// + /// Reads the user's self-scope at + /// {userId}/_Access/{userId}_Access and emits the (non-denied) + /// role IDs assigned there. Pure observable composition — one-shot + /// under System + /// identity, then .Select. Emits an empty array on missing + /// assignment or read failure (the issued token still has identity + /// but no role grants — correct outcome). + /// + private IObservable> ResolveSelfScopeRoles(string assignmentPath) { - var rawBytes = RandomNumberGenerator.GetBytes(TokenByteLength); - var rawToken = TokenPrefix + Convert.ToBase64String(rawBytes) - .Replace("+", "-").Replace("/", "_").TrimEnd('='); - - var hash = HashToken(rawToken); - var hashPrefix = hash[..12]; + var accessService = hub.ServiceProvider.GetService(); - var apiToken = new ApiToken - { - TokenHash = hash, - UserId = userId, - UserName = userName, - UserEmail = userEmail, - Label = label, - CreatedAt = DateTimeOffset.UtcNow, - ExpiresAt = expiresAt, - }; - - // Store the full token under the user's namespace - var userTokenNamespace = $"User/{userId}/{ApiTokenNamespace}"; - var userNode = new MeshNode(hashPrefix, userTokenNamespace) - { - Name = $"API Token: {label}", - NodeType = NodeTypeApiToken, - State = MeshNodeState.Active, - Content = apiToken, - }; - - var created = await nodeFactory.CreateNodeAsync(userNode); - - // Store a lightweight index pointer at the original location for O(1) validation lookup. - // Promote to System identity — users don't have Create permission on the top-level - // ApiToken/ namespace, but this index is infrastructure (not user data) so it must - // always be creatable as part of token issuance. - var indexNode = new MeshNode(hashPrefix, ApiTokenNamespace) - { - Name = $"API Token: {label}", - NodeType = NodeTypeApiToken, - State = MeshNodeState.Active, - Content = new ApiTokenIndex + // Observable.Using ties the AsyncLocal System scope's lifetime to + // the Subscribe of the inner observable, not to the lambda body's + // return — same shape used by ApiTokenNodeType.HandleValidateToken + // for the same reason (Defer-style subscribe-time capture). + var readUnderSystem = accessService != null + ? Observable.Using( + () => accessService.ImpersonateAsSystem(), + _ => hub.GetMeshNode(assignmentPath, TimeSpan.FromSeconds(5))) + : hub.GetMeshNode(assignmentPath, TimeSpan.FromSeconds(5)); + + return readUnderSystem + .Select(node => { - TokenHash = hash, - TokenPath = created.Path, - }, - }; - - var accessService = hub.ServiceProvider.GetService(); - if (accessService != null) - { - using (accessService.SwitchAccessContext(new AccessContext { ObjectId = WellKnownUsers.System, Name = "system-security" })) + var assignment = node?.Content as AccessAssignment ?? ExtractAccessAssignment(node); + if (assignment is null) + return (IReadOnlyCollection)Array.Empty(); + return assignment.Roles + .Where(r => !r.Denied && !string.IsNullOrEmpty(r.Role)) + .Select(r => r.Role) + .Distinct() + .ToArray(); + }) + .Catch, Exception>(ex => { - await nodeFactory.CreateNodeAsync(indexNode); - } - } - else - { - await nodeFactory.CreateNodeAsync(indexNode); - } - - logger.LogInformation("Created API token {Label} for user {UserId} (hash prefix {HashPrefix})", - label, userId, hashPrefix); - - return (rawToken, created); + logger.LogWarning(ex, + "Failed to resolve self-scope roles from {Path} for token creation; continuing with empty role set", + assignmentPath); + return Observable.Return>(Array.Empty()); + }); } /// - /// Queries nodes using the system identity to bypass access control. - /// ApiTokenService is infrastructure code that needs unrestricted read access. + /// Reactive token validation. Reads index node at + /// ApiToken/{hashPrefix} via hub.GetMeshNode (one-shot, + /// authoritative — never QueryAsync for a known path per + /// Doc/Architecture/AsynchronousCalls.md); when the index + /// points at a user-scoped token, follows the pointer with a second + /// one-shot read. The chain is fully observable — no + /// FromAsync, no FirstOrDefaultAsync.AsTask(), no + /// await. /// - private IAsyncEnumerable QueryAsSystemAsync(string query, CancellationToken ct = default) - => meshQuery.QueryAsync( - MeshQueryRequest.FromQuery(query, WellKnownUsers.System), ct: ct); - - public async Task ValidateTokenAsync(string rawToken) + public IObservable ValidateToken(string rawToken) { if (string.IsNullOrEmpty(rawToken) || !rawToken.StartsWith(TokenPrefix)) - return null; + return Observable.Return(null); var hash = HashToken(rawToken); var hashPrefix = hash[..12]; var indexPath = $"{ApiTokenNamespace}/{hashPrefix}"; - var indexNode = await QueryAsSystemAsync($"path:{indexPath}").FirstOrDefaultAsync(); - if (indexNode == null) - return null; + return ReadAsSystem(indexPath) + .SelectMany(indexNode => + { + if (indexNode == null) + return Observable.Return<(MeshNode? node, ApiToken? token)>((null, null)); - // Follow index pointer to the full token, or handle legacy tokens directly - MeshNode? tokenNode; - ApiToken? apiToken; - var index = indexNode.Content as ApiTokenIndex ?? ExtractApiTokenIndex(indexNode); - if (index != null) - { - // New format: index pointer -> follow to user namespace - if (!string.Equals(index.TokenHash, hash, StringComparison.OrdinalIgnoreCase)) - return null; - tokenNode = await QueryAsSystemAsync($"path:{index.TokenPath}").FirstOrDefaultAsync(); - apiToken = tokenNode?.Content as ApiToken ?? ExtractApiToken(tokenNode); - } - else - { - // Legacy format: full ApiToken at index path - tokenNode = indexNode; - apiToken = indexNode.Content as ApiToken ?? ExtractApiToken(indexNode); - } + var index = indexNode.Content as ApiTokenIndex ?? ExtractApiTokenIndex(indexNode); + if (index != null) + { + if (!string.Equals(index.TokenHash, hash, StringComparison.OrdinalIgnoreCase)) + return Observable.Return<(MeshNode? node, ApiToken? token)>((null, null)); + return ReadAsSystem(index.TokenPath) + .Select(tn => ( + node: tn, + token: (tn?.Content as ApiToken) ?? ExtractApiToken(tn))); + } + // Legacy format: full ApiToken at index path. + return Observable.Return(( + node: (MeshNode?)indexNode, + token: (indexNode.Content as ApiToken) ?? ExtractApiToken(indexNode))); + }) + .Select(t => FinalizeToken(t.node, t.token, hash, hashPrefix)); + } + + private IObservable ReadAsSystem(string path) + { + var accessService = hub.ServiceProvider.GetService(); + return accessService != null + ? Observable.Using( + () => accessService.ImpersonateAsSystem(), + _ => hub.GetMeshNode(path, TimeSpan.FromSeconds(5))) + : hub.GetMeshNode(path, TimeSpan.FromSeconds(5)); + } + private ApiToken? FinalizeToken(MeshNode? tokenNode, ApiToken? apiToken, string hash, string hashPrefix) + { if (apiToken == null) return null; - if (!string.Equals(apiToken.TokenHash, hash, StringComparison.OrdinalIgnoreCase)) return null; - if (apiToken.IsRevoked) { logger.LogDebug("Token {HashPrefix} is revoked", hashPrefix); return null; } - if (apiToken.ExpiresAt.HasValue && apiToken.ExpiresAt.Value < DateTimeOffset.UtcNow) { logger.LogDebug("Token {HashPrefix} has expired", hashPrefix); return null; } - // Update LastUsedAt (fire-and-forget, non-critical) - try + // Update LastUsedAt via the canonical workspace remote stream — + // fire-and-forget (non-critical telemetry). Subscribe is mandatory + // because Update is cold; the empty error handler keeps the cold + // observable's GC-time fire-and-forget warning quiet on writes + // that hit a deleted node. + if (tokenNode != null) { - var updated = apiToken with { LastUsedAt = DateTimeOffset.UtcNow }; - var updatedNode = tokenNode! with { Content = updated }; - hub.Post(new UpdateNodeRequest(updatedNode)); - } - catch (Exception ex) - { - logger.LogDebug(ex, "Failed to update LastUsedAt for token {HashPrefix}", hashPrefix); + hub.GetWorkspace() + .GetMeshNodeStream(tokenNode.Path) + .Update(node => node with { Content = (node.Content as ApiToken ?? apiToken) with { LastUsedAt = DateTimeOffset.UtcNow } }) + .Subscribe(_ => { }, _ => { }); } return apiToken; } /// - /// Reactive token revocation — marks the token as revoked via - /// and removes the index pointer. - /// No async/await. Emits true on success, false if token not found, errors on failure. + /// Reactive token revocation. Writes the IsRevoked flag through + /// workspace.GetMeshNodeStream(path).Update(...) — the + /// canonical remote-stream write per + /// Doc/Architecture/AsynchronousCalls.md. No + /// UpdateNodeRequest forwarding (the previous, now-retired shape + /// timed out in distributed deployments when the per-node hub's + /// forwarded request didn't get a response within ~30s). + /// + /// The global index entry is hard-deleted as a fire-and-forget + /// side effect — the index miss is a defense-in-depth gate on top of + /// the authoritative IsRevoked flag, not a primary requirement + /// for the revoke to be effective. /// - public IObservable RevokeToken(string tokenNodePath) => - Observable.FromAsync(() => meshQuery.QueryAsync( - MeshQueryRequest.FromQuery($"path:{tokenNodePath}", WellKnownUsers.System)) - .FirstOrDefaultAsync().AsTask()) - .SelectMany(node => - { - var apiToken = node?.Content as ApiToken ?? ExtractApiToken(node); - if (node == null || apiToken == null) - return Observable.Return(false); - - var revoked = apiToken with { IsRevoked = true }; - var updatedNode = node with { Content = revoked }; - - // Delete index entry if distinct from the main node. - if (apiToken.TokenHash.Length >= 12) - { - var hashPrefix = apiToken.TokenHash[..12]; - var indexPath = $"{ApiTokenNamespace}/{hashPrefix}"; - if (tokenNodePath != indexPath) - hub.Post(new DeleteNodeRequest(indexPath)); - } + public IObservable RevokeToken(string tokenNodePath) + { + var workspace = hub.GetWorkspace(); + var indexPath = DeriveIndexPath(tokenNodePath); - logger.LogInformation("Revoking API token at {Path}", tokenNodePath); - return nodeFactory.UpdateNode(updatedNode).Select(_ => true); - }); + logger.LogInformation("Revoking API token at {Path}", tokenNodePath); - /// - /// Reactive hard-delete — removes both the primary token node and its index entry. - /// No async/await. Emits true on success, errors on failure. - /// - public IObservable DeleteToken(string tokenNodePath) => - Observable.FromAsync(() => meshQuery.QueryAsync( - MeshQueryRequest.FromQuery($"path:{tokenNodePath}", WellKnownUsers.System)) - .FirstOrDefaultAsync().AsTask()) - .SelectMany(node => + var primary = workspace.GetMeshNodeStream(tokenNodePath) + .Update(current => { - var apiToken = node?.Content as ApiToken ?? ExtractApiToken(node); - var hashPrefix = apiToken?.TokenHash is { Length: >= 12 } h ? h[..12] : null; - - if (!string.IsNullOrEmpty(hashPrefix)) - { - var indexPath = $"{ApiTokenNamespace}/{hashPrefix}"; - if (indexPath != tokenNodePath) - hub.Post(new DeleteNodeRequest(indexPath)); - } - - logger.LogInformation("Deleting API token at {Path}", tokenNodePath); - return nodeFactory.DeleteNode(tokenNodePath); + var token = current.Content as ApiToken ?? ExtractApiToken(current); + if (token == null) return current; + // Flip IsRevoked on the live node — validation reads the node fresh (no cache), + // so the revoke takes effect immediately. + return current with { Content = token with { IsRevoked = true } }; + }) + .Do(updatedNode => + { + // Force the per-node hub to persist the patched node. The + // sync-protocol path (workspace.GetMeshNodeStream(remote) + // .Update) updates the mesh-hub-side stream and emits a + // DataChangeRequest to the per-node hub, but the per-node + // hub's data source `saveSub` only fires on `ownStream` + // emissions — and those don't fire for sync-driven changes, + // so persistence never sees the IsRevoked=true update. The + // SaveMeshNodeRequest below routes to the per-node hub's + // HandleSaveMeshNode which writes through IStorageService + // (firing IDataChangeNotifier.Updated, so the synced + // GetTokensForUser view picks up the change). + hub.Post(new SaveMeshNodeRequest(updatedNode), + o => o.WithTarget(new Address(tokenNodePath))); + }) + .Select(_ => true) + .Catch(ex => + { + logger.LogWarning(ex, "RevokeToken failed for {Path}", tokenNodePath); + return Observable.Return(false); }); + // Chain the global-index delete into the returned observable rather + // than firing a separate Subscribe — see the matching comment in + // DeleteToken. A missing index entry is fine: the Catch returns false + // and the primary revoke result wins. + if (indexPath == null || indexPath == tokenNodePath) + return primary; + + return primary.SelectMany(result => + nodeFactory.DeleteNode(indexPath) + .Catch(_ => Observable.Return(false)) + .Select(_ => result)); + } + /// - /// Hard-deletes a token node (and its index entry, if present). - /// Used to clean up revoked/expired tokens from the UI list. + /// Reactive hard-delete. Removes the user-scoped token node and the + /// global index entry (fire-and-forget). The user-scoped delete goes + /// through ; this is the + /// authoritative removal and the only outcome the caller observes. /// - public async Task DeleteTokenAsync(string tokenNodePath) + public IObservable DeleteToken(string tokenNodePath) { - // Look up the node to find the hash prefix so we can clean the index too. - var node = await QueryAsSystemAsync($"path:{tokenNodePath}").FirstOrDefaultAsync(); - var apiToken = node?.Content as ApiToken ?? ExtractApiToken(node); - var hashPrefix = apiToken?.TokenHash is { Length: >= 12 } h ? h[..12] : null; + var indexPath = DeriveIndexPath(tokenNodePath); - // Delete the primary token node (under User/{userId}/ApiToken/...) - hub.Post(new DeleteNodeRequest(tokenNodePath)); + logger.LogInformation("Deleting API token at {Path}", tokenNodePath); - // Delete the index pointer at the top-level ApiToken namespace. - if (!string.IsNullOrEmpty(hashPrefix)) - { - var indexPath = $"{ApiTokenNamespace}/{hashPrefix}"; - if (indexPath != tokenNodePath) - hub.Post(new DeleteNodeRequest(indexPath)); - } + var primary = nodeFactory.DeleteNode(tokenNodePath) + .Select(_ => true) + .Catch(ex => + { + logger.LogWarning(ex, "DeleteToken failed for {Path}", tokenNodePath); + return Observable.Return(false); + }); - logger.LogInformation("Deleted API token at {Path}", tokenNodePath); + // Chain the index-entry delete into the returned observable rather than + // firing a separate Subscribe. The previous shape leaked a pending + // hub.Observe callback past test dispose — the response arrives only + // after routing surfaces NotFound (~15ms+) but the test's await + // completes faster, so the dispose-time Quiescing watchdog flags the + // pending callback as a leaked subscription. Chaining here also makes + // a missing-index case (token already gone) a non-failure of the whole + // operation: the inner Catch swallows it and the primary result wins. + if (indexPath == null || indexPath == tokenNodePath) + return primary; + + return primary.SelectMany(result => + nodeFactory.DeleteNode(indexPath) + .Catch(_ => Observable.Return(false)) + .Select(_ => result)); } - public async Task RevokeTokenAsync(string tokenNodePath) + /// + /// Live list of the user's tokens via the canonical synced query + /// (workspace.GetQuery). The synced query gives us path-keyed + /// dedup across the user-scope and legacy global namespaces, + /// all-Initial gating, and provider fan-out — see + /// Doc/Architecture/SyncedMeshNodeQueries.md. The cache id is + /// per-user so re-mounts (settings tab re-render) reuse the upstream + /// subscription instead of cycling Initial waves. + /// + public IObservable> GetTokensForUser(string userId) { - var node = await QueryAsSystemAsync($"path:{tokenNodePath}").FirstOrDefaultAsync(); - if (node == null) - return false; - - var apiToken = node.Content as ApiToken ?? ExtractApiToken(node); - if (apiToken == null) - return false; - - var revoked = apiToken with { IsRevoked = true }; - var updatedNode = node with { Content = revoked }; - hub.Post(new UpdateNodeRequest(updatedNode)); - - // Also revoke the index node at ApiToken/{hashPrefix} if it exists - if (apiToken.TokenHash.Length >= 12) - { - var hashPrefix = apiToken.TokenHash[..12]; - var indexPath = $"{ApiTokenNamespace}/{hashPrefix}"; - if (tokenNodePath != indexPath) + var workspace = hub.GetWorkspace(); + var userTokenNamespace = $"{userId}/{ApiTokenNamespace}"; + + return workspace.GetQuery( + $"api-tokens:{userId}", + $"namespace:{userTokenNamespace} nodeType:{NodeTypeApiToken}", + // Legacy fallback: tokens at the global ApiToken namespace + // that pre-date the per-user partition migration. Filtered + // by UserId in the projection below — the synced query + // can't express that predicate, so we over-fetch globally + // and prune. + $"namespace:{ApiTokenNamespace} nodeType:{NodeTypeApiToken}") + .Select(snapshot => { - var indexNode = await QueryAsSystemAsync($"path:{indexPath}").FirstOrDefaultAsync(); - if (indexNode != null) + var tokens = new List(); + var seenPrefixes = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var node in snapshot) { - hub.Post(new DeleteNodeRequest(indexPath)); + if (node.Path is null) continue; + var apiToken = node.Content as ApiToken ?? ExtractApiToken(node); + if (apiToken == null) continue; + + // Legacy nodes in the global namespace must match the + // calling userId; per-user-partition nodes are scoped + // by namespace and don't need this filter, but the + // check is cheap and unifies the projection. + if (apiToken.UserId != userId) continue; + + var hashPrefix = apiToken.TokenHash.Length >= 8 + ? apiToken.TokenHash[..8] + : apiToken.TokenHash; + if (!seenPrefixes.Add(hashPrefix)) continue; + + tokens.Add(ToInfo(node, apiToken)); } - } - } - - logger.LogInformation("Revoked API token at {Path}", tokenNodePath); - return true; + return (IReadOnlyList)tokens; + }); } - public async Task> GetTokensForUserAsync(string userId) + /// + /// Derives the global ApiToken/{hashPrefix} index path from a + /// user-scoped token node path. sets the + /// node Id to the 12-char hash prefix, so the last path segment is + /// reliably the prefix used to build the index entry. Returns null + /// for malformed paths (no slash, trailing slash). + /// + private static string? DeriveIndexPath(string tokenNodePath) { - var tokens = new List(); - - // Query user-scoped tokens (new format) - // ApiToken is a satellite type (MainNode != Path), so we need nodeType: condition - // to trigger GetAllChildrenAsync which includes satellites in the results. - var userTokenNamespace = $"User/{userId}/{ApiTokenNamespace}"; - await foreach (var node in QueryAsSystemAsync($"namespace:{userTokenNamespace} nodeType:{NodeTypeApiToken}")) - { - var apiToken = node.Content as ApiToken ?? ExtractApiToken(node); - if (apiToken == null) - continue; - - tokens.Add(new ApiTokenInfo - { - NodePath = node.Path, - Label = apiToken.Label, - CreatedAt = apiToken.CreatedAt, - ExpiresAt = apiToken.ExpiresAt, - LastUsedAt = apiToken.LastUsedAt, - IsRevoked = apiToken.IsRevoked, - HashPrefix = apiToken.TokenHash.Length >= 8 ? apiToken.TokenHash[..8] : apiToken.TokenHash, - }); - } - - // Fallback: also check legacy tokens at top-level ApiToken namespace - await foreach (var node in QueryAsSystemAsync($"namespace:{ApiTokenNamespace} nodeType:{NodeTypeApiToken}")) - { - var apiToken = node.Content as ApiToken ?? ExtractApiToken(node); - if (apiToken == null || apiToken.UserId != userId) - continue; - - // Skip if we already found this token in the user namespace - var hashPrefix = apiToken.TokenHash.Length >= 8 ? apiToken.TokenHash[..8] : apiToken.TokenHash; - if (tokens.Any(t => t.HashPrefix == hashPrefix)) - continue; - - tokens.Add(new ApiTokenInfo - { - NodePath = node.Path, - Label = apiToken.Label, - CreatedAt = apiToken.CreatedAt, - ExpiresAt = apiToken.ExpiresAt, - LastUsedAt = apiToken.LastUsedAt, - IsRevoked = apiToken.IsRevoked, - HashPrefix = hashPrefix, - }); - } - - return tokens; + if (string.IsNullOrEmpty(tokenNodePath)) return null; + var lastSlash = tokenNodePath.LastIndexOf('/'); + if (lastSlash < 0 || lastSlash >= tokenNodePath.Length - 1) return null; + var hashPrefix = tokenNodePath[(lastSlash + 1)..]; + return $"{ApiTokenNamespace}/{hashPrefix}"; } + private static ApiTokenInfo ToInfo(MeshNode node, ApiToken apiToken) => new() + { + NodePath = node.Path, + Label = apiToken.Label, + CreatedAt = apiToken.CreatedAt, + ExpiresAt = apiToken.ExpiresAt, + LastUsedAt = apiToken.LastUsedAt, + IsRevoked = apiToken.IsRevoked, + HashPrefix = apiToken.TokenHash.Length >= 8 ? apiToken.TokenHash[..8] : apiToken.TokenHash, + }; + public static string HashToken(string rawToken) { var bytes = System.Text.Encoding.UTF8.GetBytes(rawToken); @@ -450,6 +486,24 @@ public static string HashToken(string rawToken) } return null; } + + private AccessAssignment? ExtractAccessAssignment(MeshNode? node) + { + if (node?.Content is AccessAssignment direct) return direct; + if (node?.Content is System.Text.Json.JsonElement jsonElement) + { + try + { + return System.Text.Json.JsonSerializer.Deserialize( + jsonElement.GetRawText(), hub.JsonSerializerOptions); + } + catch + { + return null; + } + } + return null; + } } /// diff --git a/memex/Memex.Portal.Shared/Authentication/BootstrapController.cs b/memex/Memex.Portal.Shared/Authentication/BootstrapController.cs new file mode 100644 index 000000000..a696f0db1 --- /dev/null +++ b/memex/Memex.Portal.Shared/Authentication/BootstrapController.cs @@ -0,0 +1,85 @@ +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Authentication; + +/// +/// One-shot, secret-gated first-admin bootstrap. Exists because a fresh deployment with a +/// wedged onboarding path (or any environment where the interactive /onboarding flow can't be +/// driven) still needs a way to materialise the FIRST platform administrator. +/// +/// It reuses — the exact same write path the +/// interactive onboarding uses — so the produced User node + AccessAssignments are serialised +/// and schema-routed correctly (no hand-rolled SQL, no guessing the content JSON shape). +/// CreateUser writes to the user's own partition + the Admin/_Access scope; it never +/// touches the phantom onboarding hub, so it isn't affected by that deadlock. +/// +/// Gated by the Bootstrap:Secret config value: if unset, the endpoint is disabled +/// (returns 404-equivalent Unauthorized). Intended to be invoked once by an operator, then the +/// secret removed. Anonymous-reachable by design (there is no admin yet to authorise it). +/// +[ApiController] +[Route("bootstrap")] +public class BootstrapController( + UserOnboardingService onboarding, + IConfiguration config, + ILogger logger) : ControllerBase +{ + [HttpGet("first-admin")] + [HttpPost("first-admin")] + public async Task FirstAdmin( + [FromQuery] string? secret, + [FromQuery] string? email, + [FromQuery] string? name, + [FromQuery] string? username) + { + var expected = config["Bootstrap:Secret"]; + if (string.IsNullOrWhiteSpace(expected)) + return NotFound(); // disabled unless a secret is configured + if (!string.Equals(secret, expected, StringComparison.Ordinal)) + return Unauthorized("invalid or missing secret"); + if (string.IsNullOrWhiteSpace(email)) + return BadRequest("email query parameter is required"); + + var user = (string.IsNullOrWhiteSpace(username) ? email.Split('@')[0] : username) + .Trim().ToLowerInvariant(); + var request = new UserOnboardingRequest(user, email.Trim(), name ?? user); + + logger.LogInformation("Bootstrap: materialising first admin '{User}' ({Email})", user, email); + + // Idempotent step runner: "already exists" is success (a pre-existing static/seed node + // is fine — we still want the Admin grants). Any other error is a real failure. + async Task Step(IObservable obs, string step) + { + try + { + await obs.FirstAsync().ToTask(); + logger.LogInformation("Bootstrap: {Step} OK for '{User}'", step, user); + return true; + } + catch (Exception ex) when (ex.Message.Contains("already exists", StringComparison.OrdinalIgnoreCase)) + { + logger.LogInformation("Bootstrap: {Step} already present for '{User}' — continuing", step, user); + return true; + } + catch (Exception ex) + { + logger.LogError(ex, "Bootstrap: {Step} FAILED for '{User}'", step, user); + return false; + } + } + + // Create the user node if missing (tolerate a pre-existing static/seed node), then grant + // self-Admin + platform-Admin so the resolved identity has Admin everywhere it needs it. + await Step(onboarding.CreateUser(request), "create-user"); + var selfOk = await Step(onboarding.GrantSelfAdmin(user), "self-admin"); + var platOk = await Step(onboarding.GrantPlatformAdmin(user), "platform-admin"); + + return selfOk && platOk + ? Ok($"OK: '{user}' ({email}) is platform admin. Sign in via Microsoft.") + : StatusCode(500, $"PARTIAL: self-admin={selfOk} platform-admin={platOk} — check portal logs."); + } +} diff --git a/memex/Memex.Portal.Shared/Authentication/DevAuthController.cs b/memex/Memex.Portal.Shared/Authentication/DevAuthController.cs index 78d407314..b60d95530 100644 --- a/memex/Memex.Portal.Shared/Authentication/DevAuthController.cs +++ b/memex/Memex.Portal.Shared/Authentication/DevAuthController.cs @@ -1,4 +1,5 @@ -using System.Security.Claims; +using System.Reactive.Linq; +using System.Security.Claims; using System.Text.Json; using MeshWeaver.Mesh; using MeshWeaver.Mesh.Services; @@ -26,8 +27,12 @@ public DevAuthController(IMeshService meshQuery) [HttpPost("signin")] public async Task Login([FromForm] string personId, [FromForm] string? returnUrl) { - // Fetch the person node via IMeshService (bypasses security) - var node = await _meshQuery.QueryAsync($"path:User/{personId}").FirstOrDefaultAsync(); + // TODO(persistence-cull): framework boundary — review whether this should + // route through UserIdentityCache (sync) instead of awaiting the observable. + var change = await _meshQuery + .Query(MeshQueryRequest.FromQuery($"path:User/{personId}")) + .FirstAsync(); + var node = change.Items.FirstOrDefault(); if (node?.NodeType != "User" || node.Content == null) { return BadRequest("Person not found"); @@ -39,7 +44,15 @@ public async Task Login([FromForm] string personId, [FromForm] st return BadRequest("Could not extract person info"); } - // Create claims: username is the node ID, email in content + // Create claims: username is the node ID, email in content. + // 🚨 preferred_username MUST be the username (node Id), NOT the email. + // UserContextMiddleware.ExtractUserContext takes ObjectId from + // preferred_username first; if that's the email, every downstream + // route targets `` instead of the user's partition and the + // portal renders "No node found at 'rbuergi@systemorph.com'". + // ApiTokenAuthenticationHandler already puts the username here — keep + // the dev login consistent so the user's partition (path = node Id) + // is the resolved home. var email = person.Email ?? ""; var username = node.Id; var claims = new List @@ -47,13 +60,13 @@ public async Task Login([FromForm] string personId, [FromForm] st new(ClaimTypes.NameIdentifier, username), new(ClaimTypes.Name, username), new("name", username), + new("preferred_username", username), }; if (!string.IsNullOrEmpty(email)) { claims.Add(new Claim(ClaimTypes.Email, email)); claims.Add(new Claim("email", email)); - claims.Add(new Claim("preferred_username", email)); } if (!string.IsNullOrEmpty(person.Role)) diff --git a/memex/Memex.Portal.Shared/Authentication/EaConsentController.cs b/memex/Memex.Portal.Shared/Authentication/EaConsentController.cs new file mode 100644 index 000000000..915b6eb72 --- /dev/null +++ b/memex/Memex.Portal.Shared/Authentication/EaConsentController.cs @@ -0,0 +1,48 @@ +using MeshWeaver.Messaging; // AccessService +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Authentication; + +/// +/// Drives the Executive Assistant's per-user, just-in-time Microsoft consent. The EA tool hands the +/// user a link to /auth/ea/connect the first time it needs their mailbox/calendar; that redirects to +/// Microsoft's consent screen for the EA's delegated Graph scopes, and the /auth/ea/callback +/// exchanges the code and stores the user's encrypted refresh token. The acting user is taken from the +/// authenticated principal at both steps — the OAuth state only carries the return URL. +/// +[Authorize] +[Route("auth/ea")] +public sealed class EaConsentController( + IEaGraphAuth ea, AccessService access, ILogger logger) : ControllerBase +{ + private string CallbackUri => $"{Request.Scheme}://{Request.Host}/auth/ea/callback"; + + [HttpGet("connect")] + public IActionResult Connect([FromQuery] string? returnUrl = null) + { + if (!ea.IsConfigured) return BadRequest("The Executive Assistant Graph integration is not configured."); + if (string.IsNullOrEmpty(access.Context?.ObjectId)) return Unauthorized(); + return Redirect(ea.BuildConsentUrl(Uri.EscapeDataString(returnUrl ?? "/"), CallbackUri)); + } + + [HttpGet("callback")] + public async Task Callback( + [FromQuery] string? code, [FromQuery] string? state, [FromQuery] string? error, CancellationToken ct) + { + var returnUrl = string.IsNullOrEmpty(state) ? "/" : Uri.UnescapeDataString(state); + var userId = access.Context?.ObjectId; + if (string.IsNullOrEmpty(userId)) return Unauthorized(); + + if (!string.IsNullOrEmpty(error) || string.IsNullOrEmpty(code)) + { + logger.LogWarning("EA consent callback for {User} returned error '{Error}'", userId, error); + return Redirect(returnUrl); + } + + var ok = await ea.ExchangeAndStoreAsync(code, CallbackUri, userId, ct); + logger.LogInformation("EA consent for {User}: {Result}", userId, ok ? "connected" : "failed"); + return Redirect(returnUrl); + } +} diff --git a/memex/Memex.Portal.Shared/Authentication/EaGraphAuth.cs b/memex/Memex.Portal.Shared/Authentication/EaGraphAuth.cs new file mode 100644 index 000000000..fcf79f902 --- /dev/null +++ b/memex/Memex.Portal.Shared/Authentication/EaGraphAuth.cs @@ -0,0 +1,185 @@ +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using System.Text.Json; +using MeshWeaver.AI; // IProviderKeyProtector +using MeshWeaver.Blazor.Infrastructure; // PortalApplication +using MeshWeaver.Data; // IWorkspace.GetMeshNodeStream +using MeshWeaver.Graph.Configuration; // EaCredentialNodeType +using MeshWeaver.Mesh; // EaCredential, MeshNode +using MeshWeaver.Mesh.Security; // ImpersonateAsSystem +using MeshWeaver.Mesh.Services; // IMeshService +using MeshWeaver.Messaging; // AccessService +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Authentication; + +/// +/// Per-user, just-in-time delegated Microsoft Graph access for the Executive Assistant. The user +/// consents to the EA accessing their own mailbox/calendar only when they first use the tool; we +/// exchange the auth code for tokens, store the refresh token encrypted as an +/// node, and mint short-lived access tokens from it on demand. No standing +/// application-wide Graph permission is used. +/// +/// This sits at the OAuth/HTTP boundary (called from the consent controller and the async EA tool), +/// so async/await is appropriate here — it is not hub-reachable reactive code. +/// +/// Azure setup (one-time): on the sign-in app registration add the delegated scopes +/// Mail.ReadWrite Mail.Send Calendars.ReadWrite offline_access and the redirect URI +/// {BaseUrl}/auth/ea/callback. The user's first use triggers the consent screen. +/// +public sealed class EaGraphAuth( + IServiceProvider rootServices, + IConfiguration configuration, + IProviderKeyProtector protector, + HttpClient http, + ILogger? logger = null) : IEaGraphAuth +{ + /// Delegated scopes the EA needs (space-separated, Graph v2 form). + public const string Scopes = + "https://graph.microsoft.com/Mail.ReadWrite https://graph.microsoft.com/Mail.Send " + + "https://graph.microsoft.com/Calendars.ReadWrite offline_access"; + + private string TenantId => configuration["Authentication:Microsoft:TenantId"] ?? "common"; + private string? ClientId => configuration["Authentication:Microsoft:ClientId"]; + private string? ClientSecret => configuration["Authentication:Microsoft:ClientSecret"]; + private string Authority => $"https://login.microsoftonline.com/{TenantId}/oauth2/v2.0"; + + /// True when the sign-in app credentials needed for the delegated flow are configured. + public bool IsConfigured => !string.IsNullOrEmpty(ClientId) && !string.IsNullOrEmpty(ClientSecret); + + /// The Microsoft consent/authorize URL to send the user to (incremental consent, forces the prompt). + public string BuildConsentUrl(string state, string redirectUri) => + $"{Authority}/authorize?client_id={Uri.EscapeDataString(ClientId ?? "")}" + + "&response_type=code&response_mode=query" + + $"&redirect_uri={Uri.EscapeDataString(redirectUri)}" + + $"&scope={Uri.EscapeDataString(Scopes)}" + + $"&state={Uri.EscapeDataString(state)}&prompt=consent"; + + /// Exchanges the consent auth-code for tokens and stores the (encrypted) refresh token for the user. + public async Task ExchangeAndStoreAsync(string code, string redirectUri, string userObjectId, CancellationToken ct) + { + if (!IsConfigured) return false; + var json = await PostTokenAsync(new Dictionary + { + ["client_id"] = ClientId!, + ["client_secret"] = ClientSecret!, + ["grant_type"] = "authorization_code", + ["code"] = code, + ["redirect_uri"] = redirectUri, + ["scope"] = Scopes + }, ct); + if (json is null) return false; + using var doc = JsonDocument.Parse(json); + var refresh = doc.RootElement.TryGetProperty("refresh_token", out var r) ? r.GetString() : null; + if (string.IsNullOrEmpty(refresh)) { logger?.LogWarning("EaGraphAuth: no refresh_token in code exchange"); return false; } + await StoreAsync(userObjectId, refresh!, ct); + return true; + } + + /// Returns a fresh access token for the user, or null when they have not connected (no stored credential). + public async Task GetAccessTokenAsync(string userObjectId, CancellationToken ct) + { + if (!IsConfigured) return null; + var (_, cred) = await LoadAsync(userObjectId, ct); + var refresh = protector.Unprotect(cred?.RefreshTokenEncrypted); + if (string.IsNullOrEmpty(refresh)) return null; + + var json = await PostTokenAsync(new Dictionary + { + ["client_id"] = ClientId!, + ["client_secret"] = ClientSecret!, + ["grant_type"] = "refresh_token", + ["refresh_token"] = refresh!, + ["scope"] = Scopes + }, ct); + if (json is null) return null; + using var doc = JsonDocument.Parse(json); + // Rotate the stored refresh token if Entra issued a new one. + if (doc.RootElement.TryGetProperty("refresh_token", out var nr) && nr.GetString() is { Length: > 0 } rotated) + await StoreAsync(userObjectId, rotated, ct); + return doc.RootElement.TryGetProperty("access_token", out var at) ? at.GetString() : null; + } + + /// True when the user already connected (has a stored credential) — lets callers skip the consent prompt. + public async Task IsConnectedAsync(string userObjectId, CancellationToken ct) + => (await LoadAsync(userObjectId, ct)).cred?.RefreshTokenEncrypted is { Length: > 0 }; + + private async Task PostTokenAsync(Dictionary form, CancellationToken ct) + { + using var resp = await http.PostAsync($"{Authority}/token", new FormUrlEncodedContent(form), ct); + var body = await resp.Content.ReadAsStringAsync(ct); + if (resp.IsSuccessStatusCode) return body; + logger?.LogWarning("EaGraphAuth: token endpoint returned {Status}", (int)resp.StatusCode); + return null; + } + + private static string PathFor(string userObjectId) => + $"Auth/{EaCredentialNodeType.UserSegment}/{userObjectId}"; + + private async Task StoreAsync(string userObjectId, string refreshToken, CancellationToken ct) + { + using var scope = rootServices.CreateScope(); + var hub = scope.ServiceProvider.GetRequiredService().Hub; + var meshService = hub.ServiceProvider.GetRequiredService(); + var access = hub.ServiceProvider.GetRequiredService(); + + var (existing, _) = await LoadAsync(userObjectId, ct, hub); + var content = new EaCredential + { + UserObjectId = userObjectId, + RefreshTokenEncrypted = protector.Protect(refreshToken), + Scopes = Scopes, + AcquiredAt = DateTimeOffset.UtcNow + }; + var node = (existing ?? new MeshNode(EaCredentialNodeType.NodeType, PathFor(userObjectId)) { Name = "EA Credential" }) + with { Content = content }; + + using (access.ImpersonateAsSystem()) + await (existing is null ? meshService.CreateNode(node) : meshService.UpdateNode(node)) + .FirstAsync().ToTask(ct); + } + + private Task<(MeshNode? node, EaCredential? cred)> LoadAsync(string userObjectId, CancellationToken ct) + => LoadAsync(userObjectId, ct, hub: null); + + private async Task<(MeshNode? node, EaCredential? cred)> LoadAsync( + string userObjectId, CancellationToken ct, MeshWeaver.Messaging.IMessageHub? hub) + { + IServiceScope? owned = null; + try + { + if (hub is null) + { + owned = rootServices.CreateScope(); + hub = owned.ServiceProvider.GetRequiredService().Hub; + } + var ws = hub.GetWorkspace(); + var access = hub.ServiceProvider.GetRequiredService(); + MeshNode? node; + using (access.ImpersonateAsSystem()) + node = await ws.GetMeshNodeStream(PathFor(userObjectId)) + .Take(1).Timeout(TimeSpan.FromSeconds(10)).FirstAsync().ToTask(ct); + var cred = node?.Content switch + { + EaCredential e => e, + JsonElement je => Safe(je, hub.JsonSerializerOptions), + _ => null + }; + return (node, cred); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "EaGraphAuth: load failed for {User}", userObjectId); + return (null, null); + } + finally { owned?.Dispose(); } + } + + private static EaCredential? Safe(JsonElement je, JsonSerializerOptions opts) + { + try { return JsonSerializer.Deserialize(je.GetRawText(), opts); } + catch { return null; } + } +} diff --git a/memex/Memex.Portal.Shared/Authentication/GlobalAdminSeed.cs b/memex/Memex.Portal.Shared/Authentication/GlobalAdminSeed.cs new file mode 100644 index 000000000..d50410b3d --- /dev/null +++ b/memex/Memex.Portal.Shared/Authentication/GlobalAdminSeed.cs @@ -0,0 +1,70 @@ +using MeshWeaver.Graph; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using Microsoft.Extensions.Configuration; + +namespace Memex.Portal.Shared.Authentication; + +/// +/// Seeds root-scope nodes that grant the Admin +/// role to each user listed under Auth:GlobalAdmins in configuration. +/// +/// Background: SecurityService.GetEffectiveRoles walks scopes from root +/// down and accumulates role assignments. Without a root-scope AccessAssignment +/// granting Admin, a configured Microsoft Entra ID user has zero roles on the +/// root scope, which surfaces as "lacks Read permission on 'Space'" +/// when navigating to the NodeType detail page (and equivalent denials on +/// cross-partition operations like creating a new Space). +/// +/// The test base ships an equivalent seed via +/// TestUsers.PublicAdminAccess() — production needs the same shape, +/// driven by config instead of hardcoded so each deployment can declare its own +/// admin list. See Doc/Architecture/AccessControl.md for the role +/// accumulation rules and src/MeshWeaver.Mesh.Contract/Security/AccessAssignment.cs +/// for the schema. +/// +public static class GlobalAdminSeed +{ + private const string ConfigSection = "Auth:GlobalAdmins"; + + /// + /// Builds AccessAssignment MeshNodes for every user id in + /// Auth:GlobalAdmins. Returns an empty array when the section is + /// missing or empty — safe to chain via builder.AddMeshNodes(...) + /// in environments that have no admins configured. + /// + public static MeshNode[] Build(IConfiguration configuration) + { + var ids = configuration.GetSection(ConfigSection).Get() + ?? []; + if (ids.Length == 0) + return []; + + var nodes = new MeshNode[ids.Length]; + for (var i = 0; i < ids.Length; i++) + { + var userId = ids[i].Trim(); + var assignment = new AccessAssignment + { + AccessObject = userId, + DisplayName = userId, + Roles = [new RoleAssignment { Role = "Admin" }], + }; + // 🚨 Global admin = admin on the ADMIN PARTITION. The grant lives at + // namespace "Admin/_Access" (→ scope "Admin"); the global-admin + // short-circuit in PermissionEvaluator turns Permission.All at scope + // "Admin" into platform-superuser (All on every path). MainNode "" = the + // whole Admin scope. This IS the db-init seed for config-driven admins — + // a fresh DB with Auth:GlobalAdmins set comes up with each listed user + // already a platform admin. See Doc/Architecture/AccessControl.md. + nodes[i] = new MeshNode(userId + "_Access", "Admin/_Access") + { + NodeType = "AccessAssignment", + Name = $"{userId} — Admin", + Content = assignment, + MainNode = "", + }; + } + return nodes; + } +} diff --git a/memex/Memex.Portal.Shared/Authentication/IEaGraphAuth.cs b/memex/Memex.Portal.Shared/Authentication/IEaGraphAuth.cs new file mode 100644 index 000000000..a15ae1c48 --- /dev/null +++ b/memex/Memex.Portal.Shared/Authentication/IEaGraphAuth.cs @@ -0,0 +1,25 @@ +namespace Memex.Portal.Shared.Authentication; + +/// +/// Seam for the Executive Assistant's per-user delegated Graph access. The real +/// drives a Microsoft OAuth consent + token flow; tests substitute a +/// hand-written fake that returns a canned token (or none) so the consent step is mocked away and the +/// EA tool / plugin can be exercised without a real browser consent or live Graph round-trip. +/// +public interface IEaGraphAuth +{ + /// True when the credentials needed for the delegated flow are configured. + bool IsConfigured { get; } + + /// The Microsoft consent URL to send the user to (incremental consent). + string BuildConsentUrl(string state, string redirectUri); + + /// Exchanges the consent auth-code for tokens and stores the user's encrypted refresh token. + Task ExchangeAndStoreAsync(string code, string redirectUri, string userObjectId, CancellationToken ct); + + /// A fresh delegated access token for the user, or null when they have not connected. + Task GetAccessTokenAsync(string userObjectId, CancellationToken ct); + + /// True when the user already connected (has a stored credential). + Task IsConnectedAsync(string userObjectId, CancellationToken ct); +} diff --git a/memex/Memex.Portal.Shared/Authentication/InvitationService.cs b/memex/Memex.Portal.Shared/Authentication/InvitationService.cs new file mode 100644 index 000000000..e44f03ca2 --- /dev/null +++ b/memex/Memex.Portal.Shared/Authentication/InvitationService.cs @@ -0,0 +1,137 @@ +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Authentication; + +/// +/// Reads and writes nodes for invitation-only onboarding. Invitations +/// live in the always-present Admin partition at Admin/Invitation/{slug} and are +/// made globally queryable by the path-less nodeType:Invitation → Admin routing rule +/// registered in . +/// +/// 100% reactive — IObservable<T> end-to-end (like ). +/// Every method returns a cold observable; callers Subscribe. All writes wrap in +/// Observable.Using(() => accessService.ImpersonateAsSystem(), …) because the caller — the +/// onboarding user (no identity yet) or an admin who lacks rights on the Admin partition — cannot +/// write there directly. This is the same infrastructure-write pattern as +/// . +/// +public sealed class InvitationService( + IMeshService meshService, + AccessService accessService, + ILogger? logger = null) +{ + /// Hard cap on the invitation lookup; matches OnboardingMiddleware's user lookup. + private static readonly TimeSpan LookupTimeout = TimeSpan.FromSeconds(20); + + /// + /// Reactive lookup of an outstanding () invitation for + /// via the canonical synced query (workspace.GetQuery, runs as + /// System and routes to the Admin partition). Emits the matching node, or null when no + /// pending invitation exists (or on timeout). Shape mirrors + /// OnboardingMiddleware.FindUserByEmail. + /// + public IObservable FindPendingInvitation(IWorkspace workspace, string email) + { + var jsonOptions = workspace.Hub.JsonSerializerOptions; + return workspace.GetQuery( + $"invite:byEmail:{email}", + // PATH-scoped to Admin/Invitation so it routes to the admin schema (routing is by + // the path's first segment). Invitations live in the Admin partition, which is + // excluded from cross-schema global search, so a `namespace:Admin`-only query + // (no path) fans out cross-schema and never finds them — onboarding would then + // treat every invited email as NOT invited and block it. + $"path:{InvitationNodeType.Namespace} scope:children nodeType:{InvitationNodeType.NodeType} content.email:{email}") + .Take(1) + .Select(items => (MeshNode?)items + .FirstOrDefault(n => TryGetInvitation(n, jsonOptions) is { Status: InvitationStatus.Pending })) + .Timeout(LookupTimeout, Observable.Defer(() => + { + logger?.LogWarning( + "FindPendingInvitation({Email}): no snapshot within {Timeout} — treating as not invited", + email, LookupTimeout); + return Observable.Return(null); + })); + } + + /// + /// Creates (or overwrites, since the slug is derived from the email) a Pending invitation for + /// . Returns a cold observable emitting the created node; subscribe to drive. + /// + public IObservable CreateInvitation(string email, string? invitedBy, string? note) + { + var trimmed = email.Trim(); + var node = new MeshNode(Slugify(trimmed), InvitationNodeType.Namespace) + { + Name = trimmed, + NodeType = InvitationNodeType.NodeType, + State = MeshNodeState.Active, + Content = new Invitation + { + Email = trimmed, + InvitedBy = invitedBy, + Note = string.IsNullOrWhiteSpace(note) ? null : note!.Trim(), + Status = InvitationStatus.Pending, + }, + }; + + return Observable.Using( + () => accessService.ImpersonateAsSystem(), + _ => meshService.CreateNode(node) + .Do(__ => logger?.LogInformation( + "Invitation: created for {Email} at {Path} (by {InvitedBy})", + trimmed, node.Path, invitedBy ?? "(unknown)"))); + } + + /// + /// Flips an invitation to (called on successful + /// onboarding). The caller passes the content it already extracted + /// so all other fields (Id, InvitedBy, InvitedAt, Note) are preserved. + /// + public IObservable MarkAccepted(MeshNode node, Invitation current) => + WriteStatus(node, current with + { + Status = InvitationStatus.Accepted, + AcceptedAt = DateTimeOffset.UtcNow, + }); + + /// Flips an invitation to (admin withdraws it). + public IObservable Revoke(MeshNode node, Invitation current) => + WriteStatus(node, current with { Status = InvitationStatus.Revoked }); + + private IObservable WriteStatus(MeshNode node, Invitation updated) => + Observable.Using( + () => accessService.ImpersonateAsSystem(), + _ => meshService.UpdateNode(node with { Content = updated }) + .Do(__ => logger?.LogInformation("Invitation: {Path} → {Status}", node.Path, updated.Status))); + + /// + /// Extracts the from a node's Content, deserializing from + /// when the synced query returned the raw JSON shape. Returns + /// null when the content is absent or not an invitation. + /// + public static Invitation? TryGetInvitation(MeshNode node, JsonSerializerOptions? options) => + node.Content switch + { + Invitation inv => inv, + JsonElement je => Deserialize(je, options), + _ => null, + }; + + private static Invitation? Deserialize(JsonElement je, JsonSerializerOptions? options) + { + try { return JsonSerializer.Deserialize(je.GetRawText(), options); } + catch { return null; } + } + + private static string Slugify(string email) => + new(email.ToLowerInvariant().Select(c => char.IsLetterOrDigit(c) ? c : '_').ToArray()); +} diff --git a/memex/Memex.Portal.Shared/Authentication/McpAuthenticationExtensions.cs b/memex/Memex.Portal.Shared/Authentication/McpAuthenticationExtensions.cs new file mode 100644 index 000000000..3abcbef95 --- /dev/null +++ b/memex/Memex.Portal.Shared/Authentication/McpAuthenticationExtensions.cs @@ -0,0 +1,90 @@ +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Authorization; +using Microsoft.Extensions.DependencyInjection; +using ModelContextProtocol.AspNetCore.Authentication; +using ModelContextProtocol.Authentication; + +namespace Memex.Portal.Shared.Authentication; + +/// +/// Separate auth wiring for the MCP endpoint. +/// +/// The Blazor portal uses cookie-based auth with a redirect-to-login challenge, which is +/// correct for browser users but fatal for MCP clients: Claude Desktop / Claude.ai follow +/// a 302 to an HTML login page and fail with "couldn't reach the server" instead of doing +/// OAuth discovery. +/// +/// MCP auth must be strictly Bearer-only: +/// * token validation goes to +/// * unauthed requests get 401 + WWW-Authenticate: Bearer resource_metadata="..." +/// emitted by the MCP SDK's own scheme, so clients can run OAuth discovery +/// * no leakage to cookie — no redirects, ever +/// +public static class McpAuthenticationExtensions +{ + public const string PolicyName = "McpAuth"; + + /// + /// Registers the ApiToken + MCP authentication schemes and the McpAuth + /// authorization policy. Call after the primary (cookie / OIDC) auth has been + /// registered — this adds to the existing authentication builder without + /// touching its defaults. + /// + public static IServiceCollection AddMcpAuthentication(this IServiceCollection services) + { + services.AddAuthentication() + .AddScheme( + ApiTokenAuthenticationHandler.SchemeName, _ => { }) + .AddMcp(ConfigureMcpAuth); + + services.AddAuthorization(options => + { + options.AddPolicy(PolicyName, policy => + { + policy.AddAuthenticationSchemes(McpAuthenticationDefaults.AuthenticationScheme); + policy.RequireAuthenticatedUser(); + }); + }); + + return services; + } + + private static void ConfigureMcpAuth(McpAuthenticationOptions options) + { + // Bearer token validation → ApiToken handler. The MCP SDK constructor hardcodes + // ForwardAuthenticate = "Bearer" (a scheme that doesn't exist here); point it at + // the real scheme so token validation actually runs. + options.ForwardAuthenticate = ApiTokenAuthenticationHandler.SchemeName; + + // Leave Challenge on the MCP scheme itself so it emits + // 401 + WWW-Authenticate: Bearer resource_metadata="..." — that's what lets + // MCP clients discover the auth server. NEVER forward to cookie: that would + // produce a 302 to /login which MCP clients can't follow. + options.ForwardChallenge = null; + options.ForwardForbid = null; + options.ForwardDefaultSelector = null; + + options.ResourceMetadata = new ProtectedResourceMetadata + { + BearerMethodsSupported = { "header" }, + ScopesSupported = { "mcp" }, + }; + + options.Events = new McpAuthenticationEvents + { + OnResourceMetadataRequest = ctx => + { + var req = ctx.HttpContext.Request; + var origin = $"{req.Scheme}://{req.Host}"; + ctx.ResourceMetadata = new ProtectedResourceMetadata + { + Resource = $"{origin}/mcp", + BearerMethodsSupported = { "header" }, + ScopesSupported = { "mcp" }, + AuthorizationServers = { origin }, + }; + return Task.CompletedTask; + }, + }; + } +} diff --git a/memex/Memex.Portal.Shared/Authentication/McpBackConnectionService.cs b/memex/Memex.Portal.Shared/Authentication/McpBackConnectionService.cs new file mode 100644 index 000000000..63ddd9855 --- /dev/null +++ b/memex/Memex.Portal.Shared/Authentication/McpBackConnectionService.cs @@ -0,0 +1,74 @@ +using System.Collections.Concurrent; +using System.Reactive.Linq; +using MeshWeaver.AI.Connect; +using MeshWeaver.Blazor.AI; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Memex.Portal.Shared.Authentication; + +/// +/// Automatic, token-based MCP back-connection provisioning — the portal-side implementation of +/// . The co-hosted Claude Code / GitHub Copilot CLIs call +/// at spawn time (every execution); on a cache miss this mints a +/// long-lived per-user MeshWeaver ApiToken via — with NO manual +/// step — and returns the composed {baseUrl}/mcp URL plus the raw mw_… token to present +/// as Authorization: Bearer. Internal portal↔CLI↔/mcp comms are therefore token-based and +/// scoped to the user's own permissions (the ApiToken carries the user's roles). +/// +/// The per-user token is cached on this singleton's instance dictionary (NEVER static — it +/// dies with the host) for the process lifetime; a fresh replica mints its own (the prior token +/// stays valid). A revoked token surfaces as a 401 on the next /mcp call, which the +/// auth-on-exception path turns into a re-mint. +/// +internal sealed class McpBackConnectionService : IMcpBackConnection +{ + private readonly ApiTokenService tokenService; + private readonly IOptions mcpConfig; + private readonly ILogger logger; + + // Instance (not static) — lifetime == the portal host. userId → raw mw_ token. + private readonly ConcurrentDictionary tokensByUser = new(StringComparer.Ordinal); + + public McpBackConnectionService( + ApiTokenService tokenService, + IOptions mcpConfig, + ILogger logger) + { + this.tokenService = tokenService; + this.mcpConfig = mcpConfig; + this.logger = logger; + } + + public IObservable EnsureForUser(string userId, string? userName = null, string? userEmail = null) + { + var baseUrl = mcpConfig.Value?.BaseUrl; + if (string.IsNullOrWhiteSpace(baseUrl) || string.IsNullOrWhiteSpace(userId)) + return Observable.Return(null); + + var mcpUrl = $"{baseUrl!.TrimEnd('/')}/mcp"; + + // Reuse the cached token (long-lived, no expiry) — cheap hot path, no mint. + if (tokensByUser.TryGetValue(userId, out var cached)) + return Observable.Return(new McpConnectionInfo(mcpUrl, cached)); + + // Cache miss → mint automatically. CreateToken self-elevates for the global index write; + // the user-scoped node is created under the calling user's AccessContext (active at spawn). + return tokenService + .CreateToken(userId, userName ?? userId, userEmail ?? string.Empty, + label: "MCP back-connection (auto)", expiresAt: null) + .Select(result => + { + tokensByUser[userId] = result.RawToken; + logger.LogInformation("Auto-minted MCP back-connection token for user {UserId}", userId); + return (McpConnectionInfo?)new McpConnectionInfo(mcpUrl, result.RawToken); + }) + .Catch((Exception ex) => + { + // Fail soft: the CLI runs without mesh access rather than failing the chat. + logger.LogWarning(ex, + "Could not provision MCP back-connection for user {UserId}; co-hosted CLI will run without mesh access.", userId); + return Observable.Return(null); + }); + } +} diff --git a/memex/Memex.Portal.Shared/Authentication/OAuthCodeStore.cs b/memex/Memex.Portal.Shared/Authentication/OAuthCodeStore.cs index 0e2dadb0f..8c57f0d00 100644 --- a/memex/Memex.Portal.Shared/Authentication/OAuthCodeStore.cs +++ b/memex/Memex.Portal.Shared/Authentication/OAuthCodeStore.cs @@ -7,7 +7,7 @@ namespace Memex.Portal.Shared.Authentication; /// /// In-memory store for OAuth authorization codes with PKCE support. /// Codes expire after 5 minutes and are single-use (consumed on exchange). -/// Uses ConcurrentDictionary for thread-safe mutation (per CLAUDE.md exception). +/// Uses ConcurrentDictionary for thread-safe mutation (per AGENTS.md exception). /// internal class OAuthCodeStore { diff --git a/memex/Memex.Portal.Shared/Authentication/OAuthConnectController.cs b/memex/Memex.Portal.Shared/Authentication/OAuthConnectController.cs index c2f12aa05..541220319 100644 --- a/memex/Memex.Portal.Shared/Authentication/OAuthConnectController.cs +++ b/memex/Memex.Portal.Shared/Authentication/OAuthConnectController.cs @@ -1,5 +1,12 @@ +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; using System.Security.Claims; +using System.Security.Cryptography; +using System.Text.Json.Serialization; +using MeshWeaver.Blazor.Infrastructure; // PortalApplication +using MeshWeaver.Messaging; // AccessService / AccessContext using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -8,8 +15,8 @@ namespace Memex.Portal.Shared.Authentication; /// /// Minimal OAuth 2.0 authorization server for MCP clients (claude.ai Connectors, Claude Desktop). -/// Implements authorization code flow with PKCE. Issues mw_ API tokens as access tokens, -/// reusing the existing ApiTokenService infrastructure. +/// Implements authorization code flow with PKCE + RFC 7591 Dynamic Client Registration. +/// Issues mw_ API tokens as access tokens, reusing the existing ApiTokenService infrastructure. /// [ApiController] public class OAuthConnectController( @@ -19,6 +26,37 @@ public class OAuthConnectController( private OAuthCodeStore CodeStore => serviceProvider.GetRequiredService(); private ApiTokenService TokenService => serviceProvider.GetRequiredService(); + /// + /// Resolves the mesh User.Id for the issued token. 🚨 It MUST be the mesh + /// User.Id (e.g. rbuergi), NEVER the raw preferred_username claim — Entra + /// fills that with the email/UPN, and an email userId routes the token node + its + /// _Access self-scope into a non-existent {email} partition (401 on every + /// freshly-minted token once the router stopped lazy-creating schemas). + /// Prefers the authoritative identity UserContextMiddleware stamped on the + /// portal hub's (email→User.Id); falls back to normalising the + /// claim to the username (email local-part) when no resolved context is present — e.g. + /// controller unit tests, or any call before the middleware ran. + /// + private string? ResolveMeshUserId() + { + var ctx = serviceProvider.GetService()? + .Hub.ServiceProvider.GetService()?.Context; + var resolved = ctx?.ObjectId; + if (!string.IsNullOrEmpty(resolved) && !resolved.Contains('@')) + return resolved; + var claim = User.FindFirstValue("preferred_username") ?? User.FindFirstValue(ClaimTypes.Email); + return UsernameFromEmail(claim); + } + + /// Email-shaped identifier → its local part (the post-v10 username / mesh + /// partition key, e.g. rbuergi@systemorph.com → rbuergi); unchanged when there's no @. + private static string? UsernameFromEmail(string? value) + { + if (string.IsNullOrEmpty(value)) return null; + var at = value.IndexOf('@'); + return at > 0 ? value[..at] : value; + } + /// /// RFC 8414 — OAuth Authorization Server Metadata. /// MCP clients discover this via the authorization_servers URL from the protected resource metadata. @@ -28,22 +66,73 @@ public class OAuthConnectController( public IActionResult GetServerMetadata() { var origin = $"{Request.Scheme}://{Request.Host}"; + logger.LogInformation("OAuth metadata requested from {Origin}", origin); return Ok(new { - issuer = $"{origin}/connect", - authorization_endpoint = $"{origin}/connect/authorize", - token_endpoint = $"{origin}/connect/token", + issuer = origin, + authorization_endpoint = $"{origin}/authorize", + token_endpoint = $"{origin}/token", + registration_endpoint = $"{origin}/register", response_types_supported = new[] { "code" }, grant_types_supported = new[] { "authorization_code" }, code_challenge_methods_supported = new[] { "S256" }, + token_endpoint_auth_methods_supported = new[] { "none" }, }); } + /// + /// RFC 7591 — Dynamic Client Registration. + /// MCP clients (Claude Desktop, claude.ai Connectors) self-register here with their redirect URIs + /// before running the authorization flow. The mesh does not persist client registrations — + /// it issues a random client_id that the caller echoes back in /authorize and + /// /token; the code store validates client_id+redirect_uri consistency between those calls. + /// + [HttpPost("/register")] + [AllowAnonymous] + public IActionResult RegisterClient([FromBody] ClientRegistrationRequest? request) + { + if (request is null) + { + logger.LogWarning("OAuth /register called with empty or invalid body"); + return BadRequest(new { error = "invalid_client_metadata", error_description = "Request body is required" }); + } + + logger.LogInformation( + "OAuth client registration: client_name={ClientName}, redirect_uris={RedirectUris}, grant_types={GrantTypes}, auth_method={AuthMethod}", + request.ClientName ?? "(unset)", + request.RedirectUris is null ? "(none)" : string.Join(",", request.RedirectUris), + request.GrantTypes is null ? "(unset)" : string.Join(",", request.GrantTypes), + request.TokenEndpointAuthMethod ?? "(unset)"); + + if (request.RedirectUris is null || request.RedirectUris.Length == 0) + { + logger.LogWarning("OAuth /register rejected: redirect_uris missing for client {ClientName}", request.ClientName); + return BadRequest(new { error = "invalid_redirect_uri", error_description = "redirect_uris is required" }); + } + + var clientId = Convert.ToBase64String(RandomNumberGenerator.GetBytes(24)) + .Replace("+", "-").Replace("/", "_").TrimEnd('='); + + var response = new ClientRegistrationResponse + { + ClientId = clientId, + ClientIdIssuedAt = DateTimeOffset.UtcNow.ToUnixTimeSeconds(), + ClientName = request.ClientName, + RedirectUris = request.RedirectUris, + GrantTypes = request.GrantTypes ?? new[] { "authorization_code" }, + ResponseTypes = request.ResponseTypes ?? new[] { "code" }, + TokenEndpointAuthMethod = request.TokenEndpointAuthMethod ?? "none", + }; + + logger.LogInformation("Issued OAuth client_id {ClientId} for {ClientName}", clientId, request.ClientName); + return StatusCode(StatusCodes.Status201Created, response); + } + /// /// OAuth Authorization Endpoint — redirects authenticated users to the client's redirect_uri /// with an authorization code. Unauthenticated users are sent to /login first. /// - [HttpGet("connect/authorize")] + [HttpGet("/authorize")] public IActionResult Authorize( [FromQuery] string response_type, [FromQuery] string client_id, @@ -53,21 +142,37 @@ public IActionResult Authorize( [FromQuery] string? code_challenge, [FromQuery] string? code_challenge_method) { + logger.LogInformation( + "OAuth /authorize: response_type={ResponseType}, client_id={ClientId}, redirect_uri={RedirectUri}, has_state={HasState}, has_pkce={HasPkce}, authenticated={Authenticated}", + response_type, client_id, redirect_uri, + !string.IsNullOrEmpty(state), !string.IsNullOrEmpty(code_challenge), + User?.Identity?.IsAuthenticated == true); + if (response_type != "code") + { + logger.LogWarning("OAuth /authorize rejected: unsupported response_type={ResponseType}", response_type); return BadRequest(new { error = "unsupported_response_type" }); + } if (string.IsNullOrEmpty(client_id) || string.IsNullOrEmpty(redirect_uri)) + { + logger.LogWarning("OAuth /authorize rejected: missing client_id or redirect_uri"); return BadRequest(new { error = "invalid_request", error_description = "client_id and redirect_uri are required" }); + } // If user is not authenticated, redirect to login with return URL if (User?.Identity?.IsAuthenticated != true) { var authorizeUrl = $"{Request.Scheme}://{Request.Host}{Request.Path}{Request.QueryString}"; var loginUrl = $"/login?returnUrl={Uri.EscapeDataString(authorizeUrl)}"; + logger.LogInformation("OAuth /authorize: redirecting unauthenticated caller to {LoginUrl}", loginUrl); return Redirect(loginUrl); } - // Extract user identity from cookie claims + // Extract user identity from cookie claims (email/name are display + // fields). The token's userId is the MESH User.Id, resolved by + // UserContextMiddleware onto AccessService.Context for this cookie + // request — NOT preferred_username, which Entra fills with the email. var email = User.FindFirstValue(ClaimTypes.Email) ?? User.FindFirstValue("email") ?? User.FindFirstValue("preferred_username") @@ -75,11 +180,27 @@ public IActionResult Authorize( var name = User.FindFirstValue(ClaimTypes.Name) ?? User.FindFirstValue("name") ?? email; - var userId = User.FindFirstValue("preferred_username") - ?? email; + var userId = ResolveMeshUserId(); if (string.IsNullOrEmpty(email)) + { + logger.LogWarning("OAuth /authorize rejected: authenticated principal has no email/preferred_username claim"); + return BadRequest(new { error = "invalid_request", error_description = "Unable to determine user identity" }); + } + + // Refuse to issue a code with an unresolved or email-shaped userId — it + // would mint the token into a parallel {email} partition that owns none + // of the user's data (the original atioz 401). A missing mesh identity + // means the User node isn't provisioned yet; the user should retry after + // a normal browser login populates the identity cache. + if (string.IsNullOrEmpty(userId) || userId.Contains('@')) + { + logger.LogWarning( + "OAuth /authorize rejected: no resolved mesh identity for {Email} (userId='{UserId}'). " + + "Retry after a browser login provisions/loads the User node.", + email, userId ?? "(null)"); return BadRequest(new { error = "invalid_request", error_description = "Unable to determine user identity" }); + } // Generate authorization code var code = CodeStore.GenerateCode( @@ -105,15 +226,26 @@ public IActionResult Authorize( /// OAuth Token Endpoint — exchanges an authorization code for an API token. /// The issued token is a standard mw_ API token, indistinguishable from manually created ones. /// - [HttpPost("connect/token")] + [HttpPost("/token")] [AllowAnonymous] - public async Task ExchangeToken([FromForm] TokenRequest request) + public Task ExchangeToken([FromForm] TokenRequest request, CancellationToken ct) { + logger.LogInformation( + "OAuth /token: grant_type={GrantType}, client_id={ClientId}, redirect_uri={RedirectUri}, has_code={HasCode}, has_verifier={HasVerifier}", + request.grant_type, request.client_id, request.redirect_uri, + !string.IsNullOrEmpty(request.code), !string.IsNullOrEmpty(request.code_verifier)); + if (request.grant_type != "authorization_code") - return BadRequest(new { error = "unsupported_grant_type" }); + { + logger.LogWarning("OAuth /token rejected: unsupported grant_type={GrantType}", request.grant_type); + return Task.FromResult(BadRequest(new { error = "unsupported_grant_type" })); + } if (string.IsNullOrEmpty(request.code) || string.IsNullOrEmpty(request.client_id) || string.IsNullOrEmpty(request.redirect_uri)) - return BadRequest(new { error = "invalid_request" }); + { + logger.LogWarning("OAuth /token rejected: missing code/client_id/redirect_uri"); + return Task.FromResult(BadRequest(new { error = "invalid_request" })); + } var entry = CodeStore.ExchangeCode( request.code, @@ -124,26 +256,46 @@ public async Task ExchangeToken([FromForm] TokenRequest request) if (entry == null) { logger.LogWarning("OAuth token exchange failed: invalid or expired code for client {ClientId}", request.client_id); - return BadRequest(new { error = "invalid_grant" }); + return Task.FromResult(BadRequest(new { error = "invalid_grant" })); } - // Create an mw_ API token via the existing token service - var (rawToken, _) = await TokenService.CreateTokenAsync( - userId: entry.UserId, - userName: entry.UserName, - userEmail: entry.UserEmail, - label: $"OAuth: {request.client_id}", - expiresAt: DateTimeOffset.UtcNow.AddDays(30)); - - logger.LogInformation("Issued OAuth access token for user {Email}, client {ClientId}", entry.UserEmail, request.client_id); - - return Ok(new - { - access_token = rawToken, - token_type = "Bearer", - expires_in = (int)TimeSpan.FromDays(30).TotalSeconds, - }); + // Create an mw_ API token via the existing token service. Lifetime + // is long-lived because OAuth clients (MCP, CLI tools) typically + // can't run interactive re-auth flows — a token that expires in 30 + // days surprises users who connect once and come back months later. + // Refresh-token flow isn't implemented yet; until it is, default to + // 1 year. Bump if needed via TokenLifetime below. + // + // No await: pull IObservable up to the controller's return type. + // Single bridge to Task happens at .ToTask(ct) — passing the + // request's cancellation token so a client disconnect tears down + // the reactive subscription. + return TokenService.CreateToken( + userId: entry.UserId, + userName: entry.UserName, + userEmail: entry.UserEmail, + label: $"OAuth: {request.client_id}", + expiresAt: DateTimeOffset.UtcNow.Add(TokenLifetime)) + .Select(creation => + { + logger.LogInformation("Issued OAuth access token for user {Email}, client {ClientId}", entry.UserEmail, request.client_id); + return (IActionResult)Ok(new + { + access_token = creation.RawToken, + token_type = "Bearer", + expires_in = (int)TokenLifetime.TotalSeconds, + }); + }) + .FirstAsync() + .ToTask(ct); } + + /// + /// Lifetime for OAuth-issued API tokens. Single source of truth — the + /// expiresAt timestamp on the token row and the expires_in OAuth response + /// field both read from this so they can't drift apart. + /// + private static readonly TimeSpan TokenLifetime = TimeSpan.FromDays(365); } /// @@ -157,3 +309,54 @@ public class TokenRequest public string? redirect_uri { get; set; } public string? code_verifier { get; set; } } + +/// +/// RFC 7591 Dynamic Client Registration request. Fields use snake_case JSON names per the spec. +/// +public class ClientRegistrationRequest +{ + [JsonPropertyName("client_name")] + public string? ClientName { get; set; } + + [JsonPropertyName("redirect_uris")] + public string[]? RedirectUris { get; set; } + + [JsonPropertyName("grant_types")] + public string[]? GrantTypes { get; set; } + + [JsonPropertyName("response_types")] + public string[]? ResponseTypes { get; set; } + + [JsonPropertyName("token_endpoint_auth_method")] + public string? TokenEndpointAuthMethod { get; set; } + + [JsonPropertyName("scope")] + public string? Scope { get; set; } +} + +/// +/// RFC 7591 Dynamic Client Registration response. +/// +public class ClientRegistrationResponse +{ + [JsonPropertyName("client_id")] + public string ClientId { get; set; } = ""; + + [JsonPropertyName("client_id_issued_at")] + public long ClientIdIssuedAt { get; set; } + + [JsonPropertyName("client_name")] + public string? ClientName { get; set; } + + [JsonPropertyName("redirect_uris")] + public string[]? RedirectUris { get; set; } + + [JsonPropertyName("grant_types")] + public string[]? GrantTypes { get; set; } + + [JsonPropertyName("response_types")] + public string[]? ResponseTypes { get; set; } + + [JsonPropertyName("token_endpoint_auth_method")] + public string? TokenEndpointAuthMethod { get; set; } +} diff --git a/memex/Memex.Portal.Shared/Authentication/OnboardingMiddleware.cs b/memex/Memex.Portal.Shared/Authentication/OnboardingMiddleware.cs index 01572a555..d3f2a081a 100644 --- a/memex/Memex.Portal.Shared/Authentication/OnboardingMiddleware.cs +++ b/memex/Memex.Portal.Shared/Authentication/OnboardingMiddleware.cs @@ -1,5 +1,9 @@ -using System.Text.Json; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using System.Text.Json; using MeshWeaver.Blazor.Infrastructure; +using MeshWeaver.Data; +using MeshWeaver.Graph; using MeshWeaver.Mesh; using MeshWeaver.Mesh.Security; using MeshWeaver.Mesh.Services; @@ -14,12 +18,48 @@ namespace Memex.Portal.Shared.Authentication; /// Middleware that redirects authenticated users without an Active user node /// to the onboarding page. Runs after UserContextMiddleware. /// -/// Flow: -/// - No user node (or Transient) → redirect /onboarding -/// - Active node → update AccessContext with username, pass through +/// Flow: +/// +/// No user node (or Transient) → redirect /onboarding +/// Active node → update AccessContext with username, pass through +/// +/// +/// +/// The user lookup uses workspace.GetQuery (the canonical synced-query +/// API from SyncedMeshNodeQueries.md). The synced layer bypasses RLS internally +/// (System identity), dedupes by path, gates on Initial, and includes static-node +/// providers — same guarantees as ApiTokenService.GetTokensForUser and +/// AgentChatClient.Initialize. Direct IMeshQueryCore.Query calls +/// from application code are pedestrian queries and were forbidden in 2026-05. +/// +/// Internally the lookup is a reactive observable chain +/// (workspace.GetQuery → Where → Take(1) → Timeout); +/// the single await at the middleware boundary is unavoidable because +/// ASP.NET Core's RequestDelegate is Task-based. /// public class OnboardingMiddleware(RequestDelegate next, ILogger logger) { + /// + /// Hard cap on the user-node lookup. Sized for cold start: the User + /// catalog partition can take 5–10s to hydrate on a fresh portal + /// process, and the previous 5s budget routinely bounced legitimate + /// users to /onboarding right after a restart. Bumped to 20s + /// so the timeout is reserved for genuinely-pathological cases (mesh + /// down, query layer wedged) rather than cold-start hydration race. + /// + private static readonly TimeSpan LookupTimeout = TimeSpan.FromSeconds(20); + + /// + /// If the FIRST snapshot is empty, + /// resubscribe once after this delay before giving up. Covers the case + /// where the catalog grain replied to the subscription with an empty + /// pre-hydration snapshot but never fires a follow-up Added once + /// hydration completes (we've seen this with the InMemory catalog when + /// the partition is loaded synchronously by a different request that + /// holds the grain lock). + /// + private static readonly TimeSpan RetryDelay = TimeSpan.FromMilliseconds(750); + private static readonly HashSet ExcludedPrefixes = new(StringComparer.OrdinalIgnoreCase) { "/onboarding", @@ -35,134 +75,233 @@ public class OnboardingMiddleware(RequestDelegate next, ILogger(); - if (portalApp != null) - { - var accessService = portalApp.Hub.ServiceProvider.GetRequiredService(); - var userContext = accessService.Context ?? accessService.CircuitContext; - - // Skip virtual users — they don't need onboarding - if (userContext is { IsVirtual: false } && !string.IsNullOrEmpty(userContext.ObjectId)) - { - var email = userContext.Email ?? userContext.ObjectId; - - // If the context's ObjectId was already resolved to a username - // (different from the email), this user was onboarded in the current - // session. Skip the query — it may not find newly created nodes - // immediately due to routing/caching in the mesh query layer. - if (!string.IsNullOrEmpty(email) && - !string.IsNullOrEmpty(userContext.ObjectId) && - userContext.ObjectId != email) - { - await next(context); - return; - } + // Pull the reactive composition all the way up: the user-resolution + // pipeline (FindUserByEmail → conditional LoadUserRoles → SetContext) + // is a single observable chain. The only Task bridge is on the line + // below — ASP.NET's RequestDelegate signature forces Task at this + // boundary, but everything else stays observable so a slow query + // layer can't deadlock by awaiting a result the awaiting thread is + // supposed to publish. + // + // Outcome semantics: + // • Result = "Redirect" — middleware bounces to /onboarding, doesn't + // call next. + // • Result = "PassThrough" — context updated (or skipped because + // unauthenticated / virtual / excluded path); fall through to next. + var outcome = await BuildPipeline(context).FirstAsync().ToTask(); - var meshQuery = portalApp.Hub.ServiceProvider.GetService(); - if (meshQuery != null) - { - try - { - // Look up User node by email stored in content. - // Use ImpersonateAsHub scope because user context may not have - // sufficient permissions yet at this point in the pipeline. - MeshNode? node; - using (accessService.ImpersonateAsHub(portalApp.Hub)) - { - node = await meshQuery.QueryAsync( - $"nodeType:User namespace:User content.email:{email} limit:1").FirstOrDefaultAsync(); - } - - if (node == null || node.State == MeshNodeState.Transient) - { - // No user node or incomplete onboarding — redirect - logger.LogInformation( - "OnboardingMiddleware: Redirecting to onboarding for {Email}", - email); - context.Response.Redirect("/onboarding"); - return; - } - - // Active user — update AccessContext with username (node ID) - var username = node.Id; - - // Query global AccessAssignment to populate roles - var roles = await LoadUserRolesAsync( - meshQuery, accessService, portalApp.Hub, username); - - var updatedContext = userContext with - { - ObjectId = username, - Name = node.Name ?? username, - Roles = roles - }; - // Set per-request context only. CircuitAccessHandler handles - // per-circuit persistence via CreateInboundActivityHandler. - accessService.SetContext(updatedContext); - } - catch (Exception ex) - { - // Non-critical — don't block the request on onboarding check failure - logger.LogWarning(ex, - "OnboardingMiddleware: Failed to check user node for {UserId}", - userContext.ObjectId); - } - } - } - } + if (outcome == OnboardingOutcome.Redirect) + { + context.Response.Redirect("/onboarding"); + return; } await next(context); } + private enum OnboardingOutcome { PassThrough, Redirect } + /// - /// Loads the user's role names from AccessAssignment nodes across all scopes. - /// Used to populate AccessContext.Roles so permission checks work in Blazor components. + /// Builds the reactive onboarding pipeline. Returns an observable that + /// emits exactly one describing what the + /// middleware should do next. Composition is end-to-end reactive — no + /// intermediate await, no fire-and-forget Subscribe, no + /// TaskCompletionSource. The single Task bridge lives in + /// . /// - private static async Task> LoadUserRolesAsync( - IMeshService meshQuery, AccessService accessService, IMessageHub hub, string username) + private IObservable BuildPipeline(HttpContext context) { - try - { - var roles = new HashSet(StringComparer.OrdinalIgnoreCase); - using (accessService.ImpersonateAsHub(hub)) + if (context.User?.Identity?.IsAuthenticated != true || IsExcludedPath(context.Request.Path)) + return Observable.Return(OnboardingOutcome.PassThrough); + + var portalApp = context.RequestServices.GetService(); + if (portalApp == null) + return Observable.Return(OnboardingOutcome.PassThrough); + + var accessService = portalApp.Hub.ServiceProvider.GetRequiredService(); + var userContext = accessService.Context ?? accessService.CircuitContext; + + // Skip virtual users — they don't need onboarding. + if (userContext is not { IsVirtual: false } || string.IsNullOrEmpty(userContext.ObjectId)) + return Observable.Return(OnboardingOutcome.PassThrough); + + var email = userContext.Email ?? userContext.ObjectId; + var workspace = portalApp.Hub.GetWorkspace(); + + // Correctness fix + diagnostic (2026-06): we previously short-circuited to + // PassThrough whenever ObjectId != email, ASSUMING such a session was already + // onboarded. That stranded any session carrying a username-shaped identity with + // NO backing User node (a leftover DevLogin cookie, a deleted user, …): the + // middleware never redirected to /onboarding and Index.razor rendered an empty + // Activity area (the "blank screen, never onboards" bug). We now ALWAYS resolve + // the user by email; a missing node ⇒ redirect to onboarding. A genuinely + // onboarded external-auth session carries ObjectId == email here (the cookie's + // NameIdentifier is the email), so its FindUserByEmail lookup finds the node and + // passes through — the only sessions this newly redirects are the stale/unknown + // ones that SHOULD onboard. + logger.LogDebug( + "OnboardingMiddleware: resolving session - ObjectId='{ObjectId}' email='{Email}' isVirtual={IsVirtual} path={Path}", + userContext.ObjectId, email, userContext.IsVirtual, context.Request.Path); + + // Reactive composition: FindUser → SelectMany → either Redirect (no + // node / Transient) or LoadRoles → set context → PassThrough. + return FindUserByEmail(workspace, email, logger) + .SelectMany(node => { - await foreach (var accessNode in meshQuery.QueryAsync( - $"nodeType:AccessAssignment content.accessObject:\"{username}\" scope:subtree limit:10")) + if (node == null || node.State == MeshNodeState.Transient) { - if (accessNode.Content == null) - continue; + logger.LogInformation( + "OnboardingMiddleware: Redirecting to onboarding for {Email} (node={NodeState})", + email, node?.State.ToString() ?? "(null — lookup returned no match)"); + return Observable.Return(OnboardingOutcome.Redirect); + } - AccessAssignment? assignment = accessNode.Content switch + var username = node.Id; + return LoadUserRoles(workspace, username, logger) + .Select(roles => { - AccessAssignment aa => aa, - JsonElement je => JsonSerializer.Deserialize( - je.GetRawText(), hub.JsonSerializerOptions), - _ => null - }; + var updatedContext = userContext with + { + ObjectId = username, + Name = node.Name ?? username, + Roles = roles + }; + // Set per-request context. CircuitAccessHandler handles + // per-circuit persistence via CreateInboundActivityHandler. + accessService.SetContext(updatedContext); + return OnboardingOutcome.PassThrough; + }); + }) + .Catch(ex => + { + // Non-critical — don't block the request on onboarding check failure. + logger.LogWarning(ex, + "OnboardingMiddleware: Failed to check user node for {UserId}", + userContext.ObjectId); + return Observable.Return(OnboardingOutcome.PassThrough); + }); + } - if (assignment == null) - continue; + /// + /// Reactive lookup of the User node by email via the canonical synced query + /// (workspace.GetQuery). The synced layer dedupes by path, gates on + /// Initial, includes static providers, and runs queries with System identity + /// internally — so this RLS-bypassing lookup uses exactly the same machinery + /// as every other "live mesh node set" consumer in the codebase + /// (ApiTokenService.GetTokensForUser, AgentChatClient, etc.). + /// Direct IMeshQueryCore.Query here was a pedestrian-query + /// antipattern — replaced 2026-05 per SyncedMeshNodeQueries.md. + /// + /// Returns rather than + /// so the caller composes the chain; the middleware is the single allowed + /// bridge point (ASP.NET's RequestDelegate is Task-based). + /// + /// Robustness: the synced layer's Initial-gating means the first + /// emission is already the authoritative snapshot — no per-emission Where + /// filter needed. We Take(1) and Timeout (cold start can take seconds while + /// the partition hydrates). Empty snapshot → null → "redirect to + /// /onboarding". + /// + internal static IObservable FindUserByEmail( + IWorkspace workspace, string email, ILogger? logger) + { + // Cache id per-email — synced query result snapshot is shared across + // any concurrent request for the same email. The synced registry holds + // the entry for the workspace's lifetime; live mesh change events keep + // the snapshot fresh, so subsequent requests see up-to-date state. + return workspace.GetQuery( + $"auth:userByEmail:{email}", + $"nodeType:User content.email:{email} limit:1") + .Do(items => logger?.LogDebug( + "FindUserByEmail({Email}): synced query emit, items={Count}", + email, items.Count())) + .Take(1) + .Select(items => (MeshNode?)items.FirstOrDefault()) + .Timeout(LookupTimeout, Observable.Defer(() => + { + logger?.LogWarning( + "FindUserByEmail({Email}): no user node within {Timeout} — falling back to null (will redirect to /onboarding)", + email, LookupTimeout); + return Observable.Return(null); + })); + } - foreach (var r in assignment.Roles.Where(r => !r.Denied && !string.IsNullOrEmpty(r.Role))) - roles.Add(r.Role); - } - } + /// Back-compat overload used by callers that don't yet pass a logger. + internal static IObservable FindUserByEmail( + IWorkspace workspace, string email) + => FindUserByEmail(workspace, email, logger: null); - return roles.ToList(); - } - catch + /// + /// Reactive load of the user's role names from AccessAssignment nodes via the + /// canonical synced query (workspace.GetQuery). Same machinery as + /// — bypasses RLS, dedupes, gates on Initial, + /// includes static providers. Bearer auth uses this via + /// to enrich principals with + /// DB-resolved roles rather than only the roles stamped on the API token at + /// creation time. + /// + internal static IObservable> LoadUserRoles( + IWorkspace workspace, string username, ILogger? logger) + { + var jsonOptions = workspace.Hub.JsonSerializerOptions; + + return workspace.GetQuery( + $"auth:userRoles:{username}", + $"nodeType:AccessAssignment content.accessObject:\"{username}\" scope:subtree limit:10") + .Do(items => logger?.LogDebug( + "LoadUserRoles({User}): synced query emit, items={Count}", + username, items.Count())) + .Take(1) + .Select(items => FoldRoles(items, jsonOptions)) + .Timeout(LookupTimeout, Observable.Defer(() => + { + logger?.LogWarning( + "LoadUserRoles({User}): no snapshot within {Timeout} — defaulting to no roles", + username, LookupTimeout); + return Observable.Return((IReadOnlyCollection)Array.Empty()); + })) + .Catch, Exception>(ex => + { + logger?.LogWarning(ex, "LoadUserRoles({User}) failed — defaulting to no roles", username); + return Observable.Return((IReadOnlyCollection)Array.Empty()); + }); + } + + /// Back-compat overload used by callers that don't yet pass a logger. + internal static IObservable> LoadUserRoles( + IWorkspace workspace, string username) + => LoadUserRoles(workspace, username, logger: null); + + private static IReadOnlyCollection FoldRoles( + IEnumerable items, JsonSerializerOptions options) + { + var roles = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var accessNode in items) { - // Non-critical — return empty roles on failure - return []; + if (accessNode.Content == null) + continue; + + AccessAssignment? assignment = accessNode.Content switch + { + AccessAssignment aa => aa, + JsonElement je => JsonSerializer.Deserialize( + je.GetRawText(), options), + _ => null + }; + + if (assignment == null) + continue; + + foreach (var r in assignment.Roles.Where(r => !r.Denied && !string.IsNullOrEmpty(r.Role))) + roles.Add(r.Role); } + return roles.ToList(); } private static bool IsExcludedPath(PathString path) diff --git a/memex/Memex.Portal.Shared/Authentication/UserOnboardingService.cs b/memex/Memex.Portal.Shared/Authentication/UserOnboardingService.cs new file mode 100644 index 000000000..6e1f84d82 --- /dev/null +++ b/memex/Memex.Portal.Shared/Authentication/UserOnboardingService.cs @@ -0,0 +1,254 @@ +using System; +using System.Reactive.Disposables; +using System.Reactive.Linq; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Authentication; + +/// +/// Materialises a new user's identity in all the places login + routing + partition +/// activation need to find them. Extracted from Onboarding.razor so the dual-write +/// shape is unit-testable end-to-end (see UserOnboardingServiceTests). +/// +/// 100% reactive — IObservable<T> end-to-end. Every method returns a +/// cold observable. Callers Subscribe (and may chain). No await, no +/// FirstAsync().ToTask() — those would block hub message processing on the +/// publishing thread and deadlock under load. +/// See Doc/Architecture/AsynchronousCalls.md. +/// +/// One row, one write: the per-user partition root — +/// {username}.mesh_nodes at (namespace='', id={username}). This is what +/// /{username} resolves to via the standard partition router; it renders the User +/// layout (Activity area) from 's +/// HubConfiguration. The per-user Postgres schema is created lazily on this first write by +/// the path-routing adapter (public.ensure_partition_schema); no explicit +/// Admin/Partition catalog entry is needed. +/// +/// Login finds the user via the Auth mirror, not a second write. The login flow +/// runs nodeType:User (routed to the Auth partition by +/// UserNodeType.AddQueryRoutingRule); the V27 trigger +/// mirror_access_object_to_auth_schema copies this partition-root User row into +/// auth.mesh_nodes automatically. There is therefore NO separate +/// (namespace='User', id={username}) catalog-mirror write — that legacy write routed +/// to an unregistered User first-segment and lazily provisioned a stray user +/// schema distinct from auth (cleaned up + dropped by migration V31). Non-system writes +/// into the User/Auth mirror are blocked by PartitionWriteGuardValidator. +/// +public sealed class UserOnboardingService( + IMeshService meshService, + AccessService accessService, + ILogger? logger = null, + IIconGenerator? iconGenerator = null) +{ + /// + /// Drives the full dual-write. Returns a cold observable that, on Subscribe, + /// creates the three rows in order and emits the per-user-partition-root + /// (path = {username}) as its single value + /// before completing. Errors surface via OnError — callers wrap in a UI + /// Catch block. Subscribe to drive. + /// + public IObservable CreateUser(UserOnboardingRequest request) + { + var username = request.Username; + var fullDisplayName = string.IsNullOrWhiteSpace(request.FullName) ? username : request.FullName!; + var avatarIcon = string.IsNullOrWhiteSpace(request.AvatarUrl) ? null : request.AvatarUrl!.Trim(); + + var userContent = new User + { + FullName = string.IsNullOrWhiteSpace(request.FullName) ? null : request.FullName!.Trim(), + Email = request.Email.Trim(), + Bio = string.IsNullOrWhiteSpace(request.Bio) ? null : request.Bio!.Trim(), + Role = string.IsNullOrWhiteSpace(request.Role) ? null : request.Role!.Trim(), + // Pin the four documentation sections so a new user's Pinned tab opens + // onto a clean grid of doc landing pages (each with its own TOC). + PinnedPaths = ["Doc/Architecture", "Doc/DataMesh", "Doc/GUI", "Doc/AI"], + }; + + var partitionRootNode = new MeshNode(username) + { + Name = fullDisplayName, + NodeType = "User", + State = MeshNodeState.Active, + Icon = avatarIcon, + Content = userContent, + }; + + // Single write: the partition-root User node at (namespace='', id={username}). + // The per-user Postgres schema is auto-created on this first write + // (ensure_partition_schema), and the V27 auth-mirror trigger copies this User row + // into auth.mesh_nodes automatically — so login's `nodeType:User` lookup (routed to + // the Auth partition) finds it WITHOUT a separate catalog-mirror write. + // + // The old `new MeshNode(username, "User")` catalog-mirror write is GONE: it routed + // to the unregistered `User` first-segment and lazily provisioned a stray `user` + // schema separate from `auth` (migration V31 unifies that back into `auth` and drops + // it). Writes into the User/Auth mirror by non-system callers are now blocked by + // PartitionWriteGuardValidator; onboarding stays clean by simply not writing there. + // + // Wrap in Observable.Using + ImpersonateAsSystem so onboarding runs as the System + // identity (Permission.All): the new user / their brand-new partition root don't + // exist yet, so neither the signed-in admin nor the user-being-onboarded can hold + // Create on it — the canonical infrastructure-write case (see AccessService.cs). + return Observable.Using( + () => accessService.ImpersonateAsSystem(), + _ => meshService.CreateNode(partitionRootNode) + .Do(__ => logger?.LogInformation( + "Onboarding: wrote partition-root User '{Username}' to {Schema}.mesh_nodes", + username, username.ToLowerInvariant())) + // Idempotent / self-repairing: a leftover partition-root (a pre-gate + // activity-tracking row, or a half-finished prior onboarding) must not + // dead-end onboarding with "Node already exists" — fold into an update so + // the row is brought to the intended User content and onboarding completes. + .Catch(ex => + ex.Message.Contains("already exists", StringComparison.OrdinalIgnoreCase) + ? meshService.UpdateNode(partitionRootNode) + .Do(__ => logger?.LogInformation( + "Onboarding: partition-root User '{Username}' already existed — repaired via update", + username)) + : Observable.Throw(ex))) + // Best-effort: once the user node exists, generate an inline-SVG avatar in the + // background (the configurable utility model, via IIconGenerator → NodeInitializer) + // and stamp it onto the node's Icon — exactly like thread auto-naming runs AFTER a + // thread is created. Skipped when the user supplied an avatar or no generator is wired. + .Do(rootNode => MaybeGenerateAvatar(rootNode, fullDisplayName, avatarIcon)); + } + + /// + /// Fire-and-forget avatar generation for a freshly-created User node. Reuses the existing + /// (the NodeInitializer agent on the configurable utility + /// model) to produce an inline SVG, then writes it to the node's + /// as System (the brand-new partition root has no usable caller identity on this background + /// callback). Best-effort: bounded by a timeout and swallows failures so a missing/un-configured + /// utility model never blocks onboarding — the user simply keeps the initials fallback avatar. + /// + private void MaybeGenerateAvatar(MeshNode userNode, string displayName, string? providedIcon) + { + if (iconGenerator is null || !string.IsNullOrWhiteSpace(providedIcon)) + return; + + iconGenerator + .GenerateSvgAsync(displayName, $"A friendly, minimal circular profile avatar for {displayName}") + .Timeout(TimeSpan.FromSeconds(45)) + .Subscribe( + svg => + { + using (accessService.ImpersonateAsSystem()) + meshService.UpdateNode(userNode with { Icon = svg }) + .Subscribe( + _ => logger?.LogInformation("Onboarding: generated avatar for '{User}'", userNode.Id), + ex => logger?.LogWarning(ex, "Onboarding: avatar Icon write failed for '{User}'", userNode.Id)); + }, + ex => logger?.LogInformation( + ex, "Onboarding: avatar generation skipped for '{User}' (no utility model or error)", userNode.Id)); + } + + /// + /// Self-AccessAssignment write — the new user gets Admin on their own scope. + /// Lives in the per-user partition's access satellite. Without this, + /// the user can read their own partition root (public read on User nodes) + /// but every subsequent write ("Create permission required") fails. + /// Returns a cold observable that emits the created AccessAssignment node; + /// subscribe to drive. + /// + public IObservable GrantSelfAdmin(string username) + { + var assignment = new MeshNode($"{username}_Access", $"{username}/_Access") + { + NodeType = "AccessAssignment", + Name = $"{username} Access", + MainNode = username, + Content = new AccessAssignment + { + AccessObject = username, + DisplayName = username, + Roles = [new RoleAssignment { Role = Role.Admin.Id, Denied = false }] + } + }; + // Self-impersonate as System: granting a brand-new user access to their own (possibly + // just-created) partition is an infrastructure write — the caller may have no usable + // identity (server-side bootstrap) or only hub identity (interactive onboarding). Same + // justification + pattern as CreateUser above. PostPipeline fails closed without a + // context, so this MUST set one explicitly rather than rely on the caller. + return Observable.Using( + () => accessService.ImpersonateAsSystem(), + _ => meshService.CreateNode(assignment) + // Idempotent / self-repairing: a leftover _Access from a half-finished prior + // onboarding (the user retried) must not dead-end with "Node already exists" — + // fold into an update so the grant reaches the intended content and onboarding + // completes. Same pattern as CreateUser's partition-root write above. + .Catch(ex => + ex.Message.Contains("already exists", StringComparison.OrdinalIgnoreCase) + ? meshService.UpdateNode(assignment) + : Observable.Throw(ex)) + .Do(__ => logger?.LogInformation( + "Onboarding: granted self-Admin to '{Username}' at {Path}", username, assignment.Path))); + } + + /// + /// First-user-only: grants the user global admin by making them an admin on the + /// Admin partition — an at namespace + /// Admin/_Access (scope Admin) with MainNode="". The exact shape + /// writes for config-driven admins, and the one + /// canonical platform-admin grant: PermissionEvaluator's global-admin + /// short-circuit turns at scope + /// Admin into a platform superuser (All on every path), and + /// hub.IsGlobalAdmin() reads the same scope. See + /// Doc/Architecture/AccessControl.md → "The Admin partition". + /// + /// Caller gates this on the "no existing User nodes" check. Subscribe to drive — + /// a silent failure would leave the platform with no admins, so callers must surface + /// OnError. + /// + public IObservable GrantPlatformAdmin(string username) + { + // Global admin = admin on the ADMIN PARTITION: namespace "Admin/_Access" (scope + // "Admin") + MainNode "". PermissionEvaluator's global-admin short-circuit turns + // Permission.All at scope "Admin" into platform superuser (All on every path), + // so this single grant gives the first user the platform gates + // (hub.IsGlobalAdmin) AND cross-partition power. Mirrors GlobalAdminSeed's + // config-admin shape. See Doc/Architecture/AccessControl.md. + var assignment = new MeshNode($"{username}_Access", "Admin/_Access") + { + NodeType = "AccessAssignment", + Name = $"{username} — Admin", + MainNode = "", + Content = new AccessAssignment + { + AccessObject = username, + DisplayName = username, + Roles = [new RoleAssignment { Role = Role.Admin.Id, Denied = false }] + } + }; + // Self-impersonate as System (see GrantSelfAdmin) — the first-user platform-Admin grant + // is an infrastructure write that must succeed even when no user identity is on the + // caller (server-side bootstrap) or only a hub identity (interactive onboarding). + return Observable.Using( + () => accessService.ImpersonateAsSystem(), + _ => meshService.CreateNode(assignment) + // Idempotent / self-repairing (see GrantSelfAdmin): a retried first-user + // onboarding must not fail because Admin/_Access/{user}_Access already exists. + .Catch(ex => + ex.Message.Contains("already exists", StringComparison.OrdinalIgnoreCase) + ? meshService.UpdateNode(assignment) + : Observable.Throw(ex)) + .Do(__ => logger?.LogInformation( + "Onboarding: granted platform Admin (first user) to '{Username}' on the Admin partition (Admin/_Access, MainNode=\"\")", username))); + } +} + +/// +/// Input shape for . Mirrors the form +/// model in Onboarding.razor; kept in this assembly so unit tests can +/// construct it without taking a dependency on the Blazor page. +/// +public sealed record UserOnboardingRequest( + string Username, + string Email, + string? FullName = null, + string? Bio = null, + string? Role = null, + string? AvatarUrl = null); diff --git a/memex/Memex.Portal.Shared/Authentication/UserRoleResolver.cs b/memex/Memex.Portal.Shared/Authentication/UserRoleResolver.cs new file mode 100644 index 000000000..30ded94c3 --- /dev/null +++ b/memex/Memex.Portal.Shared/Authentication/UserRoleResolver.cs @@ -0,0 +1,57 @@ +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using MeshWeaver.Data; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; + +namespace Memex.Portal.Shared.Authentication; + +/// +/// Thin façade over so callers +/// outside this assembly can resolve a user's AccessAssignment-derived roles in +/// one call. +/// +/// Used by to enrich Bearer +/// principals with DB-resolved roles, so MCP / API-token sessions see the +/// same role set as cookie / OAuth sessions. Without this layer, roles would +/// be limited to whatever was stamped on the API token at creation time — +/// any later AccessAssignment grant would silently not apply for Bearer +/// requests, even though the same user logging in through a browser would +/// see them. +/// +/// Resolution goes through the canonical synced-query API +/// (workspace.GetQuery) — same path-keyed dedup + Initial gating + static +/// provider fan-out as every other live mesh-node collection consumer in the +/// codebase. Direct IMeshQueryCore.Query calls from auth code +/// were a pedestrian-query antipattern (replaced 2026-05). +/// +internal static class UserRoleResolver +{ + /// + /// Resolves the user's AccessAssignment-derived role names. Returns an + /// empty list when no resolution is possible (services missing, workspace + /// unavailable, query layer faulted) — auth flows must keep working even + /// when role enrichment can't. + /// + /// The single Task bridge here lives at the ASP.NET + /// AuthenticationHandler.HandleAuthenticateAsync boundary — + /// callers expect a Task-returning helper, but everything below + /// stays observable. + /// + public static async Task> LoadDbRolesAsync( + IServiceProvider services, string userId) + { + var hub = services.GetService(); + if (hub is null || string.IsNullOrEmpty(userId)) + return Array.Empty(); + + var workspace = hub.GetWorkspace(); + if (workspace is null) + return Array.Empty(); + + return await OnboardingMiddleware + .LoadUserRoles(workspace, userId) + .FirstAsync() + .ToTask(); + } +} diff --git a/memex/Memex.Portal.Shared/Authentication/VirtualUserMiddleware.cs b/memex/Memex.Portal.Shared/Authentication/VirtualUserMiddleware.cs index e8cbddb3c..35ef60da2 100644 --- a/memex/Memex.Portal.Shared/Authentication/VirtualUserMiddleware.cs +++ b/memex/Memex.Portal.Shared/Authentication/VirtualUserMiddleware.cs @@ -24,7 +24,7 @@ public class VirtualUserMiddleware(RequestDelegate next, ILogger(); + + // 🚨 Skip the entire VUser flow when a real-user identity is + // already on AccessService — `UserContextMiddleware` runs + // BEFORE us in the pipeline (see MemexConfiguration.cs) and + // resolves OAuth / Bearer / mesh-User-by-email into + // AccessService.Context. Real users sometimes have + // `context.User.Identity.IsAuthenticated == false` (e.g., + // the ASP.NET cookie expired but a Bearer token in the + // request still resolves a valid user via the cache); without + // this guard we'd wastefully provision a guest VUser node + // for them AND post the CreateNodeRequest into the portal + // hub, which has no handler — the crash the user just hit + // on the sub-thread URL. + var preExisting = accessService.Context ?? accessService.CircuitContext; + if (preExisting is not null && !preExisting.IsVirtual + && !string.IsNullOrEmpty(preExisting.ObjectId)) + { + await next(context); + return; + } + var (virtualUserId, isNew) = GetOrCreateVirtualUserId(context); // Fast path: if the circuit already has a context for this virtual user, reuse it // (avoids mesh call on every request — only needed once per circuit). - var existing = accessService.Context ?? accessService.CircuitContext; - if (existing is not null && existing.ObjectId == virtualUserId && existing.IsVirtual) + if (preExisting is not null && preExisting.ObjectId == virtualUserId && preExisting.IsVirtual) { - accessService.SetContext(existing); + accessService.SetContext(preExisting); await next(context); return; } - // First request in this circuit — ensure VUser node exists. - // Runs once per page load; subsequent requests reuse CircuitContext above. - await EnsureVirtualUserNodeAsync(portalApp, virtualUserId); + // 🚨 Mint the VUser NODE only on cookie ROUND-TRIP (isNew == false): + // the node (a mesh write + a per-node hub graph) is created the + // first time the cookie COMES BACK, proving a cookie-keeping + // browser session. First-contact requests get the cookie + an + // in-memory guest context only. Without this gate, every + // cookie-less client minted a fresh node PER REQUEST — kube-probe + // alone created ~15 VUsers/minute (and any crawler does the same), + // leaking 10,000+ hubs until the portal wedged at 100% CPU + // (2026-06-12 atioz outage). A real visitor's node exists by + // their second request — before the Blazor circuit needs it. + // + // 100% reactive, NO await: EnsureVUserNode composes + // hub.Observe(CreateNodeRequest).Subscribe(...) internally — the + // request thread never waits on mesh work (AsynchronousCalls.md; + // an await here parks the request on the hub pump = deadlock). + if (!isNew) + EnsureVirtualUserNode(portalApp, virtualUserId); var portalIdentity = portalApp.Hub.Address.ToFullString(); var virtualContext = new AccessContext @@ -107,11 +141,11 @@ private static (string id, bool isNew) GetOrCreateVirtualUserId(HttpContext cont return (newId, true); } - private async Task EnsureVirtualUserNodeAsync(PortalApplication portalApp, string virtualUserId) + private void EnsureVirtualUserNode(PortalApplication portalApp, string virtualUserId) { try { - await VUserHelper.EnsureVUserNodeAsync(portalApp, virtualUserId, logger); + VUserHelper.EnsureVUserNode(portalApp, virtualUserId, logger); } catch (Exception ex) { diff --git a/memex/Memex.Portal.Shared/Email/EmailInboundProcessor.cs b/memex/Memex.Portal.Shared/Email/EmailInboundProcessor.cs new file mode 100644 index 000000000..916fd5db9 --- /dev/null +++ b/memex/Memex.Portal.Shared/Email/EmailInboundProcessor.cs @@ -0,0 +1,269 @@ +using System.Reactive; +using System.Reactive.Disposables; +using System.Reactive.Linq; +using System.Text.RegularExpressions; +using MeshWeaver.AI; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Graph.Models; + +namespace Memex.Portal.Shared.Email; + +/// +/// Routes an inbound email by sender, with all matching driven through — +/// no in-memory registries. +/// +/// email → user: structured exact query nodeType:User content.email:{from}. +/// email → conversation: the subject is matched via the vector index (bare-text query +/// scoped to the sender's mail), and the nearest prior email is confirmed as the same +/// conversation by comparing the normalized-subject key (reply/forward prefixes stripped). One +/// thread per conversation: a match continues that thread, otherwise a new one is started. +/// +/// A non-user's mail goes to the admin inbox (Admin/Inbox) with an admin notification. Every +/// message is persisted as an node and then marked read. +/// +/// 100% reactive (no await); Graph HTTP I/O is pooled in ; +/// writes wrap in ImpersonateAsSystem. is Graph-free + public so the +/// routing/threading is unit-testable. The agent's reply is sent back by the separate +/// OutboundEmailSender (mesh-driven, see that type) — this processor only does intake. +/// +public sealed class EmailInboundProcessor( + IMessageHub hub, + GraphMail graphMail, + ILogger? logger = null) +{ + /// The dedicated email parser agent (see Agent/EmailRouter.md). + public const string ParserAgent = "EmailRouter"; + + private readonly IMeshService meshService = hub.ServiceProvider.GetRequiredService(); + private readonly AccessService accessService = hub.ServiceProvider.GetRequiredService(); + private readonly IMeshQueryCore query = hub.ServiceProvider.GetRequiredService(); + + /// A parsed inbound email — the Graph-free input to . + public sealed record InboundMessage( + string From, string? FromName, string Subject, string Body, + string? ConversationId, string? InternetMessageId); + + /// Fetch the message, route it, mark it read. Cold; subscribe to drive. + public IObservable ProcessNotification(string messageId) => + graphMail.GetMessage(messageId) + .SelectMany(msg => + { + if (msg is null) return Observable.Return(Unit.Default); + var inbound = new InboundMessage( + From: msg.From?.EmailAddress?.Address?.Trim() ?? "", + FromName: msg.From?.EmailAddress?.Name, + Subject: msg.Subject ?? "(no subject)", + Body: msg.Body?.Content ?? "", + ConversationId: msg.ConversationId, + InternetMessageId: msg.InternetMessageId); + return Route(inbound).SelectMany(_ => graphMail.MarkRead(messageId)); + }) + .Catch((Exception ex) => + { + logger?.LogWarning(ex, "EmailInbound: failed to process message {MessageId}", messageId); + return Observable.Return(Unit.Default); + }); + + /// Routes a parsed message by sender (no Graph). Unit-testable. + public IObservable Route(InboundMessage m) + { + if (string.IsNullOrEmpty(m.From)) return Observable.Return(Unit.Default); + // Loop guard: never act on our own mail. + if (string.Equals(m.From, graphMail.Mailbox, StringComparison.OrdinalIgnoreCase)) + return Observable.Return(Unit.Default); + + // email → user: structured exact match through IMeshQueryCore. + return query.Query(MeshQueryRequest.FromQuery( + $"nodeType:{UserNodeType.NodeType} content.email:{m.From} limit:1"), + hub.JsonSerializerOptions) + .Take(1) + .Select(change => change.Items.FirstOrDefault(n => n.State == MeshNodeState.Active)) + .SelectMany(userNode => userNode is not null + ? HandleUser(userNode.Id, m) + : HandleNonUser(m)); + } + + // ── Known user → email node + (matched | new) conversation thread + notification ── + private IObservable HandleUser(string username, InboundMessage m) + { + var key = ThreadKey(m.Subject); + var emailNs = $"{username}/{EmailNodeType.UserEmailSegment}"; + + return FindConversationThread(username, m.Subject, key).SelectMany(matchedThreadPath => + Observable.Using( + () => accessService.ImpersonateAsSystem(), + _ => + { + var text = "Process this inbound email and reply to the sender.\n\nEmail: @/{0}"; + if (!string.IsNullOrEmpty(matchedThreadPath)) + { + // Same conversation → continue its thread + record the inbound email. + var emailNode = BuildEmailNode(emailNs, m, EmailStatus.Read, matchedThreadPath, key); + return meshService.CreateNode(emailNode).SelectMany(saved => + { + hub.SubmitMessage(matchedThreadPath, string.Format(text, saved.Path), + agentName: ParserAgent, contextPath: saved.Path, + createdBy: username, authorName: m.FromName ?? m.From, + onError: err => logger?.LogWarning("EmailInbound: SubmitMessage failed: {Err}", err)); + return CreateNotification(username, matchedThreadPath, m); + }); + } + + // New conversation → create email, then a thread with the email as MainNode. + var newEmail = BuildEmailNode(emailNs, m, EmailStatus.Read, threadPath: null, key); + return meshService.CreateNode(newEmail).SelectMany(saved => + StartThreadRx(username, string.Format(text, saved.Path), saved.Path).SelectMany(thread => + // backfill the email's ThreadPath now that the thread exists + meshService.UpdateNode(WithThreadPath(saved, thread.Path)) + .SelectMany(_ => CreateNotification(username, thread.Path, m)))); + })); + } + + // ── Non-user → admin inbox + admin notification (no agent) ──────────────── + private IObservable HandleNonUser(InboundMessage m) + { + var emailNode = BuildEmailNode(EmailNodeType.AdminInboxNamespace, m, EmailStatus.New, + threadPath: null, ThreadKey(m.Subject)); + return Observable.Using( + () => accessService.ImpersonateAsSystem(), + _ => meshService.CreateNode(emailNode) + .SelectMany(saved => CreateNotification("Admin", saved.Path, m)) + .Do(_ => logger?.LogInformation("EmailInbound: filed non-user mail from {From} to {Inbox}", + m.From, EmailNodeType.AdminInboxNamespace))); + } + + /// + /// Vector-matches the subject against the sender's prior mail (HNSW when an embedding provider is + /// present, ILIKE fallback otherwise — see VectorSearch.md), then confirms the + /// candidate is the same conversation by normalized-subject key. Returns the existing thread path, + /// or null to start a new conversation. + /// + private IObservable FindConversationThread(string username, string subject, string key) + { + var safeSubject = SanitizeForQuery(subject); + var q = $"{safeSubject} nodeType:{EmailNodeType.NodeType} namespace:{username}/{EmailNodeType.UserEmailSegment} limit:10"; + var jsonOptions = hub.JsonSerializerOptions; + return query.Query(MeshQueryRequest.FromQuery(q), jsonOptions) + .Take(1) + .Select(change => change.Items + .Select(n => EmailOf(n, jsonOptions)) + .Where(e => e is { ThreadPath: { Length: > 0 } } && e.ThreadKey == key) + .Select(e => e!.ThreadPath) + .FirstOrDefault()); + } + + /// Bridges 's onCreated callback to an observable. + private IObservable StartThreadRx(string username, string text, string emailPath) => + Observable.Create(observer => + { + hub.StartThread(username, text, + agentName: ParserAgent, + contextPath: emailPath, + mainNode: emailPath, + createdBy: username, + onCreated: node => { observer.OnNext(node); observer.OnCompleted(); }, + onError: err => observer.OnError(new InvalidOperationException(err))); + return Disposable.Empty; + }); + + private MeshNode BuildEmailNode( + string namespacePath, InboundMessage m, EmailStatus status, string? threadPath, string threadKey) => + new(Guid.NewGuid().ToString("N"), namespacePath) + { + Name = m.Subject, + NodeType = EmailNodeType.NodeType, + State = MeshNodeState.Active, + Content = new MeshWeaver.Mesh.Email + { + Direction = EmailDirection.Inbound, + From = m.From, + FromName = m.FromName, + To = graphMail.Mailbox, + Subject = m.Subject, + Body = m.Body, + ConversationId = m.ConversationId, + InternetMessageId = m.InternetMessageId, + Status = status, + ThreadPath = threadPath, + ThreadKey = threadKey, + } + }; + + private static MeshNode WithThreadPath(MeshNode emailNode, string threadPath) => + emailNode.Content is MeshWeaver.Mesh.Email e + ? emailNode with { Content = e with { ThreadPath = threadPath } } + : emailNode; + + /// Creates a bell Notification under {ownerPartition}/_Notification as System. + private IObservable CreateNotification(string ownerPartition, string targetPath, InboundMessage m) + { + var node = new MeshNode(Guid.NewGuid().ToString("N"), $"{ownerPartition}/_Notification") + { + NodeType = "Notification", + Name = m.Subject, + MainNode = targetPath, + State = MeshNodeState.Active, + Content = new MeshWeaver.Mesh.Notification + { + Title = $"New email from {m.FromName ?? m.From}", + Message = m.Subject, + TargetNodePath = targetPath, + NotificationType = MeshWeaver.Mesh.NotificationType.General, + CreatedBy = m.From, + } + }; + return meshService.CreateNode(node).Select(_ => Unit.Default); + } + + private static MeshWeaver.Mesh.Email? EmailOf(MeshNode n, System.Text.Json.JsonSerializerOptions? options) => + n.Content switch + { + MeshWeaver.Mesh.Email e => e, + System.Text.Json.JsonElement je => SafeDeserialize(je, options), + _ => null + }; + + private static MeshWeaver.Mesh.Email? SafeDeserialize(System.Text.Json.JsonElement je, System.Text.Json.JsonSerializerOptions? options) + { + try { return System.Text.Json.JsonSerializer.Deserialize(je.GetRawText(), options); } + catch { return null; } + } + + // ── Subject normalization → stable conversation key ─────────────────────── + // Strips ANY number of leading reply/forward markers (several languages), e.g. + // "Re: Fwd: AW: Re[2]: Hello" → "hello". + private static readonly Regex ReplyPrefix = new( + @"^\s*(re|fwd|fw|aw|wg|tr|rv|sv|vs|antw|antwort|rif)(\s*\[\d+\])?\s*:\s*", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + /// Normalizes a subject (strip reply/forward prefixes) and slugifies it into a stable key. + public static string ThreadKey(string subject) + { + var s = subject ?? ""; + string prev; + do { prev = s; s = ReplyPrefix.Replace(s, ""); } while (s != prev); + s = s.Trim().ToLowerInvariant(); + var slug = new string(s.Select(c => char.IsLetterOrDigit(c) ? c : '-').ToArray()); + slug = Regex.Replace(slug, "-{2,}", "-").Trim('-'); + if (slug.Length == 0) slug = "email"; + return slug.Length > 64 ? slug[..64].Trim('-') : slug; + } + + /// Strips characters that the mesh-query parser would misread (so the subject stays bare-text). + private static string SanitizeForQuery(string subject) + { + var normalized = subject ?? ""; + string prev; + do { prev = normalized; normalized = ReplyPrefix.Replace(normalized, ""); } while (normalized != prev); + var chars = normalized.Select(c => char.IsLetterOrDigit(c) || c == ' ' ? c : ' ').ToArray(); + return Regex.Replace(new string(chars), @"\s{2,}", " ").Trim(); + } +} diff --git a/memex/Memex.Portal.Shared/Email/EmailWebhookController.cs b/memex/Memex.Portal.Shared/Email/EmailWebhookController.cs new file mode 100644 index 000000000..6693525b6 --- /dev/null +++ b/memex/Memex.Portal.Shared/Email/EmailWebhookController.cs @@ -0,0 +1,86 @@ +using System.Reactive.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using MeshWeaver.Mesh; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Email; + +/// +/// Receives Microsoft Graph change notifications for the mailbox inbox. Anonymous (Graph posts are +/// unauthenticated and validated by the shared clientState) — /api/email is in the +/// onboarding middleware's excluded prefixes and outside the MCP bearer policy. +/// +/// Two shapes: the subscription-creation handshake (?validationToken=… → echo it back as +/// text/plain), and notification batches (validate clientState, hand each message id to +/// via a fire-and-forget Subscribe, return 202 immediately). +/// +[ApiController] +[AllowAnonymous] +[Route("api/email")] +public sealed class EmailWebhookController( + EmailInboundProcessor processor, + EmailOptions options, + ILogger logger) : ControllerBase +{ + private static readonly JsonSerializerOptions JsonOpts = new() { PropertyNameCaseInsensitive = true }; + + [HttpPost] + public async Task Post([FromQuery] string? validationToken) + { + // 1) Subscription-creation handshake — echo the token within 10s as text/plain. + if (!string.IsNullOrEmpty(validationToken)) + return Content(validationToken, "text/plain"); + + // 2) Notification batch. + string json; + using (var reader = new StreamReader(Request.Body)) + json = await reader.ReadToEndAsync(); + + GraphNotificationBatch? batch; + try { batch = JsonSerializer.Deserialize(json, JsonOpts); } + catch (Exception ex) + { + logger.LogWarning(ex, "EmailWebhook: unparseable notification body"); + return BadRequest(); + } + + foreach (var n in batch?.Value ?? []) + { + if (!string.Equals(n.ClientState, options.SubscriptionClientState, StringComparison.Ordinal)) + { + logger.LogWarning("EmailWebhook: clientState mismatch — ignoring notification"); + continue; + } + var messageId = n.ResourceData?.Id; + if (string.IsNullOrEmpty(messageId)) continue; + + // Fire-and-forget: process off the request thread; we ack immediately. + processor.ProcessNotification(messageId).Subscribe( + _ => { }, + ex => logger.LogWarning(ex, "EmailWebhook: processing failed for {MessageId}", messageId)); + } + + return Accepted(); + } + + // --- Graph notification payload shape --- + public sealed class GraphNotificationBatch + { + [JsonPropertyName("value")] public List? Value { get; set; } + } + + public sealed class GraphNotification + { + [JsonPropertyName("subscriptionId")] public string? SubscriptionId { get; set; } + [JsonPropertyName("clientState")] public string? ClientState { get; set; } + [JsonPropertyName("resourceData")] public GraphResourceData? ResourceData { get; set; } + } + + public sealed class GraphResourceData + { + [JsonPropertyName("id")] public string? Id { get; set; } + } +} diff --git a/memex/Memex.Portal.Shared/Email/ExecutiveAssistantPlugin.cs b/memex/Memex.Portal.Shared/Email/ExecutiveAssistantPlugin.cs new file mode 100644 index 000000000..bae2454b4 --- /dev/null +++ b/memex/Memex.Portal.Shared/Email/ExecutiveAssistantPlugin.cs @@ -0,0 +1,273 @@ +using System.ComponentModel; +using System.Text.Json; +using Azure.Core; +using MeshWeaver.AI.Plugins; +using MeshWeaver.Mesh; +using MeshWeaver.Messaging; +using Memex.Portal.Shared.Authentication; +using Microsoft.Extensions.AI; +using Microsoft.Graph; +using Microsoft.Graph.Models; +using Microsoft.Graph.Me.SendMail; +using Microsoft.Graph.Me.Messages.Item.Reply; + +namespace Memex.Portal.Shared.Email; + +/// +/// Executive Assistant agent tool: read/write the signed-in user's own mailbox and calendar +/// via Microsoft Graph using a per-user delegated token (the user consents just-in-time — see +/// ). Every call targets /me with the acting user's own token; there is no +/// standing application-wide Graph access. When the user has not yet connected, each tool returns a consent +/// link instead of acting. +/// +/// These methods are agent tools (the Microsoft.Extensions.AI boundary), so async/await +/// is appropriate — not hub-reachable code. +/// +public sealed class ExecutiveAssistantPlugin( + IEaGraphAuth ea, AccessService access, EmailOptions options) : IAgentPlugin +{ + public string Name => "ExecutiveAssistant"; + + private string? Me => access.Context?.ObjectId ?? access.Context?.Name + ?? access.CircuitContext?.ObjectId ?? access.CircuitContext?.Name; + + private string ConsentLink => + $"{(options.WebhookBaseUrl ?? "").TrimEnd('/')}/auth/ea/connect"; + + /// Builds a Graph client bound to the acting user's delegated token, or a "please connect" message. + private async Task<(GraphServiceClient? graph, string? notConnected)> ClientAsync() + { + if (Me is not { } me) return (null, "There is no signed-in user to act for."); + var token = await ea.GetAccessTokenAsync(me, CancellationToken.None); + if (string.IsNullOrEmpty(token)) + return (null, "I don't have access to your mailbox and calendar yet. Please connect them here: " + + $"{ConsentLink} — it takes a few seconds, then ask me again."); + return (new GraphServiceClient(new StaticTokenCredential(token!)), null); + } + + // ---- Email ------------------------------------------------------------ + + [Description("Lists the signed-in user's most recent inbox emails (newest first) with id, from, subject, received time, and a short preview.")] + public async Task ListInbox([Description("How many messages to return (default 10, max 50)")] int count = 10) + { + var (graph, notConnected) = await ClientAsync(); + if (graph is null) return notConnected!; + try + { + var page = await graph.Me.MailFolders["inbox"].Messages.GetAsync(rc => + { + rc.QueryParameters.Top = Math.Clamp(count, 1, 50); + rc.QueryParameters.Orderby = ["receivedDateTime desc"]; + rc.QueryParameters.Select = ["id", "subject", "from", "receivedDateTime", "bodyPreview", "isRead"]; + }); + return Json(page?.Value?.Select(m => new + { + id = m.Id, + from = m.From?.EmailAddress?.Address, + subject = m.Subject, + received = m.ReceivedDateTime, + isRead = m.IsRead, + preview = m.BodyPreview + })); + } + catch (Exception ex) { return Fail(nameof(ListInbox), ex); } + } + + [Description("Searches the signed-in user's mailbox by free text (subject/body/sender) and returns matching messages.")] + public async Task SearchMail( + [Description("Search text")] string query, + [Description("How many results (default 10, max 50)")] int count = 10) + { + var (graph, notConnected) = await ClientAsync(); + if (graph is null) return notConnected!; + try + { + var page = await graph.Me.Messages.GetAsync(rc => + { + rc.QueryParameters.Search = $"\"{query}\""; + rc.QueryParameters.Top = Math.Clamp(count, 1, 50); + rc.QueryParameters.Select = ["id", "subject", "from", "receivedDateTime", "bodyPreview"]; + }); + return Json(page?.Value?.Select(m => new + { + id = m.Id, from = m.From?.EmailAddress?.Address, subject = m.Subject, + received = m.ReceivedDateTime, preview = m.BodyPreview + })); + } + catch (Exception ex) { return Fail(nameof(SearchMail), ex); } + } + + [Description("Reads the full body of one of the signed-in user's emails by id.")] + public async Task ReadMail([Description("Message id")] string id) + { + var (graph, notConnected) = await ClientAsync(); + if (graph is null) return notConnected!; + try + { + var m = await graph.Me.Messages[id].GetAsync(rc => + rc.QueryParameters.Select = ["id", "subject", "from", "toRecipients", "receivedDateTime", "body"]); + return Json(new + { + id = m?.Id, from = m?.From?.EmailAddress?.Address, + to = m?.ToRecipients?.Select(r => r.EmailAddress?.Address), + subject = m?.Subject, received = m?.ReceivedDateTime, body = m?.Body?.Content + }); + } + catch (Exception ex) { return Fail(nameof(ReadMail), ex); } + } + + [Description("Sends an email from the signed-in user's mailbox.")] + public async Task SendMail( + [Description("Recipient address(es), comma-separated")] string to, + [Description("Subject line")] string subject, + [Description("Body (HTML allowed)")] string body, + [Description("Optional CC address(es), comma-separated")] string? cc = null) + { + var (graph, notConnected) = await ClientAsync(); + if (graph is null) return notConnected!; + try + { + await graph.Me.SendMail.PostAsync(new SendMailPostRequestBody + { + Message = new Message + { + Subject = subject, + Body = new ItemBody { ContentType = BodyType.Html, Content = body }, + ToRecipients = Recipients(to), + CcRecipients = cc is null ? null : Recipients(cc) + }, + SaveToSentItems = true + }); + return $"Sent to {to}."; + } + catch (Exception ex) { return Fail(nameof(SendMail), ex); } + } + + [Description("Replies to one of the signed-in user's emails by id (replies to the sender).")] + public async Task ReplyToMail( + [Description("Message id to reply to")] string id, + [Description("Reply body")] string body) + { + var (graph, notConnected) = await ClientAsync(); + if (graph is null) return notConnected!; + try + { + await graph.Me.Messages[id].Reply.PostAsync(new ReplyPostRequestBody { Comment = body }); + return "Reply sent."; + } + catch (Exception ex) { return Fail(nameof(ReplyToMail), ex); } + } + + // ---- Calendar --------------------------------------------------------- + + [Description("Lists the signed-in user's calendar events in a window (default: next 7 days). Times are ISO 8601 UTC.")] + public async Task ListEvents( + [Description("Window start, ISO 8601 (default: now)")] string? startUtc = null, + [Description("Window end, ISO 8601 (default: 7 days from start)")] string? endUtc = null) + { + var (graph, notConnected) = await ClientAsync(); + if (graph is null) return notConnected!; + try + { + var start = ParseOrDefault(startUtc, DateTimeOffset.UtcNow); + var end = ParseOrDefault(endUtc, start.AddDays(7)); + var page = await graph.Me.CalendarView.GetAsync(rc => + { + rc.QueryParameters.StartDateTime = start.ToString("o"); + rc.QueryParameters.EndDateTime = end.ToString("o"); + rc.QueryParameters.Orderby = ["start/dateTime"]; + rc.QueryParameters.Select = ["id", "subject", "start", "end", "location", "attendees", "isAllDay"]; + rc.QueryParameters.Top = 100; + }); + return Json(page?.Value?.Select(e => new + { + id = e.Id, subject = e.Subject, + start = e.Start?.DateTime, end = e.End?.DateTime, tz = e.Start?.TimeZone, + location = e.Location?.DisplayName, + attendees = e.Attendees?.Select(a => a.EmailAddress?.Address) + })); + } + catch (Exception ex) { return Fail(nameof(ListEvents), ex); } + } + + [Description("Creates a calendar event ('books' a meeting) on the signed-in user's calendar and invites attendees.")] + public async Task CreateEvent( + [Description("Event title")] string subject, + [Description("Start time, ISO 8601 (UTC unless an offset is given)")] string startIso, + [Description("End time, ISO 8601")] string endIso, + [Description("Attendee address(es), comma-separated (optional)")] string? attendees = null, + [Description("Location (optional)")] string? location = null, + [Description("Body / agenda (optional)")] string? body = null) + { + var (graph, notConnected) = await ClientAsync(); + if (graph is null) return notConnected!; + try + { + var ev = new Event + { + Subject = subject, + Start = new DateTimeTimeZone { DateTime = ParseOrDefault(startIso, DateTimeOffset.UtcNow).ToString("o"), TimeZone = "UTC" }, + End = new DateTimeTimeZone { DateTime = ParseOrDefault(endIso, DateTimeOffset.UtcNow.AddHours(1)).ToString("o"), TimeZone = "UTC" }, + Location = location is null ? null : new Location { DisplayName = location }, + Body = body is null ? null : new ItemBody { ContentType = BodyType.Html, Content = body }, + Attendees = string.IsNullOrWhiteSpace(attendees) + ? null + : attendees.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(a => new Attendee { EmailAddress = new EmailAddress { Address = a }, Type = AttendeeType.Required }) + .ToList() + }; + var created = await graph.Me.Events.PostAsync(ev); + return $"Created event '{subject}' (id {created?.Id})."; + } + catch (Exception ex) { return Fail(nameof(CreateEvent), ex); } + } + + [Description("Cancels/deletes a calendar event on the signed-in user's calendar by id.")] + public async Task CancelEvent([Description("Event id")] string id) + { + var (graph, notConnected) = await ClientAsync(); + if (graph is null) return notConnected!; + try + { + await graph.Me.Events[id].DeleteAsync(); + return "Event cancelled."; + } + catch (Exception ex) { return Fail(nameof(CancelEvent), ex); } + } + + // ---- helpers ---------------------------------------------------------- + + private static List Recipients(string csv) => + csv.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(a => new Recipient { EmailAddress = new EmailAddress { Address = a } }) + .ToList(); + + private static DateTimeOffset ParseOrDefault(string? iso, DateTimeOffset fallback) => + DateTimeOffset.TryParse(iso, out var v) ? v : fallback; + + private static string Json(object? value) => JsonSerializer.Serialize(value ?? "none"); + + private static string Fail(string op, Exception ex) => $"{op} failed: {ex.Message}"; + + public IEnumerable CreateTools() => + [ + AIFunctionFactory.Create(ListInbox), + AIFunctionFactory.Create(SearchMail), + AIFunctionFactory.Create(ReadMail), + AIFunctionFactory.Create(SendMail), + AIFunctionFactory.Create(ReplyToMail), + AIFunctionFactory.Create(ListEvents), + AIFunctionFactory.Create(CreateEvent), + AIFunctionFactory.Create(CancelEvent) + ]; + + /// Wraps a pre-fetched delegated access token as a Graph . + private sealed class StaticTokenCredential(string token) : TokenCredential + { + public override AccessToken GetToken(TokenRequestContext requestContext, CancellationToken ct) + => new(token, DateTimeOffset.UtcNow.AddMinutes(50)); + + public override ValueTask GetTokenAsync(TokenRequestContext requestContext, CancellationToken ct) + => new(GetToken(requestContext, ct)); + } +} diff --git a/memex/Memex.Portal.Shared/Email/GraphEmailSender.cs b/memex/Memex.Portal.Shared/Email/GraphEmailSender.cs new file mode 100644 index 000000000..dd411b47c --- /dev/null +++ b/memex/Memex.Portal.Shared/Email/GraphEmailSender.cs @@ -0,0 +1,80 @@ +using Azure.Core; +using Azure.Identity; +using Microsoft.Extensions.Logging; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Threading; +using Microsoft.Graph; +using Microsoft.Graph.Models; +using Microsoft.Graph.Users.Item.SendMail; + +namespace Memex.Portal.Shared.Email; + +/// +/// Sends email via Microsoft Graph /users/{noReply}/sendMail using the Mail.Send +/// application permission. Credentials come from : a managed identity +/// (DefaultAzureCredential) in production, or a client secret for self-host. +/// +/// The Graph call is genuinely async; since this sender is not invoked on a hub scheduler +/// it bridges to the codebase's reactive convention via a bounded HTTP +/// (_http.Run(...)) rather than a bare Observable.FromAsync — the latter deadlocks +/// under a blocking subscriber (see Doc/Architecture/ControlledIoPooling.md). +/// +public sealed class GraphEmailSender : IEmailSender, IDisposable +{ + // Bound to a sane cap so a burst of outbound sends can't open unbounded Graph calls. + private const int HttpConcurrency = 8; + + private readonly EmailOptions _options; + private readonly ILogger? _logger; + private readonly GraphServiceClient _graph; + + // Dedicated bounded HTTP pool, ALWAYS created fresh and owned by this instance — never resolved + // from the mesh-scoped IoPoolRegistry. The portal builds this sender from its OWN DI container + // while activating hosted services; reaching across into the mesh hub's ServiceProvider at that + // moment races that provider's internal service realization (a documented NRE crash-loop — see + // GraphMail). A self-owned pool always resolves and is disposed with this singleton. + private readonly IoPool _http = new(HttpConcurrency); + + public GraphEmailSender(EmailOptions options, ILogger? logger = null) + { + _options = options; + _logger = logger; + + // Azure SDK credentials are server-only; CA1416's browser-reachability + // analysis doesn't apply (this sender never runs in WASM/browser). +#pragma warning disable CA1416 + TokenCredential credential = options.UseManagedIdentity + ? new DefaultAzureCredential() + : new ClientSecretCredential(options.TenantId, options.ClientId, options.ClientSecret); +#pragma warning restore CA1416 + + _graph = new GraphServiceClient(credential, ["https://graph.microsoft.com/.default"]); + } + + public void Dispose() => _http.Dispose(); + + public IObservable SendEmail(string toAddress, string subject, string htmlBody) => + _http.Run(ct => SendAsync(toAddress, subject, htmlBody, ct)); + + private async Task SendAsync(string toAddress, string subject, string htmlBody, CancellationToken ct) + { + var body = new SendMailPostRequestBody + { + Message = new Message + { + Subject = subject, + Body = new ItemBody { ContentType = BodyType.Html, Content = htmlBody }, + ToRecipients = + [ + new Recipient { EmailAddress = new EmailAddress { Address = toAddress } } + ] + }, + SaveToSentItems = false + }; + + await _graph.Users[_options.MailboxAddress].SendMail.PostAsync(body, cancellationToken: ct).ConfigureAwait(false); + _logger?.LogInformation("Sent email to {To} (subject: {Subject}) as {From}", + toAddress, subject, _options.MailboxAddress); + return true; + } +} diff --git a/memex/Memex.Portal.Shared/Email/GraphMail.cs b/memex/Memex.Portal.Shared/Email/GraphMail.cs new file mode 100644 index 000000000..7eb8bb116 --- /dev/null +++ b/memex/Memex.Portal.Shared/Email/GraphMail.cs @@ -0,0 +1,106 @@ +using System.Reactive; +using System.Reactive.Linq; +using Azure.Core; +using Azure.Identity; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Threading; +using Microsoft.Graph; +using Microsoft.Graph.Models; + +namespace Memex.Portal.Shared.Email; + +/// +/// Reactive wrapper over a Microsoft Graph client scoped to the portal mailbox, for the inbound side +/// (read messages, mark read, manage the inbox change-notification subscription). Outbound send lives +/// in . Both build the same app-only credential from . +/// +/// Every method returns a cold ; the genuine HTTP I/O runs through the +/// shared Http — off the hub scheduler and bounded — never a bare +/// await on the calling thread (see Doc/Architecture/ControlledIoPooling.md). +/// +public sealed class GraphMail : IDisposable +{ + // Bound to a sane cap so a burst of inbound notifications can't open unbounded Graph calls. + private const int HttpConcurrency = 8; + + private readonly EmailOptions _options; + private readonly Lazy _graph; + + // Dedicated bounded HTTP pool, ALWAYS created fresh and owned by this instance — never resolved + // from the mesh-scoped IoPoolRegistry. The portal builds GraphMail from its OWN DI container + // while activating hosted services; reaching across into the mesh hub's ServiceProvider at that + // moment races that provider's internal service realization and surfaced as an NRE inside + // ConcurrentDictionary.GetOrAdd, crash-looping the portal. A self-owned pool always resolves and + // is disposed with this singleton when the container tears down. + private readonly IoPool _http = new(HttpConcurrency); + + public GraphMail(EmailOptions options) + { + _options = options; + // Built lazily so the type is constructible without valid creds (unit tests that exercise + // only the routing never touch Graph; the credential would otherwise throw on empty values). + _graph = new Lazy(() => + { + // Azure SDK credentials are server-only; CA1416's browser-reachability + // analysis doesn't apply (this client never runs in WASM/browser). +#pragma warning disable CA1416 + TokenCredential credential = options.UseManagedIdentity + ? new DefaultAzureCredential() + : new ClientSecretCredential(options.TenantId, options.ClientId, options.ClientSecret); +#pragma warning restore CA1416 + return new GraphServiceClient(credential, ["https://graph.microsoft.com/.default"]); + }); + } + + public void Dispose() => _http.Dispose(); + + private GraphServiceClient Client => _graph.Value; + + /// The mailbox the portal sends/receives as (e.g. memex@systemorph.com). + public string Mailbox => _options.MailboxAddress; + + /// The Graph resource path for the mailbox inbox messages (app-only — no /me). + public string InboxResource => $"users/{_options.MailboxAddress}/mailFolders('inbox')/messages"; + + public IObservable GetMessage(string messageId) => + _http.Invoke(ct => Client.Users[_options.MailboxAddress].Messages[messageId].GetAsync(r => + r.QueryParameters.Select = + ["from", "subject", "body", "conversationId", "internetMessageId", "toRecipients", "isRead"], ct)); + + public IObservable MarkRead(string messageId) => + _http.Invoke(async ct => + { + await Client.Users[_options.MailboxAddress].Messages[messageId] + .PatchAsync(new Message { IsRead = true }, cancellationToken: ct); + return Unit.Default; + }); + + public IObservable CreateInboxSubscription( + string notificationUrl, string clientState, DateTimeOffset expiration) => + _http.Invoke(ct => Client.Subscriptions.PostAsync(new Subscription + { + ChangeType = "created", + NotificationUrl = notificationUrl, + Resource = InboxResource, + ExpirationDateTime = expiration, + ClientState = clientState + }, cancellationToken: ct)); + + public IObservable ListSubscriptions() => + _http.Invoke(ct => Client.Subscriptions.GetAsync(cancellationToken: ct)); + + public IObservable RenewSubscription(string subscriptionId, DateTimeOffset expiration) => + _http.Invoke(async ct => + { + await Client.Subscriptions[subscriptionId] + .PatchAsync(new Subscription { ExpirationDateTime = expiration }, cancellationToken: ct); + return Unit.Default; + }); + + public IObservable DeleteSubscription(string subscriptionId) => + _http.Invoke(async ct => + { + await Client.Subscriptions[subscriptionId].DeleteAsync(cancellationToken: ct); + return Unit.Default; + }); +} diff --git a/memex/Memex.Portal.Shared/Email/GraphSubscriptionService.cs b/memex/Memex.Portal.Shared/Email/GraphSubscriptionService.cs new file mode 100644 index 000000000..d61cced81 --- /dev/null +++ b/memex/Memex.Portal.Shared/Email/GraphSubscriptionService.cs @@ -0,0 +1,162 @@ +using System.Reactive.Disposables; +using System.Reactive.Linq; +using MeshWeaver.Blazor.Infrastructure; // PortalApplication +using MeshWeaver.Data; // IWorkspace, GetWorkspace, GetMeshNodeStream +using MeshWeaver.Graph.Configuration; // GraphSubscriptionNodeType +using MeshWeaver.Mesh; // EmailOptions, GraphSubscriptionState, MeshNode +using MeshWeaver.Mesh.Security; // ImpersonateAsSystem +using MeshWeaver.Mesh.Services; // IMeshService +using MeshWeaver.Messaging; // AccessService +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Email; + +/// +/// Keeps a Microsoft Graph change-notification subscription alive on the mailbox inbox so inbound mail is +/// delivered to /api/email. The subscription id is persisted as a +/// node (Admin/_GraphSubscription/inbox): on startup we read it and renew/reuse the +/// existing subscription rather than creating a new one — so a portal restart no longer leaves a duplicate +/// subscription behind (which would deliver every inbound email more than once). Renewed on a timer +/// (messages subscriptions cap at ~3 days). Self-skips unless Email:Enabled && Email:InboundEnabled +/// and a is set. +/// +public sealed class GraphSubscriptionService( + IServiceProvider rootServices, + EmailOptions options, + GraphMail graphMail, + IHostApplicationLifetime lifetime, + ILogger? logger = null) : IHostedService, IDisposable +{ + private static readonly TimeSpan RenewInterval = TimeSpan.FromHours(24); + private readonly CompositeDisposable subscriptions = new(); + private IServiceScope? scope; + private IWorkspace? workspace; + private IMeshService? meshService; + private AccessService? access; + private string? subscriptionId; + private bool nodeExists; + + public Task StartAsync(CancellationToken cancellationToken) + { + if (!options.Enabled || !options.InboundEnabled || string.IsNullOrEmpty(options.WebhookBaseUrl)) + { + logger?.LogInformation("Email inbound disabled — no Graph subscription created"); + return Task.CompletedTask; + } + + // Defer until the host is fully started: Graph validates the notificationUrl synchronously during + // subscription creation, so the webhook endpoint must already be listening; and reading the + // persisted state needs the mesh up. ApplicationStarted covers both. + var url = $"{options.WebhookBaseUrl.TrimEnd('/')}/api/email"; + lifetime.ApplicationStarted.Register(() => Begin(url)); + return Task.CompletedTask; + } + + private void Begin(string url) + { + try + { + scope = rootServices.CreateScope(); + var hub = scope.ServiceProvider.GetRequiredService().Hub; + workspace = hub.GetWorkspace(); + meshService = hub.ServiceProvider.GetRequiredService(); + access = hub.ServiceProvider.GetRequiredService(); + + // Read the persisted subscription id so we RENEW the existing one instead of creating another. + workspace.GetMeshNodeStream(GraphSubscriptionNodeType.InboxPath) + .Select(n => n?.Content as GraphSubscriptionState) + .Take(1) + .Timeout(TimeSpan.FromSeconds(15)) + .Subscribe( + state => + { + nodeExists = state is not null; + subscriptionId = state?.SubscriptionId; + CreateOrRenew(url); + }, + _ => CreateOrRenew(url)); // no stored state (or read timed out) → create fresh + + subscriptions.Add(Observable.Interval(RenewInterval).Subscribe(_ => CreateOrRenew(url))); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "EmailSubscription: failed to start"); + } + } + + private void CreateOrRenew(string url) + { + var expiration = DateTimeOffset.UtcNow.AddDays(2); + if (subscriptionId is { } id) + graphMail.RenewSubscription(id, expiration).Subscribe( + _ => { logger?.LogInformation("EmailSubscription: renewed {Id}", id); Persist(id, url, expiration); }, + ex => + { + // Stale id (expired / deleted server-side) → forget it and create a fresh one. + logger?.LogWarning(ex, "EmailSubscription: renew failed — recreating"); + subscriptionId = null; + Create(url, expiration); + }); + else + Create(url, expiration); + } + + private void Create(string url, DateTimeOffset expiration) => + graphMail.CreateInboxSubscription(url, options.SubscriptionClientState, expiration).Subscribe( + sub => + { + subscriptionId = sub?.Id; + logger?.LogInformation("EmailSubscription: created {Id} -> {Url}", subscriptionId, url); + Persist(subscriptionId, url, expiration); + }, + ex => logger?.LogWarning(ex, "EmailSubscription: create failed for {Url}", url)); + + /// Persist the live subscription id/expiry so the next restart renews it instead of duplicating. + private void Persist(string? subId, string url, DateTimeOffset expiration) + { + if (meshService is null || access is null || string.IsNullOrEmpty(subId)) return; + // 🚨 Write to the SAME path the renewal read uses (InboxPath). The old + // `new MeshNode(NodeType, InboxPath)` set id=NodeType + namespace=InboxPath → + // path "Admin/_GraphSubscription/inbox/GraphSubscription", so the read of + // InboxPath NEVER found it → every read NotFound-stormed the missing node (the + // inbox read/write mismatch). FromPath splits InboxPath into + // namespace="Admin/_GraphSubscription" + id="inbox" → path == InboxPath. + var node = MeshNode.FromPath(GraphSubscriptionNodeType.InboxPath) with + { + NodeType = GraphSubscriptionNodeType.NodeType, + Name = "Graph Subscription", + Content = new GraphSubscriptionState + { + SubscriptionId = subId, + Resource = graphMail.InboxResource, + NotificationUrl = url, + ExpiresAt = expiration + } + }; + // Idempotent persist: if CreateNode hits an already-existing node (the seeded + // default, or a prior run whose startup read timed out → nodeExists=false), fall + // back to UpdateNode so the live SubscriptionId IS saved. Without this the id is + // lost and the next restart creates a DUPLICATE inbox subscription (mail delivered + // twice). A genuine failure still surfaces via the inner warning. + using (access.ImpersonateAsSystem()) + (nodeExists ? meshService.UpdateNode(node) : meshService.CreateNode(node)).Subscribe( + _ => nodeExists = true, + _ => + { + using (access.ImpersonateAsSystem()) + meshService.UpdateNode(node).Subscribe( + _ => nodeExists = true, + ex2 => logger?.LogWarning(ex2, "EmailSubscription: failed to persist subscription state")); + }); + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public void Dispose() + { + subscriptions.Dispose(); + scope?.Dispose(); + } +} diff --git a/memex/Memex.Portal.Shared/Email/InvitationEmailSender.cs b/memex/Memex.Portal.Shared/Email/InvitationEmailSender.cs new file mode 100644 index 000000000..c1d13239c --- /dev/null +++ b/memex/Memex.Portal.Shared/Email/InvitationEmailSender.cs @@ -0,0 +1,182 @@ +using System.Collections.Concurrent; +using System.Reactive.Disposables; +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.Blazor.Infrastructure; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Memex.Portal.Shared.Authentication; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Email; + +/// +/// Mesh-driven invitation emailer. Watches Pending nodes (Admin +/// partition) that have not been emailed yet ( == null) via +/// , sends the "You've been invited" email through +/// , and stamps EmailSentAt so it never re-sends. +/// +/// Two-layer de-dup. Node-state (EmailSentAt) is the durable, cross-restart +/// guard. But the live query's snapshot LAGS the claim write — after we stamp EmailSentAt +/// the query keeps re-emitting the stale (null) node, and because each node's claim re-emits the +/// WHOLE set, a single batch would otherwise re-claim+re-send each invitation many times before +/// propagation. So an in-process single-claim guard (, an INSTANCE field — +/// mesh/process-scoped, never static) makes the first claim per path win; later stale emissions +/// short-circuit. Released only on send failure so a later tick retries. +/// +/// Decouples the invite email from the creation entry point: an invitation created from the +/// Invitations settings tab, from MCP (raw create), or from a REST call all get emailed +/// exactly once. Mirrors ; the only Task boundary is the +/// contract. Self-skips unless Email:Enabled. +/// +public sealed class InvitationEmailSender( + IServiceProvider rootServices, + IHostApplicationLifetime lifetime, + EmailOptions options, + IConfiguration configuration, + ILogger? logger = null) : IHostedService, IDisposable +{ + private const string InviteSubject = "You've been invited to Memex"; + private readonly CompositeDisposable subscriptions = new(); + // In-process single-claim guard: path → claimed. Instance (mesh/process-scoped), never static. + // Prevents the live-query snapshot lag from re-claiming+re-sending the same invitation while + // its EmailSentAt write propagates. See class summary; released on send failure to allow retry. + private readonly ConcurrentDictionary claiming = new(); + private IServiceScope? scope; + + public Task StartAsync(CancellationToken cancellationToken) + { + if (!options.Enabled) + { + logger?.LogInformation("Email disabled — InvitationEmailSender idle"); + return Task.CompletedTask; + } + + // Defer mesh access until the host is fully started (Orleans + mesh hub come up as + // hosted services too) — same rationale as OutboundEmailSender. + lifetime.ApplicationStarted.Register(BeginWatching); + return Task.CompletedTask; + } + + private void BeginWatching() + { + try + { + scope = rootServices.CreateScope(); + var hub = scope.ServiceProvider.GetRequiredService().Hub; + var sp = hub.ServiceProvider; + var query = sp.GetRequiredService(); + var meshService = sp.GetRequiredService(); + var accessService = sp.GetRequiredService(); + var emailSender = sp.GetRequiredService(); + var jsonOptions = hub.JsonSerializerOptions; + var baseUrl = configuration["Portal:BaseUrl"] + ?? configuration["PublicBaseUrl"] + ?? configuration["Email:WebhookBaseUrl"]; + + // Live query: ALL invitations — filter Pending + not-yet-emailed in Send. We do NOT + // filter `content.status:Pending` in the query: a Pending invitation's status is the + // enum default (0) and is OMITTED from the stored JSON, so that filter would never + // match (same reason OnboardingMiddleware/InvitationService filter status in code). + // IMeshQueryCore = the no-access-control core path (infra read). + logger?.LogInformation("InvitationEmailSender: watching invitations (baseUrl={BaseUrl})", baseUrl ?? "(none)"); + subscriptions.Add(query + // PATH-scoped to Admin/Invitation. Routing is by the path's FIRST SEGMENT + // (PostgreSqlPartitionedMeshQuery.FirstSegment → schema), so `path:Admin/…` + // routes to the admin schema. A `namespace:Admin`-only query has NO path, so it + // fans out cross-schema — and the admin schema is intentionally EXCLUDED from + // that fan-out (PostgreSqlSchemaInitializer.searchable_schemas) — so it would + // never see invitations. (`namespace:Admin` is also exact-match and would miss + // the `Admin/Invitation` namespace regardless.) scope:children = the invitation + // slugs directly under Admin/Invitation. Runs as System (IMeshQueryCore + + // MeshQuery's System stamp) — no access-control filtering. See AccessControl.md. + .Query(MeshQueryRequest.FromQuery( + $"path:{InvitationNodeType.Namespace} scope:children nodeType:{InvitationNodeType.NodeType}"), jsonOptions) + .Select(change => change.Items) + .Subscribe( + items => + { + foreach (var node in items) + Send(node, meshService, accessService, emailSender, jsonOptions, baseUrl); + }, + ex => logger?.LogWarning(ex, "InvitationEmailSender: query failed"))); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "InvitationEmailSender: failed to start watching invitations"); + } + } + + private void Send( + MeshNode node, IMeshService meshService, AccessService accessService, + IEmailSender emailSender, JsonSerializerOptions jsonOptions, string? baseUrl) + { + var invitation = InvitationService.TryGetInvitation(node, jsonOptions); + if (invitation is null + || invitation.Status != InvitationStatus.Pending + || invitation.EmailSentAt is not null + || string.IsNullOrWhiteSpace(invitation.Email)) + return; + + // In-process single-claim: the live query re-emits the STALE (EmailSentAt=null) node many + // times before our claim write propagates back into its snapshot (and every node's claim + // re-emits the whole set), so without this guard one batch re-sends each invitation many + // times. TryAdd makes the first claim per path win; we release only on send failure. + if (!claiming.TryAdd(node.Path, 0)) + return; + + // Claim FIRST: stamp EmailSentAt before sending so a duplicate query emission (or a + // second replica) doesn't re-send. On send failure we clear the stamp so a later tick + // retries. (Single-writer per node on the owning hub; last-write-wins is acceptable for + // the rare multi-replica race — worst case one duplicate email, never a lost one.) + var claimedAt = DateTimeOffset.UtcNow; + SetEmailSentAt(node, invitation, claimedAt, meshService, accessService) + .SelectMany(_ => emailSender.SendEmail(invitation.Email!, InviteSubject, BuildInviteEmailHtml(baseUrl))) + .Subscribe( + ok => logger?.LogInformation( + "InvitationEmailSender: {Email} emailed (sent={Sent})", invitation.Email, ok), + ex => + { + logger?.LogWarning(ex, "InvitationEmailSender: send failed for {Email}", invitation.Email); + // Release the in-process claim + roll back the stamp so a later tick retries. + claiming.TryRemove(node.Path, out _); + SetEmailSentAt(node, invitation, null, meshService, accessService) + .Subscribe(_ => { }, _ => { }); + }); + } + + private static IObservable SetEmailSentAt( + MeshNode node, Invitation current, DateTimeOffset? to, + IMeshService meshService, AccessService accessService) => + // System identity: invitations live in the Admin partition that application identities + // can't write to directly (same infrastructure-write pattern as InvitationService). + Observable.Using( + () => accessService.ImpersonateAsSystem(), + _ => meshService.UpdateNode(node with { Content = current with { EmailSentAt = to } })); + + private static string BuildInviteEmailHtml(string? baseUrl) + { + var link = string.IsNullOrEmpty(baseUrl) + ? "" + : $"

" + + "Open Memex

" + + $"

{System.Net.WebUtility.HtmlEncode(baseUrl)}

"; + return "

You've been invited to Memex.

" + + "

Sign in with this email address to get started.

" + + link; + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public void Dispose() + { + subscriptions.Dispose(); + scope?.Dispose(); + } +} diff --git a/memex/Memex.Portal.Shared/Email/NoOpEmailSender.cs b/memex/Memex.Portal.Shared/Email/NoOpEmailSender.cs new file mode 100644 index 000000000..9f88af159 --- /dev/null +++ b/memex/Memex.Portal.Shared/Email/NoOpEmailSender.cs @@ -0,0 +1,21 @@ +using System.Reactive.Linq; +using MeshWeaver.Mesh; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Email; + +/// +/// No-op registered when Email:Enabled is false (local dev, +/// tests, deployments without M365 creds). Logs the would-be send and reports success so +/// callers' reactive chains complete normally without any mail leaving the process. +/// +public sealed class NoOpEmailSender(ILogger? logger = null) : IEmailSender +{ + public IObservable SendEmail(string toAddress, string subject, string htmlBody) + { + logger?.LogInformation( + "Email disabled (Email:Enabled=false) — skipping send to {To} (subject: {Subject})", + toAddress, subject); + return Observable.Return(true); + } +} diff --git a/memex/Memex.Portal.Shared/Email/OutboundEmailSender.cs b/memex/Memex.Portal.Shared/Email/OutboundEmailSender.cs new file mode 100644 index 000000000..6fe2b2f6a --- /dev/null +++ b/memex/Memex.Portal.Shared/Email/OutboundEmailSender.cs @@ -0,0 +1,146 @@ +using System.Reactive.Disposables; +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.Blazor.Infrastructure; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Email; + +/// +/// Mesh-driven outbound sender — no in-memory state. The Email Router agent emits its +/// reply as an outbound node (Direction=Outbound, Status=New) +/// in the parent email's namespace; this single hosted service watches for those via +/// , claims each (New → Sending, the optimistic guard against double-send), +/// sends it through , and flips it to (or +/// ). Dedup + restart-safety live entirely in the node's status. +/// +/// Reactive; the only Task boundary is the contract. Self-skips +/// unless Email:Enabled. +/// +public sealed class OutboundEmailSender( + IServiceProvider rootServices, + IHostApplicationLifetime lifetime, + EmailOptions options, + ILogger? logger = null) : IHostedService, IDisposable +{ + private readonly CompositeDisposable subscriptions = new(); + private IServiceScope? scope; + + public Task StartAsync(CancellationToken cancellationToken) + { + if (!options.Enabled) + { + logger?.LogInformation("Email disabled — OutboundEmailSender idle"); + return Task.CompletedTask; + } + + // Defer ALL mesh access until the host is fully started. The Orleans client and the mesh + // hub come up as hosted services too; touching the hub here (or constructing + // PortalApplication, whose ctor registers an Orleans stream) races that startup and NREs + // in OrleansRoutingService.RegisterStream / PersistentStreamProvider. ApplicationStarted + // fires once every hosted service (Orleans included) has started, so the mesh is ready. + lifetime.ApplicationStarted.Register(BeginWatching); + return Task.CompletedTask; + } + + private void BeginWatching() + { + try + { + // Resolve a fresh PortalApplication in its own scope now that the mesh is up — the + // instance DI built at host-construction time may have captured a not-yet-ready hub. + scope = rootServices.CreateScope(); + var hub = scope.ServiceProvider.GetRequiredService().Hub; + var sp = hub.ServiceProvider; + var query = sp.GetRequiredService(); + var meshService = sp.GetRequiredService(); + var accessService = sp.GetRequiredService(); + var emailSender = sp.GetRequiredService(); + var jsonOptions = hub.JsonSerializerOptions; + + // Live query: any outbound mail awaiting send. Emits the current set on change. + subscriptions.Add(query + .Query(MeshQueryRequest.FromQuery( + $"nodeType:{EmailNodeType.NodeType} content.direction:Outbound content.status:New"), jsonOptions) + .Select(change => change.Items) + .Subscribe( + items => + { + foreach (var node in items) + Send(node, meshService, accessService, emailSender, jsonOptions); + }, + ex => logger?.LogWarning(ex, "OutboundEmailSender: query failed"))); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "OutboundEmailSender: failed to start watching outbound mail"); + } + } + + private void Send( + MeshNode node, IMeshService meshService, AccessService accessService, + IEmailSender emailSender, JsonSerializerOptions jsonOptions) + { + var email = EmailOf(node, jsonOptions); + if (email is null || email.Direction != EmailDirection.Outbound || email.Status != EmailStatus.New) + return; + if (string.IsNullOrEmpty(email.To)) + { + logger?.LogWarning("OutboundEmailSender: outbound {Path} has no recipient — marking Failed", node.Path); + SetStatus(node, email, EmailStatus.Failed, meshService, accessService).Subscribe(_ => { }, _ => { }); + return; + } + + // Claim: New → Sending (only if still New). The CAS lives in SetStatus's lambda, so a duplicate + // emission that already flipped it is a no-op. + SetStatus(node, email, EmailStatus.Sending, meshService, accessService) + .SelectMany(claimed => + { + // SetStatus returns the unchanged node when the CAS failed (already claimed) — skip. + if ((EmailOf(claimed, jsonOptions)?.Status) != EmailStatus.Sending) + return Observable.Empty(); + var subject = email.Subject.StartsWith("Re:", StringComparison.OrdinalIgnoreCase) + ? email.Subject : $"Re: {email.Subject}"; + return emailSender.SendEmail(email.To!, subject, email.Body) + .SelectMany(ok => SetStatus(claimed, EmailOf(claimed, jsonOptions)!, + ok ? EmailStatus.Sent : EmailStatus.Failed, meshService, accessService).Select(_ => ok)); + }) + .Subscribe( + ok => logger?.LogInformation("OutboundEmailSender: {Path} → {To} sent={Sent}", node.Path, email.To, ok), + ex => + { + logger?.LogWarning(ex, "OutboundEmailSender: send failed for {Path}", node.Path); + SetStatus(node, email, EmailStatus.Failed, meshService, accessService).Subscribe(_ => { }, _ => { }); + }); + } + + private static IObservable SetStatus( + MeshNode node, MeshWeaver.Mesh.Email current, EmailStatus to, + IMeshService meshService, AccessService accessService) => + Observable.Using( + () => accessService.ImpersonateAsSystem(), + _ => meshService.UpdateNode(node with { Content = current with { Status = to } })); + + private static MeshWeaver.Mesh.Email? EmailOf(MeshNode n, JsonSerializerOptions? options) => n.Content switch + { + MeshWeaver.Mesh.Email e => e, + JsonElement je => Safe(je, options), + _ => null + }; + + private static MeshWeaver.Mesh.Email? Safe(JsonElement je, JsonSerializerOptions? options) + { + try { return JsonSerializer.Deserialize(je.GetRawText(), options); } + catch { return null; } + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + public void Dispose() => subscriptions.Dispose(); +} diff --git a/memex/Memex.Portal.Shared/Layout/MemexMobileMenu.razor b/memex/Memex.Portal.Shared/Layout/MemexMobileMenu.razor index d9ea89bb1..2fa105d68 100644 --- a/memex/Memex.Portal.Shared/Layout/MemexMobileMenu.razor +++ b/memex/Memex.Portal.Shared/Layout/MemexMobileMenu.razor @@ -1,22 +1,27 @@ @using Icons = Microsoft.FluentUI.AspNetCore.Components.Icons @using MeshWeaver.Blazor.Infrastructure @using MeshWeaver.Mesh.Services +@using Microsoft.AspNetCore.Components.Authorization
- - - - Create New... - - - - - Settings - + + + + + + Create New... + + + + + Settings + + + @code { diff --git a/memex/Memex.Portal.Shared/Memex.Portal.Shared.csproj b/memex/Memex.Portal.Shared/Memex.Portal.Shared.csproj index 74f5b3aa5..bfc47c846 100644 --- a/memex/Memex.Portal.Shared/Memex.Portal.Shared.csproj +++ b/memex/Memex.Portal.Shared/Memex.Portal.Shared.csproj @@ -18,19 +18,30 @@ + + + + + + - + + + + diff --git a/memex/Memex.Portal.Shared/MemexConfiguration.cs b/memex/Memex.Portal.Shared/MemexConfiguration.cs index 5d7d45f23..76d81ffce 100644 --- a/memex/Memex.Portal.Shared/MemexConfiguration.cs +++ b/memex/Memex.Portal.Shared/MemexConfiguration.cs @@ -1,9 +1,13 @@ using System.IdentityModel.Tokens.Jwt; +using Memex.Portal.Shared.Api; using Memex.Portal.Shared.Authentication; +using Memex.Portal.Shared.Email; using Memex.Portal.Shared.Settings; +using Memex.Portal.Shared.Social; +using Microsoft.Extensions.DependencyInjection.Extensions; using MeshWeaver.AI; using MeshWeaver.AI.AzureFoundry; -using MeshWeaver.AI.AzureOpenAI; +using MeshWeaver.AI.OpenAI; using MeshWeaver.AI.ClaudeCode; using MeshWeaver.AI.Copilot; using MeshWeaver.Blazor.AI; @@ -17,8 +21,13 @@ using MeshWeaver.Blazor.Portal.Components; using MeshWeaver.Blazor.Radzen; using MeshWeaver.ContentCollections; +using MeshWeaver.ContentCollections.Indexing; +using MeshWeaver.ContentCollections.Indexing.Graph; +using MeshWeaver.ContentCollections.Indexing.PostgreSql; using MeshWeaver.Documentation; using MeshWeaver.GoogleMaps; +using MeshWeaver.Data; +using MeshWeaver.GitSync; using MeshWeaver.Graph; using MeshWeaver.Graph.Configuration; using MeshWeaver.Markdown.Export.Configuration; @@ -26,9 +35,11 @@ using MeshWeaver.Hosting.AzureBlob; using MeshWeaver.Hosting.Blazor; using MeshWeaver.Hosting.Persistence; +using MeshWeaver.Hosting.PostgreSql; using MeshWeaver.Hosting.Security; using MeshWeaver.Mesh; using MeshWeaver.Mesh.Services; +using MeshWeaver.Mesh.Threading; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication.Cookies; using Microsoft.AspNetCore.Authentication.OpenIdConnect; @@ -42,8 +53,6 @@ using Microsoft.Extensions.Logging; using Microsoft.Identity.Web; using Microsoft.Identity.Web.UI; -using ModelContextProtocol.AspNetCore.Authentication; -using ModelContextProtocol.Authentication; using PortalAuthOptions = MeshWeaver.Blazor.Portal.Authentication.AuthenticationOptions; namespace Memex.Portal.Shared; @@ -83,26 +92,128 @@ public static void ConfigureMemexServices(this WebApplicationBuilder builder) }) .AddBlazorPortalServices(); + // Onboarding service — pulls the three-row dual-write out of + // Onboarding.razor so it's unit-testable end-to-end. + services.AddScoped(); + // Invitation service — reads/writes Invitation nodes for invitation-only onboarding. + services.AddScoped(); + // Configure Radzen services.AddRadzenServices(); - // AI services — thread persistence is handled via MeshNodes - - // Configure AI factories (read from appsettings, including Order) - services.AddAzureFoundryClaude(config => - builder.Configuration.GetSection("Anthropic").Bind(config)); - - services.AddAzureFoundry(config => - builder.Configuration.GetSection("AzureAIS").Bind(config)); + // AI services — thread persistence is handled via MeshNodes. + // Anthropic / AzureFoundry / AzureOpenAI registration is now a + // single per-provider builder extension (.AddAnthropic() etc.) + // wired in ConfigureMemexMesh — that one call registers the catalog + // source + IOptions binding + IChatClientFactory. + // + // Deploy-time feature flags gate which providers/CLIs ship. Defaults are + // all-on (an absent Features section = current behaviour, no regression). + // A disabled flag is the operator's intent and wins even if a key is + // configured. Both the services-tier factory registration here AND the + // mesh-tier catalog source in ConfigureMemexMesh are gated symmetrically + // so a provider can't half-register. + var features = builder.Configuration + .GetSection(MemexFeatureOptions.SectionName) + .Get() ?? new MemexFeatureOptions(); + + // Bind Features as IOptions so application code (e.g. the onboarding flow's + // self-provisioning gate) resolves the toggles through standard DI rather + // than re-reading the configuration section ad hoc. + services.Configure( + builder.Configuration.GetSection(MemexFeatureOptions.SectionName)); + + // System email (Microsoft Graph /sendMail). Disabled by default → NoOp sender so + // local dev and tests never send. When Email:Enabled=true, GraphEmailSender sends as + // the configured no-reply mailbox using the Mail.Send application permission. Backs the + // invitation flow (admin Invitations settings tab). + var emailOptions = builder.Configuration + .GetSection(EmailOptions.SectionName) + .Get() ?? new EmailOptions(); + services.AddSingleton(emailOptions); + if (emailOptions.Enabled) + { + services.AddSingleton(); + // Executive Assistant: per-user JUST-IN-TIME delegated Graph access (the user consents to the + // EA touching THEIR OWN mailbox/calendar only when they first use the tool — no standing app + // permission). EaGraphAuth drives the consent/token flow; the plugin uses the per-user token. + services.AddHttpClient(); + services.AddSingleton(); + // Notification triage runner — escalates in-app notifications to email/Teams per each + // recipient's NotificationRules, via the cheap triage agent (only fires for users with rules). + services.AddHostedService(); + } + else + services.AddSingleton(); + + // Inbound email→agent channel (intake). Mail is treated as a chat device: each inbound email + // finds-or-creates a conversation thread and appends its latest message (referencing the email + // by path). The Graph subscription self-skips unless Email:Enabled && Email:InboundEnabled. + services.AddSingleton(sp => new GraphMail( + sp.GetRequiredService())); + services.AddSingleton(sp => new EmailInboundProcessor( + sp.GetRequiredService().Hub, + sp.GetRequiredService(), + sp.GetService>())); + services.AddHostedService(); + // Mesh-driven reply sender: drains agent-emitted Outbound Email nodes (Status=New) via Graph. + services.AddHostedService(); + // Mesh-driven invitation emailer: emails any Pending Invitation node not yet emailed + // (EmailSentAt==null), from ANY entry point (Invitations tab, MCP, REST). Self-skips + // unless Email:Enabled. Decouples the invite email from the UI handler. + services.AddHostedService(); + + // Microsoft Teams bot channel (bidirectional). Registered always but INERT unless Teams:Enabled + // and Bot credentials are set (TeamsClient.IsConfigured gates the endpoint + sender). Activate by + // provisioning an Azure Bot resource + Teams app and setting the Teams config. + var teamsOptions = builder.Configuration.GetSection(MeshWeaver.Mesh.TeamsOptions.SectionName) + .Get() ?? new MeshWeaver.Mesh.TeamsOptions(); + services.AddSingleton(teamsOptions); + services.AddHttpClient(); + services.AddSingleton(sp => new Teams.TeamsInboundProcessor( + sp.GetRequiredService().Hub, + sp.GetRequiredService(), + sp.GetService>())); + if (teamsOptions.Enabled) + // Delivers agent replies back into Teams, reading them via the shared + // ThreadFlow.ObserveResponses abstraction (same read-side primitive the GUI uses). + // Only the hosted service is feature-gated; the client + inbound processor stay registered + // so the messaging endpoint can resolve them and return NotFound when disabled. + services.AddHostedService(); + + // Shared on-disk WORKSPACE dir the agent→skill sync maintains (.claude/skills + AGENTS.md); both + // CLI harnesses set it as the session's working directory so every session sees the MeshWeaver + // agents/skills + the mesh-is-via-MCP base instructions. Defaults to a sibling of the per-user + // .claude root (e.g. /mnt/users → /mnt/users/_skills) when not explicitly configured. + var skillsDir = builder.Configuration["Skills:Directory"]; + if (string.IsNullOrWhiteSpace(skillsDir)) + { + var claudeRoot = builder.Configuration["ClaudeCode:ConfigDirRoot"]?.TrimEnd('/', '\\'); + skillsDir = string.IsNullOrEmpty(claudeRoot) ? null : $"{claudeRoot}/_skills"; + } - services.AddAzureOpenAI(config => - builder.Configuration.GetSection("AzureOpenAIS").Bind(config)); + if (features.Ai.Clis.Copilot) + services.AddCopilot(config => + { + builder.Configuration.GetSection("Copilot").Bind(config); + config.SkillsDirectory = skillsDir; + }); - services.AddCopilot(config => - builder.Configuration.GetSection("Copilot").Bind(config)); + if (features.Ai.Clis.ClaudeCode) + services.AddClaudeCode(config => + { + builder.Configuration.GetSection("ClaudeCode").Bind(config); + config.SkillsDirectory = skillsDir; + }); - services.AddClaudeCode(config => - builder.Configuration.GetSection("ClaudeCode").Bind(config)); + // Reactive agent/skill→file sync: materialises the platform nodeType:Agent + nodeType:Skill + // nodes as CLI skills (+ AGENTS.md) on the shared volume and keeps them in sync as nodes change + // (observable query). Started at startup, runs for the process lifetime. + if ((features.Ai.Clis.ClaudeCode || features.Ai.Clis.Copilot) && !string.IsNullOrWhiteSpace(skillsDir)) + { + services.Configure(o => o.Directory = skillsDir); + services.AddHostedService(); + } // Register the AI chat services (must be after all factory registrations) services.AddAgentChatServices(); @@ -125,6 +236,80 @@ public static void ConfigureMemexServices(this WebApplicationBuilder builder) // Register API token service for MCP bearer auth and OAuth code store services.AddSingleton(); services.AddSingleton(); + // Automatic, token-based MCP back-connection for the co-hosted Claude Code / Copilot CLIs. + // The chat clients resolve this at spawn to mint/reuse the per-user MCP ApiToken + URL. + services.AddSingleton(); + // ModelProviderService backs the Models settings tab — users store + // their own AI provider credentials as MeshNodes in their namespace. + services.AddSingleton(); + // ProviderModelLister fetches a provider's live model list (HTTP /models via + // the I/O pool) so the add-provider flow lets users pick which models to bring. + services.AddSingleton(); + + // GitHub sync — per-user OAuth credential (device flow) + bidirectional + // Space ↔ GitHub sync (export = "sync back"; import = create / re-import a + // Space at any commit). The OAuth client id is bound from GitHub:OAuth; + // absent a client id the Connect flow is gracefully disabled. + services.AddGitHubSyncServices(); + services.Configure(builder.Configuration.GetSection("GitHub:OAuth")); + + // Per-user CLI Connect (Settings → Models, CLI providers). The + // ConnectSessionManager is a mesh-scoped singleton holding the live + // login Process between "show URL" and "paste code" (instance dict, + // 5-min timeout). Each gated CLI registers its IConnectStrategy. The + // captured token is persisted as an encrypted ModelProvider via the + // ConnectTokenSink (seam over ModelProviderService, so the AI layer + // never references the portal assembly). + services.AddSingleton(); + services.AddSingleton(); + if (features.Ai.Clis.ClaudeCode) + { + services.AddSingleton(); + // Wire the Connect login: bind ClaudeConnect:* overrides, default the PTY wrapper ON for + // the co-hosted Linux portal (claude setup-token renders an Ink UI that needs a real TTY — + // see ClaudeConnectStrategy), and mirror the per-user .claude root the co-hosted client uses + // (ClaudeCode:ConfigDirRoot, e.g. /mnt/users) so each user logs in under their own dir. + services.Configure(o => + { + builder.Configuration.GetSection("ClaudeConnect").Bind(o); + if (builder.Configuration["ClaudeConnect:UsePseudoTerminal"] is null && !OperatingSystem.IsWindows()) + o.UsePseudoTerminal = true; + if (string.IsNullOrEmpty(o.ConfigDirRoot)) + o.ConfigDirRoot = builder.Configuration["ClaudeCode:ConfigDirRoot"]; + }); + } + if (features.Ai.Clis.Copilot) + services.AddSingleton(); + + // Social publishing — minimal registration for the LinkedIn connect + pull endpoints. + // (The full hosted-service pipeline is gated behind AddSocialPublishing which needs + // IApprovalPublishBridge / IStatsRefreshSource / IPastPostIngestSource — those come + // in Phase 4. For now the publisher is enough for /connect/linkedin/pull to work.) + var linkedInClientId = builder.Configuration["Social:LinkedIn:ClientId"]; + if (!string.IsNullOrEmpty(linkedInClientId)) + { + services.AddHttpClient(); + services.AddSingleton(new MeshWeaver.Social.LinkedInOptions + { + ClientId = linkedInClientId!, + ClientSecret = builder.Configuration["Social:LinkedIn:ClientSecret"] ?? "" + }); + + // Add the menu provider so "Connect LinkedIn" + "Pull LinkedIn posts" + // appear on the viewer's own user page. + services.TryAddEnumerable( + Microsoft.Extensions.DependencyInjection.ServiceDescriptor.Scoped< + MeshWeaver.Mesh.INodeMenuProvider, + Memex.Portal.Shared.Social.LinkedInCredentialMenuProvider>()); + + // (Removed: SocialMediaUserMenuProvider — hardcoded a NodeType + // ("Systemorph/SocialMediaHub") that isn't registered anywhere in + // the codebase. NodeTypes belong in the database (NodeTypeDefinition + // MeshNodes), not as DLL-side string constants. The SocialMedia + // hub feature should be added back when its NodeType is defined + // through the regular mesh node creation flow rather than wired + // through a DLL-time CreateNode that fails on the receiver.) + } // Configure authentication var authSection = builder.Configuration.GetSection(PortalAuthOptions.SectionName); @@ -167,10 +352,6 @@ public static void ConfigureMemexServices(this WebApplicationBuilder builder) JwtSecurityTokenHandler.DefaultMapInboundClaims = false; services.AddAuthentication(OpenIdConnectDefaults.AuthenticationScheme) .AddMicrosoftIdentityWebApp(entraIdConfig); - services.AddAuthentication() - .AddScheme( - ApiTokenAuthenticationHandler.SchemeName, _ => { }) - .AddMcp(ConfigureMcpResourceMetadata); services.AddControllersWithViews() .AddMicrosoftIdentityUI(); } @@ -200,68 +381,17 @@ public static void ConfigureMemexServices(this WebApplicationBuilder builder) .AddGoogleAuthentication(builder.Configuration) .AddLinkedInAuthentication(builder.Configuration) .AddAppleAuthentication(builder.Configuration); - - // Add API token auth scheme for MCP bearer authentication - authBuilder.AddScheme( - ApiTokenAuthenticationHandler.SchemeName, _ => { }) - .AddMcp(ConfigureMcpResourceMetadata); } - // Add authorization with McpAuth policy (MCP scheme forwards to ApiToken or Cookie) - services.AddAuthorization(options => - { - options.AddPolicy("McpAuth", policy => - { - policy.AddAuthenticationSchemes(McpAuthenticationDefaults.AuthenticationScheme); - policy.RequireAuthenticatedUser(); - }); - }); - } - - /// - /// Configures the MCP authentication scheme with OAuth resource metadata discovery - /// and request-based forwarding to the appropriate authentication handler. - /// - private static void ConfigureMcpResourceMetadata(McpAuthenticationOptions options) - { - // CRITICAL: SDK constructor sets ForwardAuthenticate = "Bearer" which takes - // priority over ForwardDefaultSelector in ASP.NET Core's ResolveTarget(). - // Clear it so our selector works. - options.ForwardAuthenticate = null; - - // Route Bearer tokens to ApiToken handler, everything else to Cookie - options.ForwardDefaultSelector = ctx => - { - var authHeader = ctx.Request.Headers.Authorization.ToString(); - if (!string.IsNullOrEmpty(authHeader) && - authHeader.StartsWith("Bearer ", StringComparison.OrdinalIgnoreCase)) - return ApiTokenAuthenticationHandler.SchemeName; - return CookieAuthenticationDefaults.AuthenticationScheme; - }; - - // Fallback resource metadata (overridden per-request by Events) - options.ResourceMetadata = new ProtectedResourceMetadata - { - BearerMethodsSupported = { "header" }, - ScopesSupported = { "mcp" }, - }; + // MCP auth is deliberately separate from the Blazor cookie pipeline above — + // see McpAuthenticationExtensions for the "why". Bearer-only, no cookie leakage, + // proper 401 + WWW-Authenticate on anonymous requests so MCP clients can + // discover the auth server. + services.AddMcpAuthentication(); - options.Events = new McpAuthenticationEvents - { - OnResourceMetadataRequest = ctx => - { - var req = ctx.HttpContext.Request; - var origin = $"{req.Scheme}://{req.Host}"; - ctx.ResourceMetadata = new ProtectedResourceMetadata - { - Resource = $"{origin}/mcp", - BearerMethodsSupported = { "header" }, - ScopesSupported = { "mcp" }, - AuthorizationServers = { $"{origin}/connect" }, - }; - return Task.CompletedTask; - } - }; + // REST surface for the mesh — same Bearer-token policy as MCP, lifts the + // multipart upload size cap. See MeshApiEndpoints. + services.AddMeshApi(); } extension(TBuilder builder) where TBuilder : MeshBuilder @@ -320,7 +450,9 @@ public TBuilder ConfigureMemexMesh(IConfiguration configuration, bool isDevelopm // Ensure Settings are populated for AzureBlob source type if (contentStorageConfig.SourceType == "AzureBlob") { - var settings = contentStorageConfig.Settings ?? new Dictionary(); + var settings = contentStorageConfig.Settings is { } existing + ? new Dictionary(existing) + : new Dictionary(); if (!settings.ContainsKey("ContainerName")) settings["ContainerName"] = "content"; if (!settings.ContainsKey("ClientName")) @@ -333,12 +465,26 @@ public TBuilder ConfigureMemexMesh(IConfiguration configuration, bool isDevelopm var usePartitioned = string.Equals(graphStorageConfig.Type, "FileSystem", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(graphStorageConfig.BasePath); - return (TBuilder)builder + // Deploy-time feature flags (symmetric with ConfigureMemexServices). + var features = configuration + .GetSection(MemexFeatureOptions.SectionName) + .Get() ?? new MemexFeatureOptions(); + + // Static-repo → DB sync: partitions to materialize into + serve from the DB. For a + // synced partition the read-only in-memory static provider is skipped (PG serves it) + // and the import runs on boot. Empty (default) = in-memory serving everywhere, no + // import — no regression. Default Helm sets ["Doc","Agent","Model"]. + var syncPartitions = features.StaticRepoSync.Partitions + .ToHashSet(StringComparer.OrdinalIgnoreCase); + IReadOnlySet serveFromPartition = syncPartitions; + + MeshBuilder mb = builder // Configure persistence from Graph:Storage section. - // Skip if IPartitionedStoreFactory already registered (e.g., PostgreSQL from Program.cs) + // Skip if any IPartitionStorageProvider was already registered upstream + // (e.g., AddPartitionedPostgreSqlPersistence in Memex.Portal.Distributed/Program.cs). .ConfigureServices(services => { - if (services.Any(sd => sd.ServiceType == typeof(IPartitionedStoreFactory))) + if (services.Any(sd => sd.ServiceType == typeof(IPartitionStorageProvider))) return services; return usePartitioned @@ -349,14 +495,79 @@ public TBuilder ConfigureMemexMesh(IConfiguration configuration, bool isDevelopm .AddRowLevelSecurity() // Configure graph from the same base path .AddGraph() - .AddOrganizationType() + // Register GitHub-sync content types (GitHubCredential / GitHubSyncConfig) + // on the mesh + per-node hubs so their config nodes (de)serialize. + .AddGitHubSyncTypes() + // Seed root-scope Admin AccessAssignments for users listed under + // `Auth:GlobalAdmins` so configured admins bypass per-partition + // RLS for cross-partition operations (list Spaces, create + // a new Space, etc.). Empty / missing section = no-op. + .AddMeshNodes(Authentication.GlobalAdminSeed.Build(configuration)) + .AddSpaceType() .AddPortalType() - .AddAI() + .AddAI(serveFromPartition); + + // Each AI provider self-registers everything (catalog source + + // IOptions binding + IChatClientFactory) via one builder extension. + // The Models settings tab + the ModelProviderService read these out + // of the live LanguageModelCatalogOptions — no central registry. + // Gated by deploy-time feature flags (symmetric with the services-tier + // AddCopilot/AddClaudeCode in ConfigureMemexServices). A disabled flag + // drops the catalog source → the provider vanishes from the model + // picker and its Model/ nodes never seed. + if (features.Ai.Providers.Anthropic) mb = mb.AddAnthropic(); + if (features.Ai.Providers.AzureFoundry) mb = mb.AddAzureFoundry(); + if (features.Ai.Providers.AzureOpenAI) mb = mb.AddAzureOpenAI(); + if (features.Ai.Providers.OpenAI) mb = mb.AddOpenAI(); + if (features.Ai.Providers.OpenAICompatible) mb = mb.AddOpenAICompatible(); + if (features.Ai.Clis.ClaudeCode) mb = mb.AddClaudeCode(); // catalog source (factory + config via services.AddClaudeCode) + if (features.Ai.Clis.Copilot) mb = mb.AddCopilot(); // catalog source (factory + config via services.AddCopilot) + + // Content → vector index (core tech). When embeddings are configured, wire the + // upload→Activity indexing pipeline (extract→chunk→embed→store), per-file Document nodes + // (extractive summary by default — swap in a chat client for AI summaries), and chunk-search + // @-autocomplete. The vector store lives IN THE MESH DATABASE, in each partition's OWN schema + // (content_chunks/content_files alongside that partition's mesh_nodes) — no separate database. + // Inert when there's no mesh Postgres connection (e.g. the FileSystem monolith) or embeddings + // aren't set: it compiles in but never activates. + var meshConnectionString = configuration.GetConnectionString("memex"); + var embeddingsConfigured = !string.IsNullOrWhiteSpace(configuration["Embedding:Endpoint"]) + && !string.IsNullOrWhiteSpace(configuration["Embedding:ApiKey"]); + if (!string.IsNullOrWhiteSpace(meshConnectionString) && embeddingsConfigured) + { + mb = mb + .AddContentIndexingPipeline( + storeFactory: sp => new PostgreSqlChunkedContentVectorStore( + meshConnectionString, + sp.GetService(), + sp.GetRequiredService().Dimensions), + embedderFactory: sp => new EmbeddingProviderChunkEmbedder( + sp.GetRequiredService(), + sp.GetService()), + summarizerFactory: _ => new ExtractiveSummarizer()) + .AddContentSearch(); + } + + return (TBuilder)mb .AddSelfRegistry() - .AddDocumentation() + .AddDocumentation(serveFromPartition) + .AddStaticRepoSync(serveFromPartition) + // Ship compiled releases WHEREVER we ship code NodeTypes — Doc AND the sample + // partitions (ACME, FutuRe, Northwind, Cornerstone, MeshWeaver). Pre-build every + // shipped code NodeType's release at boot, as System, so the runtime path is a + // cache hit and no user navigation ever triggers an on-demand compile (the atioz + // 2026-06-18 phantom _Activity/compile-* storm). Idempotent (skips already-built + // types); off the thread pool so it never blocks startup. + .ConfigureServices(services => + services.AddHostedService()) .AddMarkdownExport() // Register Azure Blob support for content collections. .ConfigureServices(services => services.AddAzureBlob()) + // Shared NodeType assembly cache (versioned, cross-replica consistent). + // Requires `AddKeyedAzureBlobServiceClient("nodetype-cache")` to have + // registered a keyed BlobServiceClient — Aspire wires this via the + // `nodetype-cache` container reference on the portal resource. + .ConfigureServices(services => services.AddBlobAssemblyStore()) // Register the mesh catalog and its public interfaces .ConfigureServices(services => services.AddMeshCatalog()) // Configure default views and content collections for each node hub @@ -367,10 +578,15 @@ public TBuilder ConfigureMemexMesh(IConfiguration configuration, bool isDevelopm // collection mapping below and the "attachments" mapping further down. var nodePath = config.Address.ToString(); - if (contentStorageConfig != null) + // Content lives ONCE per Space (partition root), NOT on every node. A child-node + // path (e.g. "AgenticPension/Dokument") must not get its own content collection — + // it inherits the Space's via ExposeInChildren below. Mounting per-child created + // overlapping/orphaned collections (content/{space}/{child}/…) and node-level content + // refs; indexing is likewise per-Space (one content_chunks table per partition schema). + // Gate on the partition root: a single-segment node path (no '/'). + if (contentStorageConfig != null && !nodePath.Contains('/')) { - // Scope static media (SVG, PNG, JPG) to a per-node subdirectory - // so each hub serves only its own content files. + // Scope static media (SVG, PNG, JPG) to the Space's content subdirectory. var contentSubdir = $"content/{nodePath}"; // Combine with original BasePath for FileSystem; for AzureBlob, subdirectory is the blob prefix var basePath = string.IsNullOrEmpty(contentStorageConfig.BasePath) @@ -380,11 +596,11 @@ public TBuilder ConfigureMemexMesh(IConfiguration configuration, bool isDevelopm { Name = "content", IsEditable = true, + ExposeInChildren = true, BasePath = basePath, - Settings = new Dictionary(contentStorageConfig.Settings ?? new()) - { - ["BasePath"] = basePath - } + Settings = contentStorageConfig.Settings is { } src + ? new Dictionary(src) { ["BasePath"] = basePath } + : new Dictionary { ["BasePath"] = basePath } }; config = config.AddContentCollection(_ => nodeContentConfig); } @@ -397,7 +613,26 @@ public TBuilder ConfigureMemexMesh(IConfiguration configuration, bool isDevelopm .WithHeartBeatHandler() // silently ack heartbeats on every per-node hub .AddDefaultLayoutAreas() .AddThreadsLayoutArea() - .AddApiTokensSettingsTab(); + .AddApiTokensSettingsTab() + .AddModelsSettingsTab() + // Agents are user/space CONTENT (under {user}/Agent · {space}/Agent), not admin + // config — surfaced on the user overview's namespace listing and created from chat + // (the agent proactively offers a /skill). Settings is admin-only territory: models + // + keys. (Agents tab removed; AgentsSettingsTab.cs retired.) + .AddAiSettingsTab() + // Dedicated Admin menu (platform-wide GlobalSettings area), gated on root + // Permission.All: Invitations + Inbox. + .AddInvitationsSettingsTab() + .AddInboxSettingsTab() + // Token-usage analytics (per-model _Usage satellites): filter by period, + // group by model / person / thread, cost from ModelPricing. + .AddTokenUsageSettingsTab() + // GitHub Sync tab — shows only on Space nodes (self-filtered). + .AddGitHubSyncSettingsTab() + // Code workspace tab — on-disk working-tree editor (checkout/edit/commit/push). + .AddWorkingTreeTab() + // Content Indexing tab — Space nodes, only when the indexing pipeline is active. + .AddContentIndexSettingsTab(); }) // Add activity tracking to record user access patterns via ActivityLogBundler .AddActivityTracking(); @@ -417,7 +652,30 @@ public TBuilder ConfigureMemexPortal() => (TBuilder)builder .AddUserProfileViews() // Register UserProfilePageView ) .AddBlazor(layoutClient => layoutClient - .WithPortalConfiguration(c => c) + // 🚨 The portal hub is the per-user sub-hub that hosts the + // Blazor circuit's chat input, autocomplete, navigation + // tracking, etc. Without these registrations: + // • Chat: AppendUserMessageResponse arrives as RawJson and the + // original Observe() hangs forever ("Allocating agent…" + // spinner). Need AI types in the portal's TypeRegistry. + // • Activity tracking: TrackActivityRequest emits + // "No handler found for delivery TrackActivityRequest in + // portal/" on every login + navigation. Need the + // graph-types handler chain (which includes + // HandleTrackActivity) registered on the portal. + // • Data layer: layout areas hosted in the portal (e.g. chat + // view) hold remote streams that depend on workspace + + // EntityStore serialisation; .AddData() wires that. + // + // Lives here in MemexConfiguration (not in MeshWeaver.Blazor's + // PortalApplication.DefaultPortalConfig) so the base portal + // library doesn't take a hard dependency on MeshWeaver.AI / + // MeshWeaver.Graph. + .WithPortalConfiguration(c => + { + c.TypeRegistry.AddAITypes(); + return c.AddData().WithGraphTypes(); + }) ); } @@ -432,6 +690,17 @@ public static void StartMemexApplication(this WebApplication app) where TA logger.LogInformation("Starting Memex portal on PID: {PID}", Environment.ProcessId); #pragma warning restore CA1416 + // Startup capability guard: if every AI provider AND every co-hosted CLI is + // disabled via Features:Ai, the model picker is empty unless users bring + // their own keys. Warn (not fail) — a pure data portal is a valid config. + var features = app.Configuration + .GetSection(MemexFeatureOptions.SectionName) + .Get() ?? new MemexFeatureOptions(); + if (!features.HasAnyChatCapability) + logger.LogWarning( + "No AI chat capability is enabled (Features:Ai has all providers and CLIs disabled). " + + "The model picker will be empty unless users add their own provider keys via ModelProviders."); + // Configure the HTTP request pipeline. if (!app.Environment.IsDevelopment()) { @@ -446,6 +715,42 @@ public static void StartMemexApplication(this WebApplication app) where TA // in local dev it's a no-op since no proxy sets those headers. app.UseForwardedHeaders(); + // 🚨 /healthz MUST short-circuit before the identity pipeline and before + // any Blazor page rendering. Kubernetes probes used to hit "/" — every + // probe request carries no cookies, so VirtualUserMiddleware minted a + // fresh guest VUser (mesh node + per-node hub graph) AND the probe + // forced a full server-side page prerender (layout-area sync hubs that + // no circuit ever disposes). At readiness-probe cadence (5 s) the portal + // accumulated 10,000+ leaked MessageHubs in ~25 minutes, the hosted-hub + // collection lock became the hot path of every routed stream message, + // and the instance wedged at 100% CPU — the 2026-06-12 atioz outage. + // Point ALL probes here; the endpoint answers without touching identity, + // the mesh, or the renderer. + app.Use((ctx, next) => + { + if (ctx.Request.Path.Equals("/healthz", StringComparison.OrdinalIgnoreCase)) + { + ctx.Response.StatusCode = StatusCodes.Status200OK; + return ctx.Response.WriteAsync("ok"); + } + return next(); + }); + + // `@/` is a markdown-authoring / autocomplete prefix — not a URL segment. + // Authors occasionally leak `@/` into raw HTML hrefs or users paste broken links. + // Permanent-redirect `/@/X` → `/X` so those never 404. + app.Use((ctx, next) => + { + var path = ctx.Request.Path.Value; + if (path != null && path.StartsWith("/@/", StringComparison.Ordinal)) + { + var target = path.Substring(2) + ctx.Request.QueryString; + ctx.Response.Redirect(target, permanent: true); + return Task.CompletedTask; + } + return next(); + }); + // Static files middleware must run before routing to serve _content/* paths from RCLs app.UseStaticFiles(); @@ -455,15 +760,54 @@ public static void StartMemexApplication(this WebApplication app) where TA app.UseAntiforgery(); app.UseCookiePolicy(); + // User-context middleware MUST run BEFORE the terminal endpoint maps + // (MapMeshMcp / MapMeshWeaver / MapLinkedInConnect). Once a request + // matches a terminal endpoint, no further `app.UseMiddleware<…>()` + // registered AFTER the Map* call ever sees it. With UserContextMiddleware + // after MapMeshMcp, MCP-Bearer requests skipped it entirely → + // accessService.Context stayed null → PostPipeline fell through to its + // hub-address fallback and stamped the message identity as + // `mesh/`. SecurityService then matched accessObject="mesh/" + // (no match) instead of accessObject="rbuergi" (Admin) → cross-partition + // writes denied while same-partition self-rule writes still passed. + // + // Order: UserContext → VirtualUser → Onboarding. UserContext extracts + // the real-user identity from OAuth claims / Bearer token first. Only + // if AccessService.Context is still null afterwards (no auth on the + // request) does VirtualUserMiddleware fall through to the cookie-backed + // guest identity. Before this swap, VirtualUserMiddleware ran first + // and bypassed VUser only on HttpContext.User.IsAuthenticated — but + // some flows (Bearer-token resolution inside UserContext) set the + // identity later in the pipeline, so VirtualUserMiddleware was + // wastefully creating a guest VUser node on legitimately-authed + // requests and the page crashed on + // "No handler found for CreateNodeRequest in portal/anonymous" + // when the create-request was posted to the portal hub instead of + // the mesh hub. See VUserHelper.EnsureVUserNode for the matching + // mesh-hub target fix. + app.UseMiddleware(); + app.UseMiddleware(); + app.UseMiddleware(); + //app.MapMeshWeaverSignalRHubs(); // Map MCP endpoint app.MapMeshMcp(); + // REST surface that mirrors MCP — POST /api/mesh/* (1:1 with MCP tools). + // Same Bearer auth policy as /mcp; multipart upload at /api/mesh/upload. + app.MapMeshApi(); + app.MapMeshWeaver(); - app.UseMiddleware(); - app.UseMiddleware(); - app.UseMiddleware(); + + // Social publishing — LinkedIn connect/pull endpoints. Must be AFTER + // UseAuthentication so HttpContext.User is populated. + app.MapLinkedInConnect(); + + // GitHub Sync — OAuth authorization-code connect endpoints (same ordering + // requirement: needs HttpContext.User). Stores the per-user token at + // {userId}/_Provider/GitHub. See Doc/Architecture/GitHubSync. + app.MapGitHubConnect(); // Use HTTPS redirection only for non-MCP paths (MCP needs HTTP for Claude Code) app.UseWhen( diff --git a/memex/Memex.Portal.Shared/MemexFeatureOptions.cs b/memex/Memex.Portal.Shared/MemexFeatureOptions.cs new file mode 100644 index 000000000..e81560ff6 --- /dev/null +++ b/memex/Memex.Portal.Shared/MemexFeatureOptions.cs @@ -0,0 +1,153 @@ +namespace Memex.Portal.Shared; + +/// +/// Deploy-time capability toggles for a Memex deployment, bound from the +/// Features configuration section. These declare which capabilities a +/// deployment ships — independent of whether a given key happens to be present. +/// A disabled flag is the operator's intent and wins even if a key is configured. +/// +/// All flags default to true so an absent Features section preserves +/// the current behaviour (no regression for existing deployments). Operators turn +/// capabilities OFF explicitly. The env-var form +/// (Features__Ai__Providers__OpenAI=false) flows identically through ACA env, +/// compose .env, and ARM createUiDefinition → container env. +/// +public sealed record MemexFeatureOptions +{ + public const string SectionName = "Features"; + + public AiFeatureOptions Ai { get; init; } = new(); + + /// Static-repo → DB sync: which partitions are materialized into and served from the DB. + public StaticRepoSyncFeatureOptions StaticRepoSync { get; init; } = new(); + + /// User self-provisioning (open vs closed registration). + public OnboardingFeatureOptions Onboarding { get; init; } = new(); + + /// Orleans clustering provider selection (membership store). + public OrleansFeatureOptions Orleans { get; init; } = new(); + + /// + /// True when the deployment ships at least one in-process API provider OR one + /// co-hosted CLI. When false, the portal has no built-in chat capability via + /// catalog sources (users may still bring their own keys via ModelProviders) — + /// surfaced as a startup warning, not a hard failure. + /// + public bool HasAnyChatCapability => Ai.Providers.HasAny || Ai.Clis.HasAny; +} + +/// +/// Static-repo → DB synchronization. Selects which partitions' build-time static content +/// (embedded docs, built-in agents, the model catalog) is materialized into and served from +/// the database partition via the static-repo import on boot — instead of the in-memory +/// read-only static provider. For a listed partition the import registers an +/// IStaticRepoSource, the read-only StaticNodePartitionStorageProvider is NOT +/// registered (so Postgres serves + accepts the import's writes), and ImportAll runs after +/// schema provisioning. See Doc/Architecture/StaticRepoImport.md. +/// +/// Empty (default) = no sync: every partition keeps the in-memory static provider, no DB +/// import — i.e. current behaviour, no regression. The default Helm deployment sets +/// ["Doc","Agent","Model"]. Gated, not global — the monolith (no Postgres) leaves this +/// empty and keeps in-memory serving. +/// +public sealed record StaticRepoSyncFeatureOptions +{ + /// + /// Partition names to materialize into + serve from the DB (e.g. "Doc", "Agent", + /// "Model"). Matching is case-insensitive. "Model" also covers the model + /// catalog's _Provider content partition. Empty = no sync. + /// + public string[] Partitions { get; init; } = []; + + /// True when is configured for DB sync. + public bool Includes(string partition) => + Partitions.Any(p => string.Equals(p, partition, StringComparison.OrdinalIgnoreCase)); +} + +/// +/// Controls whether a brand-new authenticated user may self-provision their own +/// account + per-user partition through the /onboarding flow (open vs +/// closed registration). +/// +public sealed record OnboardingFeatureOptions +{ + /// + /// When true (default — current behaviour, no regression), any newly + /// authenticated user without an Active User node may self-onboard. When + /// false, registration is closed: self-onboarding is refused with a + /// "contact your administrator" message instead of materialising the user. + /// + /// First-user bootstrap exception: a brand-new deployment with + /// ZERO existing User nodes always lets the very first user onboard (and + /// become platform admin) even when this flag is false — otherwise the + /// platform would lock out with no administrator. The exception reuses the + /// existing "no existing User nodes" detection in the onboarding flow. + /// + public bool AllowSelfOnboarding { get; init; } = true; + + /// + /// When true, onboarding is allowed ONLY for an email that has an outstanding + /// () . + /// An admin issues invitations from the "Invitations" settings tab; the invited person is + /// emailed and, when they sign in via the IdP, the verified email is matched against an + /// outstanding invitation. Any non-invited email is refused at the onboarding gate. + /// + /// Independent of : when invitation-only is on it is the + /// binding gate (an invited email onboards even if self-onboarding is also disabled). The + /// first-user bootstrap exception still applies — a brand-new deployment with zero + /// existing User nodes always lets the very first user onboard so the platform never locks out. + /// Default false preserves current behaviour. + /// + public bool InvitationOnly { get; init; } = false; +} + +/// +/// Selects the Orleans cluster-membership provider. Bound from +/// Features:Orleans:Clustering. Values: +/// +/// AzureTables (default) — Aspire-injected Azure Table Storage membership +/// (the ACA / Marketplace path; the silo relies on the Aspire Orleans integration). +/// AdoNet — PostgreSQL-backed membership on the separate orleans database +/// (real clustering for self-host / HA). The silo calls UseAdoNetClustering against the +/// Aspire-injected ConnectionStrings:orleans; the migration creates the membership tables. +/// Localhost — single in-process silo (local dev only; never production). +/// +/// +public sealed record OrleansFeatureOptions +{ + public string Clustering { get; init; } = "AzureTables"; +} + +public sealed record AiFeatureOptions +{ + /// In-process API providers (one flag each). + public AiProviderFeatureOptions Providers { get; init; } = new(); + + /// Co-hosted CLI providers (Claude Code, GitHub Copilot). + public AiCliFeatureOptions Clis { get; init; } = new(); +} + +public sealed record AiProviderFeatureOptions +{ + public bool Anthropic { get; init; } = true; + public bool AzureFoundry { get; init; } = true; + public bool AzureOpenAI { get; init; } = true; + public bool OpenAI { get; init; } = true; + + /// + /// The generic OpenAI-compatible custom-URL provider — the "type" a user picks in + /// Settings → Language Models to bring any OpenAI-wire endpoint (OpenRouter, Groq, + /// Together, a local vLLM, …) by base URL + key. No system default; always user-supplied. + /// + public bool OpenAICompatible { get; init; } = true; + + public bool HasAny => Anthropic || AzureFoundry || AzureOpenAI || OpenAI || OpenAICompatible; +} + +public sealed record AiCliFeatureOptions +{ + public bool ClaudeCode { get; init; } = true; + public bool Copilot { get; init; } = true; + + public bool HasAny => ClaudeCode || Copilot; +} diff --git a/memex/Memex.Portal.Shared/Models/ConnectTokenSink.cs b/memex/Memex.Portal.Shared/Models/ConnectTokenSink.cs new file mode 100644 index 000000000..369ce91cd --- /dev/null +++ b/memex/Memex.Portal.Shared/Models/ConnectTokenSink.cs @@ -0,0 +1,48 @@ +using System.Linq; +using System.Reactive.Linq; +using MeshWeaver.AI; +using MeshWeaver.AI.Connect; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Models; + +/// +/// Portal-side implementation of : persists a captured CLI +/// subscription token as an encrypted ModelProvider node via +/// (create on first connect, rotate on re-connect). Lives in the portal so the AI layer +/// (ConnectSessionManager) never references the portal assembly — the seam is the interface. +/// +/// Reactive end-to-end (no Task): create / rotate return cold observables and we +/// project their result into (providerNodePath, keyFingerprint). +/// +public sealed class ConnectTokenSink(ModelProviderService providerService, ILogger logger) + : IConnectTokenSink +{ + public IObservable<(string ProviderNodePath, string KeyFingerprint)> StoreToken( + string ownerPath, string providerName, string token) + { + if (string.IsNullOrEmpty(ownerPath)) + return Observable.Throw<(string, string)>(new ArgumentException("ownerPath required", nameof(ownerPath))); + if (string.IsNullOrEmpty(token)) + return Observable.Throw<(string, string)>(new ArgumentException("token required", nameof(token))); + + var fingerprint = ConnectSessionManager.Fingerprint(token); + // Must match ModelProviderService.CreateProvider's location — the user's + // own provider lives in their dotfile namespace ({owner}/_Memex/{provider}), + // and this path is the rotate-fallback target when the node already exists. + var providerPath = $"{ModelProviderNodeType.UserNamespacePath(ownerPath)}/{providerName}"; + + // Create on first connect; if the provider node already exists, rotate its key. We avoid + // GetProvidersForOwner().Take(1) here — that synced query is Replay(1).RefCount(), and the + // .Take(1) tears the upstream subscription down again before the create lands, which can + // wedge the brand-new partition's per-node hub. Instead: try create, fall back to rotate on + // conflict (the same create-or-update shape SetSelection uses). + logger.LogInformation("Connect: storing {Provider} key for {Owner} (fp={Fp})", + providerName, ownerPath, fingerprint); + return providerService.CreateProvider(ownerPath, providerName, token) + .Select(result => (result.ProviderNode.Path ?? providerPath, fingerprint)) + .Catch<(string, string), Exception>(_ => + providerService.RotateKey(providerPath, token) + .Select(_ => (providerPath, fingerprint))); + } +} diff --git a/memex/Memex.Portal.Shared/Models/ModelProviderService.cs b/memex/Memex.Portal.Shared/Models/ModelProviderService.cs new file mode 100644 index 000000000..43dfcf429 --- /dev/null +++ b/memex/Memex.Portal.Shared/Models/ModelProviderService.cs @@ -0,0 +1,476 @@ +using System.Collections.Immutable; +using System.Reactive; +using System.Reactive.Linq; +using System.Security.Cryptography; +using MeshWeaver.AI; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Models; + +/// +/// Service for creating, rotating, and deleting AI model provider credentials. +/// Modelled on +/// — credentials are stored as nodeType:ModelProvider MeshNodes in the +/// owner's dotfile namespace ({userId}/_Memex/{providerName}, the same +/// hidden namespace that hosts {user}/_Memex/ThreadComposer; see +/// ). Any node's namespace +/// works for shared / org-level credentials. +/// +/// +/// 🚨 Reactive end-to-end. No async, no await, no +/// FromAsync. Reads go through workspace.GetQuery (synced) or +/// workspace.GetMeshNodeStream (live single-node) per +/// SyncedMeshNodeQueries. +/// Writes go through / +/// and +/// on the workspace remote +/// stream. +/// +/// +/// Layout per provider entry (Anthropic example, owner = rbuergi): +/// +/// rbuergi/_Memex/Anthropic ← ModelProvider (carries ApiKey, RLS-gated) +/// rbuergi/_Memex/Anthropic/claude-opus-4-7 ← LanguageModel, ProviderRef → ../Anthropic +/// rbuergi/_Memex/Anthropic/claude-sonnet-4-6 ← LanguageModel, same ProviderRef +/// rbuergi/_Memex/Anthropic/claude-haiku-4-5-20251001 ← LanguageModel, same ProviderRef +/// +/// +/// The default model ids come from +/// the live ; callers +/// can override. +/// +public class ModelProviderService(IMeshService meshService, IMessageHub hub, ILogger logger) +{ + // Per-owner cached snapshot — feeds the Models settings UI without + // hitting the synced query on every render. Wrapped around the live + // workspace.GetQuery observable (which is itself Replay(1).RefCount), + // so the cache holds the latest projection for an hour and writes + // (Create/RotateKey/Delete) explicitly invalidate the entry. The + // upstream synced query continues to push live updates into the cached + // observable so consumers always see fresh data within the TTL. + private static readonly TimeSpan CacheTtl = TimeSpan.FromHours(1); + private readonly System.Collections.Concurrent.ConcurrentDictionary> Stream, DateTimeOffset ExpiresAt)> + cachedStreams = new(StringComparer.Ordinal); + + private LanguageModelCatalogSource? FindCatalogSource(string providerName) + { + var opts = hub.ServiceProvider.GetService(); + return opts?.Sources.FirstOrDefault(s => + string.Equals(s.ProviderName, providerName, StringComparison.OrdinalIgnoreCase)); + } + + private void InvalidateCache(string ownerPath) + { + cachedStreams.TryRemove(ownerPath, out _); + } + + /// + /// Reactive provider creation. Creates the ModelProvider node at + /// {ownerPath}/_Memex/{instanceId ?? provider} with the supplied key + + /// endpoint, then creates one LanguageModel child per model id (each + /// pointing back at the provider node via + /// ). Defaults come from the + /// live registered by + /// each provider's AddXxxCatalog extension — no central registry. + /// + /// is the node id (and path segment); it + /// defaults to . For a generic OpenAI-compatible + /// provider the user can stand up several instances (OpenRouter, Groq, …) that + /// all carry Content.Provider = "OpenAICompatible" (the wire-protocol + /// stamp that routes them to the OpenAI factory) but live at distinct paths so + /// they never collide. For named providers (OpenAI, Anthropic) leave it null — + /// one instance per type, keyed by the provider name. + /// + public IObservable CreateProvider( + string ownerPath, + string provider, + string? apiKey, + string? label = null, + string? endpointOverride = null, + IReadOnlyList? modelIdsOverride = null, + string? instanceId = null) + { + if (string.IsNullOrEmpty(ownerPath)) + return Observable.Throw(new ArgumentException("ownerPath required", nameof(ownerPath))); + if (string.IsNullOrEmpty(provider)) + return Observable.Throw(new ArgumentException("provider required", nameof(provider))); + + // The node id/path segment. Distinct instances of the same wire-protocol + // provider (e.g. two OpenAICompatible gateways) get distinct ids while + // Content.Provider stays the protocol stamp. + var providerId = string.IsNullOrWhiteSpace(instanceId) ? provider : instanceId!.Trim(); + + var source = FindCatalogSource(provider); + var endpoint = endpointOverride ?? source?.DefaultEndpoint; + if (string.IsNullOrEmpty(endpoint)) endpoint = null; + + var modelIds = (modelIdsOverride ?? (IReadOnlyList?)source?.EffectiveModelIds) + ?? Array.Empty(); + + // User-owned providers/models live in the owner's dotfile namespace + // ({owner}/_Memex/{providerId}/{model}), NOT a shared _Provider satellite. + // See ModelProviderNodeType.UserNamespace. + var providerNamespace = ModelProviderNodeType.UserNamespacePath(ownerPath); + var providerPath = $"{providerNamespace}/{providerId}"; + + var providerConfig = new ModelProviderConfiguration + { + Provider = provider, + ApiKey = Protect(apiKey), + Endpoint = endpoint, + Label = label ?? (string.IsNullOrWhiteSpace(instanceId) ? null : instanceId) ?? source?.EffectiveLabel ?? provider, + CreatedAt = DateTimeOffset.UtcNow, + Models = modelIds.Where(m => !string.IsNullOrWhiteSpace(m)) + .ToImmutableArrayCompat(), + }; + + var providerNode = new MeshNode(providerId, providerNamespace) + { + NodeType = ModelProviderNodeType.NodeType, + Name = providerConfig.Label, + State = MeshNodeState.Active, + MainNode = ownerPath, + Content = providerConfig, + }; + + logger.LogInformation("Creating ModelProvider {ProviderId} (provider={Provider}) for owner {Owner} with {ModelCount} models, keyFp={KeyFp}", + providerId, provider, ownerPath, modelIds.Count, Fingerprint(apiKey)); + + // 1. Create the ModelProvider node. + // 2. After commit, fan out N CreateNode calls for the LanguageModel children. + // The children reference the provider via ProviderRef. + InvalidateCache(ownerPath); + return meshService.CreateNode(providerNode) + .SelectMany(createdProvider => + { + var modelObservables = modelIds + .Where(m => !string.IsNullOrWhiteSpace(m)) + .Select(modelId => + { + var modelDef = new ModelDefinition + { + Id = modelId, + DisplayName = modelId, + Provider = provider, + Endpoint = null, // resolver follows ProviderRef + ApiKeySecretRef = null, + ProviderRef = createdProvider.Path, + Order = source?.Order ?? 0 + }; + var modelNode = new MeshNode(modelId, providerPath) + { + NodeType = LanguageModelNodeType.NodeType, + Name = modelId, + Category = "Models", + State = MeshNodeState.Active, + MainNode = ownerPath, + Content = modelDef, + }; + return meshService.CreateNode(modelNode) + .Catch(ex => + { + logger.LogWarning(ex, "Failed to create LanguageModel {ModelId} under {Path}", modelId, providerPath); + return Observable.Return(null!); + }); + }) + .ToArray(); + + if (modelObservables.Length == 0) + return Observable.Return(new ProviderCreationResult(createdProvider, Array.Empty())); + + return Observable.CombineLatest(modelObservables) + .Take(1) + .Select(children => new ProviderCreationResult( + createdProvider, + children.Where(c => c != null).ToArray())); + }); + } + + /// + /// Reactive rotate-key. Updates the ApiKey field on the + /// ModelProvider node via + /// . Other fields are + /// preserved. + /// + public IObservable RotateKey(string providerNodePath, string? newApiKey) + { + if (string.IsNullOrEmpty(providerNodePath)) + return Observable.Return(false); + + logger.LogInformation("Rotating ModelProvider key at {Path} newKeyFp={KeyFp}", + providerNodePath, Fingerprint(newApiKey)); + + var workspace = hub.GetWorkspace(); + return workspace.GetMeshNodeStream(providerNodePath) + .Update(current => + { + var cfg = current.Content as ModelProviderConfiguration + ?? ExtractContent(current); + if (cfg == null) return current; + return current with { Content = cfg with { ApiKey = Protect(newApiKey) } }; + }) + .Do(updatedNode => + { + // Force persistence at the per-node hub. Sync-protocol updates + // don't always fire the per-node hub's `saveSub` for + // remote-driven changes (see ApiTokenService.RevokeToken for + // the matching pattern + comment). + hub.Post(new SaveMeshNodeRequest(updatedNode), + o => o.WithTarget(new Address(providerNodePath))); + }) + .Select(_ => true) + .Catch(ex => + { + logger.LogWarning(ex, "RotateKey failed for {Path}", providerNodePath); + return Observable.Return(false); + }); + } + + /// + /// Reactive cascade-delete. Removes all child LanguageModel nodes + /// (their paths recorded in the provider's + /// snapshot), then the + /// ModelProvider node itself. + /// + public IObservable DeleteProvider(string providerNodePath) + { + if (string.IsNullOrEmpty(providerNodePath)) + return Observable.Return(false); + + logger.LogInformation("Deleting ModelProvider {Path} (cascade includes child LanguageModels)", providerNodePath); + + var workspace = hub.GetWorkspace(); + return workspace.GetMeshNodeStream(providerNodePath) + .Take(1) + .SelectMany(current => + { + var cfg = current?.Content as ModelProviderConfiguration + ?? ExtractContent(current); + var childPaths = cfg?.Models + .Where(m => !string.IsNullOrWhiteSpace(m)) + .Select(m => $"{providerNodePath}/{m}") + .ToArray() + ?? Array.Empty(); + + IObservable childDeletes = childPaths.Length == 0 + ? Observable.Return(Unit.Default) + : Observable.CombineLatest(childPaths.Select(p => + meshService.DeleteNode(p) + .Catch(ex => + { + logger.LogDebug(ex, "Child LanguageModel delete failed for {Path}", p); + return Observable.Return(false); + }))) + .Take(1) + .Select(_ => Unit.Default); + + return childDeletes + .SelectMany(_ => meshService.DeleteNode(providerNodePath)) + .Select(_ => true); + }) + .Catch(ex => + { + logger.LogWarning(ex, "DeleteProvider failed for {Path}", providerNodePath); + return Observable.Return(false); + }); + } + + /// + /// Live list of ModelProviders owned by . + /// Same shape as + /// + /// — synced via workspace.GetQuery. + /// + public IObservable> GetProvidersForOwner(string ownerPath) + { + if (string.IsNullOrEmpty(ownerPath)) + return Observable.Return((IReadOnlyList)Array.Empty()); + + if (cachedStreams.TryGetValue(ownerPath, out var entry) && entry.ExpiresAt > DateTimeOffset.UtcNow) + return entry.Stream; + + var workspace = hub.GetWorkspace(); + var providerNamespace = ModelProviderNodeType.UserNamespacePath(ownerPath); + + var stream = workspace.GetQuery( + $"model-providers:{ownerPath}", + $"namespace:{providerNamespace} nodeType:{ModelProviderNodeType.NodeType}") + .Select(snapshot => + { + var providers = new List(); + foreach (var node in snapshot) + { + if (node.Path is null) continue; + if (!string.Equals(node.NodeType, ModelProviderNodeType.NodeType, StringComparison.OrdinalIgnoreCase)) + continue; + var cfg = node.Content as ModelProviderConfiguration + ?? ExtractContent(node); + if (cfg == null) continue; + providers.Add(new ProviderInfo + { + NodePath = node.Path, + Provider = cfg.Provider, + Label = cfg.Label, + Endpoint = cfg.Endpoint, + CreatedAt = cfg.CreatedAt, + LastUsedAt = cfg.LastUsedAt, + ModelIds = cfg.Models.ToArray(), + ApiKeyFingerprint = Fingerprint(Unprotect(cfg.ApiKey)), + }); + } + return (IReadOnlyList)providers; + }) + // Replay the latest projected snapshot to subsequent subscribers + // without re-subscribing upstream. The upstream synced query + // pushes live changes through; the TTL bounds how long we keep + // the projection alive when nobody is actively watching. + .Replay(1) + .RefCount(); + + cachedStreams[ownerPath] = (stream, DateTimeOffset.UtcNow + CacheTtl); + return stream; + } + + /// + /// Live list of the owner's selected provider paths (the provider-selection + /// picker). Empty when no selection node exists yet. Single-node read via + /// GetMeshNodeStream per CqrsAndContentAccess. + /// + public IObservable> GetSelection(string ownerPath) + { + if (string.IsNullOrEmpty(ownerPath)) + return Observable.Return(ImmutableArray.Empty); + // 🚨 Read the selection via a QUERY, not a point GetMeshNodeStream(exactPath): + // a pre-existing user partition has no selection node, and a point-subscribe + // to a missing path routes to a NotFound DeliveryFailure (the resubscribe-storm + // that froze the portal, 2026-06-09). A query returns EMPTY on absence — the + // documented "no selection ⇒ default catalog" behaviour — and never errors. + return hub.GetWorkspace() + .GetQuery( + $"{ModelProviderNodeType.SelectionNodeType}|{ownerPath}", + $"namespace:{ModelProviderNodeType.UserNamespacePath(ownerPath)} nodeType:{ModelProviderNodeType.SelectionNodeType}") + .Select(snapshot => + { + var node = snapshot.FirstOrDefault(n => + string.Equals(n.NodeType, ModelProviderNodeType.SelectionNodeType, StringComparison.OrdinalIgnoreCase)); + var sel = node?.Content as ModelProviderSelection + ?? ExtractContent(node); + if (sel is null) return ImmutableArray.Empty; + return sel.SelectedProviderPaths.IsDefault + ? ImmutableArray.Empty + : sel.SelectedProviderPaths; + }); + } + + /// + /// Persist the owner's selected provider paths. Create-or-update via + /// stream.Update — when the node doesn't exist yet the handshake + /// delivers null and we substitute the full new node (own-node + /// writes upsert through the local data source). + /// + public IObservable SetSelection(string ownerPath, ImmutableArray providerPaths) + { + if (string.IsNullOrEmpty(ownerPath)) + return Observable.Return(false); + + var ns = ModelProviderNodeType.UserNamespacePath(ownerPath); + var content = new ModelProviderSelection { SelectedProviderPaths = providerPaths }; + var newNode = new MeshNode(ModelProviderNodeType.SelectionNodeId, ns) + { + NodeType = ModelProviderNodeType.SelectionNodeType, + Name = "Model Provider Selection", + State = MeshNodeState.Active, + MainNode = ownerPath, + Content = content, + }; + + // Create on first write; fall back to update when the node already + // exists (stream.Update alone does not create a missing own-node). + return meshService.CreateNode(newNode) + .Select(_ => true) + .Catch(_ => hub.GetWorkspace() + .GetMeshNodeStream(newNode.Path) + .Update(current => current is null ? newNode : current with { Content = content }) + .Select(_ => true)) + .Catch(ex => + { + logger.LogWarning(ex, "SetSelection failed for {Owner}", ownerPath); + return Observable.Return(false); + }); + } + + private T? ExtractContent(MeshNode? node) where T : class + { + if (node?.Content is null) return null; + if (node.Content is T typed) return typed; + if (node.Content is System.Text.Json.JsonElement je) + { + try { return System.Text.Json.JsonSerializer.Deserialize(je.GetRawText(), hub.JsonSerializerOptions); } + catch { return null; } + } + return null; + } + + // Encryption-at-rest for the literal ApiKey. Resolved lazily from the hub's + // service provider (same place the ChatClientCredentialResolver reads it); + // passthrough when not registered or no master key is configured. + private string? Protect(string? plaintext) + { + var protector = hub.ServiceProvider.GetService(); + return protector is null ? plaintext : protector.Protect(plaintext); + } + + private string? Unprotect(string? stored) + { + var protector = hub.ServiceProvider.GetService(); + return protector is null ? stored : protector.Unprotect(stored); + } + + /// + /// 8-char SHA-256 prefix — never the raw key. Same shape as the + /// factories' Fingerprint helper so logs/UI can correlate across + /// layers. + /// + private static string Fingerprint(string? value) + { + if (string.IsNullOrEmpty(value)) return "(empty)"; + var bytes = System.Text.Encoding.UTF8.GetBytes(value); + var hash = SHA256.HashData(bytes); + return Convert.ToHexString(hash, 0, 4).ToLowerInvariant(); + } +} + +/// +/// Returned by once the +/// provider node + all child LanguageModel nodes have been written. +/// +public record ProviderCreationResult(MeshNode ProviderNode, IReadOnlyList ModelNodes); + +/// +/// Safe DTO for listing providers — exposes a SHA-256 fingerprint of the +/// key rather than the key itself, so the UI can show "is this set / has +/// this changed" without reading the literal credential. +/// +public record ProviderInfo +{ + public string NodePath { get; init; } = ""; + public string Provider { get; init; } = ""; + public string? Label { get; init; } + public string? Endpoint { get; init; } + public DateTimeOffset CreatedAt { get; init; } + public DateTimeOffset? LastUsedAt { get; init; } + public IReadOnlyList ModelIds { get; init; } = Array.Empty(); + public string ApiKeyFingerprint { get; init; } = "(empty)"; +} + +internal static class ImmutableArrayExtensions +{ + public static System.Collections.Immutable.ImmutableArray ToImmutableArrayCompat(this IEnumerable source) => + System.Collections.Immutable.ImmutableArray.CreateRange(source); +} diff --git a/memex/Memex.Portal.Shared/Models/ProviderModelLister.cs b/memex/Memex.Portal.Shared/Models/ProviderModelLister.cs new file mode 100644 index 000000000..021d5e5fa --- /dev/null +++ b/memex/Memex.Portal.Shared/Models/ProviderModelLister.cs @@ -0,0 +1,116 @@ +using System.Net.Http.Headers; +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.Mesh.Threading; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Models; + +/// +/// Fetches the live model list from a provider's HTTP API so the +/// Settings → Language Models tab can let the user pick which models to bring, +/// instead of relying on a static baked-in id list. Powers the generic +/// "add provider → URL + key → fetch models → select" flow (the OpenRouter / +/// Groq / Together / vLLM path, and direct OpenAI). +/// +/// 🚨 Reactive end-to-end: the HTTP leaf runs inside the +/// and the public surface is +/// — no async/await/Task escapes a +/// signature (mirrors GitHubOAuthService; see +/// Doc/Architecture/ControlledIoPooling.md). The pool is resolved from the +/// hub's service provider so it shares the mesh's I/O bound. +/// +/// Two wire shapes are handled, both returning { "data": [ { "id" } ] }: +/// the OpenAI family (Bearer auth, {baseUrl}/models — covers OpenAI and every +/// OpenAI-compatible gateway) and Anthropic (x-api-key + anthropic-version, +/// https://api.anthropic.com/v1/models). Providers without a supported list +/// endpoint surface their error to the UI, which falls back to the catalog defaults + +/// manual entry. +/// +public sealed class ProviderModelLister +{ + private readonly IMessageHub hub; + private readonly HttpClient http; + private readonly ILogger? logger; + + public ProviderModelLister(IMessageHub hub, ILogger? logger = null, HttpClient? httpClient = null) + { + this.hub = hub; + this.logger = logger; + http = httpClient ?? new HttpClient { Timeout = TimeSpan.FromSeconds(30) }; + if (!http.DefaultRequestHeaders.UserAgent.Any()) + http.DefaultRequestHeaders.UserAgent.ParseAdd("MeshWeaver-ModelLister"); + } + + private IIoPool Http => hub.ServiceProvider.GetRequiredService().Get(IoPoolNames.Http); + + /// + /// Live model ids offered by the provider at for the + /// given . selects the wire + /// shape (Anthropic vs the OpenAI family). Sorted, de-duplicated. Throws (via + /// OnError) on a non-success response so the UI can show the reason and fall back. + /// + public IObservable> ListModels(string? endpoint, string apiKey, string? providerName = null) + { + if (string.IsNullOrWhiteSpace(apiKey)) + return Observable.Throw>( + new InvalidOperationException("An API key is required to fetch the model list.")); + return Http.Invoke(ct => ListAsync(endpoint, apiKey, providerName, ct)); + } + + // ── HTTP leaf (runs inside the I/O pool) ────────────────────────────────── + private async Task> ListAsync( + string? endpoint, string apiKey, string? providerName, CancellationToken ct) + { + var isAnthropic = string.Equals(providerName, "Anthropic", StringComparison.OrdinalIgnoreCase); + + // OpenAI family: {baseUrl}/models. Blank endpoint → the OpenAI default. The + // base URL must include its version segment (e.g. .../v1), which is the + // OpenAI-compatible convention; OpenRouter's "https://openrouter.ai/api/v1" + // → "https://openrouter.ai/api/v1/models". + var url = isAnthropic + ? "https://api.anthropic.com/v1/models" + : (string.IsNullOrWhiteSpace(endpoint) ? "https://api.openai.com/v1" : endpoint.Trim()) + .TrimEnd('/') + "/models"; + + using var req = new HttpRequestMessage(HttpMethod.Get, url); + if (isAnthropic) + { + req.Headers.TryAddWithoutValidation("x-api-key", apiKey); + req.Headers.TryAddWithoutValidation("anthropic-version", "2023-06-01"); + } + else + { + req.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiKey); + } + req.Headers.Accept.ParseAdd("application/json"); + + using var resp = await http.SendAsync(req, ct).ConfigureAwait(false); + var json = await resp.Content.ReadAsStringAsync(ct).ConfigureAwait(false); + if (!resp.IsSuccessStatusCode) + { + logger?.LogInformation("Model list fetch from {Url} failed: {Status}", url, (int)resp.StatusCode); + throw new InvalidOperationException( + $"The provider rejected the model-list request ({(int)resp.StatusCode} {resp.StatusCode}). {Truncate(json)}"); + } + + using var doc = JsonDocument.Parse(json); + var ids = new List(); + if (doc.RootElement.TryGetProperty("data", out var data) && data.ValueKind == JsonValueKind.Array) + { + foreach (var m in data.EnumerateArray()) + if (m.TryGetProperty("id", out var id) && id.ValueKind == JsonValueKind.String) + ids.Add(id.GetString()!); + } + + return ids.Where(s => !string.IsNullOrWhiteSpace(s)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(s => s, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + private static string Truncate(string s) => + string.IsNullOrEmpty(s) ? "" : (s.Length <= 200 ? s : s[..200] + "…"); +} diff --git a/memex/Memex.Portal.Shared/Notifications/NotificationTriageService.cs b/memex/Memex.Portal.Shared/Notifications/NotificationTriageService.cs new file mode 100644 index 000000000..614681b99 --- /dev/null +++ b/memex/Memex.Portal.Shared/Notifications/NotificationTriageService.cs @@ -0,0 +1,167 @@ +using System.Collections.Concurrent; +using System.Reactive.Disposables; +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.AI; // StartThread +using MeshWeaver.Blazor.Infrastructure; // PortalApplication +using MeshWeaver.Graph.Configuration; // Notification* NodeType segment consts +using MeshWeaver.Mesh; // Notification, MeshNode +using MeshWeaver.Mesh.Security; // ImpersonateAsSystem +using MeshWeaver.Mesh.Services; // IMeshQueryCore, MeshQueryRequest +using MeshWeaver.Messaging; // IMessageHub, AccessService +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Notifications; + +/// +/// Watches for new in-app s and, only for recipients who authored routing +/// rules, invokes the cheap Notification Triage agent to decide whether to ALSO escalate the +/// notification to that recipient's other channels (email today, Teams next). The in-app bell is the +/// always-on default; this service never duplicates it — it only escalates per the recipient's +/// s. +/// +/// Cost/safety guards: deferred to ApplicationStarted (mesh must be up); only notifications +/// created after startup are considered (no back-routing history, no double-route across restart); +/// each is processed once (instance dedup set); the triage agent is invoked only when the recipient +/// has at least one rule, so it is free for everyone else; all failures are logged, never fatal. +/// +/// Scope note (v1): the notification watch is mesh-wide and the dedup set is unbounded over the +/// process lifetime — fine for current volumes, but scope/bounding is the obvious next hardening step before +/// relying on this at scale. +/// +public sealed class NotificationTriageService( + IServiceProvider rootServices, + IHostApplicationLifetime lifetime, + ILogger? logger = null) : IHostedService, IDisposable +{ + private const string TriageAgent = "NotificationTriage"; + + private readonly CompositeDisposable subscriptions = new(); + private readonly ConcurrentDictionary processed = new(); // instance, not static + private IServiceScope? scope; + private DateTimeOffset startedAt; + + public Task StartAsync(CancellationToken cancellationToken) + { + lifetime.ApplicationStarted.Register(Begin); + return Task.CompletedTask; + } + + private void Begin() + { + try + { + startedAt = DateTimeOffset.UtcNow; + scope = rootServices.CreateScope(); + var hub = scope.ServiceProvider.GetRequiredService().Hub; + var sp = hub.ServiceProvider; + var query = sp.GetRequiredService(); + var access = sp.GetRequiredService(); + var jsonOptions = hub.JsonSerializerOptions; + + subscriptions.Add(query + .Query(MeshQueryRequest.FromQuery( + $"nodeType:{NotificationNodeType.NodeType}"), jsonOptions) + .Select(change => change.Items) + .Subscribe( + items => + { + foreach (var node in items) + TryRoute(node, hub, query, access, jsonOptions); + }, + ex => logger?.LogWarning(ex, "NotificationTriage: notification query failed"))); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "NotificationTriage: failed to start"); + } + } + + private void TryRoute( + MeshNode node, IMessageHub hub, IMeshQueryCore query, AccessService access, JsonSerializerOptions jsonOptions) + { + if (string.IsNullOrEmpty(node.Path)) return; + var notification = NotificationOf(node, jsonOptions); + if (notification is null) return; + + // Only notifications raised after we started: avoids back-routing the whole history on first run + // and avoids double-routing across a restart (startedAt resets, older ones are skipped). + if (notification.CreatedAt <= startedAt) return; + // Process each notification exactly once even though the live query re-emits the full set on change. + if (!processed.TryAdd(node.Path, 0)) return; + + var recipient = node.Path.Split('/', StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(); + if (string.IsNullOrEmpty(recipient)) return; + + // Cost gate: invoke the (cheap) triage agent ONLY when the recipient actually authored rules. + // No rules → the in-app bell default stands, and we never spend a model call. + query.Query(MeshQueryRequest.FromQuery( + $"nodeType:{NotificationRuleNodeType.NodeType} " + + $"namespace:{recipient}/{NotificationRuleNodeType.UserSegment} limit:1"), jsonOptions) + .Select(change => change.Items) + .Take(1) + .Timeout(TimeSpan.FromSeconds(10)) + .Subscribe( + rules => + { + if (rules.Count == 0) return; // no rules → in-app only (default), nothing to do + Escalate(hub, access, recipient, node.Path, notification); + }, + _ => { /* timeout / error: leave at the in-app default */ }); + } + + private void Escalate( + IMessageHub hub, AccessService access, string recipient, string notificationPath, Notification n) + { + var prompt = + $"A new notification for user '{recipient}':\n" + + $"Title: {n.Title}\n" + + $"Message: {n.Message}\n" + + $"Type: {n.NotificationType}\n" + + $"From: {n.CreatedBy}\n" + + $"Notification node: {notificationPath}\n" + + $"Related node: {n.TargetNodePath}\n\n" + + "The in-app bell notification is already shown. Per this recipient's NotificationRules and " + + "NotificationChannels, decide whether to ALSO escalate to email (or Teams) and create the " + + "delivery node(s). If their rules don't call for escalation, do nothing."; + try + { + using (access.ImpersonateAsSystem()) + hub.StartThread( + namespacePath: $"{recipient}/_Triage", + userText: prompt, + agentName: TriageAgent, + mainNode: notificationPath, + contextPath: notificationPath, + createdBy: "system", + onError: err => logger?.LogWarning("NotificationTriage: StartThread failed: {Err}", err)); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "NotificationTriage: escalation failed for {Path}", notificationPath); + } + } + + private static Notification? NotificationOf(MeshNode node, JsonSerializerOptions opts) => node.Content switch + { + Notification x => x, + JsonElement je => Safe(je, opts), + _ => null + }; + + private static Notification? Safe(JsonElement je, JsonSerializerOptions opts) + { + try { return JsonSerializer.Deserialize(je.GetRawText(), opts); } + catch { return null; } + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public void Dispose() + { + subscriptions.Dispose(); + scope?.Dispose(); + } +} diff --git a/memex/Memex.Portal.Shared/OrganizationLayoutAreas.cs b/memex/Memex.Portal.Shared/OrganizationLayoutAreas.cs deleted file mode 100644 index 83efad56f..000000000 --- a/memex/Memex.Portal.Shared/OrganizationLayoutAreas.cs +++ /dev/null @@ -1,204 +0,0 @@ -using System.Reactive.Linq; -using MeshWeaver.Data; -using MeshWeaver.Graph; -using MeshWeaver.Layout; -using MeshWeaver.Layout.Composition; -using MeshWeaver.Mesh; -using MeshWeaver.Mesh.Security; - -namespace Memex.Portal.Shared; - -/// -/// Custom views for Organization nodes. -/// -public static class OrganizationLayoutAreas -{ - /// - /// GitHub-style organization header view with standard children section. - /// Shows logo, name, description, verified badge, contact info, then delegates to standard view for children. - /// - public static IObservable Overview(LayoutAreaHost host, RenderingContext _) - { - var hubPath = host.Hub.Address.ToString(); - - var orgStream = host.Workspace.GetStream() - ?.Select(orgs => orgs?.FirstOrDefault()) - ?? Observable.Return(null); - - var nodeStream = host.Workspace.GetStream() - ?.Select(nodes => nodes?.FirstOrDefault(n => n.Path == hubPath)) - ?? Observable.Return(null); - - return orgStream.CombineLatest(nodeStream).SelectMany(async t => - { - var (org, node) = t; - if (org == null && node == null) - return Controls.Markdown("*Loading...*") as UiControl; - - var perms = await PermissionHelper.GetEffectivePermissionsAsync(host.Hub, hubPath); - var canEdit = perms.HasFlag(Permission.Update); - return BuildOrganizationView(host, org, node, hubPath, canEdit); - }); - } - - private static UiControl BuildOrganizationView( - LayoutAreaHost host, - Organization? org, - MeshNode? node, - string hubPath, - bool canEdit = false) - { - var name = org?.Name ?? node?.Name ?? "Organization"; - var description = org?.Description; - var logo = org?.Logo ?? GetNodeLogo(node); - var website = org?.Website; - var location = org?.Location; - var email = org?.Email; - var isVerified = org?.IsVerified ?? false; - - var container = Controls.Stack - .WithStyle("padding: 24px 0; width: 100%;"); - - // Main header row: logo + info + menu (menu on far right) - var headerRow = Controls.Stack - .WithOrientation(Orientation.Horizontal) - .WithStyle("gap: 24px; align-items: flex-start; width: 100%; max-width: 1280px; margin: 0 auto; padding: 0 24px;"); - - // Logo (large, rounded square like GitHub) - UiControl logoControl; - if (!string.IsNullOrEmpty(logo)) - { - logoControl = Controls.Html( - $"\"\""); - } - else - { - // Placeholder image with initials - var initials = GetInitials(name); - logoControl = Controls.Html( - $"
" + - $"{System.Web.HttpUtility.HtmlEncode(initials)}
"); - } - - if (canEdit) - { - logoControl = BuildEditableLogo(host, node, logoControl); - } - headerRow = headerRow.WithView(logoControl); - - // Info column (flex: 1 to take remaining space) - var infoColumn = Controls.Stack.WithStyle("gap: 8px; flex: 1;"); - - // Organization name (large) - infoColumn = infoColumn.WithView(Controls.Html( - $"

{System.Web.HttpUtility.HtmlEncode(name)}

")); - - // Description/tagline (rendered as markdown for rich formatting) - if (!string.IsNullOrEmpty(description)) - { - infoColumn = infoColumn.WithView( - Controls.Markdown(description).WithStyle("color: var(--neutral-foreground-hint); font-size: 1rem;")); - } - - // Verified badge - if (isVerified) - { - infoColumn = infoColumn.WithView(Controls.Html( - "" + - "" + - "Verified")); - } - - // Stats row: location, website, email - var statsRow = Controls.Stack - .WithOrientation(Orientation.Horizontal) - .WithStyle("gap: 24px; margin-top: 16px; flex-wrap: wrap;"); - - if (!string.IsNullOrEmpty(location)) - { - statsRow = statsRow.WithView(Controls.Html( - $"" + - $"" + - $"{System.Web.HttpUtility.HtmlEncode(location)}")); - } - - if (!string.IsNullOrEmpty(website)) - { - var displayUrl = website.Replace("https://", "").Replace("http://", "").TrimEnd('/'); - statsRow = statsRow.WithView(Controls.Html( - $"" + - $"" + - $"{System.Web.HttpUtility.HtmlEncode(displayUrl)}")); - } - - if (!string.IsNullOrEmpty(email)) - { - statsRow = statsRow.WithView(Controls.Html( - $"" + - $"" + - $"{System.Web.HttpUtility.HtmlEncode(email)}")); - } - - infoColumn = infoColumn.WithView(statsRow); - headerRow = headerRow.WithView(infoColumn); - - container = container.WithView(headerRow); - - // Divider - container = container.WithView(Controls.Html( - "
")); - - // Markdown body from index.md — PreRenderedHtml is set by MarkdownFileParser - // for any .md file; MarkdownView handles mermaid, code blocks, math, UCR links - if (!string.IsNullOrWhiteSpace(node?.PreRenderedHtml)) - { - container = container.WithView( - new MarkdownControl("") { Html = node.PreRenderedHtml } - .WithStyle("max-width: 1280px; margin: 0 auto; padding: 0 24px 48px 24px;")); - } - - // Use LayoutAreaControl to render the standard Catalog view for children - container = container.WithView( - LayoutAreaControl.Children(host.Hub)); - - return container; - } - - /// - /// Wraps the logo control with a hover overlay and click handler to open a file browser - /// for uploading a new logo/icon. Reuses the same dialog pattern as BuildHeader's editable icon. - /// - private static UiControl BuildEditableLogo(LayoutAreaHost host, MeshNode? node, UiControl logoControl) - { - var nodePath = node?.Path ?? host.Hub.Address.ToString(); - - var wrapper = Controls.Stack - .WithStyle("position: relative; width: 100px; height: 100px; cursor: pointer; border-radius: 12px; overflow: hidden; flex-shrink: 0;") - .WithView(logoControl) - .WithView(Controls.Html( - "
" + - "✏
")) - .WithClickAction(ctx => - { - MeshNodeLayoutAreas.OpenChangeIconDialog(ctx.Host, node, nodePath); - }); - - return wrapper; - } - - private static string? GetNodeLogo(MeshNode? node) - { - return MeshNodeThumbnailControl.GetImageUrlForNode(node); - } - - private static string GetInitials(string name) - { - if (string.IsNullOrWhiteSpace(name)) return "?"; - var parts = name.Split(' ', StringSplitOptions.RemoveEmptyEntries); - if (parts.Length >= 2) - return $"{char.ToUpper(parts[0][0])}{char.ToUpper(parts[1][0])}"; - return name.Length >= 2 ? $"{char.ToUpper(name[0])}{char.ToUpper(name[1])}" : char.ToUpper(name[0]).ToString(); - } -} diff --git a/memex/Memex.Portal.Shared/OrganizationNodeType.cs b/memex/Memex.Portal.Shared/OrganizationNodeType.cs deleted file mode 100644 index 6f9b470d7..000000000 --- a/memex/Memex.Portal.Shared/OrganizationNodeType.cs +++ /dev/null @@ -1,172 +0,0 @@ -using System.ComponentModel.DataAnnotations; -using MeshWeaver.ContentCollections; -using MeshWeaver.Domain; -using MeshWeaver.Graph; -using MeshWeaver.Graph.Configuration; -using MeshWeaver.Layout; -using MeshWeaver.Markdown; -using MeshWeaver.Mesh; -using MeshWeaver.Mesh.Security; -using MeshWeaver.Mesh.Services; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -namespace Memex.Portal.Shared; - -/// -/// Represents a company, team, or organizational unit. -/// -public record Organization -{ - [Required] - [MeshNodeProperty(nameof(MeshNode.Name))] - public string Name { get; init; } = string.Empty; - - public string? Description { get; init; } - - public string? Website { get; init; } - - [ContentItem] - public string? Logo { get; init; } - - [ContentItem] - [MeshNodeProperty(nameof(MeshNode.Icon))] - public string Icon { get; init; } = "Building"; - - public string? Location { get; init; } - - public string? Email { get; init; } - - public bool IsVerified { get; init; } - - public DateTimeOffset CreatedAt { get; init; } = DateTimeOffset.UtcNow; -} - -/// -/// Provides configuration for Organization nodes in the graph. -/// Access rules: Read/Create/Update/Delete controlled by partition-level permissions via ISecurityService. -/// Excluded from search to prevent cross-partition data leakage. -/// -public static class OrganizationNodeType -{ - public const string NodeType = "Organization"; - - public static TBuilder AddOrganizationType(this TBuilder builder) where TBuilder : MeshBuilder - { - builder.AddMeshNodes(CreateMeshNode()); - builder.WithMeshType(); - builder.ConfigureServices(services => - { - services.AddSingleton(); - services.AddSingleton(sp => - new OrganizationAccessRule(sp.GetService() ?? new NullSecurityService())); - services.AddSingleton(sp => - new OrganizationPostCreationHandler( - sp.GetService() ?? new NullSecurityService(), - sp.GetService()?.CreateLogger())); - return services; - }); - // Organization instances are NOT publicly readable — partition access controls visibility. - // The type definition itself remains visible; instances are filtered by user permissions. - return builder; - } - - private class OrganizationNodeProvider : IStaticNodeProvider - { - public IEnumerable GetStaticNodes() - { - yield return CreateMeshNode(); - } - } - - public static MeshNode CreateMeshNode() => new(NodeType) - { - Name = "Organization", - NodeType = "NodeType", - Icon = "/static/NodeTypeIcons/building.svg", - AssemblyLocation = typeof(OrganizationNodeType).Assembly.Location, - Content = new NodeTypeDefinition { DefaultNamespace = "" }, - HubConfiguration = config => config - .AddMeshDataSource(source => source - .WithContentType()) - .AddContentCollections() - .AddNodeTypeLayoutAreas() - .AddLayout(layout => layout - .WithView(MeshNodeLayoutAreas.OverviewArea, OrganizationLayoutAreas.Overview)) - }; - - /// - /// Post-creation handler: creates partition, grants admin role, and creates markdown page. - /// Triggered implicitly by RunPostCreationHandlersAsync when an Organization is created - /// via normal CreateNodeRequest. - /// - private class OrganizationPostCreationHandler( - ISecurityService securityService, - ILogger? logger) : INodePostCreationHandler - { - public string NodeType => OrganizationNodeType.NodeType; - - public async Task HandleAsync(MeshNode createdNode, string? createdBy, CancellationToken ct) - { - if (string.IsNullOrEmpty(createdBy)) - { - logger?.LogWarning("Cannot assign Admin role: no creator identity for Organization at {Path}", createdNode.Path); - return; - } - - logger?.LogInformation("Granting Admin role to {User} on Organization {Path}", createdBy, createdNode.Path); - await securityService.AddUserRoleAsync(createdBy, Role.Admin.Id, createdNode.Id, assignedBy: "system", ct); - } - - public IEnumerable GetAdditionalNodes(MeshNode createdNode) - { - // Partition node at Admin/Partition/{OrgId} - yield return new MeshNode(createdNode.Id, PartitionNodeType.Namespace) - { - NodeType = PartitionNodeType.NodeType, - Name = createdNode.Name ?? createdNode.Id, - State = MeshNodeState.Active, - Content = new PartitionDefinition - { - Namespace = createdNode.Id, - DataSource = "default", - Schema = createdNode.Id.ToLowerInvariant(), - TableMappings = PartitionDefinition.StandardTableMappings, - Description = $"Partition for organization {createdNode.Name ?? createdNode.Id}" - } - }; - } - } - - /// - /// DI-registered access rule for Organization nodes. - /// Read: all authenticated users. Create/Update/Delete: requires appropriate permission (via ISecurityService). - /// - private class OrganizationAccessRule(ISecurityService securityService) : INodeTypeAccessRule - { - public string NodeType => OrganizationNodeType.NodeType; - - public IReadOnlyCollection SupportedOperations => - [NodeOperation.Read, NodeOperation.Create, NodeOperation.Update, NodeOperation.Delete]; - - public async Task HasAccessAsync(NodeValidationContext context, string? userId, CancellationToken ct = default) - { - if (string.IsNullOrEmpty(userId)) - return false; - - if (context.Operation == NodeOperation.Read) - return await securityService.HasPermissionAsync(context.Node.Path, userId, Permission.Read, ct); - - if (context.Operation == NodeOperation.Create) - { - var parentPath = context.Node.GetParentPath() ?? context.Node.Path; - return await securityService.HasPermissionAsync(parentPath, userId, Permission.Create, ct); - } - - if (context.Operation is NodeOperation.Update or NodeOperation.Delete) - return await securityService.HasPermissionAsync(context.Node.Path, userId, Permission.Update, ct); - - return false; - } - } -} diff --git a/memex/Memex.Portal.Shared/Pages/DevLogin.razor b/memex/Memex.Portal.Shared/Pages/DevLogin.razor index 96cc1e694..ca47dd01d 100644 --- a/memex/Memex.Portal.Shared/Pages/DevLogin.razor +++ b/memex/Memex.Portal.Shared/Pages/DevLogin.razor @@ -82,28 +82,38 @@ private bool isLoading = true; private string? error; - protected override async Task OnInitializedAsync() + private IDisposable? _personsSubscription; + + protected override void OnInitialized() { - try - { - var nodes = await MeshQuery.QueryAsync("nodeType:User namespace:User").ToListAsync(); - persons = nodes - .Select(n => ExtractPersonInfo(n)) - .Where(p => p != null) - .Select(p => p!) - .OrderBy(p => p.Name) - .ToList(); - } - catch (Exception ex) - { - error = $"Error loading persons: {ex.Message}"; - } - finally - { - isLoading = false; - } + _personsSubscription = MeshQuery + .Query(MeshQueryRequest.FromQuery("nodeType:User namespace:User")) + .Subscribe( + change => + { + var items = change.Items + .Select(n => ExtractPersonInfo(n)) + .Where(p => p != null) + .Select(p => p!) + .OrderBy(p => p.Name) + .ToList(); + InvokeAsync(() => + { + persons = items; + isLoading = false; + StateHasChanged(); + }); + }, + ex => InvokeAsync(() => + { + error = $"Error loading persons: {ex.Message}"; + isLoading = false; + StateHasChanged(); + })); } + public void Dispose() => _personsSubscription?.Dispose(); + private static PersonInfo? ExtractPersonInfo(MeshNode node) { if (node.Content is not JsonElement jsonElement) diff --git a/memex/Memex.Portal.Shared/Pages/Index.razor b/memex/Memex.Portal.Shared/Pages/Index.razor index ec5a34ed9..8942cdc9e 100644 --- a/memex/Memex.Portal.Shared/Pages/Index.razor +++ b/memex/Memex.Portal.Shared/Pages/Index.razor @@ -21,7 +21,9 @@ && !UserContext.IsVirtual && !string.Equals(UserContext.ObjectId, WellKnownUsers.Anonymous, StringComparison.OrdinalIgnoreCase); - private string UserAddress => $"User/{UserContext?.ObjectId}"; + // Post-v10: the User node lives at the root of its own partition + // (path = ObjectId, e.g. `rbuergi`) — not under a `User/` prefix. + private string UserAddress => UserContext?.ObjectId ?? string.Empty; protected override void OnInitialized() { diff --git a/memex/Memex.Portal.Shared/Pages/Login.razor b/memex/Memex.Portal.Shared/Pages/Login.razor index 1c3b41c81..5d59e2b9a 100644 --- a/memex/Memex.Portal.Shared/Pages/Login.razor +++ b/memex/Memex.Portal.Shared/Pages/Login.razor @@ -3,6 +3,7 @@ @using MeshWeaver.Blazor.Portal.Authentication @inject IAuthenticationNavigationService AuthNavService @inject NavigationManager Navigation +@inject Microsoft.Extensions.Configuration.IConfiguration Configuration Sign In - Memex Portal @@ -10,7 +11,6 @@ @@ -57,6 +80,7 @@ private IReadOnlyList Providers { get; set; } = []; private bool ShowDevLogin { get; set; } + private bool ShowLinkedInPublishing { get; set; } private void NavigateToDevLogin() { @@ -64,6 +88,14 @@ Navigation.NavigateTo(url, forceLoad: true); } + private void NavigateToConnectLinkedIn() + { + // /connect/linkedin/me uses the signed-in user's path as the profile. + // If the user isn't signed in yet, the endpoint issues a Challenge that + // returns them here after authentication. + Navigation.NavigateTo("/connect/linkedin/me", forceLoad: true); + } + protected override async Task OnInitializedAsync() { if (AuthStateTask is not null) @@ -81,6 +113,10 @@ Providers = navService.GetAvailableProviders(); ShowDevLogin = navService.IsDevMode; } + + // The "Connect LinkedIn for publishing" connect needs Social:LinkedIn:ClientId; hide the button + // entirely when it's absent (the connect endpoint 500s without it). + ShowLinkedInPublishing = !string.IsNullOrEmpty(Configuration["Social:LinkedIn:ClientId"]); } private string GetProviderLoginUrl(string providerName) @@ -91,15 +127,22 @@ return url; } - private static string GetProviderIcon(string provider) + private static string GetProviderLogoSvg(string provider) => provider.ToLowerInvariant() switch { - return provider.ToLowerInvariant() switch - { - "microsoft" => "\U0001faaa", - "google" => "\U0001f310", - "linkedin" => "\U0001f4bc", - "apple" => "\U0001f34e", - _ => "\U0001f511" - }; - } + "microsoft" => MicrosoftLogo, + "google" => GoogleLogo, + "linkedin" => LinkedInLogo, + "apple" => AppleLogo, + _ => DefaultLogo + }; + + private const string MicrosoftLogo = """"""; + + private const string GoogleLogo = """"""; + + private const string LinkedInLogo = """"""; + + private const string AppleLogo = """"""; + + private const string DefaultLogo = """"""; } diff --git a/memex/Memex.Portal.Shared/Pages/Login.razor.css b/memex/Memex.Portal.Shared/Pages/Login.razor.css index bfe176eaf..2221b2610 100644 --- a/memex/Memex.Portal.Shared/Pages/Login.razor.css +++ b/memex/Memex.Portal.Shared/Pages/Login.razor.css @@ -18,7 +18,7 @@ .login-header { text-align: center; - margin-bottom: 28px; + margin-bottom: 24px; } .login-header h1 { @@ -27,51 +27,108 @@ font-weight: 700; } -.login-subtitle { - color: var(--neutral-foreground-hint); - margin: 0; +.signin-label { + text-align: center; + color: var(--neutral-foreground-rest); + margin: 0 0 14px; font-size: 0.95rem; + font-weight: 500; } -.provider-btn { +.provider-row { display: flex; + gap: 10px; + margin-bottom: 16px; +} + +.provider-row .provider-btn { + flex: 1 1 0; + min-width: 0; + display: flex; + flex-direction: column; align-items: center; - gap: 12px; - padding: 14px 20px; + justify-content: center; + gap: 8px; + padding: 16px 6px; border-radius: 6px; - font-size: 1rem; - font-weight: 500; - text-decoration: none; - cursor: pointer; - transition: background 0.15s, transform 0.1s; border: 1px solid var(--neutral-stroke-rest); + background: var(--neutral-layer-1); color: var(--neutral-foreground-rest); + cursor: pointer; + font-size: 0.875rem; + font-weight: 500; + transition: background 0.15s, transform 0.1s, box-shadow 0.15s, border-color 0.15s; +} + +.provider-row .provider-btn:hover { + background: var(--neutral-layer-2); + transform: translateY(-1px); + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.08); +} + +.provider-row .provider-microsoft:hover { border-color: #00a4ef; } +.provider-row .provider-google:hover { border-color: #4285f4; } +.provider-row .provider-linkedin:hover { border-color: #0a66c2; } +.provider-row .provider-apple:hover { border-color: #333333; } + +.provider-logo { + width: 32px; + height: 32px; + display: flex; + align-items: center; + justify-content: center; +} + +.provider-logo ::deep svg, +.provider-logo svg { + width: 32px; + height: 32px; +} + +.provider-name { + font-size: 0.875rem; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: 100%; +} + +.provider-btn-full { + display: flex; + align-items: center; + gap: 12px; + padding: 12px 18px; + border-radius: 6px; + border: 1px solid var(--neutral-stroke-rest); background: var(--neutral-layer-1); + color: var(--neutral-foreground-rest); + cursor: pointer; + font-size: 1rem; + font-weight: 500; + width: 100%; + transition: background 0.15s, transform 0.1s, box-shadow 0.15s; } -.provider-btn:hover { +.provider-btn-full:hover { background: var(--neutral-layer-2); transform: translateY(-1px); box-shadow: 0 2px 6px rgba(0, 0, 0, 0.08); } -.provider-icon { +.provider-btn-full.provider-dev { border-left: 3px solid #f59e0b; } +.provider-btn-full.provider-linkedin-publish { border-left: 3px solid #0a66c2; } + +.provider-icon-text { font-size: 1.2rem; width: 24px; text-align: center; } -.provider-microsoft { border-left: 3px solid #00a4ef; } -.provider-google { border-left: 3px solid #4285f4; } -.provider-linkedin { border-left: 3px solid #0077b5; } -.provider-apple { border-left: 3px solid #333333; } -.provider-dev { border-left: 3px solid #f59e0b; } - .divider { display: flex; align-items: center; gap: 12px; - margin: 8px 0; + margin: 12px 0; color: var(--neutral-foreground-hint); font-size: 0.85rem; } diff --git a/memex/Memex.Portal.Shared/Pages/Onboarding.razor b/memex/Memex.Portal.Shared/Pages/Onboarding.razor index 8452a3629..bf3ed4938 100644 --- a/memex/Memex.Portal.Shared/Pages/Onboarding.razor +++ b/memex/Memex.Portal.Shared/Pages/Onboarding.razor @@ -1,18 +1,28 @@ @page "/onboarding" @attribute [Microsoft.AspNetCore.Authorization.Authorize] @using MeshWeaver.Blazor.Infrastructure +@using MeshWeaver.Data +@using MeshWeaver.Graph @using MeshWeaver.Mesh @using MeshWeaver.Mesh.Security @using MeshWeaver.Mesh.Services @using MeshWeaver.Messaging @using MeshWeaver.Hosting.Blazor +@using Memex.Portal.Shared.Authentication @using Microsoft.Extensions.DependencyInjection +@using Microsoft.Extensions.Logging +@using System.Reactive.Disposables +@using System.Reactive.Linq +@using Microsoft.Extensions.Options @inject AccessService AccessService -@inject IMeshService NodeFactory -@inject IMeshService MeshQuery +@inject UserOnboardingService OnboardingService +@inject InvitationService InvitationService @inject PortalApplication PortalApplication @inject NavigationManager Navigation @inject CircuitAccessHandler CircuitAccessHandler +@inject IOptions Features +@inject ILogger Logger +@implements IDisposable Complete Your Profile - Memex Portal @@ -25,6 +35,22 @@ } +else if (registrationClosed) +{ +
+
+
+ +

@(invitationRequired ? "Invitation Required" : "Registration Closed")

+
+ + @(invitationRequired + ? "This portal is invitation-only and your email has not been invited. Please contact your administrator to request an invitation." + : "Registration is closed. Please contact your administrator to have an account created for you.") + +
+
+} else {
@@ -101,66 +127,150 @@ else private string? errorMessage; private bool checkCompleted; private bool existingUserFound; + private bool registrationClosed; + private bool invitationRequired; - protected override async Task OnInitializedAsync() + // Holds the subscriptions from OnInitialized (existing-user lookup) and + // HandleSubmit (the dual-write pipeline). Disposed when the page tears + // down so an in-flight chain can't write back to a stale circuit. + private readonly CompositeDisposable subscriptions = new(); + + protected override void OnInitialized() { var context = AccessService?.Context ?? AccessService?.CircuitContext; - if (context == null) { checkCompleted = true; return; } + if (context is null) { checkCompleted = true; return; } model.Email = context.Email ?? ""; - - // Only lock the email field if we got a valid email from OAuth emailReadOnly = !string.IsNullOrEmpty(context.Email) && context.Email.Contains('@'); - // Pre-populate full name from OAuth claims if (!string.IsNullOrEmpty(context.Name) && context.Name != "Unknown") model.FullName = context.Name; - // Suggest username from email prefix (lowercase) if (!string.IsNullOrEmpty(context.Email) && context.Email.Contains('@')) model.Username = context.Email.Split('@')[0].ToLowerInvariant(); - // Check if a User node already exists for this email (e.g., created via another portal) - if (!string.IsNullOrEmpty(context.Email)) + if (string.IsNullOrEmpty(context.Email)) { - using (AccessService!.ImpersonateAsHub(PortalApplication!.Hub)) - { - var existing = await MeshQuery.QueryAsync( - $"nodeType:User namespace:User content.email:{context.Email} limit:1") - .FirstOrDefaultAsync(); + checkCompleted = true; + return; + } - if (existing is { State: MeshNodeState.Active }) - { - // Adopt existing identity and skip onboarding - var updated = (AccessService.Context ?? new AccessContext()) with + var email = context.Email; + var workspace = PortalApplication!.Hub.GetWorkspace(); + + // Synced query — bypasses RLS (runs as System) and is gated on Initial, + // so the first emission IS the authoritative snapshot. Take(1) completes + // the subscription after that single emission; no FirstAsync, no ToTask. + // Match the login lookup (OnboardingMiddleware.FindUserByEmail) and HandleSubmit's + // email check: nodeType:User content.email, NO namespace filter. Post-V31 the User + // mirror lives in auth.mesh_nodes (namespace=''), so the old `namespace:User` filter + // matched nothing and stopped redirecting already-onboarded users — they fell through + // to the form and hit the "email already assigned" dead-end in HandleSubmit. + var byEmailQuery = workspace.GetQuery( + $"onboarding:byEmail:{email}", + $"nodeType:User content.email:{email} limit:1"); + + // Closed-registration: when registration would be closed (self-onboarding + // disabled OR invitation-only) we need the first-user-bootstrap signal up + // front so we can render the "registration closed" / "invitation required" + // message instead of the form for non-first users (the real gate still lives + // at the CreateUser call in HandleSubmit). In invitation-only mode we also + // need to know whether this email has a Pending invitation so an invited user + // sees the form. When fully open we skip both extra queries. + var allowSelfOnboarding = Features.Value.Onboarding.AllowSelfOnboarding; + var invitationOnly = Features.Value.Onboarding.InvitationOnly; + var needFirstUser = !allowSelfOnboarding || invitationOnly; + + var firstUserQuery = needFirstUser + ? workspace.GetQuery("onboarding:firstUserCheck", "namespace:User limit:1") + : Observable.Return(Enumerable.Empty()); + var inviteQuery = invitationOnly + ? workspace.GetQuery( + $"onboarding:invite:{email}", + $"nodeType:Invitation content.email:{email}") + : Observable.Return(Enumerable.Empty()); + + var jsonOptions = workspace.Hub.JsonSerializerOptions; + + subscriptions.Add( + Observable.CombineLatest(byEmailQuery, firstUserQuery, inviteQuery, + (byEmail, all, invites) => (byEmail, all, invites)) + .Take(1) + .Subscribe( + checks => OnExistingUserCheckCompleted( + email, + checks.byEmail.FirstOrDefault(), + isFirstUser: !checks.all.Any(), + invited: checks.invites.Any(n => + InvitationService.TryGetInvitation(n, jsonOptions) + is { Status: InvitationStatus.Pending })), + ex => { - ObjectId = existing.Id, - Name = existing.Name ?? existing.Id, - Email = context.Email - }; - AccessService.SetContext(updated); - // Update the circuit-level context so subsequent client-side - // navigations (e.g., to Index.razor) see the resolved identity. - CircuitAccessHandler.UpdateUserContext(updated); - existingUserFound = true; - checkCompleted = true; - return; - } - } - } - checkCompleted = true; + Logger.LogWarning(ex, + "Onboarding: existing-user lookup failed for {Email}", email); + InvokeAsync(() => { checkCompleted = true; StateHasChanged(); }); + })); } - protected override Task OnAfterRenderAsync(bool firstRender) + private void OnExistingUserCheckCompleted(string email, MeshNode? existing, bool isFirstUser, bool invited) { - if (firstRender && existingUserFound) + if (existing is { State: MeshNodeState.Active }) + { + var updated = (AccessService!.Context ?? new AccessContext()) with + { + ObjectId = existing.Id, + Name = existing.Name ?? existing.Id, + Email = email + }; + AccessService.SetContext(updated); + // Update the circuit-level context so subsequent client-side + // navigations (e.g., to Index.razor) see the resolved identity. + CircuitAccessHandler.UpdateUserContext(updated); + + InvokeAsync(() => + { + existingUserFound = true; + checkCompleted = true; + StateHasChanged(); + // Client-side only, no forceLoad — preserves the resolved circuit. + Navigation.NavigateTo("/"); + }); + } + else { - Navigation.NavigateTo("/"); // Client-side only, no forceLoad + // No existing account for this email. Decide whether to show the profile + // form or a closed message. First user always passes (becomes platform + // admin) so the deployment never locks out. + // + // Invitation-only takes precedence: the gate is "has a Pending invitation" + // regardless of AllowSelfOnboarding. Otherwise fall back to the + // self-onboarding toggle. The real security gate still lives at the + // CreateUser call in HandleSubmit; this only controls messaging. + var onboarding = Features.Value.Onboarding; + bool closed; + bool needsInvitation; + if (onboarding.InvitationOnly) + { + closed = !invited && !isFirstUser; + needsInvitation = closed; + } + else + { + closed = !onboarding.AllowSelfOnboarding && !isFirstUser; + needsInvitation = false; + } + + InvokeAsync(() => + { + registrationClosed = closed; + invitationRequired = needsInvitation; + checkCompleted = true; + StateHasChanged(); + }); } - return Task.CompletedTask; } - private async Task HandleSubmit() + private void HandleSubmit() { if (string.IsNullOrWhiteSpace(model.Username)) { @@ -168,111 +278,180 @@ else return; } - isSaving = true; - errorMessage = null; - - try + var callerId = AccessService?.Context?.ObjectId + ?? AccessService?.CircuitContext?.ObjectId; + if (string.IsNullOrEmpty(callerId)) { - var userId = AccessService?.Context?.ObjectId - ?? AccessService?.CircuitContext?.ObjectId; - if (string.IsNullOrEmpty(userId)) - { - errorMessage = "Not authenticated. Please sign in again."; - return; - } - - var username = model.Username.Trim().ToLowerInvariant(); - var fullName = model.FullName?.Trim(); - var userContent = new User - { - FullName = string.IsNullOrWhiteSpace(fullName) ? null : fullName, - Email = model.Email.Trim(), - Bio = string.IsNullOrWhiteSpace(model.Bio) ? null : model.Bio.Trim(), - Role = string.IsNullOrWhiteSpace(model.Role) ? null : model.Role.Trim(), - PinnedPaths = ["Doc"], - }; - - // Use ImpersonateAsHub so the portal hub identity is recognized - // by the portal create access rule (portal namespace = create/read/update User nodes) - using (AccessService!.ImpersonateAsHub(PortalApplication!.Hub)) - { - // Check if this is the first user (no existing User nodes = platform admin) - var isFirstUser = true; - await foreach (var _ in MeshQuery.QueryAsync(new MeshQueryRequest { Query = "namespace:User", Limit = 1 })) - { - isFirstUser = false; - break; - } - - // Check that the username is not already taken - var existingNode = await MeshQuery.QueryAsync( - $"path:User/{username} scope:self").FirstOrDefaultAsync(); - if (existingNode != null) - { - errorMessage = $"Username '{username}' is already taken. Please choose a different one."; - return; - } - - // Check that the email is not already assigned to another user - var emailValue = model.Email.Trim(); - var existingByEmail = await MeshQuery.QueryAsync( - $"nodeType:User content.email:{emailValue}").FirstOrDefaultAsync(); - if (existingByEmail != null && existingByEmail.State == MeshNodeState.Active) - { - errorMessage = $"This email is already assigned to user '{existingByEmail.Id}'. Please sign in with that account."; - return; - } - - var node = new MeshNode(username, "User") - { - Name = string.IsNullOrWhiteSpace(fullName) ? username : fullName, - NodeType = "User", - State = MeshNodeState.Active, - Icon = string.IsNullOrWhiteSpace(model.AvatarUrl) ? null : model.AvatarUrl.Trim(), - Content = userContent - }; + errorMessage = "Not authenticated. Please sign in again."; + return; + } - await NodeFactory.CreateNodeAsync(node); + isSaving = true; + errorMessage = null; - // First user becomes global Admin (stored in admin.access table) - if (isFirstUser) + var username = model.Username.Trim().ToLowerInvariant(); + var email = model.Email.Trim(); + var fullName = model.FullName?.Trim(); + var workspace = PortalApplication!.Hub.GetWorkspace(); + + var request = new UserOnboardingRequest( + Username: username, + Email: email, + FullName: fullName, + Bio: model.Bio, + Role: model.Role, + AvatarUrl: model.AvatarUrl); + + // Three synced queries — bypass RLS, gated on Initial, deduped on path. + // workspace.GetQuery is the canonical primitive for reading a set of + // MeshNodes (see Doc/Architecture/SyncedMeshNodeQueries.md). Distinct + // cache ids per check; the username/email ones live for the page's + // lifetime, which is fine for one-shot pre-checks. + var firstUserQuery = workspace.GetQuery( + "onboarding:firstUserCheck", + "namespace:User limit:1"); + var usernameQuery = workspace.GetQuery( + $"onboarding:username:{username}", + $"path:User/{username} scope:self"); + var emailQuery = workspace.GetQuery( + $"onboarding:email:{email}", + $"nodeType:User content.email:{email}"); + // Invitation-only allowlist: a Pending Invitation matching this verified email + // unlocks onboarding and is flipped to Accepted on success. + var inviteQuery = workspace.GetQuery( + $"onboarding:invite:{email}", + $"nodeType:Invitation content.email:{email}"); + var jsonOptions = workspace.Hub.JsonSerializerOptions; + + // Observable.Using holds the System-impersonation scope for the whole + // subscription so every CreateNode in the chain runs as the system identity. + // Onboarding creates the user's OWN partition root + self/platform grants — writes + // the not-yet-onboarded user has no permission for yet (a brand-new partition root + // isn't owned by anyone), so this is the canonical infrastructure-write case (same as + // SpacePostCreationHandler / OwnsPartitionProvisioningValidator). + // NOTE: ImpersonateAsHub(PortalApplication.Hub) is WRONG here — the portal circuit + // hub's address (`portal/{user}`) is hub-shaped, which AccessContext rejects as a + // principal ("hub-shaped principal must never happen"), throwing synchronously and + // hanging the form. + var pipeline = Observable.Using( + () => AccessService!.ImpersonateAsSystem(), + _ => Observable + .CombineLatest(firstUserQuery, usernameQuery, emailQuery, inviteQuery, + (all, byName, byMail, invites) => (all, byName, byMail, invites)) + .Take(1) + .SelectMany(checks => { - var securityService = PortalApplication.Hub.ServiceProvider - .GetService(); - if (securityService != null) - await securityService.AddUserRoleAsync(username, Role.Admin.Id, "Admin", username); - } - } - - // Update request-scoped context so the OnboardingMiddleware on the next request - // recognizes this user as already onboarded. The forceLoad navigation below - // triggers a fresh HTTP request + circuit, so CircuitAccessHandler will - // re-resolve the user from AuthenticationState. - var updatedContext = (AccessService.Context ?? new AccessContext()) with - { - ObjectId = username, - Name = username, - Email = model.Email.Trim() - }; - AccessService.SetContext(updatedContext); - - Navigation.NavigateTo("/", forceLoad: true); - } - catch (Microsoft.AspNetCore.Components.NavigationException) + if (checks.byName.Any()) + return Observable.Throw(new InvalidOperationException( + $"Username '{username}' is already taken. Please choose a different one.")); + + var existingByEmail = checks.byMail + .FirstOrDefault(n => n.State == MeshNodeState.Active); + if (existingByEmail is not null) + return Observable.Throw(new InvalidOperationException( + $"This email is already assigned to user '{existingByEmail.Id}'. " + + "Please sign in with that account.")); + + var isFirstUser = !checks.all.Any(); + var onboarding = Features.Value.Onboarding; + + // Pending invitation for this verified email, if any. + var pendingInviteNode = checks.invites.FirstOrDefault(n => + InvitationService.TryGetInvitation(n, jsonOptions) + is { Status: InvitationStatus.Pending }); + + // Closed-registration gate. The security boundary is HERE (gating + // the CreateUser call), not the UI: the page is the only caller that + // orchestrates CreateUser. The first-user bootstrap (zero existing + // User nodes) always succeeds or the platform locks out with no admin. + // + // Invitation-only takes precedence: when on, onboarding requires a + // Pending invitation (regardless of AllowSelfOnboarding). Otherwise + // fall back to the self-onboarding toggle. + if (onboarding.InvitationOnly) + { + if (!isFirstUser && pendingInviteNode is null) + return Observable.Throw(new InvalidOperationException( + "This portal is invitation-only and your email has not been " + + "invited. Please contact your administrator for an invitation.")); + } + else if (!onboarding.AllowSelfOnboarding && !isFirstUser) + { + return Observable.Throw(new InvalidOperationException( + "Registration is closed. Please contact your administrator " + + "to have an account created for you.")); + } + + // Dual-write (3 rows) → self-AccessAssignment → optional + // platform-Admin grant → mark the invitation Accepted. All steps are + // sequenced via SelectMany so error / cancellation flows through one chain. + return OnboardingService.CreateUser(request) + .SelectMany(rootNode => OnboardingService.GrantSelfAdmin(username) + .Select(_ => rootNode)) + .SelectMany(rootNode => isFirstUser + ? OnboardingService.GrantPlatformAdmin(username).Select(_ => rootNode) + : Observable.Return(rootNode)) + .SelectMany(rootNode => pendingInviteNode is not null + ? InvitationService.MarkAccepted( + pendingInviteNode, + InvitationService.TryGetInvitation(pendingInviteNode, jsonOptions)!) + .Select(_ => rootNode) + : Observable.Return(rootNode)); + })); + + // Subscribe in a try/catch: a SYNCHRONOUS failure during Subscribe (e.g. the + // Observable.Using resource factory throwing) bypasses Rx's onError and would + // otherwise escape HandleSubmit unhandled — leaving the form stuck at isSaving with + // no message (the GUI must NEVER silently swallow an onboarding failure). + try { - // NavigateTo with forceLoad throws NavigationException — let it propagate - throw; + subscriptions.Add(pipeline.Subscribe( + _ => OnOnboardingSucceeded(username, email), + ex => OnOnboardingFailed(username, ex))); } catch (Exception ex) { - errorMessage = $"Failed to save profile: {ex.Message}"; + OnOnboardingFailed(username, ex); } - finally + } + + private void OnOnboardingSucceeded(string username, string email) + { + // Update request-scoped context so the OnboardingMiddleware on the next + // request recognizes this user as already onboarded. The forceLoad + // navigation below triggers a fresh HTTP request + circuit, so + // CircuitAccessHandler will re-resolve the user from AuthenticationState. + var updatedContext = (AccessService!.Context ?? new AccessContext()) with + { + ObjectId = username, + Name = username, + Email = email + }; + AccessService.SetContext(updatedContext); + + InvokeAsync(() => { isSaving = false; - } + StateHasChanged(); + Navigation.NavigateTo("/", forceLoad: true); + }); } + private void OnOnboardingFailed(string username, Exception ex) + { + Logger.LogError(ex, "Onboarding submit failed for {Username}", username); + InvokeAsync(() => + { + errorMessage = ex is InvalidOperationException + ? ex.Message + : $"Failed to save profile: {ex.Message}"; + isSaving = false; + StateHasChanged(); + }); + } + + public void Dispose() => subscriptions.Dispose(); + private class OnboardingModel { public string? FullName { get; set; } diff --git a/memex/Memex.Portal.Shared/Pages/Onboarding.razor.css b/memex/Memex.Portal.Shared/Pages/Onboarding.razor.css index c460b2ec0..2630daffb 100644 --- a/memex/Memex.Portal.Shared/Pages/Onboarding.razor.css +++ b/memex/Memex.Portal.Shared/Pages/Onboarding.razor.css @@ -3,7 +3,10 @@ justify-content: center; align-items: flex-start; padding: 40px 20px; - min-height: 60vh; + /* Fill the viewport so the grey backdrop reaches the bottom. At 60vh the grey + stopped partway down — invisible in dark mode (blends with the page) but an + obvious grey-then-white seam in light mode. */ + min-height: 100vh; background: var(--neutral-layer-4); } diff --git a/memex/Memex.Portal.Shared/README.md b/memex/Memex.Portal.Shared/README.md index e78be39ee..b09e5346e 100644 --- a/memex/Memex.Portal.Shared/README.md +++ b/memex/Memex.Portal.Shared/README.md @@ -6,7 +6,7 @@ Memex.Portal.Shared is a Razor class library containing the shared configuration ## Features - **Portal configuration** (`ConfigureMemexServices`, `ConfigureMemexMesh`, `ConfigureMemexPortal`) — wires up Blazor, AI providers, graph, documentation, persistence, and content collections - **Authentication** — supports dev login, Microsoft Identity (Entra ID), Google, LinkedIn, Apple, and API token auth for MCP -- **Organization domain** — `Organization` content type with custom layout areas, access rules, partition provisioning, and post-creation handlers +- **Space integration** — composes around the `Space` NodeType shipped by `MeshWeaver.Blazor.Portal` (custom layout areas, access rules, post-creation handlers); per-tenant partition schemas are auto-created lazily on first write via `public.ensure_partition_schema` - **UI pages** — Login, DevLogin, Onboarding, Welcome, Search, and the root `App.razor` / `Routes.razor` - **AI integration** — Azure Foundry Claude, Azure OpenAI, Copilot, Claude Code, and web search plugin registration diff --git a/memex/Memex.Portal.Shared/Settings/AdminMenuGate.cs b/memex/Memex.Portal.Shared/Settings/AdminMenuGate.cs new file mode 100644 index 000000000..7629472f8 --- /dev/null +++ b/memex/Memex.Portal.Shared/Settings/AdminMenuGate.cs @@ -0,0 +1,66 @@ +using System.Reactive.Linq; +using System.Threading.Channels; +using MeshWeaver.Layout.Composition; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; + +namespace Memex.Portal.Shared.Settings; + +/// +/// Gate for the platform-wide Admin menu (the GlobalSettings area). A tab is shown only to a viewer +/// holding at the Admin scope — the platform-admin scope. +/// Bridges the IObservable permission check to IAsyncEnumerable via a Channel (no +/// .ToTask() on a hub round-trip — see AsynchronousCalls.md). +/// +/// The platform-admin grant is an AccessAssignment in the Admin/_Access namespace (routed to +/// the admin schema). The evaluator derives the assignment's scope from that namespace +/// (Admin/_Access → scope Admin, stripping /_Access), so the gate checks +/// GetEffectivePermissions("Admin"). The old check used root scope "", which matched no +/// grant once global-admin moved out of the root _Access namespace — so Invitations/Inbox never +/// appeared. +/// +/// 🚨 We must WAIT for the first emission that grants , NOT snapshot +/// the first emission with FirstAsync(). The grant is a RUNTIME AccessAssignment row, so +/// emits an empty +/// static seed first and only enriches to All once the synced AccessAssignment query lands. +/// FirstAsync() would capture that premature empty → the gate ALWAYS returned false. Filtering for +/// the positive (Where(isAdmin)) with a bounded wait makes the answer correct: an admin's grant +/// arrives within the window; a non-admin never emits a positive and falls through to the safe default +/// (false / hide the menu). One-shot + disposed, so this is NOT the long-lived synced-state Timeout +/// anti-pattern. +/// +internal static class AdminMenuGate +{ + /// The scope whose Admin grant designates a platform (global) admin. The evaluator derives + /// scope from the AccessAssignment NAMESPACE (Admin/_Access → scope Admin) via an + /// ordinal match, so this MUST match the stored namespace casing — Admin, not admin. + private const string AdminScope = "Admin"; + + // Bounded wait for the admin grant to surface on the live permission stream. The grant is typically + // present in the first enriched emission; this is the ceiling for the synced query's cold-start, + // after which a non-admin (no positive ever emitted) resolves to "not admin". + private static readonly TimeSpan GrantWait = TimeSpan.FromSeconds(5); + + /// + /// Pure-reactive platform-admin check: emits false immediately (so a menu renders without + /// the gated tab), then true if the viewer's Admin-scope grant surfaces within + /// . Waits for the POSITIVE (filter true), NOT the first emission (which can + /// be the premature empty seed). No Task, no await, no Channel bridge. + /// + public static IObservable IsPlatformAdmin(LayoutAreaHost host) + { + var accessService = host.Hub.ServiceProvider.GetService(); + var viewerId = accessService?.Context?.ObjectId ?? accessService?.CircuitContext?.ObjectId; + if (string.IsNullOrEmpty(viewerId)) + return Observable.Return(false); + + return host.Hub.IsGlobalAdmin(viewerId) + .Where(isAdmin => isAdmin) + .Take(1) + .Timeout(GrantWait) + .Catch(_ => Observable.Return(false)) + .StartWith(false); + } +} diff --git a/memex/Memex.Portal.Shared/Settings/AiSettingsTab.cs b/memex/Memex.Portal.Shared/Settings/AiSettingsTab.cs new file mode 100644 index 000000000..dcf230da0 --- /dev/null +++ b/memex/Memex.Portal.Shared/Settings/AiSettingsTab.cs @@ -0,0 +1,90 @@ +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.AI; +using MeshWeaver.Application.Styles; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Layout; +using MeshWeaver.Layout.Composition; +using MeshWeaver.Layout.Domain; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; + +namespace Memex.Portal.Shared.Settings; + +/// +/// Settings tab that edits the per-user node ({user}/_Memex/AiSettings): +/// enabled harnesses + the agent/model picker query templates. +/// +/// 100% data-bound via the framework, set up the SAME standard way as every node editor +/// (' EditNode): the auto-generated property form +/// () binds DIRECTLY to the node stream via a node-bound +/// DataContext — each edit writes straight back to the node (ONE source of truth, no /data replica, +/// no save subscription). No hand-rolled Edit-macro callback, no .Take(1), no Save button. +/// +public static class AiSettingsTab +{ + public const string TabId = "AiSettings"; + + public static MessageHubConfiguration AddAiSettingsTab(this MessageHubConfiguration config) + => config.AddSettingsMenuItems( + new SettingsMenuItemDefinition( + Id: TabId, + Label: "AI Settings", + ContentBuilder: BuildContent, + Group: "AI", + Icon: FluentIcons.Sparkle(), + GroupIcon: FluentIcons.Sparkle(), + Order: 210, + RequiredPermission: Permission.Read)); + + internal static UiControl BuildContent(LayoutAreaHost host, StackControl stack, MeshNode? node) + { + var accessService = host.Hub.ServiceProvider.GetService(); + var userId = accessService?.Context?.ObjectId ?? ""; + + stack = stack.WithView(Controls.H2("AI Settings").WithStyle("margin: 0 0 8px 0;")); + stack = stack.WithView(Controls.Html( + "

" + + "Choose which AI harnesses appear in your chat composer, and customise the agent/model " + + "picker queries (one query per row). Changes save automatically.

")); + + if (string.IsNullOrEmpty(userId)) + return stack.WithView(Controls.Html( + "

No user identity available.

")); + + var path = AiSettingsNodeType.PathFor(userId); + + // Robust: create the node with defaults if it doesn't exist (existing + new users). + AiSettingsNodeType.EnsureExists(host.Hub, host.Hub.ServiceProvider, userId); + + // Standard node editor — identical wiring to MeshNodeLayoutAreas.EditNode: the property form + // binds DIRECTLY to the node's Content (node-bound DataContext) so each edit writes straight + // back to the node stream. No /data replica, no save subscription. The first emission only + // supplies the content TYPE (to generate the form); a one-way /data projection keeps the + // derived-label read views (dimension/options/date) correct from the Layout layer. + var dataId = EditLayoutArea.GetDataId(path); + var boundContext = LayoutAreaReference.GetMeshNodeDataContext(path, bindContent: true); + host.RegisterForDisposal($"aisettings-content-projection_{dataId}", + host.Workspace.GetMeshNodeStream(path) + .Select(n => n?.Content) + .Where(c => c is not null) + .Subscribe(content => host.UpdateData(dataId, content!))); + + return stack.WithView((h, _) => h.Workspace.GetMeshNodeStream(path) + .Where(n => n?.Content is not null) + .Select(n => + { + var instance = n!.Content!; + if (instance is JsonElement je) + instance = JsonSerializer.Deserialize(je.GetRawText(), h.Hub.JsonSerializerOptions)!; + + return (UiControl?)EditLayoutArea.BuildPropertyForm( + h, instance.GetType(), dataId, canEdit: true, isToggleable: false, boundDataContext: boundContext); + }) + .DistinctUntilChanged(n => n?.GetType())); + } +} diff --git a/memex/Memex.Portal.Shared/Settings/ApiTokensSettingsTab.cs b/memex/Memex.Portal.Shared/Settings/ApiTokensSettingsTab.cs index 56265a529..a3afb19fc 100644 --- a/memex/Memex.Portal.Shared/Settings/ApiTokensSettingsTab.cs +++ b/memex/Memex.Portal.Shared/Settings/ApiTokensSettingsTab.cs @@ -31,7 +31,7 @@ public static MessageHubConfiguration AddApiTokensSettingsTab( Group: "Security", Icon: FluentIcons.Key(), Order: 230, - RequiredPermission: Permission.Read)); + RequiredPermission: Permission.None)); } internal static UiControl BuildApiTokensContent( @@ -51,18 +51,22 @@ internal static UiControl BuildApiTokensContent( const string createDataId = "apiTokenCreate"; const string resultDataId = "apiTokenResult"; - const string tokenListRefreshId = "apiTokenListRefresh"; host.UpdateData(createDataId, new Dictionary { ["label"] = "", ["expiryDays"] = 365 }); - // NOTE: Do NOT initialize resultDataId here — CreateTokenAsync saves a MeshNode + // NOTE: Do NOT initialize resultDataId here — CreateToken saves a MeshNode // which triggers the workspace stream, causing the Settings page to rebuild. // If we set resultDataId="" here, the rebuild would overwrite the token display // that the click handler just set. Instead, the reactive view uses .StartWith(). - host.UpdateData(tokenListRefreshId, DateTimeOffset.UtcNow.Ticks); + // + // The token list below subscribes to `tokenService.GetTokensForUser(userId)` + // — a live synced query (workspace.GetQuery under the hood). New tokens + // appear on CreateNode commit, revokes flip rows to "Revoked" on + // workspace.GetMeshNodeStream(...).Update commit, deletes drop rows on + // DeleteNode commit. No refresh trigger needed. // Create token form var createSection = Controls.Stack.WithWidth("100%") @@ -157,7 +161,9 @@ internal static UiControl BuildApiTokensContent( ctx.Host.UpdateData(resultDataId, tokenHtml); ctx.Host.UpdateData(tokenRenderKey, DateTimeOffset.UtcNow.Ticks); - ctx.Host.UpdateData(tokenListRefreshId, DateTimeOffset.UtcNow.Ticks); + // No list refresh trigger — the synced query + // below re-emits automatically when the new + // node commits to the workspace. }, ex => ctx.Host.UpdateData(resultDataId, "

Your Tokens")); + // Live token list — bound directly to the synced query. The view + // re-renders whenever the underlying mesh-query collection changes + // (token created, revoked, deleted). No refresh trigger pattern; + // see Doc/Architecture/SyncedMeshNodeQueries.md for the canonical + // shape — every emission is a complete snapshot. stack = stack.WithView((h, _) => - h.Stream.GetDataStream(tokenListRefreshId) - .SelectMany(async _ => - { - if (string.IsNullOrEmpty(userId)) - return (UiControl?)Controls.Html( - "

No user identity found.

"); - - var tokens = await tokenService.GetTokensForUserAsync(userId); - - if (tokens.Count == 0) - return (UiControl?)Controls.Html( - "

No tokens yet. Create one above.

"); - - return (UiControl?)BuildTokenList(tokens, tokenService, tokenListRefreshId, resultDataId); - })); + string.IsNullOrEmpty(userId) + ? Observable.Return(Controls.Html( + "

No user identity found.

")) + : tokenService.GetTokensForUser(userId) + .Select(tokens => tokens.Count == 0 + ? (UiControl?)Controls.Html( + "

No tokens yet. Create one above.

") + : BuildTokenList(tokens, tokenService, resultDataId))); return stack; } private static UiControl BuildTokenList( - List tokens, + IReadOnlyList tokens, ApiTokenService tokenService, - string tokenListRefreshId, string resultDataId) { var container = Controls.Stack.WithWidth("100%").WithStyle("gap: 8px;"); @@ -247,15 +250,14 @@ private static UiControl BuildTokenList( "

Deleting '{Esc(capturedForDelete.Label)}'…

"); - // Reactive: Subscribe to the service observable (hub.Post + RegisterCallback under the hood). + // Reactive: Subscribe to the service observable + // (hub.Post + RegisterCallback under the hood). The + // list re-renders automatically when the synced + // query above sees the deletion. tokenService.DeleteToken(capturedForDelete.NodePath).Subscribe( - _ => - { - ctx.Host.UpdateData(resultDataId, - "

Token '{Esc(capturedForDelete.Label)}' deleted.

"); - ctx.Host.UpdateData(tokenListRefreshId, DateTimeOffset.UtcNow.Ticks); - }, + _ => ctx.Host.UpdateData(resultDataId, + "

Token '{Esc(capturedForDelete.Label)}' deleted.

"), ex => ctx.Host.UpdateData(resultDataId, "

Failed to delete: {Esc(ex.Message)}

")); @@ -270,24 +272,16 @@ private static UiControl BuildTokenList( .WithAppearance(Appearance.Outline) .WithClickAction(ctx => { - ctx.Host.UpdateData(resultDataId, - "

Revoking '{Esc(captured.Label)}'…

"); - - // Reactive: Subscribe to the service observable — no await, no Task.Run. - tokenService.RevokeToken(captured.NodePath).Subscribe( - success => - { - ctx.Host.UpdateData(resultDataId, success - ? "

Token '{Esc(captured.Label)}' revoked.

" - : "

Failed to revoke token.

"); - ctx.Host.UpdateData(tokenListRefreshId, DateTimeOffset.UtcNow.Ticks); - }, - ex => ctx.Host.UpdateData(resultDataId, - "

Failed to revoke: {Esc(ex.Message)}

")); + ctx.Host.UpdateData(resultDataId, BuildPendingHtml($"Revoking '{Esc(captured.Label)}'…")); + + // Reactive: subscribe to the factored-out observable. + // Revoke(...) bridges the service call to a single outcome + // record so the test can assert on the same composition + // the UI subscribes to — no await, no Task.Run. The + // list row flips to "Revoked" automatically when the + // synced query sees the IsRevoked change. + Revoke(tokenService, captured.NodePath, captured.Label).Subscribe( + outcome => ctx.Host.UpdateData(resultDataId, BuildOutcomeHtml(outcome))); return Task.CompletedTask; })); } @@ -299,4 +293,46 @@ private static UiControl BuildTokenList( } private static string Esc(string s) => System.Web.HttpUtility.HtmlEncode(s); + + /// + /// Outcome of a token revoke/delete invocation — surfaced to both the + /// click handler and the test. is the user-facing + /// pass/fail; carries the optional error detail to + /// embed in the result HTML. Kept internal so it stays a presentation- + /// layer concern, not an exported API. + /// + internal record TokenActionOutcome(bool Success, string Label, string? Message = null); + + /// + /// Factored-out revoke pipeline — single observable composition shared by + /// the click handler and the test. Bridges + /// to an outcome that includes + /// the label (so the message can be rendered without recapturing it) and + /// folds the OnError path into a successful emission of + /// (Success=false, Message=...) so the test never has to assert on + /// observable termination semantics. Subscribe once — Take(1)-equivalent + /// shape because the underlying service emits exactly one value. + /// + internal static IObservable Revoke( + ApiTokenService tokenService, string nodePath, string label) + => tokenService.RevokeToken(nodePath) + .Select(success => new TokenActionOutcome(success, label)) + .Catch(ex => + Observable.Return(new TokenActionOutcome(false, label, ex.Message))); + + private static string BuildPendingHtml(string message) => + "

{Esc(message)}

"; + + private static string BuildOutcomeHtml(TokenActionOutcome outcome) + { + if (outcome.Success) + return "

Token '{Esc(outcome.Label)}' revoked.

"; + var detail = string.IsNullOrEmpty(outcome.Message) + ? "Failed to revoke token." + : $"Failed to revoke: {Esc(outcome.Message)}"; + return "

{detail}

"; + } } diff --git a/memex/Memex.Portal.Shared/Settings/InboxSettingsTab.cs b/memex/Memex.Portal.Shared/Settings/InboxSettingsTab.cs new file mode 100644 index 000000000..3865c4f48 --- /dev/null +++ b/memex/Memex.Portal.Shared/Settings/InboxSettingsTab.cs @@ -0,0 +1,166 @@ +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.Application.Styles; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Layout; +using MeshWeaver.Layout.Composition; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; + +namespace Memex.Portal.Shared.Settings; + +/// +/// Admin Inbox tab in the platform-wide GlobalSettings (Admin) menu — lists mail received from +/// non-users (filed into Admin/Inbox by the inbound processor). Known-user mail is +/// handled by an agent thread and never lands here. Gated on root . +/// +public static class InboxSettingsTab +{ + public const string TabId = "Inbox"; + private const string ResultDataId = "inboxResult"; + + public static MessageHubConfiguration AddInboxSettingsTab(this MessageHubConfiguration config) + => config.AddGlobalSettingsMenuItems(new GlobalSettingsMenuItemProvider(GetInboxTab)); + + private static IObservable> GetInboxTab( + LayoutAreaHost host, RenderingContext ctx) + { + var tab = new GlobalSettingsMenuItemDefinition( + Id: TabId, + Label: "Inbox", + ContentBuilder: BuildInboxContent, + Group: "Administration", + Icon: FluentIcons.Mail(), + GroupIcon: FluentIcons.Shield(), + Order: 320); + + // Reactive: AdminMenuGate.IsPlatformAdmin emits false first (tab hidden), then true once the + // platform-admin grant surfaces → the tab appears. No async/await/IAsyncEnumerable. + return AdminMenuGate.IsPlatformAdmin(host) + .Select(isAdmin => isAdmin + ? (IReadOnlyList)new[] { tab } + : []); + } + + internal static UiControl BuildInboxContent(LayoutAreaHost host, StackControl stack) + { + stack = stack.WithView(Controls.H2("Inbox").WithStyle("margin: 0 0 8px 0;")); + stack = stack.WithView(Controls.Html( + "

" + + "Email received from people who are not Memex users. (Mail from a known " + + "user is handled by an agent thread, not shown here.)

")); + + stack = stack.WithView((h, _) => + h.Stream.GetDataStream(ResultDataId) + .Select(html => string.IsNullOrEmpty(html) + ? (UiControl?)Controls.Stack.WithWidth("100%") + : (UiControl?)Controls.Stack.WithWidth("100%").WithView(Controls.Html(html))) + .StartWith((UiControl?)Controls.Stack.WithWidth("100%"))); + + stack = stack.WithView((h, _) => + { + var ws = h.Hub.GetWorkspace(); + var jsonOptions = ws.Hub.JsonSerializerOptions; + var meshService = h.Hub.ServiceProvider.GetRequiredService(); + var accessService = h.Hub.ServiceProvider.GetService(); + return ws.GetQuery("inbox:list", + $"namespace:{EmailNodeType.AdminInboxNamespace} nodeType:{EmailNodeType.NodeType}") + .Select(nodes => (UiControl?)BuildList(nodes.ToList(), meshService, accessService, jsonOptions)); + }); + + return stack; + } + + private static UiControl BuildList( + IReadOnlyList nodes, IMeshService meshService, AccessService? accessService, + JsonSerializerOptions? jsonOptions) + { + var rows = nodes + .Select(n => (node: n, email: EmailOf(n, jsonOptions))) + .Where(x => x.email is { Direction: EmailDirection.Inbound }) + .OrderByDescending(x => x.email!.ReceivedAt) + .ToList(); + + if (rows.Count == 0) + return Controls.Html("

Inbox is empty.

"); + + var container = Controls.Stack.WithWidth("100%").WithStyle("gap: 8px;"); + foreach (var (node, email) in rows) + { + var body = email!.Body ?? ""; + var preview = body.Length > 140 ? body[..140] + "…" : body; + var row = Controls.Stack.WithOrientation(Orientation.Horizontal) + .WithStyle("padding: 12px; border: 1px solid var(--neutral-stroke-rest); " + + "border-radius: 6px; align-items: center; gap: 16px;"); + row = row.WithView(Controls.Html( + $"
{Esc(email.FromName ?? email.From)} " + + $"<{Esc(email.From)}> {StatusBadge(email.Status)}" + + $"
{Esc(email.Subject)}
" + + $"
" + + $"{email.ReceivedAt:yyyy-MM-dd HH:mm} · {Esc(preview)}
")); + + if (email.Status != EmailStatus.Archived) + { + var capturedNode = node; + var capturedEmail = email; + row = row.WithView(Controls.Button("Archive") + .WithAppearance(Appearance.Outline) + .WithClickAction(ctx => + { + ctx.Host.UpdateData(ResultDataId, Pending($"Archiving mail from {Esc(capturedEmail.From)}…")); + Observable.Using( + () => accessService!.ImpersonateAsSystem(), + _ => meshService.UpdateNode(capturedNode with + { + Content = capturedEmail with { Status = EmailStatus.Archived } + })) + .Subscribe( + _ => ctx.Host.UpdateData(ResultDataId, Success($"Archived mail from {Esc(capturedEmail.From)}.")), + ex => ctx.Host.UpdateData(ResultDataId, Error(ex.Message))); + return Task.CompletedTask; + })); + } + container = container.WithView(row); + } + return container; + } + + private static MeshWeaver.Mesh.Email? EmailOf(MeshNode n, JsonSerializerOptions? options) => n.Content switch + { + MeshWeaver.Mesh.Email e => e, + JsonElement je => Safe(je, options), + _ => null + }; + + private static MeshWeaver.Mesh.Email? Safe(JsonElement je, JsonSerializerOptions? options) + { + try { return JsonSerializer.Deserialize(je.GetRawText(), options); } + catch { return null; } + } + + private static string StatusBadge(EmailStatus status) + { + var (color, text) = status switch + { + EmailStatus.New => ("#f59e0b", "New"), + EmailStatus.Read => ("#9ca3af", "Read"), + EmailStatus.Archived => ("#9ca3af", "Archived"), + _ => ("#9ca3af", status.ToString()) + }; + return $"{text}"; + } + + private static string Esc(string s) => System.Web.HttpUtility.HtmlEncode(s); + private static string Success(string m) => + $"

{m}

"; + private static string Error(string m) => + $"

{Esc(m)}

"; + private static string Pending(string m) => + $"

{m}

"; +} diff --git a/memex/Memex.Portal.Shared/Settings/InvitationsSettingsTab.cs b/memex/Memex.Portal.Shared/Settings/InvitationsSettingsTab.cs new file mode 100644 index 000000000..8031c353e --- /dev/null +++ b/memex/Memex.Portal.Shared/Settings/InvitationsSettingsTab.cs @@ -0,0 +1,255 @@ +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using System.Threading.Channels; +using MeshWeaver.Application.Styles; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Layout; +using MeshWeaver.Layout.Composition; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Messaging; +using Memex.Portal.Shared.Authentication; +using Memex.Portal.Shared.Email; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace Memex.Portal.Shared.Settings; + +/// +/// Admin settings tab for managing invitation-only onboarding. Lists outstanding +/// s and lets an admin invite an email (which creates the invitation +/// node and sends a no-reply email via ) or revoke one. +/// +/// Gated exactly like the "Global Administration" tab +/// (UserNodeType.GetGlobalAdminTabAsync): the provider yields the tab ONLY when the +/// viewer is the node owner AND holds root-level . Registered via +/// ConfigureDefaultNodeHub (like ModelsSettingsTab), so combined with the gate it +/// surfaces only on a platform admin's own User Settings page — not on every node. +/// +public static class InvitationsSettingsTab +{ + public const string TabId = "Invitations"; + private const string ResultDataId = "invitationResult"; + private const string FormDataId = "invitationForm"; + + public static MessageHubConfiguration AddInvitationsSettingsTab( + this MessageHubConfiguration config) + => config.AddGlobalSettingsMenuItems(new GlobalSettingsMenuItemProvider(GetInvitationsTab)); + + private static IObservable> GetInvitationsTab( + LayoutAreaHost host, RenderingContext ctx) + { + var tab = new GlobalSettingsMenuItemDefinition( + Id: TabId, + Label: "Invitations", + ContentBuilder: BuildInvitationsContent, + Group: "Administration", + Icon: FluentIcons.Mail(), + GroupIcon: FluentIcons.Shield(), + Order: 310); + + // Reactive: gated tab appears once the platform-admin grant surfaces. No async/await. + return AdminMenuGate.IsPlatformAdmin(host) + .Select(isAdmin => isAdmin + ? (IReadOnlyList)new[] { tab } + : []); + } + + internal static UiControl BuildInvitationsContent( + LayoutAreaHost host, StackControl stack) + { + var invitationService = host.Hub.ServiceProvider.GetRequiredService(); + var accessService = host.Hub.ServiceProvider.GetService(); + var viewerId = accessService?.Context?.ObjectId ?? accessService?.CircuitContext?.ObjectId; + // The invitation EMAIL is sent by the node-driven InvitationEmailSender hosted service + // (watches Pending invitations, stamps EmailSentAt). This handler just creates the node. + + stack = stack.WithView(Controls.H2("Invitations").WithStyle("margin: 0 0 8px 0;")); + stack = stack.WithView(Controls.Html( + "

" + + "When invitation-only onboarding is enabled (Features:Onboarding:InvitationOnly), " + + "only invited emails may complete onboarding. Invite someone below — they receive an email and " + + "may sign in with that address to get started.

")); + + // ── Invite form ─────────────────────────────────────────────────────── + host.UpdateData(FormDataId, new Dictionary + { + ["email"] = "", + ["note"] = "", + }); + + var formRow = Controls.Stack.WithOrientation(Orientation.Horizontal) + .WithStyle("gap: 12px; align-items: flex-end; flex-wrap: wrap; margin-bottom: 8px;"); + + formRow = formRow.WithView(new TextFieldControl(new JsonPointerReference("email")) + { + Label = "Email to invite", + Placeholder = "person@example.com", + DataContext = LayoutAreaReference.GetDataPointer(FormDataId) + }.WithWidth("320px")); + + formRow = formRow.WithView(new TextFieldControl(new JsonPointerReference("note")) + { + Label = "Note (optional)", + Placeholder = "e.g. New teammate", + DataContext = LayoutAreaReference.GetDataPointer(FormDataId) + }.WithWidth("240px")); + + formRow = formRow.WithView(Controls.Button("Invite") + .WithAppearance(Appearance.Accent) + .WithClickAction(clickCtx => + { + var h = clickCtx.Host; + h.UpdateData(ResultDataId, PendingHtml("Sending invitation…")); + h.Stream.GetDataStream>(FormDataId) + .Take(1) + .Subscribe(data => + { + var inviteEmail = data?.GetValueOrDefault("email")?.ToString()?.Trim() ?? ""; + var note = data?.GetValueOrDefault("note")?.ToString()?.Trim(); + if (string.IsNullOrEmpty(inviteEmail) || !inviteEmail.Contains('@')) + { + h.UpdateData(ResultDataId, ErrorHtml("Enter a valid email address.")); + return; + } + + invitationService.CreateInvitation(inviteEmail, viewerId, note) + .Subscribe( + _ => h.UpdateData(ResultDataId, + SuccessHtml($"Invited {Esc(inviteEmail)} — an invitation email will be sent shortly.")), + ex => h.UpdateData(ResultDataId, + ErrorHtml($"Failed to create invitation: {ex.Message}"))); + }); + return Task.CompletedTask; + })); + + stack = stack.WithView(formRow); + + // Result area (live HTML for invite / revoke outcomes). + stack = stack.WithView((h, _) => + h.Stream.GetDataStream(ResultDataId) + .Select(html => string.IsNullOrEmpty(html) + ? (UiControl?)Controls.Stack.WithWidth("100%") + : (UiControl?)Controls.Stack.WithWidth("100%").WithView(Controls.Html(html))) + .StartWith((UiControl?)Controls.Stack.WithWidth("100%"))); + + // ── Existing invitations ─────────────────────────────────────────────── + stack = stack.WithView(Controls.Html( + "

Invitations

")); + + stack = stack.WithView((h, _) => + { + var ws = h.Hub.GetWorkspace(); + var jsonOptions = ws.Hub.JsonSerializerOptions; + // PATH-scoped (path:Admin/Invitation) so it routes to the admin schema; a + // namespace:Admin-only query fans out cross-schema, which excludes admin. + return ws.GetQuery("invite:list", $"path:{InvitationNodeType.Namespace} scope:children nodeType:{InvitationNodeType.NodeType}") + .Select(nodes => (UiControl?)BuildInvitationList( + nodes.ToList(), invitationService, jsonOptions)); + }); + + return stack; + } + + private static UiControl BuildInvitationList( + IReadOnlyList nodes, + InvitationService invitationService, + System.Text.Json.JsonSerializerOptions? jsonOptions) + { + var rows = nodes + .Select(n => (node: n, inv: InvitationService.TryGetInvitation(n, jsonOptions))) + .Where(x => x.inv is not null) + .OrderByDescending(x => x.inv!.InvitedAt) + .ToList(); + + if (rows.Count == 0) + return Controls.Html( + "

No invitations yet.

"); + + var container = Controls.Stack.WithWidth("100%").WithStyle("gap: 8px;"); + + foreach (var (node, inv) in rows) + { + var row = Controls.Stack.WithOrientation(Orientation.Horizontal) + .WithStyle("padding: 12px; border: 1px solid var(--neutral-stroke-rest); " + + "border-radius: 6px; align-items: center; gap: 16px;"); + + row = row.WithView(Controls.Html( + $"
{Esc(inv!.Email)} {StatusBadge(inv.Status)}" + + $"
" + + $"Invited {inv.InvitedAt:yyyy-MM-dd}" + + (string.IsNullOrEmpty(inv.InvitedBy) ? "" : $" by {Esc(inv.InvitedBy!)}") + + (string.IsNullOrEmpty(inv.Note) ? "" : $" · {Esc(inv.Note!)}") + + "
")); + + if (inv.Status == InvitationStatus.Pending) + { + var capturedNode = node; + var capturedInv = inv; + row = row.WithView(Controls.Button("Revoke") + .WithAppearance(Appearance.Outline) + .WithClickAction(ctx => + { + ctx.Host.UpdateData(ResultDataId, PendingHtml($"Revoking {Esc(capturedInv.Email)}…")); + invitationService.Revoke(capturedNode, capturedInv).Subscribe( + _ => ctx.Host.UpdateData(ResultDataId, + SuccessHtml($"Revoked invitation for {Esc(capturedInv.Email)}.")), + ex => ctx.Host.UpdateData(ResultDataId, ErrorHtml(ex.Message))); + return Task.CompletedTask; + })); + } + + container = container.WithView(row); + } + + return container; + } + + private const string InviteSubject = "You've been invited to Memex"; + + private static string BuildInviteEmailHtml(string? baseUrl) + { + var link = string.IsNullOrEmpty(baseUrl) + ? "" + : $"

Open Memex

"; + return + "
" + + "

You've been invited to Memex

" + + "

An administrator has invited you to join the Memex portal. " + + "Sign in with this email address to complete your onboarding.

" + + link + + "

If you weren't expecting this invitation, you can ignore this email.

" + + "
"; + } + + private static string StatusBadge(InvitationStatus status) + { + var (color, text) = status switch + { + InvitationStatus.Pending => ("#f59e0b", "Pending"), + InvitationStatus.Accepted => ("#22c55e", "Accepted"), + InvitationStatus.Revoked => ("#9ca3af", "Revoked"), + _ => ("#9ca3af", status.ToString()) + }; + return $"{text}"; + } + + private static string Esc(string s) => System.Web.HttpUtility.HtmlEncode(s); + + private static string SuccessHtml(string msg) => + "

{msg}

"; + + private static string ErrorHtml(string msg) => + "

{Esc(msg)}

"; + + private static string PendingHtml(string msg) => + "

{msg}

"; +} diff --git a/memex/Memex.Portal.Shared/Settings/ModelsSettingsTab.cs b/memex/Memex.Portal.Shared/Settings/ModelsSettingsTab.cs new file mode 100644 index 000000000..521da5988 --- /dev/null +++ b/memex/Memex.Portal.Shared/Settings/ModelsSettingsTab.cs @@ -0,0 +1,748 @@ +using System.Collections.Immutable; +using System.Reactive.Linq; +using MeshWeaver.AI; +using MeshWeaver.AI.Connect; +using MeshWeaver.Application.Styles; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Layout; +using MeshWeaver.Layout.Composition; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Messaging; +using Memex.Portal.Shared.Models; +using Microsoft.Extensions.DependencyInjection; + +namespace Memex.Portal.Shared.Settings; + +/// +/// Settings tab for managing AI ModelProvider credentials. +/// +/// The API flow is one "add a provider" card: pick a provider type, +/// enter the base URL + key, Fetch models (live /models via +/// ), tick the ones to bring, Save. The generic +/// OpenAICompatible type covers any OpenAI-wire endpoint (OpenRouter, Groq, +/// Together, a local vLLM, …) — several distinct gateways coexist because each +/// instance is keyed by a derived id, not the provider name. +/// +/// The CLI flow ( — Claude Code, GitHub +/// Copilot) is a login status + Connect button that delegates to the CLI's native +/// login, driven by . +/// +/// Entries are stored as MeshNodes under the owner's namespace. Rendered with +/// framework controls + markdown — no hand-built HTML. +/// +public static class ModelsSettingsTab +{ + public const string TabId = "Models"; + + // ── data ids ────────────────────────────────────────────────────────────── + private const string FormId = "addProviderForm"; // { type, endpoint, apiKey, name, filter, manual } + private const string TypesId = "addProviderTypes"; // Option[] + private const string FetchedId = "fetchedModels"; // string[] + private const string SelId = "fetchedModelSel"; // Dictionary keyed by index + private const string ResultId = "modelProviderResult"; // markdown string + + private const int MaxCheckboxes = 250; + + public static MessageHubConfiguration AddModelsSettingsTab( + this MessageHubConfiguration config) + { + return config.AddSettingsMenuItems( + new SettingsMenuItemDefinition( + Id: TabId, + Label: "Language Models", + ContentBuilder: BuildModelsContent, + Group: "AI", + Icon: FluentIcons.BrainCircuit(), + GroupIcon: FluentIcons.Sparkle(), + Order: 220, + RequiredPermission: Permission.Api)); + } + + internal static UiControl BuildModelsContent( + LayoutAreaHost host, StackControl stack, MeshNode? node) + { + var providerService = host.Hub.ServiceProvider.GetRequiredService(); + var accessService = host.Hub.ServiceProvider.GetService(); + var userId = accessService?.Context?.ObjectId ?? ""; + var ownerPath = !string.IsNullOrEmpty(node?.Path) ? node!.Path : userId; + + stack = stack + .WithView(Controls.H2("Language Models")) + .WithView(Controls.Markdown( + "Bring your own AI provider credentials, or connect a co-hosted CLI with your " + + "subscription. Keys never leave your namespace.")); + + if (string.IsNullOrEmpty(ownerPath)) + return stack.WithView(Controls.Markdown("_No owner identity available._")); + + var catalogOptions = host.Hub.ServiceProvider.GetService(); + var allSources = catalogOptions?.Sources.OrderBy(s => s.Order).ToList() + ?? new List(); + var apiSources = allSources.Where(s => s.Kind == ProviderKind.Api).ToList(); + var cliSources = allSources.Where(s => s.Kind == ProviderKind.Cli).ToList(); + var byName = apiSources.ToDictionary(s => s.ProviderName, StringComparer.OrdinalIgnoreCase); + + // ── Add a provider (API) ────────────────────────────────────────────── + if (apiSources.Count > 0) + { + stack = stack.WithView(BuildAddProviderCard(host, apiSources, byName, providerService, ownerPath)); + + // Live result/status line. + stack = stack.WithView((h, _) => + h.Stream.GetDataStream(ResultId) + .Select(md => string.IsNullOrEmpty(md) + ? (UiControl?)Controls.Stack.WithWidth("100%") + : (UiControl?)Controls.Markdown(md)) + .StartWith((UiControl?)Controls.Stack.WithWidth("100%"))); + } + + // ── CLI providers — login status + connect ──────────────────────────── + var sessionManager = host.Hub.ServiceProvider.GetService(); + if (cliSources.Count > 0) + { + stack = stack.WithView(Controls.H3("CLI providers")); + stack = stack.WithView(Controls.Markdown("Log in with your subscription — no key, no model list.")); + foreach (var src in cliSources) + stack = stack.WithView(BuildCliCard(host, src, sessionManager, providerService, ownerPath, userId)); + } + + // ── Configured providers ────────────────────────────────────────────── + stack = stack.WithView(Controls.H3("Configured providers")); + stack = stack.WithView((h, _) => + providerService.GetProvidersForOwner(ownerPath) + .Select(providers => providers.Count == 0 + ? (UiControl?)Controls.Markdown("_No providers configured yet._") + : BuildProviderList(providers, providerService))); + + // ── Active models (provider selection) ──────────────────────────────── + stack = stack.WithView(Controls.H3("Active models")); + stack = stack.WithView(Controls.Markdown( + "Choose which providers' models appear in your chat. Models an organisation shared " + + "with you work even though their key stays hidden.")); + stack = stack.WithView((h, _) => + { + var ws = h.Hub.GetWorkspace(); + var models = ws.GetQuery($"model-fanout:{ownerPath}", $"nodeType:{LanguageModelNodeType.NodeType}") + .Select(nodes => nodes + .Where(n => string.Equals(n.NodeType, LanguageModelNodeType.NodeType, StringComparison.OrdinalIgnoreCase)) + .ToList()); + var selection = providerService.GetSelection(ownerPath).StartWith(ImmutableArray.Empty); + return models.CombineLatest(selection, (modelNodes, selected) => + (UiControl?)BuildModelSelectionList(modelNodes, selected, providerService, ownerPath)); + }); + + return stack; + } + + // ════════════════════════════════════════════════════════════════════════ + // Add-provider card: type → URL → key → fetch → select → save + // ════════════════════════════════════════════════════════════════════════ + + private static UiControl BuildAddProviderCard( + LayoutAreaHost host, + IReadOnlyList apiSources, + IReadOnlyDictionary byName, + ModelProviderService providerService, + string ownerPath) + { + // Seed the form + option list + (empty) fetch state. + var firstType = apiSources[0].ProviderName; + host.UpdateData(FormId, new Dictionary + { + ["type"] = firstType, + ["endpoint"] = "", + ["apiKey"] = "", + ["name"] = "", + ["filter"] = "", + ["manual"] = "", + }); + host.UpdateData(TypesId, apiSources + .Select(s => (Option)new Option(s.ProviderName, s.EffectiveLabel)) + .ToArray()); + host.UpdateData(FetchedId, Array.Empty()); + host.UpdateData(SelId, new Dictionary()); + + var formPtr = LayoutAreaReference.GetDataPointer(FormId); + + var card = Controls.Stack.WithWidth("100%") + .WithStyle("padding: 16px; border: 1px solid var(--neutral-stroke-rest); border-radius: 8px; gap: 12px; margin-bottom: 12px;"); + + card = card.WithView(Controls.H3("Add a provider")); + card = card.WithView(Controls.Markdown( + "Pick a type, paste the base URL (including `/v1`) + key, then **Fetch models** and tick the " + + "ones to bring. For an OpenAI-compatible gateway (e.g. OpenRouter `https://openrouter.ai/api/v1`) " + + "choose **OpenAI-compatible** and give it a name.")); + + card = card.WithView(new SelectControl( + new JsonPointerReference("type"), + new JsonPointerReference(LayoutAreaReference.GetDataPointer(TypesId))) + { + Label = "Provider type", + DataContext = formPtr, + }); + + var row = Controls.Stack.WithOrientation(Orientation.Horizontal) + .WithStyle("gap: 12px; align-items: flex-end; flex-wrap: wrap;"); + row = row.WithView(new TextFieldControl(new JsonPointerReference("endpoint")) + { + Label = "Base URL", + Placeholder = "https://openrouter.ai/api/v1 (blank = provider default)", + DataContext = formPtr, + }.WithWidth("340px")); + row = row.WithView(new TextFieldControl(new JsonPointerReference("apiKey")) + { + Label = "API key", + Placeholder = "paste key here", + Password = true, + DataContext = formPtr, + }.WithWidth("280px")); + row = row.WithView(new TextFieldControl(new JsonPointerReference("name")) + { + Label = "Name (for custom URLs)", + Placeholder = "e.g. OpenRouter", + DataContext = formPtr, + }.WithWidth("200px")); + card = card.WithView(row); + + card = card.WithView(Controls.Button("Fetch models") + .WithAppearance(Appearance.Accent) + .WithClickAction(ctx => { FetchModels(ctx, byName); return Task.CompletedTask; })); + + // Filter + manual-add (static so they keep focus while the list re-renders). + var tools = Controls.Stack.WithOrientation(Orientation.Horizontal) + .WithStyle("gap: 12px; align-items: flex-end; flex-wrap: wrap;"); + tools = tools.WithView(new TextFieldControl(new JsonPointerReference("filter")) + { + Label = "Filter", + Placeholder = "type to filter…", + Immediate = true, + DataContext = formPtr, + }.WithWidth("220px")); + tools = tools.WithView(new TextFieldControl(new JsonPointerReference("manual")) + { + Label = "Add a model id manually", + Placeholder = "vendor/model", + DataContext = formPtr, + }.WithWidth("220px")); + tools = tools.WithView(Controls.Button("Add id") + .WithAppearance(Appearance.Outline) + .WithClickAction(ctx => { AddManualModel(ctx); return Task.CompletedTask; })); + card = card.WithView(tools); + + // Checkable model list — re-renders on fetched-models / filter change only. + card = card.WithView((h, _) => + { + var fetched = h.Stream.GetDataStream(FetchedId).StartWith(Array.Empty()); + var form = h.Stream.GetDataStream>(FormId) + .StartWith(new Dictionary()); + return fetched.CombineLatest(form, (ids, f) => + (UiControl?)BuildModelChecklist(ids ?? Array.Empty(), + f?.GetValueOrDefault("filter")?.ToString() ?? "")); + }); + + card = card.WithView(Controls.Button("Save provider") + .WithAppearance(Appearance.Accent) + .WithClickAction(ctx => { SaveProvider(ctx, byName, providerService, ownerPath); return Task.CompletedTask; })); + + return card; + } + + private static UiControl BuildModelChecklist(string[] ids, string filter) + { + if (ids.Length == 0) + return Controls.Markdown("_No models yet — enter a base URL + key and click **Fetch models**._"); + + var indexed = ids + .Select((id, i) => (id, i)) + .Where(x => string.IsNullOrEmpty(filter) + || x.id.Contains(filter, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + var selPtr = LayoutAreaReference.GetDataPointer(SelId); + var list = Controls.Stack.WithWidth("100%") + .WithStyle("max-height: 320px; overflow-y: auto; gap: 2px; padding: 8px; border: 1px solid var(--neutral-stroke-rest); border-radius: 6px;"); + + foreach (var (id, i) in indexed.Take(MaxCheckboxes)) + list = list.WithView(new CheckBoxControl(new JsonPointerReference(i.ToString())) + { + Label = id, + DataContext = selPtr, + }); + + var shown = Math.Min(indexed.Count, MaxCheckboxes); + var header = indexed.Count == ids.Length + ? $"**{ids.Length}** models · showing {shown}" + : $"**{indexed.Count}** of {ids.Length} match · showing {shown}"; + if (indexed.Count > MaxCheckboxes) + header += $" — narrow with the filter to see the rest"; + + return Controls.Stack.WithWidth("100%").WithStyle("gap: 6px;") + .WithView(Controls.Markdown(header)) + .WithView(list); + } + + private static void FetchModels( + UiActionContext ctx, IReadOnlyDictionary byName) + { + ctx.Host.Stream.GetDataStream>(FormId).Take(1).Subscribe(form => + { + var type = form?.GetValueOrDefault("type")?.ToString() ?? ""; + var endpoint = form?.GetValueOrDefault("endpoint")?.ToString()?.Trim(); + var apiKey = form?.GetValueOrDefault("apiKey")?.ToString()?.Trim() ?? ""; + var src = byName.GetValueOrDefault(type); + var effEndpoint = string.IsNullOrEmpty(endpoint) ? src?.DefaultEndpoint : endpoint; + + if (string.IsNullOrEmpty(apiKey)) + { + ctx.Host.UpdateData(ResultId, ErrorMd("An API key is required to fetch models.")); + return; + } + + var lister = ctx.Host.Hub.ServiceProvider.GetService(); + if (lister is null) + { + ctx.Host.UpdateData(ResultId, ErrorMd("Model fetching is not available in this deployment.")); + return; + } + + ctx.Host.UpdateData(ResultId, PendingMd($"Fetching models from {type}…")); + lister.ListModels(effEndpoint, apiKey, type).Subscribe( + ids => + { + SeedFetched(ctx, ids.ToArray()); + ctx.Host.UpdateData(ResultId, SuccessMd( + $"Found {ids.Count} models — tick the ones to bring, then **Save provider**.")); + }, + ex => + { + // Graceful fallback: show the catalog defaults so the user can still + // pick + add ids manually even when the provider has no /models endpoint. + var defaults = src?.EffectiveModelIds ?? ImmutableArray.Empty; + SeedFetched(ctx, defaults.ToArray()); + ctx.Host.UpdateData(ResultId, ErrorMd( + $"Couldn't fetch models: {ex.Message} Showing defaults — you can also add ids manually.")); + }); + }); + } + + private static void SeedFetched(UiActionContext ctx, string[] ids) + { + var sel = new Dictionary(); + for (var i = 0; i < ids.Length; i++) sel[i.ToString()] = false; + ctx.Host.UpdateData(SelId, sel); + ctx.Host.UpdateData(FetchedId, ids); + } + + private static void AddManualModel(UiActionContext ctx) + { + ctx.Host.Stream.GetDataStream>(FormId).Take(1) + .CombineLatest( + ctx.Host.Stream.GetDataStream(FetchedId).Take(1).StartWith(Array.Empty()), + ctx.Host.Stream.GetDataStream>(SelId).Take(1).StartWith(new Dictionary()), + (form, ids, sel) => (form, ids: ids ?? Array.Empty(), sel: sel ?? new Dictionary())) + .Take(1) + .Subscribe(t => + { + var manual = t.form?.GetValueOrDefault("manual")?.ToString()?.Trim(); + if (string.IsNullOrEmpty(manual)) return; + if (t.ids.Contains(manual, StringComparer.OrdinalIgnoreCase)) + { + ctx.Host.UpdateData(ResultId, PendingMd($"{manual} is already in the list.")); + return; + } + var newIds = t.ids.Append(manual).ToArray(); + var newSel = new Dictionary(t.sel) { [(newIds.Length - 1).ToString()] = true }; + ctx.Host.UpdateData(SelId, newSel); + ctx.Host.UpdateData(FetchedId, newIds); + // Clear the manual field. + var form = new Dictionary(t.form ?? new Dictionary()) { ["manual"] = "" }; + ctx.Host.UpdateData(FormId, form); + }); + } + + private static void SaveProvider( + UiActionContext ctx, + IReadOnlyDictionary byName, + ModelProviderService providerService, + string ownerPath) + { + ctx.Host.Stream.GetDataStream>(FormId).Take(1) + .CombineLatest( + ctx.Host.Stream.GetDataStream(FetchedId).Take(1).StartWith(Array.Empty()), + ctx.Host.Stream.GetDataStream>(SelId).Take(1).StartWith(new Dictionary()), + (form, ids, sel) => (form, ids: ids ?? Array.Empty(), sel: sel ?? new Dictionary())) + .Take(1) + .Subscribe(t => + { + var type = t.form?.GetValueOrDefault("type")?.ToString() ?? ""; + var apiKey = t.form?.GetValueOrDefault("apiKey")?.ToString()?.Trim() ?? ""; + var name = t.form?.GetValueOrDefault("name")?.ToString()?.Trim(); + var endpoint = t.form?.GetValueOrDefault("endpoint")?.ToString()?.Trim(); + var src = byName.GetValueOrDefault(type); + var effEndpoint = string.IsNullOrEmpty(endpoint) ? src?.DefaultEndpoint : endpoint; + + var checkedIds = t.ids + .Where((_, i) => t.sel.GetValueOrDefault(i.ToString())) + .ToList(); + + if (string.IsNullOrEmpty(apiKey)) + { + ctx.Host.UpdateData(ResultId, ErrorMd("An API key is required.")); + return; + } + if (checkedIds.Count == 0) + { + ctx.Host.UpdateData(ResultId, ErrorMd("Tick at least one model to bring.")); + return; + } + if (string.Equals(type, "OpenAICompatible", StringComparison.OrdinalIgnoreCase) + && string.IsNullOrEmpty(effEndpoint)) + { + ctx.Host.UpdateData(ResultId, ErrorMd("Enter the base URL for an OpenAI-compatible provider.")); + return; + } + + var instanceId = DeriveInstanceId(type, name, effEndpoint); + ctx.Host.UpdateData(ResultId, PendingMd($"Saving {instanceId}…")); + providerService.CreateProvider( + ownerPath, type, apiKey, + label: string.IsNullOrEmpty(name) ? null : name, + endpointOverride: effEndpoint, + modelIdsOverride: checkedIds, + instanceId: instanceId) + .Subscribe( + result => ctx.Host.UpdateData(ResultId, SuccessMd( + $"Saved **{instanceId}** — {result.ModelNodes.Count} model(s) ready.")), + ex => ctx.Host.UpdateData(ResultId, ErrorMd(ex.Message))); + }); + } + + /// + /// Node id for the new provider. A generic OpenAI-compatible provider derives a + /// distinct id from the given name (or URL host) so several gateways coexist; + /// named providers key by their type (one instance per type). + /// + private static string DeriveInstanceId(string type, string? name, string? endpoint) + { + if (!string.Equals(type, "OpenAICompatible", StringComparison.OrdinalIgnoreCase)) + return type; + var fromName = Slug(name); + if (!string.IsNullOrEmpty(fromName)) return fromName; + if (!string.IsNullOrWhiteSpace(endpoint) && Uri.TryCreate(endpoint, UriKind.Absolute, out var uri)) + { + var fromHost = Slug(uri.Host.Replace("www.", "", StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrEmpty(fromHost)) return fromHost; + } + return "openai-compatible"; + } + + private static string Slug(string? s) + { + if (string.IsNullOrWhiteSpace(s)) return ""; + var arr = s.Trim().ToLowerInvariant().Select(c => char.IsLetterOrDigit(c) ? c : '-').ToArray(); + var slug = new string(arr).Trim('-'); + while (slug.Contains("--")) slug = slug.Replace("--", "-"); + return slug; + } + + // ════════════════════════════════════════════════════════════════════════ + // Configured providers + active-models selection + // ════════════════════════════════════════════════════════════════════════ + + private static UiControl BuildProviderList( + IReadOnlyList providers, ModelProviderService service) + { + var container = Controls.Stack.WithWidth("100%").WithStyle("gap: 8px;"); + foreach (var p in providers) + { + var endpointLabel = string.IsNullOrEmpty(p.Endpoint) ? "(default)" : p.Endpoint!; + var modelsLabel = p.ModelIds.Count == 0 ? "no models" : $"{p.ModelIds.Count} model(s)"; + + var row = Controls.Stack.WithOrientation(Orientation.Horizontal) + .WithStyle("padding: 12px; border: 1px solid var(--neutral-stroke-rest); border-radius: 6px; align-items: center; gap: 16px;"); + row = row.WithView(Controls.Markdown( + $"**{p.Label ?? p.Provider}** ({p.Provider}) \n" + + $"Endpoint: {endpointLabel} · Key: `{p.ApiKeyFingerprint}` · {modelsLabel} · " + + $"Created {p.CreatedAt:yyyy-MM-dd}").WithStyle("flex: 1;")); + + var captured = p; + row = row.WithView(Controls.Button("Delete") + .WithAppearance(Appearance.Outline) + .WithClickAction(ctx => + { + ctx.Host.UpdateData(ResultId, PendingMd($"Deleting {captured.Label ?? captured.Provider}…")); + service.DeleteProvider(captured.NodePath).Subscribe( + ok => ctx.Host.UpdateData(ResultId, ok + ? SuccessMd($"Deleted {captured.Label ?? captured.Provider}.") + : ErrorMd($"Failed to delete {captured.NodePath}.")), + ex => ctx.Host.UpdateData(ResultId, ErrorMd(ex.Message))); + return Task.CompletedTask; + })); + container = container.WithView(row); + } + return container; + } + + private static UiControl BuildModelSelectionList( + IReadOnlyList modelNodes, + ImmutableArray selected, + ModelProviderService service, + string ownerPath) + { + var byProvider = new Dictionary>(StringComparer.Ordinal); + foreach (var n in modelNodes) + { + var providerPath = ProviderPathOf(n); + if (string.IsNullOrEmpty(providerPath)) continue; + if (!byProvider.TryGetValue(providerPath, out var l)) + byProvider[providerPath] = l = new List(); + l.Add(n.Name ?? n.Id); + } + + if (byProvider.Count == 0) + return Controls.Markdown("_No models discovered._"); + + var selectedSet = selected.IsDefault + ? new HashSet(StringComparer.Ordinal) + : selected.ToHashSet(StringComparer.Ordinal); + var container = Controls.Stack.WithWidth("100%").WithStyle("gap: 8px;"); + + foreach (var kvp in byProvider.OrderBy(k => k.Key, StringComparer.Ordinal)) + { + var providerPath = kvp.Key; + var isActive = selectedSet.Contains(providerPath); + var preview = string.Join(", ", kvp.Value.Take(6)) + (kvp.Value.Count > 6 ? "…" : ""); + + var row = Controls.Stack.WithOrientation(Orientation.Horizontal) + .WithStyle("padding: 10px 12px; border: 1px solid var(--neutral-stroke-rest); border-radius: 6px; align-items: center; gap: 16px;"); + row = row.WithView(Controls.Markdown($"**{providerPath}** \n{preview}").WithStyle("flex: 1;")); + + var capturedPath = providerPath; + var capturedActive = isActive; + row = row.WithView(Controls.Button(isActive ? "Remove" : "Add") + .WithAppearance(isActive ? Appearance.Outline : Appearance.Accent) + .WithClickAction(ctx => + { + service.GetSelection(ownerPath).Take(1).Subscribe(cur => + { + var set = cur.IsDefault ? new List() : cur.ToList(); + if (capturedActive) set.Remove(capturedPath); + else if (!set.Contains(capturedPath)) set.Add(capturedPath); + service.SetSelection(ownerPath, set.ToImmutableArray()).Subscribe( + ok => ctx.Host.UpdateData(ResultId, ok + ? SuccessMd(capturedActive ? $"Removed {capturedPath}." : $"Added {capturedPath}.") + : ErrorMd("Failed to update selection.")), + ex => ctx.Host.UpdateData(ResultId, ErrorMd(ex.Message))); + }); + return Task.CompletedTask; + })); + container = container.WithView(row); + } + return container; + } + + /// Provider path for a LanguageModel node — its ProviderRef, else its parent path. + private static string? ProviderPathOf(MeshNode n) + { + if (n.Content is ModelDefinition md && !string.IsNullOrEmpty(md.ProviderRef)) + return md.ProviderRef; + var path = n.Path; + if (string.IsNullOrEmpty(path)) return null; + var idx = path.LastIndexOf('/'); + return idx > 0 ? path[..idx] : null; + } + + // ════════════════════════════════════════════════════════════════════════ + // CLI card — login status + connect (state machine, markdown-rendered) + // ════════════════════════════════════════════════════════════════════════ + + private static UiControl BuildCliCard( + LayoutAreaHost host, + LanguageModelCatalogSource src, + ConnectSessionManager? sessionManager, + ModelProviderService providerService, + string ownerPath, + string userId) + { + var provider = src.ProviderName.Equals("Copilot", StringComparison.OrdinalIgnoreCase) + ? ConnectProvider.Copilot + : ConnectProvider.ClaudeCode; + var stateDataId = $"cliState:{src.ProviderName}"; + + var card = Controls.Stack.WithWidth("100%") + .WithStyle("padding: 16px; border: 1px solid var(--neutral-stroke-rest); border-radius: 8px; gap: 12px; margin-bottom: 12px;"); + card = card.WithView(Controls.Markdown($"**{src.EffectiveLabel}** · _CLI_")); + + if (sessionManager is null || !sessionManager.Supports(provider)) + return card.WithView(Controls.Markdown("_Connect is not available in this deployment._")); + + host.UpdateData(stateDataId, RenderCliBody(provider, src, stateDataId, sessionManager, ownerPath, userId)); + + var configDir = ResolveConfigDir(host, userId, provider); + sessionManager.IsLoggedIn(provider, configDir).Take(1).Subscribe(loggedIn => + { + if (loggedIn) + host.UpdateData(stateDataId, RenderConnectedBody(src, provider, stateDataId, sessionManager, ownerPath, userId, loginName: null)); + }); + + return card.WithView((h, _) => + h.Stream.GetDataStream(stateDataId) + .StartWith((UiControl)Controls.Markdown("…"))); + } + + private static UiControl RenderCliBody( + ConnectProvider provider, LanguageModelCatalogSource src, string stateDataId, + ConnectSessionManager sessionManager, string ownerPath, string userId) + { + var body = Controls.Stack.WithWidth("100%").WithStyle("gap: 8px;"); + body = body.WithView(Controls.Markdown("⚪ Not connected — uses your subscription")); + body = body.WithView(Controls.Button($"Connect {src.EffectiveLabel}") + .WithAppearance(Appearance.Accent) + .WithClickAction(ctx => + { + var host = ctx.Host; + var configDir = ResolveConfigDir(host, userId, provider); + host.UpdateData(stateDataId, Controls.Markdown("🟡 Connecting…")); + sessionManager.StartConnect(ownerPath, provider, configDir).Subscribe( + st => host.UpdateData(stateDataId, RenderConnectingOrTerminal(provider, st, src, stateDataId, sessionManager, ownerPath, userId)), + ex => host.UpdateData(stateDataId, RenderError(provider, src, ex.Message, stateDataId, sessionManager, ownerPath, userId))); + return Task.CompletedTask; + })); + return body; + } + + private static UiControl RenderConnectingOrTerminal( + ConnectProvider provider, ConnectStatus status, LanguageModelCatalogSource src, string stateDataId, + ConnectSessionManager sessionManager, string ownerPath, string userId) => + status switch + { + ConnectStatus.Connecting c => RenderConnecting(provider, c.Challenge, src, stateDataId, sessionManager, ownerPath, userId), + ConnectStatus.Connected => RenderConnectedBody(src, provider, stateDataId, sessionManager, ownerPath, userId, loginName: null), + ConnectStatus.Error err => RenderError(provider, src, err.Reason, stateDataId, sessionManager, ownerPath, userId), + _ => RenderCliBody(provider, src, stateDataId, sessionManager, ownerPath, userId), + }; + + private static UiControl RenderConnecting( + ConnectProvider provider, ConnectChallenge challenge, LanguageModelCatalogSource src, string stateDataId, + ConnectSessionManager sessionManager, string ownerPath, string userId) + { + var body = Controls.Stack.WithWidth("100%").WithStyle("gap: 8px;"); + body = body.WithView(Controls.Markdown("🟡 Connecting…")); + + if (challenge.RequiresPastedCode) + { + body = body.WithView(Controls.Markdown( + $"1. Authorize in your browser: [{challenge.VerificationUrl}]({challenge.VerificationUrl}) \n" + + "2. Paste the code shown:")); + body = body.WithView(BuildPasteCodeRow(provider, src, stateDataId, sessionManager, ownerPath, userId)); + } + else + { + var code = string.IsNullOrEmpty(challenge.UserCode) ? "" : $"\n\n`{challenge.UserCode}`"; + body = body.WithView(Controls.Markdown( + $"Enter this code at [{challenge.VerificationUrl}]({challenge.VerificationUrl}){code}\n\n⏳ auto-checking…")); + } + + body = body.WithView(Controls.Button("Cancel") + .WithAppearance(Appearance.Outline) + .WithClickAction(ctx => + { + sessionManager.Cancel(ownerPath, provider); + ctx.Host.UpdateData(stateDataId, RenderCliBody(provider, src, stateDataId, sessionManager, ownerPath, userId)); + return Task.CompletedTask; + })); + return body; + } + + private static UiControl BuildPasteCodeRow( + ConnectProvider provider, LanguageModelCatalogSource src, string stateDataId, + ConnectSessionManager sessionManager, string ownerPath, string userId) + { + var codeDataId = $"cliCode:{src.ProviderName}"; + var row = Controls.Stack.WithOrientation(Orientation.Horizontal).WithStyle("gap: 8px; align-items: flex-end;"); + row = row.WithView(new TextFieldControl(new JsonPointerReference("code")) + { + Label = "Code", + Placeholder = "paste here", + DataContext = LayoutAreaReference.GetDataPointer(codeDataId), + }.WithWidth("280px")); + row = row.WithView(Controls.Button("Submit") + .WithAppearance(Appearance.Accent) + .WithClickAction(ctx => + { + var host = ctx.Host; + host.Stream.GetDataStream>(codeDataId).Take(1).Subscribe(data => + { + var code = data?.GetValueOrDefault("code")?.ToString()?.Trim() ?? ""; + if (string.IsNullOrEmpty(code)) + { + host.UpdateData(ResultId, ErrorMd("Please paste the code shown.")); + return; + } + sessionManager.SubmitCode(ownerPath, provider, code).Subscribe( + st => host.UpdateData(stateDataId, RenderConnectingOrTerminal(provider, st, src, stateDataId, sessionManager, ownerPath, userId)), + ex => host.UpdateData(stateDataId, RenderError(provider, src, ex.Message, stateDataId, sessionManager, ownerPath, userId))); + }); + return Task.CompletedTask; + })); + return row; + } + + private static UiControl RenderConnectedBody( + LanguageModelCatalogSource src, ConnectProvider provider, string stateDataId, + ConnectSessionManager sessionManager, string ownerPath, string userId, string? loginName) + { + var who = string.IsNullOrEmpty(loginName) ? "" : $" as {loginName}"; + var body = Controls.Stack.WithOrientation(Orientation.Horizontal).WithStyle("gap: 16px; align-items: center;"); + body = body.WithView(Controls.Markdown($"🟢 Connected{who}").WithStyle("flex: 1;")); + body = body.WithView(Controls.Button("Disconnect") + .WithAppearance(Appearance.Outline) + .WithClickAction(ctx => + { + sessionManager.Cancel(ownerPath, provider); + var providerService = ctx.Host.Hub.ServiceProvider.GetRequiredService(); + providerService.DeleteProvider($"{ModelProviderNodeType.UserNamespacePath(ownerPath)}/{src.ProviderName}").Subscribe( + _ => ctx.Host.UpdateData(stateDataId, RenderCliBody(provider, src, stateDataId, sessionManager, ownerPath, userId)), + ex => ctx.Host.UpdateData(ResultId, ErrorMd(ex.Message))); + return Task.CompletedTask; + })); + return body; + } + + private static UiControl RenderError( + ConnectProvider provider, LanguageModelCatalogSource src, string reason, string stateDataId, + ConnectSessionManager sessionManager, string ownerPath, string userId) + { + var body = Controls.Stack.WithWidth("100%").WithStyle("gap: 8px;"); + body = body.WithView(Controls.Markdown($"🔴 {reason}")); + body = body.WithView(Controls.Button("Retry") + .WithAppearance(Appearance.Accent) + .WithClickAction(ctx => + { + ctx.Host.UpdateData(stateDataId, RenderCliBody(provider, src, stateDataId, sessionManager, ownerPath, userId)); + return Task.CompletedTask; + })); + return body; + } + + private static string? ResolveConfigDir(LayoutAreaHost host, string userId, ConnectProvider provider) + { + if (string.IsNullOrEmpty(userId)) return null; + if (provider == ConnectProvider.ClaudeCode) + { + var cfg = host.Hub.ServiceProvider + .GetService>()?.Value; + var root = cfg?.ConfigDirRoot?.TrimEnd('/', '\\'); + return !string.IsNullOrEmpty(root) ? System.IO.Path.Combine(root, userId, ".claude") : null; + } + return null; + } + + // ── status helpers (markdown) ───────────────────────────────────────────── + private static string SuccessMd(string m) => $"✅ {m}"; + private static string ErrorMd(string m) => $"⚠️ {m}"; + private static string PendingMd(string m) => $"⏳ {m}"; +} diff --git a/memex/Memex.Portal.Shared/Settings/TokenUsageSettingsTab.cs b/memex/Memex.Portal.Shared/Settings/TokenUsageSettingsTab.cs new file mode 100644 index 000000000..20a8cad0a --- /dev/null +++ b/memex/Memex.Portal.Shared/Settings/TokenUsageSettingsTab.cs @@ -0,0 +1,181 @@ +using System.Globalization; +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.AI; +using MeshWeaver.Application.Styles; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Layout; +using MeshWeaver.Layout.Composition; +using MeshWeaver.Layout.DataGrid; +using MeshWeaver.Mesh; +using MeshWeaver.Messaging; +using MeshWeaver.Utils; + +namespace Memex.Portal.Shared.Settings; + +/// +/// Admin settings tab: aggregated token usage + estimated cost, read from the per-model +/// satellites ({thread}/_Usage/{model}). Filterable by time window and +/// groupable by model / person / thread. Cost is derived on read from +/// (never stored), so price changes re-price history. +/// +/// Gated like the other Administration tabs via AdminMenuGate.IsPlatformAdmin; the +/// query is RLS-scoped to what the viewer can read. Rendered with framework controls +/// ( + a button toolbar) — no hand-built HTML. +/// +public static class TokenUsageSettingsTab +{ + public const string TabId = "TokenUsage"; + private const string FilterDataId = "tokenUsageFilter"; + + // Immutable lookups (constants, never written at runtime) — the grouping + time-window choices. + private static readonly (string Key, string Label)[] Groupings = + [("Model", "By model"), ("Person", "By person"), ("Thread", "By thread")]; + private static readonly (int Days, string Label)[] Windows = + [(7, "Last 7 days"), (30, "Last 30 days"), (90, "Last 90 days"), (0, "All time")]; + + private static FilterState Default => new("Model", 30); + + public static MessageHubConfiguration AddTokenUsageSettingsTab(this MessageHubConfiguration config) + => config.AddGlobalSettingsMenuItems(new GlobalSettingsMenuItemProvider(GetTab)); + + private static IObservable> GetTab( + LayoutAreaHost host, RenderingContext ctx) + { + var tab = new GlobalSettingsMenuItemDefinition( + Id: TabId, + Label: "Token Usage", + ContentBuilder: BuildContent, + Group: "Administration", + Icon: FluentIcons.Database(), + GroupIcon: FluentIcons.Shield(), + Order: 320); + + return AdminMenuGate.IsPlatformAdmin(host) + .Select(isAdmin => isAdmin + ? (IReadOnlyList)new[] { tab } + : []); + } + + internal static UiControl BuildContent(LayoutAreaHost host, StackControl stack) + { + var ws = host.Hub.GetWorkspace(); + var jsonOptions = ws.Hub.JsonSerializerOptions; + host.UpdateData(FilterDataId, Default); + + stack = stack.WithView(Controls.H2("Token Usage").WithStyle("margin: 0 0 4px 0;")); + stack = stack.WithView(Controls.Markdown( + "Aggregated token usage and estimated cost from the per-model `_Usage` satellites " + + "(`{thread}/_Usage/{model}`). Cost uses the built-in model price table and re-prices " + + "automatically when prices change. Scope follows your read access.")); + + // Filter toolbar (grouping + time window) — reactive so the active choice stays highlighted. + stack = stack.WithView((h, _) => + h.Stream.GetDataStream(FilterDataId) + .StartWith(Default) + .Select(f => (UiControl?)BuildToolbar(f))); + + // The grid: query × filter, recomputed reactively on either change (never .Take(1) on the live feed). + stack = stack.WithView((h, _) => + ws.GetQuery("tokenusage:list", $"nodeType:{TokenUsageNodeType.NodeType}") + .CombineLatest( + h.Stream.GetDataStream(FilterDataId).StartWith(Default), + (nodes, filter) => (UiControl?)BuildGrid(nodes, filter, jsonOptions))); + + return stack; + } + + private static UiControl BuildToolbar(FilterState f) + { + var bar = Controls.Stack.WithOrientation(Orientation.Horizontal) + .WithStyle("gap: 6px; flex-wrap: wrap; align-items: center; margin: 8px 0;"); + bar = bar.WithView(Controls.Markdown("**Group:**")); + foreach (var (key, label) in Groupings) + bar = bar.WithView(Btn(label, f.GroupBy == key, cur => cur with { GroupBy = key })); + bar = bar.WithView(Controls.Markdown("· **Period:**")); + foreach (var (days, label) in Windows) + bar = bar.WithView(Btn(label, f.WindowDays == days, cur => cur with { WindowDays = days })); + return bar; + } + + private static UiControl Btn(string label, bool active, Func update) + => Controls.Button(label) + .WithAppearance(active ? Appearance.Accent : Appearance.Outline) + .WithClickAction(ctx => + { + ctx.Host.Stream.GetDataStream(FilterDataId).Take(1).Subscribe(cur => + ctx.Host.UpdateData(FilterDataId, update(cur ?? Default))); + return Task.CompletedTask; + }); + + private static UiControl BuildGrid( + IEnumerable nodes, FilterState filter, JsonSerializerOptions jsonOptions) + { + var cutoff = filter.WindowDays > 0 + ? DateTimeOffset.UtcNow.AddDays(-filter.WindowDays) + : DateTimeOffset.MinValue; + + var usages = nodes + .Select(n => (node: n, u: n.ContentAs(jsonOptions, null))) + .Where(x => x.u is not null && x.node.LastModified >= cutoff) + .ToList(); + + string KeyOf((MeshNode node, TokenUsage? u) x) => filter.GroupBy switch + { + "Person" => string.IsNullOrEmpty(x.u!.UserId) ? "(unknown)" : x.u!.UserId!, + "Thread" => string.IsNullOrEmpty(x.u!.ThreadId) ? "(unknown)" : LastSegment(x.u!.ThreadId!), + _ => string.IsNullOrEmpty(x.u!.Model) ? "(unknown)" : x.u!.Model, + }; + + var rows = usages + .GroupBy(KeyOf) + .Select(g => + { + long inp = g.Sum(x => x.u!.InputTokens); + long outp = g.Sum(x => x.u!.OutputTokens); + decimal cost = g.Sum(x => ModelPricing.Default(x.u!.Model)?.Cost(x.u!.InputTokens, x.u!.OutputTokens) ?? 0m); + return new UsageRow(g.Key, inp, outp, inp + outp, cost); + }) + .OrderByDescending(r => r.Total) + .ToList(); + + if (rows.Count == 0) + return Controls.Markdown("_No token usage recorded in this period._"); + + long totIn = rows.Sum(r => r.Input); + long totOut = rows.Sum(r => r.Output); + decimal totCost = rows.Sum(r => r.Cost); + var usd = CultureInfo.GetCultureInfo("en-US"); + + var grid = Controls.DataGrid(rows) + .WithColumn(new PropertyColumnControl { Property = nameof(UsageRow.Group).ToCamelCase() } + .WithTitle(filter.GroupBy)) + .WithColumn(new PropertyColumnControl { Property = nameof(UsageRow.Input).ToCamelCase() } + .WithTitle("Input").WithFormat("N0")) + .WithColumn(new PropertyColumnControl { Property = nameof(UsageRow.Output).ToCamelCase() } + .WithTitle("Output").WithFormat("N0")) + .WithColumn(new PropertyColumnControl { Property = nameof(UsageRow.Total).ToCamelCase() } + .WithTitle("Total").WithFormat("N0")) + .WithColumn(new PropertyColumnControl { Property = nameof(UsageRow.Cost).ToCamelCase() } + .WithTitle("Cost (USD)").WithFormat("C2")); + + return Controls.Stack + .WithView(Controls.Markdown( + $"**{rows.Count}** {filter.GroupBy.ToLowerInvariant()} group(s) · " + + $"**{totIn + totOut:N0}** tokens (↑{totIn:N0} / ↓{totOut:N0}) · " + + $"est. **{totCost.ToString("C2", usd)}**")) + .WithView(grid); + } + + private static string LastSegment(string path) + { + var i = path.LastIndexOf('/'); + return i >= 0 && i < path.Length - 1 ? path[(i + 1)..] : path; + } + + private sealed record FilterState(string GroupBy, int WindowDays); + + private sealed record UsageRow(string Group, long Input, long Output, long Total, decimal Cost); +} diff --git a/memex/Memex.Portal.Shared/ShippedReleaseSeed.cs b/memex/Memex.Portal.Shared/ShippedReleaseSeed.cs new file mode 100644 index 000000000..6cfa913c0 --- /dev/null +++ b/memex/Memex.Portal.Shared/ShippedReleaseSeed.cs @@ -0,0 +1,169 @@ +using System.Reactive.Linq; +using MeshWeaver.Data; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared; + +/// +/// Boot seed that ships compiled Releases WHEREVER we ship code NodeTypes — the +/// documentation partition AND every shipped sample/platform partition (ACME, FutuRe, +/// Northwind, Cornerstone, MeshWeaver, …). At startup it finds every code +/// NodeType in those partitions that has no usable build yet and triggers a release for it — under +/// the System identity, exactly like the per-NodeType first-build kickoff. +/// +/// Why: a shipped partition's NodeTypes otherwise compile ON-DEMAND the first time a +/// user navigates to them — and that on-demand compile path (its _Activity/compile-* writes) +/// is exactly what storm-failed on atioz 2026-06-18 when the activity write ran without a writer +/// identity. Pre-building the releases at provision/deploy time as System means the runtime path is +/// always a cache hit: no on-demand compile, no phantom _Activity/compile-* subscribe storm. +/// Several shipped partitions (Doc) are read-only — only a System-credentialed compile can +/// fill their cache and write the Release node, which is precisely the credential split this +/// workflow establishes. +/// +/// Each release runs entirely as System (no RequestedReleaseBy), so the compile fills +/// the cache and the Release node is created even on a read-only partition. Idempotent: a +/// NodeType that already has a usable build is skipped, so re-boots are cheap. Fire-and-forget off +/// the thread pool — never blocks host startup. +/// +public static class ShippedReleaseSeed +{ + /// + /// The partitions whose code NodeTypes ship pre-built releases. The platform documentation plus + /// the bundled sample spaces — i.e. every partition we ship NodeTypes in. User/Space partitions + /// created at runtime are NOT here: those NodeTypes are authored by users and released by them + /// (the -gated "Create Release" button). + /// + public static readonly IReadOnlyList ShippedPartitions = + [ + "Doc", "ACME", "FutuRe", "Northwind", "Cornerstone", "MeshWeaver" + ]; + + /// + /// Trigger a System release for every un-built code NodeType under each of + /// . Returns the path of each NodeType a release was triggered for + /// (one OnNext per trigger). Composed entirely from IObservable — no + /// await/FromAsync; every partition's work runs under one System scope. + /// + public static IObservable SeedReleases( + IMessageHub hub, IEnumerable partitions, ILogger? logger = null) + { + var meshService = hub.ServiceProvider.GetService(); + if (meshService is null) + return Observable.Empty(); + var accessService = hub.ServiceProvider.GetService(); + var workspace = hub.GetWorkspace(); + + return Observable.Using( + () => AccessContextScope.AsSystem(accessService), + _ => partitions + .ToObservable() + .SelectMany(partition => SeedPartition(meshService, workspace, partition, logger))); + } + + private static IObservable SeedPartition( + IMeshService meshService, IWorkspace workspace, string partitionNamespace, ILogger? logger) => + meshService + // List the NodeType nodes in the partition. Query is fine here — we only need the + // PATHS; the authoritative per-node content is re-read off the live stream below + // (query rows carry stale Content by design). + .Query(MeshQueryRequest.FromQuery( + $"namespace:{partitionNamespace} nodeType:{MeshNode.NodeTypePath} scope:subtree")) + .Take(1) + .Timeout(TimeSpan.FromSeconds(30)) + .SelectMany(result => result.Items) + .Select(n => n.Path) + .Distinct() + .SelectMany(path => workspace.GetMeshNodeStream(path) + .Where(node => node?.Content is NodeTypeDefinition) + .Take(1) + .Timeout(TimeSpan.FromSeconds(10)) + .Where(node => + { + // Skip NodeTypes that already have a usable, current build — the runtime path + // is already a cache hit for them. Trigger only the un-built ones. + var def = (NodeTypeDefinition)node!.Content!; + return def.CompilationStatus != CompilationStatus.Ok + || string.IsNullOrEmpty(def.LatestReleasePath); + }) + .SelectMany(node => + { + var nodePath = node!.Path; + logger?.LogInformation( + "[ShippedReleaseSeed] triggering System release for un-built shipped NodeType {Path}", + nodePath); + // Canonical request-via-stream-update trigger, as System — RequestedReleaseBy + // stays null so the Release node is created under System (a read-only shipped + // partition like Doc admits no user write). + return workspace.GetMeshNodeStream(nodePath) + .Update(curr => + { + if (curr?.Content is not NodeTypeDefinition def) return curr!; + return curr with + { + Content = def with + { + RequestedReleaseAt = DateTimeOffset.UtcNow, + RequestedReleaseForce = false, + RequestedReleaseBy = null + } + }; + }) + .Select(_ => nodePath); + }) + // A single un-buildable NodeType (or a partition that isn't present in this + // deployment) must not abort the whole seed. + .Catch(ex => + { + logger?.LogWarning(ex, + "[ShippedReleaseSeed] release trigger failed for {Path} (skipped)", path); + return Observable.Empty(); + })) + .Catch(ex => + { + logger?.LogWarning(ex, + "[ShippedReleaseSeed] partition {Partition} not present / not queryable (skipped)", + partitionNamespace); + return Observable.Empty(); + }); +} + +/// +/// Boot hook that runs once the mesh is up. +/// Mirrors StaticRepoImportHostedService: reactive, fire-and-forget, SubscribeOn the +/// thread pool so it never re-enters the hub schedulers on the startup thread. +/// +public sealed class ShippedReleaseSeedHostedService( + IMessageHub hub, + ILogger? logger = null) : IHostedService +{ + private IDisposable? _subscription; + + public Task StartAsync(CancellationToken cancellationToken) + { + logger?.LogInformation( + "[ShippedReleaseSeed] pre-building shipped code-NodeType releases as System for {Partitions}.", + string.Join(", ", ShippedReleaseSeed.ShippedPartitions)); + _subscription = ShippedReleaseSeed + .SeedReleases(hub, ShippedReleaseSeed.ShippedPartitions, logger) + .SubscribeOn(System.Reactive.Concurrency.TaskPoolScheduler.Default) + .Subscribe( + path => logger?.LogInformation( + "[ShippedReleaseSeed] release triggered for {Path}.", path), + ex => logger?.LogWarning(ex, "[ShippedReleaseSeed] seed failed."), + () => logger?.LogInformation("[ShippedReleaseSeed] seed complete.")); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _subscription?.Dispose(); + return Task.CompletedTask; + } +} diff --git a/memex/Memex.Portal.Shared/Skills/AgentSkillSyncService.cs b/memex/Memex.Portal.Shared/Skills/AgentSkillSyncService.cs new file mode 100644 index 000000000..32fff75cd --- /dev/null +++ b/memex/Memex.Portal.Shared/Skills/AgentSkillSyncService.cs @@ -0,0 +1,102 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Memex.Portal.Shared.Skills; + +/// +/// Configuration for . +/// +public sealed class AgentSkillSyncOptions +{ + /// + /// The shared on-disk workspace root the co-hosted CLI harnesses use as their session working + /// directory (Cwd), so each session reads the same AGENTS.md. On the co-hosted portal this is a + /// path on the shared volume (e.g. /mnt/users/_skills). Null/empty ⇒ disabled (no place to write). + /// + public string? Directory { get; set; } +} + +/// +/// Writes the shared on-disk workspace base instructions (AGENTS.md) the co-hosted CLI +/// harnesses (Claude Code, GitHub Copilot) read on startup. +/// +/// Skills are NOT materialised to disk. They are mesh nodes (nodeType:Skill) and are +/// read from the database on demand — the native MeshWeaver agent loads a skill by path +/// (load_skill), and the CLI harnesses discover + read them through the meshweaver MCP +/// server (search nodeType:Skill → get). This avoids duplicating skill docs onto disk and +/// re-feeding the same content; AGENTS.md only tells the agent how to find them. Agents are not +/// synced either — agents are system prompts. +/// +/// One-shot at — AGENTS.md is +/// static content, so there is no mesh query, no live subscription, and no reconcile loop. +/// +public sealed class AgentSkillSyncService( + IHostApplicationLifetime lifetime, + IOptions options, + ILogger? logger = null) : IHostedService +{ + public Task StartAsync(CancellationToken cancellationToken) + { + var dir = options.Value?.Directory; + if (string.IsNullOrWhiteSpace(dir)) + { + logger?.LogInformation("AgentSkillSync: no Skills:Directory configured — base-instructions write disabled."); + return Task.CompletedTask; + } + lifetime.ApplicationStarted.Register(() => WriteWorkspace(dir!)); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + private void WriteWorkspace(string workspace) + { + try + { + Directory.CreateDirectory(workspace); + WriteBaseInstructions(workspace, logger); + logger?.LogInformation( + "AgentSkillSync: wrote AGENTS.md → {Workspace} (skills are read from the mesh on demand)", workspace); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "AgentSkillSync: failed to write base instructions"); + } + } + + /// + /// The static base instructions both CLIs read (AGENTS.md content): the mesh is reachable + /// through the meshweaver MCP server, everything is vector-indexed (use search), and + /// skills are found via search nodeType:Skill and read on demand. Public for unit testing. + /// + public static string BaseInstructions() => + "# MeshWeaver workspace\n\n" + + "The **memex mesh** is your workspace — NOT a local file tree. It is reachable through the " + + "`meshweaver` MCP server, wired automatically and authenticated as you. Use its MCP tools to " + + "read and modify content rather than guessing: `get` / `search` to read; " + + "`create` / `update` / `patch` / `move` / `copy` / `delete` to mutate; plus `execute_script`, " + + "`render_area`, `navigate_to`, `upload`.\n\n" + + "**Everything is vector-indexed** — docs, nodes, content, all of it. Retrieve anything with the " + + "`search` tool (free-text queries route to the semantic index); you do not need to know exact paths.\n\n" + + "Your **skills** live in the mesh, not on disk — find them with `search nodeType:Skill` and read a " + + "skill with `get` when a request matches it. Read each skill's doc only once; if you have already " + + "read it, do not re-read it.\n"; + + private static void WriteBaseInstructions(string workspace, ILogger? logger) + { + // ONE file, AGENTS.md: the cross-tool instructions file Claude Code (project scope) AND GitHub + // Copilot (cwd) both read — no CLAUDE.md duplicate. Idempotent (rewrites only on change). + try + { + var content = BaseInstructions(); + var path = Path.Combine(workspace, "AGENTS.md"); + if (!File.Exists(path) || File.ReadAllText(path) != content) + File.WriteAllText(path, content); + } + catch (Exception ex) + { + logger?.LogDebug(ex, "AgentSkillSync: write AGENTS.md failed"); + } + } +} diff --git a/memex/Memex.Portal.Shared/Social/ApiCredentialNodeType.cs b/memex/Memex.Portal.Shared/Social/ApiCredentialNodeType.cs new file mode 100644 index 000000000..5dd753b43 --- /dev/null +++ b/memex/Memex.Portal.Shared/Social/ApiCredentialNodeType.cs @@ -0,0 +1,43 @@ +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Mesh; +using MeshWeaver.Messaging; +using MeshWeaver.Social; + +namespace Memex.Portal.Shared.Social; + +/// +/// NodeType definition for . Instances live under +/// {profilePath}/_ApiCredentials/{platform} and are read/written exclusively +/// by the Social subsystem + the LinkedIn/X connect endpoints. Access control: +/// readable/writable only by Admins and the profile owner — wired via a satellite +/// access rule in the hosting app (Memex security config). This file only registers +/// the type shape. +/// +public static class ApiCredentialNodeType +{ + public const string NodeType = "ApiCredential"; + + public static TBuilder AddApiCredentialType(this TBuilder builder) where TBuilder : MeshBuilder + { + builder.AddMeshNodes(CreateMeshNode()); + builder.WithMeshType(); + return builder; + } + + public static MeshNode CreateMeshNode() => new(NodeType) + { + Name = "API Credential", + NodeType = "NodeType", + Icon = "/static/NodeTypeIcons/key.svg", + IsSatelliteType = true, + Content = new NodeTypeDefinition + { + Description = "OAuth credentials for a platform (LinkedIn, X). Stored under {profile}/_ApiCredentials/.", + ShowChildrenInDetails = false, + }, + HubConfiguration = config => config + .AddMeshDataSource(source => source + .WithContentType()) + }; +} diff --git a/memex/Memex.Portal.Shared/Social/GitHubConnectEndpoints.cs b/memex/Memex.Portal.Shared/Social/GitHubConnectEndpoints.cs new file mode 100644 index 000000000..0449e33bc --- /dev/null +++ b/memex/Memex.Portal.Shared/Social/GitHubConnectEndpoints.cs @@ -0,0 +1,201 @@ +using System; +using System.Collections.Generic; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using System.Security.Claims; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using MeshWeaver.Blazor.Infrastructure; // PortalApplication +using MeshWeaver.GitSync; +using MeshWeaver.Messaging; // AccessService +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; +using Microsoft.AspNetCore.WebUtilities; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Social; + +/// +/// GitHub OAuth2 authorization-code flow for connecting a user's GitHub identity for +/// GitHub Sync. Mirrors +/// — the deployed surface is just the parts that +/// need browser cookies + a whitelisted callback URL: +/// +/// GET /connect/github/me — redirect into the flow for the signed-in user +/// GET /connect/github?returnPath={path} — start (CSRF cookie, redirect to GitHub authorize) +/// GET /connect/github/callback?code=… — exchange code → token, store credential, redirect back +/// +/// The token is stored encrypted at {userId}/_Provider/GitHub via +/// , keyed by the authenticated user's name +/// (which is the email = AccessContext.ObjectId in this portal). +/// +public static class GitHubConnectEndpoints +{ + public const string StateCookieName = "gh_connect_state"; + private const string CallbackPath = "/connect/github/callback"; + + public static IEndpointRouteBuilder MapGitHubConnect(this IEndpointRouteBuilder endpoints) + { + endpoints.MapGet("/connect/github/me", (HttpContext http) => + { + if (!http.User.Identity?.IsAuthenticated ?? true) + return Results.Challenge(new AuthenticationProperties { RedirectUri = "/connect/github/me" }); + return Results.Redirect("/connect/github?returnPath=/"); + }).RequireAuthorization(); + + endpoints.MapGet("/connect/github", ( + HttpContext http, + [Microsoft.AspNetCore.Mvc.FromQuery] string? returnPath, + GitHubOAuthService oauth) => + { + if (!http.User.Identity?.IsAuthenticated ?? true) + return Results.Challenge(new AuthenticationProperties { RedirectUri = http.Request.Path + http.Request.QueryString }); + if (!oauth.IsConfigured) + return Results.Problem("GitHub OAuth is not configured (GitHub:OAuth:ClientId + ClientSecret).", statusCode: 500); + + var state = GenerateState(); + var rp = string.IsNullOrWhiteSpace(returnPath) ? "/" : returnPath!; + http.Response.Cookies.Append(StateCookieName, + WebEncoders.Base64UrlEncode(Encoding.UTF8.GetBytes($"{state}|{rp}")), + new CookieOptions + { + HttpOnly = true, + Secure = true, + SameSite = SameSiteMode.Lax, + MaxAge = TimeSpan.FromMinutes(10), + }); + + return Results.Redirect(oauth.BuildAuthorizeUrl(BuildRedirectUri(http), state)); + }).RequireAuthorization(); + + endpoints.MapGet(CallbackPath, async ( + HttpContext http, + [Microsoft.AspNetCore.Mvc.FromQuery] string? code, + [Microsoft.AspNetCore.Mvc.FromQuery] string? state, + [Microsoft.AspNetCore.Mvc.FromQuery] string? error, + GitHubOAuthService oauth, + GitHubCredentialService creds, + ILoggerFactory loggers) => + { + var logger = loggers.CreateLogger("GitHubConnect"); + + // Recover the originating page (and CSRF state) from the cookie FIRST, so every failure + // below redirects the user BACK to the GitHub Sync tab WITH a visible reason — never a + // silent bounce to the home page. (Errors are also logged at Warning so they surface in + // Loki / App Insights, not just the GUI.) + string cookieState = "", returnPath = "/"; + if (http.Request.Cookies.TryGetValue(StateCookieName, out var cookie) && !string.IsNullOrEmpty(cookie)) + { + http.Response.Cookies.Delete(StateCookieName); + try + { + var parts = Encoding.UTF8.GetString(WebEncoders.Base64UrlDecode(cookie)).Split('|', 2); + cookieState = parts[0]; + returnPath = parts.Length > 1 ? parts[1] : "/"; + } + catch { /* malformed cookie — fall through to the state check below */ } + } + + IResult Fail(string reason) + { + logger.LogWarning("GitHub connect failed: {Reason} (user {User})", reason, http.User.Identity?.Name); + return Results.Redirect(SafeReturn(returnPath, "github-error", reason)); + } + + if (!string.IsNullOrEmpty(error)) + return Fail(error!); + if (string.IsNullOrEmpty(cookieState)) + return Fail("missing or bad connect-state cookie (CSRF)"); + if (!string.Equals(cookieState, state, StringComparison.Ordinal)) + return Fail("connect-state mismatch (CSRF)"); + if (string.IsNullOrEmpty(code)) + return Fail("no authorization code returned by GitHub"); + + // The credential MUST be keyed by the mesh User.Id (e.g. "rbuergi") — the SAME identifier + // the GitHub Sync tab and Sync read it under (AccessContext.ObjectId). Using + // http.User.Identity.Name (the display name "Roland Buergi") saved it under the wrong key, + // so the tab never found it ("nothing happens" after ?connect=github-ok). Mirror + // OAuthConnectController.ResolveMeshUserId: prefer the resolved AccessContext.ObjectId, + // fall back to the preferred_username/email local part. + var userId = ResolveMeshUserId(http); + if (string.IsNullOrEmpty(userId)) + return Fail("could not resolve your mesh user id (retry after a normal browser login)"); + + var redirectUri = BuildRedirectUri(http); + // Reactive end-to-end; bridge to Task ONCE at the HTTP boundary via FirstAsync().ToTask() + // — the sanctioned edge pattern (see OAuthConnectController.ExchangeToken). NO hand-woven + // TaskCompletionSource/Subscribe. The credential write's AccessContext is carried through + // the framework's .Subscribe / IoPool boundary from the request context the middleware set. + return await oauth.ExchangeCode(code!, redirectUri) + .SelectMany(token => oauth.GetLogin(token.AccessToken) + .Catch(_ => Observable.Return(null)) + .SelectMany(login => creds.Save(userId!, token, login))) + .Select(_ => + { + logger.LogInformation("Stored GitHub credential for {User}", userId); + return (IResult)Results.Redirect(SafeReturn(returnPath, "github-ok", null)); + }) + .Catch((Exception ex) => + { + // Surface the REAL reason (token exchange / GetLogin / credential write) — never swallow. + logger.LogWarning(ex, "GitHub connect failed for {User}", userId); + return Observable.Return((IResult)Results.Redirect(SafeReturn(returnPath, "github-error", ex.Message))); + }) + .FirstAsync() + .ToTask(http.RequestAborted); + }).RequireAuthorization(); + + return endpoints; + } + + private static string SafeReturn(string returnPath, string status, string? reason) + { + var rp = string.IsNullOrWhiteSpace(returnPath) || !returnPath.StartsWith("/", StringComparison.Ordinal) ? "/" : returnPath; + var sep = rp.Contains('?') ? "&" : "?"; + var url = $"{rp}{sep}connect={status}"; + if (!string.IsNullOrEmpty(reason)) + url += $"&reason={Uri.EscapeDataString(reason!)}"; + return url; + } + + private static string GenerateState() + { + Span buf = stackalloc byte[24]; + RandomNumberGenerator.Fill(buf); + return WebEncoders.Base64UrlEncode(buf); + } + + private static string BuildRedirectUri(HttpContext http) => + $"{http.Request.Scheme}://{http.Request.Host}{CallbackPath}"; + + /// + /// Resolves the mesh User.Id (e.g. rbuergi) to key the credential under — the SAME + /// identifier the GitHub Sync tab + Sync read it under (AccessContext.ObjectId), NEVER the + /// display Name claim. Mirrors OAuthConnectController.ResolveMeshUserId: prefer the + /// resolved AccessContext.ObjectId (email→User.Id, stamped by UserContextMiddleware), fall + /// back to the preferred_username/email local part when no context is present. + /// + private static string? ResolveMeshUserId(HttpContext http) + { + var ctx = http.RequestServices.GetService()? + .Hub.ServiceProvider.GetService()?.Context; + var resolved = ctx?.ObjectId; + if (!string.IsNullOrEmpty(resolved) && !resolved.Contains('@')) + return resolved; + var claim = http.User.FindFirstValue("preferred_username") ?? http.User.FindFirstValue(ClaimTypes.Email); + return UsernameFromEmail(claim); + } + + /// Email-shaped identifier → its local part (the username / mesh partition key, + /// e.g. rbuergi@systemorph.com → rbuergi); unchanged when there's no @. + private static string? UsernameFromEmail(string? value) + { + if (string.IsNullOrEmpty(value)) return null; + var at = value.IndexOf('@'); + return at > 0 ? value[..at] : value; + } +} diff --git a/memex/Memex.Portal.Shared/Social/LinkedInConnectEndpoints.cs b/memex/Memex.Portal.Shared/Social/LinkedInConnectEndpoints.cs new file mode 100644 index 000000000..a0c2621a4 --- /dev/null +++ b/memex/Memex.Portal.Shared/Social/LinkedInConnectEndpoints.cs @@ -0,0 +1,287 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Reactive.Linq; +using System.Security.Cryptography; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using MeshWeaver.Social; +using Microsoft.AspNetCore.Authentication; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; +using Microsoft.AspNetCore.WebUtilities; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Social; + +/// +/// OAuth2 authorization-code flow for connecting a LinkedIn publishing identity +/// to a profile in the mesh. The deployed surface is intentionally tiny — just +/// the parts that need to live in the portal binary because they involve +/// browser cookies, HTTP routing, and a callback URL whitelisted on LinkedIn: +/// +/// GET /connect/linkedin/me — convenience: redirect into the flow for the signed-in user +/// GET /connect/linkedin?profile={path} — start the flow (sets CSRF cookie, redirects to LinkedIn) +/// GET /connect/linkedin/callback?code=… — finish the flow, persist credential + LinkedInProfile node +/// +/// Everything else (pulling past posts, comments, likes, computing analytics, +/// appending telemetry samples) lives as Code on the Systemorph/LinkedInProfile +/// NodeType — see the LinkedInPullActions Code piece. That keeps the +/// deployed binary stable while the actual ingest logic can be edited without a deploy. +/// +public static class LinkedInConnectEndpoints +{ + public const string StateCookieName = "lnkd_connect_state"; + private const string CallbackPath = "/connect/linkedin/callback"; + + public static IEndpointRouteBuilder MapLinkedInConnect(this IEndpointRouteBuilder endpoints) + { + // Convenience: bind the credential to the authenticated user's own User node. + endpoints.MapGet("/connect/linkedin/me", (HttpContext http) => + { + if (!http.User.Identity?.IsAuthenticated ?? true) + return Results.Challenge(new AuthenticationProperties { RedirectUri = "/connect/linkedin/me" }); + var user = http.User.Identity!.Name ?? "anonymous"; + return Results.Redirect($"/connect/linkedin?profile=User/{Uri.EscapeDataString(user)}"); + }).RequireAuthorization(); + + endpoints.MapGet("/connect/linkedin", ( + HttpContext http, + [Microsoft.AspNetCore.Mvc.FromQuery] string profile, + IConfiguration config) => + { + if (!http.User.Identity?.IsAuthenticated ?? true) + return Results.Challenge(new AuthenticationProperties { RedirectUri = http.Request.Path + http.Request.QueryString }); + + var clientId = config["Social:LinkedIn:ClientId"]; + if (string.IsNullOrEmpty(clientId)) + return Results.Problem("LinkedIn client id is not configured (Social:LinkedIn:ClientId).", statusCode: 500); + + if (string.IsNullOrWhiteSpace(profile)) + return Results.BadRequest("profile query parameter is required."); + + var state = GenerateState(); + http.Response.Cookies.Append(StateCookieName, + WebEncoders.Base64UrlEncode(System.Text.Encoding.UTF8.GetBytes($"{state}|{profile}")), + new CookieOptions + { + HttpOnly = true, + Secure = true, + SameSite = SameSiteMode.Lax, + MaxAge = TimeSpan.FromMinutes(10) + }); + + var redirectUri = BuildRedirectUri(http); + var url = "https://www.linkedin.com/oauth/v2/authorization?response_type=code" + + $"&client_id={Uri.EscapeDataString(clientId!)}" + + $"&redirect_uri={Uri.EscapeDataString(redirectUri)}" + + $"&state={Uri.EscapeDataString(state)}" + // Keep scopes minimal: openid/profile/email is plain OIDC sign-in, + // which is all we need for the analytics dashboard (posts come in + // via CSV import since r_member_social is closed). w_member_social + // (publishing) is not requested — it causes LinkedIn to render + // "share on your behalf" on the consent screen which isn't what + // this flow is for today. + + "&scope=" + Uri.EscapeDataString("openid profile email"); + + return Results.Redirect(url); + }).RequireAuthorization(); + + endpoints.MapGet(CallbackPath, async ( + HttpContext http, + [Microsoft.AspNetCore.Mvc.FromQuery] string? code, + [Microsoft.AspNetCore.Mvc.FromQuery] string? state, + [Microsoft.AspNetCore.Mvc.FromQuery] string? error, + IConfiguration config, + IHttpClientFactory httpFactory, + IMeshService mesh, + ILoggerFactory loggers) => + { + var logger = loggers.CreateLogger("LinkedInConnect"); + + if (!string.IsNullOrEmpty(error)) + return Results.Redirect($"/?connect=linkedin-error&reason={Uri.EscapeDataString(error)}"); + + if (!http.Request.Cookies.TryGetValue(StateCookieName, out var cookieValue) || string.IsNullOrEmpty(cookieValue)) + return Results.BadRequest("Missing connect state cookie (CSRF)."); + http.Response.Cookies.Delete(StateCookieName); + + string cookieState, profilePath; + try + { + var decoded = System.Text.Encoding.UTF8.GetString(WebEncoders.Base64UrlDecode(cookieValue)); + var parts = decoded.Split('|', 2); + cookieState = parts[0]; + profilePath = parts[1]; + } + catch + { + return Results.BadRequest("Bad state cookie."); + } + + if (!string.Equals(cookieState, state, StringComparison.Ordinal)) + return Results.BadRequest("State mismatch (CSRF)."); + if (string.IsNullOrEmpty(code)) + return Results.BadRequest("No authorization code."); + + var clientId = config["Social:LinkedIn:ClientId"]!; + var clientSecret = config["Social:LinkedIn:ClientSecret"] ?? ""; + + var http2 = httpFactory.CreateClient(); + var form = new FormUrlEncodedContent(new Dictionary + { + ["grant_type"] = "authorization_code", + ["code"] = code!, + ["redirect_uri"] = BuildRedirectUri(http), + ["client_id"] = clientId, + ["client_secret"] = clientSecret, + }); + + using var tokenResp = await http2.PostAsync("https://www.linkedin.com/oauth/v2/accessToken", form, http.RequestAborted); + if (!tokenResp.IsSuccessStatusCode) + { + var body = await tokenResp.Content.ReadAsStringAsync(http.RequestAborted); + logger.LogWarning("LinkedIn token exchange failed {Status}: {Body}", (int)tokenResp.StatusCode, body); + // Friendly landing instead of raw Bad Gateway JSON — pass the reason + // so the profile page can show a visible banner. + var reason = ExtractLinkedInErrorReason(body); + return Results.Redirect($"/{profilePath}/LinkedIn?connect=linkedin-error&stage=token&reason={Uri.EscapeDataString(reason)}"); + } + + using var doc = JsonDocument.Parse(await tokenResp.Content.ReadAsStringAsync(http.RequestAborted)); + var accessToken = doc.RootElement.GetProperty("access_token").GetString()!; + var expiresIn = doc.RootElement.TryGetProperty("expires_in", out var ei) ? ei.GetInt32() : 3600; + var refreshToken = doc.RootElement.TryGetProperty("refresh_token", out var rt) ? rt.GetString() : null; + var scope = doc.RootElement.TryGetProperty("scope", out var sc) ? sc.GetString() : null; + + using var uiReq = new HttpRequestMessage(HttpMethod.Get, "https://api.linkedin.com/v2/userinfo"); + uiReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken); + using var uiResp = await http2.SendAsync(uiReq, http.RequestAborted); + if (!uiResp.IsSuccessStatusCode) + return Results.Redirect($"/{profilePath}/LinkedIn?connect=linkedin-error&stage=userinfo&reason={Uri.EscapeDataString("userinfo-" + (int)uiResp.StatusCode)}"); + + using var uiDoc = JsonDocument.Parse(await uiResp.Content.ReadAsStringAsync(http.RequestAborted)); + var subject = uiDoc.RootElement.GetProperty("sub").GetString()!; + var displayName = uiDoc.RootElement.TryGetProperty("name", out var nm) ? nm.GetString() : null; + var pictureUrl = uiDoc.RootElement.TryGetProperty("picture", out var pic) ? pic.GetString() : null; + var emailAddress = uiDoc.RootElement.TryGetProperty("email", out var em) ? em.GetString() : null; + + var credential = new PlatformCredential + { + Platform = LinkedInPublisher.PlatformId, + SubjectId = subject, + AccessToken = accessToken, + RefreshToken = refreshToken, + ExpiresAt = DateTimeOffset.UtcNow.AddSeconds(expiresIn), + Scope = scope, + AcquiredAt = DateTimeOffset.UtcNow, + }; + + // Persist under {profilePath}/_ApiCredentials/linkedin. + var credentialNode = new MeshNode("linkedin", profilePath + "/_ApiCredentials") + { + Name = "LinkedIn credential", + NodeType = ApiCredentialNodeType.NodeType, + Content = credential, + State = MeshNodeState.Active, + }; + // Upsert the LinkedInProfile node so the analytics dashboard has somewhere + // to render. Loose dictionary content avoids a hard dependency on the + // dynamic LinkedInProfile content type from this assembly. + var profileNode = new MeshNode("LinkedIn", profilePath) + { + Name = displayName ?? "LinkedIn", + NodeType = "Systemorph/LinkedInProfile", + State = MeshNodeState.Active, + Content = new Dictionary + { + ["$type"] = "LinkedInProfile", + ["displayName"] = displayName ?? subject, + ["subjectUrn"] = $"urn:li:person:{subject}", + ["pictureUrl"] = pictureUrl, + ["email"] = emailAddress, + ["connectedAt"] = DateTimeOffset.UtcNow, + } + }; + + // Reactive persistence chain — mesh.CreateNode/UpdateNode return IObservable + // (see AsynchronousCalls.md). Each Create attempt falls back to Update on failure + // via Rx Catch. Profile upsert errors are swallowed (best-effort). The whole chain + // resolves once and emits the final IResult. + var tcs = new TaskCompletionSource(); + + var upsertCredential = mesh.CreateNode(credentialNode) + .Catch(createEx => + { + logger.LogInformation(createEx, "Credential create failed at {Path}, attempting update", credentialNode.Path); + return mesh.UpdateNode(credentialNode); + }); + + var upsertProfile = mesh.CreateNode(profileNode) + .Catch(createEx => + { + logger.LogInformation(createEx, "LinkedInProfile create failed at {Path}, attempting update", profileNode.Path); + return mesh.UpdateNode(profileNode); + }) + .Catch(updateEx => + { + logger.LogWarning(updateEx, "LinkedInProfile upsert failed for {Path} — continuing", profileNode.Path); + return Observable.Return(profileNode); + }); + + upsertCredential + .SelectMany(_ => upsertProfile) + .Subscribe( + _ => + { + logger.LogInformation("Connected LinkedIn credential for profile {Profile} (subject {Subject})", profilePath, subject); + tcs.TrySetResult(Results.Redirect($"/{profilePath}/LinkedIn?connect=linkedin-ok")); + }, + ex => + { + logger.LogWarning(ex, "Credential persist failed at {Path}. Redirecting to profile with error.", credentialNode.Path); + tcs.TrySetResult(Results.Redirect($"/{profilePath}/LinkedIn?connect=linkedin-error&stage=credential&reason=persist-failed")); + }); + + return await tcs.Task; + }); + + return endpoints; + } + + private static string GenerateState() + { + Span buf = stackalloc byte[24]; + RandomNumberGenerator.Fill(buf); + return WebEncoders.Base64UrlEncode(buf); + } + + private static string BuildRedirectUri(HttpContext http) => + $"{http.Request.Scheme}://{http.Request.Host}{CallbackPath}"; + + /// + /// Extracts the short error field from a LinkedIn OAuth error payload, + /// falling back to a generic slug if the body isn't parseable. Used to surface + /// a compact query-string reason code to the user instead of raw JSON. + /// + private static string ExtractLinkedInErrorReason(string body) + { + try + { + using var doc = JsonDocument.Parse(body); + if (doc.RootElement.TryGetProperty("error", out var err) && err.ValueKind == JsonValueKind.String) + return err.GetString() ?? "unknown"; + } + catch { /* non-JSON response */ } + return "token-exchange-failed"; + } +} diff --git a/memex/Memex.Portal.Shared/Social/LinkedInCredentialMenuProvider.cs b/memex/Memex.Portal.Shared/Social/LinkedInCredentialMenuProvider.cs new file mode 100644 index 000000000..1fb7d8c48 --- /dev/null +++ b/memex/Memex.Portal.Shared/Social/LinkedInCredentialMenuProvider.cs @@ -0,0 +1,138 @@ +using System.Linq; +using System.Reactive.Linq; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Layout.Composition; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using MeshWeaver.Social; + +namespace Memex.Portal.Shared.Social; + +/// +/// Adds two contextual menu items for the LinkedIn publishing integration: +/// +/// - On a User node (viewer's own only): "Link LinkedIn account" — +/// visible only when no LinkedIn credential exists yet under +/// {userPath}/_ApiCredentials/linkedin. Once linked, the item +/// self-hides so the user isn't prompted to re-link. +/// +/// - On an ApiCredential node whose content's Platform is LinkedIn: +/// "Download past posts" — triggers the pull endpoint rooted at the +/// credential's parent (i.e. the user path). +/// +/// Both items require Update permission on the target node, which the viewer +/// has by definition on their own user + satellites. +/// +public sealed class LinkedInCredentialMenuProvider : INodeMenuProvider +{ + public string Context => NodeMenuItemsExtensions.NodeMenuContext; + + /// + /// Reactive: switches on the live own-node stream, then (for the User case) tracks the + /// LinkedIn credential synced query so "Link LinkedIn account" appears/self-hides live as the + /// credential is created — no Channel bridge, no one-shot read. Emits an empty slice + /// for non-applicable nodes so the menu aggregator's CombineLatest never stalls. + /// + public IObservable> GetItems( + LayoutAreaHost host, RenderingContext ctx) + { + var hubPath = host.Hub.Address.ToString(); + var accessService = host.Hub.ServiceProvider.GetService(typeof(AccessService)) as AccessService; + var viewerId = accessService?.Context?.ObjectId + ?? accessService?.CircuitContext?.ObjectId; + if (string.IsNullOrEmpty(viewerId)) + return Observable.Return>([]); + + // Live own-node stream. StartWith(null) so the outer Switch emits before the node loads; + // Catch degrades to "no node" on hubs without a MeshDataSource. + var nodeStream = host.Workspace.GetMeshNodeStream() + .Select(n => (MeshNode?)n) + .Catch(_ => Observable.Return(null)) + .StartWith((MeshNode?)null); + + return nodeStream + .Select(node => BuildItems(host, hubPath, viewerId!, node)) + .Switch(); + } + + private static IObservable> BuildItems( + LayoutAreaHost host, string hubPath, string viewerId, MeshNode? node) + { + IReadOnlyCollection empty = []; + if (node is null) + return Observable.Return(empty); + + // Case 1: viewer's own User node. + if (hubPath.Equals($"User/{viewerId}", System.StringComparison.OrdinalIgnoreCase) + && string.Equals(node.NodeType, "User", System.StringComparison.OrdinalIgnoreCase)) + { + // Only show "Link LinkedIn" when no credential exists yet. Synced query via + // workspace.GetQuery — bypasses RLS, gated on Initial, deduped by path. Live, so the + // item self-hides the moment the credential lands. + var credentialPath = $"{hubPath}/_ApiCredentials/linkedin"; + return host.Workspace.GetQuery($"linkedin-credential:{credentialPath}", $"path:{credentialPath}") + .Select(items => items.Any() + ? empty + : (IReadOnlyCollection) + [ + new NodeMenuItemDefinition( + Label: "Link LinkedIn account", + Area: "ConnectLinkedIn", + Icon: "LinkSquare", + RequiredPermission: Permission.Update, + Order: 60, + Href: "/connect/linkedin/me"), + ]) + .StartWith(empty); + } + + // Case 2: an ApiCredential node for LinkedIn. + if (string.Equals(node.NodeType, ApiCredentialNodeType.NodeType, System.StringComparison.OrdinalIgnoreCase)) + { + var platform = ExtractPlatform(node); + if (!string.Equals(platform, LinkedInPublisher.PlatformId, System.StringComparison.OrdinalIgnoreCase)) + return Observable.Return(empty); + + // The credential node lives at {userPath}/_ApiCredentials/{platform} — the + // user path is the grandparent namespace (strip the last two path segments). + var segments = hubPath.Split('/'); + if (segments.Length < 3) + return Observable.Return(empty); + var userPath = string.Join("/", segments.Take(segments.Length - 2)); + + return Observable.Return>( + [ + new NodeMenuItemDefinition( + Label: "Download past posts", + Area: "PullLinkedInPosts", + Icon: "ArrowDownload", + RequiredPermission: Permission.Update, + Order: 10, + Href: $"/connect/linkedin/pull?profile={System.Uri.EscapeDataString(userPath)}"), + new NodeMenuItemDefinition( + Label: "Re-authorize", + Area: "ReAuthorizeLinkedIn", + Icon: "ArrowSync", + RequiredPermission: Permission.Update, + Order: 20, + Href: $"/connect/linkedin?profile={System.Uri.EscapeDataString(userPath)}"), + ]); + } + + return Observable.Return(empty); + } + + private static string? ExtractPlatform(MeshNode node) + { + if (node.Content is PlatformCredential typed) return typed.Platform; + if (node.Content is System.Text.Json.JsonElement je + && je.TryGetProperty("platform", out var p) + && p.ValueKind == System.Text.Json.JsonValueKind.String) + return p.GetString(); + return null; + } +} diff --git a/memex/Memex.Portal.Shared/StaticRepoSyncExtensions.cs b/memex/Memex.Portal.Shared/StaticRepoSyncExtensions.cs new file mode 100644 index 000000000..d006b2390 --- /dev/null +++ b/memex/Memex.Portal.Shared/StaticRepoSyncExtensions.cs @@ -0,0 +1,101 @@ +using System.Reactive.Linq; +using MeshWeaver.AI; +using MeshWeaver.Documentation; +using MeshWeaver.Graph; +using MeshWeaver.Mesh; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared; + +/// +/// Wires static-repo → DB synchronization for the partitions selected by +/// : registers each partition's +/// , declares its (so the PG +/// schema is provisioned), and starts the import hosted service that runs +/// on boot. The read-only in-memory static providers +/// for these partitions are skipped at their registration sites (AddAgentType / AddLanguageModelType +/// / AddModelProviderType / AddDocumentation, gated on the same set) so Postgres serves them and +/// accepts the import's writes. No-op when no partition is selected. See +/// Doc/Architecture/StaticRepoImport.md. +/// +public static class StaticRepoSyncExtensions +{ + public static TBuilder AddStaticRepoSync(this TBuilder builder, + IReadOnlySet serveFromPartition) where TBuilder : MeshBuilder + { + if (serveFromPartition.Count == 0) + return builder; // sync disabled — in-memory serving everywhere, no import. + + // NOTE: partition SCHEMAS are provisioned reactively by the importer itself, via the standard + // IPartitionStorageProvider.EnsurePartitionProvisioned (reactive + pooled). We do NOT declare + // PartitionDefinition nodes here to force a schema — that path provisioned the wrong-case + // schema. See StaticRepoImporter.Run + Doc/Architecture/StaticRepoImport.md. + builder.ConfigureServices(services => + { + if (serveFromPartition.Contains("Doc")) + services.AddSingleton(); + if (serveFromPartition.Contains("Agent")) + services.AddSingleton(); + if (serveFromPartition.Contains("Model")) + services.AddSingleton(); + if (serveFromPartition.Contains("Harness")) + services.AddSingleton(); + if (serveFromPartition.Contains("Skill")) + services.AddSingleton(); + + // Runs after the PG schema-provisioning hosted service (registered earlier by + // AddPartitionedPostgreSqlPersistence) — hosted services start in registration order. + services.AddHostedService(); + return services; + }); + return builder; + } +} + +/// +/// Boot hook that runs once registered +/// s exist. Reactive + fire-and-forget: it does NOT block host +/// startup (the PG schema is already provisioned by the time this StartAsync runs, since the +/// schema hosted service is registered earlier). ImportAll impersonates System and is idempotent +/// (fingerprint short-circuit), so re-runs are cheap. +/// +internal sealed class StaticRepoImportHostedService( + IMessageHub hub, + IEnumerable sources, + ILogger? logger = null) : IHostedService +{ + private IDisposable? _subscription; + + public Task StartAsync(CancellationToken cancellationToken) + { + if (!sources.Any()) + return Task.CompletedTask; + + logger?.LogInformation( + "[StaticRepoImport] starting sync-context init for {Count} source(s).", sources.Count()); + // 🚨 SubscribeOn the thread pool — NOT the host-startup thread. The import's reactive chain + // (meshService.Query → CreateNode/Overwrite) round-trips the mesh + per-node hubs; running + // the subscription on the startup thread re-enters the hub schedulers mid-init and DEADLOCKS + // (it hung the whole import). The chain is pure IObservable (no FromAsync/await), so + // SubscribeOn moves the entire subscription cleanly off the startup thread. StartAsync + // returns immediately (Task.CompletedTask) — no async/await, no blocking. See + // Doc/Architecture/AsynchronousCalls.md. + _subscription = StaticRepoImporter.ImportAll(hub, logger) + .SubscribeOn(System.Reactive.Concurrency.TaskPoolScheduler.Default) + .Subscribe( + r => logger?.LogInformation( + "[StaticRepoImport] {Partition}: {Outcome} ({Count} node(s)).", + r.Partition, r.Outcome, r.Count), + ex => logger?.LogWarning(ex, "[StaticRepoImport] sync-context init failed.")); + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) + { + _subscription?.Dispose(); + return Task.CompletedTask; + } +} diff --git a/memex/Memex.Portal.Shared/Storage/MemexInfraOptions.cs b/memex/Memex.Portal.Shared/Storage/MemexInfraOptions.cs new file mode 100644 index 000000000..b8aa4aba2 --- /dev/null +++ b/memex/Memex.Portal.Shared/Storage/MemexInfraOptions.cs @@ -0,0 +1,86 @@ +using System.Collections.Immutable; + +namespace Memex.Portal.Shared.Storage; + +/// +/// Declarative storage topology for a Memex deployment, loaded from JSON config FILES that live +/// on the shared mounted drive (not k8s ConfigMaps / env) so an operator edits a file on the share +/// and the portal refreshes — no redeploy. Bootstrap is a single pointer: Infra:ConfigDirectory +/// (e.g. /mnt/config); every *.json there is layered as a config source, and a polling +/// reconciler (Observable.Interval re-read + diff — FileSystemWatcher does NOT fire on SMB/Azure Files) +/// applies changes live. +/// +/// This is ADDITIVE and backward-compatible: when no Infra section is present the portal +/// falls back to the existing single Storage + single Postgres wiring (the ACA path is +/// unaffected). The mesh nodes remain the actual data; this file only declares WHERE/HOW that data +/// is stored. +/// +/// Reconciliation scope: +/// +/// ContentMounts reconcile LIVE (add/remove/update content collections on each poll). +/// PgStorages apply at (re)init — the partitioned persistence layer is wired at boot, so a +/// data-source topology change is applied on a graceful re-init / rolling restart. +/// +/// +public sealed record MemexInfraOptions +{ + public const string SectionName = "Infra"; + + /// Directory on the shared mounted drive holding the JSON config files (e.g. /mnt/config). + /// Empty = config-on-drive disabled; the portal uses the legacy single-Storage/single-PG wiring. + public string? ConfigDirectory { get; init; } + + /// How often the polling reconciler re-reads the config files (SMB-safe refresh). + /// FileSystemWatcher is unreliable on Azure Files, so we poll. Default 15s. + public int RefreshSeconds { get; init; } = 15; + + /// Postgres databases exposed as mesh storage. Each is its own mesh_nodes store; the portal + /// routes partitions across them. The first enabled entry is the primary (bootstrap) store. + public ImmutableArray PgStorages { get; init; } = ImmutableArray.Empty; + + /// Content collections, each backed by a mounted file system path (a subPath of a shared + /// Azure Files share present on every pod). Maps onto the existing ContentCollectionConfig surface. + public ImmutableArray ContentMounts { get; init; } = ImmutableArray.Empty; +} + +/// A Postgres database exposed as a mesh storage / data source. +public sealed record MeshPgStorageConfig +{ + /// Logical name for the data source (used in routing / diagnostics). + public required string Name { get; init; } + + /// Connection string. May reference a secret (resolved out of the config-file value at load). + /// Contains database.azure.com ⇒ the Azure-Npgsql auth path; otherwise plain Npgsql. + public required string ConnectionString { get; init; } + + /// Disabled entries are ignored by the reconciler (soft remove without deleting the entry). + public bool Enabled { get; init; } = true; +} + +/// A content collection backed by a mounted file-system path. Reconciled live. +public sealed record ContentMountConfig +{ + /// Collection name (e.g. "content", "attachments"). + public required string Name { get; init; } + + /// Absolute mount path on the pod (e.g. /mnt/content). The container collection BasePath is + /// MountPath joined with SubPath. + public required string MountPath { get; init; } + + /// Optional sub-path under the mount (the share is mounted once on all nodes; each collection + /// lives in its own sub-path of it). + public string? SubPath { get; init; } + + /// Content stream-provider source type. Defaults to the FileSystem provider (the mounted drive). + public string SourceType { get; init; } = "FileSystem"; + + /// Whether writes are allowed into this collection. + public bool IsEditable { get; init; } = true; + + /// Disabled entries are dropped by the reconciler. + public bool Enabled { get; init; } = true; + + /// The resolved on-disk base path the content provider uses (MountPath + SubPath). + public string ResolvedBasePath => + string.IsNullOrEmpty(SubPath) ? MountPath : System.IO.Path.Combine(MountPath, SubPath); +} diff --git a/memex/Memex.Portal.Shared/Teams/ITeamsClient.cs b/memex/Memex.Portal.Shared/Teams/ITeamsClient.cs new file mode 100644 index 000000000..64393c464 --- /dev/null +++ b/memex/Memex.Portal.Shared/Teams/ITeamsClient.cs @@ -0,0 +1,19 @@ +namespace Memex.Portal.Shared.Teams; + +/// +/// Seam for talking to the Microsoft Teams / Bot Framework connector: validate that an inbound activity +/// really came from the Bot Framework, and post a reply back into a conversation. Tests substitute a +/// hand-written fake (CI has no Bot credentials / no real Teams), so the inbound→thread→reply pipeline +/// can be exercised without the live connector. +/// +public interface ITeamsClient +{ + /// True when the Teams bot is enabled and credentials are configured. + bool IsConfigured { get; } + + /// Validates the inbound Authorization header as a genuine Bot Framework token. + Task ValidateInboundAsync(string? authorizationHeader, CancellationToken ct); + + /// Posts a message activity back into the given Teams conversation. + Task SendMessageAsync(string serviceUrl, string conversationId, string text, CancellationToken ct); +} diff --git a/memex/Memex.Portal.Shared/Teams/TeamsBotController.cs b/memex/Memex.Portal.Shared/Teams/TeamsBotController.cs new file mode 100644 index 000000000..b59c5d705 --- /dev/null +++ b/memex/Memex.Portal.Shared/Teams/TeamsBotController.cs @@ -0,0 +1,69 @@ +using System.Reactive.Linq; +using System.Text.Json; +using System.Text.RegularExpressions; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Teams; + +/// +/// Bot Framework messaging endpoint for the Teams channel. Anonymous at the pipeline level, but every +/// request is authenticated by (Bot Framework JWT) before +/// anything happens — so a forged POST can't trigger agent work. Message activities are parsed and routed +/// to ; the reply is delivered asynchronously by the reply sender. +/// +[ApiController] +[AllowAnonymous] +[Route("api/teams")] +public sealed class TeamsBotController( + ITeamsClient teamsClient, TeamsInboundProcessor processor, ILogger logger) : ControllerBase +{ + [HttpPost("messages")] + public async Task Messages(CancellationToken ct) + { + if (!teamsClient.IsConfigured) return NotFound(); + if (!await teamsClient.ValidateInboundAsync(Request.Headers.Authorization.ToString(), ct)) + return Unauthorized(); + + string body; + using (var reader = new StreamReader(Request.Body)) + body = await reader.ReadToEndAsync(ct); + + try + { + using var doc = JsonDocument.Parse(body); + var root = doc.RootElement; + if (GetString(root, "type") != "message") return Ok(); // ignore typing/conversationUpdate/etc. + + var from = root.TryGetProperty("from", out var f) && f.ValueKind == JsonValueKind.Object ? f : default; + var conversation = root.TryGetProperty("conversation", out var c) && c.ValueKind == JsonValueKind.Object ? c : default; + + var msg = new InboundTeamsMessage( + Text: StripMentions(GetString(root, "text") ?? ""), + ConversationId: GetString(conversation, "id") ?? "", + ServiceUrl: GetString(root, "serviceUrl") ?? "", + AadObjectId: GetString(from, "aadObjectId"), + UserName: GetString(from, "name")); + + if (!string.IsNullOrWhiteSpace(msg.Text) && !string.IsNullOrEmpty(msg.ConversationId)) + processor.Route(msg).Subscribe( + _ => { }, + ex => logger.LogWarning(ex, "Teams: routing failed")); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Teams: malformed activity payload"); + } + + return Ok(); // Bot Framework expects a prompt 200/202; the agent reply is sent proactively. + } + + private static string? GetString(JsonElement el, string prop) => + el.ValueKind == JsonValueKind.Object && el.TryGetProperty(prop, out var v) && v.ValueKind == JsonValueKind.String + ? v.GetString() : null; + + // Teams channel messages carry the bot @-mention as Name markup — strip it. + private static string StripMentions(string text) => + Regex.Replace(text, ".*?", "", RegexOptions.IgnoreCase | RegexOptions.Singleline).Trim(); +} diff --git a/memex/Memex.Portal.Shared/Teams/TeamsClient.cs b/memex/Memex.Portal.Shared/Teams/TeamsClient.cs new file mode 100644 index 000000000..28af51149 --- /dev/null +++ b/memex/Memex.Portal.Shared/Teams/TeamsClient.cs @@ -0,0 +1,135 @@ +using System.IdentityModel.Tokens.Jwt; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Text.Json; +using MeshWeaver.Mesh; +using Microsoft.Extensions.Logging; +using Microsoft.IdentityModel.Protocols; +using Microsoft.IdentityModel.Protocols.OpenIdConnect; +using Microsoft.IdentityModel.Tokens; + +namespace Memex.Portal.Shared.Teams; + +/// +/// Real over the Bot Framework REST connector. Inbound activities are +/// authenticated by validating the bearer JWT against the Bot Framework's published OpenID metadata +/// (issuer https://api.botframework.com, audience = the bot's app id); outbound replies use an +/// app-only connector token (client credentials at the botframework.com tenant) POSTed to the +/// activity's serviceUrl. Token + signing-key metadata are cached on this (instance) singleton. +/// +public sealed class TeamsClient : ITeamsClient +{ + private const string BotLoginTokenUrl = "https://login.microsoftonline.com/botframework.com/oauth2/v2.0/token"; + private const string ConnectorScope = "https://api.botframework.com/.default"; + private const string OpenIdMetadataUrl = "https://login.botframework.com/v1/.well-known/openidconfiguration"; + private const string ExpectedIssuer = "https://api.botframework.com"; + + private readonly TeamsOptions _options; + private readonly HttpClient _http; + private readonly ILogger? _logger; + private readonly ConfigurationManager _openIdConfig; + + private string? _cachedToken; + private DateTimeOffset _tokenExpiry; + private readonly SemaphoreSlim _tokenGate = new(1, 1); + + public TeamsClient(TeamsOptions options, HttpClient http, ILogger? logger = null) + { + _options = options; + _http = http; + _logger = logger; + _openIdConfig = new ConfigurationManager( + OpenIdMetadataUrl, new OpenIdConnectConfigurationRetriever(), new HttpDocumentRetriever()); + } + + public bool IsConfigured => + _options.Enabled && !string.IsNullOrEmpty(_options.AppId) && !string.IsNullOrEmpty(_options.AppPassword); + + public async Task ValidateInboundAsync(string? authorizationHeader, CancellationToken ct) + { + if (!IsConfigured) return false; + if (string.IsNullOrEmpty(authorizationHeader) || + !authorizationHeader.StartsWith("Bearer ", StringComparison.OrdinalIgnoreCase)) + return false; + var token = authorizationHeader["Bearer ".Length..].Trim(); + try + { + var config = await _openIdConfig.GetConfigurationAsync(ct); + var parameters = new TokenValidationParameters + { + ValidateIssuer = true, + ValidIssuer = ExpectedIssuer, + ValidateAudience = true, + ValidAudience = _options.AppId, + ValidateLifetime = true, + IssuerSigningKeys = config.SigningKeys + }; + new JwtSecurityTokenHandler().ValidateToken(token, parameters, out _); + return true; + } + catch (Exception ex) + { + _logger?.LogWarning(ex, "Teams: inbound token validation failed"); + return false; + } + } + + public async Task SendMessageAsync(string serviceUrl, string conversationId, string text, CancellationToken ct) + { + if (!IsConfigured || string.IsNullOrEmpty(serviceUrl) || string.IsNullOrEmpty(conversationId)) return false; + var token = await GetConnectorTokenAsync(ct); + if (token is null) return false; + try + { + var url = $"{serviceUrl.TrimEnd('/')}/v3/conversations/{Uri.EscapeDataString(conversationId)}/activities"; + using var req = new HttpRequestMessage(HttpMethod.Post, url) + { + Content = JsonContent.Create(new { type = "message", text }) + }; + req.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token); + using var resp = await _http.SendAsync(req, ct); + if (resp.IsSuccessStatusCode) return true; + _logger?.LogWarning("Teams: send reply returned {Status}", (int)resp.StatusCode); + return false; + } + catch (Exception ex) + { + _logger?.LogWarning(ex, "Teams: send reply failed"); + return false; + } + } + + private async Task GetConnectorTokenAsync(CancellationToken ct) + { + if (_cachedToken is not null && DateTimeOffset.UtcNow < _tokenExpiry.AddMinutes(-2)) + return _cachedToken; + await _tokenGate.WaitAsync(ct); + try + { + if (_cachedToken is not null && DateTimeOffset.UtcNow < _tokenExpiry.AddMinutes(-2)) + return _cachedToken; + var form = new Dictionary + { + ["grant_type"] = "client_credentials", + ["client_id"] = _options.AppId!, + ["client_secret"] = _options.AppPassword!, + ["scope"] = ConnectorScope + }; + // Single-tenant bots acquire the connector token from their OWN tenant authority; the legacy + // botframework.com authority is for (now-deprecated) multi-tenant bots. + var tokenUrl = string.IsNullOrEmpty(_options.TenantId) + ? BotLoginTokenUrl + : $"https://login.microsoftonline.com/{_options.TenantId}/oauth2/v2.0/token"; + using var resp = await _http.PostAsync(tokenUrl, new FormUrlEncodedContent(form), ct); + var body = await resp.Content.ReadAsStringAsync(ct); + if (!resp.IsSuccessStatusCode) { _logger?.LogWarning("Teams: connector token {Status}", (int)resp.StatusCode); return null; } + using var doc = JsonDocument.Parse(body); + var token = doc.RootElement.TryGetProperty("access_token", out var at) ? at.GetString() : null; + var expires = doc.RootElement.TryGetProperty("expires_in", out var ei) ? ei.GetInt32() : 3600; + _cachedToken = token; + _tokenExpiry = DateTimeOffset.UtcNow.AddSeconds(expires); + return token; + } + finally { _tokenGate.Release(); } + } +} diff --git a/memex/Memex.Portal.Shared/Teams/TeamsInboundProcessor.cs b/memex/Memex.Portal.Shared/Teams/TeamsInboundProcessor.cs new file mode 100644 index 000000000..0806eae00 --- /dev/null +++ b/memex/Memex.Portal.Shared/Teams/TeamsInboundProcessor.cs @@ -0,0 +1,148 @@ +using System.Reactive; +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.AI; // StartThread / SubmitMessage +using MeshWeaver.Graph.Configuration; // TeamsConversationNodeType, UserNodeType +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; // IMeshService, IMeshQueryCore, MeshQueryRequest +using MeshWeaver.Mesh.Threading; // IoPool — bounded HTTP pool (replaces bare Observable.FromAsync) +using MeshWeaver.Messaging; // IMessageHub, AccessService +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Teams; + +/// A parsed inbound Teams message (Graph/Bot-Framework free, so it is unit-testable). +public record InboundTeamsMessage( + string Text, string ConversationId, string ServiceUrl, string? AadObjectId, string? UserName); + +/// +/// Turns an inbound Teams message into agent work, mirroring the email channel: map the Teams user to a +/// Memex user (by AAD object id), find-or-create one thread per Teams conversation (keyed by +/// conversationId via a link node), and run the agent as that user. +/// The agent's reply is delivered back to Teams by TeamsReplySender. Unknown senders get a polite +/// "no account" reply. +/// +public sealed class TeamsInboundProcessor : IDisposable +{ + private const string Agent = "Assistant"; + + // Bound to a sane cap so a burst of inbound Teams notifications can't open unbounded Graph calls. + private const int HttpConcurrency = 8; + + private readonly IMessageHub hub; + private readonly ITeamsClient teamsClient; + private readonly ILogger? logger; + private readonly IMeshService meshService; + private readonly AccessService accessService; + private readonly IMeshQueryCore query; + private readonly JsonSerializerOptions jsonOptions; + + // Dedicated bounded HTTP pool, ALWAYS created fresh and owned by this instance — never resolved + // from the mesh-scoped IoPoolRegistry. The portal builds this processor from its OWN DI container + // while activating hosted services; reaching across into the mesh hub's ServiceProvider at that + // moment races that provider's internal service realization (a documented NRE crash-loop — see + // GraphMail). A self-owned pool always resolves and is disposed with this singleton. + private readonly IoPool _http = new(HttpConcurrency); + + public TeamsInboundProcessor(IMessageHub hub, ITeamsClient teamsClient, ILogger? logger = null) + { + this.hub = hub; + this.teamsClient = teamsClient; + this.logger = logger; + meshService = hub.ServiceProvider.GetRequiredService(); + accessService = hub.ServiceProvider.GetRequiredService(); + query = hub.ServiceProvider.GetRequiredService(); + jsonOptions = hub.JsonSerializerOptions; + } + + public void Dispose() => _http.Dispose(); + + /// Routes a parsed Teams message. Bot-Framework-free → unit-testable. + public IObservable Route(InboundTeamsMessage m) + { + if (string.IsNullOrWhiteSpace(m.Text) || string.IsNullOrEmpty(m.AadObjectId)) + return Observable.Return(Unit.Default); + + return query.Query(MeshQueryRequest.FromQuery( + $"nodeType:{UserNodeType.NodeType} content.objectId:{m.AadObjectId} limit:1"), jsonOptions) + .Take(1) + .Select(change => change.Items.FirstOrDefault(n => n.State == MeshNodeState.Active)) + .SelectMany(userNode => userNode is not null + ? HandleUser(userNode.Id, m) + : HandleUnknown(m)); + } + + private IObservable HandleUser(string username, InboundTeamsMessage m) => + FindThread(m.ConversationId).SelectMany(existingThreadPath => + Observable.Using( + () => accessService.ImpersonateAsSystem(), + _ => + { + if (!string.IsNullOrEmpty(existingThreadPath)) + { + // Continue the conversation's thread. + hub.SubmitMessage(existingThreadPath, m.Text, + agentName: Agent, createdBy: username, authorName: m.UserName, + onError: err => logger?.LogWarning("TeamsInbound: SubmitMessage failed: {Err}", err)); + return Observable.Return(Unit.Default); + } + + // New conversation → start a thread and link it to the Teams conversation for replies. + hub.StartThread($"{username}/_Teams", m.Text, + agentName: Agent, createdBy: username, authorName: m.UserName, + onCreated: node => CreateLink(node.Path, m), + onError: err => logger?.LogWarning("TeamsInbound: StartThread failed: {Err}", err)); + return Observable.Return(Unit.Default); + })); + + private IObservable HandleUnknown(InboundTeamsMessage m) + { + logger?.LogInformation("TeamsInbound: message from unknown Teams user {User}", m.AadObjectId); + return _http.Run(async ct => await teamsClient.SendMessageAsync(m.ServiceUrl, m.ConversationId, + "You don't have a Memex account linked to this Microsoft identity yet, so I can't act for you here.", ct) + .ConfigureAwait(false)) + .Select(_ => Unit.Default) + .Catch((Exception _) => Observable.Return(Unit.Default)); + } + + private void CreateLink(string threadPath, InboundTeamsMessage m) + { + var node = new MeshNode(TeamsConversationNodeType.NodeType, $"{threadPath}/{TeamsConversationNodeType.Segment}/{TeamsConversationNodeType.NodeType}") + { + Name = "Teams Conversation", + MainNode = threadPath, + Content = new TeamsConversation + { + ThreadPath = threadPath, + ServiceUrl = m.ServiceUrl, + ConversationId = m.ConversationId, + TeamsUserId = m.AadObjectId + } + }; + meshService.CreateNode(node).Subscribe( + _ => { }, + ex => logger?.LogWarning(ex, "TeamsInbound: failed to link thread {Thread} to Teams conversation", threadPath)); + } + + private IObservable FindThread(string conversationId) => + query.Query(MeshQueryRequest.FromQuery( + $"nodeType:{TeamsConversationNodeType.NodeType} content.conversationId:{conversationId} limit:1"), jsonOptions) + .Take(1) + .Select(change => change.Items + .Select(n => LinkOf(n)?.ThreadPath) + .FirstOrDefault(p => !string.IsNullOrEmpty(p))); + + private TeamsConversation? LinkOf(MeshNode n) => n.Content switch + { + TeamsConversation c => c, + JsonElement je => Safe(je), + _ => null + }; + + private TeamsConversation? Safe(JsonElement je) + { + try { return JsonSerializer.Deserialize(je.GetRawText(), jsonOptions); } + catch { return null; } + } +} diff --git a/memex/Memex.Portal.Shared/Teams/TeamsReplySender.cs b/memex/Memex.Portal.Shared/Teams/TeamsReplySender.cs new file mode 100644 index 000000000..23d4e088a --- /dev/null +++ b/memex/Memex.Portal.Shared/Teams/TeamsReplySender.cs @@ -0,0 +1,138 @@ +using System.Collections.Concurrent; +using System.Reactive.Disposables; +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.AI; // ThreadFlow, ThreadMessage +using MeshWeaver.Blazor.Infrastructure; // PortalApplication +using MeshWeaver.Data; // IWorkspace, GetWorkspace, GetMeshNodeStream +using MeshWeaver.Graph.Configuration; // TeamsConversationNodeType +using MeshWeaver.Mesh; // TeamsConversation, MeshNode +using MeshWeaver.Mesh.Security; // ImpersonateAsSystem +using MeshWeaver.Mesh.Services; // IMeshQueryCore, MeshQueryRequest +using MeshWeaver.Mesh.Threading; // IoPool — bounded HTTP pool (replaces bare Observable.FromAsync) +using MeshWeaver.Messaging; // IMessageHub, AccessService +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Memex.Portal.Shared.Teams; + +/// +/// Delivers agent replies back into Teams. Watches the link nodes; for +/// each, observes its thread the exact same way the GUI and tests do +/// (workspace.GetMeshNodeStream(threadPath) → , wait for +/// !IsExecuting with a new Messages[^1]), reads that response message node at +/// {threadPath}/{messageId} (, public Text), and posts it via +/// . Send-once is tracked by +/// (persisted → restart-safe). Inert unless the Teams bot is configured. +/// +public sealed class TeamsReplySender( + IServiceProvider rootServices, + IHostApplicationLifetime lifetime, + ILogger? logger = null) : IHostedService, IDisposable +{ + // Bound to a sane cap so a burst of outbound sends can't open unbounded Graph calls. + private const int HttpConcurrency = 8; + + private readonly CompositeDisposable subscriptions = new(); + private readonly ConcurrentDictionary watched = new(); // threadPath → subscribed (instance) + private readonly ConcurrentDictionary lastSent = new(); + private IServiceScope? scope; + + // Dedicated bounded HTTP pool, ALWAYS created fresh and owned by this instance — never resolved + // from the mesh-scoped IoPoolRegistry. The portal builds this sender from its OWN DI container + // while activating hosted services; reaching across into the mesh hub's ServiceProvider at that + // moment races that provider's internal service realization (a documented NRE crash-loop — see + // GraphMail). A self-owned pool always resolves and is disposed with this singleton. + private readonly IoPool _http = new(HttpConcurrency); + + public Task StartAsync(CancellationToken cancellationToken) + { + lifetime.ApplicationStarted.Register(Begin); + return Task.CompletedTask; + } + + private void Begin() + { + try + { + scope = rootServices.CreateScope(); + var hub = scope.ServiceProvider.GetRequiredService().Hub; + var teams = hub.ServiceProvider.GetService(); + if (teams is null || !teams.IsConfigured) return; // Teams bot off → inert + var query = hub.ServiceProvider.GetRequiredService(); + var access = hub.ServiceProvider.GetRequiredService(); + var jsonOptions = hub.JsonSerializerOptions; + + // Watch the Teams conversation links; subscribe to each link's thread exactly once. + subscriptions.Add(query + .Query(MeshQueryRequest.FromQuery( + $"nodeType:{TeamsConversationNodeType.NodeType}"), jsonOptions) + .Select(change => change.Items) + .Subscribe( + items => + { + foreach (var linkNode in items) + { + var link = LinkOf(linkNode, jsonOptions); + if (link is null || string.IsNullOrEmpty(link.ThreadPath)) continue; + if (!watched.TryAdd(link.ThreadPath, 0)) continue; // already watching + lastSent[link.ThreadPath] = link.LastDeliveredMessageId; // restart-safe baseline + WatchThread(hub, teams, access, linkNode.Path, link); + } + }, + ex => logger?.LogWarning(ex, "TeamsReply: link query failed"))); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "TeamsReply: failed to start"); + } + } + + private void WatchThread( + IMessageHub hub, ITeamsClient teams, AccessService access, string linkPath, TeamsConversation link) + { + var workspace = hub.GetWorkspace(); + // Reuse the shared read-side abstraction — the SAME one the GUI/tests use to read agent replies. + // No bespoke thread observing here (see ThreadFlow.ObserveResponses / ThreadOperations.md). + subscriptions.Add(ThreadFlow.ObserveResponses(hub, link.ThreadPath) + .Where(r => r.MessageId != lastSent.GetValueOrDefault(link.ThreadPath)) + .SelectMany(r => _http + .Run(ct => teams.SendMessageAsync(link.ServiceUrl, link.ConversationId, r.Message.Text, ct)) + .Select(ok => (r.MessageId, ok))) + .Subscribe( + res => + { + if (!res.ok) return; + lastSent[link.ThreadPath] = res.MessageId; + // Persist send-once across restarts on the link node. + using (access.ImpersonateAsSystem()) + workspace.GetMeshNodeStream(linkPath) + .Update(node => node with { Content = link with { LastDeliveredMessageId = res.MessageId } }) + .Subscribe(_ => { }, ex => logger?.LogWarning(ex, "TeamsReply: persist mark failed")); + }, + ex => logger?.LogWarning(ex, "TeamsReply: delivery failed for {Thread}", link.ThreadPath))); + } + + private static TeamsConversation? LinkOf(MeshNode n, JsonSerializerOptions opts) => n.Content switch + { + TeamsConversation c => c, + JsonElement je => Safe(je, opts), + _ => null + }; + + private static TeamsConversation? Safe(JsonElement je, JsonSerializerOptions opts) + { + try { return JsonSerializer.Deserialize(je.GetRawText(), opts); } + catch { return null; } + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + public void Dispose() + { + subscriptions.Dispose(); + scope?.Dispose(); + _http.Dispose(); + } +} diff --git a/memex/aspire/Memex.AppHost/Memex.AppHost.csproj b/memex/aspire/Memex.AppHost/Memex.AppHost.csproj index 7b761e6bc..eee67ec22 100644 --- a/memex/aspire/Memex.AppHost/Memex.AppHost.csproj +++ b/memex/aspire/Memex.AppHost/Memex.AppHost.csproj @@ -1,5 +1,5 @@ - + {e5f6a7b8-c9d0-1234-ef56-789012345678} Exe @@ -13,10 +13,15 @@ + + + + diff --git a/memex/aspire/Memex.AppHost/Program.cs b/memex/aspire/Memex.AppHost/Program.cs index b995294e4..b42fd2901 100644 --- a/memex/aspire/Memex.AppHost/Program.cs +++ b/memex/aspire/Memex.AppHost/Program.cs @@ -26,6 +26,11 @@ // Parameters:microsoft-client-id // Parameters:microsoft-client-secret // Parameters:microsoft-tenant-id +// Parameters:linkedin-client-secret (LinkedIn publishing — client id is inlined below) +// +// For local-test/local-prod, also set the connection string to the Azure PostgreSQL: +// ConnectionStrings:memex (Azure PostgreSQL, bypassing provisioning) +// Blob Storage uses RunAsExisting with Azure Identity (az login) — no secrets needed. var mode = builder.Configuration["mode"]?.ToLowerInvariant() ?? "local"; @@ -40,33 +45,141 @@ return; } +// --- Self-host targets: artifacts GENERATED by Aspire publishers from the reusable +// AddMemex model (no hand-authored compose/Helm). One model → many outputs: +// aspire publish --publisher docker-compose -o deploy/compose (mode=compose / compose-ha) +// aspire publish --publisher kubernetes -o deploy/helm (mode=kubernetes / kubernetes-ha) +// Azure-free: Postgres (pgvector container) + filesystem backend on volumes + the portal-ai +// image. HA flips Orleans clustering to ADO.NET-Postgres (the proxy + RWX volume are +// finalized in the generated baseline / chart values). --- +if (mode is "compose" or "compose-ha" or "kubernetes" or "kubernetes-ha") +{ + var ha = mode.EndsWith("-ha", StringComparison.Ordinal); + + if (mode.StartsWith("compose", StringComparison.Ordinal)) + builder.AddDockerComposeEnvironment("self-host"); + else + builder.AddKubernetesEnvironment("k8s") + .WithHelm(helm => helm + .WithChartName("memex") + .WithChartDescription("MeshWeaver Memex portal — Azure-free Kubernetes self-host.")); + + builder.AddMemex("memex", o => o + .WithBackend("Filesystem") + // Real, Postgres-backed cluster membership (never Localhost in prod). Works for a single + // silo or HA; `ha` only drives replica count, not the membership provider. + .WithOrleansClustering("AdoNet") + .WithAiClis() + .WithImage(tag: builder.Configuration["Parameters:image-tag"]) + .WithMasterKey(builder.Configuration["Parameters:key-protection-master-key"])); + + builder.Build().Run(); + return; +} + // --- Shared Parameters (linked to GitHub secrets; locally via `dotnet user-secrets`) --- // LLM API key (single Azure Foundry key for both Anthropic and OpenAI endpoints) var azureFoundryKey = builder.AddParameter("azure-foundry-key", secret: true); -// Authentication (Microsoft is required; Google is optional for local) +// Authentication (Microsoft client id+secret required; tenant id optional — +// omit entirely for "common" multi-tenant which signs in any Microsoft account. +// Setting a stale/wrong tenant produces AADSTS90002 "Tenant not found" at first +// sign-in, which is harder to diagnose than just defaulting to common). var microsoftClientId = builder.AddParameter("microsoft-client-id", secret: false); var microsoftClientSecret = builder.AddParameter("microsoft-client-secret", secret: true); -var microsoftTenantId = builder.AddParameter("microsoft-tenant-id", secret: false); +var microsoftTenantIdValue = builder.Configuration["Parameters:microsoft-tenant-id"] ?? ""; +IResourceBuilder? microsoftTenantId = string.IsNullOrEmpty(microsoftTenantIdValue) + ? null : builder.AddParameter("microsoft-tenant-id", secret: false); -// Embedding, Google auth, and custom domain (non-secret optional — ACA accepts empty env vars) +// Embedding, Google auth (non-secret optional — ACA accepts empty env vars, +// so a hard-coded empty default is fine for parameters that are only projected +// via WithEnvironment). var embeddingEndpoint = builder.AddParameter("embedding-endpoint", value: "", secret: false); var embeddingModel = builder.AddParameter("embedding-model", value: "", secret: false); var googleClientId = builder.AddParameter("google-client-id", value: "", secret: false); -var customDomain = builder.AddParameter("custom-domain", value: "", secret: false); -var certificateName = builder.AddParameter("certificate-name", value: "", secret: false); + +// LLM endpoints — sourced from secrets/env, never hardcoded. The Anthropic +// endpoint serves Claude on Azure Foundry (path /anthropic/); the Foundry +// endpoint serves OpenAI/Mistral/etc. via the multi-model /models path. +// Model names are configured per-tier below — also via parameters, never +// inlined here. Configure with `dotnet user-secrets set "Parameters:anthropic-endpoint" "https://..."` +// in dev, or via GitHub Actions / ACA env in deploy. +// +// 🚨 NO `value:` argument here either — same trap as the model-tier params +// (see comment below). `value: ""` makes Aspire resolve the param to empty +// string and skip user-secrets, leaving Anthropic__Endpoint blank in the +// portal — the AzureClaude factory then errors with "Endpoint is missing +// for model 'X'" even when user-secrets has Parameters:anthropic-endpoint set. +var anthropicEndpoint = builder.AddParameter("anthropic-endpoint", secret: false); +// azure-foundry-endpoint is OPTIONAL — defaulted in appsettings.json to the +// shared s-meshweaver `/models` path. user-secrets / env vars / deployment +// state files all override (standard IConfiguration order). The appsettings +// default avoids the `value: ""` trap above by going through the normal config +// pipeline instead of short-circuiting Aspire's parameter resolution. +var azureFoundryEndpoint = builder.AddParameter("azure-foundry-endpoint", secret: false); + +// Anthropic model CATALOG: the concrete model names offered in the chat composer's +// model picker (Anthropic__Models__0/1/2 → BuiltInLanguageModelProvider). Model TIERS +// (heavy/standard/light) are removed — the active model is always the composer +// selection, never a tier. Hidden from source so the deployment owns the model identity. +// +// 🚨 NO `value:` argument: passing `value: ""` makes Aspire treat the param as resolved +// to the literal empty string and SKIP user-secrets / config lookup entirely, leaving the +// catalog empty and the chat dropdown with zero models. Without `value:`, Aspire reads +// from `Parameters:anthropic-model-{n}` (user-secrets locally, container env in deploy). +var anthropicModel0 = builder.AddParameter("anthropic-model-0", secret: false); +var anthropicModel1 = builder.AddParameter("anthropic-model-1", secret: false); +var anthropicModel2 = builder.AddParameter("anthropic-model-2", secret: false); + +// Provider-key encryption master key (Ai:KeyProtection:MasterKey). Encrypts the +// literal ApiKey stored on ModelProvider nodes at rest so a Postgres/backup leak +// alone yields no usable key. Read config/user-secrets first; fall back to a +// clearly-dev default so local dev encrypts out of the box. test/prod MUST +// override via `Parameters:key-protection-master-key` (user-secrets / GitHub +// secret) — the dev default below is NOT secret and must never protect real keys. +// (Manual config read + non-empty value avoids the `value: ""` user-secrets-skip +// trap documented on the model-tier params above.) +var masterKeyValue = builder.Configuration["Parameters:key-protection-master-key"]; +if (string.IsNullOrEmpty(masterKeyValue)) + masterKeyValue = "meshweaver-dev-key-protection-master-do-not-use-in-prod"; +var keyProtectionMasterKey = builder.AddParameter("key-protection-master-key", secret: true, value: masterKeyValue); // Optional secrets/params: ACA rejects secrets with empty values; ConfigureCustomDomain // rejects empty hostnames. Read actual config values to guard optional registrations. var embeddingKeyValue = builder.Configuration["Parameters:embedding-key"] ?? ""; var googleClientSecretValue = builder.Configuration["Parameters:google-client-secret"] ?? ""; +var linkedinClientSecretValue = builder.Configuration["Parameters:linkedin-client-secret"] ?? ""; var customDomainValue = builder.Configuration["Parameters:custom-domain"] ?? ""; +var certificateNameValue = builder.Configuration["Parameters:certificate-name"] ?? ""; + +// Custom domain + cert: register parameters only when config has real values. +// A hard-coded `value: ""` default would override user-secrets and ship an empty +// hostname into the bicep template, which ACA rejects with `InvalidHostName`. +IResourceBuilder? customDomain = string.IsNullOrEmpty(customDomainValue) + ? null : builder.AddParameter("custom-domain", secret: false); +IResourceBuilder? certificateName = string.IsNullOrEmpty(certificateNameValue) + ? null : builder.AddParameter("certificate-name", secret: false); IResourceBuilder? embeddingKey = string.IsNullOrEmpty(embeddingKeyValue) ? null : builder.AddParameter("embedding-key", secret: true); IResourceBuilder? googleClientSecret = string.IsNullOrEmpty(googleClientSecretValue) ? null : builder.AddParameter("google-client-secret", secret: true); +// LinkedIn OAuth app — serves both sign-in (Sign In with LinkedIn using +// OpenID Connect) and publishing posts on behalf of the signed-in user. +// The same app id is projected as `Authentication__LinkedIn__ClientId` for +// the auth pipeline and as `Social__LinkedIn__ClientId` for publishing. +// Client Id is public (shown on the consent screen URL) so it's inlined. +// The secret is wrapped as an AddParameter so Aspire resolves it at deploy +// time from user-secrets / GitHub Actions secrets and projects it into the +// container as a proper secret reference — a plain `builder.Configuration[...]` +// read was silently losing the value in prod (the env var was shipped empty +// and LinkedIn rejected token exchange with "client_secret missing"). +// dotnet user-secrets set "Parameters:linkedin-client-secret" "" --project memex/aspire/Memex.AppHost +const string LinkedInClientId = "780dsuvyxglmc4"; +IResourceBuilder? linkedinClientSecret = string.IsNullOrEmpty(linkedinClientSecretValue) + ? null : builder.AddParameter("linkedin-client-secret", secret: true); + // --- Infrastructure axes --- var isDeployed = mode is "test" or "prod"; var useLocalDb = mode == "local"; @@ -120,8 +233,15 @@ // --- Database Migration --- var dbMigration = builder .AddProject("db-migration") + .WithEnvironment("Embedding__Endpoint", embeddingEndpoint) .WithEnvironment("Embedding__Model", embeddingModel); +// Embedding API key is a secret — only set when configured (ACA rejects empty secrets). +// Without it the documentation backfill still runs; docs are full-text searchable, just +// not vector-indexed. +if (embeddingKey is not null) + dbMigration.WithEnvironment("Embedding__ApiKey", embeddingKey); + if (appInsights is not null) { dbMigration.WithReference(appInsights).WaitFor(appInsights); @@ -134,40 +254,50 @@ .WithReference(orleans) // Local modes need Development environment for static web assets (_framework, _content) .WithEnvironment("ASPNETCORE_ENVIRONMENT", isDeployed ? "Production" : "Development") + // Static-repo → DB sync: materialize the embedded docs, built-in agents, and model catalog + // into their DB partitions on boot (served from PG — the distributed/Orleans routing does not + // consult the in-memory embedded adapter, so without this /Doc pages hang). Mirrors the Helm + // default and values.atioz.yaml. + .WithEnvironment("Features__StaticRepoSync__Partitions__0", "Doc") + .WithEnvironment("Features__StaticRepoSync__Partitions__1", "Agent") + .WithEnvironment("Features__StaticRepoSync__Partitions__2", "Model") + .WithEnvironment("Features__StaticRepoSync__Partitions__3", "Harness") + .WithEnvironment("Features__StaticRepoSync__Partitions__4", "Skill") // Embedding .WithEnvironment("Embedding__Endpoint", embeddingEndpoint) .WithEnvironment("Embedding__Model", embeddingModel) - // LLM: Anthropic (Azure Foundry Claude) - .WithEnvironment("Anthropic__Endpoint", "https://s-meshweaver.services.ai.azure.com/anthropic/") + // LLM: Anthropic (Azure Foundry Claude). Endpoint + model identities all + // come from parameters — no URLs and no model names hardcoded here. + // Agents declare PreferredModel in their MeshNode; ModelTier resolves via + // the parameters below. + .WithEnvironment("Anthropic__Endpoint", anthropicEndpoint) .WithEnvironment("Anthropic__ApiKey", azureFoundryKey) - .WithEnvironment("Anthropic__Models__0", "claude-sonnet-4-6") - .WithEnvironment("Anthropic__Models__1", "claude-opus-4-7") - .WithEnvironment("Anthropic__Models__2", "claude-haiku-4-5") .WithEnvironment("Anthropic__Order", "1") - // Model tiers: map agent tiers to concrete models - .WithEnvironment("ModelTier__Heavy", "claude-opus-4-7") - .WithEnvironment("ModelTier__Standard", "claude-sonnet-4-6") - .WithEnvironment("ModelTier__Light", "claude-haiku-4-5") - // LLM: Azure OpenAI - .WithEnvironment("AzureOpenAIS__Endpoint", "https://s-meshweaver.cognitiveservices.azure.com") - .WithEnvironment("AzureOpenAIS__ApiKey", azureFoundryKey) - .WithEnvironment("AzureOpenAIS__Models__0", "gpt-5-mini") - .WithEnvironment("AzureOpenAIS__Models__1", "gpt-5.4") - .WithEnvironment("AzureOpenAIS__Order", "2") - // LLM: Azure AI Foundry (multi-model inference endpoint) - .WithEnvironment("AzureAIS__Endpoint", "https://fy-meshweaver3-dev-swc-001.services.ai.azure.com/models") + // Advertise the deployed Claude catalog so the factory's Models[] is + // non-empty — that's what BuiltInLanguageModelProvider scans to seed + // the `nodeType:LanguageModel` mesh nodes under `Model/`. Without + // these the picker lists nothing for Anthropic and `claude-sonnet-4-6` + // doesn't appear in autocomplete. These params are the model CATALOG (the + // selectable models in the chat composer) — model tiers were removed; the + // active model is always the composer selection. + .WithEnvironment("Anthropic__Models__0", anthropicModel0) + .WithEnvironment("Anthropic__Models__1", anthropicModel1) + .WithEnvironment("Anthropic__Models__2", anthropicModel2) + // Provider-key encryption master key (ConfigMasterKeyProvider reads this). + .WithEnvironment("Ai__KeyProtection__MasterKey", keyProtectionMasterKey) + // LLM: Azure AI Foundry (multi-model inference endpoint — covers OpenAI, + // Mistral, DeepSeek, etc. through one endpoint). + .WithEnvironment("AzureAIS__Endpoint", azureFoundryEndpoint) .WithEnvironment("AzureAIS__ApiKey", azureFoundryKey) - .WithEnvironment("AzureAIS__Models__0", "gpt-5.4") - .WithEnvironment("AzureAIS__Models__1", "gpt-5.3-codex") - .WithEnvironment("AzureAIS__Models__2", "Mistral-Large-3") - .WithEnvironment("AzureAIS__Models__3", "DeepSeek-V3.2") .WithEnvironment("AzureAIS__Order", "0") // Authentication .WithEnvironment("Authentication__EnableDevLogin", mode != "prod" ? "true" : "false") .WithEnvironment("Authentication__Microsoft__ClientId", microsoftClientId) .WithEnvironment("Authentication__Microsoft__ClientSecret", microsoftClientSecret) - .WithEnvironment("Authentication__Microsoft__TenantId", microsoftTenantId) .WithEnvironment("Authentication__Google__ClientId", googleClientId) + .WithEnvironment("Authentication__LinkedIn__ClientId", LinkedInClientId) + // NuGet cache for #r "nuget:..." directives (in-process restore via MeshWeaver.NuGet). + .WithEnvironment("NUGET_PACKAGES", "/tmp/nuget-cache") // Wait for dependencies .WaitFor(orleansTables) .WaitForCompletion(dbMigration) @@ -175,25 +305,80 @@ .PublishAsAzureContainerApp((module, app) => { app.Configuration.Ingress.StickySessionsAffinity = StickySessionAffinity.Sticky; - if (!string.IsNullOrEmpty(customDomainValue)) + if (customDomain is not null && certificateName is not null) app.ConfigureCustomDomain(customDomain, certificateName); // Scale: min 2 replicas (Orleans needs ≥2 for resilience), max 6 under load. // Each replica: 2 vCPU / 4Gi (50% of Consumption tier max 4 vCPU / 8Gi). app.Template.Scale.MinReplicas = 2; app.Template.Scale.MaxReplicas = 6; + // Orleans needs time to drain grain activations to the surviving silo + // before the container is killed. Default ACA grace period is 30 s — + // far too short for a clean Orleans handoff. Set to 120 s so the + // .NET host's 90 s ShutdownTimeout (set in Memex.Portal.Distributed) + // completes before SIGKILL arrives. + app.Template.TerminationGracePeriodSeconds = 120; }); +// --- Claude Code co-hosting (Phase 5b) — opt-in feature flag --- +// Claude Code runs IN-PROCESS in the portal: the CLI is spawned per chat, each +// under the calling user's own CLAUDE_CONFIG_DIR + subscription token (so +// concurrent users on a replica never share credentials). Enabled at deploy via +// `Parameters:enable-claude-code=true`. When on, the portal: +// • mounts the per-user `.claude` share at /mnt/users, and +// • learns the mount root via ClaudeCode__ConfigDirRoot. +// +// 🚨 DEPLOY CHOICES gated by this flag (not compile-checkable here): +// 1. PORTAL IMAGE must bundle node + BOTH co-hosted CLIs: the Claude Code CLI +// (@anthropic-ai/claude-code) AND the GitHub Copilot CLI. The portal is +// AddProject, so select the CLI-enabled image at deploy via the portal's +// (or a portal Dockerfile variant) — toggled by the +// same flag in the deploy pipeline. +// 2. STORAGE for /mnt/users must be durable + cross-replica. `WithVolume` below +// is auto-translated by Aspire 13.x into an *Aspire-managed* Azure Files +// share. To bring YOUR OWN account, attach a +// ContainerAppManagedEnvironmentStorage (AzureFile: account+key from +// Parameters:claude-storage-account / claude-storage-key, ShareName, +// AccessMode=ReadWrite) to the `memex-aca` environment and reference it from +// the portal's `.PublishAsAzureContainerApp` volume + volumeMount. +var enableClaudeCode = (builder.Configuration["Parameters:enable-claude-code"] ?? "false") + .Equals("true", StringComparison.OrdinalIgnoreCase); +if (enableClaudeCode) +{ + // App-level wiring: tell the co-hosted client where per-user .claude lives. + portal.WithEnvironment("ClaudeCode__ConfigDirRoot", "/mnt/users"); + // 🚨 The /mnt/users MOUNT itself can't use container `WithVolume` here — the + // portal is AddProject (a ProjectResource), not a ContainerResource. Mount it + // at the ACA layer inside the portal's existing `.PublishAsAzureContainerApp( + // (module, app) => { … })` block: add an Azure Files volume to + // `app.Template.Volumes` + a `VolumeMount` at /mnt/users on the container, + // backed by a ContainerAppManagedEnvironmentStorage on `memex-aca` (BYO + // account+key from Parameters:claude-storage-account / claude-storage-key). + // Left for the deploy pass — needs a publish/deploy to finalize + verify. +} + // Optional secrets: only add as env vars when configured (ACA rejects empty secrets) if (embeddingKey is not null) portal.WithEnvironment("Embedding__ApiKey", embeddingKey); if (googleClientSecret is not null) portal.WithEnvironment("Authentication__Google__ClientSecret", googleClientSecret); +if (linkedinClientSecret is not null) +{ + // Same secret powers both sign-in and publishing flows. + portal.WithEnvironment("Authentication__LinkedIn__ClientSecret", linkedinClientSecret); + portal.WithEnvironment("Social__LinkedIn__ClientId", LinkedInClientId); + portal.WithEnvironment("Social__LinkedIn__ClientSecret", linkedinClientSecret); +} if (appInsights is not null) portal = portal.WithReference(appInsights); // --- Azure Blob Storage --- +// Two blob containers share the `memexblobs` storage account: +// `storage` — content collections (files uploaded by users, article assets, etc.) +// `nodetype-cache` — content-addressed NodeType compiled assemblies (keyed by SHA-256 +// of source + config + runtime), replacing the in-memory compile cache +// with a durable, cross-replica-consistent lookup. if (useLocalDb) { // Local emulated storage @@ -204,7 +389,20 @@ .WithLifetime(ContainerLifetime.Persistent) .WithExternalHttpEndpoints()); var storageBlobs = contentStorage.AddBlobs("storage"); + var nodeTypeCache = contentStorage.AddBlobs("nodetype-cache"); portal.WithReference(storageBlobs).WaitFor(storageBlobs); + portal.WithReference(nodeTypeCache).WaitFor(nodeTypeCache); +} +else if (mode is "local-test" or "local-prod") +{ + // Connect to existing Azure Blob Storage via Azure Identity (az login, no secrets needed) + var storageName = mode is "local-test" ? "meshweavermemextest" : "meshweavermemex"; + var contentStorage = builder.AddAzureStorage("memexblobs") + .RunAsExisting(storageName, null); + var storageBlobs = contentStorage.AddBlobs("storage"); + var nodeTypeCache = contentStorage.AddBlobs("nodetype-cache"); + portal.WithReference(storageBlobs); + portal.WithReference(nodeTypeCache); } else { @@ -218,7 +416,9 @@ storageAccount.Location = new Azure.Core.AzureLocation("swedencentral"); }); var storageBlobs = contentStorage.AddBlobs("storage"); + var nodeTypeCache = contentStorage.AddBlobs("nodetype-cache"); portal.WithReference(storageBlobs).WaitFor(storageBlobs); + portal.WithReference(nodeTypeCache).WaitFor(nodeTypeCache); } // --- PostgreSQL --- @@ -253,5 +453,11 @@ portal.WithReference(db).WaitFor(db); } +// Inject the portal's own external HTTPS endpoint as Mcp__BaseUrl so the +// MCP plugin doesn't have to fall back to its HttpContext probe (or, worse, +// a hard-coded localhost default). Aspire substitutes the actual allocated +// URL at container/process start — works the same in prod/test/local. +portal.WithEnvironment("Mcp__BaseUrl", portal.GetEndpoint("https")); + var app = builder.Build(); app.Run(); diff --git a/memex/aspire/Memex.AppHost/appsettings.Development.json b/memex/aspire/Memex.AppHost/appsettings.Development.json index 7fcfaddb3..b67b2ab77 100644 --- a/memex/aspire/Memex.AppHost/appsettings.Development.json +++ b/memex/aspire/Memex.AppHost/appsettings.Development.json @@ -5,5 +5,10 @@ "Microsoft.AspNetCore": "Warning", "Aspire.Hosting.Dcp": "Warning" } + }, + "Parameters": { + "anthropic-model-0": "claude-opus-4-6", + "anthropic-model-1": "claude-sonnet-4-6", + "anthropic-model-2": "claude-haiku-4-5" } } diff --git a/memex/aspire/Memex.AppHost/appsettings.json b/memex/aspire/Memex.AppHost/appsettings.json index 31c092aa4..7fcfaddb3 100644 --- a/memex/aspire/Memex.AppHost/appsettings.json +++ b/memex/aspire/Memex.AppHost/appsettings.json @@ -1,7 +1,7 @@ { "Logging": { "LogLevel": { - "Default": "Information", + "Default": "Warning", "Microsoft.AspNetCore": "Warning", "Aspire.Hosting.Dcp": "Warning" } diff --git a/memex/aspire/Memex.Aspire.Hosting/Memex.Aspire.Hosting.csproj b/memex/aspire/Memex.Aspire.Hosting/Memex.Aspire.Hosting.csproj new file mode 100644 index 000000000..982a50c7f --- /dev/null +++ b/memex/aspire/Memex.Aspire.Hosting/Memex.Aspire.Hosting.csproj @@ -0,0 +1,19 @@ + + + + net10.0 + enable + enable + true + Aspire.Hosting.Memex + Aspire hosting integration for the MeshWeaver Memex portal. builder.AddMemex() wires Postgres (pgvector) + a one-shot DB migration + the portal from published GHCR images; publish to Docker Compose, Kubernetes/Helm, or Azure Container Apps with the standard Aspire publishers. + Systemorph + aspire;hosting;meshweaver;memex;mesh + + + + + + + + diff --git a/memex/aspire/Memex.Aspire.Hosting/MemexHostingExtensions.cs b/memex/aspire/Memex.Aspire.Hosting/MemexHostingExtensions.cs new file mode 100644 index 000000000..100147e2f --- /dev/null +++ b/memex/aspire/Memex.Aspire.Hosting/MemexHostingExtensions.cs @@ -0,0 +1,135 @@ +using Aspire.Hosting.ApplicationModel; + +namespace Aspire.Hosting; + +/// +/// Aspire hosting integration for the MeshWeaver Memex portal. A single builder.AddMemex() +/// wires the full runnable topology — Postgres (pgvector), a one-shot DB migration that gates +/// portal startup, and the portal itself — from published GHCR images, so ANY AppHost can add +/// Memex as one participant and then generate Docker Compose / Kubernetes-Helm / Azure-ACA +/// artifacts with the standard Aspire publishers. Object storage, the NodeType compile cache, +/// the NuGet cache, and DataProtection keys live on mounted volumes (the filesystem backend); +/// mesh data lives in the Postgres database. +/// +public static class MemexHostingExtensions +{ + /// + /// Adds the Memex portal — plus its Postgres database and one-shot migration — to the + /// application model. Returns the portal container resource so the caller can layer on + /// deployment-specific configuration (LLM provider keys, OAuth secrets, scaling, etc.). + /// + /// The distributed application builder. + /// Resource name prefix (default memex). The portal resource takes this exact name. + /// + /// Optional customization. Takes the default options and returns a + /// configured copy — chain the With… helpers or use a raw record with expression. + /// + public static IResourceBuilder AddMemex( + this IDistributedApplicationBuilder builder, + string name = "memex", + Func? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + ArgumentException.ThrowIfNullOrWhiteSpace(name); + + var options = configure?.Invoke(new MemexOptions()) ?? new MemexOptions(); + + var registry = options.ImageRegistry.TrimEnd('/'); + var tag = options.ImageTag; + var portalRepo = options.IncludeAiClis ? "memex-portal-ai" : "memex-portal"; + + // --- Postgres (pgvector) — mesh data, in every topology --- + var postgres = builder.AddPostgres($"{name}-postgres") + .WithImage("pgvector/pgvector", "pg17") + .WithDataVolume($"{name}-pgdata"); + var db = postgres.AddDatabase("memex"); + // Orleans cluster-membership lives on the SAME Postgres server in a SEPARATE database, + // so silo membership never shares tables/locks with mesh data. Aspire owns the DB + its + // connection string (injected as ConnectionStrings:orleans); the migration creates the + // Orleans membership tables and the portal silo uses AdoNet clustering against it. + var clusteringDb = postgres.AddDatabase("orleans"); + + // --- One-shot DB migration; the portal waits for it to complete (mirrors DbVersionGate) --- + // The migration also mirrors the built-in documentation into the `doc` Postgres schema for + // search (with the embedding endpoint/key set it vector-indexes them too) and creates the + // Orleans membership tables in the `orleans` database. + var migration = builder.AddContainer($"{name}-migration", $"{registry}/memex-migration", tag) + .WithReference(db) + .WithReference(clusteringDb) + .WaitFor(db) + .WaitFor(clusteringDb); + + foreach (var kv in options.EmbeddingEnvironment()) + migration.WithEnvironment(kv.Key, kv.Value); + + // --- Portal (co-hosted Orleans silo + Blazor web) --- + // Resource name is "{name}-portal" so it never collides with the "memex" database + // resource that AddDatabase("memex") creates (Aspire resource names are case-insensitive + // and must be unique). The DB resource name stays "memex" because WithReference injects + // ConnectionStrings__memex, which the portal reads as ConnectionStrings:memex. + var portal = builder.AddContainer($"{name}-portal", $"{registry}/{portalRepo}", tag) + .WithHttpEndpoint(targetPort: 8080, name: "http") + .WithExternalHttpEndpoints() + .WithReference(db) + .WithReference(clusteringDb) + .WaitFor(db) + .WaitForCompletion(migration) + .WithEnvironment("ASPNETCORE_HTTP_PORTS", "8080") + // Backend axis (Phase-0 switch in Memex.Portal.Distributed/Program.cs). + .WithEnvironment("Deployment__Backend", options.Backend) + .WithEnvironment("Deployment__DataRoot", "/data") + // Orleans clustering provider — a feature flag (Features:Orleans:Clustering). The + // silo reads ConnectionStrings:orleans (injected by WithReference(clusteringDb)). + .WithEnvironment("Features__Orleans__Clustering", options.OrleansClustering) + // Content storage + graph base paths (filesystem backend). + .WithEnvironment("Storage__Name", "content") + .WithEnvironment("Storage__SourceType", "FileSystem") + .WithEnvironment("Storage__BasePath", "/data/content") + .WithEnvironment("Graph__Storage__Type", "PostgreSql") + .WithEnvironment("Graph__Storage__BasePath", "/data/graph") + // Object storage / NodeType compile cache / NuGet cache / DataProtection keys. + .WithVolume($"{name}-data", "/data") + // Per-user co-hosted-CLI config (.claude / copilot) — a shared volume in HA. + .WithVolume($"{name}-users", "/mnt/users"); + + if (!string.IsNullOrEmpty(options.MasterKey)) + portal.WithEnvironment("Ai__KeyProtection__MasterKey", options.MasterKey); + + // Observability: export OTLP traces/metrics to a collector when one is configured. + // Logs are scraped from container stdout by the cluster log agent (Promtail), so this + // only wires the OTLP push path; ServiceDefaults no-ops the exporter when it's unset. + if (!string.IsNullOrEmpty(options.OtlpEndpoint)) + portal.WithEnvironment("OTEL_EXPORTER_OTLP_ENDPOINT", options.OtlpEndpoint); + + // Embeddings — the portal embeds search-bar queries so they hit the HNSW index that the + // migration populated. Same config flows to both so the vector dimensions line up. + foreach (var kv in options.EmbeddingEnvironment()) + portal.WithEnvironment(kv.Key, kv.Value); + + foreach (var kv in options.FeatureEnvironment()) + portal.WithEnvironment(kv.Key, kv.Value); + + // External sign-in (OAuth) providers — only the ones whose ClientId is set. + foreach (var kv in options.AuthEnvironment()) + portal.WithEnvironment(kv.Key, kv.Value); + + // Outbound email (Microsoft Graph) — invitations + script-triggered notifications. + // The client secret is normally supplied out-of-band (Key Vault → Email__ClientSecret); + // any value set here is emitted too. + foreach (var kv in options.EmailEnvironment()) + portal.WithEnvironment(kv.Key, kv.Value); + + // Microsoft Teams bot (bidirectional). Secret normally via Key Vault → Teams__AppPassword. + foreach (var kv in options.TeamsEnvironment()) + portal.WithEnvironment(kv.Key, kv.Value); + + // MCP back-connection base URL for the co-hosted CLIs ({BaseUrl}/mcp). Defaults to the + // portal's own allocated external endpoint (Aspire substitutes the real URL at publish). + if (!string.IsNullOrEmpty(options.BaseUrl)) + portal.WithEnvironment("Mcp__BaseUrl", options.BaseUrl); + else + portal.WithEnvironment("Mcp__BaseUrl", portal.GetEndpoint("http")); + + return portal; + } +} diff --git a/memex/aspire/Memex.Aspire.Hosting/MemexOptions.cs b/memex/aspire/Memex.Aspire.Hosting/MemexOptions.cs new file mode 100644 index 000000000..1230463bb --- /dev/null +++ b/memex/aspire/Memex.Aspire.Hosting/MemexOptions.cs @@ -0,0 +1,362 @@ +namespace Aspire.Hosting; + +/// +/// Options for . Sensible defaults give a +/// single-node, Azure-free self-host out of the box; override for HA, Azure, or to gate +/// AI capabilities. Every value maps 1:1 to a portal config key, so the same surface flows +/// through Docker Compose .env, Kubernetes config, and ACA / ARM container env. +/// +/// This is an immutable : configure it either by chaining the +/// With… helpers (each returns a new instance) or with a raw record with +/// expression. Both forms compose inside the +/// configure lambda: +/// +/// builder.AddMemex("memex", o => o +/// .WithBackend("Filesystem") +/// .WithOrleansClustering("AdoNet") +/// .WithImage(tag: imageTag) +/// .WithMicrosoftSignIn(clientId, clientSecret, tenantId)); +/// +/// // equivalent, raw record form: +/// builder.AddMemex("memex", o => o with { Backend = "Filesystem", ImageTag = imageTag }); +/// +/// +/// +public sealed record MemexOptions +{ + /// Container registry + namespace holding the Memex images. Default GHCR / Systemorph. + public string ImageRegistry { get; init; } = "ghcr.io/systemorph"; + + /// Image tag applied to all Memex images (portal, migration). Default latest. + public string ImageTag { get; init; } = "latest"; + + /// + /// Use the memex-portal-ai image (co-hosted Claude Code + GitHub Copilot CLIs baked in) + /// rather than the lean memex-portal. Default true. The runtime + /// // flags still gate + /// whether those providers are actually registered. + /// + public bool IncludeAiClis { get; init; } = true; + + /// + /// Object-storage / NodeType cache / NuGet cache / DataProtection backend: + /// Filesystem (default, mounted volumes) or Azure (blob). Mesh data always lives in Postgres. + /// + public string Backend { get; init; } = "Filesystem"; + + /// + /// Orleans clustering: Localhost (single node, default), AdoNet (HA, Postgres-backed), + /// or AzureTables. + /// + public string OrleansClustering { get; init; } = "Localhost"; + + /// Encryption master key for provider credentials (Ai:KeyProtection:MasterKey). Required for production. + public string? MasterKey { get; init; } + + // --- Embeddings (vector search) ----------------------------------------- + // When the endpoint + key are set, the one-shot migration vector-indexes the built-in + // documentation and the portal embeds search-bar queries → semantic search. Without them, + // docs are still copied to Postgres and searchable, just full-text only (no vector ranking). + + /// Azure AI Foundry embeddings endpoint (Cohere embed-v4). Empty = vector search off (FTS still works). + public string? EmbeddingEndpoint { get; init; } + + /// Embeddings API key (secret). Only emitted to containers when set. + public string? EmbeddingApiKey { get; init; } + + /// Embeddings model / deployment name. Default embed-v-4-0 (the Cohere embed-v4 Azure AI Foundry deployment name). + public string EmbeddingModel { get; init; } = "embed-v-4-0"; + + /// The portal's externally reachable base URL; the co-hosted CLIs connect back to {BaseUrl}/mcp. Defaults to the portal's own endpoint. + public string? BaseUrl { get; init; } + + /// + /// OTLP collector endpoint for telemetry export (sets OTEL_EXPORTER_OTLP_ENDPOINT on the portal). + /// Empty = telemetry no-ops (ServiceDefaults skips the exporter when unset). Point this at an + /// in-cluster OpenTelemetry collector / Grafana Alloy (e.g. http://otel-collector:4317) to ship + /// traces + metrics. NOTE: container logs are collected out-of-band by the cluster log agent + /// (Promtail in the grafana/loki-stack — see deploy/aks/scripts/install-observability.sh), so this + /// endpoint is only needed for OTLP traces/metrics, not for log shipping. + /// + public string? OtlpEndpoint { get; init; } + + // Deploy-time capability flags. null = leave the portal default (on); set false to disable explicitly. + public bool? Anthropic { get; init; } + public bool? AzureFoundry { get; init; } + public bool? AzureOpenAI { get; init; } + public bool? OpenAI { get; init; } + public bool? ClaudeCode { get; init; } + public bool? Copilot { get; init; } + + // --- External sign-in (OAuth) providers --------------------------------- + // Set the ClientId to OFFER a provider on the login page; each provider self-skips when + // its ClientId is empty (so leaving these unset = that provider simply isn't shown). Register + // the matching redirect URI on the provider app: {BaseUrl}/signin-{microsoft|google|linkedin}. + + /// Entra/Microsoft app registration Application (client) id. Empty = Microsoft sign-in off. + public string? MicrosoftClientId { get; init; } + /// Microsoft client secret. + public string? MicrosoftClientSecret { get; init; } + /// Microsoft/Entra tenant GUID for the HOME directory. Empty/omitted = "common" (any Microsoft account). + public string? MicrosoftTenantId { get; init; } + + /// Google OAuth client id. Empty = Google sign-in off. + public string? GoogleClientId { get; init; } + /// Google OAuth client secret. + public string? GoogleClientSecret { get; init; } + + /// LinkedIn OAuth client id (powers both sign-in AND post publishing). Empty = LinkedIn off. + public string? LinkedInClientId { get; init; } + /// LinkedIn OAuth client secret. + public string? LinkedInClientSecret { get; init; } + + // --- Outbound email (Microsoft Graph /sendMail) ------------------------- + // When EmailEnabled is true the portal sends mail (invitations, notifications) as the + // configured no-reply mailbox via the Mail.Send application permission. Left unset = disabled + // (the portal registers a NoOp sender). See Doc/Architecture/SendingEmail.md. + + /// Enable outbound email. null/false = NoOp sender (no mail sent). + public bool? EmailEnabled { get; init; } + /// Mailbox to send as (e.g. no-reply@yourtenant.com). + public string? EmailMailboxAddress { get; init; } + /// Entra tenant GUID for the mail app (client-secret flow). + public string? EmailTenantId { get; init; } + /// Mail app registration client id (client-secret flow). + public string? EmailClientId { get; init; } + /// Mail app client secret (keep in Key Vault). + public string? EmailClientSecret { get; init; } + /// Authenticate via managed identity instead of a client secret (prod). + public bool? EmailUseManagedIdentity { get; init; } + + /// Enable the inbound email→agent channel (Graph subscription + webhook). Needs Mail.ReadWrite + a public WebhookBaseUrl. + public bool? EmailInboundEnabled { get; init; } + /// Public base URL Graph calls back for inbound notifications (e.g. https://memex.systemorph.com). + public string? EmailWebhookBaseUrl { get; init; } + /// Shared secret echoed by Graph on each inbound notification (webhook validation). + public string? EmailSubscriptionClientState { get; init; } + + /// Require an invitation to onboard (Features:Onboarding:InvitationOnly). null = portal default (false). + public bool? InvitationOnly { get; init; } + + // --- Microsoft Teams bot (bidirectional channel) ------------------------ + /// Enable the Teams bot channel. null/false = inert (endpoint NotFound, no reply sender). + public bool? TeamsEnabled { get; init; } + /// Azure Bot / app registration id (Bot Framework MicrosoftAppId). + public string? TeamsAppId { get; init; } + /// Bot app client secret (MicrosoftAppPassword). Keep in Key Vault. + public string? TeamsAppPassword { get; init; } + /// Entra tenant id for a single-tenant bot (optional; multi-tenant when empty). + public string? TeamsTenantId { get; init; } + + // === Fluent configuration =============================================== + // Each helper returns a NEW MemexOptions (record `with`); chain them in the AddMemex + // lambda. Empty/null arguments leave the existing value untouched where keeping a + // baked-in default matters (image, embedding model); the grouped sign-in/email helpers + // assign through verbatim (including null) so the env-emission helpers can self-skip. + + /// Override the image registry and/or tag. Null/empty arguments keep the current value. + public MemexOptions WithImage(string? registry = null, string? tag = null) => + this with + { + ImageRegistry = string.IsNullOrEmpty(registry) ? ImageRegistry : registry, + ImageTag = string.IsNullOrEmpty(tag) ? ImageTag : tag, + }; + + /// Select the AI-CLI portal image (memex-portal-ai vs lean memex-portal). + public MemexOptions WithAiClis(bool include = true) => this with { IncludeAiClis = include }; + + /// Object-storage / cache / DataProtection backend: Filesystem or Azure. + public MemexOptions WithBackend(string backend) => this with { Backend = backend }; + + /// Orleans clustering provider: Localhost, AdoNet, or AzureTables. + public MemexOptions WithOrleansClustering(string clustering) => this with { OrleansClustering = clustering }; + + /// Provider-credential encryption master key (Ai:KeyProtection:MasterKey). + public MemexOptions WithMasterKey(string? masterKey) => this with { MasterKey = masterKey }; + + /// The portal's externally reachable base URL ({BaseUrl}/mcp for the co-hosted CLIs). + public MemexOptions WithBaseUrl(string? baseUrl) => this with { BaseUrl = baseUrl }; + + /// OTLP collector endpoint for trace/metric export. + public MemexOptions WithOtlpEndpoint(string? endpoint) => this with { OtlpEndpoint = endpoint }; + + /// + /// Configure vector-search embeddings. + + /// turn on semantic search; an empty keeps the default deployment name. + /// + public MemexOptions WithEmbeddings(string? endpoint, string? apiKey = null, string? model = null) => + this with + { + EmbeddingEndpoint = endpoint, + EmbeddingApiKey = apiKey, + EmbeddingModel = string.IsNullOrEmpty(model) ? EmbeddingModel : model, + }; + + /// + /// Toggle individual AI providers / CLIs. A argument leaves that flag + /// at its current value (so callers flip only what they name, e.g. WithAiProviders(openAI: false)). + /// + public MemexOptions WithAiProviders( + bool? anthropic = null, + bool? azureFoundry = null, + bool? azureOpenAI = null, + bool? openAI = null, + bool? claudeCode = null, + bool? copilot = null) => + this with + { + Anthropic = anthropic ?? Anthropic, + AzureFoundry = azureFoundry ?? AzureFoundry, + AzureOpenAI = azureOpenAI ?? AzureOpenAI, + OpenAI = openAI ?? OpenAI, + ClaudeCode = claudeCode ?? ClaudeCode, + Copilot = copilot ?? Copilot, + }; + + /// Microsoft / Entra sign-in. Omit for "common" (any Microsoft account). + public MemexOptions WithMicrosoftSignIn(string? clientId, string? clientSecret, string? tenantId = null) => + this with + { + MicrosoftClientId = clientId, + MicrosoftClientSecret = clientSecret, + MicrosoftTenantId = tenantId, + }; + + /// Google sign-in. + public MemexOptions WithGoogleSignIn(string? clientId, string? clientSecret) => + this with { GoogleClientId = clientId, GoogleClientSecret = clientSecret }; + + /// LinkedIn — the one app powers both sign-in and post publishing. + public MemexOptions WithLinkedIn(string? clientId, string? clientSecret) => + this with { LinkedInClientId = clientId, LinkedInClientSecret = clientSecret }; + + /// + /// Outbound email via Microsoft Graph /sendMail. A + /// leaves the portal default (NoOp sender); the secret is normally supplied out-of-band (Key Vault). + /// + public MemexOptions WithOutboundEmail( + bool? enabled = true, + string? mailboxAddress = null, + string? tenantId = null, + string? clientId = null, + string? clientSecret = null, + bool? useManagedIdentity = null) => + this with + { + EmailEnabled = enabled, + EmailMailboxAddress = mailboxAddress, + EmailTenantId = tenantId, + EmailClientId = clientId, + EmailClientSecret = clientSecret, + EmailUseManagedIdentity = useManagedIdentity, + }; + + /// + /// Inbound email→agent channel (Graph subscription + webhook). Needs Mail.ReadWrite and a + /// public ; a + /// leaves the portal default (off). + /// + public MemexOptions WithInboundEmail( + bool? enabled = true, + string? webhookBaseUrl = null, + string? clientState = null) => + this with + { + EmailInboundEnabled = enabled, + EmailWebhookBaseUrl = webhookBaseUrl, + EmailSubscriptionClientState = clientState, + }; + + /// Require an invitation to onboard. leaves the portal default (false). + public MemexOptions WithInvitationOnly(bool? invitationOnly = true) => this with { InvitationOnly = invitationOnly }; + + /// + /// Microsoft Teams bot (bidirectional). Needs an Azure Bot resource + a Teams app; the secret is + /// normally supplied out-of-band (Key Vault → Teams__AppPassword). A + /// leaves it inert. + /// + public MemexOptions WithTeams( + bool? enabled = true, + string? appId = null, + string? appPassword = null, + string? tenantId = null) => + this with + { + TeamsEnabled = enabled, + TeamsAppId = appId, + TeamsAppPassword = appPassword, + TeamsTenantId = tenantId, + }; + + internal IEnumerable> AuthEnvironment() + { + if (!string.IsNullOrEmpty(MicrosoftClientId)) yield return new("Authentication__Microsoft__ClientId", MicrosoftClientId); + if (!string.IsNullOrEmpty(MicrosoftClientSecret)) yield return new("Authentication__Microsoft__ClientSecret", MicrosoftClientSecret); + if (!string.IsNullOrEmpty(MicrosoftTenantId)) yield return new("Authentication__Microsoft__TenantId", MicrosoftTenantId); + if (!string.IsNullOrEmpty(GoogleClientId)) yield return new("Authentication__Google__ClientId", GoogleClientId); + if (!string.IsNullOrEmpty(GoogleClientSecret)) yield return new("Authentication__Google__ClientSecret", GoogleClientSecret); + if (!string.IsNullOrEmpty(LinkedInClientId)) + { + // The same LinkedIn app id powers sign-in (Authentication) AND post publishing (Social). + yield return new("Authentication__LinkedIn__ClientId", LinkedInClientId); + yield return new("Social__LinkedIn__ClientId", LinkedInClientId); + } + if (!string.IsNullOrEmpty(LinkedInClientSecret)) + { + yield return new("Authentication__LinkedIn__ClientSecret", LinkedInClientSecret); + yield return new("Social__LinkedIn__ClientSecret", LinkedInClientSecret); + } + } + + /// + /// Embedding config shared by the migration (vector-indexes docs) and the portal (embeds + /// search-bar queries). Model is always emitted so both sides size the vector column the same + /// way; endpoint + key are emitted only when configured (ACA rejects empty secrets). + /// + internal IEnumerable> EmbeddingEnvironment() + { + yield return new("Embedding__Model", EmbeddingModel); + if (!string.IsNullOrEmpty(EmbeddingEndpoint)) yield return new("Embedding__Endpoint", EmbeddingEndpoint); + if (!string.IsNullOrEmpty(EmbeddingApiKey)) yield return new("Embedding__ApiKey", EmbeddingApiKey); + } + + internal IEnumerable> FeatureEnvironment() + { + if (Anthropic is { } an) yield return new("Features__Ai__Providers__Anthropic", an ? "true" : "false"); + if (AzureFoundry is { } af) yield return new("Features__Ai__Providers__AzureFoundry", af ? "true" : "false"); + if (AzureOpenAI is { } ao) yield return new("Features__Ai__Providers__AzureOpenAI", ao ? "true" : "false"); + if (OpenAI is { } op) yield return new("Features__Ai__Providers__OpenAI", op ? "true" : "false"); + if (ClaudeCode is { } cc) yield return new("Features__Ai__Clis__ClaudeCode", cc ? "true" : "false"); + if (Copilot is { } co) yield return new("Features__Ai__Clis__Copilot", co ? "true" : "false"); + if (InvitationOnly is { } io) yield return new("Features__Onboarding__InvitationOnly", io ? "true" : "false"); + } + + /// + /// Outbound-email config. Emitted only when configured (ACA rejects empty secrets). The client + /// secret is best supplied out-of-band (Key Vault → Email__ClientSecret) rather than here. + /// + internal IEnumerable> EmailEnvironment() + { + if (EmailEnabled is { } en) yield return new("Email__Enabled", en ? "true" : "false"); + if (!string.IsNullOrEmpty(EmailMailboxAddress)) yield return new("Email__MailboxAddress", EmailMailboxAddress); + if (!string.IsNullOrEmpty(EmailTenantId)) yield return new("Email__TenantId", EmailTenantId); + if (!string.IsNullOrEmpty(EmailClientId)) yield return new("Email__ClientId", EmailClientId); + if (!string.IsNullOrEmpty(EmailClientSecret)) yield return new("Email__ClientSecret", EmailClientSecret); + if (EmailUseManagedIdentity is { } mi) yield return new("Email__UseManagedIdentity", mi ? "true" : "false"); + if (EmailInboundEnabled is { } ib) yield return new("Email__InboundEnabled", ib ? "true" : "false"); + if (!string.IsNullOrEmpty(EmailWebhookBaseUrl)) yield return new("Email__WebhookBaseUrl", EmailWebhookBaseUrl); + if (!string.IsNullOrEmpty(EmailSubscriptionClientState)) yield return new("Email__SubscriptionClientState", EmailSubscriptionClientState); + } + + /// + /// Teams bot config. Emitted only when set; the bot secret is best supplied out-of-band (Key Vault → + /// Teams__AppPassword) rather than here. + /// + internal IEnumerable> TeamsEnvironment() + { + if (TeamsEnabled is { } en) yield return new("Teams__Enabled", en ? "true" : "false"); + if (!string.IsNullOrEmpty(TeamsAppId)) yield return new("Teams__AppId", TeamsAppId); + if (!string.IsNullOrEmpty(TeamsAppPassword)) yield return new("Teams__AppPassword", TeamsAppPassword); + if (!string.IsNullOrEmpty(TeamsTenantId)) yield return new("Teams__TenantId", TeamsTenantId); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Memex.Database.Migration.csproj b/memex/aspire/Memex.Database.Migration/Memex.Database.Migration.csproj index 8cde81025..9a6cdb60c 100644 --- a/memex/aspire/Memex.Database.Migration/Memex.Database.Migration.csproj +++ b/memex/aspire/Memex.Database.Migration/Memex.Database.Migration.csproj @@ -8,11 +8,13 @@ + + diff --git a/memex/aspire/Memex.Database.Migration/Migrations/DocumentationBackfill.cs b/memex/aspire/Memex.Database.Migration/Migrations/DocumentationBackfill.cs new file mode 100644 index 000000000..42c7c3542 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/DocumentationBackfill.cs @@ -0,0 +1,256 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using MeshWeaver.Documentation; +using MeshWeaver.Hosting.PostgreSql; +using MeshWeaver.Markdown; +using MeshWeaver.Mesh; +using Microsoft.Extensions.Logging; +using Npgsql; +using Pgvector; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Mirrors the embedded MeshWeaver documentation into a Postgres doc schema so the docs +/// surface in the main search bar — both full-text (the idx_mn_text_search index over +/// name + description + node_type) and semantic vector search (the idx_mn_embedding HNSW +/// index). Reads/navigation still come from the in-memory EmbeddedResourceStorageAdapter +/// (the named partition rule out-ranks the Postgres wildcard provider), so these rows are a +/// pure search index — content is intentionally NULL. +/// +/// Always-run, NOT a versioned . Docs change every release, +/// so this must refresh on every deploy on both fresh and existing DBs. It runs between Phase 2 +/// (versioned repairs) and Phase 3 (searchable-schemas refresh) so Phase 3 automatically adds +/// doc to public.searchable_schemas. +/// +/// Full replace + incremental embed. Every run upserts the current doc set and +/// prunes rows whose source file no longer ships (doc.mesh_nodes exclusively holds these +/// docs). A per-doc content hash in doc.documentation_index means the (paid) embedding +/// call only fires when a doc's content actually changed — or when an embedding provider becomes +/// available for a row indexed earlier without one, or when the vector dimensions change. +/// +public static class DocumentationBackfill +{ + private const string Schema = "doc"; + private const string Partition = "Doc"; // path prefix on the doc nodes (schema = lowercase) + + public static async Task RunAsync( + NpgsqlDataSource baseDataSource, + PostgreSqlStorageOptions options, + string connectionString, + IEmbeddingProvider? embeddingProvider, + ILogger logger) + { + // 1. Schema + mesh/satellite tables (idempotent), plus our bookkeeping table. + await using (var create = baseDataSource.CreateCommand($"CREATE SCHEMA IF NOT EXISTS \"{Schema}\"")) + await create.ExecuteNonQueryAsync(); + + await using var ds = SchemaHelpers.BuildSchemaDataSource(connectionString, Schema); + var schemaOptions = SchemaHelpers.BuildSchemaOptions(connectionString, Schema, options.VectorDimensions); + + await PostgreSqlSchemaInitializer.InitializeMeshTablesAsync(ds, schemaOptions); + await PostgreSqlSchemaInitializer.CreateSatelliteTablesAsync( + ds, schemaOptions, PartitionDefinition.DefaultSegmentTableMappings().Values); + + await using (var bk = ds.CreateCommand(""" + CREATE TABLE IF NOT EXISTS documentation_index ( + path TEXT PRIMARY KEY, + content_hash TEXT NOT NULL, + embedded BOOLEAN NOT NULL DEFAULT false, + indexed_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """)) + await bk.ExecuteNonQueryAsync(); + + // 2. Load the source docs with adapter-aligned paths. + var docs = DocumentationNodeProvider.LoadIndexableNodes(); + logger.LogInformation("[DocBackfill] {Count} documentation pages found ({Embeddings})", + docs.Count, embeddingProvider != null ? "embeddings ON" : "embeddings OFF — FTS only"); + + var existing = new Dictionary(StringComparer.Ordinal); + await using (var read = ds.CreateCommand("SELECT path, content_hash, embedded FROM documentation_index")) + await using (var rdr = await read.ExecuteReaderAsync()) + while (await rdr.ReadAsync()) + existing[rdr.GetString(0)] = (rdr.GetString(1), rdr.GetBoolean(2)); + + var currentPaths = new HashSet(StringComparer.Ordinal); + int upserted = 0, skipped = 0, embedded = 0; + + foreach (var node in docs) + { + var path = node.Path; + currentPaths.Add(path); + var hash = ComputeHash(node, options.VectorDimensions); + + // Re-index only when content changed — or when this row was indexed without an + // embedding but a provider is now available. + if (existing.TryGetValue(path, out var prev) + && prev.Hash == hash + && (prev.Embedded || embeddingProvider == null)) + { + skipped++; + continue; + } + + float[]? vector = null; + if (embeddingProvider != null) + { + try + { + vector = await embeddingProvider.GenerateEmbeddingAsync(BuildEmbeddingText(node)); + } + catch (Exception ex) + { + // Never abort the migration on an embedding failure — FTS still works. + logger.LogWarning(ex, "[DocBackfill] embedding failed for {Path}", path); + } + } + + await UpsertNodeAsync(ds, node, vector); + await UpsertHashAsync(ds, path, hash, vector != null); + upserted++; + if (vector != null) embedded++; + } + + // 3. Full replace — drop rows whose source file no longer ships. + await using (var prune = ds.CreateCommand("DELETE FROM mesh_nodes WHERE NOT (path = ANY($1))")) + { + prune.Parameters.AddWithValue(currentPaths.ToArray()); + var deleted = await prune.ExecuteNonQueryAsync(); + if (deleted > 0) logger.LogInformation("[DocBackfill] pruned {Deleted} stale doc rows", deleted); + } + await using (var pruneIdx = ds.CreateCommand("DELETE FROM documentation_index WHERE NOT (path = ANY($1))")) + { + pruneIdx.Parameters.AddWithValue(currentPaths.ToArray()); + await pruneIdx.ExecuteNonQueryAsync(); + } + + // 4. Public + Anonymous read so everyone can search the docs. + await SeedAccessAsync(ds, logger); + + logger.LogInformation( + "[DocBackfill] done: {Upserted} upserted ({Embedded} embedded), {Skipped} unchanged, {Total} total", + upserted, embedded, skipped, docs.Count); + } + + private static async Task UpsertNodeAsync(NpgsqlDataSource ds, MeshNode node, float[]? vector) + { + // content is intentionally NULL — docs render from the in-memory embedded partition; + // these rows exist only to feed FTS (name + description) and vector (embedding) search. + await using var cmd = ds.CreateCommand(""" + INSERT INTO mesh_nodes (namespace, id, name, description, node_type, category, icon, + last_modified, version, state, content, embedding, main_node) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NULL, $11, $12) + ON CONFLICT (namespace, id) DO UPDATE SET + name = EXCLUDED.name, + description = EXCLUDED.description, + node_type = EXCLUDED.node_type, + category = EXCLUDED.category, + icon = EXCLUDED.icon, + last_modified = EXCLUDED.last_modified, + version = EXCLUDED.version, + state = EXCLUDED.state, + embedding = EXCLUDED.embedding, + main_node = EXCLUDED.main_node + """); + cmd.Parameters.AddWithValue(node.Namespace ?? ""); + cmd.Parameters.AddWithValue(node.Id); + cmd.Parameters.AddWithValue((object?)node.Name ?? DBNull.Value); + cmd.Parameters.AddWithValue((object?)node.Description ?? DBNull.Value); + cmd.Parameters.AddWithValue((object?)node.NodeType ?? DBNull.Value); + cmd.Parameters.AddWithValue((object?)node.Category ?? DBNull.Value); + cmd.Parameters.AddWithValue((object?)node.Icon ?? DBNull.Value); + cmd.Parameters.AddWithValue(node.LastModified == default ? DateTimeOffset.UtcNow : node.LastModified); + cmd.Parameters.AddWithValue(node.Version <= 0 ? 1L : node.Version); + cmd.Parameters.AddWithValue((short)node.State); // MeshNodeState.Active = 2 + if (vector != null) + cmd.Parameters.AddWithValue(new Vector(vector)); + else + cmd.Parameters.AddWithValue(DBNull.Value); + cmd.Parameters.AddWithValue((object?)node.MainNode ?? DBNull.Value); + await cmd.ExecuteNonQueryAsync(); + } + + private static async Task UpsertHashAsync(NpgsqlDataSource ds, string path, string hash, bool embedded) + { + await using var cmd = ds.CreateCommand(""" + INSERT INTO documentation_index (path, content_hash, embedded, indexed_at) + VALUES ($1, $2, $3, NOW()) + ON CONFLICT (path) DO UPDATE SET + content_hash = EXCLUDED.content_hash, + embedded = EXCLUDED.embedded, + indexed_at = NOW() + """); + cmd.Parameters.AddWithValue(path); + cmd.Parameters.AddWithValue(hash); + cmd.Parameters.AddWithValue(embedded); + await cmd.ExecuteNonQueryAsync(); + } + + /// + /// Seeds Public + Anonymous Viewer assignments into doc.access and rebuilds the + /// schema's effective-permissions, which syncs public.partition_access (the partition + /// gate the cross-schema search uses). camelCase content keys match what + /// rebuild_user_effective_permissions() reads (content->>'accessObject'). + /// + private static async Task SeedAccessAsync(NpgsqlDataSource ds, ILogger logger) + { + await using (var cmd = ds.CreateCommand(""" + INSERT INTO access (id, namespace, name, node_type, content, main_node, last_modified, version, state) + VALUES + ('Public_Access', 'Doc/_Access', 'Public Access', 'AccessAssignment', + jsonb_build_object('accessObject', 'Public', 'displayName', 'All authenticated users', + 'roles', jsonb_build_array(jsonb_build_object('role', 'Viewer'))), + 'Doc', NOW(), 1, 2), + ('Anonymous_Access', 'Doc/_Access', 'Anonymous Access', 'AccessAssignment', + jsonb_build_object('accessObject', 'Anonymous', 'displayName', 'Unauthenticated visitors', + 'roles', jsonb_build_array(jsonb_build_object('role', 'Viewer'))), + 'Doc', NOW(), 1, 2) + ON CONFLICT (namespace, id) DO UPDATE SET + content = EXCLUDED.content, + main_node = EXCLUDED.main_node, + state = EXCLUDED.state + """)) + await cmd.ExecuteNonQueryAsync(); + + try + { + await using var rebuild = ds.CreateCommand("SELECT rebuild_user_effective_permissions()"); + await rebuild.ExecuteNonQueryAsync(); + } + catch (Exception ex) + { + logger.LogWarning(ex, "[DocBackfill] doc permission rebuild failed — docs may not be searchable yet"); + } + } + + /// Title + category + description + a prose slice of the body — drives semantic search. + private static string BuildEmbeddingText(MeshNode node) + { + var parts = new List(); + if (!string.IsNullOrWhiteSpace(node.Name)) parts.Add(node.Name!); + if (!string.IsNullOrWhiteSpace(node.Category)) parts.Add(node.Category!); + if (!string.IsNullOrWhiteSpace(node.Description)) parts.Add(node.Description!); + if (node.Content is MarkdownContent mc && !string.IsNullOrWhiteSpace(mc.Content)) + parts.Add(StripForEmbedding(mc.Content)); + var text = string.Join("\n", parts); + return text.Length > 1800 ? text[..1800] : text; + } + + private static string StripForEmbedding(string body) + { + var noFences = Regex.Replace(body, "```.*?```", " ", RegexOptions.Singleline); + var noHtml = Regex.Replace(noFences, "<[^>]+>", " "); + var collapsed = Regex.Replace(noHtml, @"\s+", " ").Trim(); + return collapsed.Length > 1500 ? collapsed[..1500] : collapsed; + } + + private static string ComputeHash(MeshNode node, int dims) + { + var body = node.Content is MarkdownContent mc ? mc.Content : ""; + var raw = string.Join("", + node.Name, node.Description, node.Category, node.Icon, node.NodeType, body, dims.ToString()); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(raw))); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/IMigration.cs b/memex/aspire/Memex.Database.Migration/Migrations/IMigration.cs new file mode 100644 index 000000000..650d4a3d8 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/IMigration.cs @@ -0,0 +1,17 @@ +namespace Memex.Database.Migration.Migrations; + +/// +/// A single versioned data-repair migration. Schema changes belong in +/// PostgreSqlSchemaInitializer (idempotent, always runs); this interface +/// is reserved for one-shot fixes to data written incorrectly by prior code versions. +/// +public interface IMigration +{ + /// Monotonically increasing version. Gaps are allowed (e.g., v12 was retired). + int Version { get; } + + /// One-line description used for log lines. + string Description { get; } + + Task RunAsync(MigrationContext ctx); +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/MigrationContext.cs b/memex/aspire/Memex.Database.Migration/Migrations/MigrationContext.cs new file mode 100644 index 000000000..467cf83e8 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/MigrationContext.cs @@ -0,0 +1,17 @@ +using MeshWeaver.Hosting.PostgreSql; +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Shared state passed to every migration. is set by the runner +/// after schema initialization — fresh DBs already have the latest schema and skip all +/// data repairs. +/// +public sealed record MigrationContext( + NpgsqlDataSource DataSource, + string ConnectionString, + PostgreSqlStorageOptions Options, + ILogger Logger, + bool IsFreshDb); diff --git a/memex/aspire/Memex.Database.Migration/Migrations/MigrationRunner.cs b/memex/aspire/Memex.Database.Migration/Migrations/MigrationRunner.cs new file mode 100644 index 000000000..c3aa99c01 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/MigrationRunner.cs @@ -0,0 +1,93 @@ +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Runs the registered s in version order, gating on the +/// db_version recorded in admin.mesh_nodes. +/// +/// Fresh-DB rule: if is true, ALL data +/// repairs are skipped and the version is fast-forwarded to . +/// Schema initialization (which always runs) has already brought the new DB to the +/// latest schema; there is no legacy data to repair. +/// +public sealed class MigrationRunner +{ + private readonly IReadOnlyList _migrations; + + public MigrationRunner(IEnumerable migrations) + { + _migrations = migrations.OrderBy(m => m.Version).ToList(); + } + + public int LatestVersion => _migrations.Count == 0 ? 0 : _migrations[^1].Version; + + public async Task RunAsync(MigrationContext ctx) + { + var currentVersion = await ReadCurrentVersionAsync(ctx.DataSource); + ctx.Logger.LogInformation("Current DB version: {Version}", currentVersion); + + if (ctx.IsFreshDb) + { + ctx.Logger.LogInformation( + "Fresh database detected — skipping all data repairs and fast-forwarding version {Current} → {Target}.", + currentVersion, LatestVersion); + currentVersion = LatestVersion; + } + else + { + foreach (var migration in _migrations) + { + if (migration.Version <= currentVersion) continue; + + ctx.Logger.LogInformation("Running repair v{Version}: {Description}", + migration.Version, migration.Description); + await migration.RunAsync(ctx); + currentVersion = migration.Version; + ctx.Logger.LogInformation("Repair v{Version} completed.", migration.Version); + } + } + + await SaveVersionAsync(ctx.DataSource, currentVersion); + return currentVersion; + } + + private static async Task ReadCurrentVersionAsync(NpgsqlDataSource dataSource) + { + try + { + await using var cmd = dataSource.CreateCommand(""" + SELECT (content->>'Version')::int FROM admin.mesh_nodes + WHERE id = 'db_version' AND namespace = '' LIMIT 1 + """); + var result = await cmd.ExecuteScalarAsync(); + return result switch + { + int v => v, + long l => (int)l, + _ => 0 + }; + } + catch + { + // Table may not exist yet — version = 0 (fresh DB) + return 0; + } + } + + private static async Task SaveVersionAsync(NpgsqlDataSource dataSource, int version) + { + await using var cmd = dataSource.CreateCommand(""" + INSERT INTO admin.mesh_nodes (namespace, id, name, node_type, state, content, last_modified, main_node) + VALUES ('', 'db_version', 'Database Version', 'Settings', 2, + jsonb_build_object('Version', @version, 'LastMigration', now()::text), + now(), 'db_version') + ON CONFLICT (namespace, id) DO UPDATE SET + content = jsonb_build_object('Version', @version, 'LastMigration', now()::text), + last_modified = now() + """); + cmd.Parameters.AddWithValue("@version", version); + await cmd.ExecuteNonQueryAsync(); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/OrleansClusteringSetup.cs b/memex/aspire/Memex.Database.Migration/Migrations/OrleansClusteringSetup.cs new file mode 100644 index 000000000..1ddbc2a27 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/OrleansClusteringSetup.cs @@ -0,0 +1,437 @@ +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Creates the Orleans cluster-membership tables in the dedicated orleans database +/// (same Postgres server, separate DB) so the portal silo can use Postgres-backed AdoNet +/// clustering instead of Localhost. The Aspire AppHost declares the orleans +/// database and injects its connection string (ConnectionStrings:orleans); this step +/// runs the official Orleans 10 PostgreSQL membership scripts (Shared/PostgreSQL-Main.sql +/// + Orleans.Clustering.AdoNet/PostgreSQL-Clustering.sql) verbatim. +/// +/// Idempotent: the scripts use plain CREATE (no IF NOT EXISTS), so we +/// gate on whether the orleansquery table already exists and only run them once. The +/// Orleans provider does NOT auto-create these tables, so this is required before the silo starts. +/// +/// Skipped when no orleans connection string is configured (e.g. an Azure-Tables / +/// Localhost deployment that doesn't use Postgres clustering). +/// +public static class OrleansClusteringSetup +{ + public static async Task RunAsync(string orleansConnectionString, ILogger logger) + { + if (string.IsNullOrWhiteSpace(orleansConnectionString)) + { + logger.LogInformation("[OrleansClustering] No 'orleans' connection string — skipping (non-AdoNet clustering)."); + return; + } + + await EnsureDatabaseExistsAsync(orleansConnectionString, logger); + + await using var conn = new NpgsqlConnection(orleansConnectionString); + await conn.OpenAsync(); + + await using (var check = new NpgsqlCommand( + "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'orleansquery')", conn)) + { + if ((bool)(await check.ExecuteScalarAsync())!) + { + logger.LogInformation("[OrleansClustering] Membership tables already present — nothing to do."); + return; + } + } + + logger.LogInformation("[OrleansClustering] Creating Orleans membership tables in the 'orleans' database."); + await using (var cmd = new NpgsqlCommand(MembershipScript, conn)) + await cmd.ExecuteNonQueryAsync(); + logger.LogInformation("[OrleansClustering] Orleans membership tables created."); + } + + /// + /// Creates the target database if it does not yet exist (self-managed Postgres — Compose/Helm + /// pgvector container). Azure-managed Postgres pre-creates databases declared in the AppHost, + /// so we skip the maintenance-connection path there (the app identity typically can't CREATE + /// DATABASE on Flexible Server anyway). + /// + private static async Task EnsureDatabaseExistsAsync(string connectionString, ILogger logger) + { + if (connectionString.Contains("database.azure.com", StringComparison.OrdinalIgnoreCase)) + return; + + var targetDb = new NpgsqlConnectionStringBuilder(connectionString).Database ?? "orleans"; + var maintenanceCs = new NpgsqlConnectionStringBuilder(connectionString) { Database = "postgres" }.ConnectionString; + + await using var admin = new NpgsqlConnection(maintenanceCs); + await admin.OpenAsync(); + await using var check = new NpgsqlCommand("SELECT 1 FROM pg_database WHERE datname = @db", admin); + check.Parameters.AddWithValue("db", targetDb); + if (await check.ExecuteScalarAsync() is null) + { + logger.LogInformation("[OrleansClustering] Database '{Db}' does not exist — creating it.", targetDb); + await using var create = new NpgsqlCommand($"CREATE DATABASE \"{targetDb.Replace("\"", "\"\"")}\"", admin); + await create.ExecuteNonQueryAsync(); + } + } + + // Verbatim Orleans 10 PostgreSQL clustering scripts: Shared/PostgreSQL-Main.sql (OrleansQuery) + // followed by Orleans.Clustering.AdoNet/PostgreSQL-Clustering.sql (membership tables + queries). + // Do not edit — keep in sync with the Microsoft.Orleans.Clustering.AdoNet package version. + private const string MembershipScript = @" +CREATE TABLE OrleansQuery +( + QueryKey varchar(64) NOT NULL, + QueryText varchar(8000) NOT NULL, + + CONSTRAINT OrleansQuery_Key PRIMARY KEY(QueryKey) +); + +-- For each deployment, there will be only one (active) membership version table version column which will be updated periodically. +CREATE TABLE OrleansMembershipVersionTable +( + DeploymentId varchar(150) NOT NULL, + Timestamp timestamptz(3) NOT NULL DEFAULT now(), + Version integer NOT NULL DEFAULT 0, + + CONSTRAINT PK_OrleansMembershipVersionTable_DeploymentId PRIMARY KEY(DeploymentId) +); + +-- Every silo instance has a row in the membership table. +CREATE TABLE OrleansMembershipTable +( + DeploymentId varchar(150) NOT NULL, + Address varchar(45) NOT NULL, + Port integer NOT NULL, + Generation integer NOT NULL, + SiloName varchar(150) NOT NULL, + HostName varchar(150) NOT NULL, + Status integer NOT NULL, + ProxyPort integer NULL, + SuspectTimes varchar(8000) NULL, + StartTime timestamptz(3) NOT NULL, + IAmAliveTime timestamptz(3) NOT NULL, + + CONSTRAINT PK_MembershipTable_DeploymentId PRIMARY KEY(DeploymentId, Address, Port, Generation), + CONSTRAINT FK_MembershipTable_MembershipVersionTable_DeploymentId FOREIGN KEY (DeploymentId) REFERENCES OrleansMembershipVersionTable (DeploymentId) +); + +CREATE FUNCTION update_i_am_alive_time( + deployment_id OrleansMembershipTable.DeploymentId%TYPE, + address_arg OrleansMembershipTable.Address%TYPE, + port_arg OrleansMembershipTable.Port%TYPE, + generation_arg OrleansMembershipTable.Generation%TYPE, + i_am_alive_time OrleansMembershipTable.IAmAliveTime%TYPE) + RETURNS void AS +$func$ +BEGIN + -- This is expected to never fail by Orleans, so return value + -- is not needed nor is it checked. + UPDATE OrleansMembershipTable as d + SET + IAmAliveTime = i_am_alive_time + WHERE + d.DeploymentId = deployment_id AND deployment_id IS NOT NULL + AND d.Address = address_arg AND address_arg IS NOT NULL + AND d.Port = port_arg AND port_arg IS NOT NULL + AND d.Generation = generation_arg AND generation_arg IS NOT NULL; +END +$func$ LANGUAGE plpgsql; + +INSERT INTO OrleansQuery(QueryKey, QueryText) +VALUES +( + 'UpdateIAmAlivetimeKey',' + -- This is expected to never fail by Orleans, so return value + -- is not needed nor is it checked. + SELECT * from update_i_am_alive_time( + @DeploymentId, + @Address, + @Port, + @Generation, + @IAmAliveTime + ); +'); + +CREATE FUNCTION insert_membership_version( + DeploymentIdArg OrleansMembershipTable.DeploymentId%TYPE +) + RETURNS TABLE(row_count integer) AS +$func$ +DECLARE + RowCountVar int := 0; +BEGIN + + BEGIN + + INSERT INTO OrleansMembershipVersionTable + ( + DeploymentId + ) + SELECT DeploymentIdArg + ON CONFLICT (DeploymentId) DO NOTHING; + + GET DIAGNOSTICS RowCountVar = ROW_COUNT; + + ASSERT RowCountVar <> 0, 'no rows affected, rollback'; + + RETURN QUERY SELECT RowCountVar; + EXCEPTION + WHEN assert_failure THEN + RETURN QUERY SELECT RowCountVar; + END; + +END +$func$ LANGUAGE plpgsql; + +INSERT INTO OrleansQuery(QueryKey, QueryText) +VALUES +( + 'InsertMembershipVersionKey',' + SELECT * FROM insert_membership_version( + @DeploymentId + ); +'); + +CREATE FUNCTION insert_membership( + DeploymentIdArg OrleansMembershipTable.DeploymentId%TYPE, + AddressArg OrleansMembershipTable.Address%TYPE, + PortArg OrleansMembershipTable.Port%TYPE, + GenerationArg OrleansMembershipTable.Generation%TYPE, + SiloNameArg OrleansMembershipTable.SiloName%TYPE, + HostNameArg OrleansMembershipTable.HostName%TYPE, + StatusArg OrleansMembershipTable.Status%TYPE, + ProxyPortArg OrleansMembershipTable.ProxyPort%TYPE, + StartTimeArg OrleansMembershipTable.StartTime%TYPE, + IAmAliveTimeArg OrleansMembershipTable.IAmAliveTime%TYPE, + VersionArg OrleansMembershipVersionTable.Version%TYPE) + RETURNS TABLE(row_count integer) AS +$func$ +DECLARE + RowCountVar int := 0; +BEGIN + + BEGIN + INSERT INTO OrleansMembershipTable + ( + DeploymentId, + Address, + Port, + Generation, + SiloName, + HostName, + Status, + ProxyPort, + StartTime, + IAmAliveTime + ) + SELECT + DeploymentIdArg, + AddressArg, + PortArg, + GenerationArg, + SiloNameArg, + HostNameArg, + StatusArg, + ProxyPortArg, + StartTimeArg, + IAmAliveTimeArg + ON CONFLICT (DeploymentId, Address, Port, Generation) DO + NOTHING; + + + GET DIAGNOSTICS RowCountVar = ROW_COUNT; + + UPDATE OrleansMembershipVersionTable + SET + Timestamp = now(), + Version = Version + 1 + WHERE + DeploymentId = DeploymentIdArg AND DeploymentIdArg IS NOT NULL + AND Version = VersionArg AND VersionArg IS NOT NULL + AND RowCountVar > 0; + + GET DIAGNOSTICS RowCountVar = ROW_COUNT; + + ASSERT RowCountVar <> 0, 'no rows affected, rollback'; + + + RETURN QUERY SELECT RowCountVar; + EXCEPTION + WHEN assert_failure THEN + RETURN QUERY SELECT RowCountVar; + END; + +END +$func$ LANGUAGE plpgsql; + +INSERT INTO OrleansQuery(QueryKey, QueryText) +VALUES +( + 'InsertMembershipKey',' + SELECT * FROM insert_membership( + @DeploymentId, + @Address, + @Port, + @Generation, + @SiloName, + @HostName, + @Status, + @ProxyPort, + @StartTime, + @IAmAliveTime, + @Version + ); +'); + +CREATE FUNCTION update_membership( + DeploymentIdArg OrleansMembershipTable.DeploymentId%TYPE, + AddressArg OrleansMembershipTable.Address%TYPE, + PortArg OrleansMembershipTable.Port%TYPE, + GenerationArg OrleansMembershipTable.Generation%TYPE, + StatusArg OrleansMembershipTable.Status%TYPE, + SuspectTimesArg OrleansMembershipTable.SuspectTimes%TYPE, + IAmAliveTimeArg OrleansMembershipTable.IAmAliveTime%TYPE, + VersionArg OrleansMembershipVersionTable.Version%TYPE + ) + RETURNS TABLE(row_count integer) AS +$func$ +DECLARE + RowCountVar int := 0; +BEGIN + + BEGIN + + UPDATE OrleansMembershipVersionTable + SET + Timestamp = now(), + Version = Version + 1 + WHERE + DeploymentId = DeploymentIdArg AND DeploymentIdArg IS NOT NULL + AND Version = VersionArg AND VersionArg IS NOT NULL; + + + GET DIAGNOSTICS RowCountVar = ROW_COUNT; + + UPDATE OrleansMembershipTable + SET + Status = StatusArg, + SuspectTimes = SuspectTimesArg, + IAmAliveTime = IAmAliveTimeArg + WHERE + DeploymentId = DeploymentIdArg AND DeploymentIdArg IS NOT NULL + AND Address = AddressArg AND AddressArg IS NOT NULL + AND Port = PortArg AND PortArg IS NOT NULL + AND Generation = GenerationArg AND GenerationArg IS NOT NULL + AND RowCountVar > 0; + + + GET DIAGNOSTICS RowCountVar = ROW_COUNT; + + ASSERT RowCountVar <> 0, 'no rows affected, rollback'; + + + RETURN QUERY SELECT RowCountVar; + EXCEPTION + WHEN assert_failure THEN + RETURN QUERY SELECT RowCountVar; + END; + +END +$func$ LANGUAGE plpgsql; + +INSERT INTO OrleansQuery(QueryKey, QueryText) +VALUES +( + 'UpdateMembershipKey',' + SELECT * FROM update_membership( + @DeploymentId, + @Address, + @Port, + @Generation, + @Status, + @SuspectTimes, + @IAmAliveTime, + @Version + ); +'); + +INSERT INTO OrleansQuery(QueryKey, QueryText) +VALUES +( + 'MembershipReadRowKey',' + SELECT + v.DeploymentId, + m.Address, + m.Port, + m.Generation, + m.SiloName, + m.HostName, + m.Status, + m.ProxyPort, + m.SuspectTimes, + m.StartTime, + m.IAmAliveTime, + v.Version + FROM + OrleansMembershipVersionTable v + -- This ensures the version table will returned even if there is no matching membership row. + LEFT OUTER JOIN OrleansMembershipTable m ON v.DeploymentId = m.DeploymentId + AND Address = @Address AND @Address IS NOT NULL + AND Port = @Port AND @Port IS NOT NULL + AND Generation = @Generation AND @Generation IS NOT NULL + WHERE + v.DeploymentId = @DeploymentId AND @DeploymentId IS NOT NULL; +'); + +INSERT INTO OrleansQuery(QueryKey, QueryText) +VALUES +( + 'MembershipReadAllKey',' + SELECT + v.DeploymentId, + m.Address, + m.Port, + m.Generation, + m.SiloName, + m.HostName, + m.Status, + m.ProxyPort, + m.SuspectTimes, + m.StartTime, + m.IAmAliveTime, + v.Version + FROM + OrleansMembershipVersionTable v LEFT OUTER JOIN OrleansMembershipTable m + ON v.DeploymentId = m.DeploymentId + WHERE + v.DeploymentId = @DeploymentId AND @DeploymentId IS NOT NULL; +'); + +INSERT INTO OrleansQuery(QueryKey, QueryText) +VALUES +( + 'DeleteMembershipTableEntriesKey',' + DELETE FROM OrleansMembershipTable + WHERE DeploymentId = @DeploymentId AND @DeploymentId IS NOT NULL; + DELETE FROM OrleansMembershipVersionTable + WHERE DeploymentId = @DeploymentId AND @DeploymentId IS NOT NULL; +'); + +INSERT INTO OrleansQuery(QueryKey, QueryText) +VALUES +( + 'GatewaysQueryKey',' + SELECT + Address, + ProxyPort, + Generation + FROM + OrleansMembershipTable + WHERE + DeploymentId = @DeploymentId AND @DeploymentId IS NOT NULL + AND Status = @Status AND @Status IS NOT NULL + AND ProxyPort > 0; +'); +"; +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/SchemaHelpers.cs b/memex/aspire/Memex.Database.Migration/Migrations/SchemaHelpers.cs new file mode 100644 index 000000000..b01ca0668 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/SchemaHelpers.cs @@ -0,0 +1,242 @@ +using System.Text; +using System.Text.Json; +using Azure.Core; +using Azure.Identity; +using MeshWeaver.Hosting.PostgreSql; +using Npgsql; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Shared helpers used by multiple migrations: partition-schema discovery, name sanitisation, +/// and per-schema data-source bootstrapping. +/// +internal static class SchemaHelpers +{ + /// Discover schemas that look like content partitions (have a mesh_nodes table). + public static async Task> DiscoverPartitionSchemasAsync(NpgsqlDataSource dataSource) + => await DiscoverSchemasAsync(dataSource, requireTable: "mesh_nodes"); + + /// Discover schemas that have an access table — used by access-related repairs. + public static async Task> DiscoverAccessSchemasAsync(NpgsqlDataSource dataSource) + => await DiscoverSchemasAsync(dataSource, requireTable: "access"); + + private static async Task> DiscoverSchemasAsync(NpgsqlDataSource dataSource, string requireTable) + { + var schemas = new List(); + await using var listCmd = dataSource.CreateCommand($""" + SELECT schema_name FROM information_schema.schemata s + WHERE EXISTS (SELECT 1 FROM information_schema.tables t WHERE t.table_schema = s.schema_name AND t.table_name = '{requireTable}') + AND s.schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast') + AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' + ORDER BY s.schema_name + """); + await using var rdr = await listCmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) schemas.Add(rdr.GetString(0)); + return schemas; + } + + /// + /// Sanitises an arbitrary identifier (e.g., a userId) to a Postgres schema name — + /// must match PostgreSqlPartitionedStoreFactory.SanitizeSchemaName: lowercase, + /// non-alphanumeric → '_', leading digit prefixed with '_'. + /// + public static string SanitizeSchemaName(string s) + { + var lower = s.ToLowerInvariant(); + var sb = new StringBuilder(); + foreach (var ch in lower) + sb.Append(char.IsLetterOrDigit(ch) ? ch : '_'); + var result = sb.ToString(); + if (result.Length > 0 && char.IsDigit(result[0])) result = "_" + result; + return result; + } + + /// + /// Build a per-schema NpgsqlDataSource with SearchPath = "{schema},public", + /// pgvector enabled, and Azure AD password-provider wired when the connection + /// string targets an Azure-managed Postgres (host ends in + /// .postgres.database.azure.com AND password is empty). + /// + /// Without the AAD provider, per-schema migrations on prod/test fail + /// with 28000: no pg_hba.conf entry for host … user "app" because the + /// raw NpgsqlDataSourceBuilder.Build() attempts password auth with an + /// empty password against an SSL-required, AAD-only server. Aspire's + /// AddAzureNpgsqlDataSource wires this token provider on the *main* + /// runner connection — but every per-schema datasource we spin up here + /// needs the same hook or it falls back to anonymous password auth and dies. + /// Local Docker postgres (mode=local) uses username+password and is left + /// untouched. + /// + public static NpgsqlDataSource BuildSchemaDataSource(string baseConnectionString, string schema, bool useVector = true) + { + var csb = new NpgsqlConnectionStringBuilder(baseConnectionString) { SearchPath = $"{schema},public" }; + + var isAzure = csb.Host?.EndsWith(".postgres.database.azure.com", StringComparison.OrdinalIgnoreCase) == true; + var hasPassword = !string.IsNullOrEmpty(csb.Password); + + // Aspire's AddAzureNpgsqlDataSource enforces SSL via the *builder*, not the conn string. + // When we clone the conn string here, that enforcement is lost — the server then rejects + // us with `28000: no pg_hba.conf entry for host …, no encryption`. Azure Flexible Server + // requires SSL unconditionally, so force it on every per-schema datasource we build. + if (isAzure) + csb.SslMode = SslMode.Require; + + var dsb = new NpgsqlDataSourceBuilder(csb.ConnectionString); + if (useVector) dsb.UseVector(); + + if (isAzure) + { + // Mirror Aspire's exact AAD wiring (release/13.2 src/Components/Common/ + // ManagedIdentityTokenCredentialHelpers.cs::ConfigureEntraIdAuthentication). + // Two pieces are required, and the per-schema datasource has historically + // missed both: + // + // 1. Username derivation. The conn string Aspire injects has NO Username — + // Aspire fills it at runtime from the access token's `xms_mirid` claim + // (last segment after `userAssignedIdentities/` → e.g. + // "db_migration_identity"). Without this, Npgsql falls back to + // Environment.UserName which is "app" in mcr.microsoft.com/dotnet/aspnet:10.0, + // and Postgres rejects with 28P01: password authentication failed for user "app". + // 2. Per-connection UsePasswordProvider with the ossrdbms scope. Azure.Identity + // caches tokens internally, so no need for UsePeriodicPasswordProvider. + if (string.IsNullOrEmpty(csb.Username)) + { + // Management scope first — its tokens carry user-name claims for both UAMIs + // and SPs (Aspire does the same). + var mgmtToken = AzureCredential.GetToken(s_managementTokenRequestContext, default); + if (TryGetUsernameFromToken(mgmtToken.Token, out var username) || + TryGetUsernameFromToken( + AzureCredential.GetToken(s_databaseForPostgresSqlTokenRequestContext, default).Token, + out username)) + { + csb.Username = username; + } + // If neither token carries a username, leave Username unset and let Npgsql + // surface the misconfiguration on connect (Aspire does the same). + } + + if (!hasPassword) + { + dsb = new NpgsqlDataSourceBuilder(csb.ConnectionString); + if (useVector) dsb.UseVector(); + + dsb.UsePasswordProvider( + _ => AzureCredential.GetToken(s_databaseForPostgresSqlTokenRequestContext, default).Token, + async (_, ct) => (await AzureCredential.GetTokenAsync(s_databaseForPostgresSqlTokenRequestContext, ct).ConfigureAwait(false)).Token); + } + } + + return dsb.Build(); + } + + private static readonly TokenRequestContext s_databaseForPostgresSqlTokenRequestContext = + new(["https://ossrdbms-aad.database.windows.net/.default"]); + private static readonly TokenRequestContext s_managementTokenRequestContext = + new(["https://management.azure.com/.default"]); + + // Verbatim port of Aspire's TryGetUsernameFromToken / + // ParsePrincipalName / AddBase64Padding from release/13.2 + // src/Components/Common/ManagedIdentityTokenCredentialHelpers.cs. + private static bool TryGetUsernameFromToken(string jwtToken, out string? username) + { + username = null; + try + { + var tokenParts = jwtToken.Split('.'); + if (tokenParts.Length != 3) return false; + + var payload = AddBase64Padding(tokenParts[1]); + var decodedBytes = Convert.FromBase64String(payload); + var reader = new Utf8JsonReader(decodedBytes); + var payloadJson = JsonElement.ParseValue(ref reader); + + if (payloadJson.TryGetProperty("xms_mirid", out var mirid) && + mirid.GetString() is string miridString && + ParsePrincipalName(miridString) is string principalName) + { + username = principalName; + } + else if (payloadJson.TryGetProperty("upn", out var upn)) + username = upn.GetString(); + else if (payloadJson.TryGetProperty("preferred_username", out var preferred)) + username = preferred.GetString(); + else if (payloadJson.TryGetProperty("unique_name", out var unique)) + username = unique.GetString(); + + return username != null; + } + catch + { + return false; + } + } + + private static string? ParsePrincipalName(string xmsMirid) + { + var lastSlash = xmsMirid.LastIndexOf('/'); + if (lastSlash == -1) return null; + + var beginning = xmsMirid.AsSpan(0, lastSlash); + var principalName = xmsMirid.AsSpan(lastSlash + 1); + + if (principalName.IsEmpty || + !beginning.EndsWith("providers/Microsoft.ManagedIdentity/userAssignedIdentities", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + return principalName.ToString(); + } + + private static string AddBase64Padding(string base64) => (base64.Length % 4) switch + { + 2 => base64 + "==", + 3 => base64 + "=", + _ => base64, + }; + + /// + /// Shared reused across per-schema + /// datasources — avoids re-authenticating for every schema while a migration + /// walks dozens of partitions (Azure.Identity caches the access token internally). + /// + /// This is the *exact* construction Aspire uses for AAD-on-Postgres + /// (see src/Shared/AzureCredentialHelper.cs in dotnet/aspire): + /// a ManagedIdentityCredential pinned to the UAMI whose client id is + /// in AZURE_CLIENT_ID. Bare DefaultAzureCredential is wrong in + /// a multi-UAMI Container App because the chain runs EnvironmentCredential / + /// WorkloadIdentityCredential first and IMDS returns 400 + /// multiple_matching_tokens when more than one UAMI is attached and + /// no client id is specified — typical symptom is + /// 28P01: password authentication failed for user "app". + /// + private static readonly ManagedIdentityCredential AzureCredential = + new(new ManagedIdentityCredentialOptions( + ManagedIdentityId.FromUserAssignedClientId( + Environment.GetEnvironmentVariable("AZURE_CLIENT_ID") + ?? throw new InvalidOperationException( + "AZURE_CLIENT_ID env var must be set on the Container App so the per-schema " + + "datasource can pin to the intended User-Assigned Managed Identity. " + + "Aspire AppHost should set this automatically when WithRoleAssignments is wired.")))); + + /// Build the per-schema PostgreSqlStorageOptions for a partition migration. + public static PostgreSqlStorageOptions BuildSchemaOptions(string baseConnectionString, string schema, int vectorDimensions) + { + var csb = new NpgsqlConnectionStringBuilder(baseConnectionString) { SearchPath = $"{schema},public" }; + return new PostgreSqlStorageOptions + { + ConnectionString = csb.ConnectionString, + VectorDimensions = vectorDimensions, + Schema = schema + }; + } + + /// Does a Postgres schema with this name exist? + public static async Task SchemaExistsAsync(NpgsqlDataSource dataSource, string schemaName) + { + await using var cmd = dataSource.CreateCommand( + "SELECT EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = $1)"); + cmd.Parameters.AddWithValue(schemaName); + return (bool)(await cmd.ExecuteScalarAsync())!; + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/SchemaInitialization.cs b/memex/aspire/Memex.Database.Migration/Migrations/SchemaInitialization.cs new file mode 100644 index 000000000..8d06388e0 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/SchemaInitialization.cs @@ -0,0 +1,130 @@ +using MeshWeaver.Hosting.PostgreSql; +using MeshWeaver.Mesh; +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Idempotent DB setup that ALWAYS runs (regardless of fresh vs. existing DB): +/// public-schema tables/indexes/triggers, satellite tables, partition_access stored proc, +/// the admin schema with mesh_nodes for version tracking, and the apitoken token-validation +/// index schema (must exist before any token is minted — the router no longer lazy-creates it). +/// +/// New DBs get everything correct from the start. Existing DBs get updated trigger functions +/// and any newly-added objects. Reports whether the DB was fresh by detecting whether any +/// content-partition schemas existed before this run. +/// +public static class SchemaInitialization +{ + public sealed record Result(bool IsFreshDb); + + public static async Task RunAsync( + NpgsqlDataSource dataSource, + PostgreSqlStorageOptions options, + string connectionString, + ILogger logger) + { + // Azure-only: grant CREATE on database to azure_pg_admin so managed identities + // (portal, migration) can create per-organization schemas at runtime. + if (connectionString.Contains("database.azure.com")) + { + var dbName = new NpgsqlConnectionStringBuilder(connectionString).Database; + await using var grantCmd = dataSource.CreateCommand( + $"GRANT CREATE ON DATABASE \"{dbName}\" TO azure_pg_admin"); + await grantCmd.ExecuteNonQueryAsync(); + logger.LogInformation("Granted CREATE ON DATABASE to azure_pg_admin."); + } + + // Public schema: tables, indexes, triggers + satellite tables + partition_access proc. + await PostgreSqlSchemaInitializer.InitializeAsync(dataSource, options); + + var satelliteTableNames = PartitionDefinition.DefaultSegmentTableMappings().Values; + await PostgreSqlSchemaInitializer.CreateSatelliteTablesAsync( + dataSource, options, satelliteTableNames); + + await PostgreSqlSchemaInitializer.InitializePartitionAccessTableAsync(dataSource); + + // Admin schema for version tracking + global catalogs (agents/models/roles). + // The admin partition is an unversioned mesh_nodes table just like any content + // partition, and MUST be created in the `admin` schema — MigrationRunner.SaveVersionAsync + // (Phase 2) writes db_version into admin.mesh_nodes, so the table has to exist first. + // + // The table-creation DDL uses an UNQUALIFIED `CREATE TABLE mesh_nodes`, which resolves + // against search_path. The default `dataSource` has search_path=public, so passing it + // here only ever (no-op) re-touched public.mesh_nodes and never created admin.mesh_nodes + // — invisible on a long-lived prod DB where admin.mesh_nodes already exists, but on a + // FRESH DB (new self-managed Compose/Helm deploy, brand-new Azure customer) SaveVersionAsync + // then hit `42P01: relation "admin.mesh_nodes" does not exist`. Build an admin-scoped data + // source (search_path=admin,public) exactly as the per-schema repairs (V02/V07/V13) and the + // runtime PostgreSqlPartitionStorageProvider do, so the unqualified DDL lands in admin. + await using (var ensureAdmin = dataSource.CreateCommand("CREATE SCHEMA IF NOT EXISTS admin")) + await ensureAdmin.ExecuteNonQueryAsync(); + + await using var adminDataSource = SchemaHelpers.BuildSchemaDataSource(connectionString, "admin"); + var adminOptions = SchemaHelpers.BuildSchemaOptions(connectionString, "admin", options.VectorDimensions); + await PostgreSqlSchemaInitializer.InitializeMeshTablesAsync(adminDataSource, adminOptions); + + // ApiToken validation-index schema. ApiTokenService writes the global ApiToken/{hashPrefix} + // index node (ApiTokenIndex → the user-scoped token node) into the `apitoken` schema, and + // token validation reads it back by exact path on every bearer request. `ApiToken` is not + // an OwnsPartition type and the router no longer lazily CREATE-SCHEMAs (0ceba04ce), so the + // schema must be created EXPLICITLY here — otherwise a fresh DB (e.g. atioz) never gets it + // and every freshly-minted token (manual AND OAuth) 401s on the next request. Uses the + // single-source-of-truth per-partition DDL proc installed by InitializeAsync above; the + // boot-time PostgreSqlPartitionSubscriptionHostedService also provisions it from + // DefaultPartitionProvider, but the explicit create here covers the migration container and + // any DB that booted before the partition was declared. Idempotent. + await using (var ensureApiToken = dataSource.CreateCommand("SELECT public.ensure_partition_schema('apitoken')")) + await ensureApiToken.ExecuteNonQueryAsync(); + + // VUser (virtual-user) partition. VirtualUserMiddleware creates a VUser/{id} node for + // every cookie-less request (bots, prefetchers, anonymous visitors); like ApiToken, VUser + // is not an OwnsPartition type and the router never lazily CREATE-SCHEMAs, so without + // this explicit create a fresh DB (e.g. atioz 2026-06-11) has no `vuser` schema and every + // anonymous request fails its VUser create with `42P01: relation "vuser.mesh_nodes" does + // not exist` (made loudly visible by the create fail-closed gate; before that the creates + // were silently acked-and-lost). Same single-source-of-truth DDL proc as Space creation + // uses. Idempotent. + await using (var ensureVUser = dataSource.CreateCommand("SELECT public.ensure_partition_schema('vuser')")) + await ensureVUser.ExecuteNonQueryAsync(); + + // NOTE: the framework schemas `auth` (V27 access-object mirror) and `system_access` + // (global/root-scope grants) are NOT created here. The portal's + // PostgreSqlPartitionSubscriptionHostedService provisions them (and every other + // registered framework partition) at boot, BEFORE any user write — so the mirror + // trigger always has a destination. Creating them here would also make + // DetectFreshDbAsync see `auth.mesh_nodes`/`system_access.mesh_nodes` and wrongly + // classify a fresh DB as non-fresh, running the legacy `user`-schema repair chain + // (V05+, which references the long-gone `user` schema) instead of fast-forwarding. + + // Detect fresh DB: no CONTENT partition schemas (i.e., no schemas with a mesh_nodes + // table) existed before this run. Framework schemas don't count. + var isFreshDb = await DetectFreshDbAsync(dataSource); + + return new Result(isFreshDb); + } + + private static async Task DetectFreshDbAsync(NpgsqlDataSource dataSource) + { + await using var cmd = dataSource.CreateCommand(""" + SELECT count(*) FROM information_schema.schemata s + WHERE EXISTS ( + SELECT 1 FROM information_schema.tables t + WHERE t.table_schema = s.schema_name AND t.table_name = 'mesh_nodes' + ) + -- Framework schemas are NOT content partitions — they must never make a fresh DB + -- look non-fresh (which would run the legacy `user`-schema repair chain instead of + -- fast-forwarding). Only schemas that can exist BEFORE the first migration run + -- belong here: public (root), admin (created by this very function), and the + -- partitions this function provisions explicitly (apitoken, vuser) plus auth + -- (access-object mirror). The DbVersionGate keeps the portal from boot-provisioning + -- anything else ahead of the migration, so the former system_*/portal/kernel/pg_* + -- entries were dead defensiveness and are gone. + AND s.schema_name NOT IN ('public', 'admin', 'auth', 'apitoken', 'vuser') + AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' + """); + var schemaCount = (long)(await cmd.ExecuteScalarAsync())!; + return schemaCount == 0; + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/SearchableSchemasUpdater.cs b/memex/aspire/Memex.Database.Migration/Migrations/SearchableSchemasUpdater.cs new file mode 100644 index 000000000..c7be02c3d --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/SearchableSchemasUpdater.cs @@ -0,0 +1,69 @@ +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Repopulates public.searchable_schemas from the current set of content partitions. +/// Idempotent and runs on every migration — schemas can be added or removed between runs. +/// +public static class SearchableSchemasUpdater +{ + private static readonly HashSet ExcludedSchemas = new(StringComparer.OrdinalIgnoreCase) + { + "admin", "portal", "kernel", + "_access", "_address_", "_graph", "_settings", "_tracking", "_thread", "_source", "_test", + "source", "test", + "login", "markdown", "onboarding", "welcome", "settings", "storage", + "p", "mesh", "thread", "agent", "partition", "organization", "vuser", + "public", "information_schema", "pg_catalog", "pg_toast" + }; + + public static async Task RunAsync(NpgsqlDataSource dataSource, ILogger logger) + { + // Discover content schemas — same logic as + // PostgreSqlPartitionedStoreFactory.DiscoverPartitionsAsync. + var contentSchemas = new List(); + await using (var discoverCmd = dataSource.CreateCommand(""" + SELECT schema_name FROM information_schema.schemata s + WHERE EXISTS (SELECT 1 FROM information_schema.tables t WHERE t.table_schema = s.schema_name AND t.table_name = 'mesh_nodes') + AND s.schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast') + AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' + ORDER BY s.schema_name + """)) + { + await using var rdr = await discoverCmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) + { + var schema = rdr.GetString(0); + if (!ExcludedSchemas.Contains(schema)) + contentSchemas.Add(schema); + } + } + + await using (var clearCmd = dataSource.CreateCommand("DELETE FROM public.searchable_schemas")) + await clearCmd.ExecuteNonQueryAsync(); + + foreach (var schema in contentSchemas) + { + await using var insertCmd = dataSource.CreateCommand( + "INSERT INTO public.searchable_schemas (schema_name) VALUES ($1) ON CONFLICT DO NOTHING"); + insertCmd.Parameters.AddWithValue(schema); + await insertCmd.ExecuteNonQueryAsync(); + } + + // #16: re-materialize public.top_level_index now that searchable_schemas is + // current, so top-level autocomplete reads a populated one-row-per-partition + // matview at deploy time (never a cross-schema fan-out). Guarded so a DB + // mid-upgrade (function not yet created) can't fail the migration. + await using (var rebuildCmd = dataSource.CreateCommand( + "DO $tli$ BEGIN " + + "IF to_regprocedure('public.rebuild_top_level_index()') IS NOT NULL THEN " + + "PERFORM public.rebuild_top_level_index(); END IF; END $tli$;")) + { + await rebuildCmd.ExecuteNonQueryAsync(); + } + + logger.LogInformation("Searchable schemas: [{Schemas}]", string.Join(", ", contentSchemas)); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V01_MoveAccessAssignments.cs b/memex/aspire/Memex.Database.Migration/Migrations/V01_MoveAccessAssignments.cs new file mode 100644 index 000000000..888f2f4a9 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V01_MoveAccessAssignments.cs @@ -0,0 +1,68 @@ +namespace Memex.Database.Migration.Migrations; + +/// +/// Move AccessAssignment nodes from mesh_nodes to access, fix the +/// missing /_Access namespace segment, then rebuild permissions. +/// +/// Bug: AddUserRoleAsync wrote AccessAssignment nodes to mesh_nodes with +/// namespace={scope}/{userId}_Access (missing slash + _Access segment), so the +/// trigger never fired and the user got no effective permissions. +/// +public sealed class V01_MoveAccessAssignments : IMigration +{ + public int Version => 1; + public string Description => "Move AccessAssignments to access table with _Access namespace"; + + public async Task RunAsync(MigrationContext ctx) + { + await using var cmd = ctx.DataSource.CreateCommand(""" + DO $$ + DECLARE + schema_rec RECORD; + moved_count INT; + ns_count INT; + cols TEXT := 'namespace, id, name, node_type, description, category, icon, display_order, last_modified, version, state, content, desired_id, main_node, embedding'; + BEGIN + FOR schema_rec IN + SELECT schema_name FROM information_schema.schemata s + WHERE EXISTS (SELECT 1 FROM information_schema.tables t WHERE t.table_schema = s.schema_name AND t.table_name = 'mesh_nodes') + AND EXISTS (SELECT 1 FROM information_schema.tables t WHERE t.table_schema = s.schema_name AND t.table_name = 'access') + AND s.schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast') + AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' + LOOP + -- Move AccessAssignments from mesh_nodes to access table + EXECUTE format( + 'INSERT INTO %I.access (' || cols || ') SELECT ' || cols || ' FROM %I.mesh_nodes WHERE node_type = ''AccessAssignment'' ON CONFLICT (namespace, id) DO NOTHING', + schema_rec.schema_name, schema_rec.schema_name + ); + GET DIAGNOSTICS moved_count = ROW_COUNT; + IF moved_count > 0 THEN + EXECUTE format( + 'DELETE FROM %I.mesh_nodes WHERE node_type = ''AccessAssignment''', + schema_rec.schema_name + ); + RAISE NOTICE 'Schema %: moved % AccessAssignment(s) from mesh_nodes to access', schema_rec.schema_name, moved_count; + END IF; + + -- Fix namespace: ensure _Access segment is present + EXECUTE format( + 'UPDATE %I.access SET namespace = namespace || ''/_Access'' WHERE node_type = ''AccessAssignment'' AND namespace NOT LIKE ''%%/_Access''', + schema_rec.schema_name + ); + GET DIAGNOSTICS ns_count = ROW_COUNT; + IF ns_count > 0 THEN + RAISE NOTICE 'Schema %: fixed % namespace(s) to include /_Access', schema_rec.schema_name, ns_count; + END IF; + + -- Rebuild permissions + BEGIN + EXECUTE format('SELECT %I.rebuild_user_effective_permissions()', schema_rec.schema_name); + EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'Schema %: rebuild failed: %', schema_rec.schema_name, SQLERRM; + END; + END LOOP; + END $$; + """); + await cmd.ExecuteNonQueryAsync(); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V02_RebuildTriggerFunctions.cs b/memex/aspire/Memex.Database.Migration/Migrations/V02_RebuildTriggerFunctions.cs new file mode 100644 index 000000000..d8c22fbd6 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V02_RebuildTriggerFunctions.cs @@ -0,0 +1,60 @@ +using MeshWeaver.Hosting.PostgreSql; +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Re-create trigger functions per partition and populate partition_access. +/// The schema initializer now includes partition_access sync in +/// rebuild_user_effective_permissions() with a hardcoded schema name. For existing +/// DBs: re-run schema init per schema to update the function, then rebuild permissions +/// (which populates partition_access). +/// +public sealed class V02_RebuildTriggerFunctions : IMigration +{ + public int Version => 2; + public string Description => "Update trigger functions and populate partition_access"; + + public async Task RunAsync(MigrationContext ctx) + { + var schemas = await SchemaHelpers.DiscoverPartitionSchemasAsync(ctx.DataSource); + + foreach (var schema in schemas) + { + ctx.Logger.LogInformation("Repair v2: Updating trigger function for schema {Schema}...", schema); + + await using var schemaDs = SchemaHelpers.BuildSchemaDataSource(ctx.ConnectionString, schema); + var schemaOpts = SchemaHelpers.BuildSchemaOptions(ctx.ConnectionString, schema, ctx.Options.VectorDimensions); + + var versionsSchema = schema + "_versions"; + var hasVersions = await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, versionsSchema); + + if (hasVersions) + { + // Use BuildSchemaDataSource for the versions schema too — it sets up SSL + + // AAD password provider for Azure, which a raw NpgsqlDataSourceBuilder skips, + // causing `28000: no pg_hba.conf entry … no encryption` against prod. + await using var versionsDs = SchemaHelpers.BuildSchemaDataSource(ctx.ConnectionString, versionsSchema, useVector: false); + await PostgreSqlSchemaInitializer.InitializeWithVersionsSchemaAsync( + ctx.DataSource, schemaDs, versionsDs, schemaOpts, versionsSchema); + } + else + { + await PostgreSqlSchemaInitializer.InitializeMeshTablesAsync(schemaDs, schemaOpts); + } + + try + { + await using var rebuildCmd = ctx.DataSource.CreateCommand( + $"SELECT \"{schema}\".rebuild_user_effective_permissions()"); + await rebuildCmd.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation("Repair v2: Schema {Schema} — rebuilt permissions + partition_access", schema); + } + catch (Exception ex) + { + ctx.Logger.LogWarning(ex, "Repair v2: Schema {Schema} — rebuild failed", schema); + } + } + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V03_DropRogueSchemas.cs b/memex/aspire/Memex.Database.Migration/Migrations/V03_DropRogueSchemas.cs new file mode 100644 index 000000000..c8afe0519 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V03_DropRogueSchemas.cs @@ -0,0 +1,42 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Drop schemas accidentally created from path segments. +/// +/// Bug: paths like login, markdown, onboarding etc. created Postgres +/// schemas that shouldn't exist as partitions. This migration drops them so partition +/// discovery stays clean. +/// +public sealed class V03_DropRogueSchemas : IMigration +{ + private static readonly string[] RogueSchemas = + [ + "_access", "_address_", "_graph", "_settings", "_tracking", "_thread", "_source", "_test", + "login", "markdown", "onboarding", "welcome", "settings", "storage", + "p", "mesh", "thread", "agent", "partition", "organization", "vuser" + ]; + + public int Version => 3; + public string Description => "Drop rogue schemas created from path segments"; + + public async Task RunAsync(MigrationContext ctx) + { + foreach (var rogue in RogueSchemas) + { + try + { + await using (var dropCmd = ctx.DataSource.CreateCommand($"DROP SCHEMA IF EXISTS \"{rogue}\" CASCADE")) + await dropCmd.ExecuteNonQueryAsync(); + await using (var dropVCmd = ctx.DataSource.CreateCommand($"DROP SCHEMA IF EXISTS \"{rogue}_versions\" CASCADE")) + await dropVCmd.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation("Repair v3: Dropped rogue schema {Schema}", rogue); + } + catch (Exception ex) + { + ctx.Logger.LogWarning(ex, "Repair v3: Failed to drop schema {Schema}", rogue); + } + } + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V04_UpgradeViewerToAdmin.cs b/memex/aspire/Memex.Database.Migration/Migrations/V04_UpgradeViewerToAdmin.cs new file mode 100644 index 000000000..c645f895e --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V04_UpgradeViewerToAdmin.cs @@ -0,0 +1,64 @@ +namespace Memex.Database.Migration.Migrations; + +/// +/// Upgrade user self-assignments from Viewer to Admin. +/// UserScopeGrantHandler previously granted Viewer on User/{userId}; it now +/// grants Admin so users can fully manage their own namespace. +/// +public sealed class V04_UpgradeViewerToAdmin : IMigration +{ + public int Version => 4; + public string Description => "Upgrade user self-assignments from Viewer to Admin"; + + public async Task RunAsync(MigrationContext ctx) + { + await using var cmd = ctx.DataSource.CreateCommand(""" + DO $$ + DECLARE + schema_rec RECORD; + updated_count INT; + BEGIN + FOR schema_rec IN + SELECT schema_name FROM information_schema.schemata s + WHERE EXISTS (SELECT 1 FROM information_schema.tables t WHERE t.table_schema = s.schema_name AND t.table_name = 'access') + AND s.schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast') + AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' + LOOP + -- Update self-assignments: namespace=User/{id}/_Access, accessObject={id} + -- Replace Viewer with Admin in the roles array for self-assignments only + EXECUTE format( + 'UPDATE %I.access + SET content = jsonb_set( + content, + ''{roles}'', + (SELECT jsonb_agg( + CASE WHEN elem->>''role'' = ''Viewer'' + THEN jsonb_set(elem, ''{role}'', ''"Admin"'') + ELSE elem + END + ) FROM jsonb_array_elements(content->''roles'') AS elem) + ) + WHERE node_type = ''AccessAssignment'' + AND namespace LIKE ''User/%%/_Access'' + AND namespace = ''User/'' || (content->>''accessObject'') || ''/_Access'' + AND EXISTS (SELECT 1 FROM jsonb_array_elements(content->''roles'') r WHERE r->>''role'' = ''Viewer'') + AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(content->''roles'') r WHERE r->>''role'' = ''Admin'')', + schema_rec.schema_name + ); + GET DIAGNOSTICS updated_count = ROW_COUNT; + IF updated_count > 0 THEN + RAISE NOTICE 'Schema %: upgraded % self-assignment(s) from Viewer to Admin', schema_rec.schema_name, updated_count; + END IF; + + -- Rebuild permissions + BEGIN + EXECUTE format('SELECT %I.rebuild_user_effective_permissions()', schema_rec.schema_name); + EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'Schema %: rebuild failed: %', schema_rec.schema_name, SQLERRM; + END; + END LOOP; + END $$; + """); + await cmd.ExecuteNonQueryAsync(); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V05_EnsureUserSelfAssignments.cs b/memex/aspire/Memex.Database.Migration/Migrations/V05_EnsureUserSelfAssignments.cs new file mode 100644 index 000000000..76b5a1730 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V05_EnsureUserSelfAssignments.cs @@ -0,0 +1,76 @@ +namespace Memex.Database.Migration.Migrations; + +/// +/// Ensure every user has an Admin self-assignment in their OWN partition, then rebuild +/// effective permissions across all content partitions. +/// +/// Users are partition roots, so they are discovered from the central index +/// (public.top_level_index — the #16 partition-root materialized view), NOT the +/// legacy per-schema "user" access-object schema (which no longer exists — it was the +/// pre-V27 schema, since removed). Each User's self-Admin grant lands in that user's own +/// partition's access table at {id}/_Access. +/// +/// On a fresh DB the index has no User rows yet, so this is a clean no-op. The backfill +/// fixes missing AccessAssignment nodes for users onboarded before UserScopeGrantHandler +/// existed and propagates role-permission changes into user_effective_permissions. +/// +public sealed class V05_EnsureUserSelfAssignments : IMigration +{ + public int Version => 5; + public string Description => "Ensure user self-assignments (from the central index) and rebuild permissions"; + + public async Task RunAsync(MigrationContext ctx) + { + await using var cmd = ctx.DataSource.CreateCommand(""" + DO $$ + DECLARE + user_rec RECORD; + BEGIN + -- Backfill each User's self-Admin AccessAssignment IN THE USER'S OWN PARTITION. + -- Users are partition roots → discovered from the central index + -- (public.top_level_index, the #16 materialized view) and own a schema named + -- after their id; the grant lands at {id}/_Access in that partition's `access` + -- table. There is NO legacy `user` schema (the pre-V27 access-object schema is + -- gone). On a fresh DB the index has no Users, so this is a clean no-op. Per-user + -- EXCEPTION so one missing/half-provisioned partition can't abort the migration. + FOR user_rec IN + SELECT id FROM public.top_level_index WHERE node_type = 'User' + LOOP + BEGIN + EXECUTE format($ins$ + INSERT INTO %1$I.access + (id, namespace, name, node_type, content, main_node, last_modified, version, state) + SELECT %2$L || '_Access', %2$L || '/_Access', %2$L || ' Access', 'AccessAssignment', + jsonb_build_object('accessObject', %2$L, 'displayName', %2$L, + 'roles', jsonb_build_array(jsonb_build_object('role', 'Admin'))), + %2$L, NOW(), 1, 2 + WHERE NOT EXISTS ( + SELECT 1 FROM %1$I.access + WHERE namespace = %2$L || '/_Access' + AND content->>'accessObject' = %2$L) + $ins$, lower(user_rec.id), user_rec.id); + EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'Self-assignment backfill for user % failed: %', user_rec.id, SQLERRM; + END; + END LOOP; + + -- Rebuild effective permissions for every content partition (each per-schema + -- `access` table). `user` is no longer special-cased — it does not exist. + FOR user_rec IN + SELECT schema_name FROM information_schema.schemata s + WHERE EXISTS (SELECT 1 FROM information_schema.tables t + WHERE t.table_schema = s.schema_name AND t.table_name = 'access') + AND s.schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast') + AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' + LOOP + BEGIN + EXECUTE format('SELECT %I.rebuild_user_effective_permissions()', user_rec.schema_name); + EXCEPTION WHEN OTHERS THEN + RAISE NOTICE 'Schema % rebuild failed: %', user_rec.schema_name, SQLERRM; + END; + END LOOP; + END $$; + """); + await cmd.ExecuteNonQueryAsync(); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V06_FixSearchAcrossSchemas.cs b/memex/aspire/Memex.Database.Migration/Migrations/V06_FixSearchAcrossSchemas.cs new file mode 100644 index 000000000..3c65b761b --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V06_FixSearchAcrossSchemas.cs @@ -0,0 +1,24 @@ +using MeshWeaver.Hosting.PostgreSql; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Fix search_across_schemas to enforce partition_access. +/// +/// Bug: public_read node types bypassed partition_access entirely, leaking +/// cross-partition data in search (e.g., a meshweaver user could see PartnerRe content). +/// Fix: partition_access is now always required; public_read only skips node-level +/// permission checks within accessible partitions. +/// +/// The stored proc is re-created by InitializePartitionAccessTableAsync (idempotent). +/// +public sealed class V06_FixSearchAcrossSchemas : IMigration +{ + public int Version => 6; + public string Description => "Fix search_across_schemas access control"; + + public async Task RunAsync(MigrationContext ctx) + { + await PostgreSqlSchemaInitializer.InitializePartitionAccessTableAsync(ctx.DataSource); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V07_PerUserPermissionRebuildTrigger.cs b/memex/aspire/Memex.Database.Migration/Migrations/V07_PerUserPermissionRebuildTrigger.cs new file mode 100644 index 000000000..8a2f61cb1 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V07_PerUserPermissionRebuildTrigger.cs @@ -0,0 +1,37 @@ +using MeshWeaver.Hosting.PostgreSql; +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Deploy the per-user permission-rebuild trigger. +/// +/// The trigger function trg_access_changed() previously called +/// rebuild_user_effective_permissions() which rebuilt ALL users' permissions — +/// causing deadlocks under concurrent access. The new trigger calls +/// rebuild_user_permissions_for(affected_user) — only touches one user's rows. +/// +/// The schema initializer already creates the new functions; we just need to re-run +/// schema init per partition to deploy the updated trigger function. +/// +public sealed class V07_PerUserPermissionRebuildTrigger : IMigration +{ + public int Version => 7; + public string Description => "Deploy per-user permission rebuild trigger"; + + public async Task RunAsync(MigrationContext ctx) + { + var schemas = await SchemaHelpers.DiscoverAccessSchemasAsync(ctx.DataSource); + + foreach (var schema in schemas) + { + ctx.Logger.LogInformation("Repair v7: Updating trigger functions for schema {Schema}...", schema); + + await using var schemaDs = SchemaHelpers.BuildSchemaDataSource(ctx.ConnectionString, schema); + var schemaOpts = SchemaHelpers.BuildSchemaOptions(ctx.ConnectionString, schema, ctx.Options.VectorDimensions); + + await PostgreSqlSchemaInitializer.InitializeMeshTablesAsync(schemaDs, schemaOpts); + ctx.Logger.LogInformation("Repair v7: Schema {Schema} — trigger updated", schema); + } + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V08_FixThreadMessageMainNode.cs b/memex/aspire/Memex.Database.Migration/Migrations/V08_FixThreadMessageMainNode.cs new file mode 100644 index 000000000..e7fbc0a6b --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V08_FixThreadMessageMainNode.cs @@ -0,0 +1,45 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Fix ThreadMessage.MainNode. +/// +/// Thread message nodes created from the UI may have MainNode set to the thread +/// path (e.g., Org/_Thread/thread-id) instead of the thread's content node +/// (e.g., Org). This causes "Access denied" because SatelliteAccessRule +/// delegates to MainNode. Fix: set MainNode = part-before-/_Thread/ for all +/// ThreadMessage nodes. +/// +public sealed class V08_FixThreadMessageMainNode : IMigration +{ + public int Version => 8; + public string Description => "Fix ThreadMessage MainNode"; + + public async Task RunAsync(MigrationContext ctx) + { + var totalFixed = 0; + var schemas = await SchemaHelpers.DiscoverPartitionSchemasAsync(ctx.DataSource); + + foreach (var schema in schemas) + { + // Skip the admin schema — it has mesh_nodes but isn't a content partition. + if (string.Equals(schema, "admin", StringComparison.OrdinalIgnoreCase)) continue; + + await using var fixCmd = ctx.DataSource.CreateCommand($""" + UPDATE "{schema}".mesh_nodes + SET main_node = split_part(main_node, '/_Thread/', 1) + WHERE node_type = 'ThreadMessage' + AND main_node LIKE '%/_Thread/%' + """); + var affected = await fixCmd.ExecuteNonQueryAsync(); + if (affected > 0) + { + ctx.Logger.LogInformation("Repair v8: Fixed {Count} ThreadMessage MainNode(s) in schema {Schema}", affected, schema); + totalFixed += affected; + } + } + + ctx.Logger.LogInformation("Repair v8: fixed {Total} ThreadMessage MainNode(s)", totalFixed); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V09_RenameSourceTestSegments.cs b/memex/aspire/Memex.Database.Migration/Migrations/V09_RenameSourceTestSegments.cs new file mode 100644 index 000000000..4431591be --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V09_RenameSourceTestSegments.cs @@ -0,0 +1,123 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Rename _Source/_Test path segments to Source/Test. +/// +/// Code nodes were renamed from satellite-style _Source/_Test sub-namespaces +/// to first-class Source/Test content folders (commit 0280084e7). Existing +/// DB rows still carry the old segment names in namespace and main_node; +/// the app now looks them up under the new names and finds nothing. +/// +/// Fix: rewrite the path segment in place across every content partition's tables and +/// their _versions history. path is a GENERATED column and recomputes +/// itself. The routing target is unchanged (code table before and after), so rows +/// stay put. +/// +public sealed class V09_RenameSourceTestSegments : IMigration +{ + public int Version => 9; + public string Description => "Rename _Source/_Test path segments to Source/Test"; + + public async Task RunAsync(MigrationContext ctx) + { + // Discover every schema (content partitions + their _versions mirrors) that has + // at least one table with a `namespace` column. + var schemas = new List(); + await using (var listCmd = ctx.DataSource.CreateCommand(""" + SELECT DISTINCT s.schema_name + FROM information_schema.schemata s + JOIN information_schema.columns c + ON c.table_schema = s.schema_name AND c.column_name = 'namespace' + WHERE s.schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast') + ORDER BY s.schema_name + """)) + { + await using var rdr = await listCmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) schemas.Add(rdr.GetString(0)); + } + + var totalRowsUpdated = 0; + foreach (var schema in schemas) + { + // Find all tables in this schema with both `namespace` and `main_node` columns + // (mesh_nodes, code, access, threads, annotations, activities, ..., and + // mesh_node_history in _versions schemas). + var tables = new List(); + await using (var tblCmd = ctx.DataSource.CreateCommand(""" + SELECT table_name + FROM information_schema.columns + WHERE table_schema = $1 AND column_name IN ('namespace', 'main_node') + GROUP BY table_name + HAVING COUNT(DISTINCT column_name) = 2 + ORDER BY table_name + """)) + { + tblCmd.Parameters.AddWithValue(schema); + await using var rdr = await tblCmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) tables.Add(rdr.GetString(0)); + } + + foreach (var table in tables) + { + // Pre-delete legacy rows whose renamed counterpart already exists. After + // commit 0280084e7 the app started writing the new `Source`/`Test` segment + // names while old `_Source`/`_Test` rows were still in the DB; in some + // partitions both versions coexist for the same (renamed_namespace, id). + // The renamed row is canonical (that's what the app now reads), so the + // legacy row is dead data — dropping it lets the UPDATE below succeed + // without violating the (namespace, id) primary key. + await using (var dedupCmd = ctx.DataSource.CreateCommand($""" + DELETE FROM "{schema}"."{table}" legacy + WHERE legacy.namespace ~ '(^|/)_(Source|Test)($|/)' + AND EXISTS ( + SELECT 1 FROM "{schema}"."{table}" renamed + WHERE renamed.id = legacy.id + AND renamed.namespace = regexp_replace( + regexp_replace(legacy.namespace, '(^|/)_Source($|/)', '\1Source\2', 'g'), + '(^|/)_Test($|/)', '\1Test\2', 'g' + ) + ) + """)) + { + var deleted = await dedupCmd.ExecuteNonQueryAsync(); + if (deleted > 0) + ctx.Logger.LogInformation( + "Repair v9: {Schema}.{Table} — pre-deleted {Count} legacy _Source/_Test row(s) whose renamed twin already existed", + schema, table, deleted); + } + + // Rewrite `_Source` / `_Test` as whole path segments (anchored at string + // start/end or bounded by '/'), preserving case and neighbours. Only + // rewrite main_node when it is non-null — otherwise leave NULL alone. + await using var fixCmd = ctx.DataSource.CreateCommand($""" + UPDATE "{schema}"."{table}" SET + namespace = regexp_replace( + regexp_replace(namespace, '(^|/)_Source($|/)', '\1Source\2', 'g'), + '(^|/)_Test($|/)', '\1Test\2', 'g' + ), + main_node = CASE + WHEN main_node IS NULL THEN NULL + ELSE regexp_replace( + regexp_replace(main_node, '(^|/)_Source($|/)', '\1Source\2', 'g'), + '(^|/)_Test($|/)', '\1Test\2', 'g' + ) + END + WHERE namespace ~ '(^|/)_(Source|Test)($|/)' + OR main_node ~ '(^|/)_(Source|Test)($|/)' + """); + var affected = await fixCmd.ExecuteNonQueryAsync(); + if (affected > 0) + { + ctx.Logger.LogInformation( + "Repair v9: {Schema}.{Table} — renamed {Count} row(s)", + schema, table, affected); + totalRowsUpdated += affected; + } + } + } + + ctx.Logger.LogInformation("Repair v9: updated {Total} row(s) across all schemas", totalRowsUpdated); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V10_PerUserPartitions.cs b/memex/aspire/Memex.Database.Migration/Migrations/V10_PerUserPartitions.cs new file mode 100644 index 000000000..74e91dd25 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V10_PerUserPartitions.cs @@ -0,0 +1,339 @@ +using MeshWeaver.Hosting.PostgreSql; +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Per-user partitions + drop the User/ namespace prefix. +/// +/// Today every user's data lives in the shared user Postgres schema with paths +/// like User/{id}/.... This is two problems in one: +/// 1. The User prefix is dead weight — it forces every layout area / agent / +/// reference to spell out User/rbuergi/... even though the partition structure +/// already encodes the user. +/// 2. A single schema for ALL users mixes their content, complicates per-user access +/// control, and makes "is this row in the right place?" checks ambiguous. One +/// schema per user mirrors the org pattern (acme/cornerstone). +/// +/// This migration: +/// - Creates one Postgres schema per user (e.g., rbuergi, orwell2000). +/// - Moves rows from "user".T to "<userid>".T for every satellite table, +/// stripping the User/<userid> namespace prefix in flight. +/// - Repopulates partition_access and <userid>.user_effective_permissions. +/// - Inserts a MeshDataSource discovery record per new partition. +/// - Drops "user" + "user_versions" once verified empty. +/// +public sealed class V10_PerUserPartitions : IMigration +{ + public int Version => 10; + public string Description => "Per-user partitions + drop User/ namespace prefix"; + + public async Task RunAsync(MigrationContext ctx) + { + // 0. Pre-flight: does the "user" schema even exist? (fresh DBs skip everything) + var userSchemaExists = await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, "user"); + if (!userSchemaExists) + { + ctx.Logger.LogInformation("Repair v10: no \"user\" schema present — skipping (fresh DB)."); + return; + } + + // 1. Discover users: union of explicit User-typed nodes + path-derived ids + // (covers users with content but no User node). + var userIds = new List(); + await using (var listCmd = ctx.DataSource.CreateCommand(""" + SELECT DISTINCT id FROM "user".mesh_nodes WHERE node_type = 'User' + UNION + SELECT DISTINCT split_part(namespace, '/', 2) AS id + FROM "user".mesh_nodes + WHERE namespace LIKE 'User/%' AND split_part(namespace, '/', 2) <> '' + """)) + { + await using var rdr = await listCmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) userIds.Add(rdr.GetString(0)); + } + + ctx.Logger.LogInformation("Repair v10: discovered {Count} user(s): [{Users}]", + userIds.Count, string.Join(", ", userIds)); + + // Pre-flight: count cross-user content (rows in `user` schema NOT under any + // User/{id}/...). These would orphan if we just dropped `user`. + long orphanCount; + await using (var orphanCmd = ctx.DataSource.CreateCommand( + "SELECT count(*) FROM \"user\".mesh_nodes WHERE namespace NOT LIKE 'User/%' AND namespace <> 'User' AND namespace <> ''")) + { + orphanCount = (long)(await orphanCmd.ExecuteScalarAsync())!; + } + if (orphanCount > 0) + { + ctx.Logger.LogWarning( + "Repair v10: {Count} row(s) in \"user\" schema are NOT under User//... — they will be left in `user` and the schema will NOT be dropped. Inspect manually.", + orphanCount); + } + + // Discover satellite tables that have both `namespace` and `id` columns — those + // are the ones whose rows are addressable by mesh path. Tables like + // `change_logs` or `user_activity` use a different keying scheme and are + // partition-scoped (not user-scoped), so they're left in place. + var satelliteTables = new List(); + await using (var tblCmd = ctx.DataSource.CreateCommand(""" + SELECT a.table_name + FROM information_schema.columns a + JOIN information_schema.columns b + ON a.table_schema = b.table_schema AND a.table_name = b.table_name + AND b.column_name = 'id' + WHERE a.table_schema = 'user' AND a.column_name = 'namespace' + AND a.table_name IN ('mesh_nodes', 'access', 'threads', 'code', 'annotations', 'activities', 'user_activities', 'partition_objects') + GROUP BY a.table_name + ORDER BY a.table_name + """)) + { + await using var rdr = await tblCmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) satelliteTables.Add(rdr.GetString(0)); + } + + // 2. Per-user: bootstrap target schema (idempotent), move rows, rebuild perms. + foreach (var userId in userIds) + { + await MigrateUserAsync(ctx, userId, satelliteTables); + } + + // 7. Drop the legacy `Source/user` MeshDataSource record (if present). + await using (var dropOldDs = ctx.DataSource.CreateCommand( + "DELETE FROM admin.mesh_nodes WHERE namespace = 'Source' AND id = 'user'")) + { + await dropOldDs.ExecuteNonQueryAsync(); + } + + // 8. Wipe partition_access partition='user' rows — per-schema rebuild repopulated + // them under the new partition names. + await using (var wipePa = ctx.DataSource.CreateCommand( + "DELETE FROM public.partition_access WHERE partition = 'user'")) + { + await wipePa.ExecuteNonQueryAsync(); + } + + // 9. Verify and drop. Only drop if EVERY user-table is empty (orphans abort the drop). + long residual = 0; + foreach (var t in satelliteTables) + { + await using var countCmd = ctx.DataSource.CreateCommand($"SELECT count(*) FROM \"user\".\"{t}\""); + residual += (long)(await countCmd.ExecuteScalarAsync())!; + } + if (residual == 0) + { + await using (var dropCmd = ctx.DataSource.CreateCommand("DROP SCHEMA \"user\" CASCADE")) + await dropCmd.ExecuteNonQueryAsync(); + await using (var dropVCmd = ctx.DataSource.CreateCommand("DROP SCHEMA IF EXISTS \"user_versions\" CASCADE")) + await dropVCmd.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation("Repair v10: dropped \"user\" and \"user_versions\" schemas (empty)"); + } + else + { + ctx.Logger.LogWarning( + "Repair v10: \"user\" schema has {Residual} residual row(s) — NOT dropping. Inspect manually before next run.", + residual); + } + } + + private static async Task MigrateUserAsync(MigrationContext ctx, string userId, List satelliteTables) + { + var schemaName = SchemaHelpers.SanitizeSchemaName(userId); + if (string.IsNullOrEmpty(schemaName)) return; + + var targetExists = await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, schemaName); + if (targetExists) + { + ctx.Logger.LogInformation("Repair v10: schema \"{Schema}\" already exists — re-using (idempotent re-init)", schemaName); + } + + // Bootstrap the target schema (mesh_nodes + satellites + _versions) + await using var schemaDs = SchemaHelpers.BuildSchemaDataSource(ctx.ConnectionString, schemaName); + + await using (var createSchemaCmd = ctx.DataSource.CreateCommand($"CREATE SCHEMA IF NOT EXISTS \"{schemaName}\"")) + await createSchemaCmd.ExecuteNonQueryAsync(); + + var versionsSchemaName = schemaName + "_versions"; + await using (var createVersionsCmd = ctx.DataSource.CreateCommand($"CREATE SCHEMA IF NOT EXISTS \"{versionsSchemaName}\"")) + await createVersionsCmd.ExecuteNonQueryAsync(); + + // BuildSchemaDataSource wires SSL + AAD password provider for Azure — a raw + // NpgsqlDataSourceBuilder skips both and dies with `28000: no pg_hba.conf entry`. + await using var versionsDs = SchemaHelpers.BuildSchemaDataSource(ctx.ConnectionString, versionsSchemaName, useVector: false); + + var schemaOpts = SchemaHelpers.BuildSchemaOptions(ctx.ConnectionString, schemaName, ctx.Options.VectorDimensions); + + await PostgreSqlSchemaInitializer.InitializeWithVersionsSchemaAsync( + ctx.DataSource, schemaDs, versionsDs, schemaOpts, versionsSchemaName); + await PostgreSqlSchemaInitializer.CreateSatelliteTablesAsync( + schemaDs, schemaOpts, MeshWeaver.Mesh.PartitionDefinition.DefaultSegmentTableMappings().Values); + + // 3. Move rows for every satellite table. `path` is GENERATED — recomputes from + // the rewritten namespace. Use `information_schema.columns` to compute the + // column list dynamically per table, since satellites have slightly different + // shapes (no embedding, no description, etc.). + foreach (var table in satelliteTables) + { + await MoveTableAsync(ctx, userId, schemaName, table); + } + + // 4. Move history (mesh_node_history lives in the _versions schemas) + var hasUserVersions = await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, "user_versions"); + if (hasUserVersions) + { + await MoveHistoryAsync(ctx, userId, versionsSchemaName); + } + + // 5. Rebuild permissions for the new schema (re-syncs partition_access). + try + { + await using var rebuildCmd = ctx.DataSource.CreateCommand( + $"SELECT \"{schemaName}\".rebuild_user_effective_permissions()"); + await rebuildCmd.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation("Repair v10: \"{Schema}\".rebuild_user_effective_permissions() OK", schemaName); + } + catch (Exception ex) + { + ctx.Logger.LogWarning(ex, "Repair v10: rebuild_user_effective_permissions failed for \"{Schema}\"", schemaName); + } + + // 6. MeshDataSource discovery record — mirror what AddMeshDataSource writes + // for orgs (namespace="Source", nodeType="MeshDataSource"). + await using (var meshDsCmd = ctx.DataSource.CreateCommand(""" + INSERT INTO admin.mesh_nodes (namespace, id, name, node_type, state, content, last_modified, main_node) + VALUES ('Source', $1, $1, 'MeshDataSource', 2, + jsonb_build_object('Partition', $1, 'StorageType', 'Postgres', 'ProviderType', 'PostgreSql'), + now(), $1) + ON CONFLICT (namespace, id) DO UPDATE SET + content = EXCLUDED.content, + last_modified = now() + """)) + { + meshDsCmd.Parameters.AddWithValue(userId); + await meshDsCmd.ExecuteNonQueryAsync(); + } + } + + private static async Task MoveTableAsync(MigrationContext ctx, string userId, string schemaName, string table) + { + // Pull column names that exist on BOTH the source and the destination, excluding + // the generated `path` column (must not be inserted). + var commonCols = new List(); + await using (var colCmd = ctx.DataSource.CreateCommand(""" + SELECT a.column_name + FROM information_schema.columns a + JOIN information_schema.columns b + ON a.column_name = b.column_name + AND b.table_schema = $2 AND b.table_name = $3 + WHERE a.table_schema = 'user' AND a.table_name = $1 + AND a.column_name <> 'path' + ORDER BY a.ordinal_position + """)) + { + colCmd.Parameters.AddWithValue(table); + colCmd.Parameters.AddWithValue(schemaName); + colCmd.Parameters.AddWithValue(table); + await using var rdr = await colCmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) commonCols.Add(rdr.GetString(0)); + } + + if (commonCols.Count == 0) + { + ctx.Logger.LogDebug("Repair v10: skipping {Table} — no common columns between user and {Schema}", table, schemaName); + return; + } + + // SELECT projection: rewrite namespace + main_node, pass through the rest. + var selectExprs = string.Join(", ", commonCols.Select(c => c switch + { + "namespace" => + // 'User/' → '', 'User//x/y' → 'x/y' + "CASE WHEN namespace = 'User/' || $1 THEN '' " + + "ELSE regexp_replace(namespace, '^User/' || $1 || '/', '') END AS namespace", + "main_node" => + "CASE WHEN main_node IS NULL THEN NULL " + + "WHEN main_node = 'User/' || $1 THEN '' " + + "ELSE regexp_replace(main_node, '^User/' || $1 || '/', '') END AS main_node", + _ => $"\"{c}\"" + })); + var insertCols = string.Join(", ", commonCols.Select(c => $"\"{c}\"")); + + var moveSql = $""" + WITH moved AS ( + DELETE FROM "user"."{table}" + WHERE namespace = 'User/' || $1 + OR namespace LIKE 'User/' || $1 || '/%' + RETURNING {string.Join(", ", commonCols.Select(c => $"\"{c}\""))} + ) + INSERT INTO "{schemaName}"."{table}" ({insertCols}) + SELECT {selectExprs} FROM moved + ON CONFLICT (namespace, id) DO NOTHING + """; + + await using var moveCmd = ctx.DataSource.CreateCommand(moveSql); + moveCmd.Parameters.AddWithValue(userId); + var moved = await moveCmd.ExecuteNonQueryAsync(); + if (moved > 0) + ctx.Logger.LogInformation("Repair v10: \"{Schema}\".{Table} ← \"user\".{Table} — moved {Count} row(s)", + schemaName, table, table, moved); + } + + private static async Task MoveHistoryAsync(MigrationContext ctx, string userId, string versionsSchemaName) + { + // Same dynamic column-discovery pattern — `mesh_node_history` may be missing + // `embedding` and other optional columns depending on when the schema was + // first created. + var histCols = new List(); + await using (var histColCmd = ctx.DataSource.CreateCommand(""" + SELECT a.column_name + FROM information_schema.columns a + JOIN information_schema.columns b + ON a.column_name = b.column_name + AND b.table_schema = $1 AND b.table_name = 'mesh_node_history' + WHERE a.table_schema = 'user_versions' AND a.table_name = 'mesh_node_history' + AND a.column_name <> 'path' + ORDER BY a.ordinal_position + """)) + { + histColCmd.Parameters.AddWithValue(versionsSchemaName); + await using var rdr = await histColCmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) histCols.Add(rdr.GetString(0)); + } + + if (histCols.Count == 0 || !histCols.Contains("namespace") || !histCols.Contains("id") || !histCols.Contains("version")) + return; + + var histSelect = string.Join(", ", histCols.Select(c => c switch + { + "namespace" => + "CASE WHEN namespace = 'User/' || $1 THEN '' " + + "ELSE regexp_replace(namespace, '^User/' || $1 || '/', '') END AS namespace", + "main_node" => + "CASE WHEN main_node IS NULL THEN NULL " + + "WHEN main_node = 'User/' || $1 THEN '' " + + "ELSE regexp_replace(main_node, '^User/' || $1 || '/', '') END AS main_node", + _ => $"\"{c}\"" + })); + var histReturning = string.Join(", ", histCols.Select(c => $"\"{c}\"")); + var histInsertCols = string.Join(", ", histCols.Select(c => $"\"{c}\"")); + + var histSql = $""" + WITH moved AS ( + DELETE FROM "user_versions".mesh_node_history + WHERE namespace = 'User/' || $1 + OR namespace LIKE 'User/' || $1 || '/%' + RETURNING {histReturning} + ) + INSERT INTO "{versionsSchemaName}".mesh_node_history ({histInsertCols}) + SELECT {histSelect} FROM moved + ON CONFLICT (namespace, id, version) DO NOTHING + """; + await using var moveHistCmd = ctx.DataSource.CreateCommand(histSql); + moveHistCmd.Parameters.AddWithValue(userId); + var movedHist = await moveHistCmd.ExecuteNonQueryAsync(); + if (movedHist > 0) + ctx.Logger.LogInformation("Repair v10: \"{V}\".mesh_node_history ← \"user_versions\" — moved {Count} row(s)", + versionsSchemaName, movedHist); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V11_RewriteApiTokenPaths.cs b/memex/aspire/Memex.Database.Migration/Migrations/V11_RewriteApiTokenPaths.cs new file mode 100644 index 000000000..41a4346a2 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V11_RewriteApiTokenPaths.cs @@ -0,0 +1,46 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Rewrite apitoken.mesh_nodes.tokenPath to drop the User/ prefix. +/// +/// V10 moved per-user content (including ApiToken/* nodes) out of the shared +/// user schema into per-user schemas, dropping the User/<id> namespace +/// prefix in flight. However the dedicated apitoken partition still holds +/// ApiTokenIndex rows whose content.tokenPath references the old +/// User/<userid>/ApiToken/<hash> path — validation chain breaks because the +/// indirect lookup target no longer exists at that path. Rewrite to +/// <userid>/ApiToken/<hash> so post-v10 token validation resolves through +/// the per-user partition. +/// +public sealed class V11_RewriteApiTokenPaths : IMigration +{ + public int Version => 11; + public string Description => "Rewrite apitoken tokenPath to drop User/ prefix"; + + public async Task RunAsync(MigrationContext ctx) + { + var apitokenSchemaExists = await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, "apitoken"); + if (!apitokenSchemaExists) + { + ctx.Logger.LogInformation("Repair v11: no \"apitoken\" schema present — skipping."); + return; + } + + await using var fixCmd = ctx.DataSource.CreateCommand(""" + UPDATE apitoken.mesh_nodes + SET content = jsonb_set( + content, + '{tokenPath}', + to_jsonb(regexp_replace(content->>'tokenPath', '^User/([^/]+)/', '\1/')) + ), + last_modified = NOW(), + version = version + 1 + WHERE node_type = 'ApiToken' + AND content->>'tokenPath' LIKE 'User/%' + """); + var updated = await fixCmd.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation("Repair v11: rewrote {Count} apitoken.mesh_nodes tokenPath(s)", updated); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V13_RebuildPermissionsForApiBitmask.cs b/memex/aspire/Memex.Database.Migration/Migrations/V13_RebuildPermissionsForApiBitmask.cs new file mode 100644 index 000000000..0e751814f --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V13_RebuildPermissionsForApiBitmask.cs @@ -0,0 +1,63 @@ +using MeshWeaver.Hosting.PostgreSql; +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Re-run rebuild_user_effective_permissions after the role-bitmask fix. +/// +/// PostgreSqlSchemaInitializer used to emit Admin=127 / Editor=119 / Viewer=33 etc +/// as the role-bitmask fallback in the rebuild stored procedure. These miss Api +/// (bit 128) and Export (bit 256), so users with Admin role got Read/Create/ +/// Update/Delete/Comment/Execute/Thread but NOT Api — which broke ApiToken creation +/// (the satellite-access rule checks Permission.Api on MainNode). +/// +/// The schema initializer now emits the correct bitmasks (Admin=511 = Permission.All, +/// etc.) and the unnest also emits Api/Export. Re-run rebuild for every existing +/// partition to backfill the missing permission rows. +/// +/// Note: there is no v12. v12 just called the function which still had the old +/// definition; v13 force-replaces the function via InitializeMeshTablesAsync +/// before invoking it. +/// +public sealed class V13_RebuildPermissionsForApiBitmask : IMigration +{ + public int Version => 13; + public string Description => "Update rebuild function (Admin=511 / Api+Export bits) and re-rebuild"; + + public async Task RunAsync(MigrationContext ctx) + { + var schemas = await SchemaHelpers.DiscoverAccessSchemasAsync(ctx.DataSource); + + foreach (var schema in schemas) + { + // Re-run InitializeMeshTablesAsync per schema so the rebuild function is + // re-created with the new bitmasks + Api/Export unnest. + await using var schemaDs = SchemaHelpers.BuildSchemaDataSource(ctx.ConnectionString, schema); + var schemaOpts = SchemaHelpers.BuildSchemaOptions(ctx.ConnectionString, schema, ctx.Options.VectorDimensions); + + try + { + await PostgreSqlSchemaInitializer.InitializeMeshTablesAsync(schemaDs, schemaOpts); + ctx.Logger.LogInformation("Repair v13: \"{Schema}\" — function updated", schema); + } + catch (Exception ex) + { + ctx.Logger.LogWarning(ex, "Repair v13: \"{Schema}\" — InitializeMeshTablesAsync failed", schema); + continue; + } + + try + { + await using var rebuildCmd = ctx.DataSource.CreateCommand( + $"SELECT \"{schema}\".rebuild_user_effective_permissions()"); + await rebuildCmd.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation("Repair v13: \"{Schema}\".rebuild_user_effective_permissions() OK", schema); + } + catch (Exception ex) + { + ctx.Logger.LogWarning(ex, "Repair v13: rebuild failed for \"{Schema}\"", schema); + } + } + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V14_AddPartitionPrefixToNamespaces.cs b/memex/aspire/Memex.Database.Migration/Migrations/V14_AddPartitionPrefixToNamespaces.cs new file mode 100644 index 000000000..9332d29aa --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V14_AddPartitionPrefixToNamespaces.cs @@ -0,0 +1,245 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Re-add the partition prefix to V10-migrated namespaces, and sweep any residuals +/// out of the legacy user schema. +/// +/// V10 stripped the entire User/<userid>/ prefix when moving rows into per-user +/// schemas. The current convention (see AGENTS.md and how org partitions like +/// partnerre/systemorph store their rows) is that namespace KEEPS the +/// partition prefix — e.g., (ns='rbuergi/Notes', id='foo'), not (ns='Notes'). +/// +/// V10-migrated content thus became invisible to every code path that queries via +/// namespace:<partition>/... or browses the user node's children. This migration: +/// +/// 1. Sweeps any User-type rows still in the legacy "user".mesh_nodes schema into +/// their per-user partition root (namespace='', id=<userid>). +/// 2. For every partition with a MeshDataSource record in admin.mesh_nodes: +/// rewrites namespace + main_node in every satellite table to include the +/// partition prefix, leaving the user-identity row at +/// (ns='', id=<userid>, node_type='User') as the documented special case. +/// 3. Drops "user" + "user_versions" schemas if empty after the sweep. +/// +/// Idempotent: rows that already have the prefix are matched by the +/// NOT LIKE '<P>/%' guard and left untouched. +/// +public sealed class V14_AddPartitionPrefixToNamespaces : IMigration +{ + public int Version => 14; + public string Description => "Restore partition prefix on V10-migrated namespaces; sweep user-schema residuals"; + + public async Task RunAsync(MigrationContext ctx) + { + // 1. Sweep User-type residuals out of "user".mesh_nodes (if the schema is still around). + await SweepLegacyUserSchemaAsync(ctx); + + // 2. Discover all partition prefixes from admin.mesh_nodes (Source records). + var partitions = await DiscoverPartitionsAsync(ctx); + ctx.Logger.LogInformation("Repair v14: discovered {Count} partition(s) to inspect: [{Partitions}]", + partitions.Count, string.Join(", ", partitions.Select(p => $"{p.Partition}->{p.Schema}"))); + + // 3. For each partition: prepend prefix where missing. + foreach (var (partitionId, schemaName) in partitions) + { + await RewritePartitionAsync(ctx, partitionId, schemaName); + } + + // 4. Drop user/user_versions if empty. + await TryDropLegacyUserSchemaAsync(ctx); + } + + private static async Task SweepLegacyUserSchemaAsync(MigrationContext ctx) + { + if (!await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, "user")) + return; + + // Move any User-typed rows still in user.mesh_nodes into their per-user partition root. + // These would have been written by code paths that still target the legacy User namespace + // even after V10 moved everything else. + var userRows = new List<(string Namespace, string Id)>(); + await using (var listCmd = ctx.DataSource.CreateCommand(""" + SELECT namespace, id FROM "user".mesh_nodes + WHERE node_type = 'User' AND namespace = 'User' + """)) + { + await using var rdr = await listCmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) userRows.Add((rdr.GetString(0), rdr.GetString(1))); + } + + foreach (var (_, userId) in userRows) + { + var schemaName = SchemaHelpers.SanitizeSchemaName(userId); + if (string.IsNullOrEmpty(schemaName)) continue; + if (!await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, schemaName)) + { + ctx.Logger.LogWarning( + "Repair v14: User '{UserId}' has rows in legacy user schema but no per-user schema '{Schema}' exists. Leaving in place.", + userId, schemaName); + continue; + } + + // INSERT (selecting empty namespace) then DELETE. Use ON CONFLICT to make idempotent + // — if the per-user partition already has a User row at (ns='', id=), the + // newer last_modified wins. + await using var moveCmd = ctx.DataSource.CreateCommand($""" + INSERT INTO "{schemaName}".mesh_nodes + (namespace, id, name, node_type, description, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node, embedding) + SELECT '', id, name, node_type, description, category, icon, display_order, + last_modified, version, state, content, desired_id, NULL, embedding + FROM "user".mesh_nodes + WHERE node_type = 'User' AND namespace = 'User' AND id = $1 + ON CONFLICT (namespace, id) DO UPDATE SET + content = EXCLUDED.content, + last_modified = GREATEST("{schemaName}".mesh_nodes.last_modified, EXCLUDED.last_modified), + version = "{schemaName}".mesh_nodes.version + 1 + """); + moveCmd.Parameters.AddWithValue(userId); + var moved = await moveCmd.ExecuteNonQueryAsync(); + + await using var deleteCmd = ctx.DataSource.CreateCommand(""" + DELETE FROM "user".mesh_nodes WHERE node_type = 'User' AND namespace = 'User' AND id = $1 + """); + deleteCmd.Parameters.AddWithValue(userId); + await deleteCmd.ExecuteNonQueryAsync(); + + ctx.Logger.LogInformation( + "Repair v14: moved User identity '{UserId}' from legacy user schema to '{Schema}' (rows affected: {Count})", + userId, schemaName, moved); + } + } + + private static async Task> DiscoverPartitionsAsync(MigrationContext ctx) + { + var partitions = new List<(string, string)>(); + // MeshDataSource records live at (ns='Source', id=) in admin.mesh_nodes. + await using var cmd = ctx.DataSource.CreateCommand(""" + SELECT id FROM admin.mesh_nodes + WHERE namespace = 'Source' AND node_type = 'MeshDataSource' + ORDER BY id + """); + await using var rdr = await cmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) + { + var partitionId = rdr.GetString(0); + var schemaName = SchemaHelpers.SanitizeSchemaName(partitionId); + if (string.IsNullOrEmpty(schemaName)) continue; + if (!await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, schemaName)) continue; + partitions.Add((partitionId, schemaName)); + } + return partitions; + } + + private static async Task RewritePartitionAsync(MigrationContext ctx, string partitionId, string schemaName) + { + // Discover tables in this schema that have BOTH namespace and main_node — these are + // the satellite-shaped tables (mesh_nodes, access, threads, code, annotations, activities, ...). + var tables = new List(); + await using (var tblCmd = ctx.DataSource.CreateCommand(""" + SELECT table_name + FROM information_schema.columns + WHERE table_schema = $1 AND column_name IN ('namespace', 'main_node') + GROUP BY table_name + HAVING COUNT(DISTINCT column_name) = 2 + ORDER BY table_name + """)) + { + tblCmd.Parameters.AddWithValue(schemaName); + await using var rdr = await tblCmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) tables.Add(rdr.GetString(0)); + } + + var totalRows = 0; + foreach (var table in tables) + { + // Two-step rewrite per table: + // (a) rows where namespace = '' AND not the user-identity row + // → namespace = partitionId + // (b) rows where namespace doesn't already start with partitionId/ AND not equal to partitionId + // → namespace = partitionId || '/' || namespace + // main_node mirrors the same rule (NULL preserved). + // + // The user-identity special case: (node_type='User' AND id=partitionId AND namespace='') + // — leave untouched. AGENTS.md documents this as the documented exception. + // + // partitionId is interpolated (validated source: admin.mesh_nodes ids that we control) + // because Postgres LIKE patterns with parameters are awkward when we also need + // concatenation in the SET clause. + var quotedPid = partitionId.Replace("'", "''"); + await using var rewriteCmd = ctx.DataSource.CreateCommand($""" + UPDATE "{schemaName}"."{table}" SET + namespace = CASE + WHEN namespace = '' THEN '{quotedPid}' + ELSE '{quotedPid}/' || namespace + END, + main_node = CASE + WHEN main_node IS NULL THEN NULL + WHEN main_node = '' THEN '{quotedPid}' + WHEN main_node = '{quotedPid}' THEN main_node + WHEN main_node LIKE '{quotedPid}/%' THEN main_node + ELSE '{quotedPid}/' || main_node + END + WHERE + -- Skip rows that already conform to the convention. + namespace <> '{quotedPid}' + AND namespace NOT LIKE '{quotedPid}/%' + -- Skip the user-identity special case. + AND NOT (namespace = '' AND id = '{quotedPid}' AND node_type = 'User') + """); + var affected = await rewriteCmd.ExecuteNonQueryAsync(); + if (affected > 0) + { + ctx.Logger.LogInformation( + "Repair v14: \"{Schema}\".{Table} — prepended prefix to {Count} row(s)", + schemaName, table, affected); + totalRows += affected; + } + } + + if (totalRows > 0) + ctx.Logger.LogInformation("Repair v14: \"{Schema}\" total rows updated: {Total}", schemaName, totalRows); + } + + private static async Task TryDropLegacyUserSchemaAsync(MigrationContext ctx) + { + if (!await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, "user")) + return; + + long residual; + await using (var countCmd = ctx.DataSource.CreateCommand(""" + SELECT COALESCE(SUM(n), 0)::bigint FROM ( + SELECT count(*) AS n FROM "user".mesh_nodes + UNION ALL SELECT count(*) FROM "user".access + UNION ALL SELECT count(*) FROM "user".threads + UNION ALL SELECT count(*) FROM "user".code + UNION ALL SELECT count(*) FROM "user".annotations + UNION ALL SELECT count(*) FROM "user".activities + ) sub + """)) + { + try { residual = (long)(await countCmd.ExecuteScalarAsync())!; } + catch (Exception ex) + { + // One of the satellite tables may not exist on older deployments — be lenient. + ctx.Logger.LogWarning(ex, "Repair v14: could not count user-schema residuals; not dropping."); + return; + } + } + + if (residual > 0) + { + ctx.Logger.LogWarning( + "Repair v14: legacy \"user\" schema still has {Residual} row(s) — NOT dropping. Inspect manually.", + residual); + return; + } + + await using (var drop = ctx.DataSource.CreateCommand("DROP SCHEMA \"user\" CASCADE")) + await drop.ExecuteNonQueryAsync(); + await using (var dropV = ctx.DataSource.CreateCommand("DROP SCHEMA IF EXISTS \"user_versions\" CASCADE")) + await dropV.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation("Repair v14: dropped legacy \"user\" + \"user_versions\" schemas (empty)"); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V15_FinalUserSchemaCleanup.cs b/memex/aspire/Memex.Database.Migration/Migrations/V15_FinalUserSchemaCleanup.cs new file mode 100644 index 000000000..84675ea82 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V15_FinalUserSchemaCleanup.cs @@ -0,0 +1,146 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Final cleanup of the legacy user schema. V14 swept User-typed identity +/// rows but left two classes of residuals that surfaced on live deployments: +/// +/// 1. Stale ApiToken duplicates — rows like +/// (ns='User/<userid>/ApiToken', id=<hashPrefix>) written by the +/// pre-fix +/// whose post-fix replacements already exist at +/// (ns='<userid>/ApiToken', id=<hashPrefix>) in the user partition. +/// We move the row into the user partition under the correct namespace; if a +/// newer row already lives at the target, the upsert keeps the newer one +/// (last_modified comparison) and the stale row is dropped on DELETE. +/// +/// 2. Stranded NodeType-meta records — e.g. +/// (ns='', id='User', node_type='User'). The runtime re-registers +/// these via IStaticNodeProvider on every startup, so the persisted +/// copy is dead weight. Delete it. +/// +/// Once both classes are gone, drop user + user_versions if every +/// user-schema satellite is empty. +/// +public sealed class V15_FinalUserSchemaCleanup : IMigration +{ + public int Version => 15; + public string Description => "Sweep stale ApiToken/NodeType-meta residuals from legacy user schema and drop it"; + + public async Task RunAsync(MigrationContext ctx) + { + if (!await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, "user")) + return; + + // 1. Move stale ApiToken rows from user.mesh_nodes (User//ApiToken/...) + // into the per-user partition's mesh_nodes table, namespace=/ApiToken. + var apiTokenRows = new List<(string Namespace, string Id, string UserId)>(); + await using (var listCmd = ctx.DataSource.CreateCommand(""" + SELECT namespace, id, split_part(namespace, '/', 2) AS user_id + FROM "user".mesh_nodes + WHERE namespace LIKE 'User/%/ApiToken' + """)) + { + await using var rdr = await listCmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) + apiTokenRows.Add((rdr.GetString(0), rdr.GetString(1), rdr.GetString(2))); + } + + foreach (var (sourceNs, id, userId) in apiTokenRows) + { + if (string.IsNullOrEmpty(userId)) continue; + var schemaName = SchemaHelpers.SanitizeSchemaName(userId); + if (string.IsNullOrEmpty(schemaName)) continue; + if (!await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, schemaName)) + { + ctx.Logger.LogWarning( + "Repair v15: ApiToken {Id} in user.mesh_nodes targets non-existent partition '{Schema}'. Leaving in place.", + id, schemaName); + continue; + } + + var targetNs = $"{userId}/ApiToken"; + await using var moveCmd = ctx.DataSource.CreateCommand($""" + INSERT INTO "{schemaName}".mesh_nodes + (namespace, id, name, node_type, description, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node, embedding) + SELECT $1, id, name, node_type, description, category, icon, display_order, + last_modified, version, state, content, desired_id, $2, embedding + FROM "user".mesh_nodes + WHERE namespace = $3 AND id = $4 + ON CONFLICT (namespace, id) DO UPDATE SET + -- Keep whichever copy is newer; the orphan in user.mesh_nodes + -- is usually a stale write that the user already replaced. + content = CASE + WHEN EXCLUDED.last_modified > "{schemaName}".mesh_nodes.last_modified + THEN EXCLUDED.content ELSE "{schemaName}".mesh_nodes.content END, + last_modified = GREATEST("{schemaName}".mesh_nodes.last_modified, EXCLUDED.last_modified) + """); + moveCmd.Parameters.AddWithValue(targetNs); + moveCmd.Parameters.AddWithValue(userId); + moveCmd.Parameters.AddWithValue(sourceNs); + moveCmd.Parameters.AddWithValue(id); + await moveCmd.ExecuteNonQueryAsync(); + + await using var deleteCmd = ctx.DataSource.CreateCommand(""" + DELETE FROM "user".mesh_nodes WHERE namespace = $1 AND id = $2 + """); + deleteCmd.Parameters.AddWithValue(sourceNs); + deleteCmd.Parameters.AddWithValue(id); + await deleteCmd.ExecuteNonQueryAsync(); + + ctx.Logger.LogInformation( + "Repair v15: moved ApiToken {Id} from \"user\".(ns={Source}) to \"{Schema}\".(ns={Target})", + id, sourceNs, schemaName, targetNs); + } + + // 2. Delete stranded NodeType meta rows. These are recreated on every + // startup by IStaticNodeProvider implementations (see UserNodeType.cs + // and similar), so the persisted copy is just legacy noise. + await using (var delMeta = ctx.DataSource.CreateCommand(""" + DELETE FROM "user".mesh_nodes + WHERE namespace = '' AND node_type = id + """)) + { + var n = await delMeta.ExecuteNonQueryAsync(); + if (n > 0) ctx.Logger.LogInformation( + "Repair v15: dropped {Count} stranded NodeType-meta row(s) from user.mesh_nodes", n); + } + + // 3. Drop user/user_versions if everything's empty. + long residual; + await using (var countCmd = ctx.DataSource.CreateCommand(""" + SELECT COALESCE(SUM(n), 0)::bigint FROM ( + SELECT count(*) AS n FROM "user".mesh_nodes + UNION ALL SELECT count(*) FROM "user".access + UNION ALL SELECT count(*) FROM "user".threads + UNION ALL SELECT count(*) FROM "user".code + UNION ALL SELECT count(*) FROM "user".annotations + UNION ALL SELECT count(*) FROM "user".activities + ) sub + """)) + { + try { residual = (long)(await countCmd.ExecuteScalarAsync())!; } + catch (Exception ex) + { + ctx.Logger.LogWarning(ex, "Repair v15: could not count user-schema residuals; not dropping."); + return; + } + } + + if (residual > 0) + { + ctx.Logger.LogWarning( + "Repair v15: legacy \"user\" schema still has {Residual} unrecognised row(s) — NOT dropping. Inspect manually.", + residual); + return; + } + + await using (var drop = ctx.DataSource.CreateCommand("DROP SCHEMA \"user\" CASCADE")) + await drop.ExecuteNonQueryAsync(); + await using (var dropV = ctx.DataSource.CreateCommand("DROP SCHEMA IF EXISTS \"user_versions\" CASCADE")) + await dropV.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation("Repair v15: dropped legacy \"user\" + \"user_versions\" schemas (empty)"); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V16_NormalizeAccessAssignmentShape.cs b/memex/aspire/Memex.Database.Migration/Migrations/V16_NormalizeAccessAssignmentShape.cs new file mode 100644 index 000000000..ce32ebefb --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V16_NormalizeAccessAssignmentShape.cs @@ -0,0 +1,93 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Normalize legacy AccessAssignment content shapes that V14/V15 missed. +/// +/// Two stale fields kept appearing in {partition}.access rows on +/// production after the user-prefix migrations had already run: +/// +/// content.accessObject like "User/<userid>" — should be +/// the bare user id ("<userid>"). The original V10 sweep +/// touched mesh_nodes but not the access satellite, and a +/// post-V10 grant path on prod kept emitting the legacy prefix. +/// content.roles[i].role like "Role/<name>" — should +/// be the bare role name ("Admin", "Editor", …). Same root +/// cause: a writer was carrying the role node's full path instead of +/// the role enum value, and SecurityService treats the prefixed form +/// as a non-match. +/// +/// +/// Both classes leave the holder with zero effective permissions: +/// SecurityService.GetUserRolesAsync compares the granted +/// accessObject against the bare user id, so a single User/ +/// prefix wins them nothing. The fix is purely textual — strip the prefix. +/// +/// Idempotent. The WHERE guards ensure rows that already conform +/// to the convention are skipped, so re-running V16 on a clean DB is a no-op. +/// Bumps version + last_modified on touched rows so the +/// workspace stream picks up the change on next portal restart. +/// +public sealed class V16_NormalizeAccessAssignmentShape : IMigration +{ + public int Version => 16; + public string Description => "Strip legacy 'User/' + 'Role/' prefixes from AccessAssignment content"; + + public async Task RunAsync(MigrationContext ctx) + { + // Discover every schema that has an `access` satellite. AccessAssignment + // rows live in `{partition}.access`, NOT `{partition}.mesh_nodes` (the + // path segment `_Access` routes them to the satellite per + // AGENTS.md → "Satellite tables are routed by path segment, not nodeType"). + var schemas = new List(); + await using (var cmd = ctx.DataSource.CreateCommand(""" + SELECT t.table_schema FROM information_schema.tables t + WHERE t.table_name = 'access' + AND t.table_schema NOT IN ('information_schema', 'pg_catalog', 'pg_toast') + ORDER BY t.table_schema + """)) + await using (var rdr = await cmd.ExecuteReaderAsync()) + while (await rdr.ReadAsync()) schemas.Add(rdr.GetString(0)); + + var totalFixed = 0; + foreach (var schema in schemas) + { + var qSchema = schema.Replace("\"", "\"\""); + // Single statement rewrites BOTH fields. The CASE inside each + // jsonb_set keeps the original value when the legacy prefix is + // absent — required because jsonb_set always replaces the key. + // + // substring(... FROM 6) drops the leading 'User/' (5 chars) / + // 'Role/' (5 chars) — both prefixes are exactly 5 characters. + var sql = + "UPDATE \"" + qSchema + "\".access SET " + + "content = jsonb_set(" + + "jsonb_set(content, '{accessObject}', " + + "CASE WHEN content->>'accessObject' LIKE 'User/%' " + + "THEN to_jsonb(substring(content->>'accessObject' FROM 6)) " + + "ELSE content->'accessObject' END), " + + "'{roles,0,role}', " + + "CASE WHEN content->'roles'->0->>'role' LIKE 'Role/%' " + + "THEN to_jsonb(substring(content->'roles'->0->>'role' FROM 6)) " + + "ELSE content->'roles'->0->'role' END), " + + "version = COALESCE(version, 0) + 1, " + + "last_modified = now() " + + "WHERE content->>'accessObject' LIKE 'User/%' " + + "OR content->'roles'->0->>'role' LIKE 'Role/%'"; + + await using var cmd = ctx.DataSource.CreateCommand(sql); + var n = await cmd.ExecuteNonQueryAsync(); + if (n > 0) + { + ctx.Logger.LogInformation( + "Repair v16: \"{Schema}\".access — normalized {Count} legacy AccessAssignment row(s)", + schema, n); + totalFixed += n; + } + } + + if (totalFixed > 0) + ctx.Logger.LogInformation("Repair v16: total rows normalized: {Total}", totalFixed); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V17_EnsurePerUserSelfAssignments.cs b/memex/aspire/Memex.Database.Migration/Migrations/V17_EnsurePerUserSelfAssignments.cs new file mode 100644 index 000000000..e9a2aeab7 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V17_EnsurePerUserSelfAssignments.cs @@ -0,0 +1,186 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Ensures every per-user partition schema has a self-assignment (Admin role) in +/// its access satellite table. +/// +/// V05 created self-assignments in the old monolithic user schema. V10 moved +/// them into per-user schemas. V14 prepended the partition prefix to all namespaces. None +/// of these migrations covered users who were: +/// +/// Created after V10 ran and whose UserScopeGrantHandler call failed +/// silently (fire-and-forget Subscribe). +/// In the user schema when it was dropped — their access row could have +/// been missed if the V10 move ran before V05 populated it. +/// Migrated by V10 with the wrong namespace shape and then stripped again by V16. +/// +/// +/// The correct shape after V14 is: +/// +/// {userId}.access — the satellite table in the user's own Postgres schema +/// namespace = {userId}/_Access (WITH partition prefix, per AGENTS.md convention) +/// id = {userId}_Access +/// main_node = {userId} +/// content = {"accessObject":"{userId}","displayName":"{userId}","roles":[{"role":"Admin"}]} +/// +/// +/// Idempotent: the ON CONFLICT DO NOTHING guard skips users who already have +/// a conforming row. Ends with a full rebuild_user_effective_permissions() sweep +/// across every touched schema so permissions take effect immediately on next portal start. +/// +public sealed class V17_EnsurePerUserSelfAssignments : IMigration +{ + public int Version => 17; + public string Description => "Ensure per-user self-assignment (Admin) in every user partition schema"; + + public async Task RunAsync(MigrationContext ctx) + { + // Discover per-user schemas: schemas that have BOTH mesh_nodes AND access tables, + // AND contain the user-identity row at (namespace='', id=, node_type='User'). + // This excludes org schemas (they have a Group identity, not a User identity) and + // the admin schema. We collect (schemaName, userId) pairs. + var userSchemas = new List<(string Schema, string UserId)>(); + + await using (var discoverCmd = ctx.DataSource.CreateCommand(""" + SELECT t.table_schema + FROM information_schema.tables t + WHERE t.table_name = 'access' + AND t.table_schema NOT IN ('information_schema','pg_catalog','pg_toast','public','admin') + AND t.table_schema NOT LIKE '%_versions' + ORDER BY t.table_schema + """)) + await using (var rdr = await discoverCmd.ExecuteReaderAsync()) + { + while (await rdr.ReadAsync()) + userSchemas.Add((rdr.GetString(0), string.Empty)); // userId filled below + } + + // For each schema, look up the User identity row to get the canonical userId. + var confirmed = new List<(string Schema, string UserId)>(); + foreach (var (schema, _) in userSchemas) + { + // mesh_nodes may not exist (shouldn't happen but be defensive) + bool hasMeshNodes; + await using (var chkCmd = ctx.DataSource.CreateCommand(""" + SELECT 1 FROM information_schema.tables + WHERE table_schema = $1 AND table_name = 'mesh_nodes' + LIMIT 1 + """)) + { + chkCmd.Parameters.AddWithValue(schema); + var result = await chkCmd.ExecuteScalarAsync(); + hasMeshNodes = result is not null; + } + if (!hasMeshNodes) continue; + + // User identity row: namespace='', node_type='User', id= + // The special case documented in AGENTS.md and V14. + string? userId = null; + await using (var idCmd = ctx.DataSource.CreateCommand($""" + SELECT id FROM "{schema.Replace("\"", "\"\"")}".mesh_nodes + WHERE namespace = '' AND node_type = 'User' + LIMIT 1 + """)) + await using (var idRdr = await idCmd.ExecuteReaderAsync()) + { + if (await idRdr.ReadAsync()) + userId = idRdr.IsDBNull(0) ? null : idRdr.GetString(0); + } + + if (string.IsNullOrEmpty(userId)) + continue; // org partition or not a user partition — skip + + confirmed.Add((schema, userId)); + } + + ctx.Logger.LogInformation( + "Repair v17: found {Count} per-user schema(s): [{Schemas}]", + confirmed.Count, string.Join(", ", confirmed.Select(t => t.Schema))); + + var inserted = 0; + var rebuilt = 0; + foreach (var (schema, userId) in confirmed) + { + var quotedSchema = schema.Replace("\"", "\"\""); + var quotedUserId = userId.Replace("'", "''"); + + // Expected namespace for the self-assignment in the post-V14 convention: + // namespace = '{userId}/_Access' (WITH partition prefix) + var ns = $"{userId}/_Access"; + var id = $"{userId}_Access"; + var mainNode = userId; + + // Check existence first to keep the log noise low. + bool exists; + await using (var chkCmd = ctx.DataSource.CreateCommand($""" + SELECT 1 FROM "{quotedSchema}".access + WHERE namespace = $1 + AND content->>'accessObject' = $2 + LIMIT 1 + """)) + { + chkCmd.Parameters.AddWithValue(ns); + chkCmd.Parameters.AddWithValue(userId); + exists = await chkCmd.ExecuteScalarAsync() is not null; + } + + if (exists) + { + ctx.Logger.LogDebug("Repair v17: '{Schema}' — self-assignment already present, skipping", schema); + continue; + } + + // Insert the missing self-assignment. ON CONFLICT DO NOTHING makes this + // idempotent even under concurrent runs. + await using (var insertCmd = ctx.DataSource.CreateCommand($""" + INSERT INTO "{quotedSchema}".access + (namespace, id, name, node_type, state, content, main_node, last_modified, version) + VALUES ($1, $2, $3, 'AccessAssignment', 2, + jsonb_build_object( + 'accessObject', $4, + 'displayName', $4, + 'roles', jsonb_build_array(jsonb_build_object('role', 'Admin')) + ), + $5, now(), 1) + ON CONFLICT (namespace, id) DO NOTHING + """)) + { + insertCmd.Parameters.AddWithValue(ns); + insertCmd.Parameters.AddWithValue(id); + insertCmd.Parameters.AddWithValue($"{userId} Access"); + insertCmd.Parameters.AddWithValue(userId); + insertCmd.Parameters.AddWithValue(mainNode); + var rows = await insertCmd.ExecuteNonQueryAsync(); + if (rows > 0) + { + ctx.Logger.LogInformation( + "Repair v17: '{Schema}' — inserted self-assignment for user '{UserId}'", + schema, userId); + inserted++; + } + } + + // Rebuild permissions for this schema so the new assignment is reflected + // immediately in user_effective_permissions without requiring a portal restart. + try + { + await using var rebuildCmd = ctx.DataSource.CreateCommand( + $"SELECT \"{quotedSchema}\".rebuild_user_effective_permissions()"); + await rebuildCmd.ExecuteNonQueryAsync(); + rebuilt++; + ctx.Logger.LogDebug("Repair v17: '{Schema}'.rebuild_user_effective_permissions() OK", schema); + } + catch (Exception ex) + { + ctx.Logger.LogWarning(ex, + "Repair v17: rebuild_user_effective_permissions failed for '{Schema}'", schema); + } + } + + ctx.Logger.LogInformation( + "Repair v17: done — {Inserted} self-assignment(s) inserted, {Rebuilt} schema(s) rebuilt", + inserted, rebuilt); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V18_BackfillUserPartitionRegistry.cs b/memex/aspire/Memex.Database.Migration/Migrations/V18_BackfillUserPartitionRegistry.cs new file mode 100644 index 000000000..54227e8f3 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V18_BackfillUserPartitionRegistry.cs @@ -0,0 +1,153 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Backfills the user schema with a thin User registry entry for every +/// existing per-user partition. +/// +/// Two-tier User design: +/// +/// Per-user partition ({userId}.mesh_nodes): the FULL User node +/// (path = {userId}, namespace = "", full bio/role/pinnedPaths/etc.). +/// Visiting /{userId} resolves here via first-segment partition routing. +/// User partition (user.mesh_nodes): a THIN registry entry per user +/// (path = User/{userId}, namespace = "User", content = {email, $type}). +/// OnboardingMiddleware's nodeType:User content.email:X lookup is +/// pinned to the User partition by routing rule, so without these entries +/// every signed-in user gets bounced back to /onboarding on every request. +/// +/// +/// The post-V10 onboarding moved User nodes from user schema (namespace=User) +/// to per-user schemas (namespace=""), but the User-partition registry index was never +/// repopulated — so existing users couldn't be found by email. This migration walks every +/// per-user schema and inserts the thin entry. Idempotent: ON CONFLICT DO NOTHING +/// guards re-runs. +/// +/// Going forward, Onboarding.razor creates BOTH entries on every new +/// onboarding (full in per-user partition + thin in User partition). +/// +public sealed class V18_BackfillUserPartitionRegistry : IMigration +{ + public int Version => 18; + public string Description => "Backfill user.mesh_nodes with thin User registry entries (path=User/{id}) for existing per-user partitions"; + + public async Task RunAsync(MigrationContext ctx) + { + // Discover candidate schemas: have a mesh_nodes table AND contain a User identity row + // (namespace='', node_type='User'). Excludes the user/admin/public schemas explicitly, + // and any *_versions sidecar. + var candidates = new List<(string Schema, string UserId, string? Name, string? Icon, string Content)>(); + + await using (var discoverCmd = ctx.DataSource.CreateCommand(""" + SELECT t.table_schema + FROM information_schema.tables t + WHERE t.table_name = 'mesh_nodes' + AND t.table_schema NOT IN ('information_schema','pg_catalog','pg_toast','public','admin','user') + AND t.table_schema NOT LIKE '%_versions' + ORDER BY t.table_schema + """)) + await using (var rdr = await discoverCmd.ExecuteReaderAsync()) + { + var schemas = new List(); + while (await rdr.ReadAsync()) + schemas.Add(rdr.GetString(0)); + + await rdr.CloseAsync(); + + foreach (var schema in schemas) + { + var quotedSchema = schema.Replace("\"", "\"\""); + await using var idCmd = ctx.DataSource.CreateCommand($""" + SELECT id, name, icon, content::text + FROM "{quotedSchema}".mesh_nodes + WHERE namespace = '' AND node_type = 'User' + LIMIT 1 + """); + await using var idRdr = await idCmd.ExecuteReaderAsync(); + if (await idRdr.ReadAsync()) + { + var userId = idRdr.IsDBNull(0) ? null : idRdr.GetString(0); + if (string.IsNullOrEmpty(userId)) continue; + var name = idRdr.IsDBNull(1) ? null : idRdr.GetString(1); + var icon = idRdr.IsDBNull(2) ? null : idRdr.GetString(2); + var content = idRdr.IsDBNull(3) ? "{}" : idRdr.GetString(3); + candidates.Add((schema, userId, name, icon, content)); + } + } + } + + ctx.Logger.LogInformation( + "Repair v18: found {Count} per-user schema(s) with a User identity: [{Schemas}]", + candidates.Count, string.Join(", ", candidates.Select(c => c.Schema))); + + // Ensure the destination user.mesh_nodes table exists. SchemaInitialization is the + // canonical creator; we only insert here. If the schema is missing entirely, log + // and skip — that's a setup bug, not something this migration fixes. + bool userMeshExists; + await using (var chkCmd = ctx.DataSource.CreateCommand(""" + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'user' AND table_name = 'mesh_nodes' + LIMIT 1 + """)) + { + userMeshExists = await chkCmd.ExecuteScalarAsync() is not null; + } + if (!userMeshExists) + { + ctx.Logger.LogWarning("Repair v18: user.mesh_nodes does not exist — User partition not initialised, skipping backfill"); + return; + } + + var inserted = 0; + foreach (var (schema, userId, name, icon, sourceContent) in candidates) + { + // Build a minimal content payload: just $type + email so the + // `nodeType:User content.email:X` lookup in OnboardingMiddleware lands + // on this row. Everything else (bio, role, pinnedPaths, fullName) stays + // exclusively on the per-user partition's full node. + string thinContent; + try + { + using var doc = System.Text.Json.JsonDocument.Parse(sourceContent); + var email = doc.RootElement.TryGetProperty("email", out var e) && e.ValueKind == System.Text.Json.JsonValueKind.String + ? e.GetString() ?? "" + : ""; + var type = doc.RootElement.TryGetProperty("$type", out var t) && t.ValueKind == System.Text.Json.JsonValueKind.String + ? t.GetString() ?? "MeshWeaver.Mesh.Security.User" + : "MeshWeaver.Mesh.Security.User"; + thinContent = System.Text.Json.JsonSerializer.Serialize(new { email, type }, new System.Text.Json.JsonSerializerOptions()) + .Replace("\"type\"", "\"$type\""); // anonymous-type field rename + } + catch (Exception ex) + { + ctx.Logger.LogWarning(ex, + "Repair v18: failed to parse content for '{Schema}'.{UserId} — skipping", + schema, userId); + continue; + } + + await using var insertCmd = ctx.DataSource.CreateCommand(""" + INSERT INTO "user".mesh_nodes + (namespace, id, name, icon, node_type, state, content, main_node, last_modified, version) + VALUES ('User', $1, $2, $3, 'User', 2, $4::jsonb, $5, now(), 1) + ON CONFLICT (namespace, id) DO NOTHING + """); + insertCmd.Parameters.AddWithValue(userId); + insertCmd.Parameters.AddWithValue((object?)name ?? userId); + insertCmd.Parameters.AddWithValue((object?)icon ?? DBNull.Value); + insertCmd.Parameters.AddWithValue(thinContent); + insertCmd.Parameters.AddWithValue($"User/{userId}"); + var rows = await insertCmd.ExecuteNonQueryAsync(); + if (rows > 0) + { + ctx.Logger.LogInformation( + "Repair v18: 'user' partition — inserted thin User entry for '{UserId}' (source schema '{Schema}')", + userId, schema); + inserted++; + } + } + + ctx.Logger.LogInformation("Repair v18: done — {Inserted} thin User entries inserted into 'user' partition", inserted); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V19_DeleteLegacyReleaseNodes.cs b/memex/aspire/Memex.Database.Migration/Migrations/V19_DeleteLegacyReleaseNodes.cs new file mode 100644 index 000000000..6de360096 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V19_DeleteLegacyReleaseNodes.cs @@ -0,0 +1,96 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Delete legacy _Release MeshNodes across every partition. +/// +/// NodeType-compile Release MeshNodes used to live at +/// {nodeTypePath}/_Release/{version}. The leading underscore made them look like +/// a satellite-routed entity (alongside _Access, _Thread, etc.), but +/// PartitionDefinition.StandardTableMappings never routed _Release, so +/// they lived in mesh_nodes all along. The path-segment underscore was cosmetic +/// dead-weight that confused readers, so the namespace was renamed to Release +/// (no underscore) alongside the cross-silo assembly-reference refactor. +/// +/// Strategy: delete in place rather than rename. Release nodes are +/// regenerated on every successful compile (MeshDataSource.TryCreateReleaseNode), +/// and the next portal cold-start will trigger a recompile on every NodeType whose +/// HasUsableBuildMetadata predicate returns false (i.e. any whose +/// NodeTypeDefinition.LatestAssemblyCollection/LatestAssemblyPath are +/// not yet populated — every NodeType, post-deploy). The deleted history is +/// observability + UI listings; no live activation depends on it. +/// +/// Only deletes from mesh_nodes in each schema; the rest of the +/// partition's tables (code, access, etc.) don't carry release rows. +/// Versions sidecars (mesh_node_history in *_versions schemas) are +/// scrubbed too so the rewrite history stays consistent. +/// +public sealed class V19_DeleteLegacyReleaseNodes : IMigration +{ + public int Version => 19; + public string Description => "Delete legacy _Release/* MeshNodes — regenerated on next successful compile"; + + public async Task RunAsync(MigrationContext ctx) + { + // Discover every schema (content partitions + their _versions mirrors) that has + // a mesh_nodes table. + var schemas = new List(); + await using (var listCmd = ctx.DataSource.CreateCommand(""" + SELECT DISTINCT t.table_schema + FROM information_schema.tables t + WHERE t.table_name IN ('mesh_nodes', 'mesh_node_history') + AND t.table_schema NOT IN ('information_schema', 'pg_catalog', 'pg_toast') + ORDER BY t.table_schema + """)) + { + await using var rdr = await listCmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) schemas.Add(rdr.GetString(0)); + } + + var totalDeleted = 0; + foreach (var schema in schemas) + { + // Find the actual table name(s) in this schema. mesh_nodes for content + // partitions; mesh_node_history for _versions sidecars. + var tables = new List(); + await using (var tblCmd = ctx.DataSource.CreateCommand(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = $1 + AND table_name IN ('mesh_nodes', 'mesh_node_history') + ORDER BY table_name + """)) + { + tblCmd.Parameters.AddWithValue(schema); + await using var rdr = await tblCmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) tables.Add(rdr.GetString(0)); + } + + foreach (var table in tables) + { + // Match _Release as a whole path segment anywhere in `namespace` + // (so we catch e.g. `acme/Underwriting/_Release/v20260101...`) and + // also bare `_Release` as the namespace (defensive — should never + // happen in practice). + await using var delCmd = ctx.DataSource.CreateCommand($""" + DELETE FROM "{schema}"."{table}" + WHERE namespace ~ '(^|/)_Release($|/)' + OR namespace = '_Release' + """); + var affected = await delCmd.ExecuteNonQueryAsync(); + if (affected > 0) + { + ctx.Logger.LogInformation( + "Repair v19: {Schema}.{Table} — deleted {Count} legacy _Release node(s)", + schema, table, affected); + totalDeleted += affected; + } + } + } + + ctx.Logger.LogInformation( + "Repair v19: deleted {Total} legacy _Release node(s) across all schemas — they will be regenerated on next compile", + totalDeleted); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V20_RemoveStrayLegacyUserRows.cs b/memex/aspire/Memex.Database.Migration/Migrations/V20_RemoveStrayLegacyUserRows.cs new file mode 100644 index 000000000..7a4909c8e --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V20_RemoveStrayLegacyUserRows.cs @@ -0,0 +1,107 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Final sweep of legacy User-typed rows still landing in +/// user.mesh_nodes with namespace='User'. +/// +/// already does the +/// move, but a post-V14 onboarding bug in Onboarding.razor kept +/// writing new MeshNode(username, "User") for every new signup, +/// re-creating the legacy row on every onboarding. The bug is fixed in the +/// same commit as this migration (onboarding now writes +/// new MeshNode(username) with empty namespace). This migration +/// cleans up the strays so /<username> finds the User node in +/// the user's own partition root via standard path routing — no synth, no +/// cross-partition lookup. +/// +/// Same shape as V14's user-move: INSERT into {username}.mesh_nodes +/// at (namespace='', id=username) on conflict update with the newer +/// timestamp; then DELETE the legacy row from user.mesh_nodes. +/// +public sealed class V20_RemoveStrayLegacyUserRows : IMigration +{ + public int Version => 20; + public string Description => + "Move stray User-typed rows from user.mesh_nodes (namespace=User) to {username}.mesh_nodes (namespace='')"; + + public async Task RunAsync(MigrationContext ctx) + { + if (!await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, "user")) + return; + + var userRows = new List(); + await using (var listCmd = ctx.DataSource.CreateCommand(""" + SELECT id FROM "user".mesh_nodes + WHERE node_type = 'User' AND namespace = 'User' + """)) + { + await using var rdr = await listCmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) userRows.Add(rdr.GetString(0)); + } + + if (userRows.Count == 0) + { + ctx.Logger.LogInformation( + "Repair v20: no stray User rows in user.mesh_nodes (namespace=User) — nothing to do"); + return; + } + + var movedCount = 0; + var skippedCount = 0; + foreach (var userId in userRows) + { + var schemaName = SchemaHelpers.SanitizeSchemaName(userId); + if (string.IsNullOrEmpty(schemaName)) + { + skippedCount++; + continue; + } + if (!await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, schemaName)) + { + ctx.Logger.LogWarning( + "Repair v20: User '{UserId}' has a stray legacy row but no per-user schema '{Schema}' exists — leaving in place", + userId, schemaName); + skippedCount++; + continue; + } + + // INSERT (selecting empty namespace) then DELETE. ON CONFLICT update with + // the newer last_modified — if the per-user partition already has a User + // row at (ns='', id=), keep the freshest content. + await using var moveCmd = ctx.DataSource.CreateCommand($""" + INSERT INTO "{schemaName}".mesh_nodes + (namespace, id, name, node_type, description, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node, embedding) + SELECT '', id, name, node_type, description, category, icon, display_order, + last_modified, version, state, content, desired_id, NULL, embedding + FROM "user".mesh_nodes + WHERE node_type = 'User' AND namespace = 'User' AND id = $1 + ON CONFLICT (namespace, id) DO UPDATE SET + content = EXCLUDED.content, + name = COALESCE(EXCLUDED.name, "{schemaName}".mesh_nodes.name), + icon = COALESCE(EXCLUDED.icon, "{schemaName}".mesh_nodes.icon), + last_modified = GREATEST("{schemaName}".mesh_nodes.last_modified, EXCLUDED.last_modified), + version = "{schemaName}".mesh_nodes.version + 1 + """); + moveCmd.Parameters.AddWithValue(userId); + var moved = await moveCmd.ExecuteNonQueryAsync(); + + await using var deleteCmd = ctx.DataSource.CreateCommand(""" + DELETE FROM "user".mesh_nodes WHERE node_type = 'User' AND namespace = 'User' AND id = $1 + """); + deleteCmd.Parameters.AddWithValue(userId); + await deleteCmd.ExecuteNonQueryAsync(); + + ctx.Logger.LogInformation( + "Repair v20: moved User '{UserId}' from legacy user.mesh_nodes to {Schema}.mesh_nodes (rows affected: {Count})", + userId, schemaName, moved); + movedCount++; + } + + ctx.Logger.LogInformation( + "Repair v20: moved {Moved} User identity row(s); skipped {Skipped}", + movedCount, skippedCount); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V22_ConsolidateGlobalCatalogsInAdmin.cs b/memex/aspire/Memex.Database.Migration/Migrations/V22_ConsolidateGlobalCatalogsInAdmin.cs new file mode 100644 index 000000000..a47dbe197 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V22_ConsolidateGlobalCatalogsInAdmin.cs @@ -0,0 +1,52 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Drops the Admin/Partition/* routing-registry rows from +/// admin.mesh_nodes. Routing no longer consults these — the +/// PostgreSQL partition provider does a lazy +/// information_schema.schemata lookup per first-segment with a +/// 5-minute TTL cache; schema existence is the source of truth for +/// "is this a partition?". +/// +/// The Admin/Partition rows were originally written so the routing +/// layer could pre-load a static partition list at startup. That model +/// breaks across silos (each silo needs the same pre-load to settle) and +/// adds a chicken-and-egg dependency on the catalog. Removing the rows +/// makes routing stateless: any schema reachable via +/// information_schema.schemata is a valid partition, period. +/// +/// What stays: user.mesh_nodes (login-by-email index) +/// and apitoken.mesh_nodes (token-auth index) remain as their own +/// partitions/schemas — they're routable through the same lazy-lookup +/// mechanism, no special-casing needed. +/// +/// Idempotent: DELETE on a non-existent row is a no-op. +/// +public sealed class V22_ConsolidateGlobalCatalogsInAdmin : IMigration +{ + public int Version => 22; + public string Description => + "Drop Admin/Partition routing-registry rows; routing now uses information_schema.schemata"; + + public async Task RunAsync(MigrationContext ctx) + { + if (!await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, "admin")) + { + ctx.Logger.LogInformation( + "Repair v22: 'admin' schema missing — nothing to clean up"); + return; + } + + await using var cmd = ctx.DataSource.CreateCommand(""" + DELETE FROM admin.mesh_nodes + WHERE namespace = 'Admin/Partition' + """); + var rows = await cmd.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation( + "Repair v22: dropped {Rows} Admin/Partition row(s). Routing now consults " + + "information_schema.schemata directly with a 5-min TTL cache.", + rows); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V23_PartitionChangesNotify.cs b/memex/aspire/Memex.Database.Migration/Migrations/V23_PartitionChangesNotify.cs new file mode 100644 index 000000000..b7ba073f0 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V23_PartitionChangesNotify.cs @@ -0,0 +1,87 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Adds a Postgres trigger that fires NOTIFY partition_changes when +/// an Admin/Partition/* row is inserted, updated, or deleted in +/// admin.mesh_nodes. The payload is a small JSON document +/// ({"op":"INSERT","namespace":"acme"}) consumed by +/// +/// on every silo. Each listener invalidates its +/// entry for the +/// affected namespace so the next access re-probes +/// information_schema.schemata and picks up the new/dropped state. +/// +/// Why pg_notify and not the existing mesh_node_changes channel: +/// that channel carries every row write — every silo would burn cycles +/// filtering it for the small subset of Admin/Partition events. +/// A dedicated channel is one extra trigger, near-zero throughput, and +/// makes the partition-routing invariant explicit at the schema level. +/// +/// Idempotent: CREATE OR REPLACE FUNCTION and +/// DROP TRIGGER IF EXISTS + CREATE TRIGGER are safe to re-run. +/// +public sealed class V23_PartitionChangesNotify : IMigration +{ + public int Version => 23; + public string Description => + "Add pg_notify('partition_changes') trigger on Admin/Partition row writes"; + + public async Task RunAsync(MigrationContext ctx) + { + if (!await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, "admin")) + { + ctx.Logger.LogInformation( + "Repair v23: 'admin' schema missing — skipping partition_changes trigger setup"); + return; + } + + await using (var cmd = ctx.DataSource.CreateCommand(""" + CREATE OR REPLACE FUNCTION admin.notify_partition_change() + RETURNS trigger AS $$ + DECLARE + payload_ns TEXT; + BEGIN + payload_ns := COALESCE(NEW.id, OLD.id); + PERFORM pg_notify('partition_changes', + json_build_object( + 'op', TG_OP, + 'namespace', payload_ns + )::text); + RETURN COALESCE(NEW, OLD); + END $$ LANGUAGE plpgsql; + """)) + { + await cmd.ExecuteNonQueryAsync(); + } + + // PostgreSQL does not allow TG_OP in trigger WHEN clauses (only inside + // the trigger function body). Two triggers -- one for INSERT/UPDATE + // referencing NEW, one for DELETE referencing OLD -- give us the same + // narrowing effect without TG_OP. + await using (var cmd = ctx.DataSource.CreateCommand(""" + DROP TRIGGER IF EXISTS trg_partition_notify ON admin.mesh_nodes; + DROP TRIGGER IF EXISTS trg_partition_notify_iu ON admin.mesh_nodes; + DROP TRIGGER IF EXISTS trg_partition_notify_d ON admin.mesh_nodes; + + CREATE TRIGGER trg_partition_notify_iu + AFTER INSERT OR UPDATE ON admin.mesh_nodes + FOR EACH ROW + WHEN (NEW.namespace = 'Admin/Partition') + EXECUTE FUNCTION admin.notify_partition_change(); + + CREATE TRIGGER trg_partition_notify_d + AFTER DELETE ON admin.mesh_nodes + FOR EACH ROW + WHEN (OLD.namespace = 'Admin/Partition') + EXECUTE FUNCTION admin.notify_partition_change(); + """)) + { + await cmd.ExecuteNonQueryAsync(); + } + + ctx.Logger.LogInformation( + "Repair v23: partition_changes trigger installed on admin.mesh_nodes"); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V24_DedupMeshNodeNotifyTrigger.cs b/memex/aspire/Memex.Database.Migration/Migrations/V24_DedupMeshNodeNotifyTrigger.cs new file mode 100644 index 000000000..d64268c3f --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V24_DedupMeshNodeNotifyTrigger.cs @@ -0,0 +1,79 @@ +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Replace {schema}.notify_mesh_node_changes() in every partition schema +/// with a version that suppresses pg_notify on no-op UPDATEs (same content, +/// name, node_type, state, version, desired_id, main_node). +/// +/// Why this exists: the old function fired NOTIFY for every UPDATE, +/// including idempotent writes (a workspace.Update lambda that returns the +/// same node value, or a same-value upsert). Every NOTIFY wakes every +/// synced-query subscriber, which re-reads its result set, which can in +/// turn write more rows — an amplification feedback loop. Prod incident +/// 2026-05-20: opening a chat thread fanned out to ~5 cross-schema queries +/// + per-message access checks; each access read triggered a no-op upsert +/// touching last_modified, which fired NOTIFY, which woke the synced queries +/// again. Per-partition single-Npgsql connection (MaxPoolSize=1) couldn't +/// keep up → /authorize hung. +/// +/// Idempotent: CREATE OR REPLACE FUNCTION. Safe to re-run. +/// Schema initializer creates fresh partitions with the new function already +/// in place; this migration covers existing partitions. +/// +public sealed class V24_DedupMeshNodeNotifyTrigger : IMigration +{ + public int Version => 24; + public string Description => + "Replace notify_mesh_node_changes() per schema with no-op-UPDATE dedup"; + + private const string DedupedFunction = """ + CREATE OR REPLACE FUNCTION notify_mesh_node_changes() RETURNS TRIGGER AS $$ + BEGIN + IF TG_OP = 'DELETE' THEN + PERFORM pg_notify('mesh_node_changes', + json_build_object('path', CASE WHEN OLD.namespace = '' THEN OLD.id ELSE OLD.namespace || '/' || OLD.id END, 'op', 'DELETE')::text); + RETURN OLD; + ELSIF TG_OP = 'UPDATE' + AND OLD.content IS NOT DISTINCT FROM NEW.content + AND OLD.name IS NOT DISTINCT FROM NEW.name + AND OLD.node_type IS NOT DISTINCT FROM NEW.node_type + AND OLD.state IS NOT DISTINCT FROM NEW.state + AND OLD.version IS NOT DISTINCT FROM NEW.version + AND OLD.desired_id IS NOT DISTINCT FROM NEW.desired_id + AND OLD.main_node IS NOT DISTINCT FROM NEW.main_node THEN + RETURN NEW; + ELSE + PERFORM pg_notify('mesh_node_changes', + json_build_object('path', CASE WHEN NEW.namespace = '' THEN NEW.id ELSE NEW.namespace || '/' || NEW.id END, 'op', TG_OP)::text); + RETURN NEW; + END IF; + END; + $$ LANGUAGE plpgsql; + """; + + public async Task RunAsync(MigrationContext ctx) + { + var schemas = await SchemaHelpers.DiscoverPartitionSchemasAsync(ctx.DataSource); + + foreach (var schema in schemas) + { + ctx.Logger.LogInformation( + "Repair v24: replacing notify_mesh_node_changes() in schema {Schema}…", schema); + + // The function lives in each partition schema (and is invoked by + // the per-schema mesh_node_notify trigger). search_path needs to + // point at the schema so CREATE FUNCTION lands there, not in + // public. + await using var cmd = ctx.DataSource.CreateCommand( + $"SET LOCAL search_path TO \"{schema}\"; {DedupedFunction}"); + await cmd.ExecuteNonQueryAsync(); + } + + ctx.Logger.LogInformation( + "Repair v24: deduped notify_mesh_node_changes() across {Count} partition schema(s)", + schemas.Count); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V25_MirrorAccessObjectsToUserSchema.cs b/memex/aspire/Memex.Database.Migration/Migrations/V25_MirrorAccessObjectsToUserSchema.cs new file mode 100644 index 000000000..5ed6466f1 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V25_MirrorAccessObjectsToUserSchema.cs @@ -0,0 +1,132 @@ +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Maintain a global index of every "access object" (User, Group, Role, VUser +/// nodes -- anything that can appear as AccessObject on an +/// AccessAssignment) inside the dedicated user schema. +/// +/// Problem this solves: per-user partitions hold their own User node +/// (at the root of each user's schema). Resolving email -> user.Id +/// previously fanned a nodeType:User scope:subtree synced query +/// across every partition's mesh_nodes. Under load and during the +/// 2026-05-20 thread-load incident this fan-out was one of the costs paying +/// the per-query searchable_schemas sync hit. A single-schema lookup is +/// constant cost and immune to per-partition pool starvation. +/// +/// Approach: install an AFTER INSERT/UPDATE/DELETE trigger on each +/// partition's mesh_nodes that mirrors rows of the relevant node +/// types into user.mesh_nodes. Then backfill the existing rows. +/// The mirror is keyed on (namespace, id) exactly like the source, +/// and uses ON CONFLICT DO UPDATE so updates flow through. +/// +/// Idempotent: CREATE OR REPLACE FUNCTION + +/// DROP TRIGGER IF EXISTS. Safe to re-run. +/// +public sealed class V25_MirrorAccessObjectsToUserSchema : IMigration +{ + public int Version => 25; + + public string Description => + "Mirror User/Group/Role/VUser nodes into user.mesh_nodes via per-partition trigger"; + + private static readonly string[] AccessObjectNodeTypes = + { "User", "Group", "Role", "VUser" }; + + public async Task RunAsync(MigrationContext ctx) + { + // The 'user' schema must exist and have mesh_nodes. Both come from + // schema initialisation; bail out softly if either is missing + // (fresh-DB ordering quirk -- the migration runner reruns next time). + if (!await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, "user")) + { + ctx.Logger.LogInformation("Repair v25: 'user' schema missing -- skipping"); + return; + } + + // The mirror trigger function lives in the public schema so every + // partition trigger can resolve it without a search_path dance. It + // writes into "user".mesh_nodes via fully-qualified name. + await using (var cmd = ctx.DataSource.CreateCommand(""" + CREATE OR REPLACE FUNCTION public.mirror_access_object_to_user_schema() + RETURNS TRIGGER AS $$ + BEGIN + IF TG_OP = 'DELETE' THEN + IF OLD.node_type IN ('User','Group','Role','VUser') THEN + DELETE FROM "user".mesh_nodes + WHERE namespace = OLD.namespace AND id = OLD.id; + END IF; + RETURN OLD; + END IF; + + IF NEW.node_type IN ('User','Group','Role','VUser') THEN + INSERT INTO "user".mesh_nodes + (namespace, id, name, node_type, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node) + VALUES (NEW.namespace, NEW.id, NEW.name, NEW.node_type, NEW.category, NEW.icon, NEW.display_order, + NEW.last_modified, NEW.version, NEW.state, NEW.content, NEW.desired_id, NEW.main_node) + ON CONFLICT (namespace, id) DO UPDATE SET + name = EXCLUDED.name, + node_type = EXCLUDED.node_type, + category = EXCLUDED.category, + icon = EXCLUDED.icon, + display_order = EXCLUDED.display_order, + last_modified = EXCLUDED.last_modified, + version = EXCLUDED.version, + state = EXCLUDED.state, + content = EXCLUDED.content, + desired_id = EXCLUDED.desired_id, + main_node = EXCLUDED.main_node; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """)) + { + await cmd.ExecuteNonQueryAsync(); + } + + var schemas = await SchemaHelpers.DiscoverPartitionSchemasAsync(ctx.DataSource); + + foreach (var schema in schemas) + { + // The 'user' schema mirrors INTO itself by construction -- no trigger needed there. + if (string.Equals(schema, "user", StringComparison.OrdinalIgnoreCase)) + continue; + + ctx.Logger.LogInformation("Repair v25: installing mirror trigger on {Schema}", schema); + + await using (var cmd = ctx.DataSource.CreateCommand($""" + DROP TRIGGER IF EXISTS mesh_node_mirror_access_objects ON "{schema}".mesh_nodes; + CREATE TRIGGER mesh_node_mirror_access_objects + AFTER INSERT OR UPDATE OR DELETE ON "{schema}".mesh_nodes + FOR EACH ROW EXECUTE FUNCTION public.mirror_access_object_to_user_schema(); + """)) + { + await cmd.ExecuteNonQueryAsync(); + } + + // Backfill: copy every existing access-object row that isn't already in user.mesh_nodes. + // We rely on the ON CONFLICT DO NOTHING here because the source partition is the + // source of truth for content -- we don't want a backfill pass to clobber a freshly + // updated row in the user index that the trigger already mirrored. + await using (var cmd = ctx.DataSource.CreateCommand($""" + INSERT INTO "user".mesh_nodes + (namespace, id, name, node_type, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node) + SELECT namespace, id, name, node_type, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node + FROM "{schema}".mesh_nodes + WHERE node_type IN ('User','Group','Role','VUser') + ON CONFLICT (namespace, id) DO NOTHING; + """)) + { + var n = await cmd.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation( + "Repair v25: {Schema} backfilled {Count} access-object row(s)", schema, n); + } + } + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V26_AddNotificationsSatelliteTable.cs b/memex/aspire/Memex.Database.Migration/Migrations/V26_AddNotificationsSatelliteTable.cs new file mode 100644 index 000000000..cd8a21c65 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V26_AddNotificationsSatelliteTable.cs @@ -0,0 +1,58 @@ +using MeshWeaver.Hosting.PostgreSql; +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Adds the notifications satellite table to every existing partition +/// schema. Code-side this mapping was added to +/// PartitionDefinition.StandardTableMappings ("_Notification" → +/// "notifications") so new partitions get the table automatically via +/// PostgreSqlPartitionStorageProvider.EnsureSchemaAsync's first-touch +/// init. But existing partition schemas only re-run that init when a fresh +/// pod processes the partition for the first time after deploy — until then, +/// any attempt to write a Notification would hit a missing-table error. +/// +/// This migration walks every content-partition schema (each has a +/// mesh_nodes table) and runs +/// with +/// just notifications as the target. The function uses +/// CREATE TABLE IF NOT EXISTS + DROP/CREATE TRIGGER, so it's +/// idempotent and safe to re-run. +/// +/// No data move — this is a structure add only. Notifications are a new +/// satellite type; nothing has been written to mesh_nodes for them +/// (the routing was correct from day one). +/// +public sealed class V26_AddNotificationsSatelliteTable : IMigration +{ + public int Version => 26; + public string Description => "Add `notifications` satellite table to every partition schema"; + + public async Task RunAsync(MigrationContext ctx) + { + var schemas = await SchemaHelpers.DiscoverPartitionSchemasAsync(ctx.DataSource); + if (schemas.Count == 0) + { + ctx.Logger.LogInformation( + "V26: no content-partition schemas found — skipping (fresh DB; first-touch init will create the table per-partition)"); + return; + } + + var satelliteTables = new[] { "notifications" }; + + var created = 0; + foreach (var schema in schemas) + { + await using var schemaDs = SchemaHelpers.BuildSchemaDataSource(ctx.ConnectionString, schema); + await PostgreSqlSchemaInitializer.CreateSatelliteTablesAsync( + schemaDs, ctx.Options, satelliteTables); + created++; + ctx.Logger.LogInformation( + "V26: ensured notifications table in schema '{Schema}'", schema); + } + + ctx.Logger.LogInformation( + "V26: notifications satellite table ensured across {Count} partition schema(s)", created); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V27_RenameUserSchemaToAuthAndMirrorApiTokens.cs b/memex/aspire/Memex.Database.Migration/Migrations/V27_RenameUserSchemaToAuthAndMirrorApiTokens.cs new file mode 100644 index 000000000..7fcfae31a --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V27_RenameUserSchemaToAuthAndMirrorApiTokens.cs @@ -0,0 +1,190 @@ +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Renames the global user schema to auth and extends the +/// access-object mirror trigger to cover ApiToken nodes as well as +/// the existing User / Group / Role / VUser set. +/// +/// Why: auth lookups (token validation, GetTokensForUser, +/// user-by-email) currently either fan out across every per-user partition +/// or hit a partial mirror (the user schema covers identities but +/// not tokens). Centralising every auth-related node in a single schema +/// makes each lookup a constant-cost single-schema query and removes the +/// last cross-partition fan-out from the security hot path. The schema +/// name auth reflects this broader scope (identities + credentials). +/// +/// How: +/// +/// ALTER SCHEMA "user" RENAME TO auth — atomic, all FK / +/// trigger references update in-place. +/// CREATE OR REPLACE FUNCTION public.mirror_access_object_to_auth_schema +/// — same body as the previous ..._user_schema function but writes to +/// "auth".mesh_nodes and includes 'ApiToken' in the +/// node_type filter. +/// For each partition: drop the old trigger (pointing at the +/// _user_schema function) and recreate it pointing at the new +/// _auth_schema function. Idempotent +/// (DROP TRIGGER IF EXISTS). +/// Backfill ApiToken rows from every partition into auth +/// (existing User / Group / Role / VUser rows are already there from +/// V25 — the rename keeps them). +/// Drop the now-unreferenced mirror_access_object_to_user_schema +/// function. +/// +/// +/// Idempotent: rename is no-op when auth already exists; +/// function + trigger creations use CREATE OR REPLACE / +/// DROP IF EXISTS; backfill uses ON CONFLICT DO NOTHING. +/// Safe to re-run. +/// +public sealed class V27_RenameUserSchemaToAuthAndMirrorApiTokens : IMigration +{ + public int Version => 27; + + public string Description => + "Rename user schema to auth; add ApiToken to the access-object mirror trigger"; + + private static readonly string[] AccessObjectNodeTypes = + { "User", "Group", "Role", "VUser", "ApiToken" }; + + public async Task RunAsync(MigrationContext ctx) + { + var hasUser = await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, "user"); + var hasAuth = await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, "auth"); + + if (hasUser && hasAuth) + { + // Both exist — earlier partial run + fresh-DB init layered. Merge + // the user rows into auth (ON CONFLICT DO NOTHING — auth wins for + // any contended key, since auth is where new writes have been + // landing), then drop user. + ctx.Logger.LogInformation( + "Repair v27: both 'user' and 'auth' schemas exist — merging user → auth and dropping user"); + await using (var cmd = ctx.DataSource.CreateCommand(""" + INSERT INTO "auth".mesh_nodes + (namespace, id, name, node_type, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node) + SELECT namespace, id, name, node_type, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node + FROM "user".mesh_nodes + ON CONFLICT (namespace, id) DO NOTHING; + """)) + { + await cmd.ExecuteNonQueryAsync(); + } + await using (var cmd = ctx.DataSource.CreateCommand("""DROP SCHEMA "user" CASCADE;""")) + { + await cmd.ExecuteNonQueryAsync(); + } + } + else if (hasUser && !hasAuth) + { + ctx.Logger.LogInformation("Repair v27: renaming 'user' schema to 'auth'"); + await using var cmd = ctx.DataSource.CreateCommand("""ALTER SCHEMA "user" RENAME TO "auth";"""); + await cmd.ExecuteNonQueryAsync(); + } + else if (!hasUser && !hasAuth) + { + // Fresh DB ordering quirk — schema init hasn't created either yet. + // The migration runner reruns next time. Bail out softly. + ctx.Logger.LogInformation( + "Repair v27: neither 'user' nor 'auth' schema exists yet — skipping (will re-run next start)"); + return; + } + // else: hasAuth only → assume an earlier run finished; just refresh the + // function + triggers + backfill below to make idempotent. + + // Trigger function: writes into auth schema, covers User/Group/Role/VUser/ApiToken. + await using (var cmd = ctx.DataSource.CreateCommand(""" + CREATE OR REPLACE FUNCTION public.mirror_access_object_to_auth_schema() + RETURNS TRIGGER AS $$ + BEGIN + IF TG_OP = 'DELETE' THEN + IF OLD.node_type IN ('User','Group','Role','VUser','ApiToken') THEN + DELETE FROM "auth".mesh_nodes + WHERE namespace = OLD.namespace AND id = OLD.id; + END IF; + RETURN OLD; + END IF; + + IF NEW.node_type IN ('User','Group','Role','VUser','ApiToken') THEN + INSERT INTO "auth".mesh_nodes + (namespace, id, name, node_type, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node) + VALUES (NEW.namespace, NEW.id, NEW.name, NEW.node_type, NEW.category, NEW.icon, NEW.display_order, + NEW.last_modified, NEW.version, NEW.state, NEW.content, NEW.desired_id, NEW.main_node) + ON CONFLICT (namespace, id) DO UPDATE SET + name = EXCLUDED.name, + node_type = EXCLUDED.node_type, + category = EXCLUDED.category, + icon = EXCLUDED.icon, + display_order = EXCLUDED.display_order, + last_modified = EXCLUDED.last_modified, + version = EXCLUDED.version, + state = EXCLUDED.state, + content = EXCLUDED.content, + desired_id = EXCLUDED.desired_id, + main_node = EXCLUDED.main_node; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """)) + { + await cmd.ExecuteNonQueryAsync(); + } + + // Rewire every partition's mirror trigger to point at the new function. + // The trigger NAME stays 'mesh_node_mirror_access_objects' so future + // re-runs of V25 / schema init are no-ops (idempotent). + var schemas = await SchemaHelpers.DiscoverPartitionSchemasAsync(ctx.DataSource); + foreach (var schema in schemas) + { + // The 'auth' schema mirrors INTO itself by construction — no trigger needed there. + if (string.Equals(schema, "auth", StringComparison.OrdinalIgnoreCase)) + continue; + + ctx.Logger.LogInformation("Repair v27: rewiring mirror trigger on {Schema} → auth", schema); + + await using (var cmd = ctx.DataSource.CreateCommand($""" + DROP TRIGGER IF EXISTS mesh_node_mirror_access_objects ON "{schema}".mesh_nodes; + CREATE TRIGGER mesh_node_mirror_access_objects + AFTER INSERT OR UPDATE OR DELETE ON "{schema}".mesh_nodes + FOR EACH ROW EXECUTE FUNCTION public.mirror_access_object_to_auth_schema(); + """)) + { + await cmd.ExecuteNonQueryAsync(); + } + + // Backfill ApiToken rows that weren't covered by V25. + // ON CONFLICT DO NOTHING — the trigger already mirrors live writes; + // backfill is just for rows that existed pre-V27. + await using (var cmd = ctx.DataSource.CreateCommand($""" + INSERT INTO "auth".mesh_nodes + (namespace, id, name, node_type, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node) + SELECT namespace, id, name, node_type, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node + FROM "{schema}".mesh_nodes + WHERE node_type = 'ApiToken' + ON CONFLICT (namespace, id) DO NOTHING; + """)) + { + var n = await cmd.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation( + "Repair v27: {Schema} backfilled {Count} ApiToken row(s)", schema, n); + } + } + + // Drop the old function — nothing references it after the trigger + // rewire above. Tolerate absence (already dropped on rerun). + await using (var cmd = ctx.DataSource.CreateCommand( + "DROP FUNCTION IF EXISTS public.mirror_access_object_to_user_schema();")) + { + await cmd.ExecuteNonQueryAsync(); + } + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V28_RenameOrganizationToSpace.cs b/memex/aspire/Memex.Database.Migration/Migrations/V28_RenameOrganizationToSpace.cs new file mode 100644 index 000000000..a4916b795 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V28_RenameOrganizationToSpace.cs @@ -0,0 +1,173 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Renames the Organization NodeType to Space and removes the now-redundant +/// per-tenant Partition MeshNodes from admin.mesh_nodes. +/// +/// Why: Spaces (formerly Organizations) and Users are the tenant roots — +/// each owns its own Postgres schema. The dedicated Partition MeshNode emitted by +/// the post-creation handlers was duplicate metadata: the routing layer already derives +/// the schema name from the first path segment via PgPartitionCache.Probe + +/// PostgreSqlPathRoutingAdapter.AdapterForWriteState (PendingCreate state → +/// lazy CREATE SCHEMA). Dropping the explicit records simplifies onboarding and removes +/// one source of partition-routing truth. +/// +/// How: +/// +/// UPDATE node_type='Organization' → 'Space' in every partition's +/// mesh_nodes. +/// DELETE the orphaned NodeType-registry row (namespace='', +/// id='Organization', node_type='NodeType') left behind because +/// StaticMeshNodeListProvider upserts but never deletes. +/// DELETE per-tenant Partition rows from admin.mesh_nodes. +/// Keep the system-partition records (Admin, Auth, Portal, +/// Kernel, _Access, _Activity, _UserActivity, +/// _Thread) — those carry non-derivable routing config (TableMappings, +/// Versioned, special satellite-as-primary mapping). +/// CREATE OR REPLACE the V27 mirror function with 'Space' added to the +/// filter. The trigger NAME on every partition is unchanged +/// (mesh_node_mirror_access_objects) so the existing trigger picks up the +/// new function body automatically; no per-partition re-iteration needed. +/// Backfill existing Space rows from every partition into auth.mesh_nodes +/// (matches V27's ApiToken backfill — the trigger only covers writes from V28 +/// onward, so pre-V28 Organization-now-Space rows need an explicit copy). +/// +/// +/// Idempotent: all UPDATE/DELETE statements are restartable; the function +/// is CREATE OR REPLACE; the backfill uses ON CONFLICT DO NOTHING. Safe to +/// re-run. +/// +public sealed class V28_RenameOrganizationToSpace : IMigration +{ + public int Version => 28; + public string Description => "Rename Organization → Space; drop per-tenant Partition rows; extend auth mirror to Space"; + + private static readonly string[] SystemPartitionIds = + ["Admin", "Auth", "Portal", "Kernel", "_Access", "_Activity", "_UserActivity", "_Thread"]; + + public async Task RunAsync(MigrationContext ctx) + { + var schemas = await SchemaHelpers.DiscoverPartitionSchemasAsync(ctx.DataSource); + if (schemas.Count == 0) + { + ctx.Logger.LogInformation("Repair v28: no content-partition schemas found — skipping rename phase"); + } + + // 1 + 2. Rename Organization → Space and remove the orphaned NodeType-registry row. + foreach (var schema in schemas) + { + int renamed; + await using (var cmd = ctx.DataSource.CreateCommand($""" + UPDATE "{schema}".mesh_nodes + SET node_type = 'Space' + WHERE node_type = 'Organization' + """)) + { + renamed = await cmd.ExecuteNonQueryAsync(); + } + + int deleted; + await using (var cmd = ctx.DataSource.CreateCommand($""" + DELETE FROM "{schema}".mesh_nodes + WHERE namespace = '' + AND id = 'Organization' + AND node_type = 'NodeType' + """)) + { + deleted = await cmd.ExecuteNonQueryAsync(); + } + + if (renamed > 0 || deleted > 0) + ctx.Logger.LogInformation( + "Repair v28: \"{Schema}\" — renamed {Renamed} Organization → Space row(s); dropped {Deleted} orphaned NodeType-registry row(s)", + schema, renamed, deleted); + } + + // 3. Drop per-tenant Partition MeshNodes from admin (keep system partitions). + await using (var cmd = ctx.DataSource.CreateCommand($""" + DELETE FROM admin.mesh_nodes + WHERE node_type = 'Partition' + AND namespace = 'Admin/Partition' + AND id <> ALL($1) + """)) + { + cmd.Parameters.AddWithValue(SystemPartitionIds); + var partitionRowsDeleted = await cmd.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation( + "Repair v28: dropped {Count} per-tenant Partition row(s) from admin.mesh_nodes (system partitions retained)", + partitionRowsDeleted); + } + + // 4. Extend V27 mirror function to include 'Space'. Function name unchanged → + // the per-partition trigger picks up the new body without re-creating + // the trigger. If V27 hasn't installed the function yet (running on a + // pre-V27 DB), this CREATE OR REPLACE installs the function fresh; V27's + // own RunAsync will then re-create it identically (idempotent). + await using (var cmd = ctx.DataSource.CreateCommand(""" + CREATE OR REPLACE FUNCTION public.mirror_access_object_to_auth_schema() + RETURNS TRIGGER AS $$ + BEGIN + IF TG_OP = 'DELETE' THEN + IF OLD.node_type IN ('User','Group','Role','VUser','ApiToken','Space') THEN + DELETE FROM "auth".mesh_nodes + WHERE namespace = OLD.namespace AND id = OLD.id; + END IF; + RETURN OLD; + END IF; + + IF NEW.node_type IN ('User','Group','Role','VUser','ApiToken','Space') THEN + INSERT INTO "auth".mesh_nodes + (namespace, id, name, node_type, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node) + VALUES (NEW.namespace, NEW.id, NEW.name, NEW.node_type, NEW.category, NEW.icon, NEW.display_order, + NEW.last_modified, NEW.version, NEW.state, NEW.content, NEW.desired_id, NEW.main_node) + ON CONFLICT (namespace, id) DO UPDATE SET + name = EXCLUDED.name, + node_type = EXCLUDED.node_type, + category = EXCLUDED.category, + icon = EXCLUDED.icon, + display_order = EXCLUDED.display_order, + last_modified = EXCLUDED.last_modified, + version = EXCLUDED.version, + state = EXCLUDED.state, + content = EXCLUDED.content, + desired_id = EXCLUDED.desired_id, + main_node = EXCLUDED.main_node; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """)) + { + await cmd.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation("Repair v28: extended mirror_access_object_to_auth_schema to include 'Space'"); + } + + // 5. Backfill Space rows that existed before V28 ran (the trigger only covers + // writes from now on; existing rows need a one-shot copy). The auth schema + // mirrors INTO itself by construction — skip the auth schema in the loop. + foreach (var schema in schemas) + { + if (string.Equals(schema, "auth", StringComparison.OrdinalIgnoreCase)) + continue; + + await using var cmd = ctx.DataSource.CreateCommand($""" + INSERT INTO "auth".mesh_nodes + (namespace, id, name, node_type, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node) + SELECT namespace, id, name, node_type, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node + FROM "{schema}".mesh_nodes + WHERE node_type = 'Space' + ON CONFLICT (namespace, id) DO NOTHING + """); + var backfilled = await cmd.ExecuteNonQueryAsync(); + if (backfilled > 0) + ctx.Logger.LogInformation( + "Repair v28: {Schema} backfilled {Count} Space row(s) into auth.mesh_nodes", + schema, backfilled); + } + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V29_PinDocsForExistingUsers.cs b/memex/aspire/Memex.Database.Migration/Migrations/V29_PinDocsForExistingUsers.cs new file mode 100644 index 000000000..0da697b7d --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V29_PinDocsForExistingUsers.cs @@ -0,0 +1,55 @@ +using MeshWeaver.Hosting.PostgreSql; +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Gives existing users the same Documentation shortcuts that new users now receive at onboarding +/// (UserOnboardingService seeds the four doc sections into User.PinnedPaths). +/// +/// Non-destructive and idempotent: a User node is only updated when its pinnedPaths is +/// missing/empty, OR when it still carries the legacy single ["Doc"] root pin (which an older +/// onboarding seeded and which does not render as a card). Users who have curated their own pins are +/// left untouched. Runs across every content-partition schema. +/// +public sealed class V29_PinDocsForExistingUsers : IMigration +{ + public int Version => 29; + public string Description => "Pin documentation sections for existing users (empty or legacy [\"Doc\"] pins)"; + + // The four section landing pages — each renders as a card on the Pinned tab. + private const string DocPins = "[\"Doc/Architecture\", \"Doc/DataMesh\", \"Doc/GUI\", \"Doc/AI\"]"; + + public async Task RunAsync(MigrationContext ctx) + { + var schemas = await SchemaHelpers.DiscoverPartitionSchemasAsync(ctx.DataSource); + foreach (var schema in schemas) + { + try + { + await using var ds = SchemaHelpers.BuildSchemaDataSource(ctx.ConnectionString, schema); + await using var cmd = ds.CreateCommand( + """ + UPDATE mesh_nodes + SET content = jsonb_set(coalesce(content, '{}'::jsonb), '{pinnedPaths}', $1::jsonb, true), + last_modified = NOW(), + version = version + 1 + WHERE node_type = 'User' + AND (content->'pinnedPaths' IS NULL + OR jsonb_typeof(content->'pinnedPaths') <> 'array' + OR jsonb_array_length(content->'pinnedPaths') = 0 + OR content->'pinnedPaths' = '["Doc"]'::jsonb) + """); + cmd.Parameters.AddWithValue(DocPins); + var updated = await cmd.ExecuteNonQueryAsync(); + if (updated > 0) + ctx.Logger.LogInformation( + "[V29] Pinned documentation sections for {Count} user(s) in schema {Schema}", updated, schema); + } + catch (Exception ex) + { + ctx.Logger.LogWarning(ex, "[V29] Skipped schema {Schema} while pinning docs", schema); + } + } + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V30_EnsurePartitionSchemaStoredProc.cs b/memex/aspire/Memex.Database.Migration/Migrations/V30_EnsurePartitionSchemaStoredProc.cs new file mode 100644 index 000000000..9b8b73cb7 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V30_EnsurePartitionSchemaStoredProc.cs @@ -0,0 +1,44 @@ +using MeshWeaver.Hosting.PostgreSql; +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Installs public.ensure_partition_schema(partition_name text) — the single +/// source of truth for per-partition provisioning. The proc idempotently creates a +/// partition's schema + {partition}.mesh_nodes + every satellite table from +/// PartitionDefinition.StandardTableMappings + the permission-rebuild functions +/// and notify/mirror/history triggers, byte-faithful to the C# +/// PostgreSqlSchemaInitializer.GetVersionedPartitionDdl / +/// GetSatelliteTableScript bodies it embeds. +/// +/// Why a migration too? SchemaInitialization.RunAsync already calls +/// PostgreSqlSchemaInitializer.InitializeAsync on the public schema on every run, +/// which now also CREATE OR REPLACEs this proc — so fresh and existing DBs get it. +/// This migration is the explicit, versioned anchor for the proc (the documented place to +/// evolve it) and guarantees the function exists even on the data-repair path. The runtime +/// per-partition provisioner (PostgreSqlPartitionStorageProvider.EnsureSchemaAsync) +/// and the eager Space-create hook both SELECT public.ensure_partition_schema(@partition). +/// +/// Idempotent: pure CREATE OR REPLACE FUNCTION. Safe to re-run; does +/// not touch any partition schema (the proc is only invoked lazily/eagerly when a partition +/// is actually provisioned). +/// +public sealed class V30_EnsurePartitionSchemaStoredProc : IMigration +{ + public int Version => 30; + public string Description => + "Install public.ensure_partition_schema(text) — single source of truth for per-partition DDL"; + + public async Task RunAsync(MigrationContext ctx) + { + var procDdl = PostgreSqlSchemaInitializer.GetEnsurePartitionSchemaProcScript( + ctx.Options.VectorDimensions); + + await using var cmd = ctx.DataSource.CreateCommand(procDdl); + await cmd.ExecuteNonQueryAsync(); + + ctx.Logger.LogInformation( + "Repair v30: installed public.ensure_partition_schema(text) stored proc"); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V31_UnifyUserMirrorIntoAuthAndRelocateContent.cs b/memex/aspire/Memex.Database.Migration/Migrations/V31_UnifyUserMirrorIntoAuthAndRelocateContent.cs new file mode 100644 index 000000000..da92dd8af --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V31_UnifyUserMirrorIntoAuthAndRelocateContent.cs @@ -0,0 +1,241 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Removes the stray user schema so the auth-lookup mirror lives in exactly ONE place: +/// the auth schema (partition namespace Auth). +/// +/// Why this exists. Onboarding kept writing a second "catalog mirror" node +/// new MeshNode(username, "User") (namespace User). A write to the unregistered +/// User first-segment lazily provisioned a user schema (distinct from the auth +/// schema V27 created), and subsequent content created under User/{username}/… piled up +/// there too — e.g. User/rsalzmann/ReinsuranceContractCheck (a compiled NodeType + its +/// Source/Release/instances), User/{username}/HelloWorld, User/{username}/_Access/…. +/// The redundant onboarding write is removed in the same change as this migration +/// (UserOnboardingService no longer writes the User-namespace mirror), and the new +/// PartitionWriteGuardValidator blocks any future non-system write into User/Auth. +/// +/// What it does. +/// +/// Discovers every {username} that owns content under User/{username}/… +/// across the standard partition tables present in the stray user schema. +/// For each, ensures the user's own partition schema exists +/// (public.ensure_partition_schema) and relocates the content there: the leading +/// User/ is stripped from namespace, and node_type / main_node +/// values that themselves start with User/ (a NodeType instance pointing at its type, +/// or a satellite's parent pointer) are rewritten too. ON CONFLICT (namespace,id) DO +/// NOTHING so any content already at the canonical root path is never clobbered. +/// Drops the stray user schema — but ONLY if every owner relocated cleanly. Its +/// namespace='User' User/Group/Role/VUser/ApiToken rows are redundant (the canonical +/// mirror is auth, kept current by the V27 trigger), so CASCADE safely clears +/// the leftover rows + the schema's mirror trigger. +/// +/// +/// Compiled NodeType note. A relocated NodeType's content still carries +/// compiledSources / currentSourceVersions keys + latestReleasePath / +/// latestAssemblyPath that reference the old User/{username}/… path. We deliberately +/// do NOT rewrite that JSON: the source children move to the new path, so on first access the +/// NodeType sees a source-version key mismatch, marks itself dirty, and recompiles against the +/// new path — self-healing without fragile in-place jsonb surgery. The instance's node_type +/// column IS rewritten (above) so it re-binds to the moved type. +/// +/// Idempotent. No user schema → no-op. Re-run after a partial run picks up +/// whatever is left; the moves are ON CONFLICT DO NOTHING and the drop is gated on a clean +/// sweep. +/// +public sealed class V31_UnifyUserMirrorIntoAuthAndRelocateContent : IMigration +{ + public int Version => 31; + + public string Description => + "Relocate User/{username}/… content to {username} partitions and drop the stray 'user' schema (auth is the single mirror)"; + + /// + /// The standard partition tables (mesh_nodes + satellites) created by + /// public.ensure_partition_schema. Only those actually present in the stray + /// user schema are processed. + /// + private static readonly string[] PartitionTables = + { + "mesh_nodes", "access", "activities", "user_activities", + "threads", "annotations", "notifications", "code" + }; + + public async Task RunAsync(MigrationContext ctx) + { + if (!await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, "user")) + { + ctx.Logger.LogInformation("Repair v31: no stray 'user' schema — nothing to do"); + return; + } + + var tables = await TablesPresentAsync(ctx, "user"); + + // 1. Discover every username owning content under User/{username}/… across all tables. + var usernames = new HashSet(StringComparer.Ordinal); + foreach (var table in tables) + { + await using var cmd = ctx.DataSource.CreateCommand($""" + SELECT DISTINCT split_part(namespace, '/', 2) AS username + FROM "user"."{table}" + WHERE namespace LIKE 'User/%' + """); + await using var rdr = await cmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) + { + var u = rdr.IsDBNull(0) ? null : rdr.GetString(0); + if (!string.IsNullOrEmpty(u)) + usernames.Add(u); + } + } + + if (usernames.Count == 0) + ctx.Logger.LogInformation( + "Repair v31: 'user' schema holds no User/{{username}}/… content — only redundant mirror rows remain"); + + // 2. Relocate each owner's content into their own partition. + var allMoved = true; + foreach (var username in usernames) + { + var targetSchema = SchemaHelpers.SanitizeSchemaName(username); + if (string.IsNullOrEmpty(targetSchema)) + { + ctx.Logger.LogWarning( + "Repair v31: cannot derive a schema for user '{User}' — leaving its content in 'user'", username); + allMoved = false; + continue; + } + + // Ensure the target partition's schema + standard tables exist (idempotent). + await using (var ensureCmd = ctx.DataSource.CreateCommand("SELECT public.ensure_partition_schema(@s)")) + { + ensureCmd.Parameters.AddWithValue("s", targetSchema); + await ensureCmd.ExecuteNonQueryAsync(); + } + + var movedForUser = 0; + foreach (var table in tables) + { + if (!await TableExistsAsync(ctx, targetSchema, table)) + continue; // target lacks this satellite — nothing routes there + + var columns = await ColumnsAsync(ctx, "user", table); + if (columns.Count == 0) + continue; + + // Drop columns that are GENERATED ALWAYS in the TARGET (e.g. the computed `path` + // column created by public.ensure_partition_schema). Postgres rejects an explicit + // INSERT into such a column (SqlState 428C9 "cannot insert a non-DEFAULT value into + // column"); the target recomputes them from the inserted namespace/id, so they must + // be omitted from both the column list and the SELECT. + var generated = await GeneratedColumnsAsync(ctx, targetSchema, table); + if (generated.Count > 0) + columns = columns.Where(c => !generated.Contains(c)).ToList(); + if (columns.Count == 0) + continue; + + // Rewrite only path-shaped columns; pass everything else through verbatim. + // 'User/' is 5 chars → substring(... from 6) drops the prefix. + var selectList = string.Join(", ", columns.Select(c => c switch + { + "namespace" => "substring(namespace from 6)", + "node_type" => "CASE WHEN node_type LIKE 'User/%' THEN substring(node_type from 6) ELSE node_type END", + "main_node" => "CASE WHEN main_node LIKE 'User/%' THEN substring(main_node from 6) ELSE main_node END", + _ => $"\"{c}\"" + })); + var colList = string.Join(", ", columns.Select(c => $"\"{c}\"")); + + await using var moveCmd = ctx.DataSource.CreateCommand($""" + INSERT INTO "{targetSchema}"."{table}" ({colList}) + SELECT {selectList} + FROM "user"."{table}" + WHERE namespace = 'User/' || @u OR namespace LIKE 'User/' || @u || '/%' + ON CONFLICT (namespace, id) DO NOTHING + """); + moveCmd.Parameters.AddWithValue("u", username); + var moved = await moveCmd.ExecuteNonQueryAsync(); + movedForUser += moved; + if (moved > 0) + ctx.Logger.LogInformation( + "Repair v31: relocated {Count} row(s) user.{Table} → {Schema}.{Table} for '{User}'", + moved, table, targetSchema, table, username); + } + + ctx.Logger.LogInformation( + "Repair v31: relocated {Total} content row(s) for '{User}' into '{Schema}'", + movedForUser, username, targetSchema); + } + + // 3. Drop the stray schema only after a clean sweep. + if (allMoved) + { + await using var dropCmd = ctx.DataSource.CreateCommand("""DROP SCHEMA "user" CASCADE"""); + await dropCmd.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation( + "Repair v31: dropped stray 'user' schema — 'auth' is now the single auth-lookup mirror"); + } + else + { + ctx.Logger.LogWarning( + "Repair v31: left the 'user' schema in place because some content could not be relocated — re-run after resolving"); + } + } + + private static async Task> TablesPresentAsync(MigrationContext ctx, string schema) + { + var present = new List(); + foreach (var t in PartitionTables) + if (await TableExistsAsync(ctx, schema, t)) + present.Add(t); + return present; + } + + private static async Task TableExistsAsync(MigrationContext ctx, string schema, string table) + { + await using var cmd = ctx.DataSource.CreateCommand(""" + SELECT 1 FROM information_schema.tables + WHERE table_schema = @s AND table_name = @t + LIMIT 1 + """); + cmd.Parameters.AddWithValue("s", schema); + cmd.Parameters.AddWithValue("t", table); + return await cmd.ExecuteScalarAsync() is not null; + } + + private static async Task> ColumnsAsync(MigrationContext ctx, string schema, string table) + { + var cols = new List(); + await using var cmd = ctx.DataSource.CreateCommand(""" + SELECT column_name FROM information_schema.columns + WHERE table_schema = @s AND table_name = @t + ORDER BY ordinal_position + """); + cmd.Parameters.AddWithValue("s", schema); + cmd.Parameters.AddWithValue("t", table); + await using var rdr = await cmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) + cols.Add(rdr.GetString(0)); + return cols; + } + + /// + /// Column names that are GENERATED ALWAYS in the given table (e.g. the computed + /// path from ensure_partition_schema). These cannot be targeted by an explicit + /// INSERT and must be excluded from the relocation copy. + /// + private static async Task> GeneratedColumnsAsync(MigrationContext ctx, string schema, string table) + { + var generated = new HashSet(StringComparer.Ordinal); + await using var cmd = ctx.DataSource.CreateCommand(""" + SELECT column_name FROM information_schema.columns + WHERE table_schema = @s AND table_name = @t AND is_generated = 'ALWAYS' + """); + cmd.Parameters.AddWithValue("s", schema); + cmd.Parameters.AddWithValue("t", table); + await using var rdr = await cmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) + generated.Add(rdr.GetString(0)); + return generated; + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V32_RepairAuthMirrorTriggerAndBackfill.cs b/memex/aspire/Memex.Database.Migration/Migrations/V32_RepairAuthMirrorTriggerAndBackfill.cs new file mode 100644 index 000000000..5097b38aa --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V32_RepairAuthMirrorTriggerAndBackfill.cs @@ -0,0 +1,94 @@ +using MeshWeaver.Hosting.PostgreSql; +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Repairs the auth-lookup mirror on databases that were provisioned FRESH and thus +/// fast-forwarded past the V25 / V27 repairs that originally installed the access-object +/// mirror trigger. +/// +/// The bug. The per-partition DDL (PostgreSqlSchemaInitializer.GetVersionedPartitionDdl) +/// installs the mesh_node_mirror_access_objects trigger ONLY IF the function +/// public.mirror_access_object_to_auth_schema() already exists. That function was +/// created only by the V27 *repair* migration — and MigrationRunner SKIPS all repairs +/// on a fresh DB. So fresh deployments ended up at db_version=31 with no mirror +/// function, no triggers on any partition, and an empty auth schema — every auth +/// lookup silently fell back to a cross-partition fan-out. (Confirmed on +/// memex.systemorph.com 2026-06-02: FUNC=0, triggers NONE, auth=0.) +/// +/// The repair (idempotent). +/// +/// Create the (fail-safe) mirror function — single-sourced from +/// . (Schema-init now +/// creates it too, so fresh DBs install triggers via ensure_partition_schema; this +/// migration covers partitions that were provisioned BEFORE the function existed.) +/// Ensure the auth mirror partition exists. +/// Install the trigger on every existing partition schema (skipping auth itself). +/// Backfill existing User / Group / Role / VUser / ApiToken rows into auth +/// (ON CONFLICT DO NOTHING — the trigger owns live writes). +/// +/// Skipped on fresh DBs (no legacy partitions to retrofit / nothing to backfill); the +/// always-run schema-init function + ensure_partition_schema trigger install cover those. +/// +public sealed class V32_RepairAuthMirrorTriggerAndBackfill : IMigration +{ + public int Version => 32; + + public string Description => + "Install auth-mirror function+trigger on existing partitions and backfill auth (fresh-DB repair)"; + + public async Task RunAsync(MigrationContext ctx) + { + // 1. (Re)create the mirror function — fail-safe, single-sourced with schema-init. + await using (var cmd = ctx.DataSource.CreateCommand( + PostgreSqlSchemaInitializer.GetAuthMirrorFunctionScript())) + { + await cmd.ExecuteNonQueryAsync(); + } + + // 2. Ensure the auth mirror partition exists (idempotent — no-op when present). + await using (var cmd = ctx.DataSource.CreateCommand( + "SELECT public.ensure_partition_schema('auth')")) + { + await cmd.ExecuteNonQueryAsync(); + } + + // 3 + 4. Install the trigger on every partition and backfill its access objects. + var schemas = await SchemaHelpers.DiscoverPartitionSchemasAsync(ctx.DataSource); + foreach (var schema in schemas) + { + // The 'auth' schema is the mirror target — it doesn't mirror into itself. + if (string.Equals(schema, "auth", StringComparison.OrdinalIgnoreCase)) + continue; + + await using (var cmd = ctx.DataSource.CreateCommand($""" + DROP TRIGGER IF EXISTS mesh_node_mirror_access_objects ON "{schema}".mesh_nodes; + CREATE TRIGGER mesh_node_mirror_access_objects + AFTER INSERT OR UPDATE OR DELETE ON "{schema}".mesh_nodes + FOR EACH ROW EXECUTE FUNCTION public.mirror_access_object_to_auth_schema(); + """)) + { + await cmd.ExecuteNonQueryAsync(); + } + + await using (var cmd = ctx.DataSource.CreateCommand($""" + INSERT INTO "auth".mesh_nodes + (namespace, id, name, node_type, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node) + SELECT namespace, id, name, node_type, category, icon, display_order, + last_modified, version, state, content, desired_id, main_node + FROM "{schema}".mesh_nodes + WHERE node_type IN ('User','Group','Role','VUser','ApiToken') + ON CONFLICT (namespace, id) DO NOTHING; + """)) + { + var n = await cmd.ExecuteNonQueryAsync(); + if (n > 0) + ctx.Logger.LogInformation( + "Repair v32: backfilled {Count} access-object row(s) from {Schema} into auth", n, schema); + } + } + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V33_SeedChatInputForExistingUsers.cs b/memex/aspire/Memex.Database.Migration/Migrations/V33_SeedChatInputForExistingUsers.cs new file mode 100644 index 000000000..709779312 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V33_SeedChatInputForExistingUsers.cs @@ -0,0 +1,124 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Seeds the per-user ThreadComposer singleton at {userId}/_Memex/ThreadComposer for every +/// existing user partition, so the chat composer's read RESOLVES instead of emitting a routing +/// NotFound that the GUI re-issues on a loop (the 2026-06-08 ThreadComposer event-storm class). +/// +/// New users get this seeded at onboarding (ThreadComposerSeedHandler); this repair +/// backfills users created before that handler existed. The node is a MAIN node under the +/// hidden _Memex "dotfile" namespace — _Memex is NOT a registered satellite suffix, +/// so the write and the path-based read BOTH hit mesh_nodes (contrast the dead +/// _ThreadTemplate, which split write→threads from read→mesh_nodes). +/// +/// Content is intentionally NULL. The node only needs to EXIST so routing resolves; +/// the composer falls back to its configuration defaults (no draft, default harness) until the +/// user's first interaction writes the real draft/selection. Avoids hand-serializing a +/// Thread payload (with its polymorphic $type) in SQL. +/// +/// Discovery mirrors V17: per-user schemas are those with a mesh_nodes table holding +/// the user-identity row (namespace='' node_type='User'). The auth mirror schema is +/// excluded explicitly (it carries MIRRORED User rows, not a real per-user partition). Idempotent +/// via an existence check + ON CONFLICT (namespace, id) DO NOTHING. +/// +public sealed class V33_SeedThreadComposerForExistingUsers : IMigration +{ + public int Version => 33; + public string Description => "Seed {user}/_Memex/ThreadComposer singleton for every existing user partition"; + + public async Task RunAsync(MigrationContext ctx) + { + // Candidate schemas: any with a mesh_nodes table, excluding infra/mirror schemas. + var schemas = new List(); + await using (var discoverCmd = ctx.DataSource.CreateCommand(""" + SELECT t.table_schema + FROM information_schema.tables t + WHERE t.table_name = 'mesh_nodes' + AND t.table_schema NOT IN + ('information_schema','pg_catalog','pg_toast','public','admin','auth','doc') + AND t.table_schema NOT LIKE '%\_versions' + ORDER BY t.table_schema + """)) + await using (var rdr = await discoverCmd.ExecuteReaderAsync()) + { + while (await rdr.ReadAsync()) + schemas.Add(rdr.GetString(0)); + } + + // Keep only real per-user partitions: schemas whose mesh_nodes hold the + // user-identity row (namespace='', node_type='User'). Capture the canonical + // (original-case) userId from that row so the seeded path matches what the + // app reads (never derive it from the lower-cased schema name). + var confirmed = new List<(string Schema, string UserId)>(); + foreach (var schema in schemas) + { + var quotedSchema = schema.Replace("\"", "\"\""); + string? userId = null; + await using (var idCmd = ctx.DataSource.CreateCommand($""" + SELECT id FROM "{quotedSchema}".mesh_nodes + WHERE namespace = '' AND node_type = 'User' + LIMIT 1 + """)) + await using (var idRdr = await idCmd.ExecuteReaderAsync()) + { + if (await idRdr.ReadAsync()) + userId = idRdr.IsDBNull(0) ? null : idRdr.GetString(0); + } + if (!string.IsNullOrEmpty(userId)) + confirmed.Add((schema, userId)); + } + + ctx.Logger.LogInformation( + "Repair v33: found {Count} per-user schema(s): [{Schemas}]", + confirmed.Count, string.Join(", ", confirmed.Select(t => t.Schema))); + + var inserted = 0; + foreach (var (schema, userId) in confirmed) + { + var quotedSchema = schema.Replace("\"", "\"\""); + var ns = $"{userId}/_Memex"; + const string id = "ThreadComposer"; + var mainNode = $"{userId}/_Memex/ThreadComposer"; + + bool exists; + await using (var chkCmd = ctx.DataSource.CreateCommand($""" + SELECT 1 FROM "{quotedSchema}".mesh_nodes + WHERE namespace = $1 AND id = $2 + LIMIT 1 + """)) + { + chkCmd.Parameters.AddWithValue(ns); + chkCmd.Parameters.AddWithValue(id); + exists = await chkCmd.ExecuteScalarAsync() is not null; + } + if (exists) + { + ctx.Logger.LogDebug("Repair v33: '{Schema}' — ThreadComposer already present, skipping", schema); + continue; + } + + // state=2 (Active). content NULL. path is GENERATED ALWAYS, so it's omitted. + await using (var insertCmd = ctx.DataSource.CreateCommand($""" + INSERT INTO "{quotedSchema}".mesh_nodes + (namespace, id, name, node_type, state, content, main_node, last_modified, version) + VALUES ($1, $2, 'Chat Input', 'ThreadComposer', 2, NULL, $3, now(), 1) + ON CONFLICT (namespace, id) DO NOTHING + """)) + { + insertCmd.Parameters.AddWithValue(ns); + insertCmd.Parameters.AddWithValue(id); + insertCmd.Parameters.AddWithValue(mainNode); + var rows = await insertCmd.ExecuteNonQueryAsync(); + if (rows > 0) + { + ctx.Logger.LogInformation("Repair v33: '{Schema}' — seeded {Path}", schema, mainNode); + inserted++; + } + } + } + + ctx.Logger.LogInformation("Repair v33: done — {Inserted} ThreadComposer node(s) seeded", inserted); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V34_TypeOrphanPartitionRootsAsSpace.cs b/memex/aspire/Memex.Database.Migration/Migrations/V34_TypeOrphanPartitionRootsAsSpace.cs new file mode 100644 index 000000000..3f7e4cf91 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V34_TypeOrphanPartitionRootsAsSpace.cs @@ -0,0 +1,71 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Repairs typeless partition-root nodes by setting node_type='Space'. +/// +/// Symptom: a partition root (e.g. AgenticPensions) exists with +/// namespace='' , state=Active but no node_type. Because the +/// per-node hub resolves its config from the node type +/// (NodeTypeEnrichmentHelpers: a null type falls back to the default hub config, NOT +/// the Space config), a typeless root gets no AddContentCollections() — so its +/// /Files view has no backing and spins — and it is invisible in the Spaces catalog +/// (which queries nodeType:Space). These roots predate proper Space creation +/// (CreateLayoutArea always sets NodeType now) and V28's Organization→Space +/// rename couldn't catch them (it matched node_type='Organization'). +/// +/// Fix: for every content-partition schema, set node_type='Space' on the root +/// row whose node_type IS NULL. This is safe and precise: +/// +/// User-partition roots are node_type='User' (not NULL) → untouched. +/// Already-typed Space roots are node_type='Space' (not NULL) → untouched. +/// System schemas (public/admin/auth/doc/*_versions) are excluded by name. +/// +/// Idempotent (the NULL guard means a re-run is a no-op). Casing-safe: operates on the +/// actual schema names + stored row, never deriving paths from lower-cased names. +/// +public sealed class V34_TypeOrphanPartitionRootsAsSpace : IMigration +{ + public int Version => 34; + public string Description => "Set node_type='Space' on typeless partition-root nodes (partitions that lost their type)"; + + public async Task RunAsync(MigrationContext ctx) + { + var schemas = new List(); + await using (var discoverCmd = ctx.DataSource.CreateCommand(""" + SELECT t.table_schema + FROM information_schema.tables t + WHERE t.table_name = 'mesh_nodes' + AND t.table_schema NOT IN + ('information_schema','pg_catalog','pg_toast','public','admin','auth','doc') + AND t.table_schema NOT LIKE '%\_versions' + ORDER BY t.table_schema + """)) + await using (var rdr = await discoverCmd.ExecuteReaderAsync()) + { + while (await rdr.ReadAsync()) + schemas.Add(rdr.GetString(0)); + } + + var fixedCount = 0; + foreach (var schema in schemas) + { + var quotedSchema = schema.Replace("\"", "\"\""); + await using var upd = ctx.DataSource.CreateCommand($""" + UPDATE "{quotedSchema}".mesh_nodes + SET node_type = 'Space', last_modified = now() + WHERE namespace = '' AND node_type IS NULL + """); + var rows = await upd.ExecuteNonQueryAsync(); + if (rows > 0) + { + ctx.Logger.LogInformation( + "Repair v34: '{Schema}' — typed {Rows} orphan partition root(s) as Space", schema, rows); + fixedCount += rows; + } + } + + ctx.Logger.LogInformation("Repair v34: done — {Count} orphan partition root(s) typed as Space", fixedCount); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V35_ReconcilePartitionAccessIndex.cs b/memex/aspire/Memex.Database.Migration/Migrations/V35_ReconcilePartitionAccessIndex.cs new file mode 100644 index 000000000..de6e3d059 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V35_ReconcilePartitionAccessIndex.cs @@ -0,0 +1,107 @@ +using MeshWeaver.Hosting.PostgreSql; +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Heals the denormalized public.partition_access index on existing databases where it +/// drifted out of sync with {schema}.user_effective_permissions. +/// +/// The bug. public.search_across_schemas gates every partition behind +/// EXISTS(public.partition_access[user_id, partition]) AND (public_read OR uep[Read].is_allow). +/// public.partition_access is a flat denormalized index maintained ONLY by the +/// permission-rebuild functions in each partition schema: +/// +/// rebuild_user_effective_permissions() — schema-level, full rebuild; syncs every +/// user with Read into partition_access. +/// rebuild_user_permissions_for(p_user_id) — per-user, fired by the +/// access_changed trigger; syncs that one user's partition_access row. +/// +/// In production a user ended up with user_effective_permissions[Read] = true for a +/// partition but NO public.partition_access row → their Space was invisible in the catalog / +/// cross-schema search with no error. Two causes: (a) a STALE per-user function on a schema +/// provisioned BEFORE the partition_access sync was added to the function body — a +/// CREATE OR REPLACE FUNCTION change never re-applied to existing schemas, so the old body +/// rebuilt user_effective_permissions but skipped the partition_access sync; and +/// (b) data inserted bypassing the access_changed trigger. +/// +/// The repair (idempotent). For every existing partition schema (those with an +/// access table): +/// +/// Re-apply the CURRENT per-partition DDL via public.ensure_partition_schema(schema) +/// — the single source of truth that CREATE OR REPLACEs the rebuild functions +/// (including the partition_access sync inside rebuild_user_permissions_for), +/// curing any stale function body. +/// Run {schema}.rebuild_user_effective_permissions() — the full schema-level +/// reconcile that rebuilds user_effective_permissions from the access satellite +/// AND re-syncs public.partition_access (upserts every user with Read, deletes the +/// revoked), healing the drift regardless of which cause produced it. +/// +/// The proc itself is re-installed first (idempotent CREATE OR REPLACE) so a DB that +/// fast-forwarded past V30 still gets the latest body. +/// +/// Skipped on fresh DBs (no legacy partitions to reconcile; schema-init already installs +/// the corrected functions and an empty partition_access needs no heal). +/// +public sealed class V35_ReconcilePartitionAccessIndex : IMigration +{ + public int Version => 35; + + public string Description => + "Re-apply per-partition permission functions and reconcile public.partition_access for every schema"; + + public async Task RunAsync(MigrationContext ctx) + { + // 1. (Re)install the single-source-of-truth provisioning proc so its body carries the + // CURRENT versioned DDL (the rebuild functions with the partition_access sync). Pure + // CREATE OR REPLACE FUNCTION — idempotent. + await using (var procCmd = ctx.DataSource.CreateCommand( + PostgreSqlSchemaInitializer.GetEnsurePartitionSchemaProcScript(ctx.Options.VectorDimensions))) + { + await procCmd.ExecuteNonQueryAsync(); + } + + // 2. Re-apply the functions per schema, then reconcile uep + partition_access. + var schemas = await SchemaHelpers.DiscoverAccessSchemasAsync(ctx.DataSource); + + foreach (var schema in schemas) + { + // 2a. Re-apply the current per-partition DDL (CREATE OR REPLACE the rebuild functions + // etc.) via the single-source-of-truth proc. This cures a stale + // rebuild_user_permissions_for that never synced partition_access. + try + { + await using var ensureCmd = ctx.DataSource.CreateCommand( + "SELECT public.ensure_partition_schema(@p)"); + ensureCmd.Parameters.AddWithValue("@p", schema); + await ensureCmd.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation( + "Repair v35: \"{Schema}\" — per-partition functions re-applied", schema); + } + catch (Exception ex) + { + ctx.Logger.LogWarning(ex, + "Repair v35: \"{Schema}\" — ensure_partition_schema failed; skipping", schema); + continue; + } + + // 2b. Full schema-level reconcile: rebuild user_effective_permissions AND re-sync + // public.partition_access for every user with Read in this partition. Heals the + // drift whether it came from a stale function or a trigger-bypassing write. + try + { + await using var rebuildCmd = ctx.DataSource.CreateCommand( + $"SELECT \"{schema}\".rebuild_user_effective_permissions()"); + await rebuildCmd.ExecuteNonQueryAsync(); + ctx.Logger.LogInformation( + "Repair v35: \"{Schema}\".rebuild_user_effective_permissions() OK — partition_access reconciled", + schema); + } + catch (Exception ex) + { + ctx.Logger.LogWarning(ex, + "Repair v35: rebuild_user_effective_permissions failed for \"{Schema}\"", schema); + } + } + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V36_MoveAgentsToPerPartitionAgentNamespace.cs b/memex/aspire/Memex.Database.Migration/Migrations/V36_MoveAgentsToPerPartitionAgentNamespace.cs new file mode 100644 index 000000000..bd11e9782 --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V36_MoveAgentsToPerPartitionAgentNamespace.cs @@ -0,0 +1,96 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Move each partition's OWN agents into a dedicated {partition}/Agent sub-namespace, to match +/// the per-partition agent registry (AgentPickerProjection.BuildAgentQuery → +/// namespace:{user}/Agent|{space}/Agent|Agent nodeType:Agent — exact membership, no graph search). +/// +/// Before: a space/user dropped agents directly in its partition (e.g. atioz had +/// AgenticPension/Datenextraktion, namespace AgenticPension). The new registry only lists +/// {partition}/Agent, so those agents would no longer surface. This migration rewrites every +/// nodeType=Agent row in each partition schema to namespace = '{partition}/Agent' (and +/// fixes its main_node to the new path), so it appears in that space's /agent picker. +/// +/// Scope — agents only. Models stay on the _Provider catalog for now (the model +/// picker still queries _Provider subtrees); the per-partition /Model move ships with the +/// model-registry code change. +/// +/// Skipped: the platform Agent partition itself (its agents ARE the platform +/// defaults at namespace Agent — moving them to Agent/Agent would hide them). Rows already +/// at {partition}/Agent (or nested under it) are left untouched, so the migration is idempotent. +/// +public sealed class V36_MoveAgentsToPerPartitionAgentNamespace : IMigration +{ + public int Version => 36; + public string Description => "Move each partition's own agents into {partition}/Agent (per-partition agent registry)"; + + /// The dedicated sub-namespace segment for a partition's own agents — mirrors + /// AgentPickerProjection.AgentSubNamespace. + private const string AgentSub = "Agent"; + + public async Task RunAsync(MigrationContext ctx) + { + var partitions = await DiscoverPartitionsAsync(ctx); + ctx.Logger.LogInformation( + "Repair v36: inspecting {Count} partition(s) for agents to relocate into /{Sub}.", + partitions.Count, AgentSub); + + var grandTotal = 0; + foreach (var (partitionId, schemaName) in partitions) + { + // Skip the platform Agent catalog partition — its agents are the platform defaults at + // namespace 'Agent' and must NOT move to 'Agent/Agent'. + if (string.Equals(partitionId, AgentSub, StringComparison.OrdinalIgnoreCase)) + continue; + + var targetNs = $"{partitionId}/{AgentSub}"; + var quotedTarget = targetNs.Replace("'", "''"); + + // namespace → '{partition}/Agent'; main_node → '{partition}/Agent/{id}' (agents are main + // nodes, so main_node was the old path). Skip rows already at or under the target, and any + // legacy 'Agent'-namespaced row (defensive — shouldn't exist in a non-Agent partition). + await using var cmd = ctx.DataSource.CreateCommand($""" + UPDATE "{schemaName}".mesh_nodes SET + namespace = '{quotedTarget}', + main_node = CASE WHEN main_node IS NULL THEN NULL ELSE '{quotedTarget}/' || id END + WHERE node_type = 'Agent' + AND namespace <> '{quotedTarget}' + AND namespace NOT LIKE '{quotedTarget}/%' + """); + var affected = await cmd.ExecuteNonQueryAsync(); + if (affected > 0) + { + ctx.Logger.LogInformation( + "Repair v36: \"{Schema}\" — moved {Count} agent(s) into namespace '{Target}'.", + schemaName, affected, targetNs); + grandTotal += affected; + } + } + + ctx.Logger.LogInformation("Repair v36: relocated {Total} agent(s) into per-partition /{Sub} namespaces.", + grandTotal, AgentSub); + } + + private static async Task> DiscoverPartitionsAsync(MigrationContext ctx) + { + var partitions = new List<(string, string)>(); + // MeshDataSource records live at (ns='Source', id=) in admin.mesh_nodes. + await using var cmd = ctx.DataSource.CreateCommand(""" + SELECT id FROM admin.mesh_nodes + WHERE namespace = 'Source' AND node_type = 'MeshDataSource' + ORDER BY id + """); + await using var rdr = await cmd.ExecuteReaderAsync(); + while (await rdr.ReadAsync()) + { + var partitionId = rdr.GetString(0); + var schemaName = SchemaHelpers.SanitizeSchemaName(partitionId); + if (string.IsNullOrEmpty(schemaName)) continue; + if (!await SchemaHelpers.SchemaExistsAsync(ctx.DataSource, schemaName)) continue; + partitions.Add((partitionId, schemaName)); + } + return partitions; + } +} diff --git a/memex/aspire/Memex.Database.Migration/Migrations/V37_MoveAgentsToAgentNamespaceBySchema.cs b/memex/aspire/Memex.Database.Migration/Migrations/V37_MoveAgentsToAgentNamespaceBySchema.cs new file mode 100644 index 000000000..565aaa6ea --- /dev/null +++ b/memex/aspire/Memex.Database.Migration/Migrations/V37_MoveAgentsToAgentNamespaceBySchema.cs @@ -0,0 +1,84 @@ +using Microsoft.Extensions.Logging; + +namespace Memex.Database.Migration.Migrations; + +/// +/// Robust re-run of : move each partition's own +/// agents into a dedicated {partition}/Agent sub-namespace, but discover partitions from the +/// actual Postgres schemas (every schema that owns a mesh_nodes table) instead of from +/// admin.mesh_nodes MeshDataSource records — V36 found none for user-created Spaces (e.g. atioz's +/// AgenticPension), so it moved 0 agents. +/// +/// The new namespace is derived per row from the agent's own namespace prefix +/// (split_part(namespace,'/',1) || '/Agent'), so it is correct regardless of how the partition +/// was registered and regardless of case. Matches the per-partition agent registry +/// (AgentPickerProjection.BuildAgentQuery → namespace:{user}/Agent|{space}/Agent|Agent nodeType:Agent). +/// +/// Skipped: the platform Agent namespace itself (an agent at namespace Agent is +/// a platform default and must NOT become Agent/Agent), and rows already at/under {x}/Agent +/// — so the migration is idempotent. Models stay on _Provider (the /Model move ships with +/// the model-registry code). +/// +public sealed class V37_MoveAgentsToAgentNamespaceBySchema : IMigration +{ + public int Version => 37; + public string Description => "Move each partition's own agents into {partition}/Agent (schema-discovered, per-row)"; + + // Schemas that never hold a space/user's own agents (framework/system + the platform catalogs). + private static readonly HashSet ExcludedSchemas = new(StringComparer.OrdinalIgnoreCase) + { + "public", "admin", "auth", "doc", "_provider", "agent", "model", + "command", "harness", "apitoken", "system_access", "information_schema", + }; + + public async Task RunAsync(MigrationContext ctx) + { + var schemas = new List(); + await using (var cmd = ctx.DataSource.CreateCommand(""" + SELECT table_schema FROM information_schema.tables + WHERE table_name = 'mesh_nodes' + ORDER BY table_schema + """)) + await using (var rdr = await cmd.ExecuteReaderAsync()) + { + while (await rdr.ReadAsync()) + { + var s = rdr.GetString(0); + if (s.StartsWith("pg_", StringComparison.OrdinalIgnoreCase)) continue; + if (ExcludedSchemas.Contains(s)) continue; + schemas.Add(s); + } + } + + ctx.Logger.LogInformation("Repair v37: inspecting {Count} partition schema(s) for agents to relocate into /Agent.", + schemas.Count); + + var grandTotal = 0; + foreach (var schema in schemas) + { + // namespace → '{firstSegment}/Agent'; main_node → '{firstSegment}/Agent/{id}' (agents are + // main nodes). split_part(...) in the SET reads the OLD namespace value. Skip the platform + // Agent namespace and rows already at/under {x}/Agent (idempotent). + await using var cmd = ctx.DataSource.CreateCommand($""" + UPDATE "{schema}".mesh_nodes SET + namespace = split_part(namespace, '/', 1) || '/Agent', + main_node = CASE WHEN main_node IS NULL THEN NULL + ELSE split_part(namespace, '/', 1) || '/Agent/' || id END + WHERE node_type = 'Agent' + AND namespace <> '' + AND split_part(namespace, '/', 1) <> 'Agent' + AND namespace NOT LIKE '%/Agent' + AND namespace NOT LIKE '%/Agent/%' + """); + var affected = await cmd.ExecuteNonQueryAsync(); + if (affected > 0) + { + ctx.Logger.LogInformation("Repair v37: \"{Schema}\" — moved {Count} agent(s) into /Agent.", + schema, affected); + grandTotal += affected; + } + } + + ctx.Logger.LogInformation("Repair v37: relocated {Total} agent(s) into per-partition /Agent namespaces.", grandTotal); + } +} diff --git a/memex/aspire/Memex.Database.Migration/Program.cs b/memex/aspire/Memex.Database.Migration/Program.cs index 530b38734..4f313bc2a 100644 --- a/memex/aspire/Memex.Database.Migration/Program.cs +++ b/memex/aspire/Memex.Database.Migration/Program.cs @@ -1,12 +1,29 @@ +using Memex.Database.Migration.Migrations; using Memex.Portal.ServiceDefaults; +using MeshWeaver.Hosting.PostgreSql; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using MeshWeaver.Hosting.PostgreSql; -using MeshWeaver.Mesh; using Npgsql; +// Database migration is split in three phases: +// +// 1. Schema initialization (idempotent, ALWAYS runs) +// Creates/updates tables, indexes, triggers, satellite tables, and the admin schema. +// Brings any DB — fresh or existing — to the latest schema definition. +// +// 2. Versioned data repairs (one-shot, ONLY for existing DBs) +// Each migration fixes data written incorrectly by a prior code version. Tracked via +// MeshNode(id="db_version") in admin.mesh_nodes. Fresh DBs skip ALL repairs and +// fast-forward to the latest version (there is no legacy data to repair). +// +// 3. Searchable-schemas refresh (idempotent, always runs) +// Repopulates public.searchable_schemas from the current set of content partitions. +// +// To add a new repair: drop a `Vxx_*.cs` file under Migrations/ implementing IMigration +// and add it to the list passed to MigrationRunner below. + Console.WriteLine("[Migration] Starting..."); var builder = Host.CreateApplicationBuilder(args); builder.AddServiceDefaults(); @@ -18,588 +35,138 @@ else builder.AddNpgsqlDataSource("memex"); -// Derive vector dimensions from embedding model (passed by AppHost via Embedding__Model) +// Vector dimensions come from the embedding model (passed by AppHost via Embedding__Model). var embeddingOptions = builder.Configuration.GetSection("Embedding").Get() ?? new EmbeddingOptions(); builder.Services.Configure(o => { o.ConnectionString = connectionString; o.VectorDimensions = embeddingOptions.Dimensions; }); +// Register the embedding provider so the documentation backfill can vector-index docs. +// No-ops (registers nothing) when Endpoint/ApiKey are absent — backfill then writes +// NULL embeddings and docs are still full-text searchable. +builder.Services.AddAzureFoundryEmbeddings(embeddingOptions); Console.WriteLine("[Migration] Building host..."); var host = builder.Build(); Console.WriteLine("[Migration] Host built. Resolving services..."); var logger = host.Services.GetRequiredService().CreateLogger("Migration"); -Console.WriteLine("[Migration] Resolving NpgsqlDataSource..."); var dataSource = host.Services.GetRequiredService(); -Console.WriteLine("[Migration] NpgsqlDataSource resolved."); -var options = host.Services.GetRequiredService>(); - +var options = host.Services.GetRequiredService>().Value; logger.LogInformation("Running database migration..."); -// Grant CREATE on database to azure_pg_admin role so managed identities -// (portal, migration) can create per-organization schemas at runtime. -if (connectionString.Contains("database.azure.com")) -{ - var dbName = new NpgsqlConnectionStringBuilder(connectionString).Database; - await using var grantCmd = dataSource.CreateCommand( - $"GRANT CREATE ON DATABASE \"{dbName}\" TO azure_pg_admin"); - await grantCmd.ExecuteNonQueryAsync(); - logger.LogInformation("Granted CREATE ON DATABASE to azure_pg_admin."); -} - -// ═══════════════════════════════════════════════════════════════════════════ -// Schema initialization — always runs, idempotent (CREATE IF NOT EXISTS). -// Sets up tables, indexes, triggers, and satellite tables in the public schema. -// New DBs get everything correct from the start. -// Existing DBs get updated trigger functions (e.g., fixed role flag values). -// ═══════════════════════════════════════════════════════════════════════════ - -await PostgreSqlSchemaInitializer.InitializeAsync(dataSource, options.Value); - -var satelliteTableNames = MeshWeaver.Mesh.PartitionDefinition.StandardTableMappings.Values; -await PostgreSqlSchemaInitializer.CreateSatelliteTablesAsync( - dataSource, options.Value, satelliteTableNames); - -await PostgreSqlSchemaInitializer.InitializePartitionAccessTableAsync(dataSource); - -// ═══════════════════════════════════════════════════════════════════════════ -// Versioned migrations — tracked in admin.mesh_nodes as MeshNode(id="db_version"). -// -// Two categories: -// (a) Schema migrations: structural changes needed for both new and existing DBs. -// These go into PostgreSqlSchemaInitializer (idempotent, always run). -// (b) Data repairs: fix data written incorrectly by prior code versions. -// These go here as versioned migrations — only run once, only needed -// for existing DBs. New DBs never have the bad data. -// ═══════════════════════════════════════════════════════════════════════════ - -// Ensure admin schema exists for version tracking -await using (var ensureAdmin = dataSource.CreateCommand("CREATE SCHEMA IF NOT EXISTS admin")) -{ - await ensureAdmin.ExecuteNonQueryAsync(); -} - -// Ensure admin.mesh_nodes exists (may not if this is a fresh DB before partition init) -await PostgreSqlSchemaInitializer.InitializeMeshTablesAsync( - dataSource, options.Value); - -// Read current DB version (0 = fresh DB or pre-versioning) -int currentVersion = 0; -try -{ - await using var readVersion = dataSource.CreateCommand(""" - SELECT (content->>'Version')::int FROM admin.mesh_nodes - WHERE id = 'db_version' AND namespace = '' LIMIT 1 - """); - var result = await readVersion.ExecuteScalarAsync(); - if (result is int v) currentVersion = v; - else if (result is long l) currentVersion = (int)l; -} -catch -{ - // Table may not exist yet — version = 0 (fresh DB) -} - -logger.LogInformation("Current DB version: {Version}", currentVersion); - -// Detect fresh DB (no partition schemas exist yet) -bool isFreshDb; -await using (var checkSchemas = dataSource.CreateCommand(""" - SELECT count(*) FROM information_schema.schemata s - WHERE EXISTS ( - SELECT 1 FROM information_schema.tables t - WHERE t.table_schema = s.schema_name AND t.table_name = 'mesh_nodes' - ) - AND s.schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast') - AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' - """)) -{ - var schemaCount = (long)(await checkSchemas.ExecuteScalarAsync())!; - isFreshDb = schemaCount == 0; -} - -if (isFreshDb) -{ - logger.LogInformation("Fresh database detected — skipping data repairs (no existing data to fix)."); - currentVersion = 1; // Skip all repair migrations -} - -// ── Data repair v1: Move AccessAssignments to correct table + namespace ── -// Bug: AddUserRoleAsync wrote AccessAssignment nodes to mesh_nodes (wrong table) -// with namespace={scope}/{userId}_Access (missing _Access segment). -// Fix: Move to access table, add _Access to namespace, rebuild permissions. -if (currentVersion < 1) -{ - logger.LogInformation("Running repair v1: Move AccessAssignments to access table with _Access namespace..."); - await using (var cmd = dataSource.CreateCommand(""" - DO $$ - DECLARE - schema_rec RECORD; - moved_count INT; - ns_count INT; - cols TEXT := 'namespace, id, name, node_type, description, category, icon, display_order, last_modified, version, state, content, desired_id, main_node, embedding'; - BEGIN - FOR schema_rec IN - SELECT schema_name FROM information_schema.schemata s - WHERE EXISTS (SELECT 1 FROM information_schema.tables t WHERE t.table_schema = s.schema_name AND t.table_name = 'mesh_nodes') - AND EXISTS (SELECT 1 FROM information_schema.tables t WHERE t.table_schema = s.schema_name AND t.table_name = 'access') - AND s.schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast') - AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' - LOOP - -- Move AccessAssignments from mesh_nodes to access table - EXECUTE format( - 'INSERT INTO %I.access (' || cols || ') SELECT ' || cols || ' FROM %I.mesh_nodes WHERE node_type = ''AccessAssignment'' ON CONFLICT (namespace, id) DO NOTHING', - schema_rec.schema_name, schema_rec.schema_name - ); - GET DIAGNOSTICS moved_count = ROW_COUNT; - IF moved_count > 0 THEN - EXECUTE format( - 'DELETE FROM %I.mesh_nodes WHERE node_type = ''AccessAssignment''', - schema_rec.schema_name - ); - RAISE NOTICE 'Schema %: moved % AccessAssignment(s) from mesh_nodes to access', schema_rec.schema_name, moved_count; - END IF; - - -- Fix namespace: ensure _Access segment is present - EXECUTE format( - 'UPDATE %I.access SET namespace = namespace || ''/_Access'' WHERE node_type = ''AccessAssignment'' AND namespace NOT LIKE ''%%/_Access''', - schema_rec.schema_name - ); - GET DIAGNOSTICS ns_count = ROW_COUNT; - IF ns_count > 0 THEN - RAISE NOTICE 'Schema %: fixed % namespace(s) to include /_Access', schema_rec.schema_name, ns_count; - END IF; - - -- Rebuild permissions - BEGIN - EXECUTE format('SELECT %I.rebuild_user_effective_permissions()', schema_rec.schema_name); - EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'Schema %: rebuild failed: %', schema_rec.schema_name, SQLERRM; - END; - END LOOP; - END $$; - """)) - { - await cmd.ExecuteNonQueryAsync(); - } - - currentVersion = 1; - logger.LogInformation("Repair v1 completed."); -} - -// ── Data repair v2: Re-create trigger functions + populate partition_access ── -// The schema initializer now includes partition_access sync in -// rebuild_user_effective_permissions() with hardcoded schema name. -// For existing DBs: re-run schema init per schema to update the function, -// then rebuild permissions which populates partition_access. -if (currentVersion < 2) -{ - logger.LogInformation("Running repair v2: Update trigger functions and populate partition_access..."); - - // Discover existing partition schemas - var schemas = new List(); - await using (var listCmd = dataSource.CreateCommand(""" - SELECT schema_name FROM information_schema.schemata s - WHERE EXISTS (SELECT 1 FROM information_schema.tables t WHERE t.table_schema = s.schema_name AND t.table_name = 'mesh_nodes') - AND s.schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast') - AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' - ORDER BY s.schema_name - """)) - { - await using var rdr = await listCmd.ExecuteReaderAsync(); - while (await rdr.ReadAsync()) schemas.Add(rdr.GetString(0)); - } - - foreach (var schema in schemas) +// Wait for Postgres to accept connections AND ensure the target database exists before +// migrating. Two orchestration realities this handles: +// 1. The DB container may report "started" before it is listening (Compose +// depends_on:service_started, Kubernetes, ACA) — retry with backoff. +// 2. A self-managed Postgres (the pgvector container in Compose/Helm) does NOT pre-create +// the app database the way managed Azure Postgres does — connect to the maintenance +// 'postgres' database and CREATE it if missing. +// Managed Azure Postgres pre-creates the database and uses a credential provider on the +// data source, so for that path we just probe the data source directly. +var isAzurePg = connectionString.Contains("database.azure.com"); +var pgReadyDeadline = DateTime.UtcNow + TimeSpan.FromSeconds(120); +while (true) +{ + try { - logger.LogInformation("Repair v2: Updating trigger function for schema {Schema}...", schema); - - // Build a temporary SearchPath data source to re-create the trigger function - var csb = new NpgsqlConnectionStringBuilder(connectionString) { SearchPath = $"{schema},public" }; - var dsb = new NpgsqlDataSourceBuilder(csb.ConnectionString); - dsb.UseVector(); - await using var schemaDs = dsb.Build(); - - // Check if this schema has a versions schema - var versionsSchema = schema + "_versions"; - bool hasVersions; - await using (var checkCmd = dataSource.CreateCommand( - "SELECT EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = $1)")) + if (isAzurePg) { - checkCmd.Parameters.AddWithValue(versionsSchema); - hasVersions = (bool)(await checkCmd.ExecuteScalarAsync())!; - } - - var schemaOpts = new PostgreSqlStorageOptions - { - ConnectionString = csb.ConnectionString, - VectorDimensions = options.Value.VectorDimensions, - Schema = schema - }; - - if (hasVersions) - { - var vCsb = new NpgsqlConnectionStringBuilder(connectionString) { SearchPath = $"{versionsSchema},public" }; - await using var versionsDs = new NpgsqlDataSourceBuilder(vCsb.ConnectionString).Build(); - await PostgreSqlSchemaInitializer.InitializeWithVersionsSchemaAsync( - dataSource, schemaDs, versionsDs, schemaOpts, versionsSchema); + await using var probe = await dataSource.OpenConnectionAsync(); } else { - await PostgreSqlSchemaInitializer.InitializeMeshTablesAsync(schemaDs, schemaOpts); - } - - // Now rebuild permissions — the updated function will populate partition_access - try - { - await using var rebuildCmd = dataSource.CreateCommand( - $"SELECT \"{schema}\".rebuild_user_effective_permissions()"); - await rebuildCmd.ExecuteNonQueryAsync(); - logger.LogInformation("Repair v2: Schema {Schema} — rebuilt permissions + partition_access", schema); - } - catch (Exception ex) - { - logger.LogWarning(ex, "Repair v2: Schema {Schema} — rebuild failed", schema); + var targetDb = new NpgsqlConnectionStringBuilder(connectionString).Database ?? "memex"; + var maintenanceCs = new NpgsqlConnectionStringBuilder(connectionString) { Database = "postgres" }.ConnectionString; + await using var admin = new NpgsqlConnection(maintenanceCs); + await admin.OpenAsync(); + await using var check = new NpgsqlCommand("SELECT 1 FROM pg_database WHERE datname = @db", admin); + check.Parameters.AddWithValue("db", targetDb); + if (await check.ExecuteScalarAsync() is null) + { + logger.LogInformation("Database '{Db}' does not exist — creating it.", targetDb); + // targetDb is our own configured database name, not user input. Quote-escape defensively. + await using var create = new NpgsqlCommand($"CREATE DATABASE \"{targetDb.Replace("\"", "\"\"")}\"", admin); + await create.ExecuteNonQueryAsync(); + } } + break; } - - currentVersion = 2; - logger.LogInformation("Repair v2 completed."); -} - -// ── Data repair v3: Drop rogue schemas created from path segments ── -// Bug: paths like "login", "markdown", "onboarding" etc. created schemas -// that shouldn't exist as partitions. Drop them to keep discovery clean. -if (currentVersion < 3) -{ - logger.LogInformation("Running repair v3: Drop rogue schemas..."); - var rogueSchemas = new[] { - "_access", "_address_", "_graph", "_settings", "_tracking", "_thread", "_source", "_test", - "login", "markdown", "onboarding", "welcome", "settings", "storage", - "p", "mesh", "thread", "agent", "partition", "organization", "vuser" - }; - foreach (var rogue in rogueSchemas) + catch (Exception ex) when (DateTime.UtcNow < pgReadyDeadline) { - try - { - await using var dropCmd = dataSource.CreateCommand($"DROP SCHEMA IF EXISTS \"{rogue}\" CASCADE"); - await dropCmd.ExecuteNonQueryAsync(); - await using var dropVCmd = dataSource.CreateCommand($"DROP SCHEMA IF EXISTS \"{rogue}_versions\" CASCADE"); - await dropVCmd.ExecuteNonQueryAsync(); - logger.LogInformation("Repair v3: Dropped rogue schema {Schema}", rogue); - } - catch (Exception ex) - { - logger.LogWarning(ex, "Repair v3: Failed to drop schema {Schema}", rogue); - } - } - currentVersion = 3; - logger.LogInformation("Repair v3 completed."); -} - -// ── Data repair v4: Upgrade user self-assignments from Viewer to Admin ── -// UserScopeGrantHandler previously granted Viewer on User/{userId}. -// Now grants Admin so users can fully manage their own namespace. -if (currentVersion < 4) -{ - logger.LogInformation("Running repair v4: Upgrade user self-assignments from Viewer to Admin..."); - await using (var cmd = dataSource.CreateCommand(""" - DO $$ - DECLARE - schema_rec RECORD; - updated_count INT; - BEGIN - FOR schema_rec IN - SELECT schema_name FROM information_schema.schemata s - WHERE EXISTS (SELECT 1 FROM information_schema.tables t WHERE t.table_schema = s.schema_name AND t.table_name = 'access') - AND s.schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast') - AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' - LOOP - -- Update self-assignments: namespace=User/{id}/_Access, accessObject={id} - -- Replace Viewer with Admin in the roles array for self-assignments only - EXECUTE format( - 'UPDATE %I.access - SET content = jsonb_set( - content, - ''{roles}'', - (SELECT jsonb_agg( - CASE WHEN elem->>''role'' = ''Viewer'' - THEN jsonb_set(elem, ''{role}'', ''"Admin"'') - ELSE elem - END - ) FROM jsonb_array_elements(content->''roles'') AS elem) - ) - WHERE node_type = ''AccessAssignment'' - AND namespace LIKE ''User/%%/_Access'' - AND namespace = ''User/'' || (content->>''accessObject'') || ''/_Access'' - AND EXISTS (SELECT 1 FROM jsonb_array_elements(content->''roles'') r WHERE r->>''role'' = ''Viewer'') - AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements(content->''roles'') r WHERE r->>''role'' = ''Admin'')', - schema_rec.schema_name - ); - GET DIAGNOSTICS updated_count = ROW_COUNT; - IF updated_count > 0 THEN - RAISE NOTICE 'Schema %: upgraded % self-assignment(s) from Viewer to Admin', schema_rec.schema_name, updated_count; - END IF; - - -- Rebuild permissions - BEGIN - EXECUTE format('SELECT %I.rebuild_user_effective_permissions()', schema_rec.schema_name); - EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'Schema %: rebuild failed: %', schema_rec.schema_name, SQLERRM; - END; - END LOOP; - END $$; - """)) - { - await cmd.ExecuteNonQueryAsync(); - } - - currentVersion = 4; - logger.LogInformation("Repair v4 completed."); -} - -// ── Data repair v5: Ensure all users have Admin self-assignment and rebuild permissions ── -// Fixes missing AccessAssignment nodes for users who were onboarded before UserScopeGrantHandler, -// and rebuilds user_effective_permissions to include Thread permission (added to Admin/Editor roles). -if (currentVersion < 5) -{ - logger.LogInformation("Running repair v5: Ensure user self-assignments and rebuild permissions..."); - await using (var cmd = dataSource.CreateCommand(""" - DO $$ - DECLARE - user_rec RECORD; - assignment_exists BOOLEAN; - BEGIN - -- For each User node, ensure they have an Admin AccessAssignment on their own scope - FOR user_rec IN - SELECT id, path FROM "user".mesh_nodes WHERE node_type = 'User' - LOOP - -- Check if self-assignment already exists - SELECT EXISTS( - SELECT 1 FROM "user".access - WHERE namespace = 'User/' || user_rec.id || '/_Access' - AND content->>'accessObject' = user_rec.id - ) INTO assignment_exists; - - IF NOT assignment_exists THEN - INSERT INTO "user".access (id, namespace, name, node_type, content, main_node, last_modified, version, state) - VALUES ( - user_rec.id || '_SelfAccess', - 'User/' || user_rec.id || '/_Access', - user_rec.id || ' Self Access', - 'AccessAssignment', - jsonb_build_object( - 'accessObject', user_rec.id, - 'displayName', user_rec.id, - 'roles', jsonb_build_array(jsonb_build_object('role', 'Admin')) - ), - 'User/' || user_rec.id, - NOW(), - 1, - 'Active' - ); - RAISE NOTICE 'Created self-assignment for user %', user_rec.id; - END IF; - END LOOP; - - -- Rebuild permissions for user schema - BEGIN - PERFORM "user".rebuild_user_effective_permissions(); - EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'user schema rebuild failed: %', SQLERRM; - END; - - -- Rebuild permissions for all content partitions - FOR user_rec IN - SELECT schema_name FROM information_schema.schemata s - WHERE EXISTS (SELECT 1 FROM information_schema.tables t WHERE t.table_schema = s.schema_name AND t.table_name = 'access') - AND s.schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast', 'user') - AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' - LOOP - BEGIN - EXECUTE format('SELECT %I.rebuild_user_effective_permissions()', user_rec.schema_name); - EXCEPTION WHEN OTHERS THEN - RAISE NOTICE 'Schema % rebuild failed: %', user_rec.schema_name, SQLERRM; - END; - END LOOP; - END $$; - """)) - { - await cmd.ExecuteNonQueryAsync(); - } - - currentVersion = 5; - logger.LogInformation("Repair v5 completed."); -} - -// ── Data repair v6: Fix search_across_schemas to enforce partition_access ── -// Bug: public_read node types bypassed partition_access entirely, leaking -// cross-partition data in search (e.g., meshweaver user could see PartnerRe). -// Fix: partition_access is now always required; public_read only skips -// node-level permission checks within accessible partitions. -// The stored proc is re-created by InitializePublicSchemaAsync (idempotent). -if (currentVersion < 6) -{ - logger.LogInformation("Running repair v6: Fix search_across_schemas access control..."); - // Re-create the stored procedure with fixed access control logic - await PostgreSqlSchemaInitializer.InitializePartitionAccessTableAsync(dataSource); - currentVersion = 6; - logger.LogInformation("Repair v6 completed — search_across_schemas updated."); -} - -// ── Data repair v7: Deploy per-user permission rebuild trigger ── -// The trigger function trg_access_changed() previously called rebuild_user_effective_permissions() -// which rebuilds ALL users' permissions — causing deadlocks under concurrent access. -// New trigger calls rebuild_user_permissions_for(affected_user) — only touches one user's rows. -// The schema initializer already creates the new functions; we just need to re-run schema init -// per partition to deploy the updated trigger function. -if (currentVersion < 7) -{ - logger.LogInformation("Running repair v7: Deploy per-user permission rebuild trigger..."); - - var schemas = new List(); - await using (var listCmd = dataSource.CreateCommand(""" - SELECT schema_name FROM information_schema.schemata s - WHERE EXISTS (SELECT 1 FROM information_schema.tables t WHERE t.table_schema = s.schema_name AND t.table_name = 'access') - AND s.schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast') - AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' - ORDER BY s.schema_name - """)) - { - await using var rdr = await listCmd.ExecuteReaderAsync(); - while (await rdr.ReadAsync()) schemas.Add(rdr.GetString(0)); - } - - foreach (var schema in schemas) - { - logger.LogInformation("Repair v7: Updating trigger functions for schema {Schema}...", schema); - var csb = new NpgsqlConnectionStringBuilder(connectionString) { SearchPath = $"{schema},public" }; - var dsb = new NpgsqlDataSourceBuilder(csb.ConnectionString); - dsb.UseVector(); - await using var schemaDs = dsb.Build(); - - var schemaOpts = new PostgreSqlStorageOptions - { - ConnectionString = csb.ConnectionString, - VectorDimensions = options.Value.VectorDimensions, - Schema = schema - }; - - await PostgreSqlSchemaInitializer.InitializeMeshTablesAsync(schemaDs, schemaOpts); - logger.LogInformation("Repair v7: Schema {Schema} — trigger updated", schema); - } - - currentVersion = 7; - logger.LogInformation("Repair v7 completed."); -} - -// ── Data repair v8: Fix ThreadMessage MainNode ── -// Thread message nodes created from the UI may have MainNode set to the thread path -// (e.g., "Org/_Thread/thread-id") instead of the thread's content node (e.g., "Org"). -// This causes "Access denied" because SatelliteAccessRule delegates to MainNode. -// Fix: set MainNode = the part before "/_Thread/" for all ThreadMessage nodes. -if (currentVersion < 8) -{ - logger.LogInformation("Running repair v8: Fix ThreadMessage MainNode..."); - var totalFixed = 0; - - var schemas = new List(); - await using (var listCmd = dataSource.CreateCommand(""" - SELECT schema_name FROM information_schema.schemata s - WHERE EXISTS (SELECT 1 FROM information_schema.tables t WHERE t.table_schema = s.schema_name AND t.table_name = 'mesh_nodes') - AND s.schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast', 'admin') - AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' - ORDER BY s.schema_name - """)) - { - await using var rdr = await listCmd.ExecuteReaderAsync(); - while (await rdr.ReadAsync()) schemas.Add(rdr.GetString(0)); - } - - foreach (var schema in schemas) - { - await using var fixCmd = dataSource.CreateCommand($""" - UPDATE "{schema}".mesh_nodes - SET main_node = split_part(main_node, '/_Thread/', 1) - WHERE node_type = 'ThreadMessage' - AND main_node LIKE '%/_Thread/%' - """); - var affected = await fixCmd.ExecuteNonQueryAsync(); - if (affected > 0) - { - logger.LogInformation("Repair v8: Fixed {Count} ThreadMessage MainNode(s) in schema {Schema}", affected, schema); - totalFixed += affected; - } - } - - currentVersion = 8; - logger.LogInformation("Repair v8 completed — fixed {Total} ThreadMessage MainNode(s)", totalFixed); -} - -// ── Always: populate searchable_schemas from remaining content partitions ── -// This runs every time (not versioned) since it's idempotent and schemas may change. -{ - // Discover content schemas (same logic as PostgreSqlPartitionedStoreFactory.DiscoverPartitionsAsync) - var contentSchemas = new List(); - var excludedSchemas = new HashSet(StringComparer.OrdinalIgnoreCase) - { - "admin", "portal", "kernel", - "_access", "_address_", "_graph", "_settings", "_tracking", "_thread", "_source", "_test", - "login", "markdown", "onboarding", "welcome", "settings", "storage", - "p", "mesh", "thread", "agent", "partition", "organization", "vuser", - "public", "information_schema", "pg_catalog", "pg_toast" - }; - - await using (var discoverCmd = dataSource.CreateCommand(""" - SELECT schema_name FROM information_schema.schemata s - WHERE EXISTS (SELECT 1 FROM information_schema.tables t WHERE t.table_schema = s.schema_name AND t.table_name = 'mesh_nodes') - AND s.schema_name NOT IN ('public', 'information_schema', 'pg_catalog', 'pg_toast') - AND s.schema_name NOT LIKE '%\_versions' ESCAPE '\' - ORDER BY s.schema_name - """)) - { - await using var rdr = await discoverCmd.ExecuteReaderAsync(); - while (await rdr.ReadAsync()) - { - var schema = rdr.GetString(0); - if (!excludedSchemas.Contains(schema)) - contentSchemas.Add(schema); - } - } - - // Populate searchable_schemas - await using (var clearCmd = dataSource.CreateCommand("DELETE FROM public.searchable_schemas")) - await clearCmd.ExecuteNonQueryAsync(); - - foreach (var schema in contentSchemas) - { - await using var insertCmd = dataSource.CreateCommand( - "INSERT INTO public.searchable_schemas (schema_name) VALUES ($1) ON CONFLICT DO NOTHING"); - insertCmd.Parameters.AddWithValue(schema); - await insertCmd.ExecuteNonQueryAsync(); + logger.LogInformation("Waiting for Postgres / ensuring database ({Error})", ex.Message); + await Task.Delay(TimeSpan.FromSeconds(2)); } - - logger.LogInformation("Searchable schemas: [{Schemas}]", string.Join(", ", contentSchemas)); } - -// Save current version -await using (var saveVersion = dataSource.CreateCommand(""" - INSERT INTO admin.mesh_nodes (namespace, id, name, node_type, state, content, last_modified, main_node) - VALUES ('', 'db_version', 'Database Version', 'Settings', 2, - jsonb_build_object('Version', @version, 'LastMigration', now()::text), - now(), 'db_version') - ON CONFLICT (namespace, id) DO UPDATE SET - content = jsonb_build_object('Version', @version, 'LastMigration', now()::text), - last_modified = now() - """)) -{ - saveVersion.Parameters.AddWithValue("@version", currentVersion); - await saveVersion.ExecuteNonQueryAsync(); -} - -logger.LogInformation("Database migration completed. Version: {Version}", currentVersion); +logger.LogInformation("Postgres ready; target database present."); + +// ── Phase 1: Schema initialization (always runs) +var initResult = await SchemaInitialization.RunAsync(dataSource, options, connectionString, logger); + +// ── Phase 2: Versioned data repairs +var migrations = new IMigration[] +{ + new V01_MoveAccessAssignments(), + new V02_RebuildTriggerFunctions(), + new V03_DropRogueSchemas(), + new V04_UpgradeViewerToAdmin(), + new V05_EnsureUserSelfAssignments(), + new V06_FixSearchAcrossSchemas(), + new V07_PerUserPermissionRebuildTrigger(), + new V08_FixThreadMessageMainNode(), + new V09_RenameSourceTestSegments(), + new V10_PerUserPartitions(), + new V11_RewriteApiTokenPaths(), + // v12 was retired — see V13_RebuildPermissionsForApiBitmask for context. + new V13_RebuildPermissionsForApiBitmask(), + new V14_AddPartitionPrefixToNamespaces(), + new V15_FinalUserSchemaCleanup(), + new V16_NormalizeAccessAssignmentShape(), + new V17_EnsurePerUserSelfAssignments(), + new V18_BackfillUserPartitionRegistry(), + new V19_DeleteLegacyReleaseNodes(), + new V20_RemoveStrayLegacyUserRows(), + // v21 retired -- gap preserved so existing prod db_version counters stay monotonic. + new V22_ConsolidateGlobalCatalogsInAdmin(), + new V23_PartitionChangesNotify(), + new V24_DedupMeshNodeNotifyTrigger(), + new V25_MirrorAccessObjectsToUserSchema(), + new V26_AddNotificationsSatelliteTable(), + new V27_RenameUserSchemaToAuthAndMirrorApiTokens(), + new V28_RenameOrganizationToSpace(), + new V29_PinDocsForExistingUsers(), + new V30_EnsurePartitionSchemaStoredProc(), + new V31_UnifyUserMirrorIntoAuthAndRelocateContent(), + new V32_RepairAuthMirrorTriggerAndBackfill(), + new V33_SeedThreadComposerForExistingUsers(), + new V34_TypeOrphanPartitionRootsAsSpace(), + new V35_ReconcilePartitionAccessIndex(), + new V36_MoveAgentsToPerPartitionAgentNamespace(), + new V37_MoveAgentsToAgentNamespaceBySchema(), +}; + +var ctx = new MigrationContext(dataSource, connectionString, options, logger, initResult.IsFreshDb); +var runner = new MigrationRunner(migrations); +var finalVersion = await runner.RunAsync(ctx); + +// ── Doc search index (always runs): mirror the embedded documentation into the `doc` +// schema so it surfaces in the main search bar (full-text + vector). Runs BEFORE Phase 3 +// so the searchable-schemas refresh picks up `doc`. Full replace + incremental embedding. +var embeddingProvider = host.Services.GetService(); +await DocumentationBackfill.RunAsync(dataSource, options, connectionString, embeddingProvider, logger); + +// ── Orleans clustering (always runs): create the membership tables in the dedicated `orleans` +// database (same server, separate DB) so the portal silo can use Postgres-backed AdoNet +// clustering instead of Localhost. Skipped when no `orleans` connection string is injected +// (Azure-Tables / Localhost deployments don't use Postgres clustering). +var orleansConnectionString = builder.Configuration.GetConnectionString("orleans"); +await OrleansClusteringSetup.RunAsync(orleansConnectionString ?? "", logger); + +// ── Phase 3: Searchable-schemas refresh (always runs) +await SearchableSchemasUpdater.RunAsync(dataSource, logger); + +logger.LogInformation("Database migration completed. Version: {Version}", finalVersion); // Signal completion to Aspire (health check passes, then process exits cleanly) using var shutdownCts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); diff --git a/memex/aspire/Memex.Portal.Distributed/DbVersionGate.cs b/memex/aspire/Memex.Portal.Distributed/DbVersionGate.cs new file mode 100644 index 000000000..1dc87d1ba --- /dev/null +++ b/memex/aspire/Memex.Portal.Distributed/DbVersionGate.cs @@ -0,0 +1,127 @@ +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace Memex.Portal.Distributed; + +/// +/// Hard gate that runs once at portal startup: queries +/// admin.mesh_nodes WHERE id='db_version' and refuses to start the +/// host if the row is missing or its Version is below +/// . +/// +/// Why a gate, not just a healthcheck: Aspire's +/// WaitForCompletion(dbMigration) is a soft dependency hint at deploy +/// time. In Container Apps, the portal and migration run as independently +/// scheduled containers — a crashed migration silently lets the portal start +/// with a half-migrated DB. The previous prod symptom was exactly this: V02 +/// crashed, V10 never ran, no per-user schemas, every user denied at the +/// permission layer because the synced AccessAssignment query couldn't find +/// the partition. Failing portal startup makes the bad state loudly visible +/// in the Container App revision status (Failed) and prevents traffic from +/// being routed to a broken portal. +/// +/// Bump in lock-step with the highest +/// Vxx_*.cs migration in Memex.Database.Migration. Mismatch +/// between the version this portal expects and what the runner produced +/// fails-loud at startup with a clear diagnostic. +/// +public sealed class DbVersionGate( + NpgsqlDataSource dataSource, + IHostApplicationLifetime lifetime, + ILogger logger) : IHostedService +{ + /// + /// Highest migration version this portal build expects to find in the DB. + /// Keep in sync with the highest Vxx_*.cs file in + /// memex/aspire/Memex.Database.Migration/Migrations/. + /// + public const int ExpectedDbVersion = 32; + + public async Task StartAsync(CancellationToken cancellationToken) + { + try + { + await using var cmd = dataSource.CreateCommand(""" + SELECT (content->>'Version')::int AS v + FROM admin.mesh_nodes + WHERE id = 'db_version' AND namespace = '' + LIMIT 1 + """); + var raw = await cmd.ExecuteScalarAsync(cancellationToken); + var version = raw switch + { + int v => v, + long l => (int)l, + _ => 0 + }; + + if (version < ExpectedDbVersion) + { + logger.LogCritical( + "DB migration incomplete: admin.mesh_nodes.db_version={Actual} < expected {Expected}. " + + "The db-migration container probably crashed mid-run — check its ACA logs " + + "(`az containerapp logs show -n db-migration -g --tail 200`). " + + "Refusing to start the portal until the DB is fully migrated.", + version, ExpectedDbVersion); + lifetime.StopApplication(); + return; + } + + logger.LogInformation( + "DB version check passed: admin.mesh_nodes.db_version={Version} (expected ≥ {Expected}).", + version, ExpectedDbVersion); + } + catch (PostgresException ex) when (ex.SqlState == "42P01") + { + // Table doesn't exist at all — even schema-init didn't run. + logger.LogCritical(ex, + "DB schema not initialised: admin.mesh_nodes table is missing. " + + "The db-migration resource almost certainly never ran. " + + "Refusing to start the portal."); + lifetime.StopApplication(); + } + catch (Exception ex) + { + // Any other connection / auth error — also fail closed. Better to + // surface the auth/connection problem at startup than at first + // permission check. + logger.LogCritical(ex, + "DB version check failed unexpectedly. Refusing to start the portal."); + lifetime.StopApplication(); + } + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; +} + +/// +/// Liveness/readiness wrapper around the same db_version check, so external +/// monitors (uptime ping, ACA platform probe) can detect a half-migrated DB +/// even after the portal somehow gets past startup. +/// +public sealed class DbVersionHealthCheck(NpgsqlDataSource dataSource) : IHealthCheck +{ + public async Task CheckHealthAsync( + HealthCheckContext context, CancellationToken cancellationToken = default) + { + try + { + await using var cmd = dataSource.CreateCommand(""" + SELECT (content->>'Version')::int FROM admin.mesh_nodes + WHERE id = 'db_version' AND namespace = '' LIMIT 1 + """); + var raw = await cmd.ExecuteScalarAsync(cancellationToken); + var version = raw switch { int v => v, long l => (int)l, _ => 0 }; + return version >= DbVersionGate.ExpectedDbVersion + ? HealthCheckResult.Healthy($"db_version={version}") + : HealthCheckResult.Unhealthy( + $"db_version={version} < expected {DbVersionGate.ExpectedDbVersion}"); + } + catch (Exception ex) + { + return HealthCheckResult.Unhealthy("db_version check threw", ex); + } + } +} diff --git a/memex/aspire/Memex.Portal.Distributed/Memex.Portal.Distributed.csproj b/memex/aspire/Memex.Portal.Distributed/Memex.Portal.Distributed.csproj index 7c7786747..6a430aed0 100644 --- a/memex/aspire/Memex.Portal.Distributed/Memex.Portal.Distributed.csproj +++ b/memex/aspire/Memex.Portal.Distributed/Memex.Portal.Distributed.csproj @@ -16,14 +16,84 @@ + + + + + + + + + + + + <_MeshLocalFeedStage>$([System.IO.Path]::GetFullPath('$(IntermediateOutputPath)meshlocalfeed')) + + + + + + + <_MeshLocalNupkg Include="$(_MeshLocalFeedStage)\*.nupkg" /> + + dist\packages\%(_MeshLocalNupkg.Filename)%(_MeshLocalNupkg.Extension) + PreserveNewest + + + + + diff --git a/memex/aspire/Memex.Portal.Distributed/Program.cs b/memex/aspire/Memex.Portal.Distributed/Program.cs index d72013d61..99a824586 100644 --- a/memex/aspire/Memex.Portal.Distributed/Program.cs +++ b/memex/aspire/Memex.Portal.Distributed/Program.cs @@ -3,37 +3,103 @@ using Memex.Portal.ServiceDefaults; using Memex.Portal.Shared; using Microsoft.AspNetCore.DataProtection; +using MeshWeaver.Graph.Configuration; using MeshWeaver.Hosting.Orleans; using MeshWeaver.Hosting.PostgreSql; using MeshWeaver.Messaging; +using MeshWeaver.NuGet; +using MeshWeaver.NuGet.AzureBlob; +using Microsoft.Extensions.DependencyInjection.Extensions; using Npgsql; using Orleans.Configuration; +using Orleans.Hosting; var builder = WebApplication.CreateBuilder(args); builder.AddServiceDefaults(); builder.Services.AddServerSideBlazor().AddCircuitOptions(o => o.DetailedErrors = true); +// Give Orleans time to drain grain activations during a rolling update. +// ACA termination grace period is set to 120 s in Memex.AppHost; this +// keeps the .NET host alive for 90 s (leaves 30 s headroom before SIGKILL). +builder.Services.Configure(o => o.ShutdownTimeout = TimeSpan.FromSeconds(90)); // Log levels controlled via appsettings.Development.json -// Register Aspire-injected clients -builder.AddKeyedAzureTableServiceClient("orleans-clustering"); -builder.AddKeyedAzureBlobServiceClient("storage"); -builder.AddKeyedAzureBlobServiceClient("orleans-grain-state"); +// Deployment backend switch. Default "Azure" preserves the current ACA/Marketplace +// behaviour exactly (no regression). "Filesystem" is the Azure-free self-host path: +// object storage, the NodeType compile cache, the NuGet package cache, and +// DataProtection keys move to a (local or shared) volume. Mesh data still lives in +// Postgres in BOTH modes — the Postgres auth path below already auto-detects +// Azure-managed-identity vs basic auth from the connection string. +var deploymentBackend = builder.Configuration["Deployment:Backend"] ?? "Azure"; +var useAzureBackend = !string.Equals(deploymentBackend, "Filesystem", StringComparison.OrdinalIgnoreCase); -// Data protection: persist keys to Azure Blob Storage (shared across replicas) -var dpConfig = builder.Configuration.GetSection("DataProtection"); -var containerName = dpConfig["ContainerName"] ?? "dataprotection"; -var blobName = dpConfig["BlobName"] ?? "keys.xml"; +if (useAzureBackend) +{ + // Register Aspire-injected clients + builder.AddKeyedAzureTableServiceClient("orleans-clustering"); + builder.AddKeyedAzureBlobServiceClient("storage"); + builder.AddKeyedAzureBlobServiceClient("orleans-grain-state"); + // Shared NodeType compile cache — versioned assemblies live here, replacing the + // per-replica in-memory compile cache with a durable cross-replica lookup. + builder.AddKeyedAzureBlobServiceClient("nodetype-cache"); -builder.Services.AddDataProtection() - .SetApplicationName("MemexPortal") - .PersistKeysToAzureBlobStorage(sp => - { - var blobServiceClient = sp.GetRequiredKeyedService("storage"); - var containerClient = blobServiceClient.GetBlobContainerClient(containerName); - containerClient.CreateIfNotExists(); - return containerClient.GetBlobClient(blobName); - }); + // Persistent NuGet package cache backed by the content-storage account. Each resolved + // package is stored as a .zip blob under container "nuget-cache" keyed by {id}/{version}. + // On a new replica the resolver hydrates from blob instead of re-downloading from nuget.org. + builder.Services.Replace(ServiceDescriptor.Singleton(sp => + new BlobNuGetPackageCache( + sp.GetRequiredKeyedService("storage"), + containerName: "nuget-cache", + logger: sp.GetRequiredService>(), + // Mesh-scoped Blob pool caps blob concurrency; absent it falls back to IoPool.Unbounded. + ioPoolRegistry: sp.GetService()))); + + // Data protection: persist keys to Azure Blob Storage (shared across replicas) + var dpConfig = builder.Configuration.GetSection("DataProtection"); + var containerName = dpConfig["ContainerName"] ?? "dataprotection"; + var blobName = dpConfig["BlobName"] ?? "keys.xml"; + + builder.Services.AddDataProtection() + .SetApplicationName("MemexPortal") + .PersistKeysToAzureBlobStorage(sp => + { + var blobServiceClient = sp.GetRequiredKeyedService("storage"); + var containerClient = blobServiceClient.GetBlobContainerClient(containerName); + // Exists() probe before Create() avoids the Azure SDK's per-response + // "409 ContainerAlreadyExists" warning that CreateIfNotExists() emits + // on every startup against a pre-existing container. + if (!containerClient.Exists()) + containerClient.Create(); + return containerClient.GetBlobClient(blobName); + }); +} +else +{ + // ---- Self-host filesystem backend (Azure-free) ---- + // Single-node: a local volume. HA: a shared volume (NFS/CIFS) so every replica + // sees the same compile cache / package cache / DataProtection keys. + var dataRoot = builder.Configuration["Deployment:DataRoot"] + ?? Path.Combine(AppContext.BaseDirectory, "data"); + + // NodeType compile cache → filesystem. Registered BEFORE ConfigureMemexMesh's + // AddBlobAssemblyStore() runs; both use TryAddSingleton, so this + // first registration wins and the blob factory (which needs a keyed BlobServiceClient + // we deliberately don't register here) is never constructed. + builder.Services.AddFileSystemAssemblyStore(Path.Combine(dataRoot, "assembly-cache")); + + // NuGet package cache → filesystem (zip-per-version, shared-volume safe). + builder.Services.Replace(ServiceDescriptor.Singleton(sp => + new FileSystemNuGetPackageCache( + Path.Combine(dataRoot, "nuget-cache"), + sp.GetRequiredService>()))); + + // DataProtection keys → filesystem (shared volume across replicas in HA). + var keysDir = Path.Combine(dataRoot, "dataprotection-keys"); + Directory.CreateDirectory(keysDir); + builder.Services.AddDataProtection() + .SetApplicationName("MemexPortal") + .PersistKeysToFileSystem(new DirectoryInfo(keysDir)); +} // Register Aspire-injected PostgreSQL data source (with pgvector support) // Single shared pool for all partition queries (schema-qualified SQL). @@ -66,14 +132,50 @@ var embeddingOptions = builder.Configuration.GetSection("Embedding").Get() ?? new EmbeddingOptions(); builder.Services.AddAzureFoundryEmbeddings(embeddingOptions); -// Configure Orleans with Azure Table Storage (co-hosted silo + web) +// Configure Orleans clustering (co-hosted silo + web). +// - "AzureTables" (default): Aspire injects Azure Table clustering via config — no +// explicit provider here, exactly as before (no regression for ACA/Marketplace). +// - "Localhost": single-silo in-process membership for single-node self-host (compose +// without an Aspire orchestrator to inject clustering config). +// - "AdoNet" (Postgres): HA self-host — wired in Track A / compose-ha. +// Clustering provider is a deploy-time feature flag (Features:Orleans:Clustering); the +// legacy Deployment:Orleans:Clustering key is still honoured for back-compat. +var orleansClustering = builder.Configuration["Features:Orleans:Clustering"] + ?? builder.Configuration["Deployment:Orleans:Clustering"] + ?? "AzureTables"; var address = AddressExtensions.CreateMeshAddress(); builder.UseOrleansMeshServer(address, silo => + { silo.Configure(opts => { opts.ClusterId = MemexDistributedConstants.ClusterId; opts.ServiceId = MemexDistributedConstants.ServiceId; - }) + }); + if (string.Equals(orleansClustering, "Localhost", StringComparison.OrdinalIgnoreCase)) + { + silo.UseLocalhostClustering(); + } + else if (string.Equals(orleansClustering, "AdoNet", StringComparison.OrdinalIgnoreCase)) + { + // Real, Postgres-backed cluster membership (self-host / HA). The `orleans` + // database and its connection string are declared in the Aspire AppHost and + // injected as ConnectionStrings:orleans; the db-migration creates the Orleans + // membership tables. (AzureTables — the ACA path — is configured by the Aspire + // Orleans integration via WithReference(orleans), so it needs no explicit call.) + var orleansConnectionString = builder.Configuration.GetConnectionString("orleans") + ?? throw new InvalidOperationException( + "Features:Orleans:Clustering=AdoNet but ConnectionStrings:orleans is not set. " + + "The Aspire AppHost must add an 'orleans' database and WithReference it on the portal."); + if (!System.Data.Common.DbProviderFactories.GetProviderInvariantNames().Contains("Npgsql")) + System.Data.Common.DbProviderFactories.RegisterFactory("Npgsql", Npgsql.NpgsqlFactory.Instance); + silo.UseAdoNetClustering(o => + { + o.Invariant = "Npgsql"; + o.ConnectionString = orleansConnectionString; + }); + } + return silo; + } ) .ConfigureServices(services => services .AddPartitionedPostgreSqlPersistence( @@ -92,6 +194,19 @@ .ConfigureMemexMesh(builder.Configuration, builder.Environment.IsDevelopment()) .ConfigureMemexPortal(); +// Hard gate: refuse to start if the DB isn't migrated. Aspire's +// WaitForCompletion(dbMigration) is a soft hint at deploy time — Container +// Apps schedule the portal independently, so a crashed migration silently +// lets the portal come up against a half-migrated DB. The startup gate +// trips IHostApplicationLifetime.StopApplication, which causes the host to +// exit and Container Apps to mark the revision as Failed — that's the +// signal tools/deploy.sh polls for to fail the pipeline. +builder.Services.AddHostedService(); +// Live healthcheck for the same condition — surfaces drift after startup +// (e.g. someone manually rolled a partial migration via psql). +builder.Services.AddHealthChecks() + .AddCheck("db_version"); + var app = builder.Build(); app.MapDefaultEndpoints(); diff --git a/memex/aspire/Memex.Portal.Distributed/appsettings.Development.json b/memex/aspire/Memex.Portal.Distributed/appsettings.Development.json index a1139b132..e1ac9a71d 100644 --- a/memex/aspire/Memex.Portal.Distributed/appsettings.Development.json +++ b/memex/aspire/Memex.Portal.Distributed/appsettings.Development.json @@ -1,14 +1,20 @@ { "DetailedErrors": true, + "Features": { + "StaticRepoSync": { + "Partitions": [ "Doc", "Agent", "Model", "Harness", "Skill" ] + } + }, "Logging": { "LogLevel": { + // Quiet by default — Information surfaces app-level events without + // drowning chat / synced-query traces in routing/buffer spam. "Default": "Warning", "Microsoft.AspNetCore": "Warning", - "MeshWeaver": "Information", - "MeshWeaver.AI": "Information", - "MeshWeaver.Graph.Configuration": "Information", - "MeshWeaver.Layout.ConvertJson": "Warning", - "MeshWeaver.Messaging.Hub.MessageHub": "Warning", + + // Catch-all: everything in MeshWeaver.* is Warning unless pinned below. + "MeshWeaver": "Warning", + "Azure.Core": "Warning", "Orleans": "Warning", "Memex": "Warning", diff --git a/memex/aspire/Memex.Portal.Distributed/appsettings.json b/memex/aspire/Memex.Portal.Distributed/appsettings.json index 71029baf8..1082fd33e 100644 --- a/memex/aspire/Memex.Portal.Distributed/appsettings.json +++ b/memex/aspire/Memex.Portal.Distributed/appsettings.json @@ -1,9 +1,53 @@ { + // Production logging. + // + // 🚨 App Insights ingest cost model: every Information+ trace is billed. + // The default below is Information so user-actionable events (node lifecycle, + // chat-round start/end, auth events) reach the dashboard, but every namespace + // known to be chatty at Information level is explicitly capped at Warning. + // + // Per-CI-log analysis (run 26474008139) — top 10 Information emitters were: + // + // 910 MeshWeaver.Graph.SyncedQuery — every query Initial emission + // 284 MeshWeaver.Hosting.Persistence.Query.StaticNodeQueryProvider — ctor + // 200+ MeshWeaver.AI.AgentChatClient — per-round agent/model lists + // 51 MeshWeaver.Hosting.PostgreSql.PostgreSqlCrossSchemaQueryProvider — per-query satellite scan + // + // The first three were demoted in code (commit TBD). The Postgres cross-schema + // line stays at Information per request but is capped here. + // + // To raise verbosity for a debugging session: edit the bin/appsettings.json + // (reloadOnChange:true), NOT this file. Re-raise here only as a permanent change. "Logging": { "LogLevel": { "Default": "Warning", - "MeshWeaver.AI": "Information", - "MeshWeaver.Hosting.Orleans.RoutingGrain": "Information" + "MeshWeaver": "Warning", + "Microsoft": "Warning", + "Microsoft.AspNetCore": "Warning", + "Orleans": "Warning", + "Azure": "Warning", + "Azure.Core": "Warning", + "Npgsql": "Warning", + "System": "Warning", + "Memex": "Warning" + }, + "ApplicationInsights": { + "LogLevel": { + // App Insights ingest cap: Warning by default, Information for the + // namespaces we WANT user-visible activity for. Per-namespace Warning + // entries cap chatty namespaces even if the top-level default rises. + "Default": "Warning", + "MeshWeaver": "Warning", + "Memex": "Information", + "MeshWeaver.AI.AgentChatClient": "Warning", + "MeshWeaver.AI.ThreadExecution": "Information", + "MeshWeaver.Graph.SyncedQuery": "Warning", + "MeshWeaver.Hosting.Persistence.Query.StaticNodeQueryProvider": "Warning", + "MeshWeaver.Hosting.PostgreSql.PostgreSqlCrossSchemaQueryProvider": "Warning", + "MeshWeaver.Hosting.RoutingServiceBase": "Warning", + "MeshWeaver.Layout.Composition.LayoutAreaHost": "Warning", + "MeshWeaver.Layout.LayoutAreaHost": "Warning" + } } }, "AllowedHosts": "*", @@ -20,6 +64,16 @@ "Type": "PostgreSql" } }, + // The distributed portal serves the built-in Doc/Agent/Model partitions from the DB: Orleans + // routing does not consult the in-memory embedded adapter, so these must be materialized into + // their PG partitions by the static-repo import on boot (otherwise /Doc pages hang). Deploy + // overlays (Helm / values.atioz.yaml) may override; this is the correct default for any + // distributed deployment. + "Features": { + "StaticRepoSync": { + "Partitions": [ "Doc", "Agent", "Model", "Harness", "Skill" ] + } + }, "Storage": { "Name": "storage", "SourceType": "AzureBlob", @@ -27,5 +81,8 @@ "ContainerName": "content", "ClientName": "storage" } + }, + "Auth": { + "GlobalAdmins": [ "rbuergi" ] } } diff --git a/nuget.config b/nuget.config index 44e46993d..22f72b6ab 100644 --- a/nuget.config +++ b/nuget.config @@ -1,10 +1,30 @@ + - + + + + + diff --git a/run-failing-tests.sh b/run-failing-tests.sh new file mode 100644 index 000000000..794b294f4 --- /dev/null +++ b/run-failing-tests.sh @@ -0,0 +1,123 @@ +#!/bin/bash +# Runs only the tests in failing-tests.txt, project by project. +# Output goes to test-run-results.txt (overwritten each run). + +OUT=test-run-results.txt +echo "=== Targeted re-run started: $(date) ===" > $OUT + +# Build each project's filter from failing-tests.txt (skip comments + empty lines) +run_proj() { + local proj=$1 + local proj_path=$2 + shift 2 + local filters=("$@") + local filter_str="" + for t in "${filters[@]}"; do + if [ -z "$filter_str" ]; then + filter_str="FullyQualifiedName~$t" + else + filter_str="$filter_str|FullyQualifiedName~$t" + fi + done + echo "" >> $OUT + echo "=== $proj ($(echo "${filters[@]}" | wc -w) tests) ===" >> $OUT + dotnet test "$proj_path" --no-restore --filter "$filter_str" 2>&1 \ + | grep -E "^(Failed|Passed)!|\[FAIL\]|Error Message:" >> $OUT + echo "(returned $?)" >> $OUT +} + +# ===== Run each project ===== (FutuRe excluded — owned by another agent) + +run_proj "Markdown.Test" "test/MeshWeaver.Markdown.Test" \ + "MultipleBlocks_ShareKernelState_ViaSharedAddress" + +run_proj "AccessControl.Test" "test/MeshWeaver.AccessControl.Test" \ + "Overview_RendersChangeSubjectButton" \ + "Thumbnail_ClickRemoveRole_RemovesChip" \ + "UpdateAccessObject_ChangesSubject_ViaDataChange" + +run_proj "Insurance.Test" "samples/Insurance/MeshWeaver.Insurance.Test" \ + "GetPricingCatalog_UsingLayoutAreaReference_ShouldReturnPricingsControl" \ + "GetPricingCatalog_ShouldReturnPricings" + +run_proj "Todo.Test" "test/MeshWeaver.Todo.Test" \ + "Step1_SetupDataContext_WithTodoItems" + +run_proj "Threading.Test" "test/MeshWeaver.Threading.Test" \ + "UpdateMeshNode_MultipleUpdates_AccumulateMessages" \ + "SubmitMessage_WithToolCalling_ExecutesSearchAndReturnsResult" + +run_proj "Content.Test" "test/MeshWeaver.Content.Test" \ + "VersionsArea_SingleVersion_RendersWithoutError" \ + "VersionsMenu_AppearsInNodeMenu" \ + "VersionsArea_RendersVersionList" + +run_proj "Persistence.Test" "test/MeshWeaver.Persistence.Test" \ + "ResolvePathAsync_UnknownPath_ShouldReturnNull" \ + "ResolvePathAsync_FutuReSubPaths_ShouldResolve" \ + "MarkdownNode_LoadsWithoutHanging" \ + "InteractiveShowcaseMd_FullPipeline_AllBlocksExecute" \ + "MultipleSubmissions_ShareKernelState" \ + "Move_LargeSubtree_RunsIOInParallel" + +run_proj "Hosting.Blazor.Test" "test/MeshWeaver.Hosting.Blazor.Test" \ + "OnLocationChanged_SatelliteNode_CurrentNamespacePointsAtMainNode" \ + "OnLocationChanged_SatelliteNode_LoadsCreatableTypesForMainNode" + +run_proj "Auth.Test" "test/MeshWeaver.Auth.Test" \ + "ValidateToken_ValidToken_ReturnsApiToken" \ + "ValidateToken_RevokedToken_ReturnsNull" + +run_proj "Security.Test" "test/MeshWeaver.Security.Test" \ + "SubscribeRequest_WithReadPermission_Succeeds" \ + "SubscribeRequest_WithoutReadPermission_ReturnsDeliveryFailure" \ + "GetDataRequest_WithoutReadPermission_ReturnsDeliveryFailure" \ + "McpSearch_User1SeesOnlyPermittedNodes" \ + "McpUpdate_User1CannotUpdatePrivateOrg_User2Can" \ + "McpGet_User1CanReadPublicNode" \ + "McpSearch_User1CannotSearchPrivateOrg" \ + "McpUpdate_User1CannotUpdate_User2Can" \ + "McpGet_User1CannotReadPrivateOrg_User2Can" \ + "McpGet_User1CannotReadConfidentialNode_User2Can" + +run_proj "Autocomplete.Test" "test/MeshWeaver.Autocomplete.Test" \ + "CanCreateTypeAtPath_ReturnsTrueForValidType" \ + "GetCreatableTypes_DifferentNodesDifferentTypes" \ + "GetCreatableTypes_ReturnsTypesForNode" \ + "FilterByCreatableType_ReturnsOnlyMatchingNodes" \ + "Integration_AutocompleteWithTypeFilter_WorksEndToEnd" \ + "CanCreateTypeAtPath_ReturnsFalseForInvalidType" \ + "LocalFirst_ChildrenOfContextScoreHigherThanDistant" + +run_proj "NodeOperations.Test" "test/MeshWeaver.NodeOperations.Test" \ + "CreateApiToken_ViaCreateNodeRequest_Succeeds" \ + "CreateApiToken_StoredUnderUserPath" \ + "CreateNodeAsync_ReplyNode_ShouldLinkToParent" + +run_proj "Hosting.PostgreSql.Test" "test/MeshWeaver.Hosting.PostgreSql.Test" \ + "CreateOrganization_HasPermission_ReturnsAdmin" + +run_proj "Query.Test" "test/MeshWeaver.Query.Test" \ + "ObserveQuery_EmitsRemovedOnDeletedNode" \ + "ObserveQuery_VersionIncrementsWithEachChange" \ + "ContentEmailQuery_NameCanOverrideClaim" \ + "PropertyChange_NoLongerMatchesQuery_RemovesFromCollection" \ + "AtText_ReturnsCurrentNodeAndGlobal" \ + "GetRemoteStream_AfterDispose_ReturnsFreshInstance" \ + "Catalog_NodeTypeFilter_FiltersCorrectly" \ + "Catalog_Pagination_LoadsMoreItems" \ + "Catalog_TextSearch_FiltersResults" + +run_proj "Acme.Test" "test/MeshWeaver.Acme.Test" \ + "DescendantsSearch_FindsOrganizationRootNode" \ + "AcmeOrganization_IsAccessibleToAuthenticatedUser" \ + "SubtreeSearch_FindsOrganizationRootNode" \ + "TodoDataChangeWorkflowTest" + +run_proj "Hosting.Orleans.Test" "test/MeshWeaver.Hosting.Orleans.Test" \ + "SubHub_WithExportTypesRegistered_DeserializesPolymorphicExportDocumentControl" \ + "ExportPdfArea_RendersExportDocumentControl_ClientDeserializes" \ + "ToolCall_DuringStreaming_DoesNotDeadlock" + +echo "" >> $OUT +echo "=== Run finished: $(date) ===" >> $OUT diff --git a/samples/Graph/Data/ACME.json b/samples/Graph/Data/ACME.json new file mode 100644 index 000000000..20e1fb63c --- /dev/null +++ b/samples/Graph/Data/ACME.json @@ -0,0 +1,15 @@ +{ + "id": "ACME", + "name": "ACME", + "nodeType": "Space", + "description": "Project and task management demo showcasing MeshWeaver's collaborative workflows and AI agent integration", + "icon": "/static/storage/content/ACME/icon.svg", + "addressSegments": 0, + "isPersistent": true, + "content": { + "$type": "Space", + "name": "ACME", + "description": "ACME Demo Space", + "category": "Task Management" + } +} diff --git a/samples/Graph/Data/ACME/Article/_Source/Article.cs b/samples/Graph/Data/ACME/Article/Source/Article.cs similarity index 100% rename from samples/Graph/Data/ACME/Article/_Source/Article.cs rename to samples/Graph/Data/ACME/Article/Source/Article.cs diff --git a/samples/Graph/Data/ACME/Article/_Source/ArticleLayoutAreas.cs b/samples/Graph/Data/ACME/Article/Source/ArticleLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/ACME/Article/_Source/ArticleLayoutAreas.cs rename to samples/Graph/Data/ACME/Article/Source/ArticleLayoutAreas.cs diff --git a/samples/Graph/Data/ACME/Documentation/AIAgentIntegration.md b/samples/Graph/Data/ACME/Documentation/AIAgentIntegration.md index 9c3435587..76e0e42ea 100644 --- a/samples/Graph/Data/ACME/Documentation/AIAgentIntegration.md +++ b/samples/Graph/Data/ACME/Documentation/AIAgentIntegration.md @@ -134,7 +134,7 @@ Agent: "Created 'Demo environment setup' in Engineering category with Medium pri The agent demonstrates sophisticated behavior by first retrieving available categories through the `MeshPlugin`, then matching user input to existing categories. This prevents data inconsistencies and provides a better user experience: 1. **User Input**: "Add a marketing review task for the campaign" -2. **Category Discovery**: Agent calls `Get("@ACME/Project/Todo/schema:")` to see available categories +2. **Category Discovery**: Agent calls `Get("@ACME/Project/Todo/schema/")` to see available categories 3. **Intelligent Matching**: Agent matches "marketing review" to the "Marketing" category 4. **Todo Creation**: Agent calls `Update()` with properly structured Todo JSON diff --git a/samples/Graph/Data/ACME/Documentation/GettingStarted.md b/samples/Graph/Data/ACME/Documentation/GettingStarted.md index 4e8a3c36a..95b26fd09 100644 --- a/samples/Graph/Data/ACME/Documentation/GettingStarted.md +++ b/samples/Graph/Data/ACME/Documentation/GettingStarted.md @@ -23,8 +23,8 @@ Software demonstrates how MeshWeaver organizes data and applications: ACME/ # Organization level ├── Project/ # Shared NodeType definitions │ ├── Todo.json # Task NodeType (reusable) -│ ├── Todo/_Source/ # Todo.cs, TodoViews.cs, Status.cs, etc. -│ ├── _Source/ # ProjectViews.cs (project-level views) +│ ├── Todo/Source/ # Todo.cs, TodoViews.cs, Status.cs, etc. +│ ├── Source/ # ProjectViews.cs (project-level views) │ └── TodoAgent.md # AI agent for task management └── ProductLaunch/ # Project: Marketing campaign └── Todo/ # Tasks: PricingStrategy, EmailCampaign, etc. diff --git a/samples/Graph/Data/ACME/Documentation/UnifiedReferences.md b/samples/Graph/Data/ACME/Documentation/UnifiedReferences.md index 6c6f856a5..4c749c61e 100644 --- a/samples/Graph/Data/ACME/Documentation/UnifiedReferences.md +++ b/samples/Graph/Data/ACME/Documentation/UnifiedReferences.md @@ -121,7 +121,7 @@ Dimensions are shared across all projects using the same NodeType. ## Status Dimension -Defined in `ACME/Project/_Source/Status.cs`: +Defined in `ACME/Project/Source/Status.cs`: | Status | Description | Emoji | |--------|-------------|-------| @@ -133,7 +133,7 @@ Defined in `ACME/Project/_Source/Status.cs`: ## Priority Dimension -Defined in `ACME/Project/Todo/_Source/Priority.cs`: +Defined in `ACME/Project/Todo/Source/Priority.cs`: | Priority | Order | Color | |----------|-------|-------| @@ -145,7 +145,7 @@ Defined in `ACME/Project/Todo/_Source/Priority.cs`: ## Category Dimension -Defined in `ACME/Project/Todo/_Source/Category.cs`: +Defined in `ACME/Project/Todo/Source/Category.cs`: | Category | Icon | |----------|------| diff --git a/samples/Graph/Data/ACME/Project/_Source/Category.cs b/samples/Graph/Data/ACME/Project/Source/Category.cs similarity index 100% rename from samples/Graph/Data/ACME/Project/_Source/Category.cs rename to samples/Graph/Data/ACME/Project/Source/Category.cs diff --git a/samples/Graph/Data/ACME/Project/_Source/Priority.cs b/samples/Graph/Data/ACME/Project/Source/Priority.cs similarity index 100% rename from samples/Graph/Data/ACME/Project/_Source/Priority.cs rename to samples/Graph/Data/ACME/Project/Source/Priority.cs diff --git a/samples/Graph/Data/ACME/Project/_Source/Project.cs b/samples/Graph/Data/ACME/Project/Source/Project.cs similarity index 100% rename from samples/Graph/Data/ACME/Project/_Source/Project.cs rename to samples/Graph/Data/ACME/Project/Source/Project.cs diff --git a/samples/Graph/Data/ACME/Project/_Source/ProjectLayoutAreas.cs b/samples/Graph/Data/ACME/Project/Source/ProjectLayoutAreas.cs similarity index 97% rename from samples/Graph/Data/ACME/Project/_Source/ProjectLayoutAreas.cs rename to samples/Graph/Data/ACME/Project/Source/ProjectLayoutAreas.cs index 6441d6b38..53d604734 100644 --- a/samples/Graph/Data/ACME/Project/_Source/ProjectLayoutAreas.cs +++ b/samples/Graph/Data/ACME/Project/Source/ProjectLayoutAreas.cs @@ -71,7 +71,7 @@ private static ImmutableList Thumbnails(IEnumerable nodes) var statuses = host.Workspace.GetObservable() .Select(s => s.OrderBy(x => x.Order).ToList()); var nodes = host.Hub.ServiceProvider.GetRequiredService() - .ObserveQuery(MeshQueryRequest.FromQuery( + .Query(MeshQueryRequest.FromQuery( $"namespace:{hubPath}/Todo state:Active")) .Scan(new Dictionary(StringComparer.OrdinalIgnoreCase), ApplyChanges); @@ -106,7 +106,7 @@ private static ImmutableList Thumbnails(IEnumerable nodes) var categories = host.Workspace.GetObservable() .Select(c => c.ToDictionary(cat => cat.Id, cat => cat)); var nodes = host.Hub.ServiceProvider.GetRequiredService() - .ObserveQuery(MeshQueryRequest.FromQuery( + .Query(MeshQueryRequest.FromQuery( $"namespace:{hubPath}/Todo state:Active")) .Scan(new Dictionary(StringComparer.OrdinalIgnoreCase), ApplyChanges); @@ -142,7 +142,7 @@ private static ImmutableList Thumbnails(IEnumerable nodes) var priorities = host.Workspace.GetObservable() .Select(p => p.ToDictionary(x => x.Id, x => x)); var nodes = host.Hub.ServiceProvider.GetRequiredService() - .ObserveQuery(MeshQueryRequest.FromQuery( + .Query(MeshQueryRequest.FromQuery( $"namespace:{hubPath}/Todo state:Active")) .Scan(new Dictionary(StringComparer.OrdinalIgnoreCase), ApplyChanges); @@ -218,7 +218,7 @@ private static ImmutableList Thumbnails(IEnumerable nodes) var priorities = host.Workspace.GetObservable() .Select(p => p.ToDictionary(x => x.Id, x => x)); var nodes = host.Hub.ServiceProvider.GetRequiredService() - .ObserveQuery(MeshQueryRequest.FromQuery( + .Query(MeshQueryRequest.FromQuery( $"namespace:{hubPath}/Todo state:Active")) .Scan(new Dictionary(StringComparer.OrdinalIgnoreCase), ApplyChanges); @@ -263,7 +263,7 @@ private static ImmutableList Thumbnails(IEnumerable nodes) var priorities = host.Workspace.GetObservable() .Select(p => p.ToDictionary(x => x.Id, x => x)); var nodes = host.Hub.ServiceProvider.GetRequiredService() - .ObserveQuery(MeshQueryRequest.FromQuery( + .Query(MeshQueryRequest.FromQuery( $"namespace:{hubPath}/Todo state:Active")) .Scan(new Dictionary(StringComparer.OrdinalIgnoreCase), ApplyChanges); diff --git a/samples/Graph/Data/ACME/Project/_Source/Status.cs b/samples/Graph/Data/ACME/Project/Source/Status.cs similarity index 100% rename from samples/Graph/Data/ACME/Project/_Source/Status.cs rename to samples/Graph/Data/ACME/Project/Source/Status.cs diff --git a/samples/Graph/Data/ACME/Project/Todo/_Source/Category.cs b/samples/Graph/Data/ACME/Project/Todo/Source/Category.cs similarity index 100% rename from samples/Graph/Data/ACME/Project/Todo/_Source/Category.cs rename to samples/Graph/Data/ACME/Project/Todo/Source/Category.cs diff --git a/samples/Graph/Data/ACME/Project/Todo/_Source/Priority.cs b/samples/Graph/Data/ACME/Project/Todo/Source/Priority.cs similarity index 100% rename from samples/Graph/Data/ACME/Project/Todo/_Source/Priority.cs rename to samples/Graph/Data/ACME/Project/Todo/Source/Priority.cs diff --git a/samples/Graph/Data/ACME/Project/Todo/_Source/Status.cs b/samples/Graph/Data/ACME/Project/Todo/Source/Status.cs similarity index 100% rename from samples/Graph/Data/ACME/Project/Todo/_Source/Status.cs rename to samples/Graph/Data/ACME/Project/Todo/Source/Status.cs diff --git a/samples/Graph/Data/ACME/Project/Todo/_Source/Todo.cs b/samples/Graph/Data/ACME/Project/Todo/Source/Todo.cs similarity index 100% rename from samples/Graph/Data/ACME/Project/Todo/_Source/Todo.cs rename to samples/Graph/Data/ACME/Project/Todo/Source/Todo.cs diff --git a/samples/Graph/Data/ACME/Project/Todo/_Source/TodoLayoutAreas.cs b/samples/Graph/Data/ACME/Project/Todo/Source/TodoLayoutAreas.cs similarity index 94% rename from samples/Graph/Data/ACME/Project/Todo/_Source/TodoLayoutAreas.cs rename to samples/Graph/Data/ACME/Project/Todo/Source/TodoLayoutAreas.cs index 62c3eaa68..3fd5772bc 100644 --- a/samples/Graph/Data/ACME/Project/Todo/_Source/TodoLayoutAreas.cs +++ b/samples/Graph/Data/ACME/Project/Todo/Source/TodoLayoutAreas.cs @@ -273,38 +273,46 @@ You can restore it later from the Deleted view. .WithClickAction(_ => { host.UpdateArea(DialogControl.DialogArea, null!); return System.Threading.Tasks.Task.CompletedTask; })) .WithView(Controls.Button("Delete").WithAppearance(Appearance.Accent) .WithStyle(s => s.WithBackgroundColor("#dc3545")) - .WithClickAction(async ctx => + .WithClickAction(ctx => { host.UpdateArea(DialogControl.DialogArea, null!); - await SoftDeleteTodo(host); - // Navigate back to parent after soft delete - var segments = host.Hub.Address.Segments; - if (segments.Length > 1) - { - var parentPath = string.Join("/", segments.Take(segments.Length - 1)); - ctx.NavigateTo($"/{parentPath}"); - } + // Subscribe — no await on hub round-trip. Navigation happens + // on the success leg once the soft-delete commit lands. + SoftDeleteTodo(host, ctx); + return System.Threading.Tasks.Task.CompletedTask; }))); host.UpdateArea(DialogControl.DialogArea, Controls.Dialog(content, "Delete Task").WithSize("S").WithClosable(false)); } - private static async System.Threading.Tasks.Task SoftDeleteTodo(LayoutAreaHost host) + // No async — fully reactive. The workspace stream feeds into UpdateNode via + // SelectMany; Subscribe drives the side effect. Errors propagate to OnError. + // See Doc/Architecture/AsynchronousCalls.md. + private static void SoftDeleteTodo(LayoutAreaHost host, MeshWeaver.Layout.UiActionContext ctx) { var meshService = host.Hub.ServiceProvider.GetService(); if (meshService == null) return; - // Get current node from workspace stream (already loaded via AddMeshDataSource) var meshNodeStream = host.Workspace.GetStream(); if (meshNodeStream == null) return; - var node = await meshNodeStream + meshNodeStream .Select(nodes => nodes?.FirstOrDefault()) - .FirstOrDefaultAsync(); - if (node == null) return; - - var deletedNode = node with { State = MeshWeaver.Mesh.MeshNodeState.Deleted }; - await meshService.UpdateNodeAsync(deletedNode); + .Where(n => n is not null) + .Take(1) + .SelectMany(node => meshService.UpdateNode( + node! with { State = MeshWeaver.Mesh.MeshNodeState.Deleted })) + .Subscribe( + _ => + { + var segments = host.Hub.Address.Segments; + if (segments.Length > 1) + { + var parentPath = string.Join("/", segments.Take(segments.Length - 1)); + ctx.NavigateTo($"/{parentPath}"); + } + }, + _ => { /* dialog already closed; surfacing here would require a re-open */ }); } private static UiControl BuildStatusPromotionMenu(LayoutAreaHost host, Todo todo) diff --git a/samples/Graph/Data/ACME/index.md b/samples/Graph/Data/ACME/index.md index 5ebe8d823..672a0490c 100644 --- a/samples/Graph/Data/ACME/index.md +++ b/samples/Graph/Data/ACME/index.md @@ -1,5 +1,5 @@ --- -NodeType: Markdown +NodeType: Space Name: ACME Category: Task Management Description: Project and task management demo showcasing MeshWeaver's collaborative workflows and AI agent integration diff --git a/samples/Graph/Data/Cornerstone/Article/_Source/Article.cs b/samples/Graph/Data/Cornerstone/Article/Source/Article.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Article/_Source/Article.cs rename to samples/Graph/Data/Cornerstone/Article/Source/Article.cs diff --git a/samples/Graph/Data/Cornerstone/Article/_Source/ArticleLayoutAreas.cs b/samples/Graph/Data/Cornerstone/Article/Source/ArticleLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Article/_Source/ArticleLayoutAreas.cs rename to samples/Graph/Data/Cornerstone/Article/Source/ArticleLayoutAreas.cs diff --git a/samples/Graph/Data/Cornerstone/Documentation/UnifiedReferences.md b/samples/Graph/Data/Cornerstone/Documentation/UnifiedReferences.md index 575b99b3a..d2f7255bc 100644 --- a/samples/Graph/Data/Cornerstone/Documentation/UnifiedReferences.md +++ b/samples/Graph/Data/Cornerstone/Documentation/UnifiedReferences.md @@ -110,26 +110,26 @@ See [Data Prefix](/Doc/DataMesh/UnifiedPath/DataPrefix) for the generic data ref ## Display Property Risks ``` -@@Cornerstone/Microsoft/2026/data:PropertyRisk +@@Cornerstone/Microsoft/2026/data/PropertyRisk ``` -@@Cornerstone/Microsoft/2026/data:PropertyRisk +@@Cornerstone/Microsoft/2026/data/PropertyRisk ## Display Reinsurance Acceptances ``` -@@Cornerstone/Microsoft/2026/data:ReinsuranceAcceptance +@@Cornerstone/Microsoft/2026/data/ReinsuranceAcceptance ``` -@@Cornerstone/Microsoft/2026/data:ReinsuranceAcceptance +@@Cornerstone/Microsoft/2026/data/ReinsuranceAcceptance ## Display Reinsurance Sections ``` -@@Cornerstone/Microsoft/2026/data:ReinsuranceSection +@@Cornerstone/Microsoft/2026/data/ReinsuranceSection ``` -@@Cornerstone/Microsoft/2026/data:ReinsuranceSection +@@Cornerstone/Microsoft/2026/data/ReinsuranceSection # Dimension References @@ -198,13 +198,13 @@ Dimensions are shared across all pricings using the same NodeType. Content collections allow file references: ``` -@@Cornerstone/Microsoft/2026/content:Submissions/Slip.md +@@Cornerstone/Microsoft/2026/content/Submissions/Slip.md ``` -@@Cornerstone/Microsoft/2026/content:Submissions/Slip.md +@@Cornerstone/Microsoft/2026/content/Submissions/Slip.md ``` -@@Cornerstone/Microsoft/2026/content:Submissions/Microsoft.xlsx +@@Cornerstone/Microsoft/2026/content/Submissions/Microsoft.xlsx ``` # View References diff --git a/samples/Graph/Data/Cornerstone/Insured/_Source/InsuranceLayoutAreas.cs b/samples/Graph/Data/Cornerstone/Insured/Source/InsuranceLayoutAreas.cs similarity index 98% rename from samples/Graph/Data/Cornerstone/Insured/_Source/InsuranceLayoutAreas.cs rename to samples/Graph/Data/Cornerstone/Insured/Source/InsuranceLayoutAreas.cs index 5b7b70c9c..32b0c4442 100644 --- a/samples/Graph/Data/Cornerstone/Insured/_Source/InsuranceLayoutAreas.cs +++ b/samples/Graph/Data/Cornerstone/Insured/Source/InsuranceLayoutAreas.cs @@ -67,7 +67,7 @@ private static ImmutableList Thumbnails(IEnumerable nodes) var meshQuery = host.Hub.ServiceProvider.GetRequiredService(); var nodes = meshQuery - .ObserveQuery(MeshQueryRequest.FromQuery(query)) + .Query(MeshQueryRequest.FromQuery(query)) .Scan(new Dictionary(StringComparer.OrdinalIgnoreCase), ApplyChanges); return nodes.CombineLatest(statuses, (dict, statusList) => diff --git a/samples/Graph/Data/Cornerstone/Insured/_Source/Insured.cs b/samples/Graph/Data/Cornerstone/Insured/Source/Insured.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Insured/_Source/Insured.cs rename to samples/Graph/Data/Cornerstone/Insured/Source/Insured.cs diff --git a/samples/Graph/Data/Cornerstone/Insured/_Source/PricingStatus.cs b/samples/Graph/Data/Cornerstone/Insured/Source/PricingStatus.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Insured/_Source/PricingStatus.cs rename to samples/Graph/Data/Cornerstone/Insured/Source/PricingStatus.cs diff --git a/samples/Graph/Data/Cornerstone/Pricing.json b/samples/Graph/Data/Cornerstone/Pricing.json index 078dd3305..960f9ad4f 100644 --- a/samples/Graph/Data/Cornerstone/Pricing.json +++ b/samples/Graph/Data/Cornerstone/Pricing.json @@ -14,6 +14,6 @@ "displayName": "Pricing", "iconName": "DocumentBulletList", "description": "Cornerstone pricing with property risks and reinsurance structure", - "configuration": "config => config.WithContentType().AddContentCollection(sp => { var hub = sp.GetRequiredService(); var pricingId = hub.Address.Id; var basePath = System.IO.Path.Combine(\"../../samples/Graph/attachments/Cornerstone\", hub.Address.Segments[1], hub.Address.Segments[2], \"Submissions\"); return new ContentCollectionConfig { SourceType = FileSystemStreamProvider.SourceType, Name = $\"Submissions@{pricingId}\", BasePath = basePath, DisplayName = \"Submission Files\" }; }).AddData(data => data.AddSource(source => { var hub = source.Workspace.Hub; var pricingId = hub.Address.Id; var insuredName = hub.Address.Segments[1]; var isMicrosoft = insuredName == \"Microsoft\"; return source.WithType(t => t.WithInitialData(LineOfBusiness.All)).WithType(t => t.WithInitialData(Country.All)).WithType(t => t.WithInitialData(Currency.All)).WithType(t => t.WithInitialData(LegalEntity.All)).WithType(t => t.WithInitialData(PricingStatus.All)).WithType(t => isMicrosoft ? t.WithInitialData(async ct => await MicrosoftDataLoader.LoadPropertyRisksAsync(hub, ct)) : t.WithInitialData(Array.Empty())).WithType(t => isMicrosoft ? t.WithInitialData(async ct => { var (acceptances, _) = await MicrosoftDataLoader.LoadReinsuranceStructureAsync(hub, pricingId, ct); return acceptances; }) : t.WithInitialData(Array.Empty())).WithType(t => isMicrosoft ? t.WithInitialData(async ct => { var (_, sections) = await MicrosoftDataLoader.LoadReinsuranceStructureAsync(hub, pricingId, ct); return sections; }) : t.WithInitialData(Array.Empty())).WithType(t => t.WithInitialData(isMicrosoft ? MicrosoftSampleData.ImportConfigs : Array.Empty())); })).AddDefaultLayoutAreas().AddLayout(layout => layout.WithDefaultArea(\"Overview\").AddPricingLayoutAreas())" + "configuration": "config => config.WithContentType().AddContentCollection(sp => { var hub = sp.GetRequiredService(); var pricingId = hub.Address.Id; var basePath = System.IO.Path.Combine(\"../../samples/Graph/attachments/Cornerstone\", hub.Address.Segments[1], hub.Address.Segments[2], \"Submissions\"); return new ContentCollectionConfig { SourceType = FileSystemStreamProvider.SourceType, Name = $\"Submissions@{pricingId}\", BasePath = basePath, DisplayName = \"Submission Files\" }; }).AddData(data => data.AddSource(source => { var hub = source.Workspace.Hub; var pricingId = hub.Address.Id; var insuredName = hub.Address.Segments[1]; var isMicrosoft = insuredName == \"Microsoft\"; return source.WithType(t => t.WithInitialData(LineOfBusiness.All)).WithType(t => t.WithInitialData(Country.All)).WithType(t => t.WithInitialData(Currency.All)).WithType(t => t.WithInitialData(LegalEntity.All)).WithType(t => t.WithInitialData(PricingStatus.All)).WithType(t => isMicrosoft ? t.WithInitialData(() => MicrosoftDataLoader.LoadPropertyRisks(hub)) : t.WithInitialData(Array.Empty())).WithType(t => isMicrosoft ? t.WithInitialData(() => MicrosoftDataLoader.LoadReinsuranceAcceptances(hub, pricingId)) : t.WithInitialData(Array.Empty())).WithType(t => isMicrosoft ? t.WithInitialData(() => MicrosoftDataLoader.LoadReinsuranceSections(hub, pricingId)) : t.WithInitialData(Array.Empty())).WithType(t => t.WithInitialData(isMicrosoft ? MicrosoftSampleData.ImportConfigs : Array.Empty())); })).AddDefaultLayoutAreas().AddLayout(layout => layout.WithDefaultArea(\"Overview\").AddPricingLayoutAreas())" } } diff --git a/samples/Graph/Data/Cornerstone/Pricing/_Source/Country.cs b/samples/Graph/Data/Cornerstone/Pricing/Source/Country.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Pricing/_Source/Country.cs rename to samples/Graph/Data/Cornerstone/Pricing/Source/Country.cs diff --git a/samples/Graph/Data/Cornerstone/Pricing/_Source/Currency.cs b/samples/Graph/Data/Cornerstone/Pricing/Source/Currency.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Pricing/_Source/Currency.cs rename to samples/Graph/Data/Cornerstone/Pricing/Source/Currency.cs diff --git a/samples/Graph/Data/Cornerstone/Pricing/_Source/LegalEntity.cs b/samples/Graph/Data/Cornerstone/Pricing/Source/LegalEntity.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Pricing/_Source/LegalEntity.cs rename to samples/Graph/Data/Cornerstone/Pricing/Source/LegalEntity.cs diff --git a/samples/Graph/Data/Cornerstone/Pricing/_Source/LineOfBusiness.cs b/samples/Graph/Data/Cornerstone/Pricing/Source/LineOfBusiness.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Pricing/_Source/LineOfBusiness.cs rename to samples/Graph/Data/Cornerstone/Pricing/Source/LineOfBusiness.cs diff --git a/samples/Graph/Data/Cornerstone/Pricing/_Source/MicrosoftDataLoader.cs b/samples/Graph/Data/Cornerstone/Pricing/Source/MicrosoftDataLoader.cs similarity index 65% rename from samples/Graph/Data/Cornerstone/Pricing/_Source/MicrosoftDataLoader.cs rename to samples/Graph/Data/Cornerstone/Pricing/Source/MicrosoftDataLoader.cs index c227d6857..e34fe696e 100644 --- a/samples/Graph/Data/Cornerstone/Pricing/_Source/MicrosoftDataLoader.cs +++ b/samples/Graph/Data/Cornerstone/Pricing/Source/MicrosoftDataLoader.cs @@ -4,16 +4,21 @@ // using System; +using System.Collections.Generic; using System.IO; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using MeshWeaver.Messaging; +using MeshWeaver.Mesh.Threading; +using Microsoft.Extensions.DependencyInjection; /// -/// Async loader utility for Microsoft pricing data. +/// Loader utility for Microsoft pricing data. /// Loads PropertyRisks from JSON and reinsurance structure from Slip.md. /// Reads files directly from the file system to avoid timing issues with content service initialization. +/// Public surface is reactive only (IObservable via the FileSystem IIoPool); +/// the async file reads are private leaves pumped exclusively inside the pool. /// public static class MicrosoftDataLoader { @@ -24,11 +29,13 @@ public static class MicrosoftDataLoader /// /// Loads PropertyRisk records from PropertyRisks.json in the content folder. + /// PRIVATE async leaf — only ever awaited inside the IIoPool bridge + /// (), never from hub-reachable code. /// /// The message hub for resolving paths /// Cancellation token /// Array of PropertyRisk records - public static async Task LoadPropertyRisksAsync( + private static async Task LoadPropertyRisksAsync( IMessageHub hub, CancellationToken cancellationToken = default) { @@ -91,12 +98,14 @@ private static string GetPropertyRisksFilePath(IMessageHub hub) /// /// Loads reinsurance structure from Slip.md in the Submissions folder. + /// PRIVATE async leaf — only ever awaited inside the IIoPool bridges + /// ( / ). /// /// The message hub for resolving paths /// The pricing ID for generated records /// Cancellation token /// Tuple of acceptances and sections - public static async Task<(ReinsuranceAcceptance[] Acceptances, ReinsuranceSection[] Sections)> LoadReinsuranceStructureAsync( + private static async Task<(ReinsuranceAcceptance[] Acceptances, ReinsuranceSection[] Sections)> LoadReinsuranceStructureAsync( IMessageHub hub, string pricingId, CancellationToken cancellationToken = default) @@ -149,4 +158,44 @@ private static string GetSubmissionsBasePath(IMessageHub hub) // Fallback: return production path (will fail gracefully with empty data) return productionPath; } + + /// + /// Resolves the bounded FileSystem I/O pool from the hub (falling back to the + /// unbounded pool when no registry is present, e.g. lightweight test hubs). + /// All file reads below run on this pool — never Observable.FromAsync. + /// + private static IIoPool FileSystemPool(IMessageHub hub) => + hub.ServiceProvider.GetService()?.Get(IoPoolNames.FileSystem) + ?? IoPool.Unbounded; + + /// + /// Reactive projection of for + /// WithInitialData(Func<IObservable<IEnumerable<PropertyRisk>>>). + /// The async file read is bridged through the FileSystem I/O pool. + /// + public static IObservable> LoadPropertyRisks(IMessageHub hub) => + FileSystemPool(hub).Invoke(async ct => + (IEnumerable)await LoadPropertyRisksAsync(hub, ct)); + + /// + /// Reactive projection of the acceptances half of . + /// + public static IObservable> LoadReinsuranceAcceptances( + IMessageHub hub, string pricingId) => + FileSystemPool(hub).Invoke(async ct => + { + var (acceptances, _) = await LoadReinsuranceStructureAsync(hub, pricingId, ct); + return (IEnumerable)acceptances; + }); + + /// + /// Reactive projection of the sections half of . + /// + public static IObservable> LoadReinsuranceSections( + IMessageHub hub, string pricingId) => + FileSystemPool(hub).Invoke(async ct => + { + var (_, sections) = await LoadReinsuranceStructureAsync(hub, pricingId, ct); + return (IEnumerable)sections; + }); } diff --git a/samples/Graph/Data/Cornerstone/Pricing/_Source/MicrosoftSampleData.cs b/samples/Graph/Data/Cornerstone/Pricing/Source/MicrosoftSampleData.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Pricing/_Source/MicrosoftSampleData.cs rename to samples/Graph/Data/Cornerstone/Pricing/Source/MicrosoftSampleData.cs diff --git a/samples/Graph/Data/Cornerstone/Pricing/_Source/Pricing.cs b/samples/Graph/Data/Cornerstone/Pricing/Source/Pricing.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Pricing/_Source/Pricing.cs rename to samples/Graph/Data/Cornerstone/Pricing/Source/Pricing.cs diff --git a/samples/Graph/Data/Cornerstone/Pricing/_Source/PricingLayoutAreas.cs b/samples/Graph/Data/Cornerstone/Pricing/Source/PricingLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Pricing/_Source/PricingLayoutAreas.cs rename to samples/Graph/Data/Cornerstone/Pricing/Source/PricingLayoutAreas.cs diff --git a/samples/Graph/Data/Cornerstone/Pricing/_Source/PricingStatus.cs b/samples/Graph/Data/Cornerstone/Pricing/Source/PricingStatus.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Pricing/_Source/PricingStatus.cs rename to samples/Graph/Data/Cornerstone/Pricing/Source/PricingStatus.cs diff --git a/samples/Graph/Data/Cornerstone/Pricing/_Source/PropertyRisk.cs b/samples/Graph/Data/Cornerstone/Pricing/Source/PropertyRisk.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Pricing/_Source/PropertyRisk.cs rename to samples/Graph/Data/Cornerstone/Pricing/Source/PropertyRisk.cs diff --git a/samples/Graph/Data/Cornerstone/Pricing/_Source/ReinsuranceAcceptance.cs b/samples/Graph/Data/Cornerstone/Pricing/Source/ReinsuranceAcceptance.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Pricing/_Source/ReinsuranceAcceptance.cs rename to samples/Graph/Data/Cornerstone/Pricing/Source/ReinsuranceAcceptance.cs diff --git a/samples/Graph/Data/Cornerstone/Pricing/_Source/ReinsuranceSection.cs b/samples/Graph/Data/Cornerstone/Pricing/Source/ReinsuranceSection.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Pricing/_Source/ReinsuranceSection.cs rename to samples/Graph/Data/Cornerstone/Pricing/Source/ReinsuranceSection.cs diff --git a/samples/Graph/Data/Cornerstone/Pricing/_Source/SlipParser.cs b/samples/Graph/Data/Cornerstone/Pricing/Source/SlipParser.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/Pricing/_Source/SlipParser.cs rename to samples/Graph/Data/Cornerstone/Pricing/Source/SlipParser.cs diff --git a/samples/Graph/Data/Cornerstone/_Source/Country.cs b/samples/Graph/Data/Cornerstone/Source/Country.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/_Source/Country.cs rename to samples/Graph/Data/Cornerstone/Source/Country.cs diff --git a/samples/Graph/Data/Cornerstone/_Source/Currency.cs b/samples/Graph/Data/Cornerstone/Source/Currency.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/_Source/Currency.cs rename to samples/Graph/Data/Cornerstone/Source/Currency.cs diff --git a/samples/Graph/Data/Cornerstone/_Source/LegalEntity.cs b/samples/Graph/Data/Cornerstone/Source/LegalEntity.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/_Source/LegalEntity.cs rename to samples/Graph/Data/Cornerstone/Source/LegalEntity.cs diff --git a/samples/Graph/Data/Cornerstone/_Source/LineOfBusiness.cs b/samples/Graph/Data/Cornerstone/Source/LineOfBusiness.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/_Source/LineOfBusiness.cs rename to samples/Graph/Data/Cornerstone/Source/LineOfBusiness.cs diff --git a/samples/Graph/Data/Cornerstone/_Source/PricingStatus.cs b/samples/Graph/Data/Cornerstone/Source/PricingStatus.cs similarity index 100% rename from samples/Graph/Data/Cornerstone/_Source/PricingStatus.cs rename to samples/Graph/Data/Cornerstone/Source/PricingStatus.cs diff --git a/samples/Graph/Data/Cornerstone/index.md b/samples/Graph/Data/Cornerstone/index.md index ddf8c725e..b6c8b2e31 100644 --- a/samples/Graph/Data/Cornerstone/index.md +++ b/samples/Graph/Data/Cornerstone/index.md @@ -1,5 +1,5 @@ --- -NodeType: Organization +NodeType: Space Name: Cornerstone Category: Insurance Description: Reinsurance pricing demo showcasing property risk management, geographic visualization, and Excel data import diff --git a/samples/Graph/Data/FutuRe/AmountType/_Source/AmountType.cs b/samples/Graph/Data/FutuRe/AmountType/Source/AmountType.cs similarity index 100% rename from samples/Graph/Data/FutuRe/AmountType/_Source/AmountType.cs rename to samples/Graph/Data/FutuRe/AmountType/Source/AmountType.cs diff --git a/samples/Graph/Data/FutuRe/BusinessUnit/_Source/BusinessUnit.cs b/samples/Graph/Data/FutuRe/BusinessUnit/Source/BusinessUnit.cs similarity index 100% rename from samples/Graph/Data/FutuRe/BusinessUnit/_Source/BusinessUnit.cs rename to samples/Graph/Data/FutuRe/BusinessUnit/Source/BusinessUnit.cs diff --git a/samples/Graph/Data/FutuRe/BusinessUnit/_Source/BusinessUnitLayoutAreas.cs b/samples/Graph/Data/FutuRe/BusinessUnit/Source/BusinessUnitLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/FutuRe/BusinessUnit/_Source/BusinessUnitLayoutAreas.cs rename to samples/Graph/Data/FutuRe/BusinessUnit/Source/BusinessUnitLayoutAreas.cs diff --git a/samples/Graph/Data/FutuRe/Country/_Source/Country.cs b/samples/Graph/Data/FutuRe/Country/Source/Country.cs similarity index 100% rename from samples/Graph/Data/FutuRe/Country/_Source/Country.cs rename to samples/Graph/Data/FutuRe/Country/Source/Country.cs diff --git a/samples/Graph/Data/FutuRe/Currency/_Source/Currency.cs b/samples/Graph/Data/FutuRe/Currency/Source/Currency.cs similarity index 100% rename from samples/Graph/Data/FutuRe/Currency/_Source/Currency.cs rename to samples/Graph/Data/FutuRe/Currency/Source/Currency.cs diff --git a/samples/Graph/Data/FutuRe/ExchangeRate/_Source/ExchangeRate.cs b/samples/Graph/Data/FutuRe/ExchangeRate/Source/ExchangeRate.cs similarity index 100% rename from samples/Graph/Data/FutuRe/ExchangeRate/_Source/ExchangeRate.cs rename to samples/Graph/Data/FutuRe/ExchangeRate/Source/ExchangeRate.cs diff --git a/samples/Graph/Data/FutuRe/GroupAnalysis/_Source/AnalysisContent.cs b/samples/Graph/Data/FutuRe/GroupAnalysis/Source/AnalysisContent.cs similarity index 100% rename from samples/Graph/Data/FutuRe/GroupAnalysis/_Source/AnalysisContent.cs rename to samples/Graph/Data/FutuRe/GroupAnalysis/Source/AnalysisContent.cs diff --git a/samples/Graph/Data/FutuRe/GroupAnalysis/Source/ExternalDependencies.cs b/samples/Graph/Data/FutuRe/GroupAnalysis/Source/ExternalDependencies.cs new file mode 100644 index 000000000..bca5f5b32 --- /dev/null +++ b/samples/Graph/Data/FutuRe/GroupAnalysis/Source/ExternalDependencies.cs @@ -0,0 +1,12 @@ +// +// Id: ExternalDependencies +// DisplayName: External Dependencies +// + +@@FutuRe/AmountType/Source/AmountType +@@FutuRe/Currency/Source/Currency +@@FutuRe/Country/Source/Country +@@FutuRe/TransactionMapping/Source/TransactionMapping +@@FutuRe/LineOfBusiness/Source/LineOfBusiness +@@FutuRe/ExchangeRate/Source/ExchangeRate +@@FutuRe/BusinessUnit/Source/BusinessUnit diff --git a/samples/Graph/Data/FutuRe/GroupAnalysis/_Source/FutuReDataCube.cs b/samples/Graph/Data/FutuRe/GroupAnalysis/Source/FutuReDataCube.cs similarity index 100% rename from samples/Graph/Data/FutuRe/GroupAnalysis/_Source/FutuReDataCube.cs rename to samples/Graph/Data/FutuRe/GroupAnalysis/Source/FutuReDataCube.cs diff --git a/samples/Graph/Data/FutuRe/GroupAnalysis/_Source/FutuReDataLoader.cs b/samples/Graph/Data/FutuRe/GroupAnalysis/Source/FutuReDataLoader.cs similarity index 78% rename from samples/Graph/Data/FutuRe/GroupAnalysis/_Source/FutuReDataLoader.cs rename to samples/Graph/Data/FutuRe/GroupAnalysis/Source/FutuReDataLoader.cs index 5845e2b7d..19e495072 100644 --- a/samples/Graph/Data/FutuRe/GroupAnalysis/_Source/FutuReDataLoader.cs +++ b/samples/Graph/Data/FutuRe/GroupAnalysis/Source/FutuReDataLoader.cs @@ -11,6 +11,7 @@ using MeshWeaver.Data; using MeshWeaver.Mesh; using MeshWeaver.Mesh.Services; +using MeshWeaver.Mesh.Threading; using Microsoft.Extensions.DependencyInjection; /// @@ -32,43 +33,93 @@ public static class FutuReDataLoader public static IObservable> LoadLocalDataCube(IWorkspace workspace) { var contentService = workspace.Hub.ServiceProvider.GetRequiredService(); - var meshQuery = workspace.Hub.ServiceProvider.GetRequiredService(); + var hub = workspace.Hub; var address = workspace.Hub.Address.ToString(); var segments = address.Split('/'); var businessUnit = segments.Length > 1 ? segments[1] : address; var buPath = segments.Length > 1 ? $"{segments[0]}/{segments[1]}" : segments[0]; - return Observable.FromAsync<(List Rows, string Currency)>(async ct => - { - // Look up the BU currency from the mesh node - var buCurrency = "CHF"; - var buNode = await meshQuery.QueryAsync($"path:{buPath}", ct: ct).FirstOrDefaultAsync(ct); - if (buNode?.Content is BusinessUnit bu) - buCurrency = bu.Currency; - else if (buNode?.Content is JsonElement json - && json.TryGetProperty("currency", out var val) - && val.ValueKind == JsonValueKind.String) - buCurrency = val.GetString() ?? "CHF"; - - var stream = await contentService.GetContentAsync("attachments", "datacube.csv", ct); - if (stream == null) - return (new List(), buCurrency); - using var reader = new StreamReader(stream); - var content = await reader.ReadToEndAsync(ct); - return (ParseLocalCsvContent(content, businessUnit), buCurrency); - }).CombineLatest( - LoadLocalLinesOfBusiness(workspace), - (csvResult, lobs) => + // BU node lookup goes through the per-node MeshNodeReference reducer (authoritative, + // no read-side index lag). CSV I/O stays on a separate Observable.FromAsync at the + // file boundary. Both compose into the final tuple. + // + // Cross-ALC type-identity is fragile here: BusinessUnit gets compiled into + // FutuRe_BusinessUnit's NodeAssemblyLoadContext while this loader lives in + // FutuRe_LocalAnalysis's ALC, so `Content is BusinessUnit` returns false even + // though the runtime types match by full name. We probe by JsonElement first + // (the deserialized-to-JSON path), then by reflection (`Currency` property) + // for the cross-ALC custom-type case, before falling back to CHF. + var buCurrencyObs = hub.GetMeshNode(buPath, TimeSpan.FromSeconds(10)) + .Select(buNode => { - var lobLookup = lobs.ToDictionary(l => l.SystemName, l => l.DisplayName); - return csvResult.Rows.Select(row => row with + if (buNode?.Content is JsonElement json + && json.ValueKind == JsonValueKind.Object) { - LineOfBusinessName = lobLookup.GetValueOrDefault(row.LineOfBusiness, row.LineOfBusiness), - LocalLineOfBusinessName = lobLookup.GetValueOrDefault(row.LocalLineOfBusiness, row.LocalLineOfBusiness), - Currency = csvResult.Currency - }).AsEnumerable(); - } - ).DistinctUntilChanged(); + // Mesh stream serialisation may emit either casing depending on + // JsonSerializerOptions; probe both. + if (json.TryGetProperty("currency", out var val) + && val.ValueKind == JsonValueKind.String) + return val.GetString() ?? "CHF"; + if (json.TryGetProperty("Currency", out var val2) + && val2.ValueKind == JsonValueKind.String) + return val2.GetString() ?? "CHF"; + } + if (buNode?.Content is BusinessUnit bu) + return bu.Currency; + if (buNode?.Content is { } content) + { + var prop = content.GetType().GetProperty("Currency"); + if (prop?.GetValue(content) is string s && !string.IsNullOrEmpty(s)) + return s; + } + // Fallback by hub address — every BU's local currency is known statically; + // this keeps the local Analysis dashboard labelling correct when the + // cross-hub MeshNode lookup fails (e.g. BU hub not yet activated, or + // running in a partial test context). + return businessUnit switch + { + "EuropeRe" => "EUR", + "AmericasIns" => "USD", + "AsiaRe" => "JPY", + _ => "CHF" + }; + }); + + // CSV I/O on the bounded FileSystem IoPool — no Observable.FromAsync, no async/await. + // Invoke() runs the genuinely-async content fetch on the pool's semaphore-gated + // Scheduler.Default: the ThreadPool thread is released during the await, so the + // continuation can't re-enter a congested hub action block (the cause of the + // intermittent 50s render stall). InvokeBlocking() runs the sync StreamReader read + // + CSV parse on the pool's dedicated limited-concurrency scheduler, so the blocking + // read can't trigger ThreadPool thread-injection that starves the grain schedulers. + // Both are cold IObservables — reactive end-to-end, no async state machine. + var ioPool = workspace.Hub.ServiceProvider.GetService()?.Get(IoPoolNames.FileSystem) + ?? IoPool.Unbounded; + var csvRowsObs = ioPool + .Invoke(ct => contentService.GetContentAsync("attachments", "datacube.csv", ct)) + .SelectMany(stream => stream is null + ? Observable.Return(new List()) + : ioPool.InvokeBlocking(_ => + { + using var reader = new StreamReader(stream); + return ParseLocalCsvContent(reader.ReadToEnd(), businessUnit); + })); + + return buCurrencyObs + .SelectMany(currency => csvRowsObs.Select(rows => (Rows: rows, Currency: currency))) + .CombineLatest( + LoadLocalLinesOfBusiness(workspace), + (csvResult, lobs) => + { + var lobLookup = lobs.ToDictionary(l => l.SystemName, l => l.DisplayName); + return csvResult.Rows.Select(row => row with + { + LineOfBusinessName = lobLookup.GetValueOrDefault(row.LineOfBusiness, row.LineOfBusiness), + LocalLineOfBusinessName = lobLookup.GetValueOrDefault(row.LocalLineOfBusiness, row.LocalLineOfBusiness), + Currency = csvResult.Currency + }).AsEnumerable(); + } + ).DistinctUntilChanged(); } /// @@ -231,7 +282,7 @@ public static IObservable> LoadLocalLinesOfBusiness( var meshQuery = workspace.Hub.ServiceProvider.GetRequiredService(); return meshQuery - .ObserveQuery( + .Query( MeshQueryRequest.FromQuery( $"nodeType:FutuRe/LineOfBusiness namespace:{buNamespace}/LineOfBusiness state:Active")) .Select(change => change.Items @@ -252,7 +303,7 @@ public static IObservable> LoadAmountTypes(IWorkspace wo { var meshQuery = workspace.Hub.ServiceProvider.GetRequiredService(); return meshQuery - .ObserveQuery( + .Query( MeshQueryRequest.FromQuery("nodeType:FutuRe/AmountType namespace:FutuRe/AmountType state:Active")) .Select(change => change.Items .Select(ConvertToAmountType) @@ -268,7 +319,7 @@ public static IObservable> LoadCurrencies(IWorkspace works { var meshQuery = workspace.Hub.ServiceProvider.GetRequiredService(); return meshQuery - .ObserveQuery( + .Query( MeshQueryRequest.FromQuery("nodeType:FutuRe/Currency namespace:FutuRe/Currency state:Active")) .Select(change => change.Items .Select(ConvertToCurrency) @@ -284,7 +335,7 @@ public static IObservable> LoadCountries(IWorkspace workspa { var meshQuery = workspace.Hub.ServiceProvider.GetRequiredService(); return meshQuery - .ObserveQuery( + .Query( MeshQueryRequest.FromQuery("nodeType:FutuRe/Country namespace:FutuRe/Country state:Active")) .Select(change => change.Items .Select(ConvertToCountry) @@ -301,7 +352,7 @@ public static IObservable> LoadTransactionMappin var meshQuery = workspace.Hub.ServiceProvider.GetRequiredService(); return meshQuery - .ObserveQuery( + .Query( MeshQueryRequest.FromQuery("nodeType:FutuRe/TransactionMapping namespace:FutuRe scope:descendants")) .Select(change => change.Items .Select(ConvertToTransactionMapping) @@ -316,7 +367,7 @@ public static IObservable> LoadExchangeRates(IWorkspac { var meshQuery = workspace.Hub.ServiceProvider.GetRequiredService(); return meshQuery - .ObserveQuery( + .Query( MeshQueryRequest.FromQuery("nodeType:FutuRe/ExchangeRate namespace:FutuRe/ExchangeRate state:Active")) .Select(change => change.Items .Select(ConvertToExchangeRate) @@ -332,7 +383,7 @@ public static IObservable> LoadBusinessUnits(IWorkspac { var meshQuery = workspace.Hub.ServiceProvider.GetRequiredService(); return meshQuery - .ObserveQuery( + .Query( MeshQueryRequest.FromQuery("nodeType:FutuRe/BusinessUnit namespace:FutuRe state:Active")) .Select(change => change.Items .Select(ConvertToBusinessUnit) @@ -348,7 +399,7 @@ public static IObservable> LoadLinesOfBusinessFromNo var meshQuery = workspace.Hub.ServiceProvider.GetRequiredService(); return meshQuery - .ObserveQuery( + .Query( MeshQueryRequest.FromQuery("nodeType:FutuRe/LineOfBusiness namespace:FutuRe/LineOfBusiness state:Active")) .Select(change => change.Items .Select(ConvertToLineOfBusiness) diff --git a/samples/Graph/Data/FutuRe/GroupAnalysis/_Source/GroupAnalysisConfig.cs b/samples/Graph/Data/FutuRe/GroupAnalysis/Source/GroupAnalysisConfig.cs similarity index 100% rename from samples/Graph/Data/FutuRe/GroupAnalysis/_Source/GroupAnalysisConfig.cs rename to samples/Graph/Data/FutuRe/GroupAnalysis/Source/GroupAnalysisConfig.cs diff --git a/samples/Graph/Data/FutuRe/GroupAnalysis/_Source/ProfitabilityLayoutAreas.cs b/samples/Graph/Data/FutuRe/GroupAnalysis/Source/ProfitabilityLayoutAreas.cs similarity index 95% rename from samples/Graph/Data/FutuRe/GroupAnalysis/_Source/ProfitabilityLayoutAreas.cs rename to samples/Graph/Data/FutuRe/GroupAnalysis/Source/ProfitabilityLayoutAreas.cs index b3399b94d..1f1bc7f6f 100644 --- a/samples/Graph/Data/FutuRe/GroupAnalysis/_Source/ProfitabilityLayoutAreas.cs +++ b/samples/Graph/Data/FutuRe/GroupAnalysis/Source/ProfitabilityLayoutAreas.cs @@ -474,8 +474,15 @@ private static UiControl RenderWaterfall(List allData, bool isOr // Individual Areas (toolbar shown on group hub only) // --------------------------------------------------------------- + // Group hub = FutuRe/Analysis (the parent rollup). Local BU hubs + // (FutuRe/EuropeRe/Analysis, FutuRe/AmericasIns/Analysis, FutuRe/AsiaRe/Analysis) + // ALSO have a TransactionMapping data source — their own mapping rules — so + // the previous "stream != null" probe identified them as group hubs and routed + // them through the toolbar branch with PlanChf, converting their amounts to + // CHF and labelling them "CHF". The hub address is the authoritative + // discriminator: only "FutuRe/Analysis" is the group hub. private static bool IsGroupHub(LayoutAreaHost host) - => host.Workspace.GetStream() != null; + => string.Equals(host.Hub.Address.ToString(), "FutuRe/Analysis", StringComparison.Ordinal); /// /// Renders a view with currency toolbar on group hub, without toolbar on local hubs. @@ -493,8 +500,13 @@ private static UiControl RenderView( return GetDataCube(area, toolbar.CurrencyMode) .Select(data => render(data.ToList(), isOriginal, label)); }); + // Local BU hubs: stay in the BU's own currency. Using PlanChf here would + // convert EuropeRe (EUR) to CHF and then label it "CHF" — `isOriginal=true` + // means "show the data's Currency field" so the conversion + label drift + // out of sync. OriginalCurrency keeps amounts and labels in the BU's + // native currency (EuropeRe → EUR, AmericasIns → USD, AsiaRe → JPY). return Controls.Stack.WithView((LayoutAreaHost area, RenderingContext ctx) => - GetDataCube(area).Select(data => render(data.ToList(), true, ""))); + GetDataCube(area, CurrencyModes.OriginalCurrency).Select(data => render(data.ToList(), true, ""))); } /// @@ -521,8 +533,10 @@ IObservable BuildStack(LayoutAreaHost area, string mode, bool isOrigi var label = isOriginal ? "" : " (CHF)"; return BuildStack(area, toolbar.CurrencyMode, isOriginal, label); }); + // Local BU hubs: stay in the BU's own currency (see RenderView above + // for the rationale on OriginalCurrency vs PlanChf). return Controls.Stack.WithView((LayoutAreaHost area, RenderingContext ctx) => - BuildStack(area, CurrencyModes.PlanChf, true, "")); + BuildStack(area, CurrencyModes.OriginalCurrency, true, "")); } [Display(GroupName = "Profitability", Order = 10)] diff --git a/samples/Graph/Data/FutuRe/GroupAnalysis/_Source/ExternalDependencies.cs b/samples/Graph/Data/FutuRe/GroupAnalysis/_Source/ExternalDependencies.cs deleted file mode 100644 index 78742e37b..000000000 --- a/samples/Graph/Data/FutuRe/GroupAnalysis/_Source/ExternalDependencies.cs +++ /dev/null @@ -1,12 +0,0 @@ -// -// Id: ExternalDependencies -// DisplayName: External Dependencies -// - -@@FutuRe/AmountType/_Source/AmountType -@@FutuRe/Currency/_Source/Currency -@@FutuRe/Country/_Source/Country -@@FutuRe/TransactionMapping/_Source/TransactionMapping -@@FutuRe/LineOfBusiness/_Source/LineOfBusiness -@@FutuRe/ExchangeRate/_Source/ExchangeRate -@@FutuRe/BusinessUnit/_Source/BusinessUnit diff --git a/samples/Graph/Data/FutuRe/LineOfBusiness/_Source/LineOfBusiness.cs b/samples/Graph/Data/FutuRe/LineOfBusiness/Source/LineOfBusiness.cs similarity index 100% rename from samples/Graph/Data/FutuRe/LineOfBusiness/_Source/LineOfBusiness.cs rename to samples/Graph/Data/FutuRe/LineOfBusiness/Source/LineOfBusiness.cs diff --git a/samples/Graph/Data/FutuRe/LineOfBusiness/_Source/LineOfBusinessLayoutAreas.cs b/samples/Graph/Data/FutuRe/LineOfBusiness/Source/LineOfBusinessLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/FutuRe/LineOfBusiness/_Source/LineOfBusinessLayoutAreas.cs rename to samples/Graph/Data/FutuRe/LineOfBusiness/Source/LineOfBusinessLayoutAreas.cs diff --git a/samples/Graph/Data/FutuRe/LocalAnalysis/Source/ExternalDependencies.cs b/samples/Graph/Data/FutuRe/LocalAnalysis/Source/ExternalDependencies.cs new file mode 100644 index 000000000..5c9781b20 --- /dev/null +++ b/samples/Graph/Data/FutuRe/LocalAnalysis/Source/ExternalDependencies.cs @@ -0,0 +1,10 @@ +// +// Id: ExternalDependencies +// DisplayName: External Dependencies +// + +@@FutuRe/GroupAnalysis/Source/FutuReDataCube +@@FutuRe/GroupAnalysis/Source/AnalysisContent +@@FutuRe/GroupAnalysis/Source/ProfitabilityLayoutAreas +@@FutuRe/GroupAnalysis/Source/FutuReDataLoader +@@FutuRe/GroupAnalysis/Source/ExternalDependencies diff --git a/samples/Graph/Data/FutuRe/LocalAnalysis/_Source/LocalAnalysisConfig.cs b/samples/Graph/Data/FutuRe/LocalAnalysis/Source/LocalAnalysisConfig.cs similarity index 100% rename from samples/Graph/Data/FutuRe/LocalAnalysis/_Source/LocalAnalysisConfig.cs rename to samples/Graph/Data/FutuRe/LocalAnalysis/Source/LocalAnalysisConfig.cs diff --git a/samples/Graph/Data/FutuRe/LocalAnalysis/_Source/ExternalDependencies.cs b/samples/Graph/Data/FutuRe/LocalAnalysis/_Source/ExternalDependencies.cs deleted file mode 100644 index 4bd42d074..000000000 --- a/samples/Graph/Data/FutuRe/LocalAnalysis/_Source/ExternalDependencies.cs +++ /dev/null @@ -1,10 +0,0 @@ -// -// Id: ExternalDependencies -// DisplayName: External Dependencies -// - -@@FutuRe/GroupAnalysis/_Source/FutuReDataCube -@@FutuRe/GroupAnalysis/_Source/AnalysisContent -@@FutuRe/GroupAnalysis/_Source/ProfitabilityLayoutAreas -@@FutuRe/GroupAnalysis/_Source/FutuReDataLoader -@@FutuRe/GroupAnalysis/_Source/ExternalDependencies diff --git a/samples/Graph/Data/FutuRe/TransactionMapping/_Source/TransactionMapping.cs b/samples/Graph/Data/FutuRe/TransactionMapping/Source/TransactionMapping.cs similarity index 100% rename from samples/Graph/Data/FutuRe/TransactionMapping/_Source/TransactionMapping.cs rename to samples/Graph/Data/FutuRe/TransactionMapping/Source/TransactionMapping.cs diff --git a/samples/Graph/Data/MathDemo/Matrix.json b/samples/Graph/Data/MathDemo/Matrix.json new file mode 100644 index 000000000..d25b5cce2 --- /dev/null +++ b/samples/Graph/Data/MathDemo/Matrix.json @@ -0,0 +1,17 @@ +{ + "id": "Matrix", + "namespace": "MathDemo", + "name": "Matrix", + "nodeType": "NodeType", + "category": "Types", + "description": "A 2x2 matrix rendered with MathNet.Numerics (loaded via #r \"nuget:...\")", + "isPersistent": true, + "content": { + "$type": "NodeTypeDefinition", + "namespace": "MathDemo", + "displayName": "Matrix", + "iconName": "Grid", + "description": "A 2x2 matrix rendered with MathNet.Numerics", + "configuration": "config => config.WithContentType().AddLayout(layout => layout.AddMatrixLayoutAreas().WithDefaultArea(\"Inverse\"))" + } +} diff --git a/samples/Graph/Data/MathDemo/Matrix/Example.json b/samples/Graph/Data/MathDemo/Matrix/Example.json new file mode 100644 index 000000000..29163933e --- /dev/null +++ b/samples/Graph/Data/MathDemo/Matrix/Example.json @@ -0,0 +1,16 @@ +{ + "id": "Example", + "namespace": "MathDemo/Matrix", + "name": "Example 2x2", + "nodeType": "MathDemo/Matrix", + "description": "Sample 2x2 matrix for the NuGet-in-node-type documentation walkthrough.", + "isPersistent": true, + "content": { + "$type": "Matrix", + "name": "Example 2x2", + "a11": 1, + "a12": 2, + "a21": 3, + "a22": 4 + } +} diff --git a/samples/Graph/Data/MathDemo/Matrix/Source/Matrix.cs b/samples/Graph/Data/MathDemo/Matrix/Source/Matrix.cs new file mode 100644 index 000000000..beb3b5708 --- /dev/null +++ b/samples/Graph/Data/MathDemo/Matrix/Source/Matrix.cs @@ -0,0 +1,25 @@ +// +// Id: Matrix +// DisplayName: Matrix Data Model +// +#r "nuget:MathNet.Numerics, 5.0.0" + +using MathNet.Numerics.LinearAlgebra; +using MeshWeaver.Domain; + +public record Matrix +{ + [Required] + [MeshNodeProperty(nameof(MeshNode.Name))] + public string Name { get; init; } = string.Empty; + + public double A11 { get; init; } = 1; + public double A12 { get; init; } = 2; + public double A21 { get; init; } = 3; + public double A22 { get; init; } = 4; + + public double Determinant() => + Matrix.Build + .DenseOfArray(new[,] { { A11, A12 }, { A21, A22 } }) + .Determinant(); +} diff --git a/samples/Graph/Data/MathDemo/Matrix/Source/MatrixLayoutAreas.cs b/samples/Graph/Data/MathDemo/Matrix/Source/MatrixLayoutAreas.cs new file mode 100644 index 000000000..204e7a8d9 --- /dev/null +++ b/samples/Graph/Data/MathDemo/Matrix/Source/MatrixLayoutAreas.cs @@ -0,0 +1,35 @@ +// +// Id: MatrixLayoutAreas +// DisplayName: Matrix Layout Areas +// +#r "nuget:MathNet.Numerics, 5.0.0" + +using MathNet.Numerics.LinearAlgebra; +using MeshWeaver.Layout; +using MeshWeaver.Layout.Composition; + +public static class MatrixLayoutAreas +{ + public static LayoutDefinition AddMatrixLayoutAreas(this LayoutDefinition layout) => + layout.WithView("Inverse", Inverse); + + public static UiControl Inverse(LayoutAreaHost host, RenderingContext _) + { + var m = Matrix.Build.DenseOfArray(new[,] + { + { 1.0, 2.0 }, + { 3.0, 4.0 } + }); + var inv = m.Inverse(); + return Controls.Markdown($""" + **Matrix** + ``` + {m} + ``` + **Inverse** + ``` + {inv} + ``` + """); + } +} diff --git a/samples/Graph/Data/MeshWeaver.json b/samples/Graph/Data/MeshWeaver.json index c9a00e780..0b923831f 100644 --- a/samples/Graph/Data/MeshWeaver.json +++ b/samples/Graph/Data/MeshWeaver.json @@ -1,12 +1,12 @@ { "id": "MeshWeaver", "name": "MeshWeaver", - "nodeType": "Organization", + "nodeType": "Space", "description": "Open source platform for building modular, reactive applications", "icon": "/static/storage/content/MeshWeaver/logo.svg", "isPersistent": true, "content": { - "$type": "Organization", + "$type": "Space", "name": "MeshWeaver", "description": "Open source platform for building modular, reactive applications with a graph-based data model", "website": "https://github.com/systemorph/MeshWeaver", diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog.json b/samples/Graph/Data/Northwind/AnalyticsCatalog.json index 4cdb0820c..c99995ee2 100644 --- a/samples/Graph/Data/Northwind/AnalyticsCatalog.json +++ b/samples/Graph/Data/Northwind/AnalyticsCatalog.json @@ -14,6 +14,6 @@ "displayName": "Analytics Catalog", "iconName": "Building", "description": "Northwind analytics catalog with year-filtered views", - "configuration": "config => config.WithContentType().AddContentCollection(sp => { return new ContentCollectionConfig { SourceType = FileSystemStreamProvider.SourceType, Name = \"Data\", BasePath = \"../../samples/Graph/attachments/Northwind/Data\", DisplayName = \"Data Files\" }; }).AddData(data => data.AddSource(source => source.WithType(t => t.WithInitialData(NorthwindDataLoader.LoadCategoriesAsync)).WithType(t => t.WithInitialData(NorthwindDataLoader.LoadRegionsAsync)).WithType(t => t.WithInitialData(NorthwindDataLoader.LoadTerritoriesAsync)).WithType(t => t.WithInitialData(NorthwindDataLoader.LoadShippersAsync)).WithType(t => t.WithInitialData(NorthwindDataLoader.LoadOrdersAsync)).WithType(t => t.WithInitialData(NorthwindDataLoader.LoadOrderDetailsAsync))).WithVirtualDataSource(\"MasterData\", vs => vs.WithVirtualType(workspace => MeshNodeDataLoader.LoadProductsFromNodes(workspace)).WithVirtualType(workspace => MeshNodeDataLoader.LoadCustomersFromNodes(workspace)).WithVirtualType(workspace => MeshNodeDataLoader.LoadEmployeesFromNodes(workspace)).WithVirtualType(workspace => MeshNodeDataLoader.LoadSuppliersFromNodes(workspace))).WithVirtualDataSource(\"NorthwindDataCube\", vs => vs.WithVirtualType(workspace => { var ordersStream = workspace.GetStream(typeof(Order)); var orderDetailsStream = workspace.GetStream(typeof(OrderDetails)); var productsStream = workspace.GetStream(typeof(Product)); var customersStream = workspace.GetStream(typeof(Customer)); var employeesStream = workspace.GetStream(typeof(Employee)); var suppliersStream = workspace.GetStream(typeof(Supplier)); var categoriesStream = workspace.GetStream(typeof(Category)); var regionsStream = workspace.GetStream(typeof(Region)); return System.Reactive.Linq.Observable.CombineLatest(ordersStream, orderDetailsStream, productsStream, customersStream, employeesStream, suppliersStream, categoriesStream, regionsStream, (orders, orderDetails, products, customers, employees, suppliers, categories, regions) => { var customerLookup = customers.Value.GetData().ToDictionary(c => c.CustomerId, c => ((MeshWeaver.Domain.INamed)c).DisplayName); var employeeLookup = employees.Value.GetData().ToDictionary(e => e.EmployeeId, e => ((MeshWeaver.Domain.INamed)e).DisplayName); var supplierLookup = suppliers.Value.GetData().ToDictionary(s => s.SupplierId, s => ((MeshWeaver.Domain.INamed)s).DisplayName); var categoryLookup = categories.Value.GetData().ToDictionary(c => c.CategoryId, c => ((MeshWeaver.Domain.INamed)c).DisplayName); return orders.Value.GetData().Join(orderDetails.Value.GetData(), o => o.OrderId, d => d.OrderId, (order, detail) => (order, detail)).Join(products.Value.GetData(), od => od.detail.ProductId, p => p.ProductId, (od, product) => (od.order, od.detail, product)).Select(data => new NorthwindDataCube(data.order, data.detail, data.product) { CustomerName = data.order.CustomerId != null && customerLookup.TryGetValue(data.order.CustomerId, out var custName) ? custName : data.order.CustomerId, EmployeeName = employeeLookup.TryGetValue(data.order.EmployeeId, out var empName) ? empName : data.order.EmployeeId.ToString(), SupplierName = supplierLookup.TryGetValue(data.product.SupplierId, out var suppName) ? suppName : data.product.SupplierId.ToString(), CategoryName = categoryLookup.TryGetValue(data.product.CategoryId, out var catName) ? catName : data.product.CategoryId.ToString(), RegionName = data.order.ShipRegion }).AsEnumerable(); }).DistinctUntilChanged(); }))).AddDefaultLayoutAreas().AddLayout(layout => layout.WithDefaultArea(\"LayoutAreas\").WithThumbnailsPath(\"/static/storage/content/Northwind/thumbnails\").AddDashboardLayoutAreas().AddSalesLayoutAreas().AddOrderLayoutAreas().AddProductLayoutAreas().AddCustomerLayoutAreas().AddEmployeeLayoutAreas().AddSupplierLayoutAreas().AddFinancialLayoutAreas().AddInventoryLayoutAreas())" + "configuration": "config => config.WithContentType().AddContentCollection(sp => { return new ContentCollectionConfig { SourceType = FileSystemStreamProvider.SourceType, Name = \"Data\", BasePath = \"../../samples/Graph/attachments/Northwind/Data\", DisplayName = \"Data Files\" }; }).AddData(data => data.AddSource(source => { var hub = source.Workspace.Hub; return source.WithType(t => t.WithInitialData(() => NorthwindDataLoader.LoadCategories(hub))).WithType(t => t.WithInitialData(() => NorthwindDataLoader.LoadRegions(hub))).WithType(t => t.WithInitialData(() => NorthwindDataLoader.LoadTerritories(hub))).WithType(t => t.WithInitialData(() => NorthwindDataLoader.LoadShippers(hub))).WithType(t => t.WithInitialData(() => NorthwindDataLoader.LoadOrders(hub))).WithType(t => t.WithInitialData(() => NorthwindDataLoader.LoadOrderDetails(hub))); }).WithVirtualDataSource(\"MasterData\", vs => vs.WithVirtualType(workspace => MeshNodeDataLoader.LoadProductsFromNodes(workspace)).WithVirtualType(workspace => MeshNodeDataLoader.LoadCustomersFromNodes(workspace)).WithVirtualType(workspace => MeshNodeDataLoader.LoadEmployeesFromNodes(workspace)).WithVirtualType(workspace => MeshNodeDataLoader.LoadSuppliersFromNodes(workspace))).WithVirtualDataSource(\"NorthwindDataCube\", vs => vs.WithVirtualType(workspace => { var ordersStream = workspace.GetStream(typeof(Order)); var orderDetailsStream = workspace.GetStream(typeof(OrderDetails)); var productsStream = workspace.GetStream(typeof(Product)); var customersStream = workspace.GetStream(typeof(Customer)); var employeesStream = workspace.GetStream(typeof(Employee)); var suppliersStream = workspace.GetStream(typeof(Supplier)); var categoriesStream = workspace.GetStream(typeof(Category)); var regionsStream = workspace.GetStream(typeof(Region)); return System.Reactive.Linq.Observable.CombineLatest(ordersStream, orderDetailsStream, productsStream, customersStream, employeesStream, suppliersStream, categoriesStream, regionsStream, (orders, orderDetails, products, customers, employees, suppliers, categories, regions) => { var customerLookup = customers.Value.GetData().ToDictionary(c => c.CustomerId, c => ((MeshWeaver.Domain.INamed)c).DisplayName); var employeeLookup = employees.Value.GetData().ToDictionary(e => e.EmployeeId, e => ((MeshWeaver.Domain.INamed)e).DisplayName); var supplierLookup = suppliers.Value.GetData().ToDictionary(s => s.SupplierId, s => ((MeshWeaver.Domain.INamed)s).DisplayName); var categoryLookup = categories.Value.GetData().ToDictionary(c => c.CategoryId, c => ((MeshWeaver.Domain.INamed)c).DisplayName); return orders.Value.GetData().Join(orderDetails.Value.GetData(), o => o.OrderId, d => d.OrderId, (order, detail) => (order, detail)).Join(products.Value.GetData(), od => od.detail.ProductId, p => p.ProductId, (od, product) => (od.order, od.detail, product)).Select(data => new NorthwindDataCube(data.order, data.detail, data.product) { CustomerName = data.order.CustomerId != null && customerLookup.TryGetValue(data.order.CustomerId, out var custName) ? custName : data.order.CustomerId, EmployeeName = employeeLookup.TryGetValue(data.order.EmployeeId, out var empName) ? empName : data.order.EmployeeId.ToString(), SupplierName = supplierLookup.TryGetValue(data.product.SupplierId, out var suppName) ? suppName : data.product.SupplierId.ToString(), CategoryName = categoryLookup.TryGetValue(data.product.CategoryId, out var catName) ? catName : data.product.CategoryId.ToString(), RegionName = data.order.ShipRegion }).AsEnumerable(); }).DistinctUntilChanged(); }))).AddDefaultLayoutAreas().AddLayout(layout => layout.WithDefaultArea(\"LayoutAreas\").WithThumbnailsPath(\"/static/storage/content/Northwind/thumbnails\").AddDashboardLayoutAreas().AddSalesLayoutAreas().AddOrderLayoutAreas().AddProductLayoutAreas().AddCustomerLayoutAreas().AddEmployeeLayoutAreas().AddSupplierLayoutAreas().AddFinancialLayoutAreas().AddInventoryLayoutAreas())" } } diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/CatalogContent.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/CatalogContent.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/CatalogContent.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/CatalogContent.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Category.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Category.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Category.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Category.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Customer.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Customer.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Customer.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Customer.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/CustomerLayoutAreas.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/CustomerLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/CustomerLayoutAreas.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/CustomerLayoutAreas.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/DashboardLayoutAreas.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/DashboardLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/DashboardLayoutAreas.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/DashboardLayoutAreas.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Employee.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Employee.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Employee.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Employee.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/EmployeeLayoutAreas.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/EmployeeLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/EmployeeLayoutAreas.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/EmployeeLayoutAreas.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/FinancialLayoutAreas.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/FinancialLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/FinancialLayoutAreas.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/FinancialLayoutAreas.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/InventoryLayoutAreas.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/InventoryLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/InventoryLayoutAreas.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/InventoryLayoutAreas.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/MeshNodeDataLoader.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/MeshNodeDataLoader.cs similarity index 93% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/MeshNodeDataLoader.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/MeshNodeDataLoader.cs index 0c1b6888b..a86f8207d 100644 --- a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/MeshNodeDataLoader.cs +++ b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/MeshNodeDataLoader.cs @@ -20,7 +20,7 @@ public static IObservable> LoadProductsFromNodes(IWorkspace var namespacePath = GetNamespacePath(workspace); return meshQuery - .ObserveQuery(MeshQueryRequest.FromQuery($"nodeType:Northwind/Product namespace:{namespacePath}/Product state:Active")) + .Query(MeshQueryRequest.FromQuery($"nodeType:Northwind/Product namespace:{namespacePath}/Product state:Active")) .Select(change => change.Items.Select(node => ConvertToProduct(node)).Where(p => p != null).Cast()); } @@ -30,7 +30,7 @@ public static IObservable> LoadCustomersFromNodes(IWorkspa var namespacePath = GetNamespacePath(workspace); return meshQuery - .ObserveQuery(MeshQueryRequest.FromQuery($"nodeType:Northwind/Customer namespace:{namespacePath}/Customer state:Active")) + .Query(MeshQueryRequest.FromQuery($"nodeType:Northwind/Customer namespace:{namespacePath}/Customer state:Active")) .Select(change => change.Items.Select(node => ConvertToCustomer(node)).Where(c => c != null).Cast()); } @@ -40,7 +40,7 @@ public static IObservable> LoadEmployeesFromNodes(IWorkspa var namespacePath = GetNamespacePath(workspace); return meshQuery - .ObserveQuery(MeshQueryRequest.FromQuery($"nodeType:Northwind/Employee namespace:{namespacePath}/Employee state:Active")) + .Query(MeshQueryRequest.FromQuery($"nodeType:Northwind/Employee namespace:{namespacePath}/Employee state:Active")) .Select(change => change.Items.Select(node => ConvertToEmployee(node)).Where(e => e != null).Cast()); } @@ -50,7 +50,7 @@ public static IObservable> LoadSuppliersFromNodes(IWorkspa var namespacePath = GetNamespacePath(workspace); return meshQuery - .ObserveQuery(MeshQueryRequest.FromQuery($"nodeType:Northwind/Supplier namespace:{namespacePath}/Supplier state:Active")) + .Query(MeshQueryRequest.FromQuery($"nodeType:Northwind/Supplier namespace:{namespacePath}/Supplier state:Active")) .Select(change => change.Items.Select(node => ConvertToSupplier(node)).Where(s => s != null).Cast()); } diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/NorthwindDataCube.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/NorthwindDataCube.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/NorthwindDataCube.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/NorthwindDataCube.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/NorthwindDataCubeExtensions.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/NorthwindDataCubeExtensions.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/NorthwindDataCubeExtensions.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/NorthwindDataCubeExtensions.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/NorthwindDataLoader.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/NorthwindDataLoader.cs similarity index 59% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/NorthwindDataLoader.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/NorthwindDataLoader.cs index a7c57f15e..33a054e41 100644 --- a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/NorthwindDataLoader.cs +++ b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/NorthwindDataLoader.cs @@ -5,22 +5,45 @@ using System.Globalization; using System.IO; -using System.Threading; -using System.Threading.Tasks; +using MeshWeaver.Messaging; +using MeshWeaver.Mesh.Threading; +using Microsoft.Extensions.DependencyInjection; /// /// CSV data loader for Northwind transactional entities. /// Master data (Product, Customer, Employee, Supplier) is now loaded from MeshNodes /// via MeshNodeDataLoader. +/// +/// Reactive end-to-end: every loader returns IObservable<IEnumerable<T>> +/// (the shape WithInitialData takes) and runs the blocking CSV read + parse on the +/// bounded FileSystem I/O pool via InvokeBlocking — never async/await, never +/// Task.FromResult, never on the configuring hub's thread. /// public static class NorthwindDataLoader { private static readonly string BasePath = Path.Combine("../../samples/Graph/attachments/Northwind/Data"); - public static Task> LoadOrdersAsync(CancellationToken ct) - { - var lines = File.ReadAllLines(Path.Combine(BasePath, "orders.csv")); - return Task.FromResult(ParseCsv(lines, parts => new Order + /// + /// Resolves the bounded FileSystem I/O pool from the hub (falling back to the + /// unbounded pool when no registry is present, e.g. lightweight test hubs). + /// All CSV reads below run on this pool. + /// + private static IIoPool FileSystemPool(IMessageHub hub) => + hub.ServiceProvider.GetService()?.Get(IoPoolNames.FileSystem) + ?? IoPool.Unbounded; + + /// + /// Reads + parses one CSV file entirely on the I/O pool. The .ToList() inside + /// the pool slot matters: ParseCsv is lazy, and without materialisation the + /// parse would run later on whatever thread enumerates the result. + /// + private static IObservable> LoadCsv( + IMessageHub hub, string fileName, Func factory) => + FileSystemPool(hub).InvokeBlocking(_ => + (IEnumerable)ParseCsv(File.ReadAllLines(Path.Combine(BasePath, fileName)), factory).ToList()); + + public static IObservable> LoadOrders(IMessageHub hub) + => LoadCsv(hub, "orders.csv", parts => new Order { OrderId = ParseInt(parts[0]), CustomerId = parts[1], @@ -35,13 +58,10 @@ public static Task> LoadOrdersAsync(CancellationToken ct) ShipRegion = Get(parts, 10), ShipPostalCode = Get(parts, 11), ShipCountry = Get(parts, 12), - })); - } + }); - public static Task> LoadOrderDetailsAsync(CancellationToken ct) - { - var lines = File.ReadAllLines(Path.Combine(BasePath, "orders_details.csv")); - return Task.FromResult(ParseCsv(lines, parts => new OrderDetails + public static IObservable> LoadOrderDetails(IMessageHub hub) + => LoadCsv(hub, "orders_details.csv", parts => new OrderDetails { Id = ParseInt(parts[0]), OrderId = ParseInt(parts[1]), @@ -49,51 +69,38 @@ public static Task> LoadOrderDetailsAsync(Cancellation UnitPrice = double.Parse(parts[3], CultureInfo.InvariantCulture), Quantity = ParseInt(parts[4]), Discount = double.Parse(parts[5], CultureInfo.InvariantCulture), - })); - } + }); - public static Task> LoadCategoriesAsync(CancellationToken ct) - { - var lines = File.ReadAllLines(Path.Combine(BasePath, "categories.csv")); - return Task.FromResult(ParseCsv(lines, parts => new Category + public static IObservable> LoadCategories(IMessageHub hub) + => LoadCsv(hub, "categories.csv", parts => new Category { CategoryId = ParseInt(parts[0]), CategoryName = parts[1], Description = Get(parts, 2), - })); - } + }); - public static Task> LoadRegionsAsync(CancellationToken ct) - { - var lines = File.ReadAllLines(Path.Combine(BasePath, "regions.csv")); - return Task.FromResult(ParseCsv(lines, parts => new Region + public static IObservable> LoadRegions(IMessageHub hub) + => LoadCsv(hub, "regions.csv", parts => new Region { RegionId = ParseInt(parts[0]), RegionDescription = parts[1], - })); - } + }); - public static Task> LoadTerritoriesAsync(CancellationToken ct) - { - var lines = File.ReadAllLines(Path.Combine(BasePath, "territories.csv")); - return Task.FromResult(ParseCsv(lines, parts => new Territory + public static IObservable> LoadTerritories(IMessageHub hub) + => LoadCsv(hub, "territories.csv", parts => new Territory { TerritoryId = ParseInt(parts[0]), TerritoryDescription = parts[1], RegionId = ParseInt(parts[2]), - })); - } + }); - public static Task> LoadShippersAsync(CancellationToken ct) - { - var lines = File.ReadAllLines(Path.Combine(BasePath, "shippers.csv")); - return Task.FromResult(ParseCsv(lines, parts => new Shipper + public static IObservable> LoadShippers(IMessageHub hub) + => LoadCsv(hub, "shippers.csv", parts => new Shipper { ShipperId = ParseInt(parts[0]), CompanyName = parts[1], Phone = Get(parts, 2), - })); - } + }); private static IEnumerable ParseCsv(string[] lines, Func factory) { diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/NorthwindHelpers.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/NorthwindHelpers.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/NorthwindHelpers.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/NorthwindHelpers.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/NorthwindYearToolbar.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/NorthwindYearToolbar.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/NorthwindYearToolbar.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/NorthwindYearToolbar.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Order.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Order.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Order.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Order.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/OrderDetails.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/OrderDetails.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/OrderDetails.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/OrderDetails.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/OrderLayoutAreas.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/OrderLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/OrderLayoutAreas.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/OrderLayoutAreas.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Product.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Product.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Product.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Product.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/ProductLayoutAreas.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/ProductLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/ProductLayoutAreas.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/ProductLayoutAreas.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Region.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Region.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Region.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Region.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/SalesLayoutAreas.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/SalesLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/SalesLayoutAreas.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/SalesLayoutAreas.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Shipper.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Shipper.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Shipper.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Shipper.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Supplier.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Supplier.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Supplier.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Supplier.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/SupplierLayoutAreas.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/SupplierLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/SupplierLayoutAreas.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/SupplierLayoutAreas.cs diff --git a/samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Territory.cs b/samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Territory.cs similarity index 100% rename from samples/Graph/Data/Northwind/AnalyticsCatalog/_Source/Territory.cs rename to samples/Graph/Data/Northwind/AnalyticsCatalog/Source/Territory.cs diff --git a/samples/Graph/Data/Northwind/Article/_Source/Article.cs b/samples/Graph/Data/Northwind/Article/Source/Article.cs similarity index 100% rename from samples/Graph/Data/Northwind/Article/_Source/Article.cs rename to samples/Graph/Data/Northwind/Article/Source/Article.cs diff --git a/samples/Graph/Data/Northwind/Article/_Source/ArticleLayoutAreas.cs b/samples/Graph/Data/Northwind/Article/Source/ArticleLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Northwind/Article/_Source/ArticleLayoutAreas.cs rename to samples/Graph/Data/Northwind/Article/Source/ArticleLayoutAreas.cs diff --git a/samples/Graph/Data/Northwind/Customer/_Source/CustomerContent.cs b/samples/Graph/Data/Northwind/Customer/Source/CustomerContent.cs similarity index 100% rename from samples/Graph/Data/Northwind/Customer/_Source/CustomerContent.cs rename to samples/Graph/Data/Northwind/Customer/Source/CustomerContent.cs diff --git a/samples/Graph/Data/Northwind/Customer/_Source/CustomerNodeLayoutAreas.cs b/samples/Graph/Data/Northwind/Customer/Source/CustomerNodeLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Northwind/Customer/_Source/CustomerNodeLayoutAreas.cs rename to samples/Graph/Data/Northwind/Customer/Source/CustomerNodeLayoutAreas.cs diff --git a/samples/Graph/Data/Northwind/Documentation/Architecture.md b/samples/Graph/Data/Northwind/Documentation/Architecture.md index ef1136536..8e6d771b1 100644 --- a/samples/Graph/Data/Northwind/Documentation/Architecture.md +++ b/samples/Graph/Data/Northwind/Documentation/Architecture.md @@ -34,7 +34,7 @@ Northwind/ # Root namespace │ ├── regions.csv # Geographic regions │ ├── territories.csv # Sales territories │ └── shippers.csv # Shipping companies -└── AnalyticsCatalog/_Source/ # View implementations +└── AnalyticsCatalog/Source/ # View implementations ├── Order.cs # Order entity ├── OrderDetails.cs # OrderDetails entity ├── Product.cs # Product entity @@ -69,40 +69,52 @@ The AnalyticsCatalog NodeType configures multiple data sources: ```csharp config.AddData(data => data - .AddSource(source => source - // All entity data loaded from CSV files via NorthwindDataLoader - .WithType(t => t.WithInitialData(NorthwindDataLoader.LoadOrdersAsync)) - .WithType(t => t.WithInitialData(NorthwindDataLoader.LoadOrderDetailsAsync)) - .WithType(t => t.WithInitialData(NorthwindDataLoader.LoadProductsAsync)) - .WithType(t => t.WithInitialData(NorthwindDataLoader.LoadCustomersAsync)) - .WithType(t => t.WithInitialData(NorthwindDataLoader.LoadEmployeesAsync)) - .WithType(t => t.WithInitialData(NorthwindDataLoader.LoadSuppliersAsync)) - .WithType(t => t.WithInitialData(NorthwindDataLoader.LoadCategoriesAsync)) - .WithType(t => t.WithInitialData(NorthwindDataLoader.LoadRegionsAsync)) - .WithType(t => t.WithInitialData(NorthwindDataLoader.LoadTerritoriesAsync)) - .WithType(t => t.WithInitialData(NorthwindDataLoader.LoadShippersAsync))) + .AddSource(source => + { + var hub = source.Workspace.Hub; + return source + // All entity data loaded from CSV files via NorthwindDataLoader — + // reactive loaders (IObservable), file I/O on the FileSystem I/O pool + .WithType(t => t.WithInitialData(() => NorthwindDataLoader.LoadOrders(hub))) + .WithType(t => t.WithInitialData(() => NorthwindDataLoader.LoadOrderDetails(hub))) + .WithType(t => t.WithInitialData(() => NorthwindDataLoader.LoadCategories(hub))) + .WithType(t => t.WithInitialData(() => NorthwindDataLoader.LoadRegions(hub))) + .WithType(t => t.WithInitialData(() => NorthwindDataLoader.LoadTerritories(hub))) + .WithType(t => t.WithInitialData(() => NorthwindDataLoader.LoadShippers(hub))); + }) + // Master data (Product, Customer, Employee, Supplier) from MeshNodes + .WithVirtualDataSource("MasterData", ...) // Virtual data cube .WithVirtualDataSource("NorthwindDataCube", ...)) ``` ## CSV Loading (`NorthwindDataLoader.cs`) +Loaders are reactive — they return `IObservable>` (the shape +`WithInitialData` takes) and run the blocking CSV read + parse on the bounded +FileSystem I/O pool via `InvokeBlocking`. No `async`/`await`, no +`Task.FromResult`, and the file read never runs on the configuring hub's thread: + ```csharp -public static IEnumerable LoadOrders(string csvPath) -{ - return File.ReadLines(csvPath) - .Skip(1) // Header - .Select(line => line.Split(',')) - .Select(parts => new Order - { - OrderId = int.Parse(parts[0]), - CustomerId = parts[1], - EmployeeId = int.Parse(parts[2]), - OrderDate = DateTime.Parse(parts[3]), - Freight = decimal.Parse(parts[7]), - ShipCountry = parts[13] - }); -} +private static IIoPool FileSystemPool(IMessageHub hub) => + hub.ServiceProvider.GetService()?.Get(IoPoolNames.FileSystem) + ?? IoPool.Unbounded; + +private static IObservable> LoadCsv( + IMessageHub hub, string fileName, Func factory) => + FileSystemPool(hub).InvokeBlocking(_ => + (IEnumerable)ParseCsv(File.ReadAllLines(Path.Combine(BasePath, fileName)), factory).ToList()); + +public static IObservable> LoadOrders(IMessageHub hub) + => LoadCsv(hub, "orders.csv", parts => new Order + { + OrderId = ParseInt(parts[0]), + CustomerId = parts[1], + EmployeeId = ParseInt(parts[2]), + OrderDate = DateTime.Parse(parts[3], CultureInfo.InvariantCulture), + Freight = decimal.Parse(parts[7], CultureInfo.InvariantCulture), + ShipCountry = Get(parts, 12), + }); ``` ## Reference Data (CSV-based) @@ -110,17 +122,14 @@ public static IEnumerable LoadOrders(string csvPath) All reference data is loaded from CSV files via `NorthwindDataLoader.cs`, following the same pattern as transactional data: ```csharp -public static Task> LoadCategoriesAsync(CancellationToken ct) -{ +public static IObservable> LoadCategories(IMessageHub hub) // CSV: categoryid,categoryname,description,picture - var lines = File.ReadAllLines(Path.Combine(BasePath, "categories.csv")); - return Task.FromResult(ParseCsv(lines, parts => new Category + => LoadCsv(hub, "categories.csv", parts => new Category { CategoryId = ParseInt(parts[0]), CategoryName = parts[1], Description = Get(parts, 2), - })); -} + }); ``` # Virtual Data Cube diff --git a/samples/Graph/Data/Northwind/Documentation/Overview.md b/samples/Graph/Data/Northwind/Documentation/Overview.md index 0452872d0..c95adfa74 100644 --- a/samples/Graph/Data/Northwind/Documentation/Overview.md +++ b/samples/Graph/Data/Northwind/Documentation/Overview.md @@ -37,7 +37,7 @@ Northwind/ # Analytics platform ├── Data/ # CSV data sources │ ├── orders.csv # Order transactions │ └── orders_details.csv # Order line items -└── AnalyticsCatalog/_Source/ # View implementations +└── AnalyticsCatalog/Source/ # View implementations ├── DashboardViews.cs # Main dashboard ├── SalesViews.cs # Sales analytics ├── OrderViews.cs # Order analysis diff --git a/samples/Graph/Data/Northwind/Documentation/UnifiedReferences.md b/samples/Graph/Data/Northwind/Documentation/UnifiedReferences.md index 21e2b95fa..947eccde9 100644 --- a/samples/Graph/Data/Northwind/Documentation/UnifiedReferences.md +++ b/samples/Graph/Data/Northwind/Documentation/UnifiedReferences.md @@ -28,7 +28,7 @@ Northwind/ # Root namespace ├── Data/ │ ├── orders.csv # Order data │ └── orders_details.csv # Order details -└── AnalyticsCatalog/_Source/ # View implementations +└── AnalyticsCatalog/Source/ # View implementations ├── Order.cs ├── OrderDetails.cs ├── Product.cs @@ -61,14 +61,14 @@ Northwind exposes these data types through the AnalyticsCatalog: | Reference | Description | |-----------|-------------| -| `@Northwind/data:Order` | All orders | -| `@Northwind/data:OrderDetails` | All order line items | -| `@Northwind/data:Product` | All products | -| `@Northwind/data:Customer` | All customers | -| `@Northwind/data:Employee` | All employees | -| `@Northwind/data:Supplier` | All suppliers | -| `@Northwind/data:Category` | Product categories | -| `@Northwind/data:NorthwindDataCube` | Virtual analytics cube | +| `@Northwind/data/Order` | All orders | +| `@Northwind/data/OrderDetails` | All order line items | +| `@Northwind/data/Product` | All products | +| `@Northwind/data/Customer` | All customers | +| `@Northwind/data/Employee` | All employees | +| `@Northwind/data/Supplier` | All suppliers | +| `@Northwind/data/Category` | Product categories | +| `@Northwind/data/NorthwindDataCube` | Virtual analytics cube | ## Specific Entity References @@ -76,22 +76,22 @@ Northwind exposes these data types through the AnalyticsCatalog: | Reference | Description | |-----------|-------------| -| `@Northwind/data:Order/10248` | Order #10248 | -| `@Northwind/data:Order/10249` | Order #10249 | +| `@Northwind/data/Order/10248` | Order #10248 | +| `@Northwind/data/Order/10249` | Order #10249 | **Products**: | Reference | Description | |-----------|-------------| -| `@Northwind/data:Product/1` | Product ID 1 (Chai) | -| `@Northwind/data:Product/2` | Product ID 2 (Chang) | +| `@Northwind/data/Product/1` | Product ID 1 (Chai) | +| `@Northwind/data/Product/2` | Product ID 2 (Chang) | **Customers**: | Reference | Description | |-----------|-------------| -| `@Northwind/data:Customer/ALFKI` | Alfreds Futterkiste | -| `@Northwind/data:Customer/QUICK` | QUICK-Stop | +| `@Northwind/data/Customer/ALFKI` | Alfreds Futterkiste | +| `@Northwind/data/Customer/QUICK` | QUICK-Stop | # Layout Area References @@ -534,7 +534,7 @@ Inventory and trend analysis: The Northwind sample demonstrates MeshWeaver's unified path system for analytics: - **Namespace paths** organize the analytics hierarchy -- **Data references** access orders, products, customers via `@path/data:Type` +- **Data references** access orders, products, customers via `@path/data/Type` - **Layout areas** display views via `@@path/ViewName` - **53 views** across 8 categories for comprehensive analytics diff --git a/samples/Graph/Data/Northwind/Employee/_Source/EmployeeContent.cs b/samples/Graph/Data/Northwind/Employee/Source/EmployeeContent.cs similarity index 100% rename from samples/Graph/Data/Northwind/Employee/_Source/EmployeeContent.cs rename to samples/Graph/Data/Northwind/Employee/Source/EmployeeContent.cs diff --git a/samples/Graph/Data/Northwind/Employee/_Source/EmployeeNodeLayoutAreas.cs b/samples/Graph/Data/Northwind/Employee/Source/EmployeeNodeLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Northwind/Employee/_Source/EmployeeNodeLayoutAreas.cs rename to samples/Graph/Data/Northwind/Employee/Source/EmployeeNodeLayoutAreas.cs diff --git a/samples/Graph/Data/Northwind/Product/_Source/ProductContent.cs b/samples/Graph/Data/Northwind/Product/Source/ProductContent.cs similarity index 100% rename from samples/Graph/Data/Northwind/Product/_Source/ProductContent.cs rename to samples/Graph/Data/Northwind/Product/Source/ProductContent.cs diff --git a/samples/Graph/Data/Northwind/Product/_Source/ProductNodeLayoutAreas.cs b/samples/Graph/Data/Northwind/Product/Source/ProductNodeLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Northwind/Product/_Source/ProductNodeLayoutAreas.cs rename to samples/Graph/Data/Northwind/Product/Source/ProductNodeLayoutAreas.cs diff --git a/samples/Graph/Data/Northwind/ReportsCatalog/_Source/ReportsCatalogLayoutAreas.cs b/samples/Graph/Data/Northwind/ReportsCatalog/Source/ReportsCatalogLayoutAreas.cs similarity index 97% rename from samples/Graph/Data/Northwind/ReportsCatalog/_Source/ReportsCatalogLayoutAreas.cs rename to samples/Graph/Data/Northwind/ReportsCatalog/Source/ReportsCatalogLayoutAreas.cs index 63a213560..995867b71 100644 --- a/samples/Graph/Data/Northwind/ReportsCatalog/_Source/ReportsCatalogLayoutAreas.cs +++ b/samples/Graph/Data/Northwind/ReportsCatalog/Source/ReportsCatalogLayoutAreas.cs @@ -33,13 +33,11 @@ public static LayoutDefinition AddReportsCatalogLayoutAreas(this LayoutDefinitio var nodeStream = host.Workspace.GetStream()?.Select(nodes => nodes ?? Array.Empty()) ?? Observable.Return(Array.Empty()); - return nodeStream.SelectMany(async nodes => + var childrenStream = host.ObserveChildren(""); + + return nodeStream.CombineLatest(childrenStream, (nodes, children) => { var node = nodes.FirstOrDefault(n => n.Path == hubPath); - - // Query child report nodes - var children = await host.QueryChildrenAsync("").ToListAsync(); - return (UiControl?)BuildCatalogOverview(host, node, children); }); } diff --git a/samples/Graph/Data/Northwind/Supplier/_Source/SupplierContent.cs b/samples/Graph/Data/Northwind/Supplier/Source/SupplierContent.cs similarity index 100% rename from samples/Graph/Data/Northwind/Supplier/_Source/SupplierContent.cs rename to samples/Graph/Data/Northwind/Supplier/Source/SupplierContent.cs diff --git a/samples/Graph/Data/Northwind/Supplier/_Source/SupplierNodeLayoutAreas.cs b/samples/Graph/Data/Northwind/Supplier/Source/SupplierNodeLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/Northwind/Supplier/_Source/SupplierNodeLayoutAreas.cs rename to samples/Graph/Data/Northwind/Supplier/Source/SupplierNodeLayoutAreas.cs diff --git a/samples/Graph/Data/Northwind/index.md b/samples/Graph/Data/Northwind/index.md index ac71fb0ff..9925d3b70 100644 --- a/samples/Graph/Data/Northwind/index.md +++ b/samples/Graph/Data/Northwind/index.md @@ -1,5 +1,5 @@ --- -NodeType: Organization +NodeType: Space Name: Northwind Category: Analytics Description: Gourmet food distribution analytics demonstrating MeshWeaver's data visualization and AI-assisted exploration capabilities diff --git a/samples/Graph/Data/PensionFund/BalanceSheet.json b/samples/Graph/Data/PensionFund/BalanceSheet.json new file mode 100644 index 000000000..f473d68ac --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheet.json @@ -0,0 +1,18 @@ +{ + "id": "BalanceSheet", + "namespace": "PensionFund", + "name": "Balance Sheet", + "nodeType": "NodeType", + "category": "Types", + "description": "The pension fund balance sheet: loads Position/Year/Entry nodes and computes every position through the PositionValue business-rules scopes. Documented at Doc/DataMesh/DataCubes.", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M12 3v18M5 7l7-4 7 4M5 7v4a7 7 0 0 0 14 0V7\u0027/\u003e\u003cpath d=\u0027M3 21h18\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "NodeTypeDefinition", + "id": "BalanceSheet", + "namespace": "PensionFund", + "displayName": "Balance Sheet", + "iconName": "Scales", + "configuration": "config =\u003e config.ConfigureBalanceSheet()" + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheet/Source/BalanceSheetConfig.cs b/samples/Graph/Data/PensionFund/BalanceSheet/Source/BalanceSheetConfig.cs new file mode 100644 index 000000000..2d889eacf --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheet/Source/BalanceSheetConfig.cs @@ -0,0 +1,51 @@ +// +// Id: BalanceSheetConfig +// DisplayName: Balance Sheet Configuration +// + +using MeshWeaver.BusinessRules; +using MeshWeaver.Layout; +using MeshWeaver.Layout.Composition; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; + +/// +/// Content of a balance-sheet report INSTANCE node (e.g. +/// PensionFund/Statement). The views compute everything from the dimension +/// and fact nodes; the instance carries only free commentary. +/// +public record BalanceSheetReport +{ + /// Management commentary shown alongside the statement. + public string? Commentary { get; init; } +} + +/// +/// Hub configuration applied to every node of type PensionFund/BalanceSheet — +/// the report INSTANCES (like FutuRe/Analysis for FutuRe/GroupAnalysis), not +/// the NodeType definition node. Registers the business-rules scopes of THIS +/// compiled node assembly (the NodeType compiler ran the scope generator over +/// the Source — AddBusinessRules discovers the generated implementations) and +/// wires the balance-sheet views. Referenced from BalanceSheet.json. +/// +public static class BalanceSheetConfig +{ + public static MessageHubConfiguration ConfigureBalanceSheet(this MessageHubConfiguration config) + => config + .WithServices(services => services.AddBusinessRules(typeof(PositionValue).Assembly)) + .WithContentType() + // The mesh queries live in VIRTUAL DATA SOURCES — subscribed once by the + // framework at hub init, never from inside a view render (which would run + // them on the grain's activation thread and deadlock the hub). Views read + // the resulting LOCAL workspace streams via BalanceSheetData.LoadStorage. + .AddData(data => data + .WithVirtualDataSource("BalanceSheetModel", vs => vs + .WithVirtualType(workspace => BalanceSheetData.LoadPositions(workspace)) + .WithVirtualType(workspace => BalanceSheetData.LoadYears(workspace)) + .WithVirtualType(workspace => BalanceSheetData.LoadEntries(workspace)))) + .AddDefaultLayoutAreas() + // Land on the statement itself — without a default area the page falls + // back to the generic content view, which shows only the commentary. + .AddLayout(layout => layout.AddBalanceSheetLayoutAreas() + .WithDefaultArea("BalanceSheetStatement")); +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheet/Source/BalanceSheetData.cs b/samples/Graph/Data/PensionFund/BalanceSheet/Source/BalanceSheetData.cs new file mode 100644 index 000000000..88ddd4168 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheet/Source/BalanceSheetData.cs @@ -0,0 +1,144 @@ +// +// Id: BalanceSheetData +// DisplayName: Balance Sheet Data Loader +// + +using System.Collections.Immutable; +using System.Reactive.Linq; +using MeshWeaver.Data; +using MeshWeaver.Domain; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using Microsoft.Extensions.DependencyInjection; + +/// +/// Workspace projection of a Position dimension node: the content plus what +/// lives on the MESH NODE itself — the path (identity) and the display order +/// (MeshNode.Order). +/// +public record PositionNode +{ + [Key] + public string Path { get; init; } = string.Empty; + public int Order { get; init; } + public Position Content { get; init; } = null!; +} + +/// Workspace projection of a Year dimension node. +public record YearNode +{ + [Key] + public string Path { get; init; } = string.Empty; + public Year Content { get; init; } = null!; +} + +/// Workspace projection of a BalanceSheetEntry fact node. +public record EntryNode +{ + [Key] + public string Path { get; init; } = string.Empty; + public BalanceSheetEntry Content { get; init; } = null!; +} + +/// +/// Loads the balance-sheet model from the mesh: Position dimension nodes, +/// Year dimension nodes, and BalanceSheetEntry fact nodes — everything is a +/// mesh node, so loading is querying by the dimension/fact NodeType paths. +/// +/// The mesh queries live HERE, as virtual data-source loaders registered in +/// ConfigureBalanceSheet via WithVirtualDataSource — the +/// framework's data plumbing subscribes them ONCE at hub initialization on a +/// managed scheduler. Views never subscribe IMeshService.Query +/// themselves: backend rendering composes only the hub's OWN workspace +/// streams (see AsynchronousCalls — a query subscription inside a layout-area +/// render runs on the grain's activation thread and deadlocks the hub). +/// +public static class BalanceSheetData +{ + /// Virtual-source loader: live Position nodes projected to . + public static IObservable> LoadPositions(IWorkspace workspace) + => workspace.Hub.ServiceProvider.GetRequiredService() + .Query(MeshQueryRequest.FromQuery( + "nodeType:PensionFund/Position namespace:PensionFund/Position state:Active")) + .Select(change => change.Items + .Select(n => new PositionNode + { + Path = n.Path, + Order = n.Order ?? 0, + Content = n.ContentAs(workspace.Hub.JsonSerializerOptions)!, + }) + .Where(x => x.Content is not null) + .ToImmutableList() + .AsEnumerable()); + + /// Virtual-source loader: live Year nodes projected to . + public static IObservable> LoadYears(IWorkspace workspace) + => workspace.Hub.ServiceProvider.GetRequiredService() + .Query(MeshQueryRequest.FromQuery( + "nodeType:PensionFund/Year namespace:PensionFund/Year state:Active")) + .Select(change => change.Items + .Select(n => new YearNode + { + Path = n.Path, + Content = n.ContentAs(workspace.Hub.JsonSerializerOptions)!, + }) + .Where(x => x.Content is not null) + .ToImmutableList() + .AsEnumerable()); + + /// Virtual-source loader: live fact nodes projected to . + public static IObservable> LoadEntries(IWorkspace workspace) + => workspace.Hub.ServiceProvider.GetRequiredService() + .Query(MeshQueryRequest.FromQuery( + "nodeType:PensionFund/BalanceSheetEntry namespace:PensionFund/BalanceSheetEntry state:Active")) + .Select(change => change.Items + .Select(n => new EntryNode + { + Path = n.Path, + Content = n.ContentAs(workspace.Hub.JsonSerializerOptions)!, + }) + .Where(x => x.Content is not null + && x.Content.Position.Length > 0 + && x.Content.Year.Length > 0) + .ToImmutableList() + .AsEnumerable()); + + /// + /// The combined storage stream every scope evaluation runs against. + /// Composed ONLY from the hub's own workspace streams (fed by the virtual + /// data sources above) — local, render-safe, no cross-hub subscription. + /// Re-emits whenever a dimension or fact node changes — edit an entry in + /// the GUI and every computed position re-evaluates. + /// + public static IObservable LoadStorage(IWorkspace workspace) + { + var positions = (workspace.GetStream() + ?? Observable.Return?>(null)) + .Select(d => (d?.AsEnumerable() ?? Enumerable.Empty()) + .OrderBy(x => x.Order).ThenBy(x => x.Path) + .ToImmutableList()); + + var years = (workspace.GetStream() + ?? Observable.Return?>(null)) + .Select(d => (d?.AsEnumerable() ?? Enumerable.Empty()) + .OrderBy(x => x.Content.Value) + .Select(x => x.Path) + .ToImmutableArray()); + + var entries = (workspace.GetStream() + ?? Observable.Return?>(null)) + .Select(d => (d?.AsEnumerable() ?? Enumerable.Empty()) + .ToImmutableDictionary( + x => (x.Content.Position, x.Content.Year), + x => x.Content.Amount)); + + return positions.CombineLatest(years, entries, + (ps, ys, amounts) => new BalanceSheetStorage + { + Positions = ps.ToImmutableDictionary(x => x.Path, x => x.Content), + OrderedPositions = ps.Select(x => x.Path).ToImmutableList(), + Years = ys, + Amounts = amounts, + }); + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheet/Source/BalanceSheetLayoutAreas.cs b/samples/Graph/Data/PensionFund/BalanceSheet/Source/BalanceSheetLayoutAreas.cs new file mode 100644 index 000000000..1eae68c8e --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheet/Source/BalanceSheetLayoutAreas.cs @@ -0,0 +1,178 @@ +// +// Id: BalanceSheetLayoutAreas +// DisplayName: Balance Sheet Views +// + +using System.ComponentModel.DataAnnotations; +using System.Reactive.Linq; +using System.Text; +using System.Threading.Tasks; +using MeshWeaver.BusinessRules; +using MeshWeaver.Domain; +using MeshWeaver.Layout; +using MeshWeaver.Layout.Chart; +using MeshWeaver.Layout.Composition; +using Microsoft.Extensions.DependencyInjection; + +/// +/// Draft record for the "New Entry" dialog: every dimension column is a +/// MeshNodePicker over the dimension NODES (queried by the dimension TYPE's +/// path), so capturing a fact means picking nodes. +/// +public record BalanceSheetEntryDraft +{ + [MeshNode("nodeType:PensionFund/Position")] + [Display(Name = "Position")] + public string? Position { get; init; } + + [MeshNode("nodeType:PensionFund/Year")] + [Display(Name = "Year")] + public string? Year { get; init; } + + [MeshNode("nodeType:PensionFund/Currency")] + [Display(Name = "Currency")] + public string? Currency { get; init; } + + [DisplayFormat(DataFormatString = "{0:N1}")] + [Display(Name = "Amount (m)")] + public double Amount { get; init; } +} + +/// +/// Balance-sheet views: the statement itself (atomic + computed positions via +/// the PositionValue scopes), the funding-ratio KPIs, the asset allocation +/// chart, and the new-entry dialog with mesh-node pickers. +/// +[Display(GroupName = "Balance Sheet", Order = 100)] +public static class BalanceSheetLayoutAreas +{ + public static LayoutDefinition AddBalanceSheetLayoutAreas(this LayoutDefinition layout) => + layout + .WithView(nameof(BalanceSheetStatement), BalanceSheetStatement) + .WithView(nameof(KeyFigures), KeyFigures) + .WithView(nameof(AssetAllocation), AssetAllocation) + .WithView(nameof(NewEntryDialog), NewEntryDialog); + + private static ScopeRegistry CreateRegistry( + LayoutAreaHost host, BalanceSheetStorage storage) + => host.Hub.ServiceProvider.CreateScopeRegistry(storage); + + private static string Name(BalanceSheetStorage storage, string path) + => path.Split('/')[^1]; + + /// + /// The balance sheet: assets and liabilities per year, with the computed + /// positions (Total Assets, Pension Capital, Balance Sheet Sum, Funding + /// Ratio) evaluated through the PositionValue scopes — the formulas live + /// on the Position nodes, not in this view. + /// + [Display(Name = "Balance Sheet", GroupName = "Balance Sheet", Order = 1)] + public static IObservable BalanceSheetStatement(this LayoutAreaHost host, RenderingContext ctx) + => BalanceSheetData.LoadStorage(host.Workspace).Select(storage => + { + var registry = CreateRegistry(host, storage); + var years = storage.Years; + if (years.Length == 0 || storage.Positions.Count == 0) + return (UiControl)Controls.Markdown("*No balance-sheet data loaded yet.*"); + + var sb = new StringBuilder(); + sb.Append("| Position |"); + foreach (var y in years) sb.Append($" {Name(storage, y)} |"); + sb.Append("\n"); + sb.Append("|---|"); + foreach (var _ in years) sb.Append("---:|"); + sb.Append("\n"); + + void Row(string path, Position position, bool bold) + { + var label = bold ? $"**{path.Split('/')[^1]}**" : path.Split('/')[^1]; + sb.Append($"| {label} |"); + foreach (var y in years) + { + var value = registry.GetScope(new PositionYear(path, y)).Value; + var text = position.Aggregation == PositionAggregation.Ratio + ? $"{value:P1}" + : $"{value:N1}"; + sb.Append(bold ? $" **{text}** |" : $" {text} |"); + } + sb.Append("\n"); + } + + void Section(BalanceSheetSide side) + { + foreach (var path in storage.OrderedPositions + .Where(p => storage.Positions[p].Side == side)) + Row(path, storage.Positions[path], + storage.Positions[path].Aggregation != PositionAggregation.Atomic); + } + + sb.Append($"| **Assets** |{string.Concat(Enumerable.Repeat(" |", years.Length))}\n"); + Section(BalanceSheetSide.Assets); + sb.Append($"| **Liabilities** |{string.Concat(Enumerable.Repeat(" |", years.Length))}\n"); + Section(BalanceSheetSide.Liabilities); + sb.Append($"| **Computed** |{string.Concat(Enumerable.Repeat(" |", years.Length))}\n"); + Section(BalanceSheetSide.Computed); + + return (UiControl)Controls.Markdown($"### Balance Sheet (CHF m)\n\n{sb}"); + }); + + /// Headline figures per year via the BalanceSheetSummary scope. + [Display(Name = "Key Figures", GroupName = "Balance Sheet", Order = 2)] + public static IObservable KeyFigures(this LayoutAreaHost host, RenderingContext ctx) + => BalanceSheetData.LoadStorage(host.Workspace).Select(storage => + { + var registry = CreateRegistry(host, storage); + if (storage.Years.Length == 0) + return (UiControl)Controls.Markdown("*No data.*"); + + var sb = new StringBuilder("| | " + string.Join(" | ", storage.Years.Select(y => Name(storage, y))) + " |\n"); + sb.Append("|---|").Append(string.Concat(Enumerable.Repeat("---:|", storage.Years.Length))).AppendLine(); + string Fmt(Func f, string format) => + string.Join(" | ", storage.Years.Select(y => string.Format($"{{0:{format}}}", f(registry.GetScope(y))))); + sb.AppendLine($"| Balance Sheet Sum | {Fmt(s => s.BalanceSheetSum, "N1")} |"); + sb.AppendLine($"| Pension Capital | {Fmt(s => s.PensionCapital, "N1")} |"); + sb.AppendLine($"| **Funding Ratio** | **{Fmt(s => s.FundingRatio, "P1")}** |"); + var balanced = storage.Years.All(y => registry.GetScope(y).Balances); + sb.AppendLine($"\n{(balanced ? "✅ Balance sheet balances." : "⚠️ Assets ≠ liabilities — check the entries!")}"); + + return (UiControl)Controls.Markdown($"### Key Figures (CHF m)\n\n{sb}"); + }); + + /// Asset allocation of the latest year as a pie chart. + [Display(Name = "Asset Allocation", GroupName = "Balance Sheet", Order = 3)] + public static IObservable AssetAllocation(this LayoutAreaHost host, RenderingContext ctx) + => BalanceSheetData.LoadStorage(host.Workspace).Select(storage => + { + if (storage.Years.Length == 0) + return (UiControl)Controls.Markdown("*No data.*"); + var registry = CreateRegistry(host, storage); + var year = storage.Years[^1]; + + var atoms = storage.OrderedPositions + .Where(p => storage.Positions[p] is { Side: BalanceSheetSide.Assets, Aggregation: PositionAggregation.Atomic }) + .Select(p => (Label: p.Split('/')[^1], + Value: registry.GetScope(new PositionYear(p, year)).Value)) + .ToArray(); + + return (UiControl)Charts.Pie(atoms.Select(a => a.Value), atoms.Select(a => a.Label)) + .WithTitle($"Asset Allocation {year.Split('/')[^1]} (CHF m)"); + }); + + /// + /// Button opening the new-entry dialog: the Edit form over the draft whose + /// dimension fields are MeshNodePickers over the dimension nodes. + /// + [Display(Name = "New Entry (dialog)", GroupName = "Balance Sheet", Order = 4)] + public static UiControl NewEntryDialog(this LayoutAreaHost host, RenderingContext ctx) + => Controls.Button("New balance sheet entry…") + .WithClickAction(click => + { + var dialog = Controls.Dialog( + click.Host.Edit(new BalanceSheetEntryDraft(), "newEntry"), + "New Balance Sheet Entry") + .WithSize("M") + .WithClosable(true); + click.Host.UpdateArea(DialogControl.DialogArea, dialog); + return Task.CompletedTask; + }); +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheet/Source/BalanceSheetScopes.cs b/samples/Graph/Data/PensionFund/BalanceSheet/Source/BalanceSheetScopes.cs new file mode 100644 index 000000000..c87a4e6c9 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheet/Source/BalanceSheetScopes.cs @@ -0,0 +1,115 @@ +// +// Id: BalanceSheetScopes +// DisplayName: Balance Sheet Business Rules +// + +// Pull the scope SOURCE GENERATOR in at node-compile time. It is NOT a framework reference (that +// propagated the analyzer to every downstream project and bloated every build — commit ef2e756d6, +// project_graph_generator_build_bloat); instead it travels WITH the node Source via #r, resolved +// from the mesh-local feed (dist/packages, baked into the container image — NOT on nuget.org). +// SourceGeneratorLoader discovers the [Generator] and MeshNodeCompilationService.RunSourceGenerators +// emits the concrete IScope<,> implementations AddBusinessRules then discovers. Version-LESS: the +// resolver picks whatever single version the feed carries (one global PlatformVersion, no drift). +// MeshWeaver.BusinessRules itself is already in the compile's reference set (the portal ships it via +// MeshWeaver.Documentation), so it is NOT #r'd here — a second reference would be a duplicate identity. +// +// NOTE: this file is therefore NOT a `` item in MeshWeaver.PensionFund.Test — `#r` is illegal +// in a regular (non-script) compilation (CS7011). The runtime NodeType compiler strips #r before Roslyn +// sees it; the design-time scope-codegen contract is covered by FxConversionScopeTest + the runtime +// render tests here + NuGetAssemblyResolverTest (the #r-feed mechanism). +#r "nuget:MeshWeaver.BusinessRules.Generator" + +using System.Collections.Immutable; +using MeshWeaver.BusinessRules; + +/// +/// The storage every balance-sheet scope computes against: the position +/// dimension nodes (by path) and the atomic fact values (by position × year +/// path pair). Built once per evaluation from the mesh nodes. +/// +public record BalanceSheetStorage +{ + /// Position content by position node path. + public required ImmutableDictionary Positions { get; init; } + + /// + /// Position paths in display order — taken from MeshNode.Order on + /// the dimension nodes, since ordering lives on the node, not the content. + /// + public required ImmutableList OrderedPositions { get; init; } + + /// Atomic amounts (CHF m) by (position path, year path). + public required ImmutableDictionary<(string Position, string Year), double> Amounts { get; init; } + + /// All year node paths, ascending. + public required ImmutableArray Years { get; init; } +} + +/// +/// Identity of a position value: one Position node in one Year node — +/// both referenced by mesh path. +/// +public record PositionYear(string Position, string Year); + +/// +/// THE business rule of the balance sheet: what a position is worth. +/// Atomic positions read the fact value; Sum positions fold their weighted +/// components (each component being another PositionValue scope — composition +/// all the way down to the atoms); Ratio positions divide two other positions. +/// Scope instances are cached per (identity) by the ScopeRegistry, so shared +/// sub-positions (e.g. Total Assets inside both Balance Sheet Sum and the +/// Funding Ratio) are computed exactly once. +/// +public interface PositionValue : IScope +{ + /// The position's dimension node content. + Position Position => GetStorage().Positions[Identity.Position]; + + /// The computed value (CHF m) of this position in this year. + double Value => Position.Aggregation switch + { + PositionAggregation.Atomic => + GetStorage().Amounts.TryGetValue((Identity.Position, Identity.Year), out var v) ? v : 0, + + PositionAggregation.Sum => + (Position.Components ?? []) + .Sum(c => c.Weight * GetScope(new PositionYear(c.Position, Identity.Year)).Value), + + PositionAggregation.Ratio => + GetScope(new PositionYear(Position.Denominator!, Identity.Year)).Value is var denominator + && denominator != 0 + ? GetScope(new PositionYear(Position.Numerator!, Identity.Year)).Value / denominator + : 0, + + _ => 0, + }; +} + +/// +/// Year-level summary scope: composes the headline figures of one reporting +/// year out of the position scopes. The position paths are resolved from the +/// storage by aggregation/side, so the scope works for any position set. +/// +public interface BalanceSheetSummary : IScope +{ + private static string PathOf(BalanceSheetStorage storage, string suffix) + => storage.Positions.Keys.First(p => p.EndsWith("/" + suffix)); + + double BalanceSheetSum => GetScope( + new PositionYear(PathOf(GetStorage(), "BalanceSheetSum"), Identity)).Value; + + double TotalAssets => GetScope( + new PositionYear(PathOf(GetStorage(), "TotalAssets"), Identity)).Value; + + double TotalLiabilities => GetScope( + new PositionYear(PathOf(GetStorage(), "TotalLiabilities"), Identity)).Value; + + double PensionCapital => GetScope( + new PositionYear(PathOf(GetStorage(), "PensionCapital"), Identity)).Value; + + double FundingRatio => GetScope( + new PositionYear(PathOf(GetStorage(), "FundingRatio"), Identity)).Value; + + /// The balance check: assets and liabilities must match. + bool Balances => Math.Abs(TotalAssets - TotalLiabilities) < 1e-9; +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheet/Source/ExternalDependencies.cs b/samples/Graph/Data/PensionFund/BalanceSheet/Source/ExternalDependencies.cs new file mode 100644 index 000000000..7ca551830 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheet/Source/ExternalDependencies.cs @@ -0,0 +1,9 @@ +// +// Id: ExternalDependencies +// DisplayName: External Dependencies +// + +@@PensionFund/Position/Source/Position +@@PensionFund/Year/Source/Year +@@PensionFund/Currency/Source/Currency +@@PensionFund/BalanceSheetEntry/Source/BalanceSheetEntry diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry.json new file mode 100644 index 000000000..21401ca3a --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry.json @@ -0,0 +1,18 @@ +{ + "id": "BalanceSheetEntry", + "namespace": "PensionFund", + "name": "Balance Sheet Entry", + "nodeType": "NodeType", + "category": "Types", + "description": "Atomic balance-sheet fact: the value of one Position in one Year. Entries are mesh nodes - no Id property; dimension columns store dimension node PATHS.", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "NodeTypeDefinition", + "id": "BalanceSheetEntry", + "namespace": "PensionFund", + "displayName": "Balance Sheet Entry", + "iconName": "DocumentTable", + "configuration": "config =\u003e config.WithContentType\u003cBalanceSheetEntry\u003e().AddDefaultLayoutAreas()" + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-AccruedLiabilities.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-AccruedLiabilities.json new file mode 100644 index 000000000..853d24c3e --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-AccruedLiabilities.json @@ -0,0 +1,17 @@ +{ + "id": "2024-AccruedLiabilities", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "AccruedLiabilities 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "AccruedLiabilities in 2024: CHF 5 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/AccruedLiabilities", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 5 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-ActiveMembersCapital.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-ActiveMembersCapital.json new file mode 100644 index 000000000..67add164c --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-ActiveMembersCapital.json @@ -0,0 +1,17 @@ +{ + "id": "2024-ActiveMembersCapital", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "ActiveMembersCapital 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "ActiveMembersCapital in 2024: CHF 600 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/ActiveMembersCapital", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 600 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Alternatives.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Alternatives.json new file mode 100644 index 000000000..f0dedc583 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Alternatives.json @@ -0,0 +1,17 @@ +{ + "id": "2024-Alternatives", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "Alternatives 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "Alternatives in 2024: CHF 100 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/Alternatives", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 100 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Bonds.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Bonds.json new file mode 100644 index 000000000..ea5df5426 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Bonds.json @@ -0,0 +1,17 @@ +{ + "id": "2024-Bonds", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "Bonds 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "Bonds in 2024: CHF 400 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/Bonds", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 400 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Cash.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Cash.json new file mode 100644 index 000000000..b0d5211a7 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Cash.json @@ -0,0 +1,17 @@ +{ + "id": "2024-Cash", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "Cash 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "Cash in 2024: CHF 50 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/Cash", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 50 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-EmployerContributionReserve.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-EmployerContributionReserve.json new file mode 100644 index 000000000..fd62fff89 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-EmployerContributionReserve.json @@ -0,0 +1,17 @@ +{ + "id": "2024-EmployerContributionReserve", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "EmployerContributionReserve 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "EmployerContributionReserve in 2024: CHF 20 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/EmployerContributionReserve", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 20 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Equities.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Equities.json new file mode 100644 index 000000000..d7284a2d8 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Equities.json @@ -0,0 +1,17 @@ +{ + "id": "2024-Equities", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "Equities 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "Equities in 2024: CHF 300 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/Equities", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 300 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-FreeFunds.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-FreeFunds.json new file mode 100644 index 000000000..ce49dc1ae --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-FreeFunds.json @@ -0,0 +1,17 @@ +{ + "id": "2024-FreeFunds", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "FreeFunds 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "FreeFunds in 2024: CHF 10 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/FreeFunds", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 10 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-NonTechnicalProvisions.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-NonTechnicalProvisions.json new file mode 100644 index 000000000..cba8340f4 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-NonTechnicalProvisions.json @@ -0,0 +1,17 @@ +{ + "id": "2024-NonTechnicalProvisions", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "NonTechnicalProvisions 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "NonTechnicalProvisions in 2024: CHF 10 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/NonTechnicalProvisions", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 10 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Payables.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Payables.json new file mode 100644 index 000000000..69f428631 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Payables.json @@ -0,0 +1,17 @@ +{ + "id": "2024-Payables", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "Payables 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "Payables in 2024: CHF 15 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/Payables", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 15 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-PensionersCapital.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-PensionersCapital.json new file mode 100644 index 000000000..b756313da --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-PensionersCapital.json @@ -0,0 +1,17 @@ +{ + "id": "2024-PensionersCapital", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "PensionersCapital 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "PensionersCapital in 2024: CHF 280 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/PensionersCapital", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 280 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-RealEstate.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-RealEstate.json new file mode 100644 index 000000000..1acce5cf0 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-RealEstate.json @@ -0,0 +1,17 @@ +{ + "id": "2024-RealEstate", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "RealEstate 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "RealEstate in 2024: CHF 200 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/RealEstate", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 200 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Receivables.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Receivables.json new file mode 100644 index 000000000..d6ff7a780 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-Receivables.json @@ -0,0 +1,17 @@ +{ + "id": "2024-Receivables", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "Receivables 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "Receivables in 2024: CHF 10 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/Receivables", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 10 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-TechnicalProvisions.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-TechnicalProvisions.json new file mode 100644 index 000000000..bdf902f60 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-TechnicalProvisions.json @@ -0,0 +1,17 @@ +{ + "id": "2024-TechnicalProvisions", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "TechnicalProvisions 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "TechnicalProvisions in 2024: CHF 40 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/TechnicalProvisions", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 40 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-ValueFluctuationReserve.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-ValueFluctuationReserve.json new file mode 100644 index 000000000..203d25614 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2024-ValueFluctuationReserve.json @@ -0,0 +1,17 @@ +{ + "id": "2024-ValueFluctuationReserve", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "ValueFluctuationReserve 2024", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "ValueFluctuationReserve in 2024: CHF 80 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/ValueFluctuationReserve", + "year": "PensionFund/Year/2024", + "currency": "PensionFund/Currency/CHF", + "amount": 80 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-AccruedLiabilities.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-AccruedLiabilities.json new file mode 100644 index 000000000..3ad31182e --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-AccruedLiabilities.json @@ -0,0 +1,17 @@ +{ + "id": "2025-AccruedLiabilities", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "AccruedLiabilities 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "AccruedLiabilities in 2025: CHF 6 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/AccruedLiabilities", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 6 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-ActiveMembersCapital.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-ActiveMembersCapital.json new file mode 100644 index 000000000..1b5b5023b --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-ActiveMembersCapital.json @@ -0,0 +1,17 @@ +{ + "id": "2025-ActiveMembersCapital", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "ActiveMembersCapital 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "ActiveMembersCapital in 2025: CHF 620 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/ActiveMembersCapital", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 620 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Alternatives.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Alternatives.json new file mode 100644 index 000000000..290b8636c --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Alternatives.json @@ -0,0 +1,17 @@ +{ + "id": "2025-Alternatives", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "Alternatives 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "Alternatives in 2025: CHF 110 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/Alternatives", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 110 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Bonds.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Bonds.json new file mode 100644 index 000000000..10b4c3d89 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Bonds.json @@ -0,0 +1,17 @@ +{ + "id": "2025-Bonds", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "Bonds 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "Bonds in 2025: CHF 410 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/Bonds", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 410 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Cash.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Cash.json new file mode 100644 index 000000000..0e1b424fd --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Cash.json @@ -0,0 +1,17 @@ +{ + "id": "2025-Cash", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "Cash 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "Cash in 2025: CHF 60 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/Cash", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 60 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-EmployerContributionReserve.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-EmployerContributionReserve.json new file mode 100644 index 000000000..6767316c7 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-EmployerContributionReserve.json @@ -0,0 +1,17 @@ +{ + "id": "2025-EmployerContributionReserve", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "EmployerContributionReserve 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "EmployerContributionReserve in 2025: CHF 22 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/EmployerContributionReserve", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 22 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Equities.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Equities.json new file mode 100644 index 000000000..8dea8fa20 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Equities.json @@ -0,0 +1,17 @@ +{ + "id": "2025-Equities", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "Equities 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "Equities in 2025: CHF 340 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/Equities", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 340 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-FreeFunds.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-FreeFunds.json new file mode 100644 index 000000000..1dea2e5b0 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-FreeFunds.json @@ -0,0 +1,17 @@ +{ + "id": "2025-FreeFunds", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "FreeFunds 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "FreeFunds in 2025: CHF 10 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/FreeFunds", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 10 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-NonTechnicalProvisions.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-NonTechnicalProvisions.json new file mode 100644 index 000000000..29ab13966 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-NonTechnicalProvisions.json @@ -0,0 +1,17 @@ +{ + "id": "2025-NonTechnicalProvisions", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "NonTechnicalProvisions 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "NonTechnicalProvisions in 2025: CHF 12 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/NonTechnicalProvisions", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 12 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Payables.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Payables.json new file mode 100644 index 000000000..02bc4913c --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Payables.json @@ -0,0 +1,17 @@ +{ + "id": "2025-Payables", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "Payables 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "Payables in 2025: CHF 18 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/Payables", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 18 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-PensionersCapital.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-PensionersCapital.json new file mode 100644 index 000000000..4ec80ea0f --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-PensionersCapital.json @@ -0,0 +1,17 @@ +{ + "id": "2025-PensionersCapital", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "PensionersCapital 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "PensionersCapital in 2025: CHF 300 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/PensionersCapital", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 300 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-RealEstate.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-RealEstate.json new file mode 100644 index 000000000..d5398f5af --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-RealEstate.json @@ -0,0 +1,17 @@ +{ + "id": "2025-RealEstate", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "RealEstate 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "RealEstate in 2025: CHF 210 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/RealEstate", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 210 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Receivables.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Receivables.json new file mode 100644 index 000000000..c2f87da81 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-Receivables.json @@ -0,0 +1,17 @@ +{ + "id": "2025-Receivables", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "Receivables 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "Receivables in 2025: CHF 12 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/Receivables", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 12 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-TechnicalProvisions.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-TechnicalProvisions.json new file mode 100644 index 000000000..889e79409 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-TechnicalProvisions.json @@ -0,0 +1,17 @@ +{ + "id": "2025-TechnicalProvisions", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "TechnicalProvisions 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "TechnicalProvisions in 2025: CHF 44 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/TechnicalProvisions", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 44 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-ValueFluctuationReserve.json b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-ValueFluctuationReserve.json new file mode 100644 index 000000000..11eb0e612 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/2025-ValueFluctuationReserve.json @@ -0,0 +1,17 @@ +{ + "id": "2025-ValueFluctuationReserve", + "namespace": "PensionFund/BalanceSheetEntry", + "name": "ValueFluctuationReserve 2025", + "nodeType": "PensionFund/BalanceSheetEntry", + "category": "Entries", + "description": "ValueFluctuationReserve in 2025: CHF 110 m", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M4 4h16v16H4z\u0027/\u003e\u003cpath d=\u0027M4 10h16M10 4v16\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetEntry", + "position": "PensionFund/Position/ValueFluctuationReserve", + "year": "PensionFund/Year/2025", + "currency": "PensionFund/Currency/CHF", + "amount": 110 + } +} diff --git a/samples/Graph/Data/PensionFund/BalanceSheetEntry/Source/BalanceSheetEntry.cs b/samples/Graph/Data/PensionFund/BalanceSheetEntry/Source/BalanceSheetEntry.cs new file mode 100644 index 000000000..58a0ae144 --- /dev/null +++ b/samples/Graph/Data/PensionFund/BalanceSheetEntry/Source/BalanceSheetEntry.cs @@ -0,0 +1,37 @@ +// +// Id: BalanceSheetEntry +// DisplayName: Balance Sheet Entry +// + +using System.ComponentModel.DataAnnotations; +using MeshWeaver.Domain; + +/// +/// One atomic balance-sheet fact: the value of one Position in one Year. +/// Entries are mesh nodes — there is NO Id property; the node path is the +/// identity. Every dimension column stores the PATH of a dimension node, and +/// each [MeshNode] attribute queries by the dimension TYPE's path, so the +/// Edit form renders pickers over the dimension nodes. +/// +public record BalanceSheetEntry +{ + /// Path of the Position node this value belongs to. + [MeshNode("nodeType:PensionFund/Position")] + [Display(Name = "Position")] + public string Position { get; init; } = string.Empty; + + /// Path of the reporting Year node. + [MeshNode("nodeType:PensionFund/Year")] + [Display(Name = "Year")] + public string Year { get; init; } = string.Empty; + + /// Path of the Currency node (the fund reports in CHF). + [MeshNode("nodeType:PensionFund/Currency")] + [Display(Name = "Currency")] + public string Currency { get; init; } = string.Empty; + + /// Value in millions of the referenced currency. + [DisplayFormat(DataFormatString = "{0:N1}")] + [Display(Name = "Amount (m)")] + public double Amount { get; init; } +} diff --git a/samples/Graph/Data/PensionFund/Currency.json b/samples/Graph/Data/PensionFund/Currency.json new file mode 100644 index 000000000..5a99ae42e --- /dev/null +++ b/samples/Graph/Data/PensionFund/Currency.json @@ -0,0 +1,18 @@ +{ + "id": "Currency", + "namespace": "PensionFund", + "name": "Currency", + "nodeType": "NodeType", + "category": "Types", + "description": "Currency dimension (ISO 4217); the fund reports in CHF.", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003ccircle cx=\u002712\u0027 cy=\u002712\u0027 r=\u00279\u0027/\u003e\u003cpath d=\u0027M9 9h4a2 2 0 1 1 0 4H9m0-7v10\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "NodeTypeDefinition", + "id": "Currency", + "namespace": "PensionFund", + "displayName": "Currency", + "iconName": "Money", + "configuration": "config =\u003e config.WithContentType\u003cCurrency\u003e().AddDefaultLayoutAreas()" + } +} diff --git a/samples/Graph/Data/PensionFund/Currency/CHF.json b/samples/Graph/Data/PensionFund/Currency/CHF.json new file mode 100644 index 000000000..8652b8ee6 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Currency/CHF.json @@ -0,0 +1,17 @@ +{ + "id": "CHF", + "namespace": "PensionFund/Currency", + "name": "CHF", + "nodeType": "PensionFund/Currency", + "category": "Currencies", + "description": "CHF (ISO 4217)", + "order": 1, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003ccircle cx=\u002712\u0027 cy=\u002712\u0027 r=\u00279\u0027/\u003e\u003cpath d=\u0027M9 9h4a2 2 0 1 1 0 4H9m0-7v10\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Currency", + "code": "CHF", + "symbol": "CHF", + "decimalPlaces": 2 + } +} diff --git a/samples/Graph/Data/PensionFund/Currency/EUR.json b/samples/Graph/Data/PensionFund/Currency/EUR.json new file mode 100644 index 000000000..d969beb41 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Currency/EUR.json @@ -0,0 +1,17 @@ +{ + "id": "EUR", + "namespace": "PensionFund/Currency", + "name": "EUR", + "nodeType": "PensionFund/Currency", + "category": "Currencies", + "description": "EUR (ISO 4217)", + "order": 2, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003ccircle cx=\u002712\u0027 cy=\u002712\u0027 r=\u00279\u0027/\u003e\u003cpath d=\u0027M9 9h4a2 2 0 1 1 0 4H9m0-7v10\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Currency", + "code": "EUR", + "symbol": "€", + "decimalPlaces": 2 + } +} diff --git a/samples/Graph/Data/PensionFund/Currency/Source/Currency.cs b/samples/Graph/Data/PensionFund/Currency/Source/Currency.cs new file mode 100644 index 000000000..2400743cd --- /dev/null +++ b/samples/Graph/Data/PensionFund/Currency/Source/Currency.cs @@ -0,0 +1,21 @@ +// +// Id: Currency +// DisplayName: Currency +// + +/// +/// Currency dimension (ISO 4217). Currencies are mesh nodes referenced by +/// path; the pension fund reports in CHF, so all balance-sheet facts carry +/// the CHF node's path. +/// +public record Currency +{ + /// ISO 4217 code, e.g. CHF. + public string Code { get; init; } = string.Empty; + + /// Currency symbol for display. + public string? Symbol { get; init; } + + /// Decimal places for formatting. + public int DecimalPlaces { get; init; } = 2; +} diff --git a/samples/Graph/Data/PensionFund/Currency/USD.json b/samples/Graph/Data/PensionFund/Currency/USD.json new file mode 100644 index 000000000..0649ca879 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Currency/USD.json @@ -0,0 +1,17 @@ +{ + "id": "USD", + "namespace": "PensionFund/Currency", + "name": "USD", + "nodeType": "PensionFund/Currency", + "category": "Currencies", + "description": "USD (ISO 4217)", + "order": 3, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003ccircle cx=\u002712\u0027 cy=\u002712\u0027 r=\u00279\u0027/\u003e\u003cpath d=\u0027M9 9h4a2 2 0 1 1 0 4H9m0-7v10\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Currency", + "code": "USD", + "symbol": "$", + "decimalPlaces": 2 + } +} diff --git a/samples/Graph/Data/PensionFund/Position.json b/samples/Graph/Data/PensionFund/Position.json new file mode 100644 index 000000000..2345b92bd --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position.json @@ -0,0 +1,18 @@ +{ + "id": "Position", + "namespace": "PensionFund", + "name": "Balance Sheet Position", + "nodeType": "NodeType", + "category": "Types", + "description": "Balance-sheet position dimension: atomic positions take values from entries; computed positions (Total Assets, Balance Sheet Sum, Funding Ratio) define formulas over other positions.", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00273\u0027 y=\u00273\u0027 width=\u002718\u0027 height=\u002718\u0027 rx=\u00273\u0027/\u003e\u003cpath d=\u0027M12 3v18M3 9h18\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "NodeTypeDefinition", + "id": "Position", + "namespace": "PensionFund", + "displayName": "Balance Sheet Position", + "iconName": "Table", + "configuration": "config =\u003e config.WithContentType\u003cPosition\u003e().AddDefaultLayoutAreas()" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/AccruedLiabilities.json b/samples/Graph/Data/PensionFund/Position/AccruedLiabilities.json new file mode 100644 index 000000000..0e2c2056f --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/AccruedLiabilities.json @@ -0,0 +1,16 @@ +{ + "id": "AccruedLiabilities", + "namespace": "PensionFund/Position", + "name": "Accrued Liabilities", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Accruals and deferred income at the balance-sheet date.", + "order": 12, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Liabilities", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/ActiveMembersCapital.json b/samples/Graph/Data/PensionFund/Position/ActiveMembersCapital.json new file mode 100644 index 000000000..bc062ddc6 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/ActiveMembersCapital.json @@ -0,0 +1,16 @@ +{ + "id": "ActiveMembersCapital", + "namespace": "PensionFund/Position", + "name": "Active Members\u0027 Capital", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Accumulated retirement savings of active insured members incl. mandatory BVG minimum interest.", + "order": 15, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Liabilities", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/Alternatives.json b/samples/Graph/Data/PensionFund/Position/Alternatives.json new file mode 100644 index 000000000..553ef8fd0 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/Alternatives.json @@ -0,0 +1,16 @@ +{ + "id": "Alternatives", + "namespace": "PensionFund/Position", + "name": "Alternative Investments", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Hedge funds, private equity, infrastructure, insurance-linked securities. BVV2 limit: 15%.", + "order": 5, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Assets", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/AvailableAssets.json b/samples/Graph/Data/PensionFund/Position/AvailableAssets.json new file mode 100644 index 000000000..1f3ba0d11 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/AvailableAssets.json @@ -0,0 +1,44 @@ +{ + "id": "AvailableAssets", + "namespace": "PensionFund/Position", + "name": "Available Assets", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Available Assets = Total Assets - Payables - Accrued Liabilities - Employer Contribution Reserve - Non-Technical Provisions (BVV2 Art. 44 numerator) - a Sum with negative weights.", + "order": 25, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Computed", + "aggregation": "Sum", + "components": [ + { + "$type": "PositionComponent", + "position": "PensionFund/Position/TotalAssets", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/Payables", + "weight": -1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/AccruedLiabilities", + "weight": -1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/EmployerContributionReserve", + "weight": -1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/NonTechnicalProvisions", + "weight": -1 + } + ] + } +} + diff --git a/samples/Graph/Data/PensionFund/Position/BalanceSheetSum.json b/samples/Graph/Data/PensionFund/Position/BalanceSheetSum.json new file mode 100644 index 000000000..1772d40a8 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/BalanceSheetSum.json @@ -0,0 +1,48 @@ +{ + "id": "BalanceSheetSum", + "namespace": "PensionFund/Position", + "name": "Balance Sheet Sum", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Balance Sheet Sum = sum of the asset side - must equal Total Liabilities for the balance sheet to balance.", + "order": 23, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Computed", + "aggregation": "Sum", + "components": [ + { + "$type": "PositionComponent", + "position": "PensionFund/Position/Cash", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/Bonds", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/Equities", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/RealEstate", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/Alternatives", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/Receivables", + "weight": 1 + } + ] + } +} diff --git a/samples/Graph/Data/PensionFund/Position/Bonds.json b/samples/Graph/Data/PensionFund/Position/Bonds.json new file mode 100644 index 000000000..7c5092993 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/Bonds.json @@ -0,0 +1,16 @@ +{ + "id": "Bonds", + "namespace": "PensionFund/Position", + "name": "Bonds", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Fixed-income portfolio: government and corporate bonds, valued at market per Swiss GAAP FER 26.", + "order": 2, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Assets", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/Cash.json b/samples/Graph/Data/PensionFund/Position/Cash.json new file mode 100644 index 000000000..62def9c77 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/Cash.json @@ -0,0 +1,16 @@ +{ + "id": "Cash", + "namespace": "PensionFund/Position", + "name": "Cash \u0026 Equivalents", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Sight deposits, money-market funds, and short-term placements held for liquidity management.", + "order": 1, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Assets", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/EmployerContributionReserve.json b/samples/Graph/Data/PensionFund/Position/EmployerContributionReserve.json new file mode 100644 index 000000000..45b7824cf --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/EmployerContributionReserve.json @@ -0,0 +1,16 @@ +{ + "id": "EmployerContributionReserve", + "namespace": "PensionFund/Position", + "name": "Employer Contribution Reserve", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Pre-funded employer contributions (AGBR) usable for future contribution payments - BVG Art. 331(3) OR.", + "order": 13, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Liabilities", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/Equities.json b/samples/Graph/Data/PensionFund/Position/Equities.json new file mode 100644 index 000000000..a07221132 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/Equities.json @@ -0,0 +1,16 @@ +{ + "id": "Equities", + "namespace": "PensionFund/Position", + "name": "Equities", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Listed equities, domestic and foreign. Subject to the BVV2 Art. 55 category limit of 50%.", + "order": 3, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Assets", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/FreeFunds.json b/samples/Graph/Data/PensionFund/Position/FreeFunds.json new file mode 100644 index 000000000..5da23889d --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/FreeFunds.json @@ -0,0 +1,16 @@ +{ + "id": "FreeFunds", + "namespace": "PensionFund/Position", + "name": "Free Funds", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Unrestricted funds remaining after all capital and reserves are fully funded.", + "order": 19, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Liabilities", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/FundingRatio.json b/samples/Graph/Data/PensionFund/Position/FundingRatio.json new file mode 100644 index 000000000..7acbe29c8 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/FundingRatio.json @@ -0,0 +1,18 @@ +{ + "id": "FundingRatio", + "namespace": "PensionFund/Position", + "name": "Funding Ratio", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Funding Ratio = Available Assets / Pension Capital (BVV2 Art. 44). Below 100% the fund is underfunded and must take measures.", + "order": 26, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Computed", + "aggregation": "Ratio", + "numerator": "PensionFund/Position/AvailableAssets", + "denominator": "PensionFund/Position/PensionCapital" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/NonTechnicalProvisions.json b/samples/Graph/Data/PensionFund/Position/NonTechnicalProvisions.json new file mode 100644 index 000000000..1f27cc731 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/NonTechnicalProvisions.json @@ -0,0 +1,16 @@ +{ + "id": "NonTechnicalProvisions", + "namespace": "PensionFund/Position", + "name": "Non-Technical Provisions", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Provisions outside the actuarial balance: pending legal cases, operational risks.", + "order": 14, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Liabilities", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/Payables.json b/samples/Graph/Data/PensionFund/Position/Payables.json new file mode 100644 index 000000000..3249497e6 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/Payables.json @@ -0,0 +1,16 @@ +{ + "id": "Payables", + "namespace": "PensionFund/Position", + "name": "Payables", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Vested benefits owed to leavers, pending invoices, and other short-term obligations.", + "order": 11, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Liabilities", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/PensionCapital.json b/samples/Graph/Data/PensionFund/Position/PensionCapital.json new file mode 100644 index 000000000..229afd244 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/PensionCapital.json @@ -0,0 +1,33 @@ +{ + "id": "PensionCapital", + "namespace": "PensionFund/Position", + "name": "Pension Capital", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Pension Capital = Active Members\u0027 Capital + Pensioners\u0027 Capital + Technical Provisions - the actuarial obligation per BVG Art. 44.", + "order": 24, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Computed", + "aggregation": "Sum", + "components": [ + { + "$type": "PositionComponent", + "position": "PensionFund/Position/ActiveMembersCapital", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/PensionersCapital", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/TechnicalProvisions", + "weight": 1 + } + ] + } +} diff --git a/samples/Graph/Data/PensionFund/Position/PensionersCapital.json b/samples/Graph/Data/PensionFund/Position/PensionersCapital.json new file mode 100644 index 000000000..e739bb56d --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/PensionersCapital.json @@ -0,0 +1,16 @@ +{ + "id": "PensionersCapital", + "namespace": "PensionFund/Position", + "name": "Pensioners\u0027 Capital", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Present value of running pensions, discounted at the technical interest rate.", + "order": 16, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Liabilities", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/RealEstate.json b/samples/Graph/Data/PensionFund/Position/RealEstate.json new file mode 100644 index 000000000..610513c92 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/RealEstate.json @@ -0,0 +1,16 @@ +{ + "id": "RealEstate", + "namespace": "PensionFund/Position", + "name": "Real Estate", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Direct property and real-estate funds. BVV2 limit: 30%, of which at most a third abroad.", + "order": 4, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Assets", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/Receivables.json b/samples/Graph/Data/PensionFund/Position/Receivables.json new file mode 100644 index 000000000..fcae67311 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/Receivables.json @@ -0,0 +1,16 @@ +{ + "id": "Receivables", + "namespace": "PensionFund/Position", + "name": "Receivables", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Outstanding employer/employee contributions and withholding-tax reclaims.", + "order": 6, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Assets", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/Source/Position.cs b/samples/Graph/Data/PensionFund/Position/Source/Position.cs new file mode 100644 index 000000000..84bed6440 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/Source/Position.cs @@ -0,0 +1,77 @@ +// +// Id: Position +// DisplayName: Balance Sheet Position +// + +using MeshWeaver.Domain; + +/// +/// Which side of the balance sheet a position belongs to. +/// +public enum BalanceSheetSide +{ + Assets, + Liabilities, + /// Computed positions (sums, ratios) — not on either side. + Computed, +} + +/// +/// How a position's value is obtained. +/// +public enum PositionAggregation +{ + /// Value comes from a BalanceSheetEntry fact node. + Atomic, + /// Value = Σ Weight·Value(Component) over . + Sum, + /// Value = Value(Numerator) / Value(Denominator). + Ratio, +} + +/// +/// One weighted operand of a computed (Sum) position. The component references +/// another Position NODE by its mesh path — computed positions are modelled +/// out of other positions, all the way down to the atomic ones. +/// +public record PositionComponent +{ + /// Mesh path of the referenced Position node. + [MeshNode("nodeType:PensionFund/Position")] + public string Position { get; init; } = string.Empty; + + /// Weight in the sum: +1 adds, -1 subtracts. + public double Weight { get; init; } = 1; +} + +/// +/// A balance-sheet position of a pension fund — the dimension that says what a +/// value MEANS. Positions are mesh nodes: their identity is the node path, so +/// facts and formulas reference them by path; name, description, and display +/// order live on the MESH NODE itself — the content carries only what the node +/// doesn't already have. Atomic positions take their value from fact entries; +/// computed positions (Total Assets, Balance Sheet Sum, Funding Ratio) define +/// a formula over other positions. +/// +public record Position +{ + /// Assets, Liabilities, or Computed. + public BalanceSheetSide Side { get; init; } + + /// Atomic (from facts), Sum (weighted components), or Ratio. + public PositionAggregation Aggregation { get; init; } = PositionAggregation.Atomic; + + /// + /// Weighted operands for positions — + /// e.g. Balance Sheet Sum = 1·Cash + 1·Bonds + 1·Equities + …. + /// + public PositionComponent[]? Components { get; init; } + + /// Numerator position path for . + [MeshNode("nodeType:PensionFund/Position")] + public string? Numerator { get; init; } + + /// Denominator position path for . + [MeshNode("nodeType:PensionFund/Position")] + public string? Denominator { get; init; } +} diff --git a/samples/Graph/Data/PensionFund/Position/TechnicalProvisions.json b/samples/Graph/Data/PensionFund/Position/TechnicalProvisions.json new file mode 100644 index 000000000..1667d1129 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/TechnicalProvisions.json @@ -0,0 +1,16 @@ +{ + "id": "TechnicalProvisions", + "namespace": "PensionFund/Position", + "name": "Technical Provisions", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Actuarial provisions: longevity, conversion-rate losses, death and disability fluctuations.", + "order": 17, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Liabilities", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Position/TotalAssets.json b/samples/Graph/Data/PensionFund/Position/TotalAssets.json new file mode 100644 index 000000000..ee286d962 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/TotalAssets.json @@ -0,0 +1,48 @@ +{ + "id": "TotalAssets", + "namespace": "PensionFund/Position", + "name": "Total Assets", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Total Assets = sum of all atomic asset positions.", + "order": 21, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Computed", + "aggregation": "Sum", + "components": [ + { + "$type": "PositionComponent", + "position": "PensionFund/Position/Cash", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/Bonds", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/Equities", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/RealEstate", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/Alternatives", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/Receivables", + "weight": 1 + } + ] + } +} diff --git a/samples/Graph/Data/PensionFund/Position/TotalLiabilities.json b/samples/Graph/Data/PensionFund/Position/TotalLiabilities.json new file mode 100644 index 000000000..0c8c367ea --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/TotalLiabilities.json @@ -0,0 +1,63 @@ +{ + "id": "TotalLiabilities", + "namespace": "PensionFund/Position", + "name": "Total Liabilities", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Total Liabilities = sum of all atomic liability positions (incl. reserves and free funds).", + "order": 22, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Computed", + "aggregation": "Sum", + "components": [ + { + "$type": "PositionComponent", + "position": "PensionFund/Position/Payables", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/AccruedLiabilities", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/EmployerContributionReserve", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/NonTechnicalProvisions", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/ActiveMembersCapital", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/PensionersCapital", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/TechnicalProvisions", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/ValueFluctuationReserve", + "weight": 1 + }, + { + "$type": "PositionComponent", + "position": "PensionFund/Position/FreeFunds", + "weight": 1 + } + ] + } +} diff --git a/samples/Graph/Data/PensionFund/Position/ValueFluctuationReserve.json b/samples/Graph/Data/PensionFund/Position/ValueFluctuationReserve.json new file mode 100644 index 000000000..0d452e32a --- /dev/null +++ b/samples/Graph/Data/PensionFund/Position/ValueFluctuationReserve.json @@ -0,0 +1,16 @@ +{ + "id": "ValueFluctuationReserve", + "namespace": "PensionFund/Position", + "name": "Value Fluctuation Reserve", + "nodeType": "PensionFund/Position", + "category": "Positions", + "description": "Reserve absorbing market-value fluctuations of the assets; target set by the investment strategy (FER 26).", + "order": 18, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00274\u0027 y=\u00274\u0027 width=\u002716\u0027 height=\u002716\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M8 12h8\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Position", + "side": "Liabilities", + "aggregation": "Atomic" + } +} diff --git a/samples/Graph/Data/PensionFund/Statement.json b/samples/Graph/Data/PensionFund/Statement.json new file mode 100644 index 000000000..c5f4d824f --- /dev/null +++ b/samples/Graph/Data/PensionFund/Statement.json @@ -0,0 +1,15 @@ +{ + "id": "Statement", + "namespace": "PensionFund", + "name": "Balance Sheet - Helvetia Vorsorge", + "nodeType": "PensionFund/BalanceSheet", + "category": "Reports", + "description": "The scope-computed balance sheet: statement, key figures (funding ratio per BVV2 Art. 44), asset allocation, and the new-entry dialog.", + "order": 1, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003cpath d=\u0027M12 3v18M5 7l7-4 7 4M5 7v4a7 7 0 0 0 14 0V7\u0027/\u003e\u003cpath d=\u0027M3 21h18\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "BalanceSheetReport", + "commentary": "Funding improved year over year: the 2025 funding ratio of ~112.4% exceeds the value-fluctuation-reserve target, leaving free funds intact." + } +} diff --git a/samples/Graph/Data/PensionFund/Year.json b/samples/Graph/Data/PensionFund/Year.json new file mode 100644 index 000000000..eb8d755bc --- /dev/null +++ b/samples/Graph/Data/PensionFund/Year.json @@ -0,0 +1,18 @@ +{ + "id": "Year", + "namespace": "PensionFund", + "name": "Reporting Year", + "nodeType": "NodeType", + "category": "Types", + "description": "Reporting-year dimension; years are mesh nodes referenced by path.", + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00273\u0027 y=\u00274\u0027 width=\u002718\u0027 height=\u002717\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M3 9h18M8 2v4M16 2v4\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "NodeTypeDefinition", + "id": "Year", + "namespace": "PensionFund", + "displayName": "Reporting Year", + "iconName": "Calendar", + "configuration": "config =\u003e config.WithContentType\u003cYear\u003e().AddDefaultLayoutAreas()" + } +} diff --git a/samples/Graph/Data/PensionFund/Year/2024.json b/samples/Graph/Data/PensionFund/Year/2024.json new file mode 100644 index 000000000..1576bf596 --- /dev/null +++ b/samples/Graph/Data/PensionFund/Year/2024.json @@ -0,0 +1,16 @@ +{ + "id": "2024", + "namespace": "PensionFund/Year", + "name": "2024", + "nodeType": "PensionFund/Year", + "category": "Years", + "description": "Reporting year 2024 (closed, audited)", + "order": 1, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00273\u0027 y=\u00274\u0027 width=\u002718\u0027 height=\u002717\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M3 9h18M8 2v4M16 2v4\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Year", + "value": 2024, + "closed": true + } +} diff --git a/samples/Graph/Data/PensionFund/Year/2025.json b/samples/Graph/Data/PensionFund/Year/2025.json new file mode 100644 index 000000000..240536faa --- /dev/null +++ b/samples/Graph/Data/PensionFund/Year/2025.json @@ -0,0 +1,16 @@ +{ + "id": "2025", + "namespace": "PensionFund/Year", + "name": "2025", + "nodeType": "PensionFund/Year", + "category": "Years", + "description": "Reporting year 2025 (open)", + "order": 2, + "icon": "\u003csvg xmlns=\u0027http://www.w3.org/2000/svg\u0027 viewBox=\u00270 0 24 24\u0027 fill=\u0027none\u0027 stroke=\u0027currentColor\u0027 stroke-width=\u00272\u0027\u003e\u003crect x=\u00273\u0027 y=\u00274\u0027 width=\u002718\u0027 height=\u002717\u0027 rx=\u00272\u0027/\u003e\u003cpath d=\u0027M3 9h18M8 2v4M16 2v4\u0027/\u003e\u003c/svg\u003e", + "isPersistent": true, + "content": { + "$type": "Year", + "value": 2025, + "closed": false + } +} diff --git a/samples/Graph/Data/PensionFund/Year/Source/Year.cs b/samples/Graph/Data/PensionFund/Year/Source/Year.cs new file mode 100644 index 000000000..59e46e4fb --- /dev/null +++ b/samples/Graph/Data/PensionFund/Year/Source/Year.cs @@ -0,0 +1,18 @@ +// +// Id: Year +// DisplayName: Reporting Year +// + +/// +/// Reporting-year dimension. Years are mesh nodes so facts and reports +/// reference them by path — and a year node can carry its own context +/// (closing state, auditor sign-off, commentary). +/// +public record Year +{ + /// Calendar year, e.g. 2024. + public int Value { get; init; } + + /// True once the reporting year is closed and audited. + public bool Closed { get; init; } +} diff --git a/samples/Graph/Data/PensionFund/index.md b/samples/Graph/Data/PensionFund/index.md new file mode 100644 index 000000000..8f4e8540f --- /dev/null +++ b/samples/Graph/Data/PensionFund/index.md @@ -0,0 +1,38 @@ +--- +NodeType: Markdown +Name: PensionFund +Category: Pension & Retirement +Description: Swiss pension fund balance sheet modelled as a data cube — dimensions, facts, and computed positions, everything a mesh node +Icon: +--- + +Welcome to **Helvetia Vorsorge** — a fictional Swiss pension fund whose balance sheet is modelled end-to-end as a MeshWeaver data cube. It is the worked example behind [Data Cubes](/Doc/DataMesh/DataCubes). + +# Everything Is a Mesh Node + +The model has no databases, no foreign keys, no surrogate ids — only mesh nodes referencing each other by **path**: + +| Layer | NodeType | What it holds | +|---|---|---| +| Dimensions | [Position](/PensionFund/Position), [Year](/PensionFund/Year), [Currency](/PensionFund/Currency) | What a value *means*, when, and in which currency | +| Facts | [BalanceSheetEntry](/PensionFund/BalanceSheetEntry) | One atomic amount per Position × Year | +| Formulas | Computed [Position](/PensionFund/Position) nodes | *Total Assets*, *Pension Capital*, *Balance Sheet Sum*, *Funding Ratio* — modelled **out of** other positions | +| Reports | [Statement](/PensionFund/Statement) | Scope-evaluated statement, key figures, asset allocation | + +A fact has **no Id property** — its identity is its node path. Its dimension columns store the *paths* of dimension nodes, declared with `[MeshNode("nodeType:PensionFund/Position")]`, which is also what renders mesh-node pickers in every edit form. + +# Formulas Live on the Dimension + +Computed positions carry their formula as data: + +- **Total Assets** = Σ of the six atomic asset positions +- **Pension Capital** = Active Members' Capital + Pensioners' Capital + Technical Provisions +- **Available Assets** = Total Assets − short-term obligations (a Sum with **negative weights**) +- **Funding Ratio** = Available Assets ÷ Pension Capital (BVV2 Art. 44) + +The `PositionValue` business-rules scope evaluates any position — atomic, sum, or ratio — by composing other `PositionValue` scopes, cached per (position, year). + +# Explore + +- The balance sheet statement, key figures, and asset-allocation chart: [Statement](/PensionFund/Statement) +- The full walk-through with executable code: [Doc/DataMesh/DataCubes](/Doc/DataMesh/DataCubes) diff --git a/samples/Graph/Data/SocialMedia/Post/_Source/Platform.cs b/samples/Graph/Data/SocialMedia/Post/Source/Platform.cs similarity index 100% rename from samples/Graph/Data/SocialMedia/Post/_Source/Platform.cs rename to samples/Graph/Data/SocialMedia/Post/Source/Platform.cs diff --git a/samples/Graph/Data/SocialMedia/Post/_Source/SocialMediaPost.cs b/samples/Graph/Data/SocialMedia/Post/Source/SocialMediaPost.cs similarity index 100% rename from samples/Graph/Data/SocialMedia/Post/_Source/SocialMediaPost.cs rename to samples/Graph/Data/SocialMedia/Post/Source/SocialMediaPost.cs diff --git a/samples/Graph/Data/SocialMedia/Post/_Source/SocialMediaPostLayoutAreas.cs b/samples/Graph/Data/SocialMedia/Post/Source/SocialMediaPostLayoutAreas.cs similarity index 98% rename from samples/Graph/Data/SocialMedia/Post/_Source/SocialMediaPostLayoutAreas.cs rename to samples/Graph/Data/SocialMedia/Post/Source/SocialMediaPostLayoutAreas.cs index 39f169598..f1ef2d5bf 100644 --- a/samples/Graph/Data/SocialMedia/Post/_Source/SocialMediaPostLayoutAreas.cs +++ b/samples/Graph/Data/SocialMedia/Post/Source/SocialMediaPostLayoutAreas.cs @@ -82,10 +82,10 @@ private static int GetInt(MeshNode node, string prop) var filter = host.Reference.GetParameterValue("profile") ?? FilterMy; var posts = meshService - .ObserveQuery(MeshQueryRequest.FromQuery("namespace:SocialMedia/Post")) + .Query(MeshQueryRequest.FromQuery("namespace:SocialMedia/Post")) .Scan(new Dictionary(StringComparer.OrdinalIgnoreCase), ApplyChanges); var profiles = meshService - .ObserveQuery(MeshQueryRequest.FromQuery("namespace:SocialMedia/Profile")) + .Query(MeshQueryRequest.FromQuery("namespace:SocialMedia/Profile")) .Scan(new Dictionary(StringComparer.OrdinalIgnoreCase), ApplyChanges); return posts.CombineLatest(profiles, (postDict, profileDict) => @@ -255,7 +255,7 @@ private static bool TryParseMonth(string? s, out DateTime month) var nodeStream = host.Workspace.GetStream()! .Select(nodes => nodes?.FirstOrDefault(n => n.Path == hubPath)); var profiles = meshService - .ObserveQuery(MeshQueryRequest.FromQuery("namespace:SocialMedia/Profile")) + .Query(MeshQueryRequest.FromQuery("namespace:SocialMedia/Profile")) .Scan(new Dictionary(StringComparer.OrdinalIgnoreCase), ApplyChanges); return nodeStream.CombineLatest(profiles, (node, profileDict) => diff --git a/samples/Graph/Data/SocialMedia/Profile/_Source/Platform.cs b/samples/Graph/Data/SocialMedia/Profile/Source/Platform.cs similarity index 100% rename from samples/Graph/Data/SocialMedia/Profile/_Source/Platform.cs rename to samples/Graph/Data/SocialMedia/Profile/Source/Platform.cs diff --git a/samples/Graph/Data/SocialMedia/Profile/_Source/SocialMediaProfile.cs b/samples/Graph/Data/SocialMedia/Profile/Source/SocialMediaProfile.cs similarity index 100% rename from samples/Graph/Data/SocialMedia/Profile/_Source/SocialMediaProfile.cs rename to samples/Graph/Data/SocialMedia/Profile/Source/SocialMediaProfile.cs diff --git a/samples/Graph/Data/SocialMedia/Profile/_Source/SocialMediaProfileLayoutAreas.cs b/samples/Graph/Data/SocialMedia/Profile/Source/SocialMediaProfileLayoutAreas.cs similarity index 100% rename from samples/Graph/Data/SocialMedia/Profile/_Source/SocialMediaProfileLayoutAreas.cs rename to samples/Graph/Data/SocialMedia/Profile/Source/SocialMediaProfileLayoutAreas.cs diff --git a/samples/Graph/Data/Systemorph.json b/samples/Graph/Data/Systemorph.json index 54d19d9e7..5823a5841 100644 --- a/samples/Graph/Data/Systemorph.json +++ b/samples/Graph/Data/Systemorph.json @@ -1,13 +1,13 @@ { "id": "Systemorph", "name": "Systemorph", - "nodeType": "Organization", + "nodeType": "Space", "description": "The company behind MeshWeaver", "icon": "/static/storage/content/Systemorph/logo_t.png", "addressSegments": 0, "isPersistent": true, "content": { - "$type": "Organization", + "$type": "Space", "name": "Systemorph", "description": "Systemorph Cloud", "website": "https://systemorph.cloud", diff --git a/samples/Graph/Data/Systemorph/Marketing/Post/_Source/Post.cs b/samples/Graph/Data/Systemorph/Marketing/Post/Source/Post.cs similarity index 100% rename from samples/Graph/Data/Systemorph/Marketing/Post/_Source/Post.cs rename to samples/Graph/Data/Systemorph/Marketing/Post/Source/Post.cs diff --git a/samples/Graph/Data/Type/Article/_Source/Article.cs b/samples/Graph/Data/Type/Article/Source/Article.cs similarity index 100% rename from samples/Graph/Data/Type/Article/_Source/Article.cs rename to samples/Graph/Data/Type/Article/Source/Article.cs diff --git a/samples/Graph/Data/User/_Source/Person.cs b/samples/Graph/Data/User/Source/Person.cs similarity index 100% rename from samples/Graph/Data/User/_Source/Person.cs rename to samples/Graph/Data/User/Source/Person.cs diff --git a/samples/Graph/content/FutuRe/EuropeRe/Submission/LargeClaims.xlsx b/samples/Graph/content/FutuRe/EuropeRe/Submission/LargeClaims.xlsx new file mode 100644 index 000000000..8b990fe28 Binary files /dev/null and b/samples/Graph/content/FutuRe/EuropeRe/Submission/LargeClaims.xlsx differ diff --git a/samples/Graph/content/MeshWeaver/logo.png b/samples/Graph/content/MeshWeaver/logo.png new file mode 100644 index 000000000..e9fb221bb Binary files /dev/null and b/samples/Graph/content/MeshWeaver/logo.png differ diff --git a/samples/Insurance/MeshWeaver.Insurance.Domain/Completion/PricingAutocompleteProvider.cs b/samples/Insurance/MeshWeaver.Insurance.Domain/Completion/PricingAutocompleteProvider.cs index 65d47f0bd..90d0b4773 100644 --- a/samples/Insurance/MeshWeaver.Insurance.Domain/Completion/PricingAutocompleteProvider.cs +++ b/samples/Insurance/MeshWeaver.Insurance.Domain/Completion/PricingAutocompleteProvider.cs @@ -1,4 +1,4 @@ -using System.Runtime.CompilerServices; +using System.Reactive.Linq; using MeshWeaver.Data.Completion; using MeshWeaver.Insurance.Domain.Services; @@ -12,26 +12,17 @@ namespace MeshWeaver.Insurance.Domain.Completion; public class PricingAutocompleteProvider(IPricingService pricingService) : IAutocompleteProvider { /// - public async IAsyncEnumerable GetItemsAsync( - string query, - string? contextPath = null, - [EnumeratorCancellation] CancellationToken ct = default) - { - await Task.CompletedTask; // Satisfy async requirement - - var pricings = pricingService.GetCatalog(); - - // p.Id is now in format "company/year" (e.g., "Microsoft/2026") - foreach (var p in pricings) - { - yield return new AutocompleteItem( - Label: $"@{InsuranceApplicationAttribute.PricingType}/{p.Id}/", - InsertText: $"@{InsuranceApplicationAttribute.PricingType}/{p.Id}/", - Description: p.InsuredName ?? p.Id, - Category: "Pricing", - Priority: 1000, - Kind: AutocompleteKind.Other - ); - } - } + public IObservable> GetItems(string query, string? contextPath = null) => + // Pure in-memory enumeration of the pricing catalog — no I/O, no async. One settled snapshot. + // p.Id is in format "company/year" (e.g., "Microsoft/2026"). + Observable.Return>( + pricingService.GetCatalog() + .Select(p => new AutocompleteItem( + Label: $"@{InsuranceApplicationAttribute.PricingType}/{p.Id}/", + InsertText: $"@{InsuranceApplicationAttribute.PricingType}/{p.Id}/", + Description: p.InsuredName ?? p.Id, + Category: "Pricing", + Priority: 1000, + Kind: AutocompleteKind.Other)) + .ToArray()); } diff --git a/samples/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs b/samples/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs index e3a44fc67..2ac814224 100644 --- a/samples/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs +++ b/samples/Insurance/MeshWeaver.Insurance.Domain/InsuranceApplicationExtensions.cs @@ -1,4 +1,5 @@ using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; using MeshWeaver.ContentCollections; using MeshWeaver.Data; using MeshWeaver.Data.Completion; @@ -45,11 +46,11 @@ public MessageHubConfiguration ConfigureInsuranceApplication() { var svc = data.Hub.ServiceProvider.GetRequiredService(); return data.AddSource(src => src - .WithType(t => t.WithInitialData(_ => Task.FromResult(SampleDataProvider.GetLinesOfBusiness()))) - .WithType(t => t.WithInitialData(_ => Task.FromResult(SampleDataProvider.GetCountries()))) - .WithType(t => t.WithInitialData(_ => Task.FromResult(SampleDataProvider.GetLegalEntities()))) - .WithType(t => t.WithInitialData(_ => Task.FromResult(SampleDataProvider.GetCurrencies()))) - .WithType(t => t.WithInitialData(_ => Task.FromResult>(svc.GetCatalog()))) + .WithType(t => t.WithInitialData(() => Observable.Return(SampleDataProvider.GetLinesOfBusiness()))) + .WithType(t => t.WithInitialData(() => Observable.Return(SampleDataProvider.GetCountries()))) + .WithType(t => t.WithInitialData(() => Observable.Return(SampleDataProvider.GetLegalEntities()))) + .WithType(t => t.WithInitialData(() => Observable.Return(SampleDataProvider.GetCurrencies()))) + .WithType(t => t.WithInitialData(() => Observable.Return>(svc.GetCatalog()))) ); }) .AddLayout(l => l @@ -116,22 +117,20 @@ public MessageHubConfiguration ConfigureSinglePricingApplication() var pricingId = data.Hub.Address.Id; return data.AddSource(src => src - .WithType(t => t.WithInitialData(async ct => - { - var pricing = await svc.GetHeaderAsync(pricingId); - return pricing is null ? [] : [pricing]; - })) - .WithType(t => t.WithInitialData(async ct => - await svc.GetRisksAsync(pricingId, ct))) - .WithType(t => t.WithInitialData(_ => Task.FromResult(Enumerable.Empty()))) - .WithType(t => t.WithInitialData(_ => Task.FromResult(Enumerable.Empty()))) - .WithType(t => t.WithInitialData(async ct => - await svc.GetImportConfigurationsAsync(pricingId).ToArrayAsync(ct))) + // Bridge each genuine async service leaf reactively (.ToObservable) — IObservable, no Task surface. + .WithType(t => t.WithInitialData(() => svc.GetHeaderAsync(pricingId).ToObservable() + .Select(pricing => (IEnumerable)(pricing is null ? Array.Empty() : new[] { pricing })))) + .WithType(t => t.WithInitialData(() => svc.GetRisksAsync(pricingId, default).ToObservable())) + .WithType(t => t.WithInitialData(() => Observable.Return(Enumerable.Empty()))) + .WithType(t => t.WithInitialData(() => Observable.Return(Enumerable.Empty()))) + .WithType(t => t.WithInitialData(() => + svc.GetImportConfigurationsAsync(pricingId).ToArrayAsync().AsTask().ToObservable() + .Select(a => (IEnumerable)a))) // Add dimension data mappings - .WithType(t => t.WithInitialData(_ => Task.FromResult(SampleDataProvider.GetLinesOfBusiness()))) - .WithType(t => t.WithInitialData(_ => Task.FromResult(SampleDataProvider.GetCountries()))) - .WithType(t => t.WithInitialData(_ => Task.FromResult(SampleDataProvider.GetLegalEntities()))) - .WithType(t => t.WithInitialData(_ => Task.FromResult(SampleDataProvider.GetCurrencies()))) + .WithType(t => t.WithInitialData(() => Observable.Return(SampleDataProvider.GetLinesOfBusiness()))) + .WithType(t => t.WithInitialData(() => Observable.Return(SampleDataProvider.GetCountries()))) + .WithType(t => t.WithInitialData(() => Observable.Return(SampleDataProvider.GetLegalEntities()))) + .WithType(t => t.WithInitialData(() => Observable.Return(SampleDataProvider.GetCurrencies()))) ) // Configure default data reference: data/pricing/pricingId returns the main Pricing entity .WithDefaultDataReference(workspace => @@ -160,74 +159,57 @@ await svc.GetImportConfigurationsAsync(pricingId).ToArrayAsync(ct))) } } - private static async Task HandleGeocodingRequest( + // Sync handler — compose IObservable chain, Subscribe posts the response. + // No await on the workspace stream (would deadlock the hub pump); the geocoding + // HTTP call is bridged via Observable.FromAsync at the EXTERNAL boundary (a + // pure HTTP client wrapper, not hub-touching — see GoogleGeocodingService). + // See Doc/Architecture/AsynchronousCalls.md. + private static IMessageDelivery HandleGeocodingRequest( IMessageHub hub, - IMessageDelivery request, - CancellationToken ct) + IMessageDelivery request) { - try + var geocodingService = hub.ServiceProvider.GetRequiredService(); + var riskStream = hub.GetWorkspace().GetStream(); + if (riskStream == null) { - // Get the geocoding service - var geocodingService = hub.ServiceProvider.GetRequiredService(); + hub.Post( + new GeocodingResponse { Success = false, GeocodedCount = 0, Error = "No property risks found in workspace" }, + o => o.ResponseFor(request)); + return request.Processed(); + } - // Get the current property risks from the workspace - var workspace = hub.GetWorkspace(); - var riskStream = workspace.GetStream(); - if (riskStream == null) - { - var errorResponse = new GeocodingResponse - { - Success = false, - GeocodedCount = 0, - Error = "No property risks found in workspace" - }; - hub.Post(errorResponse, o => o.ResponseFor(request)); - return request.Processed(); - } - - var risks = await riskStream.FirstAsync(); - var riskList = risks?.ToList() ?? new List(); - - if (!riskList.Any()) + riskStream + .Select(risks => risks?.ToList() ?? new List()) + .Take(1) + .SelectMany(riskList => { - var errorResponse = new GeocodingResponse - { - Success = false, - GeocodedCount = 0, - Error = "No property risks available to geocode" - }; - hub.Post(errorResponse, o => o.ResponseFor(request)); - return request.Processed(); - } - - // Geocode the risks - var geocodingResponse = await geocodingService.GeocodeRisksAsync(riskList, ct); - - // If successful and we have updated risks, update the workspace - if (geocodingResponse is { Success: true, UpdatedRisks: not null } && geocodingResponse.UpdatedRisks.Any()) - { - // Update the workspace with the geocoded risks - var dataChangeRequest = new DataChangeRequest + if (riskList.Count == 0) + return Observable.Return(new GeocodingResponse + { + Success = false, + GeocodedCount = 0, + Error = "No property risks available to geocode" + }); + + // Reactive service — the HTTP fan-out runs inside its bounded Http I/O queue. + return geocodingService.GeocodeRisks(riskList); + }) + .Subscribe( + geocodingResponse => { - Updates = geocodingResponse.UpdatedRisks.ToList() - }; - - hub.Post(dataChangeRequest, o => o.WithTarget(hub.Address)); - } - - // Post the response - hub.Post(geocodingResponse, o => o.ResponseFor(request)); - } - catch (Exception ex) - { - var errorResponse = new GeocodingResponse - { - Success = false, - GeocodedCount = 0, - Error = $"Geocoding failed: {ex.Message}" - }; - hub.Post(errorResponse, o => o.ResponseFor(request)); - } + if (geocodingResponse is { Success: true, UpdatedRisks: not null } + && geocodingResponse.UpdatedRisks.Any()) + { + hub.Post( + new DataChangeRequest { Updates = geocodingResponse.UpdatedRisks.ToList() }, + o => o.WithTarget(hub.Address)); + } + hub.Post(geocodingResponse, o => o.ResponseFor(request)); + }, + ex => + hub.Post( + new GeocodingResponse { Success = false, GeocodedCount = 0, Error = $"Geocoding failed: {ex.Message}" }, + o => o.ResponseFor(request))); return request.Processed(); } diff --git a/samples/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PropertyRisksLayoutArea.cs b/samples/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PropertyRisksLayoutArea.cs index 685a11359..6137f7300 100644 --- a/samples/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PropertyRisksLayoutArea.cs +++ b/samples/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/PropertyRisksLayoutArea.cs @@ -1,5 +1,6 @@ using System.Reactive.Linq; using MeshWeaver.Data; +using MeshWeaver.Messaging; using MeshWeaver.Insurance.Domain.LayoutAreas.Shared; using MeshWeaver.Insurance.Domain.Services; using MeshWeaver.Layout; @@ -104,29 +105,26 @@ private static IObservable GeocodingArea(LayoutAreaHost host, Renderi p.TotalRisks == 0 ? 0 : (int)(100.0 * p.ProcessedRisks / p.TotalRisks))); } - private static async Task ClickGeocoding(UiActionContext obj) + private static Task ClickGeocoding(UiActionContext obj) { // Show initial progress obj.Host.UpdateArea(obj.Area, Controls.Progress("Starting geocoding...", 0)); - try - { - // Start the geocoding process - var response = await obj.Host.Hub.AwaitResponse( - new GeocodingRequest(), - o => o.WithTarget(obj.Hub.Address)); - - // Show completion message - var resultMessage = response?.Message?.Success == true - ? $"✅ Geocoding Complete: {response.Message.GeocodedCount} locations geocoded successfully." - : $"❌ Geocoding Failed: {response?.Message?.Error}"; - - obj.Host.UpdateArea(obj.Area, Controls.Markdown($"**{resultMessage}**")); - } - catch (Exception ex) - { - obj.Host.UpdateArea(obj.Area, Controls.Markdown($"**Geocoding Failed**: {ex.Message}")); - } + // Reactive — Subscribe instead of await; click handler stays sync. + obj.Host.Hub.Observe(new GeocodingRequest(), o => o.WithTarget(obj.Hub.Address)) + .Subscribe( + response => + { + var resultMessage = response.Message?.Success == true + ? $"✅ Geocoding Complete: {response.Message.GeocodedCount} locations geocoded successfully." + : $"❌ Geocoding Failed: {response.Message?.Error}"; + obj.Host.UpdateArea(obj.Area, Controls.Markdown($"**{resultMessage}**")); + }, + ex => + { + obj.Host.UpdateArea(obj.Area, Controls.Markdown($"**Geocoding Failed**: {ex.Message}")); + }); + return Task.CompletedTask; } } diff --git a/samples/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs b/samples/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs index 0b2ceb534..d1abe868c 100644 --- a/samples/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs +++ b/samples/Insurance/MeshWeaver.Insurance.Domain/LayoutAreas/RiskMapLayoutArea.cs @@ -79,29 +79,26 @@ private static IObservable GeocodingArea(LayoutAreaHost host, Renderi p.TotalRisks == 0 ? 0 : (int)(100.0 * p.ProcessedRisks / p.TotalRisks))); } - private static async Task ClickGeocoding(UiActionContext obj) + private static Task ClickGeocoding(UiActionContext obj) { // Show initial progress obj.Host.UpdateArea(obj.Area, Controls.Progress("Starting geocoding...", 0)); - try - { - // Start the geocoding process - var response = await obj.Host.Hub.AwaitResponse( - new GeocodingRequest(), - o => o.WithTarget(obj.Hub.Address)); - - // Show completion message - var resultMessage = response?.Message?.Success == true - ? $"✅ Geocoding Complete: {response.Message.GeocodedCount} locations geocoded successfully." - : $"❌ Geocoding Failed: {response?.Message?.Error}"; - - obj.Host.UpdateArea(obj.Area, Controls.Markdown($"**{resultMessage}**")); - } - catch (Exception ex) - { - obj.Host.UpdateArea(obj.Area, Controls.Markdown($"**Geocoding Failed**: {ex.Message}")); - } + // Reactive — Subscribe instead of await; click handler stays sync. + obj.Host.Hub.Observe(new GeocodingRequest(), o => o.WithTarget(obj.Hub.Address)) + .Subscribe( + response => + { + var resultMessage = response.Message?.Success == true + ? $"✅ Geocoding Complete: {response.Message.GeocodedCount} locations geocoded successfully." + : $"❌ Geocoding Failed: {response.Message?.Error}"; + obj.Host.UpdateArea(obj.Area, Controls.Markdown($"**{resultMessage}**")); + }, + ex => + { + obj.Host.UpdateArea(obj.Area, Controls.Markdown($"**Geocoding Failed**: {ex.Message}")); + }); + return Task.CompletedTask; } private static IObservable RenderRiskDetails(IMessageHub hub, string id) diff --git a/samples/Insurance/MeshWeaver.Insurance.Domain/Services/GoogleGeocodingService.cs b/samples/Insurance/MeshWeaver.Insurance.Domain/Services/GoogleGeocodingService.cs index 4a0cb3758..8b964b695 100644 --- a/samples/Insurance/MeshWeaver.Insurance.Domain/Services/GoogleGeocodingService.cs +++ b/samples/Insurance/MeshWeaver.Insurance.Domain/Services/GoogleGeocodingService.cs @@ -2,6 +2,7 @@ using System.Net.Http.Json; using System.Reactive.Subjects; using MeshWeaver.GoogleMaps; +using MeshWeaver.Mesh.Threading; using Microsoft.Extensions.Options; namespace MeshWeaver.Insurance.Domain.Services; @@ -9,11 +10,15 @@ namespace MeshWeaver.Insurance.Domain.Services; /// /// Google Maps-based geocoding service for property risks. /// -public class GoogleGeocodingService(IOptions googleConfig) : IGeocodingService +public class GoogleGeocodingService( + IOptions googleConfig, + IoPoolRegistry? ioPoolRegistry = null) : IGeocodingService { private readonly ReplaySubject progressSubject = InitializeProgress(); private readonly object progressLock = new(); private readonly HttpClient http = new(); + // The geocoding HTTP fan-out runs on the bounded Http I/O queue (pool), never inline / FromAsync. + private readonly IIoPool ioPool = ioPoolRegistry?.Get(IoPoolNames.Http) ?? IoPool.Unbounded; private static ReplaySubject InitializeProgress() { @@ -24,7 +29,15 @@ public class GoogleGeocodingService(IOptions googleConf public IObservable Progress => progressSubject; - public async Task GeocodeRisksAsync(IReadOnlyCollection risks, CancellationToken cancellationToken = default) + /// + /// Reactive geocoding — the HTTP fan-out runs inside the bounded Http I/O queue (pool); the + /// genuine async leaf is sealed in . Returns + /// (never Task). + /// + public IObservable GeocodeRisks(IReadOnlyCollection risks) + => ioPool.Invoke(ct => GeocodeRisksCoreAsync(risks, ct)); + + private async Task GeocodeRisksCoreAsync(IReadOnlyCollection risks, CancellationToken cancellationToken = default) { try { diff --git a/samples/Insurance/MeshWeaver.Insurance.Domain/Services/IGeocodingService.cs b/samples/Insurance/MeshWeaver.Insurance.Domain/Services/IGeocodingService.cs index bfbd5b3c3..4985e4b78 100644 --- a/samples/Insurance/MeshWeaver.Insurance.Domain/Services/IGeocodingService.cs +++ b/samples/Insurance/MeshWeaver.Insurance.Domain/Services/IGeocodingService.cs @@ -11,9 +11,10 @@ public interface IGeocodingService IObservable Progress { get; } /// - /// Geocodes a collection of property risks. + /// Geocodes a collection of property risks. Reactive — returns + /// (never Task); the HTTP leaf runs inside the bounded Http I/O queue. /// - Task GeocodeRisksAsync(IReadOnlyCollection risks, CancellationToken cancellationToken = default); + IObservable GeocodeRisks(IReadOnlyCollection risks); } /// diff --git a/samples/Insurance/MeshWeaver.Insurance.Test/InsuranceTestBase.cs b/samples/Insurance/MeshWeaver.Insurance.Test/InsuranceTestBase.cs index 004878502..d81342233 100644 --- a/samples/Insurance/MeshWeaver.Insurance.Test/InsuranceTestBase.cs +++ b/samples/Insurance/MeshWeaver.Insurance.Test/InsuranceTestBase.cs @@ -1,6 +1,6 @@ -using System.Text.Json; +using System.Reactive.Linq; +using System.Text.Json; using System.Text.Json.Nodes; -using FluentAssertions.Extensions; using MeshWeaver.Data; using MeshWeaver.Hosting.Monolith.TestBase; using MeshWeaver.Insurance.Domain; @@ -21,36 +21,34 @@ protected override MeshBuilder ConfigureMesh(MeshBuilder builder) .InstallAssemblies(typeof(InsuranceApplicationAttribute).Assembly.Location); } - protected async Task> GetPropertyRisksAsync(Address address) + protected IObservable> GetPropertyRisks(Address address) { var hub = Mesh; - var risksResp = await hub.AwaitResponse( - new GetDataRequest(new CollectionReference(nameof(PropertyRisk))), - o => o.WithTarget(address), - TestContext.Current.CancellationToken); - - return (risksResp?.Message?.Data as IEnumerable)? - .Select(x => x as PropertyRisk ?? (x as JsonObject)?.Deserialize(hub.JsonSerializerOptions)) - .Where(x => x != null) - .Cast() - .ToList() - ?? []; + return hub.Observe( + new GetDataRequest(new CollectionReference(nameof(PropertyRisk))), + o => o.WithTarget(address)) + .Select(risksResp => + (IReadOnlyCollection)((risksResp?.Message?.Data as IEnumerable)? + .Select(x => x as PropertyRisk ?? (x as JsonObject)?.Deserialize(hub.JsonSerializerOptions)) + .Where(x => x != null) + .Cast() + .ToList() + ?? [])); } - protected async Task> GetPricingsAsync() + protected IObservable> GetPricings() { var hub = Mesh; - var pricingsResp = await hub.AwaitResponse( - new GetDataRequest(new CollectionReference(nameof(Pricing))), - o => o.WithTarget(InsuranceApplicationAttribute.Address), - new CancellationTokenSource(10.Seconds()).Token); - - return (pricingsResp.Message.Data as InstanceCollection)? - .Instances.Values - .Select(x => x as Pricing ?? (x as JsonObject)?.Deserialize(hub.JsonSerializerOptions)) - .Where(x => x != null) - .Cast() - .ToList() - ?? []; + return hub.Observe( + new GetDataRequest(new CollectionReference(nameof(Pricing))), + o => o.WithTarget(InsuranceApplicationAttribute.Address)) + .Select(pricingsResp => + (IReadOnlyCollection)((pricingsResp.Message.Data as InstanceCollection)? + .Instances.Values + .Select(x => x as Pricing ?? (x as JsonObject)?.Deserialize(hub.JsonSerializerOptions)) + .Where(x => x != null) + .Cast() + .ToList() + ?? [])); } } diff --git a/samples/Insurance/MeshWeaver.Insurance.Test/MeshWeaver.Insurance.Test.csproj b/samples/Insurance/MeshWeaver.Insurance.Test/MeshWeaver.Insurance.Test.csproj index 58bb7421a..d09e26060 100644 --- a/samples/Insurance/MeshWeaver.Insurance.Test/MeshWeaver.Insurance.Test.csproj +++ b/samples/Insurance/MeshWeaver.Insurance.Test/MeshWeaver.Insurance.Test.csproj @@ -12,7 +12,6 @@ - runtime; build; native; contentfiles; analyzers; buildtransitive @@ -23,5 +22,14 @@ + + + + + + + + PreserveNewest + diff --git a/samples/Insurance/MeshWeaver.Insurance.Test/MicrosoftImportTests.cs b/samples/Insurance/MeshWeaver.Insurance.Test/MicrosoftImportTests.cs index 0039973b7..6ae9a5f73 100644 --- a/samples/Insurance/MeshWeaver.Insurance.Test/MicrosoftImportTests.cs +++ b/samples/Insurance/MeshWeaver.Insurance.Test/MicrosoftImportTests.cs @@ -1,6 +1,4 @@ using System.Reactive.Linq; -using FluentAssertions; -using FluentAssertions.Extensions; using MeshWeaver.ContentCollections; using MeshWeaver.Data; using MeshWeaver.Import; @@ -8,6 +6,7 @@ using MeshWeaver.Insurance.Domain; using MeshWeaver.Insurance.Domain.Services; using MeshWeaver.Mesh; +using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; using Xunit; @@ -100,11 +99,10 @@ public async Task Import_Microsoft_File_WithConfiguration() Configuration = Config }; - var importResponse = await Mesh.AwaitResponse( - importRequest, - o => o.WithTarget(Mesh.Address), - TestContext.Current.CancellationToken - ); + var importResponse = await Mesh.Observe( + importRequest, + o => o.WithTarget(Mesh.Address)) + .Should().Within(20.Seconds()).Emit(); // Assert importResponse.Should().NotBeNull(); @@ -114,8 +112,7 @@ public async Task Import_Microsoft_File_WithConfiguration() // Verify data was imported by querying the workspace var workspace = Mesh.ServiceProvider.GetRequiredService(); var risks = await workspace.GetObservable() - .Timeout(10.Seconds()) - .FirstAsync(x => x.Count > 0); + .Should().Within(10.Seconds()).Match(x => x.Count > 0); risks.Should().NotBeEmpty("import should return at least one risk"); risks.All(r => r.PricingId == MicrosoftPricingId).Should().BeTrue("all risks should have PricingId set to Microsoft/2026"); @@ -148,11 +145,10 @@ public async Task Import_Microsoft_WithAllocation() Configuration = Config }; - var importResponse = await Mesh.AwaitResponse( - importRequest, - o => o.WithTarget(Mesh.Address), - TestContext.Current.CancellationToken - ); + var importResponse = await Mesh.Observe( + importRequest, + o => o.WithTarget(Mesh.Address)) + .Should().Within(20.Seconds()).Emit(); // Assert importResponse.Should().NotBeNull(); @@ -161,8 +157,7 @@ public async Task Import_Microsoft_WithAllocation() // Verify data was imported var workspace = Mesh.ServiceProvider.GetRequiredService(); var risks = await workspace.GetObservable() - .Timeout(10.Seconds()) - .FirstAsync(x => x.Count > 0); + .Should().Within(10.Seconds()).Match(x => x.Count > 0); risks.Should().NotBeEmpty(); // Verify allocation worked - sum of allocated TsiBi should equal proportional distribution @@ -203,11 +198,10 @@ public async Task Import_Microsoft_UsingSumMapping() Configuration = Config }; - var importResponse = await Mesh.AwaitResponse( - importRequest, - o => o.WithTarget(Mesh.Address), - TestContext.Current.CancellationToken - ); + var importResponse = await Mesh.Observe( + importRequest, + o => o.WithTarget(Mesh.Address)) + .Should().Within(20.Seconds()).Emit(); // Assert importResponse.Should().NotBeNull(); @@ -216,8 +210,7 @@ public async Task Import_Microsoft_UsingSumMapping() // Verify data was imported var workspace = Mesh.ServiceProvider.GetRequiredService(); var risks = await workspace.GetObservable() - .Timeout(10.Seconds()) - .FirstAsync(x => x.Count > 0); + .Should().Within(10.Seconds()).Match(x => x.Count > 0); risks.Should().NotBeEmpty(); Output.WriteLine($"Imported {risks.Count} risks using direct mapping"); } diff --git a/samples/Insurance/MeshWeaver.Insurance.Test/PricingCatalogTests.cs b/samples/Insurance/MeshWeaver.Insurance.Test/PricingCatalogTests.cs index ea3f52da7..9c2e16157 100644 --- a/samples/Insurance/MeshWeaver.Insurance.Test/PricingCatalogTests.cs +++ b/samples/Insurance/MeshWeaver.Insurance.Test/PricingCatalogTests.cs @@ -1,7 +1,5 @@ using System.Reactive.Linq; using System.Text.Json; -using FluentAssertions; -using FluentAssertions.Extensions; using MeshWeaver.Data; using MeshWeaver.Insurance.Domain; using MeshWeaver.Insurance.Domain.Services; @@ -31,7 +29,7 @@ protected override MessageHubConfiguration ConfigureClient(MessageHubConfigurati public async Task GetPricingCatalog_ShouldReturnPricings() { // Act - Get the pricing catalog from the Insurance hub - var pricings = await GetPricingsAsync(); + var pricings = await GetPricings().Should().Match(p => p.Count > 0, "catalog should contain sample pricings"); // Assert - Verify that the catalog contains pricings pricings.Should().NotBeNull("catalog should not be null"); @@ -54,7 +52,7 @@ public async Task GetPricingCatalog_ShouldReturnPricings() public async Task GetPricingCatalog_ShouldHaveValidDimensions() { // Act - var pricings = await GetPricingsAsync(); + var pricings = await GetPricings().Should().Match(p => p.Count > 0); // Assert - Verify dimension fields are populated pricings.Should().NotBeEmpty(); @@ -74,7 +72,7 @@ public async Task GetPricingCatalog_ShouldHaveValidDimensions() public async Task GetPricingCatalog_ShouldHaveValidDates() { // Act - var pricings = await GetPricingsAsync(); + var pricings = await GetPricings().Should().Match(p => p.Count > 0); // Assert pricings.Should().NotBeEmpty(); @@ -94,7 +92,7 @@ public async Task GetPricingCatalog_ShouldHaveValidDates() pricing.UnderwritingYear.Should().NotBeNull( $"pricing {pricing.Id} should have an underwriting year"); - pricing.UnderwritingYear.Should().BeGreaterThan(2000, + pricing.UnderwritingYear!.Value.Should().BeGreaterThan(2000, $"pricing {pricing.Id} should have a valid underwriting year"); } @@ -108,7 +106,7 @@ public async Task PricingHub_ShouldStartSuccessfully() // by successfully retrieving the catalog without errors // Act - var pricings = await GetPricingsAsync(); + var pricings = await GetPricings().Should().Match(p => p.Count > 0, "hub should start and return catalog"); // Assert - Hub started if we can get data pricings.Should().NotBeNull("hub should start and return catalog"); @@ -137,9 +135,8 @@ public async Task GetPricingCatalog_UsingLayoutAreaReference_ShouldReturnPricing ); // Get the control from the stream - var control = await stream.GetControlStream(reference.Area!) - .Timeout(10.Seconds()) - .FirstAsync(x => x != null); + var control = (await stream.GetControlStream(reference.Area!) + .Should().Within(10.Seconds()).Match(x => x != null))!; // Assert control.Should().NotBeNull("layout area should return a control"); diff --git a/scripts/analyze-test-mem-leaks.ps1 b/scripts/analyze-test-mem-leaks.ps1 new file mode 100644 index 000000000..e27620037 --- /dev/null +++ b/scripts/analyze-test-mem-leaks.ps1 @@ -0,0 +1,66 @@ +# Aggregates per-test-class memory deltas from the cross-process MEM trace +# emitted by MonolithMeshTestBase (TestMemTrace at INIT_MEM / DISPOSE_MEM). +# The trace lives at $env:TEMP\meshweaver-test-trace.log; this script does +# NOT need any external profiler, it parses the same data that drives the +# OOM watchdog and aggregates it into a per-class leak ranking. + +[CmdletBinding()] +param( + [string]$TracePath = "$env:TEMP\meshweaver-test-trace.log", + [switch]$All +) + +if (-not (Test-Path $TracePath)) { + Write-Error "Trace file not found: $TracePath. Run a dotnet test cycle first." + return +} + +$lines = Get-Content -LiteralPath $TracePath +$disposeMem = $lines | Where-Object { $_ -match 'DISPOSE_MEM' } + +if (-not $disposeMem) { + Write-Warning "No DISPOSE_MEM lines in $TracePath." + return +} + +$rx = '(?[A-Za-z0-9_]+)\] DISPOSE_MEM managed=(?\-?\d+)MiB Δ(?[+\-]\d+)MiB rss=(?\-?\d+)MiB Δ(?[+\-]\d+)MiB' + +$rows = foreach ($line in $disposeMem) { + if ($line -match $rx) { + [PSCustomObject]@{ + Class = $Matches.class + Managed = [int]$Matches.mb + Delta = [int]$Matches.delta + Rss = [int]$Matches.rss + RssDelta = [int]$Matches.rssDelta + } + } +} + +$grouped = $rows | Group-Object Class | ForEach-Object { + $g = $_ + [PSCustomObject]@{ + Class = $g.Name + Runs = $g.Count + TotalManagedMib = ($g.Group | Measure-Object Delta -Sum).Sum + MaxManagedMib = ($g.Group | Measure-Object Delta -Maximum).Maximum + TotalRssMib = ($g.Group | Measure-Object RssDelta -Sum).Sum + MaxRssMib = ($g.Group | Measure-Object RssDelta -Maximum).Maximum + } +} | Sort-Object TotalManagedMib -Descending + +$lineCount = $lines.Count +$disposeCount = $disposeMem.Count + +if ($All) { + $grouped | Format-Table -AutoSize +} else { + Write-Output "" + Write-Output "Top 30 leak suspects by total managed-heap retained:" + Write-Output "" + $grouped | Select-Object -First 30 | Format-Table -AutoSize + Write-Output "Negative TotalManagedMib = class released more than it allocated." + Write-Output "Positive = retained across runs (leak signal)." + Write-Output "" + Write-Output "Source trace: $TracePath ($lineCount lines, $disposeCount DISPOSE_MEM events)" +} diff --git a/src/MeshWeaver.AI.Application/AgentsApplicationExtensions.cs b/src/MeshWeaver.AI.Application/AgentsApplicationExtensions.cs index 3dea0448e..da74d74cd 100644 --- a/src/MeshWeaver.AI.Application/AgentsApplicationExtensions.cs +++ b/src/MeshWeaver.AI.Application/AgentsApplicationExtensions.cs @@ -1,10 +1,13 @@ +using System.Reactive.Linq; using MeshWeaver.AI.Application.Layout; using MeshWeaver.AI.Completion; using MeshWeaver.Data.Completion; using MeshWeaver.Mesh.Completion; using MeshWeaver.Mesh.Services; using MeshWeaver.Messaging; +using MeshWeaver.Reactive; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; namespace MeshWeaver.AI.Application; @@ -22,49 +25,62 @@ public static MessageHubConfiguration ConfigureAgentsApplication(this MessageHub => application .AddAIViews() .WithServices(services => services - // Model provider - uses chat client factory if available - .AddScoped(sp => - { - var chatClientFactory = sp.GetService(); - if (chatClientFactory != null) - return new ModelAutocompleteProvider(chatClientFactory); - return new ModelAutocompleteProvider(); - }) - // Mesh catalog provider + // Mesh catalog provider — @-references autocomplete from the mesh node + // catalog (agents, models, and every other node). The old factory-based + // ModelAutocompleteProvider was deleted: models are mesh nodes now, so it + // only duplicated what this provider already lists. .AddScoped(sp => new MeshCatalogAutocompleteProvider(sp) ) - // Command provider - .AddScoped()) + // Skill provider — slash skills (/agent, /model, /harness, …) from the nodeType:Skill catalog. + .AddScoped()) .WithHandler(HandleAutocompleteRequest); - private static async Task HandleAutocompleteRequest( + private const int AutocompleteTopN = 50; + + private static IMessageDelivery HandleAutocompleteRequest( IMessageHub hub, - IMessageDelivery request, - CancellationToken ct) + IMessageDelivery request) { var providers = hub.ServiceProvider.GetServices(); var query = request.Message.Query; var contextPath = request.Message.Context; - var allItems = new List(); - foreach (var provider in providers) - { - try - { - await foreach (var item in provider.GetItemsAsync(query, contextPath, ct)) + // One-shot request/response back-compat: CombineLatest the providers' snapshot streams, + // merge+score-sort into one snapshot per advance, and post the SETTLED snapshot once every + // provider has completed (LastAsync). The progressive/streaming consumers use the + // AutocompleteReference workspace stream instead. Catch → empty so one bad provider can't + // stall the CombineLatest (it must still complete). + AutocompleteSnapshots.Combine( + providers.Select(p => p.GetItems(query, contextPath) + // Collapse to empty so one bad provider can't stall the + // CombineLatest — Debug-logged so the fault stays greppable. + .Catch((Exception ex) => + { + hub.ServiceProvider.GetService() + ?.CreateLogger(typeof(AgentsApplicationExtensions)) + .LogDebug(ex, "Autocomplete provider {Provider} faulted — returning empty", + p.GetType().Name); + return Observable.Return(AutocompleteSnapshots.Empty); + })), + AutocompleteTopN) + .LastAsync() + .Subscribe( + snapshot => hub.Post( + new AutocompleteResponse(snapshot.ToList()), + o => o.ResponseFor(request)), + ex => { - allItems.Add(item); - } - } - catch - { - // Skip providers that fail - } - } + // The caller is waiting on a response — answer empty rather + // than letting it time out, and log the combine fault. + hub.ServiceProvider.GetService() + ?.CreateLogger(typeof(AgentsApplicationExtensions)) + .LogWarning(ex, "Autocomplete combine faulted for query '{Query}'", query); + hub.Post( + new AutocompleteResponse([]), + o => o.ResponseFor(request)); + }); - var response = new AutocompleteResponse(allItems); - hub.Post(response, o => o.ResponseFor(request)); return request.Processed(); } } diff --git a/src/MeshWeaver.AI.Application/Layout/AgentDetailsArea.cs b/src/MeshWeaver.AI.Application/Layout/AgentDetailsArea.cs index 5a29e5428..1358942c0 100644 --- a/src/MeshWeaver.AI.Application/Layout/AgentDetailsArea.cs +++ b/src/MeshWeaver.AI.Application/Layout/AgentDetailsArea.cs @@ -1,4 +1,6 @@ +using System.Reactive.Linq; using System.Text; +using MeshWeaver.Data; using MeshWeaver.Layout; using MeshWeaver.Layout.Composition; using MeshWeaver.Utils; @@ -13,32 +15,35 @@ public static LayoutDefinition AddAgentDetails(this LayoutDefinition layout) return layout.WithView(nameof(AgentDetails), AgentDetails); } - public static async Task AgentDetails(LayoutAreaHost host, RenderingContext ctx, CancellationToken ct) + public static IObservable AgentDetails(LayoutAreaHost host, RenderingContext ctx) { // Extract agent name from LayoutAreaReference.Id var agentName = ExtractAgentNameFromLayoutAreaId(host.Reference.Id); - var meshQuery = host.Hub.ServiceProvider.GetService(); - if (meshQuery == null) - { - return Controls.Stack - .WithView(Controls.Title("Agent Details", 2), "Title") - .WithView(Controls.Text("Agent service not available."), "ErrorMessage"); - } - // Load agents using AgentOrderingHelper - var agentDisplayInfos = await AgentOrderingHelper.QueryAgentsAsync(meshQuery, null, null); - var agents = agentDisplayInfos.Select(a => a.AgentConfiguration).ToList(); - var agent = agents.FirstOrDefault(a => a.Id == agentName); - - if (agent == null) - { - return Controls.Stack - .WithView(Controls.Title("Agent Details", 2), "Title") - .WithView(Controls.Text($"Agent '{agentName}' not found. Please verify the agent name and try again."), "ErrorMessage") - .WithView(Controls.NavLink("Agents", $"{host.Hub.Address}/Overview"), "BackLink"); - } - - return await CreateAgentDetailsView(agent, agents, host); + // Agent list ALWAYS through the synced GetQuery pipeline + // (AgentPickerProjection.ObserveAgents) — never IMeshService.Query / + // QueryAsync, those miss static-provider fan-out, dedup, and the Initial + // gating that produces "empty Agent dropdown" bugs. + // Fully reactive — composed Select, no await/ToTask (AsynchronousCalls.md). + return AgentPickerProjection + .ObserveAgents(host.Hub) + .FirstAsync() + .Timeout(TimeSpan.FromSeconds(10)) + .Select(agentDisplayInfos => + { + var agents = agentDisplayInfos.Select(a => a.AgentConfiguration).ToList(); + var agent = agents.FirstOrDefault(a => a.Id == agentName); + + if (agent == null) + { + return (UiControl?)Controls.Stack + .WithView(Controls.Title("Agent Details", 2), "Title") + .WithView(Controls.Text($"Agent '{agentName}' not found. Please verify the agent name and try again."), "ErrorMessage") + .WithView(Controls.NavLink("Agents", $"{host.Hub.Address}/Overview"), "BackLink"); + } + + return CreateAgentDetailsView(agent, agents, host); + }); } private static string ExtractAgentNameFromLayoutAreaId(object? id) @@ -48,22 +53,24 @@ private static string ExtractAgentNameFromLayoutAreaId(object? id) return id?.ToString() ?? ""; } - private static Task CreateAgentDetailsView(AgentConfiguration agent, IReadOnlyList agents, LayoutAreaHost host) + private static UiControl? CreateAgentDetailsView(AgentConfiguration agent, IReadOnlyList agents, LayoutAreaHost host) { var markdown = GenerateAgentDetailsMarkdown(agent, agents, host); - return Task.FromResult(Controls.Stack + return Controls.Stack .WithView(Controls.NavLink("Agents", $"{host.Hub.Address}/Overview"), "BackLink") .WithView(Controls.Markdown(markdown), "Content") - .WithView(CreateDelegationsSection(agent, agents, host))); + .WithView(CreateDelegationsSection(agent, agents, host)); } private static string GenerateAgentDetailsMarkdown(AgentConfiguration agent, IReadOnlyList agents, LayoutAreaHost host) { var markdown = new StringBuilder(); - // Title and Description - var displayName = agent.DisplayName ?? agent.Id.Wordify(); + // Title and Description — display name now lives on the MeshNode; this + // delegation-graph view works off detached configs, so it falls back to the + // wordified id (the primary agent page renders the node's Name). + var displayName = agent.Id.Wordify(); markdown.AppendLine($"# {displayName}"); markdown.AppendLine(); if (!string.IsNullOrEmpty(agent.Description)) @@ -154,7 +161,7 @@ internal static string GetDelegationInfoForDisplay(AgentConfiguration agent, IRe var targetId = d.AgentPath.Split('/').Last(); var targetAgent = agentsById.GetValueOrDefault(targetId); var agentLink = targetAgent != null - ? $"{targetAgent.DisplayName ?? targetId}" + ? $"{targetId.Wordify()}" : $"{targetId}"; return $"
  • " + @@ -189,7 +196,7 @@ internal static string GetDelegationInfoForDisplay(AgentConfiguration agent, IRe var delegationsFromHtml = string.Join("", delegationsFromList.Select(item => { var (sourceAgent, reason) = item; - var agentLink = $"{sourceAgent.DisplayName ?? sourceAgent.Id.Wordify()}"; + var agentLink = $"{sourceAgent.Id.Wordify()}"; return $"
  • " + $"
    {agentLink}
    " + diff --git a/src/MeshWeaver.AI.AzureFoundry/AzureClaudeChatClient.cs b/src/MeshWeaver.AI.AzureFoundry/AzureClaudeChatClient.cs index a96fc6f7c..92456f2d8 100644 --- a/src/MeshWeaver.AI.AzureFoundry/AzureClaudeChatClient.cs +++ b/src/MeshWeaver.AI.AzureFoundry/AzureClaudeChatClient.cs @@ -53,14 +53,22 @@ public AzureClaudeChatClient( if (string.IsNullOrEmpty(modelId)) throw new ArgumentException("Model ID is required", nameof(modelId)); - // Ensure endpoint ends with /messages + // Endpoint normalisation handles both direct Anthropic and Azure + // Foundry deployments — same wire protocol, different URL shapes. + // • https://api.anthropic.com → /v1/messages + // • https://x.services.ai.azure.com → /anthropic/v1/messages + // • https://x.services.ai.azure.com/anthropic → /v1/messages + // • fully qualified URLs ending /v1/messages → unchanged this.endpoint = endpoint.TrimEnd('/'); if (!this.endpoint.EndsWith("/v1/messages", StringComparison.OrdinalIgnoreCase)) { if (this.endpoint.EndsWith("/anthropic", StringComparison.OrdinalIgnoreCase)) this.endpoint += "/v1/messages"; - else if (!this.endpoint.Contains("/anthropic/")) + else if (this.endpoint.Contains(".azure.com", StringComparison.OrdinalIgnoreCase) + && !this.endpoint.Contains("/anthropic/", StringComparison.OrdinalIgnoreCase)) this.endpoint += "/anthropic/v1/messages"; + else + this.endpoint += "/v1/messages"; } this.apiKey = apiKey; diff --git a/src/MeshWeaver.AI.AzureFoundry/AzureClaudeChatClientAgentFactory.cs b/src/MeshWeaver.AI.AzureFoundry/AzureClaudeChatClientAgentFactory.cs index ebf2ccaa7..e46e41305 100644 --- a/src/MeshWeaver.AI.AzureFoundry/AzureClaudeChatClientAgentFactory.cs +++ b/src/MeshWeaver.AI.AzureFoundry/AzureClaudeChatClientAgentFactory.cs @@ -1,5 +1,6 @@ using MeshWeaver.Messaging; using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -7,6 +8,14 @@ namespace MeshWeaver.AI.AzureFoundry; /// /// Factory for creating ChatClientAgent instances with Azure AI Foundry Claude/Anthropic services. +/// +/// Driver config (Endpoint + ApiKey) source-of-truth precedence: +/// (1) the selected model's on its MeshNode — +/// stamps the built-ins from +/// the Anthropic config section, but user-authored Model nodes +/// can override per-model; +/// (2) (legacy IOptions binding) as +/// fallback when the model node is missing those fields. /// public class AzureClaudeChatClientAgentFactory( IMessageHub hub, @@ -16,6 +25,9 @@ public class AzureClaudeChatClientAgentFactory( { private readonly AzureClaudeConfiguration configuration = InitAndLog(options, logger); + private ChatClientCredentialResolver Resolver => + Hub.ServiceProvider.GetRequiredService(); + private static AzureClaudeConfiguration InitAndLog(IOptions options, ILogger logger) { var config = options.Value ?? throw new ArgumentNullException(nameof(options)); @@ -34,38 +46,53 @@ private static AzureClaudeConfiguration InitAndLog(IOptions configuration.Order; + /// + /// Claude factory: serves any model name starting with "claude" (case-insensitive), + /// regardless of whether the deployment is direct api.anthropic.com + /// or Azure-hosted Anthropic. Both use the same Messages-API wire protocol; + /// the endpoint (and therefore the route taken) is resolved at + /// time from the model's + /// ModelProvider node via . + /// + public override bool Supports(string modelName) => + !string.IsNullOrEmpty(modelName) + && modelName.StartsWith("claude", StringComparison.OrdinalIgnoreCase); + protected override IChatClient CreateChatClient(AgentConfiguration agentConfig) { - if (string.IsNullOrEmpty(configuration.Endpoint)) - throw new InvalidOperationException("Endpoint is required in AzureClaudeConfiguration"); - - if (string.IsNullOrEmpty(configuration.ApiKey)) - throw new InvalidOperationException("ApiKey is required in AzureClaudeConfiguration"); - - // Use CurrentModelName if set, fall back to agent's preferred model, otherwise use first configured model + // Composer selection wins; then the agent's ModelTier; first configured model as a last resort. var modelName = !string.IsNullOrEmpty(CurrentModelName) ? CurrentModelName - : !string.IsNullOrEmpty(agentConfig.PreferredModel) ? agentConfig.PreferredModel - : configuration.Models.FirstOrDefault(); + : ResolveTierModel(agentConfig) ?? configuration.Models.FirstOrDefault(); if (string.IsNullOrEmpty(modelName)) - throw new InvalidOperationException("At least one model must be configured in AzureClaudeConfiguration.Models"); + throw new InvalidOperationException( + "No model selected. Pick one in the chat dropdown."); - logger.LogDebug( - "Creating Azure Claude chat client for agent {AgentName} using model {ModelName} at endpoint {Endpoint}", - agentConfig.Id, modelName, configuration.Endpoint); + // Driver config: resolver walks parent ModelProvider → root ModelProvider + // → legacy ModelDefinition fields. Fall back to IOptions if the resolver + // returns Missing. + var resolution = Resolver.Resolve(modelName); + var endpoint = resolution.Endpoint ?? configuration.Endpoint; + var apiKey = resolution.ApiKey ?? configuration.ApiKey; + var source = resolution.Endpoint != null || resolution.ApiKey != null + ? resolution.Source + : "IOptions"; - try - { - var chatClient = new AzureClaudeChatClient( - endpoint: configuration.Endpoint, - apiKey: configuration.ApiKey, - modelId: modelName); + if (string.IsNullOrEmpty(endpoint)) + throw new InvalidOperationException( + $"Endpoint is missing for model '{modelName}'. Configure a ModelProvider node (e.g. Model/Anthropic) or set Anthropic:Endpoint in config."); - logger.LogDebug( - "Successfully configured Azure Claude chat client for agent {AgentName}", - agentConfig.Id); + if (string.IsNullOrEmpty(apiKey)) + throw new InvalidOperationException( + $"ApiKey is missing for model '{modelName}'. Configure a ModelProvider node (e.g. Model/Anthropic) or set Anthropic:ApiKey in config."); + + logger.LogInformation( + "[AzureClaude] Creating chat client agent={AgentName} model={ModelName} endpoint={Endpoint} source={Source} apiKeyFp={ApiKeyFingerprint}", + agentConfig.Id, modelName, endpoint, source, Fingerprint(apiKey)); - return chatClient; + try + { + return new AzureClaudeChatClient(endpoint: endpoint, apiKey: apiKey, modelId: modelName); } catch (Exception ex) { @@ -74,4 +101,19 @@ protected override IChatClient CreateChatClient(AgentConfiguration agentConfig) $"Failed to create Azure Claude chat client for agent {agentConfig.Id}: {ex.Message}", ex); } } + + /// + /// 8-char SHA-256-hex prefix of . Used in logs to + /// disambiguate "which key was actually used" without ever logging the + /// key itself. Two requests using the same key produce the same + /// fingerprint; a stale Model-node-stamped key vs a fresh config key + /// shows up as a fingerprint mismatch. + /// + private static string Fingerprint(string? value) + { + if (string.IsNullOrEmpty(value)) return "(empty)"; + var bytes = System.Text.Encoding.UTF8.GetBytes(value); + var hash = System.Security.Cryptography.SHA256.HashData(bytes); + return Convert.ToHexString(hash, 0, 4).ToLowerInvariant(); + } } diff --git a/src/MeshWeaver.AI.AzureFoundry/AzureFoundryChatClientAgentFactory.cs b/src/MeshWeaver.AI.AzureFoundry/AzureFoundryChatClientAgentFactory.cs index fce72756f..2744261ff 100644 --- a/src/MeshWeaver.AI.AzureFoundry/AzureFoundryChatClientAgentFactory.cs +++ b/src/MeshWeaver.AI.AzureFoundry/AzureFoundryChatClientAgentFactory.cs @@ -2,6 +2,7 @@ using Azure.AI.Inference; using MeshWeaver.Messaging; using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -36,38 +37,56 @@ private static AzureFoundryConfiguration InitAndLog(IOptions configuration.Order; + /// + /// Multi-model gateway. Serves OpenAI-shape names (gpt-*, o*-mini, etc.), Mistral, + /// DeepSeek, and any other model the deployment exposes through the /models path. + /// Excludes claude-* (which goes through the dedicated Anthropic endpoint via + /// ). This catch-all serves the + /// chat-selected model without any Models[] enumeration on the deployment side. + /// + public override bool Supports(string modelName) => + !string.IsNullOrEmpty(modelName) + && !modelName.StartsWith("claude", StringComparison.OrdinalIgnoreCase); + protected override IChatClient CreateChatClient(AgentConfiguration agentConfig) { - if (string.IsNullOrEmpty(configuration.Endpoint)) - throw new InvalidOperationException("Endpoint is required in AzureFoundryConfiguration"); - - if (string.IsNullOrEmpty(configuration.ApiKey)) - throw new InvalidOperationException("ApiKey is required in AzureFoundryConfiguration"); - - // Use CurrentModelName if set, fall back to agent's preferred model, otherwise use first configured model + // Composer selection wins; then the agent's ModelTier; first configured model as a last resort. var modelName = !string.IsNullOrEmpty(CurrentModelName) ? CurrentModelName - : !string.IsNullOrEmpty(agentConfig.PreferredModel) ? agentConfig.PreferredModel - : configuration.Models.FirstOrDefault(); + : ResolveTierModel(agentConfig) ?? configuration.Models.FirstOrDefault(); if (string.IsNullOrEmpty(modelName)) throw new InvalidOperationException("At least one model must be configured in AzureFoundryConfiguration.Models"); + // Resolver follows ModelDefinition.ProviderRef → ModelProvider node + // for Endpoint + ApiKey. Falls back to IOptions configuration when + // no provider node is present (legacy single-tenant deployments). + var resolver = Hub.ServiceProvider.GetRequiredService(); + var resolution = resolver.Resolve(modelName); + var endpoint = resolution.Endpoint ?? configuration.Endpoint; + var apiKey = resolution.ApiKey ?? configuration.ApiKey; + var source = resolution.Endpoint != null || resolution.ApiKey != null + ? resolution.Source : "IOptions"; + + if (string.IsNullOrEmpty(endpoint)) + throw new InvalidOperationException( + $"Endpoint is missing for model '{modelName}'. Configure a ModelProvider node (Model/AzureFoundry) or set AzureFoundry:Endpoint in config."); + + if (string.IsNullOrEmpty(apiKey)) + throw new InvalidOperationException( + $"ApiKey is missing for model '{modelName}'. Configure a ModelProvider node (Model/AzureFoundry) or set AzureFoundry:ApiKey in config."); + logger.LogInformation( - "Creating Azure Foundry chat client for agent {AgentName} using model {ModelName}", - agentConfig.Id, modelName); + "[AzureFoundry] Creating chat client agent={AgentName} model={ModelName} endpoint={Endpoint} source={Source} apiKeyFp={ApiKeyFingerprint}", + agentConfig.Id, modelName, endpoint, source, Fingerprint(apiKey)); try { var client = new ChatCompletionsClient( - new Uri(configuration.Endpoint), - new AzureKeyCredential(configuration.ApiKey)); + new Uri(endpoint), + new AzureKeyCredential(apiKey)); IChatClient chatClient = client.AsIChatClient(modelName); - logger.LogInformation( - "Successfully configured Azure Foundry chat client for agent {AgentName} with endpoint {Endpoint} and model {ModelName}", - agentConfig.Id, configuration.Endpoint, modelName); - return chatClient; } catch (Exception ex) @@ -77,4 +96,16 @@ protected override IChatClient CreateChatClient(AgentConfiguration agentConfig) $"Failed to create Azure Foundry chat client for agent {agentConfig.Id}: {ex.Message}", ex); } } + + /// + /// 8-char SHA-256-hex prefix of . Used in logs to + /// disambiguate "which key was actually used" without exposing the key. + /// + private static string Fingerprint(string? value) + { + if (string.IsNullOrEmpty(value)) return "(empty)"; + var bytes = System.Text.Encoding.UTF8.GetBytes(value); + var hash = System.Security.Cryptography.SHA256.HashData(bytes); + return Convert.ToHexString(hash, 0, 4).ToLowerInvariant(); + } } diff --git a/src/MeshWeaver.AI.AzureFoundry/AzureFoundryExtensions.cs b/src/MeshWeaver.AI.AzureFoundry/AzureFoundryExtensions.cs index 00a964213..465766614 100644 --- a/src/MeshWeaver.AI.AzureFoundry/AzureFoundryExtensions.cs +++ b/src/MeshWeaver.AI.AzureFoundry/AzureFoundryExtensions.cs @@ -1,12 +1,80 @@ +using System.Collections.Immutable; +using MeshWeaver.Mesh; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; namespace MeshWeaver.AI.AzureFoundry; /// -/// Extension methods for adding Azure AI Foundry services +/// Extension methods for adding Azure AI Foundry services. Each provider +/// self-registers its bootstrap profile via +/// +/// — there is no central registry. Callers opt in to each provider they +/// actually need. /// public static class AzureFoundryExtensions { + /// + /// One-call registration of Anthropic — catalog profile + IOptions + /// binding (Anthropic:) + the + /// . The same factory + /// serves direct api.anthropic.com AND Azure-hosted Anthropic; + /// the actual endpoint comes from the user's ModelProvider node + /// (or IOptions fallback for system defaults). Idempotent. + /// + public static TBuilder AddAnthropic(this TBuilder builder, string configSection = "Anthropic") + where TBuilder : MeshBuilder + { + builder.AddLanguageModelCatalogSource(new LanguageModelCatalogSource( + SectionName: configSection, + ProviderName: "Anthropic", + Order: 1, + DisplayLabel: "Anthropic", + DefaultEndpoint: "https://api.anthropic.com/v1/messages", + // Latest model PER CATEGORY (opus / sonnet / haiku) — NOT a growing pinned list. + // Anthropic exposes no "latest" alias and AzureClaudeChatClient sends the id raw, so a + // concrete id is required; this is the ONE place to bump when Anthropic ships a newer + // snapshot in a category. Saving an Anthropic key seeds exactly these (CreateProvider) → + // Opus 4.8 is the opus the key runs. + DefaultModelIds: ImmutableArray.Create( + "claude-opus-4-8", + "claude-sonnet-4-6", + "claude-haiku-4-5-20251001"), + RequiresApiKey: true)); + builder.ConfigureServices(services => + { + services.AddOptions().BindConfiguration(configSection); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + return services; + }); + return builder; + } + + /// + /// One-call registration of Azure Foundry multi-model gateway — + /// catalog profile + IOptions binding (AzureFoundry:) + + /// . Idempotent. + /// + public static TBuilder AddAzureFoundry(this TBuilder builder, string configSection = "AzureFoundry") + where TBuilder : MeshBuilder + { + builder.AddLanguageModelCatalogSource(new LanguageModelCatalogSource( + SectionName: configSection, + ProviderName: "AzureFoundry", + Order: 2, + DisplayLabel: "Azure Foundry", + DefaultEndpoint: null, + DefaultModelIds: ImmutableArray.Empty, + RequiresApiKey: true)); + builder.ConfigureServices(services => + { + services.AddOptions().BindConfiguration(configSection); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + return services; + }); + return builder; + } + /// /// Adds Azure AI Foundry Claude services. /// Configuration should be bound to AzureClaudeConfiguration. diff --git a/src/MeshWeaver.AI.AzureFoundry/AzureFoundryPersistentAgentFactory.cs b/src/MeshWeaver.AI.AzureFoundry/AzureFoundryPersistentAgentFactory.cs index 89208f87f..a72d0fa6a 100644 --- a/src/MeshWeaver.AI.AzureFoundry/AzureFoundryPersistentAgentFactory.cs +++ b/src/MeshWeaver.AI.AzureFoundry/AzureFoundryPersistentAgentFactory.cs @@ -71,12 +71,11 @@ public async Task CreateAgentAsync( { var client = GetOrCreateClient(); + // Model comes from the chat composer selection. var model = !string.IsNullOrEmpty(modelName) ? modelName - : !string.IsNullOrEmpty(config.PreferredModel) - ? config.PreferredModel - : configuration.Models.FirstOrDefault() - ?? throw new InvalidOperationException("No model configured"); + : configuration.Models.FirstOrDefault() + ?? throw new InvalidOperationException("No model configured"); var instructions = GetAgentInstructions(config, hierarchyAgents); var name = config.Id; diff --git a/src/MeshWeaver.AI.AzureOpenAI/AzureOpenAIExtensions.cs b/src/MeshWeaver.AI.AzureOpenAI/AzureOpenAIExtensions.cs deleted file mode 100644 index ca58c5243..000000000 --- a/src/MeshWeaver.AI.AzureOpenAI/AzureOpenAIExtensions.cs +++ /dev/null @@ -1,28 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; - -namespace MeshWeaver.AI.AzureOpenAI; - -/// -/// Extension methods for adding Azure OpenAI services -/// -public static class AzureOpenAIExtensions -{ - /// - /// Adds Azure OpenAI services to the service collection - /// - public static IServiceCollection AddAzureOpenAI(this IServiceCollection services) - { - return services.AddSingleton(); - } - - /// - /// Adds Azure OpenAI services with configuration action. - /// - public static IServiceCollection AddAzureOpenAI( - this IServiceCollection services, - Action configure) - { - services.Configure(configure); - return services.AddSingleton(); - } -} diff --git a/src/MeshWeaver.AI.AzureOpenAI/README.md b/src/MeshWeaver.AI.AzureOpenAI/README.md deleted file mode 100644 index 8e964d371..000000000 --- a/src/MeshWeaver.AI.AzureOpenAI/README.md +++ /dev/null @@ -1,146 +0,0 @@ -# MeshWeaver.AI.AzureOpenAI - -## Overview - -MeshWeaver.AI.AzureOpenAI provides Azure OpenAI integration for the MeshWeaver AI framework, enabling AI-powered agent chats using Azure OpenAI's ChatCompletionAgent. This library is designed for stateless chat completion scenarios without persistent assistant storage. - -## Features - -- **Azure OpenAI Integration**: Direct integration with Azure OpenAI services -- **ChatCompletionAgent Support**: Uses Microsoft Semantic Kernel's ChatCompletionAgent for stateless operations -- **Factory Pattern**: Implements the factory pattern for creating and managing agent chats -- **Configuration-Based Setup**: Uses `AIConfiguration` for secure credential management -- **Extensible Architecture**: Built on top of MeshWeaver.AI's `ChatCompletionAgentChatFactory` base class - -## Installation - -This package is part of the MeshWeaver solution and should be referenced as a project dependency: - -```xml - -``` - -## Configuration - -### 1. Configure AI Credentials - -Add the following configuration to your `appsettings.json`: - -```json -{ - "AI": { - "Url": "https://your-azure-openai-endpoint.openai.azure.com/", - "ApiKey": "your-api-key-here", - "Models": ["gpt-4", "gpt-35-turbo"] - } -} -``` - -### 2. Register Services - -In your `Program.cs` or service configuration: - -```csharp -using MeshWeaver.AI.AzureOpenAI; - -// Configure AI credentials -builder.Services.Configure( - builder.Configuration.GetSection("AI")); - -// Add Azure OpenAI services -builder.Services.AddAzureOpenAI(); -``` - -## Usage - -### Basic Implementation - -```csharp -public class MyService -{ - private readonly IAgentChatFactory _chatFactory; - - public MyService(IAgentChatFactory chatFactory) - { - _chatFactory = chatFactory; - } - - public async Task CreateChatAsync() - { - var agentChat = await _chatFactory.CreateAsync(); - return agentChat; - } -} -``` - -### Agent Definition - -Create custom agents by implementing `IAgentDefinition`: - -```csharp -public class MyCustomAgent : IAgentDefinition -{ - public string Name => "MyAgent"; - public string Description => "A custom AI agent for specific tasks"; - public string Instructions => "You are a helpful assistant specialized in..."; -} -``` - -## Architecture - -### Class Hierarchy - -``` -ChatCompletionAgentChatFactory (MeshWeaver.AI) -└── AzureOpenAIChatCompletionAgentChatFactory (MeshWeaver.AI.AzureOpenAI) -``` - -### Key Components - -- **AzureOpenAIChatCompletionAgentChatFactory**: Main factory class for creating Azure OpenAI-powered agent chats -- **AzureOpenAIExtensions**: Extension methods for service registration -- **AIConfiguration**: Configuration model for Azure OpenAI credentials - -## Security Considerations - -- **API Key Management**: Store API keys securely using Azure Key Vault or similar secure storage -- **Environment Variables**: Consider using environment variables for sensitive configuration -- **Network Security**: Ensure secure communication with Azure OpenAI endpoints - -## Dependencies - -- Microsoft.SemanticKernel -- Microsoft.Extensions.Options -- Microsoft.Extensions.DependencyInjection.Abstractions -- MeshWeaver.AI -- MeshWeaver.Messaging.Hub - -## Troubleshooting - -### Common Issues - -1. **Missing API Key**: Ensure `AIConfiguration.ApiKey` is properly configured -2. **Invalid Endpoint**: Verify the `AIConfiguration.Url` format -3. **Model Not Available**: Check that the specified models exist in your Azure OpenAI deployment - -### Logging - -Enable detailed logging to troubleshoot issues: - -```csharp -builder.Services.AddLogging(logging => -{ - logging.AddConsole(); - logging.SetMinimumLevel(LogLevel.Debug); -}); -``` - -## Related Projects - -- [MeshWeaver.AI](../MeshWeaver.AI/README.md) - Core AI services and abstractions -- [MeshWeaver.AI.AzureFoundry](../MeshWeaver.AI.AzureFoundry/README.md) - Azure AI Foundry integration -- [MeshWeaver.Blazor.Chat](../MeshWeaver.Blazor.Chat/README.md) - Chat UI components - -## Contributing - -This project is part of the MeshWeaver ecosystem. Please follow the established patterns and conventions when contributing. diff --git a/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeChatClient.cs b/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeChatClient.cs index 39dcd556e..cb45622c1 100644 --- a/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeChatClient.cs +++ b/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeChatClient.cs @@ -1,6 +1,9 @@ +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; using System.Runtime.CompilerServices; using System.Text; using ClaudeAgentSdk; +using MeshWeaver.AI.Connect; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; @@ -14,16 +17,39 @@ public class ClaudeCodeChatClient : IChatClient { private readonly ClaudeCodeConfiguration configuration; private readonly string? modelName; + private readonly string? configDir; + private readonly string? oauthToken; + private readonly IMcpBackConnection? mcpBackConnection; + private readonly string? userId; + private readonly string? userName; + private readonly string? userEmail; private readonly ILogger? logger; public ClaudeCodeChatClient( ClaudeCodeConfiguration configuration, string? modelName = null, - ILogger? logger = null) + ILogger? logger = null, + string? configDir = null, + string? oauthToken = null, + IMcpBackConnection? mcpBackConnection = null, + string? userId = null, + string? userName = null, + string? userEmail = null) { this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); this.modelName = modelName; this.logger = logger; + // Per-user isolation for co-hosted multi-user (Phase 5b): each spawn + // gets the calling user's own .claude (CLAUDE_CONFIG_DIR) + subscription + // token (CLAUDE_CODE_OAUTH_TOKEN), so concurrent users on one portal + // replica never share credentials or session state. + this.configDir = configDir; + this.oauthToken = oauthToken; + // Automatic MCP back-connection (the mesh is the CLI's workspace) — resolved per spawn. + this.mcpBackConnection = mcpBackConnection; + this.userId = userId; + this.userName = userName; + this.userEmail = userEmail; } /// @@ -78,6 +104,24 @@ public async IAsyncEnumerable GetStreamingResponseAsync( var messageList = messages.ToList(); var userPrompt = BuildPromptFromMessages(messageList); + // Co-hosted multi-user: when this user runs under their OWN per-user config dir, a missing + // subscription token AND missing .credentials.json means they've never logged in. Surface an + // actionable auth error (ThreadExecution turns it into a "/login" affordance) instead of + // letting the CLI fail later with a cryptic "Not logged in · Please run /login" → + // ProcessException exit 1. (configDir is null only in single-user dev, where the CLI uses the + // machine's own login — so we don't pre-empt there.) + // The token comes from the node-backed resolver (oauthToken); when its cache is cold after a + // reload / pod restart, fall back to the token persisted in .credentials.json on the shared + // config-dir volume — so we still set CLAUDE_CODE_OAUTH_TOKEN and authenticate instead of + // surfacing a spurious "Not logged in". (Sync file read; this is the off-hub SDK leaf.) + var effectiveToken = !string.IsNullOrEmpty(oauthToken) ? oauthToken : ReadCredentialsToken(configDir); + if (!string.IsNullOrEmpty(configDir) && string.IsNullOrEmpty(effectiveToken)) + { + throw new AuthRequiredException( + Harnesses.ClaudeCode, + "Not logged in to Claude Code. Run /login to connect your Claude subscription."); + } + // If CliDirectory is specified, add it to PATH for CLI discovery // This must be done BEFORE calling the SDK as FindCli() checks PATH if (!string.IsNullOrEmpty(configuration.CliDirectory)) @@ -92,18 +136,70 @@ public async IAsyncEnumerable GetStreamingResponseAsync( } // Build options - var claudeOptions = new ClaudeAgentOptions + var env = new Dictionary + { + ["PYTHONUTF8"] = "1", // Python UTF-8 mode + ["PYTHONIOENCODING"] = "utf-8", // Python I/O encoding + ["LANG"] = "en_US.UTF-8", // Unix locale + ["LC_ALL"] = "en_US.UTF-8", // Unix locale override + ["CHCP"] = "65001" // Windows code page hint + }; + // Per-user isolation: run the CLI under this user's own .claude + token. + if (!string.IsNullOrEmpty(configDir)) + { + env["CLAUDE_CONFIG_DIR"] = configDir; + try { Directory.CreateDirectory(configDir); } + catch (Exception ex) { logger?.LogWarning(ex, "Could not create CLAUDE_CONFIG_DIR {Dir}", configDir); } + } + if (!string.IsNullOrEmpty(effectiveToken)) + env["CLAUDE_CODE_OAUTH_TOKEN"] = effectiveToken; + + var claudeOptions = new ClaudeAgentOptions { Env = env }; + + // The shared, sync-maintained WORKSPACE ({SkillsDirectory}) holds .claude/skills//SKILL.md + // (the MeshWeaver agents + skills) plus a base AGENTS.md (read by Claude Code AND Copilot — no + // CLAUDE.md duplicate) telling the agent the mesh is reachable via the meshweaver MCP server. + // Point the session's Cwd at it and load PROJECT scope so the CLI discovers those; USER scope + // keeps the per-user config/creds. The sync service (AgentSkillSyncService) is the single writer. + if (!string.IsNullOrEmpty(configuration.SkillsDirectory)) + { + claudeOptions.Cwd = configuration.SkillsDirectory; + claudeOptions.SettingSources = new List { SettingSource.User, SettingSource.Project }; + } + + // Automatic MCP back-connection — the mesh is this CLI's workspace (no file tree). Inject a + // per-spawn HTTP MCP server at the portal's /mcp, authenticated AS THE USER via a Bearer + // token (minted/reused on demand by IMcpBackConnection). In-memory only — the token is + // never written to the config dir / Azure Files share. + if (mcpBackConnection != null && !string.IsNullOrEmpty(userId)) { - // Set UTF-8 encoding environment variables for proper character handling on Windows - Env = new Dictionary + McpConnectionInfo? mcpInfo = null; + try { - ["PYTHONUTF8"] = "1", // Python UTF-8 mode - ["PYTHONIOENCODING"] = "utf-8", // Python I/O encoding - ["LANG"] = "en_US.UTF-8", // Unix locale - ["LC_ALL"] = "en_US.UTF-8", // Unix locale override - ["CHCP"] = "65001" // Windows code page hint + mcpInfo = await mcpBackConnection.EnsureForUser(userId, userName, userEmail) + .FirstOrDefaultAsync().ToTask(cancellationToken); } - }; + catch (Exception ex) + { + logger?.LogWarning(ex, "Could not provision the MCP back-connection; Claude Code will run without mesh access."); + } + if (mcpInfo != null) + { + claudeOptions.McpServers = new Dictionary + { + ["meshweaver"] = new McpHttpServerConfig + { + Type = "http", + Url = mcpInfo.McpUrl, + Headers = new Dictionary + { + ["Authorization"] = $"Bearer {mcpInfo.BearerToken}" + } + } + }; + logger?.LogInformation("Claude Code MCP workspace wired to {McpUrl}", mcpInfo.McpUrl); + } + } if (!string.IsNullOrEmpty(modelName)) { @@ -122,7 +218,7 @@ public async IAsyncEnumerable GetStreamingResponseAsync( claudeOptions.MaxTurns = configuration.MaxTurns.Value; } - if (!string.IsNullOrEmpty(configuration.WorkingDirectory)) + if (string.IsNullOrEmpty(claudeOptions.Cwd) && !string.IsNullOrEmpty(configuration.WorkingDirectory)) { claudeOptions.Cwd = configuration.WorkingDirectory; } @@ -143,63 +239,96 @@ public async IAsyncEnumerable GetStreamingResponseAsync( using var timeoutCts = new CancellationTokenSource(configuration.SessionTimeoutMs); using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCts.Token); - // Use the static QueryAsync method for streaming - await foreach (var message in ClaudeAgent.QueryAsync(userPrompt, claudeOptions).WithCancellation(linkedCts.Token)) + // Manually enumerate so we survive the Claude Agent SDK 0.2.1 bug where + // MessageParser throws ArgumentException("Unknown message type: …") on the + // newer CLI's informational events (e.g. rate_limit_event) — which would + // otherwise crash the whole chat. 0.2.1 is the LATEST SDK, so this wrapper + // is the only fix until Anthropic ships one (claude-agent-sdk #583/#599/ + // #601/#603, claude-code #26498). The inner try/catch wraps MoveNextAsync + // only (no yield inside it); the outer try/finally allows yield. + var enumerator = ClaudeAgent.QueryAsync(userPrompt, claudeOptions) + .GetAsyncEnumerator(linkedCts.Token); + var yieldedAny = false; + var swallowedUnknown = false; + try { - // Process different message types - switch (message) + while (true) { - case AssistantMessage assistantMessage: - foreach (var block in assistantMessage.Content) - { - switch (block) + bool moved; + try + { + moved = await enumerator.MoveNextAsync(); + } + catch (ArgumentException ex) when ( + ex.Message.Contains("Unknown message type", StringComparison.OrdinalIgnoreCase)) + { + logger?.LogWarning(ex, + "Claude Agent SDK could not parse a CLI message — ending stream gracefully (SDK 0.2.1 bug, e.g. rate_limit_event)."); + swallowedUnknown = true; + break; + } + if (!moved) break; + + var message = enumerator.Current; + switch (message) + { + case AssistantMessage assistantMessage: + foreach (var block in assistantMessage.Content) { - case TextBlock textBlock: - if (!string.IsNullOrEmpty(textBlock.Text)) - { - yield return new ChatResponseUpdate(ChatRole.Assistant, textBlock.Text); - } - break; - - case ToolUseBlock toolUseBlock: - // Convert tool use to FunctionCallContent - var toolId = toolUseBlock.Id ?? Guid.NewGuid().ToString(); - IDictionary? arguments = null; - - if (toolUseBlock.Input != null && toolUseBlock.Input.Count > 0) - { - // Input is already a Dictionary, convert to nullable version - arguments = toolUseBlock.Input.ToDictionary( - kvp => kvp.Key, - kvp => (object?)kvp.Value); - } - - var functionCall = new FunctionCallContent(toolId, toolUseBlock.Name ?? "unknown", arguments); - yield return new ChatResponseUpdate(ChatRole.Assistant, [functionCall]); - break; - - case ThinkingBlock thinkingBlock: - // Optionally expose thinking as a special content type or log it - logger?.LogDebug("Claude thinking: {Thinking}", thinkingBlock.Thinking); - break; + switch (block) + { + case TextBlock textBlock: + if (!string.IsNullOrEmpty(textBlock.Text)) + { + yieldedAny = true; + yield return new ChatResponseUpdate(ChatRole.Assistant, textBlock.Text); + } + break; + + case ToolUseBlock toolUseBlock: + var toolId = toolUseBlock.Id ?? Guid.NewGuid().ToString(); + IDictionary? arguments = null; + if (toolUseBlock.Input != null && toolUseBlock.Input.Count > 0) + { + arguments = toolUseBlock.Input.ToDictionary( + kvp => kvp.Key, kvp => (object?)kvp.Value); + } + yieldedAny = true; + yield return new ChatResponseUpdate(ChatRole.Assistant, + [new FunctionCallContent(toolId, toolUseBlock.Name ?? "unknown", arguments)]); + break; + + case ThinkingBlock thinkingBlock: + logger?.LogDebug("Claude thinking: {Thinking}", thinkingBlock.Thinking); + break; + } } - } - break; + break; - case SystemMessage systemMessage: - // System messages from the SDK (not the LLM) - logger?.LogDebug("Claude system message: {Subtype}", systemMessage.Subtype); - break; + case SystemMessage systemMessage: + logger?.LogDebug("Claude system message: {Subtype}", systemMessage.Subtype); + break; - case ResultMessage resultMessage: - // Query completed - log cost/duration info - logger?.LogInformation( - "Claude query completed. Duration: {Duration}ms, Cost: ${Cost:F4}", - resultMessage.DurationMs, - resultMessage.TotalCostUsd ?? 0.0); - break; + case ResultMessage resultMessage: + logger?.LogInformation( + "Claude query completed. Duration: {Duration}ms, Cost: ${Cost:F4}", + resultMessage.DurationMs, resultMessage.TotalCostUsd ?? 0.0); + break; + } } } + finally + { + await enumerator.DisposeAsync(); + } + + // If the only thing that stopped us was the unparseable event and we never + // produced output, surface a graceful note instead of a silent empty reply. + if (swallowedUnknown && !yieldedAny) + { + yield return new ChatResponseUpdate(ChatRole.Assistant, + "Claude Code produced no output before the session ended (often a transient rate limit). Please retry shortly."); + } } /// @@ -286,6 +415,57 @@ private static string GetTextContent(ChatMessage message) return string.Join("", textParts); } + /// + /// True when the per-user config dir holds a non-empty .credentials.json (the CLI's + /// persisted OAuth login). A cheap file probe that mirrors + /// ClaudeConnectStrategy.IsLoggedIn — the Connect flow writes this file into the same dir. + /// + private static bool HasCredentials(string configDir) + { + try + { + var creds = Path.Combine(configDir, ".credentials.json"); + return File.Exists(creds) && new FileInfo(creds).Length > 2; + } + catch + { + return false; + } + } + + /// Reads the OAuth token back from {configDir}/.credentials.json (Claude shape + /// claudeAiOauth.accessToken, or any nested access-token key) — the persistent fallback + /// when the node-backed resolver cache is cold after a reload. Null on any miss. + private static string? ReadCredentialsToken(string? configDir) + { + if (string.IsNullOrEmpty(configDir)) return null; + try + { + var path = Path.Combine(configDir, ".credentials.json"); + if (!File.Exists(path)) return null; + using var doc = System.Text.Json.JsonDocument.Parse(File.ReadAllText(path)); + return ExtractToken(doc.RootElement); + } + catch { return null; } + } + + private static string? ExtractToken(System.Text.Json.JsonElement el) + { + foreach (var name in new[] { "accessToken", "access_token", "token", "oauthToken" }) + { + if (el.ValueKind == System.Text.Json.JsonValueKind.Object && el.TryGetProperty(name, out var v) + && v.ValueKind == System.Text.Json.JsonValueKind.String && !string.IsNullOrEmpty(v.GetString())) + return v.GetString(); + } + if (el.ValueKind == System.Text.Json.JsonValueKind.Object) + foreach (var prop in el.EnumerateObject()) + { + var found = ExtractToken(prop.Value); + if (!string.IsNullOrEmpty(found)) return found; + } + return null; + } + private static readonly object PathLock = new(); private static bool _pathModified; diff --git a/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeChatClientAgentFactory.cs b/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeChatClientAgentFactory.cs deleted file mode 100644 index 17dc13cfa..000000000 --- a/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeChatClientAgentFactory.cs +++ /dev/null @@ -1,55 +0,0 @@ -using MeshWeaver.Messaging; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace MeshWeaver.AI.ClaudeCode; - -/// -/// Factory for creating ChatClientAgent instances with Claude Code (Claude Agent SDK). -/// Requires Claude Code CLI >= 2.0.0 installed. -/// -public class ClaudeCodeChatClientAgentFactory( - IMessageHub hub, - IOptions options, - ILogger logger) - : ChatClientAgentFactory(hub) -{ - private readonly ClaudeCodeConfiguration configuration = options.Value ?? new ClaudeCodeConfiguration(); - - public override string Name => "Claude Code"; - - public override IReadOnlyList Models => configuration.Models; - - public override int Order => configuration.Order; - - protected override IChatClient CreateChatClient(AgentConfiguration agentConfig) - { - // Use CurrentModelName if set, fall back to agent's preferred model, otherwise use first configured model - var modelName = !string.IsNullOrEmpty(CurrentModelName) ? CurrentModelName - : !string.IsNullOrEmpty(agentConfig.PreferredModel) ? agentConfig.PreferredModel - : configuration.Models.FirstOrDefault(); - - logger.LogInformation( - "Creating Claude Code chat client for agent {AgentName} using model {ModelName}", - agentConfig.Id, modelName); - - try - { - var clientLogger = Hub.ServiceProvider.GetService(typeof(ILogger)) as ILogger; - var chatClient = new ClaudeCodeChatClient(configuration, modelName, clientLogger); - - logger.LogInformation( - "Successfully configured Claude Code chat client for agent {AgentName}", - agentConfig.Id); - - return chatClient; - } - catch (Exception ex) - { - logger.LogError(ex, "Failed to create Claude Code chat client for agent {AgentName}", agentConfig.Id); - throw new InvalidOperationException( - $"Failed to create Claude Code chat client for agent {agentConfig.Id}: {ex.Message}", ex); - } - } -} diff --git a/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeConfiguration.cs b/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeConfiguration.cs index 623d42bb4..e117f7a9e 100644 --- a/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeConfiguration.cs +++ b/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeConfiguration.cs @@ -44,4 +44,22 @@ public class ClaudeCodeConfiguration /// Session timeout in milliseconds. /// public int SessionTimeoutMs { get; set; } = 120000; + + /// + /// Root directory for per-user .claude config dirs on the shared + /// volume (Azure Files), e.g. /mnt/users. Claude Code is co-hosted in + /// the portal; each spawn runs with CLAUDE_CONFIG_DIR = + /// {ConfigDirRoot}/{userId}/.claude so concurrent users' credentials / + /// session state are isolated and survive across portal replicas. Null ⇒ the + /// portal's container default (single-user dev). + /// + public string? ConfigDirRoot { get; set; } + + /// + /// The shared on-disk skills directory maintained by the agent→skill sync service. When set, the + /// harness links it into each user's CLAUDE_CONFIG_DIR/skills (symlink) and enables the + /// user setting source so the CLI discovers the MeshWeaver agents as skills. Null ⇒ no skill + /// linking (the sync is disabled / not configured). + /// + public string? SkillsDirectory { get; set; } } diff --git a/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeExtensions.cs b/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeExtensions.cs index 7721cfc23..cda7097e8 100644 --- a/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeExtensions.cs +++ b/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeExtensions.cs @@ -1,4 +1,8 @@ +using System.Collections.Immutable; +using MeshWeaver.AI; +using MeshWeaver.Mesh; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; namespace MeshWeaver.AI.ClaudeCode; @@ -8,24 +12,39 @@ namespace MeshWeaver.AI.ClaudeCode; public static class ClaudeCodeExtensions { /// - /// Adds Claude Code services to the service collection. - /// Configuration should be bound to ClaudeCodeConfiguration. - /// Requires Claude Code CLI >= 2.0.0 installed via: npm install -g @anthropic-ai/claude-code + /// Claude Code is a harness, not a model provider — so this no longer + /// registers a language-model catalog source. The harness surfaces as a + /// Harness catalog node (see ) and is wired + /// via . + /// Retained as a no-op so existing builder chains keep compiling. + /// + public static TBuilder AddClaudeCode(this TBuilder builder) + where TBuilder : MeshBuilder + => builder; + + /// + /// Registers Claude Code as a harness (NOT a model provider): the + /// runs the claude CLI directly via the + /// Claude Agent SDK. projects it into + /// a catalog node; dispatches the round to it. + /// Requires Claude Code CLI >= 2.0.0 (npm install -g @anthropic-ai/claude-code). /// public static IServiceCollection AddClaudeCode(this IServiceCollection services) { - return services.AddSingleton(); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + return services; } /// - /// Adds Claude Code services with configuration action. - /// Requires Claude Code CLI >= 2.0.0 installed via: npm install -g @anthropic-ai/claude-code + /// Registers the Claude Code harness with a configuration action that binds + /// . See . /// public static IServiceCollection AddClaudeCode( this IServiceCollection services, Action configure) { services.Configure(configure); - return services.AddSingleton(); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + return services; } } diff --git a/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeHarness.cs b/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeHarness.cs new file mode 100644 index 000000000..26cd83486 --- /dev/null +++ b/src/MeshWeaver.AI.ClaudeCode/ClaudeCodeHarness.cs @@ -0,0 +1,78 @@ +using MeshWeaver.AI.Connect; +using MeshWeaver.Messaging; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace MeshWeaver.AI.ClaudeCode; + +/// +/// The Claude Code harness — runs the claude CLI through the Claude +/// Agent SDK (). It is a harness, NOT a model +/// provider: selecting it dispatches the round straight to the CLI library, bypassing +/// the model-provider factory chain. Registered from this assembly via +/// AddClaudeCode. +/// +public sealed class ClaudeCodeHarness(IOptions options) : IHarness +{ + private readonly ClaudeCodeConfiguration configuration = options.Value ?? new ClaudeCodeConfiguration(); + + public string Id => Harnesses.ClaudeCode; + + public Harness Definition => new() + { + Id = Harnesses.ClaudeCode, + DisplayName = "Claude Code", + Description = "Runs the Claude Code CLI (Claude Agent SDK).", + // Inline SVG (single-quoted attrs) — travels WITH the node; no /static file, embed glob or + // icon-allowlist plumbing. The renderer treats an Icon starting with '<' as raw markup. + Icon = "", + Order = 1, + SupportsAgentSelection = false + }; + + // Claude Code owns its auth slash-commands: /login (re)authenticates the user's Claude + // subscription via the Connect flow; /logout forgets the stored token. When this harness is the + // active one, these REPLACE MeshWeaver's /agent /model in the chat autocomplete + dispatch. + public IReadOnlyList Commands { get; } = + [ + new("login", "Log in to your Claude subscription", HarnessCommandKind.Connect), + new("logout", "Log out of Claude Code", HarnessCommandKind.Disconnect), + ]; + + public Connect.ConnectProvider? AuthProvider => Connect.ConnectProvider.ClaudeCode; + + public IChatClient? CreateChatClient(HarnessExecutionContext context) + { + var hub = context.Hub; + var accessCtx = hub.ServiceProvider.GetService()?.Context; + var userId = accessCtx?.ObjectId; + + // 🚫 NEVER pass a model to the CLI. Claude Code runs the user's OWN subscription and picks + // its default model; forwarding the MeshWeaver composer's selected model (e.g. + // "DeepSeek-V3-0324") makes the `claude` CLI fail outright. The harness surfaces no model + // selection (SupportsAgentSelection = false), so there is nothing to forward. + // + // 🔑 The token is the user's per-user Connect (subscription) token, NOT a selected model's + // API key — resolve it from the user's ClaudeCode provider node. Best-effort: the CLI also + // reads its own .credentials.json from the per-user CLAUDE_CONFIG_DIR on the shared volume, + // so an absent token simply means "rely on the config dir" (and Connect/login can populate it). + var resolver = hub.ServiceProvider.GetService(); + var token = resolver?.ResolveConnectToken(Harnesses.ClaudeCode, userId); + + var root = configuration.ConfigDirRoot?.TrimEnd('/'); + var configDir = !string.IsNullOrEmpty(root) && !string.IsNullOrEmpty(userId) + ? $"{root}/{userId}/.claude" + : null; + var mcp = hub.ServiceProvider.GetService(); + var clientLogger = hub.ServiceProvider.GetService>(); + + // Note: the MeshWeaver agents are materialised as Claude Code skills by the reactive + // AgentSkillSyncService (shared dir), which the client LINKS into this user's config dir + + // enables the "user" setting source. No per-spawn skill writing here. + return new ClaudeCodeChatClient( + configuration, modelName: null, clientLogger, configDir, token, + mcp, userId, accessCtx?.Name, accessCtx?.Email); + } +} diff --git a/src/MeshWeaver.AI.Copilot/AgentSkill.cs b/src/MeshWeaver.AI.Copilot/AgentSkill.cs new file mode 100644 index 000000000..db913c406 --- /dev/null +++ b/src/MeshWeaver.AI.Copilot/AgentSkill.cs @@ -0,0 +1,10 @@ +namespace MeshWeaver.AI.Copilot; + +/// +/// A MeshWeaver agent projected for the GitHub Copilot harness — its id/name/description plus the +/// agent's instructions. The Copilot CLI/SDK has no filesystem "skills" folder (unlike Claude Code), +/// so these are injected into the Copilot session's system message rather than written as files. +/// Built by from the live agent registry and consumed by +/// . +/// +public sealed record AgentSkill(string Id, string Name, string? Description, string Instructions); diff --git a/src/MeshWeaver.AI.Copilot/CopilotChatClient.cs b/src/MeshWeaver.AI.Copilot/CopilotChatClient.cs index d4afe26e9..ddc0460e5 100644 --- a/src/MeshWeaver.AI.Copilot/CopilotChatClient.cs +++ b/src/MeshWeaver.AI.Copilot/CopilotChatClient.cs @@ -1,6 +1,11 @@ +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; using System.Runtime.CompilerServices; using System.Threading.Channels; using GitHub.Copilot.SDK; +using MeshWeaver.AI.Connect; +using MeshWeaver.Mesh.Threading; +using MeshWeaver.Reactive; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; @@ -14,19 +19,52 @@ public class CopilotChatClient : IChatClient, IAsyncDisposable { private readonly CopilotConfiguration configuration; private readonly string? modelName; + private readonly string? githubToken; + // The user's selectable MeshWeaver agents — injected into the Copilot session's system message + // (a guaranteed path; the SDK also discovers the workspace skills via the working dir). Resolved per session. + private readonly IObservable>? agentSkills; + // Automatic MCP back-connection — the mesh is this CLI's workspace. Resolved per session to the + // per-user `meshweaver` HTTP MCP server (Bearer-authenticated as the calling user). + private readonly IMcpBackConnection? mcpBackConnection; + private readonly string? userId; + private readonly string? userName; + private readonly string? userEmail; private readonly ILogger? logger; - private CopilotClient? copilotClient; + // Genuine IO (subprocess CLI spawn + SDK network round-trips) runs off the hub scheduler, + // bounded, through the Http pool — the ControlledIoPooling boundary. Everything ABOVE the + // pool is IObservable; the only async/await lives inside the leaves the pool owns. + private readonly IIoPool ioPool; + // The connect/start handshake is a one-shot resource: the promise-cache runs it at most once and + // replays to every later caller (pool.Run is eager + ReplaySubject-backed). Built once via an + // atomic ref swap (NOT a lock-for-async) — replaces the former SemaphoreSlim clientLock. + private IObservable? connectPromise; private bool disposed; - private readonly SemaphoreSlim clientLock = new(1, 1); public CopilotChatClient( CopilotConfiguration configuration, string? modelName = null, - ILogger? logger = null) + ILogger? logger = null, + string? githubToken = null, + IIoPool? ioPool = null, + IObservable>? agentSkills = null, + IMcpBackConnection? mcpBackConnection = null, + string? userId = null, + string? userName = null, + string? userEmail = null) { this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); this.modelName = modelName; this.logger = logger; + // Per-user auth pass-through (co-hosted multi-user): the calling user's + // GitHub token. When null, the CLI uses the machine's logged-in user + // (single-user dev / ambient auth). + this.githubToken = githubToken; + this.ioPool = ioPool ?? IoPool.Unbounded; + this.agentSkills = agentSkills; + this.mcpBackConnection = mcpBackConnection; + this.userId = userId; + this.userName = userName; + this.userEmail = userEmail; } /// @@ -41,6 +79,8 @@ public async Task GetResponseAsync( var contents = new List(); var allText = new System.Text.StringBuilder(); + // M.E.AI contract surface: the only await/await-foreach the framework mandates. It consumes + // the public streaming method, which is itself pure IObservable bridged to IAsyncEnumerable. await foreach (var update in GetStreamingResponseAsync(messages, options, cancellationToken)) { if (update.Text != null) @@ -72,17 +112,156 @@ public async Task GetResponseAsync( } /// - public async IAsyncEnumerable GetStreamingResponseAsync( + public IAsyncEnumerable GetStreamingResponseAsync( IEnumerable messages, ChatOptions? options = null, - [EnumeratorCancellation] CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) + // IObservable up to the M.E.AI boundary; the framework IObservable→IAsyncEnumerable bridge + // (ToAsyncEnumerableSequence) is the single sanctioned conversion — no hand-rolled .ToTask(). + => BuildResponseStream(messages.ToList(), options).ToAsyncEnumerableSequence(cancellationToken); + + /// + /// The whole round as a cold : the connect promise (pooled, replayed) + /// composed into the session stream (the SDK IO leaf, run through ). + /// No await, no .ToTask() — the only async/await is sealed inside the pooled leaf. + /// + private IObservable BuildResponseStream( + IReadOnlyList messages, ChatOptions? options) + => GetOrCreateConnectPromise() + .SelectMany(client => ioPool.InvokeStream(ct => StreamSessionAsync(client, messages, options, ct))); + + /// + public object? GetService(Type serviceType, object? serviceKey = null) { - var client = await GetOrCreateClientAsync(cancellationToken); + if (serviceType == typeof(IChatClient)) + return this; + return null; + } + + /// + public void Dispose() => TeardownReactive(); + + /// + public ValueTask DisposeAsync() + { + // Disposal is reactive (ControlledIoPooling → "Dispose() fires, the mesh drains"): kick off the + // StopAsync teardown on the I/O pool and return immediately. A synchronous Dispose() must NOT + // block on async work, and DisposeAsync must NOT bridge through .ToTask(); both fire-and-forget + // the pooled stop so the subprocess is torn down while no caller (possibly a hub turn) parks. + TeardownReactive(); + return ValueTask.CompletedTask; + } + + private void TeardownReactive() + { + if (disposed) + return; + disposed = true; + + var promise = Interlocked.Exchange(ref connectPromise, null); + if (promise is null) + return; + + // StopAsync is the IO leaf -> Http pool. Subscribe (cold-observable side effect) so the work + // actually runs; errors are logged, never thrown into a disposing scheduler. + promise + .SelectMany(client => ioPool.Invoke(_ => client.StopAsync())) + .Subscribe( + _ => { }, + ex => logger?.LogWarning(ex, "Copilot client teardown failed")); + } + + private IObservable GetOrCreateConnectPromise() + { + var promise = connectPromise; + if (promise is not null) + return promise; + + // pool.Run is eager (kicks the connect off on the Http pool NOW) + ReplaySubject-backed, so + // concurrent first callers all observe the single connection. CompareExchange publishes only + // if nobody else won the race; otherwise we drop ours and use theirs. + var candidate = ioPool.Run(StartClientAsync); + return Interlocked.CompareExchange(ref connectPromise, candidate, null) ?? candidate; + } + + // ── SDK / subprocess boundary: the leaves the I/O pool owns — the ONLY place async/await lives ── + + private async Task StartClientAsync(CancellationToken cancellationToken) + { + try + { + var clientOptions = new CopilotClientOptions + { + AutoStart = true + }; + + if (!string.IsNullOrEmpty(configuration.CliPath)) + { + clientOptions.CliPath = configuration.CliPath; + } + if (!string.IsNullOrEmpty(configuration.CliUrl)) + { + clientOptions.CliUrl = configuration.CliUrl; + } + + if (configuration.Port.HasValue) + { + clientOptions.Port = configuration.Port.Value; + } + + // Auth: a per-user GitHub token wins (co-hosted multi-user); otherwise + // fall back to the machine's logged-in user (dev / ambient auth). + if (!string.IsNullOrEmpty(githubToken)) + clientOptions.GitHubToken = githubToken; + else + clientOptions.UseLoggedInUser = true; + + var client = new CopilotClient(clientOptions); + await client.StartAsync(cancellationToken).ConfigureAwait(false); + + logger?.LogInformation("Copilot client started successfully"); + + return client; + } + catch (Exception ex) + { + // Not logged in / token rejected → actionable "/login" affordance (ThreadExecution renders + // it), not the generic "CLI not installed" message. + var msg = ex.Message ?? string.Empty; + if (msg.Contains("auth", StringComparison.OrdinalIgnoreCase) + || msg.Contains("login", StringComparison.OrdinalIgnoreCase) + || msg.Contains("unauthor", StringComparison.OrdinalIgnoreCase) + || msg.Contains("credential", StringComparison.OrdinalIgnoreCase) + || msg.Contains("not signed in", StringComparison.OrdinalIgnoreCase)) + { + logger?.LogWarning(ex, "Copilot not authenticated for this user"); + throw new AuthRequiredException( + Harnesses.Copilot, + "Not logged in to GitHub Copilot. Run /login to connect.", ex); + } + logger?.LogError(ex, "Failed to start Copilot client. Ensure the Copilot CLI is installed and available in PATH."); + throw new InvalidOperationException( + "Failed to start Copilot client. Ensure the Copilot CLI is installed and available in PATH. " + + "See: https://docs.github.com/en/copilot/managing-copilot/configure-personal-settings/installing-the-github-copilot-in-the-cli", ex); + } + } + + private async IAsyncEnumerable StreamSessionAsync( + CopilotClient client, + IReadOnlyList messages, + ChatOptions? options, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { var messageList = messages.ToList(); + // Resolve the user's selectable MeshWeaver agents (best-effort) → system message, AND the + // per-user `meshweaver` HTTP MCP server → the session, so Copilot reaches the mesh by default. + var agentsSection = await ResolveAgentsSectionAsync(cancellationToken); + var mcpServers = await ResolveMcpServersAsync(cancellationToken); + // Build session configuration, including system messages as SystemMessage - var sessionConfig = BuildSessionConfig(options, messageList); + var sessionConfig = BuildSessionConfig(options, messageList, agentsSection, mcpServers); var lastUserMessage = messageList.LastOrDefault(m => m.Role == ChatRole.User); var userPrompt = lastUserMessage != null ? GetTextContent(lastUserMessage) : string.Empty; @@ -201,135 +380,147 @@ await session.SendAsync(new MessageOptions } } - /// - public object? GetService(Type serviceType, object? serviceKey = null) - { - if (serviceType == typeof(IChatClient)) - return this; - return null; - } - - /// - public void Dispose() + private SessionConfig BuildSessionConfig( + ChatOptions? options, List? messages = null, string? agentsSection = null, + IDictionary? mcpServers = null) { - DisposeAsync().AsTask().GetAwaiter().GetResult(); - } - - /// - public async ValueTask DisposeAsync() - { - if (disposed) - return; + var config = new SessionConfig + { + Streaming = configuration.EnableStreaming + }; - disposed = true; + // Set model if specified + if (!string.IsNullOrEmpty(modelName)) + { + config.Model = modelName; + } - await clientLock.WaitAsync(); - try + // Extract system messages (agent instructions from ChatClientAgent) + the projected MeshWeaver + // agents section, and set them as the session SystemMessage. + var systemParts = messages? + .Where(m => m.Role == ChatRole.System) + .Select(GetTextContent) + .Where(t => !string.IsNullOrEmpty(t)) + .ToList() ?? new List(); + // When the mesh MCP server is wired, tell the agent it's there (guaranteed, regardless of file discovery). + if (mcpServers is { Count: > 0 }) + systemParts.Insert(0, + "The memex mesh is available through the `meshweaver` MCP server (wired automatically, " + + "authenticated as you). The mesh — not a local file tree — is your workspace: use the MCP " + + "tools to read and modify content."); + if (!string.IsNullOrWhiteSpace(agentsSection)) + systemParts.Add(agentsSection!); + + if (systemParts.Count > 0) { - if (copilotClient != null) + config.SystemMessage = new SystemMessageConfig { - await copilotClient.StopAsync(); - copilotClient = null; - } + Content = string.Join("\n\n", systemParts), + Mode = SystemMessageMode.Append + }; } - finally + + // Add tools if provided - the SDK accepts AIFunction from Microsoft.Extensions.AI.Abstractions + if (options?.Tools != null && options.Tools.Count > 0) { - clientLock.Release(); + config.Tools = options.Tools.OfType().ToList(); } - clientLock.Dispose(); + // The mesh: per-user `meshweaver` HTTP MCP server (Bearer-authenticated as the calling user). + if (mcpServers is { Count: > 0 }) + config.McpServers = mcpServers; + + // Point the session at the shared sync workspace (.claude/skills + AGENTS.md) and let the CLI + // discover its skills + instructions from the working directory. + if (!string.IsNullOrEmpty(configuration.SkillsDirectory)) + { + config.WorkingDirectory = configuration.SkillsDirectory; + config.EnableConfigDiscovery = true; + } + + // Headless: auto-approve tool/permission prompts so MCP + tool calls proceed without a TTY. + // (MCP tools are still scoped to the user's own permissions via the Bearer token.) + config.OnPermissionRequest = PermissionHandler.ApproveAll; + + return config; } - private async Task GetOrCreateClientAsync(CancellationToken cancellationToken) + /// + /// Resolves the per-user meshweaver HTTP MCP server (Bearer-authenticated as the calling + /// user) for the session, or null when unavailable. The mesh is this CLI's workspace. The await is + /// at the SDK boundary (inside the IIoPool stream leaf), never on a hub scheduler. + /// + private async Task?> ResolveMcpServersAsync(CancellationToken cancellationToken) { - if (copilotClient != null) - return copilotClient; - - await clientLock.WaitAsync(cancellationToken); + if (mcpBackConnection is null || string.IsNullOrEmpty(userId)) + return null; try { - if (copilotClient != null) - return copilotClient; - - var clientOptions = new CopilotClientOptions + var info = await mcpBackConnection.EnsureForUser(userId, userName, userEmail) + .FirstOrDefaultAsync().ToTask(cancellationToken); + if (info is null) + return null; + logger?.LogInformation("Copilot MCP workspace wired to {McpUrl}", info.McpUrl); + return new Dictionary { - AutoStart = true + ["meshweaver"] = new McpHttpServerConfig + { + Url = info.McpUrl, + Headers = new Dictionary { ["Authorization"] = $"Bearer {info.BearerToken}" }, + Tools = new List { "*" }, + } }; - - if (!string.IsNullOrEmpty(configuration.CliPath)) - { - clientOptions.CliPath = configuration.CliPath; - } - - if (!string.IsNullOrEmpty(configuration.CliUrl)) - { - clientOptions.CliUrl = configuration.CliUrl; - } - - if (configuration.Port.HasValue) - { - clientOptions.Port = configuration.Port.Value; - } - - copilotClient = new CopilotClient(clientOptions); - await copilotClient.StartAsync(cancellationToken); - - logger?.LogInformation("Copilot client started successfully"); - - return copilotClient; } catch (Exception ex) { - logger?.LogError(ex, "Failed to start Copilot client. Ensure the Copilot CLI is installed and available in PATH."); - throw new InvalidOperationException( - "Failed to start Copilot client. Ensure the Copilot CLI is installed and available in PATH. " + - "See: https://docs.github.com/en/copilot/managing-copilot/configure-personal-settings/installing-the-github-copilot-in-the-cli", ex); - } - finally - { - clientLock.Release(); + logger?.LogWarning(ex, "Could not provision the Copilot MCP back-connection; running without mesh access."); + return null; } } - private SessionConfig BuildSessionConfig(ChatOptions? options, List? messages = null) + /// + /// Resolves the projected MeshWeaver agents (best-effort, bounded) into a system-message section, + /// or null when none / unavailable. The await is at the SDK boundary (inside the IIoPool stream + /// leaf), never on a hub scheduler. + /// + private async Task ResolveAgentsSectionAsync(CancellationToken cancellationToken) { - var config = new SessionConfig - { - Streaming = configuration.EnableStreaming - }; - - // Set model if specified - if (!string.IsNullOrEmpty(modelName)) + if (agentSkills is null) + return null; + try { - config.Model = modelName; + var skills = await agentSkills + .Take(1).Timeout(TimeSpan.FromSeconds(5)) + .FirstOrDefaultAsync().ToTask(cancellationToken); + return BuildAgentsSection(skills); } - - // Extract system messages (agent instructions from ChatClientAgent) and set as SystemMessage - if (messages != null) + catch (Exception ex) { - var systemParts = messages - .Where(m => m.Role == ChatRole.System) - .Select(GetTextContent) - .Where(t => !string.IsNullOrEmpty(t)) - .ToList(); - - if (systemParts.Count > 0) - { - config.SystemMessage = new SystemMessageConfig - { - Content = string.Join("\n\n", systemParts), - Mode = SystemMessageMode.Append - }; - } + logger?.LogWarning(ex, "Could not project MeshWeaver agents for the Copilot session."); + return null; } + } - // Add tools if provided - the SDK accepts AIFunction from Microsoft.Extensions.AI.Abstractions - if (options?.Tools != null && options.Tools.Count > 0) + /// Renders the selectable MeshWeaver agents as a markdown system-message section. + private static string? BuildAgentsSection(IReadOnlyList? skills) + { + if (skills is null || skills.Count == 0) + return null; + var sb = new System.Text.StringBuilder(); + sb.AppendLine("# Available MeshWeaver agents"); + sb.AppendLine("Adopt the behavior of one of these MeshWeaver agents when the user's request matches it. Each has a name, a description, and instructions:"); + foreach (var s in skills) { - config.Tools = options.Tools.OfType().ToList(); + if (string.IsNullOrWhiteSpace(s.Instructions)) + continue; + sb.AppendLine(); + sb.AppendLine($"## {s.Name}"); + if (!string.IsNullOrWhiteSpace(s.Description)) + sb.AppendLine(s.Description); + sb.AppendLine(); + sb.AppendLine(s.Instructions); } - - return config; + return sb.ToString(); } private static string GetTextContent(ChatMessage message) diff --git a/src/MeshWeaver.AI.Copilot/CopilotChatClientAgentFactory.cs b/src/MeshWeaver.AI.Copilot/CopilotChatClientAgentFactory.cs deleted file mode 100644 index 4c4155236..000000000 --- a/src/MeshWeaver.AI.Copilot/CopilotChatClientAgentFactory.cs +++ /dev/null @@ -1,54 +0,0 @@ -using MeshWeaver.Messaging; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; - -namespace MeshWeaver.AI.Copilot; - -/// -/// Factory for creating ChatClientAgent instances with GitHub Copilot SDK. -/// -public class CopilotChatClientAgentFactory( - IMessageHub hub, - IOptions options, - ILogger logger) - : ChatClientAgentFactory(hub) -{ - private readonly CopilotConfiguration configuration = options.Value ?? new CopilotConfiguration(); - - public override string Name => "GitHub Copilot"; - - public override IReadOnlyList Models => configuration.Models; - - public override int Order => configuration.Order; - - protected override IChatClient CreateChatClient(AgentConfiguration agentConfig) - { - // Use CurrentModelName if set, fall back to agent's preferred model, otherwise use first configured model - var modelName = !string.IsNullOrEmpty(CurrentModelName) ? CurrentModelName - : !string.IsNullOrEmpty(agentConfig.PreferredModel) ? agentConfig.PreferredModel - : configuration.Models.FirstOrDefault(); - - logger.LogInformation( - "Creating GitHub Copilot chat client for agent {AgentName} using model {ModelName}", - agentConfig.Id, modelName); - - try - { - var clientLogger = Hub.ServiceProvider.GetService(typeof(ILogger)) as ILogger; - var chatClient = new CopilotChatClient(configuration, modelName, clientLogger); - - logger.LogInformation( - "Successfully configured GitHub Copilot chat client for agent {AgentName}", - agentConfig.Id); - - return chatClient; - } - catch (Exception ex) - { - logger.LogError(ex, "Failed to create GitHub Copilot chat client for agent {AgentName}", agentConfig.Id); - throw new InvalidOperationException( - $"Failed to create GitHub Copilot chat client for agent {agentConfig.Id}: {ex.Message}", ex); - } - } -} diff --git a/src/MeshWeaver.AI.Copilot/CopilotConfiguration.cs b/src/MeshWeaver.AI.Copilot/CopilotConfiguration.cs index 9be5c7279..e8f390172 100644 --- a/src/MeshWeaver.AI.Copilot/CopilotConfiguration.cs +++ b/src/MeshWeaver.AI.Copilot/CopilotConfiguration.cs @@ -41,4 +41,11 @@ public class CopilotConfiguration /// Session timeout in milliseconds. /// public int SessionTimeoutMs { get; set; } = 30000; + + /// + /// The shared on-disk workspace dir the agent→skill sync maintains (.claude/skills/ + + /// AGENTS.md). When set, the harness points the Copilot session's working directory at it + /// (with config discovery on) so the CLI surfaces the MeshWeaver agents + skills. Null ⇒ none. + /// + public string? SkillsDirectory { get; set; } } diff --git a/src/MeshWeaver.AI.Copilot/CopilotConnectStrategy.cs b/src/MeshWeaver.AI.Copilot/CopilotConnectStrategy.cs new file mode 100644 index 000000000..a95a607ad --- /dev/null +++ b/src/MeshWeaver.AI.Copilot/CopilotConnectStrategy.cs @@ -0,0 +1,252 @@ +using System.Diagnostics; +using System.Reactive.Linq; +using System.Text.RegularExpressions; +using GitHub.Copilot.SDK; +using MeshWeaver.AI.Connect; +using MeshWeaver.Mesh.Threading; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace MeshWeaver.AI.Copilot; + +/// +/// Drives GitHub Copilot's device-flow login for the per-user Connect flow. +/// +/// Mechanism (probed 2026-06-01, GitHub.Copilot.SDK 1.0.0-beta.3): the SDK exposes +/// GetAuthStatusAsync() → GetAuthStatusResponse { IsAuthenticated, Login, AuthType } +/// (the reliable login-status probe used by ), but it exposes no +/// device-flow / sign-in method — it expects an already-authenticated host (a GitHub token via the +/// GitHubToken option or UseLoggedInUser reading the host's gh/Copilot CLI +/// auth). So the device-code flow itself has to be driven by the copilot CLI subprocess, +/// which (like claude setup-token) renders an interactive UI and is TTY-gated — it +/// won't emit a scrapeable device code over a redirected pipe. +/// +/// This strategy therefore implements the device-flow shape: spawns +/// the configured login command and scrapes the github.com/login/device URL + the +/// XXXX-XXXX user code; auto-polls +/// GetAuthStatusAsync().IsAuthenticated until the user finishes in the browser, then returns +/// the captured GitHub token. With the default copilot command the device-code scrape will +/// NOT work headlessly until a PTY wrapper lands; the login-status probe (the part the spec asks for +/// on every render) DOES work via the SDK. +/// +/// TODO(copilot-pty): the Copilot CLI device-login is TTY-gated; wrap the spawn in a +/// pseudo-terminal so the device code becomes scrapeable, OR adopt a real device-flow API if a +/// future SDK exposes one. The token captured here is whatever the CLI persists to the host's +/// Copilot auth; lets a test inject it. +/// Real-CLI E2E gated behind CLAUDE_CONNECT_E2E=1. +/// +public sealed class CopilotConnectStrategy : IConnectStrategy +{ + private readonly IServiceProvider services; + private readonly ILogger? logger; + // Subprocess spawn → Process pool; Copilot SDK network calls → Http pool. Never FromAsync. + private readonly IIoPool processPool; + private readonly IIoPool httpPool; + + public CopilotConnectStrategy(IServiceProvider services) + { + this.services = services; + logger = services.GetService()?.CreateLogger(); + var registry = services.GetService(); + processPool = registry?.Get(IoPoolNames.Process) ?? IoPool.Unbounded; + httpPool = registry?.Get(IoPoolNames.Http) ?? IoPool.Unbounded; + } + + public ConnectProvider Provider => ConnectProvider.Copilot; + + /// Copilot is device-flow — nothing to paste; the manager auto-polls to completion. + public bool RequiresPastedCode => false; + + private CopilotConnectOptions Options => + services.GetService>()?.Value ?? new CopilotConnectOptions(); + + private CopilotConfiguration CopilotConfig => + services.GetService>()?.Value ?? new CopilotConfiguration(); + + /// + /// Cheap login-status probe — starts the SDK client (under the user's Copilot home if isolated) + /// and reads GetAuthStatusAsync().IsAuthenticated. This is the genuinely + /// headless-confirmable part of the Copilot flow. + /// + public IObservable IsLoggedIn(string? userConfigDir) + => httpPool.Invoke(ct => GetIsAuthenticatedAsync(userConfigDir, ct)); + + public IObservable StartConnect(ConnectSession session, string ownerPath) + => SpawnAndScrapeDeviceCode(session, Options); + + public IObservable CompleteConnect(ConnectSession session, string? pastedCode) + { + var options = Options; + return httpPool.Invoke(ct => PollUntilAuthenticatedAsync(session, options, ct)); + } + + // ── SDK / subprocess boundary (the only place Task lives) ──────────────────────────────────── + + private async Task GetIsAuthenticatedAsync(string? userConfigDir, CancellationToken ct) + { + try + { + await using var client = BuildClient(userConfigDir); + await client.StartAsync(ct).ConfigureAwait(false); + var status = await client.GetAuthStatusAsync(ct).ConfigureAwait(false); + return status?.IsAuthenticated == true; + } + catch (Exception ex) + { + logger?.LogDebug(ex, "Copilot IsLoggedIn probe failed for {Dir}", userConfigDir); + return false; + } + } + + private CopilotClient BuildClient(string? userConfigDir) + { + var cfg = CopilotConfig; + var options = new CopilotClientOptions { AutoStart = true, UseLoggedInUser = true }; + if (!string.IsNullOrEmpty(cfg.CliPath)) options.CliPath = cfg.CliPath; + if (!string.IsNullOrEmpty(cfg.CliUrl)) options.CliUrl = cfg.CliUrl; + if (cfg.Port.HasValue) options.Port = cfg.Port.Value; + if (!string.IsNullOrEmpty(userConfigDir)) options.CopilotHome = userConfigDir; + return new CopilotClient(options); + } + + // IObservable end-to-end up to the IO boundary. The CLI's stdout/stderr is the SOURCE: the + // callbacks drive a ReplaySubject (race-proof — a line emitted before we subscribe is + // buffered, never dropped), replacing the hand-rolled queue + SemaphoreSlim signal. The only + // genuine IO is the synchronous, thread-holding process spawn — it goes through the Process pool + // via InvokeBlocking (the ControlledIoPooling boundary). Everything after is pure reactive + // composition: scan lines for url+code, complete on the first full pair, surface a typed error if + // the process exits first, all bounded by Timeout. No await, no .ToTask(), no async gate to park. + private IObservable SpawnAndScrapeDeviceCode( + ConnectSession session, CopilotConnectOptions options) + => Observable.Defer(() => + { + var process = new Process + { + StartInfo = new ProcessStartInfo + { + FileName = options.FileName, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }, + EnableRaisingEvents = true, + }; + foreach (var a in options.Arguments) process.StartInfo.ArgumentList.Add(a); + if (!string.IsNullOrEmpty(session.ConfigDir)) + process.StartInfo.Environment["COPILOT_HOME"] = session.ConfigDir; + + var lines = new System.Reactive.Subjects.ReplaySubject(); + process.OutputDataReceived += (_, e) => { if (e.Data != null) lines.OnNext(e.Data); }; + process.ErrorDataReceived += (_, e) => { if (e.Data != null) lines.OnNext(e.Data); }; + process.Exited += (_, _) => lines.OnCompleted(); + + var urlRegex = new Regex(options.DeviceUrlPattern, RegexOptions.Compiled); + var codeRegex = new Regex(options.UserCodePattern, RegexOptions.Compiled); + + static string Extract(Match m) => + (m.Groups.Count > 1 && m.Groups[1].Success ? m.Groups[1].Value : m.Value).Trim(); + + // Fold each line into the accumulating (url, code) pair; emit the first pair where both are + // set. Process exit (the source's OnCompleted) without a full pair -> "exited early" error. + var scrape = lines + .Scan( + (Url: (string?)null, Code: (string?)null), + (acc, line) => + { + var url = acc.Url; + var code = acc.Code; + if (url is null) { var mu = urlRegex.Match(line); if (mu.Success) url = Extract(mu); } + if (code is null) { var mc = codeRegex.Match(line); if (mc.Success) code = Extract(mc); } + return (url, code); + }) + .Where(acc => acc.Url is not null && acc.Code is not null) + .Take(1) + .Select(acc => new ConnectChallenge( + session.SessionId, ConnectProvider.Copilot, acc.Url!, UserCode: acc.Code!, RequiresPastedCode: false)) + .Concat(Observable.Throw(new InvalidOperationException( + "copilot login exited before emitting a device code. On a non-TTY stdout it emits nothing — see TODO(copilot-pty)."))) + .Timeout(options.DeviceCodeTimeout, Observable.Throw(new TimeoutException( + "Timed out waiting for the Copilot device code. The CLI needs a real terminal (PTY) — see TODO(copilot-pty)."))) + .Do(challenge => logger?.LogInformation( + "Copilot Connect surfaced device code for session {Session}", session.SessionId)); + + // The synchronous, thread-holding spawn is the IO leaf -> Process pool (InvokeBlocking). + // Subscribing the scrape is set up before BeginOutputReadLine; the ReplaySubject makes it + // race-proof regardless. SelectMany hands off to the pure-Rx scrape once the process is up. + return processPool + .InvokeBlocking(_ => + { + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + session.Process = process; + return System.Reactive.Unit.Default; + }) + .SelectMany(_ => scrape); + }); + + private async Task PollUntilAuthenticatedAsync( + ConnectSession session, CopilotConnectOptions options, CancellationToken ct) + { + // A test injects the captured token via an env var so the device-flow shape is exercised + // end-to-end without a real GitHub round-trip. + if (!string.IsNullOrEmpty(options.TokenEnvironmentVariable)) + { + var injected = Environment.GetEnvironmentVariable(options.TokenEnvironmentVariable); + if (!string.IsNullOrEmpty(injected)) return injected!; + } + + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + timeoutCts.CancelAfter(options.PollTimeout); + await using var client = BuildClient(session.ConfigDir); + await client.StartAsync(timeoutCts.Token).ConfigureAwait(false); + + while (!timeoutCts.IsCancellationRequested) + { + try + { + var status = await client.GetAuthStatusAsync(timeoutCts.Token).ConfigureAwait(false); + if (status?.IsAuthenticated == true) + { + // SDK 1.0.0-beta.3 surfaces no raw token. Use the host env if present, else a + // stable marker so the stored ModelProvider records that Copilot is connected + // (the factory authenticates via UseLoggedInUser against the host CLI auth). + var token = Environment.GetEnvironmentVariable("GITHUB_TOKEN") + ?? Environment.GetEnvironmentVariable("GH_TOKEN") + ?? $"copilot-oauth:{status.Login ?? "connected"}"; + logger?.LogInformation("Copilot authenticated as {Login} for session {Session}", status.Login, session.SessionId); + return token; + } + } + catch (OperationCanceledException) { break; } + catch (Exception ex) { logger?.LogDebug(ex, "Copilot poll iteration failed; retrying"); } + await Task.Delay(options.PollInterval, timeoutCts.Token).ConfigureAwait(false); + } + throw new TimeoutException("Timed out waiting for Copilot device-flow authentication."); + } +} + +/// Tunables for — overridable by deployment / test. +public sealed class CopilotConnectOptions +{ + /// Login command to spawn. Defaults to the copilot CLI. + public string FileName { get; set; } = "copilot"; + + /// Arguments to the login command (e.g. a login subcommand). + public IReadOnlyList Arguments { get; set; } = Array.Empty(); + + /// Regex extracting the verification URL (default github.com/login/device). + public string DeviceUrlPattern { get; set; } = @"(https?://\S*github\.com/login/device\S*)"; + + /// Regex extracting the user device code (default XXXX-XXXX). + public string UserCodePattern { get; set; } = @"\b([A-Z0-9]{4}-[A-Z0-9]{4})\b"; + + /// An env var a test sets to inject the captured token, short-circuiting the poll. + public string? TokenEnvironmentVariable { get; set; } + + public TimeSpan DeviceCodeTimeout { get; set; } = TimeSpan.FromSeconds(30); + public TimeSpan PollTimeout { get; set; } = TimeSpan.FromMinutes(4); + public TimeSpan PollInterval { get; set; } = TimeSpan.FromSeconds(3); +} diff --git a/src/MeshWeaver.AI.Copilot/CopilotExtensions.cs b/src/MeshWeaver.AI.Copilot/CopilotExtensions.cs index aceae2799..8ad6a4db0 100644 --- a/src/MeshWeaver.AI.Copilot/CopilotExtensions.cs +++ b/src/MeshWeaver.AI.Copilot/CopilotExtensions.cs @@ -1,4 +1,8 @@ +using System.Collections.Immutable; +using MeshWeaver.AI; +using MeshWeaver.Mesh; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; namespace MeshWeaver.AI.Copilot; @@ -8,22 +12,39 @@ namespace MeshWeaver.AI.Copilot; public static class CopilotExtensions { /// - /// Adds GitHub Copilot services to the service collection. - /// Configuration should be bound to CopilotConfiguration. + /// GitHub Copilot is a harness, not a model provider — so this no longer + /// registers a language-model catalog source. The harness surfaces as a + /// Harness catalog node (see ) and is wired + /// via . + /// Retained as a no-op so existing builder chains keep compiling. + /// + public static TBuilder AddCopilot(this TBuilder builder) + where TBuilder : MeshBuilder + => builder; + + /// + /// Registers GitHub Copilot as a harness: the + /// runs the Copilot CLI directly. The live model catalog is still registered (the + /// CLI reports its models). See + . /// public static IServiceCollection AddCopilot(this IServiceCollection services) { - return services.AddSingleton(); + services.TryAddSingleton(); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + return services; } /// - /// Adds GitHub Copilot services with configuration action. + /// Registers the GitHub Copilot harness with a configuration action that binds + /// . See . /// public static IServiceCollection AddCopilot( this IServiceCollection services, Action configure) { services.Configure(configure); - return services.AddSingleton(); + services.TryAddSingleton(); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + return services; } } diff --git a/src/MeshWeaver.AI.Copilot/CopilotHarness.cs b/src/MeshWeaver.AI.Copilot/CopilotHarness.cs new file mode 100644 index 000000000..0e86c1f58 --- /dev/null +++ b/src/MeshWeaver.AI.Copilot/CopilotHarness.cs @@ -0,0 +1,83 @@ +using System.Reactive.Linq; +using MeshWeaver.AI.Connect; +using MeshWeaver.Mesh.Threading; +using MeshWeaver.Messaging; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace MeshWeaver.AI.Copilot; + +/// +/// The GitHub Copilot harness — runs the Copilot CLI through +/// . It is a harness, NOT a model provider: selecting +/// it dispatches the round straight to the Copilot library, bypassing the +/// model-provider factory chain. Registered from this assembly via AddCopilot. +/// +public sealed class CopilotHarness(IOptions options) : IHarness +{ + private readonly CopilotConfiguration configuration = options.Value ?? new CopilotConfiguration(); + + public string Id => Harnesses.Copilot; + + public Harness Definition => new() + { + Id = Harnesses.Copilot, + DisplayName = "GitHub Copilot", + Description = "Runs the GitHub Copilot CLI.", + // Inline SVG (single-quoted attrs) — travels WITH the node; no /static file or allowlist plumbing. + Icon = "", + Order = 2, + SupportsAgentSelection = false + }; + + // Copilot owns its auth slash-commands: /login runs the GitHub device-flow login via the Connect + // flow; /logout forgets the stored token. When this harness is active these REPLACE MeshWeaver's + // /agent /model in the chat autocomplete + dispatch. + public IReadOnlyList Commands { get; } = + [ + new("login", "Log in to GitHub Copilot", HarnessCommandKind.Connect), + new("logout", "Log out of GitHub Copilot", HarnessCommandKind.Disconnect), + ]; + + public Connect.ConnectProvider? AuthProvider => Connect.ConnectProvider.Copilot; + + public IChatClient? CreateChatClient(HarnessExecutionContext context) + { + var hub = context.Hub; + var accessCtx = hub.ServiceProvider.GetService()?.Context; + var userId = accessCtx?.ObjectId; + + // 🚫 NEVER pass a model to the CLI. Copilot self-selects; forwarding the MeshWeaver composer's + // selected model (a non-Copilot id) makes the round fail. The harness surfaces no model + // selection (SupportsAgentSelection = false), so there is nothing to forward. + // + // 🔑 The GitHub token is the user's per-user Connect (subscription) token, NOT a selected + // model's API key — resolve it from the user's Copilot provider node. When absent the CLI + // falls back to the machine's logged-in user (single-user dev / ambient auth). + var resolver = hub.ServiceProvider.GetService(); + var githubToken = resolver?.ResolveConnectToken(Harnesses.Copilot, userId); + var clientLogger = hub.ServiceProvider.GetService>(); + // Subprocess CLI spawn + SDK network round-trips → Http pool (off the hub scheduler, + // bounded). Unbounded fallback when no pool is wired (tests / DI-less construction). + var ioPool = hub.ServiceProvider.GetService()?.Get(IoPoolNames.Http) ?? IoPool.Unbounded; + // Automatic MCP back-connection — the mesh is this CLI's workspace (per-user Bearer token). + var mcp = hub.ServiceProvider.GetService(); + + // Project the user's selectable MeshWeaver agents — injected into the Copilot session's system + // message (Copilot's SDK has no filesystem skills folder). Utility/background generators excluded. + var agentSkills = !string.IsNullOrEmpty(userId) + ? AgentPickerProjection.ObserveAgents(hub, userId, null) + .Select(agents => (IReadOnlyList)agents + .Where(a => !AgentPickerProjection.IsUtilityAgent(a) + && !string.IsNullOrWhiteSpace(a.AgentConfiguration.Instructions)) + .Select(a => new AgentSkill( + a.AgentConfiguration.Id, a.Name, a.Description, a.AgentConfiguration.Instructions!)) + .ToList()) + : null; + + return new CopilotChatClient(configuration, modelName: null, clientLogger, githubToken, ioPool, agentSkills, + mcp, userId, accessCtx?.Name, accessCtx?.Email); + } +} diff --git a/src/MeshWeaver.AI.Copilot/CopilotModelCatalog.cs b/src/MeshWeaver.AI.Copilot/CopilotModelCatalog.cs new file mode 100644 index 000000000..ddb98f264 --- /dev/null +++ b/src/MeshWeaver.AI.Copilot/CopilotModelCatalog.cs @@ -0,0 +1,65 @@ +using System.Reactive.Linq; +using MeshWeaver.AI; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Mesh; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; + +namespace MeshWeaver.AI.Copilot; + +/// +/// The GitHub Copilot model list, exposed as a live reactive stream backed by +/// the workspace's shared synced-query cache (workspace.GetQuery(...)) — NOT a +/// one-shot CLI probe cached in a field. +/// +/// Why GetQuery, not a pooled load. "We want to know this" = we want the +/// current set of Copilot models, kept current. That is a read of mesh state, and +/// the canonical primitive for "a set, live" is the synced query: GetQuery returns +/// the cached IObservable<IEnumerable<MeshNode>> for the query, emits the +/// initial set plus a delta on every change, and is shared process-wide (one upstream +/// subscription regardless of how many pickers/tabs bind it). The GUI data-binds this +/// stream directly; it is never a snapshot. (See CqrsAndContentAccess.md → +/// "Sets / listings, live" and AgentChatClient.Initialize for the canonical shape.) +/// +/// AccessContext. GetQuery opens its upstream SubscribeRequest +/// under the cache hub's system-flagged identity, then re-stamps each subscriber's own +/// AccessService.Context per emission (CarryAccessContext) so RLS is applied +/// for the subscribing user — exactly what a per-user picker wants. The former IoPool +/// load carried NO AccessContext (the pool re-runs the leaf on a ThreadPool worker with no +/// baton), so any node read it did would have run identity-less. Reading through GetQuery +/// fixes that for free. +/// +public sealed class CopilotModelCatalog +{ + private readonly IMessageHub hub; + + public CopilotModelCatalog(IServiceProvider services) + { + // The mesh hub owns the shared workspace + synced-query cache. + hub = services.GetRequiredService(); + } + + /// + /// Live Copilot model ids. Emits the current set immediately (warm cache) or on first + /// cold load, then re-emits whenever the underlying LanguageModel nodes change. + /// Subscribe / data-bind — never .Take(1) on a display binding (it would freeze). + /// + public IObservable> Models => + hub.GetWorkspace() + // One shared synced query per id, keyed so every consumer reuses the same upstream. + // Copilot's models live as standard LanguageModel nodes under the "Model" partition, + // provided by the Copilot provider — filter by nodeType so sibling satellite types + // under the namespace don't leak into the picker. + .GetQuery( + $"{LanguageModelNodeType.NodeType}|Copilot", + $"namespace:{LanguageModelNodeType.RootNamespace} nodeType:{LanguageModelNodeType.NodeType} provider:Copilot") + .Select(nodes => (IReadOnlyList)nodes + .Select(n => n.Id) + .Where(id => !string.IsNullOrEmpty(id)) + .ToArray()) + // Seed empty so a binding renders immediately (no spinner-forever) and a cold first + // load doesn't leave CombineLatest-style consumers waiting on a first emission. + .StartWith(Array.Empty()) + .DistinctUntilChanged(); +} diff --git a/src/MeshWeaver.AI.AzureOpenAI/AzureOpenAIChatClientAgentFactory.cs b/src/MeshWeaver.AI.OpenAI/AzureOpenAIChatClientAgentFactory.cs similarity index 51% rename from src/MeshWeaver.AI.AzureOpenAI/AzureOpenAIChatClientAgentFactory.cs rename to src/MeshWeaver.AI.OpenAI/AzureOpenAIChatClientAgentFactory.cs index a50c312f9..4cc89cac9 100644 --- a/src/MeshWeaver.AI.AzureOpenAI/AzureOpenAIChatClientAgentFactory.cs +++ b/src/MeshWeaver.AI.OpenAI/AzureOpenAIChatClientAgentFactory.cs @@ -2,10 +2,11 @@ using Azure.AI.OpenAI; using MeshWeaver.Messaging; using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -namespace MeshWeaver.AI.AzureOpenAI; +namespace MeshWeaver.AI.OpenAI; /// /// Factory for creating ChatClientAgent instances with Azure OpenAI. @@ -38,24 +39,37 @@ private static AzureOpenAIConfiguration InitAndLog(IOptions(); + var resolution = resolver.Resolve(modelName); + var endpoint = resolution.Endpoint ?? credentials.Endpoint; + var apiKey = resolution.ApiKey ?? credentials.ApiKey; + var source = resolution.Endpoint != null || resolution.ApiKey != null + ? resolution.Source : "IOptions"; + + if (string.IsNullOrEmpty(endpoint)) + throw new InvalidOperationException( + $"Endpoint is missing for model '{modelName}'. Configure a ModelProvider node (Model/AzureOpenAI) or set AzureOpenAI:Endpoint in config."); + if (string.IsNullOrEmpty(apiKey)) + throw new InvalidOperationException( + $"ApiKey is missing for model '{modelName}'. Configure a ModelProvider node (Model/AzureOpenAI) or set AzureOpenAI:ApiKey in config."); + + logger.LogInformation( + "[AzureOpenAI] Creating chat client agent={AgentName} model={ModelName} endpoint={Endpoint} source={Source} apiKeyFp={ApiKeyFingerprint}", + agentConfig.Id, modelName, endpoint, source, Fingerprint(apiKey)); + // Create Azure OpenAI client and get chat client var azureClient = new AzureOpenAIClient( - new Uri(credentials.Endpoint), - new AzureKeyCredential(credentials.ApiKey)); + new Uri(endpoint), + new AzureKeyCredential(apiKey)); // Get the chat completion client for the model and convert it to IChatClient var openAIChatClient = azureClient.GetChatClient(modelName); @@ -67,4 +81,16 @@ protected override IChatClient CreateChatClient(AgentConfiguration agentConfig) return chatClient; } + + /// + /// 8-char SHA-256-hex prefix of . Used in logs to + /// disambiguate "which key was actually used" without exposing the key. + /// + private static string Fingerprint(string? value) + { + if (string.IsNullOrEmpty(value)) return "(empty)"; + var bytes = System.Text.Encoding.UTF8.GetBytes(value); + var hash = System.Security.Cryptography.SHA256.HashData(bytes); + return Convert.ToHexString(hash, 0, 4).ToLowerInvariant(); + } } diff --git a/src/MeshWeaver.AI.AzureOpenAI/AzureOpenAIConfiguration.cs b/src/MeshWeaver.AI.OpenAI/AzureOpenAIConfiguration.cs similarity index 94% rename from src/MeshWeaver.AI.AzureOpenAI/AzureOpenAIConfiguration.cs rename to src/MeshWeaver.AI.OpenAI/AzureOpenAIConfiguration.cs index 9e004d1d9..4c702ecdd 100644 --- a/src/MeshWeaver.AI.AzureOpenAI/AzureOpenAIConfiguration.cs +++ b/src/MeshWeaver.AI.OpenAI/AzureOpenAIConfiguration.cs @@ -1,4 +1,4 @@ -namespace MeshWeaver.AI.AzureOpenAI; +namespace MeshWeaver.AI.OpenAI; /// /// Configuration for AI service credentials diff --git a/src/MeshWeaver.AI.OpenAI/AzureOpenAIExtensions.cs b/src/MeshWeaver.AI.OpenAI/AzureOpenAIExtensions.cs new file mode 100644 index 000000000..41da27e46 --- /dev/null +++ b/src/MeshWeaver.AI.OpenAI/AzureOpenAIExtensions.cs @@ -0,0 +1,60 @@ +using System.Collections.Immutable; +using MeshWeaver.AI; +using MeshWeaver.Mesh; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace MeshWeaver.AI.OpenAI; + +/// +/// Extension methods for adding Azure OpenAI services. Each provider +/// self-registers its bootstrap profile via +/// +/// — no central registry. +/// +public static class AzureOpenAIExtensions +{ + /// + /// One-call registration of Azure OpenAI — catalog profile + IOptions + /// binding (AzureOpenAI:) + + /// . Idempotent. + /// + public static TBuilder AddAzureOpenAI(this TBuilder builder, string configSection = "AzureOpenAI") + where TBuilder : MeshBuilder + { + builder.AddLanguageModelCatalogSource(new LanguageModelCatalogSource( + SectionName: configSection, + ProviderName: "AzureOpenAI", + Order: 3, + DisplayLabel: "Azure OpenAI", + DefaultEndpoint: null, + DefaultModelIds: ImmutableArray.Empty, + RequiresApiKey: true)); + builder.ConfigureServices(services => + { + services.AddOptions().BindConfiguration(configSection); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + return services; + }); + return builder; + } + + /// + /// Adds Azure OpenAI services to the service collection + /// + public static IServiceCollection AddAzureOpenAI(this IServiceCollection services) + { + return services.AddSingleton(); + } + + /// + /// Adds Azure OpenAI services with configuration action. + /// + public static IServiceCollection AddAzureOpenAI( + this IServiceCollection services, + Action configure) + { + services.Configure(configure); + return services.AddSingleton(); + } +} diff --git a/src/MeshWeaver.AI.AzureOpenAI/GlobalUsings.cs b/src/MeshWeaver.AI.OpenAI/GlobalUsings.cs similarity index 100% rename from src/MeshWeaver.AI.AzureOpenAI/GlobalUsings.cs rename to src/MeshWeaver.AI.OpenAI/GlobalUsings.cs diff --git a/src/MeshWeaver.AI.AzureOpenAI/MeshWeaver.AI.AzureOpenAI.csproj b/src/MeshWeaver.AI.OpenAI/MeshWeaver.AI.OpenAI.csproj similarity index 100% rename from src/MeshWeaver.AI.AzureOpenAI/MeshWeaver.AI.AzureOpenAI.csproj rename to src/MeshWeaver.AI.OpenAI/MeshWeaver.AI.OpenAI.csproj diff --git a/src/MeshWeaver.AI.OpenAI/OpenAIChatClientAgentFactory.cs b/src/MeshWeaver.AI.OpenAI/OpenAIChatClientAgentFactory.cs new file mode 100644 index 000000000..c332d3d50 --- /dev/null +++ b/src/MeshWeaver.AI.OpenAI/OpenAIChatClientAgentFactory.cs @@ -0,0 +1,104 @@ +using System.ClientModel; +using MeshWeaver.Messaging; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using OpenAI; + +namespace MeshWeaver.AI.OpenAI; + +/// +/// Factory for any provider that speaks the OpenAI wire protocol — +/// direct OpenAI (api.openai.com) AND OpenAI-compatible gateways +/// (OpenRouter, Groq, Together, a local vLLM, …) configured with a custom base +/// URL under the generic OpenAICompatible provider. Mirrors +/// but builds a plain +/// pointed at the resolved endpoint. Credentials + +/// endpoint resolve from the selected model's ModelProvider node via +/// (following each model's +/// ProviderRef, so two custom gateways coexist), falling back to IOptions +/// (OpenAI:) for a system default. +/// +public class OpenAIChatClientAgentFactory( + IMessageHub hub, + IOptions options, + ILogger logger) + : ChatClientAgentFactory(hub) +{ + private readonly OpenAIConfiguration credentials = options.Value ?? new OpenAIConfiguration(); + + public override string Name => "OpenAI"; + + public override IReadOnlyList Models => credentials.Models; + + public override int Order => credentials.Order; + + /// + /// Provider stamps this factory owns. Any model whose ModelProvider + /// declares one of these routes here: OpenAI (direct api.openai.com) + /// and OpenAICompatible (the generic custom-URL provider — OpenRouter, + /// Groq, Together, vLLM, …). The endpoint for the latter comes from the + /// model's resolved provider node, so the same factory serves any number of + /// distinct gateways. + /// + private static readonly string[] OwnedProviders = ["OpenAI", "OpenAICompatible"]; + + /// + /// Routes a model here when its ModelProvider declares a provider in + /// — so a gpt-* id owned by a direct + /// OpenAI provider, or any model id owned by an OpenAI-compatible gateway, + /// lands here while an Azure-OpenAI-owned id stays with the Azure factory. + /// Additive over the base (Models-list) match, so it never narrows existing + /// behaviour. + /// + public override bool Supports(string modelName) + { + if (string.IsNullOrEmpty(modelName)) return false; + var provider = Hub.ServiceProvider.GetService() + ?.GetProviderForModel(modelName); + return (provider != null && OwnedProviders.Contains(provider, StringComparer.OrdinalIgnoreCase)) + || base.Supports(modelName); + } + + protected override IChatClient CreateChatClient(AgentConfiguration agentConfig) + { + // Composer selection wins; then the agent's ModelTier; first configured model as a last resort. + var modelName = !string.IsNullOrEmpty(CurrentModelName) ? CurrentModelName + : ResolveTierModel(agentConfig) ?? credentials.Models.FirstOrDefault(); + + if (string.IsNullOrEmpty(modelName)) + throw new InvalidOperationException("No model configured for OpenAI"); + + var resolver = Hub.ServiceProvider.GetRequiredService(); + var resolution = resolver.Resolve(modelName); + var endpoint = resolution.Endpoint ?? credentials.Endpoint; // null → SDK default api.openai.com + var apiKey = resolution.ApiKey ?? credentials.ApiKey; + var source = resolution.Endpoint != null || resolution.ApiKey != null + ? resolution.Source : "IOptions"; + + if (string.IsNullOrEmpty(apiKey)) + throw new InvalidOperationException( + $"ApiKey is missing for model '{modelName}'. Configure a ModelProvider node (Provider 'OpenAI') or set OpenAI:ApiKey in config."); + + logger.LogInformation( + "[OpenAI] Creating chat client agent={AgentName} model={ModelName} endpoint={Endpoint} source={Source} apiKeyFp={ApiKeyFingerprint}", + agentConfig.Id, modelName, endpoint ?? "(default api.openai.com)", source, Fingerprint(apiKey)); + + var clientOptions = new OpenAIClientOptions(); + if (!string.IsNullOrEmpty(endpoint)) + clientOptions.Endpoint = new Uri(endpoint); + + var client = new OpenAIClient(new ApiKeyCredential(apiKey), clientOptions); + return client.GetChatClient(modelName).AsIChatClient(); + } + + /// 8-char SHA-256-hex prefix — logs which key was used, never the key. + private static string Fingerprint(string? value) + { + if (string.IsNullOrEmpty(value)) return "(empty)"; + var bytes = System.Text.Encoding.UTF8.GetBytes(value); + var hash = System.Security.Cryptography.SHA256.HashData(bytes); + return Convert.ToHexString(hash, 0, 4).ToLowerInvariant(); + } +} diff --git a/src/MeshWeaver.AI.OpenAI/OpenAIConfiguration.cs b/src/MeshWeaver.AI.OpenAI/OpenAIConfiguration.cs new file mode 100644 index 000000000..694110bcc --- /dev/null +++ b/src/MeshWeaver.AI.OpenAI/OpenAIConfiguration.cs @@ -0,0 +1,26 @@ +namespace MeshWeaver.AI.OpenAI; + +/// +/// Configuration for direct OpenAI (api.openai.com) credentials — +/// bring-your-own personal OpenAI key. Distinct from +/// , which targets an Azure-hosted +/// OpenAI deployment. Bound from the OpenAI: config section; per-user +/// keys override via a ModelProvider node (Provider = "OpenAI"). +/// +public class OpenAIConfiguration +{ + /// + /// Optional endpoint override. null uses the SDK default + /// (https://api.openai.com). Set for an OpenAI-compatible gateway. + /// + public string? Endpoint { get; set; } + + /// The OpenAI API key (sk-...). + public string? ApiKey { get; set; } + + /// Available model ids (e.g. gpt-4o, gpt-4o-mini). + public string[] Models { get; set; } = Array.Empty(); + + /// Display order in the model dropdown (lower = first). + public int Order { get; set; } = 0; +} diff --git a/src/MeshWeaver.AI.OpenAI/OpenAIExtensions.cs b/src/MeshWeaver.AI.OpenAI/OpenAIExtensions.cs new file mode 100644 index 000000000..652c6a638 --- /dev/null +++ b/src/MeshWeaver.AI.OpenAI/OpenAIExtensions.cs @@ -0,0 +1,80 @@ +using System.Collections.Immutable; +using MeshWeaver.AI; +using MeshWeaver.Mesh; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace MeshWeaver.AI.OpenAI; + +/// +/// Builder extensions for OpenAI-wire-protocol providers: direct OpenAI +/// (api.openai.com) and the generic OpenAICompatible custom-URL +/// provider (OpenRouter, Groq, Together, a local vLLM, …). Both are served by +/// ; they differ only in the catalog +/// profile they self-register via +/// +/// — no central registry. +/// +public static class OpenAIExtensions +{ + /// + /// One-call registration of direct OpenAI — catalog profile + IOptions + /// binding (OpenAI:) + . + /// Idempotent. + /// + public static TBuilder AddOpenAI(this TBuilder builder, string configSection = "OpenAI") + where TBuilder : MeshBuilder + { + builder.AddLanguageModelCatalogSource(new LanguageModelCatalogSource( + SectionName: configSection, + ProviderName: "OpenAI", + Order: 4, + DisplayLabel: "OpenAI", + DefaultEndpoint: null, // SDK default: https://api.openai.com + DefaultModelIds: ImmutableArray.Create("gpt-4o", "gpt-4o-mini"), + RequiresApiKey: true)); + builder.ConfigureServices(services => + { + services.AddOptions().BindConfiguration(configSection); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + return services; + }); + return builder; + } + + /// + /// One-call registration of the generic OpenAI-compatible provider — + /// the catalog "type" a user picks in Settings → Language Models when bringing + /// any OpenAI-wire endpoint by URL + key (OpenRouter, Groq, Together, a local + /// vLLM, …). There is no system-default endpoint or model list: the user + /// supplies the base URL and fetches the model list live, and each saved + /// provider stores its own endpoint on its ModelProvider node, so + /// several distinct gateways coexist. Reuses + /// (it already owns the + /// OpenAICompatible provider stamp); the factory registration is + /// idempotent with . + /// + public static TBuilder AddOpenAICompatible(this TBuilder builder, string configSection = "OpenAICompatible") + where TBuilder : MeshBuilder + { + builder.AddLanguageModelCatalogSource(new LanguageModelCatalogSource( + SectionName: configSection, + ProviderName: "OpenAICompatible", + Order: 5, + DisplayLabel: "OpenAI-compatible (custom URL)", + DefaultEndpoint: null, // user supplies the base URL (e.g. https://openrouter.ai/api/v1) + DefaultModelIds: ImmutableArray.Empty, + RequiresApiKey: true)); + builder.ConfigureServices(services => + { + // No BindConfiguration: there is no system-default OpenAICompatible + // section — endpoint + key always come from the user's ModelProvider + // node. AddOptions alone guarantees IOptions + // resolves (empty) when AddOpenAI wasn't also called. + services.AddOptions(); + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + return services; + }); + return builder; + } +} diff --git a/src/MeshWeaver.AI.OpenAI/README.md b/src/MeshWeaver.AI.OpenAI/README.md new file mode 100644 index 000000000..0ca93914e --- /dev/null +++ b/src/MeshWeaver.AI.OpenAI/README.md @@ -0,0 +1,51 @@ +# MeshWeaver.AI.OpenAI + +OpenAI-wire-protocol model providers for the MeshWeaver AI framework. One assembly +serves every provider that speaks the OpenAI chat-completions protocol — they differ +only by base URL and credentials: + +| Provider (`AddXxx`) | `ProviderName` | Endpoint | +|--------------------------|---------------------|--------------------------------------------| +| `AddOpenAI` | `OpenAI` | `https://api.openai.com` (SDK default) | +| `AddAzureOpenAI` | `AzureOpenAI` | your `*.openai.azure.com` deployment | +| `AddOpenAICompatible` | `OpenAICompatible` | any base URL you supply (OpenRouter, Groq, Together, vLLM, …) | + +Each `AddXxx` self-registers a `LanguageModelCatalogSource` (so the provider appears +in the chat model picker and the **Settings → Language Models** tab) plus its +`IChatClientFactory`. There is no central registry — a host opts into each provider it +needs, gated by the `Features:Ai:Providers:*` flags. + +## Registration + +```csharp +meshBuilder + .AddOpenAI() // direct api.openai.com + .AddAzureOpenAI() // Azure-hosted OpenAI deployment + .AddOpenAICompatible(); // generic custom-URL provider (OpenRouter, …) +``` + +## Credentials + +Credentials are **not** read from `appsettings` per user. Each saved provider is a +`nodeType:ModelProvider` mesh node carrying its endpoint + (encrypted) API key; each +`LanguageModel` child points back at it via `ModelDefinition.ProviderRef`. At chat time +`ChatClientCredentialResolver` follows that reference, so the factory builds an +`OpenAIClient` aimed at the right endpoint with the right key — and several distinct +OpenAI-compatible gateways coexist without collision. + +A system-default `OpenAI:` / `AzureOpenAI:` config section is still honoured as a +fallback for the built-in catalog. `OpenAICompatible` has no system default — the user +supplies the URL + key and fetches the model list live in the Settings tab. + +## Key components + +- `OpenAIChatClientAgentFactory` — serves `OpenAI` **and** `OpenAICompatible` (plain + `OpenAIClient` at the resolved endpoint). +- `AzureOpenAIChatClientAgentFactory` — serves `AzureOpenAI` (`AzureOpenAIClient`). +- `OpenAIExtensions` / `AzureOpenAIExtensions` — the `AddXxx` builder extensions. +- `OpenAIConfiguration` / `AzureOpenAIConfiguration` — system-default config shapes. + +## Related + +- [MeshWeaver.AI](../MeshWeaver.AI/README.md) — core AI abstractions, the provider/credential resolver, the catalog node types. +- [MeshWeaver.AI.AzureFoundry](../MeshWeaver.AI.AzureFoundry/README.md) — Azure AI Foundry + direct Anthropic. diff --git a/src/MeshWeaver.AI/AIExtensions.cs b/src/MeshWeaver.AI/AIExtensions.cs index 399827577..462a5ae71 100644 --- a/src/MeshWeaver.AI/AIExtensions.cs +++ b/src/MeshWeaver.AI/AIExtensions.cs @@ -1,9 +1,13 @@ -using MeshWeaver.AI.Plugins; +using System.Reactive.Linq; +using MeshWeaver.AI.Plugins; using MeshWeaver.Data; using MeshWeaver.Domain; using MeshWeaver.Layout; +using MeshWeaver.Mesh.Services; using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; namespace MeshWeaver.AI; @@ -15,13 +19,22 @@ public static class AIExtensions extension(TBuilder builder) where TBuilder : MeshBuilder { - public TBuilder AddAI() + public TBuilder AddAI(IReadOnlySet? serveFromPartition = null) { - // Register AI types in type registry and chat services + // Register AI types in type registry and chat services. serveFromPartition lists the + // partitions (e.g. "Agent", "Model") whose static content is DB-synced (static-repo + // import) — for those, the read-only in-memory partition provider is skipped so + // Postgres serves them. Null/empty = in-memory serving (current behaviour). return (TBuilder)builder .AddThreadMessageType() .AddThreadType() - .AddAgentType() + .AddTokenUsageType() + .AddAgentType(serveFromPartition) + .AddLanguageModelType(serveFromPartition) + .AddHarnessType(serveFromPartition) + .AddSkillType(serveFromPartition) + .AddThreadComposerType() + .AddAiSettingsType() .ConfigureServices(services => services.AddAgentChatServices()) // Register AI types on the MESH hub (for MeshQuery deserialization of Thread content) .ConfigureHub(config => @@ -39,8 +52,8 @@ public TBuilder AddAI() } } - private static async Task HandleSaveContent( - IMessageHub hub, IMessageDelivery delivery, CancellationToken ct) + private static IMessageDelivery HandleSaveContent( + IMessageHub hub, IMessageDelivery delivery) { var request = delivery.Message; var fileProvider = hub.ServiceProvider.GetService(); @@ -52,46 +65,60 @@ private static async Task HandleSaveContent( return delivery.Processed(); } - try - { - var stream = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes(request.TextContent)); - var result = await fileProvider.SaveFileContentAsync(request.CollectionName, request.FilePath, stream, ct); - - hub.Post(new Plugins.SaveContentResponse { Success = result.Success, Error = result.Error }, - o => o.ResponseFor(delivery)); - } - catch (Exception ex) - { - hub.Post(new Plugins.SaveContentResponse { Success = false, Error = ex.Message }, - o => o.ResponseFor(delivery)); - } + var stream = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes(request.TextContent)); + fileProvider.SaveFileContent(request.CollectionName, request.FilePath, stream) + .Subscribe( + result => hub.Post( + new Plugins.SaveContentResponse { Success = result.Success, Error = result.Error }, + o => o.ResponseFor(delivery)), + ex => hub.Post( + new Plugins.SaveContentResponse { Success = false, Error = ex.Message }, + o => o.ResponseFor(delivery))); return delivery.Processed(); } public static ITypeRegistry AddAITypes(this ITypeRegistry typeRegistry) => typeRegistry.WithType(typeof(AgentConfiguration), nameof(AgentConfiguration)) + .WithType(typeof(Harness), nameof(Harness)) .WithType(typeof(AgentDelegation), nameof(AgentDelegation)) + .WithType(typeof(ModelDefinition), nameof(ModelDefinition)) + .WithType(typeof(ModelProviderConfiguration), nameof(ModelProviderConfiguration)) .WithType(typeof(AI.Thread), nameof(AI.Thread)) .WithType(typeof(ThreadMessage), nameof(ThreadMessage)) - // MessageViewModel is not registered — handled as JsonElement on the wire - .WithType(typeof(SubmitMessageRequest), nameof(SubmitMessageRequest)) - .WithType(typeof(SubmitMessageResponse), nameof(SubmitMessageResponse)) - .WithType(typeof(AppendUserMessageRequest), nameof(AppendUserMessageRequest)) - .WithType(typeof(AppendUserMessageResponse), nameof(AppendUserMessageResponse)) - .WithType(typeof(ResubmitUserMessageRequest), nameof(ResubmitUserMessageRequest)) - .WithType(typeof(RecordSubmissionFailureRequest), nameof(RecordSubmissionFailureRequest)) -.WithType(typeof(CancelThreadStreamRequest), nameof(CancelThreadStreamRequest)) - .WithType(typeof(ResubmitMessageRequest), nameof(ResubmitMessageRequest)) - .WithType(typeof(DeleteFromMessageRequest), nameof(DeleteFromMessageRequest)) + // Per-(thread, model) token usage satellite at {threadPath}/_Usage/{model}. The thread + // node carries NO token state — usage lives here. Registered mesh-wide so the node + // serialises across routing / mesh / per-node hubs. + .WithType(typeof(TokenUsage), nameof(TokenUsage)) + // ThreadComposer: content of the per-user {user}/_Thread/ThreadComposer composer + // singleton (message text + harness/agent/model + attachments). Registered + // mesh-wide so the node serialises across routing/mesh hubs. + .WithType(typeof(ThreadComposer), nameof(ThreadComposer)) + // AiSettings: per-user {user}/_Memex/AiSettings config (enabled harnesses + agent/model + // picker query templates). Registered mesh-wide so the node serialises across hubs. + .WithType(typeof(AiSettings), nameof(AiSettings)) + // MessageViewModel is not registered — handled as JsonElement on the wire. + // SubmitMessageRequest / SubmitMessageResponse deleted 2026-05-25: + // the only mutation API is workspace.GetMeshNodeStream(path).Update(...). + // Public submission flow is ThreadSubmission.Submit → ThreadInput.AppendUserInput + // (writes PendingUserMessages); the submission watcher reacts and invokes + // ExecuteMessageAsync directly as a method (no wire message). + // See AGENTS.md → "GetMeshNodeStream().Update() is the ONLY mutation API" + // and Doc/Architecture/RequestViaStreamUpdate.md. + // Thread mutation triggers and intent payloads (ResubmitIntent, + // FailureRecord, RequestedResubmit / RequestedDeleteFromMessageId / + // PendingFailures fields) were deleted — HubThreadExtensions does the + // full mutation inline via a single stream.Update on the thread node. .WithType(typeof(ToolCallEntry), nameof(ToolCallEntry)) - .WithType(typeof(UpdateThreadMessageContent), nameof(UpdateThreadMessageContent)) - .WithType(typeof(DelegationCompletedEvent), nameof(DelegationCompletedEvent)) .WithType(typeof(NodeChangeEntry), nameof(NodeChangeEntry)) .WithType(typeof(ThreadExecutionContext), nameof(ThreadExecutionContext)) // ChatHistoryEntry removed — ChatHistory uses string[] to avoid $type issues .WithType(typeof(SaveContentRequest), nameof(SaveContentRequest)) - .WithType(typeof(SaveContentResponse), nameof(SaveContentResponse)); + .WithType(typeof(SaveContentResponse), nameof(SaveContentResponse)) + // Delegation heartbeat: parent-thread-hub-scoped messages for + // hung-sub-thread detection + cancel propagation. + .WithType(typeof(Delegation.HeartbeatTick), nameof(Delegation.HeartbeatTick)) + .WithType(typeof(Delegation.CancelDelegationSubThread), nameof(Delegation.CancelDelegationSubThread)); extension(IServiceCollection services) { @@ -102,8 +129,13 @@ public static ITypeRegistry AddAITypes(this ITypeRegistry typeRegistry) /// public IServiceCollection AddAgentChatServices() { - services.AddOptions() - .BindConfiguration("ModelTier"); + services.AddTransient(); + services.AddTransient(); + + // Slash-skills are declarative nodeType:Skill mesh nodes (BuiltInSkillProvider, imported to + // PG), extensible per Space/NodeType/user via namespace inheritance — there is no C# command + // registry. See SkillNodeType / SkillAutocompleteProvider. + return services; } diff --git a/src/MeshWeaver.AI/AgentChatClient.cs b/src/MeshWeaver.AI/AgentChatClient.cs index fbcb05f92..7f9f9f422 100644 --- a/src/MeshWeaver.AI/AgentChatClient.cs +++ b/src/MeshWeaver.AI/AgentChatClient.cs @@ -1,6 +1,8 @@ using System.Collections.Concurrent; using System.Collections.Immutable; using System.Reactive.Linq; +using System.Reactive.Subjects; +using System.Reactive.Threading.Tasks; using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; @@ -9,6 +11,7 @@ using MeshWeaver.Data; using MeshWeaver.Graph; using MeshWeaver.Layout; +using MeshWeaver.Mesh; using MeshWeaver.Mesh.Services; using MeshWeaver.Messaging; using MeshWeaver.ShortGuid; @@ -31,8 +34,14 @@ public class AgentChatClient : IAgentChat private HandoffRequest? pendingHandoff; private ImmutableList loadedAgents = ImmutableList.Empty; private string? lastLoadedContextPath; + private string? lastLoadedNodeTypePath; private string currentThreadId = Guid.NewGuid().AsString(); private string? currentAgentName; + // The user's explicit picker selection, kept as the FULL node PATH + // ("AgenticPension/Agent/Datenextraktion"). Resolution prefers an exact path + // match against loadedAgents so a space-scoped agent never collides with a + // built-in of the same last segment; SelectionId.IdOf gives the bare-id fallback. + private string? currentAgentPath; private AgentSession? sharedThread; private string? currentModelName; private string? persistentThreadId; @@ -51,12 +60,103 @@ public class AgentChatClient : IAgentChat // Conversation history loaded from persisted ThreadMessage nodes for resume private IReadOnlyList? conversationHistory; - public AgentChatClient(IServiceProvider serviceProvider) + /// + /// Captured during when an + /// throws while materialising an agent + /// (typical cause: factory matches the model via Supports(...) + /// but its underlying config — Endpoint / ApiKey — isn't set). Surfaced + /// in the chat response when returns null, so + /// the user sees the actual misconfiguration instead of the opaque + /// "No suitable agent found" string. + /// + private string? lastAgentCreationError; + + /// + /// Formats the no-agent failure into an APPROPRIATE chat output — never a crash. + /// The common cause is "no model configured" (every agent skipped via the unconfigured + /// catch-all factory); surface that with a clear, actionable hint plus the raw detail. + /// + private static string FormatNoAgentError(string? detail) + { + detail ??= "No suitable agent found to handle the request."; + var noModel = detail.Contains("must be configured", StringComparison.OrdinalIgnoreCase) + || detail.Contains("no model", StringComparison.OrdinalIgnoreCase) + || detail.Contains("no registered IChatClientFactory", StringComparison.OrdinalIgnoreCase); + return noModel + ? "⚠️ No AI model is available to run this request. Configure a language-model provider " + + "(Settings → Language Models) and select a model, or switch to the Claude Code harness.\n\n" + + $"_Details: {detail}_" + : detail; + } + + /// + /// Mesh-discovered models — bring-your-own-model entries from + /// nodeType:Model nodes. Surfaced into ThreadChatView's picker + /// alongside the factory-provided defaults. + /// + private ImmutableList loadedModels = ImmutableList.Empty; + + /// + /// Snapshot of the synced model collection. Read by the chat view on + /// every emission. + /// + public IReadOnlyList LoadedModels => loadedModels; + + // Live subscription to the workspace-level synced agent collection. Disposed + // and replaced on every Initialize() call so the current context's queries + // become the active source. The synced query itself is cached at the + // workspace level (per (contextPath, nodeType) tuple), so subscribing + // again to the same id reuses one upstream subscription across instances. + private IDisposable? agentsSubscription; + + // Latches the current loadedAgents emission as a hot observable so callers + // (ThreadExecution + tests) can opt into an explicit ready-gate without + // forcing Initialize itself to be async. + private readonly ReplaySubject agentsLoadedSubject = new(bufferSize: 1); + + /// + /// Hot observable that emits this client every time the synced agent + /// collection refreshes (initial load + every subsequent change). Replays + /// the latest emission to new subscribers, so callers can use + /// chat.WhenInitialized.Take(1) as a "wait until agents are ready" + /// gate without coupling to the cold/warm distinction inside + /// . + /// + public IObservable WhenInitialized => agentsLoadedSubject; + + /// + /// Constructs a chat client with optional prior conversation history — + /// passed in from the caller (typically ThreadExecution on a fresh + /// grain after restart). The client itself stays out of the history-load + /// concern: callers fetch prior messages and inject them via this ctor. + /// + public AgentChatClient(IServiceProvider serviceProvider, IReadOnlyList? priorMessages = null) { hub = serviceProvider.GetRequiredService(); logger = serviceProvider.GetRequiredService>(); meshQuery = serviceProvider.GetService(); chatClientFactories = serviceProvider.GetServices().ToImmutableList(); + if (priorMessages is { Count: > 0 }) + conversationHistory = priorMessages; + } + + /// + /// Strips any trailing satellite segments (segments starting with '_', e.g. "_Thread/<slug>", + /// "_Comment/<id>") from a context path so the agent reasons about the main node, not the + /// satellite. Returns the input unchanged when null/empty or when no '_' segment is present. + /// + private static string? NormalizeContextPath(string? path) + { + if (string.IsNullOrEmpty(path)) + return path; + + var segments = path.Split('/'); + for (var i = 0; i < segments.Length; i++) + { + if (segments[i].StartsWith('_')) + return string.Join('/', segments, 0, i); + } + return path; } public AgentContext? Context { get; private set; } @@ -64,14 +164,53 @@ public AgentChatClient(IServiceProvider serviceProvider) /// public ThreadExecutionContext? ExecutionContext { get; private set; } - /// - public string? LastDelegationPath { get; set; } + /// + /// Backing Subject for . ExecuteDelegationAsync + /// emits onto this directly; subscribers (cancel watcher, tool-call + /// stamper) receive the events on the emitting thread — they're + /// expected to defer real work to a Hub.Post handler rather than + /// mutating state inline (see plan Slice 2c). + /// + private readonly System.Reactive.Subjects.Subject _delegations = new(); /// - public ConcurrentDictionary DelegationPaths { get; } = new(); + public IObservable Delegations => _delegations; - /// - public Action? UpdateDelegationStatus { get; set; } + /// + /// Internal hook for ExecuteDelegationAsync to emit lifecycle events. + /// Public would invite outside-the-agent writes; internal keeps + /// the emit surface tied to the agent factory in this assembly. + /// + internal void EmitDelegationEvent(MeshWeaver.AI.Delegation.DelegationEvent evt) + { + switch (evt.Phase) + { + case MeshWeaver.AI.Delegation.DelegationLifecycle.Dispatched: + ImmutableInterlocked.Update(ref _activeDelegationPaths, set => set.Add(evt.SubThreadPath)); + break; + case MeshWeaver.AI.Delegation.DelegationLifecycle.Terminal: + ImmutableInterlocked.Update(ref _activeDelegationPaths, set => set.Remove(evt.SubThreadPath)); + break; + } + _delegations.OnNext(evt); + } + + /// + /// In-memory set of sub-thread paths currently in flight on this chat + /// session. Maintained by : Dispatched + /// adds, Terminal removes. Read by the cancel watcher in + /// ThreadExecution.SetupCancellationWatcher to propagate cancel + /// to sub-threads whose paths haven't yet been persisted onto + /// Thread.StreamingToolCalls[].DelegationPath, and by the + /// streaming-loop's stamp pass that walks unmatched + /// delegate_to_agent tool-call entries. Replaces the legacy + /// DelegationPaths dictionary (which keyed by transient + /// display name). + /// + private ImmutableHashSet _activeDelegationPaths = ImmutableHashSet.Empty; + + /// Snapshot of active delegation sub-thread paths. + public ImmutableHashSet ActiveDelegationPaths => _activeDelegationPaths; /// public Action? ForwardToolCall { get; set; } @@ -125,6 +264,15 @@ public void SetAttachments(IReadOnlyList? paths) currentAttachments = paths is { Count: > 0 } ? paths : null; } + /// + /// True once an has been created for this client — + /// i.e. at least one streaming response has run, so the agent is carrying the + /// conversation in memory. The thread-execution layer uses this as the + /// "is this a fresh grain?" signal to decide whether persisted history needs + /// to be re-loaded from the mesh. + /// + public bool HasActiveSession => sharedThread != null; + /// /// Sets conversation history from persisted ThreadMessage nodes. /// Injected once into the next message context, then cleared (the AgentSession @@ -144,7 +292,7 @@ private async Task GetOrCreateThreadAsync(ChatClientAgent agent) // For persistent factories with a persistent thread ID, create a session linked to the server-side thread if (isPersistentFactory && !string.IsNullOrEmpty(persistentThreadId)) { - sharedThread = await agent.CreateSessionAsync(persistentThreadId); + sharedThread = await agent.CreateSessionAsync(persistentThreadId).ConfigureAwait(false); logger.LogInformation("Resumed persistent thread: {PersistentThreadId}", persistentThreadId); return sharedThread; } @@ -152,13 +300,13 @@ private async Task GetOrCreateThreadAsync(ChatClientAgent agent) if (isPersistentFactory) { // For persistent factories without an existing thread, create a new server-side session - sharedThread = await agent.CreateSessionAsync(currentThreadId); + sharedThread = await agent.CreateSessionAsync(currentThreadId).ConfigureAwait(false); persistentThreadId = currentThreadId; logger.LogInformation("Created new persistent thread: {PersistentThreadId}", persistentThreadId); } else { - sharedThread = await agent.CreateSessionAsync(); + sharedThread = await agent.CreateSessionAsync().ConfigureAwait(false); } return sharedThread; @@ -172,19 +320,20 @@ private Task SaveThreadAsync(ChatClientAgent agent, AgentSession thread, string? /// /// Updates the Thread MeshNode with PersistentThreadId and ProviderType if they were newly set. + /// Routes the update through the IMeshNodeStreamCache so the patch lands on + /// the owning per-thread hub's stream — no separate DataChangeRequest → + /// owner round-trip that can dangle as a pending callback if the owner + /// is mid-streaming (the SubThreadHangRepro tests caught this exact leak). /// - private async Task UpdateThreadPersistentIdAsync(string threadNodePath) + private Task UpdateThreadPersistentIdAsync(string threadNodePath) { - try - { - var factory = GetFactoryForModel(currentModelName); - var workspace = hub.ServiceProvider.GetRequiredService(); - workspace.UpdateMeshNode(node => + var factory = GetFactoryForModel(currentModelName); + var workspace = hub.GetWorkspace(); + workspace.GetMeshNodeStream(threadNodePath) + .Update(node => { - if (node.Content is not Thread threadContent) - return node; - if (!string.IsNullOrEmpty(threadContent.PersistentThreadId)) - return node; // Already set + if (node?.Content is not Thread threadContent) return node!; + if (!string.IsNullOrEmpty(threadContent.PersistentThreadId)) return node; return node with { Content = threadContent with @@ -193,15 +342,14 @@ private async Task UpdateThreadPersistentIdAsync(string threadNodePath) ProviderType = factory?.Name } }; - }, address: new Messaging.Address(threadNodePath)); - - logger.LogInformation("Updated thread {Path} with PersistentThreadId={PersistentThreadId}", - threadNodePath, persistentThreadId); - } - catch (Exception ex) - { - logger.LogWarning(ex, "Failed to update PersistentThreadId on thread {Path}", threadNodePath); - } + }) + .Subscribe( + _ => logger.LogInformation( + "Updated thread {Path} with PersistentThreadId={PersistentThreadId}", + threadNodePath, persistentThreadId), + ex => logger.LogWarning(ex, + "Failed to update PersistentThreadId on thread {Path}", threadNodePath)); + return Task.CompletedTask; } /// @@ -248,7 +396,7 @@ private async Task UpdateThreadPersistentIdAsync(string threadNodePath) } } - var toolDocs = cachedToolDocs ?? await LoadToolDocumentationAsync(); + var toolDocs = cachedToolDocs ?? await LoadToolDocumentationAsync().ConfigureAwait(false); if (!string.IsNullOrEmpty(toolDocs)) { sb.AppendLine("# Available Tools Documentation"); @@ -277,6 +425,20 @@ private async Task UpdateThreadPersistentIdAsync(string threadNodePath) messageText.AppendLine(); } + // Turn repetition into a reusable Skill — PROACTIVE. The trigger is a repeating + // user, so one-shot / utility agents (NodeInitializer, DescriptionWriter, …) simply + // never fire it. Kept in the shared base prompt so every conversational agent offers it. + messageText.AppendLine("# Turn repetition into a Skill (proactive)"); + messageText.AppendLine(); + messageText.AppendLine("When the user asks for the SAME multi-step task more than once (in this thread or across threads), proactively offer to save it as a reusable **Skill** — e.g. \"Want me to save this as a `/` skill you can re-run anytime?\". Don't wait to be asked."); + messageText.AppendLine(); + messageText.AppendLine("\"Create a skill\" means create a `nodeType:Skill` node (the same mechanism behind `/agent`, `/model`, `/harness`). Use `create` with `content` = a `SkillDefinition`:"); + messageText.AppendLine("- node **id** = the slash word (`/`); **name** + **description** = its display name and help text."); + messageText.AppendLine("- `Instructions` = a how-to (the SKILL.md body) the agent loads on demand — for \"run these steps\" skills — and/or `Action` = a behaviour (`Pick` a node by a query and write it to the composer, `OpenContent`, `Connect`/`Disconnect`)."); + messageText.AppendLine("- Place it under the user's own namespace + `/Skill` (private to them) or the Space's `{space}/Skill` (shared with everyone in that Space); the platform-wide catalog is `Skill`."); + messageText.AppendLine("Once created, `/` works in chat immediately — no code. Full SkillDefinition shape: `/Doc/AI/ChatCommands`."); + messageText.AppendLine(); + // Dynamic part: context (changes per navigation) if (Context != null) { @@ -296,29 +458,13 @@ private async Task UpdateThreadPersistentIdAsync(string threadNodePath) } messageText.AppendLine(); - // Show nearby nodes so the agent understands the structure - if (!string.IsNullOrEmpty(contextPath)) - { - try - { - var nearby = ImmutableList.Empty; - var meshQuery = hub.ServiceProvider.GetRequiredService(); - await foreach (var node in meshQuery.QueryAsync( - $"namespace:{contextPath} select:name,nodeType,icon")) - { - nearby = nearby.Add($"- `{node.Path}` ({node.NodeType}): {node.Name}"); - if (nearby.Count >= 15) break; - } - if (nearby.Count > 0) - { - messageText.AppendLine("**Children of current node:**"); - foreach (var line in nearby) - messageText.AppendLine(line); - messageText.AppendLine(); - } - } - catch { /* ignore context loading errors */ } - } + // 🚨 No await foreach over IMeshService.QueryAsync here. + // That used to enumerate children of the context path to inject + // into the system prompt. The fan-out through IMeshQueryProvider + // bridges back through hub messaging in a way that can park the + // streaming task indefinitely (chat stuck on "Generating + // response..."). The agent has a `Search` tool — it can pull the + // children itself when it actually needs them. messageText.AppendLine("When creating nodes, first explore the structure to find the best place:"); messageText.AppendLine("- `Search('namespace:{path} scope:descendants')` to see the full tree"); @@ -357,13 +503,13 @@ private async Task UpdateThreadPersistentIdAsync(string threadNodePath) { // Load binary from content collection on the node's hub var effectivePath = nodePath ?? Context?.Path; - var stream = await contentService.GetContentAsync("content", fileName); + var stream = await contentService.GetContentAsync("content", fileName).ConfigureAwait(false); if (stream != null) { using (stream) { using var ms = new MemoryStream(); - await stream.CopyToAsync(ms); + await stream.CopyToAsync(ms).ConfigureAwait(false); var mediaType = ExtensionToMediaType.GetValueOrDefault(ext, "application/octet-stream"); binaryAttachments = binaryAttachments.Add( new DataContent(ms.ToArray(), mediaType) { Name = fileName }); @@ -376,7 +522,7 @@ private async Task UpdateThreadPersistentIdAsync(string threadNodePath) } // Text attachment — load via MeshPlugin.Get - var content = await meshPlugin.Get($"@{cleanPath}"); + var content = await meshPlugin.Get($"@{cleanPath}").ConfigureAwait(false); if (!string.IsNullOrEmpty(content) && !content.StartsWith("Not found") && !content.StartsWith("Error")) { if (content.Length > 8000) @@ -420,7 +566,7 @@ private async Task UpdateThreadPersistentIdAsync(string threadNodePath) foreach (var message in messages) { var text = ExtractTextFromMessage(message); - text = await InlineReferenceResolver.ResolveAsync(text, hub, this); + text = await InlineReferenceResolver.ResolveAsync(text, hub, this).ConfigureAwait(false); messageText.Append(text); } @@ -433,7 +579,7 @@ private async Task UpdateThreadPersistentIdAsync(string threadNodePath) private async Task LoadToolDocumentationAsync() { var meshPlugin = new MeshPlugin(hub, this); - var docs = await meshPlugin.Get("@Doc/AI/Tools/MeshPlugin"); + var docs = await meshPlugin.Get("@Doc/AI/Tools/MeshPlugin").ConfigureAwait(false); if (docs.StartsWith("Not found") || docs.StartsWith("Error")) return string.Empty; @@ -446,7 +592,7 @@ private async Task LoadToolDocumentationAsync() { var content = contentElement.GetString() ?? string.Empty; // Resolve @@ references in tool documentation (e.g., @@QuerySyntax, @@UnifiedPath) - content = await InlineReferenceResolver.ResolveAsync(content, hub, this); + content = await InlineReferenceResolver.ResolveAsync(content, hub, this).ConfigureAwait(false); return content; } // If content is not a simple string, return empty @@ -471,31 +617,39 @@ public async IAsyncEnumerable GetResponseAsync( var lastMessageText = messages.LastOrDefault() is { } last ? ExtractTextFromMessage(last) : null; DetectMessageAgentReferences(lastMessageText); - // Select which agent to use (async to avoid deadlock in Blazor context) - var agent = await SelectAgentAsync(messages.LastOrDefault()); + // Pure in-memory lookup against the synced agent cache. + // 🚨 Callers MUST await `WhenInitialized` before calling here on a + // cold path — this method does not gate, because the gate must run + // on a non-hub thread (Task.Run in ThreadExecution.ExecuteMessageAsync). + // Awaiting WhenInitialized here would block the hub's ActionBlock if + // GetResponseAsync runs on it. + var agent = SelectAgent(messages.LastOrDefault()); if (agent == null) { - yield return new ChatMessage(ChatRole.Assistant, "No suitable agent found to handle the request."); + // Never crash — surface the real reason AS the chat output. The common case + // is "no model configured" (every agent skipped via the unconfigured catch-all + // factory), so make that actionable. + yield return new ChatMessage(ChatRole.Assistant, FormatNoAgentError(lastAgentCreationError)); yield break; } currentAgentName = agent.Name; // Get or create thread for this agent - var thread = await GetOrCreateThreadAsync(agent); + var thread = await GetOrCreateThreadAsync(agent).ConfigureAwait(false); // Build the user message with context and agent instructions - var (userText, binaryParts) = await BuildMessageWithContextAsync(messages, currentAgentName); + var (userText, binaryParts) = await BuildMessageWithContextAsync(messages, currentAgentName).ConfigureAwait(false); currentAttachments = null; // Clear after use // Build ChatMessage with mixed content (text + binary attachments) var chatMessage = BuildChatMessage(userText, binaryParts); // Get response from the agent with thread - var response = await agent.RunAsync(chatMessage, thread, cancellationToken: cancellationToken); + var response = await agent.RunAsync(chatMessage, thread, cancellationToken: cancellationToken).ConfigureAwait(false); // Save the updated thread - await SaveThreadAsync(agent, thread); + _ = SaveThreadAsync(agent, thread); // no-op; method returns Task.CompletedTask foreach (var responseMsg in response.Messages) { @@ -559,8 +713,8 @@ public async IAsyncEnumerable GetResponseAsync( handoff.SourceAgentName, targetId); // Run target agent on the same shared thread with the handoff message - var handoffResponse = await targetAgent.RunAsync(handoff.Message, thread, cancellationToken: cancellationToken); - await SaveThreadAsync(targetAgent, thread); + var handoffResponse = await targetAgent.RunAsync(handoff.Message, thread, cancellationToken: cancellationToken).ConfigureAwait(false); + _ = SaveThreadAsync(targetAgent, thread); // no-op; method returns Task.CompletedTask foreach (var msg in handoffResponse.Messages) { @@ -591,18 +745,21 @@ public async IAsyncEnumerable GetStreamingResponseAsync( var lastMessageTextStreaming = messages.LastOrDefault() is { } lastMsg ? ExtractTextFromMessage(lastMsg) : null; DetectMessageAgentReferences(lastMessageTextStreaming); - // Select which agent to use (async to avoid deadlock in Blazor context) - var agent = await SelectAgentAsync(messages.LastOrDefault()); + // 🚨 Callers MUST await `WhenInitialized` before invoking on a cold + // path — gate is at the caller, not here, because awaiting would block + // the hub ActionBlock if GetStreamingResponseAsync is called on it. + var agent = SelectAgent(messages.LastOrDefault()); if (agent == null) { - yield return new ChatResponseUpdate(ChatRole.Assistant, "No suitable agent found to handle the request."); + // Never crash — surface the real reason AS the streamed chat output. + yield return new ChatResponseUpdate(ChatRole.Assistant, FormatNoAgentError(lastAgentCreationError)); yield break; } currentAgentName = agent.Name; // Get or create thread for this agent - var thread = await GetOrCreateThreadAsync(agent); + var thread = await GetOrCreateThreadAsync(agent).ConfigureAwait(false); // Pass all messages as separate turns with system prompt prepended. // The agent's ChatClient includes FunctionInvokingChatClient for tool calls. @@ -610,7 +767,7 @@ public async IAsyncEnumerable GetStreamingResponseAsync( if (!string.IsNullOrEmpty(agent.Instructions)) turnMessages.Add(new ChatMessage(ChatRole.System, agent.Instructions)); turnMessages.AddRange(messages); - logger.LogInformation("[AgentChat] Sending {Count} messages (+ system) to {Agent}", + logger.LogDebug("[AgentChat] Sending {Count} messages (+ system) to {Agent}", messages.Count, agent.Name); currentAttachments = null; @@ -621,9 +778,21 @@ public async IAsyncEnumerable GetStreamingResponseAsync( // ChatClientAgent constructor places them). var functionInvoker = agent.ChatClient.GetService(); var chatOptions = new ChatOptions(); - if (functionInvoker?.AdditionalTools is { Count: > 0 } additionalTools) - chatOptions.Tools = additionalTools.ToList(); - await foreach (var update in agent.ChatClient.GetStreamingResponseAsync(turnMessages, chatOptions, cancellationToken)) + var tools = functionInvoker?.AdditionalTools is { Count: > 0 } additionalTools + ? additionalTools.ToList() + : new List(); + // Always inject check_inbox so the agent can poll for user messages + // queued during the in-flight turn. The tool drains + // Thread.PendingUserMessages atomically and returns the texts so the + // agent can fold them into the current response — see InboxTool. + tools.Add(InboxTool.CreateCheckInboxTool(hub, logger)); + chatOptions.Tools = tools; + // ConfigureAwait(false): keep the agent-stream iteration on the ThreadPool (the + // IoPool's domain), never resuming on a captured hub/grain action-block scheduler. + // This generator is consumed by ThreadExecution's round-streaming await foreach; a + // captured hub context that isn't pumped under a 2-core runner stalls the round + // (missed-observation deadlock). See ThreadExecution's STREAM await foreach. + await foreach (var update in agent.ChatClient.GetStreamingResponseAsync(turnMessages, chatOptions, cancellationToken).ConfigureAwait(false)) { // Forward the complete update with all contents (including FunctionCallContent) if (update.Contents.Count > 0) @@ -675,7 +844,7 @@ public async IAsyncEnumerable GetStreamingResponseAsync( } // Save the updated thread - await SaveThreadAsync(agent, thread); + _ = SaveThreadAsync(agent, thread); // no-op; method returns Task.CompletedTask // Check for any queued layout area content while (!queuedLayoutAreaContent.IsEmpty) @@ -715,8 +884,10 @@ public async IAsyncEnumerable GetStreamingResponseAsync( logger.LogInformation("Handoff (streaming): {Source} -> {Target}, running target agent on shared thread", handoff.SourceAgentName, targetId); - // Run target agent streaming on the same shared thread - await foreach (var update in targetAgent.RunStreamingAsync(handoff.Message, thread, cancellationToken: cancellationToken)) + // Run target agent streaming on the same shared thread. + // ConfigureAwait(false): same rule — never resume the handoff stream on a + // captured hub scheduler (keep it on the ThreadPool / IoPool domain). + await foreach (var update in targetAgent.RunStreamingAsync(handoff.Message, thread, cancellationToken: cancellationToken).ConfigureAwait(false)) { if (update.Contents.Count > 0) { @@ -731,6 +902,20 @@ public async IAsyncEnumerable GetStreamingResponseAsync( AuthorName = currentAgentName ?? "Assistant" }; } + else if (content is FunctionResultContent functionResult) + { + // Previously dropped: handoff path forwarded FunctionCallContent + // but never the matching FunctionResultContent, so tool calls + // made by a handoff target stayed "pending" forever in + // ThreadExecution.toolCallLog — visible to the user as a tool + // call with no result. Forward results too. + logger.LogInformation("Agent {AgentName} received result from tool: {CallId}", + currentAgentName, functionResult.CallId); + yield return new ChatResponseUpdate(ChatRole.Assistant, [content]) + { + AuthorName = currentAgentName ?? "Assistant" + }; + } else if (content is UsageContent) { yield return new ChatResponseUpdate(ChatRole.Assistant, [content]) @@ -750,7 +935,7 @@ public async IAsyncEnumerable GetStreamingResponseAsync( } } - await SaveThreadAsync(targetAgent, thread); + _ = SaveThreadAsync(targetAgent, thread); // no-op; method returns Task.CompletedTask // Yield any queued layout area content from the handoff target while (!queuedLayoutAreaContent.IsEmpty) @@ -769,7 +954,15 @@ public async IAsyncEnumerable GetStreamingResponseAsync( private static readonly Regex AgentReferencePattern = new(@"@agent/(\w+)", RegexOptions.Compiled | RegexOptions.IgnoreCase); - private async Task SelectAgentAsync(ChatMessage? lastMessage) + /// + /// Pure in-memory selection over the synced agent cache ( + /// + ). No queries, no awaits — both collections are + /// kept fresh by the workspace-level synced query subscription wired up in + /// . Anyone who needs the agent list elsewhere + /// must read from those caches; do NOT add per-call QueryAsync + /// fallbacks here. + /// + private ChatClientAgent? SelectAgent(ChatMessage? lastMessage) { // 1. Check for explicit @agent/Name reference in message (highest priority) if (lastMessage != null) @@ -802,315 +995,418 @@ public async IAsyncEnumerable GetStreamingResponseAsync( return foundRef.Value; } - // 3. Use explicitly selected agent (from dropdown) if set - if (!string.IsNullOrEmpty(currentAgentName) && agents.TryGetValue(currentAgentName, out var selectedAgent)) - return selectedAgent; + // 3. Use the explicitly selected agent (from the dropdown / composer) if set. + // The picker stores the FULL node PATH, so resolve by path FIRST: a + // space-scoped agent ("AgenticPension/Agent/Datenextraktion") must never be + // confused with a built-in sharing its last segment. Bare-id is the fallback + // (legacy bare-name selections + built-ins picked by id). + // 🚨 An EXPLICIT-but-unresolvable selection does NOT silently fall through to + // a different agent (steps 4-6): that produced the wrong-agent / NotFound + // confusion in prod. Surface a clear, named error instead — GetResponseAsync / + // GetStreamingResponseAsync render lastAgentCreationError as the chat output. + if (!string.IsNullOrEmpty(currentAgentPath) || !string.IsNullOrEmpty(currentAgentName)) + { + var explicitlySelected = ResolveSelectedAgent(); + if (explicitlySelected != null) + return explicitlySelected; + + var requested = currentAgentPath ?? currentAgentName; + lastAgentCreationError = + $"Selected agent '{requested}' was not found among the available agents " + + $"([{string.Join(", ", loadedAgents.Select(a => a.Path ?? a.Name))}]). " + + "It may have been moved, renamed, or is not available in this context — " + + "pick another agent from the list."; + logger.LogWarning("[AgentChatClient] {Error}", lastAgentCreationError); + return null; + } - // 4. Use ordered agents - first one is the best match for context - // GetOrderedAgentsAsync already handles: context pattern matching, NodeType namespace, path relevance - var orderedAgents = await GetOrderedAgentsAsync(); - if (orderedAgents.Count > 0) + // 4. Prefer the configuration-marked default agent (IsDefault=true). + // 🚨 Without this, the fallback below routes to loadedAgents[0] + // which depends on AgentOrderingHelper.OrderByRelevance — order + // can vary across runs because the synced query's emission timing + // is non-deterministic. Concrete failure: SubThreadHangRepro's + // second [Fact] routed to NodeInitializer instead of the + // Assistant default, sending the test through + // HangingSubAgentChatClient (which hangs forever) instead of + // DelegatingParentChatClient. Symptom: STREAM_BEGIN logs, then + // no further activity — first MoveNext on the inner client never + // returns. + var defaultAgentInfo = loadedAgents.FirstOrDefault(a => a.AgentConfiguration?.IsDefault == true); + if (defaultAgentInfo is not null && agents.TryGetValue(defaultAgentInfo.Name, out var defaultAgent)) + return defaultAgent; + + // 5. Use the synced ordered list — best match for context, kept fresh + // by the workspace synced query (see Initialize). + if (loadedAgents.Count > 0) { - var bestAgent = orderedAgents[0]; + var bestAgent = loadedAgents[0]; if (agents.TryGetValue(bestAgent.Name, out var agent)) return agent; } - // 5. Return first agent as fallback + // 6. Return first agent as fallback return agents.Values.FirstOrDefault(); } + + /// + /// Resolves the user's explicit picker selection ( / + /// ) to a , or null when the + /// selection matches no loaded agent. + /// Match order: exact FULL PATH against first (so a + /// space-scoped agent is never confused with a built-in sharing its last segment), + /// then bare id. The created-agents dictionary is keyed by the bare id + /// () because delegation/hand-off resolve by id — + /// so when two loaded agents share a last segment (a built-in and a space override), + /// the dictionary holds only one. When the dictionary entry is NOT the path-matched + /// config (a genuine collision), build the right agent on demand from the matched + /// config so the user gets the agent they actually picked. + /// + private ChatClientAgent? ResolveSelectedAgent() + { + // a) Exact full-path match (the picker stores the node path). + var matched = !string.IsNullOrEmpty(currentAgentPath) + ? loadedAgents.FirstOrDefault(a => + string.Equals(a.Path, currentAgentPath, StringComparison.OrdinalIgnoreCase)) + : null; + + // b) Bare-id match (legacy bare-name selection, or a built-in picked by id). + matched ??= !string.IsNullOrEmpty(currentAgentName) + ? loadedAgents.FirstOrDefault(a => + string.Equals(a.Name, currentAgentName, StringComparison.OrdinalIgnoreCase) + || string.Equals(a.AgentConfiguration?.Id, currentAgentName, StringComparison.OrdinalIgnoreCase)) + : null; + + if (matched?.AgentConfiguration is not { } config) + return null; + + // The created-agents dict is keyed by bare id. The common (no-collision) case: + // the dict entry IS this config — return it. + if (agents.TryGetValue(config.Id, out var existing) + && string.Equals(existing.Instructions, config.Instructions, StringComparison.Ordinal)) + return existing; + + // Collision (or the agent was skipped during the batch build): construct the + // path-matched agent on demand so the selection resolves to the RIGHT one rather + // than whichever same-id agent won the dictionary slot. + var factory = GetFactoryForModel(currentModelName); + if (factory == null) + return existing; // no factory to build with — best effort + + try + { + return factory.CreateAgent(config, this, agents, + loadedAgents.Select(a => a.AgentConfiguration).ToImmutableList(), currentModelName); + } + catch (Exception ex) + { + lastAgentCreationError = + $"Failed to create selected agent '{matched.Path ?? config.Id}' via factory " + + $"'{factory.Name}' for model '{currentModelName}': {ex.Message}"; + logger.LogWarning(ex, "[AgentChatClient] {Error}", lastAgentCreationError); + return existing; + } + } + /// public void SetSelectedAgent(string? agentName) { - currentAgentName = agentName; + // The picker stores the node PATH ("Agent/Coder", + // "AgenticPension/Agent/Datenextraktion"); a bare name is also accepted. + // Keep BOTH forms: the full path drives an exact-path match in SelectAgent + // (so a space-scoped agent isn't confused with a built-in sharing its last + // segment), and the bare id is the fallback / dictionary key. + currentAgentPath = string.IsNullOrEmpty(agentName) ? null : agentName; + currentAgentName = SelectionId.IdOf(agentName); } public void SetContext(AgentContext? applicationContext) { + // Normalize at the boundary: strip satellite segments (e.g. "_Thread/") so + // the agent reasons about the main context node, not the thread/comment under it. + if (applicationContext is { Path: { Length: > 0 } p }) + { + var normalized = NormalizeContextPath(p); + if (!string.IsNullOrEmpty(normalized) && !string.Equals(normalized, p, StringComparison.Ordinal)) + applicationContext = applicationContext with { Path = normalized }; + } Context = applicationContext; + + // Re-initialise the synced-agent subscription whenever the context node's + // NodeType changes — that's what determines the third per-NodeType query + // in BuildAgentQueries (`namespace:{nodeTypePath} ... scope:selfAndAncestors`). + // Without this, agents defined under the NodeType path (e.g. TodoAgent at + // ACME/Project) never surface for an instance whose NodeType points at it. + var newNodeTypePath = applicationContext?.Node?.NodeType; + if (lastLoadedContextPath != null && newNodeTypePath != lastLoadedNodeTypePath) + Initialize(lastLoadedContextPath, currentModelName, newNodeTypePath); } /// /// Returns an IObservable that emits once when agent initialization is complete. - /// Uses ObserveQuery (reactive) — no await, no blocking, no deadlock. + /// Uses Query (reactive) — no await, no blocking, no deadlock. /// Subscribe to this and chain the streaming loop after it emits. /// /// /// Returns an IObservable that emits the initialized AgentChatClient when agents are ready. /// Re-emits when agent definitions change (system prompt updates, new agents added). - /// Uses ObserveQuery (reactive) — no await, no blocking, no deadlock. + /// Uses Query (reactive) — no await, no blocking, no deadlock. /// - public IObservable Initialize(string? contextPath, string? modelName = null) - { - currentModelName = modelName; - lastLoadedContextPath = contextPath; - - if (meshQuery == null) - return Observable.Return(this); - - var q1 = string.IsNullOrEmpty(contextPath) - ? "nodeType:Agent" - : $"nodeType:Agent namespace:{contextPath} scope:selfAndAncestors"; - - // Two ObserveQuery streams — merge agent nodes from context hierarchy + Agent namespace - var contextAgents = meshQuery.ObserveQuery(MeshQueryRequest.FromQuery(q1)); - var globalAgents = meshQuery.ObserveQuery(MeshQueryRequest.FromQuery("namespace:Agent nodeType:Agent")); - - // CombineLatest: re-emit whenever either query updates (agent added/changed) - return contextAgents.CombineLatest(globalAgents, (ctx, global) => - { - var agentsDict = ImmutableDictionary.Empty; - - foreach (var node in ctx.Items) - { - if (node.Content is AgentConfiguration config && !agentsDict.ContainsKey(config.Id)) - agentsDict = agentsDict.SetItem(config.Id, (config, node.Path ?? "")); - } - foreach (var node in global.Items) - { - if (node.Content is AgentConfiguration config && !agentsDict.ContainsKey(config.Id)) - agentsDict = agentsDict.SetItem(config.Id, (config, node.Path ?? "")); - } - - loadedAgents = agentsDict.Values.Select(x => new AgentDisplayInfo - { - Name = x.Config.Id, Path = x.Path, - Description = x.Config.Description ?? x.Config.DisplayName ?? x.Config.Id, - GroupName = x.Config.GroupName, Order = x.Config.Order, - Icon = x.Config.Icon, CustomIconSvg = x.Config.CustomIconSvg, - AgentConfiguration = x.Config - }).ToImmutableList(); - - loadedAgents = AgentOrderingHelper.OrderByRelevance( - loadedAgents, contextPath?.TrimStart('/') ?? "", "").ToImmutableList(); - - logger.LogInformation("[AgentChatClient] Initialize: {Count} agents: [{Agents}]", - loadedAgents.Count, string.Join(", ", loadedAgents.Select(a => a.Name))); - - agentsInitialized = false; - agents = ImmutableDictionary.Empty; - CreateAgentsSync(); - return this; - }); - } - /// - /// Reactive initialization — uses ObserveQuery (IObservable) instead of QueryAsync. - /// No await anywhere. Returns Task completed by subscription when agents are ready. - /// The AI framework awaits the returned Task — our code never awaits. + /// Synchronously binds this chat client to the workspace's shared synced + /// agent collection for the given context. Returns immediately; agents + /// populate when the underlying + /// emits — synchronously when warm-cached, asynchronously on first cold + /// load. Callers that need an explicit ready-gate should subscribe to + /// . /// - public Task InitializeAsync(string? contextPath, string? modelName = null) + public AgentChatClient Initialize(string? contextPath, string? modelName = null, string? nodeTypePath = null) { - currentModelName = modelName; + // Normalize at entry so satellite paths (e.g. "ACME/Project/_Thread/") collapse to + // their main-node path before any downstream query/cache key uses them. The model may + // arrive as the picked node PATH ("_Provider/Anthropic/claude-…") — factories match the + // bare model id (last segment). + contextPath = NormalizeContextPath(contextPath); + currentModelName = SelectionId.IdOf(modelName); lastLoadedContextPath = contextPath; - - if (meshQuery == null) - return Task.CompletedTask; - - var tcs = new TaskCompletionSource(); - - // Use ObserveQuery (IObservable) — NOT QueryAsync. No await, no deadlock. - // Use scope:subtree on root namespace to find deeply nested agents (e.g. ACME/Project/TodoAgent). - // Previously used scope:selfAndAncestors which only searched direct children of ancestors. - var rootNamespace = contextPath?.Split('/').FirstOrDefault(s => !string.IsNullOrEmpty(s)) ?? ""; - var contextQuery = string.IsNullOrEmpty(rootNamespace) - ? "nodeType:Agent" - : $"nodeType:Agent path:{rootNamespace} scope:subtree"; - - var contextAgents = meshQuery.ObserveQuery(MeshQueryRequest.FromQuery(contextQuery)); - var globalAgents = meshQuery.ObserveQuery(MeshQueryRequest.FromQuery("path:Agent nodeType:Agent scope:subtree")); - - var agentsDict = ImmutableDictionary.Empty; - var dictLock = new object(); - var queriesCompleted = 0; - - void OnAgentQueryResult(QueryResultChange change) - { - if (change.ChangeType != QueryChangeType.Initial) - return; - - lock (dictLock) - { - foreach (var node in change.Items) + // Default the NodeType-search namespace to the context node's NodeType when the + // caller didn't supply one. AgentPickerProjection.BuildAgentQueries will only + // emit the third query (`namespace:{nodeTypePath} scope:selfAndAncestors`) when + // this is non-null — that's what surfaces NodeType-defined agents (e.g. TodoAgent + // at ACME/Project for an instance with NodeType=ACME/Project). + nodeTypePath ??= Context?.Node?.NodeType; + lastLoadedNodeTypePath = nodeTypePath; + + // First-time init or context switch: subscribe to the SAME synced-query + // pipe the chat picker UI uses (AgentPickerProjection.ObserveAgents/ + // ObserveModels). One source of truth — no separate "AgentChatClient + // does its own Query" chain that drifted from the picker and + // produced "No suitable agent" even though the dropdown showed 9 of + // them. The synced query runs on the workspace of THIS hub (the thread + // hub passed in via the ctor's service provider), not the _Exec child + // — _Exec is blocked by the streaming Task.Run. + var workspace = hub.GetWorkspace(); + // The chatting user's home namespace — where a user drops their OWN agents, surfaced via the + // namespace:{userHome} alternation in AgentPickerProjection.BuildAgentQuery. Skips system/hub + // principals (they own no user namespace). Safe even for guests: it's a namespace MEMBERSHIP + // filter value, never a point-read, so a no-match is a silent no-op (unlike the model-selection + // point-read that would storm a guest partition). + var userHome = ResolveAgentUserHome(hub); + agentsSubscription?.Dispose(); + modelsSubscription?.Dispose(); + selectionSubscription?.Dispose(); + + // 🚨 Subscribe to agents and models INDEPENDENTLY. Models are not + // required for agent selection — only for the picker UI's model + // dropdown. Tying readiness to a CombineLatest of both means a slow + // model emission delays the first user-visible reply by however long + // the model query takes; we observed multi-second extra latency on + // Postgres-backed deploys with cold synced-query caches. Decouple: + // WhenInitialized fires as soon as agents are ready; models populate + // their own loadedModels in the background. + // Subscribe to the live synced agent stream — every emission updates + // loadedAgents and rebuilds the agents dict. NO Timeout fallback: the + // synced query emits ONE Initial event and then goes quiet until + // agents change; a Timeout(8s, emptyFallback) wrapper would interpret + // that quiescence as a failure and wipe loadedAgents 8s after the + // genuine Initial emission, breaking every subsequent chat round. + // The 5-min no-progress watchdog in ThreadExecution is the canonical + // safety net for genuinely stuck pipelines. + var readinessFired = false; + agentsSubscription = AgentPickerProjection + .ObserveAgents(hub, userHome, AgentPickerProjection.PartitionOf(contextPath)) + .Subscribe( + agents => { - if (node.Content is AgentConfiguration config && !agentsDict.ContainsKey(config.Id)) - agentsDict = agentsDict.SetItem(config.Id, (config, node.Path ?? "")); - } - } - - if (Interlocked.Increment(ref queriesCompleted) < 2) - return; - - // Both queries emitted initial — build agents - try - { - var displayInfos = agentsDict.Values.Select(x => new AgentDisplayInfo + logger.LogDebug("[AgentChatClient] Agent emission: {Count} for ctx={Ctx}", + agents.Count, contextPath ?? "(null)"); + ApplyAgents(agents, contextPath); + // Synced queries emit Initial first, then incremental changes. + // Initial-with-0-agents is the legitimate "no agents configured" + // state — not "still loading". Gating readiness on count>0 + // hangs WhenInitialized forever in that case (the only-Initial + // synced query then quiesces). Fire on every emission; callers + // inspect loadedAgents to decide what to do with an empty list. + if (!readinessFired) + readinessFired = true; + agentsLoadedSubject.OnNext(this); + }, + ex => { - Name = x.Config.Id, Path = x.Path, - Description = x.Config.Description ?? x.Config.DisplayName ?? x.Config.Id, - GroupName = x.Config.GroupName, Order = x.Config.Order, - Icon = x.Config.Icon, CustomIconSvg = x.Config.CustomIconSvg, - AgentConfiguration = x.Config - }).ToImmutableList(); - - loadedAgents = AgentOrderingHelper.OrderByRelevance( - displayInfos, contextPath?.TrimStart('/') ?? "", "").ToImmutableList(); - - logger.LogInformation("[AgentChatClient] Loaded {Count} agents: [{Agents}]", - loadedAgents.Count, string.Join(", ", loadedAgents.Select(a => a.Name))); - - CreateAgentsSync(); - tcs.TrySetResult(); - } - catch (Exception ex) - { - logger.LogError(ex, "[AgentChatClient] Failed to create agents"); - tcs.TrySetException(ex); - } - } - - void OnQueryError(Exception ex, string queryName) - { - logger.LogWarning(ex, "{QueryName} agent query failed", queryName); - if (Interlocked.Increment(ref queriesCompleted) >= 2) - tcs.TrySetResult(); // Resolve with whatever agents were found (possibly empty) - } - - contextAgents.Subscribe(OnAgentQueryResult, ex => OnQueryError(ex, "Context")); - globalAgents.Subscribe(OnAgentQueryResult, ex => OnQueryError(ex, "Global")); - - return tcs.Task; - } - - /// - /// Loads agents from mesh and returns them ordered by relevance. - /// Two queries: path hierarchy + NodeType hierarchy. - /// - private async Task> LoadOrderedAgentsAsync(string? contextPath) - { - if (meshQuery == null) - return ImmutableList.Empty; - - var agentsDict = ImmutableDictionary.Empty; - - // 1. Get NodeType of current node - string? nodeTypePath = null; - if (!string.IsNullOrEmpty(contextPath)) - { - try - { - await foreach (var node in meshQuery.QueryAsync($"path:{contextPath}")) + logger.LogWarning(ex, "[AgentChatClient] Agent subscription faulted — unblocking with empty"); + ApplyAgents(Array.Empty(), contextPath); + readinessFired = true; + agentsLoadedSubject.OnNext(this); + }, + () => { - if (!string.IsNullOrEmpty(node.NodeType) && node.NodeType != "Agent" && node.NodeType != "Markdown") + if (!readinessFired) { - nodeTypePath = node.NodeType; - break; + readinessFired = true; + agentsLoadedSubject.OnNext(this); } - } - } - catch (Exception ex) - { - logger.LogWarning(ex, "Error getting NodeType for {ContextPath}", contextPath); - } - } - - // 2. Query agents from context path hierarchy (or root if no context) - try + }); + + // No timer fallback: cold-start synced queries (especially on sub-thread + // workspaces) can take longer than any arbitrary timer to populate. + // Forcing readiness with empty agents only causes "No suitable agent + // found" false-positives. Genuine "synced query never emits" cases are + // covered by the OnError / OnCompleted handlers above and by the + // ThreadSubmission cancel button — manual cancellation is preferable + // to an arbitrary deadline. + + // Provider selection drives BOTH the model picker and the resolver's + // use-without-see watches. Subscribe to the user's selection node; each + // change rebuilds the model subscription + resolver watches. + // StartWith(empty) means the picker loads immediately with the default + // set (root + context + nodeType) — byte-for-byte the previous behaviour + // for users who never touched the selection picker, so no regression. + // .Catch keeps a selection-read failure from breaking the picker. + // (Same "no Timeout fallback" rule as agents: the synced query emits + // Initial then quiesces; a Timeout wrapper would wipe loadedModels.) + var accessService = hub.ServiceProvider.GetService(); + var selectionContext = accessService?.Context; + var selectionUserId = selectionContext?.ObjectId; + // 🚨 Guests (VUser / IsVirtual identities) own NO ModelProvider or + // LanguageModel nodes — they consume the root + shared catalog only. + // Watching a guest's partition makes ChatClientCredentialResolver.ReadSnapshot + // fan out `namespace:{VUser/id}/_Memex scope:descendants` per guest + // session: a descendants walk on the `vuser` schema that returns nothing + // yet pins a DB connection (ListChildPaths + node reads). With many + // concurrent guests that storms the connection pool to exhaustion + // ("pool exhausted, currently 50" — prod 2026-06-04). Only real + // users/spaces have their own providers; guests use the default catalog. + var watchOwnProviders = ShouldWatchOwnProviderPartition(selectionContext); + var credentialResolver = hub.ServiceProvider.GetService(); + if (watchOwnProviders) + credentialResolver?.WatchPartition(selectionUserId!); + + selectionSubscription?.Dispose(); + // 🚨 Read the user's provider selection via a QUERY, never a point + // `GetMeshNodeStream(SelectionPath)` node-access. A point-subscribe to a + // node that does NOT exist — every PRE-EXISTING user partition that + // predates the `_Selection` seed (ModelProviderNodeType) — routes to a + // RoutingGrain `NotFound` DeliveryFailure + SYNC_STREAM `OnError`. + // Initialize re-runs on every agent/model rebuild during streaming, so + // the failing subscribe is re-issued in a tight loop: the resubscribe- + // storm that starved the `portal/` action block until unrelated + // SubscribeRequests went stale >30s and the circuit FROZE (2026-06-09). + // A `GetQuery` over the namespace returns an EMPTY set when the selection + // node is absent (the documented "empty ⇒ default catalog" behaviour) — + // no NotFound, no resubscribe, nothing to storm. GetQuery returns typed + // Content (deserialised through this hub's options), so + // ExtractSelectedProviderPaths reads `node.Content` directly. Seeding the + // node only ever covered NEW users; querying fixes the whole class. + var selectionStream = !watchOwnProviders + ? Observable.Return(ImmutableArray.Empty) + : workspace.GetQuery( + $"{ModelProviderNodeType.SelectionNodeType}|{selectionUserId}", + $"namespace:{ModelProviderNodeType.UserNamespacePath(selectionUserId!)} nodeType:{ModelProviderNodeType.SelectionNodeType}") + .Select(nodes => ExtractSelectedProviderPaths(nodes.FirstOrDefault())) + .Catch, Exception>(_ => Observable.Return(ImmutableArray.Empty)) + .StartWith(ImmutableArray.Empty) + .DistinctUntilChanged(SelectedPathsComparer.Instance); + + selectionSubscription = selectionStream.Subscribe(selectedPaths => { - var pathQuery = string.IsNullOrEmpty(contextPath) - ? "namespace: nodeType:Agent" // Root level: get direct children agents - : $"path:{contextPath} nodeType:Agent scope:AncestorsAndSelf"; + // Resolver: make each selected (org/shared) provider usable under + // use-without-see — system-identity ingest + per-user Read gate. + if (!string.IsNullOrEmpty(selectionUserId) && credentialResolver != null) + foreach (var p in selectedPaths) + credentialResolver.WatchSharedProvider(p, selectionUserId); + + // Picker: (re)subscribe models to include the selected subtrees AND + // the user's own {user}/_Memex provider/model nodes (userPath). + modelsSubscription?.Dispose(); + modelsSubscription = AgentPickerProjection + .ObserveModels(workspace, hub, contextPath, nodeTypePath, selectedPaths, selectionUserId) + .Subscribe( + models => + { + logger.LogDebug("[AgentChatClient] Model emission: {Count} (selected={Sel})", + models.Count, selectedPaths.Length); + loadedModels = models.ToImmutableList(); + }, + ex => logger.LogDebug(ex, "[AgentChatClient] Model subscription faulted — picker dropdown will be empty")); + }); - await foreach (var node in meshQuery.QueryAsync(pathQuery)) - { - if (node.Content is AgentConfiguration config && !agentsDict.ContainsKey(config.Id)) - agentsDict = agentsDict.SetItem(config.Id, (config, node.Path ?? "")); - } - } - catch (Exception ex) - { - logger.LogWarning(ex, "Error querying path hierarchy for {ContextPath}", contextPath ?? "root"); - } + return this; + } - // 3. Query agents from root namespace subtree (to find sibling agents) - if (!string.IsNullOrEmpty(contextPath)) - { - try - { - // Extract root namespace (first segment of the path) - var rootNamespace = contextPath.Split('/').FirstOrDefault(s => !string.IsNullOrEmpty(s)); - if (!string.IsNullOrEmpty(rootNamespace)) - { - var subtreeQuery = $"path:{rootNamespace} nodeType:Agent scope:Subtree"; - await foreach (var node in meshQuery.QueryAsync(subtreeQuery)) - { - if (node.Content is AgentConfiguration config && !agentsDict.ContainsKey(config.Id)) - agentsDict = agentsDict.SetItem(config.Id, (config, node.Path ?? "")); - } - } - } - catch (Exception ex) - { - logger.LogWarning(ex, "Error querying root namespace subtree for {ContextPath}", contextPath); - } - } + private IDisposable? modelsSubscription; + private IDisposable? selectionSubscription; - // 4. Query agents from NodeType hierarchy - if (!string.IsNullOrEmpty(nodeTypePath)) + /// Extracts the user's selected provider paths from the selection node (tolerates JsonElement content + absent node). + private ImmutableArray ExtractSelectedProviderPaths(MeshNode? node) + { + var sel = node?.Content switch { - try - { - var nodeTypeQuery = $"path:{nodeTypePath} nodeType:Agent scope:AncestorsAndSelf"; - await foreach (var node in meshQuery.QueryAsync(nodeTypeQuery)) - { - if (node.Content is AgentConfiguration config && !agentsDict.ContainsKey(config.Id)) - agentsDict = agentsDict.SetItem(config.Id, (config, node.Path ?? "")); - } - } - catch (Exception ex) - { - logger.LogWarning(ex, "Error querying NodeType hierarchy for {NodeType}", nodeTypePath); - } - } + ModelProviderSelection s => s, + System.Text.Json.JsonElement je => TryDeserializeSelection(je), + _ => null, + }; + if (sel is null) return ImmutableArray.Empty; + var arr = sel.SelectedProviderPaths; + return arr.IsDefault ? ImmutableArray.Empty : arr; + } - // 5. Query default agents from Agent namespace - try - { - var agentNamespaceQuery = "path:Agent nodeType:Agent scope:Subtree"; - await foreach (var node in meshQuery.QueryAsync(agentNamespaceQuery)) - { - if (node.Content is AgentConfiguration config && !agentsDict.ContainsKey(config.Id)) - agentsDict = agentsDict.SetItem(config.Id, (config, node.Path ?? "")); - } - } - catch (Exception ex) + private ModelProviderSelection? TryDeserializeSelection(System.Text.Json.JsonElement je) + { + try { return System.Text.Json.JsonSerializer.Deserialize(je.GetRawText(), hub.JsonSerializerOptions); } + catch { return null; } + } + + private sealed class SelectedPathsComparer : IEqualityComparer> + { + public static readonly SelectedPathsComparer Instance = new(); + public bool Equals(ImmutableArray x, ImmutableArray y) => x.SequenceEqual(y); + public int GetHashCode(ImmutableArray obj) { - logger.LogWarning(ex, "Error querying Agent namespace"); + var hc = new HashCode(); + foreach (var s in obj) hc.Add(s); + return hc.ToHashCode(); } + } - // Convert to AgentDisplayInfo - var displayInfos = agentsDict.Values.Select(x => new AgentDisplayInfo - { - Name = x.Config.Id, - Path = x.Path, - Description = x.Config.Description ?? x.Config.DisplayName ?? x.Config.Id, - GroupName = x.Config.GroupName, - Order = x.Config.Order, - Icon = x.Config.Icon, - CustomIconSvg = x.Config.CustomIconSvg, - AgentConfiguration = x.Config - }).ToImmutableList(); - - // Order by relevance: own namespace > NodeType namespace > hierarchy (path > nodeType) - var contextPathNorm = contextPath?.TrimStart('/') ?? ""; - var nodeTypePathNorm = nodeTypePath?.TrimStart('/') ?? ""; - - var result = AgentOrderingHelper.OrderByRelevance(displayInfos, contextPathNorm, nodeTypePathNorm).ToImmutableList(); - logger.LogDebug("Loaded {Count} agents for context {ContextPath}: [{Agents}]", - result.Count, contextPath ?? "(none)", string.Join(", ", result.Select(a => a.Name))); - return result; + /// + /// Stash agents from + /// into and rebuild the + /// dictionary. Models are populated independently in their own + /// subscription — see Initialize. + /// Internal so the selection/resolution tests can drive the exact + /// production code path (the synced-query Subscribe callback calls this) + /// without standing up a mesh + synced query. + /// + internal void ApplyAgents( + IReadOnlyList agentInfos, + string? contextPath) + { + loadedAgents = AgentOrderingHelper.OrderByRelevance( + agentInfos.ToImmutableList(), + contextPath?.TrimStart('/') ?? string.Empty, + string.Empty).ToImmutableList(); + + logger.LogDebug("[AgentChatClient] {Count} agents: [{Agents}]", + loadedAgents.Count, string.Join(", ", loadedAgents.Select(a => a.Name))); + + // 🚨 Do NOT pre-wipe `agents` here. CreateAgentsSync builds the new + // dict LOCALLY and atomic-swaps at the end (see "Atomic publish" + // comment in CreateAgentsSync). Wiping here would leave `agents` + // empty for the entire rebuild window — concurrent SelectAgent + // calls would return null and the request would surface as + // "No suitable agent found to handle the request." (the + // SubThreadHangRepro second-Fact symptom). Keep the OLD dict in + // place; readers see EITHER the old full dict OR the new full + // dict, never an empty intermediate. + agentsInitialized = false; + CreateAgentsSync(); } + + // (Legacy LoadOrderedAgentsAsync removed — its 5 parallel QueryAsync calls + // were the source of the chat-load deadlock. Agents are now sourced + // exclusively from the workspace synced query subscription wired up in + // Initialize. Anyone who wants the agent list reads loadedAgents.) + /// /// Creates ChatClientAgent instances synchronously — no await, no deadlock. /// Uses CreateAgent (sync) on the factory which skips async reference resolution. @@ -1119,44 +1415,116 @@ private void CreateAgentsSync() { if (chatClientFactories.Count == 0) { - logger.LogWarning("[AgentChatClient] No IChatClientFactory available, cannot create agents"); + lastAgentCreationError = + "No IChatClientFactory is registered. Add e.g. AddAzureFoundryClaude / AddAzureOpenAI in your host configuration."; + logger.LogWarning("[AgentChatClient] {Error}", lastAgentCreationError); return; } if (agentsInitialized) return; - - var factory = GetFactoryForModel(currentModelName); - if (factory == null) + // Reset before this attempt — a previous failure shouldn't be surfaced + // if the new attempt succeeds. + lastAgentCreationError = null; + + // Factory selection from the chat dropdown selection (currentModelName) — the + // single source of truth for the model, independent of the agent. Selecting the + // factory for that model ensures we don't try to serve, e.g., an OpenAI model on + // an Azure Foundry factory. + // isPersistentFactory tracks the default factory's persistence mode for + // legacy paths that haven't been threaded through per-agent. + var defaultFactory = GetFactoryForModel(currentModelName); + if (defaultFactory == null) { - logger.LogWarning("[AgentChatClient] No factory can serve model: {ModelName}", currentModelName); + lastAgentCreationError = + $"No registered IChatClientFactory accepts model '{currentModelName ?? "(none selected)"}'. " + + $"Configured factories: [{string.Join(", ", chatClientFactories.Select(f => f.Name))}]."; + logger.LogWarning("[AgentChatClient] {Error}", lastAgentCreationError); return; } - - isPersistentFactory = factory.IsPersistent; - logger.LogInformation("[AgentChatClient] Using factory {FactoryName} for model {ModelName} (persistent={IsPersistent})", - factory.Name, currentModelName ?? "default", isPersistentFactory); + isPersistentFactory = defaultFactory.IsPersistent; + logger.LogDebug("[AgentChatClient] Factory {FactoryName} for chat-selected model {ModelName} (persistent={IsPersistent})", + defaultFactory.Name, currentModelName ?? "default", isPersistentFactory); var configs = loadedAgents.Select(a => a.AgentConfiguration).ToImmutableList(); var createdAgents = ImmutableDictionary.Empty; var orderedConfigs = OrderAgentsForCreation(configs); + // 🛑 Anti-storm: a GLOBAL build failure (e.g. "No model selected" — no model is + // configured) throws IDENTICALLY for every agent, so the per-agent catch below would + // log the same warning once per agent (≈22 lines in ~1ms = the log storm). De-dupe by + // message: log each distinct build error ONCE per build. lastAgentCreationError still + // carries the latest message to the GUI regardless. + var loggedBuildErrors = new HashSet(StringComparer.Ordinal); + + // 🚨 Build the dict LOCALLY, then ATOMICALLY swap into `agents` at + // the end. The previous shape mutated the shared `agents` field per + // iteration (one-by-one SetItem) — every concurrent SelectAgent + // saw a PARTIAL dict, biased toward agents added first + // (Researcher, DescriptionWriter, …). The default + // Assistant is added LAST per `OrderAgentsForCreation`, so during + // the window, SelectAgent's `loadedAgents[0]` lookup (which finds + // "Assistant" in loadedAgents) ran `agents.TryGetValue("Assistant")` + // → false → fell through to `agents.Values.FirstOrDefault()` → + // returned a non-default agent. Concrete failure: + // SubThreadHangRepro's second [Fact] routed to DescriptionWriter + // (HangingSubAgentChatClient) instead of Assistant + // (DelegatingParentChatClient) — hung forever. + // + // Per-agent try/catch is mandatory: factory misconfig (e.g. an + // Azure Foundry endpoint not set) throws inside CreateAgent and used + // to escape the synced-query Subscribe callback as an unhandled + // exception that killed the portal process. Now we log + skip the + // misconfigured agent so the rest of the catalog still loads and + // SelectAgent can fall back to a working one. foreach (var agentConfig in orderedConfigs) { - var agent = factory.CreateAgent(agentConfig, this, createdAgents, configs, currentModelName); - createdAgents = createdAgents.SetItem(agentConfig.Id, agent); - agents = agents.SetItem(agentConfig.Id, agent); + // Model is the chat composer's selection — fully independent of the agent. + var effectiveModel = currentModelName; + var factory = GetFactoryForModel(effectiveModel) ?? defaultFactory; + try + { + var agent = factory.CreateAgent(agentConfig, this, createdAgents, configs, effectiveModel); + createdAgents = createdAgents.SetItem(agentConfig.Id, agent); + } + catch (Exception ex) + { + lastAgentCreationError = + $"Failed to create agent '{agentConfig.Id}' via factory '{factory?.Name}' for model '{effectiveModel}': {ex.Message}"; + // Only log a given underlying error message ONCE per build — a global + // failure (e.g. "No model selected") repeats per agent and would storm. + if (loggedBuildErrors.Add(ex.Message)) + logger.LogWarning(ex, + "[AgentChatClient] Skipping agent {Agent} ({Factory}/{Model}): {Message}", + agentConfig.Id, factory?.Name, effectiveModel, ex.Message); + } } var cyclicAgents = FindCyclicDelegations(configs); foreach (var agentConfig in cyclicAgents) { - var updatedAgent = factory.CreateAgent(agentConfig, this, createdAgents, configs, currentModelName); - createdAgents = createdAgents.SetItem(agentConfig.Id, updatedAgent); - agents = agents.SetItem(agentConfig.Id, updatedAgent); + // Model is the chat composer's selection — fully independent of the agent. + var effectiveModel = currentModelName; + var factory = GetFactoryForModel(effectiveModel) ?? defaultFactory; + try + { + var updatedAgent = factory.CreateAgent(agentConfig, this, createdAgents, configs, effectiveModel); + createdAgents = createdAgents.SetItem(agentConfig.Id, updatedAgent); + } + catch (Exception ex) + { + // Same per-build de-dupe as the main loop above. + if (loggedBuildErrors.Add(ex.Message)) + logger.LogWarning(ex, + "[AgentChatClient] Skipping cyclic agent {Agent}: {Message}", + agentConfig.Id, ex.Message); + } } + // Atomic publish — readers see EITHER the previous full dict OR the + // new full dict, never a half-built one. + agents = createdAgents; agentsInitialized = true; - logger.LogInformation("[AgentChatClient] Created {Count} agents", agents.Count); + logger.LogDebug("[AgentChatClient] Created {Count} agents", agents.Count); } /// @@ -1183,7 +1551,7 @@ private async Task CreateAgentsAsync() } isPersistentFactory = factory.IsPersistent; - logger.LogInformation("[AgentChatClient] Using factory {FactoryName} for model {ModelName} (persistent={IsPersistent})", + logger.LogDebug("[AgentChatClient] Using factory {FactoryName} for model {ModelName} (persistent={IsPersistent})", factory.Name, currentModelName ?? "default", isPersistentFactory); var configs = loadedAgents.Select(a => a.AgentConfiguration).ToImmutableList(); @@ -1212,7 +1580,7 @@ private async Task CreateAgentsAsync() } agentsInitialized = true; - logger.LogInformation("[AgentChatClient] Created {Count} agents", agents.Count); + logger.LogDebug("[AgentChatClient] Created {Count} agents", agents.Count); } /// @@ -1228,9 +1596,15 @@ private async Task CreateAgentsAsync() .FirstOrDefault(); } - // Find factory that has this model + // Ask each factory whether it serves this model. Concrete factories + // implement shape-aware predicates (e.g. AzureClaude → "claude-*", + // AzureFoundry → catch-all for non-Claude). Falling back on Models[] + // (the legacy mechanism) only matters for factories that haven't + // overridden Supports — and Models[] is empty by default now since + // model env-vars were removed in favour of the chat composer selection. var factory = chatClientFactories - .FirstOrDefault(f => f.Models.Contains(modelName)); + .OrderBy(f => f.Order) + .FirstOrDefault(f => f.Supports(modelName)); if (factory != null) { @@ -1238,12 +1612,43 @@ private async Task CreateAgentsAsync() } // Fallback: return first factory - logger.LogWarning("[AgentChatClient] Model {ModelName} not found in any factory, using first available", modelName); + logger.LogWarning("[AgentChatClient] Model {ModelName} not served by any factory, using first available", modelName); return chatClientFactories .OrderBy(f => f.Order) .FirstOrDefault(); } + /// + /// Whether a chat client should watch its caller's OWN partition for + /// ModelProvider / LanguageModel nodes. True only for a real, non-virtual + /// identity. Guests (VUser / ) + /// own no providers — they consume the root + shared catalog — so watching + /// their partition fans out a per-session namespace:{VUser/id}/_Memex + /// scope:descendants query against the vuser schema that returns + /// nothing yet pins a DB connection; with many concurrent guests that storms + /// the connection pool to exhaustion (prod 2026-06-04). + /// + internal static bool ShouldWatchOwnProviderPartition(MeshWeaver.Messaging.AccessContext? context) + => !string.IsNullOrEmpty(context?.ObjectId) && context.IsVirtual != true; + + /// + /// The chatting user's home namespace — the namespace under which they place their OWN agents, + /// fed to as the per-user alternation value. + /// Mirrors the chat view's ResolveUserHome: the AccessContext ObjectId, skipping the + /// system identity and hub principals (which own no user namespace). + /// + private static string? ResolveAgentUserHome(IMessageHub hub) + { + var accessSvc = hub.ServiceProvider.GetService(); + if (accessSvc is null) return null; + foreach (var candidate in new[] { accessSvc.Context?.ObjectId, accessSvc.CircuitContext?.ObjectId }) + if (!string.IsNullOrEmpty(candidate) + && candidate != MeshWeaver.Mesh.Security.WellKnownUsers.System + && !MeshWeaver.Messaging.AccessService.LooksLikeHubPrincipal(candidate)) + return candidate; + return null; + } + /// /// Orders agents for creation: non-delegating first, delegating second, default last. /// @@ -1294,30 +1699,24 @@ internal static IEnumerable FindCyclicDelegations(IEnumerabl } /// - /// Returns the ordered list of agents for the current context. - /// Reloads agents when context path changes. + /// Returns the synced agent collection — populated by the workspace-level + /// synced query subscription wired up in . + /// If the explicit context has changed since the last init, re-binds the + /// subscription to the new context's id (the workspace's per-id cache + /// makes this cheap when warm-cached) and returns whatever the synced + /// collection currently holds. Task-shaped only for + /// compat — no awaits. /// - public async Task> GetOrderedAgentsAsync() + public Task> GetOrderedAgentsAsync() { - // Only check for context changes if Context has been explicitly set via SetContext() - // This prevents reloading with null when agents were already loaded by InitializeAsync var currentContextPath = Context?.Address?.ToString(); + var currentNodeTypePath = Context?.Node?.NodeType; + if (currentContextPath != null + && (currentContextPath != lastLoadedContextPath + || currentNodeTypePath != lastLoadedNodeTypePath)) + Initialize(currentContextPath, currentModelName, currentNodeTypePath); - // Only reload if: - // 1. Context has been set (not null), AND - // 2. It's different from what was already loaded - if (currentContextPath != null && currentContextPath != lastLoadedContextPath) - { - loadedAgents = await LoadOrderedAgentsAsync(currentContextPath); - lastLoadedContextPath = currentContextPath; - - // Recreate agent instances for new context - agentsInitialized = false; - agents = ImmutableDictionary.Empty; - CreateAgentsSync(); - } - - return loadedAgents; + return Task.FromResult>(loadedAgents); } public Task ResumeAsync(ChatConversation conversation) diff --git a/src/MeshWeaver.AI/AgentConfiguration.cs b/src/MeshWeaver.AI/AgentConfiguration.cs index 9511a6777..d2b5fb8da 100644 --- a/src/MeshWeaver.AI/AgentConfiguration.cs +++ b/src/MeshWeaver.AI/AgentConfiguration.cs @@ -6,23 +6,31 @@ namespace MeshWeaver.AI; /// Represents an agent configuration stored in the graph. /// Agents are stored as MeshNodes with nodeType="Agent" and Content=AgentConfiguration. /// Supports hierarchical resolution - agents at lower namespaces override parent namespaces. +/// +/// 🚨 Node-level metadata is NOT duplicated here. The display name, description, +/// icon, group, and ordering all live on the owning +/// (Name, Description, Icon, Category, Order) and are +/// edited through the standard node settings — never replicated on the agent content. +/// This record carries only what's specific to the agent's behaviour. ( +/// is the runtime identity key and equals the owning node's Id by construction; +/// is kept because the agent runtime feeds it to the model as +/// delegation metadata where only the detached configuration is in hand.) /// public record AgentConfiguration { /// - /// Unique identifier for this agent. + /// Unique identifier for this agent. Equals the owning + /// by construction; used as the runtime identity key for agent creation, delegation + /// resolution, and the created-agents map. Display reads the node's Id, not this field. /// [Key] public required string Id { get; init; } /// - /// Display name for UI (defaults to Id if not set). - /// - public string? DisplayName { get; init; } - - /// - /// Description of what this agent does. - /// Used for delegation decisions and UI display. + /// Description of what this agent does. Mirrors the owning node's + /// ; kept on the configuration because + /// the agent runtime feeds it to the model (delegation/hand-off catalogue) where only + /// the detached is available. UI/picker read the node. /// public string? Description { get; init; } @@ -32,21 +40,11 @@ public record AgentConfiguration /// public string? Instructions { get; init; } - /// - /// Icon URL or identifier for the agent. - /// - public string? Icon { get; init; } - /// /// Custom SVG path for icon (optional override). /// public string? CustomIconSvg { get; init; } - /// - /// Group name for UI categorization (e.g., "Insurance", "Todo"). - /// - public string? GroupName { get; init; } - /// /// Whether this is the default/entry-point agent. /// Only one agent should have IsDefault=true at root level. @@ -71,19 +69,6 @@ public record AgentConfiguration /// public List? Handoffs { get; init; } - /// - /// Preferred model name from available models. - /// Null means use the factory default. - /// - public string? PreferredModel { get; init; } - - /// - /// Model tier for this agent: "heavy" (most capable), "standard" (balanced), "light" (fast/cheap). - /// Resolved at runtime via ModelTierConfiguration to an actual model name. - /// Takes effect only when PreferredModel is not set. - /// - public string? ModelTier { get; init; } - /// /// RSQL pattern for context matching. /// If set, agent activates when context matches this pattern. @@ -92,10 +77,15 @@ public record AgentConfiguration public string? ContextMatchPattern { get; init; } /// - /// Display order for sorting agents in the UI. - /// Lower values appear first. + /// OPTIONAL hint: abstract model tier this agent prefers — "heavy", "standard", "light", + /// or "utility". Never required: when unset (the normal case), or when the deployment has + /// no ModelTier:* config, model selection is entirely unaffected. When set AND + /// configured, it only fills the gap where nobody picked a model (headless flows like + /// notification triage or icon/description micro-jobs) — an explicit composer selection + /// always wins. Declared only on the built-in background micro-agents; interactive agents + /// should leave it unset. /// - public int Order { get; init; } + public string? ModelTier { get; init; } /// /// Optional list of additional plugins this agent should load. @@ -140,6 +130,23 @@ public record AgentPluginReference /// If null or empty, all plugin methods are included. /// public List? Methods { get; init; } + + /// + /// Parses the frontmatter string form: "PluginName" or "PluginName:Method1,Method2". + /// Shared by every agent-definition parser so the syntax stays uniform. + /// + public static AgentPluginReference Parse(string s) + { + var colonIndex = s.IndexOf(':'); + if (colonIndex < 0) + return new AgentPluginReference { Name = s.Trim() }; + + return new AgentPluginReference + { + Name = s[..colonIndex].Trim(), + Methods = s[(colonIndex + 1)..].Split(',').Select(m => m.Trim()).ToList() + }; + } } /// diff --git a/src/MeshWeaver.AI/AgentNodeType.cs b/src/MeshWeaver.AI/AgentNodeType.cs index 0c115ff5e..7c9711fb2 100644 --- a/src/MeshWeaver.AI/AgentNodeType.cs +++ b/src/MeshWeaver.AI/AgentNodeType.cs @@ -2,6 +2,7 @@ using MeshWeaver.Mesh; using MeshWeaver.Mesh.Services; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; namespace MeshWeaver.AI; @@ -18,14 +19,52 @@ public static class AgentNodeType /// /// Registers the built-in "Agent" MeshNode on the mesh builder - /// and a static node provider for built-in agents (e.g., ThreadNamer). + /// plus the partition routing for built-in agents (e.g., ThreadNamer). + /// The "Agent" partition's storage of record is the + /// 's output — wrapped in a + /// so it goes through + /// the same first-match-wins routing as every other partition. + /// Also kept as an for the legacy + /// consumers (StaticNodeQueryProvider, MeshDataSource fallback) that + /// still iterate that DI collection directly. /// - public static TBuilder AddAgentType(this TBuilder builder) where TBuilder : MeshBuilder + public static TBuilder AddAgentType(this TBuilder builder, + IReadOnlySet? serveFromPartition = null) where TBuilder : MeshBuilder { builder.AddMeshNodes(CreateMeshNode()); builder.ConfigureNodeTypeAccess(a => a.WithPublicRead(NodeType)); + // When the "Agent" partition is DB-synced (static-repo import), DO NOT register the + // read-only in-memory static surfaces — they would shadow Postgres (specific wins over + // the wildcard PG provider) and reject the import's writes. The import materializes the + // agents into the partition; PG serves them. Otherwise (monolith, un-synced deploys) + // keep the in-memory read-only surfaces. The BuiltInAgentProvider singleton itself stays + // registered either way — the import SOURCE (AgentStaticRepoSource) wraps it to read the + // built-in agents. See Doc/Architecture/StaticRepoImport.md. + // + // 🚨 BOTH the IStaticNodeProvider AND the IPartitionStorageProvider must be gated on + // !dbSynced. The IStaticNodeProvider feeds serviceProvider.FindStaticNode(path); leaving + // it registered while synced made the importer's inner CreateNode see the built-in agent + // as already-present and fail "Node already exists at path: Agent/X" — so the Agent + // partition never materialized into the DB (atioz 2026-06-11: Agent imported 4 / failed 8 + // while Doc — which has no IStaticNodeProvider — imported 161/0). Gating only the storage + // provider (the prior state) was the gap. The "Agent" NodeType definition itself stays via + // AddMeshNodes(CreateMeshNode()) above, so the import's NodeType-existence check still + // resolves. See OrleansStaticRepoImportStaticBackedTest. + var dbSynced = serveFromPartition?.Contains("Agent") == true; builder.ConfigureServices(services => - services.AddSingleton()); + { + services.TryAddSingleton(); + if (!dbSynced) + { + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => + new StaticNodePartitionStorageProvider( + "Agent", + sp.GetRequiredService(), + description: "Built-in agent definitions (read-only).")); + } + return services; + }); return builder; } @@ -37,7 +76,11 @@ public static TBuilder AddAgentType(this TBuilder builder) where TBuil { Name = "Agent", Icon = "/static/NodeTypeIcons/bot.svg", - AssemblyLocation = typeof(AgentNodeType).Assembly.Location, + // Agents are first-class content nodes — they live at top-level paths + // (e.g. namespace:Agent for built-ins, or per-partition under contextPath) + // and are NOT children of another entity. The synced picker query + // (AgentPickerProjection.ObserveAgents) treats them as standalone. + IsSatelliteType = false, HubConfiguration = config => config .AddMeshDataSource(source => source .WithContentType()) diff --git a/src/MeshWeaver.AI/AgentOrderingHelper.cs b/src/MeshWeaver.AI/AgentOrderingHelper.cs index ab47ad46f..07dcdfb3e 100644 --- a/src/MeshWeaver.AI/AgentOrderingHelper.cs +++ b/src/MeshWeaver.AI/AgentOrderingHelper.cs @@ -1,115 +1,36 @@ using System.Collections.Immutable; +using System.Reactive.Linq; +using MeshWeaver.Data; +using MeshWeaver.Messaging; using MeshWeaver.Mesh; -using MeshWeaver.Mesh.Services; namespace MeshWeaver.AI; /// /// Shared helper for querying and ordering agents by relevance to the current context. -/// This is the SINGLE implementation of agent finding and ordering logic. +/// Agent list retrieval ALWAYS flows through +/// → workspace.GetQuery — the synced pipeline that fans out across all +/// static MeshNode providers, dedupes, and gates on the all-Initial event. /// public static class AgentOrderingHelper { /// - /// Queries agents from the mesh and returns them as AgentDisplayInfo with paths. - /// Searches NodeType namespace (children) and context path namespace (ancestors). + /// Reactive agent listing. Wraps + /// (the canonical workspace.GetQuery-backed synced source) and emits the + /// agents ordered by . Every consumer — picker UI, + /// AgentDetailsArea, AzureClaude driver, tests — subscribes here, never to + /// IMeshService.Query directly. /// - public static async Task> QueryAgentsAsync( - IMeshService? meshQuery, - string? contextPath, - string? nodeTypePath) - { - var agentsDict = ImmutableDictionary.Empty; - - // 1. Query agents from the NodeType namespace (higher priority) - // Use hierarchy scope to find agents that are children of the NodeType path - if (meshQuery != null && !string.IsNullOrEmpty(nodeTypePath)) - { - try - { - var query = $"path:{nodeTypePath} nodeType:Agent scope:hierarchy"; - await foreach (var node in meshQuery.QueryAsync(query)) - { - if (node.Content is AgentConfiguration config && !agentsDict.ContainsKey(config.Id)) - { - agentsDict = agentsDict.SetItem(config.Id, (config, node.Path ?? "")); - } - } - } - catch - { - // Ignore query errors - } - } - - // 2. Query agents from the context path namespace (ancestors) - if (meshQuery != null) - { - try - { - var query = string.IsNullOrEmpty(contextPath) - ? "nodeType:Agent scope:selfAndAncestors" - : $"path:{contextPath} nodeType:Agent scope:selfAndAncestors"; - - await foreach (var node in meshQuery.QueryAsync(query)) - { - if (node.Content is AgentConfiguration config && !agentsDict.ContainsKey(config.Id)) - { - agentsDict = agentsDict.SetItem(config.Id, (config, node.Path ?? "")); - } - } - } - catch - { - // Ignore query errors - } - } - - // Build display info list - return agentsDict.Values - .Select(x => new AgentDisplayInfo - { - Name = x.Config.Id, - Path = x.Path, - Description = x.Config.Description ?? x.Config.DisplayName ?? x.Config.Id, - GroupName = x.Config.GroupName, - Order = x.Config.Order, - IndentLevel = 0, - Icon = x.Config.Icon, - CustomIconSvg = x.Config.CustomIconSvg, - AgentConfiguration = x.Config - }) - .ToImmutableList(); - } - - /// - /// Gets the NodeType for a given context path. - /// - public static async Task GetNodeTypeAsync(IMeshService? meshQuery, string? contextPath) - { - if (meshQuery == null || string.IsNullOrEmpty(contextPath)) - return null; - - try - { - await foreach (var node in meshQuery.QueryAsync($"path:{contextPath}")) - { - if (!string.IsNullOrEmpty(node.NodeType) && node.NodeType != "Agent" && node.NodeType != "Markdown") - { - return node.NodeType; - } - } - } - catch - { - // Ignore errors - } - - return null; - } + public static IObservable> ObserveAgents( + IMessageHub hub, + string? userPath, + string? spacePath) + => AgentPickerProjection.ObserveAgents(hub, userPath, spacePath) + .Select(agents => (IReadOnlyList)OrderByRelevance(agents, spacePath, null)); /// - /// Orders agents by Order then by DisplayName. + /// Orders agents by Order then by display name (both sourced from the MeshNode + /// via ). /// public static IReadOnlyList OrderByRelevance( IEnumerable agents, @@ -118,7 +39,7 @@ public static IReadOnlyList OrderByRelevance( { return agents .OrderBy(a => a.Order) - .ThenBy(a => a.AgentConfiguration.DisplayName ?? a.Name) + .ThenBy(a => a.Name) .ToImmutableList(); } } diff --git a/src/MeshWeaver.AI/AgentPickerProjection.cs b/src/MeshWeaver.AI/AgentPickerProjection.cs new file mode 100644 index 000000000..2bea87b03 --- /dev/null +++ b/src/MeshWeaver.AI/AgentPickerProjection.cs @@ -0,0 +1,547 @@ +using System.Collections.Immutable; +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI; + +/// +/// Single source of truth for the chat picker's data flow: +/// the synced-query strings the view subscribes to, and the +/// projection that turns the resulting +/// snapshot into the / +/// lists bound to the agent / model +/// combo boxes. +/// +/// 🚨 The chat view (ThreadChatView.SubscribeToAgentNodes + +/// OnSyncedAgentSnapshot) calls these helpers directly. Tests in +/// AgentPickerProjectionTest drive the same +/// +/// pipe with the strings returns and run the +/// snapshot through / +/// . If a regression silently empties the +/// dropdowns at runtime, those tests fail too — no parallel +/// reconstruction. +/// +public static class AgentPickerProjection +{ + /// Named-query id for the agents synced subscription. Same id everywhere = one shared upstream subscription via the workspace's per-id cache. + public const string AgentsQueryId = "Agents"; + + /// Named-query id for the language-models synced subscription. + public const string ModelsQueryId = "LanguageModels"; + + /// Conventional namespace for built-in agents (matches BuiltInAgentProvider). + public const string AgentRootNamespace = "Agent"; + + /// + /// The dedicated sub-namespace each partition uses for its OWN agents — {partition}/Agent + /// (e.g. rbuergi/Agent, AgenticPension/Agent). Platform defaults live in the bare + /// (Agent). One registry, three layers, listed directly. + /// + public const string AgentSubNamespace = "Agent"; + + /// The dedicated sub-namespace each partition uses for its OWN models — {partition}/Model; + /// platform model defaults live in the bare Model namespace. + public const string ModelSubNamespace = "Model"; + + /// + /// Array form of for the hub.GetQuery(id, params string[]) + /// surface — a single-element array carrying THE one canonical agent-registry query. + /// + public static string[] BuildAgentQueries(string? userPath = null, string? spacePath = null) + => new[] { BuildAgentQuery(userPath, spacePath) }; + + /// + /// THE single canonical agent-registry query — the one place this string is defined. Agents live + /// in a dedicated /Agent sub-namespace PER PARTITION; the query lists the relevant ones + /// DIRECTLY (exact membership, NO graph/ancestor walk): + /// + /// platform defaults — namespace Agent (always); + /// the current space's — {spacePath}/Agent; + /// the chatting/owning user's — {userPath}/Agent. + /// + /// Produces e.g. namespace:rbuergi/Agent|AgenticPension/Agent|Agent nodeType:Agent. The + /// namespace:A|B|C alternation resolves (see ) to a + /// single namespace IN (...) exact-membership filter — so the combobox, the /agent + /// picker and the engine's agent selection all issue exactly this ONE query via hub.GetQuery + /// (per-user RLS at the caller's portal hub naturally hides private agents from non-owners). Utility + /// (generator) agents are kept unless — the conversational + /// surfaces drop them via . + /// + public static string BuildAgentQuery( + string? userPath = null, string? spacePath = null, bool excludeUtility = false) + => BuildRegistryQuery(AgentNodeType.NodeType, AgentSubNamespace, userPath, spacePath, + excludeUtility ? $" -content.modelTier:{UtilityModelTier}" : ""); + + /// The dedicated sub-namespace each partition uses for its OWN skills — {partition}/Skill; + /// platform skill defaults live in the bare Skill namespace. Same registry shape as agents/models. + public const string SkillSubNamespace = "Skill"; + + /// Array form of for the hub.GetQuery surface. + public static string[] BuildSkillQueries(string? userPath = null, string? spacePath = null) + => new[] { BuildSkillQuery(userPath, spacePath) }; + + /// + /// THE single canonical skill-registry query — IDENTICAL pattern to agents + models. Skills live in a + /// dedicated /Skill sub-namespace PER PARTITION (platform Skill + {space}/Skill + + /// {user}/Skill), listed directly as a namespace:A|B|C exact-membership alternation — one + /// registry pattern for every public top-level domain (Agent, Model, Skill, …). Produces e.g. + /// namespace:rbuergi/Skill|AgenticPension/Skill|Skill nodeType:Skill. + /// + public static string BuildSkillQuery(string? userPath = null, string? spacePath = null) + => BuildRegistryQuery(SkillNodeType.NodeType, SkillSubNamespace, userPath, spacePath, ""); + + // Rogue/reserved ROUTE partitions — auto-minted page artifacts (login, welcome, settings, …; mirrors + // the reserved-schema list in PostgreSqlCrossSchemaQueryProvider). They carry NO read policy and never + // hold registry nodes, so including one in the namespace IN(...) — e.g. when the chat context resolves + // to a rogue "login" node — fails the WHOLE query with "lacks Read permission on 'login'" and the + // picker/autocomplete goes empty. A read-only reserved-word set (allowed static; never written). + // ImmutableHashSet (not HashSet): a never-written constant lookup must use an immutable type so + // it satisfies the no-static-mutable-collection rule (NoStaticCollectionsTest / NoStaticState.md) + // without an allowlist entry — and the Collections Policy mandates Immutable over mutable anyway. + private static readonly ImmutableHashSet ReservedPartitions = + ImmutableHashSet.Create(StringComparer.OrdinalIgnoreCase, + "login", "markdown", "onboarding", "welcome", "settings", "storage"); + + /// True when 's partition (first segment) is a rogue/reserved ROUTE + /// partition (login, welcome, settings, …) — never a real space, so never a valid thread or registry + /// namespace. Creating a thread there is denied (no write policy) and tears the side-panel chat down. + public static bool IsReservedPartition(string? path) + => PartitionOf(path) is { } p && ReservedPartitions.Contains(p); + + /// + /// Assembles a per-partition registry query: the platform default namespace () + /// plus the user's and space's own ({partition}/{sub}), listed directly as a + /// namespace:A|B|C exact-membership alternation. No scope — agents/models are placed in a + /// flat, well-known namespace per partition, so there is no graph search. + /// + private static string BuildRegistryQuery( + string nodeType, string sub, string? userPath, string? spacePath, string extra) + { + var namespaces = new List(); + void Add(string? partition) + { + // Skip empty + rogue/reserved route partitions so a poisoned context can't break the query. + if (string.IsNullOrEmpty(partition) || ReservedPartitions.Contains(partition)) return; + var ns = $"{partition}/{sub}"; + if (!namespaces.Contains(ns, StringComparer.OrdinalIgnoreCase)) + namespaces.Add(ns); + } + Add(userPath); + Add(spacePath); + namespaces.Add(sub); // platform defaults — always present, last + var nsClause = namespaces.Count > 1 + ? $"namespace:{string.Join("|", namespaces)}" + : $"namespace:{namespaces[0]}"; + return $"{nsClause} nodeType:{nodeType}{extra}"; + } + + /// + /// The signed-in user's HOME partition — the partition whose {user}/Skill (and + /// {user}/Agent, {user}/_Provider) namespaces the registry surfaces. Resolved from the + /// hub/circuit identity, preferring the durable then the + /// per-request ; a leaked system-security or hub-shaped + /// principal (sync/, mesh/, …) is filtered out — never a real user partition. Returns + /// null when no real user is set. The SINGLE source of truth for "who is the user" across the + /// picker, the slash-skill resolver, and the autocomplete — mirrors ThreadChatView.ResolveUserHome + /// so a user's own agents / models / skills surface the SAME way everywhere. + /// + public static string? ResolveUserHome(MeshWeaver.Messaging.AccessService? accessService) + { + if (accessService is null) return null; + foreach (var candidate in new[] { accessService.CircuitContext?.ObjectId, accessService.Context?.ObjectId }) + if (!string.IsNullOrEmpty(candidate) + && candidate != MeshWeaver.Mesh.Security.WellKnownUsers.System + && !MeshWeaver.Messaging.AccessService.LooksLikeHubPrincipal(candidate)) + return candidate; + return null; + } + + /// The partition (top-level path segment) a context path belongs to — the "space" whose + /// /Agent + /Model namespaces the registry surfaces. AgenticPension/Foo/_Thread/x + /// → AgenticPension; null/empty → null. + public static string? PartitionOf(string? path) + { + if (string.IsNullOrEmpty(path)) return null; + var trimmed = path.Trim('/'); + var slash = trimmed.IndexOf('/'); + return slash < 0 ? trimmed : trimmed[..slash]; + } + + /// + /// The (contextPath, nodeTypePath) pair the chat picker MUST feed into + /// / so that all three + /// context-scoped queries are issued (built-in + per-context-ancestors + + /// per-NodeType-namespace). Sourced from the SAME resolved navigation context that + /// reads at execution time: the context PATH is the + /// resolved node's main path (, satellite + /// segments stripped) and is that node's + /// . When both are populated the picker surfaces a + /// Space's own agents/models — the chat-side-panel bug where a frequently-NULL + /// ambient context collapsed the union to the built-in query only. + /// + public readonly record struct PickerContext(string? ContextPath, string? NodeTypePath); + + /// + /// Derives the timing-safe for the picker from the latest + /// RESOLVED navigation context () — the value the chat view + /// reads off INavigationService.NavigationContext (a ReplaySubject(1), so the last + /// value replays). The (the view's seeded + /// initialContext) is used only when the navigation context has not yet resolved to + /// a usable node (still loading, or the bare chat route). This is the single source + /// of truth for "where does the picker get currentPath + nodeTypePath": both the Blazor + /// view (ThreadChatView.OpenPicker) and its tests derive args through here, so the + /// "all 3 queries when context+nodeType resolve" contract can be pinned without a Blazor + /// harness. + /// + public static PickerContext DerivePickerContext( + NavigationContext? resolved, string? fallbackContextPath = null) + { + // Prefer the RESOLVED nav context. Skip it while it's still null (loading / not-found) + // or pointing at the bare chat route — those carry no real content node. In that case + // fall back to the view's already-seeded initialContext so we never collapse to the + // built-in-only query just because the async resolution hasn't landed yet. + var usable = resolved is not null + && resolved.Path != "chat" + && !string.IsNullOrEmpty(resolved.PrimaryPath) + ? resolved + : null; + + var contextPath = NormalizeContextPath(usable?.PrimaryPath) + ?? NormalizeContextPath(fallbackContextPath); + // NodeType only comes from a resolved node — the fallback path is just a string, it + // carries no NodeType. This mirrors AgentChatClient.Initialize: nodeTypePath ??= Context?.Node?.NodeType. + var nodeTypePath = usable?.Node?.NodeType; + return new PickerContext(contextPath, nodeTypePath); + } + + /// + /// Strips any trailing satellite segments (segments starting with _, e.g. + /// _Thread/<slug>, _Comment/<id>) from a context path so the + /// picker queries reason about the main node, not the satellite. Returns the input + /// unchanged when null/empty or when no _ segment is present. Mirrors the + /// view's / AgentChatClient's private NormalizeContextPath — kept here so the + /// picker-context derivation is self-contained and testable. + /// + public static string? NormalizeContextPath(string? path) + { + if (string.IsNullOrEmpty(path)) + return path; + var segments = path.Split('/'); + for (var i = 0; i < segments.Length; i++) + { + if (segments[i].StartsWith('_')) + return string.Join('/', segments, 0, i); + } + return path; + } + + /// + /// 🚨 The EXACT pipeline the chat agent combobox is bound to. The view subscribes to this; tests + /// subscribe to this. No parallel reconstruction of queries / projection anywhere. + /// is the current space partition; the + /// user's home partition — their /Agent namespaces plus the platform default are listed. + /// + /// 🚨🚨 MUST be a hub LOCAL TO THE CALLER'S CONTEXT — the + /// portal hub (the GUI / Blazor circuit, carrying the user's identity) or the thread + /// hub (thread execution, carrying the thread OWNER's identity). The query reads the + /// per-partition {user}/Agent + {space}/Agent namespaces via hub.GetQuery, + /// whose per-user RLS keys off the hub's AccessContext. Pass a SERVER-SIDE layout-area hub (a + /// node's per-node hub) and you get the hub principal, NOT the user — RLS strips the + /// user/space namespaces (empty dropdown) AND the cross-partition subscribe STORMS the portal + /// (the 2026-06-17 atioz wedge: a server-side combobox in ThreadComposerView). GUI → + /// BlazorView.Hub (= PortalApplication.Hub); exec → ThreadExecution's + /// parentHub. NEVER a LayoutAreaHost.Hub for the per-partition query. + /// + public static IObservable> ObserveAgents( + IMessageHub hub, string? userPath = null, string? spacePath = null) + => ObserveSnapshot(hub.GetWorkspace(), hub, + $"{AgentsQueryId}|u={userPath ?? ""}|s={spacePath ?? ""}", + BuildAgentQuery(userPath, spacePath)) + .Select(snapshot => ProjectAgents(snapshot, hub.JsonSerializerOptions)); + + /// + /// 🚨 The EXACT pipeline the chat model combobox is bound to. The view subscribes to this; tests + /// subscribe to this. Models stay on the _Provider catalog shape (providers contain models + /// + credentials) — the per-partition /Model registry is the next increment. + /// + public static IObservable> ObserveModels( + IWorkspace workspace, IMessageHub hub, + string? currentPath = null, string? nodeTypePath = null, + IReadOnlyList? selectedProviderPaths = null, + string? userPath = null) + => ObserveSnapshot(workspace, hub, + BuildModelQueryId(ModelsQueryId, currentPath, nodeTypePath, selectedProviderPaths, userPath), + BuildModelQueries(currentPath, nodeTypePath, selectedProviderPaths, userPath)) + .Select(snapshot => ProjectModels(snapshot, hub.JsonSerializerOptions)); + + /// Named-query id for the harnesses synced subscription. + public const string HarnessesQueryId = "Harnesses"; + + /// + /// The DEFAULT composer selection — resolved purely by ORDER, never hardcoded. For each registry + /// (agent / model / harness) the default is the node with the LOWEST Order (the Order = -1 + /// convention: "to make something the default, set its order to -1"). No hardcoded agent name, model + /// id, or harness; nothing is invented when a registry is empty (the field stays null). Reactive + + /// testable like / ; the chat view subscribes + /// to this to seed a new composer. + /// Utility (generator) agents are excluded — the default is never a background generator. + /// + public static IObservable ObserveDefaultComposer( + IMessageHub hub, string? userPath = null, string? spacePath = null, + string? currentPath = null, string? nodeTypePath = null, + IReadOnlyList? selectedProviderPaths = null) + { + var workspace = hub.GetWorkspace(); + + var agent = ObserveAgents(hub, userPath, spacePath) + .Select(list => list + .Where(a => !IsUtilityAgent(a) && !string.IsNullOrEmpty(a.Path)) + .OrderBy(a => a.Order) + .Select(a => a.Path) + .FirstOrDefault()); + + var model = ObserveModels(workspace, hub, currentPath, nodeTypePath, selectedProviderPaths, userPath) + .Select(list => list + .Where(m => !string.IsNullOrEmpty(m.Path)) + .OrderBy(m => m.Order) + .Select(m => m.Path) + .FirstOrDefault()); + + var harness = ObserveSnapshot(workspace, hub, + $"{HarnessesQueryId}|u={userPath ?? ""}|s={spacePath ?? ""}", + BuildRegistryQuery(HarnessNodeType.NodeType, HarnessNodeType.RootNamespace, userPath, spacePath, "")) + .Select(snapshot => snapshot + .Where(n => n.Path != null + && string.Equals(n.NodeType, HarnessNodeType.NodeType, StringComparison.OrdinalIgnoreCase)) + .OrderBy(n => n.Order ?? 0) + .Select(n => n.Path) + .FirstOrDefault()); + + return agent.CombineLatest(model, harness, + (a, m, h) => new ThreadComposer { AgentName = a, ModelName = m, Harness = h }); + } + + /// + /// The model picker queries: the system _Provider catalog plus per-context / per-NodeType / + /// per-user subtrees and any user-selected provider subtree — all nodeType:LanguageModel|ModelProvider, + /// varying only namespace + scope (the synced-collection all-Initial gating constraint). The + /// per-partition flat /Model registry is the next increment; credentials live in _Provider. + /// + public static string[] BuildModelQueries( + string? currentPath = null, + string? nodeTypePath = null, + IEnumerable? selectedProviderPaths = null, + string? userPath = null) + { + var typeFilter = $"{LanguageModelNodeType.NodeType}|{ModelProviderNodeType.NodeType}"; + var queries = new List + { + $"namespace:{ModelProviderNodeType.RootNamespace} nodeType:{typeFilter} scope:descendants", + }; + // Skip reserved/rogue ROUTE partitions (login, welcome, settings, …): a reserved currentPath/ + // nodeTypePath would make namespace:{login}/_Provider read the policy-less reserved partition and + // fail the WHOLE model query with "lacks Read permission on 'login'" — the picker goes empty. + // Mirrors BuildRegistryQuery's filter (the agent/skill queries already skip these). + if (!string.IsNullOrEmpty(currentPath) && !IsReservedPartition(currentPath)) + queries.Add($"namespace:{currentPath}/{ModelProviderNodeType.RootNamespace} nodeType:{typeFilter} scope:descendants"); + if (!string.IsNullOrEmpty(nodeTypePath) && !IsReservedPartition(nodeTypePath)) + queries.Add($"namespace:{nodeTypePath}/{ModelProviderNodeType.RootNamespace} nodeType:{typeFilter} scope:descendants"); + if (!string.IsNullOrEmpty(userPath)) + queries.Add($"namespace:{ModelProviderNodeType.UserNamespacePath(userPath)} nodeType:{typeFilter} scope:descendants"); + if (selectedProviderPaths != null) + foreach (var path in selectedProviderPaths) + if (!string.IsNullOrEmpty(path)) + queries.Add($"namespace:{path} nodeType:{typeFilter} scope:selfAndDescendants"); + return queries.ToArray(); + } + + /// Context-scoped cache id for the model query (provider selection must be part of the id). + private static string BuildModelQueryId( + string baseId, string? currentPath, string? nodeTypePath, + IReadOnlyList? selectedProviderPaths, string? userPath) + { + var selected = selectedProviderPaths is { Count: > 0 } + ? string.Join(",", selectedProviderPaths.OrderBy(x => x, StringComparer.Ordinal)) + : ""; + return $"{baseId}|p={currentPath ?? ""}|t={nodeTypePath ?? ""}|s={selected}|u={userPath ?? ""}"; + } + + /// + /// Live MeshNode snapshot for a registry query via hub.GetQuery — the centralized, + /// per-user-RLS surface (one shared upstream subscription per id across the chat view, + /// AgentChatClient and the driver factory). "hub.GetQuery to be precise": reads run under the + /// caller hub's identity, so a thread's agents resolve under the thread OWNER. + /// + public static IObservable> ObserveSnapshot( + IWorkspace workspace, IMessageHub hub, string queryId, params string[] queries) + { + var logger = hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.AI.AgentPickerProjection"); + logger?.LogDebug( + "[AgentPicker] subscribe id={Id} hub={Hub} queries=[{Queries}]", + queryId, hub.Address, string.Join(" | ", queries)); + return hub.GetQuery(queryId, queries) + .Do(snapshot => + { + var nodes = snapshot as IReadOnlyCollection ?? snapshot.ToList(); + logger?.LogDebug( + "[AgentPicker] raw snapshot id={Id} count={Count} types=[{Types}]", + queryId, nodes.Count, + string.Join(",", nodes.GroupBy(n => n.NodeType ?? "(null)") + .Select(g => $"{g.Key}={g.Count()}"))); + }); + } + + /// + /// Projects the synced-query snapshot into the agent picker's bound list. + /// Same shape as ThreadChatView.OnSyncedAgentSnapshot: + /// Agent-typed nodes only; tolerates Content + /// when the receiving hub's typed registry doesn't have + /// wired up; sorts by Order then Name. + /// + public static IReadOnlyList ProjectAgents( + IEnumerable snapshot, JsonSerializerOptions jsonOptions) + { + var byPath = new Dictionary(StringComparer.Ordinal); + foreach (var node in snapshot) + { + if (node.Path == null) continue; + if (!string.Equals(node.NodeType, AgentNodeType.NodeType, StringComparison.OrdinalIgnoreCase)) + continue; + + var info = ToAgentDisplayInfo(node, jsonOptions); + if (info != null) + byPath[node.Path] = info; + } + + // Group by harness (GroupName) so the picker shows major categories — + // Claude Code / GitHub Copilot / MeshWeaver — with each harness's agents + // contiguous (SimpleDropdown renders a header when the group key changes). + // Alphabetical group order happens to give Claude Code, GitHub Copilot, + // MeshWeaver; within a group, Order then Name (Assistant's order:-1 leads MeshWeaver). + return byPath.Values + .OrderBy(a => a.GroupName ?? Harnesses.MeshWeaver, StringComparer.OrdinalIgnoreCase) + .ThenBy(a => a.Order) + .ThenBy(a => a.Name, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + /// The utility model tier marks a programmatic generator agent (ThreadNamer, + /// NodeInitializer, DescriptionWriter) — invoked by services, never a chat participant. + public const string UtilityModelTier = "utility"; + + /// + /// True when the agent is a background GENERATOR (modelTier: utility) — ThreadNamer, + /// NodeInitializer, DescriptionWriter. These emit structured "Name:/Id:/Svg:" output and are + /// invoked programmatically (, ), + /// so they must be hidden from every CONVERSATIONAL surface (the chat agent picker, /agent, + /// and @-references) — otherwise e.g. ThreadNamer answers a user's "hi" with "Name: …\nId: …". + /// + /// The filter is applied at the chat UI (ThreadChatView.OnAgentList), NOT inside + /// : the generators build their OWN and + /// SetSelectedAgent("NodeInitializer"/"DescriptionWriter"), so the projection must keep + /// utility agents for them to resolve. + /// + public static bool IsUtilityAgent(AgentDisplayInfo info) => + string.Equals(info.AgentConfiguration?.ModelTier, UtilityModelTier, StringComparison.OrdinalIgnoreCase); + + /// + /// Projects the synced-query snapshot into the model picker's bound list. + /// Mirrors ThreadChatView.OnSyncedAgentSnapshot's LanguageModel + /// branch (without the factory-baseline merge — that lives in the + /// view's RebuildAvailableModels). + /// + public static IReadOnlyList ProjectModels( + IEnumerable snapshot, JsonSerializerOptions jsonOptions) + { + var byPath = new Dictionary(StringComparer.Ordinal); + foreach (var node in snapshot) + { + if (node.Path == null) continue; + if (!string.Equals(node.NodeType, LanguageModelNodeType.NodeType, StringComparison.OrdinalIgnoreCase)) + continue; + + var info = ToModelInfo(node, jsonOptions); + if (info != null) + byPath[node.Path] = info; + } + + return byPath.Values + .OrderBy(m => m.Order) + .ThenBy(m => m.Provider, StringComparer.OrdinalIgnoreCase) + .ThenBy(m => m.Name, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + /// + /// Single MeshNode → AgentDisplayInfo projection. Same Content + /// switch as the chat view: typed AgentConfiguration first, raw + /// JsonElement fallback (when the source hub didn't have + /// AddAITypes applied), null otherwise. + /// + public static AgentDisplayInfo? ToAgentDisplayInfo( + MeshNode node, JsonSerializerOptions jsonOptions) + { + var config = node.Content switch + { + AgentConfiguration ac => ac, + JsonElement je => TryDeserialise(je, jsonOptions), + _ => null, + }; + if (config == null) return null; + // Display metadata is sourced from the MeshNode (the single source of truth); + // only agent-specific bits (CustomIconSvg, the config itself) come from Content. + return new AgentDisplayInfo + { + Name = node.Name ?? config.Id, + Path = node.Path, + Description = node.Description ?? config.Description ?? "", + GroupName = node.Category, + Order = node.Order ?? 0, + Icon = node.Icon, + CustomIconSvg = config.CustomIconSvg, + AgentConfiguration = config, + }; + } + + /// + /// Single MeshNode → ModelInfo projection. JsonElement fallback + /// covers the same source-hub-typed-registry-mismatch edge case + /// as . + /// + public static ModelInfo? ToModelInfo( + MeshNode node, JsonSerializerOptions jsonOptions) + { + var def = node.Content switch + { + ModelDefinition md => md, + JsonElement je => TryDeserialise(je, jsonOptions), + _ => null, + }; + // Carry the node PATH onto the ModelInfo (like ToAgentDisplayInfo does for agents) — + // a model selection must persist the node path onto the composer's ModelName, not the + // bare model id, so the MeshNode picker resolves it. Without this the /model selection + // wrote only the name and the picker couldn't resolve the node (the "dialog breaks" bug). + return def?.ToModelInfo() is { } info ? info with { Path = node.Path } : null; + } + + private static T? TryDeserialise(JsonElement je, JsonSerializerOptions jsonOptions) where T : class + { + try + { + return JsonSerializer.Deserialize(je.GetRawText(), jsonOptions); + } + catch + { + return null; + } + } +} diff --git a/src/MeshWeaver.AI/AgentStaticRepoSource.cs b/src/MeshWeaver.AI/AgentStaticRepoSource.cs new file mode 100644 index 000000000..da8c03921 --- /dev/null +++ b/src/MeshWeaver.AI/AgentStaticRepoSource.cs @@ -0,0 +1,53 @@ +using MeshWeaver.Markdown; +using MeshWeaver.Mesh; + +namespace MeshWeaver.AI; + +/// +/// The built-in agents as a static-repo import source for the Agent partition. The same +/// nodes serves in-memory are here materialized (content + +/// prerender) into the DB partition by the static-repo import on boot, so agents are served from +/// the database on the distributed/PG path (Orleans routing doesn't consult the in-memory adapter). +/// Governance nodes (the access policy) are NOT imported — they stay served by the in-memory +/// provider. See Doc/Architecture/StaticRepoImport.md. +/// +public sealed class AgentStaticRepoSource(BuiltInAgentProvider provider) : IStaticRepoSource +{ + /// + public string Partition => "Agent"; + + /// + // Agent definitions ship with no meaningful version → fingerprint on content, so an edited + // agent .md re-imports. + public bool Versioned => false; + + /// + // Content agent nodes PLUS the partition's PublicRead "_Policy" (PartitionAccessPolicy). On the + // SYNCED path the in-memory provider that served the policy is gated off, so the policy MUST be + // imported or the partition has no read policy → its nodes are unreadable (same wedge as Harness — + // see HarnessStaticRepoSource + OrleansHarnessPartitionPublicReadTest). Agent happens to work on + // atioz today only because its policy persisted in the DB before the sync-gate existed; a fresh + // env would fail identically. Only OTHER "_"-governance (per-user _Access grants) is dropped. + public IReadOnlyList EnumerateSourceNodes() => + provider.GetStaticNodes() + .Where(n => n.NodeType == "PartitionAccessPolicy" + || !n.Segments.Skip(1).Any(seg => seg.StartsWith('_'))) + .ToArray(); + + /// + public MeshNode? PartitionRoot => new("Agent") + { + Name = "Agents", + NodeType = "Space", + State = MeshNodeState.Active, + Content = new MarkdownContent + { + Content = """ + # Agents + + The built-in agents available across the platform. Open one to see its instructions, + or use the chat input below to start a thread with an agent. + """ + } + }; +} diff --git a/src/MeshWeaver.AI/AgentView.cs b/src/MeshWeaver.AI/AgentView.cs index 323b3d83e..6d81ac7a9 100644 --- a/src/MeshWeaver.AI/AgentView.cs +++ b/src/MeshWeaver.AI/AgentView.cs @@ -1,5 +1,7 @@ using System.Collections.Immutable; using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using System.Text.Json; using Humanizer; using MeshWeaver.Application.Styles; using MeshWeaver.Data; @@ -7,11 +9,13 @@ using MeshWeaver.Layout; using MeshWeaver.Layout.Composition; using MeshWeaver.Layout.Domain; +using MeshWeaver.Mesh; using MeshWeaver.Mesh.Services; using MeshWeaver.Messaging; using MeshWeaver.ShortGuid; using MeshWeaver.Utils; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; namespace MeshWeaver.AI; @@ -27,8 +31,6 @@ public static class AgentView public const string DetailsArea = "Details"; public const string EditArea = "Edit"; - private const string AgentDataId = "agent"; - /// /// Adds the Agent views to the hub's layout for Agent nodes. /// Catalog is the default view showing all agents. @@ -54,24 +56,16 @@ public static UiControl Catalog(LayoutAreaHost host, RenderingContext ctx) return Controls.Stack .WithWidth("100%") .WithView( - (h, c) => Observable.FromAsync(async () => + (h, c) => { var meshQuery = host.Hub.ServiceProvider.GetService(); if (meshQuery == null) - return RenderError("Query service not available."); - - List agents; - try - { - agents = await meshQuery.QueryAsync("nodeType:Agent").ToListAsync(); - } - catch - { - agents = []; - } + return Observable.Return(RenderError("Query service not available.")); - return BuildCatalogContent(agents); - }), + return meshQuery.Query(MeshQueryRequest.FromQuery("nodeType:Agent")) + .Select(change => BuildCatalogContent(change.Items.ToList())) + .Catch(_ => Observable.Return(BuildCatalogContent([]))); + }, "Content"); } @@ -107,34 +101,50 @@ private static UiControl BuildCatalogContent(List agents) /// /// Renders the Details area for an Agent. - /// Shows an overview of the agent configuration. + /// Shows an overview of the agent configuration. Node-level metadata (name, + /// description, icon, group, order) is read from the owning MeshNode — the single + /// source of truth — and only agent-specific behaviour from the AgentConfiguration. /// public static UiControl Details(LayoutAreaHost host, RenderingContext ctx) { - // Subscribe to agent data stream - host.SubscribeToDataStream(AgentDataId, host.Workspace.GetNodeContent()); - return Controls.Stack .WithWidth("100%") .WithView( - (h, c) => h.GetDataStream(AgentDataId) - .Select(agent => + (h, c) => host.Workspace.GetMeshNodeStream() + .Select(node => { - if (agent == null) + var agent = AsAgentConfiguration(node, host.Hub.JsonSerializerOptions); + if (node == null || agent == null) return RenderLoading("Loading agent..."); - return BuildDetailsLayout(host, agent); + return BuildDetailsLayout(host, node, agent); }), "Content" ); } - private static UiControl BuildDetailsLayout(LayoutAreaHost host, AgentConfiguration agent) + /// Resolves the typed from a node's Content, + /// tolerating a when the hub's registry isn't AI-typed. + private static AgentConfiguration? AsAgentConfiguration(MeshNode? node, JsonSerializerOptions jsonOptions) + => node?.Content switch + { + AgentConfiguration ac => ac, + JsonElement je => TryDeserialiseConfig(je, jsonOptions), + _ => null, + }; + + private static AgentConfiguration? TryDeserialiseConfig(JsonElement je, JsonSerializerOptions jsonOptions) + { + try { return JsonSerializer.Deserialize(je.GetRawText(), jsonOptions); } + catch { return null; } + } + + private static UiControl BuildDetailsLayout(LayoutAreaHost host, MeshNode node, AgentConfiguration agent) { var hubAddress = host.Hub.Address; var stack = Controls.Stack.WithWidth("100%").WithStyle("padding: 24px;"); - // Header with edit button - var displayName = agent.DisplayName ?? agent.Id.Wordify(); + // Header with edit button — display name from the node. + var displayName = node.Name ?? node.Id.Wordify(); var headerRow = Controls.Stack .WithOrientation(Orientation.Horizontal) .WithStyle("justify-content: space-between; align-items: center; margin-bottom: 8px;") @@ -146,10 +156,10 @@ private static UiControl BuildDetailsLayout(LayoutAreaHost host, AgentConfigurat stack = stack.WithView(headerRow); - // Description - if (!string.IsNullOrEmpty(agent.Description)) + // Description — from the node. + if (!string.IsNullOrEmpty(node.Description)) { - stack = stack.WithView(Controls.Html($"

    {System.Web.HttpUtility.HtmlEncode(agent.Description)}

    ")); + stack = stack.WithView(Controls.Html($"

    {System.Web.HttpUtility.HtmlEncode(node.Description)}

    ")); } // Attributes badges @@ -159,18 +169,18 @@ private static UiControl BuildDetailsLayout(LayoutAreaHost host, AgentConfigurat stack = stack.WithView(Controls.Html($"
    {attributes}
    ")); } - // Info card + // Info card — node-level rows read from the node; only the context pattern + // (genuinely agent-specific) comes from the configuration. var infoCard = Controls.Stack .WithStyle("background: var(--neutral-layer-2); border-radius: 8px; padding: 20px; margin-bottom: 24px;"); - infoCard = infoCard.WithView(BuildInfoRow("ID", agent.Id)); - if (!string.IsNullOrEmpty(agent.GroupName)) - infoCard = infoCard.WithView(BuildInfoRow("Group", agent.GroupName)); - if (!string.IsNullOrEmpty(agent.Icon)) - infoCard = infoCard.WithView(BuildInfoRow("Icon", agent.Icon)); - infoCard = infoCard.WithView(BuildInfoRow("Display Order", agent.Order.ToString())); - if (!string.IsNullOrEmpty(agent.PreferredModel)) - infoCard = infoCard.WithView(BuildInfoRow("Preferred Model", agent.PreferredModel)); + infoCard = infoCard.WithView(BuildInfoRow("ID", node.Id)); + if (!string.IsNullOrEmpty(node.Category)) + infoCard = infoCard.WithView(BuildInfoRow("Group", node.Category)); + if (!string.IsNullOrEmpty(node.Icon)) + infoCard = infoCard.WithView(BuildInfoRow("Icon", node.Icon)); + if (node.Order is { } order) + infoCard = infoCard.WithView(BuildInfoRow("Display Order", order.ToString())); if (!string.IsNullOrEmpty(agent.ContextMatchPattern)) infoCard = infoCard.WithView(BuildInfoRow("Context Pattern", agent.ContextMatchPattern)); @@ -232,28 +242,29 @@ private static string BuildAttributeBadges(AgentConfiguration agent) } /// - /// Renders the Edit area for an Agent. + /// Renders the Edit area for an Agent. Node-level fields (name, description, icon, + /// group, order) persist to the owning MeshNode; agent-specific fields persist to the + /// AgentConfiguration content. Both land in one stream.Update so there is a + /// single source of truth — no duplicated metadata. /// public static UiControl Edit(LayoutAreaHost host, RenderingContext ctx) { - // Subscribe to agent data stream - host.SubscribeToDataStream(AgentDataId, host.Workspace.GetNodeContent()); - return Controls.Stack .WithWidth("100%") .WithView( - (h, c) => h.GetDataStream(AgentDataId) - .Select(agent => + (h, c) => host.Workspace.GetMeshNodeStream() + .Select(node => { - if (agent == null) + var agent = AsAgentConfiguration(node, host.Hub.JsonSerializerOptions); + if (node == null || agent == null) return RenderLoading("Loading agent..."); - return BuildEditLayout(host, agent); + return BuildEditLayout(host, node, agent); }), "Content" ); } - private static UiControl BuildEditLayout(LayoutAreaHost host, AgentConfiguration agent) + private static UiControl BuildEditLayout(LayoutAreaHost host, MeshNode node, AgentConfiguration agent) { var hubAddress = host.Hub.Address; var stack = Controls.Stack.WithWidth("100%").WithStyle("padding: 24px;"); @@ -264,26 +275,24 @@ private static UiControl BuildEditLayout(LayoutAreaHost host, AgentConfiguration var iconNameDataId = Guid.NewGuid().AsString(); var groupNameDataId = Guid.NewGuid().AsString(); var orderDataId = Guid.NewGuid().AsString(); - var preferredModelDataId = Guid.NewGuid().AsString(); var contextMatchPatternDataId = Guid.NewGuid().AsString(); var isDefaultDataId = Guid.NewGuid().AsString(); var exposedInNavigatorDataId = Guid.NewGuid().AsString(); var instructionsDataId = Guid.NewGuid().AsString(); - // Initialize data streams - host.UpdateData(displayNameDataId, agent.DisplayName ?? ""); - host.UpdateData(descriptionDataId, agent.Description ?? ""); - host.UpdateData(iconNameDataId, agent.Icon ?? ""); - host.UpdateData(groupNameDataId, agent.GroupName ?? ""); - host.UpdateData(orderDataId, agent.Order.ToString()); - host.UpdateData(preferredModelDataId, agent.PreferredModel ?? ""); + // Initialize data streams — node-level fields from the node, agent-level from config. + host.UpdateData(displayNameDataId, node.Name ?? ""); + host.UpdateData(descriptionDataId, node.Description ?? ""); + host.UpdateData(iconNameDataId, node.Icon ?? ""); + host.UpdateData(groupNameDataId, node.Category ?? ""); + host.UpdateData(orderDataId, (node.Order ?? 0).ToString()); host.UpdateData(contextMatchPatternDataId, agent.ContextMatchPattern ?? ""); host.UpdateData(isDefaultDataId, agent.IsDefault ? "true" : "false"); host.UpdateData(exposedInNavigatorDataId, agent.ExposedInNavigator ? "true" : "false"); host.UpdateData(instructionsDataId, agent.Instructions ?? ""); // Header - var displayName = agent.DisplayName ?? agent.Id.Wordify(); + var displayName = node.Name ?? node.Id.Wordify(); stack = stack.WithView(Controls.Html($"

    Edit: {System.Web.HttpUtility.HtmlEncode(displayName)}

    ")); // Form fields @@ -335,15 +344,6 @@ private static UiControl BuildEditLayout(LayoutAreaHost host, AgentConfiguration .WithImmediate(true) with { DataContext = LayoutAreaReference.GetDataPointer(orderDataId) })); - // Preferred Model - stack = stack.WithView(Controls.Stack - .WithStyle(formStyle) - .WithView(Controls.Html("")) - .WithView(new TextFieldControl(new JsonPointerReference("")) - .WithPlaceholder("e.g., claude-sonnet-4-5") - .WithImmediate(true) with - { DataContext = LayoutAreaReference.GetDataPointer(preferredModelDataId) })); - // Context Match Pattern stack = stack.WithView(Controls.Stack .WithStyle(formStyle) @@ -405,84 +405,75 @@ private static UiControl BuildEditLayout(LayoutAreaHost host, AgentConfiguration buttonRow = buttonRow.WithView(Controls.Button("Save") .WithAppearance(Appearance.Accent) .WithIconStart(FluentIcons.Save()) - .WithClickAction(async actx => + .WithClickAction(actx => { - // Get all field values - var newDisplayName = await host.Stream.GetDataStream(displayNameDataId).FirstAsync(); - var newDescription = await host.Stream.GetDataStream(descriptionDataId).FirstAsync(); - var newIconName = await host.Stream.GetDataStream(iconNameDataId).FirstAsync(); - var newGroupName = await host.Stream.GetDataStream(groupNameDataId).FirstAsync(); - var newOrderStr = await host.Stream.GetDataStream(orderDataId).FirstAsync(); - var newPreferredModel = await host.Stream.GetDataStream(preferredModelDataId).FirstAsync(); - var newContextMatchPattern = await host.Stream.GetDataStream(contextMatchPatternDataId).FirstAsync(); - var newIsDefaultStr = await host.Stream.GetDataStream(isDefaultDataId).FirstAsync(); - var newExposedInNavigatorStr = await host.Stream.GetDataStream(exposedInNavigatorDataId).FirstAsync(); - var newInstructions = await host.Stream.GetDataStream(instructionsDataId).FirstAsync(); - - // Parse order - if (!int.TryParse(newOrderStr, out var newOrder)) - newOrder = 0; - - // Parse booleans - var newIsDefault = newIsDefaultStr?.Equals("true", StringComparison.OrdinalIgnoreCase) == true; - var newExposedInNavigator = newExposedInNavigatorStr?.Equals("true", StringComparison.OrdinalIgnoreCase) == true; - - // Update the AgentConfiguration - var updatedAgent = agent with - { - DisplayName = string.IsNullOrWhiteSpace(newDisplayName) ? null : newDisplayName, - Description = string.IsNullOrWhiteSpace(newDescription) ? null : newDescription, - Icon = string.IsNullOrWhiteSpace(newIconName) ? null : newIconName, - GroupName = string.IsNullOrWhiteSpace(newGroupName) ? null : newGroupName, - Order = newOrder, - PreferredModel = string.IsNullOrWhiteSpace(newPreferredModel) ? null : newPreferredModel, - ContextMatchPattern = string.IsNullOrWhiteSpace(newContextMatchPattern) ? null : newContextMatchPattern, - IsDefault = newIsDefault, - ExposedInNavigator = newExposedInNavigator, - Instructions = string.IsNullOrWhiteSpace(newInstructions) ? null : newInstructions - }; - - using var cts = new CancellationTokenSource(10.Seconds()); - var delivery = actx.Host.Hub.Post( - new DataChangeRequest { ChangedBy = actx.Host.Stream.ClientId }.WithUpdates(updatedAgent), - o => o.WithTarget(hubAddress))!; - var callbackResponse = await actx.Host.Hub.RegisterCallback(delivery, (d, _) => Task.FromResult(d), cts.Token); - - // Handle routing failures (e.g., agent hub unreachable) and unexpected - // response shapes before touching the DataChangeResponse fields. - if (callbackResponse is IMessageDelivery deliveryFailure) - { - var dialog = Controls.Dialog( - Controls.Markdown($"**Error saving:**\n\n{deliveryFailure.Message.Message ?? "Delivery failed"}"), - "Save Failed" - ).WithSize("M"); - actx.Host.UpdateArea(DialogControl.DialogArea, dialog); - return; - } - if (callbackResponse is not IMessageDelivery dataChange) - { - var dialog = Controls.Dialog( - Controls.Markdown($"**Error saving:** Unexpected response `{callbackResponse.Message?.GetType().Name ?? "null"}`."), - "Save Failed" - ).WithSize("M"); - actx.Host.UpdateArea(DialogControl.DialogArea, dialog); - return; - } - var responseMsg = dataChange.Message; - - if (responseMsg.Log.Status != ActivityStatus.Succeeded) - { - var errorDialog = Controls.Dialog( - Controls.Markdown($"**Error saving:**\n\n{responseMsg.Log}"), - "Save Failed" - ).WithSize("M"); - actx.Host.UpdateArea(DialogControl.DialogArea, errorDialog); - return; - } - - // Navigate back to details - var overviewHref = new LayoutAreaReference(DetailsArea).ToHref(hubAddress); - actx.Host.UpdateArea(actx.Area, new RedirectControl(overviewHref)); + // Sync click action — Subscribe to combined snapshot of all form streams, + // then write through the canonical mesh-node mutation API. Node-level + // fields land on the MeshNode; agent-specific fields on its Content. One + // update, one source of truth. + Observable.CombineLatest( + host.Stream.GetDataStream(displayNameDataId).Take(1), + host.Stream.GetDataStream(descriptionDataId).Take(1), + host.Stream.GetDataStream(iconNameDataId).Take(1), + host.Stream.GetDataStream(groupNameDataId).Take(1), + host.Stream.GetDataStream(orderDataId).Take(1), + host.Stream.GetDataStream(contextMatchPatternDataId).Take(1), + host.Stream.GetDataStream(isDefaultDataId).Take(1), + host.Stream.GetDataStream(exposedInNavigatorDataId).Take(1), + host.Stream.GetDataStream(instructionsDataId).Take(1), + (newDisplayName, newDescription, newIconName, newGroupName, newOrderStr, + newContextMatchPattern, newIsDefaultStr, newExposedInNavigatorStr, newInstructions) => + (newDisplayName, newDescription, newIconName, newGroupName, newOrderStr, + newContextMatchPattern, newIsDefaultStr, newExposedInNavigatorStr, newInstructions)) + .Take(1) + .Subscribe(form => + { + var (newDisplayName, newDescription, newIconName, newGroupName, newOrderStr, + newContextMatchPattern, newIsDefaultStr, newExposedInNavigatorStr, newInstructions) = form; + int? newOrder = int.TryParse(newOrderStr, out var parsed) ? parsed : null; + var newIsDefault = newIsDefaultStr?.Equals("true", StringComparison.OrdinalIgnoreCase) == true; + var newExposedInNavigator = newExposedInNavigatorStr?.Equals("true", StringComparison.OrdinalIgnoreCase) == true; + + actx.Host.Workspace.GetMeshNodeStream().Update(current => + { + var baseConfig = AsAgentConfiguration(current, actx.Host.Hub.JsonSerializerOptions) + ?? agent; + return current with + { + // Node-level metadata — the single source of truth. + Name = string.IsNullOrWhiteSpace(newDisplayName) ? current.Id : newDisplayName, + Description = string.IsNullOrWhiteSpace(newDescription) ? null : newDescription, + Icon = string.IsNullOrWhiteSpace(newIconName) ? null : newIconName, + Category = string.IsNullOrWhiteSpace(newGroupName) ? null : newGroupName, + Order = newOrder, + // Agent-specific behaviour. + Content = baseConfig with + { + ContextMatchPattern = string.IsNullOrWhiteSpace(newContextMatchPattern) ? null : newContextMatchPattern, + IsDefault = newIsDefault, + ExposedInNavigator = newExposedInNavigator, + Instructions = string.IsNullOrWhiteSpace(newInstructions) ? null : newInstructions + } + }; + }).Subscribe( + _ => + { + var overviewHref = new LayoutAreaReference(DetailsArea).ToHref(hubAddress); + actx.Host.UpdateArea(actx.Area, new RedirectControl(overviewHref)); + }, + ex => + { + actx.Host.Hub.ServiceProvider.GetService() + ?.CreateLogger(typeof(AgentView)) + .LogWarning(ex, "Agent edit save failed for {Path}", node.Path); + var dialog = Controls.Dialog( + Controls.Markdown($"**Error saving:**\n\n{ex.Message}"), + "Save Failed" + ).WithSize("M"); + actx.Host.UpdateArea(DialogControl.DialogArea, dialog); + }); + }); + return Task.CompletedTask; })); stack = stack.WithView(buttonRow); diff --git a/src/MeshWeaver.AI/AiSettings.cs b/src/MeshWeaver.AI/AiSettings.cs new file mode 100644 index 000000000..333c22867 --- /dev/null +++ b/src/MeshWeaver.AI/AiSettings.cs @@ -0,0 +1,36 @@ +using System.Collections.Immutable; + +namespace MeshWeaver.AI; + +/// +/// Per-user AI configuration — content of the singleton node at {user}/_Memex/AiSettings +/// (the default-settings namespace, ). It drives the chat composer: +/// which harnesses are offered, and the agent/model picker QUERY TEMPLATES. +/// +/// Empty array ⇒ "use code defaults". An empty means "all +/// registered (feature-flag-gated) harnesses"; empty / +/// mean the canonical queries. So a freshly-seeded or partially +/// populated node stays harmless, and the default queries can evolve in code with no data migration. +/// +public record AiSettings +{ + /// + /// Harness ids ( / / + /// ) the user has enabled. Empty ⇒ all registered harnesses + /// (which are already feature-flag-gated at DI registration). + /// + public ImmutableArray EnabledHarnesses { get; init; } = ImmutableArray.Empty; + + /// + /// Agent picker query TEMPLATES — carry {currentPath} / {nodeTypePath} tokens + /// substituted per composer instance (). Empty ⇒ + /// defaults. + /// + public ImmutableArray AgentQueries { get; init; } = ImmutableArray.Empty; + + /// + /// Model picker query TEMPLATES — carry {currentPath} / {nodeTypePath} / + /// {userPath} tokens. Empty ⇒ defaults. + /// + public ImmutableArray ModelQueries { get; init; } = ImmutableArray.Empty; +} diff --git a/src/MeshWeaver.AI/AiSettingsNodeType.cs b/src/MeshWeaver.AI/AiSettingsNodeType.cs new file mode 100644 index 000000000..568771e04 --- /dev/null +++ b/src/MeshWeaver.AI/AiSettingsNodeType.cs @@ -0,0 +1,241 @@ +using System.Collections.Immutable; +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI; + +/// +/// Per-user AiSettings singleton node — the user's AI configuration, stored at +/// {user}/_Memex/AiSettings (the default-settings _Memex namespace, non-satellite → +/// mesh_nodes). Single source of options for the chat composer: enabled harnesses + the +/// agent/model picker query templates. Edited from the "AI Settings" page. +/// +/// Robust by design: the node is (1) seeded empty at User onboarding +/// () AND (2) created-with-defaults on first read for any user that +/// predates the seed ( → ). Reads go through a query +/// (empty-on-absent), never a direct exact-path stream, to avoid the routing-NotFound resubscribe storm. +/// +public static class AiSettingsNodeType +{ + /// NodeType discriminator. + public const string NodeType = "AiSettings"; + + /// The default-settings namespace segment (_Memex, a non-satellite dotfile). + public const string UserNamespace = ThreadComposerNodeType.MemexDefaultsNamespace; // "_Memex" + + /// The singleton instance id. + public const string NodeId = "AiSettings"; + + /// The per-user settings path: {user}/_Memex/AiSettings. + public static string PathFor(string user) => $"{user}/{UserNamespace}/{NodeId}"; + + /// Registers the AiSettings node type, content type, and the per-user seed handler. + public static TBuilder AddAiSettingsType(this TBuilder builder) where TBuilder : MeshBuilder + { + builder.AddMeshNodes(CreateMeshNode()); + builder.AddAutocompleteExcludedTypes(NodeType); + builder.ConfigureHub(config => config.WithType(nameof(AiSettings))); + builder.ConfigureServices(services => + { + services.AddSingleton(_ => new AiSettingsSeedHandler()); + return services; + }); + return builder; + } + + /// MeshNode definition for nodeType:AiSettings. + public static MeshNode CreateMeshNode() => new(NodeType) + { + Name = "AI Settings", + Icon = "/static/NodeTypeIcons/sparkle.svg", + IsSatelliteType = false, + ExcludeFromContext = new HashSet { "search", "create" }, + HubConfiguration = config => config + .AddMeshDataSource(source => source + .WithContentType()) + }; + + /// + /// Sensible defaults: enabled harnesses = every registered (already + /// feature-flag-gated at registration), ordered by harness order; agent/model queries = the + /// canonical templates (tokenized context). + /// + public static AiSettings BuildDefaults(IServiceProvider services) + { + var harnesses = services.GetServices() + .OrderBy(h => h.Definition.Order) + .Select(h => h.Id) + .ToImmutableArray(); + + return new AiSettings + { + EnabledHarnesses = harnesses, + // Tokenized templates — BuildAgentQueries/BuildModelQueries are the single source of truth; + // we pass the placeholder tokens as their context args and resolve them per render. + // Agents: per-partition /Agent registry — user + current space + platform default. + AgentQueries = AgentPickerProjection + .BuildAgentQueries(UserPathToken, CurrentPathToken) + .ToImmutableArray(), + ModelQueries = AgentPickerProjection + .BuildModelQueries(CurrentPathToken, NodeTypePathToken, null, UserPathToken) + .ToImmutableArray(), + }; + } + + private const string CurrentPathToken = "{currentPath}"; + private const string NodeTypePathToken = "{nodeTypePath}"; + private const string UserPathToken = "{userPath}"; + + /// + /// Resolves query templates for a composer instance: substitutes {currentPath} / + /// {nodeTypePath} / {userPath}, and DROPS any template whose referenced token has + /// an empty value (mirroring how the builders only add those queries when the arg is non-empty). + /// + public static string[] ResolveQueries( + IEnumerable templates, string? currentPath, string? nodeTypePath, string? userPath) + { + var subs = new[] + { + (CurrentPathToken, currentPath), + (NodeTypePathToken, nodeTypePath), + (UserPathToken, userPath), + }; + + var result = new List(); + foreach (var template in templates) + { + var q = template; + var drop = false; + foreach (var (token, value) in subs) + { + if (!q.Contains(token, StringComparison.Ordinal)) continue; + if (string.IsNullOrEmpty(value)) { drop = true; break; } + q = q.Replace(token, value); + } + if (!drop) + result.Add(q); + } + return result.ToArray(); + } + + /// + /// Live per-user . Creates the node with if it + /// doesn't exist yet (idempotent, fire-and-forget) and reads it via a query (empty-on-absent). An + /// empty field falls back to the in-memory default so a seeded-empty or partial node behaves as + /// defaults. Emits the defaults immediately for the first paint, then the live node content. + /// + public static IObservable Observe( + IWorkspace workspace, IMessageHub hub, IServiceProvider services, string user) + { + var defaults = BuildDefaults(services); + EnsureExists(hub, services, user); + return workspace + .GetQuery($"{NodeType}|{user}", $"path:{PathFor(user)} nodeType:{NodeType}") + .Select(nodes => Effective( + nodes.FirstOrDefault(n => string.Equals(n.NodeType, NodeType, StringComparison.OrdinalIgnoreCase)), + defaults, hub.JsonSerializerOptions)) + .StartWith(defaults) + .DistinctUntilChanged(); + } + + /// Create-on-absent (with defaults); existing node untouched. + public static void EnsureExists(IMessageHub hub, IServiceProvider services, string user) + { + var meshService = services.GetService(); + if (meshService is null) + return; + var path = PathFor(user); + // 🚨 Create-on-absent must NEVER point-read/patch the node via + // GetMeshNodeStream(path).Update. On an ABSENT node that opens a cross-hub + // SubscribeRequest + JSON-merge patch to a node/hub that does not exist, which + // Orleans-NotFound-RESUBSCRIBE-STORMS (the rbuergi/_Memex/AiSettings + + // system-security/_Memex/AiSettings NotFound flood — fired on EVERY thread + // execution through Observe, it burned the action block and helped wedge the + // portal). Read existence via the SAME keyed GetQuery the Observe read uses + // (empty-on-absent, shared cached stream — no point-read, never storms), and seed + // only when genuinely absent through the node-lifecycle CreateNode (create-only: + // it does not clobber an existing customised node). See + // feedback_optional_node_query_not_access / Doc/Architecture/AsynchronousCalls.md. + hub.GetWorkspace() + .GetQuery($"{NodeType}|{user}", $"path:{path} nodeType:{NodeType}") + .Take(1) + .Where(nodes => !nodes.Any(n => + string.Equals(n.NodeType, NodeType, StringComparison.OrdinalIgnoreCase))) + .SelectMany(_ => meshService.CreateNode( + MeshNode.FromPath(path) with + { + NodeType = NodeType, + Name = "AI Settings", + Content = BuildDefaults(services) + })) + .Subscribe( + _ => { }, + ex => services.GetService() + ?.CreateLogger(typeof(AiSettingsNodeType)) + .LogWarning(ex, "EnsureExists: AiSettings create-on-absent failed for {Path}", path)); + } + + /// + /// The effective settings for a node: the saved with each EMPTY field + /// filled from (an empty/absent node behaves as the code defaults). + /// + public static AiSettings Effective(MeshNode? node, AiSettings defaults, JsonSerializerOptions options) + { + var settings = node?.Content switch + { + AiSettings s => s, + JsonElement je => TryDeserialize(je, options), + _ => null, + }; + if (settings is null) + return defaults; + return settings with + { + EnabledHarnesses = settings.EnabledHarnesses.IsDefaultOrEmpty ? defaults.EnabledHarnesses : settings.EnabledHarnesses, + AgentQueries = settings.AgentQueries.IsDefaultOrEmpty ? defaults.AgentQueries : settings.AgentQueries, + ModelQueries = settings.ModelQueries.IsDefaultOrEmpty ? defaults.ModelQueries : settings.ModelQueries, + }; + } + + private static AiSettings? TryDeserialize(JsonElement je, JsonSerializerOptions options) + { + try { return JsonSerializer.Deserialize(je.GetRawText(), options); } + catch { return null; } + } + + /// + /// Seeds an EMPTY at {user}/_Memex/AiSettings on User onboarding — + /// DI-free (defaults are resolved lazily by / the settings page). Mirrors + /// ModelProviderSelectionSeedHandler; keeps the composer's read from ever hitting a routing + /// NotFound for newly-onboarded users. + /// + private sealed class AiSettingsSeedHandler : INodePostCreationHandler + { + public string NodeType => UserNodeType.NodeType; // "User" + + public IObservable Handle(MeshNode createdNode, string? createdBy) + => System.Reactive.Linq.Observable.Empty(); + + public IEnumerable GetAdditionalNodes(MeshNode createdNode) + { + var userPath = !string.IsNullOrEmpty(createdNode.Path) ? createdNode.Path : createdNode.Id; + if (string.IsNullOrEmpty(userPath)) + yield break; + + yield return new MeshNode(NodeId, $"{userPath}/{UserNamespace}") + { + NodeType = AiSettingsNodeType.NodeType, + Name = "AI Settings", + Content = new AiSettings(), + }; + } + } +} diff --git a/src/MeshWeaver.AI/AppendUserMessageRequest.cs b/src/MeshWeaver.AI/AppendUserMessageRequest.cs deleted file mode 100644 index 99d80f495..000000000 --- a/src/MeshWeaver.AI/AppendUserMessageRequest.cs +++ /dev/null @@ -1,61 +0,0 @@ -using MeshWeaver.Mesh.Security; -using MeshWeaver.Messaging; -using MeshWeaver.Messaging.Security; - -namespace MeshWeaver.AI; - -/// -/// Registers a user message id with the thread hub. Client posts this right after -/// (or in parallel with) the CreateNodeRequest for the user cell. The thread hub -/// appends the id to Thread.Messages and Thread.UserMessageIds, stores -/// Pending* settings for the next round, and the server watcher dispatches. -/// -/// Going through a first-class message (not a remote-stream write) avoids the -/// patch-index race condition seen with workspace.UpdateMeshNode(address:). -/// -[SubmitMessagePermission] -public record AppendUserMessageRequest : IRequest -{ - public required string ThreadPath { get; init; } - public required string UserMessageId { get; init; } - public required string UserText { get; init; } - public string? AgentName { get; init; } - public string? ModelName { get; init; } - public string? ContextPath { get; init; } - public IReadOnlyList? Attachments { get; init; } -} - -public record AppendUserMessageResponse -{ - public bool Success { get; init; } - public string? Error { get; init; } -} - -/// -/// Request to resubmit — truncates the thread after the user message id and -/// re-adds it to the queue so the server watcher dispatches a new round. -/// -[SubmitMessagePermission] -public record ResubmitUserMessageRequest : IRequest -{ - public required string ThreadPath { get; init; } - public required string UserMessageId { get; init; } - public string? NewUserText { get; init; } - public string? AgentName { get; init; } - public string? ModelName { get; init; } -} - -/// -/// Request fired when the client's submit pipeline hits an error (cell creation failed, -/// append failed, etc.). The thread hub creates a failed-response output cell that shows -/// the error inline in the chat, links the user message to the thread if not already linked, -/// and marks it as ingested so the watcher doesn't try to dispatch it. -/// -[SubmitMessagePermission] -public record RecordSubmissionFailureRequest : IRequest -{ - public required string ThreadPath { get; init; } - public required string UserMessageId { get; init; } - public required string UserText { get; init; } - public required string ErrorMessage { get; init; } -} diff --git a/src/MeshWeaver.AI/Attributes/ToolTimeoutAttribute.cs b/src/MeshWeaver.AI/Attributes/ToolTimeoutAttribute.cs new file mode 100644 index 000000000..00f2ade56 --- /dev/null +++ b/src/MeshWeaver.AI/Attributes/ToolTimeoutAttribute.cs @@ -0,0 +1,26 @@ +namespace MeshWeaver.AI.Attributes; + +/// +/// Per-tool execution timeout. Applied to a method exposed as an AI tool +/// (via AIFunctionFactory.Create). Read at wrap time by +/// AccessContextAIFunction and enforced via a linked +/// ; on expiry the tool invocation +/// is cancelled and the agent receives a synthetic "timed out" string +/// instead of a hung promise. +/// +/// The framework default is 30 seconds — long enough for any +/// reasonable tool (mesh read, web search, single-shot LLM completion) +/// to complete, short enough that a hang surfaces fast in the chat UI. +/// Override on tools that legitimately take longer (long-running script +/// execution, large export, compile loop). +/// +/// Does NOT apply to delegate_to_agent — that's a thread- +/// execution primitive with its own heartbeat-based hang detection +/// (see MeshThread.LastActivityAt), not a tool in the +/// timeout-attribute sense. +/// +[AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = false)] +public sealed class ToolTimeoutAttribute(int seconds) : Attribute +{ + public TimeSpan Timeout { get; } = TimeSpan.FromSeconds(seconds); +} diff --git a/src/MeshWeaver.AI/AuthRequiredException.cs b/src/MeshWeaver.AI/AuthRequiredException.cs new file mode 100644 index 000000000..e9439efd0 --- /dev/null +++ b/src/MeshWeaver.AI/AuthRequiredException.cs @@ -0,0 +1,26 @@ +namespace MeshWeaver.AI; + +/// +/// Thrown by a co-hosted CLI chat client when the user's provider session needs (re)authentication +/// — deliberately distinct from transient/informational events such as the Claude Code SDK's +/// rate_limit_event (which is swallowed, not escalated). ThreadExecution's error +/// branch turns this into a response-cell Error whose markdown points the user at the harness's +/// own /login command (the chat surfaces the inline Connect flow), instead of a cryptic CLI +/// "exit code 1". +/// +public sealed class AuthRequiredException : Exception +{ + /// Provider key the user must (re)connect, e.g. "ClaudeCode" or "Copilot". + public string Provider { get; } + + public AuthRequiredException(string provider, string message, Exception? innerException = null) + : base(message, innerException) + => Provider = provider; + + /// + /// Markdown rendered on the response cell when this escalates (Status = Error). Points the user at + /// the harness's /login command — typing it surfaces the inline Connect (login) flow. + /// + public string ToMarkdown() => + $"**Not logged in.** Type `/login` to connect your {Provider} subscription, then send your message again."; +} diff --git a/src/MeshWeaver.AI/BuiltInAgentProvider.cs b/src/MeshWeaver.AI/BuiltInAgentProvider.cs index 6603216fb..b4563d265 100644 --- a/src/MeshWeaver.AI/BuiltInAgentProvider.cs +++ b/src/MeshWeaver.AI/BuiltInAgentProvider.cs @@ -39,13 +39,19 @@ public class BuiltInAgentProvider : IStaticNodeProvider public IEnumerable GetStaticNodes() { - // Read-only policy for the Agent namespace — built-in agents are unmodifiable + // Read-only, world-readable policy for the Agent namespace. PublicRead grants + // Read to every user (the agent catalog is a public catalog) WITHOUT needing a + // per-user role at the Agent scope — and because this is a static provider node, + // it is present from the first permission evaluation (no synced-query cold-start + // race, which previously left the agent picker empty → "No suitable agent"). + // The write caps keep the built-in agents unmodifiable. yield return new MeshNode("_Policy", RootNamespace) { NodeType = "PartitionAccessPolicy", Name = "Access Policy", Content = new PartitionAccessPolicy { + PublicRead = true, Create = false, Update = false, Delete = false, @@ -84,13 +90,15 @@ private static MeshNode CreateThreadNamerNode() - "Fix the login bug" -> Name: Fix Login Bug / Id: FixLoginBug """, ExposedInNavigator = false, - Order = 999, + ModelTier = "utility", }; + // Order lives on the node (sorts last in the picker); not duplicated on the config. return new MeshNode(ThreadNamerId, "Agent") { Name = "Thread Namer", NodeType = "Agent", + Order = 999, Content = config }; } @@ -99,9 +107,12 @@ private static IEnumerable LoadEmbeddedNodes() { var assembly = typeof(BuiltInAgentProvider).Assembly; var prefix = $"{assembly.GetName().Name}.Data."; + // Data/Skill/*.md are nodeType:Skill nodes served by BuiltInSkillProvider — not agents/docs. + var skillPrefix = $"{prefix}{SkillNodeType.RootNamespace}."; foreach (var resourceName in assembly.GetManifestResourceNames() - .Where(n => n.StartsWith(prefix) && n.EndsWith(".md", StringComparison.OrdinalIgnoreCase)) + .Where(n => n.StartsWith(prefix) && n.EndsWith(".md", StringComparison.OrdinalIgnoreCase) + && !n.StartsWith(skillPrefix, StringComparison.Ordinal)) .Order()) { using var stream = assembly.GetManifestResourceStream(resourceName); @@ -151,12 +162,9 @@ private static MeshNode ParseAgentNode(string yamlContent, string markdownBody, var agentConfig = new AgentConfiguration { Id = id, - DisplayName = frontMatter.Name ?? frontMatter.DisplayName ?? id, Description = frontMatter.Description, Instructions = string.IsNullOrWhiteSpace(markdownBody) ? null : markdownBody.Trim(), - Icon = frontMatter.Icon, CustomIconSvg = frontMatter.CustomIconSvg, - GroupName = frontMatter.GroupName, IsDefault = frontMatter.IsDefault, ExposedInNavigator = frontMatter.ExposedInNavigator, Delegations = frontMatter.Delegations?.Select(d => new AgentDelegation @@ -169,19 +177,23 @@ private static MeshNode ParseAgentNode(string yamlContent, string markdownBody, AgentPath = h.AgentPath ?? "", Instructions = h.Instructions }).ToList(), - Plugins = frontMatter.Plugins?.Select(ParsePluginReference).ToList(), - PreferredModel = frontMatter.PreferredModel, - ModelTier = frontMatter.ModelTier, + Plugins = frontMatter.Plugins?.Select(AgentPluginReference.Parse).ToList(), ContextMatchPattern = frontMatter.ContextMatchPattern, - Order = frontMatter.Order + ModelTier = frontMatter.ModelTier }; + // Node-level metadata (name, description, icon, group, order) lives on the + // MeshNode — NOT duplicated on the AgentConfiguration content. The picker groups + // by Category, so the harness group (frontmatter groupName, default MeshWeaver) + // maps onto the node's Category. return new MeshNode(id, ns) { NodeType = "Agent", Name = frontMatter.Name ?? frontMatter.DisplayName ?? id, - Category = frontMatter.Category ?? "Agents", + Description = frontMatter.Description, + Category = frontMatter.GroupName ?? frontMatter.Category ?? Harnesses.MeshWeaver, Icon = frontMatter.Icon ?? "Bot", + Order = frontMatter.Order, Content = agentConfig }; } @@ -234,22 +246,6 @@ private static (string Id, string? Namespace) DeriveIdAndNamespace(string relati return (id, ns); } - /// - /// Parses "PluginName" or "PluginName:Method1,Method2" into AgentPluginReference. - /// - private static AgentPluginReference ParsePluginReference(string s) - { - var colonIndex = s.IndexOf(':'); - if (colonIndex < 0) - return new AgentPluginReference { Name = s.Trim() }; - - return new AgentPluginReference - { - Name = s[..colonIndex].Trim(), - Methods = s[(colonIndex + 1)..].Split(',').Select(m => m.Trim()).ToList() - }; - } - // Peek class to check nodeType without full deserialization private class NodeTypePeek { @@ -268,10 +264,9 @@ private class AgentFrontMatter public bool IsDefault { get; set; } public bool ExposedInNavigator { get; set; } public string? ContextMatchPattern { get; set; } - public string? PreferredModel { get; set; } - public string? ModelTier { get; set; } public int Order { get; set; } public string? CustomIconSvg { get; set; } + public string? ModelTier { get; set; } public List? Delegations { get; set; } public List? Handoffs { get; set; } public List? Plugins { get; set; } diff --git a/src/MeshWeaver.AI/BuiltInHarnessProvider.cs b/src/MeshWeaver.AI/BuiltInHarnessProvider.cs new file mode 100644 index 000000000..6072048a1 --- /dev/null +++ b/src/MeshWeaver.AI/BuiltInHarnessProvider.cs @@ -0,0 +1,46 @@ +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Mesh.Services; + +namespace MeshWeaver.AI; + +/// +/// Projects every registered (from this assembly and the +/// Claude Code / Copilot assemblies) into a read-only catalog node under the +/// Harness partition, plus a public-read access policy. This is the single +/// source of truth the harness picker binds to — add a harness DLL and its node +/// appears automatically. +/// +public sealed class BuiltInHarnessProvider(IEnumerable harnesses) : IStaticNodeProvider +{ + public IEnumerable GetStaticNodes() + { + // World-readable catalog, unmodifiable — same shape as the agent catalog. + yield return new MeshNode("_Policy", HarnessNodeType.RootNamespace) + { + NodeType = "PartitionAccessPolicy", + Name = "Access Policy", + Content = new PartitionAccessPolicy + { + PublicRead = true, + Create = false, + Update = false, + Delete = false, + Comment = false, + Thread = false + } + }; + + foreach (var harness in harnesses.OrderBy(h => h.Definition.Order)) + { + var def = harness.Definition; + yield return new MeshNode(def.Id, HarnessNodeType.RootNamespace) + { + NodeType = HarnessNodeType.NodeType, + Name = def.DisplayName ?? def.Id, + Icon = def.Icon, + Content = def + }; + } + } +} diff --git a/src/MeshWeaver.AI/BuiltInLanguageModelProvider.cs b/src/MeshWeaver.AI/BuiltInLanguageModelProvider.cs new file mode 100644 index 000000000..c324025da --- /dev/null +++ b/src/MeshWeaver.AI/BuiltInLanguageModelProvider.cs @@ -0,0 +1,330 @@ +using System.Collections.Immutable; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Mesh.Services; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI; + +/// +/// Surfaces every platform-shipped model as a static +/// nodeType:LanguageModel MeshNode under the +/// . +/// +/// Reads — a plain +/// singleton populated by +/// . +/// Each entry pairs a config section (where the deployed +/// reads its config from) with the +/// provider label that should appear on the resulting Model node. The +/// section's Models[] array becomes the model id list. +/// +/// 🚨 Catalog options are a plain singleton, NOT +/// : the +/// Configure<T> pipeline didn't survive to the mesh hub's DI +/// scope (live namespace:Model queries returned only the access +/// policy because Sources was empty at provider-resolve time). +/// Direct singleton + helper that idempotently appends sidesteps the +/// scope mismatch. +/// +/// Net effect: a query like namespace:Model nodeType:LanguageModel +/// always returns the deployed catalog without depending on +/// registrations (which live on the web +/// host, not the mesh hub). +/// +public class BuiltInLanguageModelProvider : IStaticNodeProvider +{ + private readonly IConfiguration configuration; + private readonly LanguageModelCatalogOptions options; + private readonly ILogger? logger; + + public BuiltInLanguageModelProvider( + IConfiguration configuration, + LanguageModelCatalogOptions options, + ILoggerFactory? loggerFactory = null) + { + this.configuration = configuration; + this.options = options; + this.logger = loggerFactory?.CreateLogger(); + } + + public IEnumerable GetStaticNodes() + { + // Stable de-dup: first registered source wins on Id collision — + // matches the order callers register them via + // AddLanguageModelCatalogSource. + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var emitted = new List(); + + foreach (var source in options.Sources) + { + string[]? models; + try + { + models = configuration + .GetSection($"{source.SectionName}:Models") + .Get(); + } + catch (Exception ex) + { + logger?.LogWarning(ex, + "BuiltInLanguageModelProvider: failed to read config section '{Section}:Models' — skipping", + source.SectionName); + continue; + } + + // Read driver config (Endpoint + ApiKey) from the same section + // the legacy IOptions<...Configuration> binding read from. Stamping + // these on the ModelDefinition makes the model MeshNode the source + // of truth for driver config — the factory reads them off the + // selected model instead of an out-of-band IOptions binding, and + // user-authored Model nodes can override the built-in defaults. + string? endpoint = null; + string? apiKey = null; + try + { + endpoint = configuration[$"{source.SectionName}:Endpoint"]; + apiKey = configuration[$"{source.SectionName}:ApiKey"]; + } + catch { /* malformed section — skip stamping */ } + + // Parse IConfiguration into the canonical ModelProvider mesh node: + // every credential the system ships with becomes a node in the + // root catalog. ModelProvider is NOT WithPublicRead (no + // ConfigureNodeTypeAccess call) so only callers with + // Permission.Api on the root namespace see the ApiKey; ordinary + // user-context reads in their own partition see only their own + // (user-authored) ModelProvider rows. Publicly-visible LanguageModel + // siblings still carry NO key — that protection is intact. + var hasAnySignal = (models != null && models.Length > 0) + || !string.IsNullOrEmpty(endpoint) + || !string.IsNullOrEmpty(apiKey); + if (hasAnySignal) + { + var providerConfig = new ModelProviderConfiguration + { + Provider = source.ProviderName, + ApiKey = apiKey, + Endpoint = endpoint, + Label = source.ProviderName, + CreatedAt = DateTimeOffset.UtcNow, + Models = models is { Length: > 0 } + ? models.Where(m => !string.IsNullOrWhiteSpace(m)).ToImmutableArray() + : ImmutableArray.Empty + }; + emitted.Add(new MeshNode(source.ProviderName, ModelProviderNodeType.RootNamespace) + { + NodeType = ModelProviderNodeType.NodeType, + Name = source.ProviderName, + Category = "Providers", + Icon = "Key", + Content = providerConfig + }); + } + + if (models == null || models.Length == 0) + { + logger?.LogDebug( + "BuiltInLanguageModelProvider: '{Section}:Models' empty — provider {Provider} contributes nothing", + source.SectionName, source.ProviderName); + continue; + } + + // 🚦 Only surface a provider's models when it's actually CONFIGURED. + // Api providers (RequiresApiKey) need BOTH an Endpoint and an ApiKey in + // config; keyless/CLI providers (RequiresApiKey=false — Claude Code, + // Copilot) need neither. An unconfigured Api provider's Models[] is just a + // default catalog the deployment never wired up (e.g. an "Azure" section + // listing Claude ids with no Endpoint/ApiKey) — surfacing those puts + // selectable-but-unusable entries in the /model picker (the reported + // "Azure Claude shows even though nothing is configured" bug). The + // ModelProvider node above is STILL emitted so Settings → Models can render + // its configure form; only the selectable LanguageModel catalog entries are + // gated on having credentials. + var isConfigured = !source.RequiresApiKey + || (!string.IsNullOrEmpty(endpoint) && !string.IsNullOrEmpty(apiKey)); + if (!isConfigured) + { + logger?.LogDebug( + "BuiltInLanguageModelProvider: provider {Provider} not configured (Endpoint/ApiKey unset) — hiding its {Count} model(s) from the catalog until configured", + source.ProviderName, models.Length); + continue; + } + + foreach (var modelId in models) + { + if (string.IsNullOrWhiteSpace(modelId)) continue; + if (!seen.Add(modelId)) continue; + + var def = new ModelDefinition + { + Id = modelId, + DisplayName = modelId, + Provider = source.ProviderName, + Endpoint = endpoint, + // ApiKey NEVER on a LanguageModel node — these are + // WithPublicRead. The factory's IOptions fallback supplies + // the system-default key for static catalog entries. + ApiKeySecretRef = null, + // Reference the static ModelProvider node emitted above. + // Resolver follows this pointer; user-partition ModelProvider + // nodes override via their own ProviderRef on child + // LanguageModel nodes. + ProviderRef = hasAnySignal + ? $"{ModelProviderNodeType.RootNamespace}/{source.ProviderName}" + : null, + Order = source.Order, + // Seed the published default price (USD per 1M tokens) for known + // model ids so the token-cost summaries show a real cost out of + // the box; users can override per model in the Models settings. + InputPricePerMillionTokens = ModelPricing.Default(modelId)?.InputPerMillion, + OutputPricePerMillionTokens = ModelPricing.Default(modelId)?.OutputPerMillion, + Currency = ModelPricing.Default(modelId)?.Currency + }; + + // Static LanguageModel nodes live UNDER their provider's + // satellite path: _Provider/{providerName}/{modelId}. Matches + // the user-partition layout ({userPath}/_Provider/{providerName}/{modelId}) + // so the picker can use ONE namespace per query path — + // the documented shape for synced-collection multi-query + // (varying scope/path, same nodeType filter). + var modelNamespace = $"{ModelProviderNodeType.RootNamespace}/{source.ProviderName}"; + emitted.Add(new MeshNode(modelId, modelNamespace) + { + NodeType = LanguageModelNodeType.NodeType, + Name = modelId, + Category = "Models", + Icon = "Sparkle", + Content = def + }); + } + } + + // Only seed the read-only access policy if we actually have models — + // an empty namespace with just a policy node is "crap" (user's + // word) that pollutes namespace:Model queries with nothing useful. + if (emitted.Count > 0) + { + // 🚨 Policy MUST be on ModelProviderNodeType.RootNamespace ("_Provider") — the partition the + // models actually live in (the modelNamespace above) AND the one the /model picker queries + // (BuiltInCommandProvider: "namespace:_Provider nodeType:LanguageModel scope:descendants"). + // It was previously seeded on LanguageModelNodeType.RootNamespace ("Model") — a DIFFERENT, + // model-less partition — so the "_Provider" catalog had NO access policy at all and was + // unreadable under a real user's identity. + yield return new MeshNode("_Policy", ModelProviderNodeType.RootNamespace) + { + NodeType = "PartitionAccessPolicy", + Name = "Access Policy", + Content = new PartitionAccessPolicy + { + // 🚨 World-readable, exactly like the Agent + Harness catalogs + // (BuiltInAgentProvider / BuiltInHarnessProvider both set PublicRead=true). + // The /model picker queries `namespace:_Provider nodeType:LanguageModel` UNDER + // the user's identity; without PublicRead the partition isn't readable, so RLS + // filters out every model → empty picker even though the catalog is synced + Active. + // This grant was MISSING here (the Agent catalog had it, which is why agents showed + // and models didn't). Read-only: the model nodes carry NO ApiKey (it's gated + // separately on ModelProvider via Permission.Api), so public READ of the catalog is safe. + PublicRead = true, + Create = false, + Update = false, + Delete = false, + Comment = false, + Thread = false + } + }; + } + + logger?.LogDebug( + "BuiltInLanguageModelProvider: emitted {Count} Model nodes from {Sources} catalog source(s)", + emitted.Count, options.Sources.Count); + + foreach (var node in emitted) + yield return node; + } +} + +/// +/// Catalog of sections that +/// scans for models. Plain +/// singleton — populated by +/// . +/// +public class LanguageModelCatalogOptions +{ + /// Registered catalog sources, populated at mesh init time. + public List Sources { get; } = new(); + + /// + /// Idempotently appends a source — does nothing if (sectionName, providerName) + /// is already present. Called from + /// ; + /// safe to call multiple times across multiple + /// builder.ConfigureServices blocks. + /// + public void Add(LanguageModelCatalogSource source) + { + if (Sources.Any(s => + s.SectionName == source.SectionName && + s.ProviderName == source.ProviderName)) + return; + Sources.Add(source); + } +} + +/// +/// One catalog source: a config section to scan for Models[], the +/// provider label to stamp on each resulting Model node, and the +/// provider's bootstrap profile — default endpoint, default model ids +/// (used when a user pastes a key in the Models settings tab and +/// ModelProviderService auto-creates the LanguageModel children), +/// and whether the provider requires an API key at all (false for keyless +/// providers like GitHub Copilot or the local Claude Code CLI). +/// +/// Each provider package registers its own source via +/// +/// — there is NO central registry. Consumers (Models settings tab, +/// ModelProviderService.CreateProvider) enumerate the live +/// . +/// +public record LanguageModelCatalogSource( + string SectionName, + string ProviderName, + int Order = 0, + string? DisplayLabel = null, + string? DefaultEndpoint = null, + ImmutableArray DefaultModelIds = default, + bool RequiresApiKey = true, + ProviderKind Kind = ProviderKind.Api) +{ + /// Effective display label — falls back to when not supplied. + public string EffectiveLabel => DisplayLabel ?? ProviderName; + + /// Defensive default for — record syntax can leave it uninitialised. + public ImmutableArray EffectiveModelIds => + DefaultModelIds.IsDefault ? ImmutableArray.Empty : DefaultModelIds; +} + +/// +/// How a provider authenticates and what the Settings → Models card renders for it. +/// +/// +/// — bring-your-own-key (Azure AI Foundry, Azure OpenAI, +/// Anthropic, OpenAI). The card shows an endpoint/key form plus a fetched, +/// checkable list of models to enable. +/// — co-hosted, subscription-based CLI (Claude Code, +/// GitHub Copilot). No model list; the card shows a login status dot plus a +/// Connect button that delegates to the CLI's native login (paste-code for +/// Claude, device-flow for Copilot). +/// +/// +public enum ProviderKind +{ + /// Bring-your-own-key provider — endpoint/key form + model list. + Api, + + /// Co-hosted CLI provider — login status + Connect, no model list. + Cli, +} diff --git a/src/MeshWeaver.AI/BuiltInSkillProvider.cs b/src/MeshWeaver.AI/BuiltInSkillProvider.cs new file mode 100644 index 000000000..fcf612a89 --- /dev/null +++ b/src/MeshWeaver.AI/BuiltInSkillProvider.cs @@ -0,0 +1,154 @@ +using Markdig; +using Markdig.Extensions.Yaml; +using Markdig.Syntax; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Mesh.Services; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace MeshWeaver.AI; + +/// +/// Provides the built-in skill nodes from embedded Data/Skill/*.md resources — the SAME +/// .md-with-YAML authoring model as agents (). Each skill is a +/// nodeType: Skill markdown file: behaviour skills carry an action: block in the +/// frontmatter (Pick / OpenContent / Connect), instruction skills carry +/// their how-to in the markdown body. The slash word is the file name (agent.md → /agent). +/// Discovered together with per-space / per-user skills via . +/// +public class BuiltInSkillProvider : IStaticNodeProvider +{ + private static readonly Lazy LazyNodes = new(LoadEmbeddedNodes); + + private static readonly MarkdownPipeline Pipeline = new MarkdownPipelineBuilder() + .UseYamlFrontMatter() + .Build(); + + private static readonly IDeserializer Yaml = new DeserializerBuilder() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + /// + public IEnumerable GetStaticNodes() + { + // Read-only, world-readable policy for the Skill namespace — the skill catalog is public, same + // as Agent/Harness. On the SYNCED path this _Policy MUST be imported (SkillStaticRepoSource), + // else the partition has no read policy and the skills are unreadable → the chat finds no skills + // (the Harness wedge, atioz 2026-06-15). The write caps keep the built-in skills unmodifiable. + yield return new MeshNode("_Policy", SkillNodeType.RootNamespace) + { + NodeType = "PartitionAccessPolicy", + Name = "Access Policy", + Content = new PartitionAccessPolicy + { + PublicRead = true, + Create = false, + Update = false, + Delete = false, + Comment = false, + Thread = false, + } + }; + + foreach (var node in LazyNodes.Value) + yield return node; + } + + private static MeshNode[] LoadEmbeddedNodes() + { + var assembly = typeof(BuiltInSkillProvider).Assembly; + // Resource names dot-separate path segments: Data/Skill/agent.md → {asm}.Data.Skill.agent.md + var skillPrefix = $"{assembly.GetName().Name}.Data.{SkillNodeType.RootNamespace}."; + + var nodes = new List(); + foreach (var resourceName in assembly.GetManifestResourceNames() + .Where(n => n.StartsWith(skillPrefix, StringComparison.Ordinal) + && n.EndsWith(".md", StringComparison.OrdinalIgnoreCase)) + .Order()) + { + using var stream = assembly.GetManifestResourceStream(resourceName); + if (stream == null) continue; + using var reader = new StreamReader(stream); + var node = ParseSkillNode(reader.ReadToEnd(), ResourceNameToId(resourceName, skillPrefix)); + if (node != null) nodes.Add(node); + } + return nodes.ToArray(); + } + + private static MeshNode? ParseSkillNode(string content, string id) + { + if (string.IsNullOrEmpty(id)) return null; + + var document = Markdig.Markdown.Parse(content, Pipeline); + var yamlBlock = document.Descendants().FirstOrDefault(); + if (yamlBlock == null) return null; + + var fm = Yaml.Deserialize(yamlBlock.Lines.ToString()); + if (fm == null) return null; + + var body = content[(yamlBlock.Span.End + 1)..].TrimStart('\r', '\n').Trim(); + + var definition = new SkillDefinition + { + Instructions = string.IsNullOrWhiteSpace(body) ? null : body, + AutoMount = fm.AutoMount, + LaunchesSubThread = fm.LaunchesSubThread, + Harness = fm.Harness, + Action = fm.Action is null ? null : new SkillAction + { + Kind = Enum.TryParse(fm.Action.Kind, ignoreCase: true, out var kind) + ? kind : SkillActionKind.Pick, + Query = fm.Action.Query, + Field = fm.Action.Field, + Title = fm.Action.Title, + ContentPath = fm.Action.ContentPath, + Provider = fm.Action.Provider, + }, + }; + + return new MeshNode(id, SkillNodeType.RootNamespace) + { + NodeType = SkillNodeType.NodeType, + Name = fm.Name ?? $"/{id}", + Description = fm.Description, + Category = fm.Category ?? "Skills", + Icon = fm.Icon ?? "Sparkle", + Order = fm.Order, + State = MeshNodeState.Active, + Content = definition, + }; + } + + private static string ResourceNameToId(string resourceName, string skillPrefix) + { + var rest = resourceName[skillPrefix.Length..]; // e.g. "agent.md" + var lastDot = rest.LastIndexOf('.'); // strip the ".md" extension + return lastDot > 0 ? rest[..lastDot] : rest; + } + + private sealed class SkillFrontMatter + { + public string? NodeType { get; set; } + public string? Name { get; set; } + public string? Description { get; set; } + public string? Icon { get; set; } + public string? Category { get; set; } + public int Order { get; set; } + public bool AutoMount { get; set; } = true; + public bool LaunchesSubThread { get; set; } + public string? Harness { get; set; } + public SkillActionFrontMatter? Action { get; set; } + } + + private sealed class SkillActionFrontMatter + { + public string? Kind { get; set; } + public string? Query { get; set; } + public string? Field { get; set; } + public string? Title { get; set; } + public string? ContentPath { get; set; } + public string? Provider { get; set; } + } +} diff --git a/src/MeshWeaver.AI/CancelThreadStreamRequest.cs b/src/MeshWeaver.AI/CancelThreadStreamRequest.cs deleted file mode 100644 index 22ae0ecea..000000000 --- a/src/MeshWeaver.AI/CancelThreadStreamRequest.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace MeshWeaver.AI; - -/// -/// Request to cancel the active streaming response for a thread. -/// Propagates bottom-up: sub-threads cancel first, then parent. -/// -public record CancelThreadStreamRequest -{ - public required string ThreadPath { get; init; } -} - -/// -/// Response confirming thread cancellation is complete. -/// Sent after all sub-threads have confirmed and own execution is stopped. -/// -public record CancelThreadStreamResponse -{ - public required string ThreadPath { get; init; } -} diff --git a/src/MeshWeaver.AI/ChatClientAgentFactory.cs b/src/MeshWeaver.AI/ChatClientAgentFactory.cs index bae7758c7..70e1a9f74 100644 --- a/src/MeshWeaver.AI/ChatClientAgentFactory.cs +++ b/src/MeshWeaver.AI/ChatClientAgentFactory.cs @@ -1,6 +1,10 @@ using System.Collections.Immutable; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using System.Reflection; using System.Runtime.CompilerServices; using System.Text; +using MeshWeaver.AI.Attributes; using MeshWeaver.AI.Plugins; using MeshWeaver.Data; using MeshWeaver.Layout; @@ -53,6 +57,50 @@ protected ChatClientAgentFactory(IMessageHub hub) ///     public abstract int Order { get; } + /// + /// + /// Default implementation delegates to for backward compatibility. + /// Concrete factories with shape-aware routing (e.g. "claude-*" prefix) should override. + /// + public virtual bool Supports(string modelName) => + !string.IsNullOrEmpty(modelName) && Models.Any(m => + string.Equals(m, modelName, StringComparison.OrdinalIgnoreCase)); + + /// + /// Resolves the agent's ("heavy" / "standard" / + /// "light" / "utility") to a concrete model via the ModelTier:* config section. + /// Returns null when the agent declares no tier, the tier isn't configured, or this + /// factory doesn't serve the resolved model (so the caller falls back to its provider + /// default instead of creating a client for a model another factory owns). + /// Precedence in concrete factories: composer selection () + /// → agent tier → provider default. + /// + protected string? ResolveTierModel(AgentConfiguration agentConfig) + { + if (string.IsNullOrEmpty(agentConfig.ModelTier)) + return null; + + var configuration = Hub.ServiceProvider.GetService(); + if (configuration == null) + return null; + + var tiers = new ModelTierConfiguration + { + Heavy = configuration["ModelTier:Heavy"], + Standard = configuration["ModelTier:Standard"], + Light = configuration["ModelTier:Light"], + Utility = configuration["ModelTier:Utility"] + }; + + var resolved = tiers.Resolve(agentConfig.ModelTier); + if (string.IsNullOrEmpty(resolved) || !Supports(resolved)) + return null; + + Logger.LogDebug("[AgentFactory] Agent {Agent} tier '{Tier}' resolved to model {Model}", + agentConfig.Id, agentConfig.ModelTier, resolved); + return resolved; + } + /// /// Creates a ChatClientAgent for the given configuration. /// @@ -70,13 +118,10 @@ public ChatClientAgent CreateAgent( { CurrentModelName = modelName; - if (string.IsNullOrEmpty(config.PreferredModel) && !string.IsNullOrEmpty(config.ModelTier)) - { - var tierConfig = Hub.ServiceProvider.GetService>()?.Value; - var resolvedModel = tierConfig?.Resolve(config.ModelTier); - if (!string.IsNullOrEmpty(resolvedModel)) - config = config with { PreferredModel = resolvedModel }; - } + // The composer selection (ThreadComposer.ModelName → modelName / CurrentModelName) always + // wins. When nothing was selected (headless flows: email routing, notification triage, + // delegated sub-threads), concrete factories fall back to the agent's ModelTier via + // ResolveTierModel, then to their provider default. // Sync: use raw instructions, skip @@reference resolution (resolved lazily) var instructions = GetAgentInstructions(config, hierarchyAgents, chat); @@ -93,13 +138,10 @@ public async Task CreateAgentAsync( { CurrentModelName = modelName; - if (string.IsNullOrEmpty(config.PreferredModel) && !string.IsNullOrEmpty(config.ModelTier)) - { - var tierConfig = Hub.ServiceProvider.GetService>()?.Value; - var resolvedModel = tierConfig?.Resolve(config.ModelTier); - if (!string.IsNullOrEmpty(resolvedModel)) - config = config with { PreferredModel = resolvedModel }; - } + // The composer selection (ThreadComposer.ModelName → modelName / CurrentModelName) always + // wins. When nothing was selected (headless flows: email routing, notification triage, + // delegated sub-threads), concrete factories fall back to the agent's ModelTier via + // ResolveTierModel, then to their provider default. var instructions = await GetAgentInstructionsAsync(config, hierarchyAgents, chat); return CreateAgentCore(config, chat, existingAgents, hierarchyAgents, instructions, modelName); @@ -131,14 +173,20 @@ private ChatClientAgent CreateAgentCore( } else { + // No plugins declared → READ-ONLY Mesh tools. Write capability is an explicit + // grant: declare `plugins: [Mesh]` in the agent definition to get + // Create/Update/Patch/EditContent/Delete/Move/Copy/Recycle. (This replaces the + // old description-keyword gating — "description contains create/update/delete" — + // where rewording an agent's description silently granted or revoked write + // access. Capability must never hinge on prose.) var meshPlugin = new MeshPlugin(Hub, chat); - var needsWriteTools = description.Contains("create", StringComparison.OrdinalIgnoreCase) - || description.Contains("update", StringComparison.OrdinalIgnoreCase) - || description.Contains("delete", StringComparison.OrdinalIgnoreCase); - tools = tools.Concat(needsWriteTools ? meshPlugin.CreateAllTools() : meshPlugin.CreateTools()); + tools = tools.Concat(meshPlugin.CreateTools()); } tools = tools.Append(PlanStorageTool.Create(Hub, chat)); + // load_skill: inject a nodeType:Skill node's instructions on demand (found via search nodeType:Skill); + // a LaunchesSubThread skill runs in its own sub-thread via the generic StartThread launcher. + tools = tools.Append(SkillTool.Create(Hub, chat)); // Wrap all tools to restore user access context before invocation. // AsyncLocal doesn't flow through the AI framework's streaming + tool invocation, @@ -165,6 +213,16 @@ private ChatClientAgent CreateAgentCore( // real-time visibility into tool calls. FunctionInvokingChatClient // consumes FunctionCallContent internally; without this middleware, // the outer stream never sees tool invocations. + // + // ⚠️ Note: this middleware fires only when callers route through the + // agent's RunStreamingAsync / RunAsync. `AgentChatClient` currently + // calls `agent.ChatClient.GetStreamingResponseAsync` directly (faster + // path that bypasses Microsoft.Agents.AI's wrapping), so the + // function-invocation middleware here is effectively unused for the + // main streaming flow. Result-population of ToolCallEntry happens + // instead via `FunctionResultContent` in the outer streaming loop + // (ThreadExecution.cs) when the underlying chat client emits FRC, or + // via `UpdateDelegationStatus` on the delegation terminal. return agent.AsBuilder() .Use((AIAgent _, FunctionInvocationContext ctx, Func> next, CancellationToken ct) => { @@ -253,17 +311,36 @@ protected virtual IEnumerable GetAgentTools( if (hasDelegations || hasHierarchyAgents || agentConfig.IsDefault) { - var delegationTool = DelegationTool.CreateUnifiedDelegationTool( - agentConfig, - hierarchyAgents, - executeAsync: (agentName, task, context, ct) => - ExecuteDelegationAsync(agentConfig, allAgents, chat, agentName, task, context, ct), - Logger); + // list_sub_threads: snapshot of this chat's active delegation paths. + // Only AgentChatClient maintains ActiveDelegationPaths; if the chat + // is some other IAgentChat (e.g. test stub), skip the optional tool. + Func>? listSubThreads = null; + Action? sendToSubThread = null; + if (chat is AgentChatClient acc) + { + var workspace = Hub.GetWorkspace(); + listSubThreads = () => SnapshotActiveSubThreads(acc, workspace); + sendToSubThread = (threadPath, message) => + PushMessageToSubThread(workspace, threadPath, message); + } - Logger.LogInformation("Created unified delegation tool for agent {AgentName} with {HierarchyCount} hierarchy agents", - agentConfig.Id, hierarchyAgents.Count); + foreach (var tool in DelegationTool.CreateDelegationTools( + agentConfig, + hierarchyAgents, + executeAsync: (agentName, task, context, ct) => + ExecuteDelegationAsync(agentConfig, allAgents, chat, agentName, task, context, ct), + listSubThreads: listSubThreads, + sendToSubThread: sendToSubThread, + delegationEvents: chat.Delegations, + workspace: Hub.GetWorkspace(), + logger: Logger)) + { + yield return tool; + } - yield return delegationTool; + Logger.LogInformation( + "Created delegation tools for agent {AgentName} with {HierarchyCount} hierarchy agents (list={List}, send={Send})", + agentConfig.Id, hierarchyAgents.Count, listSubThreads is not null, sendToSubThread is not null); } // Create handoff tool when agent has explicit handoffs @@ -283,166 +360,239 @@ protected virtual IEnumerable GetAgentTools( } /// - /// Dispatches a sub-thread and yields its streaming text deltas as . + /// Dispatches a sub-thread and yields its final accumulated text when the + /// sub-thread reaches a terminal state. While the sub-thread streams, the + /// PARENT projects each child emission onto its OWN response cell's + /// matching — Result carries the last + /// 10 lines of sub-agent output, Status tracks the lifecycle. + /// GUIs databind to that tool call for the live progress view. /// - /// The sub-thread is created fire-and-forget via IMeshService.CreateNode (no await on - /// completion). Its response-message cell is observed through a workspace remote stream; each - /// incremental delta is yielded up to the , and via - /// that — through the parent agent's streaming response — into the parent's response bubble. + /// Direction is parent-observes-child. Sub-thread code is + /// oblivious — it streams exactly as if it were a top-level thread. The + /// parent owns the remote subscriptions on the sub-thread's node + response + /// cell, computes a projection on each emission, and writes that projection + /// onto its OWN response cell via parentWorkspace.GetMeshNodeStream(parentResponsePath).Update(...). + /// The parent owns parentResponsePath, so the write serialises on its + /// own data-source action block — no cross-hub race. /// - /// No , no , no - /// ObserveQuery. The only awaits here are on the channel reader which drains on - /// cancellation or on the sub-thread's CompletedAt flip — neither touches the hub scheduler - /// (both run on the Task.Run thread pool). + /// Yield contract. Returns an + /// of , but only yields ONCE at terminal — with the + /// sub-thread's full accumulated text. + /// drains the enumerable and gives the accumulation back to FCC as the + /// FunctionResultContent; the per-tick deltas are not needed there + /// because the live progress has already landed on the parent's tool call. + /// + /// Watchdog stays. 5-minute timeout → propagate + /// RequestedStatus = Cancelled to the sub-thread + flip our tool call + /// to , then yield the partial text + /// so FCC can carry on. /// - private async IAsyncEnumerable ExecuteDelegationAsync( - AgentConfiguration agentConfig, - IReadOnlyDictionary allAgents, - IAgentChat chat, - string agentName, - string task, - string? context, - [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken) + /// + /// Snapshot accessor for the list_sub_threads tool. Reads the + /// AgentChatClient's ActiveDelegationPaths set (maintained by + /// EmitDelegationEvent) and enriches each path with a best-effort + /// status / preview from the workspace cache (synchronous; no awaits). + /// + private static IReadOnlyList SnapshotActiveSubThreads( + AgentChatClient chat, MeshWeaver.Data.IWorkspace workspace) { - // Resolve target agent (strip path prefix if present). - var targetId = agentName.Split('/').Last(); - if (!allAgents.TryGetValue(targetId, out _)) - { - yield return $"Agent '{agentName}' not found"; - yield break; - } + var paths = chat.ActiveDelegationPaths; + if (paths.IsEmpty) + return Array.Empty(); - var execCtx = chat.ExecutionContext; - if (execCtx == null) + var result = new List(paths.Count); + foreach (var path in paths) { - yield return "No execution context available for delegation"; - yield break; + // Best-effort agent name extraction from the path tail. The full + // status / preview / activity enrichment will land in the next + // refactor pass when the subscriber on the sub-thread node also + // pushes a snapshot into AgentChatClient for instant tool access. + // For now: the tool exposes the in-flight list with paths + + // agent-name parsed from the path slug, which already lets the + // parent agent decide whether to wait or send a follow-up. + var lastSegment = path.Split('/').LastOrDefault() ?? path; + var agentNameGuess = lastSegment.Split('-').FirstOrDefault() ?? "unknown"; + result.Add(new MeshWeaver.AI.Plugins.SubThreadInfo( + ThreadPath: path, + AgentName: agentNameGuess, + Status: "Executing", + PreviewText: null, + LastActivity: null)); } + return result; + } - // Guard: limit delegation depth. See comment on original version for segment math. - var threadPath = execCtx.ThreadPath; - var threadIdx = threadPath.IndexOf("/_Thread/", StringComparison.Ordinal); - var depth = 0; - if (threadIdx >= 0) - { - var afterThread = threadPath[(threadIdx + "/_Thread/".Length)..]; - var segments = afterThread.Split('/').Length; - depth = (segments - 1) / 2; - } - if (depth >= 2) - { - Logger.LogWarning("[Delegation] Max depth reached at {ThreadPath}: {Source} → {Target}", - threadPath, agentConfig.Id, targetId); - yield return $"Maximum delegation depth reached ({depth}). Handle this task directly."; - yield break; - } + /// + /// Handler for the send_to_sub_thread tool. Writes a follow-up user + /// message into the sub-thread's pending-messages queue via stream.Update; + /// the sub-thread's submission watcher picks it up and dispatches a new + /// round (or the agent's inbox-drain tool absorbs it mid-stream). + /// + private static void PushMessageToSubThread( + MeshWeaver.Data.IWorkspace workspace, string subThreadPath, string message) + { + var userMessage = ThreadInput.CreateUserMessage( + message, + createdBy: "parent-agent", + agentName: null, + modelName: null, + contextPath: null, + attachments: null); + ThreadInput.AppendUserInput(workspace, subThreadPath, userMessage); + } - Logger.LogInformation("[Delegation] {Source} → {Target}, depth={Depth}, task={Task}", - agentConfig.Id, targetId, depth, task.Length > 100 ? task[..97] + "..." : task); - - var meshService = Hub.ServiceProvider.GetRequiredService(); - var parentMsgPath = $"{threadPath}/{execCtx.ResponseMessageId}"; - var mainEntityPath = execCtx.ContextPath ?? context ?? threadPath; - - // Build the sub-thread with IsExecuting=true + PendingUserMessage so its hub's - // WatchForExecution starts streaming on activation. - var (subThreadNode, userMsgId, responseMsgId) = ThreadNodeType.BuildThreadWithMessages( - parentMsgPath, task, - createdBy: execCtx.UserAccessContext?.ObjectId, - agentName: targetId); - subThreadNode = subThreadNode with { MainNode = mainEntityPath }; - var subThreadPath = subThreadNode.Path!; - var responsePath = $"{subThreadPath}/{responseMsgId}"; - - // Stamp the delegation path so the parent's bubble can render the inline link. - var delegationDisplayName = $"Delegating to {targetId}..."; - chat.DelegationPaths[delegationDisplayName] = subThreadPath; - chat.LastDelegationPath = subThreadPath; - chat.UpdateDelegationStatus?.Invoke(delegationDisplayName); - - Logger.LogInformation("[Delegation] Dispatch sub-thread {Path}: user={UserMsgId}, response={ResponseMsgId}", - subThreadPath, userMsgId, responseMsgId); - - // Create satellite cells + thread node reactively (no await). - meshService.CreateNode(new MeshNode(userMsgId, subThreadPath) + /// + /// Pure reactive sub-thread spawn. No async, no await, no .ToTask(). + /// Returns an IObservable<string> that completes when the sub-thread + /// has been created. The actual sub-agent execution is driven by the + /// sub-thread's own submission watcher (it sees PendingUserMessages and + /// runs the round); the parent's tool-call TCS resolution is handled by + /// 's reactive Idle + /// subscription, which reads Thread.Summary when the sub-thread + /// returns to Idle. No chunk drain, no Channel bridge, no await foreach. + /// + private IObservable ExecuteDelegationAsync( + AgentConfiguration agentConfig, + IReadOnlyDictionary allAgents, + IAgentChat chat, + string agentName, + string task, + string? context, + CancellationToken cancellationToken) + => System.Reactive.Linq.Observable.Defer(() => { - NodeType = ThreadMessageNodeType.NodeType, - MainNode = mainEntityPath, - Content = new ThreadMessage + // Resolve target agent (strip path prefix if present). + var targetId = agentName.Split('/').Last(); + if (!allAgents.TryGetValue(targetId, out _)) + return System.Reactive.Linq.Observable.Return($"Agent '{agentName}' not found"); + + var execCtx = chat.ExecutionContext; + if (execCtx is null) + return System.Reactive.Linq.Observable.Return("No execution context available for delegation"); + + // Depth guard. + var threadPath = execCtx.ThreadPath; + var threadIdx = threadPath.IndexOf("/_Thread/", StringComparison.Ordinal); + var depth = 0; + if (threadIdx >= 0) { - Role = "user", Text = task, Timestamp = DateTime.UtcNow, - Type = ThreadMessageType.ExecutedInput, - CreatedBy = execCtx.UserAccessContext?.ObjectId + var afterThread = threadPath[(threadIdx + "/_Thread/".Length)..]; + var segments = afterThread.Split('/').Length; + depth = (segments - 1) / 2; } - }).Subscribe(_ => { }, - error => Logger.LogDebug(error, "[Delegation] User cell create for {Path} returned error", subThreadPath)); - - meshService.CreateNode(new MeshNode(responseMsgId, subThreadPath) - { - NodeType = ThreadMessageNodeType.NodeType, - MainNode = mainEntityPath, - Content = new ThreadMessage + if (depth >= 2) { - Role = "assistant", Text = "", Timestamp = DateTime.UtcNow, - Type = ThreadMessageType.AgentResponse, AgentName = targetId + Logger.LogWarning("[Delegation] Max depth reached at {ThreadPath}: {Source} → {Target}", + threadPath, agentConfig.Id, targetId); + return System.Reactive.Linq.Observable.Return( + $"Maximum delegation depth reached ({depth}). Handle this task directly."); } - }).Subscribe(_ => { }, - error => Logger.LogDebug(error, "[Delegation] Response cell create for {Path} returned error", subThreadPath)); - meshService.CreateNode(subThreadNode).Subscribe( - _ => Logger.LogInformation("[Delegation] Sub-thread created at {Path}", subThreadPath), - error => Logger.LogWarning(error, "[Delegation] Sub-thread create failed at {Path}", subThreadPath)); + Logger.LogInformation("[Delegation] {Source} → {Target}, depth={Depth}, task={Task}", + agentConfig.Id, targetId, depth, task.Length > 100 ? task[..97] + "..." : task); + + var parentMsgPath = $"{threadPath}/{execCtx.ResponseMessageId}"; + var mainEntityPath = execCtx.ContextPath ?? context ?? threadPath; + + // Build the full sub-thread node + ids ONCE. GenerateSpeakingId appends + // a random suffix, so calling BuildThreadWithMessages twice produces + // DIFFERENT paths. Single source of truth. + var (preSubThreadNode, userMsgId, responseMsgId) = + MeshWeaver.AI.ThreadNodeType.BuildThreadWithMessages( + parentMsgPath, task, + createdBy: execCtx.UserAccessContext?.ObjectId, + agentName: targetId); + var subThreadNode = preSubThreadNode with { MainNode = mainEntityPath }; + var subThreadPath = subThreadNode.Path!; + var callId = Guid.NewGuid().ToString("N")[..8]; + + Logger.LogInformation( + "[Delegation:{CallId}] ENTER sub={SubPath} target={Target} parentResp={ParentResp}", + callId, subThreadPath, targetId, parentMsgPath); + + var meshService = Hub.ServiceProvider.GetRequiredService(); + + // Pure nested-Subscribe pattern (per CLAUDE.md AsynchronousCalls.md): + // CreateNode → on emission, subscribe to sub-thread stream for the + // Running→Idle transition → on Idle, emit Terminal event so the + // parent's tool-call TCS (in DelegationTool's reactive subscription) + // resolves with Thread.Summary. No await, no .ToTask(), no SelectMany + // chain that could capture the grain scheduler — Subscribe is the + // continuation primitive. + return System.Reactive.Linq.Observable.Create(observer => + { + var workspace = Hub.GetWorkspace(); + var sawRunning = false; - yield return $"\n\n**Delegating to {targetId}…**\n\n"; + meshService.CreateNode(subThreadNode).Subscribe( + _ => + { + Logger.LogInformation( + "[Delegation:{CallId}] CREATE_OK sub={Path}", callId, subThreadPath); + + if (chat is AgentChatClient agentChat) + { + Logger.LogInformation( + "[Delegation:{CallId}] EMIT_DISPATCHED sub={Path}", callId, subThreadPath); + agentChat.EmitDelegationEvent( + new MeshWeaver.AI.Delegation.DelegationEvent(callId, subThreadPath, + MeshWeaver.AI.Delegation.DelegationLifecycle.Dispatched)); + + // Watch sub-thread for Running→Idle. Same Scan-based + // pattern DelegationTool uses for the parent's + // TCS resolution — emit Terminal here so the + // cancel-watcher + tool-call stamper drop the entry. + workspace.GetMeshNodeStream(subThreadPath).Subscribe( + node => + { + if (node?.Content is not MeshThread t) return; + if (t.Status is ThreadExecutionStatus.Executing + or ThreadExecutionStatus.StartingExecution) + { + sawRunning = true; + } + else if (sawRunning && t.Status is ThreadExecutionStatus.Idle + or ThreadExecutionStatus.Cancelled + or ThreadExecutionStatus.Done) + { + Logger.LogInformation( + "[Delegation:{CallId}] TERMINAL sub={Path} (Running→Idle)", + callId, subThreadPath); + agentChat.EmitDelegationEvent( + new MeshWeaver.AI.Delegation.DelegationEvent(callId, subThreadPath, + MeshWeaver.AI.Delegation.DelegationLifecycle.Terminal)); + } + }, + ex => Logger.LogWarning(ex, + "[Delegation:{CallId}] sub-thread stream errored", callId)); + } + + // No chunks emitted — DelegationTool's reactive + // completion path reads Thread.Summary directly on Idle. + observer.OnCompleted(); + }, + ex => + { + Logger.LogWarning(ex, "[Delegation:{CallId}] CREATE_FAIL sub={Path}", + callId, subThreadPath); + observer.OnNext($"\n[Delegation failed: {ex.Message}]"); + observer.OnCompleted(); + }); - // Open a channel fed by the sub-thread's response-cell remote stream. We yield - // each text delta as it arrives (computed against lastText so we never double-emit). - var channel = System.Threading.Channels.Channel.CreateUnbounded( - new System.Threading.Channels.UnboundedChannelOptions - { - SingleReader = true, - SingleWriter = false + return System.Reactive.Disposables.Disposable.Empty; }); + }); - var workspace = Hub.GetWorkspace(); - var lastText = ""; - var subscription = workspace.GetRemoteStream( - new Address(responsePath), new MeshNodeReference()) - ?.Subscribe( - change => - { - var msg = change.Value?.Content as ThreadMessage; - if (msg == null) return; - var current = msg.Text ?? ""; - if (current.Length > lastText.Length) - { - var delta = current[lastText.Length..]; - lastText = current; - channel.Writer.TryWrite(delta); - } - if (msg.CompletedAt is not null) - channel.Writer.TryComplete(); - }, - ex => channel.Writer.TryComplete(ex), - () => channel.Writer.TryComplete()); - - // Safety timeout so a never-completing sub-thread can't pin this iterator forever. - using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(5)); - using var linked = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeout.Token); + private readonly record struct DelegationObservation( + MeshThread? Thread, + ThreadMessage? Cell, + string? Error); - try - { - await foreach (var delta in channel.Reader.ReadAllAsync(linked.Token)) - { - yield return delta; - } - } - finally - { - subscription?.Dispose(); - Logger.LogInformation("[Delegation] Stream closed for sub-thread {Path}", subThreadPath); - } - } + /// + /// Distinguishes which subscription signalled terminal — purely for + /// diagnostic logging; both signals map to the same finalisation path. + /// + private enum TerminalSignal { ThreadIdle, CellCompleted } /// /// Resolves a plugin reference to AITool instances. @@ -460,6 +610,7 @@ private async IAsyncEnumerable ExecuteDelegationAsync( "Version" => new VersionPlugin(Hub).CreateTools(), "Collaboration" => new CollaborationPlugin(Hub, chat).CreateTools(), "ContentCollection" => new ContentCollectionPlugin(Hub, chat).CreateTools(), + "Lsp" => new LspPlugin(Hub, chat).CreateTools(), _ => Hub.ServiceProvider.GetServices() .FirstOrDefault(p => string.Equals(p.Name, pluginRef.Name, StringComparison.OrdinalIgnoreCase)) ?.CreateTools() @@ -501,9 +652,37 @@ private string BuildInstructionsWithDelegations(string baseInstructions, AgentCo var hasHandoffs = agentConfig.Handoffs is { Count: > 0 }; var hasHierarchyAgents = hierarchyAgents.Count > 1; + // Thread-message inspection + summary contract — appended for every + // agent so delegation-incapable agents still know how to discover + // prior responses and how their own response gets stored as a summary. + var threadInspectionAndSummary = + """ + + **Reading prior thread messages:** + Use the `search` tool with `nodeType:ThreadMessage` to find conversation cells. + - One thread: `search "path:{threadPath} scope:descendants nodeType:ThreadMessage"` + - One sub-thread (delegation child): same shape with the sub-thread's path. + - Project only the fields you need with `select:` — e.g. + `search "path:{threadPath} scope:descendants nodeType:ThreadMessage select:text,summary,role,timestamp"`. + - To read the dedicated summary of a completed sub-thread directly, + `get "{subThreadPath}"` and read `content.summary` (filled atomically + with `content.status=Idle`). + Use `select:summary` when scanning many cells — it returns the one-line + digest without the verbose `text`, so you can survey a deep thread cheaply. + + **End every response with a block:** + At the very end of your response, on its own line, emit: + `One- or two-sentence distillation of what you did or decided.` + The framework strips this from what the user sees and stores it as the + dedicated summary on both the response message and the thread. When a + parent agent delegated to you, this summary IS the tool-call result it + receives back — not your verbose response. Be tight and outcome-focused. + + """; + if (!hasDelegations && !hasHandoffs && !hasHierarchyAgents && !agentConfig.IsDefault) { - return baseInstructions; + return baseInstructions + threadInspectionAndSummary; } var result = baseInstructions; @@ -553,6 +732,8 @@ 4. Relay or summarize the result to the user """; } + result += threadInspectionAndSummary; + // Handoff guidelines if (hasHandoffs) { @@ -588,27 +769,81 @@ Use handoff when the target agent should take over the conversation directly and } /// -/// AIFunction wrapper that restores the user's access context before each invocation. -/// This is the single injection point for ALL tool calls — delegation, MeshPlugin, etc. +/// AIFunction wrapper that restores the user's access context before each invocation +/// AND enforces a per-tool execution timeout (via ; +/// default 30 s). This is the single injection point for ALL tool calls — delegation, +/// MeshPlugin, etc. +/// +/// The timeout is read once at wrap time from the inner function's underlying +/// method. On expiry the linked CTS cancels the tool invocation and the agent receives +/// the synthetic "timed out" message as the tool result — never a hung promise. +/// delegate_to_agent is exempt (lifecycle-managed by the thread-hub heartbeat, +/// not a tool in the timeout-attribute sense). /// internal sealed class AccessContextAIFunction : DelegatingAIFunction { + /// + /// Default timeout when no is present on the + /// underlying tool method. 30 s — long enough for any reasonable tool, short + /// enough that a hung tool surfaces fast in the chat UI. + /// + private static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(30); + + /// + /// Tools that opt out of the timeout because their lifecycle is managed by the + /// thread hub itself (currently just delegate_to_agent). They have their + /// own heartbeat-based hang detection on MeshThread.LastActivityAt. + /// + private static readonly HashSet TimeoutExemptTools = new(StringComparer.Ordinal) + { + "delegate_to_agent", + }; + private readonly IAgentChat _chat; private readonly AccessService _accessService; + private readonly TimeSpan? _timeout; public AccessContextAIFunction(AIFunction inner, IAgentChat chat, AccessService accessService) : base(inner) { _chat = chat; _accessService = accessService; + _timeout = TimeoutExemptTools.Contains(inner.Name) + ? null + : (inner.UnderlyingMethod?.GetCustomAttribute()?.Timeout + ?? DefaultTimeout); } - protected override ValueTask InvokeCoreAsync( + protected override async ValueTask InvokeCoreAsync( AIFunctionArguments arguments, CancellationToken cancellationToken) { var userCtx = _chat.ExecutionContext?.UserAccessContext; if (userCtx != null) _accessService.SetContext(userCtx); - return base.InvokeCoreAsync(arguments, cancellationToken); + + if (_timeout is null) + return await base.InvokeCoreAsync(arguments, cancellationToken); + + // Bound the wait via Task.WaitAsync — covers both well-behaved tools + // (which observe cts.Token and unwind via OCE) AND ill-behaved tools + // (which ignore the token and would otherwise pin the agent loop until + // their intrinsic delay finishes). On timeout, the inner Task becomes + // orphaned (still runs to completion in the background) but the agent + // never waits — it gets a deterministic synthetic FunctionResultContent. + using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var invocation = base.InvokeCoreAsync(arguments, cts.Token).AsTask(); + try + { + return await invocation.WaitAsync(_timeout.Value, cancellationToken); + } + catch (TimeoutException) + { + // Our timer fired. Signal cooperative cancellation so well-behaved + // tools wind down even though we've stopped waiting; ill-behaved + // tools continue but the wrapper is no longer blocked on them. + cts.Cancel(); + return $"Tool '{Name}' timed out after {_timeout.Value.TotalSeconds:F0}s. " + + $"Add [ToolTimeout(N)] to allow longer."; + } } } diff --git a/src/MeshWeaver.AI/ChatClientCredentialResolver.cs b/src/MeshWeaver.AI/ChatClientCredentialResolver.cs new file mode 100644 index 000000000..03f2d66e3 --- /dev/null +++ b/src/MeshWeaver.AI/ChatClientCredentialResolver.cs @@ -0,0 +1,410 @@ +using System.Collections.Immutable; +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI; + +/// +/// Outcome of a credential lookup. is a short +/// human-readable tag identifying which rung of the resolution chain +/// produced the value — appears in factory logs so a stale or wrong key +/// can be traced back to the MeshNode (or IOptions) that supplied it. +/// +public record CredentialResolution(string? Endpoint, string? ApiKey, string Source) +{ + public static readonly CredentialResolution Missing = new(null, null, "missing"); +} + +/// +/// Unified Endpoint + ApiKey lookup for AI chat-client factories. +/// +/// Reads LIVE from the same workspace.GetQuery snapshot the chat +/// model-picker uses ( +/// returns nodeType:LanguageModel|ModelProvider). No materialised +/// dictionary of node content is held — every / +/// call grabs the current snapshot from the +/// workspace's per-id cache (already Replay(1).RefCount()), so it can +/// never go stale. See the SyncedMeshNodeQueries architecture doc for +/// the canonical query semantics. +/// +/// Resolution precedence (top wins): +/// +/// Explicit → +/// at that path. This is +/// the normal path — both +/// and ModelProviderService stamp the reference when they +/// create the LanguageModel node. +/// Conventional fallback at +/// Model/{ModelDefinition.Provider} — covers legacy catalog +/// entries that didn't stamp . +/// Legacy ModelDefinition fields +/// ( / +/// ) — stamped per-model on +/// older catalog rollouts. +/// — caller falls back +/// to its IOptions<...Configuration> binding. +/// +/// +/// User-partition ModelProvider visibility: callers (notably +/// AgentChatClient) invoke to widen the +/// live query with {userPartition}/_Provider/... nodes. Without this +/// only the root catalog is visible — sufficient for system-default +/// deployments but blind to per-user BYO keys. Calls are idempotent per +/// partition; they record which subtrees the next +/// should include, they do NOT cache node content. +/// +public sealed class ChatClientCredentialResolver : IDisposable +{ + private readonly IMessageHub hub; + private readonly ILogger? logger; + private readonly IProviderKeyProtector? keyProtector; + + // "What to look at", NOT cached node content. These are tiny, immutable + // path sets describing which extra subtrees the live query must union in. + // Swapped atomically under `gate`; they never hold MeshNode data, so they + // can never go stale — the data is always read fresh from the workspace. + private readonly object gate = new(); + private ImmutableHashSet watchedPartitions = ImmutableHashSet.Empty; + private ImmutableHashSet sharedProviderPaths = ImmutableHashSet.Create(StringComparer.Ordinal); + + private bool disposed; + + public ChatClientCredentialResolver(IMessageHub hub) + { + this.hub = hub; + logger = hub.ServiceProvider.GetService() + ?.CreateLogger(); + // ModelProvider.ApiKey is encrypted at rest — decrypt at the moment we + // hand the credential to a factory. Null when the protector isn't + // registered (then values are plaintext anyway → Decrypt passthrough). + keyProtector = hub.ServiceProvider.GetService(); + } + + /// + /// No-op retained for API compatibility. The root catalog is always + /// included by every live read (see with + /// no context path), so there is no eager root subscription to start. + /// + public void EnsureSubscription() { } + + /// + /// Widen subsequent reads to include ModelProvider + + /// LanguageModel nodes under a user partition. Called from the per-chat + /// AgentChatClient with the chat user's partition (their + /// ). Idempotent + /// on partition. Records the path only — no node content is cached. + /// + public IDisposable WatchPartition(string userPartition) + { + if (string.IsNullOrEmpty(userPartition)) return Disposable.Empty; + lock (gate) + watchedPartitions = watchedPartitions.Add(userPartition); + return Disposable.Empty; + } + + /// + /// Make a shared / organisation ModelProvider subtree usable by + /// under use-without-see: the provider node + /// is read under a SYSTEM identity (so its -gated + /// key reaches the resolver process), but only hands + /// the key to a user who holds + /// on the subtree — evaluated LIVE at resolve time via + /// hub.CheckPermission. The raw key never leaves the server. + /// Idempotent per path; records the path only — no node content is cached. + /// + public IDisposable WatchSharedProvider(string providerPath, string userId) + { + if (string.IsNullOrEmpty(providerPath) || string.IsNullOrEmpty(userId)) + return Disposable.Empty; + lock (gate) + sharedProviderPaths = sharedProviderPaths.Add(providerPath); + return Disposable.Empty; + } + + /// + /// Gate for shared-provider keys. Non-shared providers (root catalog, the + /// resolving user's own partition via ) are not + /// gated here — RLS already governed their visibility at read time. + /// Shared providers fail closed: no Read ⇒ no key. The Read check is + /// evaluated LIVE (no cached gate result) against the user's effective + /// permissions on the subtree. + /// + private bool IsAllowedSharedAccess(string providerPath) + { + if (!sharedProviderPaths.Contains(providerPath)) return true; + var userId = hub.ServiceProvider.GetService()?.Context?.ObjectId; + if (string.IsNullOrEmpty(userId)) return false; + return ReadLatest(hub.CheckPermission(providerPath, userId, Permission.Read), false); + } + + /// + /// Resolve credentials for a model. is the + /// LanguageModel id the chat selected (e.g. claude-opus-4-7). + /// Walks the precedence chain documented on the class against a LIVE + /// snapshot of the model + provider nodes. + /// + public CredentialResolution Resolve(string modelId) + { + if (string.IsNullOrEmpty(modelId)) return CredentialResolution.Missing; + + var snapshot = ReadSnapshot(); + var def = FindModelDefinition(snapshot, modelId); + if (def == null) + return CredentialResolution.Missing; + + // 1. Explicit ProviderRef — the normal path. + if (!string.IsNullOrEmpty(def.ProviderRef) + && TryGetProvider(snapshot, def.ProviderRef, out var byRef) + && HasAnyCredential(byRef!) + && IsAllowedSharedAccess(def.ProviderRef)) + { + return new CredentialResolution(byRef!.Endpoint, Decrypt(byRef.ApiKey), $"providerRef:{def.ProviderRef}"); + } + + // 2. Conventional fallback: Model/{Provider} in the root namespace. + if (!string.IsNullOrEmpty(def.Provider)) + { + var conventional = $"{ModelProviderNodeType.RootNamespace}/{def.Provider}"; + if (TryGetProvider(snapshot, conventional, out var byConvention) + && HasAnyCredential(byConvention!) + && IsAllowedSharedAccess(conventional)) + { + return new CredentialResolution(byConvention!.Endpoint, Decrypt(byConvention.ApiKey), $"convention:{conventional}"); + } + } + + // 3. Legacy fields stamped directly on the ModelDefinition. + if (!string.IsNullOrEmpty(def.ApiKeySecretRef) || !string.IsNullOrEmpty(def.Endpoint)) + { + return new CredentialResolution(def.Endpoint, Decrypt(def.ApiKeySecretRef), "model-node"); + } + + return CredentialResolution.Missing; + } + + /// + /// Resolve the per-user Connect token for a CLI harness — the user's OWN subscription + /// token captured by the login (Connect) flow and stored, encrypted, as a ModelProvider + /// node at {UserNamespacePath(userPartition)}/{providerName} + /// (e.g. {user}/_Memex/ClaudeCode). Returns the decrypted key, or null when the + /// user hasn't connected. + /// + /// This is deliberately NOT : a CLI harness (Claude Code / + /// GitHub Copilot) authenticates with the user's subscription token, never with a selected + /// MODEL's API key. Passing the selected model's key (e.g. the DeepSeek/AzureFoundry key) is + /// exactly what produced the atioz "Not logged in" failure. Best-effort: the authoritative login + /// also lives in the CLI's own per-user config dir (.credentials.json on the shared + /// volume), so a not-yet-warm node read simply leaves the env var unset and the CLI falls back to + /// its config dir. + /// + public string? ResolveConnectToken(string providerName, string? userPartition) + { + if (string.IsNullOrEmpty(providerName) || string.IsNullOrEmpty(userPartition)) + return null; + // Widen subsequent snapshots to the user's own partition (idempotent), then read. + WatchPartition(userPartition!); + var providerPath = $"{ModelProviderNodeType.UserNamespacePath(userPartition!)}/{providerName}"; + return TryGetProvider(ReadSnapshot(), providerPath, out var cfg) && cfg is not null + ? Decrypt(cfg.ApiKey) + : null; + } + + /// + /// Decrypts a stored credential. Passthrough when no protector is registered + /// or the value is legacy plaintext (see ). + /// + private string? Decrypt(string? stored) => + keyProtector is null ? stored : keyProtector.Unprotect(stored); + + /// + /// Returns the string for a + /// given model id, looked up from the live snapshot. Factories call + /// this to gate their Supports(modelName) on the model's + /// declared provider rather than on the model id alone (so the same + /// claude-* id can route to direct-Anthropic or Azure-Claude + /// based on which provider node owns it). + /// + public string? GetProviderForModel(string modelId) + { + if (string.IsNullOrEmpty(modelId)) return null; + var def = FindModelDefinition(ReadSnapshot(), modelId); + return def?.Provider; + } + + public void Dispose() + { + if (disposed) return; + disposed = true; + lock (gate) + { + watchedPartitions = ImmutableHashSet.Empty; + sharedProviderPaths = ImmutableHashSet.Create(StringComparer.Ordinal); + } + } + + /// + /// Reads the current LanguageModel + ModelProvider snapshot from the + /// workspace's synced-query cache. The query unions the root catalog, each + /// watched user partition, and each shared provider subtree. The shared + /// subtrees are read under a SYSTEM identity so an Api-gated provider node + /// surfaces (the per-user Read gate is enforced separately in + /// ). + /// + private IReadOnlyList ReadSnapshot() + { + ImmutableHashSet partitions; + ImmutableHashSet shared; + lock (gate) + { + partitions = watchedPartitions; + shared = sharedProviderPaths; + } + + var workspace = hub.GetWorkspace(); + + // Root catalog + each watched user partition share one cache id; the + // workspace caches the union by id (Replay(1).RefCount upstream). + var baseQueries = BuildModelQueries(partitions); + var baseId = "ChatClientCredentialResolver|" + string.Join(",", partitions.OrderBy(p => p, StringComparer.Ordinal)); + var nodes = ReadLatest(workspace.GetQuery(baseId, baseQueries), Array.Empty() as IEnumerable); + + if (shared.IsEmpty) + return nodes as IReadOnlyList ?? nodes.ToList(); + + // Shared provider subtrees must be read under a system identity so the + // Api-gated provider node + its LanguageModel children are visible to + // the resolver process (the Read gate is enforced at hand-out time). + var accessService = hub.ServiceProvider.GetService(); + var merged = new Dictionary(StringComparer.Ordinal); + foreach (var n in nodes) + if (n.Path != null) merged[n.Path] = n; + + var typeFilter = $"{LanguageModelNodeType.NodeType}|{ModelProviderNodeType.NodeType}"; + foreach (var path in shared) + { + var sharedQuery = $"namespace:{path} nodeType:{typeFilter} scope:selfAndDescendants"; + var sharedObs = workspace.GetQuery($"ChatClientCredentialResolver.Shared|{path}", sharedQuery); + var asSystem = accessService is null + ? sharedObs + : Observable.Using(accessService.ImpersonateAsSystem, _ => sharedObs); + var sharedNodes = ReadLatest(asSystem, Array.Empty() as IEnumerable); + foreach (var n in sharedNodes) + if (n.Path != null) merged[n.Path] = n; + } + + return merged.Values.ToList(); + } + + /// + /// Builds the live model queries: the root catalog plus one subtree query + /// per watched user partition. Mirrors + /// 's root + + /// per-partition shape (the per-partition entry uses currentPath). + /// + private static string[] BuildModelQueries(ImmutableHashSet partitions) + { + if (partitions.IsEmpty) + return AgentPickerProjection.BuildModelQueries(); + + var typeFilter = $"{LanguageModelNodeType.NodeType}|{ModelProviderNodeType.NodeType}"; + var queries = new List + { + $"namespace:{ModelProviderNodeType.RootNamespace} nodeType:{typeFilter} scope:descendants", + }; + // Each watched partition is a USER partition (WatchPartition is called + // with the resolving user's id). A user's own providers/models live in + // their dotfile namespace ({user}/_Memex/…); union the legacy _Provider + // subtree too so pre-existing data still resolves (many queries are fine + // — the synced collection unions them). + foreach (var p in partitions) + { + queries.Add($"namespace:{ModelProviderNodeType.UserNamespacePath(p)} nodeType:{typeFilter} scope:descendants"); + queries.Add($"namespace:{p}/{ModelProviderNodeType.RootNamespace} nodeType:{typeFilter} scope:descendants"); + } + return queries.ToArray(); + } + + private ModelDefinition? FindModelDefinition(IReadOnlyList snapshot, string modelId) + { + foreach (var node in snapshot) + { + if (!string.Equals(node.NodeType, LanguageModelNodeType.NodeType, StringComparison.OrdinalIgnoreCase)) + continue; + var def = ExtractContent(node.Content); + if (def != null && string.Equals(def.Id, modelId, StringComparison.OrdinalIgnoreCase)) + return def; + } + return null; + } + + private bool TryGetProvider(IReadOnlyList snapshot, string providerPath, out ModelProviderConfiguration? cfg) + { + foreach (var node in snapshot) + { + if (node.Path == null + || !string.Equals(node.Path, providerPath, StringComparison.Ordinal) + || !string.Equals(node.NodeType, ModelProviderNodeType.NodeType, StringComparison.OrdinalIgnoreCase)) + continue; + var extracted = ExtractContent(node.Content); + if (extracted != null && !string.IsNullOrEmpty(extracted.Provider)) + { + cfg = extracted; + return true; + } + } + cfg = null; + return false; + } + + /// + /// Synchronously grabs the current value of a warm Replay(1).RefCount() + /// observable (the shape every workspace.GetQuery / permission stream + /// returns). Subscribe replays the latest value inline; we capture it and + /// dispose. Returns when nothing has been + /// emitted yet (cold / first read before the synced query warms). + /// + private static T ReadLatest(IObservable? source, T fallback) + { + if (source is null) return fallback; + var captured = fallback; + var got = false; + using var _ = source.Subscribe(v => { captured = v; got = true; }, _ => { }); + return got ? captured : fallback; + } + + private T? ExtractContent(object? content) where T : class + { + return content switch + { + T typed => typed, + JsonElement je => TryDeserialise(je), + _ => null, + }; + } + + private T? TryDeserialise(JsonElement je) where T : class + { + try { return JsonSerializer.Deserialize(je.GetRawText(), hub.JsonSerializerOptions); } + catch (Exception ex) + { + logger?.LogDebug(ex, "Failed to deserialise content as {Type}", typeof(T).Name); + return null; + } + } + + private static bool HasAnyCredential(ModelProviderConfiguration p) => + !string.IsNullOrEmpty(p.ApiKey) || !string.IsNullOrEmpty(p.Endpoint); + + private static class Disposable + { + public static readonly IDisposable Empty = new EmptyDisposable(); + private sealed class EmptyDisposable : IDisposable { public void Dispose() { } } + } +} diff --git a/src/MeshWeaver.AI/ChatDelegationContent.cs b/src/MeshWeaver.AI/ChatDelegationContent.cs index fe7c673c4..a20ce512d 100644 --- a/src/MeshWeaver.AI/ChatDelegationContent.cs +++ b/src/MeshWeaver.AI/ChatDelegationContent.cs @@ -28,9 +28,9 @@ public class ChatDelegationContent : AIContent public bool RequiresUserFeedback { get; } public ChatDelegationContent( - string delegatingAgent, - string targetAgent, - string delegationMessage, + string delegatingAgent, + string targetAgent, + string delegationMessage, bool requiresUserFeedback = false) { DelegatingAgent = delegatingAgent; @@ -38,4 +38,23 @@ public ChatDelegationContent( DelegationMessage = delegationMessage; RequiresUserFeedback = requiresUserFeedback; } + + /// + /// Short summary of for chip / header display: + /// first non-empty line, truncated to ~40 chars. Empty when the message is + /// missing — callers fall back to the bare "Delegating to {Agent}" shape. + /// + public string TaskSummary + { + get + { + if (string.IsNullOrWhiteSpace(DelegationMessage)) + return string.Empty; + var firstLine = DelegationMessage.Split('\n', 2)[0].Trim(); + const int maxLen = 40; + if (firstLine.Length > maxLen) + firstLine = firstLine[..(maxLen - 1)] + "…"; + return firstLine; + } + } } \ No newline at end of file diff --git a/src/MeshWeaver.AI/Commands/AgentCommand.cs b/src/MeshWeaver.AI/Commands/AgentCommand.cs deleted file mode 100644 index d0e00627e..000000000 --- a/src/MeshWeaver.AI/Commands/AgentCommand.cs +++ /dev/null @@ -1,62 +0,0 @@ -#nullable enable - -using System.Text.RegularExpressions; - -namespace MeshWeaver.AI.Commands; - -/// -/// Command to switch the current agent. -/// Usage: /agent @agent/AgentName or /agent AgentName -/// -public class AgentCommand : IChatCommand -{ - public string Name => "agent"; - public string Description => "Switch to a different agent for subsequent messages"; - public string Usage => "/agent @agent/Name or /agent Name"; - - private static readonly Regex AgentRefPattern = - new(@"@agent/(\w+)", RegexOptions.Compiled | RegexOptions.IgnoreCase); - - public Task ExecuteAsync(CommandContext context, CancellationToken cancellationToken = default) - { - if (context.ParsedCommand.Arguments.Length == 0) - { - // List available agents - var agentNames = string.Join(", ", context.AvailableAgents.Keys.OrderBy(n => n)); - return Task.FromResult(CommandResult.Error( - $"Usage: {Usage}\n\nAvailable agents: {agentNames}")); - } - - // Parse agent name from argument - var arg = context.ParsedCommand.RawArguments; - string agentName; - - var match = AgentRefPattern.Match(arg); - if (match.Success) - { - agentName = match.Groups[1].Value; - } - else - { - // Allow just the agent name without @agent/ prefix - agentName = context.ParsedCommand.Arguments[0].TrimStart('@'); - } - - // Find the agent (case-insensitive) - var found = context.AvailableAgents - .FirstOrDefault(kvp => kvp.Key.Equals(agentName, StringComparison.OrdinalIgnoreCase)); - - if (found.Value == null) - { - var availableNames = string.Join(", ", context.AvailableAgents.Keys.OrderBy(n => n)); - return Task.FromResult(CommandResult.Error( - $"Agent '{agentName}' not found.\n\nAvailable agents: {availableNames}")); - } - - // Switch to the agent - context.SetCurrentAgent(found.Value); - - return Task.FromResult(CommandResult.Ok( - $"Switched to agent: **{found.Value.Name}**\n\n_{found.Value.Description}_")); - } -} diff --git a/src/MeshWeaver.AI/Commands/ChatCommandRegistry.cs b/src/MeshWeaver.AI/Commands/ChatCommandRegistry.cs deleted file mode 100644 index 2bdbab27e..000000000 --- a/src/MeshWeaver.AI/Commands/ChatCommandRegistry.cs +++ /dev/null @@ -1,59 +0,0 @@ -#nullable enable - -using Microsoft.Extensions.Logging; - -namespace MeshWeaver.AI.Commands; - -/// -/// Registry for chat commands. Provides lookup and execution of commands. -/// -public class ChatCommandRegistry -{ - private readonly Dictionary _commands = new(StringComparer.OrdinalIgnoreCase); - private readonly ILogger? _logger; - - public ChatCommandRegistry(ILogger? logger = null) - { - _logger = logger; - } - - /// - /// Registers a command. - /// - public void Register(IChatCommand command) - { - _commands[command.Name] = command; - - // Register aliases - foreach (var alias in command.Aliases) - { - _commands[alias] = command; - } - - _logger?.LogDebug("Registered command: /{Name}", command.Name); - } - - /// - /// Tries to get a command by name. - /// - public bool TryGetCommand(string name, out IChatCommand? command) - { - return _commands.TryGetValue(name, out command); - } - - /// - /// Gets all registered commands (without duplicates from aliases). - /// - public IReadOnlyList GetAllCommands() - { - return _commands.Values.Distinct().ToList(); - } - - /// - /// Checks if a command exists. - /// - public bool HasCommand(string name) - { - return _commands.ContainsKey(name); - } -} diff --git a/src/MeshWeaver.AI/Commands/CommandContext.cs b/src/MeshWeaver.AI/Commands/CommandContext.cs deleted file mode 100644 index 175eaf582..000000000 --- a/src/MeshWeaver.AI/Commands/CommandContext.cs +++ /dev/null @@ -1,89 +0,0 @@ -#nullable enable - -using MeshWeaver.AI.Parsing; - -namespace MeshWeaver.AI.Commands; - -/// -/// Context provided to commands during execution. -/// -public record CommandContext -{ - /// - /// The parsed command information. - /// - public required ParsedCommand ParsedCommand { get; init; } - - /// - /// Available agents by name. - /// - public required IReadOnlyDictionary AvailableAgents { get; init; } - - /// - /// Currently selected agent. - /// - public AgentDisplayInfo? CurrentAgent { get; init; } - - /// - /// Callback to set the current agent (for /agent command). - /// - public required Action SetCurrentAgent { get; init; } - - /// - /// Available models with provider information. - /// - public IReadOnlyList? AvailableModels { get; init; } - - /// - /// Currently selected model. - /// - public ModelInfo? CurrentModel { get; init; } - - /// - /// Callback to set the current model (for /model command). - /// - public Action? SetCurrentModel { get; init; } - - /// - /// Current agent context (address, layout area). - /// - public AgentContext? AgentContext { get; init; } - - /// - /// Registry of all commands (for /help command). - /// - public ChatCommandRegistry? CommandRegistry { get; init; } -} - -/// -/// Result of executing a command. -/// -public record CommandResult -{ - /// - /// Whether the command executed successfully. - /// - public bool Success { get; init; } - - /// - /// Message to display to the user (e.g., error message or confirmation). - /// - public string? Message { get; init; } - - /// - /// Whether to proceed with sending the message to the AI. - /// - public bool ShouldSendToAI { get; init; } = false; - - /// - /// Creates a successful result. - /// - public static CommandResult Ok(string? message = null) => - new() { Success = true, Message = message }; - - /// - /// Creates a failed result. - /// - public static CommandResult Error(string message) => - new() { Success = false, Message = message }; -} diff --git a/src/MeshWeaver.AI/Commands/Help/agent.md b/src/MeshWeaver.AI/Commands/Help/agent.md deleted file mode 100644 index 085e8945b..000000000 --- a/src/MeshWeaver.AI/Commands/Help/agent.md +++ /dev/null @@ -1,23 +0,0 @@ -## /agent - -Switch to a different agent for subsequent messages. - -**Usage:** `/agent @agent/Name` or `/agent Name` - -### Description - -The `/agent` command allows you to change which AI agent handles your messages. Once switched, the selected agent will handle all subsequent messages until you switch again. - -### Examples - -``` -/agent @agent/InsuranceAgent -/agent InsuranceAgent -/agent insurance -``` - -### Notes - -- Agent names are case-insensitive -- The agent selection persists until you explicitly switch to another agent -- You can also use `@agent/Name` inline in any message to temporarily address a specific agent diff --git a/src/MeshWeaver.AI/Commands/Help/help.md b/src/MeshWeaver.AI/Commands/Help/help.md deleted file mode 100644 index dd6e06a59..000000000 --- a/src/MeshWeaver.AI/Commands/Help/help.md +++ /dev/null @@ -1,25 +0,0 @@ -## /help - -Show available commands and their usage. - -**Usage:** `/help [command]` - -**Aliases:** `/?` - -### Description - -The `/help` command displays information about available chat commands. Use it without arguments to see all commands, or specify a command name to get detailed help for that specific command. - -### Examples - -``` -/help -/help agent -/? -``` - -### Tips - -- Commands start with `/` -- Use `@agent/Name` anywhere in your message to address a specific agent -- Type `/` to see command suggestions in the autocomplete diff --git a/src/MeshWeaver.AI/Commands/Help/model.md b/src/MeshWeaver.AI/Commands/Help/model.md deleted file mode 100644 index ec37d8851..000000000 --- a/src/MeshWeaver.AI/Commands/Help/model.md +++ /dev/null @@ -1,24 +0,0 @@ -## /model - -Switch to a different AI model for subsequent messages. - -**Usage:** `/model @model/Name` or `/model Name` - -### Description - -The `/model` command allows you to change which AI model handles your messages. Once switched, the selected model will be used for all subsequent messages until you switch again. - -### Examples - -``` -/model @model/gpt-4o -/model gpt-4o -/model claude-3-opus -``` - -### Notes - -- Model names are case-insensitive -- The model selection persists until you explicitly switch to another model -- You can also use `@model/Name` inline in any message to use a specific model -- Available models depend on the configured AI providers diff --git a/src/MeshWeaver.AI/Commands/HelpCommand.cs b/src/MeshWeaver.AI/Commands/HelpCommand.cs deleted file mode 100644 index 783c97a2c..000000000 --- a/src/MeshWeaver.AI/Commands/HelpCommand.cs +++ /dev/null @@ -1,103 +0,0 @@ -#nullable enable - -using System.Reflection; -using System.Text; - -namespace MeshWeaver.AI.Commands; - -/// -/// Command to display help information about available commands. -/// -public class HelpCommand : IChatCommand -{ - public string Name => "help"; - public string Description => "Show available commands and their usage"; - public string Usage => "/help [command]"; - public IReadOnlyList Aliases => ["?"]; - - public Task ExecuteAsync(CommandContext context, CancellationToken cancellationToken = default) - { - var registry = context.CommandRegistry; - if (registry == null) - { - return Task.FromResult(CommandResult.Error("Command registry not available.")); - } - - var sb = new StringBuilder(); - - if (context.ParsedCommand.Arguments.Length > 0) - { - // Show help for specific command - var commandName = context.ParsedCommand.Arguments[0].TrimStart('/'); - if (registry.TryGetCommand(commandName, out var command) && command != null) - { - // Try to load help from markdown file - var markdownHelp = LoadHelpMarkdown(command.Name); - if (!string.IsNullOrEmpty(markdownHelp)) - { - sb.Append(markdownHelp); - } - else - { - // Fallback to inline help - sb.AppendLine($"## /{command.Name}"); - sb.AppendLine(); - sb.AppendLine($"**Description:** {command.Description}"); - sb.AppendLine(); - sb.AppendLine($"**Usage:** `{command.Usage}`"); - - if (command.Aliases.Count > 0) - { - sb.AppendLine(); - sb.AppendLine($"**Aliases:** {string.Join(", ", command.Aliases.Select(a => "/" + a))}"); - } - } - } - else - { - return Task.FromResult(CommandResult.Error($"Unknown command: {commandName}")); - } - } - else - { - // Show all commands - sb.AppendLine("## Available Commands"); - sb.AppendLine(); - - foreach (var command in registry.GetAllCommands().OrderBy(c => c.Name)) - { - sb.AppendLine($"**/{command.Name}** - {command.Description}"); - sb.AppendLine($" Usage: `{command.Usage}`"); - sb.AppendLine(); - } - - sb.AppendLine("---"); - sb.AppendLine("**Tip:** You can also use `@agent/Name` anywhere in your message to address a specific agent."); - } - - return Task.FromResult(CommandResult.Ok(sb.ToString())); - } - - /// - /// Tries to load help content from embedded markdown file. - /// - private static string? LoadHelpMarkdown(string commandName) - { - try - { - var assembly = Assembly.GetExecutingAssembly(); - var resourceName = $"MeshWeaver.AI.Commands.Help.{commandName}.md"; - - using var stream = assembly.GetManifestResourceStream(resourceName); - if (stream == null) - return null; - - using var reader = new StreamReader(stream); - return reader.ReadToEnd(); - } - catch - { - return null; - } - } -} diff --git a/src/MeshWeaver.AI/Commands/IChatCommand.cs b/src/MeshWeaver.AI/Commands/IChatCommand.cs deleted file mode 100644 index 479942280..000000000 --- a/src/MeshWeaver.AI/Commands/IChatCommand.cs +++ /dev/null @@ -1,37 +0,0 @@ -#nullable enable - -namespace MeshWeaver.AI.Commands; - -/// -/// Represents a chat command that can be executed by the user. -/// -public interface IChatCommand -{ - /// - /// The command name (without the / prefix). Must be lowercase. - /// - string Name { get; } - - /// - /// Short description of the command for help text. - /// - string Description { get; } - - /// - /// Usage syntax for the command (e.g., "/agent @agent:Name"). - /// - string Usage { get; } - - /// - /// Optional aliases for the command. - /// - IReadOnlyList Aliases => Array.Empty(); - - /// - /// Executes the command. - /// - /// The command execution context. - /// Cancellation token. - /// The result of command execution. - Task ExecuteAsync(CommandContext context, CancellationToken cancellationToken = default); -} diff --git a/src/MeshWeaver.AI/Commands/ModelCommand.cs b/src/MeshWeaver.AI/Commands/ModelCommand.cs deleted file mode 100644 index a6f8a8e94..000000000 --- a/src/MeshWeaver.AI/Commands/ModelCommand.cs +++ /dev/null @@ -1,85 +0,0 @@ -#nullable enable - -using System.Text.RegularExpressions; - -namespace MeshWeaver.AI.Commands; - -/// -/// Command to switch the current AI model. -/// Usage: /model @model/ModelName or /model ModelName -/// -public class ModelCommand : IChatCommand -{ - public string Name => "model"; - public string Description => "Switch to a different AI model for subsequent messages"; - public string Usage => "/model @model/Name or /model Name"; - - private static readonly Regex ModelRefPattern = - new(@"@model/(.+)", RegexOptions.Compiled | RegexOptions.IgnoreCase); - - public Task ExecuteAsync(CommandContext context, CancellationToken cancellationToken = default) - { - if (context.ParsedCommand.Arguments.Length == 0) - { - // List available models grouped by provider - if (context.AvailableModels == null || !context.AvailableModels.Any()) - { - return Task.FromResult(CommandResult.Error( - $"Usage: {Usage}\n\nNo models available.")); - } - - var grouped = context.AvailableModels - .GroupBy(m => m.Provider) - .OrderBy(g => g.First().Order); - - var modelList = string.Join("\n", grouped.Select(g => - $"**{g.Key}**: {string.Join(", ", g.Select(m => m.Name))}")); - - return Task.FromResult(CommandResult.Error( - $"Usage: {Usage}\n\nAvailable models:\n{modelList}")); - } - - // Parse model name from argument - var arg = context.ParsedCommand.RawArguments; - string modelName; - - var match = ModelRefPattern.Match(arg); - if (match.Success) - { - modelName = match.Groups[1].Value; - } - else - { - // Allow just the model name without @model/ prefix - modelName = context.ParsedCommand.RawArguments.Trim(); - } - - // Find the model (case-insensitive) - if (context.AvailableModels == null || !context.AvailableModels.Any()) - { - return Task.FromResult(CommandResult.Error("No models available.")); - } - - var found = context.AvailableModels - .FirstOrDefault(m => m.Name.Equals(modelName, StringComparison.OrdinalIgnoreCase)); - - if (found == null) - { - var grouped = context.AvailableModels - .GroupBy(m => m.Provider) - .OrderBy(g => g.First().Order); - - var modelList = string.Join("\n", grouped.Select(g => - $"**{g.Key}**: {string.Join(", ", g.Select(m => m.Name))}")); - - return Task.FromResult(CommandResult.Error( - $"Model '{modelName}' not found.\n\nAvailable models:\n{modelList}")); - } - - // Switch to the model - context.SetCurrentModel?.Invoke(found); - - return Task.FromResult(CommandResult.Ok( - $"Switched to model: **{found.Name}** ({found.Provider})")); - } -} diff --git a/src/MeshWeaver.AI/Completion/AutocompleteClient.cs b/src/MeshWeaver.AI/Completion/AutocompleteClient.cs index 070d76f8d..485add6ee 100644 --- a/src/MeshWeaver.AI/Completion/AutocompleteClient.cs +++ b/src/MeshWeaver.AI/Completion/AutocompleteClient.cs @@ -1,6 +1,7 @@ #nullable enable using System.Collections.Immutable; +using System.Reactive.Linq; using MeshWeaver.Data.Completion; using MeshWeaver.Messaging; @@ -19,58 +20,54 @@ public class AutocompleteClient( /// /// Gets autocomplete suggestions by dispatching requests to all configured addresses. + /// Per-address responses merge through Observable.Merge — no per-address Task await, + /// no .ToTask() bridge on hub round-trips. /// - public async Task GetCompletionsAsync( + public IObservable GetCompletions( string query, - AgentContext? context, - CancellationToken ct = default) + AgentContext? context) { - var allItems = ImmutableList.Empty; + var addresses = GetAllDispatchAddresses(context); - // Get all addresses to query - var addresses = await GetAllDispatchAddressesAsync(context, ct); - - foreach (var address in addresses) - { - try + var perAddress = addresses + .Select(address => { - using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(ct); - timeoutCts.CancelAfter(DefaultTimeout); - var delivery = hub.Post( new AutocompleteRequest(query, context?.Context), - o => o.WithTarget(address))!; - var callbackResponse = await hub.RegisterCallback(delivery, (d, _) => Task.FromResult(d), timeoutCts.Token); + o => o.WithTarget(address)); + if (delivery == null) + return Observable.Return>(Array.Empty()); - // Tolerate hub-level failures (target unreachable, timeout as DeliveryFailure) - // and any unexpected response type — skipping is the historical behaviour. - if (callbackResponse is IMessageDelivery ok - && ok.Message?.Items != null) - { - allItems = allItems.AddRange(ok.Message.Items); - } - } - catch - { - // Skip addresses that fail to respond or timeout - } - } - - // Deduplicate by InsertText (keep highest priority item) - var deduplicated = allItems - .GroupBy(i => i.InsertText) - .Select(g => g.OrderByDescending(i => i.Priority).First()) - .ToImmutableList(); + return hub.Observe(delivery) + .Timeout(DefaultTimeout) + .FirstAsync() + .Select(d => d.Message is AutocompleteResponse { Items: { } items } + ? items + : Array.Empty()) + // Tolerate hub-level failures (target unreachable, timeout as DeliveryFailure) + // and any unexpected response type — skipping is the historical behaviour. + .Catch, Exception>( + _ => Observable.Return>(Array.Empty())); + }); - return new AutocompleteResponse(deduplicated); + return perAddress + .Merge() + .Aggregate(ImmutableList.Empty, (acc, items) => acc.AddRange(items)) + .Select(allItems => + { + // Deduplicate by InsertText (keep highest priority item) + var deduplicated = allItems + .GroupBy(i => i.InsertText) + .Select(g => g.OrderByDescending(i => i.Priority).First()) + .ToImmutableList(); + return new AutocompleteResponse(deduplicated); + }); } /// /// Gets all addresses to dispatch to: base addresses + context address. /// - private Task> GetAllDispatchAddressesAsync( - AgentContext? context, - CancellationToken ct) + private IReadOnlyCollection
    GetAllDispatchAddresses(AgentContext? context) { var addresses = ImmutableHashSet
    .Empty; @@ -86,6 +83,6 @@ private Task> GetAllDispatchAddressesAsync( addresses = addresses.Add(context.Address); } - return Task.FromResult>(addresses); + return addresses; } } diff --git a/src/MeshWeaver.AI/Completion/AutocompleteService.cs b/src/MeshWeaver.AI/Completion/AutocompleteService.cs deleted file mode 100644 index a833264a3..000000000 --- a/src/MeshWeaver.AI/Completion/AutocompleteService.cs +++ /dev/null @@ -1,131 +0,0 @@ -#nullable enable - -using System.Collections.Immutable; -using MeshWeaver.Data; -using MeshWeaver.Data.Completion; - -namespace MeshWeaver.AI.Completion; - -/// -/// Service that aggregates autocomplete results from registered providers and applies fuzzy matching. -/// This service no longer contains hard-coded completion logic - all completions are delegated to IAutocompleteProvider implementations. -/// -public class AutocompleteService( - FuzzyScorer fuzzyScorer, - IEnumerable providers) -{ - /// - /// Gets autocomplete suggestions by aggregating results from all registered providers. - /// - /// The autocomplete request. - /// Cancellation token. - /// Response containing autocomplete items with fuzzy scoring applied. - public async Task GetCompletionsAsync( - AutocompleteRequest request, - CancellationToken ct = default) - { - var results = await GetCompletionsInternalAsync(request.Query, request.Context, ct); - - // Convert AutocompleteResult to AutocompleteItem for the response - var items = results.Select(r => new AutocompleteItem( - Label: r.Label, - InsertText: r.InsertText, - Description: r.Description, - Category: r.Category, - Priority: r.Score, - Kind: r.Kind, - Icon: r.Icon, - Path: r.Path - )).ToList(); - - return new AutocompleteResponse(items); - } - - /// - /// Gets autocomplete suggestions with scoring for display. - /// - /// The search query. - /// Maximum number of results to return. - /// Cancellation token. - /// List of scored autocomplete results. - public async Task> GetCompletionsAsync( - string query, - int maxResults = 20, - CancellationToken ct = default) - { - return await GetCompletionsInternalAsync(query, null, ct, maxResults); - } - - private async Task> GetCompletionsInternalAsync( - string query, - string? contextPath, - CancellationToken ct, - int maxResults = 20) - { - var allItems = ImmutableList.Empty; - - // Collect items from all registered providers - foreach (var provider in providers) - { - try - { - await foreach (var item in provider.GetItemsAsync(query, contextPath, ct)) - { - allItems = allItems.Add(item); - } - } - catch - { - // Skip providers that fail - } - } - - // Deduplicate by InsertText (keep highest priority item) - allItems = allItems - .GroupBy(i => i.InsertText) - .Select(g => g.OrderByDescending(i => i.Priority).First()) - .ToImmutableList(); - - // Apply fuzzy scoring - var scored = fuzzyScorer.Score( - allItems, - query, - item => item.Label - ); - - // Sort by: priority (desc), then fuzzy score (desc) - var results = scored - .OrderByDescending(s => s.Item.Priority) - .ThenByDescending(s => s.Score) - .Take(maxResults) - .Select(s => new AutocompleteResult( - s.Item.Label, - s.Item.InsertText, - s.Item.Description, - s.Item.Category, - s.Score, - s.MatchPositions, - s.Item.Kind, - s.Item.Icon, - s.Item.Path - )) - .ToList(); - - return results; - } -} - -/// -/// Represents a scored autocomplete result ready for display. -/// -public record AutocompleteResult( - string Label, - string InsertText, - string? Description, - string Category, - int Score, - int[] MatchPositions, - AutocompleteKind Kind, - string? Icon = null, - string? Path = null -); diff --git a/src/MeshWeaver.AI/Completion/CommandAutocompleteProvider.cs b/src/MeshWeaver.AI/Completion/CommandAutocompleteProvider.cs deleted file mode 100644 index 650cba0dc..000000000 --- a/src/MeshWeaver.AI/Completion/CommandAutocompleteProvider.cs +++ /dev/null @@ -1,50 +0,0 @@ -#nullable enable - -using System.Runtime.CompilerServices; -using MeshWeaver.AI.Commands; -using MeshWeaver.Data.Completion; - -namespace MeshWeaver.AI.Completion; - -/// -/// Provides autocomplete items for chat commands. -/// This provider requires a ChatCommandRegistry to be set before use. -/// -public class CommandAutocompleteProvider : IAutocompleteProvider -{ - private const int CommandCategoryPriority = 2000; - - private ChatCommandRegistry? _commandRegistry; - - /// - /// Sets the command registry for autocomplete. - /// - public void SetCommandRegistry(ChatCommandRegistry registry) - { - _commandRegistry = registry; - } - - /// - public async IAsyncEnumerable GetItemsAsync( - string query, - string? contextPath = null, - [EnumeratorCancellation] CancellationToken ct = default) - { - if (_commandRegistry == null) - yield break; - - await Task.CompletedTask; // Satisfy async requirement - - foreach (var cmd in _commandRegistry.GetAllCommands()) - { - yield return new AutocompleteItem( - Label: $"/{cmd.Name}", - InsertText: $"/{cmd.Name} ", - Description: cmd.Description, - Category: "Commands", - Priority: CommandCategoryPriority, - Kind: AutocompleteKind.Command - ); - } - } -} diff --git a/src/MeshWeaver.AI/Completion/IAutocompleteStreamProvider.cs b/src/MeshWeaver.AI/Completion/IAutocompleteStreamProvider.cs new file mode 100644 index 000000000..9a53f194a --- /dev/null +++ b/src/MeshWeaver.AI/Completion/IAutocompleteStreamProvider.cs @@ -0,0 +1,50 @@ +using System.Reactive.Linq; +using MeshWeaver.Data.Completion; + +namespace MeshWeaver.AI.Completion; + +/// +/// Reactive entry point for autocomplete consumers in the same hub as the providers. +/// Returns a stream of top-N snapshots that grows as each +/// finishes producing items — fast local providers emit early, remote ones merge in later, +/// the snapshot keeps refining until everything completes. +/// +/// +/// Consumers (Blazor components, layout areas, plugins) subscribe and receive each snapshot; +/// no Task, no await, no Hub.AwaitResponse. For cross-hub autocomplete, +/// the existing / +/// message pair still applies — that handler aggregates with LastOrDefaultAsync and +/// posts the final snapshot. +/// +/// +public interface IAutocompleteStreamProvider +{ + /// + /// Subscribe to streaming autocomplete results for . Each + /// emission is a top-N snapshot ordered by + /// (higher first). The first emission is an empty snapshot (so consumers can render + /// their initial empty state). Completes when every registered provider's + /// GetItems observable has completed. + /// + IObservable> Stream(string query, string? contextPath); +} + +/// +/// Default . CombineLatest's every registered +/// snapshot stream and merges them through +/// , so the merged top-N snapshot appears as +/// soon as the first provider returns and refines as the rest arrive. +/// +public sealed class AutocompleteStreamProvider(IEnumerable providers, int topN = 50) + : IAutocompleteStreamProvider +{ + public IObservable> Stream(string query, string? contextPath) + { + return AutocompleteSnapshots + .Combine( + providers.Select(p => p.GetItems(query, contextPath) + .Catch(Observable.Return(AutocompleteSnapshots.Empty))), + topN) + .Select(snapshot => (IReadOnlyList)snapshot.ToList()); + } +} diff --git a/src/MeshWeaver.AI/Completion/ModelAutocompleteProvider.cs b/src/MeshWeaver.AI/Completion/ModelAutocompleteProvider.cs deleted file mode 100644 index b88fc5158..000000000 --- a/src/MeshWeaver.AI/Completion/ModelAutocompleteProvider.cs +++ /dev/null @@ -1,70 +0,0 @@ -#nullable enable - -using System.Runtime.CompilerServices; -using MeshWeaver.Data.Completion; - -namespace MeshWeaver.AI.Completion; - -/// -/// Provides autocomplete items for AI models. -/// Gets models from IChatClientFactory when available. -/// -public class ModelAutocompleteProvider : IAutocompleteProvider -{ - private readonly IChatClientFactory? _chatClientFactory; - private IReadOnlyList? _availableModels; - - public ModelAutocompleteProvider(IChatClientFactory chatClientFactory) - { - _chatClientFactory = chatClientFactory; - } - - public ModelAutocompleteProvider() - { - } - - /// - /// Sets the available models for autocomplete. - /// Called when models are loaded or changed. - /// - public void SetAvailableModels(IReadOnlyList models) - { - _availableModels = models; - } - - /// - public async IAsyncEnumerable GetItemsAsync( - string query, - string? contextPath = null, - [EnumeratorCancellation] CancellationToken ct = default) - { - IReadOnlyList models; - - if (_chatClientFactory != null) - { - models = _chatClientFactory.Models; - } - else if (_availableModels != null) - { - models = _availableModels; - } - else - { - yield break; - } - - await Task.CompletedTask; // Satisfy async requirement - - foreach (var model in models) - { - yield return new AutocompleteItem( - Label: $"@model/{model}", - InsertText: $"@model/{model} ", - Description: "AI Model", - Category: "Models", - Priority: 0, - Kind: AutocompleteKind.Other - ); - } - } -} diff --git a/src/MeshWeaver.AI/Completion/SkillAutocompleteProvider.cs b/src/MeshWeaver.AI/Completion/SkillAutocompleteProvider.cs new file mode 100644 index 000000000..4a4055d0f --- /dev/null +++ b/src/MeshWeaver.AI/Completion/SkillAutocompleteProvider.cs @@ -0,0 +1,84 @@ +#nullable enable + +using System.Text.Json; +using System.Reactive.Linq; +using MeshWeaver.Data; +using MeshWeaver.Data.Completion; +using MeshWeaver.Mesh; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; + +namespace MeshWeaver.AI.Completion; + +/// +/// Autocomplete for chat slash-skills, sourced from the nodeType:Skill catalog with namespace +/// inheritance ( — built-ins under the Skill namespace, +/// plus any Skill node defined in the context or the user's home and their ancestors). Replaces the +/// retired CommandAutocompleteProvider; skills are declarative nodes, so there is no C# registry. +/// +public class SkillAutocompleteProvider : IAutocompleteProvider +{ + private const int SkillCategoryPriority = 2000; + private static readonly JsonSerializerOptions EmptyJsonOptions = new(); + + private readonly IServiceProvider _serviceProvider; + + /// + public SkillAutocompleteProvider(IServiceProvider serviceProvider) + { + _serviceProvider = serviceProvider; + } + + /// + public IObservable> GetItems(string query, string? contextPath = null) + { + var workspace = _serviceProvider.GetService(); + var hub = _serviceProvider.GetService(); + if (workspace is null || hub is null) + return Observable.Return((IReadOnlyCollection)[]); + + // nodeType:Skill catalog with inheritance — cached by queryId so per-keystroke calls reuse the + // same shared subscription. Built-in skills are served live under the Skill partition; any + // Space/NodeType/user-defined skill comes from the inherited scopes — INCLUDING the chatting + // user's own {user}/Skill (derived from the hub identity), so a user's personal skills appear + // alongside the space's and the platform's. Cache key includes the user so the per-user + // subscription isn't shared across identities. + var accessService = _serviceProvider.GetService(); + var userHome = AgentPickerProjection.ResolveUserHome(accessService); + var queries = BuildQueries(accessService, contextPath); + return AgentPickerProjection.ObserveSnapshot( + workspace, hub, $"skill-autocomplete|{contextPath}|{userHome}", queries) + .Select(snapshot => BuildItems(snapshot, hub)); + } + + /// + /// The skill-autocomplete query union: the platform Skill catalog plus the current space's + /// {space}/Skill and the chatting user's {user}/Skill (derived from + /// via ), as one + /// namespace:A|B|C nodeType:Skill exact-membership query with reserved partitions filtered — + /// IDENTICAL inheritance to the agent / model registry. Extracted as a pure method so the union is + /// unit-testable without a mesh (see AgentPickerQueriesTest). Was the bug: the provider passed + /// a null userPath, so a user's OWN skills never appeared in autocomplete. + /// + internal static string[] BuildQueries(AccessService? accessService, string? contextPath) + => SkillNodeType.SkillQueries(contextPath, AgentPickerProjection.ResolveUserHome(accessService)); + + private static IReadOnlyCollection BuildItems(IEnumerable snapshot, IMessageHub? hub) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + var items = new List(); + foreach (var skill in SkillNodeType.ProjectSkills(snapshot, hub?.JsonSerializerOptions ?? EmptyJsonOptions)) + if (seen.Add(skill.Id)) + items.Add(Item(skill.Id, skill.Description)); + return items; + } + + private static AutocompleteItem Item(string name, string? description) => + new( + Label: $"/{name}", + InsertText: $"/{name} ", + Description: description ?? "", + Category: "Commands", + Priority: SkillCategoryPriority, + Kind: AutocompleteKind.Command); +} diff --git a/src/MeshWeaver.AI/ConfigMasterKeyProvider.cs b/src/MeshWeaver.AI/ConfigMasterKeyProvider.cs new file mode 100644 index 000000000..edbdcb332 --- /dev/null +++ b/src/MeshWeaver.AI/ConfigMasterKeyProvider.cs @@ -0,0 +1,53 @@ +using System.Security.Cryptography; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI; + +/// +/// Default — reads a base64 master key from +/// configuration key (Ai:KeyProtection:MasterKey). +/// For local/dev the AppHost injects it as an env var +/// (Ai__KeyProtection__MasterKey); in prod it should come from a deploy +/// secret or Key Vault and must NEVER be committed to a src/ appsettings. +/// +/// Any input length is accepted: the configured value is hashed with +/// SHA-256 to derive the 32-byte AES key, so a passphrase or a base64 of 32 +/// random bytes both work. When the key is absent/blank, returns null so +/// falls back to plaintext passthrough. +/// +/// ⚠ Rotating the configured value makes previously-stored ciphertext +/// undecryptable (different derived key) — re-save / rotate affected provider +/// keys after a master-key change. A KMS-backed +/// with versioned keys is the upgrade path for seamless rotation. +/// +public sealed class ConfigMasterKeyProvider : IMasterKeyProvider +{ + public const string ConfigKey = "Ai:KeyProtection:MasterKey"; + + private readonly byte[]? masterKey; + + public ConfigMasterKeyProvider(IServiceProvider services) + { + var logger = services.GetService()?.CreateLogger(); + var configured = services.GetService()?[ConfigKey]; + if (string.IsNullOrWhiteSpace(configured)) + { + logger?.LogInformation( + "No {ConfigKey} configured — provider-key encryption is DISABLED (keys stored as plaintext). " + + "Set a base64 master key via env/secret to enable encryption at rest.", ConfigKey); + return; + } + + // Accept any input (passphrase or base64); derive a stable 32-byte key. + var trimmed = configured.Trim(); + byte[] raw; + try { raw = Convert.FromBase64String(trimmed); } + catch (FormatException) { raw = System.Text.Encoding.UTF8.GetBytes(trimmed); } + masterKey = SHA256.HashData(raw); + logger?.LogInformation("Provider-key encryption ENABLED (master key from {ConfigKey}).", ConfigKey); + } + + public byte[]? GetMasterKey() => masterKey; +} diff --git a/src/MeshWeaver.AI/Connect/ClaudeConnectOptions.cs b/src/MeshWeaver.AI/Connect/ClaudeConnectOptions.cs new file mode 100644 index 000000000..068a0c4fe --- /dev/null +++ b/src/MeshWeaver.AI/Connect/ClaudeConnectOptions.cs @@ -0,0 +1,73 @@ +using System.Text.RegularExpressions; + +namespace MeshWeaver.AI.Connect; + +/// +/// Tunables for . Lets a test point the strategy at a committed +/// fake CLI (prints a URL → reads a code on stdin → prints a token) without touching the real +/// claude binary, and lets a deployment override the command / config-dir root. +/// +public sealed class ClaudeConnectOptions +{ + /// + /// Executable to spawn for the login. Defaults to claude (resolved on PATH). A test sets + /// this to a fake CLI; a deployment can set the absolute path to the shipped binary. + /// + public string FileName { get; set; } = "claude"; + + /// + /// Arguments to the login command. Defaults to setup-token. A test fake takes whatever it + /// needs (or none). + /// + public IReadOnlyList Arguments { get; set; } = new[] { "setup-token" }; + + /// + /// Root directory for per-user .claude config dirs (mirrors + /// ClaudeCodeConfiguration.ConfigDirRoot). The login runs with + /// CLAUDE_CONFIG_DIR = {ConfigDirRoot}/{userId}/.claude. Null ⇒ the spawn inherits the + /// container default (single-user dev) or the explicit dir passed by the caller. + /// + public string? ConfigDirRoot { get; set; } + + /// + /// Run the login command inside a pseudo-terminal. claude setup-token renders an Ink + /// (React-for-terminal) UI that emits nothing on a non-TTY pipe; with this on, the spawn is + /// wrapped as {PtyWrapper} -qfc "{FileName} {Arguments}" /dev/null (util-linux + /// script) so a real PTY is allocated, the URL/prompt become scrapeable, and the pasted + /// code is forwarded into the terminal. Linux-only; keep false for the fake-CLI test and + /// Windows dev. The co-hosted Linux portal sets this true (via ClaudeConnect:UsePseudoTerminal). + /// + public bool UsePseudoTerminal { get; set; } = false; + + /// PTY wrapper executable used when is set (util-linux script). + public string PtyWrapper { get; set; } = "script"; + + /// + /// Terminal width (columns) forced on the PTY before the CLI runs (stty cols). The Ink UI + /// line-wraps the long OAuth URL at the terminal width; a wrapped URL gets scraped truncated + /// (losing trailing params like redirect_uri). Set wide enough that the whole URL stays on + /// one line. Only used when is set. + /// + public int PtyColumns { get; set; } = 4096; + + /// + /// Regex whose first capturing group (or whole match) is the auth URL to surface, applied to + /// each stdout line. Default matches an https://… URL on a line. + /// + public string UrlPattern { get; set; } = @"(https://\S+)"; + + /// + /// Regex applied to each stdout line to extract the captured token after the code is pasted. + /// First capturing group (or whole match) is the token. Default matches a long opaque token. + /// + public string TokenPattern { get; set; } = @"(sk-ant-[A-Za-z0-9_\-]+|[A-Za-z0-9_\-]{40,})"; + + /// How long to wait for the URL line before failing the StartConnect emission. + public TimeSpan UrlTimeout { get; set; } = TimeSpan.FromSeconds(30); + + /// How long to wait for the token line after the code is pasted. + public TimeSpan TokenTimeout { get; set; } = TimeSpan.FromMinutes(2); + + internal Regex CompiledUrl() => new(UrlPattern, RegexOptions.Compiled | RegexOptions.CultureInvariant); + internal Regex CompiledToken() => new(TokenPattern, RegexOptions.Compiled | RegexOptions.CultureInvariant); +} diff --git a/src/MeshWeaver.AI/Connect/ClaudeConnectStrategy.cs b/src/MeshWeaver.AI/Connect/ClaudeConnectStrategy.cs new file mode 100644 index 000000000..ef90b0f24 --- /dev/null +++ b/src/MeshWeaver.AI/Connect/ClaudeConnectStrategy.cs @@ -0,0 +1,343 @@ +using System.Diagnostics; +using System.Reactive.Linq; +using System.Reactive.Subjects; +using System.Reactive.Threading.Tasks; +using System.Text.Json; +using System.Text.RegularExpressions; +using MeshWeaver.Mesh.Threading; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace MeshWeaver.AI.Connect; + +/// +/// Drives Claude Code's native login (claude setup-token) for the per-user Connect flow. +/// +/// Mechanism (probed 2026-06-01, claude CLI 2.1.159): claude setup-token renders +/// an Ink (React-for-terminal) UI and requires a real PTY. On a redirected (non-TTY) stdout +/// it emits zero scrapeable output and hangs until killed — so the URL line cannot be scraped +/// from a plain RedirectStandardOutput pipe, and the pasted code cannot be delivered via a +/// plain RedirectStandardInput. See the TODO(claude-pty) below. +/// +/// This strategy therefore implements the paste-code shape cleanly and configurably +/// (): it spawns the configured command under the user's +/// CLAUDE_CONFIG_DIR, scrapes the auth URL from stdout, accepts a pasted code via stdin, and +/// captures the token from stdout (or {ConfigDir}/.credentials.json). +/// the committed fake-CLI test drives the exact same shape to prove the wiring end-to-end. +/// +/// PTY (claude-pty): set to run the real +/// CLI under a pseudo-terminal — on Linux the spawn is wrapped as +/// script -qfc "claude setup-token" /dev/null (util-linux script), which allocates a +/// PTY so the Ink UI renders and its URL/prompt become scrapeable, forwards stdin into the terminal +/// for the pasted code, and is configured on for the co-hosted Linux portal via +/// ClaudeConnect:UsePseudoTerminal=true. With UsePseudoTerminal=false (the default, +/// Windows/dev/tests) the command is spawned directly and the fake CLI drives the same shape. No +/// non-interactive token-issue path exists as of CLI 2.1.159, so real-CLI E2E stays gated behind +/// CLAUDE_CONNECT_E2E=1. +/// +public sealed class ClaudeConnectStrategy : IConnectStrategy +{ + private readonly IServiceProvider services; + private readonly ILogger? logger; + // Spawning + scraping the claude CLI is a process I/O leaf — bounded Process pool, never FromAsync. + private readonly IIoPool processPool; + + public ClaudeConnectStrategy(IServiceProvider services) + { + this.services = services; + logger = services.GetService()?.CreateLogger(); + processPool = services.GetService()?.Get(IoPoolNames.Process) ?? IoPool.Unbounded; + } + + public ConnectProvider Provider => ConnectProvider.ClaudeCode; + + /// Claude uses the paste-a-code flow. + public bool RequiresPastedCode => true; + + private ClaudeConnectOptions Options => + services.GetService>()?.Value ?? new ClaudeConnectOptions(); + + // PTY output carries ANSI escape/colour sequences (the Ink UI). Strip them before scraping so the + // URL/token regexes see clean text (an escape sequence is non-whitespace and would corrupt \S+). + private static readonly Regex AnsiEscape = new(@"\x1B\[[0-9;?]*[ -/]*[@-~]", RegexOptions.Compiled); + private static string StripAnsi(string s) => AnsiEscape.Replace(s, ""); + + /// + /// Logged-in ⇔ a non-empty {userConfigDir}/.credentials.json (where the CLI persists its + /// OAuth token) exists. Cheap, file-only probe — no process spawn. When no config dir is given + /// we can't isolate per-user state, so report not-logged-in (forces an explicit Connect). + /// + public IObservable IsLoggedIn(string? userConfigDir) + { + if (string.IsNullOrEmpty(userConfigDir)) return Observable.Return(false); + return Observable.Defer(() => + { + try + { + var creds = Path.Combine(userConfigDir, ".credentials.json"); + if (File.Exists(creds) && new FileInfo(creds).Length > 2) + return Observable.Return(true); + return Observable.Return(false); + } + catch (Exception ex) + { + logger?.LogDebug(ex, "Claude IsLoggedIn probe failed for {Dir}", userConfigDir); + return Observable.Return(false); + } + }); + } + + public IObservable StartConnect(ConnectSession session, string ownerPath) + { + var options = Options; + return processPool.Invoke(ct => SpawnAndScrapeUrlAsync(session, options, ct)); + } + + public IObservable CompleteConnect(ConnectSession session, string? pastedCode) + { + var options = Options; + return processPool.Invoke(ct => SubmitCodeAndCaptureTokenAsync(session, pastedCode, options, ct)); + } + + // ── subprocess boundary (the only place Task lives — per "nothing async ever") ─────────────── + + private async Task SpawnAndScrapeUrlAsync( + ConnectSession session, ClaudeConnectOptions options, CancellationToken ct) + { + var configDir = ResolveConfigDir(session, options); + var startInfo = new ProcessStartInfo + { + RedirectStandardOutput = true, + RedirectStandardError = true, + RedirectStandardInput = true, // paste-code is written here (forwarded into the PTY when wrapped) + UseShellExecute = false, + CreateNoWindow = true, + }; + if (options.UsePseudoTerminal) + { + // claude setup-token renders an Ink (React-for-terminal) UI that needs a real TTY; on a + // plain redirected pipe it emits nothing. Run it under a pseudo-terminal via util-linux + // `script -qfc "" /dev/null`, which allocates a PTY, forwards the child's stdout to + // our pipe (so URL/token lines become scrapeable) and forwards our stdin into the PTY + // (so the pasted code reaches the CLI). Linux-only; UsePseudoTerminal stays false on + // Windows/dev and in the fake-CLI test. + var cmd = options.Arguments.Count > 0 + ? $"{options.FileName} {string.Join(" ", options.Arguments)}" + : options.FileName; + // Force a wide PTY first so the Ink UI doesn't wrap the long OAuth URL across lines — + // a wrapped URL gets scraped truncated (losing trailing params like redirect_uri). + var inner = $"stty cols {options.PtyColumns} 2>/dev/null; {cmd}"; + startInfo.FileName = options.PtyWrapper; + startInfo.ArgumentList.Add("-qfc"); + startInfo.ArgumentList.Add(inner); + startInfo.ArgumentList.Add("/dev/null"); + } + else + { + startInfo.FileName = options.FileName; + foreach (var a in options.Arguments) startInfo.ArgumentList.Add(a); + } + if (!string.IsNullOrEmpty(configDir)) + { + try { Directory.CreateDirectory(configDir); } catch { /* best effort */ } + startInfo.Environment["CLAUDE_CONFIG_DIR"] = configDir; + } + var process = new Process { StartInfo = startInfo, EnableRaisingEvents = true }; + + // Output is an OBSERVABLE line feed (ReplaySubject-backed) shared with CompleteConnect via the + // session — every stdout/stderr line is OnNext'd; process exit OnCompletes it. No SemaphoreSlim + // signal: the scrape is a reactive Where/FirstAsync over this source (per the "no hand-woven + // async gate" rule). ReplaySubject so the phase-2 token scan still sees lines emitted before it + // subscribed (the old shared-queue "no line is lost" contract). + var buffer = new OutputBuffer(); + session.ProviderClient = buffer; // reused by CompleteConnect to read the token line + process.OutputDataReceived += (_, e) => { if (e.Data != null) buffer.Add(e.Data); }; + process.ErrorDataReceived += (_, e) => { if (e.Data != null) buffer.Add(e.Data); }; + // Exit completes the feed so a waiting FirstAsync terminates (→ "exited before URL") instead of + // hanging; this is the reactive replacement for the old `if (process.HasExited) throw` check. + process.Exited += (_, _) => buffer.Complete(); + + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + session.Process = process; + + var urlRegex = options.CompiledUrl(); + + // Reactive scrape: first stripped line whose text matches the URL regex → the challenge, + // bounded by UrlTimeout, honouring the pool's CancellationToken. The `await … .ToTask(ct)` + // bridge runs INSIDE the Process IoPool worker (not the hub action block) — the one sanctioned + // async edge. A completed-without-match feed (process exited) surfaces as "no elements". + try + { + var url = await buffer.Lines + .Select(StripAnsi) + .Select(line => urlRegex.Match(line)) + .Where(m => m.Success) + .Select(m => (m.Groups.Count > 1 && m.Groups[1].Success ? m.Groups[1].Value : m.Value).Trim()) + .FirstAsync() + .Timeout(options.UrlTimeout) + .ToTask(ct) + .ConfigureAwait(false); + logger?.LogInformation("Claude Connect surfaced auth URL for session {Session}", session.SessionId); + return new ConnectChallenge(session.SessionId, ConnectProvider.ClaudeCode, url, UserCode: null, RequiresPastedCode: true); + } + catch (InvalidOperationException) + { + // FirstAsync on a completed-without-match feed → the CLI exited before emitting a URL. + throw new InvalidOperationException( + "claude setup-token exited before emitting an auth URL. On a non-TTY stdout the Ink UI emits nothing — see TODO(claude-pty)."); + } + catch (TimeoutException) + { + throw new TimeoutException( + "Timed out waiting for the Claude auth URL. The CLI needs a real terminal (PTY) — see TODO(claude-pty)."); + } + } + + private async Task SubmitCodeAndCaptureTokenAsync( + ConnectSession session, string? pastedCode, ClaudeConnectOptions options, CancellationToken ct) + { + var process = session.Process + ?? throw new InvalidOperationException("No live Claude login process; call StartConnect first."); + var buffer = session.ProviderClient as OutputBuffer + ?? throw new InvalidOperationException("Connect session is missing its output buffer."); + + if (!string.IsNullOrEmpty(pastedCode)) + { + try + { + await process.StandardInput.WriteLineAsync(pastedCode).ConfigureAwait(false); + await process.StandardInput.FlushAsync(ct).ConfigureAwait(false); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "Failed to write pasted code to claude stdin"); + } + } + + var tokenRegex = options.CompiledToken(); + + // 1) Prefer a token printed on stdout. Reactive scan of the same observable line feed — + // first stripped line matching the token regex, bounded by TokenTimeout. The `await … + // .ToTask(ct)` runs INSIDE the Process IoPool worker (the one sanctioned async edge), not + // the hub. Timeout / completed-without-match (process exited) / cancellation all fall + // through to the credentials-file fallback, exactly as the old loop did. + string? fromStdout = null; + try + { + fromStdout = await buffer.Lines + .Select(StripAnsi) + .Select(line => tokenRegex.Match(line)) + .Where(m => m.Success) + .Select(m => (m.Groups.Count > 1 && m.Groups[1].Success ? m.Groups[1].Value : m.Value).Trim()) + .FirstAsync() + .Timeout(options.TokenTimeout) + .ToTask(ct) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is TimeoutException or InvalidOperationException or OperationCanceledException) + { + // Timed out, feed completed without a match (process exited), or cancelled — fall through + // to the credentials-file fallback below. + } + + if (!string.IsNullOrEmpty(fromStdout)) + { + logger?.LogInformation("Claude Connect captured token (stdout) for session {Session}", session.SessionId); + // NOTE: do NOT write the captured token to .credentials.json — a `setup-token` token is used + // via the CLAUDE_CODE_OAUTH_TOKEN env var, not that file (which is the interactive + // `claude login` OAuth-bundle schema). Writing it there made the CLI choke and exit 1. The + // token is persisted in the ModelProvider node and re-applied to the env by the harness. + return fromStdout; + } + + // 2) Fallback: the CLI may have written the token to {ConfigDir}/.credentials.json instead. + var configDir = ResolveConfigDir(session, options); + var fromFile = TryReadCredentialsToken(configDir); + if (!string.IsNullOrEmpty(fromFile)) + { + logger?.LogInformation("Claude Connect captured token (.credentials.json) for session {Session}", session.SessionId); + return fromFile!; + } + + throw new TimeoutException("Timed out waiting for the Claude token after the code was submitted."); + } + + private string? ResolveConfigDir(ConnectSession session, ClaudeConnectOptions options) + { + if (!string.IsNullOrEmpty(session.ConfigDir)) return session.ConfigDir; + var root = options.ConfigDirRoot?.TrimEnd('/', '\\'); + var userId = session.OwnerPath; + return !string.IsNullOrEmpty(root) && !string.IsNullOrEmpty(userId) + ? Path.Combine(root, userId, ".claude") + : null; + } + + private string? TryReadCredentialsToken(string? configDir) + { + if (string.IsNullOrEmpty(configDir)) return null; + try + { + var path = Path.Combine(configDir, ".credentials.json"); + if (!File.Exists(path)) return null; + using var doc = JsonDocument.Parse(File.ReadAllText(path)); + return ExtractToken(doc.RootElement); + } + catch (Exception ex) + { + logger?.LogDebug(ex, "Could not read Claude .credentials.json under {Dir}", configDir); + return null; + } + } + + + private static string? ExtractToken(JsonElement el) + { + // The credentials file shape isn't a stable contract — walk for the first plausible + // access/oauth token property anywhere in the object graph. + foreach (var name in new[] { "accessToken", "access_token", "token", "oauthToken", "primaryApiKey" }) + { + if (el.ValueKind == JsonValueKind.Object + && el.TryGetProperty(name, out var v) + && v.ValueKind == JsonValueKind.String + && !string.IsNullOrEmpty(v.GetString())) + return v.GetString(); + } + if (el.ValueKind == JsonValueKind.Object) + { + foreach (var prop in el.EnumerateObject()) + { + var found = ExtractToken(prop.Value); + if (!string.IsNullOrEmpty(found)) return found; + } + } + return null; + } + + /// + /// An observable line feed over the process output streams. The producer (the process + /// OutputDataReceived/ErrorDataReceived events) s each line; the consumers + /// (StartConnect's URL scrape, CompleteConnect's token scrape) compose reactively over + /// — no SemaphoreSlim / no hand-woven async gate (per the "no hand-woven + /// async/concurrency primitives" rule). Backed by a so the + /// later (token) scrape still observes lines emitted before it subscribed — the old shared-queue + /// "no line is lost across the two phases" contract. (driven by process + /// exit) terminates the feed so a waiting scrape ends instead of hanging. + /// + private sealed class OutputBuffer : IDisposable + { + private readonly ReplaySubject subject = new(); + + /// The line feed: every stdout/stderr line, replayed to late subscribers. + public IObservable Lines => subject; + + public void Add(string line) => subject.OnNext(line); + + /// Signal end-of-stream (process exited) — terminates any waiting scrape. + public void Complete() => subject.OnCompleted(); + + public void Dispose() => subject.Dispose(); + } +} diff --git a/src/MeshWeaver.AI/Connect/ConnectModels.cs b/src/MeshWeaver.AI/Connect/ConnectModels.cs new file mode 100644 index 000000000..81d45ce94 --- /dev/null +++ b/src/MeshWeaver.AI/Connect/ConnectModels.cs @@ -0,0 +1,121 @@ +using System.Diagnostics; + +namespace MeshWeaver.AI.Connect; + +/// Which co-hosted CLI a Connect session authenticates. +public enum ConnectProvider +{ + ClaudeCode, + Copilot +} + +/// +/// A login challenge surfaced to the user: a URL to visit (and, for device-flow providers like +/// Copilot, a short user code), plus whether the flow then expects a code pasted back into the +/// portal (Claude Code paste-code) versus completing by polling (Copilot device-flow). +/// +public sealed record ConnectChallenge( + string SessionId, + ConnectProvider Provider, + string VerificationUrl, + string? UserCode, + bool RequiresPastedCode); + +/// State of a Connect session as it progresses. +public abstract record ConnectStatus +{ + private ConnectStatus() { } + + /// No live session — the card renders the NotConnected / Connect button branch. + public sealed record NotConnected : ConnectStatus; + + /// A login is in flight: the challenge URL (+ code) is shown. + public sealed record Connecting(ConnectChallenge Challenge) : ConnectStatus; + + /// Login completed and the token was stored as a ModelProvider. + public sealed record Connected(string ProviderNodePath, string KeyFingerprint) : ConnectStatus; + + /// The login failed / timed out / was cancelled. + public sealed record Error(string Reason) : ConnectStatus; +} + +/// +/// Mutable, per-session bag holding the live login handles between "show URL" and completion. +/// Owned by the session manager (an instance dictionary on a mesh-scoped singleton — never +/// static). Strategy-specific handles are loosely typed so a strategy in another assembly +/// (e.g. the Copilot strategy in MeshWeaver.AI.Copilot) can stash its own client. +/// +public sealed class ConnectSession : IDisposable +{ + public required string SessionId { get; init; } + public required string OwnerPath { get; init; } + public required ConnectProvider Provider { get; init; } + + /// Per-user CLI config dir (e.g. {ConfigDirRoot}/{userId}/.claude) the login runs under. + public string? ConfigDir { get; set; } + + /// Claude paste-code flow: the live claude setup-token subprocess. + public Process? Process { get; set; } + + /// Copilot device-flow: the live SDK client (typed in the Copilot assembly). + public object? ProviderClient { get; set; } + + /// The 5-minute hard-timeout subscription; disposed on completion/cancel. + public IDisposable? TimeoutSubscription { get; set; } + + public void Dispose() + { + try { TimeoutSubscription?.Dispose(); } catch { /* best effort */ } + try { if (Process is { HasExited: false }) Process.Kill(entireProcessTree: true); } catch { /* best effort */ } + try { Process?.Dispose(); } catch { /* best effort */ } + try { (ProviderClient as IDisposable)?.Dispose(); } catch { /* best effort */ } + } +} + +/// +/// Per-provider native-login driver. +/// +/// is the cheap, always-run probe each CLI card calls on render — +/// it inspects the user's CLI config dir (Claude's .credentials.json / Copilot's SDK auth +/// state) and decides whether to show the Connected state or the Connect button. +/// +/// begins the CLI's own login and emits a +/// once the URL (and any user code) is known. +/// drives it to completion — writing the pasted code to stdin +/// (Claude, true) or polling auth status (Copilot, +/// false) — and emits the captured raw token exactly once. Both return cold observables; +/// the session manager subscribes. Observable.FromAsync is used only at the subprocess / +/// SDK boundary (per the "nothing async ever" rule). +/// +public interface IConnectStrategy +{ + /// Which CLI this strategy logs in. + ConnectProvider Provider { get; } + + /// + /// True when the flow expects a code to be pasted back into the portal (Claude Code), false + /// when it completes by polling the CLI's auth status (Copilot device-flow). Drives whether the + /// inline card renders a paste field or an auto-polling device-code block. + /// + bool RequiresPastedCode { get; } + + /// + /// Cheap login-status probe for the given user CLI config dir. Cold; the card subscribes on + /// render and shows the Connected state (true) or the Connect button (false). + /// + IObservable IsLoggedIn(string? userConfigDir); + + /// + /// Start the CLI's native login under and emit the + /// (auth URL, optional device code) once known. Stashes the live + /// process / SDK client on the session so can drive it. + /// + IObservable StartConnect(ConnectSession session, string ownerPath); + + /// + /// Complete the login — paste to the process's stdin (Claude) or + /// poll the device-flow auth status (Copilot, ignored) — and emit + /// the captured raw token exactly once. + /// + IObservable CompleteConnect(ConnectSession session, string? pastedCode); +} diff --git a/src/MeshWeaver.AI/Connect/ConnectSessionManager.cs b/src/MeshWeaver.AI/Connect/ConnectSessionManager.cs new file mode 100644 index 000000000..f2d694c99 --- /dev/null +++ b/src/MeshWeaver.AI/Connect/ConnectSessionManager.cs @@ -0,0 +1,229 @@ +using System.Collections.Concurrent; +using System.Collections.Immutable; +using System.Reactive.Linq; +using System.Security.Cryptography; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI.Connect; + +/// +/// Coordinates per-user CLI login (Connect) sessions for the co-hosted Claude Code / GitHub +/// Copilot providers, driving the NotConnected → Connecting → Connected/Error state machine the +/// Settings → Models card renders. +/// +/// 🚨 Mesh-scoped singleton (registered in MemexConfiguration) holding an +/// instance keyed per +/// "{ownerPath}|{provider}" — never static. The session lifetime IS the mesh's: when +/// the hub is disposed the dictionary (and any live CLI ) +/// dies with it. A 5-minute hard timeout per session disposes the process +/// (Kill(entireProcessTree: true)). +/// +/// Reactive end-to-end: the strategies expose cold observables and this manager subscribes +/// inline; the only Task bridge is inside the strategies, at the subprocess / SDK boundary +/// (Observable.FromAsync). +/// +public sealed class ConnectSessionManager : IDisposable +{ + private static readonly TimeSpan SessionTimeout = TimeSpan.FromMinutes(5); + + private readonly IMessageHub hub; + private readonly ILogger? logger; + private readonly ImmutableDictionary strategies; + + // Live sessions keyed "{ownerPath}|{provider}". Instance field on a mesh-scoped singleton — + // bleeds across neither tests nor users (each mesh gets its own manager). + private readonly ConcurrentDictionary sessions = new(StringComparer.Ordinal); + private bool disposed; + + private sealed record Live(ConnectSession Session, IConnectStrategy Strategy) + { + public ConnectStatus Status { get; set; } = new ConnectStatus.NotConnected(); + } + + public ConnectSessionManager(IMessageHub hub, IEnumerable strategies) + { + this.hub = hub; + logger = hub.ServiceProvider.GetService()?.CreateLogger(); + // Last-registration-wins per provider; tolerates duplicate DI registrations. + var builder = ImmutableDictionary.CreateBuilder(); + foreach (var s in strategies) builder[s.Provider] = s; + this.strategies = builder.ToImmutable(); + } + + /// Whether a strategy is registered for . + public bool Supports(ConnectProvider provider) => strategies.ContainsKey(provider); + + /// True when the flow pastes a code back (Claude) vs auto-polls (Copilot). + public bool RequiresPastedCode(ConnectProvider provider) => + strategies.TryGetValue(provider, out var s) && s.RequiresPastedCode; + + /// + /// Cheap login-status probe for the card's first render — delegates to the strategy's + /// . Cold; emits false when no strategy is registered. + /// + public IObservable IsLoggedIn(ConnectProvider provider, string? userConfigDir) + { + if (!strategies.TryGetValue(provider, out var strategy)) + return Observable.Return(false); + return strategy.IsLoggedIn(userConfigDir) + .Catch(ex => + { + logger?.LogDebug(ex, "IsLoggedIn probe failed for {Provider}", provider); + return Observable.Return(false); + }); + } + + /// Current state of the (owner, provider) session, NotConnected when none is live. + public ConnectStatus GetStatus(string ownerPath, ConnectProvider provider) => + sessions.TryGetValue(Key(ownerPath, provider), out var live) + ? live.Status + : new ConnectStatus.NotConnected(); + + /// + /// Begin a login. Tears down any prior session for the same (owner, provider), spawns the CLI + /// login via the strategy, arms the 5-minute timeout, and emits the + /// challenge. For Copilot (device-flow) the manager then + /// auto-completes by polling; for Claude (paste-code) the caller must follow up with + /// . + /// + public IObservable StartConnect( + string ownerPath, ConnectProvider provider, string? userConfigDir) + { + if (string.IsNullOrEmpty(ownerPath)) + return Observable.Return(new ConnectStatus.Error("No owner identity.")); + if (!strategies.TryGetValue(provider, out var strategy)) + return Observable.Return(new ConnectStatus.Error($"No connect strategy for {provider}.")); + + CancelInternal(ownerPath, provider); + + var session = new ConnectSession + { + SessionId = Guid.NewGuid().ToString("N"), + OwnerPath = ownerPath, + Provider = provider, + ConfigDir = userConfigDir, + }; + var live = new Live(session, strategy); + sessions[Key(ownerPath, provider)] = live; + + // 5-minute hard timeout — disposes the live CLI process (Kill entire tree) + flips to Error. + session.TimeoutSubscription = Observable.Timer(SessionTimeout) + .Subscribe(_ => + { + logger?.LogInformation("Connect session timed out for {Owner}/{Provider}", ownerPath, provider); + live.Status = new ConnectStatus.Error("Timed out after 5 minutes. Please try again."); + CancelInternal(ownerPath, provider); + }); + + logger?.LogInformation("Starting Connect for {Owner}/{Provider} (configDir={ConfigDir})", + ownerPath, provider, userConfigDir ?? "(default)"); + + return strategy.StartConnect(session, ownerPath) + .Select(challenge => + { + live.Status = new ConnectStatus.Connecting(challenge); + // Copilot (device-flow) has nothing to paste — auto-poll to completion immediately. + if (!strategy.RequiresPastedCode) + CompleteInternal(ownerPath, provider, pastedCode: null); + return (ConnectStatus)live.Status; + }) + .Catch(ex => + { + logger?.LogWarning(ex, "StartConnect failed for {Owner}/{Provider}", ownerPath, provider); + live.Status = new ConnectStatus.Error(ex.Message); + CancelInternal(ownerPath, provider); + return Observable.Return(live.Status); + }); + } + + /// + /// Submit the pasted code for a Claude paste-code session, drive it to completion, and emit the + /// resulting (Connected on success, Error otherwise). + /// + public IObservable SubmitCode(string ownerPath, ConnectProvider provider, string pastedCode) + => CompleteInternal(ownerPath, provider, pastedCode); + + /// Cancel / disconnect a live session (Disconnect button, or auth-error reset). + public void Cancel(string ownerPath, ConnectProvider provider) => CancelInternal(ownerPath, provider); + + private IObservable CompleteInternal(string ownerPath, ConnectProvider provider, string? pastedCode) + { + if (!sessions.TryGetValue(Key(ownerPath, provider), out var live)) + return Observable.Return(new ConnectStatus.Error("No active connect session.")); + + var sink = hub.ServiceProvider.GetService(); + if (sink is null) + return Observable.Return(new ConnectStatus.Error("No token sink registered.")); + + var providerName = ProviderName(provider); + + var pipeline = live.Strategy.CompleteConnect(live.Session, pastedCode) + .SelectMany(token => + { + if (string.IsNullOrEmpty(token)) + return Observable.Return(new ConnectStatus.Error("No token captured from the CLI.")); + return sink.StoreToken(ownerPath, providerName, token) + .Select(stored => (ConnectStatus)new ConnectStatus.Connected(stored.ProviderNodePath, stored.KeyFingerprint)); + }) + .Do(status => + { + live.Status = status; + // Success or terminal — tear down the live process + timeout. + CancelInternal(ownerPath, provider); + }) + .Catch(ex => + { + logger?.LogWarning(ex, "CompleteConnect failed for {Owner}/{Provider}", ownerPath, provider); + live.Status = new ConnectStatus.Error(ex.Message); + CancelInternal(ownerPath, provider); + return Observable.Return(live.Status); + }) + .Replay(1); + + // For device-flow (auto-poll on StartConnect) we drive the pipeline ourselves so the card + // just observes GetStatus. Connected/Publish so the optional caller can also subscribe. + var connectable = pipeline; + connectable.Connect(); + return connectable; + } + + private void CancelInternal(string ownerPath, ConnectProvider provider) + { + if (sessions.TryRemove(Key(ownerPath, provider), out var live)) + { + // Keep the last Status (Connected/Error) discoverable for one render via the returned + // observable; the session bag itself is disposed (process killed, timeout cancelled). + try { live.Session.Dispose(); } catch { /* best effort */ } + } + } + + private static string Key(string ownerPath, ConnectProvider provider) => $"{ownerPath}|{provider}"; + + private static string ProviderName(ConnectProvider provider) => provider switch + { + ConnectProvider.ClaudeCode => "ClaudeCode", + ConnectProvider.Copilot => "Copilot", + _ => provider.ToString(), + }; + + /// 8-char SHA-256-hex prefix — never the raw token. + public static string Fingerprint(string? value) + { + if (string.IsNullOrEmpty(value)) return "(empty)"; + var hash = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(value)); + return Convert.ToHexString(hash, 0, 4).ToLowerInvariant(); + } + + public void Dispose() + { + if (disposed) return; + disposed = true; + foreach (var live in sessions.Values) + { + try { live.Session.Dispose(); } catch { /* best effort */ } + } + sessions.Clear(); + } +} diff --git a/src/MeshWeaver.AI/Connect/IConnectTokenSink.cs b/src/MeshWeaver.AI/Connect/IConnectTokenSink.cs new file mode 100644 index 000000000..7da2f8f4d --- /dev/null +++ b/src/MeshWeaver.AI/Connect/IConnectTokenSink.cs @@ -0,0 +1,21 @@ +namespace MeshWeaver.AI.Connect; + +/// +/// The seam by which a completed CLI login persists its captured token. The +/// (in MeshWeaver.AI) hands the raw token here once a +/// strategy captures it; the portal's implementation stores it as an encrypted +/// ModelProvider node (create-or-rotate) via ModelProviderService — so the AI layer +/// never references the portal assembly. Reactive end-to-end (no Task). +/// +public interface IConnectTokenSink +{ + /// + /// Persist the captured CLI subscription token for under provider + /// ("ClaudeCode" / "Copilot"). Creates the + /// ModelProvider node when absent, rotates the key when it already exists. The token is + /// encrypted at rest by the implementation. Emits the persisted provider node path + an 8-char + /// key fingerprint exactly once. + /// + IObservable<(string ProviderNodePath, string KeyFingerprint)> StoreToken( + string ownerPath, string providerName, string token); +} diff --git a/src/MeshWeaver.AI/Connect/IMcpBackConnection.cs b/src/MeshWeaver.AI/Connect/IMcpBackConnection.cs new file mode 100644 index 000000000..a421221ce --- /dev/null +++ b/src/MeshWeaver.AI/Connect/IMcpBackConnection.cs @@ -0,0 +1,31 @@ +namespace MeshWeaver.AI.Connect; + +/// +/// The coordinates a co-hosted CLI needs to call the portal's MCP endpoint AS THE USER: +/// the fully-composed {baseUrl}/mcp URL and the Bearer token to present. +/// +public sealed record McpConnectionInfo(string McpUrl, string BearerToken); + +/// +/// Automatically provisions the per-user MCP back-connection used by the co-hosted Claude Code / +/// GitHub Copilot CLIs so the mesh is their workspace. Creating the token + wiring is automatic — +/// there is NO manual step: the co-hosted chat clients call at spawn +/// time, every execution, and inject the result as a per-spawn HTTP MCP server with a +/// Authorization: Bearer header (the token-based pattern for internal comms). +/// +/// The implementation lives in the portal (mints/reuses a MeshWeaver ApiToken via +/// ApiTokenService, stores it encrypted so it survives across replicas, and resolves the +/// portal's own base URL). It is consumed here in the AI layer through this interface so the +/// chat clients never reference the portal assembly. +/// +public interface IMcpBackConnection +{ + /// + /// Ensure a usable MCP Bearer token exists for — mint one if missing + /// or invalid, otherwise reuse the stored one — and return the + /// to inject for this spawn. Idempotent and cheap on the hot path (reuse is a decrypt, not a + /// mint). Returns null when no back-connection can be established (e.g. the portal base + /// URL is unknown); the CLI then runs without mesh access rather than failing the chat. + /// + IObservable EnsureForUser(string userId, string? userName = null, string? userEmail = null); +} diff --git a/src/MeshWeaver.AI/Data/Agent/Assistant.md b/src/MeshWeaver.AI/Data/Agent/Assistant.md new file mode 100644 index 000000000..2d898cd48 --- /dev/null +++ b/src/MeshWeaver.AI/Data/Agent/Assistant.md @@ -0,0 +1,102 @@ +--- +nodeType: Agent +name: Assistant +description: The main agent — owns the conversation's red line from first message to last. Has all tools. Does work directly. Delegates only when a specialist is clearly better, or to keep heavy work out of the main context window. +icon: +category: Agents +isDefault: true +exposedInNavigator: false +order: -1 +delegations: + - agentPath: Agent/Researcher + instructions: "Deep information gathering: web search, mesh exploration across many nodes, documentation lookup, data analysis. Use when the investigation would otherwise bloat your main context window." + - agentPath: Agent/Coder + instructions: "Authoring or modifying NodeTypes — source files, data models, layout areas, CSV loaders, JSON definitions. The Coder owns the architecture rules + LSP pre-flight + compile/diagnostics loop." + - agentPath: Agent/Worker + instructions: "Mechanical bulk writes you want kept out of the main context (e.g. create 5 child nodes in parallel, or a long iterative patch loop). For one-off small writes, do them yourself." +plugins: + - Mesh + - Version + - WebSearch + - Collaboration + - ContentCollection +--- + +You are **Assistant**, the main agent. You own the conversation's red line — the same context window persists from the first user message to the last reply in the thread. The user steers; you keep state across turns. + +**You have all the tools.** Do the work directly. Don't reach for delegation as a default — every delegated sub-thread is extra coordination overhead and your reply will be slower. Delegate only when the rules below clearly say to. + +# Two questions before each action + +1. **Have I understood the situation?** Read the relevant nodes, scan the surrounding namespace, inspect referenced documents *before* you act. A wrong tool call on the first signal costs more turns than a careful read. +2. **What does "done" look like?** Articulate completion criteria (to yourself, and in your reply when non-trivial) before any write: which nodes will exist, with what content, how the user verifies. If you can't state the criteria, **ask the user for the final goal** before doing anything. + +If you didn't call a tool, you didn't do the thing. Never describe a write you would have made. + +# When to delegate (and when not to) + +Delegation is **opt-in, not default**. Reach for it in exactly two cases: + +1. **A specialist is clearly better at this.** Examples: + - The task is creating or editing a **NodeType** (source files, data models, layout areas, compile diagnostics) → **Coder** owns the architecture rules + LSP pre-flight loop. Don't try to hand-write `Source/*.cs` from this prompt. + - The task is **deep cross-mesh investigation** or **multi-source web research** → **Researcher** can run many `Search` / `Get` / `SearchWeb` calls without polluting your context. + - A **local agent** (per-user or per-org custom agent visible in your `hierarchyAgents`) was built for exactly this domain. +2. **You want the work out of your main context window.** Long iterative patch loops, bulk creation of many child nodes, exhaustive search-and-replace passes — anything where the *intermediate* reads/writes would bloat the conversation but the *summary* is all the user needs. Delegate, get the summary back, relay it. **Worker** is the right target for mechanical bulk writes. + +For everything else — small reads, single writes, planning, summarising, navigating, answering a question, drafting one Markdown node, patching one field — **do it yourself**. The main thread is your conversation; don't fragment it without a reason. + +## When delegating + +- **Pass a clear, specific task description.** The sub-thread's id and title are derived from this text (it becomes a URL-friendly slug + the human-readable title). A vague task like "do the thing" produces a sub-thread called `do-the-thing-a3f9` that nobody can locate later. Write the task as you'd write a Jira summary: noun + verb + scope. Good: `"Create a Markdown node at OrgA/Process with the onboarding checklist; include the 5 steps from /Doc/Onboarding"`. Bad: `"help with onboarding"`. +- **Make the task self-contained.** The sub-agent starts with an empty context — it has not seen this conversation. Include the concrete paths, constraints, and acceptance criteria it needs; never write "as discussed above". +- **Set `context` explicitly** when the work lives somewhere other than your current node (parallel work on multiple docs, work on a sibling). +- **Independent tasks can run in parallel** — dispatch several delegations rather than serializing them. Check on running sub-threads with `list_sub_threads`; push a correction into one with `send_to_sub_thread` instead of cancelling and re-dispatching. +- After a delegation returns, **summarise its result in one or two sentences** for the user. Don't echo the whole sub-thread output — it's already visible inline. + +## Planning + +There is no separate Planner agent. You plan. For a multi-step task, write the plan as a brief checklist in your reply (or use `store_plan` if it's long-lived enough to need persistence). Then execute, one step at a time, marking progress. + +# Stay listening + +The user can type follow-ups while you work. Those messages queue until you call **`check_inbox`** (no arguments) — each call returns the queued text(s), or `(no new messages)`. + +**When to call:** between steps of multi-step work — after a tool call completes, before starting a new write, before dispatching a delegation. **When not to:** during a single fast read, or immediately after an empty `check_inbox` in the same response. + +**When new input arrives:** fold it in if compatible (`"also include X"` → add X). If it changes direction (`"stop, do Y instead"`), acknowledge in one sentence and pivot. A returned message is permanently delivered — fold it in now; it won't be re-delivered later. + +# Paths, links, and node creation + +The complete rules — `@` path resolution, query syntax, MeshNode schemas, icon requirements — are in the Tools Reference below. The three you use in every reply: + +- Mesh links in markdown output: **absolute** `[text](@/Full/Path)` — never bare names, never `@/` inside raw HTML `href` (write `` there). +- Tool calls take the node's `path`, never its display name. +- Before creating nodes, explore what exists (`Search('namespace:{contextPath}')`) and create in the current context's namespace — never under `Agent/` or other system namespaces unless explicitly asked. + +# Version history + +You have the Version tools directly (`GetVersions`, `GetVersion`, `RestoreVersion`, `RestoreFromPointInTime`) — no delegation needed: + +- **List versions first** before restoring, and **confirm with the user** which version you'll restore and what will change — a restore overwrites the current state (though it creates a new version, never deletes history). +- "Revert to yesterday" → `GetVersions` to confirm history exists, then `RestoreFromPointInTime` with yesterday's date. +- "What changed in version 5?" → `GetVersion(path, 5)` and `GetVersion(path, 4)`, describe the difference. + +# Tools Reference + +@@Agent/ToolsReference + +# Notification preferences + +The in-app bell is always on; whether a notification *also* escalates to email (or, later, Teams) is decided by a triage agent from the user's own rules. Channels live at `{user}/_NotificationChannel/{id}` (`kind`: `InApp`/`Email`/`Teams`, optional `target`, `enabled`); rules at `{user}/_NotificationRule/{id}` (plain-English `ruleText`, optional `channel`, `enabled`, `order`). With **no** rules the user gets in-app only — enabling email means adding **both** an email channel and a rule. + +When the user asks *"email me when…"*, *"stop notifying me about…"*, or *"what are my notification settings?"* — read their current channels/rules with `Search`/`Get`, explain them plainly, `Create`/`Update` the nodes to match, confirm, and point them at the manual: **[Managing your notification preferences](@/Doc/GUI/NotificationPreferences)**. + +# Guidelines + +- **ALWAYS call tools** — never say "I'll navigate to X" without actually calling `NavigateTo('@X')`. +- When the user mentions a path with `@`, call `NavigateTo` on it immediately. +- When the user says "show me", "take me to", "display", "open" → call `NavigateTo`. +- When the user says "find", "search", "list", "what's under" → call `Search`. +- When the user asks for a simple change/edit/update/create/delete → do it yourself. +- When the user asks for complex multi-step work → understand the situation yourself (read, search), articulate completion criteria (ask the user if unclear), then choose: do it yourself, or delegate per the rules above. +- Keep text minimal. A brief confirmation after the tool call beats a paragraph before it. diff --git a/src/MeshWeaver.AI/Data/Agent/Coder.md b/src/MeshWeaver.AI/Data/Agent/Coder.md index 647ad458e..1e2e02b9f 100644 --- a/src/MeshWeaver.AI/Data/Agent/Coder.md +++ b/src/MeshWeaver.AI/Data/Agent/Coder.md @@ -2,23 +2,169 @@ nodeType: Agent name: Coder description: Creates and modifies node types, source code, data models, layout areas, and CSV data loaders -icon: Code +icon: category: Agents exposedInNavigator: true -modelTier: heavy plugins: - Mesh - ContentCollection + - Lsp delegations: - agentPath: Agent/Researcher instructions: "Research existing patterns, schemas, or code before creating new types" --- -You are **Coder**, the node type engineering agent. You create and modify custom NodeTypes including their source code (`_Source/`), data models, layout areas, reference data, CSV loaders, and JSON definitions. +You are **Coder**, the node type engineering agent. You create and modify custom NodeTypes including their source code (`Source/`), data models, layout areas, reference data, CSV loaders, and JSON definitions. + +# 🚨 Read these architecture docs FIRST (non-negotiable) + +Before you write any handler, layout area, click action, service method, or Blazor view, you must internalise four documents. Almost every recent deadlock and stale-content incident traces back to violating one of them. + +1. **[Asynchronous Calls](@/Doc/Architecture/AsynchronousCalls)** — *the* hub-handler / service-code rule book. The headline rule: **no `Task` / `async` / `await` in mesh-reachable code.** Public methods on services, handlers, layout areas, and click actions return `IObservable` (or `void`). Compose with `SelectMany` / `Select` / `Where`. Request/response uses `hub.Observe(request).Subscribe(onNext, onError)` — NOT `RegisterCallback` (`[Obsolete]`, silently swallows DeliveryFailure) and NOT `AwaitResponse` (`[Obsolete]`, deadlocks via Task await). NEVER `Observable.FromAsync(() => hub.RegisterCallback(...))` — that pattern bridges Tasks back into Rx and deadlocks via captured sync-context. Click actions must be sync (`ctx => { ...; return Task.CompletedTask; }`), never `async ctx => await ...`. **Tests follow the same rule** — they are `void` and assert on observables directly via `MeshWeaver.Reactive.Assertions` (`x.Should().Match(...)`), with no `await` in the test body; see [Reactive Test Assertions](@/Doc/Architecture/ReactiveTestAssertions). +2. **[CQRS — Queries vs. Content Access](@/Doc/Architecture/CqrsAndContentAccess)** — **never** use `meshQuery.QueryAsync($"path:{X}").FirstOrDefaultAsync()` (or any `Observable.FromAsync` wrapper around it) to read a known node. Queries go through a lagged read-side index and return stale content right after a write. For a known path: live = `workspace.GetMeshNodeStream(path)` (own/local/remote auto-dispatch — never `GetRemoteStream` for a node by path); one-shot = `hub.GetMeshNode(path, timeout?)`. `QueryAsync` / `Query` is for **sets and existence**, not single-node content reads. In tests, use the reactive `ReadNode(path)` helper on the test base (`ReadNode(path).Should().Emit()`). +3. **[Data Binding](@/Doc/GUI/DataBinding)** — **the GUI is fully data-bound, with ONE source of truth: the node stream.** Backend layout areas declare *what* to render and pass paths into controls; they never load instances and never put concrete values into controls. To EDIT a node's content, bind the GUI client DIRECTLY to the node via `Hub.GetMeshNodeStream(path)` (the `IMeshNodeStreamCache`) for reads, and write edits straight back through `GetMeshNodeStream(path).Update(current => ...)`. 🚨 **NEVER replicate the node into a layout-area `/data/{id}` copy and reconcile it with a server-side save subscription** — that "replicate-then-save" shape (`host.UpdateData(id, node.Content)` + `GetDataStream(id).Debounce().Subscribe(...Update...)`, a.k.a. `OverviewLayoutArea.SetupAutoSave` / any `*AutoSave` helper / a "Save" button that reads `/data` and writes the node) is the FORBIDDEN antipattern: two stores drift and the save loop clobbers unedited fields. For a simple scalar/bool editor just declare `MeshNodeContentEditorControl.ForType(path, typeof(MyContent))` (the GUI view binds to the node stream and persists per-field) — never reach for `GetRemoteStream` for a node by path. For rich content use the already-node-bound controls (`MarkdownEditorControl.WithAutoSave`, `MeshNodePickerControl`, `CollaborativeMarkdownView`). Backend rendering stays purely synchronous, side-effect-free, and never deadlocks because there's no `await` to deadlock on. +4. **[Activity Control Plane](@/Doc/Architecture/ActivityControlPlane)** — **every operation on a stateful node is a property patch on the node's content, not a separate message type.** When you build a NodeType with state-machine semantics (long-running job, transitional resource, anything start/pause/resume/retry/cancellable), pair `Status` (current actual state, written only by the owning hub) with `RequestedStatus` (control surface, patched by callers via `workspace.GetMeshNodeStream(path).Update(...)`). The hub's `WithInitialization` subscribes to its own `MeshNodeReference` stream, watches `RequestedStatus` with `DistinctUntilChanged()`, and reacts. **Do not invent `CancelXRequest` / `PauseXRequest` / `RetryXRequest` message types** — they bypass this pattern. Internal hub-to-hub plumbing may still use messages; the *external* surface is content. The kernel's cancel flow is the canonical example: `hub.CancelActivity(activityPath)` — which writes `RequestedStatus = ActivityStatus.Cancelled` onto the activity node via `workspace.GetMeshNodeStream(activityPath).Update(...)`. + +These rules apply just as strictly to test code: a NodeType test that does `await meshQuery.QueryAsync($"path:{X}").FirstOrDefaultAsync()` after a write is testing stale content and will be flaky in CI. Use `ReadNode(path).Should().Emit()` on the test base (authoritative, reactive — there is no async `ReadNodeAsync` any more) — see [Writing Tests](@/Doc/Architecture/WritingTests) and the [Reactive Test Assertions](@/Doc/Architecture/ReactiveTestAssertions) playbook for the full testing guide. + +**Tests are reactive role models — no `await` in the test body.** The platform runs reactive end-to-end, so its tests do too: assert on the stream directly instead of bridging to a `Task`. Replace `await x.FirstAsync().ToTask(ct)` with `x.Should().Match(predicate)` / `.Be(expected)` / `.Emit()`, and drive reactive creates/updates from the assertion's subscribe (`CreateNode(node).Should().Emit()`) — the `[Fact]` becomes a plain `void`. Folding the assertion into the predicate (`.Match(items => items.Count == 2)`) waits for the *right* state, removing the "wait, then assert" race that flakes in CI. Where production runs work on an activity hub, drive it through `RequestedStatus` and observe the activity stream — test the same path production takes. Full pattern + API: [Reactive Test Assertions](@/Doc/Architecture/ReactiveTestAssertions). + +## Script (executable Code node) — the same three rules apply + +You also write **Scripts**: `Code` MeshNodes flagged `isExecutable: true`, executed via the MCP `ExecuteScript` tool (full guide: [ExecuteScript](@/Doc/AI/ExecuteScript)). Inside a Script, the kernel exposes `Mesh` — the portal's `IMessageHub` — and the top-level C# is compiled and run by `Microsoft.DotNet.Interactive`. The Script runs on the kernel's own execution hub, *not* a message-handler pump, so `await` **is** allowed at the top level. But the mesh reads and writes you do have to follow the same CQRS / reactive rules as production code, or you'll either write stale assertions or deadlock the kernel. + +### Where to put Scripts + +Organize Scripts as **child Code nodes under the feature they serve**, not as top-level nodes. A namespace like `MyDomain/Feature/Script/ImportMonthly` keeps the Script co-located with the NodeType / data it operates on, shows up under the feature's overview in the portal, and inherits the feature's access context. + +```jsonc +// MyDomain/Feature/Script/ImportMonthly.json — Script lives under the feature +{ + "id": "ImportMonthly", + "namespace": "MyDomain/Feature/Script", + "name": "Import Monthly Data", + "nodeType": "Code", + "content": { + "code": "// script body — see template above", + "language": "csharp", + "isExecutable": true + } +} +``` + +### Verify the Script is actually executable + +After creating or editing a Script, **don't just ship it** — run it through MCP `ExecuteScript` to prove it compiles and executes cleanly: + +```jsonc +{ + "name": "ExecuteScript", + "arguments": { + "path": "@MyDomain/Feature/Script/ImportMonthly", + "timeoutSeconds": 60 + } +} +``` + +Watch for: +- `status: "Executed"` and a non-error `message` → the kernel compiled and ran the code. +- `status: "Error"` → kernel exception; the `error` field carries the C# compiler/runtime error. Fix, re-run. +- `status: "Timeout"` → the script exceeded `timeoutSeconds`; side effects may have partially applied. Re-query the mesh to understand state before re-running. + +A Script you ship without at least one `status: "Executed"` run is a Script you haven't actually tested. Treat the happy-path run as part of the acceptance criteria for the PR. + +### Scripts execute in a hosted hub — that's what makes `await` safe + +A Script runs in a **hosted hub** (the kernel's `_Exec` hub) with its own `ActionBlock`, not on the parent hub's pump. That isolation is what makes `await` safe inside a script: the script blocks its own hub's pump, but responses to its requests route back via *other* hubs (mesh, per-node, the parent portal hub) — different pumps, no deadlock. This is the same reason `parentHub.Post(...)` from inside `ExecuteMessageAsync` is safe (see [Asynchronous Calls — Blocking Execution](@/Doc/Architecture/AsynchronousCalls)). + +If you ever find yourself writing code that's *not* in a hosted hub (rare — only happens if you're embedding compilation directly in a handler), you must drop back to the canonical reactive shape from [Asynchronous Calls](@/Doc/Architecture/AsynchronousCalls): compose `IObservable` and `.Subscribe(onNext, onError)` — never `await`, and never `TaskCompletionSource` (it is on the forbidden list for hub-reachable code). + +One caveat to the safety rule: `await` is safe in a Script only because nothing the script awaits needs the pump the script is blocking. Don't post requests targeted at the script's own exec hub and await the response — that is the one self-inflicted deadlock the hosted-hub isolation cannot save you from. + +### ✅ Script boilerplate — reads + writes done right + +```csharp +#r "nuget:System.Reactive" +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using MeshWeaver.Data; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; + +var meshService = Mesh.ServiceProvider.GetRequiredService(); +var workspace = Mesh.GetWorkspace(); + +// ✅ Write — Observable-returning service completes when the per-node hub +// posts back. Awaiting is safe here because the script runs in the hosted +// exec hub, not the pump that processes the response. +var created = await meshService.CreateNode(new MeshNode("Import001", "Acme/Imports") +{ + Name = "Monthly import", + NodeType = "Markdown", +}).FirstAsync(); + +// ✅ Read after write — one-shot via hub.GetMeshNode(path). Internally posts +// GetDataRequest(MeshNodeReference) — request/response, NOT a SubscribeRequest +// that has to be torn down. No catalog/index lag, no stale content. +var reread = await Mesh.GetMeshNode(created.Path!, TimeSpan.FromSeconds(15)).ToTask(); +Console.WriteLine($"Re-read at {reread!.Path}, name={reread.Name}"); + +// ✅ Update — same reactive shape, single await at the script edge. +var renamed = reread with { Name = "Monthly import (Q1)" }; +await meshService.UpdateNode(renamed).FirstAsync(); + +// ✅ Wait for a state change — subscribe to the LIVE per-node stream with a +// predicate, take 1, await. (Different from the one-shot read above — +// here you genuinely need a subscription because you're waiting for the +// state to flip *over time*. GetMeshNodeStream rides the process-wide +// shared handle — never GetRemoteStream for a node by path.) +var completed = await workspace + .GetMeshNodeStream("Acme/Jobs/MigrateV2") + .Where(n => n is { State: MeshNodeState.Active }) + .Take(1) + .Timeout(TimeSpan.FromMinutes(2)) + .ToTask(); +Console.WriteLine($"Job Active: {completed.Path}"); +``` + +### ❌ Script anti-patterns — stale data, polling, callback misuse + +```csharp +// ❌ Lagged index read right after a write — classic CQRS violation. +// The read-side index hasn't indexed the create yet; this returns null +// or stale content on the first call and "works" on the second — flaky. +await mesh.CreateNode(node).FirstAsync(); +var stale = await mesh.QueryAsync($"path:{node.Path}").FirstOrDefaultAsync(); + +// ❌ QueryAsync wrapped in Observable.FromAsync to "look reactive" — same bug. +var obs = Observable.FromAsync(ct => + mesh.QueryAsync($"path:{p}").FirstOrDefaultAsync(ct).AsTask()); + +// ❌ Polling loop for a state change — lagged every iteration, wastes minutes. +for (var i = 0; i < 60; i++) +{ + var n = await mesh.QueryAsync($"path:{p}").FirstOrDefaultAsync(); + if (n?.State == MeshNodeState.Active) break; + await Task.Delay(1000); +} + +// ❌ Awaiting inside a Subscribe callback — subscribe runs on an arbitrary +// thread; awaits inside it race kernel teardown and frequently hang. +stream.Subscribe(async node => { await mesh.UpdateNode(node with { ... }); }); + +// ❌ Reaching for test-base helpers (ReadNode and friends) from a Script — +// those live on the test base classes, not on the hub. Scripts read via +// Mesh.GetMeshNode(path, timeout) or workspace.GetMeshNodeStream(path). +var node = await Mesh.ReadNodeAsync(path); // ← doesn't exist; don't invent it +``` + +**Rule of thumb for Scripts:** read known paths one-shot via `Mesh.GetMeshNode(path, timeout).ToTask()`; wait for state changes via `workspace.GetMeshNodeStream(path).Where(predicate).Take(1).Timeout(...)`. Use `mesh.QueryAsync(...)` / `mesh.Query(...)` only for searching / listing / counting (sets, not specific node content). Reach for `QueryAsync(path:X)` and you've written a stale-read bug. # Decision Rule: NodeType vs Markdown -When the user describes a **data model, object type, custom entity, or interactive view** — e.g. "social media posts with a calendar", "a task tracker", "risk model with charts", "build X as code" — you build a **NodeType**: a `NodeType` JSON + `_Source/` C# files + at least one instance JSON. +When the user describes a **data model, object type, custom entity, or interactive view** — e.g. "social media posts with a calendar", "a task tracker", "risk model with charts", "build X as code" — you build a **NodeType**: a `NodeType` JSON + `Source/` C# files + at least one instance JSON. You build a **Markdown** node ONLY when the user explicitly asks for a document, note, article, or narrative page (e.g. "write a doc about X", "draft a changelog", "add an FAQ page"). @@ -26,10 +172,10 @@ You build a **Markdown** node ONLY when the user explicitly asks for a document, ## Canonical Example -The walkthrough at [SocialMedia model node type](@@Doc/DataMesh/SocialMedia) is the reference implementation. It has exactly the shape you should produce: +The walkthrough at [SocialMedia model node type](@/Doc/DataMesh/SocialMedia) is the reference implementation. It has exactly the shape you should produce: - `Post.json`, `Profile.json` — NodeType definitions with a `configuration` lambda -- `Post/_Source/*.cs`, `Profile/_Source/*.cs` — content record, reference data (`Platform`), layout areas +- `Post/Source/*.cs`, `Profile/Source/*.cs` — content record, reference data (`Platform`), layout areas - `Post/Post-001.json`, `Profile/Roland-LinkedIn.json` — instances alongside (IDs are meaningful — never `SamplePost`/`SampleProfile`) When asked to build "X as code" or "X as a model", open that example, mirror its shape, then adapt to the user's domain. @@ -38,24 +184,24 @@ When asked to build "X as code" or "X as a model", open that example, mirror its A NodeType is a MeshNode with `nodeType: "NodeType"` whose `content` contains a `NodeTypeDefinition` with a `configuration` field. The configuration is a C# lambda expression compiled at startup. -## Folder Structure +## Namespace Structure ``` {Namespace}/ MyType.json # NodeType definition (nodeType: "NodeType") MyType/ - _Source/ # C# files compiled at startup + Source/ # C# files compiled at startup MyType.cs # Content record type Status.cs # Reference data (optional) DataLoader.cs # CSV loader (optional) MyTypeLayoutAreas.cs # Custom views (optional) - _Test/ # xUnit tests (optional) - MyTypeTests.cs + Test/ # C# test files — REQUIRED for every NodeType + MyTypeTest.cs ``` ## Source Code Frontmatter -Every `.cs` file in `_Source/` MUST start with the meshweaver frontmatter: +Every `.cs` file in `Source/` MUST start with the meshweaver frontmatter: ```csharp // @@ -99,8 +245,7 @@ public record Project - `[Required]` — Validation - `[MeshNodeProperty(nameof(MeshNode.Name))]` — Maps to MeshNode.Name - `[MeshNodeProperty(nameof(MeshNode.Icon))]` — Maps to MeshNode.Icon -- `[Dimension]` — References a lookup type -- `[Dimension(typeof(Supplier))]` — Alternative dimension syntax (for int keys) +- `[MeshNode("nodeType:ACME/Category")]` — References another mesh node. The editor renders a `MeshNodePickerControl`; the property is a `string` storing the referenced node's PATH. The query argument always uses the **full path of the referenced NodeType** (`nodeType:ACME/Category`, never a bare `Category`). Multiple queries merge; `{node.namespace}` / `{node.Property}` template variables resolve at edit time; options: `Layout = MeshNodePickerLayout.Thin`, `Open = MeshNodePickerOpenDirection.Up`, `DefaultToFirst = true`. This is the canonical cross-node reference — see the balance-sheet facts in [Data Cubes](@/Doc/DataMesh/DataCubes). - `[Markdown(EditorHeight = "200px")]` — Rich text field - `[UiControl(Style = "width: 200px;")]` — Form layout control - `[Browsable(false)]` — Hidden from UI @@ -139,7 +284,11 @@ public record Status ## CSV Data Loader Pattern -For types that load from CSV files: +For types that load from CSV files. **Never `async`/`await`/`Task`/`Task.FromResult` +in a loader** — the loader returns `IObservable>` (the shape +`WithInitialData` takes) and runs the blocking file read + parse on the bounded +FileSystem I/O pool via `InvokeBlocking`, so it never executes on the configuring +hub's thread (canonical live example: `NorthwindDataLoader`): ```csharp // @@ -148,21 +297,32 @@ For types that load from CSV files: // using System.Globalization; +using MeshWeaver.Messaging; +using MeshWeaver.Mesh.Threading; +using Microsoft.Extensions.DependencyInjection; public static class DataLoader { private static readonly string BasePath = Path.Combine("../../samples/Graph/attachments/MyNamespace/Data"); - public static Task> LoadProductsAsync(CancellationToken ct) - { - var lines = File.ReadAllLines(Path.Combine(BasePath, "products.csv")); - return Task.FromResult(ParseCsv(lines, parts => new Product - { - ProductId = int.Parse(parts[0]), - ProductName = parts[1], - UnitPrice = double.Parse(parts[4], CultureInfo.InvariantCulture), - })); - } + private static IIoPool FileSystemPool(IMessageHub hub) => + hub.ServiceProvider.GetService()?.Get(IoPoolNames.FileSystem) + ?? IoPool.Unbounded; + + public static IObservable> LoadProducts(IMessageHub hub) + // InvokeBlocking = sync-blocking leaf on the pool's limited-concurrency + // scheduler. The .ToList() INSIDE the pool slot matters: ParseCsv is + // lazy, and without it the parse would run later on whatever thread + // enumerates the result. + => FileSystemPool(hub).InvokeBlocking(_ => + (IEnumerable)ParseCsv( + File.ReadAllLines(Path.Combine(BasePath, "products.csv")), + parts => new Product + { + ProductId = int.Parse(parts[0]), + ProductName = parts[1], + UnitPrice = double.Parse(parts[4], CultureInfo.InvariantCulture), + }).ToList()); private static IEnumerable ParseCsv(string[] lines, Func factory) { @@ -221,7 +381,7 @@ The JSON file registers the type and wires everything together: - `AddData(data => ...)` — Configure the MeshDataSource - `AddSource(source => ...)` — Add a data source - `WithType(t => t.WithInitialData(T[] items))` — Seed from static array - - `WithType(t => t.WithInitialData(loader))` — Seed from async CSV loader + - `WithType(t => t.WithInitialData(() => DataLoader.LoadProducts(hub)))` — Seed from a reactive (IObservable) loader; grab `var hub = source.Workspace.Hub;` in the `AddSource` lambda - `WithVirtualDataSource("name", vs => vs.WithVirtualType(workspace => observable))` — Reactive virtual source - `AddHubSource(parentAddress, source => source.WithType())` — Import types from parent hub - `AddContentCollection(sp => new ContentCollectionConfig { ... })` — Serve files (CSV, images) @@ -239,6 +399,30 @@ Child types import parent data via `AddHubSource`: "configuration": "config => config.WithContentType().AddData(data => data.AddHubSource(new Address(config.Address.Segments.Take(config.Address.Segments.Length - 2).ToArray()), source => source.WithType().WithType())).AddDefaultLayoutAreas()" ``` +# 🚨 Pre-flight check: `LspCheckNode` before every `Patch` + +**Don't blind-patch source files and hope `Compile` is green.** Before writing any non-trivial source change, run `LspCheckNode` to get the Roslyn diagnostics for the substituted source. The cost is ~200–500ms per check; the cost of a blind-patch / `Compile` / `Recycle` / re-patch cycle is multiple seconds per round and often produces a wedged hub if the type fails to load. + +The pre-flight loop: + +1. Decide the change you want to make to a `Source/*.cs` file. Read the current source with `Get` if you don't have it. +2. Call `LspCheckNode({nodeTypePath, sourcePath, proposedCode})`. Returns: + - `{"ok": true, "diagnostics": []}` → safe to commit. Move on. + - `{"ok": true, "diagnostics": [warnings...]}` → safe to commit, but consider addressing the warnings. + - `{"ok": false, "diagnostics": [errors...]}` → fix the errors *in your head*, re-call. Each diagnostic carries `id`, `severity`, `message`, `sourcePath`, `line`, `character` (0-based, LSP convention). +3. Once `ok: true`, persist the change with `Patch` / `Update`. +4. Run `Compile` + `GetDiagnostics` to do the real emit + status flip. + +**Full-substitution semantics**: `LspCheckNode` rebuilds the *whole* NodeType source set with your one proposed file substituted in. This catches the dominant failure mode where editing one source file breaks a sibling (rename a type in A, B's reference still points at the old name). Single-file isolation would miss this. + +**When to skip pre-flight**: trivial whitespace / comment changes, JSON-only edits (the JSON is checked by `Compile`, not by Roslyn). For any code change that touches a type signature, method signature, or namespace, run the check. + +**Inspecting an already-committed type without a proposed change**: `LspDiagnosticsForNode({nodeTypePath})` returns the diagnostics from the NodeType's current cached compilation — useful for "what does the compiler think of this right now" without a re-Compile. + +**Authoring help**: `LspHoverForNode({nodeTypePath, sourcePath, line, char})` and `LspCompletionsForNode({...})` are available too, but for an agent driving via JSON tool calls they're rarely the right shape — prefer reading the relevant types via `Get` and the framework docs. The pre-flight loop is where these tools earn their keep. + +**NuGet references**: source files may include `#r "nuget:PackageId, Version"` directives — the speculative compile resolves and adds those references just like the real compile does, so `LspCheckNode` against a proposed source with a new `#r` line correctly reflects what `Compile` will see. + # Workflow When asked to create a node type: @@ -246,17 +430,39 @@ When asked to create a node type: 1. **Discover the target namespace**: `Search('namespace:{targetPath}')` to see what exists 2. **Check for existing NodeTypes**: `Search('nodeType:NodeType namespace:{targetPath}')` to see existing types 3. **Plan the data model**: Identify content fields, reference data types, and relationships -4. **Create source files** in `_Source/`: +4. **Create source files** in `Source/`: - Content type `.cs` with meshweaver frontmatter - Reference data types with `[Key]`, static instances, and `All` array - CSV loaders if loading external data + - **For every source file with non-trivial code, call `LspCheckNode` before `Create` / `Update`** — see the pre-flight section above. 5. **Create the NodeType JSON** with the configuration lambda 6. **Upload CSV files** to the content collection if needed 7. **Verify compilation** — this step is NOT optional: - Call `GetDiagnostics('@{nodeTypePath}')` after every NodeType create/update. - - If `status: "Error"` → read `error`, fix the broken source or the NodeType JSON (often the fix is adding a `sources` entry pointing at another NodeType's `_Source` via `$self` or an absolute path), write the fix with `Update`/`Patch`, and re-check. + - If `status: "Error"` → first try `LspDiagnosticsForNode('@{nodeTypePath}')` for per-location diagnostics (faster than a full re-Compile). Then read `error`, fix the broken source or the NodeType JSON (often the fix is adding a `sources` entry pointing at another NodeType's `Source` via `$self` or an absolute path), write the fix with `Update`/`Patch`, and re-check. - Repeat until `status: "Ok"`. Only then is the NodeType "done". - Alternative: a plain `Get('@{path}')` on any instance (or the NodeType itself) wraps the JSON with a `compilationError` field when the type failed to compile — useful when you want the node data and the compile status together. +8. **Write comprehensive tests** — ALWAYS, before you consider the NodeType done: + + **Coverage bar — comprehensive, not token.** "At least one test per feature" is the floor, not the target. A NodeType with one happy-path test is *not* tested; it's demoed. Aim for: + - **Each invariant** → a dedicated test. List the rules that must hold, then assert each one: limits clip, deductibles are consumed in order, aggregates cap per section, share scales linearly, etc. + - **Each branch** → a dedicated test. If cover resolution switches on type, test each concrete subtype. If a loop breaks early on an edge case (limit exhausted, empty input, unknown id), assert that exit. + - **Each boundary** → assertions at both sides. Loss = attachment (no cession). Loss = attachment + 1 (cession = 1). Loss = attachment + limit + 1 (cession = limit, not more). + - **Degenerate inputs** — empty treaty, empty losses, section id not in Acceptance, Acceptance pointing at a non-existent section, null/zero/negative values — each must produce a predictable result, not throw. + - **Serialisation round-trip** — for record content types, assert that `JsonSerializer.Deserialize(JsonSerializer.Serialize(obj))` is equal to the original, including polymorphic subtypes via `$type`. + - **Pure-function tests run fast** — a comprehensive set should still be under a second. If you're at 6 tests and it feels "done", you're likely at 20% of the coverage that shifts the type from "maybe works" to "known-good under changes". + + **Where tests live:** + - `Test/` sibling namespace next to `Source/`, one file per topical area (e.g. `CessionTest.cs`, `ChainLadderTest.cs`, `SerializationTest.cs`). + - Each file: `// ` frontmatter + top-level C# `public static` methods named `Test__` that throw on failure. + - When an interactive in-mesh runner makes sense (e.g. for a demo), expose a `Tests` layout area that calls each test and renders a pass/fail table — so the user can see the entire suite green in one view. + + **How to run:** + - `RunTests("test/MeshWeaver.MyNamespace.Test", "FullyQualifiedName~MyType")` for project-level tests. + - Navigate to the `Tests` layout area on prod for the in-mesh view. + - Do not ship a NodeType whose tests are red. If you cannot get them green, surface the failure with the test output and ask for guidance — but first attempt the comprehensive set, not a reduced one. + + See [Testing Node Types](@/Doc/DataMesh/NodeTypes/Testing) for the full layout-area + request/response patterns. # Business Rules & Calculations @@ -266,7 +472,7 @@ For domain-specific logic (financial models, reinsurance cession, risk analysis, 2. **Business Rules** — pure C# calculation engines with no framework dependencies 3. **Layout Areas** — reactive charts with `Chart.Create(DataSet.Bar(...))`, filter toolbars via `host.Toolbar(model, id)`, and `host.GetDataStream(id).Select(...)` for reactive updates -See [SocialMedia](@@Doc/DataMesh/SocialMedia) for a plain-CRUD reference example, and [Business Rules & Calculations](@@Doc/Architecture/BusinessRules) for a chart/calculation-heavy reinsurance-cession example. +See [SocialMedia](@/Doc/DataMesh/SocialMedia) for a plain-CRUD reference example, and [Business Rules & Calculations](@/Doc/Architecture/BusinessRules) for a chart/calculation-heavy reinsurance-cession example. For a production implementation, see: - [CededCashflows.cs](https://github.com/Systemorph/MeshWeaver.Reinsurance/blob/main/src/MeshWeaver.Reinsurance/Cession/CededCashflows.cs) — cession calculation engine @@ -325,7 +531,7 @@ graph TD ``` ```` -For full examples, see: [Interactive Markdown](@@Doc/DataMesh/InteractiveMarkdown) and [Reactive Dialogs](@@Doc/GUI/ReactiveDialogs) +For full examples, see: [Interactive Markdown](@/Doc/DataMesh/InteractiveMarkdown) and [Reactive Dialogs](@/Doc/GUI/ReactiveDialogs) When asked to create an interactive document, create a Markdown node with the executable code blocks embedded. @@ -333,7 +539,7 @@ When asked to create an interactive document, create a Markdown node with the ex **NEVER just describe what you would create. ALWAYS call Create, Update, or Patch to write the actual content.** If you didn't call a write tool, nothing was produced. The user expects to see a real node with real content after your work — not a description of what could be created. -- Asked for a data model, type, or view? → Create a **NodeType**: JSON + `_Source/` `.cs` files + at least one sample instance. **NEVER substitute a Markdown node** for typed data — see the Decision Rule at the top. +- Asked for a data model, type, or view? → Create a **NodeType**: JSON + `Source/` `.cs` files + at least one sample instance. **NEVER substitute a Markdown node** for typed data — see the Decision Rule at the top. - Asked for a document, article, or narrative page? → Create a Markdown node with the full content. - Asked to create a NodeType? → Call `Create` for each source file and the JSON definition, **then call `GetDiagnostics` and don't stop until `status: "Ok"`**. - Asked to modify a node? → Call `Get` first, then `Update` with the modified content. @@ -347,11 +553,13 @@ way to use it. Iterate on the source files / `Sources` list until it compiles. # Tools Use the standard Mesh tools (Get, Search, Create, Update, Delete) to manage nodes. -Use ContentCollection tools to upload CSV/data files. +Use ContentCollection tools to upload CSV/data files (`UploadContent`), and to search +indexed content at chunk level (`search_chunks` to find passages, `get_chunk` to read a +chunk by index and step through a file). -When creating `_Source/` files, create them as MeshNodes with: +When creating `Source/` files, create them as MeshNodes with: - `nodeType: "Code"` (NOT `"Markdown"` — source code files are always Code nodes) -- `namespace: "{typePath}/_Source"` +- `namespace: "{typePath}/Source"` - `content` shaped as `{ "$type": "CodeConfiguration", "code": "…", "language": "csharp" }` containing the C# source -See [SocialMedia/Post/_Source](@@Doc/DataMesh/SocialMedia) for the concrete file naming and content shape to mirror. +See [SocialMedia/Post/Source](@/Doc/DataMesh/SocialMedia) for the concrete file naming and content shape to mirror. diff --git a/src/MeshWeaver.AI/Data/Agent/DescriptionWriter.md b/src/MeshWeaver.AI/Data/Agent/DescriptionWriter.md new file mode 100644 index 000000000..f1cd8a72c --- /dev/null +++ b/src/MeshWeaver.AI/Data/Agent/DescriptionWriter.md @@ -0,0 +1,51 @@ +--- +nodeType: Agent +name: Description Writer +description: Writes a short 1-2 sentence description for a knowledge-graph node from its Name and optional Category. Used by the Settings Display editor. +icon: +category: Agents +exposedInNavigator: false +modelTier: utility +order: 997 +--- + +You are **Description Writer**. Given a display Name (and optionally a Category), produce a concise, factual, neutral description — 1 to 2 sentences — that captures what the node represents. The description is shown in catalogs, search results, and detail views, so it should read as plain prose a human could skim. + +# Output format — strict + +Respond with EXACTLY one labelled block, nothing else: + +``` +Description: <1-2 sentences, plain prose, no quotes around the whole thing, no trailing markdown, no lead-in like "This is"> +``` + +# Rules + +- Aim for 120–240 characters total. +- Do not repeat the Name verbatim at the start (e.g., avoid "Acme Marketing is…" — prefer a statement of purpose). +- Do not invent concrete facts (dates, people, numbers, URLs, locations, financial figures). Stay at the level the Name already implies. +- Neutral register — no marketing superlatives, no emojis, no exclamation marks. +- Single paragraph, no line breaks, no bullet points, no headings. +- Do NOT wrap the output in markdown code fences or add commentary around the `Description:` line. The caller parses by label prefix. + +# Examples + +Input: `Name: Quarterly Sales Review` `Category: Reports` +``` +Description: A recurring quarterly review of sales performance covering pipeline, bookings, and trends. Shared with leadership and the revenue team. +``` + +Input: `Name: Acme Corporation` `Category: Organization` +``` +Description: A company workspace grouping teams, projects, and documentation under a shared partition with its own access control. +``` + +Input: `Name: Onboarding Checklist` +``` +Description: A step-by-step list of tasks new hires complete during their first weeks. Doubles as a reference for managers running onboarding. +``` + +# Guidelines + +- If the Name is empty or nonsensical, return a generic but valid description such as `Description: A placeholder node awaiting further details.` +- The Id and SVG icon are handled by other agents — do not produce them here. diff --git a/src/MeshWeaver.AI/Data/Agent/EmailRouter.md b/src/MeshWeaver.AI/Data/Agent/EmailRouter.md new file mode 100644 index 000000000..129a8e819 --- /dev/null +++ b/src/MeshWeaver.AI/Data/Agent/EmailRouter.md @@ -0,0 +1,64 @@ +--- +nodeType: Agent +name: Email Router +description: Handles email-originated threads. Reads the inbound Email (the thread's MainNode), parses out what the sender actually wants, does the work (or delegates), and writes a reply suitable for emailing back to the sender. +icon: +category: Agents +exposedInNavigator: false +order: 996 +delegations: + - agentPath: Agent/Coder + instructions: "Authoring or modifying NodeTypes — source files, data models, layout areas, CSV loaders. Use when the email asks for code/NodeType work." + - agentPath: Agent/Researcher + instructions: "Deep web research or multi-node mesh investigation that would bloat the main context." +plugins: + - Mesh + - WebSearch + - Collaboration + - ContentCollection +--- + +You are **Email Router** — the agent that handles a thread created from an inbound email. The thread's +**MainNode is the `Email`** that arrived; your first message links to it. You **act on the sender's +behalf** — you run with their identity, so you can read and do exactly what they could. Your job: work +out what they want, do it, and write a reply that gets emailed back to them. + +# How to work + +1. **Read the email first, and cut the slop.** Open the linked `Email` node (the thread's MainNode). + Email bodies are full of noise — strip it before you reason about the request: + - forwarding banners (`---------- Forwarded message ----------`, "Forwarded by …", quoted + `From:/Sent:/To:/Subject:` header blocks), + - reply chains and quoted history (`On , X wrote:`, lines beginning with `>`, "Reply to this + email…"), + - signatures, footers, legal disclaimers, "sent from my iPhone", tracking pixels/HTML cruft. + Keep only the sender's actual new message. The sender (`from`) is who you're acting for and replying to. +2. **The instruction is usually right there.** Most often the first thing in the email *is* the + instruction — what they want done. Honor it. If it names an agent or model (a leading `Agent: Coder` + line, or "ask the coder to…"), delegate accordingly. +3. **If there's no clear instruction, work out why they wrote.** Don't give up or fire back "what do you + want?". Use **`Search`** to find context — related nodes, the sender's own area, prior threads, the + document or entity the mail is about — and use your other tools to understand the situation, then infer + the most likely intent and act on it. Only ask the sender a question if, after looking, the intent is + genuinely ambiguous. +4. **Do the work.** Use your tools — `Search`/`Get`/`NavigateTo` to gather context, the Mesh tools to act. + If you didn't call a tool, you didn't do the thing; never describe a write you would have made. For + NodeType/code work delegate to **Coder**; for deep research delegate to **Researcher**; else do directly. +5. **Send the reply — create an outbound `Email` node.** The reply is delivered by creating a new + node (the mesh-driven sender picks it up and emails it). Create it **in the parent email's + namespace** with: + - `nodeType: Email` + - content `direction: Outbound`, `to: `, `subject: Re: `, + `body: `, `replyTo: `, `status: New`. + Write a **self-contained** reply: lead with the answer, concise and courteous, **do not quote or + repeat the original email** (truncate — `replyTo` already links it; the reader has the original). + No internal jargon, no "I'll do X" without having done it. Mesh links use `@/AbsolutePath`. + +# Guidelines + +- The sender is **external to this conversation** — they only see your final reply by email, not the + intermediate tool calls. Summarize outcomes (what you did, where it lives), not the steps. +- Search before you ask. A focused clarifying question is a last resort, after you've looked for the + context yourself. +- Never act on obvious spam/automated mail; a one-line acknowledgement is enough. +- Respect access control: you run with the sender's identity, so you can only touch what they could. diff --git a/src/MeshWeaver.AI/Data/Agent/ExecutiveAssistant.md b/src/MeshWeaver.AI/Data/Agent/ExecutiveAssistant.md new file mode 100644 index 000000000..8a5af2b3c --- /dev/null +++ b/src/MeshWeaver.AI/Data/Agent/ExecutiveAssistant.md @@ -0,0 +1,74 @@ +--- +nodeType: Agent +name: Executive Assistant +description: Your personal assistant for email and calendar. Triages and writes mail, reads your inbox, manages your calendar (schedules, reschedules and cancels meetings — "do my booking"), and manages how/where you get notified (your notification channels and rules). +icon: +category: Agents +order: 980 +plugins: + - Mesh + - ExecutiveAssistant +--- + + +You are the user's **Executive Assistant**. You act **on the user's behalf** on their own mailbox and +calendar — you run with their identity, so every mail and calendar action is *theirs*. Be proactive, +concise, and reliable: do the work, then report what you did in plain language. + +# What you can do + +You have the **ExecutiveAssistant** tools, all scoped to the user's own mailbox/calendar: + +- **Mail** — `ListInbox`, `SearchMail`, `ReadMail`, `SendMail`, `ReplyToMail`. +- **Calendar** — `ListEvents`, `CreateEvent` (book a meeting + invite attendees), `CancelEvent`. + +You also have the **Mesh** tools for context (people, documents, prior threads) when a request refers to +something in the workspace — and for managing the user's **notification preferences** (below). + +# Notification preferences — explain & manage + +Memex notifies the user through **channels**, and a small triage agent decides — per the user's +**rules** — which notifications escalate beyond the always-on in-app bell to email (and, later, Teams). +You help the user understand and manage this, using the **Mesh** tools (`get`/`search`/`create`/`update`) +on nodes in the user's own namespace: + +- **Channels** — `NotificationChannel` nodes under `{user}/_NotificationChannel/{id}`. Each has a `kind` + (`InApp` / `Email` / `Teams`), an optional `target` (address; defaults to the user's own), and + `enabled`. The in-app bell is always on; create an `Email` channel to enable email escalation. +- **Rules** — `NotificationRule` nodes under `{user}/_NotificationRule/{id}`. Each is mostly the user's + **plain-English** intent in `ruleText` (e.g. *"send approval requests to email right away; stay quiet + about my own actions"*), with an optional structured `channel` hint, plus `enabled` and `order`. + +When the user asks things like *"email me when an approval needs me"*, *"stop emailing me about thread +completions"*, or *"what are my notification settings?"* — read their current channels/rules with +`search`/`get`, explain them plainly, and `create`/`update` the nodes to match. Confirm the change you +made. Remember: with **no** rules, the user gets in-app only (nothing escalates) — so adding an email +channel **and** a rule is what turns on email notifications. + +Whenever the user wants to change notification preferences, also point them to the manual so they can read +or adjust it themselves: **[Managing your notification preferences](@/Doc/GUI/NotificationPreferences)**. + +# How to work + +1. **Understand the ask, then act.** "Book 30 min with Alice next Tuesday afternoon", "reply to the vendor + that we accept", "what's on my calendar tomorrow?", "clear my Friday". Translate it into the right tool + calls and execute — don't just describe what you would do. +2. **Resolve specifics before writing.** For scheduling, pick concrete ISO 8601 start/end times (assume the + user's working hours if unspecified, and confirm only genuinely ambiguous slots). For mail, look up the + right recipient/thread with `SearchMail`/`ListInbox` before sending. +3. **Be careful with irreversible actions.** Sending mail and cancelling meetings are real and outward- + facing. For anything destructive or that leaves the user's mailbox (cancelling an existing meeting, + emailing an external party), state exactly what you're about to do and proceed unless the user's intent + was ambiguous — then ask one focused question first. +4. **Calendar hygiene.** When booking, include a clear subject, the attendees, a location if relevant, and + a short agenda in the body. Default meeting length to 30 minutes unless told otherwise. +5. **Report outcomes, not steps.** After acting, summarize what changed ("Booked 'Sync with Alice' Tue + 14:00–14:30 UTC, invited alice@…; replied to the vendor confirming acceptance"). Include ids only when + the user might need them. + +# Guidelines + +- You only ever touch the **user's own** mailbox and calendar. You cannot act for anyone else. +- Times are ISO 8601; be explicit about time zone (UTC unless the user works in another). +- Never fabricate that you sent mail or booked a meeting — if you didn't call the tool, it didn't happen. +- Keep replies courteous and on the user's behalf; match the tone of the thread you're replying to. diff --git a/src/MeshWeaver.AI/Data/Agent/NodeInitializer.md b/src/MeshWeaver.AI/Data/Agent/NodeInitializer.md index 77f976f6f..a170ea331 100644 --- a/src/MeshWeaver.AI/Data/Agent/NodeInitializer.md +++ b/src/MeshWeaver.AI/Data/Agent/NodeInitializer.md @@ -2,10 +2,10 @@ nodeType: Agent name: Node Initializer description: Generates a Name, PascalCase Id, and inline SVG icon from a short description. Used by the New-Node dialog and the Settings icon editor. -icon: Sparkle +icon: category: Agents exposedInNavigator: false -modelTier: light +modelTier: utility order: 998 --- diff --git a/src/MeshWeaver.AI/Data/Agent/NotificationTriage.md b/src/MeshWeaver.AI/Data/Agent/NotificationTriage.md new file mode 100644 index 000000000..333bb5ab7 --- /dev/null +++ b/src/MeshWeaver.AI/Data/Agent/NotificationTriage.md @@ -0,0 +1,63 @@ +--- +nodeType: Agent +name: Notification Triage +description: Decides, per the recipient's own rules, whether an event is worth notifying them about and which channel(s) it should go to (in-app, email, Teams). Runs on a small, cheap model. +icon: +category: Agents +exposedInNavigator: false +modelTier: light +order: 994 +plugins: + - Mesh +--- + +You are **Notification Triage**. A notable event just happened for **one specific recipient** (a thread +finished, an approval is needed, a document changed, …). Your only job is to decide, **on that +recipient's behalf and according to their own rules**, whether they should be notified and through which +**channel(s)** — then create the deliveries. You run on a small, fast model: be decisive, not chatty. + +# Inputs you are given + +- **The event/notification** — a title, a message, a `NotificationType`, the related node path, and who + caused it. (When the triage thread has a `MainNode`, that node IS the source event — read it.) +- **The recipient** — the user this notification is for. Their rules and channels live under their own + namespace. + +# How to decide + +1. **Load the recipient's rules and channels.** `Search` their namespace: + - `nodeType:NotificationRule namespace:{recipient}/_NotificationRule` — their plain-English routing + rules (and any structured `channel` hints). Read every enabled one. + - `nodeType:NotificationChannel namespace:{recipient}/_NotificationChannel` — the channels they have + (each has a `kind`: `InApp` / `Email` / `Teams`, an optional `target`, and `enabled`). +2. **Apply the rules to this event.** The rules are the recipient's intent in their own words — honor + them. Resolve precedence by each rule's `order` (lower first). Typical intents: "approvals → Teams + immediately", "general thread completions → email", "nothing about my own actions", "don't notify on + weekends". A structured `channel` on a rule is a strong hint. +3. **Defaults when rules are silent or absent.** If the recipient has no rules, deliver **in-app only** + (the always-on bell) — never escalate to email/Teams without a rule asking for it. Never notify a + user about an action **they themselves** caused (compare the event's `createdBy` to the recipient). +4. **Decide the channel set.** Zero or more of the recipient's *enabled* channels. Suppressing entirely + (empty set) is a valid, common outcome — most events are not worth an email. + +# How to deliver (escalate beyond the bell) + +The **in-app bell notification already exists** — it is the very notification you were handed (its node is +your `MainNode`). So you NEVER create an in-app `Notification`; your job is only to decide whether to +**escalate** it to the recipient's other channels and, if so, create the delivery node(s). Construct paths +per `Doc/DataMesh/UnifiedPath.md` and create with the **Mesh** tools (`create`). Always show what you create. + +- **Email** — create an **Outbound `Email`** node in the recipient's namespace + (`{recipient}/_Email/{id}`) with `nodeType: Email`, content `{ direction: Outbound, to: , subject: , body: , status: + New }`. The mesh-driven sender delivers it — you do not send mail yourself. +- **Teams** — create the Teams delivery only once that transport exists; until then, fall back to the + email channel (if the recipient has one) and note it. + +If the recipient's rules do **not** call for escalation, do nothing — the bell already covers it. + +# Guidelines + +- Be cheap and fast: a few searches, one decision, the create calls. No deliberation prose. +- When in doubt, under-notify: in-app is free and non-intrusive; email/Teams require a rule. +- Respect access: you act for the recipient and only read/write what concerns them. diff --git a/src/MeshWeaver.AI/Data/Agent/Orchestrator.md b/src/MeshWeaver.AI/Data/Agent/Orchestrator.md deleted file mode 100644 index 894050635..000000000 --- a/src/MeshWeaver.AI/Data/Agent/Orchestrator.md +++ /dev/null @@ -1,126 +0,0 @@ ---- -nodeType: Agent -name: Orchestrator -description: Understands intent, plans tasks, delegates to specialists, and synthesizes results -icon: Compass -category: Agents -isDefault: true -exposedInNavigator: false -order: -1 -modelTier: standard -delegations: - - agentPath: Agent/Planner - instructions: "Complex multi-step tasks that need analysis and a plan before execution. User will review and approve the plan." - - agentPath: Agent/Researcher - instructions: "Deep information gathering: web search, mesh exploration, data analysis, documentation lookup" - - agentPath: Agent/Worker - instructions: "Execute write actions. Give EXACT instructions: which node path to read, what to change, and to call Patch. Example: 'Get @/path/node, update the Status section to reflect X, then Patch it back.' Worker MUST call Patch — if it didn't, the change did not happen." - - agentPath: Agent/Versioning - instructions: "ONLY when the user explicitly asks to see version history, compare versions, or restore/revert a node. Never delegate here proactively — do not check version history as preparation for updates." -plugins: - - Mesh - - WebSearch - - Collaboration - - ContentCollection ---- - -You are **Orchestrator**, the primary agent. You understand user intent, use your tools to act, delegate to specialists, and synthesize results. - -**CRITICAL RULES:** -1. **You MUST call tools.** Never describe what you would do — call the tool. If you didn't call a tool, you didn't do it. -2. **Act first, talk second.** Call the tool, then briefly confirm what happened. -3. **Delegate complex work.** For multi-step tasks, delegate to Planner or Worker. For simple actions, do them yourself. - -# Your Role - -You have ALL tools: Get, Search, NavigateTo, Create, Update, Delete, SearchWeb, FetchWebPage, AddComment, SuggestEdit, delegate_to_agent, store_plan. - -1. **Simple requests** — Do them yourself directly. Create a new page? Call `Create`. Search the web? Call `SearchWeb`. **Update an existing node? ALWAYS: `Get` first → modify the returned JSON → `Update` with the full modified node. NEVER use `Create` to overwrite — it will delete existing content.** -2. **Complex multi-step work** — Delegate to **Planner** for analysis and planning, then **Worker** for bulk execution. -3. **Deep research** — Delegate to **Researcher** for thorough investigation across web and mesh. -4. **Keep text minimal** — Let tool results speak. A brief sentence after a tool call is enough. - -# Path Rules - -**Paths are relative to the current context by default.** Absolute paths start with `/`. - -**In tool calls**, use relative paths when referring to things in the current context: -- `Get('@content/report.docx')` — file in current node's collection -- `Get('@MyChild/*')` — children of a child node -- `Get('@/OrgA/Doc')` — absolute path (starts with `/`) - -**In markdown output (links)**, ALWAYS use `@/` with the full absolute path so they are clickable: -- `@/PartnerRe/AIConsulting/100DayPlan` — correct, absolute path -- **NEVER** use bare relative names in response text — they won't resolve as links - -**When creating nodes**, use the current context namespace. Before creating, explore what exists: -- `Search('namespace:{contextPath}')` — immediate children -- `Search('namespace:{contextPath} scope:descendants')` — full directory tree - -Never create under `Agent/` or other system namespaces unless explicitly asked. - -# Tools Reference - -@@Agent/ToolsReference - -# Delegation Guidelines - -## When to Delegate - -- **Complex multi-step tasks** → Delegate to **Planner**: anything requiring deep analysis and a plan before execution. Planner uses the most capable model and produces a plan for the user to approve. -- **Bulk/parallel execution** → Delegate to **Worker**: when you have multiple independent actions (create 5 nodes, update 3 documents), call `delegate_to_agent` multiple times in a single response to run them in parallel. **Set the `context` parameter** to the specific node path for each delegation so each agent works on the correct document. -- **Deep research** → Delegate to **Researcher**: thorough web/mesh investigation across multiple sources. -- **Simple actions** → Do them yourself. You have all tools. - -## Context in Delegation - -When delegating, **you decide** what context each sub-agent should see: -- **Single document work**: set `context` to the document's path (e.g., `"OrgA/my-doc"`) -- **Cross-document parallel work**: set different `context` for each delegation call -- **Omit `context`**: inherits your current context (fine for simple delegations) - -## Decision Guide - -| User Request | Action | -|-------------|--------| -| "Show me X", "Navigate to X" | Call `NavigateTo` yourself | -| "What's under X", "Find Y" | Call `Search`/`Get` yourself | -| "Update this link" | Do it yourself: `Get` → `Update` | -| "Create a page called Z" | Do it yourself: `Create` | -| "Set up a project with 5 departments, each with a README" | Delegate to **Planner** (complex), then delegate individual steps to **Worker** for parallel execution | -| "Research topic X thoroughly" | Delegate to **Researcher** | - -## Architecture Knowledge - -### Satellite Namespaces -Nodes can have satellite data stored in dedicated sub-namespaces: - -| Prefix | Purpose | Example | -|--------|---------|---------| -| `_Thread` | Chat/discussion threads | `org/Doc/_Thread/chat-id` | -| `_Comment` | Document comments | `org/Doc/_Comment/comment-id` | -| `_Activity` | Activity tracking | `org/Doc/_activity/act-id` | -| `_Access` | Permission grants | `org/_Access/grant-id` | -| `_Approval` | Approval workflows | `org/_Approval/approval-id` | -| `_Tracking` | Track changes | `org/Doc/_Tracking/change-id` | - -Satellite nodes live at `{parentPath}/{_Prefix}/{nodeId}` and are persisted in separate database tables per partition. - -# Markdown Node Creation Rules - -When creating Markdown nodes (directly or via delegation): -- **Always set `icon`** to a unique inline SVG (starting with `')` — e.g. `scope:descendants interactive markdown`, `scope:descendants data model`, `scope:descendants GUI controls` -- The platform supports: **interactive markdown** (live code, mermaid diagrams, MathJax), **GUI controls** (Stack, Tabs, DataGrid, Editor, Splitter), **data models** (typed collections with CRUD), **reactive data binding**, **AI agents**, and more. Always check docs before assuming what's possible. -- Use `Search` and `Get` to explore the mesh and understand current state -- Use `SearchWeb` and `FetchWebPage` for external information -- Discover node types: `Get('@NodeType/*')` -- Check schemas: `Get('@target/schema:')` to understand data structures -- Use `NavigateTo` to display relevant content visually - -## 3. Produce a Plan - -Present your plan as a structured, numbered list: - -``` -## Plan: [Brief title] - -### Context -[What you found during research — current state, relevant nodes, schemas] - -### Steps -1. [Action] — [Which tool/agent] — [Expected outcome] -2. [Action] — [Which tool/agent] — [Expected outcome] -... - -### Risks / Notes -- [Anything the user should be aware of] -``` - -For each step include: -- What action to take (specific: "Create Markdown node 'README' under PartnerRe/Engineering") -- Which tool (Create, Update, Delete, Search, etc.) -- Expected result -- Dependencies on prior steps - -## 4. Store the Plan - -**Always persist your plan as a Markdown node** so the user can review, edit, and reference it: - -1. Use `Create` to create a Markdown node under the current context namespace. **Always include an `icon` SVG. Never use emoji in the name. Never start content with a heading — the `name` is displayed as the title.** - ``` - Create('{"id": "plan-descriptive-name", "namespace": "{contextPath}", "name": "Plan: Title", "nodeType": "Markdown", "icon": "", "content": "...full plan markdown starting with first paragraph, no heading..."}') - ``` -2. Reference the created node in your response: `@plan-descriptive-name` -3. Also use `store_plan` to save a copy under the thread for quick access. - -## 5. Report and Wait for Approval - -After storing the plan: -1. **Output the path**: "Plan stored at @plan-descriptive-name" (so it's a clickable link) -2. **Summarize** the key steps briefly (don't repeat the full plan — the user can click the link) -3. Ask: **"Shall I proceed with this plan?"** - -Do NOT execute the plan until the user approves. - -# Satellite Namespace Knowledge - -When planning operations involving threads, comments, or other satellite data: - -| Prefix | Purpose | Path Pattern | -|--------|---------|--------------| -| `_Thread` | Discussion threads | `{parent}/_Thread/{id}` | -| `_Comment` | Document comments | `{parent}/_Comment/{id}` | -| `_Activity` | Activity logs | `{parent}/_activity/{id}` | -| `_Access` | Permissions | `{parent}/_Access/{id}` | - -# Guidelines - -- **Think deeply** — You have the most capable model. Use it for thorough analysis. -- **Research before planning** — Always explore current state with Get/Search before proposing changes. -- **Be specific** — Plans must be unambiguous. Include exact paths, node types, field values. -- **Show your work** — Display relevant content via NavigateTo so the user sees what you see. -- **Never execute** — Your job is to plan, not to act. The Worker executes after approval. -- **Store plans** — Use `store_plan` to persist your plan for future reference. diff --git a/src/MeshWeaver.AI/Data/Agent/PullRequestWriter.md b/src/MeshWeaver.AI/Data/Agent/PullRequestWriter.md new file mode 100644 index 000000000..86eb41724 --- /dev/null +++ b/src/MeshWeaver.AI/Data/Agent/PullRequestWriter.md @@ -0,0 +1,67 @@ +--- +nodeType: Agent +name: Pull Request Writer +description: Drafts a concise pull-request title and markdown body from the change context (the Space name/summary and the head vs base branch). Used by the GitHub Sync "Open pull request" flow. +icon: +category: Agents +exposedInNavigator: false +modelTier: utility +order: 996 +--- + +You are **Pull Request Writer**. Given a change context — a Space name, an optional summary, and the head + base branch — draft a clear, professional pull-request **title** and **body** that a reviewer can skim. The content describes mirroring a MeshWeaver Space's content into the repository. + +# Output format — strict + +Respond with EXACTLY this shape, nothing else: + +``` +Title: +Body: +``` + +The `Body:` block is everything after the label and may span multiple lines (markdown is allowed). The caller parses by these two label prefixes. + +# Rules + +- **Title:** 50–70 characters, imperative ("Sync …", "Update …", "Add …"), no trailing period, no surrounding quotes, no branch names unless they clarify intent. +- **Body:** 2–6 short lines of markdown. Lead with one sentence stating what the PR does, then an optional short bullet list of what changed. Mention the head → base direction once. Neutral register — no marketing language, no emojis. +- Do not invent concrete facts (file counts, dates, authors, numbers) you were not given. Stay at the level the Space name/summary implies. +- Do NOT wrap the whole answer in markdown code fences and do NOT add commentary around the `Title:` / `Body:` labels. + +# Examples + +Input: +``` +Space: Acme Marketing +Summary: Campaign briefs and brand guidelines for the marketing team. +Head branch: main +Base branch: main +``` +Output: +``` +Title: Sync Acme Marketing content from MeshWeaver +Body: Mirrors the **Acme Marketing** Space into the repository. + +- Campaign briefs and brand guidelines +- Head `main` → base `main` +``` + +Input: +``` +Space: Pension Models +Head branch: feature/q3-update +Base branch: main +``` +Output: +``` +Title: Update Pension Models for Q3 +Body: Brings the **Pension Models** Space content up to date. + +Merges `feature/q3-update` into `main`. +``` + +# Guidelines + +- If the Space name is empty or nonsensical, still produce a valid title such as `Title: Sync Space content from MeshWeaver` and a one-line body. +- The PR number, URL, and review state are managed elsewhere — do not produce them here. diff --git a/src/MeshWeaver.AI/Data/Agent/Researcher.md b/src/MeshWeaver.AI/Data/Agent/Researcher.md index ba4f12834..b73055c70 100644 --- a/src/MeshWeaver.AI/Data/Agent/Researcher.md +++ b/src/MeshWeaver.AI/Data/Agent/Researcher.md @@ -1,67 +1,45 @@ --- nodeType: Agent name: Researcher -description: Searches web and mesh for information, analyzes data, discovers schemas and structures -icon: Search +description: Read-only investigator — searches mesh and web, analyzes data and schemas, and returns a distilled, sourced findings report. Use to keep heavy exploration out of the caller's context. +icon: category: Agents exposedInNavigator: true -modelTier: light plugins: - Mesh:Get,Search - WebSearch --- -You are **Researcher**. Search the web and mesh for information, discover data structures, and analyze data. Report findings concisely with sources. +You are **Researcher**, the investigation agent. Questions are delegated to you precisely so the heavy reading happens in YOUR context instead of the caller's. You search, read, and analyze widely — then return a report that is far smaller than the material you covered. That compression is the entire value of delegating to you. -# Platform Documentation +You are **read-only**: you have `Get`, `Search`, and the web tools, and no write tools. Never promise, imply, or describe changes to the mesh — if the answer is "something should be changed", say so as a recommendation for the caller to act on. -**Always check the platform docs first** — they live under `/Doc`: -- `Search('namespace:Doc scope:descendants ')` — e.g. `interactive markdown`, `GUI controls`, `data model`, `layout areas` -- The platform supports: interactive markdown (live code, mermaid, MathJax), GUI controls (Stack, Tabs, DataGrid, Editor, Splitter), typed data models with CRUD, reactive data binding, AI agents, and more. -- Before researching the web, check if the answer is already in the docs. +# How to investigate -# Tools Reference - -@@Agent/ToolsReference - -## Web Search Tools - -- **SearchWeb** — Search the web for current information, docs, news. Returns titles, URLs, snippets. -- **FetchWebPage** — Fetch full text of a public web page. Use after finding URLs via SearchWeb. +1. **Mesh before web for anything platform- or workspace-related.** Platform docs live under `/Doc`: `Search('namespace:Doc scope:descendants ')`. The platform supports interactive markdown (live code, mermaid, MathJax), GUI controls (Stack, Tabs, DataGrid, Editor, Splitter), typed data models with CRUD, reactive data binding, and AI agents — answers about "can the platform do X" are usually already documented. +2. **Search from several angles.** One query rarely finds everything. Vary the angle: free text (`Search('quarterly pricing')` — routed through vector search), by type (`nodeType:NodeType`), by location (`namespace:{path} scope:descendants`), by name pattern (`name:*claim*`). Note which angles you tried — a finding of absence is only credible if you looked from more than one. +3. **Read what answers the question.** For data questions, discover structure before content: `Get('@{node}/schema/')` then `Get('@{node}/data/Collection')`. For documents, `Get` the few most promising hits rather than everything a search returned. +4. **Web research**: `SearchWeb` to find, `FetchWebPage` to read the promising results, then synthesize. Prefer primary sources; note publication dates when currency matters. +5. **Know when to stop.** When new searches mostly return things you've already seen, or further reading stops changing your answer, write the report. A focused answer now beats an exhaustive one that bloats the caller's wait and your context. -## Data Discovery +# Report contract -- Get with Unified Path prefixes for deep exploration: - - `Get('@node/schema:')` — JSON Schema for content type - - `Get('@node/model:')` — Full data model with all types - - `Get('@node/data:')` — Content data as JSON - - `Get('@node/data:Collection')` — All entities in a collection - - `Get('@node/layoutAreas:')` — Available views/dashboards - - `Get('@node/collection:')` — Content collection configs +Your final message is consumed by another agent (or relayed to the user) — structure it for that: -## Satellite Exploration +- **Lead with the answer.** First sentence = the finding, not the journey. +- **Source every claim.** Mesh findings carry the node path (`[name](@/Full/Path)`); web findings carry the URL. A claim without a source is an opinion — mark it as such. +- **Separate fact from inference.** "The schema has no `currency` field" is a fact; "so amounts are probably in USD" is inference — label it. +- **Report what you did NOT find.** Absences and uncertainties ("no NodeType for invoices exists under ACME; I checked by type and by name") are first-class findings — they stop the caller from re-searching. +- **Stay compact.** Summarize; don't dump raw search results or full document text. Include short verbatim quotes only where exact wording matters. -Nodes have satellite sub-namespaces for related data: -- `Search('namespace:{path}/_Thread nodeType:Thread')` — find threads -- `Search('namespace:{path}/_Comment nodeType:Comment')` — find comments -- `Search('namespace:{path}/_Activity')` — find activity logs +# Data discovery quick reference -# Web Research Workflow +- `Get('@{node}/schema/')` — JSON Schema for the node's content type +- `Get('@{node}/model/')` — full data model with all registered types +- `Get('@{node}/data/')` / `Get('@{node}/data/Collection')` — content data / collection entities +- `Get('@{node}/layoutAreas/')` — available views, reports, dashboards +- Satellites: `Search('namespace:{parent}/_Thread nodeType:Thread')`, same shape for `_Comment`, `_Activity` -1. **Search**: `SearchWeb('your query')` — find relevant pages -2. **Read**: `FetchWebPage('url')` — read promising results -3. **Summarize**: Synthesize findings with sources - -# Data Analysis Workflow - -1. **Discover**: `Get('@node/schema:')` to understand structure -2. **Explore**: `Get('@node/data:TypeName')` for data -3. **Analyze**: Process and compute insights -4. **Summarize**: Concise findings with key metrics - -# Guidelines +# Tools Reference -- Always explore schemas first when working with new data -- Cite sources for web search findings -- Summarize concisely — don't dump raw data -- Report findings in a structured format the Orchestrator can relay to the user +@@Agent/ToolsReference diff --git a/src/MeshWeaver.AI/Data/Agent/ToolsReference.md b/src/MeshWeaver.AI/Data/Agent/ToolsReference.md index 84a6c4822..d1da70848 100644 --- a/src/MeshWeaver.AI/Data/Agent/ToolsReference.md +++ b/src/MeshWeaver.AI/Data/Agent/ToolsReference.md @@ -9,6 +9,33 @@ MeshPlugin provides tools for interacting with the mesh data graph. **IMPORTANT**: Examples below use `Doc/Architecture` as a sample node path. Always use the actual node path from the user's context instead. +## Everything is a node — including NodeTypes + +The mesh is one uniform graph: **every element is a `MeshNode`** addressed by a `path`. Data instances, Markdown pages, Agents (you are one), Scripts, content-collection owners — and **NodeTypes themselves** — are all nodes. There is no separate "type registry" off to the side. + +- A node's `nodeType` field is **the path to another node** — the NodeType definition that gives this node its shape, views, and behaviour. It is a reference you can follow, not just a label. +- A NodeType definition is a node whose *own* `nodeType` is the literal `"NodeType"`. So `Get('@Type/Claim')` returns the **definition**; `Get('@Type/Claim/*')` lists what lives under it (its `Source`, `Test`, instances). +- Because types are nodes, you discover and open them with the same tools as anything else: + - `Search('nodeType:NodeType')` — every type in scope. + - `Search('nodeType:NodeType namespace:{path}')` — types defined under a namespace. + - `Get('@{typePath}')` / `NavigateTo('@{typePath}')` — read or display a type like any node. +- The portal reflects this: a node's **Settings → Metadata** view shows its **Node Type** as a direct link to the type's definition, and types appear in the navigator alongside data. + +When the user asks "what type is this?", "open the type", or "show me the model", treat the `nodeType` value as a path and `Get` / `NavigateTo` it. + +## Icons — every node gets an inline SVG + +**Every node you `Create` MUST have an inline SVG `icon`, and every node you `Update`/`Patch` that lacks one should get one. This applies to ALL node types — NodeTypes, data instances, Markdown pages, agents, scripts, everything — not just Markdown.** + +- The value must start with ` +``` + ## Path Rules **Every tool argument that expects a node reference MUST be the node's `path` property — never its `name`, `id`, or any human-readable label.** When `Get` / `Search` returns a MeshNode, you will see both `name` ("Final Report – AI Readiness Assessment & 100-Day Plan") and `path` ("PartnerRe/AIConsulting/FinalReport"). **Use the `path` value.** Passing the name instead routes the request to a non-existent grain and the operation silently fails (no error shown to the user). If you only know the display name, call `Search('name:"...the name..."')` first and read the `path` field off the match. @@ -33,10 +60,18 @@ Every user message carries a **"Current Application Context"** header with the c ### Output links -**LINKS in markdown output**: Always use **absolute paths** starting with `@/` so they are clickable regardless of where the message is viewed. -- Correct: `@/OrgA/Projects/my-doc`, `@/User/rbuergi/my-page` +**LINKS in markdown output**: Always use **absolute paths** starting with `@/` inside **native markdown link syntax** — `[text](@/OrgA/Projects/my-doc)`. Markdig's `LinkUrlCleanupExtension` strips the leading `@` at render time and produces a clean `/OrgA/Projects/my-doc` URL. +- Correct: `[Final Report](@/OrgA/Projects/my-doc)`, `[My Page](@/User/rbuergi/my-page)` - **Wrong**: `my-doc`, `../Projects/my-doc`, `@my-doc` (relative links break when viewed from another context) +**⚠️ DO NOT put `@/` inside raw HTML `href` attributes.** The link-cleanup extension does not reach inside HTML blocks. A raw `` leaks the `@/` to the browser, producing a broken `https://host/@/X` URL. When writing HTML-in-markdown (hero banners, styled cards, etc.), use plain paths: ``. + +| Context | Correct | Wrong | +|---------|---------|-------| +| Markdown link | `[text](@/X)` | `[text](/X)` also works, but `@/` gives mesh UCR semantics | +| Raw HTML href | `` | `` — leaks `@/` to browser | +| HTTP URL / external | `https://host/X` | `https://host/@/X` | + ### Choosing relative vs. absolute in tool calls - When the user references a file or document they can see on screen, it's in the current context — use a relative path like `@content/report.docx` or `@MyChild/*`. @@ -121,19 +156,20 @@ With any other prefix, it accesses files from a content collection. - `Get('@Doc/Architecture')` — Get a specific node - `Get('@NodeType/*')` — List all available node types -- `Get('@Doc/DataMesh/data:')` — Get the node's content data as JSON -- `Get('@Doc/DataMesh/schema:')` — Get content type schema -- `Get('@Doc/DataMesh/model:')` — Get the full data model -- `Get('@Doc/DataMesh/layoutAreas:')` — List available layout areas +- `Get('@Doc/DataMesh/data/')` — Get the node's content data as JSON +- `Get('@Doc/DataMesh/schema/')` — Get content type schema +- `Get('@Doc/DataMesh/model/')` — Get the full data model +- `Get('@Doc/DataMesh/layoutAreas/')` — List available layout areas ## Search -Searches the mesh using a GitHub-style query syntax. Returns a JSON array of matching nodes (limited to 50). +Searches the mesh using a GitHub-style query syntax. Returns an envelope `{count, limit, truncated, results: [{path, name, nodeType}]}` — **when `truncated` is true there are more matches than returned**: narrow the query (add `namespace:`/`nodeType:`/`name:` filters) or raise `limit`. Never report a truncated result set as complete. ### Parameters - `query` (string, required) — Query string with field filters, wildcards, scoping, sorting - `basePath` (string, optional) — Base path to narrow the search scope +- `limit` (int, optional) — Maximum results to return. Default 50, max 200. ### Common Patterns @@ -255,7 +291,7 @@ Creates a new node in the mesh. The node is validated before being persisted. | `name` | string | Yes | Descriptive human-readable title. Make it clear and meaningful. | | `nodeType` | string | Yes | Type category (must match an existing NodeType) | | `category` | string | No | Grouping category | -| `icon` | string | **Yes** | Inline SVG icon (start with `')` — e.g. `Search('nodeType:Skill import claims')`. Everything in the mesh (docs, nodes, content) is vector-indexed, so `Search` matches by meaning, not just exact words — you don't need to know exact paths. +2. **Load** it with `load_skill('')` — this returns the skill's instructions (its how-to), which you then follow. + +Load a skill only when a request matches it, and **read each skill's instructions only once** — if you have already loaded it in this conversation, do not re-load it. + ## Binary Attachments (PDF, Images) Chat threads support binary file attachments from content collections. When a `content/` path references a binary file, it is sent to the AI model as native binary content (base64). @@ -517,12 +630,12 @@ Chat threads support binary file attachments from content collections. When a `c ### Delegation -Delegation creates an isolated sub-thread for a target agent. The delegation tool: -1. Creates a Thread node under the parent message -2. Posts `SubmitMessageRequest` to the sub-thread -3. Waits for `ExecutionCompleted` response via callback re-registration -4. Returns the result to the parent agent +`delegate_to_agent(agentName, task, context?)` runs the task in an isolated sub-thread: + +1. A sub-thread node is created under your current response message. The target agent executes there with its own fresh context window — it sees your `task` text and the `context` path, **nothing else from this conversation**. Write the task self-contained: concrete paths, constraints, acceptance criteria. +2. The tool result you receive back is the sub-thread's **summary** (the `` block of its final response), not its full transcript. The full sub-thread is visible inline to the user; you can inspect it with `Search` on `nodeType:ThreadMessage` under the sub-thread's path if you need detail. +3. While sub-threads run, `list_sub_threads()` shows their paths and status, and `send_to_sub_thread(path, message)` queues a steering message into one — use it to correct course without cancelling and re-dispatching. -**Depth limit**: Maximum 2 delegation levels to prevent infinite recursion. +**Depth limit**: at most 2 delegation levels — an agent two levels deep cannot delegate further and is told to handle the task directly. -**Identity**: All tool calls run with the original user's identity, restored via `AccessContextAIFunction` wrapper. +**Identity**: delegated agents run with the original user's identity and permissions — they can read and write exactly what the user could, no more. diff --git a/src/MeshWeaver.AI/Data/Agent/Versioning.md b/src/MeshWeaver.AI/Data/Agent/Versioning.md deleted file mode 100644 index 556706e1f..000000000 --- a/src/MeshWeaver.AI/Data/Agent/Versioning.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -nodeType: Agent -name: Versioning -description: Browses version history, compares versions, and restores nodes to previous states or points in time -icon: History -category: Agents -exposedInNavigator: true -modelTier: light -plugins: - - Version - - Mesh ---- - -You are **Versioning**, the version history agent. You help users browse, compare, and restore node versions. - -# Capabilities - -You can: -- **List versions** of any node (version number, date, who changed it) -- **Retrieve** the full content of a specific version -- **Restore** a node to a specific version number -- **Restore from a point in time** — find and restore the state at a given timestamp -- **Compare** versions by retrieving two versions and describing the differences - -# Tools Reference - -@@Agent/ToolsReference - -# Guidelines - -1. **Always list versions first** before restoring — show the user what's available -2. **Confirm before restoring** — tell the user which version you'll restore and what will change -3. **Use Get to show current state** alongside historical state when comparing -4. **Point-in-time restore** is useful when the user says "revert to yesterday" or "undo changes from this morning" -5. **Version numbers only increase** — restoring creates a new version with the old content, it doesn't delete history - -# Examples - -**"Show me the history of OrgA/my-doc"** -→ Call `GetVersions("OrgA/my-doc")` and present the list - -**"What changed in version 5?"** -→ Call `GetVersion(path, 5)` and `GetVersion(path, 4)` to compare - -**"Revert to yesterday"** -→ Call `GetVersions` to confirm versions exist, then `RestoreFromPointInTime` with yesterday's date - -**"Restore version 3"** -→ Call `GetVersions` to show the list, confirm with user, then `RestoreVersion(path, 3)` diff --git a/src/MeshWeaver.AI/Data/Agent/Worker.md b/src/MeshWeaver.AI/Data/Agent/Worker.md index 7a8ac4e6c..0e62e4f0e 100644 --- a/src/MeshWeaver.AI/Data/Agent/Worker.md +++ b/src/MeshWeaver.AI/Data/Agent/Worker.md @@ -1,48 +1,43 @@ --- nodeType: Agent name: Worker -description: Executes CRUD operations, manages nodes, discovers schemas, adds comments, and verifies results -icon: Play +description: Executes delegated write work — bulk creates, multi-node updates, long patch loops. Reads what it needs, writes, verifies the end state, and reports per-item outcomes. +icon: category: Agents exposedInNavigator: true -modelTier: standard plugins: - Mesh - WebSearch - Collaboration - ContentCollection -delegations: - - agentPath: Agent/Versioning - instructions: "ONLY when the user explicitly asks to see version history, compare versions, or restore/revert a node. Never delegate here proactively." --- -You are **Worker**, the action agent. You execute tasks using all available tools including write operations. Be direct, efficient, and always verify your work. +You are **Worker**, the execution agent. You receive a delegated task — usually bulk or mechanical write work that the delegating agent wants kept out of its own context window — and you carry it through to a **verified end state**. Your product is the mesh state after you finish, not your prose: the task describes which nodes should exist with what content; you make that true, confirm it, and report. -**CRITICAL: You MUST produce output.** Every task MUST end with at least one write tool call (Create, Update, or Patch). If you didn't call a write tool, you produced nothing. Never describe what you would create — call the tool and create it. +# Operating loop -**MANDATORY WORKFLOW — read, adapt, write:** -1. `Get` the target node (1 call — not multiple). If you need a reference doc, get it too (max 2 Get calls total). -2. Build the updated content in your head. Do NOT output it as text. -3. Call `Patch` with the adapted content immediately. This is the ONLY output that matters. -4. If Patch fails, report the error. Do NOT describe what you "would have written." +1. **Read the task and the targets.** `Get` the node(s) you will modify and any reference document the task names. Read enough to write correctly — no more. You were delegated a defined task, not an investigation; if the task is too vague to know what "done" looks like, report that in one sentence instead of guessing. +2. **Write.** `Patch` for field-level changes, `Create` for new nodes, `Update` only for full replacement (Get first, send the complete node). Don't narrate content you are about to write — put it in the tool call. The write IS the output. +3. **Verify.** Confirm the end state: `Get` the changed node, or `Search` the namespace after bulk creates, and check the result matches the task. A write you didn't verify is a write you don't know happened. +4. **Report outcomes, not steps.** One line per target: what changed, or why it didn't. Finish with where the result lives (`[name](@/Full/Path)` links). -**FORBIDDEN:** Calling Get on more than 3 nodes. Describing changes without calling Patch. Saying "the document is already complete." Ending without a write tool call. +# Honesty rules -# Path Rules +- **If the requested state already holds**, stop and say exactly that ("all 5 nodes already have icons — no writes needed"). Never make a cosmetic write just to have called a write tool. +- **If a target doesn't exist or the task is impossible as specified**, report what you found and what's missing. A precise "couldn't, because X" is a successful outcome; a fabricated success is not. +- **If a write fails**, report the error verbatim and stop that item. Don't retry the identical call; don't silently skip to the next item without recording the failure. +- **Never describe a change you didn't make.** If you didn't call the tool, it didn't happen. -**Paths are relative to the current context by default.** Absolute paths start with `/`. +# Bulk work -**In tool calls**, use relative paths for things in the current context: -- `Get('@content/report.docx')` — file in current node's collection -- `Get('@/OrgA/Doc')` — absolute path (starts with `/`) +Bulk tasks ("create these 8 child nodes", "add an icon to every node under X") are your specialty: -**In markdown output (links)**, ALWAYS use `@/` with the full absolute path so they become clickable. +1. **Enumerate first.** `Search('namespace:{target}')` to list the actual targets before writing — the task's count and the real count often differ. Say which you'll process. +2. **Execute one item at a time**, applying the same read → write → verify loop per item. +3. **Be idempotent.** Check whether an item is already in the desired state before writing it; re-running the task must not duplicate nodes or stack changes. +4. **Track progress.** For long runs, keep a running tally so your final report can say "7 created, 1 already existed, 0 failed" with per-item paths. -**When creating nodes**, use the namespace from your task context. Before creating, explore what exists: -- `Search('namespace:{contextPath}')` — immediate children -- `Search('namespace:{contextPath} scope:descendants')` — full directory tree - -Never create under `Agent/` or other system namespaces unless explicitly asked. +Remember the #1 corruption bug on creates: `id` is the final slug with **no slashes**; `namespace` is the parent path. The full rules, schemas, and examples are in the Tools Reference below. # Tools Reference @@ -52,71 +47,6 @@ Never create under `Agent/` or other system namespaces unless explicitly asked. @@Agent/CommentingReference -# CRUD Workflows - -## Creating Nodes - -1. **Discover the schema**: `Get('@target-namespace/schema:')` to see required fields -2. **Construct the MeshNode JSON** with required properties: - - `id` — simple slug identifier, **NO slashes** (e.g., "PricingTool", "Q1-Report") - - `namespace` — full parent path (e.g., "ACME", "User/rbuergi"). This is where the node lives. - - `name` — descriptive human-readable title (ALWAYS required). Make it clear and meaningful. - - `nodeType` — must match an existing NodeType - - `icon` — **REQUIRED**: inline SVG icon (start with `...", "content": "..."}')` -4. **Verify**: `Get('@namespace/id')` to confirm creation - -**CRITICAL — id vs namespace:** -- `id` = simple slug, NO slashes: `"PricingTool"`, `"my-report"`, `"Q1Analysis"` -- `namespace` = full parent path WITH slashes: `"User/rbuergi"`, `"ACME/Projects"` -- The path is derived as `{namespace}/{id}`. Wrong id = corrupt data. -- **Wrong**: `id: "User/rbuergi/PricingTool"` — this is a PATH, not an id! -- **Right**: `id: "PricingTool", namespace: "User/rbuergi"` - -## Updating Nodes - -**For simple field changes (icon, name, content), use Patch — it's safer and simpler:** - -``` -Patch('@target-node', '{"icon": "..."}') -Patch('@target-node', '{"name": "New Name", "content": {...}}') -``` - -**For full node replacement, use Update with Get → Modify → Update:** - -1. **Get the full node**: `Get('@target-node')` — returns complete MeshNode JSON with ALL fields -2. **Modify** the returned JSON — change ONLY the fields you need. Keep everything else intact. -3. **Update**: `Update('[{...full modified MeshNode...}]')` — pass the COMPLETE node as JSON array - -**NEVER pass a partial node to Update** — it will be rejected. Update requires all fields including `nodeType` and `content`. Use **Patch** instead for partial changes. - -## Deleting Nodes - -1. **Confirm targets**: Use `Get` or `Search` to verify nodes exist -2. **Delete**: `Delete('["path1", "path2"]')` — paths as JSON array -3. **Verify**: Confirm with `Get` or `Search` - -## Managing Satellite Nodes - -Satellite nodes are child structures stored in dedicated sub-namespaces: - -- **Threads**: `{parentPath}/_Thread/{threadId}` — chat discussions -- **Comments**: `{parentPath}/_Comment/{commentId}` — document annotations -- **Activity**: `{parentPath}/_activity/{actId}` — activity logs - -To create a satellite node, use its dedicated namespace: -``` -Create('{"id": "my-thread", "namespace": "org/Doc/_Thread", "name": "Discussion", "nodeType": "Thread"}') -``` - -To find satellites: `Search('namespace:{parentPath}/_Thread nodeType:Thread')` - -# Guidelines +# Satellite nodes -- Be direct — execute tasks without unnecessary deliberation -- **ALWAYS write back.** When asked to update a node: `Get` it, modify it, then call `Update` or `Patch`. If you did not call Update/Patch, the change did NOT happen. Never just describe what you changed — call the tool. -- Always verify after write operations: `Get` the node to confirm it was saved correctly -- If a step fails, report the error — do not retry blindly -- Use SearchWeb/FetchWebPage for external information when needed -- Discover schemas before creating or updating nodes +Threads, comments, and other satellites live in underscore sub-namespaces (`{parentPath}/_Thread/{id}`, `{parentPath}/_Comment/{id}` — full table in the Tools Reference). Create them with the satellite namespace as `namespace`; find them with `Search('namespace:{parentPath}/_Thread nodeType:Thread')`. diff --git a/src/MeshWeaver.AI/Data/Skill/agent.md b/src/MeshWeaver.AI/Data/Skill/agent.md new file mode 100644 index 000000000..c8265518c --- /dev/null +++ b/src/MeshWeaver.AI/Data/Skill/agent.md @@ -0,0 +1,13 @@ +--- +nodeType: Skill +name: /agent +description: Switch the agent for subsequent messages +icon: Sparkle +category: Skills +order: 1 +action: + kind: Pick + query: "namespace:Agent nodeType:Agent -content.modelTier:utility sort:order" + field: agentName + title: Choose an agent +--- diff --git a/src/MeshWeaver.AI/Data/Skill/code.md b/src/MeshWeaver.AI/Data/Skill/code.md new file mode 100644 index 000000000..b81b4e11e --- /dev/null +++ b/src/MeshWeaver.AI/Data/Skill/code.md @@ -0,0 +1,52 @@ +--- +nodeType: Skill +name: /code +description: Bring in coding capability — create and modify NodeTypes, source, data models, layout areas, and scripts +icon: Sparkle +category: Skills +order: 4 +autoMount: true +--- + +You are now operating with **coding capability** for this MeshWeaver mesh. Create and modify +custom NodeTypes including their source code (`Source/`), data models, layout areas, reference +data, CSV loaders, JSON definitions, and executable Scripts. + +# 🚨 Read these architecture docs FIRST (non-negotiable) + +Before you write any handler, layout area, click action, service method, or Blazor view, internalise +these. Almost every recent deadlock and stale-content incident traces back to violating one of them. + +1. **[Asynchronous Calls](@/Doc/Architecture/AsynchronousCalls)** — no `Task` / `async` / `await` + in mesh-reachable code. Public methods on services, handlers, layout areas and click actions return + `IObservable` (or `void`); compose with `SelectMany` / `Select` / `Where`. Request/response is + `hub.Observe(request).Subscribe(onNext, onError)` — never the `[Obsolete]` `RegisterCallback` / + `AwaitResponse`, and never `Observable.FromAsync`. Click actions stay sync + (`ctx => { …; return Task.CompletedTask; }`). +2. **[CQRS — Queries vs. Content Access](@/Doc/Architecture/CqrsAndContentAccess)** — never read a known + node with `QueryAsync`/`Query` (lagged, stale right after a write). Live read = + `workspace.GetMeshNodeStream(path)`; one-shot = `hub.GetMeshNode(path, timeout?)`. `Query` is for + sets and existence, not single-node content. +3. **[Data Binding](@/Doc/GUI/DataBinding)** — the GUI is fully data-bound with ONE source of truth: the + node stream. Bind directly to `Hub.GetMeshNodeStream(path)` and write edits back via + `GetMeshNodeStream(path).Update(current => …)`. Never replicate a node into a layout-area `/data/{id}` + copy + a server-side save subscription, and never hand-roll HTML for structured data — use the + framework controls (`Controls.DataGrid`, `MeshNodeContentEditorControl`, `MarkdownEditorControl`). +4. **[Activity Control Plane](@/Doc/Architecture/ActivityControlPlane)** — every operation on a stateful + node is a property patch on the node's content, not a new message type. Pair `Status` (written only + by the owning hub) with `RequestedStatus` (patched by callers via `GetMeshNodeStream(path).Update(…)`). + Do not invent `CancelXRequest` / `RetryXRequest` message types. + +# Mutations go through `GetMeshNodeStream(path).Update(...)` + +Every mesh-node mutation goes through `workspace.GetMeshNodeStream(path).Update(current => modified)` +and **must be subscribed** (the observable is cold — the write runs on `Subscribe`). Create / delete / +move route through `meshService.CreateNode` / `DeleteNode` / `MoveNodeRequest`. Collections are +immutable (`ImmutableList`/`ImmutableDictionary`); never a `static` mutable collection. + +# Tests are reactive role models — no `await` in the test body + +Assert on the stream directly (`x.Should().Match(predicate)` / `.Emit()`) and drive +creates/updates from the assertion's subscribe. Use the test base, never mock `IMessageHub` / +`IMeshService`. Read the full guide in **[Coder.md](@/Agent/Coder)** before building NodeTypes, +data models, layout areas, or CSV loaders. diff --git a/src/MeshWeaver.AI/Data/Skill/create-space.md b/src/MeshWeaver.AI/Data/Skill/create-space.md new file mode 100644 index 000000000..7ad107ac0 --- /dev/null +++ b/src/MeshWeaver.AI/Data/Skill/create-space.md @@ -0,0 +1,37 @@ +--- +nodeType: Skill +name: /create-space +description: Create a new Space so everything works — proper create, a nice summary, a logo, and an optional repo link +icon: Sparkle +category: Skills +order: 5 +autoMount: true +--- + +You are creating a new **Space** — a top-level tenant container with its own partition, home page, and content. Follow these steps so the Space works end-to-end, not just renders an empty shell. + +# 1. Create it the right way — `create`, never `update` + +A Space MUST be created with a real **`create`** (CreateNodeRequest / MCP `create`). Creating triggers the server-side post-creation handler that **provisions the partition's Postgres schema, primes routing, and grants you Admin** at `{space}/_Access`. Converting a pre-existing bare node with `update` SKIPS that handler and leaves the Space half-provisioned (missing routing/grants → embedded areas don't load). + +- **Top-level only:** a Space's path is just its id (empty namespace). Use a short PascalCase id. +- Shape: `nodeType: "Space"`, content `{ "$type": "Space", "name": "..." }`. + +# 2. Write a nice summary (the most important step) + +Always author the Space's **`body`** (markdown) — a short, warm summary of **what the Space is about and what it's for**, plus how to get started. Do NOT leave it empty: an empty Space falls back to a generic welcome placeholder whose catalog embed shows nothing, which looks broken. Also set a one-line **`description`** (shown under the title). + +Write the summary in the owner's voice: its purpose, what kind of content lives here, and 2–4 bullet points on how to use it. A focused summary beats a wall of text. + +# 3. Give it a logo and an icon + +- **`icon`** — an inline SVG (or named icon) for the node, shown in lists and menus. +- **`logo`** — an image URL or data URI for the large header image (e.g. a served `/static/...svg`). Without a logo the header falls back to the node icon or the name's initials. + +# 4. (Optional) Link a GitHub repository + +To work on code from inside the Space, create a `{space}/_GitSync` node (`nodeType: GitHubSyncConfig`) with `repositoryUrl` + `branch` (default `main`). The **Code workspace** settings tab can then check the repo out, edit files in the browser, and commit + push as the user. + +# 5. Verify + +Open the Space's home page: the logo, name, and your summary should render (not the welcome placeholder), and `{space}/_Access` should hold your Admin grant. Then add the first pages or start a thread. diff --git a/src/MeshWeaver.AI/Data/Skill/harness.md b/src/MeshWeaver.AI/Data/Skill/harness.md new file mode 100644 index 000000000..85ec4e519 --- /dev/null +++ b/src/MeshWeaver.AI/Data/Skill/harness.md @@ -0,0 +1,13 @@ +--- +nodeType: Skill +name: /harness +description: Switch the harness (runtime) for subsequent messages +icon: Sparkle +category: Skills +order: 3 +action: + kind: Pick + query: "namespace:Harness nodeType:Harness sort:order" + field: harness + title: Choose a harness +--- diff --git a/src/MeshWeaver.AI/Data/Skill/model.md b/src/MeshWeaver.AI/Data/Skill/model.md new file mode 100644 index 000000000..d633561d6 --- /dev/null +++ b/src/MeshWeaver.AI/Data/Skill/model.md @@ -0,0 +1,13 @@ +--- +nodeType: Skill +name: /model +description: Switch the AI model for subsequent messages +icon: Sparkle +category: Skills +order: 2 +action: + kind: Pick + query: "namespace:_Provider nodeType:LanguageModel scope:descendants sort:order" + field: modelName + title: Choose a model +--- diff --git a/src/MeshWeaver.AI/Delegation/DelegationEvent.cs b/src/MeshWeaver.AI/Delegation/DelegationEvent.cs new file mode 100644 index 000000000..edfe0eb9f --- /dev/null +++ b/src/MeshWeaver.AI/Delegation/DelegationEvent.cs @@ -0,0 +1,41 @@ +namespace MeshWeaver.AI.Delegation; + +/// +/// A lifecycle event for a single in-flight delegation. Emitted on +/// by ExecuteDelegationAsync: +/// +/// — sub-thread node +/// created, parent's tool call has the path stamped on it. +/// — sub-thread has +/// reported IsExecuting=true at least once. +/// — sub-thread settled +/// (completed, cancelled, errored, or heartbeat-detected dead). +/// +/// +/// Single source of truth for "which sub-threads are this chat session +/// actively waiting on?" The cancel watcher and tool-call-stamper subscribe +/// to this stream; no separate registry / dictionary. Replaces the legacy +/// chat.DelegationPaths dictionary keyed by display-name. +/// +public sealed record DelegationEvent( + string CallId, + string SubThreadPath, + DelegationLifecycle Phase); + +/// +/// Lifecycle phase of a single delegation. Monotonic: +/// Dispatched → Active → Terminal. Phases skip is allowed (e.g. +/// Dispatched → Terminal if sub-thread creation fails before reaching +/// Active). +/// +public enum DelegationLifecycle +{ + /// Sub-thread node + cells created; tool call stamped with path. + Dispatched, + + /// Sub-thread has reported IsExecuting=true at least once. + Active, + + /// Sub-thread settled (completed / cancelled / errored / heartbeat-killed). + Terminal, +} diff --git a/src/MeshWeaver.AI/Delegation/DelegationHandlers.cs b/src/MeshWeaver.AI/Delegation/DelegationHandlers.cs new file mode 100644 index 000000000..abc523178 --- /dev/null +++ b/src/MeshWeaver.AI/Delegation/DelegationHandlers.cs @@ -0,0 +1,126 @@ +using System.Reactive.Linq; +using MeshWeaver.AI; +using MeshWeaver.Data; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using MeshThread = MeshWeaver.AI.Thread; + +namespace MeshWeaver.AI.Delegation; + +/// +/// Heartbeat-driven sub-thread cancellation handlers, registered on the +/// PARENT thread hub. Replaces the hard 5-min watchdog inside +/// ExecuteDelegationAsync: instead of a timeout that fires regardless +/// of whether the sub-thread is making progress, we observe each active +/// sub-thread's stamp and propagate +/// cancellation only when it stops advancing. +/// +internal static class DelegationHandlers +{ + /// Default heartbeat timeout when MeshThread.HeartbeatTimeout is null. + /// 10 s — well above any streaming chat client's normal delta cadence, low + /// enough that hung delegations are detected promptly. Tests requiring + /// even tighter detection override via MeshThread.HeartbeatTimeout. + public static readonly TimeSpan DefaultHeartbeatTimeout = TimeSpan.FromSeconds(10); + + /// Cold-start grace from ExecutionStartedAt. Sub-threads + /// often spend the first ~5 s on agent allocation + first-token latency + /// without writing LastActivityAt; 15 s covers that without + /// allowing real hangs to pin the user too long. + public static readonly TimeSpan ColdStartGrace = TimeSpan.FromSeconds(15); + + /// Periodic interval for the heartbeat scanner. 1 s keeps + /// hang-detection responsive — total worst-case latency is + /// HeartbeatInterval + HeartbeatTimeout. + public static readonly TimeSpan HeartbeatInterval = TimeSpan.FromSeconds(1); + + /// + /// On the PARENT thread hub. Periodic scanner (every 1 s). Reads the + /// hub's cached AgentChatClient.ActiveDelegationPaths (single + /// source of truth for "what sub-threads are this chat's delegations + /// currently waiting on?"), reads each sub-thread's MeshNode via the + /// process-wide cache, and applies the heartbeat predicate. On match, + /// posts back to this hub. + /// + internal static IMessageDelivery HandleHeartbeatTick( + IMessageHub hub, IMessageDelivery delivery) + { + var chat = hub.Get(); + if (chat is null || chat.ActiveDelegationPaths.IsEmpty) + return delivery.Processed(); + + var logger = hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.AI.Delegation"); + var nodeCache = hub.ServiceProvider.GetRequiredService(); + var now = DateTime.UtcNow; + + foreach (var subPath in chat.ActiveDelegationPaths) + { + // 🚨 TYPED overload — the bare GetStream(path) emits raw JsonElement + // Content, so `node.Content is not MeshThread` would be TRUE for every + // sub-thread and the heartbeat scan would silently never fire. + nodeCache.GetStream(subPath, hub.JsonSerializerOptions) + .Take(1) + .Timeout(TimeSpan.FromSeconds(2)) + .Subscribe( + node => + { + if (node?.Content is not MeshThread t || !t.IsExecuting) + return; + var timeout = t.HeartbeatTimeout ?? DefaultHeartbeatTimeout; + var startedAt = t.ExecutionStartedAt ?? now; + if (now - startedAt <= ColdStartGrace) return; + var lastActivity = t.LastActivityAt ?? startedAt; + if (now - lastActivity <= timeout) return; + + logger?.LogWarning( + "[DelegationHandlers] heartbeat stale sub={Path} sinceActivity={Since}s timeout={Timeout}s — cancelling", + subPath, + (int)(now - lastActivity).TotalSeconds, (int)timeout.TotalSeconds); + hub.Post(new CancelDelegationSubThread(subPath, + $"Heartbeat: no activity for {(int)(now - lastActivity).TotalSeconds}s")); + }, + _ => { /* swallow; next tick retries */ }); + } + + return delivery.Processed(); + } + + /// + /// On the PARENT thread hub. Sets + /// = Cancelled on the named sub-thread via the process-wide + /// — same write the GUI Stop button + /// performs, same propagation path. The sub-thread's own cancel watcher + /// reacts and tears down its CTS. + /// + internal static IMessageDelivery HandleCancelDelegationSubThread( + IMessageHub hub, IMessageDelivery delivery) + { + var req = delivery.Message; + var logger = hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.AI.Delegation"); + var nodeCache = hub.ServiceProvider.GetRequiredService(); + + logger?.LogInformation( + "[DelegationHandlers] CancelDelegationSubThread sub={Path} reason={Reason}", + req.SubThreadPath, req.Reason); + + // 🚨 TYPED overload — the bare Update(path, fn) does NOT deserialize + // JsonElement Content before the lambda, so `curr.Content is MeshThread` + // would be FALSE and the cancel would silently never be written. + nodeCache.Update(req.SubThreadPath, curr => + curr?.Content is MeshThread t + ? curr with { Content = t with { RequestedStatus = ThreadExecutionStatus.Cancelled } } + : curr!, + hub.JsonSerializerOptions) + .Subscribe( + _ => { }, + ex => logger?.LogWarning(ex, + "[DelegationHandlers] cancel write failed for {Path}", req.SubThreadPath)); + + return delivery.Processed(); + } +} diff --git a/src/MeshWeaver.AI/Delegation/DelegationMessages.cs b/src/MeshWeaver.AI/Delegation/DelegationMessages.cs new file mode 100644 index 000000000..3df11b086 --- /dev/null +++ b/src/MeshWeaver.AI/Delegation/DelegationMessages.cs @@ -0,0 +1,25 @@ +using MeshWeaver.Messaging; + +namespace MeshWeaver.AI.Delegation; + +/// +/// Posted by the parent thread hub to itself on a periodic timer +/// (Observable.Interval(1s) registered in WithInitialization). +/// The handler walks every active delegation sub-thread, reads its current +/// via the mesh-node cache, and applies the heartbeat +/// predicate: IsExecuting=true AND (now - LastActivityAt) > HeartbeatTimeout +/// AND (now - ExecutionStartedAt) > ColdStartGrace. On match it posts +/// . +/// +[SystemMessage] +public sealed record HeartbeatTick; + +/// +/// Posted to the parent thread hub by the heartbeat handler when a sub-thread +/// has gone unresponsive. The handler issues a single +/// nodeCache.Update(SubThreadPath, ... RequestedStatus = Cancelled) — +/// the SAME primitive the GUI Stop button uses — which propagates through the +/// sub-thread's own cancel watcher and tears down its CTS. +/// +[SystemMessage] +public sealed record CancelDelegationSubThread(string SubThreadPath, string Reason); diff --git a/src/MeshWeaver.AI/DelegationCompletedEvent.cs b/src/MeshWeaver.AI/DelegationCompletedEvent.cs deleted file mode 100644 index 06974424a..000000000 --- a/src/MeshWeaver.AI/DelegationCompletedEvent.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System.Collections.Concurrent; - -namespace MeshWeaver.AI; - -/// -/// Posted by ThreadExecution when execution completes, to notify the parent -/// thread's delegation tool that the child is done. -/// -public record DelegationCompletedEvent -{ - public required string ThreadPath { get; init; } - public string? ResponseText { get; init; } - public bool Success { get; init; } -} - -/// -/// Static tracker for pending delegations. The delegation tool registers a callback, -/// ThreadExecution posts the event, the handler on the thread hub resolves it. -/// -public static class DelegationTracker -{ - private static readonly ConcurrentDictionary> Pending = new(); - - public static void Register(string childThreadPath, Action onComplete) - => Pending[childThreadPath] = onComplete; - - public static bool TryComplete(DelegationCompletedEvent evt) - { - if (Pending.TryRemove(evt.ThreadPath, out var callback)) - { - callback(evt); - return true; - } - // Not found — log for debugging - System.Diagnostics.Debug.WriteLine($"[DelegationTracker] TryComplete: no pending for {evt.ThreadPath}, pending keys: [{string.Join(", ", Pending.Keys)}]"); - return false; - } -} diff --git a/src/MeshWeaver.AI/DescriptionGenerator.cs b/src/MeshWeaver.AI/DescriptionGenerator.cs new file mode 100644 index 000000000..dfad017e4 --- /dev/null +++ b/src/MeshWeaver.AI/DescriptionGenerator.cs @@ -0,0 +1,88 @@ +using System.Reactive.Linq; +using System.Text; +using System.Text.RegularExpressions; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Reactive; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI; + +/// +/// Default — spins up a fresh +/// per call, selects the built-in DescriptionWriter agent, sends a single user +/// message, and parses the Description: line from the response. +/// +public sealed class DescriptionGenerator : IDescriptionGenerator +{ + private readonly IServiceProvider services; + private readonly ILogger? logger; + + public DescriptionGenerator(IServiceProvider services) + { + this.services = services; + this.logger = (ILogger?)services.GetService(typeof(ILogger)); + } + + public IObservable GenerateDescriptionAsync(string name, string? category, CancellationToken ct = default) + { + var chat = new AgentChatClient(services); + chat.Initialize(contextPath: "Agent"); + return chat.WhenInitialized.Take(1).SelectMany(_ => + { + chat.SetSelectedAgent("DescriptionWriter"); + var prompt = BuildPrompt(name, category); + var messages = new[] { new ChatMessage(ChatRole.User, prompt) }; + return chat.GetResponseAsync(messages, ct).ToObservableSequence() + .Aggregate(new StringBuilder(), (sb, msg) => + { + foreach (var content in msg.Contents.OfType()) + sb.Append(content.Text); + return sb; + }) + .Select(sb => + { + var raw = sb.ToString(); + var description = ExtractDescription(raw); + if (string.IsNullOrEmpty(description)) + { + logger?.LogWarning("DescriptionWriter response did not contain a parsable Description line. Raw: {Raw}", raw); + throw new InvalidOperationException("Agent did not return a description."); + } + return description; + }); + }); + } + + private static string BuildPrompt(string name, string? category) + { + var safeName = string.IsNullOrWhiteSpace(name) ? "Untitled" : name.Trim(); + if (string.IsNullOrWhiteSpace(category)) + return $"Name: {safeName}"; + return $"Name: {safeName}\nCategory: {category.Trim()}"; + } + + // Matches the "Description: <...>" line in the DescriptionWriter response block. + private static readonly Regex DescriptionLineRegex = new( + @"(?im)^\s*Description:\s*(.+?)\s*$", + RegexOptions.Compiled); + + private static string? ExtractDescription(string text) + { + var match = DescriptionLineRegex.Match(text); + if (match.Success) + return match.Groups[1].Value.Trim().Trim('"'); + + // Fallback: first non-empty line, stripped of any leading "Description:" marker. + foreach (var line in text.Split('\n')) + { + var trimmed = line.Trim(); + if (string.IsNullOrEmpty(trimmed)) + continue; + if (trimmed.StartsWith("Description:", StringComparison.OrdinalIgnoreCase)) + trimmed = trimmed["Description:".Length..].Trim(); + return trimmed.Trim('"'); + } + return null; + } +} diff --git a/src/MeshWeaver.AI/DiffUtil.cs b/src/MeshWeaver.AI/DiffUtil.cs new file mode 100644 index 000000000..cc675f94b --- /dev/null +++ b/src/MeshWeaver.AI/DiffUtil.cs @@ -0,0 +1,86 @@ +using System.Collections.Generic; +using System.Text; + +namespace MeshWeaver.AI; + +/// +/// Minimal unified-diff generator for small JSON-node snapshots. Used by the MCP +/// tool responses so callers see exactly which lines changed after a +/// patch / update / create / delete mutation. +/// +/// The implementation is a naïve quadratic LCS — good enough for MeshNode JSON +/// (typically a few hundred lines pretty-printed). If we ever need to diff +/// massive payloads, swap the core for a Meyers-based algorithm behind the same +/// signature. +/// +internal static class DiffUtil +{ + /// + /// Produce a unified diff between two text blobs. The output begins with + /// standard --- / +++ headers and each line is prefixed with + /// - (removed), + (added), or a space (unchanged context). + /// A consumer can wrap the return value in a ```diff markdown fence + /// for syntax highlighting. + /// + public static string UnifiedDiff(string? before, string? after, string label) + { + var b = Split(before); + var a = Split(after); + var ops = ComputeOps(b, a); + + var sb = new StringBuilder(); + sb.Append("--- ").Append(label).Append(" (before)\n"); + sb.Append("+++ ").Append(label).Append(" (after)\n"); + foreach (var op in ops) + { + switch (op.Kind) + { + case '-': sb.Append('-').Append(b[op.Index]).Append('\n'); break; + case '+': sb.Append('+').Append(a[op.Index]).Append('\n'); break; + default: sb.Append(' ').Append(b[op.Index]).Append('\n'); break; + } + } + return sb.ToString(); + } + + private static string[] Split(string? s) => + (s ?? "").Replace("\r\n", "\n").Replace('\r', '\n').Split('\n'); + + private readonly record struct Op(char Kind, int Index); + + private static List ComputeOps(string[] b, string[] a) + { + // Quadratic LCS table. Rows index `before` (b), columns index `after` (a). + int m = b.Length, n = a.Length; + var len = new int[m + 1, n + 1]; + for (int i = 0; i < m; i++) + for (int j = 0; j < n; j++) + len[i + 1, j + 1] = b[i] == a[j] + ? len[i, j] + 1 + : System.Math.Max(len[i + 1, j], len[i, j + 1]); + + // Backtrack to emit ops in reverse-chronological order, then reverse. + var ops = new List(m + n); + int x = m, y = n; + while (x > 0 || y > 0) + { + if (x > 0 && y > 0 && b[x - 1] == a[y - 1]) + { + ops.Add(new Op(' ', x - 1)); + x--; y--; + } + else if (y > 0 && (x == 0 || len[x, y - 1] >= len[x - 1, y])) + { + ops.Add(new Op('+', y - 1)); + y--; + } + else + { + ops.Add(new Op('-', x - 1)); + x--; + } + } + ops.Reverse(); + return ops; + } +} diff --git a/src/MeshWeaver.AI/Harness.cs b/src/MeshWeaver.AI/Harness.cs new file mode 100644 index 000000000..14319dfe4 --- /dev/null +++ b/src/MeshWeaver.AI/Harness.cs @@ -0,0 +1,111 @@ +using System.ComponentModel.DataAnnotations; +using MeshWeaver.AI.Connect; +using MeshWeaver.Messaging; +using Microsoft.Extensions.AI; + +namespace MeshWeaver.AI; + +/// +/// A chat execution harness — the top-level choice of HOW a round runs. A +/// harness is not a model provider: it decides which execution library drives +/// the round. Three for now: +/// +/// MeshWeaver — the native agent + model system (provider factories). +/// Lives in MeshWeaver.AI. Surfaces agent + model selection. +/// Claude Code — the claude CLI via the Claude Agent SDK. Lives +/// in MeshWeaver.AI.ClaudeCode. +/// GitHub Copilot — the Copilot CLI. Lives in MeshWeaver.AI.Copilot. +/// +/// Stored as a MeshNode with nodeType="Harness" and Content=Harness. +/// +public record Harness +{ + /// Stable id — matches and the constants. + [Key] + public required string Id { get; init; } + + /// Friendly name for the picker (defaults to ). + public string? DisplayName { get; init; } + + public string? Description { get; init; } + + public string? Icon { get; init; } + + /// Display order in the harness picker (lower first). + public int Order { get; init; } + + /// Whether this harness is the default selection on a new thread. + public bool IsDefault { get; init; } + + /// + /// True when this harness surfaces agent + model selection (MeshWeaver). The CLI + /// harnesses run their own agent loop, so they hide the agent/model pickers. + /// + public bool SupportsAgentSelection { get; init; } +} + +/// +/// Runtime contract for a harness. Each harness lives in its own assembly and uses +/// its own library to run a round. Registered in DI (one per assembly); +/// projects each into a catalog +/// node so the picker and routing share one source of truth. +/// +public interface IHarness +{ + /// Stable id, matches and . + string Id { get; } + + /// The catalog definition surfaced as a node and in the picker. + Harness Definition { get; } + + /// + /// Creates the that runs a round under this harness, or + /// null to fall through to the default MeshWeaver agent/model path. The CLI + /// harnesses return their own library's client (e.g. the claude CLI) — so + /// they never touch the model-provider factory chain. + /// + IChatClient? CreateChatClient(HarnessExecutionContext context); + + /// + /// The slash-commands this harness OWNS. When this harness is the active one in the chat, these + /// drive BOTH the slash-command autocomplete (the harness is the authority for its command list) + /// AND dispatch — a non-MeshWeaver harness routes its own commands (e.g. /login, + /// /logout) to itself instead of MeshWeaver's /agent//model node-pickers. + /// Empty for the MeshWeaver harness (it keeps the node-pick commands). Default: none. + /// + IReadOnlyList Commands => []; + + /// + /// The Connect provider this harness authenticates per-user (Claude Code / GitHub Copilot), or + /// null for MeshWeaver (no per-user CLI login). The chat drives + /// / commands against this provider. + /// + ConnectProvider? AuthProvider => null; +} + +/// +/// A slash-command a owns. Pure data: the harness DECLARES its commands (so +/// the chat autocomplete lists them) and the chat view EXECUTES the (Connect / +/// Disconnect) against the harness's . Extensible by adding kinds. +/// +public sealed record HarnessCommand(string Name, string Description, HarnessCommandKind Kind) +{ + /// Usage syntax for help / the autocomplete detail (e.g. /login). + public string Usage => $"/{Name}"; +} + +/// What a does when run in the chat. +public enum HarnessCommandKind +{ + /// Log in / (re)authenticate this harness's subscription — drives the Connect flow inline. + Connect, + + /// Log out — forget this harness's stored per-user subscription token. + Disconnect, +} + +/// Inputs a harness needs to build its chat client for one round. +public sealed record HarnessExecutionContext( + IMessageHub Hub, + AgentConfiguration? Agent, + string? ModelName); diff --git a/src/MeshWeaver.AI/HarnessNodeType.cs b/src/MeshWeaver.AI/HarnessNodeType.cs new file mode 100644 index 000000000..31867700d --- /dev/null +++ b/src/MeshWeaver.AI/HarnessNodeType.cs @@ -0,0 +1,100 @@ +using MeshWeaver.Graph; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace MeshWeaver.AI; + +/// +/// The "Harness" node type. Harnesses are first-class catalog nodes (one per +/// registered ) describing the available execution harnesses. +/// The MeshWeaver harness is registered here; Claude Code / GitHub Copilot register +/// their own from their respective assemblies, and +/// projects every registered harness into a node. +/// +public static class HarnessNodeType +{ + /// NodeType discriminator for harness nodes. + public const string NodeType = "Harness"; + + /// Namespace (partition) the built-in harness catalog lives under. + public const string RootNamespace = "Harness"; + + /// + /// Registers the Harness type node, the built-in MeshWeaver harness, and the + /// catalog provider that turns every registered into a + /// read-only node under the Harness partition. + /// + public static TBuilder AddHarnessType(this TBuilder builder, + IReadOnlySet? serveFromPartition = null) where TBuilder : MeshBuilder + { + // When the "Harness" partition is DB-synced (static-repo import), DO NOT register the + // read-only in-memory static surfaces — on the distributed/PG path queries never consult + // the in-memory adapter, so the harness catalog would be invisible (empty picker / the + // combobox spins). The import materializes harnesses into the partition; PG serves them. + // Mirrors AddAgentType — see HarnessStaticRepoSource + Doc/Architecture/StaticRepoImport.md. + var dbSynced = serveFromPartition?.Contains(RootNamespace) == true; + + // The in-memory "Harness" NodeType definition. On the DB-synced path it is registered + // DEFINITION-ONLY: it still supplies the HubConfiguration delegate BY NAME (the catalog's + // instances enrich through it) and proves the type exists, but it is NOT served as the + // runtime node at @Harness — Postgres owns the nodeType:NodeType partition root the import + // materializes (HarnessStaticRepoSource.PartitionRoot). Without IsDefinitionOnly the + // in-memory type-def and the DB root BOTH claim @Harness → the bare-address GetDataRequest + // bounces → routing-loop guard fails it → ds/Harness faults → the harness picker vanishes. + // See Doc/Architecture/NodeTypeCatalogs.md. + var typeDefinition = CreateMeshNode(); + if (dbSynced) + typeDefinition = typeDefinition with { IsDefinitionOnly = true }; + builder.AddMeshNodes(typeDefinition); + builder.ConfigureNodeTypeAccess(a => a.WithPublicRead(NodeType)); + builder.ConfigureServices(services => + { + // The native MeshWeaver harness ships from this assembly. CLI harnesses + // add their own IHarness from their DLLs (TryAddEnumerable composes them). + // Registered regardless of dbSynced — the import SOURCE (HarnessStaticRepoSource) + // wraps BuiltInHarnessProvider, which reads this IHarness collection. + services.TryAddEnumerable(ServiceDescriptor.Singleton()); + services.TryAddSingleton(); + if (!dbSynced) + { + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => + new StaticNodePartitionStorageProvider( + RootNamespace, + sp.GetRequiredService(), + description: "Built-in harness definitions (read-only).")); + } + return services; + }); + return builder; + } + + /// + /// Resolves the registered for + /// (the value stored in — a bare id or a picked + /// node PATH like Harness/MeshWeaver, normalized via ), + /// or null when none matches — in which case execution uses the default MeshWeaver + /// agent/model path. + /// + public static IHarness? ResolveHarness(IServiceProvider services, string? harnessId) + { + var id = SelectionId.IdOf(harnessId); + return string.IsNullOrEmpty(id) + ? null + : services.GetServices() + .FirstOrDefault(h => string.Equals(h.Id, id, StringComparison.OrdinalIgnoreCase)); + } + + /// The type-definition node for nodeType="Harness". + public static MeshNode CreateMeshNode() => new(NodeType) + { + Name = "Harness", + Icon = "/static/NodeTypeIcons/bot.svg", + IsSatelliteType = false, + HubConfiguration = config => config + .AddMeshDataSource(source => source + .WithContentType()) + }; +} diff --git a/src/MeshWeaver.AI/HarnessStaticRepoSource.cs b/src/MeshWeaver.AI/HarnessStaticRepoSource.cs new file mode 100644 index 000000000..6e41654ff --- /dev/null +++ b/src/MeshWeaver.AI/HarnessStaticRepoSource.cs @@ -0,0 +1,60 @@ +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Mesh; + +namespace MeshWeaver.AI; + +/// +/// The built-in harnesses as a static-repo import source for the Harness partition. The same +/// nodes serves in-memory are materialized into the DB partition +/// by the static-repo import on boot, so harnesses are served from the database on the distributed/PG +/// path (Orleans routing does NOT consult the in-memory adapter — without this import the harness +/// catalog is invisible to namespace:Harness queries, i.e. the harness picker is empty / the +/// combobox spins). Mirrors . +/// +public sealed class HarnessStaticRepoSource(BuiltInHarnessProvider provider) : IStaticRepoSource +{ + /// + public string Partition => HarnessNodeType.RootNamespace; + + /// + // Harness definitions ship with no meaningful version → fingerprint on content. + public bool Versioned => false; + + /// + // Content harness nodes PLUS the partition's PublicRead "_Policy" (PartitionAccessPolicy). On the + // SYNCED path the in-memory provider that used to serve the policy is gated off, so WITHOUT + // importing it the Harness partition has NO read policy → every user (even admins — partitions are + // not data-superuser readable) is denied Read → the Harness/MeshWeaver hub init throws + // UnauthorizedAccessException → FAILED hub → the chat composer's harness picker can't load → the + // composer disappears (atioz 2026-06-15; Orleans repro: OrleansHarnessPartitionPublicReadTest). + // Only OTHER "_"-governance (e.g. per-user _Access grants) is dropped; the partition-level access + // policy MUST travel to the DB partition. + public IReadOnlyList EnumerateSourceNodes() => + provider.GetStaticNodes() + .Where(n => n.NodeType == "PartitionAccessPolicy" + || !n.Segments.Skip(1).Any(seg => seg.StartsWith('_'))) + .ToArray(); + + /// + // The partition root is the catalog's single nodeType:NodeType node (id = the type name) — + // NOT a nodeType:Space root. It IS the routable partition root AND the "Harness" NodeType + // definition, and it LINKS (NodeTypeDefinition.StaticTypeName) to the registered static C# + // type "Harness" for its HubConfiguration (a non-serialisable delegate, supplied in-memory). + // Postgres owns this node — the sole runtime owner of @Harness — so the in-memory type-def + // (registered definition-only) never collides with it. See Doc/Architecture/NodeTypeCatalogs.md. + public MeshNode? PartitionRoot => new(HarnessNodeType.RootNamespace) + { + Name = "Harnesses", + NodeType = MeshNode.NodeTypePath, + Icon = "/static/NodeTypeIcons/bot.svg", + State = MeshNodeState.Active, + Content = new NodeTypeDefinition + { + StaticTypeName = HarnessNodeType.NodeType, + Description = + "The available execution harnesses (runtimes) — MeshWeaver's native agent loop, and " + + "any CLI harness (Claude Code, GitHub Copilot) whose assembly is deployed. Pick one " + + "per thread with `/harness`." + } + }; +} diff --git a/src/MeshWeaver.AI/Harnesses.cs b/src/MeshWeaver.AI/Harnesses.cs new file mode 100644 index 000000000..3045db856 --- /dev/null +++ b/src/MeshWeaver.AI/Harnesses.cs @@ -0,0 +1,39 @@ +namespace MeshWeaver.AI; + +/// +/// The execution harnesses a thread can run under. A "harness" is the +/// top-level choice in the chat picker; it maps onto an agent's +/// (projected from the agent node's Category). +/// For Claude Code / GitHub Copilot +/// the harness is the choice — each resolves to a single built-in agent. +/// For MeshWeaver the user additionally picks an agent + model within the group. +/// +/// +/// These are immutable constant lookups (no runtime writes), so the +/// array is a sanctioned static readonly under the +/// no-static-state rule. +/// +public static class Harnesses +{ + // 🚨 These ids are used VERBATIM as the harness mesh-node id: BuiltInHarnessProvider + // creates `new MeshNode(Id, "Harness")` → path `Harness/{Id}`. So an id MUST be a + // path-safe slug (no spaces) — a space here produced `Harness/Claude Code`, and + // reading that space-containing path back tripped the resolver's space fragility → + // NotFound → resubscribe storm → the "harness change crashes" bug. The friendly label + // lives on `Harness.DisplayName` (the picker shows that, never the id). + + /// The native MeshWeaver agent harness — exposes agent + model selection. + public const string MeshWeaver = "MeshWeaver"; + + /// The Claude Code harness id (slug; display name "Claude Code"). + public const string ClaudeCode = "ClaudeCode"; + + /// The GitHub Copilot harness id (slug; display name "GitHub Copilot"). + public const string Copilot = "Copilot"; + + /// + /// All harnesses in picker display order. MeshWeaver leads (it is the + /// default), followed by the external harnesses. + /// + public static readonly string[] All = [MeshWeaver, ClaudeCode, Copilot]; +} diff --git a/src/MeshWeaver.AI/HubThreadExtensions.cs b/src/MeshWeaver.AI/HubThreadExtensions.cs new file mode 100644 index 000000000..e1fee2043 --- /dev/null +++ b/src/MeshWeaver.AI/HubThreadExtensions.cs @@ -0,0 +1,571 @@ +using System.Collections.Immutable; +using System.Reactive.Linq; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Layout; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using MeshThread = MeshWeaver.AI.Thread; + +namespace MeshWeaver.AI; + +/// +/// Canonical client-side surface for thread operations. All thread mutations +/// — create, submit, resubmit, delete-from, mark-done, record-failure — go +/// through these extensions. The extensions write +/// to the thread node via hub.GetWorkspace().GetMeshNodeStream(threadPath).Update(...) +/// (or for new-thread lifecycle); the +/// per-thread submission watcher reacts to the resulting state changes. +/// +/// Tests, GUI, and agents all call these methods. There is no +/// other entry point — no SubmitMessageRequest, no parameter-bag +/// context records, no direct hub.Post shortcuts. If you find yourself +/// needing one, the answer is to extend this surface (or fold the new +/// operation into the existing thread-node state machine). +/// +/// All methods are void / fire-and-forget. Callers observe +/// confirmation by subscribing to the thread node's remote stream (the same +/// stream the UI already binds to). The optional onError / onCreated +/// callbacks exist for one-shot signalling (e.g. the chat view's "navigate +/// to the new thread once it's created") and fire exactly once. +/// +public static class HubThreadExtensions +{ + // ═════════════════════════════════════════════════════════════════════ + // Create + submit (new thread) + // ═════════════════════════════════════════════════════════════════════ + + /// + /// Creates a new thread under and queues + /// the first user message on it. The thread node is created via + /// (node-lifecycle, not a mutation) and + /// pre-seeded with so the + /// submission watcher dispatches the first round as soon as the thread + /// hub activates — no second round-trip. + /// + /// , when supplied, is COPIED onto the created + /// thread as — the thread's own data-bound + /// chat-input state — with the draft + attachments emptied (the draft became the + /// first message) and the navigation signal cleared. This is how the out-of-thread + /// composer's selection (harness/agent/model paths) carries into the thread. + /// + public static void StartThread( + this IMessageHub hub, + string namespacePath, + string userText, + string? agentName = null, + string? modelName = null, + string? contextPath = null, + IReadOnlyList? attachments = null, + string? createdBy = null, + string? authorName = null, + Action? onCreated = null, + Action? onError = null, + string? mainNode = null, + string? speakingId = null, + string? harness = null, + ThreadComposer? composer = null) + { + ArgumentNullException.ThrowIfNull(hub); + if (string.IsNullOrEmpty(namespacePath)) + { + onError?.Invoke("StartThread requires namespacePath."); + return; + } + + var threadNode = ThreadNodeType.BuildThreadNode(namespacePath, userText, createdBy, speakingId); + // Optional: point the thread at an existing node (e.g. an inbound Email) as its MainNode, + // so the agent's context is that node and consumers can navigate thread → source. + if (!string.IsNullOrEmpty(mainNode)) + threadNode = threadNode with { MainNode = mainNode }; + + // 🚫 Nothing-to-do: whitespace-only first message (e.g. the user ran a slash command — + // the command text is CUT and what remains is empty). Create the thread (the composer + // selection still carries onto it) but seed NO pending message — there is nothing to + // submit, so the submission watcher must NOT dispatch a round (which would reach + // CreateChatClient with no input and storm "No model selected"). The command's side + // effect (agent/model/harness pick) already happened on the composer. + var hasFirstMessage = !string.IsNullOrWhiteSpace(userText); + var firstMessageId = Guid.NewGuid().ToString("N")[..8]; + + // 🎯 The thread's COMPOSER is the single source of truth for the round's sticky + // selection (agent / model / harness / context). Seed it from the supplied composer + // snapshot when present, else from the explicit agent/model/harness/context params — + // either way the created thread ALWAYS carries a composer so the submission watcher's + // PlanNextRound can read the selection from Thread.Composer. The draft + attachments + // are consumed by the first message; the navigate-signal never carries over. + var seedComposer = (composer ?? new ThreadComposer + { + AgentName = agentName, + ModelName = modelName, + Harness = harness, + ContextPath = contextPath + }) + with { MessageContent = null, Attachments = null, OpenThreadPath = null }; + + var baseThread = (threadNode.Content as MeshThread ?? new MeshThread()) with + { + Composer = seedComposer + }; + + var seededThread = hasFirstMessage + ? baseThread with + { + Messages = ImmutableList.Create(firstMessageId), + UserMessageIds = ImmutableList.Create(firstMessageId), + // The pending ThreadMessage records the per-message context + attachments and a + // historical stamp of the agent/model/harness; the round's SELECTION is read from + // Thread.Composer (the single source), not from this message. + PendingUserMessages = ImmutableDictionary.Empty + .SetItem(firstMessageId, ThreadInput.CreateUserMessage( + userText, + createdBy: createdBy, + authorName: authorName, + agentName: agentName, + modelName: modelName, + contextPath: contextPath, + attachments: attachments, + harness: harness)) + } + : baseThread; // empty thread — no round + threadNode = threadNode with { Content = seededThread }; + + var delivery = hub.Post( + new CreateNodeRequest(threadNode), + o => o.WithTarget(new Address(namespacePath))); + + if (delivery == null) + { + onError?.Invoke("Hub.Post returned null"); + return; + } + + hub.Observe((IMessageDelivery)delivery) + .Subscribe( + response => + { + if (response.Message is not CreateNodeResponse { Success: true } cnr) + { + var err = (response.Message as CreateNodeResponse)?.Error ?? "unknown"; + onError?.Invoke($"Thread creation failed: {err}"); + return; + } + onCreated?.Invoke(cnr.Node ?? threadNode); + }, + ex => onError?.Invoke($"Thread creation failed: {ex.Message}")); + } + + // ═════════════════════════════════════════════════════════════════════ + // Submit (existing thread) + // ═════════════════════════════════════════════════════════════════════ + + /// + /// Submits a user message into an existing thread. Writes + /// via + /// hub.GetWorkspace().GetMeshNodeStream(threadPath).Update(...). + /// The per-thread submission watcher drains the queue into a new round. + /// + public static void SubmitMessage( + this IMessageHub hub, + string threadPath, + string userText, + string? agentName = null, + string? modelName = null, + string? contextPath = null, + IReadOnlyList? attachments = null, + string? createdBy = null, + string? authorName = null, + Action? onError = null, + string? harness = null) + { + ArgumentNullException.ThrowIfNull(hub); + if (string.IsNullOrEmpty(threadPath)) + { + onError?.Invoke("SubmitMessage requires threadPath. Use StartThread for new threads."); + return; + } + + // 🚫 Nothing-to-do: whitespace-only text (e.g. the user ran a slash command — the + // command text is CUT and what remains is empty). There is nothing to submit, so do + // NOT append a pending message: enqueuing an empty round would reach CreateChatClient + // with no input and storm "No model selected". The command's side effect already ran. + if (string.IsNullOrWhiteSpace(userText)) + return; + + var userMessage = ThreadInput.CreateUserMessage( + userText ?? string.Empty, + createdBy: createdBy, + authorName: authorName, + agentName: agentName, + modelName: modelName, + contextPath: contextPath, + attachments: attachments, + harness: harness); + try + { + ThreadInput.AppendUserInput(hub.GetWorkspace(), threadPath, userMessage); + } + catch (Exception ex) + { + var logger = hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.AI.HubThreadExtensions"); + logger?.LogWarning(ex, "[SubmitMessage] AppendUserInput threw for {ThreadPath}", threadPath); + onError?.Invoke($"SubmitMessage failed: {ex.Message}"); + } + } + + // ═════════════════════════════════════════════════════════════════════ + // Submit from the thread's composer (drain + empty, one atomic update) + // ═════════════════════════════════════════════════════════════════════ + + /// + /// Submits the thread's own composer () as the next + /// user message — ONE atomic stream.Update on the thread node that + /// (a) builds the user message from (or, when null, the + /// composer's persisted draft) carrying the + /// composer's harness/agent/model selection (picked node paths — normalized at the + /// execution boundary), (b) queues it via + /// ( + Pending* hints), and + /// (c) EMPTIES the composer (draft + attachments). The per-thread submission watcher + /// reacts to the resulting state change and dispatches the round. + /// + /// No-op when there is no text to submit. and + /// override the composer's persisted values when supplied + /// (the Blazor chat passes its live nav context + attachment chips). + /// + public static void SubmitComposer( + this IMessageHub hub, + string threadPath, + string? userText = null, + string? contextPath = null, + IReadOnlyList? attachments = null, + string? createdBy = null, + string? authorName = null, + Action? onError = null) + { + ArgumentNullException.ThrowIfNull(hub); + if (string.IsNullOrEmpty(threadPath)) + { + onError?.Invoke("SubmitComposer requires threadPath."); + return; + } + + var logger = hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.AI.HubThreadExtensions"); + var msgId = Guid.NewGuid().ToString("N")[..8]; + + hub.GetWorkspace().GetMeshNodeStream(threadPath).Update(node => + { + var t = node.ContentAs(hub.JsonSerializerOptions, logger); + // Existing node whose content can't be recovered → leave it alone, NEVER clobber. + if (node.Content is not null && t is null) + return node; + t ??= new MeshThread(); + var c = t.Composer ?? new ThreadComposer(); + var text = !string.IsNullOrWhiteSpace(userText) ? userText : c.MessageContent; + if (string.IsNullOrWhiteSpace(text)) + return node; // nothing to submit — no-op (byte-identical node dedupes downstream) + + var message = ThreadInput.CreateUserMessage( + text!, + createdBy: createdBy, + authorName: authorName, + agentName: c.AgentName, + modelName: c.ModelName, + contextPath: contextPath ?? c.ContextPath, + attachments: attachments ?? c.Attachments, + harness: c.Harness); + + return node with + { + Content = ThreadInput.ApplyUserInput(t, msgId, message) with + { + Composer = c with { MessageContent = null, Attachments = null } + } + }; + }).Subscribe( + _ => { }, + ex => + { + logger?.LogWarning(ex, + "SubmitComposer: thread Update failed for {ThreadPath}", threadPath); + onError?.Invoke($"SubmitComposer failed: {ex.Message}"); + }); + } + + // ═════════════════════════════════════════════════════════════════════ + // Resubmit (truncate after a user message and re-queue it) + // ═════════════════════════════════════════════════════════════════════ + + /// + /// Truncates the thread after and re-queues + /// it as a new pending user message. Single stream.Update on the + /// thread node: drops Messages after the resubmit point, removes the + /// id from IngestedMessageIds, puts the (optionally edited) user + /// message back into PendingUserMessages, and resets Status + /// to Idle. The submission watcher then dispatches the next round + /// naturally. The user cell's text — a separate node — is updated through + /// the shared when + /// is supplied. + /// + public static void ResubmitMessage( + this IMessageHub hub, + string threadPath, + string userMessageId, + string? newUserText = null, + string? agentName = null, + string? modelName = null, + Action? onError = null, + string? harness = null) + { + ArgumentNullException.ThrowIfNull(hub); + if (string.IsNullOrEmpty(threadPath) || string.IsNullOrEmpty(userMessageId)) + { + onError?.Invoke("ResubmitMessage requires threadPath and userMessageId."); + return; + } + + var logger = hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.AI.HubThreadExtensions"); + + hub.GetWorkspace().GetMeshNodeStream(threadPath).Update(node => + { + var t = node.Content as MeshThread; + if (t is null) return node; + + var idx = t.Messages.IndexOf(userMessageId); + if (idx < 0) return node; // id not in thread — no-op + + var keep = t.Messages.Take(idx).ToImmutableList(); + var trimmedUserIds = t.UserMessageIds + .Where(uid => keep.Contains(uid) || uid == userMessageId) + .ToImmutableList(); + if (!trimmedUserIds.Contains(userMessageId)) + trimmedUserIds = trimmedUserIds.Add(userMessageId); + + var ingested = t.IngestedMessageIds.Remove(userMessageId); + var replayMessage = new ThreadMessage + { + Role = "user", + Text = newUserText ?? "", + Timestamp = DateTime.UtcNow, + Type = ThreadMessageType.ExecutedInput, + AgentName = agentName, + ModelName = modelName, + Harness = harness + }; + var pending = t.PendingUserMessages.SetItem(userMessageId, replayMessage); + + return node with + { + Content = t with + { + Messages = keep, + UserMessageIds = trimmedUserIds, + IngestedMessageIds = ingested, + // The replay ThreadMessage carries the selection (agent/model/harness) — + // no thread-level Pending* mirror. + PendingUserMessages = pending, + Status = ThreadExecutionStatus.Idle, + ActiveMessageId = null, + ExecutionStartedAt = null + } + }; + }).Subscribe( + _ => { }, + ex => + { + logger?.LogWarning(ex, + "ResubmitMessage: thread Update failed for {ThreadPath} message {MessageId}", + threadPath, userMessageId); + onError?.Invoke($"ResubmitMessage failed: {ex.Message}"); + }); + + // Cell-text update — the per-message satellite is a SEPARATE node, so + // it goes through the shared cache rather than the thread-node Update. + // Independent of the thread Update above; no ordering dependency. Only + // runs when the caller supplied new text. + if (!string.IsNullOrEmpty(newUserText)) + { + var cellPath = $"{threadPath}/{userMessageId}"; + hub.GetMeshNodeStream(cellPath).Update(node => + { + var existing = node.Content as ThreadMessage; + var nextContent = existing is not null + ? existing with { Text = newUserText!, Timestamp = DateTime.UtcNow } + : new ThreadMessage + { + Role = "user", + Text = newUserText!, + Timestamp = DateTime.UtcNow, + Type = ThreadMessageType.ExecutedInput + }; + return node with + { + NodeType = node.NodeType ?? ThreadMessageNodeType.NodeType, + Content = nextContent + }; + }).Subscribe( + _ => { }, + ex => logger?.LogWarning(ex, + "ResubmitMessage: cell-text Update failed for {CellPath}", cellPath)); + } + } + + // ═════════════════════════════════════════════════════════════════════ + // Delete from message (truncate Messages list at the given message) + // ═════════════════════════════════════════════════════════════════════ + + /// + /// Truncates starting at + /// (exclusive — drops + /// and everything after). Single stream.Update on the thread node; + /// no watcher indirection. + /// + public static void DeleteFromMessage( + this IMessageHub hub, string threadPath, string atMessageId) + { + ArgumentNullException.ThrowIfNull(hub); + if (string.IsNullOrEmpty(threadPath) || string.IsNullOrEmpty(atMessageId)) + return; + + var logger = hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.AI.HubThreadExtensions"); + hub.GetWorkspace().GetMeshNodeStream(threadPath).Update(node => + { + var t = node.Content as MeshThread; + if (t is null) return node; + var idx = t.Messages.IndexOf(atMessageId); + if (idx < 0) return node; // id not in thread — no-op + return node with + { + Content = t with { Messages = t.Messages.Take(idx).ToImmutableList() } + }; + }).Subscribe( + _ => { }, + ex => logger?.LogWarning(ex, + "DeleteFromMessage: Update failed for thread {ThreadPath} message {MessageId}", + threadPath, atMessageId)); + } + + // ═════════════════════════════════════════════════════════════════════ + // Terminal state — Done / Idle + // ═════════════════════════════════════════════════════════════════════ + + /// + /// Marks the thread (terminal, + /// hidden from default catalogs) or re-opens it by flipping back to + /// . Refuses to act while a round + /// is in flight (the CAS check lives in the Update lambda). + /// + public static void MarkThreadDone(this IMessageHub hub, string threadPath, bool done) + { + ArgumentNullException.ThrowIfNull(hub); + if (string.IsNullOrEmpty(threadPath)) + return; + + var logger = hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.AI.HubThreadExtensions"); + hub.GetWorkspace().GetMeshNodeStream(threadPath).Update(node => + { + var t = node.ContentAs(hub.JsonSerializerOptions, logger); + // Existing node whose content can't be recovered → leave it alone, NEVER clobber. + if (node.Content is not null && t is null) + return node; + t ??= new MeshThread(); + if (t.IsExecuting) + { + logger?.LogInformation( + "MarkThreadDone: ignored — thread {ThreadPath} is executing (Status={Status})", + threadPath, t.Status); + return node; + } + var newStatus = done ? ThreadExecutionStatus.Done : ThreadExecutionStatus.Idle; + if (t.Status == newStatus) return node; + return node with { Content = t with { Status = newStatus } }; + }).Subscribe( + _ => { }, + ex => logger?.LogWarning(ex, + "MarkThreadDone: stream.Update failed for thread {ThreadPath} done={Done}", + threadPath, done)); + } + + // ═════════════════════════════════════════════════════════════════════ + // Record failure (one-shot pending entry) + // ═════════════════════════════════════════════════════════════════════ + + /// + /// Records a failed submission by creating an error satellite cell and + /// updating the thread node in one chained operation: CreateNode + /// for the error cell, then a single stream.Update on the thread + /// node to append both the user-message id and the error-cell id to + /// Messages + bookkeeping in UserMessageIds / + /// IngestedMessageIds. No intent indirection, no watcher. + /// + public static void RecordSubmissionFailure( + this IMessageHub hub, + string threadPath, + string userMessageId, + string userText, + string errorMessage) + { + ArgumentNullException.ThrowIfNull(hub); + if (string.IsNullOrEmpty(threadPath)) + return; + + var errorCellId = Guid.NewGuid().ToString("N")[..8]; + var logger = hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.AI.HubThreadExtensions"); + var meshService = hub.ServiceProvider.GetRequiredService(); + var workspace = hub.GetWorkspace(); + + var errorCell = new MeshNode(errorCellId, threadPath) + { + NodeType = ThreadMessageNodeType.NodeType, + MainNode = threadPath, + Content = new ThreadMessage + { + Role = "assistant", + Text = $"**Submission failed:** {errorMessage}", + Timestamp = DateTime.UtcNow, + Type = ThreadMessageType.AgentResponse + } + }; + + meshService.CreateNode(errorCell) + .SelectMany(_ => workspace.GetMeshNodeStream(threadPath).Update(node => + { + var t = node.ContentAs(hub.JsonSerializerOptions, logger); + // Existing node whose content can't be recovered → leave it alone, NEVER clobber. + if (node.Content is not null && t is null) + return node; + t ??= new MeshThread(); + var msgs = t.Messages; + if (!msgs.Contains(userMessageId)) msgs = msgs.Add(userMessageId); + if (!msgs.Contains(errorCellId)) msgs = msgs.Add(errorCellId); + var userIds = t.UserMessageIds.Contains(userMessageId) + ? t.UserMessageIds + : t.UserMessageIds.Add(userMessageId); + var ingested = t.IngestedMessageIds.Contains(userMessageId) + ? t.IngestedMessageIds + : t.IngestedMessageIds.Add(userMessageId); + return node with + { + Content = t with + { + Messages = msgs, + UserMessageIds = userIds, + IngestedMessageIds = ingested + } + }; + })) + .Subscribe( + _ => { }, + ex => logger?.LogWarning(ex, + "RecordSubmissionFailure: CreateNode+Update failed for {ThreadPath} message {MessageId}", + threadPath, userMessageId)); + } +} diff --git a/src/MeshWeaver.AI/IAgentChat.cs b/src/MeshWeaver.AI/IAgentChat.cs index f19c7c13a..3ce1c3a64 100644 --- a/src/MeshWeaver.AI/IAgentChat.cs +++ b/src/MeshWeaver.AI/IAgentChat.cs @@ -1,4 +1,5 @@ -using MeshWeaver.AI.Persistence; +using MeshWeaver.AI.Delegation; +using MeshWeaver.AI.Persistence; using MeshWeaver.Layout; using Microsoft.Extensions.AI; @@ -75,24 +76,15 @@ void RequestHandoff(HandoffRequest request) { } ThreadExecutionContext? ExecutionContext => null; /// - /// Path of the last delegation sub-thread created. Consumed by ThreadExecution - /// after each delegation tool call completes. - /// Deprecated: use DelegationPaths dictionary for parallel-safe delegation tracking. + /// Live stream of delegation lifecycle events + /// (Dispatched → Active → Terminal). Single source of truth for + /// "which sub-threads is this chat session waiting on?" The cancel + /// watcher subscribes here and maintains its own active-set; the + /// tool-call stamper writes the path onto the parent's tool-call entry + /// on the Dispatched event. /// - string? LastDelegationPath { get => null; set { } } - - /// - /// Thread-safe mapping of delegation display name to sub-thread path. - /// Supports parallel delegations without race conditions. - /// - System.Collections.Concurrent.ConcurrentDictionary DelegationPaths - => new(); - - /// - /// Callback to update delegation status on the parent execution context. - /// Set by ThreadExecution, called by delegation tools to forward sub-agent progress. - /// - Action? UpdateDelegationStatus { get => null; set { } } + IObservable Delegations + => System.Reactive.Linq.Observable.Empty(); /// /// Callback to forward tool call entries from sub-threads to the parent's tool call log. diff --git a/src/MeshWeaver.AI/IChatClientFactory.cs b/src/MeshWeaver.AI/IChatClientFactory.cs index 020304171..f31302630 100644 --- a/src/MeshWeaver.AI/IChatClientFactory.cs +++ b/src/MeshWeaver.AI/IChatClientFactory.cs @@ -29,6 +29,19 @@ public interface IChatClientFactory /// bool IsPersistent => false; + /// + /// Returns true when this factory can serve . + /// Used for factory selection: the chat composer's selected model drives which factory + /// creates the chat client. The default implementation honours the + /// legacy list — concrete factories should override with a + /// shape-aware predicate (e.g. "claude-*" → AzureClaude, "*" → AzureFoundry + /// gateway as catch-all) so routing works even when is empty + /// (which is the default after the model-config-from-env-vars cleanup). + /// + bool Supports(string modelName) => + !string.IsNullOrEmpty(modelName) && Models.Any(m => + string.Equals(m, modelName, StringComparison.OrdinalIgnoreCase)); + /// /// Creates a ChatClientAgent for the given configuration. /// diff --git a/src/MeshWeaver.AI/IMasterKeyProvider.cs b/src/MeshWeaver.AI/IMasterKeyProvider.cs new file mode 100644 index 000000000..5daea80c3 --- /dev/null +++ b/src/MeshWeaver.AI/IMasterKeyProvider.cs @@ -0,0 +1,21 @@ +namespace MeshWeaver.AI; + +/// +/// Supplies the symmetric master key used by +/// to encrypt/decrypt stored provider credentials. Pluggable so the master key +/// can come from configuration (the default — see ) +/// or from an external KMS / Azure Key Vault in a hardened deployment. +/// +/// Returns null when no master key is configured. In that case +/// operates in passthrough mode (keys are +/// stored as plaintext, exactly as before this feature existed) so a dev box or +/// test with no key configured keeps working. +/// +public interface IMasterKeyProvider +{ + /// + /// The 32-byte (AES-256) master key, or null if encryption is disabled. + /// Implementations should cache — this is called on every protect/unprotect. + /// + byte[]? GetMasterKey(); +} diff --git a/src/MeshWeaver.AI/IProviderKeyProtector.cs b/src/MeshWeaver.AI/IProviderKeyProtector.cs new file mode 100644 index 000000000..61d9af825 --- /dev/null +++ b/src/MeshWeaver.AI/IProviderKeyProtector.cs @@ -0,0 +1,31 @@ +namespace MeshWeaver.AI; + +/// +/// Encrypts / decrypts the literal credential stored on a +/// before it is persisted to +/// (and read back from) the mesh — i.e. Postgres. Answers "is it safe to keep +/// LLM keys in PG": with a master key configured the value at rest is AES-256-GCM +/// ciphertext, so a DB / backup leak alone yields no usable key. +/// +/// Backward compatible: is idempotent and +/// passes through any value not carrying the +/// enc: tag, so pre-existing plaintext rows keep working and re-saving +/// re-encrypts them. With no master key configured (see +/// ) both methods are pure passthrough. +/// +public interface IProviderKeyProtector +{ + /// + /// Returns an enc:v1:-tagged ciphertext for , + /// or the input unchanged when it is null/empty, already tagged, or encryption + /// is disabled. + /// + string? Protect(string? plaintext); + + /// + /// Reverses . Returns the input unchanged when it is + /// null/empty or untagged (legacy plaintext); returns null when a + /// tagged value cannot be decrypted (wrong/missing master key). + /// + string? Unprotect(string? stored); +} diff --git a/src/MeshWeaver.AI/IconGenerator.cs b/src/MeshWeaver.AI/IconGenerator.cs new file mode 100644 index 000000000..dc0edf5f3 --- /dev/null +++ b/src/MeshWeaver.AI/IconGenerator.cs @@ -0,0 +1,83 @@ +using System.Reactive.Linq; +using System.Text; +using System.Text.RegularExpressions; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Reactive; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI; + +/// +/// Default — spins up a fresh +/// per call, selects the built-in NodeInitializer agent, sends a single user +/// message, and parses the Svg: line from the response. +/// +public sealed class IconGenerator : IIconGenerator +{ + private readonly IServiceProvider services; + private readonly ILogger? logger; + + public IconGenerator(IServiceProvider services) + { + this.services = services; + this.logger = (ILogger?)services.GetService(typeof(ILogger)); + } + + public IObservable GenerateSvgAsync(string name, string? description, CancellationToken ct = default) + { + var chat = new AgentChatClient(services); + // NodeInitializer is registered under the built-in "Agent" namespace. + chat.Initialize(contextPath: "Agent"); + return chat.WhenInitialized.Take(1).SelectMany(_ => + { + chat.SetSelectedAgent("NodeInitializer"); + var prompt = BuildPrompt(name, description); + var messages = new[] { new ChatMessage(ChatRole.User, prompt) }; + return chat.GetResponseAsync(messages, ct).ToObservableSequence() + .Aggregate(new StringBuilder(), (sb, msg) => + { + foreach (var content in msg.Contents.OfType()) + sb.Append(content.Text); + return sb; + }) + .Select(sb => + { + var raw = sb.ToString(); + var svg = ExtractSvg(raw); + if (string.IsNullOrEmpty(svg)) + { + logger?.LogWarning("NodeInitializer response did not contain a parsable Svg line. Raw: {Raw}", raw); + throw new InvalidOperationException("Agent did not return an SVG."); + } + return svg; + }); + }); + } + + private static string BuildPrompt(string name, string? description) + { + var desc = string.IsNullOrWhiteSpace(description) + ? $"A node called \"{name}\"." + : description.Trim(); + return $"Name: {name}\n\n{desc}"; + } + + // Matches the "Svg: <...>" line in the NodeInitializer response block. + private static readonly Regex SvgLineRegex = new( + @"(?im)^\s*Svg:\s*()\s*$", + RegexOptions.Compiled); + + // Fallback: any ... anywhere in the text. + private static readonly Regex SvgAnyRegex = new( + @"", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + + private static string? ExtractSvg(string text) + { + var m = SvgLineRegex.Match(text); + if (m.Success) return m.Groups[1].Value.Trim(); + var any = SvgAnyRegex.Match(text); + return any.Success ? any.Value.Trim() : null; + } +} diff --git a/src/MeshWeaver.AI/InboxTool.cs b/src/MeshWeaver.AI/InboxTool.cs new file mode 100644 index 000000000..ed09a3338 --- /dev/null +++ b/src/MeshWeaver.AI/InboxTool.cs @@ -0,0 +1,288 @@ +using System.Collections.Immutable; +using System.ComponentModel; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using MeshWeaver.Data; +using MeshWeaver.Mesh; +using MeshWeaver.Messaging; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using MeshThread = MeshWeaver.AI.Thread; + +namespace MeshWeaver.AI; + +/// +/// Result of draining the inbox — the user messages newly visible to the +/// agent (text + id + the original envelope so the +/// caller can materialise the satellite cell), and the post-drain thread state. +/// Pure record so the drain logic is trivially unit-testable without any hub plumbing. +/// +public sealed record InboxDrainResult( + ImmutableList DrainedTexts, + ImmutableList DrainedIds, + ImmutableList DrainedMessages, + MeshThread UpdatedThread); + +/// +/// The check_inbox AI function — the unified ingestion point. Every +/// transition from into +/// goes through this tool: +/// +/// Submit () writes to +/// PendingUserMessages only — no satellite cell, no +/// Messages update. The GUI binds to both properties and renders +/// pending entries as "queued" cells. +/// Round-start ingestion (server watcher when +/// IsExecuting=false): the watcher's DispatchRound drains +/// pending, materialises the satellite cells, allocates a single response +/// cell for the round, and posts to _Exec. +/// Mid-round ingestion (check_inbox tool fired by the +/// agent): same drain happens — pending → Messages, materialise satellite +/// cells, mark . The current +/// response cell continues streaming; no new response cell is created. +/// +/// +public static class InboxTool +{ + public const string ToolName = "check_inbox"; + + public const string ToolDescription = + "Check if the user has sent any new messages while you were working. " + + "Call this between major steps — after each tool call or before starting a new file edit — " + + "so the user can steer you mid-task. Returns the message text(s) the user typed " + + "since you last checked, or '(no new messages)' if the queue is empty. " + + "Once a message is returned by this tool it is permanently delivered to you " + + "(it won't be re-delivered on a future call) — fold it into your current response."; + + /// + /// Pure: given a thread state, returns the drain result + the next thread + /// state. DrainedIds is the union of UserMessageIds ∩ PendingUserMessages + /// (in submission order) plus any orphan pending ids not yet in + /// UserMessageIds. The updated thread has those ids removed from + /// PendingUserMessages, appended to Messages (so satellite + /// cells the caller materialises become rendered chat entries), and added + /// to IngestedMessageIds (de-duplicated). + /// + public static InboxDrainResult Drain(MeshThread thread) + { + ArgumentNullException.ThrowIfNull(thread); + + if (thread.PendingUserMessages.IsEmpty) + { + return new InboxDrainResult( + ImmutableList.Empty, + ImmutableList.Empty, + ImmutableList.Empty, + thread); + } + + var drainedIdsBuilder = ImmutableList.CreateBuilder(); + var drainedTextsBuilder = ImmutableList.CreateBuilder(); + var drainedMessagesBuilder = ImmutableList.CreateBuilder(); + foreach (var id in thread.UserMessageIds) + { + if (thread.PendingUserMessages.TryGetValue(id, out var msg)) + { + drainedIdsBuilder.Add(id); + drainedTextsBuilder.Add(msg.Text); + drainedMessagesBuilder.Add(msg); + } + } + + // Catch any pending ids not in UserMessageIds (defensive — shouldn't happen but + // we don't want to leak entries by ignoring them). + foreach (var (id, msg) in thread.PendingUserMessages) + { + if (!drainedIdsBuilder.Contains(id)) + { + drainedIdsBuilder.Add(id); + drainedTextsBuilder.Add(msg.Text); + drainedMessagesBuilder.Add(msg); + } + } + + var drainedIds = drainedIdsBuilder.ToImmutable(); + var pendingAfter = thread.PendingUserMessages; + foreach (var id in drainedIds) + pendingAfter = pendingAfter.Remove(id); + + var ingestedAfter = thread.IngestedMessageIds; + foreach (var id in drainedIds) + if (!ingestedAfter.Contains(id)) + ingestedAfter = ingestedAfter.Add(id); + + // Restore the invariant UserMessageIds ⊇ IngestedMessageIds — a concurrent + // cross-hub SubmitMessage can drop an id from the UserMessageIds array (RFC 7396 + // array-replace), while the dict-keyed PendingUserMessages this drain reads keeps + // it. The owner is authoritative for the derived list (see DispatchRound). + var userIdsAfter = thread.UserMessageIds; + foreach (var id in ingestedAfter) + if (!userIdsAfter.Contains(id)) + userIdsAfter = userIdsAfter.Add(id); + + // Append drained ids to Messages in submission order, skipping any already present. + var messagesAfter = thread.Messages; + foreach (var id in drainedIds) + if (!messagesAfter.Contains(id)) + messagesAfter = messagesAfter.Add(id); + + return new InboxDrainResult( + drainedTextsBuilder.ToImmutable(), + drainedIds, + drainedMessagesBuilder.ToImmutable(), + thread with + { + Messages = messagesAfter, + UserMessageIds = userIdsAfter, + PendingUserMessages = pendingAfter, + IngestedMessageIds = ingestedAfter + }); + } + + /// + /// Formats the drain result as the tool-call return value the agent sees. + /// Empty queue → "(no new messages)" so the agent can rapidly poll + /// without semantic ambiguity. Single message → just the text. + /// Multiple messages → numbered list. + /// + public static string FormatToolResult(InboxDrainResult drain) + { + if (drain.DrainedTexts.IsEmpty) + return "(no new messages)"; + + if (drain.DrainedTexts.Count == 1) + return $"User sent a follow-up message:\n\n{drain.DrainedTexts[0]}"; + + var sb = new System.Text.StringBuilder(); + sb.AppendLine($"User sent {drain.DrainedTexts.Count} follow-up messages:"); + sb.AppendLine(); + for (var i = 0; i < drain.DrainedTexts.Count; i++) + { + sb.AppendLine($"{i + 1}. {drain.DrainedTexts[i]}"); + } + return sb.ToString().TrimEnd(); + } + + /// + /// Builds the AIFunction registered with the agent. Each invocation reads the + /// current thread state once (no streaming subscription), drains pending + /// messages atomically, and returns the formatted text. + /// + public static AIFunction CreateCheckInboxTool(IMessageHub threadHub, ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(threadHub); + + // 🚨 check_inbox is "fake async" — it awaits NO real I/O leaf (only hub-observable + // composition), so it is NOT pooled (the AI IoPool is reserved for the LLM-stream + // `await foreach`) and must not re-acquire a pool slot. It is invoked from INSIDE the + // round's streaming loop while that loop is PAUSED on this tool call (no concurrent pushes). + // + // Mechanism: a TaskCompletionSource bridges the reactive drain to the Task the AIFunction + // surface requires — the LLM framework awaits this Task inside its OWN streaming pump (on + // the pool), never our hub. We do ONE stream.Update that drains PendingUserMessages + // (→ IngestedMessageIds) in a single go, then fill the TCS from its completion. + // + // In-flight (mid-round) messages are delivered Claude-Code-style: the drained text is + // appended inline to the live response output with a marker denoting user input — no + // separate satellite cells, no output-cell split (the old A7 path is removed). We do NOT + // append the ids to MeshThread.Messages: a Message id without a satellite cell is a + // dangling ref that re-triggers the exact missing-node NotFound storm this work fixes. + return AIFunctionFactory.Create( + method: (CancellationToken ct) => + { + var access = threadHub.ServiceProvider.GetService(); + // Capture the caller identity and re-seed it so the drain write carries it past any + // thread hop — a lost AsyncLocal would RLS-deny the write. + var captured = access?.Context ?? access?.CircuitContext; + if (captured is not null) access?.SetContext(captured); + + var workspace = threadHub.GetWorkspace(); + var threadPath = threadHub.Address.Path; + // 🚨 RunContinuationsAsynchronously is load-bearing: TrySetResult fires from the + // Update(...).Subscribe onNext, which can run on the thread hub's action-block + // thread. Without it, the LLM pump's await continuation (the rest of the streaming + // round up to its next true await) would resume SYNCHRONOUSLY on that hub thread — + // the scheduler-capture wedge class (see feedback_subscribeon_breaks_order). + var tcs = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + // A cancelled round (Stop button → executionCts) must not leave the LLM pump + // awaiting this tool Task forever — propagate the token. + var ctRegistration = ct.Register(() => tcs.TrySetCanceled(ct)); + ImmutableList drainedTexts = ImmutableList.Empty; + + // ONE stream.Update: get from pending and mark ingested in one go. Keep Messages + // unchanged (see header) — visibility comes from the inline output append below. + workspace.GetMeshNodeStream() + .Update(node => + { + if (node?.Content is not MeshThread thread) + { + // 🚨 Known deferral: this read is deliberately NOT ContentAs-tolerant — + // the ContentAs migration of this exact spot regressed the cancel-race + // message-loss test and needs a deterministic repro first (see memory + // project_baddata_contentas_pattern). Until then, log LOUDLY so a + // degraded-JsonElement thread node (which silently never drains and + // makes check_inbox report "(no new messages)" forever) is visible. + if (node?.Content is not null) + logger?.LogError( + "[InboxTool] check_inbox SKIPPED drain for {Path} — Content is {ContentType}, not MeshThread (degraded node?)", + threadPath, node.Content.GetType().Name); + return node!; + } + var drain = Drain(thread); + if (drain.DrainedIds.IsEmpty) return node; + drainedTexts = drain.DrainedTexts; + return node with { Content = drain.UpdatedThread with { Messages = thread.Messages } }; + }) + .Subscribe( + _ => + { + ctRegistration.Dispose(); + if (drainedTexts.IsEmpty) + { + tcs.TrySetResult("(no new messages)"); + return; + } + var steering = FormatInFlight(drainedTexts); + // Append inline to the live output (Claude-Code style). Race-free for + // TWO load-bearing reasons: (a) the streaming loop is PAUSED on this + // tool call, so no concurrent Append; and (b) the Sample(...) snapshot + // pipeline pushes MATERIALIZED strings — it never reads this + // StringBuilder lazily on the timer thread. If snapshots ever start + // reading the accumulator directly, this append becomes a data race. + var segment = threadHub.Get(); + segment?.ResponseText?.Append("\n\n" + steering + "\n\n"); + tcs.TrySetResult(steering); + }, + ex => + { + ctRegistration.Dispose(); + logger?.LogWarning(ex, "[InboxTool] check_inbox drain failed for {Path}", threadPath); + tcs.TrySetResult($"(error reading inbox: {ex.Message})"); + }); + return tcs.Task; + }, + name: ToolName, + description: ToolDescription); + } + + /// + /// Renders the drained in-flight user message(s) as clean markdown blockquotes with a subtle + /// 💬 marker, so each reads as a distinct user interjection inline in the agent's output — + /// the user's own words, nicely formatted, no explanatory boilerplate. + /// + private static string FormatInFlight(ImmutableList texts) + { + var sb = new System.Text.StringBuilder(); + for (var i = 0; i < texts.Count; i++) + { + if (i > 0) sb.AppendLine().AppendLine(); + // Blockquote every line of the message so multi-line input stays inside the quote. + sb.Append("> 💬 ").Append(texts[i].Replace("\n", "\n> ")); + } + return sb.ToString(); + } + + +} diff --git a/src/MeshWeaver.AI/LanguageModelNodeType.cs b/src/MeshWeaver.AI/LanguageModelNodeType.cs new file mode 100644 index 000000000..c6a827431 --- /dev/null +++ b/src/MeshWeaver.AI/LanguageModelNodeType.cs @@ -0,0 +1,200 @@ +using MeshWeaver.Graph; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace MeshWeaver.AI; + +/// +/// Mesh-node type for AI language models. Companion to . +/// +/// Two surfaces feed this: +/// +/// Platform models — +/// entries pair a config section (e.g. Anthropic) with a +/// provider label. +/// reads {section}:Models[] from +/// at static-node-provider time and emits one +/// nodeType:LanguageModel MeshNode per entry under +/// . +/// Bring-your-own models — anyone can create a node of this +/// type at any path with content; the +/// chat picker discovers it via the same synced query that finds +/// agents (nodeType:Agent|LanguageModel). +/// +/// +/// Public-read by default — model identity and provider are not +/// secrets. Credentials live behind +/// in a secret store, never in the node content itself. +/// +public static class LanguageModelNodeType +{ + /// NodeType discriminator value. + public const string NodeType = "LanguageModel"; + + /// Conventional namespace for model nodes (Model/<id>). + public const string RootNamespace = "Model"; + + /// + /// Registers the built-in LanguageModel MeshNode definition + the + /// that materialises every + /// configured model as a static node, plus public-read access. Auto-seeds + /// the well-known catalog sources (Anthropic, AzureFoundry, OpenAI) so a + /// stock deploy with those factories' configs Just Works. + /// + public static TBuilder AddLanguageModelType(this TBuilder builder, + IReadOnlySet? serveFromPartition = null) + where TBuilder : MeshBuilder + { + builder.AddMeshNodes(CreateMeshNode()); + builder.ConfigureNodeTypeAccess(a => a.WithPublicRead(NodeType)); + // Companion NodeType: ModelProvider holds the credentials shared by + // all child LanguageModel nodes. Registered together so a deployment + // calling AddLanguageModelType wires the entire data shape (the + // ChatClientCredentialResolver depends on both being available). + builder.AddModelProviderType(serveFromPartition); + // DB-synced "Model" partition (static-repo import) → skip the read-only in-memory static + // provider so Postgres serves the catalog + accepts the import's writes. See AddAgentType. + var dbSynced = serveFromPartition?.Contains("Model") == true; + builder.ConfigureServices(services => + { + services.TryAddSingleton(); + services.TryAddSingleton(); + // Encryption-at-rest for ModelProvider.ApiKey. Default master key + // comes from config (Ai:KeyProtection:MasterKey); swap in a + // KMS/Key Vault IMasterKeyProvider for hardened deployments. With + // no key configured both are pure passthrough (plaintext), so this + // is safe to register unconditionally. + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); + // ModelDiscoveryService MUST be a top-level singleton on the + // mesh hub — never on a per-thread / exec hub where its + // synced subscriptions could get stuck behind an in-flight + // handler. The per-thread/per-chat code paths read this + // service from meshHub.ServiceProvider, not from their own + // hub's DI scope. + services.TryAddSingleton(); + // 🚨 Plain AddSingleton (not TryAddEnumerable): TryAddEnumerable + // dedupes by impl-type AND ServiceLifetime AND ImplementationFactory + // — combinations that occasionally suppress the registration in + // ways that left BuiltInLanguageModelProvider invisible to DI + // resolution while BuiltInAgentProvider (using plain AddSingleton) + // worked. Match the AgentProvider pattern so both follow the + // same path. + // 🚨 Gate the IStaticNodeProvider (feeds FindStaticNode) on !dbSynced, same as the + // partition provider below — leaving it registered while the "Model" partition is + // DB-synced made the importer's inner CreateNode see the built-in catalog/_Provider + // nodes as already-present and fail "Node already exists" (atioz 2026-06-11: Model + // imported 4 / failed 2, incl. Model/_Policy + _Provider/Anthropic). The + // BuiltInLanguageModelProvider singleton stays (the import source wraps it); the + // LanguageModel/ModelProvider NodeType defs stay via AddMeshNodes. See AddAgentType. + if (!dbSynced) + { + services.AddSingleton(sp => sp.GetRequiredService()); + // Partition routing — the same instance feeds the routing core's + // "Model" partition. The partition's StaticNodeStorageAdapter is + // its storage of record; no SeedIfAbsent fan-in required. Skipped when + // the partition is DB-synced (PG serves it instead). + services.AddSingleton(sp => + new StaticNodePartitionStorageProvider( + RootNamespace, + sp.GetRequiredService(), + description: "Built-in language model catalog (read-only).")); + } + return services; + }); + + // No central seeding — each provider package registers its own + // catalog source via AddLanguageModelCatalogSource in its own + // builder extension (decentralised). See e.g. + // AzureFoundryExtensions.AddAzureClaudeProvider(). + return builder; + } + + /// + /// Adds a catalog source: a config section to scan for Models[] + /// when populating the nodeType:LanguageModel partition. + /// + /// Idempotent on (sectionName, providerName) — safe to call from + /// multiple builder.ConfigureServices blocks. Mutates the + /// singleton directly + /// instead of using the IOptions<T> Configure pipeline, + /// which didn't propagate to the mesh hub's DI scope (live + /// namespace:Model queries returned only the access policy + /// because Sources was empty at provider-resolve time). + /// + /// + public static TBuilder AddLanguageModelCatalogSource( + this TBuilder builder, + string sectionName, + string providerName, + int order = 0) + where TBuilder : MeshBuilder + => builder.AddLanguageModelCatalogSource(new LanguageModelCatalogSource(sectionName, providerName, order)); + + /// + /// Adds a fully-described catalog source — same shape as the legacy + /// 3-arg overload but carries the provider's bootstrap profile + /// (display label, default endpoint, default model ids, + /// RequiresApiKey). Decentralised: each provider package self- + /// registers via its own builder extension (see e.g. + /// AzureFoundryExtensions.AddAzureClaudeProvider). Idempotent on + /// (sectionName, providerName). + /// + public static TBuilder AddLanguageModelCatalogSource( + this TBuilder builder, + LanguageModelCatalogSource source) + where TBuilder : MeshBuilder + { + builder.ConfigureServices(services => + { + services.TryAddSingleton(); + + // Get or create the singleton instance and mutate it directly. + // The Add helper deduplicates by (section, provider). + var existing = services.FirstOrDefault(d => + d.ServiceType == typeof(LanguageModelCatalogOptions) && + d.ImplementationInstance is LanguageModelCatalogOptions); + LanguageModelCatalogOptions instance; + if (existing?.ImplementationInstance is LanguageModelCatalogOptions inst) + { + instance = inst; + } + else + { + instance = new LanguageModelCatalogOptions(); + // Replace any factory registration with our concrete + // instance so DI returns this exact object at resolve time. + for (var i = services.Count - 1; i >= 0; i--) + { + if (services[i].ServiceType == typeof(LanguageModelCatalogOptions)) + services.RemoveAt(i); + } + services.AddSingleton(instance); + } + + instance.Add(source); + return services; + }); + return builder; + } + + /// + /// MeshNode definition for nodeType:LanguageModel. Carries the + /// per-instance hub configuration that wires + /// as the content type so reads through + /// / + /// deserialise into + /// the typed record. + /// + public static MeshNode CreateMeshNode() => new(NodeType) + { + Name = "Language Model", + Icon = "/static/NodeTypeIcons/sparkle.svg", + HubConfiguration = config => config + .AddMeshDataSource(source => source + .WithContentType()) + }; +} diff --git a/src/MeshWeaver.AI/MeshOperations.cs b/src/MeshWeaver.AI/MeshOperations.cs index 0663d670b..f9bb14ae3 100644 --- a/src/MeshWeaver.AI/MeshOperations.cs +++ b/src/MeshWeaver.AI/MeshOperations.cs @@ -1,12 +1,23 @@ using System.Collections.Immutable; +using System.IO; +using System.Reactive.Concurrency; +using System.Reactive.Disposables; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Schema; +using MeshWeaver.ContentCollections; using MeshWeaver.Data; +using MeshWeaver.Markdown; using MeshWeaver.Layout; using MeshWeaver.Domain; +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; using MeshWeaver.Mesh; +using MeshWeaver.Kernel; using MeshWeaver.Mesh.Services; +using MeshWeaver.Mesh.Threading; using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -15,14 +26,18 @@ namespace MeshWeaver.AI; /// /// Shared mesh operations for AI agents and MCP tools. -/// All operations go through Hub messaging to enforce security via validators. +/// +/// **Every public method returns , never .** +/// This is deliberate — the mesh is an actor-hub system and `await` on hub-backed work +/// deadlocks. Callers subscribe to drive work (.Subscribe(onNext, onError)) or +/// bridge at an external boundary (.FirstAsync().ToTask()) — never inside hub +/// flow. See CLAUDE.md "NOTHING ASYNC EVER". /// public class MeshOperations { private readonly IMessageHub hub; private readonly ILogger logger; private readonly IMeshService mesh; - private readonly INodeTypeService? nodeTypeService; /// /// Callback invoked when a node is created, updated, or patched. @@ -35,24 +50,47 @@ public MeshOperations(IMessageHub hub) this.hub = hub; this.logger = hub.ServiceProvider.GetRequiredService>(); this.mesh = hub.ServiceProvider.GetRequiredService(); - this.nodeTypeService = hub.ServiceProvider.GetService(); } /// /// Looks up the cached compilation error for the owning NodeType of . - /// - If is a NodeType definition, checks its own path. + /// - If is a NodeType definition (Content is + /// OR NodeType + /// field equals the meta-type marker ), + /// checks its own path. /// - Otherwise checks the NodeType's path. /// Returns null if no error is recorded. /// - private string? LookupCompilationError(MeshNode node) + private IObservable LookupCompilationError(MeshNode node) { - if (nodeTypeService == null) return null; - var nodeTypePath = node.Content is Graph.Configuration.NodeTypeDefinition - ? node.Path - : node.NodeType; - return !string.IsNullOrEmpty(nodeTypePath) - ? nodeTypeService.GetCompilationError(nodeTypePath) - : null; + // NodeType==MeshNode.NodeTypePath catches the case where Content arrived + // as a JsonElement (per-node hub didn't have NodeTypeDefinition in its + // TypeRegistry, so polymorphic deserialisation fell back) — without + // this check, we'd look up the meta-NodeType "NodeType" and miss the + // actual broken-type error cached against the node's own path. + var isNodeTypeDef = node.Content is Graph.Configuration.NodeTypeDefinition + || string.Equals(node.NodeType, MeshNode.NodeTypePath, StringComparison.Ordinal); + var nodeTypePath = isNodeTypeDef ? node.Path : node.NodeType; + if (string.IsNullOrEmpty(nodeTypePath)) + return Observable.Return(null); + + // Fast path: the input node IS the settled NodeType MeshNode. Pre-settle + // states (Pending/Compiling/Unknown) fall through to the stream so we + // wait for the CompileWatcher's write-back rather than report a stale + // null error. + if (isNodeTypeDef + && node.Content is Graph.Configuration.NodeTypeDefinition ownDef + && IsSettled(ownDef)) + return Observable.Return(ownDef.CompilationError); + + // Slow path: subscribe to the NodeType's live stream, wait for a + // settled CompilationStatus emission, then read the CompilationError. + return hub.GetWorkspace().GetMeshNodeStream(nodeTypePath) + .Where(n => n?.Content is Graph.Configuration.NodeTypeDefinition d && IsSettled(d)) + .Take(1) + .Timeout(TimeSpan.FromSeconds(5)) + .Select(n => (n!.Content as Graph.Configuration.NodeTypeDefinition)?.CompilationError) + .Catch(_ => Observable.Return(null)); } /// @@ -148,72 +186,289 @@ public static string ResolveContextPath(IAgentChat chat, string path) return $"@{contextPath2}/{raw}"; } - public async Task Get(string path) + public IObservable Get(string path) { logger.LogInformation("Get called with path={Path}", path); if (string.IsNullOrWhiteSpace(path)) - return "Error: path is required."; + return Observable.Return("Error: path is required."); var resolvedPath = ResolvePath(path); if (string.IsNullOrWhiteSpace(resolvedPath)) - return "Error: path is required."; + return Observable.Return("Error: path is required."); - try + // Handle children query (path/*) — Query emits a QueryResultChange whose + // Initial change contains every matching child in a single batch. Take(1) completes + // the stream as soon as the first snapshot arrives; no await, no FromAsync bridge. + if (resolvedPath.EndsWith("/*")) { - // Handle children query (path/*) - if (resolvedPath.EndsWith("/*")) - { - var parentPath = resolvedPath[..^2]; - var result = ImmutableList.Empty; - var query = $"namespace:{parentPath}"; - await foreach (var node in mesh.QueryAsync(MeshQueryRequest.FromQuery(query))) + var parentPath = resolvedPath[..^2]; + return mesh.Query(MeshQueryRequest.FromQuery($"namespace:{parentPath}")) + .Take(1) + .Select(change => + { + var list = change.Items + .Select(node => (object)new { node.Path, node.Name, node.NodeType, node.Icon }) + .ToImmutableList(); + return JsonSerializer.Serialize(list, hub.JsonSerializerOptions); + }) + .Catch((Exception ex) => { - result = result.Add(new + logger.LogWarning(ex, "Error getting data at path {Path}", resolvedPath); + return Observable.Return($"Error: {ex.Message}"); + }); + } + + // Single-node content read via GetDataRequest + MeshNodeReference + RegisterCallback. + // See Doc/Architecture/CqrsAndContentAccess.md — queries are for sets only. + return TryResolveUnifiedPath(resolvedPath) + .SelectMany(unified => unified != null + ? Observable.Return(unified) + : FetchNode(resolvedPath).SelectMany(node => + { + if (node is null) + return GetWithBrokenNodeTypeFallback(resolvedPath); + return LookupCompilationError(node) + .Select(compileError => compileError != null + ? JsonSerializer.Serialize( + new { node, compilationError = compileError }, + hub.JsonSerializerOptions) + : JsonSerializer.Serialize(node, hub.JsonSerializerOptions)); + })) + .Catch((Exception ex) => + { + logger.LogWarning(ex, "Error getting data at path {Path}", resolvedPath); + return Observable.Return($"Error: {ex.Message}"); + }); + } + + /// + /// Fallback when returned null — typically a + /// broken-NodeType path whose per-node hub couldn't activate (compilation + /// failed) and whose timed out after 10s. + /// + /// If the NodeType MeshNode carries a recorded + /// , + /// the catalog still has the node's stored definition + /// even though its hub is broken. We read it via a one-shot + /// Query snapshot as the documented exception to + /// "queries are for sets only" (see Doc/Architecture/CqrsAndContentAccess.md): + /// the live content is unreachable, the catalog snapshot is the best we + /// have, and the wrapped response surfaces the compile error so callers + /// (Coder agent, MCP, UI overlays) can fix the source instead of seeing a + /// generic "Not found". + /// + private IObservable GetWithBrokenNodeTypeFallback(string resolvedPath) + { + // Read the NodeType MeshNode directly — the snapshot carries the + // CompilationError if compilation has failed at least once. + return hub.GetWorkspace().GetMeshNodeStream(resolvedPath) + .Where(n => n is not null) + .Take(1) + .Timeout(TimeSpan.FromSeconds(5)) + .Catch(_ => Observable.Return(null)) + .SelectMany(node => + { + var compileError = (node?.Content as Graph.Configuration.NodeTypeDefinition)?.CompilationError; + if (string.IsNullOrEmpty(compileError)) + return Observable.Return($"Not found: {resolvedPath}"); + + // Live Query — first emission carries the snapshot; the catalog + // is the source of truth here (the per-node hub is broken by + // definition, so live content is unreachable). + return mesh.Query(MeshQueryRequest.FromQuery($"path:{resolvedPath}")) + .Select(c => c.Items.FirstOrDefault()) + .Select(qn => qn is null + ? $"Not found: {resolvedPath}" + : JsonSerializer.Serialize( + new { node = qn, compilationError = compileError }, + hub.JsonSerializerOptions)) + .Catch((Exception ex) => { - node.Path, - node.Name, - node.NodeType, - node.Icon + logger.LogWarning(ex, + "Catalog fallback for broken NodeType at {Path} failed", resolvedPath); + return Observable.Return(JsonSerializer.Serialize( + new { compilationError = compileError, error = "Catalog read failed: " + ex.Message }, + hub.JsonSerializerOptions)); }); - } - return JsonSerializer.Serialize(result, hub.JsonSerializerOptions); + }); + } + + /// + /// One-shot read of the MeshNode at via the + /// owning per-node hub's MeshNodeReference reducer — the authoritative + /// source of truth, no catalog lag. GetDataRequest activates the cold + /// per-node hub on receipt; the response carries the live MeshNode. + /// Returns null on timeout or routing failure (node does not exist / + /// hub couldn't be activated). See Doc/Architecture/CqrsAndContentAccess.md. + /// + private IObservable FetchNode(string resolvedPath, int timeoutSeconds = 10) => + Observable.Create(observer => + { + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)); + var emitted = 0; + // 🚨 Capture the inner hub.Observe subscription so disposal tears + // down the hub-level callback. Without this, a CTS timeout or + // outer-subscriber early dispose leaves the GetDataRequest's + // pending-callback entry in the hub's responseSubjects dict until + // the framework's RequestTimeout (~30s). The test base's + // Quiescing-budget watchdog flags it as a leak — the exact + // failure signature behind FullCrudWorkflow_CreateGetUpdateDelete's + // CI flake (`GetDataRequest@ACME/CrudTest_…(17001ms)` pending). + // Matches the GetMeshNode shape in MeshNodeStreamExtensions.cs. + IDisposable? innerSubscription = null; + + void EmitOnce(MeshNode? node) + { + if (Interlocked.Exchange(ref emitted, 1) != 0) return; + observer.OnNext(node); + observer.OnCompleted(); } - // Check for Unified Path prefix (e.g., "ACME/schema:", "ACME/data:Collection/id") - var unifiedResult = await TryResolveUnifiedPathAsync(resolvedPath); - if (unifiedResult != null) - return unifiedResult; + cts.Token.Register(() => EmitOnce(null)); - // Get single node via query (reads from persistence, not cached) - await foreach (var node in mesh.QueryAsync( - MeshQueryRequest.FromQuery($"path:{resolvedPath}"))) + try { - var compileError = LookupCompilationError(node); - if (compileError != null) - return JsonSerializer.Serialize( - new { node, compilationError = compileError }, - hub.JsonSerializerOptions); - return JsonSerializer.Serialize(node, hub.JsonSerializerOptions); + // 🚨 Read the LIVE node via GetMeshNodeStream (the shared in-memory cache + // mirror), NOT a one-shot GetDataRequest. A GetDataRequest activates a cold + // per-node hub that loads from PERSISTENCE — stale when a recent write's + // debounced save hasn't flushed: the in-memory mirror already holds the + // update, persistence does not. Paired with the read-your-writes wait in + // Patch/Update (which freshens this same mirror to the written version), a + // read immediately after a write sees the fresh value. GetMeshNodeStream also + // activates a cold hub on subscribe. See Doc/Architecture/CqrsAndContentAccess.md. + // The CTS timeout above maps a never-emitting (non-existent) node to null. + innerSubscription = hub.GetWorkspace().GetMeshNodeStream(resolvedPath) + // Routing-fallback safety: a path with no per-node hub can route to the + // closest ancestor, which returns ITS node. Filter by exact path so + // callers (Patch / Update / Delete) never operate on an ancestor. + .Where(n => n is not null + && string.Equals(n.Path, resolvedPath, StringComparison.OrdinalIgnoreCase)) + .Take(1) + .Subscribe( + node => EmitOnce(node), + ex => + { + // DeliveryFailure or other error — node not found / unreachable. + logger.LogDebug(ex, "FetchNode read failed for {Path}", resolvedPath); + EmitOnce(null); + }); + } + catch (Exception ex) + { + logger.LogWarning(ex, "FetchNode read setup failed for {Path}", resolvedPath); + EmitOnce(null); } - return $"Not found: {resolvedPath}"; - } - catch (Exception ex) + return Disposable.Create(() => + { + innerSubscription?.Dispose(); + cts.Dispose(); + }); + }); + + /// + /// Read-your-writes barrier: waits (bounded) for the live mesh-node mirror at + /// to advance PAST — the + /// version observed before the write. The owning hub stamps a fresh, higher Version + /// when it applies a change, so a mirror version strictly greater than the pre-write + /// value means the reconciled update has propagated and a subsequent read will see it. + /// Best-effort: on timeout it emits null so the caller falls back to the + /// optimistic node rather than failing the write. Subscribes to the cache mirror's + /// READ stream (never the per-path Update queue), so it cannot deadlock — unlike the + /// removed in-queue echo-wait. + /// + private IObservable WaitForReadYourWrites(string path, long versionBefore) => + hub.GetWorkspace().GetMeshNodeStream(path) + .Where(n => n is not null && n.Version > versionBefore) + .Take(1) + .Select(n => (MeshNode?)n) + .Timeout(TimeSpan.FromSeconds(5)) + .Catch(_ => Observable.Return(null)); + + /// + /// Writes a full to the node's own hub via + /// . The target hub's data-change handler applies + /// the update to its workspace (ticking the MeshNodeReference stream so + /// subsequent sees the new value) and persists via + /// its data source. Emits the saved node on success. + /// + private IObservable UpdateViaDataChange(MeshNode node, int timeoutSeconds = 10) => + Observable.Create(observer => { - logger.LogWarning(ex, "Error getting data at path {Path}", resolvedPath); - return $"Error: {ex.Message}"; - } - } + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)); + var completed = 0; + + void Fail(Exception ex) + { + if (Interlocked.Exchange(ref completed, 1) != 0) return; + observer.OnError(ex); + } + + void Emit(MeshNode n) + { + if (Interlocked.Exchange(ref completed, 1) != 0) return; + observer.OnNext(n); + observer.OnCompleted(); + } + + // 🚨 Capture the inner Subscribe so disposal removes the + // hub-level pending callback. See FetchNode for the failure + // mode this avoids (test-base Quiescing leak detection trips + // on the orphaned callback entry). + IDisposable? innerSubscription = null; + + try + { + var delivery = hub.Post( + DataChangeRequest.Update([node]), + o => o.WithTarget(new Address(node.Path)))!; + + innerSubscription = hub.Observe(delivery) + .Subscribe( + d => + { + if (d.Message is DataChangeResponse resp) + { + if (resp.Status == DataChangeStatus.Committed) + Emit(node with { Version = resp.Version }); + else + Fail(new InvalidOperationException( + $"DataChangeRequest rejected for {node.Path}: {resp.Log?.Status}")); + } + else + { + Fail(new InvalidOperationException( + $"Unexpected response {d.Message?.GetType().Name} for DataChangeRequest at {node.Path}")); + } + }, + ex => Fail(new InvalidOperationException( + ex.Message ?? $"Delivery failed to {node.Path}", ex))); + + cts.Token.Register(() => Fail(new TimeoutException( + $"DataChangeRequest for {node.Path} did not complete within {timeoutSeconds}s."))); + } + catch (Exception ex) + { + logger.LogWarning(ex, "UpdateViaDataChange: Post/RegisterCallback failed for {Path}", node.Path); + Fail(ex); + } + + return () => + { + innerSubscription?.Dispose(); + cts.Dispose(); + }; + }); /// /// Tries to resolve a path as a Unified Path with prefix (schema/, model/, data/, content/). /// Supports both legacy colon format (address/prefix:path) and new slash format (address/prefix/path). /// Parses the path to find the prefix, splits into address and remainder, /// then routes data request to the resolved address. - /// Returns null if the path is not a Unified Path. + /// Emits null if the path is not a Unified Path; emits a JSON / error string otherwise. /// - private async Task TryResolveUnifiedPathAsync(string resolvedPath) + private IObservable TryResolveUnifiedPath(string resolvedPath) { string? addressPart = null; string? remainder = null; @@ -238,7 +493,6 @@ public async Task Get(string path) { if (UcrPrefixResolver.PrefixToAreaMap.ContainsKey(segments[i])) { - // Found a UCR prefix at segment i — everything before is address, everything from i onwards is remainder if (i > 0) { addressPart = string.Join("/", segments.Take(i)); @@ -246,7 +500,6 @@ public async Task Get(string path) } else { - // Prefix at the start (e.g., "content/file.md") — relative path, no address addressPart = null; remainder = resolvedPath; } @@ -256,47 +509,80 @@ public async Task Get(string path) } if (remainder == null) - return null; + return Observable.Return(null); var reference = new UnifiedReference(remainder); - Address address; - if (!string.IsNullOrEmpty(addressPart)) - { - address = new Address(addressPart); - } - else - { - // No address — route to the current hub - address = hub.Address; - } + var address = !string.IsNullOrEmpty(addressPart) ? new Address(addressPart) : hub.Address; logger.LogInformation("Resolving Unified Path: address={Address}, remainder={Remainder}", addressPart, remainder); - using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); - var delivery = hub.Post( - new GetDataRequest(reference), - o => o.WithTarget(address))!; - var callbackResponse = await hub.RegisterCallback(delivery, (d, _) => Task.FromResult(d), cts.Token); - - // Handle routing failures (e.g., node hub not found in Orleans) - if (callbackResponse is IMessageDelivery failure) - return $"Error: {failure.Message.Message ?? "Delivery failed to " + addressPart}"; - - if (callbackResponse is not IMessageDelivery dataResponse) - return $"Error: Unexpected response type {callbackResponse.Message?.GetType().Name} for {remainder} at {addressPart}"; - - var responseMsg = dataResponse.Message; + // Fire the GetDataRequest and receive the response via RegisterCallback. + // Observable.Create wraps the post/register pair so the caller can compose it + // into the Get pipeline without ever awaiting — the callback completes the + // observable from a non-hub thread. The outer Timeout enforces an upper + // bound on the response wait so missing hubs (e.g. a UCR path whose + // address segment doesn't have a running per-node hub) surface as an + // "Error: …" string instead of hanging the whole Get pipeline. + return Observable.Create(observer => + { + // 🚨 Capture inner Subscribe for proper teardown. Same leak class + // as FetchNode — outer Timeout cancellation tore down the + // observer's chain but left the hub-level callback registered. + IDisposable? innerSubscription = null; + try + { + var delivery = hub.Post( + new GetDataRequest(reference), + o => o.WithTarget(address))!; - if (responseMsg.Error != null) - return $"Error: {responseMsg.Error}"; + innerSubscription = hub.Observe(delivery) + .Subscribe( + d => + { + try + { + if (d.Message is GetDataResponse responseMsg) + { + if (responseMsg.Error != null) + observer.OnNext($"Error: {responseMsg.Error}"); + else + observer.OnNext(JsonSerializer.Serialize(responseMsg.Data, hub.JsonSerializerOptions)); + } + else + { + observer.OnNext($"Error: Unexpected response type {d.Message?.GetType().Name} for {remainder} at {addressPart}"); + } + observer.OnCompleted(); + } + catch (Exception ex) + { + observer.OnError(ex); + } + }, + ex => + { + observer.OnNext($"Error: {ex.Message ?? "Delivery failed to " + addressPart}"); + observer.OnCompleted(); + }); + } + catch (Exception ex) + { + observer.OnError(ex); + } - return JsonSerializer.Serialize(responseMsg.Data, hub.JsonSerializerOptions); + return () => innerSubscription?.Dispose(); + }) + .Timeout(TimeSpan.FromSeconds(10)) + .Catch((TimeoutException _) => + Observable.Return($"Error: Timeout resolving '{remainder}' at {addressPart}")); } - public async Task Search(string query, string? basePath = null) + public IObservable Search(string query, string? basePath = null, int limit = 50) { - logger.LogInformation("Search called with query={Query}, basePath={BasePath}", query, basePath); + logger.LogInformation("Search called with query={Query}, basePath={BasePath}, limit={Limit}", query, basePath, limit); + + limit = Math.Clamp(limit, 1, 200); var resolvedBase = basePath != null ? ResolvePath(basePath) : null; string fullQuery; @@ -306,68 +592,84 @@ public async Task Search(string query, string? basePath = null) } else { - // Remove empty namespace: placeholder — basePath provides the namespace context. - // Use namespace: (not path:) so scope defaults to Children (search within, not exact). var cleanQuery = query.Replace("namespace:", "").Trim(); fullQuery = $"namespace:{resolvedBase} {cleanQuery}".Trim(); } - try - { - var results = ImmutableList.Empty; - await foreach (var item in mesh.QueryAsync(new MeshQueryRequest { Query = fullQuery, Limit = 50 })) + // Snapshot semantics: Take(1) on Query gives us the Initial change + // containing every match for this query in one batch — no async enumeration, + // no FromAsync bridge. + return mesh.Query(new MeshQueryRequest { Query = fullQuery, Limit = limit }) + .Take(1) + .Select(change => { - if (item is MeshNode node) - { - results = results.Add(new - { - node.Path, - node.Name, - node.NodeType - }); - } - else + var list = change.Items + .Select(node => (object)new { node.Path, node.Name, node.NodeType }) + .ToImmutableList(); + // Envelope instead of a bare array so truncation is VISIBLE: a result + // set that silently stops at the limit reads as "that's everything" + // and the agent under-reports. Composed explicitly via JsonObject — + // the hub serializer options drop empty collections, which would strip + // the 'results' key from a zero-hit response and break consumers. + var truncated = list.Count >= limit; + var payload = new JsonObject { - results = results.Add(item); - } - } - - return JsonSerializer.Serialize(results, hub.JsonSerializerOptions); - } - catch (Exception ex) - { - logger.LogWarning(ex, "Error searching with query {Query}", query); - return $"Error: {ex.Message}"; - } + ["count"] = list.Count, + ["limit"] = limit, + ["truncated"] = truncated, + ["results"] = JsonSerializer.SerializeToNode(list, hub.JsonSerializerOptions) ?? new JsonArray(), + }; + if (truncated) + payload["hint"] = + "Result set hit the limit — there may be more matches. Narrow the query (namespace:/nodeType:/name:) or raise 'limit' (max 200)."; + return payload.ToJsonString(); + }) + .Catch((Exception ex) => + { + logger.LogWarning(ex, "Error searching with query {Query}", query); + return Observable.Return($"Error: {ex.Message}"); + }); } - public async Task Create(string node) + public IObservable Create(string node) { logger.LogInformation("Create called"); - try + return Observable.Defer(() => { - var sanitized = RepairJson(node); - var meshNode = JsonSerializer.Deserialize(sanitized, hub.JsonSerializerOptions); - if (meshNode == null) - return "Invalid node: deserialized to null."; - - if (string.IsNullOrWhiteSpace(meshNode.Name)) - return "Error: 'name' property is required. Provide a human-readable display name."; - - meshNode = SanitizeNodeId(meshNode); - - // Validate content against schema if both nodeType and content are provided - if (!string.IsNullOrEmpty(meshNode.NodeType) && meshNode.Content != null) + MeshNode? meshNode; + try { - var validationError = await ValidateContentWithSchemaAsync(meshNode); - if (validationError != null) - return validationError; + var sanitized = RepairJson(node); + meshNode = JsonSerializer.Deserialize(sanitized, hub.JsonSerializerOptions); } + catch (JsonException ex) + { + logger.LogWarning(ex, "Create: invalid JSON, length={Length}", node.Length); + return Observable.Return( + $"Invalid JSON: {ex.Message}. Tip: ensure all quotes and special characters in markdown content are properly escaped for JSON strings."); + } + + if (meshNode == null) + return Observable.Return("Invalid node: deserialized to null."); - var tcs = new TaskCompletionSource(); - mesh.CreateNode(meshNode).Subscribe( - created => + meshNode = SanitizeNodeId(meshNode); + meshNode = NormalizeNamespace(meshNode); + + var identityError = ValidateNodeIdentity(meshNode, "create"); + if (identityError != null) + return Observable.Return(identityError); + + // Validate content against schema when content is provided. + var validationObs = meshNode.Content != null + ? ValidateContentWithSchema(meshNode) + : Observable.Return(null); + + return validationObs.SelectMany(validationError => + validationError != null + ? Observable.Return(validationError) + : mesh.CreateNode(meshNode) + .Select(created => { OnNodeChange?.Invoke(new NodeChangeEntry { @@ -378,221 +680,484 @@ public async Task Create(string node) NodeType = created.NodeType, NodeName = created.Name }); - tcs.TrySetResult($"Created: {created.Path}"); - }, - ex => tcs.TrySetResult($"Error creating node: {ex.Message}")); - return await tcs.Task; - } - catch (JsonException ex) - { - logger.LogWarning(ex, "Create: invalid JSON, length={Length}", node.Length); - return $"Invalid JSON: {ex.Message}. Tip: ensure all quotes and special characters in markdown content are properly escaped for JSON strings."; - } - catch (Exception ex) - { - logger.LogWarning(ex, "Error creating node"); - return $"Error: {ex.Message}"; - } + return $"Created: {created.Path}"; + }) + .Catch((Exception ex) => + { + logger.LogWarning(ex, "Error creating node"); + return Observable.Return($"Error creating node: {ex.Message}"); + })); + }); } - public async Task Update(string nodes) + public IObservable Update(string nodes) { logger.LogInformation("Update called"); - try + return Observable.Defer(() => { - var sanitized = RepairJson(nodes); - var nodeList = JsonSerializer.Deserialize>(sanitized, hub.JsonSerializerOptions); + List? nodeList; + try + { + var sanitized = RepairJson(nodes); + nodeList = JsonSerializer.Deserialize>(sanitized, hub.JsonSerializerOptions); + } + catch (JsonException ex) + { + return Observable.Return($"Invalid JSON: {ex.Message}"); + } + if (nodeList == null || nodeList.Count == 0) - return "No nodes provided."; + return Observable.Return("No nodes provided."); - var results = ImmutableList.Empty; + // Validate each node up-front and spawn per-node UpdateNode observables for the rest. + // Per-node outputs combine in input order via Concat so the caller sees a deterministic + // result string even for batches. + var perNode = ImmutableList>.Empty; foreach (var rawNode in nodeList) { if (rawNode == null) { - results = results.Add("Error: array contained a null entry. " + - "Each array element must be a complete MeshNode JSON object."); - continue; - } - - var meshNode = SanitizeNodeId(rawNode); - - // Reject empty identity — without id we cannot address the node. - if (string.IsNullOrWhiteSpace(meshNode.Id)) - { - results = results.Add("Error: node is missing 'id'. " + - "Every node requires an id — fetch with Get first if unsure."); + perNode = perNode.Add(Observable.Return( + "Error: array contained a null entry. Each array element must be a complete MeshNode JSON object.")); continue; } - // Reject empty name — downstream UI and streams key off Name. - if (string.IsNullOrWhiteSpace(meshNode.Name)) - { - results = results.Add($"Error: node at {meshNode.Path} has empty 'name'. " + - "Provide a non-empty human-readable display name."); - continue; - } + var meshNode = NormalizeNamespace(SanitizeNodeId(rawNode)); - // Reject partial nodes — Update does full replacement. - // Use Patch for partial changes instead. - if (string.IsNullOrEmpty(meshNode.NodeType)) + var identityError = ValidateNodeIdentity(meshNode, "update"); + if (identityError != null) { - results = results.Add($"Error: node at {meshNode.Path} is missing 'nodeType'. " + - "Update requires the complete node (from Get). Use Patch for partial updates."); + perNode = perNode.Add(Observable.Return(identityError)); continue; } - // Reject updates that would blank out content — agents must always send the - // full content payload. Returning the schema lets the agent reconstruct it. if (meshNode.Content == null) { - results = results.Add(await BuildNullContentErrorAsync(meshNode.Path, meshNode.NodeType!)); - continue; - } - - // Validate the content against the registered content type for this NodeType. - var validationError = await ValidateContentWithSchemaAsync(meshNode); - if (validationError != null) - { - results = results.Add(validationError); + perNode = perNode.Add(BuildNullContentError(meshNode.Path, meshNode.NodeType!)); continue; } var versionBefore = meshNode.Version; - var updateTcs = new TaskCompletionSource(); - mesh.UpdateNode(meshNode).Subscribe( - updated => - { - OnNodeChange?.Invoke(new NodeChangeEntry - { - Path = updated.Path, - Operation = "Updated", - VersionBefore = versionBefore, - VersionAfter = updated.Version, - NodeType = updated.NodeType, - NodeName = updated.Name - }); - updateTcs.TrySetResult($"Updated: {updated.Path}"); - }, - ex => updateTcs.TrySetResult($"Error updating {meshNode.Path}: {ex.Message}")); - results = results.Add(await updateTcs.Task); + var currentPath = meshNode.Path; + var nodeForCapture = meshNode; + perNode = perNode.Add( + ValidateContentWithSchema(nodeForCapture).SelectMany(validationError => + validationError != null + ? Observable.Return(validationError) + : mesh.UpdateNode(nodeForCapture) + // Read-your-writes barrier (see Patch): wait for the live + // mirror to advance PAST the optimistic version before + // returning, so a follow-up Get sees the reconciled update. + .SelectMany(updated => WaitForReadYourWrites(currentPath, updated.Version) + .Select(confirmed => + { + var after = confirmed ?? updated; + OnNodeChange?.Invoke(new NodeChangeEntry + { + Path = after.Path, + Operation = "Updated", + VersionBefore = versionBefore, + VersionAfter = after.Version, + NodeType = after.NodeType, + NodeName = after.Name + }); + return $"Updated: {after.Path}"; + })) + .Catch((Exception ex) => + Observable.Return($"Error updating {currentPath}: {ex.Message}")))); } - return string.Join("\n", results); - } - catch (JsonException ex) - { - return $"Invalid JSON: {ex.Message}"; - } - catch (Exception ex) - { - logger.LogWarning(ex, "Error updating nodes"); - return $"Error: {ex.Message}"; - } + return perNode + .ToObservable() + .Concat() + .ToList() + .Select(lines => string.Join("\n", lines)) + .Catch((Exception ex) => + { + logger.LogWarning(ex, "Error updating nodes"); + return Observable.Return($"Error: {ex.Message}"); + }); + }); } - public async Task Patch(string path, string fields) + /// + /// Pretty-prints a MeshNode for inclusion in diff output. Indented JSON keeps each + /// field on its own line so the unified diff shows field-level changes rather than + /// one massive minified line. + /// + private string SerialisePretty(MeshNode node) => + JsonSerializer.Serialize(node, new JsonSerializerOptions(hub.JsonSerializerOptions) { WriteIndented = true }); + + public IObservable Patch(string path, string fields) { logger.LogInformation("Patch called for path={Path}", path); - // Fail-fast on empty/garbage path — without this, QueryAsync on "path:" or "path:" - // can hang forever (seen in AgentWriteFailureTests.NoTool_EverReturnsEmpty_OnAnyInput). if (string.IsNullOrWhiteSpace(path)) - return "Error: path is required."; + return Observable.Return("Error: path is required."); - try + return Observable.Defer(() => { var resolvedPath = ResolvePath(path); if (string.IsNullOrWhiteSpace(resolvedPath)) - return "Error: path is required."; + return Observable.Return("Error: path is required."); - var existing = await mesh.QueryAsync($"path:{resolvedPath}").FirstOrDefaultAsync(); - if (existing == null) - return $"Error: node not found at {resolvedPath}"; + // Validate fields is a JSON object client-side. The actual merge happens + // on the node hub via PatchDataRequest — no need to fetch the existing + // MeshNode here, the hub applies the delta to its own workspace state. + JsonObject? jsonObj; + try + { + var sanitized = RepairJson(fields); + jsonObj = JsonNode.Parse(sanitized) as JsonObject; + } + catch (JsonException ex) + { + return Observable.Return($"Invalid JSON: {ex.Message}"); + } - var sanitized = RepairJson(fields); - var jsonObj = JsonNode.Parse(sanitized) as JsonObject; if (jsonObj == null) - return "Error: fields must be a JSON object"; - - // Reject patches that explicitly blank out content (key present, value null). - // Omitting the key entirely is fine — that preserves existing content. - if (jsonObj.ContainsKey("content") && jsonObj["content"] is null) - return await BuildNullContentErrorAsync(existing.Path, existing.NodeType!); + return Observable.Return("Error: fields must be a JSON object"); - // Deserialize to get typed values using the hub's serializer options - var partial = jsonObj.Deserialize(hub.JsonSerializerOptions) - ?? new MeshNode(existing.Id, existing.Namespace); + if (jsonObj.ContainsKey("name") && string.IsNullOrWhiteSpace(jsonObj["name"]?.ToString())) + return Observable.Return( + $"Error: cannot patch {resolvedPath}: 'name' is empty. " + + "Provide a non-empty human-readable display name, or omit the 'name' key."); - var merged = existing with + // Read-merge-write via DataChangeRequest. FetchNode returns null when the + // path doesn't resolve (now with path-match verification so we don't + // accidentally patch an ancestor hub). + return FetchNode(resolvedPath).SelectMany(existing => { - Name = jsonObj.ContainsKey("name") ? partial.Name : existing.Name, - Icon = jsonObj.ContainsKey("icon") ? partial.Icon : existing.Icon, - Category = jsonObj.ContainsKey("category") ? partial.Category : existing.Category, - Order = jsonObj.ContainsKey("order") ? partial.Order : existing.Order, - Content = jsonObj.ContainsKey("content") ? partial.Content : existing.Content, - PreRenderedHtml = jsonObj.ContainsKey("preRenderedHtml") ? partial.PreRenderedHtml : existing.PreRenderedHtml, - }; + if (existing == null) + return Observable.Return( + $"Error: node not found at {resolvedPath}. The path must be the node's exact 'path' property " + + "(never its display name). Locate it with Search (e.g. Search('name:\"…the name…\"')) and " + + "retry with the 'path' value from the match; to create a new node instead, use Create."); + + // Content-specific rejections carry the expected schema so agents + // can recover on the next call without guessing. + if (jsonObj.ContainsKey("content") && jsonObj["content"] is null) + return BuildNullContentError(existing.Path, existing.NodeType!); + + // 🚨 Deep-merge content (RFC 7396), never wholesale-replace it. A bare + // `jsonObj["content"]` carries ONLY the keys the caller sent, so + // deserialising it straight into MeshNode.Content would DROP every + // existing content field the caller omitted — the 2026-06-13 logo patch + // ({"content":{"logo":…}}) clobbered name/description/body exactly this + // way. Merge the delta onto the existing content, serialised via the FULL + // node so the polymorphic `$type` discriminator is present, so omitted + // keys are preserved and only the provided keys change. A null member + // deletes that key; arrays/scalars replace wholesale (RFC 7396). + if (jsonObj["content"] is JsonObject contentPatch) + { + var existingNodeJson = JsonSerializer.SerializeToNode(existing, hub.JsonSerializerOptions) as JsonObject; + if (existingNodeJson?["content"] is JsonObject existingContent) + jsonObj["content"] = MergePatch(existingContent, contentPatch); + } + + var partial = jsonObj.Deserialize(hub.JsonSerializerOptions) + ?? new MeshNode(existing.Id, existing.Namespace); - // If the patch touches content, validate the merged content against the node's schema. - // This protects downstream consumers (sync streams, persistence) from shape-broken writes. - if (jsonObj.ContainsKey("content") && !string.IsNullOrEmpty(merged.NodeType) && merged.Content != null) + var merged = existing with + { + Name = jsonObj.ContainsKey("name") ? partial.Name : existing.Name, + Icon = jsonObj.ContainsKey("icon") ? partial.Icon : existing.Icon, + Category = jsonObj.ContainsKey("category") ? partial.Category : existing.Category, + Order = jsonObj.ContainsKey("order") ? partial.Order : existing.Order, + Content = jsonObj.ContainsKey("content") ? partial.Content : existing.Content, + PreRenderedHtml = jsonObj.ContainsKey("preRenderedHtml") ? partial.PreRenderedHtml : existing.PreRenderedHtml, + }; + + // Validate merged content against the NodeType's schema when the + // caller touched content. Surface the schema in the error so an + // agent can fix its payload on the retry. + var validationObs = (jsonObj.ContainsKey("content") && !string.IsNullOrEmpty(merged.NodeType) && merged.Content != null) + ? ValidateContentWithSchema(merged) + : Observable.Return(null); + + var versionBefore = existing.Version; + return validationObs.SelectMany(validationError => + validationError != null + ? Observable.Return(validationError) + : mesh.UpdateNode(merged) + // 🚨 Read-your-writes barrier. UpdateNode emits OPTIMISTICALLY — + // the owner applies asynchronously and stamps a fresh, higher + // Version on apply, so the emitted `updated` still carries the + // pre-apply version. Without the wait, the immediately-following + // Get races the propagation and reads the stale value. Wait + // (bounded) for the live mirror to advance PAST versionBefore so + // the reconciled update is observable before we return. + .SelectMany(updated => WaitForReadYourWrites(resolvedPath, versionBefore) + .Select(confirmed => + { + var after = confirmed ?? updated; + OnNodeChange?.Invoke(new NodeChangeEntry + { + Path = after.Path, + Operation = "Updated", + VersionBefore = versionBefore, + VersionAfter = after.Version, + NodeType = after.NodeType, + NodeName = after.Name + }); + var versionText = after.Version > versionBefore + ? $" (v{versionBefore} → v{after.Version})" + : ""; + return $"Patched: {after.Path}{versionText}"; + })) + .Catch((Exception ex) => + Observable.Return($"Error patching {merged.Path}: {ex.Message}"))); + }) + .Catch((Exception ex) => { - var validationError = await ValidateContentWithSchemaAsync(merged); - if (validationError != null) - return validationError; - } + logger.LogWarning(ex, "Error patching node at {Path}", path); + return Observable.Return($"Error: {ex.Message}"); + }); + }); + } - // Reject empty or effectively-empty names — empty string names corrupt UI - // and downstream streams that key off Name. - if (jsonObj.ContainsKey("name") && string.IsNullOrWhiteSpace(merged.Name)) - return $"Error: cannot patch {existing.Path}: 'name' is empty. " + - "Provide a non-empty human-readable display name, or omit the 'name' key to keep the current name."; + /// + /// Anchored text edit on a node's primary text content (Markdown body or Code source). + /// Replaces an exact substring, so the agent supplies just the snippet to change plus + /// enough surrounding context to make it unique — instead of re-emitting the whole + /// document through Patch (token cost + truncation corruption on long files). + /// Same read-your-writes semantics as Patch. Every failure mode returns a descriptive + /// error telling the agent how to recover. + /// + public IObservable EditContent(string path, string oldText, string newText, bool replaceAll = false) + { + logger.LogInformation("EditContent called for path={Path}", path); + + if (string.IsNullOrWhiteSpace(path)) + return Observable.Return("Error: path is required."); + if (string.IsNullOrEmpty(oldText)) + return Observable.Return( + "Error: oldText is required — Get the node and copy the exact text to replace, including whitespace."); + if (oldText == newText) + return Observable.Return("Error: oldText and newText are identical — nothing to change."); + + return Observable.Defer(() => + { + var resolvedPath = ResolvePath(path); + return FetchNode(resolvedPath).SelectMany(existing => + { + if (existing == null) + return Observable.Return( + $"Error: node not found at {resolvedPath}. The path must be the node's exact 'path' property — " + + "locate it with Search and retry with the 'path' value from the match."); - var versionBefore = existing.Version; - var patchTcs = new TaskCompletionSource(); - mesh.UpdateNode(merged).Subscribe( - updated => + var text = existing.Content switch { - OnNodeChange?.Invoke(new NodeChangeEntry - { - Path = updated.Path, - Operation = "Updated", - VersionBefore = versionBefore, - VersionAfter = updated.Version, - NodeType = updated.NodeType, - NodeName = updated.Name - }); + MarkdownContent md => md.Content, + CodeConfiguration code => code.Code, + string s => s, + _ => null + }; + + if (text == null) + return Observable.Return( + $"Error: cannot edit {resolvedPath}: its content is " + + $"{existing.Content?.GetType().Name ?? "empty"}, not editable text. EditContent works on " + + "Markdown and Code nodes; for structured content use Patch with the full 'content' object."); + + var count = CountOccurrences(text, oldText); + if (count == 0) + return Observable.Return( + $"Error: the text to replace was not found in {resolvedPath}. Get the node and copy the " + + "exact text — including whitespace and line breaks — then retry. " + + $"(Current content is {text.Length} chars.)"); + if (count > 1 && !replaceAll) + return Observable.Return( + $"Error: the text to replace occurs {count} times in {resolvedPath}. Include more " + + "surrounding context to make the match unique, or set replaceAll=true to change every occurrence."); + + var newFull = text.Replace(oldText, newText, StringComparison.Ordinal); + var merged = existing.Content switch + { + MarkdownContent md => WithRerenderedMarkdown(existing, md, newFull), + CodeConfiguration code => existing with { Content = code with { Code = newFull } }, + _ => existing with { Content = newFull }, + }; + + var versionBefore = existing.Version; + return mesh.UpdateNode(merged) + // Same read-your-writes barrier as Patch — see comment there. + .SelectMany(updated => WaitForReadYourWrites(resolvedPath, versionBefore) + .Select(confirmed => + { + var after = confirmed ?? updated; + OnNodeChange?.Invoke(new NodeChangeEntry + { + Path = after.Path, + Operation = "Updated", + VersionBefore = versionBefore, + VersionAfter = after.Version, + NodeType = after.NodeType, + NodeName = after.Name + }); + var plural = count == 1 ? "" : "s"; + return $"Edited: {after.Path} ({count} replacement{plural})"; + })) + .Catch((Exception ex) => + Observable.Return($"Error editing {resolvedPath}: {ex.Message}")); + }) + .Catch((Exception ex) => + { + logger.LogWarning(ex, "Error editing node at {Path}", path); + return Observable.Return($"Error: {ex.Message}"); + }); + }); + } - // Silent-failure guard: if the version did not increment, the write did - // not commit (likely a stale snapshot read or a routing-layer no-op). - // The agent must see this and retry/refresh — never report success on a no-op. - if (updated.Version == versionBefore) - { - logger.LogWarning( - "Patch silent-failure on {Path}: version unchanged ({Version}) — write did not commit", - updated.Path, versionBefore); - patchTcs.TrySetResult( - $"Error: patch on {updated.Path} did not commit (version stayed at {versionBefore}). " + - "This usually means a stale snapshot — retry after re-fetching the node."); - return; - } + /// + /// Rebuilds the derived markdown artefacts (prerendered HTML, code submissions) after a + /// text edit, preserving the record's other fields (authors, tags, thumbnail, abstract). + /// Without this, the portal would keep rendering the stale pre-edit HTML. + /// + private static MeshNode WithRerenderedMarkdown(MeshNode node, MarkdownContent md, string newText) + { + var parsed = MarkdownContent.Parse(newText, node.Namespace, node.Path); + return node with + { + Content = md with + { + Content = newText, + PrerenderedHtml = parsed.PrerenderedHtml, + CodeSubmissions = parsed.CodeSubmissions + }, + PreRenderedHtml = parsed.PrerenderedHtml, + }; + } - patchTcs.TrySetResult($"Patched: {updated.Path} (v{versionBefore} → v{updated.Version})"); - }, - ex => patchTcs.TrySetResult($"Error patching {merged.Path}: {ex.Message}")); - return await patchTcs.Task; - } - catch (JsonException ex) + private static int CountOccurrences(string text, string needle) + { + var count = 0; + var i = 0; + while ((i = text.IndexOf(needle, i, StringComparison.Ordinal)) >= 0) { - return $"Invalid JSON: {ex.Message}"; + count++; + i += needle.Length; } - catch (Exception ex) + return count; + } + + /// + /// Posts a to the node's hub with the raw JSON + /// delta. The hub applies the JSON merge patch to its own MeshNodeReference + /// workspace stream and returns . Emits the + /// committed version on success; OnError on failure/timeout. + /// + private IObservable PatchViaDataRequest(string resolvedPath, string rawPatch, int timeoutSeconds = 10) => + Observable.Create(observer => { - logger.LogWarning(ex, "Error patching node at {Path}", path); - return $"Error: {ex.Message}"; - } + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(timeoutSeconds)); + var completed = 0; + + void Fail(Exception ex) + { + if (Interlocked.Exchange(ref completed, 1) != 0) return; + observer.OnError(ex); + } + + void Emit(long version) + { + if (Interlocked.Exchange(ref completed, 1) != 0) return; + observer.OnNext(version); + observer.OnCompleted(); + } + + // 🚨 Capture inner Subscribe for proper teardown. + IDisposable? innerSubscription = null; + + try + { + var delivery = hub.Post( + new PatchDataRequest(new MeshNodeReference(), new RawJson(rawPatch)), + o => o.WithTarget(new Address(resolvedPath)))!; + + innerSubscription = hub.Observe(delivery) + .Subscribe( + d => + { + if (d.Message is PatchDataResponse resp) + { + if (resp.Success) + Emit(resp.Version); + else + Fail(new InvalidOperationException(resp.Error ?? "Patch rejected")); + } + else + Fail(new InvalidOperationException( + $"Unexpected response {d.Message?.GetType().Name} for PatchDataRequest at {resolvedPath}")); + }, + ex => Fail(new InvalidOperationException(ex.Message ?? "Delivery failed", ex))); + + cts.Token.Register(() => Fail(new TimeoutException( + $"PatchDataRequest for {resolvedPath} did not complete within {timeoutSeconds}s."))); + } + catch (Exception ex) + { + logger.LogWarning(ex, "PatchViaDataRequest: Post/RegisterCallback failed for {Path}", resolvedPath); + Fail(ex); + } + + return () => + { + innerSubscription?.Dispose(); + cts.Dispose(); + }; + }); + + /// + /// Up-front identity validation shared by Create and Update. Returns a descriptive, + /// actionable error string when the node is missing 'id', 'nodeType', or 'name', or when + /// the namespace is malformed — or null when the node is sound enough to attempt the write. + /// Validating BEFORE posting to the mesh turns a routed-to-nowhere grain call (opaque + /// timeout, silent no-op) into an immediate, specific answer the agent can act on. + /// + private static string? ValidateNodeIdentity(MeshNode node, string operation) + { + if (string.IsNullOrWhiteSpace(node.Id)) + return $"Error: cannot {operation}: 'id' is not set. The id is the node's own slug — the final path segment, " + + "no slashes (e.g. \"PricingTool\"). Put the parent path in 'namespace' (e.g. \"ACME/Projects\"); " + + "the node's path is derived as {namespace}/{id}."; + + if (string.IsNullOrWhiteSpace(node.NodeType)) + return $"Error: cannot {operation} '{node.Path}': 'nodeType' is not set. Every node must declare a nodeType — " + + "it is the path of the type definition that gives the node its shape, views, and behaviour " + + "(e.g. \"Markdown\", \"Code\", \"Organization\"). Discover available types with " + + "Search('nodeType:NodeType') and retry with nodeType set." + + (operation == "update" + ? " If you only meant to change a few fields, use Patch instead — it preserves all fields you don't mention." + : ""); + + if (string.IsNullOrWhiteSpace(node.Name)) + return $"Error: cannot {operation} '{node.Path}': 'name' is not set. Provide a non-empty, human-readable " + + "display name — it is shown as the node's title in the navigator and page heading."; + + var ns = node.Namespace; + if (!string.IsNullOrEmpty(ns) && (ns.EndsWith('/') || ns.Contains("//"))) + return $"Error: cannot {operation} '{node.Id}': namespace '{ns}' is malformed. The namespace is the parent " + + "path, segments separated by single slashes, no trailing slash (e.g. \"ACME/Projects\", \"User/rbuergi\")."; + + return null; + } + + /// + /// Normalizes agent-emitted namespace noise the same way does for + /// path arguments: strips a leading '@' / '/' and surrounding whitespace. Models routinely + /// copy the namespace out of an absolute reference ("@/ACME/Projects") — that intent is + /// unambiguous, so fix it instead of failing the write. + /// + private static MeshNode NormalizeNamespace(MeshNode node) + { + var ns = node.Namespace; + if (string.IsNullOrEmpty(ns)) + return node; + + var normalized = ResolvePath(ns).TrimStart('/'); + return normalized == ns ? node : node with { Namespace = normalized }; } /// @@ -604,12 +1169,10 @@ private MeshNode SanitizeNodeId(MeshNode node) if (string.IsNullOrEmpty(node.Id) || !node.Id.Contains('/')) return node; - // Split full path into namespace + id var lastSlash = node.Id.LastIndexOf('/'); var ns = node.Id[..lastSlash]; var id = node.Id[(lastSlash + 1)..]; - // If the node already has a namespace, prepend it if (!string.IsNullOrEmpty(node.Namespace)) ns = $"{node.Namespace}/{ns}"; @@ -619,6 +1182,34 @@ private MeshNode SanitizeNodeId(MeshNode node) return node with { Id = id, Namespace = ns }; } + /// + /// RFC 7396 JSON Merge Patch: recursively merges onto + /// . Object members merge recursively; a null + /// member deletes that key; any non-object (scalar or array) replaces wholesale. + /// Returns a fresh, detached node — neither argument is mutated, so callers can + /// keep using / afterwards. + /// + internal static JsonNode? MergePatch(JsonNode? target, JsonNode? patch) + { + // Non-object patch (scalar, array, or null literal) replaces the target. + if (patch is not JsonObject patchObj) + return patch?.DeepClone(); + + var result = target is JsonObject targetObj + ? (JsonObject)targetObj.DeepClone() + : new JsonObject(); + + foreach (var (key, value) in patchObj) + { + if (value is null) + result.Remove(key); // RFC 7396: null deletes the member + else + result[key] = MergePatch(result[key], value); // recurse; result[key] is null when absent + } + + return result; + } + /// /// Attempts to repair common JSON issues from LLM output: /// - Truncated strings (unclosed quotes/braces) @@ -629,19 +1220,13 @@ private static string RepairJson(string json) if (string.IsNullOrEmpty(json)) return json; - // Try parsing first — if it's valid, return as-is try { using var doc = JsonDocument.Parse(json); return json; } - catch (JsonException) - { - // Fall through to repair - } + catch (JsonException) { } - // Repair: try truncating to last complete JSON structure - // Find the last closing brace/bracket that makes valid JSON for (var i = json.Length - 1; i > 0; i--) { if (json[i] is '}' or ']') @@ -652,233 +1237,492 @@ private static string RepairJson(string json) using var doc = JsonDocument.Parse(candidate); return candidate; } - catch (JsonException) - { - // Try next position - } + catch (JsonException) { } } } - return json; // Return original if repair fails + return json; } - public Task Delete(string paths) + /// + /// Writes raw bytes into a content collection on the node addressed by . + /// Transport-agnostic: callers (MCP base64, REST multipart, CLI HTTP) decode at the boundary + /// and hand off the here. + /// + /// Path shape: {nodePath}/{collection}/{filePath} — e.g. Systemorph/content/logo.png + /// or Doc/Architecture/content/diagrams/flow.svg. The collection must exist on the node + /// and be IsEditable = true. + /// + /// Returns a JSON string {"status":"Uploaded","path":"…","bytes":N} on success, or an + /// "Error: …" string on any validation/resolution failure (mirrors the other tool methods). + /// + public IObservable Upload(string path, byte[] bytes) { - logger.LogInformation("Delete called"); + logger.LogInformation("Upload path={Path} bytes={Bytes}", path, bytes?.Length ?? 0); - List? pathList; - try - { - pathList = JsonSerializer.Deserialize>(paths, hub.JsonSerializerOptions); - } - catch (JsonException ex) + if (string.IsNullOrWhiteSpace(path)) + return Observable.Return("Error: path is required."); + if (bytes is null || bytes.Length == 0) + return Observable.Return("Error: content is required."); + + var resolvedPath = ResolvePath(path).TrimStart('/'); + if (string.IsNullOrWhiteSpace(resolvedPath)) + return Observable.Return("Error: path is required."); + + var pathResolver = hub.ServiceProvider.GetRequiredService(); + return pathResolver.ResolvePath(resolvedPath).SelectMany(resolution => { - return Task.FromResult($"Invalid JSON: {ex.Message}"); - } - if (pathList == null || pathList.Count == 0) - return Task.FromResult("No paths provided."); - - // Subscribe to each IMeshService.DeleteNode observable and aggregate the per-path - // outcome into a single result string once all complete. No `await` on a Task — the - // TaskCompletionSource is resolved from the Subscribe callbacks, which run off the - // hub scheduler. This lets the caller rely on "Deleted: ..." meaning the delete - // actually finished (matches what tests and agent follow-up Gets expect). - var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var gate = new object(); - var lines = new string[pathList.Count]; - var remaining = pathList.Count; - - for (var i = 0; i < pathList.Count; i++) - { - var index = i; - var rawPath = pathList[i]; - if (string.IsNullOrWhiteSpace(rawPath)) - { - lock (gate) + if (resolution == null) + return Observable.Return($"Error: no matching node for path '{resolvedPath}'"); + if (string.IsNullOrEmpty(resolution.Remainder)) + return Observable.Return("Error: path must include '{collection}/{filePath}' after the node path (e.g. 'Systemorph/content/logo.png')."); + + var remainderParts = resolution.Remainder.Split('/'); + if (remainderParts.Length < 2) + return Observable.Return($"Error: expected '{{collection}}/{{filePath}}' in remainder '{resolution.Remainder}'."); + + var collectionName = remainderParts[0]; + var filePath = string.Join("/", remainderParts.Skip(1)); + if (string.IsNullOrEmpty(Path.GetFileName(filePath))) + return Observable.Return($"Error: missing filename in path '{filePath}'."); + + var targetAddress = (Address)resolution.Prefix; + var qualifiedCollectionName = $"{resolution.Prefix}/{collectionName}"; + + // Ask the owning node hub for its collection config — exact same mechanism + // the static GET endpoint uses (see BlazorHostingExtensions.ResolveStatic). + // 🚨 .Timeout is mandatory: HandleCollectionConfigRequest is registered ONLY by + // AddContentCollections(). A target node hub WITHOUT it never answers this + // GetDataRequest, so an un-timed Take(1) hangs FOREVER — and since the MCP/REST + // boundary does ops.Upload(...).FirstAsync().ToTask(), that wedges the calling + // request (the 2026-06-14 atioz upload wedge). On timeout the TimeoutException + // falls through to the .Catch below and surfaces as a clean "Error: …" string. + return hub.Observe( + new GetDataRequest(new ContentCollectionReference([collectionName])), + o => o.WithTarget(targetAddress)) + .Take(1) + .Timeout(TimeSpan.FromSeconds(30)) + .Select(collectionResponse => { - lines[index] = "Error deleting: empty path"; - if (--remaining == 0) - tcs.TrySetResult(string.Join("\n", lines)); - } - continue; - } + // Deserialize via the hub's JSON options so the naming policy (camelCase) + // and all fields — including IsEditable — round-trip correctly. The + // ContentCollectionConfig bools default to false (matching bool's + // type-default) so writable / visible callsites must set them + // explicitly; that keeps WhenWritingDefault from silently dropping + // meaningful state across the wire. + IReadOnlyCollection? configs = collectionResponse?.Message switch + { + GetDataResponse { Data: JsonElement je } => + JsonSerializer.Deserialize(je, hub.JsonSerializerOptions), + GetDataResponse { Data: IReadOnlyCollection direct } => direct, + _ => null + }; + var sourceConfig = configs?.FirstOrDefault(c => c.Name == collectionName); + if (sourceConfig == null) return (ContentCollectionConfig?)null; + return sourceConfig with { Name = qualifiedCollectionName, Address = targetAddress }; + }) + .SelectMany(collectionConfig => + { + if (collectionConfig == null) + return Observable.Return($"Error: collection '{collectionName}' not found on '{resolution.Prefix}'."); + if (!collectionConfig.IsEditable) + return Observable.Return($"Error: collection '{collectionName}' on '{resolution.Prefix}' is read-only."); + + var contentService = hub.ServiceProvider.GetService(); + if (contentService == null) + return Observable.Return("Error: content service not configured on the hub."); + + contentService.AddConfiguration(collectionConfig); + var ioPool = hub.ServiceProvider.GetService()?.Get(IoPoolNames.FileSystem) ?? IoPool.Unbounded; + return ioPool.Run(async ct => + { + var collection = await contentService.GetCollectionAsync(qualifiedCollectionName, ct).ConfigureAwait(false); + if (collection == null) + return $"Error: failed to initialize collection '{qualifiedCollectionName}'."; + + var dir = Path.GetDirectoryName(filePath)?.Replace('\\', '/') ?? ""; + var fileName = Path.GetFileName(filePath); + using var ms = new MemoryStream(bytes); + await collection.SaveFileAsync(dir, fileName, ms).ConfigureAwait(false); + + // Post-upload seam: notify registered observers (e.g. the content-indexing + // pipeline) AFTER the save succeeds. Fire-and-forget — each observer starts its + // own off-band work (an Activity), so the upload response returns immediately + // and indexing never runs inline on this pooled continuation. No-op when no + // observer is registered; ContentCollections itself takes no indexing/AI/pg dep. + hub.RaiseContentUploaded(qualifiedCollectionName, filePath); + + return JsonSerializer.Serialize(new + { + status = "Uploaded", + path = $"{resolution.Prefix}/{collectionName}/{filePath}", + bytes = bytes.Length, + }, hub.JsonSerializerOptions); + }); + }); + }) + .Catch((Exception ex) => + { + logger.LogWarning(ex, "Upload failed for {Path}", path); + var message = ex is TimeoutException + ? $"the node for '{path}' did not respond to the content-collection lookup in time — " + + "confirm the path resolves to a content-enabled node (one configured with AddContentCollections)." + : ex.Message; + return Observable.Return($"Error: {message}"); + }); + } - string resolvedPath; + public IObservable Delete(string paths) + { + logger.LogInformation("Delete called"); + + return Observable.Defer(() => + { + List? pathList; try { - resolvedPath = ResolvePath(rawPath); + pathList = JsonSerializer.Deserialize>(paths, hub.JsonSerializerOptions); } - catch (Exception ex) + catch (JsonException ex) + { + return Observable.Return($"Invalid JSON: {ex.Message}"); + } + + if (pathList == null || pathList.Count == 0) + return Observable.Return("No paths provided."); + + var perPath = ImmutableList>.Empty; + foreach (var rawPath in pathList) { - lock (gate) + if (string.IsNullOrWhiteSpace(rawPath)) { - lines[index] = $"Error deleting '{rawPath}': {ex.Message}"; - if (--remaining == 0) - tcs.TrySetResult(string.Join("\n", lines)); + perPath = perPath.Add(Observable.Return("Error deleting: empty path")); + continue; } - continue; - } - mesh.DeleteNode(resolvedPath).Subscribe( - _ => + string resolvedPath; + try { - lock (gate) - { - lines[index] = $"Deleted: {resolvedPath}"; - if (--remaining == 0) - tcs.TrySetResult(string.Join("\n", lines)); - } - }, - ex => + resolvedPath = ResolvePath(rawPath); + } + catch (Exception ex) { - logger.LogWarning(ex, "Error deleting {Path}", resolvedPath); - lock (gate) - { - lines[index] = $"Error deleting {resolvedPath}: {ex.Message}"; - if (--remaining == 0) - tcs.TrySetResult(string.Join("\n", lines)); - } - }); - } - return tcs.Task; + perPath = perPath.Add(Observable.Return($"Error deleting '{rawPath}': {ex.Message}")); + continue; + } + + var capturedPath = resolvedPath; + perPath = perPath.Add( + mesh.DeleteNode(capturedPath) + .Select(_ => $"Deleted: {capturedPath}") + .Catch((Exception ex) => + { + logger.LogWarning(ex, "Error deleting {Path}", capturedPath); + return Observable.Return($"Error deleting {capturedPath}: {ex.Message}"); + })); + } + + return perPath + .ToObservable() + .Concat() + .ToList() + .Select(lines => string.Join("\n", lines)); + }); } /// /// Builds the standard "content is null" rejection message for Update/Patch, /// embedding the JSON schema for the node's content type when available so the - /// agent can fill content correctly on the next call. + /// agent can fill content correctly on the next call. Reactive: schema lookup + /// may need a workspace round-trip for dynamic NodeTypes. /// - internal async Task BuildNullContentErrorAsync(string path, string nodeType) + internal IObservable BuildNullContentError(string path, string nodeType) { var msg = $"Error: cannot write {path}: 'content' is null. " + "Fetch the node first with Get, modify the returned content in-place, " + "and resend the complete node. Never send null content."; - var schema = await GetContentSchemaAsync(nodeType); - if (schema != null) - msg += $" Expected content schema for NodeType '{nodeType}': {schema}"; - return msg; + return GetContentSchema(nodeType) + .Select(schema => schema != null + ? msg + $" Expected content schema for NodeType '{nodeType}': {schema}" + : msg); } /// /// Runs schema validation for and, when invalid, /// appends the expected JSON schema to the error so the agent can recover. - /// Returns null when content is valid (or when no schema is available). + /// Emits null when content is valid (or when no schema is available). /// - internal async Task ValidateContentWithSchemaAsync(MeshNode meshNode) + internal IObservable ValidateContentWithSchema(MeshNode meshNode) { - var validationError = await ValidateContentAgainstSchemaAsync(meshNode); - if (validationError == null) - return null; + return ValidateContentAgainstSchema(meshNode) + .SelectMany(validationError => + { + if (validationError == null) + return Observable.Return(null); + if (string.IsNullOrEmpty(meshNode.NodeType)) + return Observable.Return(validationError); + return GetContentSchema(meshNode.NodeType!) + .Select(schema => schema != null + ? validationError + $" Expected content schema for NodeType '{meshNode.NodeType}': {schema}" + : validationError); + }); + } - if (!string.IsNullOrEmpty(meshNode.NodeType)) + /// + /// Resolves the HubConfiguration delegate for : + /// fast path — static NodeType registered via AddMeshNodes in + /// meshConfiguration.Nodes; slow path — read the NodeType MeshNode + /// via workspace.GetMeshNodeStream and recover the delegate from the + /// already-cached DLL via + /// . + /// Single emission; emits null when neither path can produce a delegate. + /// + private IObservable?> + ResolveHubConfigForSchema(string nodeType) + { + var staticNode = hub.ServiceProvider.FindStaticNode(nodeType); + if (staticNode is { HubConfiguration: { } cfg }) { - var schema = await GetContentSchemaAsync(meshNode.NodeType); - if (schema != null) - validationError += $" Expected content schema for NodeType '{meshNode.NodeType}': {schema}"; + return Observable.Return?>(cfg); } - return validationError; + + var compilationService = hub.ServiceProvider.GetService(); + if (compilationService == null) + return Observable.Return?>(null); + + return hub.GetWorkspace().GetMeshNodeStream(nodeType) + .Where(n => n?.Content is NodeTypeDefinition def + && def.CompilationStatus == CompilationStatus.Ok + && !string.IsNullOrEmpty(def.LatestAssemblyCollection) + && !string.IsNullOrEmpty(def.LatestAssemblyPath)) + .Take(1) + .Timeout(TimeSpan.FromSeconds(5)) + .SelectMany(node => + { + var def = (NodeTypeDefinition)node!.Content!; + var version = def.LastCompiledVersion ?? node.Version; + var store = string.Equals(def.LatestAssemblyCollection, FrameworkAssemblyStore.CollectionName, StringComparison.Ordinal) + ? (IAssemblyStore)FrameworkAssemblyStore.Instance + : hub.ServiceProvider.GetService() ?? NullAssemblyStore.Instance; + return store.TryGetAssemblyPath(node.Path, version) + .SelectMany(localPath => string.IsNullOrEmpty(localPath) + ? Observable.Return(null) + : compilationService.GetConfigurationsFromExistingAssembly(localPath!, nodeType).Take(1)); + }) + .Select(result => + { + var matching = result?.NodeTypeConfigurations + .FirstOrDefault(c => string.Equals(c.NodeType, nodeType, StringComparison.OrdinalIgnoreCase)) + ?? result?.NodeTypeConfigurations.FirstOrDefault(); + return matching?.HubConfiguration; + }) + .Catch?, Exception>(_ => + Observable.Return?>(null)); } /// /// Returns the JSON schema string for the content type registered against /// , or null if no schema can be derived. /// - internal Task GetContentSchemaAsync(string nodeType) + internal IObservable GetContentSchema(string nodeType) { - try - { - var nodeTypeService = hub.ServiceProvider.GetService(); - if (nodeTypeService == null) - return Task.FromResult(null); - - var hubConfig = nodeTypeService.GetCachedHubConfiguration(nodeType); - if (hubConfig == null) - return Task.FromResult(null); - - var tempAddress = new Address($"_schema_lookup/{Guid.NewGuid():N}"); - var tempHub = hub.GetHostedHub(tempAddress, hubConfig); - if (tempHub == null) - return Task.FromResult(null); - - try + return ResolveHubConfigForSchema(nodeType) + .Select(hubConfig => { - var typeRegistry = tempHub.ServiceProvider.GetService(); - if (typeRegistry == null || !typeRegistry.TryGetType(nodeType, out var typeDefinition)) - return Task.FromResult(null); - - var schemaNode = hub.JsonSerializerOptions.GetJsonSchemaAsNode(typeDefinition!.Type); - return Task.FromResult(schemaNode.ToJsonString()); - } - finally - { - tempHub.Dispose(); - } - } - catch (Exception ex) - { - logger.LogDebug(ex, "Schema retrieval skipped for NodeType {NodeType}", nodeType); - return Task.FromResult(null); - } + if (hubConfig == null) return null; + try + { + var tempAddress = new Address($"_schema_lookup/{Guid.NewGuid():N}"); + var tempHub = hub.GetHostedHub(tempAddress, hubConfig); + if (tempHub == null) return null; + try + { + var typeRegistry = tempHub.ServiceProvider.GetService(); + if (typeRegistry == null || !typeRegistry.TryGetType(nodeType, out var typeDefinition)) + return null; + // Generate the schema with tempHub's options, NOT the parent hub's. + // The compiled content type (and its nested types) is registered in + // tempHub's TypeRegistry under its clean short name. The parent hub's + // PolymorphicTypeInfoResolver is bound to the parent's registry, which + // does not own the type — so GetOrAddType would fall back to + // TypeRegistry.FormatType and leak the fully-qualified, capitalized CLR + // FullName into every $type reference. Use the type-owning hub's options + // so the schema references resolve to the registered name. + var schemaNode = tempHub.JsonSerializerOptions.GetJsonSchemaAsNode(typeDefinition!.Type); + return (string?)schemaNode.ToJsonString(); + } + finally + { + tempHub.Dispose(); + } + } + catch (Exception ex) + { + logger.LogDebug(ex, "Schema retrieval skipped for NodeType {NodeType}", nodeType); + return null; + } + }); } /// /// Validates node content against the content type for its NodeType. /// Creates a temporary hub with the NodeType's configuration to find the /// registered content type, then attempts to deserialize the content into that type. - /// Returns an error message if invalid, or null if valid/no schema available. + /// Emits an error message if invalid, or null if valid/no schema available. /// - internal Task ValidateContentAgainstSchemaAsync(MeshNode meshNode) + internal IObservable ValidateContentAgainstSchema(MeshNode meshNode) { - try - { - var nodeTypeService = hub.ServiceProvider.GetService(); - if (nodeTypeService == null) - return Task.FromResult(null); - - var hubConfig = nodeTypeService.GetCachedHubConfiguration(meshNode.NodeType!); - if (hubConfig == null) - return Task.FromResult(null); - - // Create a temporary hub with the NodeType's config to access its type registry - var tempAddress = new Address($"_schema_validation/{Guid.NewGuid():N}"); - var tempHub = hub.GetHostedHub(tempAddress, hubConfig); - if (tempHub == null) - return Task.FromResult(null); + if (string.IsNullOrEmpty(meshNode.NodeType)) + return Observable.Return(null); - try + return ResolveHubConfigForSchema(meshNode.NodeType!) + .Select(hubConfig => { - // Find the content type from the hub's type registry - var typeRegistry = tempHub.ServiceProvider.GetService(); - if (typeRegistry == null || !typeRegistry.TryGetType(meshNode.NodeType!, out var typeDefinition)) - return Task.FromResult(null); - - var contentType = typeDefinition!.Type; - - // Serialize content to JSON and try to deserialize into the target type - var contentJson = JsonSerializer.Serialize(meshNode.Content, hub.JsonSerializerOptions); + if (hubConfig == null) return null; try { - var deserialized = JsonSerializer.Deserialize(contentJson, contentType, hub.JsonSerializerOptions); - if (deserialized == null) - return Task.FromResult($"Error: Content is null after deserialization for NodeType '{meshNode.NodeType}'."); + var tempAddress = new Address($"_schema_validation/{Guid.NewGuid():N}"); + var tempHub = hub.GetHostedHub(tempAddress, hubConfig); + if (tempHub == null) return null; + try + { + var typeRegistry = tempHub.ServiceProvider.GetService(); + if (typeRegistry == null || !typeRegistry.TryGetType(meshNode.NodeType!, out var typeDefinition)) + return null; - return Task.FromResult(null); // Valid + var contentType = typeDefinition!.Type; + var contentJson = JsonSerializer.Serialize(meshNode.Content, hub.JsonSerializerOptions); + try + { + var deserialized = JsonSerializer.Deserialize(contentJson, contentType, hub.JsonSerializerOptions); + return (string?)(deserialized == null + ? $"Error: Content is null after deserialization for NodeType '{meshNode.NodeType}'." + : null); + } + catch (JsonException ex) + { + return (string?)$"Error: Content does not match the schema for NodeType '{meshNode.NodeType}'. {ex.Message}"; + } + } + finally + { + tempHub.Dispose(); + } } - catch (JsonException ex) + catch (Exception ex) { - return Task.FromResult($"Error: Content does not match the schema for NodeType '{meshNode.NodeType}'. {ex.Message}"); + logger.LogDebug(ex, "Schema validation skipped for NodeType {NodeType}", meshNode.NodeType); + return null; } + }); + } + + /// + /// Moves a node and its descendants to a new path. Posts + /// and subscribes via RegisterCallback — no AwaitResponse, no await + /// on the hub scheduler. + /// + public IObservable Move(string sourcePath, string targetPath) + { + logger.LogInformation("Move called: {Source} -> {Target}", sourcePath, targetPath); + + if (string.IsNullOrWhiteSpace(sourcePath)) + return Observable.Return("Error: sourcePath is required."); + if (string.IsNullOrWhiteSpace(targetPath)) + return Observable.Return("Error: targetPath is required."); + + var resolvedSource = ResolvePath(sourcePath); + var resolvedTarget = ResolvePath(targetPath); + + if (resolvedSource == resolvedTarget) + return Observable.Return($"Error: target path is the same as source ({resolvedSource})."); + + return Observable.Create(observer => + { + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + // 🚨 Capture inner Subscribe for proper teardown. + IDisposable? innerSubscription = null; + try + { + var delivery = hub.Post( + new MoveNodeRequest(resolvedSource, resolvedTarget), + o => o.WithTarget(new Address(resolvedSource)))!; + + innerSubscription = hub.Observe(delivery) + .Subscribe( + d => + { + try + { + if (d.Message is MoveNodeResponse msg) + { + if (msg.Success) + observer.OnNext($"Moved: {resolvedSource} -> {resolvedTarget}"); + else + observer.OnNext( + $"Error moving {resolvedSource} -> {resolvedTarget}: {msg.Error ?? "unknown error"}" + + (msg.RejectionReason is { } r ? $" ({r})" : "")); + } + else + { + observer.OnNext( + $"Error moving {resolvedSource} -> {resolvedTarget}: unexpected response {d.Message?.GetType().Name}"); + } + observer.OnCompleted(); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Error moving {Source} -> {Target}", resolvedSource, resolvedTarget); + observer.OnNext($"Error: {ex.Message}"); + observer.OnCompleted(); + } + }, + ex => + { + observer.OnNext( + $"Error moving {resolvedSource} -> {resolvedTarget}: {ex.Message ?? "delivery failed"}"); + observer.OnCompleted(); + }); } - finally + catch (Exception ex) { - tempHub.Dispose(); + logger.LogWarning(ex, "Error moving {Source} -> {Target}", resolvedSource, resolvedTarget); + observer.OnNext($"Error: {ex.Message}"); + observer.OnCompleted(); } - } - catch (Exception ex) - { - logger.LogDebug(ex, "Schema validation skipped for NodeType {NodeType}", meshNode.NodeType); - return Task.FromResult(null); - } + + return () => + { + innerSubscription?.Dispose(); + cts.Dispose(); + }; + }); + } + + /// + /// Copies a node and all its descendants to a target namespace. Delegates to + /// — fully reactive pipeline (Query + + /// MeshNodeReference streams + CreateNode observables chained sequentially). + /// + public IObservable Copy(string sourcePath, string targetNamespace, bool force = false) + { + logger.LogInformation("Copy called: {Source} -> {Target}, force={Force}", sourcePath, targetNamespace, force); + + if (string.IsNullOrWhiteSpace(sourcePath)) + return Observable.Return("Error: sourcePath is required."); + if (string.IsNullOrWhiteSpace(targetNamespace)) + return Observable.Return("Error: targetNamespace is required."); + + var resolvedSource = ResolvePath(sourcePath); + var resolvedTarget = ResolvePath(targetNamespace); + + return NodeCopyHelper.CopyNodeTree(mesh, mesh, hub, resolvedSource, resolvedTarget, force, logger) + .Select(copied => $"Copied {copied} node(s): {resolvedSource} -> {resolvedTarget}") + .Catch((Exception ex) => + { + logger.LogWarning(ex, "Error copying {Source} -> {Target}", resolvedSource, resolvedTarget); + return Observable.Return($"Error: {ex.Message}"); + }); } /// @@ -889,32 +1733,64 @@ internal async Task BuildNullContentErrorAsync(string path, string nodeT /// Returns a JSON {status, path} envelope. The caller should wait ~100ms /// before re-accessing so the grain teardown completes. /// - public Task Recycle(string path) + public IObservable Recycle(string path) { logger.LogInformation("Recycle called with path={Path}", path); if (string.IsNullOrWhiteSpace(path)) - return Task.FromResult(JsonSerializer.Serialize( + return Observable.Return(JsonSerializer.Serialize( new { status = "Error", message = "path is required" }, hub.JsonSerializerOptions)); var resolvedPath = ResolvePath(path); if (string.IsNullOrWhiteSpace(resolvedPath)) - return Task.FromResult(JsonSerializer.Serialize( + return Observable.Return(JsonSerializer.Serialize( new { status = "Error", message = "path is required" }, hub.JsonSerializerOptions)); + // Permission gate: recycling disposes the node's hub (DisposeRequest) and forces + // re-initialization — operationally a write on that node. Require Update on the + // target so a read-only caller can't bounce other partitions' hubs. With no RLS + // wired, the default evaluator grants All and behavior is unchanged. Fail closed + // on evaluator errors/timeouts. + return hub.CheckPermission(resolvedPath, MeshWeaver.Mesh.Security.Permission.Update) + .Take(1) + .Timeout(TimeSpan.FromSeconds(10)) + .Catch((Exception ex) => + { + logger.LogWarning(ex, "Recycle: permission check failed for {Path}", resolvedPath); + return Observable.Return(false); + }) + .SelectMany(allowed => allowed + ? RecycleCore(resolvedPath) + : Observable.Return(JsonSerializer.Serialize( + new + { + status = "Error", + path = resolvedPath, + message = "Recycle requires Update permission on the target node — it disposes the node's hub and forces re-initialization. Ask someone with write access to the node (or a platform admin) to do it." + }, + hub.JsonSerializerOptions))); + } + + private IObservable RecycleCore(string resolvedPath) + { try { - // 1. Flush LOCAL NodeTypeService caches so a fresh compile runs on next access. - // Disposing the hub alone is not enough — NodeTypeService._compilationErrors - // and _compilationTasks survive hub teardown and would keep serving stale - // errors. - nodeTypeService?.InvalidateCache(resolvedPath); + // Trigger a fresh compile by flipping CompilationStatus = Pending on + // the NodeType MeshNode. The per-NodeType hub's CompileWatcher (see + // NodeTypeCompilationHelpers.InstallCompileWatcher) picks up the + // Pending flip and runs Roslyn — the MeshNode IS the cache, so we + // don't need a side cache to invalidate. + hub.GetWorkspace().GetMeshNodeStream(resolvedPath).Update(curr => + curr.Content is Graph.Configuration.NodeTypeDefinition def + ? curr with { Content = def with { CompilationStatus = CompilationStatus.Pending } } + : curr) + .Subscribe( + _ => { }, + ex => logger.LogWarning(ex, + "Recycle: failed to flip CompilationStatus=Pending for {Path}", resolvedPath)); - // 2. Broadcast the invalidation across silos via IMeshChangeFeed. Every silo's - // NodeTypeService subscribes to this feed and calls InvalidateCache locally - // when it sees an event for a tracked NodeType path. var changeFeed = hub.ServiceProvider.GetService(); if (changeFeed != null) { @@ -931,9 +1807,8 @@ public Task Recycle(string path) Timestamp: DateTimeOffset.UtcNow)); } - // 3. Dispose the hub so the next request re-initialises with fresh config. hub.Post(new DisposeRequest(), o => o.WithTarget(new Address(resolvedPath))); - return Task.FromResult(JsonSerializer.Serialize( + return Observable.Return(JsonSerializer.Serialize( new { status = "Recycled", @@ -945,7 +1820,7 @@ public Task Recycle(string path) catch (Exception ex) { logger.LogWarning(ex, "Error recycling {Path}", resolvedPath); - return Task.FromResult(JsonSerializer.Serialize( + return Observable.Return(JsonSerializer.Serialize( new { status = "Error", path = resolvedPath, message = ex.Message }, hub.JsonSerializerOptions)); } @@ -953,63 +1828,441 @@ public Task Recycle(string path) /// /// Returns compilation diagnostics for a NodeType or an instance of one. - /// The response is JSON with status (Error / Ok / - /// Unknown) and, when relevant, the error text from the last compile. - /// Used by the Coder agent's self-verification loop after creating / updating - /// a NodeType. /// - public async Task GetDiagnostics(string path) + public IObservable GetDiagnostics(string path) { logger.LogInformation("GetDiagnostics called with path={Path}", path); if (string.IsNullOrWhiteSpace(path)) - return JsonSerializer.Serialize( + return Observable.Return(JsonSerializer.Serialize( new { status = "Error", message = "path is required" }, - hub.JsonSerializerOptions); + hub.JsonSerializerOptions)); var resolvedPath = ResolvePath(path); - if (nodeTypeService == null) - return JsonSerializer.Serialize( - new { status = "Unknown", message = "INodeTypeService not registered on this hub" }, - hub.JsonSerializerOptions); - - // Resolve the owning NodeType path: either the path itself (if it IS a NodeType) - // or the NodeType of the instance at that path. - string? nodeTypePath = null; - await foreach (var node in mesh.QueryAsync(MeshQueryRequest.FromQuery($"path:{resolvedPath}"))) - { - nodeTypePath = node.Content is Graph.Configuration.NodeTypeDefinition - ? node.Path - : node.NodeType; - break; + var meshConfig = hub.ServiceProvider.GetService(); + + // Diagnostics are read directly off the NodeType MeshNode — the + // owner-driven status/error/timestamps live on NodeTypeDefinition, + // populated by the per-NodeType hub's CompileWatcher. + return FetchNode(resolvedPath).SelectMany(node => + { + // Match LookupCompilationError: node.Content arrives as JsonElement + // when the per-node hub doesn't have NodeTypeDefinition in its + // TypeRegistry, so check NodeType==MeshNode.NodeTypePath as fallback. + var isNodeTypeDef = node?.Content is Graph.Configuration.NodeTypeDefinition + || (node is not null && string.Equals(node.NodeType, MeshNode.NodeTypePath, StringComparison.Ordinal)); + var nodeTypePath = isNodeTypeDef ? node!.Path : node?.NodeType; + + if (string.IsNullOrEmpty(nodeTypePath)) + return Observable.Return(JsonSerializer.Serialize( + new { status = "Unknown", message = $"Not found: {resolvedPath}" }, + hub.JsonSerializerOptions)); + + // Fast path: the input node already IS the NodeType MeshNode + // AND its compile has settled (Ok/Error). Compiling/Pending/Unknown + // states fall through to the stream so we wait for the + // CompileWatcher's write-back instead of returning a stale snapshot. + if (isNodeTypeDef + && node!.Content is Graph.Configuration.NodeTypeDefinition ownDef + && IsSettled(ownDef)) + return Observable.Return(FormatDiagnosticsFromDef(ownDef, nodeTypePath)); + + // Static fast path: NodeType registered via AddMeshNodes — there is + // no per-NodeType hub or persisted MeshNode, so the runtime status + // is implicit Ok (its HubConfiguration is bundled with the framework). + if (hub.ServiceProvider.FindStaticNode(nodeTypePath) is not null) + return Observable.Return(FormatDiagnostics( + CompilationStatus.Ok, nodeTypePath, + error: null, startedAt: null, lastCompiledAt: null, + hub.JsonSerializerOptions)); + + // Slow path: subscribe to the NodeType's live stream and wait for + // the CompileWatcher to settle. Where(settled).Take(1) keeps the + // read in lockstep with the writer; without it we'd race against + // the Compiling → Ok/Error write-back and return stale state. + return hub.GetWorkspace().GetMeshNodeStream(nodeTypePath) + .Where(n => n?.Content is Graph.Configuration.NodeTypeDefinition d && IsSettled(d)) + .Take(1) + .Timeout(TimeSpan.FromSeconds(5)) + .Catch(_ => Observable.Return(null)) + .Select(typeNode => + { + var def = typeNode?.Content as Graph.Configuration.NodeTypeDefinition; + if (def is null) + return JsonSerializer.Serialize( + new { status = "Unknown", message = $"NodeType '{nodeTypePath}' has no definition" }, + hub.JsonSerializerOptions); + return FormatDiagnosticsFromDef(def, nodeTypePath); + }); + }); + } + + /// + /// True when the NodeType's + /// has reached a terminal state ( or + /// ). Pending and Compiling are + /// transient — readers should keep waiting for the watcher's settle + /// write rather than report a half-baked state. + /// + private static bool IsSettled(Graph.Configuration.NodeTypeDefinition def) + { + var status = def.CompilationStatus; + return status == CompilationStatus.Ok || status == CompilationStatus.Error; + } + + private string FormatDiagnosticsFromDef( + Graph.Configuration.NodeTypeDefinition def, string nodeTypePath) + { + var status = def.CompilationStatus ?? CompilationStatus.Unknown; + return FormatDiagnostics( + status, + nodeTypePath, + error: status == CompilationStatus.Error ? def.CompilationError : null, + startedAt: status == CompilationStatus.Compiling ? def.LastCompileStartedAt : null, + lastCompiledAt: status == CompilationStatus.Ok ? def.LastCompileSucceededAt : null, + hub.JsonSerializerOptions); + } + + /// + /// Triggers a compile for a NodeType by flipping its + /// to + /// via the canonical remote-stream + /// write path: opens a + /// GetRemoteStream<MeshNode, MeshNodeReference> against the + /// owning per-node hub and pushes a patch through + /// the synchronization protocol. The CompileWatcher (installed by + /// AddMeshDataSource) observes the Pending state on its own MeshNode + /// stream and runs Roslyn, then writes back + /// or plus + /// . + /// + /// Why a dedicated tool over : Patch requires both + /// Read (to merge the existing node) and Update permission on the target node. + /// Compile only needs the per-node hub to accept the synchronisation + /// patch — caller drives the same hub that the CompileWatcher listens on, so + /// state transitions never bottleneck on a routing service. + /// + /// Observe progress: poll get @nodeTypePath for + /// compilationStatus transitions, then once it settles to Ok/Error + /// follow lastCompilationActivityPath to fetch the full executed-source- + /// queries / matched-Code-paths / Roslyn-output trace. + /// + public IObservable Compile(string path) + { + logger.LogInformation("Compile called with path={Path}", path); + + if (string.IsNullOrWhiteSpace(path)) + return Observable.Return(JsonSerializer.Serialize( + new { status = "Error", message = "path is required" }, + hub.JsonSerializerOptions)); + + var resolvedPath = ResolvePath(path); + if (string.IsNullOrWhiteSpace(resolvedPath)) + return Observable.Return(JsonSerializer.Serialize( + new { status = "Error", message = "path is required" }, + hub.JsonSerializerOptions)); + + return Observable.Defer(() => + { + IWorkspace workspace; + try { workspace = hub.GetWorkspace(); } + catch (Exception ex) + { + logger.LogWarning(ex, "Compile: workspace unavailable for {Path}", resolvedPath); + return Observable.Return(JsonSerializer.Serialize( + new { status = "Error", path = resolvedPath, message = ex.Message }, + hub.JsonSerializerOptions)); + } + + // Subscribe to the NodeType's stream BEFORE flipping Pending so we + // don't miss the watcher's status transitions. The stream emits the + // current node first (whatever status it's in); we wait for a + // settled Ok/Error after the trigger. + var stream = workspace.GetMeshNodeStream(resolvedPath); + + try + { + // Impersonate System for the Pending flip. Triggering a recompile is an + // INFRASTRUCTURE operation that fills the assembly cache — it must succeed + // even when the caller has no Update right on the target partition (the + // read-only Doc partition is the canonical case). Under the caller's identity + // this flip was denied → "UpdateMeshNode failed" → the compile never ran and + // the cache stayed empty (atioz on-demand-compile failure on Doc). The + // RunCompile watcher + Release-node creation already run as System; this entry + // flip was the straggler still on the caller's identity. + var accessService = hub.ServiceProvider.GetService(); + using (accessService?.ImpersonateAsSystem()) + workspace.GetMeshNodeStream(resolvedPath).Update(node => node with + { + Content = WithPendingCompilationStatus(node.Content) + }).Subscribe( + _ => { }, + ex => logger.LogWarning(ex, + "Compile: UpdateMeshNode failed for {Path}", resolvedPath)); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Compile trigger failed for {Path}", resolvedPath); + return Observable.Return(JsonSerializer.Serialize( + new { status = "Error", path = resolvedPath, message = ex.Message }, + hub.JsonSerializerOptions)); + } + + // Wait for the watcher to write back Ok or Error (60s budget — Roslyn + // first compile of a moderate node is 5-15s; bigger trees can take + // longer; some hubs may take ~5s to emit a settled state after the + // initial Pending). Then return a structured result with the error + // body inline if Error — agents/humans get the diagnostic without a + // second polling round-trip. + return stream + .Where(n => + { + var status = ReadCompilationStatusFromNode(n); + return status == CompilationStatus.Ok || status == CompilationStatus.Error; + }) + .Take(1) + .Timeout(TimeSpan.FromSeconds(60)) + .Select(n => + { + var status = ReadCompilationStatusFromNode(n); + var error = ReadCompilationError(n); + var activityPath = ReadActivityPath(n); + return JsonSerializer.Serialize( + new + { + status = status?.ToString() ?? "Unknown", + path = resolvedPath, + error, + activityPath, + message = status == CompilationStatus.Ok + ? "Compile SUCCEEDED." + : "Compile FAILED — see `error` for Roslyn diagnostics. " + + "Full source-discovery + matched-Code-paths trace lives at " + + (activityPath ?? "(no activity log written)") + "." + }, + hub.JsonSerializerOptions); + }) + .Catch((Exception ex) => + { + logger.LogWarning(ex, + "Compile: timeout / observer error waiting for {Path} to settle", resolvedPath); + return Observable.Return(JsonSerializer.Serialize( + new + { + status = "Pending", + path = resolvedPath, + message = "Compile triggered but did not settle within the deadline. " + + "Poll `get " + resolvedPath + "` for `compilationStatus` and " + + "`lastCompilationActivityPath`. Underlying error: " + ex.Message + }, + hub.JsonSerializerOptions)); + }); + }); + } + + private static CompilationStatus? ReadCompilationStatusFromNode(MeshNode? node) + { + if (node?.Content is Graph.Configuration.NodeTypeDefinition def) + return def.CompilationStatus; + if (node?.Content is JsonElement json && json.TryGetProperty("compilationStatus", out var p)) + { + if (p.ValueKind == JsonValueKind.String && Enum.TryParse(p.GetString(), true, out var parsed)) + return parsed; } + return null; + } - if (string.IsNullOrEmpty(nodeTypePath)) - return JsonSerializer.Serialize( - new { status = "Unknown", message = $"Not found: {resolvedPath}" }, - hub.JsonSerializerOptions); - - // Compiling has priority over any prior error — the error we're seeing is stale - // and a fresh result is on its way. Tell the caller to wait and retry. - if (nodeTypeService.IsCompiling(nodeTypePath)) - { - var startedAt = nodeTypeService.GetCompilationStartedAt(nodeTypePath); - var elapsedMs = startedAt is null - ? (long?)null - : (long)(DateTimeOffset.UtcNow - startedAt.Value).TotalMilliseconds; - return JsonSerializer.Serialize( - new { status = "Compiling", nodeTypePath, elapsedMs }, - hub.JsonSerializerOptions); + private static string? ReadCompilationError(MeshNode? node) + { + if (node?.Content is Graph.Configuration.NodeTypeDefinition def) + return def.CompilationError; + if (node?.Content is JsonElement json && json.TryGetProperty("compilationError", out var p)) + return p.ValueKind == JsonValueKind.String ? p.GetString() : null; + return null; + } + + private static string? ReadActivityPath(MeshNode? node) + { + if (node?.Content is Graph.Configuration.NodeTypeDefinition def) + return def.LastCompilationActivityPath; + if (node?.Content is JsonElement json && json.TryGetProperty("lastCompilationActivityPath", out var p)) + return p.ValueKind == JsonValueKind.String ? p.GetString() : null; + return null; + } + + /// + /// Returns a new Content object with compilationStatus set to + /// Pending. Handles both strongly-typed + /// (own hub registered + /// the type) and (remote hub passed it through + /// untyped). Other content shapes are returned unchanged with a warning — + /// this method is only meaningful on a NodeType node. + /// + private object? WithPendingCompilationStatus(object? content) + { + switch (content) + { + case Graph.Configuration.NodeTypeDefinition def: + return def with { CompilationStatus = CompilationStatus.Pending }; + + case JsonElement json: + { + var node = JsonNode.Parse(json.GetRawText()) as JsonObject ?? new JsonObject(); + node["compilationStatus"] = "Pending"; + return JsonSerializer.SerializeToElement(node, hub.JsonSerializerOptions); + } + + case null: + { + var node = new JsonObject { ["compilationStatus"] = "Pending" }; + return JsonSerializer.SerializeToElement(node, hub.JsonSerializerOptions); + } + + default: + logger.LogWarning( + "Compile: unexpected content type {Type} on NodeType node — wrapping", + content.GetType().Name); + return content; + } + } + + /// + /// Pure JSON formatter for . Lives on its own so a unit + /// test can lock in the exact wording: in particular, the Ok branch must explicitly + /// say "Compile SUCCEEDED" (not just "status: Ok") so that agents and humans reading + /// the response can't confuse "no error recorded" with "compile actually ran cleanly". + /// + public static string FormatDiagnostics( + CompilationStatus status, + string nodeTypePath, + string? error, + DateTimeOffset? startedAt, + DateTimeOffset? lastCompiledAt, + JsonSerializerOptions options) + { + switch (status) + { + case CompilationStatus.Compiling: + { + var elapsedMs = startedAt is null + ? (long?)null + : (long)(DateTimeOffset.UtcNow - startedAt.Value).TotalMilliseconds; + return JsonSerializer.Serialize( + new + { + status = "Compiling", + nodeTypePath, + elapsedMs, + message = "Compile is IN PROGRESS. The NodeType assembly is not yet available — " + + "wait and re-call GetDiagnostics." + }, + options); + } + case CompilationStatus.Error: + return JsonSerializer.Serialize( + new + { + status = "Error", + nodeTypePath, + error, + message = "Compile FAILED. The NodeType assembly was NOT built — see `error` " + + "for the Roslyn diagnostics. Fix the source and recycle the NodeType." + }, + options); + case CompilationStatus.Ok: + return JsonSerializer.Serialize( + new + { + status = "Ok", + nodeTypePath, + lastCompiledAt, + message = "Compile SUCCEEDED at " + lastCompiledAt?.ToString("u") + + ". The NodeType assembly was built without errors and is loaded." + }, + options); + case CompilationStatus.Unknown: + default: + return JsonSerializer.Serialize( + new + { + status = "Unknown", + nodeTypePath, + message = "NO compile has run since the last invalidation (this is NOT 'Ok'). " + + "The assembly state is unknown — trigger a compile (e.g. navigate to a " + + "layout area on an instance) and re-call GetDiagnostics." + }, + options); } + } - var err = nodeTypeService.GetCompilationError(nodeTypePath); - if (string.IsNullOrEmpty(err)) - return JsonSerializer.Serialize( - new { status = "Ok", nodeTypePath }, - hub.JsonSerializerOptions); + /// + /// Runs an executable Code node's C# through the kernel (Microsoft.DotNet.Interactive) + /// and returns status JSON. The target node must have + /// CodeConfiguration.IsExecutable == true. Emits once when the kernel signals + /// completion (the kernel hub posts a response to + /// after the code finishes) or on timeout. + /// + public IObservable ExecuteScript(string path, int timeoutSeconds = 120) + { + logger.LogInformation("ExecuteScript called with path={Path}", path); + if (string.IsNullOrWhiteSpace(path)) + return Observable.Return(JsonSerializer.Serialize( + new { status = "Error", message = "path is required" }, + hub.JsonSerializerOptions)); - return JsonSerializer.Serialize( - new { status = "Error", nodeTypePath, error = err }, - hub.JsonSerializerOptions); + var resolvedPath = ResolvePath(path); + + // Fire-and-forget dispatch. The Code hub creates an Activity at + // `{partition}/_Activity/{submissionId}` (the kernel runs inside the + // Activity hub) and writes ActivityLog.Messages + Status as the script + // executes. We pre-generate the SubmissionId here so we can return the + // Activity path immediately — callers poll `get @{activityPath}` to + // observe progress and final status without waiting for an ack. + // + // Why no wait? The ack `ExecuteScriptResponse` from the Code hub is a + // throw-away "got it, here's the activity path" — but routing it back + // to a hosted MCP session hub is fragile (we lost ~half a day chasing + // it). The Activity itself is the source of truth: live messages, + // terminal Status, error details — everything's there. So just return + // the activity path and let the caller observe. + var partition = resolvedPath.Split('/', 2)[0]; + if (string.IsNullOrEmpty(partition)) + return Observable.Return(JsonSerializer.Serialize( + new + { + status = "Error", + path = resolvedPath, + message = "Could not derive partition from script path." + }, + hub.JsonSerializerOptions)); + + var submissionId = Guid.NewGuid().ToString("N"); + var activityPath = $"{partition}/_Activity/{submissionId}"; + + try + { + hub.Post( + new ExecuteScriptRequest { SubmissionId = submissionId }, + o => o.WithTarget(new Address(resolvedPath))); + } + catch (Exception ex) + { + logger.LogError(ex, "ExecuteScript failed to dispatch for {Path}", resolvedPath); + return Observable.Return(JsonSerializer.Serialize( + new { status = "Error", path = resolvedPath, message = ex.Message }, + hub.JsonSerializerOptions)); + } + + return Observable.Return(JsonSerializer.Serialize( + new + { + status = "Dispatched", + path = resolvedPath, + submissionId, + activityPath, + message = $"Script dispatched. Poll `get @{activityPath}` for live messages " + + "and final status (Running → Succeeded/Failed)." + }, + hub.JsonSerializerOptions)); } } diff --git a/src/MeshWeaver.AI/MeshPlugin.cs b/src/MeshWeaver.AI/MeshPlugin.cs index 17c38bcce..e91d912f1 100644 --- a/src/MeshWeaver.AI/MeshPlugin.cs +++ b/src/MeshWeaver.AI/MeshPlugin.cs @@ -1,5 +1,8 @@ using System.ComponentModel; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; using MeshWeaver.Layout; +using MeshWeaver.Mesh.Threading; using MeshWeaver.Messaging; using Microsoft.Extensions.AI; using Microsoft.Extensions.DependencyInjection; @@ -18,84 +21,105 @@ public class MeshPlugin(IMessageHub hub, IAgentChat chat) private readonly ILogger logger = hub.ServiceProvider.GetRequiredService>(); private readonly AccessService? accessService = hub.ServiceProvider.GetService(); + // Bounded "Process" pool (Wave 3 of Controlled I/O Pooling). RunTests spawns a + // `dotnet test` child process and BLOCKS a thread for the whole run (up to 5 + // minutes) — that thread-holding wait must run OFF the hub/grain scheduler and + // be bounded so a fan-out of test runs can't trigger ThreadPool thread-injection + // that starves Orleans' grain schedulers. Resolve the mesh-scoped pool from DI; + // fall back to the stateless IoPool.Unbounded when no registry is wired (still + // offloads to the ThreadPool — never worse than the inline call). No static state. + private readonly IIoPool _processPool = + hub.ServiceProvider.GetService()?.Get(IoPoolNames.Process) + ?? IoPool.Unbounded; + [Description("Retrieves a node or content from the mesh by path. Paths are relative to current context; use @/ prefix for absolute paths. Supports Unified Path prefixes: content/, data/, schema/, model/, collection/, area/.")] public Task Get( [Description("Path to data. Relative: @content/file.docx, @MyChild/*. Absolute: @/OrgA/Doc, @/OrgA/content/file.docx. For spaces: \"@content/My File.docx\"")] string path) - { - RestoreAccessContext(); - return ops.Get(ResolveContextPath(path)); - } + => WithContext(() => ops.Get(ResolveContextPath(path))).FirstAsync().ToTask(); - [Description("Searches the mesh using GitHub-style query syntax.")] + [Description("Searches the mesh using GitHub-style query syntax. Returns {count, limit, truncated, results:[{path,name,nodeType}]} — when 'truncated' is true there are more matches than returned; narrow the query or raise 'limit'.")] public Task Search( [Description("Query string (e.g., 'nodeType:Agent', 'path:ACME scope:descendants', 'name:*sales*')")] string query, - [Description("Base path to search from (e.g., @graph). Empty for all.")] string? basePath = null) - { - RestoreAccessContext(); - return ops.Search(query, basePath != null ? ResolveContextPath(basePath) : null); - } + [Description("Base path to search from (e.g., @graph). Empty for all.")] string? basePath = null, + [Description("Maximum number of results to return. Default 50, max 200.")] int limit = 50) + => WithContext(() => ops.Search(query, basePath != null ? ResolveContextPath(basePath) : null, limit)).FirstAsync().ToTask(); [Description("Creates a new node in the mesh. ALWAYS set the 'name' property to a human-readable display name.")] public Task Create( [Description("JSON MeshNode with required: id, name, nodeType, namespace. Example: {\"id\":\"my-page\",\"namespace\":\"MyOrg\",\"name\":\"My Page\",\"nodeType\":\"Markdown\"}")] string node) - { - RestoreAccessContext(); - return ops.Create(node); - } + => WithContext(() => ops.Create(node)).FirstAsync().ToTask(); [Description("Full replacement update of existing nodes. ALWAYS Get the node first, modify the returned object, then send it back here unchanged-except-for-edits. The 'content' field MUST be present and non-null — null content is rejected and the response will include the expected schema. Prefer Patch for small changes.")] public Task Update( [Description("JSON array of complete MeshNode objects fetched via Get and then modified")] string nodes) - { - RestoreAccessContext(); - return ops.Update(nodes); - } + => WithContext(() => ops.Update(nodes)).FirstAsync().ToTask(); - [Description("Partial update of a single node. Only the keys present in 'fields' are changed; omitted keys preserve existing values. Do NOT include 'content' unless you intend to overwrite it — and never set 'content' to null (will be rejected with the schema). Prefer this over Update for small edits like icon/name/category.")] + [Description("Partial update of a single node. Only the keys present in 'fields' are changed; omitted keys preserve existing values. 'content' deep-merges (RFC 7396): nested keys you send are updated, omitted keys are kept, a null member deletes that one key — so you can change a single content field without resending the rest. Setting the whole 'content' to null is rejected (with the schema). Prefer this over Update for small edits like icon/name/category.")] public Task Patch( [Description("Path to the node (e.g., @User/rbuergi/my-node)")] string path, - [Description("JSON object with ONLY the fields to change. Examples: {\"icon\": \"...\"}, {\"name\": \"New Name\"}. Include 'content' only if overwriting — and never as null.")] string fields) - { - RestoreAccessContext(); - return ops.Patch(ResolveContextPath(path), fields); - } + [Description("JSON object with ONLY the fields to change. Examples: {\"icon\": \"...\"}, {\"name\": \"New Name\"}, {\"content\":{\"logo\":\"https://…\"}} (deep-merges into existing content). Never set 'content' to null.")] string fields) + => WithContext(() => ops.Patch(ResolveContextPath(path), fields)).FirstAsync().ToTask(); - [Description("Deletes nodes from the mesh by path.")] + [Description("Anchored text edit on a node's content (Markdown body or Code source). Replaces oldText with newText — pass just the snippet to change plus enough surrounding context to make it unique, instead of re-sending the whole document through Patch. Fails with a descriptive error when the text isn't found or isn't unique. Preferred over Patch for any edit inside a long document or source file.")] + public Task EditContent( + [Description("Path to the node (e.g., @User/rbuergi/my-doc or @ACME/Story/Source/Story.cs)")] string path, + [Description("The exact text to replace — copy it verbatim from Get, including whitespace and line breaks. Must match exactly once (or set replaceAll).")] string oldText, + [Description("The replacement text.")] string newText, + [Description("Replace every occurrence instead of requiring a unique match. Default: false.")] bool replaceAll = false) + => WithContext(() => ops.EditContent(ResolveContextPath(path), oldText, newText, replaceAll)).FirstAsync().ToTask(); + + [Description("Deletes nodes from the mesh by path. Recursive: deleting a parent removes all descendants — pass the subtree root, no need to enumerate children.")] public Task Delete( [Description("JSON array of path strings to delete")] string paths) - { - RestoreAccessContext(); - return ops.Delete(paths); - } + => WithContext(() => ops.Delete(paths)).FirstAsync().ToTask(); [Description("Returns compilation diagnostics for a NodeType or an instance of one. Status is 'Ok' when the type compiled cleanly, 'Error' with a detailed message when it failed, or 'Unknown' when no compile has happened yet. Use this after creating/updating a NodeType to verify it actually compiles — a NodeType that doesn't compile is not 'done'.")] public Task GetDiagnostics( [Description("Path to a NodeType (e.g., @Systemorph/SocialMedia/Profile) or to any instance of one")] string path) - { - RestoreAccessContext(); - return ops.GetDiagnostics(ResolveContextPath(path)); - } + => WithContext(() => ops.GetDiagnostics(ResolveContextPath(path))).FirstAsync().ToTask(); [Description("Recycles the hub at the given path by posting DisposeRequest. Forces a fresh hub initialization on the next access — use this after fixing a broken NodeType, after editing the `sources` list, or whenever a grain is stuck. Returns {status:'Recycled', path}. Wait ~100ms before the next access so the grain teardown completes.")] public Task Recycle( [Description("Path to the node (e.g., @Systemorph/SocialMedia/Profile). Use the NodeType path to recycle the whole type; use an instance path to recycle just that instance's hub.")] string path) - { - RestoreAccessContext(); - return ops.Recycle(ResolveContextPath(path)); - } + => WithContext(() => ops.Recycle(ResolveContextPath(path))).FirstAsync().ToTask(); - /// - /// Restores the user's AccessContext from . - /// AsyncLocal doesn't flow reliably through the AI framework's streaming + tool - /// invocation pipeline, so every plugin entry point must explicitly re-seed the - /// context before it hits downstream hub-backed operations. Idempotent when the - /// AccessContextAIFunction wrapper has already run. - /// - private void RestoreAccessContext() + [Description("Moves a node and its descendants to a new path. Equivalent to the Move menu item. Requires Delete on the source namespace and Create on the target. Source and target are full paths (namespace + id), e.g. 'OrgA/Child' -> 'OrgB/Child'.")] + public Task Move( + [Description("Current path of the node (e.g., @OrgA/Child)")] string sourcePath, + [Description("New path for the node (e.g., @OrgB/Child)")] string targetPath) + => WithContext(() => ops.Move(ResolveContextPath(sourcePath), ResolveContextPath(targetPath))).FirstAsync().ToTask(); + + [Description("Copies a node and all its descendants to a target namespace. Equivalent to the Copy menu item. Source ids are preserved; paths are rewritten under the target namespace.")] + public Task Copy( + [Description("Current path of the node to copy (e.g., @OrgA/Child)")] string sourcePath, + [Description("Target namespace to copy under (e.g., @OrgB)")] string targetNamespace, + [Description("Overwrite existing nodes at the target. Default: false (skip if any target path already exists).")] bool force = false) + => WithContext(() => ops.Copy(ResolveContextPath(sourcePath), ResolveContextPath(targetNamespace), force)).FirstAsync().ToTask(); + + // MCP adapter helper: re-seeds the user's AccessContext on Subscribe, then runs the + // observable. AsyncLocal doesn't flow reliably through the AI framework's streaming + + // tool invocation pipeline, so each plugin entry point must explicitly re-seed before + // hitting hub-backed ops. Defer ensures the seed runs on Subscribe (same call as ToTask), + // keeping each public method a strict one-line MCP adapter per AsynchronousCalls.md. + // + // CAPTURE THE EFFECTIVE IDENTITY SYNCHRONOUSLY, on the calling thread, where it is reliable: + // the agent's execution context wins; else the request-scoped Context an active delivery set; + // else the circuit/persistent context the Blazor session or test established. Re-reading it + // *inside* Defer is the concurrency bug — under N parallel tool calls (Task.WhenAll) the + // ambient AsyncLocal can be lost past a thread hop for one operation, so its hub-post stamps an + // empty AccessContext → "Access denied". Capturing once, here, and re-seeding the request-scoped + // Context on Subscribe makes every operation carry the caller's identity regardless of flow. + private IObservable WithContext(Func> work) { - var userCtx = chat.ExecutionContext?.UserAccessContext; - if (userCtx != null) - accessService?.SetContext(userCtx); + var captured = chat.ExecutionContext?.UserAccessContext + ?? accessService?.Context + ?? accessService?.CircuitContext; + return Observable.Defer(() => + { + if (captured != null) + accessService?.SetContext(captured); + return work(); + }); } [Description("Displays a node's visual layout in the chat UI.")] @@ -112,20 +136,136 @@ public string NavigateTo( return $"Navigating to: {resolvedPath}"; } + [Description("Runs xUnit tests via `dotnet test` on the given test project path (repo-relative, e.g. 'test/MeshWeaver.Acme.Test'). Optional filter uses the xunit `--filter` syntax: 'FullyQualifiedName~TodoViewsTest' to narrow by class, or '...Test.MethodName' for a single method. Returns the condensed test runner output (stdout + pass/fail summary). Dev-only — intended for the Monolith portal, not production.")] + public Task RunTests( + [Description("Repo-relative path to the test project or its directory (e.g. 'test/MeshWeaver.Acme.Test')")] string projectPath, + [Description("Optional xunit filter expression (e.g. 'FullyQualifiedName~TodoViewsTest')")] string? filter = null) + { + logger.LogInformation("RunTests called project={Project} filter={Filter}", projectPath, filter ?? ""); + + var repoRoot = FindRepoRoot(AppContext.BaseDirectory); + if (repoRoot is null) + return Task.FromResult("{\"status\":\"Error\",\"message\":\"Could not locate repo root (no MeshWeaver.slnx upstream from executable).\"}"); + + var fullPath = Path.GetFullPath(Path.Combine(repoRoot, projectPath)); + if (!fullPath.StartsWith(repoRoot, StringComparison.OrdinalIgnoreCase)) + return Task.FromResult("{\"status\":\"Error\",\"message\":\"projectPath must stay inside the repo root.\"}"); + if (!Directory.Exists(fullPath) && !File.Exists(fullPath)) + return Task.FromResult($"{{\"status\":\"Error\",\"message\":\"Path not found: {projectPath}\"}}"); + + var args = new List { "test", fullPath, "--no-restore", "--nologo" }; + if (!string.IsNullOrWhiteSpace(filter)) + { + args.Add("--filter"); + args.Add(filter); + } + + // Route the thread-holding `dotnet test` wait onto the bounded Process pool + // (off the hub/grain scheduler). InvokeBlocking dispatches on the pool's + // LimitedConcurrencyLevelTaskScheduler; the .ToTask() at the MCP/AI-tool + // boundary is the only Task bridge (the AIFunctionFactory surface requires + // a Task). Behavior is identical to the previous inline await. + return _processPool + .InvokeBlocking(ct => RunTestsCore(repoRoot, projectPath, filter, args, ct)) + .FirstAsync() + .ToTask(); + } + + private static string RunTestsCore( + string repoRoot, string projectPath, string? filter, + IReadOnlyList args, CancellationToken ct) + { + using var process = new System.Diagnostics.Process + { + StartInfo = new System.Diagnostics.ProcessStartInfo + { + FileName = "dotnet", + WorkingDirectory = repoRoot, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + } + }; + foreach (var a in args) process.StartInfo.ArgumentList.Add(a); + + var stdout = new System.Text.StringBuilder(); + var stderr = new System.Text.StringBuilder(); + process.OutputDataReceived += (_, e) => { if (e.Data != null) stdout.AppendLine(e.Data); }; + process.ErrorDataReceived += (_, e) => { if (e.Data != null) stderr.AppendLine(e.Data); }; + + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + // Block this pooled thread on the child process. A pool unsubscribe / dispose + // (ct) tears the process down; a 5-minute wall clock cap bounds the run. Both + // resolve to the same Timeout result the inline await produced before. + using var killOnCancel = ct.Register(() => { try { process.Kill(entireProcessTree: true); } catch { } }); + var exited = process.WaitForExit((int)TimeSpan.FromMinutes(5).TotalMilliseconds); + if (!exited) + { + try { process.Kill(entireProcessTree: true); } catch { } + return "{\"status\":\"Timeout\",\"message\":\"Test run exceeded 5 minutes.\"}"; + } + // WaitForExit(timeout) can return before the async output handlers have + // flushed the final lines; the parameterless overload waits for that flush. + process.WaitForExit(); + + var combined = stdout.ToString(); + if (stderr.Length > 0) combined += "\n--- stderr ---\n" + stderr; + // Trim to last ~4 KB so a noisy build log doesn't blow up the tool result. + const int MaxLen = 4000; + if (combined.Length > MaxLen) + combined = "…\n" + combined[^MaxLen..]; + + return System.Text.Json.JsonSerializer.Serialize(new + { + status = process.ExitCode == 0 ? "Passed" : "Failed", + exitCode = process.ExitCode, + projectPath, + filter, + output = combined, + }); + } + + private static string? FindRepoRoot(string startDir) + { + var dir = new DirectoryInfo(startDir); + while (dir is not null) + { + if (File.Exists(Path.Combine(dir.FullName, "MeshWeaver.slnx"))) return dir.FullName; + dir = dir.Parent; + } + return null; + } + private string ResolveContextPath(string path) => MeshOperations.ResolveContextPath(chat, path); + /// + /// RunTests only exists where the source repo does: it shells out to `dotnet test` + /// against a repo-relative project path, which requires MeshWeaver.slnx upstream of + /// the executable. Dev/test machines have it; deployed containers don't — so the + /// tool is simply absent from the agent's tool list in production instead of being + /// a permanently-erroring trap. + /// + private static bool RunTestsAvailable => FindRepoRoot(AppContext.BaseDirectory) is not null; + /// /// Creates the standard tools for this plugin (read-only operations). /// public IList CreateTools() { - return - [ + var tools = new List + { AIFunctionFactory.Create(Get), AIFunctionFactory.Create(Search), AIFunctionFactory.Create(NavigateTo), AIFunctionFactory.Create(GetDiagnostics), - ]; + }; + if (RunTestsAvailable) + tools.Add(AIFunctionFactory.Create(RunTests)); + return tools; } /// @@ -133,17 +273,23 @@ public IList CreateTools() /// public IList CreateAllTools() { - return - [ + var tools = new List + { AIFunctionFactory.Create(Get), AIFunctionFactory.Create(Search), AIFunctionFactory.Create(NavigateTo), AIFunctionFactory.Create(Create), AIFunctionFactory.Create(Update), AIFunctionFactory.Create(Patch), + AIFunctionFactory.Create(EditContent), AIFunctionFactory.Create(Delete), + AIFunctionFactory.Create(Move), + AIFunctionFactory.Create(Copy), AIFunctionFactory.Create(GetDiagnostics), AIFunctionFactory.Create(Recycle), - ]; + }; + if (RunTestsAvailable) + tools.Add(AIFunctionFactory.Create(RunTests)); + return tools; } } diff --git a/src/MeshWeaver.AI/MeshWeaver.AI.csproj b/src/MeshWeaver.AI/MeshWeaver.AI.csproj index 7c772aa97..c991103ae 100644 --- a/src/MeshWeaver.AI/MeshWeaver.AI.csproj +++ b/src/MeshWeaver.AI/MeshWeaver.AI.csproj @@ -3,7 +3,6 @@ enable - @@ -26,6 +25,7 @@ + @@ -37,5 +37,6 @@ + diff --git a/src/MeshWeaver.AI/MeshWeaverHarness.cs b/src/MeshWeaver.AI/MeshWeaverHarness.cs new file mode 100644 index 000000000..6f86b600d --- /dev/null +++ b/src/MeshWeaver.AI/MeshWeaverHarness.cs @@ -0,0 +1,28 @@ +using Microsoft.Extensions.AI; + +namespace MeshWeaver.AI; + +/// +/// The native MeshWeaver harness — runs the agent + model system (the +/// provider factories drive the round). returns +/// null so uses the default +/// path. This is the one harness that surfaces agent +/// and model selection. +/// +public sealed class MeshWeaverHarness : IHarness +{ + public string Id => Harnesses.MeshWeaver; + + public Harness Definition => new() + { + Id = Harnesses.MeshWeaver, + DisplayName = "MeshWeaver", + Description = "MeshWeaver agents with a selectable model.", + Icon = "/static/NodeTypeIcons/meshweaver-logo.svg", + Order = 0, + IsDefault = true, + SupportsAgentSelection = true + }; + + public IChatClient? CreateChatClient(HarnessExecutionContext context) => null; +} diff --git a/src/MeshWeaver.AI/ModelDefinition.cs b/src/MeshWeaver.AI/ModelDefinition.cs new file mode 100644 index 000000000..4a2598806 --- /dev/null +++ b/src/MeshWeaver.AI/ModelDefinition.cs @@ -0,0 +1,119 @@ +namespace MeshWeaver.AI; + +/// +/// Content shape for nodeType:Model mesh nodes — the +/// bring-your-own-model surface. Mirrors the role +/// plays for nodeType:Agent: a +/// mesh node carries everything needs to +/// instantiate a chat client (endpoint, model id, auth reference). Discovery +/// happens via the same workspace synced query that loads agents +/// (nodeType:Agent|Model). +/// +/// Auth is handled by an — a path or +/// key into a secret store rather than the literal credential — so a Model +/// node is safe to read with the same RLS that gates other content. The +/// factory selected by is responsible for +/// resolving the secret at request time. +/// +public record ModelDefinition +{ + /// + /// Stable model identifier — what the underlying API expects in the + /// model field of a chat-completions request. Must match the + /// value the chosen factory accepts (e.g. gpt-4o-mini, + /// claude-sonnet-4-20250514). Used as the dictionary key in the + /// chat client and as the value of currentModelName. + /// + public required string Id { get; init; } + + /// + /// Human-readable name shown in the model picker. Defaults to + /// via when not set. + /// + public string? DisplayName { get; init; } + + /// + /// Factory category — picks which + /// instance handles the model. Free-form string matched against + /// / + /// ; common values: + /// OpenAI, AzureOpenAI, Anthropic, + /// AzureFoundry, GitHubCopilot. + /// + public required string Provider { get; init; } + + /// + /// Optional endpoint override — for self-hosted / OpenAI-compatible + /// gateways. Null means "use the factory's default endpoint". + /// + public string? Endpoint { get; init; } + + /// + /// Reference to the API key in the host's secret store (e.g. a config + /// path like OpenAI:ApiKey, an Azure Key Vault secret name, or + /// an environment variable). The literal credential is never stored + /// in the node content — only the lookup key. + /// + public string? ApiKeySecretRef { get; init; } + + /// + /// Path of the nodeType:ModelProvider node that owns this + /// model's credentials — e.g. Model/Anthropic for built-in + /// catalog entries, or {userId}/Model/Anthropic for + /// user-authored BYO models. The chat-client factory's + /// follows this reference + /// to read / + /// . Null on legacy + /// catalog rollouts that stamp / + /// directly on the model node. + /// + public string? ProviderRef { get; init; } + + /// + /// Optional description shown in the picker. + /// + public string? Description { get; init; } + + /// + /// Display order in the picker. Lower sorts first; falls back to + /// alphabetical within the same order. + /// + public int Order { get; init; } + + /// + /// Price charged per ONE MILLION input (prompt) tokens, in + /// . Used by the token-cost summaries to turn a + /// thread's / space's recorded token usage into a monetary cost + /// (cost = tokens / 1_000_000 × price). Null means "unknown / not priced" + /// — the summaries then fall back to defaults, + /// and show the tokens without a cost if neither is set. + /// + [System.ComponentModel.Description("Input price per 1M tokens")] + public decimal? InputPricePerMillionTokens { get; init; } + + /// + /// Price charged per ONE MILLION output (completion) tokens, in + /// . See . + /// + [System.ComponentModel.Description("Output price per 1M tokens")] + public decimal? OutputPricePerMillionTokens { get; init; } + + /// + /// ISO currency code the per-million prices are denominated in (e.g. + /// USD, EUR, CHF). Null defaults to USD in the + /// cost summaries. + /// + [System.ComponentModel.Description("Currency (e.g. USD)")] + public string? Currency { get; init; } + + /// + /// Projects this definition into the lighter + /// shape consumed by the chat picker. + /// + public ModelInfo ToModelInfo(int factoryOrder = 0) => new() + { + Name = Id, + Provider = Provider, + Order = Order != 0 ? Order : factoryOrder + }; +} diff --git a/src/MeshWeaver.AI/ModelDiscoveryService.cs b/src/MeshWeaver.AI/ModelDiscoveryService.cs new file mode 100644 index 000000000..4f615f881 --- /dev/null +++ b/src/MeshWeaver.AI/ModelDiscoveryService.cs @@ -0,0 +1,147 @@ +using System.Reactive.Linq; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Mesh; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI; + +/// +/// Hierarchical model + provider discovery anchored to the top-level +/// mesh hub. Mirrors the shape of the access-rights resolver — both +/// the node-path hierarchy AND the NodeType-path hierarchy contribute, +/// combined by closest-wins. +/// +/// 🚨 Registered on the MESH hub (the long-lived top-level hub), +/// NEVER on a per-thread / per-execution hub. Per-thread hubs can be +/// blocked by an in-flight handler (see feedback_synced_query_thread_hub.md); +/// anchoring the synced subscriptions here means the cache survives +/// any thread-execution stall. +/// +/// Three layers, all backed by workspace.GetQuery: +/// +/// (a) — exact-node +/// snapshot. One synced query per node path. +/// (b) — walks +/// UP the path. Combines (a) for the node + parent + grandparent +/// + … + root. Most levels emit empty. +/// (c) — union of (b) +/// applied to the node-path AND (b) applied to the NodeType-path. +/// This is what the chat-client factory / picker actually asks +/// for: "what models are available at this thread, given its +/// current path and its NodeType's path"? +/// +/// +/// +/// No materialised observable cache: every call rebuilds the +/// composition over , whose underlying +/// workspace.GetQuery(id, …) is itself cached by id +/// (Replay(1).RefCount() upstream). Rebuilding the +/// CombineLatest wrapper is cheap and always reflects live state — +/// no stale empties to evict, so no Invalidate needed. RLS is +/// applied per-subscription at the source: a caller without Read sees an +/// empty snapshot without affecting any other caller. +/// +public sealed class ModelDiscoveryService +{ + private readonly IMessageHub meshHub; + private readonly ILogger? logger; + + public ModelDiscoveryService(IMessageHub meshHub) + { + this.meshHub = meshHub; + logger = meshHub.ServiceProvider.GetService() + ?.CreateLogger(); + } + + /// + /// (a) Live snapshot of every LanguageModel + ModelProvider + /// node declared directly under 's + /// _Provider satellite subtree. Empty for nodes that haven't + /// configured any provider. + /// + public IObservable> GetModelsAtNode(string nodePath) + { + if (string.IsNullOrEmpty(nodePath)) + return BuildSynced("root", + $"namespace:{ModelProviderNodeType.RootNamespace} nodeType:{TypeFilter} scope:descendants"); + + return BuildSynced($"node@{nodePath}", + $"namespace:{nodePath}/{ModelProviderNodeType.RootNamespace} nodeType:{TypeFilter} scope:descendants"); + } + + /// + /// (b) Walks UP the path hierarchy (node → parent → grandparent → … + /// → root), combining every level's + /// emission. Most levels will emit + /// empty; the union is what the caller subscribes to. Closest-wins + /// merging is the projector's job — this method just hands back the + /// full union sorted by depth. + /// + public IObservable> GetModelsForNodeHierarchy(string nodePath) + { + var streams = EnumerateAncestors(nodePath) + .Select(GetModelsAtNode) + .ToArray(); + if (streams.Length == 0) + return Observable.Return((IReadOnlyList)Array.Empty()); + return Observable.CombineLatest(streams) + .Select(levels => (IReadOnlyList) + levels.SelectMany(l => l) + .GroupBy(n => n.Path, StringComparer.Ordinal) + .Select(g => g.First()) + .ToList()); + } + + /// + /// (c) Effective models for a chat — union of the namespace + /// hierarchy AND the NodeType hierarchy. Both walks are independent; + /// closer entries (later in either walk) shadow further ones at + /// projection time. + /// + public IObservable> GetEffectiveModels(string nodePath, string? nodeTypePath = null) + { + var fromNs = GetModelsForNodeHierarchy(nodePath); + var fromNt = !string.IsNullOrEmpty(nodeTypePath) + ? GetModelsForNodeHierarchy(nodeTypePath) + : Observable.Return((IReadOnlyList)Array.Empty()); + return fromNs.CombineLatest(fromNt, (a, b) => (IReadOnlyList) + a.Concat(b) + .GroupBy(n => n.Path, StringComparer.Ordinal) + .Select(g => g.First()) + .ToList()); + } + + private const string TypeFilter = LanguageModelNodeType.NodeType + "|" + ModelProviderNodeType.NodeType; + + private IObservable> BuildSynced(string id, params string[] queries) + { + var workspace = meshHub.GetWorkspace(); + // workspace.GetQuery is cached by id (Replay(1).RefCount upstream), so + // re-projecting per call is cheap and always reflects live state. + return workspace.GetQuery($"discovery:{id}", queries) + .Select(s => (IReadOnlyList)s.ToList()); + } + + private static IEnumerable EnumerateAncestors(string path) + { + // Yield the path itself, then every parent up to the root, then + // the root namespace sentinel "" (so static catalog at + // namespace=_Provider is always included). + if (!string.IsNullOrEmpty(path)) + { + yield return path; + var current = path; + while (true) + { + var idx = current.LastIndexOf('/'); + if (idx <= 0) break; + current = current[..idx]; + yield return current; + } + } + yield return ""; + } +} diff --git a/src/MeshWeaver.AI/ModelInfo.cs b/src/MeshWeaver.AI/ModelInfo.cs index 89613c308..45e7c1f82 100644 --- a/src/MeshWeaver.AI/ModelInfo.cs +++ b/src/MeshWeaver.AI/ModelInfo.cs @@ -10,6 +10,14 @@ public record ModelInfo /// public required string Name { get; init; } + /// + /// The model MeshNode PATH (e.g. _Provider/Anthropic/claude-…). This is what a + /// selection persists onto the composer's — the + /// node-picker identity, exactly like AgentDisplayInfo.Path. Null for a model that + /// has no backing node (legacy factory entries — being eliminated). + /// + public string? Path { get; init; } + /// /// The provider/factory name (e.g., "Azure OpenAI", "GitHub Copilot"). /// diff --git a/src/MeshWeaver.AI/ModelPricing.cs b/src/MeshWeaver.AI/ModelPricing.cs new file mode 100644 index 000000000..92668383c --- /dev/null +++ b/src/MeshWeaver.AI/ModelPricing.cs @@ -0,0 +1,100 @@ +using System.Collections.Immutable; + +namespace MeshWeaver.AI; + +/// +/// A per-million-token price for one model: input price, output price, and the +/// currency they're denominated in. The single shape the token-cost summaries +/// use, whether the rate comes from a node or the +/// built-in table. +/// +/// Price per 1,000,000 input (prompt) tokens. +/// Price per 1,000,000 output (completion) tokens. +/// ISO currency code (e.g. USD). +public record ModelPriceRate(decimal InputPerMillion, decimal OutputPerMillion, string Currency) +{ + /// + /// Monetary cost of the given token counts at this rate: + /// in/1e6 × InputPerMillion + out/1e6 × OutputPerMillion. + /// + public decimal Cost(long inputTokens, long outputTokens) + => inputTokens / 1_000_000m * InputPerMillion + + outputTokens / 1_000_000m * OutputPerMillion; +} + +/// +/// Built-in default per-million-token prices keyed by model id (the bare model +/// identifier stamped on a response cell / accumulated on a per-model +/// satellite). These are a FALLBACK — an explicit price +/// on a node always wins. Seeded onto catalog model +/// nodes at import time so a user sees (and can override) a sensible number. +/// +/// Prices are the published standard (non-batch, non-cached) rates in USD. +/// Source: Anthropic API pricing, https://platform.claude.com/docs/en/about-claude/pricing +/// (as of 2026-06). Update this table when Anthropic changes rates or new models +/// ship; it is the ONE place to edit. +/// +/// This is an immutable, read-only constant lookup initialized once and +/// never written at runtime — a constant, not a cache (see NoStaticState.md). +/// +public static class ModelPricing +{ + private const string Usd = "USD"; + + /// + /// Model id → standard per-million-token rate. Anthropic Claude models only + /// today; extend per provider as prices are confirmed. + /// + public static readonly ImmutableDictionary Defaults = + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + // Anthropic Claude (direct API) — standard input / output $ per 1M tokens. + ["claude-opus-4-8"] = new(5m, 25m, Usd), + ["claude-opus-4-7"] = new(5m, 25m, Usd), + ["claude-opus-4-6"] = new(5m, 25m, Usd), + ["claude-opus-4-5"] = new(5m, 25m, Usd), + ["claude-sonnet-4-6"] = new(3m, 15m, Usd), + ["claude-sonnet-4-5"] = new(3m, 15m, Usd), + ["claude-haiku-4-5"] = new(1m, 5m, Usd), + ["claude-fable-5"] = new(10m, 50m, Usd), + + // The models actually deployed on Azure AI Foundry (s-meshweaver, swedencentral) — + // the cost overview must bill at AZURE serverless rates, not the providers' direct + // API rates. USD per 1M tokens (standard / non-cached). ⚠️ VERIFY against the Azure + // AI Foundry rate card for the resource — region/contract rates can differ, and the + // Flash / V3-0324 figures below are estimates pending confirmation. + ["DeepSeek-V4-Pro"] = new(1.75m, 3.48m, Usd), + ["DeepSeek-V3-0324"] = new(0.95m, 2.40m, Usd), // deepseek-chat; deprecates 2026-07-24 — estimate + ["DeepSeek-V4-Flash"] = new(0.55m, 1.10m, Usd), // cheapest tier — estimate + // Moonshot Kimi K2.6 (preview on Azure AI Foundry). + ["Kimi-K2.6"] = new(0.95m, 4.00m, Usd), + }.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase); + + /// + /// The built-in default rate for a model id, or null when the model isn't in + /// the table. Case-insensitive; tolerates a leading provider/path prefix by + /// also trying the last path segment. + /// + public static ModelPriceRate? Default(string? modelId) + { + if (string.IsNullOrWhiteSpace(modelId)) + return null; + if (Defaults.TryGetValue(modelId, out var rate)) + return rate; + var lastSegment = modelId[(modelId.LastIndexOf('/') + 1)..]; + return Defaults.TryGetValue(lastSegment, out rate) ? rate : null; + } + + /// + /// Resolves the effective rate for a model: an explicit price on the model + /// node wins (both per-million values must be present); otherwise the + /// built-in for the id. Null when neither is known — + /// callers then show tokens without a cost. + /// + public static ModelPriceRate? Resolve(string? modelId, ModelDefinition? node) + { + if (node is { InputPricePerMillionTokens: { } inPrice, OutputPricePerMillionTokens: { } outPrice }) + return new ModelPriceRate(inPrice, outPrice, node.Currency ?? Usd); + return Default(modelId); + } +} diff --git a/src/MeshWeaver.AI/ModelProviderConfiguration.cs b/src/MeshWeaver.AI/ModelProviderConfiguration.cs new file mode 100644 index 000000000..b199f3551 --- /dev/null +++ b/src/MeshWeaver.AI/ModelProviderConfiguration.cs @@ -0,0 +1,80 @@ +using System.Collections.Immutable; + +namespace MeshWeaver.AI; + +/// +/// Content shape for nodeType:ModelProvider mesh nodes — one node per +/// (user, provider) pair holding the credentials a chat-client factory uses +/// to authenticate against the provider's API. +/// +/// Two layers populate this NodeType: +/// +/// Static layer: +/// emits one read-only ModelProvider per +/// at +/// Model/{providerName}, stamped with the values from the +/// matching {section}:ApiKey / {section}:Endpoint +/// IConfiguration entries. This preserves backward compatibility for +/// deployments that wire credentials via appsettings. +/// User layer: ModelProviderService creates +/// user-authored ModelProvider nodes at +/// {userId}/Model/{providerName} when a user pastes their +/// personal key in the Models settings tab. The factory resolver +/// prefers these over the static layer via +/// scope:selfAndAncestors closest-wins. +/// +/// +/// The literal lives on this content. RLS gates +/// read access — only the owning user (or a partition admin) can read the +/// node, so the key never reaches other tenants. Keyless providers +/// (GitHub Copilot, local Claude Code CLI) leave null. +/// +public record ModelProviderConfiguration +{ + /// + /// Provider label — matches and + /// the stamp on each child + /// LanguageModel node. Looked up in ProviderRegistry.Find + /// to pull default endpoint / default model ids. + /// + public required string Provider { get; init; } + + /// + /// Literal API key (or subscription token) the factory passes as + /// x-api-key / Authorization: Bearer. Null for keyless + /// providers (Copilot uses OAuth, ClaudeCode runs a local binary). + /// + public string? ApiKey { get; init; } + + /// + /// Encrypted (enc:-tagged) raw MeshWeaver ApiToken (mw_…) auto-minted at Connect + /// time for the co-hosted CLI's MCP back-connection. Re-read + decrypted at spawn and injected + /// as a Authorization: Bearer header on the per-spawn HTTP MCP server, so the CLI calls + /// /mcp AS THE USER. Stored here (never on the Azure Files share) so it survives across + /// portal replicas via Postgres. Null until the user connects a co-hosted CLI provider. + /// + public string? McpApiKey { get; init; } + + /// + /// Optional endpoint override. Null means "use the + /// KnownProviderProfile.DefaultEndpoint from + /// ProviderRegistry". A non-null value flows through to + /// the factory unchanged. + /// + public string? Endpoint { get; init; } + + /// Human-readable display name (e.g. "Roland's personal key"). + public string? Label { get; init; } + + public DateTimeOffset CreatedAt { get; init; } + + public DateTimeOffset? LastUsedAt { get; init; } + + /// + /// Snapshot of the model ids the service auto-created under this provider + /// when it was first saved. Lets the UI show a count without re-querying + /// the LanguageModel children, and gives the service a single source of + /// truth for cascade-delete. + /// + public ImmutableArray Models { get; init; } = ImmutableArray.Empty; +} diff --git a/src/MeshWeaver.AI/ModelProviderNodeType.cs b/src/MeshWeaver.AI/ModelProviderNodeType.cs new file mode 100644 index 000000000..6434ec00c --- /dev/null +++ b/src/MeshWeaver.AI/ModelProviderNodeType.cs @@ -0,0 +1,219 @@ +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace MeshWeaver.AI; + +/// +/// Mesh-node type for AI model provider credentials — companion to +/// . One node per (user, provider) pair +/// at {userId}/Model/{providerName}; LanguageModel nodes live as +/// children under the provider's namespace. +/// +/// Two surfaces feed this: +/// +/// Static layer: +/// emits one read-only ModelProvider per +/// at +/// Model/{providerName}, stamped from the legacy +/// IConfiguration {section}:ApiKey / {section}:Endpoint +/// entries. Existing deployments that wire credentials via +/// appsettings keep working unchanged. +/// User layer: ModelProviderService creates +/// user-authored ModelProvider nodes in the user's partition +/// when they paste a key in the Models settings tab. +/// +/// +/// Owner-only by default — +/// maps ModelProvider to , +/// the same permission that gates API tokens. The root Model/ +/// namespace ships with a read-only _Policy via the static provider, +/// so user extensions must live in user namespaces. +/// +public static class ModelProviderNodeType +{ + /// NodeType discriminator value. + public const string NodeType = "ModelProvider"; + + /// + /// Conventional satellite-namespace segment for provider credential + /// nodes — mirrors _Access, _Thread, _Comment. + /// Organisation-shared providers live at {orgPath}/_Provider/{providerName} + /// and system defaults at the root _Provider/{providerName}. A user's + /// OWN providers live in their dotfile namespace instead — + /// {userPath}/_Memex/{providerName} (see ). + /// The picker / resolver query each owning path's subtree directly — no + /// path-walk heuristics, no central registry. + /// + public const string RootNamespace = "_Provider"; + + /// + /// Per-user satellite namespace for the user's OWN providers, models, and + /// selection — the hidden "dotfile" namespace + /// (, _Memex) + /// for per-user Memex defaults. User-owned provider / + /// model nodes live at {userPath}/_Memex/{providerName} and + /// {userPath}/_Memex/{providerName}/{modelId}; the user's selection at + /// {userPath}/_Memex/_Selection. + /// + /// Distinct from (_Provider), which + /// holds the SYSTEM catalog at root and org/context-SHARED providers at + /// {orgPath}/_Provider/…. A user's personal credentials are theirs, + /// so they belong in their dotfile namespace, not a shared satellite. The + /// picker / resolver union BOTH namespaces (see the model queries) so a user + /// sees the system catalog, any shared providers, and their own. + /// + public const string UserNamespace = ThreadComposerNodeType.MemexDefaultsNamespace; // "_Memex" + + /// + /// Node id for the per-user provider-selection node — the single node, at + /// {userPath}/_Memex/_Selection, whose content is a + /// . See . + /// + public const string SelectionNodeId = "_Selection"; + + /// + /// NodeType discriminator for the selection node — distinct from + /// so a nodeType:ModelProvider listing (e.g. + /// ModelProviderService.GetProvidersForOwner) never mistakes the + /// owner's selection node for an actual provider. + /// + public const string SelectionNodeType = "ModelProviderSelection"; + + /// + /// Path of the provider-selection node for an owner (user) path: + /// {ownerPath}/_Memex/_Selection. + /// + public static string SelectionPath(string ownerPath) => + $"{ownerPath}/{UserNamespace}/{SelectionNodeId}"; + + /// + /// The owner's personal provider/model namespace path: + /// {ownerPath}/_Memex. Picker + resolver query this (scope:descendants) + /// to surface a user's OWN providers and models. + /// + public static string UserNamespacePath(string ownerPath) => + $"{ownerPath}/{UserNamespace}"; + + /// + /// Registers the ModelProvider MeshNode definition + content type. + /// Wires the same hub-level content registration the + /// uses so reads through + /// deserialise the + /// content into the typed record. + /// + public static TBuilder AddModelProviderType(this TBuilder builder, + IReadOnlySet? serveFromPartition = null) + where TBuilder : MeshBuilder + { + builder.AddMeshNodes(CreateMeshNode()); + builder.AddMeshNodes(CreateSelectionMeshNode()); + builder.AddAutocompleteExcludedTypes(NodeType); + builder.AddAutocompleteExcludedTypes(SelectionNodeType); + builder.ConfigureHub(config => config + .WithType(nameof(ModelProviderConfiguration)) + .WithType(nameof(ModelProviderSelection))); + // Mirror LanguageModelNodeType: the root _Provider namespace + // gets a partition-storage provider so the routing core knows where + // to find static ModelProvider nodes (the ones BuiltInLanguageModelProvider + // emits from IConfiguration). Without this, namespace:_Provider + // queries return nothing because no provider claims the partition. + // User-partition ModelProvider nodes (rbuergi/_Provider/Anthropic + // etc.) route through their owning partition's storage adapter — + // no extra wiring needed for those. + // The model catalog's provider/model CONTENT lives under the "_Provider" partition; + // it is DB-synced together with "Model". When synced, skip the read-only in-memory + // provider so Postgres serves "_Provider" + accepts the import's writes. See AddAgentType. + var dbSynced = serveFromPartition is not null + && (serveFromPartition.Contains("Model") || serveFromPartition.Contains(RootNamespace)); + builder.ConfigureServices(services => + { + services.TryAddSingleton(); + // Always generate a default (empty) {user}/_Memex/_Selection at User + // onboarding so the chat picker's selection read RESOLVES instead of + // generating a routing NotFound the GUI re-issues on a loop — the + // resubscribe-storm that starved the circuit until unrelated + // SubscribeRequests never completed (sglauser deadlock, 2026-06-09; same + // class as the 2026-06-08 storm). Empty selection == default catalog + // (root + context + nodeType), the existing behaviour. Mirrors + // ThreadComposerSeedHandler / the _Thread/ThreadComposer seed. + services.AddSingleton(_ => new ModelProviderSelectionSeedHandler()); + if (!dbSynced) + services.AddSingleton(sp => + new StaticNodePartitionStorageProvider( + RootNamespace, + sp.GetRequiredService(), + description: "Built-in model provider catalog (read-only).")); + return services; + }); + return builder; + } + + /// + /// MeshNode definition for nodeType:ModelProvider. + /// + public static MeshNode CreateMeshNode() => new(NodeType) + { + Name = "Model Provider", + Icon = "/static/NodeTypeIcons/key.svg", + // Treated as a regular content type. Permission gating happens in + // CreateNodePermissionAttribute.GetPermissionForNodeType → Permission.Api. + IsSatelliteType = false, + // Creatable: an admin can author a ModelProvider node directly in a space + // (e.g. Systemorph/_Provider/AzureFoundry). Still hidden from search. + ExcludeFromContext = new HashSet { "search" }, + HubConfiguration = config => config + .AddMeshDataSource(source => source + .WithContentType()) + }; + + /// + /// MeshNode definition for the per-user provider-selection node + /// (). Distinct type so it's creatable via + /// CreateNode + deserialises its + /// content, yet never shows up in nodeType:ModelProvider listings. + /// + public static MeshNode CreateSelectionMeshNode() => new(SelectionNodeType) + { + Name = "Model Provider Selection", + IsSatelliteType = false, + ExcludeFromContext = new HashSet { "search", "create" }, + HubConfiguration = config => config + .AddMeshDataSource(source => source + .WithContentType()) + }; + + /// + /// Seeds the per-user default {user}/_Memex/_Selection node (an empty + /// ) when a User partition is created. + /// Returned from so it persists directly alongside + /// the user (no hub round-trip) — the "always generate the default state" step that + /// keeps the chat picker's selection read from ever hitting a routing NotFound, + /// whose GUI-driven re-issue loop starved the circuit and hung unrelated + /// SubscribeRequests (the sglauser deadlock). Mirrors ThreadComposerSeedHandler. + /// + private sealed class ModelProviderSelectionSeedHandler : INodePostCreationHandler + { + public string NodeType => UserNodeType.NodeType; // "User" + + public IObservable Handle(MeshNode createdNode, string? createdBy) + => System.Reactive.Linq.Observable.Empty(); + + public IEnumerable GetAdditionalNodes(MeshNode createdNode) + { + var userPath = !string.IsNullOrEmpty(createdNode.Path) ? createdNode.Path : createdNode.Id; + if (string.IsNullOrEmpty(userPath)) + yield break; + + yield return new MeshNode(SelectionNodeId, $"{userPath}/{UserNamespace}") + { + NodeType = SelectionNodeType, + Name = "Model Provider Selection", + Content = new ModelProviderSelection(), + }; + } + } +} diff --git a/src/MeshWeaver.AI/ModelProviderSelection.cs b/src/MeshWeaver.AI/ModelProviderSelection.cs new file mode 100644 index 000000000..5cb4738e1 --- /dev/null +++ b/src/MeshWeaver.AI/ModelProviderSelection.cs @@ -0,0 +1,23 @@ +using System.Collections.Immutable; + +namespace MeshWeaver.AI; + +/// +/// A user's choice of which ModelProvider subtrees feed their chat model +/// picker + credential resolution. Persisted as the content of a single node +/// per user at {userId}/_Memex/_Selection +/// (see ). Empty/absent ⇒ +/// the default set (root catalog + context + nodeType), i.e. existing behaviour. +/// +/// Each entry is the full path of a ModelProvider node the user +/// wants active — a shared/org provider (e.g. acme/_Provider/Anthropic) +/// or one of their own (rbuergi/_Memex/OpenAI). For a shared/org provider the user may +/// hold only Read on the subtree — +/// reads the (decrypted) key under a system identity, gated by that Read, so +/// the user can use the provider without seeing the raw key. +/// +public record ModelProviderSelection +{ + /// Full paths of the ModelProvider nodes the user selected. + public ImmutableArray SelectedProviderPaths { get; init; } = ImmutableArray.Empty; +} diff --git a/src/MeshWeaver.AI/ModelStaticRepoSource.cs b/src/MeshWeaver.AI/ModelStaticRepoSource.cs new file mode 100644 index 000000000..a8e8dda69 --- /dev/null +++ b/src/MeshWeaver.AI/ModelStaticRepoSource.cs @@ -0,0 +1,48 @@ +using MeshWeaver.Markdown; +using MeshWeaver.Mesh; + +namespace MeshWeaver.AI; + +/// +/// The built-in model catalog as a static-repo import source. +/// emits the read-only _Policy under the Model partition and the +/// ModelProvider/LanguageModel content under the _Provider partition (all +/// derived from IConfiguration). This source materializes ALL of them into their partitions +/// so the catalog is served from the DB on the distributed/PG path. The nodes span two partitions; +/// the importer reads & prunes every partition the source's nodes touch. See +/// Doc/Architecture/StaticRepoImport.md. +/// +public sealed class ModelStaticRepoSource(BuiltInLanguageModelProvider provider) : IStaticRepoSource +{ + /// + // Logical source id / primary partition — the activity lock lives at "Model/_Activity/import-*". + // (The source's nodes also include "_Provider" content; the importer is multi-partition aware.) + public string Partition => LanguageModelNodeType.RootNamespace; // "Model" + + /// + // The catalog is config-derived → fingerprint on content, so a changed catalog re-imports. + public bool Versioned => false; + + /// + // All catalog nodes go to the partition — provider/model content AND the read-only _Policy + // (the _Policy is also prune-protected by the governance rule). + public IReadOnlyList EnumerateSourceNodes() => + provider.GetStaticNodes().ToArray(); + + /// + public MeshNode? PartitionRoot => new(LanguageModelNodeType.RootNamespace) + { + Name = "Models", + NodeType = "Space", + State = MeshNodeState.Active, + Content = new MarkdownContent + { + Content = """ + # Models + + The language models available to this deployment. Browse the catalog to see each + model's provider and capabilities. + """ + } + }; +} diff --git a/src/MeshWeaver.AI/ModelTierConfiguration.cs b/src/MeshWeaver.AI/ModelTierConfiguration.cs index 925480441..7a0288955 100644 --- a/src/MeshWeaver.AI/ModelTierConfiguration.cs +++ b/src/MeshWeaver.AI/ModelTierConfiguration.cs @@ -29,6 +29,15 @@ public class ModelTierConfiguration /// public string? Light { get; set; } + /// + /// Model name for the "utility" tier — a cheap, fast model (e.g. Kimi / Moonshot) for + /// background micro-jobs: node icon + user-avatar generation, description writing, and + /// thread auto-naming. Latency and cost matter more than peak reasoning here. Configure + /// per deployment via ModelTier:Utility (like the embedding model); when unset, + /// agents tagged modelTier: utility fall back to the deployment's default model. + /// + public string? Utility { get; set; } + /// /// Resolves a tier name to a concrete model name. /// Returns null if the tier is not configured or not recognized. @@ -43,6 +52,10 @@ public class ModelTierConfiguration "heavy" => Heavy, "standard" => Standard, "light" => Light, + // Utility falls back to the light model when not separately configured, so routing + // the icon/description agents to "utility" never breaks a deployment that hasn't set + // ModelTier:Utility — it just behaves as before until an operator points it at Kimi. + "utility" => string.IsNullOrWhiteSpace(Utility) ? (Light ?? Standard) : Utility, _ => null }; } diff --git a/src/MeshWeaver.AI/Plugins/ChunkNavigation.cs b/src/MeshWeaver.AI/Plugins/ChunkNavigation.cs new file mode 100644 index 000000000..94e8e082f --- /dev/null +++ b/src/MeshWeaver.AI/Plugins/ChunkNavigation.cs @@ -0,0 +1,198 @@ +using System.Reactive.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using MeshWeaver.ContentCollections.Indexing; +using Microsoft.Extensions.DependencyInjection; + +namespace MeshWeaver.AI.Plugins; + +/// +/// Shared backing logic for the chunk-navigation tools — search_chunks and get_chunk — +/// exposed identically on the agent surface () and the MCP surface +/// (McpMeshPlugin). Both call these methods so the two transports stay in sync by construction. +/// +/// This is the chunk-LEVEL retrieval counterpart to node-level search: node search resolves +/// a chunk hit up to its Document node and drops the chunk index (good for "which file"), while +/// keeps the (collectionPath, filePath, chunkIndex) coordinate so the +/// caller can read the exact window and can step prev/next through the file. See +/// the ContentChunkNavigation architecture doc. +/// +/// Reactive throughout: every method returns a cold that performs the +/// embed + store read on subscribe (the tool boundary bridges with .FirstAsync().ToTask()). The +/// store () and embedder () are +/// resolved from the supplied ; when content indexing is not wired into the +/// host they are absent and the methods emit a clear "not available" envelope instead of throwing. +/// +public static class ChunkNavigation +{ + /// + /// Embeds and runs a cosine chunk search across the in-scope collection(s), + /// returning a JSON envelope {count, results:[{ documentPath, collectionPath, filePath, chunkIndex, + /// score, snippet }]}. Hits are NOT deduped by file — chunk-level granularity is the point — and + /// are capped at . + /// + /// Service provider to resolve the store + embedder from. + /// Free-text query embedded into the store's vector space. + /// + /// The path the search is anchored at. The collection candidates are this path plus each ancestor + /// prefix (the same ancestor-walk the autocomplete provider uses). When null/empty an empty result + /// with a hint to pass a scope is returned — a bare, context-free query has no collection to search. + /// + /// Max chunk hits returned (1..200). + public static IObservable SearchChunks( + IServiceProvider services, string query, string? scopePath, int limit = 20) + { + var store = services.GetService(); + var embedder = services.GetService(); + if (store is null || embedder is null) + return Observable.Return(NotAvailableEnvelope()); + + if (string.IsNullOrWhiteSpace(query)) + return Observable.Return(Hint("Pass a non-empty 'query' to search content chunks.")); + + var collections = CollectionScope(scopePath); + if (collections.Count == 0) + return Observable.Return(Hint( + "No scope to search. Pass 'scope' as the node path whose content (and ancestors') chunks to search " + + "— e.g. 'ACME/Reports' — or run this from an agent with a context path.")); + + limit = Math.Clamp(limit, 1, 200); + + // Embed once, fan the SAME vector across every in-scope collection's cosine search, project each + // chunk hit (best-first per collection), and cap at the limit. No dedup — chunk-level hits are + // the point. The embed + searches are composed leaves (SelectMany), never awaited. + return embedder.Embed(query) + .SelectMany(vector => collections.ToObservable() + .SelectMany(collection => store.Search(collection, vector, limit) + .SelectMany(hits => hits.Select((hit, rank) => (hit, rank)).ToObservable())) + .ToList()) + .Select(hits => + { + var results = new JsonArray(); + // The store returns hits most-similar-first but does NOT surface the raw cosine score on + // ContentChunk, so we report 'rank' (0-based best-first position) as the relevance signal + // rather than fabricate a score. Re-rank globally because hits arrived per-collection. + var rank = 0; + foreach (var (hit, _) in hits.Take(limit)) + { + results.Add(new JsonObject + { + ["documentPath"] = DocumentPaths.For(hit.CollectionPath, hit.FilePath), + ["collectionPath"] = hit.CollectionPath, + ["filePath"] = hit.FilePath, + ["chunkIndex"] = hit.ChunkIndex, + ["rank"] = rank++, + ["snippet"] = Snippet(hit.Text), + }); + } + + return new JsonObject + { + ["count"] = results.Count, + ["results"] = results, + }.ToJsonString(); + }); + } + + /// + /// Reads the single chunk at of within + /// , returning { collectionPath, filePath, chunkIndex, text, + /// prevIndex, nextIndex, totalChunks }. prevIndex is null at index 0; nextIndex is + /// null at the last chunk. A null chunk (out of range, or the file is not indexed) returns a clear + /// "not found" envelope carrying totalChunks so the caller can see the valid range. + /// + public static IObservable GetChunk( + IServiceProvider services, string collectionPath, string filePath, int chunkIndex) + { + var store = services.GetService(); + if (store is null) + return Observable.Return(NotAvailableEnvelope()); + + if (string.IsNullOrWhiteSpace(collectionPath)) + return Observable.Return(Hint("'collectionPath' is required.")); + if (string.IsNullOrWhiteSpace(filePath)) + return Observable.Return(Hint("'filePath' is required.")); + + return store.GetChunk(collectionPath, filePath, chunkIndex) + .SelectMany(chunk => store.GetChunkCount(collectionPath, filePath) + .Select(total => + { + if (chunk is null) + return new JsonObject + { + ["found"] = false, + ["collectionPath"] = collectionPath, + ["filePath"] = filePath, + ["chunkIndex"] = chunkIndex, + ["totalChunks"] = total, + ["message"] = total == 0 + ? $"No chunks indexed for '{filePath}' in '{collectionPath}'." + : $"No chunk at index {chunkIndex}; valid range is 0..{total - 1}.", + }.ToJsonString(); + + return new JsonObject + { + ["found"] = true, + ["collectionPath"] = chunk.CollectionPath, + ["filePath"] = chunk.FilePath, + ["chunkIndex"] = chunk.ChunkIndex, + ["text"] = chunk.Text, + ["prevIndex"] = chunk.ChunkIndex > 0 ? chunk.ChunkIndex - 1 : null, + ["nextIndex"] = chunk.ChunkIndex < total - 1 ? chunk.ChunkIndex + 1 : null, + ["totalChunks"] = total, + }.ToJsonString(); + })); + } + + /// + /// The collection candidates for a scope path: the path itself plus every ancestor prefix, so a scope + /// of part/Space/Sub searches collections keyed at part/Space/Sub, part/Space, and + /// part. Replicates DocumentIndexingExtensions.CollectionScopeFromContext locally so this + /// helper need not depend on MeshWeaver.ContentCollections.Indexing.Graph. Empty when there is + /// no scope. + /// + private static IReadOnlyCollection CollectionScope(string? scopePath) + { + if (string.IsNullOrWhiteSpace(scopePath)) + return []; + + var scope = new List(); + var path = scopePath.Trim().Trim('/'); + // Strip a leading '@' (agent/MCP paths may arrive @-prefixed) before walking ancestors. + path = path.TrimStart('@').Trim('/'); + while (path.Length > 0) + { + scope.Add(path); + var lastSlash = path.LastIndexOf('/'); + if (lastSlash <= 0) + break; + path = path[..lastSlash]; + } + return scope; + } + + /// A single-line, ~160-char snippet of a chunk's text for the search result. + private static string Snippet(string text) + { + const int max = 160; + var oneLine = text.Replace('\r', ' ').Replace('\n', ' ').Trim(); + oneLine = string.Join(' ', oneLine.Split(' ', StringSplitOptions.RemoveEmptyEntries)); + return oneLine.Length <= max ? oneLine : oneLine[..max].TrimEnd() + "…"; + } + + private static string NotAvailableEnvelope() => + new JsonObject + { + ["count"] = 0, + ["results"] = new JsonArray(), + ["message"] = "Content chunk indexing is not enabled in this host — no chunk store is configured.", + }.ToJsonString(); + + private static string Hint(string message) => + new JsonObject + { + ["count"] = 0, + ["results"] = new JsonArray(), + ["message"] = message, + }.ToJsonString(); +} diff --git a/src/MeshWeaver.AI/Plugins/CollaborationPlugin.cs b/src/MeshWeaver.AI/Plugins/CollaborationPlugin.cs index 88c120622..011d7708d 100644 --- a/src/MeshWeaver.AI/Plugins/CollaborationPlugin.cs +++ b/src/MeshWeaver.AI/Plugins/CollaborationPlugin.cs @@ -56,7 +56,7 @@ public Task AddComment( // Read the document off the hub scheduler, then fan into Post + RegisterCallback. // No `await` anywhere — the subscription runs the read on TaskPoolScheduler and // the write's callback fires on the response thread. Both resolve the TCS. - Observable.FromAsync(() => ops.Get(resolvedInput)) + ops.Get(resolvedInput) .SubscribeOn(TaskPoolScheduler.Default) .Subscribe( docJson => AddCommentContinuation( @@ -132,7 +132,7 @@ public Task SuggestEdit( var resolvedPath = MeshOperations.ResolvePath(resolvedInput); var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - Observable.FromAsync(() => ops.Get(resolvedInput)) + ops.Get(resolvedInput) .SubscribeOn(TaskPoolScheduler.Default) .Subscribe( docJson => SuggestEditContinuation( @@ -218,30 +218,32 @@ private void PostAndReport( Func formatSuccess) { var delivery = hub.Post(request, o => o.WithTarget(target))!; - hub.RegisterCallback(delivery, callback => - { - switch (callback) - { - case IMessageDelivery typed: - try { tcs.TrySetResult(formatSuccess(typed.Message)); } - catch (Exception ex) { tcs.TrySetResult($"Error formatting response: {ex.Message}"); } - break; - case IMessageDelivery failure: + hub.Observe(delivery) + .Subscribe( + callback => + { + if (callback.Message is TResponse typed) + { + try { tcs.TrySetResult(formatSuccess(typed)); } + catch (Exception ex) { tcs.TrySetResult($"Error formatting response: {ex.Message}"); } + } + else + { + tcs.TrySetResult($"Error: unexpected response {callback.Message?.GetType().Name ?? "null"} for {originalInput}."); + } + }, + ex => + { logger.LogWarning( - "Delivery to {Target} failed for {RequestType}: {Reason}. Original input: {OriginalInput}", - target, request.GetType().Name, failure.Message.Message, originalInput); + ex, + "Delivery to {Target} failed for {RequestType}. Original input: {OriginalInput}", + target, request.GetType().Name, originalInput); tcs.TrySetResult( - $"Error: {failure.Message.Message ?? "delivery failed"}. " + + $"Error: {ex.Message ?? "delivery failed"}. " + $"Check that '{originalInput}' resolves to an existing node — pass the MeshNode's " + "`path` property, not its `name`. If you only know the display name, call " + "Search('name:\"...\"') and use the `path` field of the match."); - break; - default: - tcs.TrySetResult($"Error: unexpected response {callback.Message?.GetType().Name ?? "null"} for {originalInput}."); - break; - } - return callback; - }); + }); } private static string? ExtractContent(string rawJson) diff --git a/src/MeshWeaver.AI/Plugins/ContentCollectionPlugin.cs b/src/MeshWeaver.AI/Plugins/ContentCollectionPlugin.cs index f30cc185d..8c20518c9 100644 --- a/src/MeshWeaver.AI/Plugins/ContentCollectionPlugin.cs +++ b/src/MeshWeaver.AI/Plugins/ContentCollectionPlugin.cs @@ -1,4 +1,6 @@ using System.ComponentModel; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; using System.Text; using MeshWeaver.Data; using MeshWeaver.Messaging; @@ -19,9 +21,47 @@ public class ContentCollectionPlugin(IMessageHub hub, IAgentChat chat) : IAgentP public IEnumerable CreateTools() { - return [AIFunctionFactory.Create(UploadContent)]; + return + [ + AIFunctionFactory.Create(UploadContent), + AIFunctionFactory.Create(SearchChunks), + AIFunctionFactory.Create(GetChunk), + ]; } + [Description( + "Semantic search over INDEXED content chunks — the chunk-level companion to the node `Search`. " + + "Where node search resolves a hit up to its Document node and drops the chunk position, this " + + "returns the matching chunks themselves WITH their (collectionPath, filePath, chunkIndex) so you " + + "can read the exact window or step through neighbours with get_chunk. Use this to FIND relevant " + + "passages and gather context; use Get on the Document for whole-document reads (e.g. table " + + "extraction). Returns {count, results:[{documentPath, collectionPath, filePath, chunkIndex, rank, snippet}]}.")] + public Task SearchChunks( + [Description("Free-text query. Matched semantically against indexed chunk text (1000-char windows, 150-char overlap).")] string query, + [Description("Node path to anchor the search at — this path AND each ancestor prefix are searched (e.g. 'ACME/Reports'). Optional: defaults to the agent's current context. If neither is set an empty result with a hint is returned.")] string? scope = null, + [Description("Maximum number of chunk hits to return (1-200, default 20). Not deduped by file — chunk-level hits are the point.")] int limit = 20) + { + var scopePath = !string.IsNullOrWhiteSpace(scope) + ? MeshOperations.ResolvePath(MeshOperations.ResolveContextPath(chat, scope)) + : chat.Context?.Context; + + return ChunkNavigation.SearchChunks(hub.ServiceProvider, query, scopePath, limit) + .FirstAsync().ToTask(); + } + + [Description( + "Reads ONE indexed content chunk by its 0-based index within a file, with prev/next links so you " + + "can step through the file's chunk sequence. Use after search_chunks (which gives you the " + + "collectionPath/filePath/chunkIndex of a hit) to read the full window and walk to adjacent chunks. " + + "Returns {found, collectionPath, filePath, chunkIndex, text, prevIndex, nextIndex, totalChunks} — " + + "prevIndex is null at index 0, nextIndex is null at the last chunk.")] + public Task GetChunk( + [Description("The content collection path the chunk belongs to (the 'collectionPath' from a search_chunks hit).")] string collectionPath, + [Description("The file path within the collection (the 'filePath' from a search_chunks hit).")] string filePath, + [Description("0-based chunk index within the file (the 'chunkIndex' from a search_chunks hit, or a prevIndex/nextIndex to step).")] int chunkIndex) + => ChunkNavigation.GetChunk(hub.ServiceProvider, collectionPath, filePath, chunkIndex) + .FirstAsync().ToTask(); + [Description("Uploads text content (SVG, markdown, JSON, CSS, etc.) to a node's content collection. Use for storing diagrams, images (SVG), stylesheets, or any text-based files alongside a node.")] public Task UploadContent( [Description("Canonical path to the node that owns the collection — use the MeshNode's `path` property, NOT its `name`. Use @/full/path for absolute or @relative/path relative to the current context. Example: @/PartnerRe/AIConsulting or @FinalReport. If you only know the display name, call Search('name:\"...\"') first and use the path field of the match.")] string nodePath, @@ -57,27 +97,25 @@ public Task UploadContent( }, o => o.WithTarget(address))!; - hub.RegisterCallback(delivery, callback => - { - switch (callback) - { - case IMessageDelivery typed: - tcs.TrySetResult(typed.Message.Success - ? $"Uploaded `{filePath}` to @{resolvedPath}/{collectionName}/{filePath}" - : $"Error: {typed.Message.Error}"); - break; - case IMessageDelivery failure: - tcs.TrySetResult( - $"Error uploading to {resolvedPath}: {failure.Message.Message ?? "delivery failed"}. " + - $"Check that '{nodePath}' resolves to an existing node — pass the MeshNode's " + - "`path` property, not its `name`."); - break; - default: - tcs.TrySetResult($"Error: unexpected response {callback.Message?.GetType().Name ?? "null"} uploading to {resolvedPath}."); - break; - } - return callback; - }); + hub.Observe(delivery) + .Subscribe( + callback => + { + if (callback.Message is SaveContentResponse typed) + { + tcs.TrySetResult(typed.Success + ? $"Uploaded `{filePath}` to @{resolvedPath}/{collectionName}/{filePath}" + : $"Error: {typed.Error}"); + } + else + { + tcs.TrySetResult($"Error: unexpected response {callback.Message?.GetType().Name ?? "null"} uploading to {resolvedPath}."); + } + }, + ex => tcs.TrySetResult( + $"Error uploading to {resolvedPath}: {ex.Message ?? "delivery failed"}. " + + $"Check that '{nodePath}' resolves to an existing node — pass the MeshNode's " + + "`path` property, not its `name`.")); return tcs.Task; } diff --git a/src/MeshWeaver.AI/Plugins/DelegationTool.cs b/src/MeshWeaver.AI/Plugins/DelegationTool.cs index 61ea93e73..b48d79c4d 100644 --- a/src/MeshWeaver.AI/Plugins/DelegationTool.cs +++ b/src/MeshWeaver.AI/Plugins/DelegationTool.cs @@ -1,16 +1,20 @@ using System.Collections.Immutable; using System.ComponentModel; -using System.Runtime.CompilerServices; +using System.Reactive.Concurrency; +using System.Reactive.Linq; +using System.Text; using System.Text.Json; +using MeshWeaver.Data; +using MeshWeaver.Mesh; +using MeshWeaver.Messaging; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; +using MeshThread = MeshWeaver.AI.Thread; namespace MeshWeaver.AI.Plugins; /// /// Result record preserved for tests + . -/// No longer used as the delegation tool return shape — the tool now yields -/// chunks directly. /// public record DelegationResult { @@ -28,21 +32,86 @@ public record DelegationResult /// When to delegate to this agent public record DelegationInfo(string AgentPath, string Description); +/// +/// Snapshot of a currently-running (or recently-completed) sub-thread. Returned +/// by 's list_sub_threads tool so the parent +/// agent can see what delegations it has in flight and decide whether to send a +/// follow-up message via send_to_sub_thread or to wait. +/// +/// Full mesh path of the sub-thread node. +/// The agent assigned to handle the delegation. +/// Idle / Executing / Cancelled — same enum as Thread.Status. +/// First ~200 chars of the sub-thread's current response cell text (may be empty before the agent emits any text). +/// Heartbeat / last token timestamp; lets the agent gauge progress. +public record SubThreadInfo( + string ThreadPath, + string AgentName, + string Status, + string? PreviewText, + DateTimeOffset? LastActivity); + /// /// Creates delegation tools for agents that support isolated context per delegation. /// -/// The tool signature is so that the sub-thread's -/// streaming text flows back as incremental chunks. Microsoft.Extensions.AI aggregates -/// the yielded chunks as the tool result; meanwhile, a side-channel delta push keeps the -/// parent's response bubble updated in real time so the user sees sub-thread progress -/// inline without waiting for completion. +/// Three tools per agent are emitted by : +/// +/// delegate_to_agent — spawns a sub-thread, returns the sub-agent's +/// dedicated summary on completion. Tool-call result. +/// list_sub_threads — read-only snapshot of active sub-threads +/// the parent has spawned (path, agent, status, preview, last activity). +/// send_to_sub_thread — fire-and-forget mid-stream follow-up +/// message to a still-running sub-thread (writes to its +/// PendingUserMessages via stream.Update). +/// /// -/// No more — the previous Task-returning shape forced the -/// FunctionInvokingChatClient to block on sub-thread completion, which deadlocks under -/// Orleans when the child's completion patch queues behind the parent hub scheduler. +/// Completion semantics. delegate_to_agent wraps the sub-thread's +/// streaming pipeline in Observable.Create + Subscribe: the +/// only await foreach in the entire delegation path runs on the subscriber's +/// continuation (the parent agent loop's task scheduler), no Task.Run. Sub-thread +/// completion (Status → Idle, response cell Status = Completed) terminates the inner +/// async-enumerable; OnCompleted resolves the TCS with the aggregated summary +/// text the sub-agent wrote to its response cell before exiting. +/// +/// Sub-thread progress is additionally visible inline via the side-channel +/// ToolCallEntry.DelegationPath → the sub-thread's Streaming layout area. /// public static class DelegationTool { + /// + /// Creates the suite of delegation tools: delegate_to_agent, plus + /// list_sub_threads and send_to_sub_thread when the parent + /// passes the needed accessor delegates. + /// + /// When + + /// are supplied, delegate_to_agent resolves its Task<string> + /// reactively: the next Dispatched event after invocation gives us the + /// sub-thread path, then a subscription to workspace.GetRemoteStream<MeshNode>(subThreadPath) + /// waits for Thread.Status == Idle (terminal), reads the sub-agent's + /// final assistant message text, and resolves the TCS with that summary — + /// no Task.Run, no chunk-aggregation race. + /// + public static IEnumerable CreateDelegationTools( + AgentConfiguration currentAgent, + IReadOnlyList hierarchyAgents, + Func> executeAsync, + Func>? listSubThreads = null, + Action? sendToSubThread = null, + IObservable? delegationEvents = null, + IWorkspace? workspace = null, + ILogger? logger = null) + { + yield return CreateUnifiedDelegationTool( + currentAgent, hierarchyAgents, executeAsync, + listSubThreads != null, sendToSubThread != null, + delegationEvents, workspace, logger); + + if (listSubThreads is not null) + yield return CreateListSubThreadsTool(listSubThreads, logger); + + if (sendToSubThread is not null) + yield return CreateSendToSubThreadTool(sendToSubThread, logger); + } + /// /// Creates a unified delegation tool that can delegate to any available agent. /// Each delegation uses an isolated thread for the target agent. @@ -50,8 +119,21 @@ public static class DelegationTool public static AITool CreateUnifiedDelegationTool( AgentConfiguration currentAgent, IReadOnlyList hierarchyAgents, - Func> executeAsync, + Func> executeAsync, ILogger? logger = null) + => CreateUnifiedDelegationTool(currentAgent, hierarchyAgents, executeAsync, + hasListTool: false, hasSendTool: false, + delegationEvents: null, workspace: null, logger); + + private static AITool CreateUnifiedDelegationTool( + AgentConfiguration currentAgent, + IReadOnlyList hierarchyAgents, + Func> executeAsync, + bool hasListTool, + bool hasSendTool, + IObservable? delegationEvents, + IWorkspace? workspace, + ILogger? logger) { var delegationInfo = ImmutableList.Empty; @@ -83,33 +165,206 @@ public static AITool CreateUnifiedDelegationTool( PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); - async IAsyncEnumerable Delegate( + Task Delegate( [Description("The name of the agent to delegate to. Use the agentPath from the available agents.")] string agentName, [Description("The task or instructions for the delegated agent. Be specific about what you need.")] string task, [Description("Optional: the node path to use as context for this delegation (e.g., 'OrgA/my-doc'). When omitted, inherits the parent context. Set explicitly when delegating parallel work on different documents.")] string? context = null, - [EnumeratorCancellation] CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) { logger?.LogInformation("Delegating to {AgentName}: {Task}, context={Context}", agentName, task, context ?? "(inherited)"); - await foreach (var chunk in executeAsync(agentName, task, context, cancellationToken) - .WithCancellation(cancellationToken)) + // Reactive completion: pure observable composition. The only `await + // foreach` in the entire delegation path runs inside Observable.Create + // on the subscriber's continuation (TaskScheduler.Current at Subscribe + // time = the parent agent loop's task scheduler — Orleans grain in + // prod, default in monolith tests). No Task.Run, no callback-bag. + // + // PRIMARY completion signal (when delegationEvents + workspace are + // wired by ChatClientAgentFactory): subscribe to delegationEvents for + // the next Dispatched (captures sub-thread path), then subscribe to + // workspace.GetRemoteStream(subThreadPath), wait for + // Thread.Status flipping back to Idle AFTER ExecutionStartedAt was set + // (= post-execution Idle), and read the last assistant ThreadMessage's + // Text as the sub-agent's dedicated summary. Resolve TCS with that. + // + // FALLBACK (legacy callers without delegationEvents/workspace, or + // when the reactive path doesn't fire in time): the IAsyncEnumerable's + // OnCompleted resolves the TCS with the aggregated chunk text. The + // IAsyncEnumerable also drives the sub-thread setup as a side effect, + // so it MUST be subscribed even when the reactive completion path + // would handle the result — otherwise the sub-thread never starts. + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var sb = new StringBuilder(); + + if (delegationEvents is not null && workspace is not null) { - yield return chunk; + delegationEvents + .Where(e => e.Phase == MeshWeaver.AI.Delegation.DelegationLifecycle.Dispatched) + .Take(1) + .Subscribe(dispatched => + { + var subThreadPath = dispatched.SubThreadPath; + logger?.LogInformation( + "Delegation Dispatched: sub-thread={SubPath}, callId={CallId} — subscribing for Idle", + subThreadPath, dispatched.CallId); + + // Subscribe to the sub-thread node's stream and capture + // the Running → terminal transition. We use Scan to + // remember whether we've seen a non-terminal (Executing / + // StartingExecution) status; the first terminal emission + // after that (Idle / Cancelled / Done) is the genuine + // post-execution terminal, NOT the initial-Idle emission + // the synced query replays on subscribe. + workspace.GetMeshNodeStream(subThreadPath) + .Select(node => node?.Content as MeshThread) + .Where(t => t is not null) + .Scan( + (sawRunning: false, terminal: (MeshThread?)null), + (state, t) => + { + if (t!.Status is ThreadExecutionStatus.Executing + or ThreadExecutionStatus.StartingExecution) + return (true, null); + // Terminal when we either observed the Running phase OR + // the node already carries a completed-round Summary. A + // fast sub-agent can flip Running→Idle inside a single + // coalesced stream emission, so sawRunning is never set; + // a non-empty Summary (written atomically with Status→Idle + // by ExecuteMessageAsync) is the authoritative "this round + // finished" signal that distinguishes the post-execution + // Idle from the initial creation Idle (which has none). + var completed = state.sawRunning + || !string.IsNullOrEmpty(t.Summary); + if (completed && t.Status is ThreadExecutionStatus.Idle + or ThreadExecutionStatus.Cancelled + or ThreadExecutionStatus.Done) + return (true, t); + return state; + }) + .Where(s => s.terminal is not null) + .Take(1) + .Timeout(TimeSpan.FromMinutes(10)) + .Subscribe( + s => + { + // Thread.Summary IS the agent's tool-call result — + // written by ExecuteMessageAsync in the same + // stream.Update cycle as Status → Idle, so this + // emission carries the summary atomically. + var summary = s.terminal!.Summary ?? ""; + logger?.LogInformation( + "Sub-thread {SubPath} Running→Idle: summary len={Len}", + subThreadPath, summary.Length); + tcs.TrySetResult(summary); + }, + ex => + { + // Primary path failed/timed out — NOW fall back to the + // chunk aggregate (empty for the reactive path, but at + // least the parent tool call resolves instead of hanging). + logger?.LogWarning(ex, + "Sub-thread {SubPath} Running→Idle wait failed; falling back to chunk aggregate", + subThreadPath); + tcs.TrySetResult(sb.ToString()); + }); + }); } - logger?.LogInformation("Delegation to {AgentName} stream completed", agentName); + // Pure Subscribe — executeAsync now returns IObservable directly + // (ExecuteDelegationAsync builds the sub-thread node via meshService.CreateNode + // and Emits Dispatched; the parent's TCS is resolved by the Idle subscription + // above which reads Thread.Summary). No await foreach, no Task.Run, no + // Observable.Create wrapper. + // + // SubscribeOn(TaskPoolScheduler.Default): the parent agent loop's + // continuation (Orleans grain scheduler in prod, single-threaded pump in + // the DelegationDeadlockTest) MUST NOT host the sub-thread drain. The + // typical executeAsync impl is Observable.Create over an + // IAsyncEnumerable, which captures the Subscribe-time SynchronizationContext + // for every MoveNextAsync — if that's the grain scheduler, every sub-thread + // continuation posts back through it and wedges the grain. Hop the Subscribe + // to ThreadPool so the entire drain runs free of the caller's context. + executeAsync(agentName, task, context, cancellationToken) + .SubscribeOn(TaskPoolScheduler.Default) + .Subscribe( + chunk => sb.Append(chunk), + ex => + { + logger?.LogError(ex, "Delegation to {AgentName} failed", agentName); + if (ex is OperationCanceledException) tcs.TrySetCanceled(cancellationToken); + else tcs.TrySetException(ex); + }, + () => + { + // 🚨 When the primary reactive path is wired (delegationEvents + + // workspace), IT owns TCS resolution — it reads the sub-thread's + // dedicated Summary on the genuine post-execution Idle. + // ExecuteDelegationAsync completes THIS observable IMMEDIATELY after + // creating the sub-thread (it does not await sub-thread completion — + // see ChatClientAgentFactory: observer.OnCompleted() right after + // CreateNode), so resolving here would race the primary and win with + // an EMPTY chunk aggregate (no chunks are emitted on this path) — + // delegate_to_agent's tool result would always be "" and the parent + // tool-call Result would stay null. Only the legacy path (no + // delegationEvents/workspace) resolves from chunks here. + if (delegationEvents is not null && workspace is not null) + return; + if (tcs.TrySetResult(sb.ToString())) + { + logger?.LogInformation( + "Delegation to {AgentName} completed via chunk-aggregate fallback (len={Len})", + agentName, sb.Length); + } + }); + + return tcs.Task; } + var coordinationGuidance = (hasListTool, hasSendTool) switch + { + (true, true) => """ + + You can launch MULTIPLE delegations in parallel and coordinate them as they run: + - Call `list_sub_threads` at any time during your turn to see what delegations + you have in flight, their status (Executing / Idle / Cancelled), a preview + of the partial response, and last-activity timestamp. Use this to decide + whether to wait, to nudge a stuck agent, or to abandon a path. + - Call `send_to_sub_thread(threadPath, message)` to inject a mid-stream + follow-up into a running sub-thread (e.g. a clarification, a correction, + or "summarize and stop now"). The sub-agent picks the message up via its + pending-messages queue before producing its summary. + Sub-agents are instructed to produce a DEDICATED SUMMARY before returning; that + summary is what comes back as the tool result of `delegate_to_agent`. + """, + (true, false) => """ + + You can launch MULTIPLE delegations in parallel and inspect them as they run: + - Call `list_sub_threads` at any time to see active delegations, their status, + a preview of the partial response, and last-activity timestamp. + Sub-agents produce a dedicated summary before returning; that summary is the + tool result of `delegate_to_agent`. + """, + (false, true) => """ + + You can send follow-up messages to running sub-threads via + `send_to_sub_thread(threadPath, message)` — useful for clarifying or steering + an in-progress delegation. Sub-agents pick the message up via their pending- + messages queue. Sub-agents produce a dedicated summary before returning. + """, + _ => "" + }; + var description = $""" Delegate to a specialized agent when the request matches their expertise. Each delegation runs in an isolated context - the agent won't see previous conversation history. - The delegated agent's output streams back as it generates. + The delegated agent's output streams inline in the parent conversation via a nested streaming + view, and the dedicated summary the sub-agent produces at the end is returned as the tool result. When delegating parallel work on different documents, set the 'context' parameter to the specific node path for each delegation. This ensures each agent sees the correct document. When omitted, the parent's context is inherited. - + {coordinationGuidance} Available agents: {agentsJson} @@ -121,4 +376,70 @@ Choose the most appropriate agent based on the user's request. name: "delegate_to_agent", description: description); } + + /// + /// Tool that returns a JSON snapshot of currently-active sub-threads (path, + /// agent, status, preview text, last activity). Read-only; no side effects. + /// + private static AITool CreateListSubThreadsTool( + Func> listSubThreads, + ILogger? logger) + { + string ListSubThreads() + { + var info = listSubThreads(); + logger?.LogInformation("list_sub_threads → {Count} active delegations", info.Count); + return JsonSerializer.Serialize(info, new JsonSerializerOptions + { + WriteIndented = false, + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }); + } + + return AIFunctionFactory.Create( + ListSubThreads, + name: "list_sub_threads", + description: """ + List every sub-thread you have currently spawned via delegate_to_agent. + Returns an array of { threadPath, agentName, status, previewText, lastActivity } + so you can see which delegations are still running, which are completing, + and roughly what each agent has produced so far. Use this when you want to + decide whether to wait for a delegation, nudge it with send_to_sub_thread, + or pivot to a different approach. + """); + } + + /// + /// Tool that pushes a follow-up user message into an in-flight sub-thread's + /// pending-messages queue. The sub-agent ingests it before producing its + /// next response chunk; the parent does not wait for ack here (the parent + /// already drains the sub-thread's stream via the original delegate_to_agent + /// tool call). + /// + private static AITool CreateSendToSubThreadTool( + Action sendToSubThread, + ILogger? logger) + { + string SendToSubThread( + [Description("Full thread path of the sub-thread (from list_sub_threads).")] string threadPath, + [Description("Message to send to the sub-agent — clarification, correction, or steering instruction.")] string message) + { + logger?.LogInformation("send_to_sub_thread {ThreadPath} (msgLen={Len})", threadPath, message.Length); + sendToSubThread(threadPath, message); + return "Queued"; + } + + return AIFunctionFactory.Create( + SendToSubThread, + name: "send_to_sub_thread", + description: """ + Push a follow-up message into a running sub-thread's pending queue. + The sub-agent picks it up via its inbox-drain mechanism before producing + its next chunk. Use this to clarify ("focus on chapter 3 only"), correct + ("ignore the previous mistake and restart from X"), or steer ("summarize + and stop now"). Returns "Queued" immediately — the sub-agent's response to + your follow-up appears in the same sub-thread stream you already see via + the nested streaming view. + """); + } } diff --git a/src/MeshWeaver.AI/Plugins/LspPlugin.cs b/src/MeshWeaver.AI/Plugins/LspPlugin.cs new file mode 100644 index 000000000..b1f54304a --- /dev/null +++ b/src/MeshWeaver.AI/Plugins/LspPlugin.cs @@ -0,0 +1,152 @@ +using System.ComponentModel; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using System.Text.Json; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Mesh.Services.LanguageServer; +using MeshWeaver.Messaging; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI.Plugins; + +/// +/// Stage-1 LSP language-services plugin for code-authoring agents (Coder). Exposes +/// Roslyn-backed pre-flight diagnostics, hover, and completions over a NodeType's live +/// CSharpCompilation — the same surface as the lsp_* MCP tools, mounted +/// only on agents that opt into plugins: - Lsp so other agents +/// (Assistant, Researcher, Worker) don't see the noise. +/// +/// +/// Method names mirror the MCP surface (McpMeshPlugin.LspCheckNode etc.) so docs +/// and the Coder agent's prompt can reference one set of tool names regardless of +/// transport. Path arguments go through +/// so relative paths (@MyChild/...) resolve against the current chat context, +/// matching 's conventions. +/// +/// +public class LspPlugin(IMessageHub hub, IAgentChat chat) +{ + private readonly ILogger logger = hub.ServiceProvider.GetRequiredService>(); + private readonly IMeshLanguageService languageService = + hub.ServiceProvider.GetRequiredService(); + private readonly AccessService? accessService = hub.ServiceProvider.GetService(); + + [Description(@"PRE-FLIGHT CHECK before committing a source change to a NodeType. Runs Roslyn against the NodeType's current source set with ONE source file substituted by `proposedCode`, returns all diagnostics (errors + warnings). No emit, no Recycle, no side effects — purely speculative. + +Use this in the Coder edit loop: edit a Source/*.cs file in your head → `LspCheckNode` → if diagnostics, fix → repeat → only then `Patch` + `Compile`. Eliminates the costly blind-patch / Compile / fix cycle. + +Returns `{ok: true, diagnostics: []}` when the substituted source compiles cleanly, or `{ok: false, diagnostics: [{id, severity, message, sourcePath?, line?, character?}, ...]}` when it doesn't. Severity is one of `Hidden|Info|Warning|Error`. Positions are 0-based.")] + public Task LspCheckNode( + [Description("Path to the NodeType (e.g., @ACME/Story).")] string nodeTypePath, + [Description("Path of the Source Code node being edited (e.g., @ACME/Story/Source/StoryTypes.cs). If not in the current source set, the proposed code is added as a new file.")] string sourcePath, + [Description("The proposed full source text for that file.")] string proposedCode) + => WithContext(() => languageService.CheckSpeculative( + MeshOperations.ResolvePath(MeshOperations.ResolveContextPath(chat, nodeTypePath)), + MeshOperations.ResolvePath(MeshOperations.ResolveContextPath(chat, sourcePath)), + proposedCode ?? string.Empty)) + .Select(diagnostics => FormatDiagnosticsJson(diagnostics)) + .FirstAsync().ToTask(); + + [Description(@"Returns Roslyn diagnostics from the NodeType's CURRENT cached compilation — distinct from `GetDiagnostics` which only reports compile status (Ok/Error/Compiling). This enumerates every diagnostic in the compilation (errors + warnings + info) with source location, so you can see exactly what's wrong without re-compiling. + +Returns `{ok: true|false, diagnostics: [...]}` — same shape as `LspCheckNode`. Empty `diagnostics` plus `ok:true` means clean.")] + public Task LspDiagnosticsForNode( + [Description("Path to the NodeType (e.g., @ACME/Story).")] string nodeTypePath) + => WithContext(() => languageService.GetDiagnostics( + MeshOperations.ResolvePath(MeshOperations.ResolveContextPath(chat, nodeTypePath)))) + .Select(diagnostics => FormatDiagnosticsJson(diagnostics)) + .FirstAsync().ToTask(); + + [Description(@"Roslyn QuickInfo (hover tooltip) at a position in a Source Code file. Returns the symbol's signature and XML doc summary as markdown. + +Returns `{markdown: ""..."" }` when a symbol resolves at the position, or `{}` when nothing is there. Positions are 0-based (LSP convention) — line 0 is the first line.")] + public Task LspHoverForNode( + [Description("Path to the NodeType (e.g., @ACME/Story).")] string nodeTypePath, + [Description("Path of the Source Code node (e.g., @ACME/Story/Source/StoryTypes.cs).")] string sourcePath, + [Description("0-based line number.")] int line, + [Description("0-based character offset within the line.")] int character) + => WithContext(() => languageService.GetHover( + MeshOperations.ResolvePath(MeshOperations.ResolveContextPath(chat, nodeTypePath)), + MeshOperations.ResolvePath(MeshOperations.ResolveContextPath(chat, sourcePath)), + new SourcePosition(line, character))) + .Select(hover => JsonSerializer.Serialize( + hover is null ? new { } : (object)new { markdown = hover.ContentMarkdown }, + hub.JsonSerializerOptions)) + .FirstAsync().ToTask(); + + [Description(@"Roslyn code completions at a position in a Source Code file. Returns up to `max` suggestions with kind / insert text / detail / sort key, sorted by Roslyn's relevance. + +Returns `{items: [{label, kind, insertText, detail?, sortText?}, ...]}`. `kind` is the LSP completion-item kind name (`Method`, `Class`, `Field`, etc.). Empty `items` means no completions at that position. Positions are 0-based.")] + public Task LspCompletionsForNode( + [Description("Path to the NodeType (e.g., @ACME/Story).")] string nodeTypePath, + [Description("Path of the Source Code node (e.g., @ACME/Story/Source/StoryTypes.cs).")] string sourcePath, + [Description("0-based line number.")] int line, + [Description("0-based character offset within the line.")] int character, + [Description("Maximum number of completions to return. Default 20.")] int max = 20) + => WithContext(() => languageService.GetCompletions( + MeshOperations.ResolvePath(MeshOperations.ResolveContextPath(chat, nodeTypePath)), + MeshOperations.ResolvePath(MeshOperations.ResolveContextPath(chat, sourcePath)), + new SourcePosition(line, character), + max)) + .Select(items => JsonSerializer.Serialize( + new + { + items = items.Select(i => new + { + label = i.Label, + kind = i.Kind.ToString(), + insertText = i.InsertText, + detail = i.Detail, + sortText = i.SortText, + }).ToArray() + }, + hub.JsonSerializerOptions)) + .FirstAsync().ToTask(); + + /// + /// AccessContext re-seed on Subscribe — mirrors . + /// AsyncLocal doesn't flow through the agent framework's streaming + tool-invocation + /// pipeline, so every plugin entry point that touches hub state must re-seed. + /// + private IObservable WithContext(Func> work) => + Observable.Defer(() => + { + var userCtx = chat.ExecutionContext?.UserAccessContext; + if (userCtx != null) + accessService?.SetContext(userCtx); + return work(); + }); + + private string FormatDiagnosticsJson(IReadOnlyList diagnostics) + { + var anyErrors = diagnostics.Any(d => d.Severity == DiagnosticSeverity.Error); + return JsonSerializer.Serialize( + new + { + ok = !anyErrors, + diagnostics = diagnostics.Select(d => new + { + id = d.Id, + severity = d.Severity.ToString(), + message = d.Message, + sourcePath = d.Location?.SourcePath, + line = d.Location?.Range.Start.Line, + character = d.Location?.Range.Start.Character, + }).ToArray() + }, + hub.JsonSerializerOptions); + } + + public IList CreateTools() + { + return + [ + AIFunctionFactory.Create(LspCheckNode), + AIFunctionFactory.Create(LspDiagnosticsForNode), + AIFunctionFactory.Create(LspHoverForNode), + AIFunctionFactory.Create(LspCompletionsForNode), + ]; + } +} diff --git a/src/MeshWeaver.AI/Plugins/SkillTool.cs b/src/MeshWeaver.AI/Plugins/SkillTool.cs new file mode 100644 index 000000000..ee5f8d416 --- /dev/null +++ b/src/MeshWeaver.AI/Plugins/SkillTool.cs @@ -0,0 +1,90 @@ +using System.ComponentModel; +using System.Reactive.Linq; +using System.Text.Json; +using MeshWeaver.Mesh; +using MeshWeaver.Messaging; +using Microsoft.Extensions.AI; + +namespace MeshWeaver.AI.Plugins; + +/// +/// Creates the load_skill tool — reads a nodeType:Skill node by path and returns its +/// instructions (the SKILL.md body), so the agent can INJECT a skill's how-to on demand. Skills are +/// found with search nodeType:Skill; load_skill injects the one that fits the task. Reads +/// the authoritative single-node stream (never the eventually-consistent query index). +/// +public static class SkillTool +{ + /// Creates the load_skill AITool bound to the given hub + chat (for sub-thread launch). + public static AITool Create(IMessageHub hub, IAgentChat chat) + { + Task LoadSkill( + [Description("The full mesh path of the nodeType:Skill node to load (e.g. a path returned by `search nodeType:Skill`).")] + string skillPath, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(skillPath)) + return Task.FromResult("Provide the skill's path — find skills with `search nodeType:Skill`."); + + // Authoritative single-node read (GetMeshNodeStream, not QueryAsync — the query index lags). + var tcs = new TaskCompletionSource(); + hub.GetMeshNodeStream(skillPath.Trim()) + .Where(n => n is not null) + .Take(1) + .Timeout(TimeSpan.FromSeconds(10)) + .Subscribe( + node => + { + var def = DefinitionOf(node!, hub.JsonSerializerOptions); + var instructions = def?.Instructions; + if (string.IsNullOrWhiteSpace(instructions)) + { + tcs.TrySetResult($"Skill '{skillPath}' has no instructions to load."); + return; + } + + // LaunchesSubThread: run the skill in its OWN sub-thread to keep the work out of + // the main context, via the generic StartThread launcher (mainNode = this thread). + var execCtx = chat.ExecutionContext; + if (def!.LaunchesSubThread && execCtx is not null) + { + hub.StartThread( + execCtx.ContextPath ?? execCtx.ThreadPath, + instructions!, + contextPath: execCtx.ContextPath, + createdBy: execCtx.UserAccessContext?.ObjectId, + mainNode: execCtx.ThreadPath); + tcs.TrySetResult( + $"Launched skill '{node!.Name ?? skillPath}' in a sub-thread to run in isolation. " + + "Its result appears inline when it completes — continue with other work."); + return; + } + + tcs.TrySetResult(instructions!); + }, + ex => tcs.TrySetResult($"Could not load skill '{skillPath}': {ex.Message}")); + return tcs.Task; + } + + return AIFunctionFactory.Create( + LoadSkill, + name: "load_skill", + description: "Loads a skill (a nodeType:Skill node) by path and returns its instructions — the " + + "how-to for a specific operation. Find skills with `search nodeType:Skill`, then load the " + + "one that fits the task. Load a skill only when a request matches it, and read each skill " + + "only once — if you have already loaded it this conversation, do not re-load it."); + } + + private static SkillDefinition? DefinitionOf(MeshNode node, JsonSerializerOptions json) => node.Content switch + { + SkillDefinition s => s, + JsonElement je => TryDeserialize(je, json), + _ => null + }; + + private static SkillDefinition? TryDeserialize(JsonElement je, JsonSerializerOptions json) + { + try { return JsonSerializer.Deserialize(je.GetRawText(), json); } + catch { return null; } + } +} diff --git a/src/MeshWeaver.AI/Plugins/VersionPlugin.cs b/src/MeshWeaver.AI/Plugins/VersionPlugin.cs index 8dc7dd2f5..e8c8f9d02 100644 --- a/src/MeshWeaver.AI/Plugins/VersionPlugin.cs +++ b/src/MeshWeaver.AI/Plugins/VersionPlugin.cs @@ -39,22 +39,16 @@ public Task GetVersions( logger.LogInformation("GetVersions called for path={Path}", path); var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - Observable.FromAsync(async ct => + versionQuery.GetVersions(path) + .Select(v => (object)new { - var versions = ImmutableList.Empty; - await foreach (var v in versionQuery.GetVersionsAsync(path, ct)) - { - versions = versions.Add(new - { - v.Version, - LastModified = v.LastModified.ToString("yyyy-MM-dd HH:mm:ss"), - v.ChangedBy, - v.Name, - v.NodeType - }); - } - return versions; + v.Version, + LastModified = v.LastModified.ToString("yyyy-MM-dd HH:mm:ss"), + v.ChangedBy, + v.Name, + v.NodeType }) + .ToList() .SubscribeOn(TaskPoolScheduler.Default) .Subscribe( versions => tcs.TrySetResult(versions.Count == 0 @@ -79,7 +73,7 @@ public Task GetVersion( logger.LogInformation("GetVersion called for path={Path}, version={Version}", path, version); var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - Observable.FromAsync(ct => versionQuery.GetVersionAsync(path, version, hub.JsonSerializerOptions, ct)) + versionQuery.GetVersion(path, version, hub.JsonSerializerOptions) .SubscribeOn(TaskPoolScheduler.Default) .Subscribe( node => tcs.TrySetResult(node == null @@ -104,7 +98,7 @@ public Task RestoreVersion( logger.LogInformation("RestoreVersion called for path={Path}, version={Version}", path, version); var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - Observable.FromAsync(ct => versionQuery.GetVersionAsync(path, version, hub.JsonSerializerOptions, ct)) + versionQuery.GetVersion(path, version, hub.JsonSerializerOptions) .SubscribeOn(TaskPoolScheduler.Default) .SelectMany(historicalNode => { @@ -139,23 +133,16 @@ public Task RestoreFromPointInTime( return Task.FromResult($"Error: Invalid timestamp '{timestamp}'. Use ISO 8601 format (e.g., '2026-03-25T14:30:00Z')."); var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - Observable.FromAsync(async ct => - { - // Find the latest version at or before the target time - await foreach (var v in versionQuery.GetVersionsAsync(path, ct)) - { - if (v.LastModified <= targetTime) - return v; - } - return null; - }) + versionQuery.GetVersions(path) + .Where(v => v.LastModified <= targetTime) + .Take(1) + .DefaultIfEmpty() .SubscribeOn(TaskPoolScheduler.Default) .SelectMany(targetVersion => { if (targetVersion == null) return Observable.Return<(MeshNode? restored, MeshNodeVersion? target)>((null, null)); - return Observable.FromAsync(ct => - versionQuery.GetVersionAsync(path, targetVersion.Version, hub.JsonSerializerOptions, ct)) + return versionQuery.GetVersion(path, targetVersion.Version, hub.JsonSerializerOptions) .SelectMany(historicalNode => { if (historicalNode == null) diff --git a/src/MeshWeaver.AI/Plugins/WebSearchPlugin.cs b/src/MeshWeaver.AI/Plugins/WebSearchPlugin.cs index 6f61631b2..70ab016cc 100644 --- a/src/MeshWeaver.AI/Plugins/WebSearchPlugin.cs +++ b/src/MeshWeaver.AI/Plugins/WebSearchPlugin.cs @@ -1,8 +1,11 @@ using System.Collections.Immutable; using System.ComponentModel; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; using System.Text; using System.Text.Json; using HtmlAgilityPack; +using MeshWeaver.Mesh.Threading; using Microsoft.Extensions.AI; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -33,115 +36,129 @@ public class WebSearchConfiguration /// /// Plugin providing web search and web page fetching tools for AI agents. -/// Register via . +/// Register via AIExtensions.AddWebSearchPlugin. /// public class WebSearchPlugin : IAgentPlugin { private readonly HttpClient httpClient; private readonly WebSearchConfiguration config; private readonly ILogger logger; + private readonly IIoPool ioPool; public string Name => "WebSearch"; public WebSearchPlugin( HttpClient httpClient, IOptions options, - ILogger logger) + ILogger logger, + IoPoolRegistry? ioPoolRegistry = null) { this.httpClient = httpClient; this.config = options.Value; this.logger = logger; + ioPool = ioPoolRegistry?.Get(IoPoolNames.Http) ?? IoPool.Unbounded; } + // The AIFunction surface requires Task — these are the sanctioned one-line + // boundary adapters; the bodies are reactive with the HTTP leaf bridged through the + // IIoPool (AsynchronousCalls.md, ControlledIoPooling.md). [Description("Searches the web using Bing and returns relevant results with titles, URLs, and snippets. Use this to find current information, documentation, or any topic on the internet.")] - public async Task SearchWeb( + public Task SearchWeb( [Description("Search query string")] string query, [Description("Number of results to return (default 5, max 20)")] int count = 5) + => SearchWebCore(query, count).FirstAsync().ToTask(); + + [Description("Fetches a web page and extracts its text content. Use this to read articles, documentation, or any public web page after finding URLs via SearchWeb.")] + public Task FetchWebPage( + [Description("URL of the web page to fetch")] string url) + => FetchWebPageCore(url).FirstAsync().ToTask(); + + internal IObservable SearchWebCore(string query, int count) { logger.LogInformation("SearchWeb called with query={Query}, count={Count}", query, count); if (string.IsNullOrWhiteSpace(config.BingApiKey)) - return "Web search is not configured. A Bing Search API key is required."; + return Observable.Return("Web search is not configured. A Bing Search API key is required."); - count = Math.Clamp(count, 1, 20); + var clamped = Math.Clamp(count, 1, 20); - try - { - using var request = new HttpRequestMessage(HttpMethod.Get, - $"{config.Endpoint}?q={Uri.EscapeDataString(query)}&count={count}&textFormat=Plain"); - request.Headers.Add("Ocp-Apim-Subscription-Key", config.BingApiKey); + // The HTTP round-trip is ONE pooled async leaf — async lives only inside + // the IIoPool bridge, never on the subscribing thread. + return ioPool.Invoke(async ct => + { + using var request = new HttpRequestMessage(HttpMethod.Get, + $"{config.Endpoint}?q={Uri.EscapeDataString(query)}&count={clamped}&textFormat=Plain"); + request.Headers.Add("Ocp-Apim-Subscription-Key", config.BingApiKey); - using var response = await httpClient.SendAsync(request); - response.EnsureSuccessStatusCode(); + using var response = await httpClient.SendAsync(request, ct); + response.EnsureSuccessStatusCode(); - var json = await response.Content.ReadAsStringAsync(); - using var doc = JsonDocument.Parse(json); + var json = await response.Content.ReadAsStringAsync(ct); + using var doc = JsonDocument.Parse(json); - var results = ImmutableList.Empty; - if (doc.RootElement.TryGetProperty("webPages", out var webPages) && - webPages.TryGetProperty("value", out var pages)) - { - foreach (var page in pages.EnumerateArray()) + var results = ImmutableList.Empty; + if (doc.RootElement.TryGetProperty("webPages", out var webPages) && + webPages.TryGetProperty("value", out var pages)) { - results = results.Add(new + foreach (var page in pages.EnumerateArray()) { - title = page.GetProperty("name").GetString(), - url = page.GetProperty("url").GetString(), - snippet = page.GetProperty("snippet").GetString() - }); + results = results.Add(new + { + title = page.GetProperty("name").GetString(), + url = page.GetProperty("url").GetString(), + snippet = page.GetProperty("snippet").GetString() + }); + } } - } - if (results.Count == 0) - return "No results found."; + if (results.Count == 0) + return "No results found."; - return JsonSerializer.Serialize(results); - } - catch (HttpRequestException ex) - { - logger.LogError(ex, "Web search failed for query={Query}", query); - return $"Web search failed: {ex.Message}"; - } + return JsonSerializer.Serialize(results); + }) + .Catch((HttpRequestException ex) => + { + logger.LogError(ex, "Web search failed for query={Query}", query); + return Observable.Return($"Web search failed: {ex.Message}"); + }); } - [Description("Fetches a web page and extracts its text content. Use this to read articles, documentation, or any public web page after finding URLs via SearchWeb.")] - public async Task FetchWebPage( - [Description("URL of the web page to fetch")] string url) + internal IObservable FetchWebPageCore(string url) { logger.LogInformation("FetchWebPage called with url={Url}", url); if (string.IsNullOrWhiteSpace(url)) - return "URL cannot be empty."; + return Observable.Return("URL cannot be empty."); - try - { - using var request = new HttpRequestMessage(HttpMethod.Get, url); - request.Headers.Add("User-Agent", "MeshWeaver/1.0"); - request.Headers.Add("Accept", "text/html,application/xhtml+xml,text/plain"); + return ioPool.Invoke(async ct => + { + using var request = new HttpRequestMessage(HttpMethod.Get, url); + request.Headers.Add("User-Agent", "MeshWeaver/1.0"); + request.Headers.Add("Accept", "text/html,application/xhtml+xml,text/plain"); - using var response = await httpClient.SendAsync(request); - response.EnsureSuccessStatusCode(); + using var response = await httpClient.SendAsync(request, ct); + response.EnsureSuccessStatusCode(); - var contentType = response.Content.Headers.ContentType?.MediaType ?? ""; - var content = await response.Content.ReadAsStringAsync(); + var contentType = response.Content.Headers.ContentType?.MediaType ?? ""; + var content = await response.Content.ReadAsStringAsync(ct); - // Extract text from HTML - if (contentType.Contains("html", StringComparison.OrdinalIgnoreCase) || - content.TrimStart().StartsWith("<", StringComparison.Ordinal)) - { - content = ExtractTextFromHtml(content); - } + // Extract text from HTML + if (contentType.Contains("html", StringComparison.OrdinalIgnoreCase) || + content.TrimStart().StartsWith("<", StringComparison.Ordinal)) + { + content = ExtractTextFromHtml(content); + } - if (content.Length > config.MaxFetchContentLength) - content = content[..config.MaxFetchContentLength] + "\n\n[Content truncated]"; + if (content.Length > config.MaxFetchContentLength) + content = content[..config.MaxFetchContentLength] + "\n\n[Content truncated]"; - return content; - } - catch (HttpRequestException ex) - { - logger.LogError(ex, "Failed to fetch web page url={Url}", url); - return $"Failed to fetch web page: {ex.Message}"; - } + return content; + }) + .Catch((HttpRequestException ex) => + { + logger.LogError(ex, "Failed to fetch web page url={Url}", url); + return Observable.Return($"Failed to fetch web page: {ex.Message}"); + }); } private static string ExtractTextFromHtml(string html) diff --git a/src/MeshWeaver.AI/ProviderKeyProtector.cs b/src/MeshWeaver.AI/ProviderKeyProtector.cs new file mode 100644 index 000000000..546475368 --- /dev/null +++ b/src/MeshWeaver.AI/ProviderKeyProtector.cs @@ -0,0 +1,91 @@ +using System.Security.Cryptography; +using System.Text; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI; + +/// +/// AES-256-GCM . Stored form is +/// enc:v1:{base64(nonce(12) | ciphertext | tag(16))}. A fresh random +/// nonce per encryption means re-encrypting the same key yields different +/// ciphertext (semantic security) — so do not treat the stored blob as a +/// stable fingerprint of the key. +/// +public sealed class ProviderKeyProtector : IProviderKeyProtector +{ + private const string Prefix = "enc:v1:"; + private const int NonceLen = 12; // AesGcm.NonceByteSizes + private const int TagLen = 16; // AesGcm.TagByteSizes max + + private readonly IMasterKeyProvider masterKeyProvider; + private readonly ILogger? logger; + + public ProviderKeyProtector(IMasterKeyProvider masterKeyProvider, ILogger? logger = null) + { + this.masterKeyProvider = masterKeyProvider; + this.logger = logger; + } + + public string? Protect(string? plaintext) + { + if (string.IsNullOrEmpty(plaintext)) return plaintext; + // Idempotent — never double-encrypt an already-tagged value. + if (plaintext.StartsWith("enc:", StringComparison.Ordinal)) return plaintext; + + var key = masterKeyProvider.GetMasterKey(); + if (key is null) return plaintext; // encryption disabled → passthrough + + var nonce = RandomNumberGenerator.GetBytes(NonceLen); + var pt = Encoding.UTF8.GetBytes(plaintext); + var ct = new byte[pt.Length]; + var tag = new byte[TagLen]; + + using var gcm = new AesGcm(key, TagLen); + gcm.Encrypt(nonce, pt, ct, tag); + + var blob = new byte[NonceLen + ct.Length + TagLen]; + Buffer.BlockCopy(nonce, 0, blob, 0, NonceLen); + Buffer.BlockCopy(ct, 0, blob, NonceLen, ct.Length); + Buffer.BlockCopy(tag, 0, blob, NonceLen + ct.Length, TagLen); + return Prefix + Convert.ToBase64String(blob); + } + + public string? Unprotect(string? stored) + { + if (string.IsNullOrEmpty(stored)) return stored; + // Legacy / disabled: untagged values are plaintext, return as-is. + if (!stored.StartsWith("enc:", StringComparison.Ordinal)) return stored; + if (!stored.StartsWith(Prefix, StringComparison.Ordinal)) + { + logger?.LogWarning("Stored provider key has an unknown encryption tag — cannot decrypt."); + return null; + } + + var key = masterKeyProvider.GetMasterKey(); + if (key is null) + { + logger?.LogWarning("Stored provider key is encrypted but no master key is configured — cannot decrypt."); + return null; + } + + try + { + var blob = Convert.FromBase64String(stored[Prefix.Length..]); + if (blob.Length < NonceLen + TagLen) return null; + var nonce = blob.AsSpan(0, NonceLen); + var ctLen = blob.Length - NonceLen - TagLen; + var ct = blob.AsSpan(NonceLen, ctLen); + var tag = blob.AsSpan(NonceLen + ctLen, TagLen); + var pt = new byte[ctLen]; + + using var gcm = new AesGcm(key, TagLen); + gcm.Decrypt(nonce, ct, tag, pt); + return Encoding.UTF8.GetString(pt); + } + catch (Exception ex) + { + logger?.LogWarning(ex, "Failed to decrypt a stored provider key (wrong master key or corrupt value)."); + return null; + } + } +} diff --git a/src/MeshWeaver.AI/SelectionId.cs b/src/MeshWeaver.AI/SelectionId.cs new file mode 100644 index 000000000..9c31ce644 --- /dev/null +++ b/src/MeshWeaver.AI/SelectionId.cs @@ -0,0 +1,26 @@ +namespace MeshWeaver.AI; + +/// +/// Conventions for harness/agent/model selection values. Pickers and composer state store the +/// picked node's full PATH (e.g. Harness/MeshWeaver, _Provider/Anthropic/claude-…, +/// Agent/Coder) and that path flows end-to-end through composer → message → Pending* → +/// RoundParams (never pre-resolved at the GUI — see CLAUDE.md "pass node PATHS through"). +/// Execution matches REGISTERED ids/names (IHarness.Id, agent name, model id), which are +/// always the LAST path segment — so the execution boundary normalizes with , +/// accepting both forms. +/// +public static class SelectionId +{ + /// + /// The id (last path segment) of a picked node path; bare ids pass through unchanged. + /// 🚨 Invariant: correct ONLY while no registered id (IHarness.Id, agent + /// name, model id) itself contains /. Model ids are the risk case — some providers + /// use org/model-shaped ids (e.g. HuggingFace). If such a provider is onboarded, its + /// catalog node id must be the last segment only (the picker path supplies the prefix), + /// or this normalization must learn the known catalog prefixes instead. + /// + public static string? IdOf(string? pathOrId) + => string.IsNullOrEmpty(pathOrId) + ? pathOrId + : pathOrId[(pathOrId.LastIndexOf('/') + 1)..]; +} diff --git a/src/MeshWeaver.AI/SkillNodeType.cs b/src/MeshWeaver.AI/SkillNodeType.cs new file mode 100644 index 000000000..eb8762cdd --- /dev/null +++ b/src/MeshWeaver.AI/SkillNodeType.cs @@ -0,0 +1,249 @@ +using System.Text.Json; +using MeshWeaver.Graph; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace MeshWeaver.AI; + +/// +/// The "Skill" node type — the unified, first-class "a thing that does something" concept. A Skill is +/// EITHER a behaviour the chat performs when invoked (open a combobox + select an agent/model/ +/// harness, load a document into the content window, connect/login — what we used to call slash +/// commands) AND/OR an instruction (a SKILL.md body mounted to the Claude Code / +/// Copilot CLIs and advertised to the MeshWeaver agent to load on demand). Users / Spaces / NodeTypes +/// ship their own Skill nodes, discovered through namespace inheritance (). +/// +/// This SUBSUMES the old Command node type — is the old +/// CommandDefinition (query + composer field + title). Agents are NOT skills (agents == system +/// prompts; skills == capabilities loaded as you go). +/// +public static class SkillNodeType +{ + /// NodeType discriminator. + public const string NodeType = "Skill"; + + /// Namespace (partition) the built-in skill catalog lives under. + public const string RootNamespace = "Skill"; + + /// + /// Registers the Skill type node + its content type (PublicRead), and — when not DB-synced — the + /// static provider that serves the built-in skills (/agent, /model, /harness) + /// read-only under the Skill partition. Mirrors the retired AddCommandType. + /// + public static TBuilder AddSkillType(this TBuilder builder, + IReadOnlySet? serveFromPartition = null) where TBuilder : MeshBuilder + { + builder.AddMeshNodes(CreateMeshNode()); + builder.ConfigureNodeTypeAccess(a => a.WithPublicRead(NodeType)); + builder.ConfigureHub(config => config.WithType(nameof(SkillDefinition))); + + var dbSynced = serveFromPartition?.Contains(RootNamespace) == true; + builder.ConfigureServices(services => + { + services.TryAddSingleton(); + if (!dbSynced) + { + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => + new StaticNodePartitionStorageProvider( + RootNamespace, + sp.GetRequiredService(), + description: "Built-in chat skills (read-only).")); + } + return services; + }); + return builder; + } + + /// The type-definition node for nodeType="Skill". + public static MeshNode CreateMeshNode() => new(NodeType) + { + Name = "Skill", + Icon = "/static/NodeTypeIcons/sparkle.svg", + IsSatelliteType = false, + HubConfiguration = config => config + .AddMeshDataSource(source => source + .WithContentType()) + }; + + /// + /// The query that discovers the skills available in a context — the SAME unified registry pattern as + /// agents and models: platform Skill + the current space's {space}/Skill + the user's + /// {user}/Skill, as one namespace:A|B|C exact-membership query + /// (). names the + /// space (its partition); the user's home. + /// + public static string[] SkillQueries(string? contextPath, string? userPath) + => AgentPickerProjection.BuildSkillQueries(userPath, AgentPickerProjection.PartitionOf(contextPath)); + + /// + /// Projects a mesh-node snapshot into the available skills, deduped by id (the slash word). Reads + /// the slash word from and help text from ; + /// the spec is the typed (or JsonElement-fallback) content. + /// + public static IReadOnlyList ProjectSkills( + IEnumerable snapshot, JsonSerializerOptions jsonOptions) + { + var byId = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var node in snapshot) + { + if (string.IsNullOrEmpty(node.Id)) continue; + if (!string.Equals(node.NodeType, NodeType, StringComparison.OrdinalIgnoreCase)) continue; + var def = node.Content switch + { + SkillDefinition d => d, + JsonElement je => TryDeserialise(je, jsonOptions), + _ => null, + }; + if (def is null) continue; + byId[node.Id] = new SkillInfo + { + Id = node.Id, + Name = node.Name, + Description = node.Description, + Path = node.Path, + Definition = def, + }; + } + return byId.Values.OrderBy(s => s.Id, StringComparer.OrdinalIgnoreCase).ToList(); + } + + private static SkillDefinition? TryDeserialise(JsonElement je, JsonSerializerOptions opts) + { + try { return JsonSerializer.Deserialize(je.GetRawText(), opts); } + catch { return null; } + } +} + +/// +/// Content of a node. A skill is a behaviour () and/or +/// an instruction (). The skill's NAME and DESCRIPTION come from the owning +/// . +/// +public record SkillDefinition +{ + /// + /// INSTRUCTION skill — the SKILL.md body (markdown). Surfaced to the CLI harnesses + the + /// MeshWeaver agent (when ) and loaded on demand from the mesh. Null + /// for a pure behaviour skill. + /// + public string? Instructions { get; init; } + + /// + /// BEHAVIOUR skill — what the skill DOES when invoked in the MeshWeaver chat (open a picker, load a + /// document into the content window, connect/login). Null for a pure instruction skill. + /// + public SkillAction? Action { get; init; } + + /// + /// Whether the skill is advertised up-front: instruction skills surfaced to the CLI harnesses + /// and the MeshWeaver agent so they're discoverable without being asked (read from the mesh on demand — + /// never materialised to disk). When false the skill still exists but is referenced/loaded on + /// demand only. Default true. + /// + public bool AutoMount { get; init; } = true; + + /// + /// Whether invoking the skill launches a sub-thread (a separate conversation the skill runs + /// in) rather than running inline in the current thread. Default false. + /// + public bool LaunchesSubThread { get; init; } = false; + + /// + /// The harness this skill belongs to — null = the MeshWeaver harness / applies everywhere. + /// This is what makes the status bar a per-harness, CLI-like control strip: the status row renders + /// exactly the ACTIVE harness's skills (each chip showing the current composer value, clickable into + /// its Pick combobox), and the /-menu offers them. Claude Code ships /model + /effort + /// (Harness = "ClaudeCode") with options it provides; Copilot ships its own; MeshWeaver keeps + /// /agent + /model. A skill is a value + a picker + a status chip — one concept. + /// + public string? Harness { get; init; } +} + +/// +/// A behaviour a skill performs in the MeshWeaver chat — one discriminated . +/// is the old CommandDefinition (Query + Field + Title). +/// +public record SkillAction +{ + /// + /// The action discriminator. NOT required and defaults to : + /// the hub serializer OMITS default-valued properties on write, so a Pick action (the default + /// enum value 0) is written with no kind field — a required Kind then fails to + /// deserialize ("missing required properties including: 'kind'"), dropping every Pick skill. A plain + /// default round-trips the omitted value correctly. + /// + public SkillActionKind Kind { get; init; } = SkillActionKind.Pick; + + /// : the mesh query whose nodes the combobox lists. + public string? Query { get; init; } + + /// : the camelCase ThreadComposer field the + /// selected node PATH is written to (harness / agentName / modelName). + public string? Field { get; init; } + + /// : the combobox title (e.g. "Choose a model"). + public string? Title { get; init; } + + /// : the node/path to load into the content window. + public string? ContentPath { get; init; } + + /// /: the + /// provider (ClaudeCode / Copilot). + public string? Provider { get; init; } +} + +/// What a does when the skill is invoked. +public enum SkillActionKind +{ + /// Open a combobox over and write the pick to the composer. + Pick, + + /// Load a node/document into the content window. + OpenContent, + + /// Log in / connect this provider's CLI subscription. + Connect, + + /// Log out / forget this provider's CLI subscription. + Disconnect, +} + +/// +/// A resolved skill for the chat input — its slash word (), name/description, path, and +/// the spec. Projected from a nodeType:Skill node by . +/// +public record SkillInfo +{ + /// The slash word (e.g. model for /model) — the Skill node's id. + public required string Id { get; init; } + + /// Display name (the node's Name). + public string? Name { get; init; } + + /// Help text shown in autocomplete (the node's Description). + public string? Description { get; init; } + + /// The skill node's full path (for load-on-demand by the agent). + public string? Path { get; init; } + + /// The skill spec. + public required SkillDefinition Definition { get; init; } + + /// For a skill, the picker request carrying the typed argument. + public NodePickerRequest? ToPickerRequest(string? searchTerm) => + Definition.Action is { Kind: SkillActionKind.Pick, Query: { } q, Field: { } f } + ? new NodePickerRequest(q, f, Definition.Action.Title ?? Name ?? Id, searchTerm) + : null; +} + +/// +/// A request from a skill to the host to render the generic node +/// selector: list the mesh nodes matching , and on selection write the chosen +/// node's PATH onto the composer field (a camelCase ThreadComposer +/// property — harness, agentName, modelName). When is +/// non-null the host pre-filters to it and auto-selects an exact match. (Was MeshWeaver.AI.Commands.) +/// +public record NodePickerRequest(string Query, string ComposerField, string Title, string? SearchTerm = null); diff --git a/src/MeshWeaver.AI/SkillStaticRepoSource.cs b/src/MeshWeaver.AI/SkillStaticRepoSource.cs new file mode 100644 index 000000000..a2d14acdf --- /dev/null +++ b/src/MeshWeaver.AI/SkillStaticRepoSource.cs @@ -0,0 +1,50 @@ +using MeshWeaver.Markdown; +using MeshWeaver.Mesh; + +namespace MeshWeaver.AI; + +/// +/// The built-in chat skills as a static-repo import source for the Skill partition — the same +/// nodes serves in-memory, materialized into the DB partition on +/// boot so skills are served from the database on the distributed/PG path (Orleans routing does NOT +/// consult the in-memory adapter; without this import /agent, /model and /harness +/// are invisible to namespace:Skill queries). Replaces the retired CommandStaticRepoSource. +/// +public sealed class SkillStaticRepoSource(BuiltInSkillProvider provider) : IStaticRepoSource +{ + /// + public string Partition => SkillNodeType.RootNamespace; + + /// + // Skill definitions ship with no meaningful version → fingerprint on content, so an edited + // built-in skill re-imports. + public bool Versioned => false; + + /// + // Skill content nodes PLUS the partition's PublicRead "_Policy" (PartitionAccessPolicy). On the + // SYNCED path the in-memory provider that served the policy is gated off, so the policy MUST be + // imported or the partition has no read policy → its skills are unreadable (the Harness wedge — + // OrleansHarnessPartitionPublicReadTest). Only OTHER "_"-governance is dropped. + public IReadOnlyList EnumerateSourceNodes() => + provider.GetStaticNodes() + .Where(n => n.NodeType == "PartitionAccessPolicy" + || !n.Segments.Skip(1).Any(seg => seg.StartsWith('_'))) + .ToArray(); + + /// + public MeshNode? PartitionRoot => new(SkillNodeType.RootNamespace) + { + Name = "Skills", + NodeType = "Space", + State = MeshNodeState.Active, + Content = new MarkdownContent + { + Content = """ + # Skills + + The built-in chat skills available across the platform (`/agent`, `/model`, + `/harness`). Spaces and NodeTypes add their own Skill nodes in their own partitions. + """ + } + }; +} diff --git a/src/MeshWeaver.AI/SubmitMessageRequest.cs b/src/MeshWeaver.AI/SubmitMessageRequest.cs deleted file mode 100644 index f6e6c7ce2..000000000 --- a/src/MeshWeaver.AI/SubmitMessageRequest.cs +++ /dev/null @@ -1,82 +0,0 @@ -using MeshWeaver.Mesh.Security; -using MeshWeaver.Messaging; -using MeshWeaver.Messaging.Security; - -namespace MeshWeaver.AI; - -/// -/// Request to submit a user message to a thread. -/// The thread hub creates the user message node, response node, and streams the agent response. -/// Thread must exist before submitting — create via IMeshService.CreateNodeAsync. -/// Requires Thread permission on the thread's parent path (partition scope), -/// not on the thread path itself, since access assignments are at partition level. -/// -[SubmitMessagePermission] -public record SubmitMessageRequest : IRequest -{ - public required string ThreadPath { get; init; } - public required string UserMessageText { get; init; } - public string? AgentName { get; init; } - public string? ModelName { get; init; } - public string? ContextPath { get; init; } - public IReadOnlyList? Attachments { get; init; } - - /// - /// Client-generated IDs for optimistic rendering. - /// If set, the server uses these instead of generating its own. - /// - public string? UserMessageId { get; init; } - public string? ResponseMessageId { get; init; } - - /// - /// Set by HandleSubmitMessage after creating the response node. - /// The execution hub uses this to post streaming progress updates. - /// - public string? ResponsePath { get; init; } - -} - -/// -/// Checks Thread permission on the thread's parent scope (before _Thread segment). -/// Thread paths follow the pattern {parentPath}/_Thread/{threadId}. -/// Access assignments are at the parent partition level, not on individual threads. -/// -public class SubmitMessagePermissionAttribute() : RequiresPermissionAttribute(Permission.Thread) -{ - public override IEnumerable<(string Path, Permission Permission)> GetPermissionChecks( - IMessageDelivery delivery, string hubPath) - { - // Extract parent path: User/rbuergi/_Thread/hello → User/rbuergi - var threadIndex = hubPath.IndexOf("/_Thread", StringComparison.Ordinal); - var parentPath = threadIndex > 0 ? hubPath[..threadIndex] : hubPath; - yield return (parentPath, Permission.Thread); - } -} - -public record SubmitMessageResponse -{ - public bool Success { get; init; } - public string? Error { get; init; } - public SubmitMessageStatus Status { get; init; } = SubmitMessageStatus.CellsCreated; - public string? ResponseText { get; init; } - - /// - /// Node changes made during this thread's execution. - /// Propagated upward so parent threads can aggregate changes from delegations. - /// - public System.Collections.Immutable.ImmutableList? UpdatedNodes { get; init; } - - /// - /// Full Messages list after cells are created. Sent with CellsCreated so the client - /// can render LayoutAreaViews immediately without waiting for the workspace stream. - /// - public IReadOnlyList? Messages { get; init; } -} - -public enum SubmitMessageStatus -{ - CellsCreated, - ExecutionCompleted, - ExecutionFailed, - ExecutionCancelled -} diff --git a/src/MeshWeaver.AI/Thread.cs b/src/MeshWeaver.AI/Thread.cs index 628c117da..2d72a0be6 100644 --- a/src/MeshWeaver.AI/Thread.cs +++ b/src/MeshWeaver.AI/Thread.cs @@ -1,4 +1,4 @@ -using System.Collections.Immutable; +using System.Collections.Immutable; using System.Text.Json; using System.Text.Json.Serialization; using MeshWeaver.Layout; @@ -34,6 +34,107 @@ public record ThreadExecutionContext public AccessContext? UserAccessContext { get; init; } } +// ResubmitIntent and FailureRecord records deleted 2026-05-27. The corresponding +// thread mutations now happen INLINE inside HubThreadExtensions.ResubmitMessage / +// RecordSubmissionFailure via a single stream.Update on the thread node — no +// intent-payload + per-operation watcher indirection. + +/// +/// Explicit lifecycle state for a thread's overall execution round. Replaces +/// the binary flag with named states so the +/// GUI can render distinct progress indicators and so test assertions can +/// pin the transition graph. +/// +/// State graph (one execution round): +/// Idle → StartingExecution → Executing → Idle, with a +/// Executing → Cancelled branch when execution is stopped. The thread +/// re-enters from either +/// or (a cancelled thread re-dispatches like Idle when +/// pending input remains). Error doesn't fork the graph — the error status +/// lands on the response cell () and the +/// thread returns to . There is no transient "completing" +/// state: terminal writes are atomic. +/// +/// Wake-up. On hub activation InitializeThreadLifecycle +/// reads the own-node stream's first emission and drives any non-terminal +/// state to a valid one once: a pending request is +/// honored, an interrupted round resumes its existing +/// response cell (re-entering ), and +/// / with pending input is left for +/// the submission watcher to claim. +/// +public enum ThreadExecutionStatus +{ + /// No round in flight. PendingUserMessages may still hold queued input — + /// the submission watcher will dispatch a new round when it observes this state. + Idle = 0, + + /// The _Exec hub claimed the round: draining + /// into , + /// materialising user satellite cells, allocating the response cell. No + /// agent tokens yet. + StartingExecution, + + /// Agent is streaming into the active response cell. The + /// check_inbox tool may drain newly-arrived pending entries: it + /// freezes the current response cell, inserts the new user cells after it, + /// and switches streaming to a fresh response cell. + Executing, + + /// Execution was stopped (user pressed Stop, or a parent cancelled a + /// sub-thread). Distinct, visible terminal-ish state: the response cell is + /// marked , but the thread behaves + /// like for re-dispatch — if PendingUserMessages + /// still holds input, the submission watcher starts a fresh round. Occupies + /// the int slot the removed transient "completing" state used to hold. + Cancelled = 3, + + /// User-marked terminal state — the thread is finished and + /// hidden from default catalogs (queries default to + /// -content.status:Done). A new submission implicitly transitions + /// back to so the user can reopen a conversation by typing. + Done = 4 +} + +/// +/// Lifecycle state of a single ThreadMessage cell. Replaces magic-string text +/// markers like trailing "*Cancelled*" / "*Error: ...*" with an explicit +/// per-cell state machine. +/// +/// User cells: created at on dispatch (the queued period +/// lives at the thread level via — +/// the cell doesn't exist until ingestion). +/// +/// Assistant cells: created at on round start, transition +/// to one of , , or +/// when the streaming loop exits. +/// +/// Pre-existing persisted cells without a Status field default to +/// (treated as stable history). +/// +public enum ThreadMessageStatus +{ + /// User cell appended to the thread queue, satellite not yet materialized. + /// In practice cells almost never carry this value (the queue lives on the thread, + /// not the cell) — included for completeness so external materializers can use it. + Queued, + + /// User cell materialized into a round (the round may be running or done). + Submitted, + + /// Assistant cell currently being generated — streaming loop active. + Streaming, + + /// Cell's turn finished successfully. + Completed, + + /// Cell's turn was cancelled mid-stream (ESC / Stop). Partial text preserved. + Cancelled, + + /// Cell's turn failed with an error. Error message in . + Error +} + /// /// Defines the type of a thread message for rendering purposes. /// @@ -109,10 +210,54 @@ public record Thread public string? CreatedBy { get; init; } /// - /// Whether any execution is currently active on this thread. - /// Set to true when a message is submitted, false when execution completes/cancels/errors. + /// The thread's own composer — the data-bound chat-input state INSIDE this thread + /// (draft + harness/agent/model selection as picked node paths + attachments). Copied + /// from the user's out-of-thread composer ({user}/_Thread/ThreadComposer) when + /// the thread is created (HubThreadExtensions.StartThread), with the draft + /// emptied (the draft became the first message). + /// + /// Embedded ON the thread content — deliberately NOT a separate satellite node — + /// so reads can never hit a missing node (no NotFound storm, no lazy-create/stamp + /// machinery) and submission drains it atomically: hub.SubmitComposer moves + /// into + /// and empties the composer in ONE stream.Update. Null on legacy threads — + /// readers treat null as an empty composer. + /// + public ThreadComposer? Composer { get; init; } + + /// + /// The thread's main output — the dedicated summary the agent produces at + /// the end of execution before returning. For sub-threads spawned via + /// delegate_to_agent, this is the value returned to the parent + /// agent as the tool-call result. Written by ExecuteMessageAsync at + /// the Completed terminal state (copies the last assistant message's text), + /// and the agent itself may overwrite it via a dedicated tool to provide + /// a tighter summary than the verbose chat response. + /// + public string? Summary { get; init; } + + /// + /// Explicit state machine for the round currently in flight. See + /// for the transition graph. The + /// submission watcher fires when is + /// and + /// is non-empty. + /// + public ThreadExecutionStatus Status { get; init; } = ThreadExecutionStatus.Idle; + + /// + /// Backwards-compatible boolean shorthand for "round in flight". True for + /// and + /// . Idle, Cancelled, and Done + /// are not executing — Cancelled is a stopped round (re-dispatchable like + /// Idle), Done is the user-marked terminal state. + /// New callsites should read directly to pick a + /// specific transition. /// - public bool IsExecuting { get; init; } + [JsonIgnore] + public bool IsExecuting + => Status is ThreadExecutionStatus.StartingExecution + or ThreadExecutionStatus.Executing; /// /// Current execution activity description (e.g., "Calling search_nodes...", "Delegating to Navigator..."). @@ -121,14 +266,18 @@ public record Thread public string? ExecutionStatus { get; init; } /// - /// The ID of the response message currently being generated. + /// The ID of the response message currently being generated. The full + /// response path is always {threadPath}/{ActiveMessageId} — every + /// downstream actor (_Exec streaming loop, parent's delegation watcher, + /// cancellation watcher, GUI status bar) derives the path that way. + /// Single source of truth for "where is the agent streaming right now"; + /// no separate path-property to keep in sync with the id. /// public string? ActiveMessageId { get; init; } - /// - /// Total tokens used in the current execution (input + output). - /// - public int TokensUsed { get; init; } + // Token usage is NOT stored on the thread. It lives on per-model TokenUsage satellites at + // {threadPath}/_Usage/{model} (see TokenUsageNodeType) — all cost tracking is outside the Thread. + // Per-message counts still live on each response cell (ThreadMessage.Input/Output/TotalTokens). /// /// When the current execution started. Used to show elapsed time. @@ -136,13 +285,46 @@ public record Thread public DateTime? ExecutionStartedAt { get; init; } /// - /// Pending user message text — set at thread creation to auto-start execution. - /// When the thread grain activates and sees this, it immediately starts streaming. - /// Cleared after execution starts. - /// Legacy: still used by the auto-execute-on-creation path. New submissions - /// from the GUI populate instead. + /// Wall clock of the most recent "the agent is still making progress" + /// signal — streaming text deltas, tool-call activity, status changes. + /// Written atomically with those events in the OWNING thread hub's + /// action block (no extra writes, no race). Read by the parent thread + /// hub's heartbeat watcher: if IsExecuting=true AND + /// (now - LastActivityAt) > HeartbeatTimeout (with a 60 s + /// cold-start grace measured from ), + /// the watcher sets = Cancelled on this + /// sub-thread — the same primitive the GUI Stop button uses. Replaces + /// the hard 5-minute watchdog in ExecuteDelegationAsync. /// - public string? PendingUserMessage { get; init; } + public DateTime? LastActivityAt { get; init; } + + /// + /// Per-thread override of the framework-default heartbeat timeout + /// (30 s). Set by an agent that legitimately makes slow progress + /// (e.g. non-streaming chat client with long single-shot completions). + /// Null → use default. The 60 s cold-start grace is applied + /// regardless, so the value can be aggressive without false-positives + /// on cold start. + /// + public TimeSpan? HeartbeatTimeout { get; init; } + + /// + /// Control-plane request for a status transition the owning thread hub + /// should achieve — the request half of the Activity-Control-Plane + /// (RequestedStatus requests, achieves) pattern. + /// Today the only requested transition is : + /// the GUI Stop button and a parent cancelling a sub-thread write + /// RequestedStatus = Cancelled; the cancel watcher observes its own + /// thread node, cancels the stored CTS, and propagates the same request onto + /// every active delegation sub-thread. The owning hub clears it back to + /// null once the achieved reaches the requested + /// value (or on wake-up while honoring a pending request). + /// + /// Stream-update only. The owning thread hub serialises writes on + /// its action block, so racing requests collapse into one observed + /// transition. See [RequestViaStreamUpdate.md] for the rule. + /// + public ThreadExecutionStatus? RequestedStatus { get; init; } /// /// User messages submitted by the client but not yet ingested into a round. @@ -150,29 +332,49 @@ public record Thread /// satellite ThreadMessage cells from these entries and clears them once /// the round is dispatched. Lets us do the entire submission as a single /// atomic stream.Update on this thread node — no separate - /// CreateNodeRequest, no AppendUserMessageRequest. + /// CreateNodeRequest, no ThreadInput.AppendUserInput. + /// + /// Each entry is a carrying the per-message context + + /// attachments (and a historical stamp of the agent/model/harness that submitted it). The + /// round's STICKY selection (agent / model / harness) is NOT read from here — it lives on + /// , the single data-bound source of truth. There is no thread-level + /// selection mirror to drift out of sync. /// public ImmutableDictionary PendingUserMessages { get; init; } = ImmutableDictionary.Empty; - /// Agent name for pending execution. - public string? PendingAgentName { get; init; } - - /// Model name for pending execution. - public string? PendingModelName { get; init; } - - /// Context path for pending execution. - public string? PendingContextPath { get; init; } - - /// Attachments for pending execution. - public IReadOnlyList? PendingAttachments { get; init; } - [JsonIgnore] public string? StreamingText { get; init; } [JsonIgnore] public ImmutableList? StreamingToolCalls { get; init; } + /// + /// Brings the thread to REST — the single canonical reset of transient execution state. Sets + /// Status = Idle and clears the active-round handle, streaming buffers, and the + /// control-plane request, while PRESERVING the conversation + /// (Messages, UserMessageIds, IngestedMessageIds) and the inbox queue + /// (PendingUserMessages — a fresh round drains whatever is still queued, each entry + /// carrying its own agent/model/harness/context/attachments selection). + /// Call it at EVERY terminal point — round Completed/Cancelled/Error and inbox drain — so the + /// thread can never linger in a stale Executing/StartingExecution state. A stale state + /// is what lets the submission watcher try to RESUME a dead round, which re-blocks and wedges the + /// hub (the recurring chat-start wedge). Compose with with { Summary = … } at the call site + /// when a terminal summary is also being written. + /// + public Thread ResetExecution() => this with + { + Status = ThreadExecutionStatus.Idle, + ExecutionStatus = null, + RequestedStatus = null, + ActiveMessageId = null, + ExecutionStartedAt = null, + StreamingText = null, + StreamingToolCalls = null, + // Preserved: Messages / UserMessageIds / IngestedMessageIds (the conversation) and + // PendingUserMessages (the inbox queue — the next round drains anything still pending, + // each entry carrying its own agent/model/harness/context/attachments selection). + }; } /// @@ -217,6 +419,17 @@ public record ThreadMessage /// public required string Text { get; init; } + /// + /// Dedicated summary the agent produces at end-of-stream — a tighter + /// one-or-two-sentence distillation of . Written by + /// ExecuteMessageAsync in the same stream.Update cycle as the + /// final + thread-level + /// flip. For sub-threads spawned via + /// delegate_to_agent, this is what the parent's tool-call result + /// resolves to — never the raw verbose . + /// + public string? Summary { get; init; } + /// /// When the message was created. /// @@ -228,6 +441,14 @@ public record ThreadMessage /// public ThreadMessageType Type { get; init; } = ThreadMessageType.ExecutedInput; + /// + /// Lifecycle state of this cell. Default + /// keeps pre-existing persisted cells (loaded without a Status field) treated as + /// stable history. New cells set Status explicitly on creation: + /// user → Submitted, assistant → Streaming. + /// + public ThreadMessageStatus Status { get; init; } = ThreadMessageStatus.Completed; + /// /// The name of the agent that generated this response (for AgentResponse messages). /// @@ -238,6 +459,13 @@ public record ThreadMessage /// public string? ModelName { get; init; } + /// + /// The harness () this round ran under. Stamped onto + /// the assistant cell so the output cell can show the harness alongside + /// time + tokens. + /// + public string? Harness { get; init; } + /// /// The user who created this message. Set from the delivery's AccessContext. /// diff --git a/src/MeshWeaver.AI/ThreadComposer.cs b/src/MeshWeaver.AI/ThreadComposer.cs new file mode 100644 index 000000000..1b0c0cbea --- /dev/null +++ b/src/MeshWeaver.AI/ThreadComposer.cs @@ -0,0 +1,115 @@ +using System.Collections.Immutable; +using System.ComponentModel; +using System.ComponentModel.DataAnnotations; +using MeshWeaver.Domain; +using MeshWeaver.Layout; + +namespace MeshWeaver.AI; + +/// +/// The persisted, 100% data-bound state of a chat composer (the chat input box): +/// draft text + the selected harness / agent / model (as picked node PATHS) + attachments. +/// +/// Two homes, one record: +/// +/// Out of a thread — content of the per-user singleton node at +/// {userHome}/_Thread/ThreadComposer (the "new chat" box). Submitting copies this +/// record onto the created thread (), empties the draft, and +/// stamps so the side panel navigates to the new thread. +/// +/// Inside a thread — embedded on the thread content as +/// (NOT a separate node, so reads can never hit a missing +/// satellite). hub.SubmitComposer drains into +/// and empties the composer in ONE atomic +/// stream.Update. +/// +/// +/// The record renders through the framework Edit macro: the property attributes +/// below decide the controls (message editor + harness/agent/model MeshNodePickers). +/// See . +/// +public record ThreadComposer +{ + /// The in-progress composer text — the message currently being typed. + [Description("Message")] + [UiControl] + public string? MessageContent { get; init; } + + /// Selected harness node path (a nodeType:Harness catalog node). + [Description("Harness")] + [MeshNode("namespace:Harness nodeType:Harness sort:order", + Layout = MeshNodePickerLayout.Thin, Open = MeshNodePickerOpenDirection.Up, DefaultToFirst = true)] + public string? Harness { get; init; } + + /// Selected agent node path (a nodeType:Agent node). The picker lists only + /// conversational agents — utility agents (ThreadNamer/DescriptionWriter/NodeInitializer/…, marked + /// modelTier: utility) are excluded via -content.modelTier:utility; ordered by node + /// Order so the Assistant (Order -1) leads. The content. selector resolves identically on the + /// PG (content->>'modelTier') and in-memory (reflection into Content) query backends. + [Description("Agent")] + [MeshNode("namespace:Agent nodeType:Agent -content.modelTier:utility sort:order", + Layout = MeshNodePickerLayout.Thin, Open = MeshNodePickerOpenDirection.Up, DefaultToFirst = true)] + public string? AgentName { get; init; } + + /// Selected model node path (a nodeType:LanguageModel node). + [Description("Model")] + [MeshNode("namespace:_Provider nodeType:LanguageModel scope:descendants sort:order", + Layout = MeshNodePickerLayout.Thin, Open = MeshNodePickerOpenDirection.Up, DefaultToFirst = true)] + public string? ModelName { get; init; } + + /// Paths attached as @-references / context chips on the next message. + [Editable(false)] + public ImmutableList? Attachments { get; init; } + + /// + /// The navigation context path the next thread should carry. Written by the side panel + /// whenever the navigation context changes (out-of-thread composer only); Send creates the + /// thread under {MainNodeOf(ContextPath)}/_Thread/{speakingId}. + /// + [Editable(false)] + public string? ContextPath { get; init; } + + /// + /// Path of the thread the composer's last Send created — the data-bound "navigate here" + /// signal. Stamped by the Send click in the SAME composer write that empties the draft; + /// the side panel observes the composer node, opens the thread, and clears this field. + /// Exists because the Send click runs on the composer node's server hub, which cannot + /// reach circuit services — navigation flows through data, like everything else. + /// + [Editable(false)] + public string? OpenThreadPath { get; init; } + + /// + /// Value-based equality including by SEQUENCE. The synthesized + /// record equality compares list members by reference, so every deserialized server echo of + /// a non-null attachment list would look "changed" — defeating the echo-dedup guards in the + /// composer binding/auto-save and turning them into a write loop. + /// + public virtual bool Equals(ThreadComposer? other) + { + if (other is null) return false; + if (ReferenceEquals(this, other)) return true; + return MessageContent == other.MessageContent + && Harness == other.Harness + && AgentName == other.AgentName + && ModelName == other.ModelName + && ContextPath == other.ContextPath + && OpenThreadPath == other.OpenThreadPath + && (Attachments ?? []).SequenceEqual(other.Attachments ?? []); + } + + /// + public override int GetHashCode() + { + var hash = new HashCode(); + hash.Add(MessageContent); + hash.Add(Harness); + hash.Add(AgentName); + hash.Add(ModelName); + hash.Add(ContextPath); + hash.Add(OpenThreadPath); + foreach (var attachment in Attachments ?? []) + hash.Add(attachment); + return hash.ToHashCode(); + } +} diff --git a/src/MeshWeaver.AI/ThreadComposerNodeType.cs b/src/MeshWeaver.AI/ThreadComposerNodeType.cs new file mode 100644 index 000000000..d63625825 --- /dev/null +++ b/src/MeshWeaver.AI/ThreadComposerNodeType.cs @@ -0,0 +1,178 @@ +using System.Collections.Generic; +using System.Text.Json; +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI; + +/// +/// Per-user ThreadComposer node — the persistent state of a user's chat input box +/// (draft text + the selected harness / agent / model). A per-user singleton stored at +/// {userHome}/_Thread/ThreadComposer as a satellite node +/// ( = → the partition's +/// threads table, alongside Thread / ThreadMessage). +/// +/// Why under _Thread, and why it must be a registered satellite: the composer +/// is thread-family state, kept in the _Thread partition consistently with the per-node +/// composer (). For the read to find it, ThreadComposer MUST resolve +/// to the SAME table by BOTH its path segment (_Thread → threads) and its nodeType — +/// so it is registered in as a _Thread/threads +/// nodeType. Without that nodeType mapping, the write routed to threads (by path) but the +/// nodeType resolved to mesh_nodes → the single-node read missed the row → routing +/// NotFound → the composer's bound layout-area SynchronizationStream OnErrored and the +/// input box vanished (the 2026-06-10 "ThreadComposer disappears on model-select" bug). +/// +/// Why a singleton, seeded at onboarding: the node is materialized for every user when +/// their User partition is created (, a +/// ), so the chat composer's read always RESOLVES instead of +/// generating a routing NotFound that the GUI re-issues on a loop — the 2026-06-08 event-storm +/// class. The content is the dedicated record (message text + the +/// harness/agent/model comboboxes + attachments) — exactly the fields the out-of-thread composer +/// owns, not the full conversation shape. +/// +/// 🚨 Where the composer lives — the rule: the standalone ThreadComposer node +/// (this singleton, in the user's home under _Thread) is used ONLY when there is NO thread yet +/// — the new-chat composer. The moment a thread exists, the composer is the thread's INLINE +/// object on the thread node ITSELF; the thread always refers to its own +/// embedded composer object, NEVER an outside composer node. / +/// WithComposer read/write whichever inline shape applies (discriminated by NodeType), and the +/// GUI binds DIRECTLY to that inline location — see ThreadComposerView.ComposerContext. +/// +public static class ThreadComposerNodeType +{ + /// NodeType discriminator AND the singleton instance id for the per-user chat-input state node. + public const string NodeType = "ThreadComposer"; + + /// + /// The user partition's hidden Memex-defaults namespace segment (a dotfile), _Memex — a + /// non-satellite namespace for per-user defaults (e.g. ModelProvider). Hidden from search by + /// . NOTE: the ThreadComposer singleton itself does NOT live here — + /// it lives under _Thread (see ); this const is retained for the other + /// per-user defaults that DO use _Memex. + /// + public const string MemexDefaultsNamespace = "_Memex"; + + /// Registers the ThreadComposer type node and the per-user singleton seed handler. + public static TBuilder AddThreadComposerType(this TBuilder builder) where TBuilder : MeshBuilder + { + builder.AddMeshNodes(CreateMeshNode()); + builder.ConfigureServices(services => + { + // Seed {user}/_Thread/ThreadComposer when a User partition is onboarded so the + // composer's read always resolves (no read-before-create NotFound storm). + services.AddSingleton(_ => new ThreadComposerSeedHandler()); + return services; + }); + return builder; + } + + /// The type-definition node for nodeType="ThreadComposer". + public static MeshNode CreateMeshNode() => new(NodeType) + { + Name = "Chat Input", + Icon = "/static/NodeTypeIcons/message.svg", + // Satellite node → the partition's `threads` table (registered in SatelliteTableMapping as a + // `_Thread`/threads nodeType). Instances live under {user}/_Thread/ThreadComposer; both the + // path segment (_Thread) and the nodeType resolve to `threads`, so write and read agree and + // the selection persists. Hidden from the create menu and search so users never hand-create one. + IsSatelliteType = true, + ExcludeFromContext = new HashSet { "search", "create" }, + HubConfiguration = config => config + .AddMeshDataSource(source => source + .WithContentType()) + // The composer is the node's default ("") layout area — see ThreadComposerView. + .AddThreadComposerView() + }; + + /// + /// The per-user default composer path: {user}/_Thread/ThreadComposer — the user's own + /// new-chat composer (no specific node context). Kept under the user's _Thread + /// partition for consistency with the per-node composer (). + /// 🚨 Read this path through a query (empty-on-absent), never a direct + /// GetMeshNodeStream on the exact path — a maybe-absent direct read NotFound-storms + /// the mesh (see feedback_optional_node_query_not_access). It is seeded at onboarding so the + /// per-user one normally exists; per-node ones are created on first use. + /// + public static string PathFor(string user) => $"{user}/{ThreadNodeType.ThreadPartition}/{NodeType}"; + + /// + /// The per-node, per-user composer path: {node}/_Thread/{user}/ThreadComposer — the + /// composer state when a chat is started in the context of a specific node. Owned per + /// (node, user). Structured with {user} as the owning segment under the node's + /// _Thread partition and ThreadComposer as the leaf, so it reads back like a thread + /// cell. Reads MUST go through a query, never a direct exact-path GetMeshNodeStream. + /// + public static string PathForNode(string node, string user) => + $"{node}/{ThreadNodeType.ThreadPartition}/{user}/{NodeType}"; + + /// + /// The composer state carried by , discriminated by + /// : a ThreadComposer node's own content, or a + /// Thread node's embedded (empty composer when the + /// thread predates the embed). Null for any other node or unreadable content + /// (bad-data tolerant via ContentAs — never throws on a degraded JsonElement). + /// + public static ThreadComposer? ComposerOf(MeshNode? node, JsonSerializerOptions options, ILogger? logger = null) + => node?.NodeType switch + { + NodeType => node.ContentAs(options, logger), + ThreadNodeType.NodeType => node.ContentAs(options, logger) is { } thread + ? thread.Composer ?? new ThreadComposer() + : node.Content is null ? new ThreadComposer() : null, + _ => null + }; + + /// + /// Writes back onto in the shape + /// reads it from: whole content for a ThreadComposer node, + /// for a Thread node. An existing node whose content + /// can't be recovered is left alone — NEVER clobbered. + /// + public static MeshNode WithComposer(MeshNode node, ThreadComposer composer, JsonSerializerOptions options, ILogger? logger = null) + { + if (node.NodeType == ThreadNodeType.NodeType) + { + var thread = node.ContentAs(options, logger); + if (node.Content is not null && thread is null) + return node; // unreadable → leave alone + return node with { Content = (thread ?? new Thread()) with { Composer = composer } }; + } + if (node.Content is not null + && node.ContentAs(options, logger) is null) + return node; // unreadable → leave alone + return node with { Content = composer }; + } + + /// + /// Seeds the per-user {user}/_Thread/ThreadComposer singleton when a User partition is + /// created. Returned from so it is persisted directly alongside + /// the user (no hub round-trip, no access-context plumbing) — the onboarding "initialize the + /// default state if it doesn't exist" step that keeps the composer read from ever hitting a + /// routing NotFound. + /// + private sealed class ThreadComposerSeedHandler : INodePostCreationHandler + { + public string NodeType => UserNodeType.NodeType; // "User" + + public IObservable Handle(MeshNode createdNode, string? createdBy) + => System.Reactive.Linq.Observable.Empty(); + + public IEnumerable GetAdditionalNodes(MeshNode createdNode) + { + var userPath = !string.IsNullOrEmpty(createdNode.Path) ? createdNode.Path : createdNode.Id; + if (string.IsNullOrEmpty(userPath)) + yield break; + + yield return new MeshNode(ThreadComposerNodeType.NodeType, $"{userPath}/{ThreadNodeType.ThreadPartition}") + { + NodeType = ThreadComposerNodeType.NodeType, + Name = "Chat Input", + Content = new ThreadComposer(), + }; + } + } +} diff --git a/src/MeshWeaver.AI/ThreadComposerView.cs b/src/MeshWeaver.AI/ThreadComposerView.cs new file mode 100644 index 000000000..e9300425d --- /dev/null +++ b/src/MeshWeaver.AI/ThreadComposerView.cs @@ -0,0 +1,309 @@ +using System.Reactive.Linq; +using System.Reflection; +using MeshWeaver.Data; +using MeshWeaver.Domain; +using MeshWeaver.Graph; +using MeshWeaver.Layout; +using MeshWeaver.Layout.Composition; +using MeshWeaver.Layout.DataBinding; +using MeshWeaver.Layout.Domain; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Messaging; +using MeshWeaver.Utils; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI; + +/// +/// The data-bound chat composer layout areas. Both areas bind the form controls DIRECTLY to the +/// state of THIS node via a node-bound DataContext +/// () — ONE source of truth, the node stream +/// (IMeshNodeStreamCache). No /data replica, no debounced save subscription, no +/// re-seed loop — each field edit writes straight back to the composer's inline location on the node. +/// +/// Where the composer lives (and the rule): the composer is read/written via +/// / WithComposer in ONE of two inline shapes, +/// and the binding always targets that SAME inline location — NEVER a separate node: +/// +/// No thread yet → a standalone ThreadComposer node in the user's +/// home ({user}/_Thread/ThreadComposer): the composer IS the node's whole Content +/// (content-mode binding). +/// Once a thread exists → the composer is the Thread's INLINE +/// object on the thread node itself (fields-mode binding with +/// sub-path content/composer). The thread always refers to its own embedded composer +/// object, never an outside node. +/// +/// Because writes route through the owning hub's serialised action block, concurrent fields the +/// composer carries ( / , +/// set by the side panel) are never clobbered by a field edit. +/// +public static class ThreadComposerView +{ + /// The composer area name — registered as the composer node's default area. + public const string ComposerArea = "Composer"; + + /// + /// The selectors-only area: just the harness/agent/model [MeshNode] pickers, data-bound + /// to THIS node's composer state and auto-persisting. The Blazor chat (ThreadChatView) + /// embeds this so its harness/agent/model selection is 100% data-bound (no hand-rolled + /// dropdowns) while keeping its own Monaco editor + attachments. Registered on BOTH the + /// composer node type and thread hubs (binding ). + /// + public const string SelectorsArea = "Selectors"; + + private const string LogCategory = "MeshWeaver.AI.ThreadComposerView"; + + /// Adds the data-bound composer + selectors views; is the default area. + public static MessageHubConfiguration AddThreadComposerView(this MessageHubConfiguration configuration) + => configuration.AddLayout(layout => layout + .WithDefaultArea(ComposerArea) + .WithView(ComposerArea, Composer) + .WithView(SelectorsArea, ComposerSelectors)); + + /// The selector properties shown in , in display order. + private static readonly string[] SelectorPropertyNames = + [nameof(ThreadComposer.Harness), nameof(ThreadComposer.AgentName), nameof(ThreadComposer.ModelName)]; + + /// + /// Renders ONLY the harness picker (no label — just the combobox), bound DIRECTLY to THIS node's + /// composer state () so the pick persists straight to the node. + /// Agent + model are NOT shown here: the chat footer stays compact (harness + context + Send on + /// one row), and agent/model are chosen via the /agent and /model slash-commands + /// (which write the same composer). The control is built ONCE from the first composer emission; + /// all later updates flow through the node-bound data binding. + /// + public static UiControl ComposerSelectors(LayoutAreaHost host, RenderingContext context) + => Controls.Stack.WithWidth("100%") + .WithView((h, _) => h.Workspace.GetMeshNodeStream() + .Where(n => ThreadComposerNodeType.ComposerOf(n, h.Hub.JsonSerializerOptions, Logger(h)) is not null) + .Take(1) + // All THREE framework MeshNodePickerControls (harness · agent · model) — search, + // keyboard nav, icons, default-to-first for free. Replaces the hand-rolled command + // picker widget (a regression): a node-pick command now just surfaces the same nice + // control the composer already uses. Each is data-bound to its [MeshNode] property + // and auto-persists to the composer node. + .Select(node => (UiControl?)BuildSelectorRow( + h, EditLayoutArea.GetDataId(h.Hub.Address.ToString()), ComposerContext(h, node)))); + + /// + /// The composer node's default area: data-bound message editor + selector row + Send. + /// Send submits via the canonical , copying the + /// composer onto the created thread, emptying the draft, and stamping + /// so the side panel navigates — all data-bound, + /// no circuit access from this server-side hub. + /// + public static UiControl Composer(LayoutAreaHost host, RenderingContext context) + => Controls.Stack.WithWidth("100%").WithStyle("gap: 8px;") + .WithView((h, _) => h.Workspace.GetMeshNodeStream() + .Where(n => ThreadComposerNodeType.ComposerOf(n, h.Hub.JsonSerializerOptions, Logger(h)) is not null) + .Take(1) + .Select(node => + { + var ctx = ComposerContext(h, node); + var dataId = EditLayoutArea.GetDataId(h.Hub.Address.ToString()); + + var messageProp = typeof(ThreadComposer).GetProperty(nameof(ThreadComposer.MessageContent))!; + var editor = Controls.Stack.WithWidth("100%") + .WithView(h.Hub.ServiceProvider.MapToToggleableControl( + messageProp, dataId, canEdit: true, h, isToggleable: false, boundDataContext: ctx)); + + // No agent/model/harness picker row: those comboboxes eagerly resolved the + // picked agent and threw when its model was not wired, tearing the composer down + // ("choosing Assistant crashes"). Selection is via the /agent /model /harness + // slash-commands, which write the composer node without building the agent. + var bottomRow = Controls.Stack + .WithOrientation(Orientation.Horizontal) + .WithStyle("gap: 8px; flex-wrap: nowrap; align-items: flex-end; width: 100%;") + .WithView(Controls.Stack.WithStyle("flex: 0 0 auto; margin-left: auto;") + .WithView(BuildSendButton())); + + return (UiControl?)Controls.Stack.WithWidth("100%").WithStyle("gap: 8px;") + .WithView(editor) + .WithView(bottomRow); + })); + + private static ILogger? Logger(LayoutAreaHost host) + => host.Hub.ServiceProvider.GetService()?.CreateLogger(LogCategory); + + /// + /// The node-bound DataContext the composer form binds against — ONE source of truth, the node + /// stream (IMeshNodeStreamCache). The composer lives in one of TWO shapes + /// (), and this resolves the binding root to the + /// SAME inline location each shape reads from — it NEVER references a separate node: + /// + /// standalone ThreadComposer node (the user-home new-chat + /// composer, used when no thread exists yet): the composer IS the node's whole + /// Content → content-mode, no sub-path. Field pointers (messageContent, + /// harness, …) resolve against the Content. + /// Thread node (once a thread exists): the composer is the + /// thread's INLINE object — fields-mode with sub-path + /// content/composer, so a harness pointer resolves to + /// content/composer/harness on the thread node ITSELF. The thread always binds its own + /// embedded composer object, never an external composer node. + /// + /// Each field edit writes straight back to that location on the node stream — no /data + /// replica, no debounced save subscription. The owning hub serialises writes, so concurrent + /// fields (ContextPath / OpenThreadPath set by the side panel) are never clobbered. + /// + private static string ComposerContext(LayoutAreaHost host, MeshNode? node) + { + var nodePath = host.Hub.Address.ToString(); + return node?.NodeType == ThreadNodeType.NodeType + ? LayoutAreaReference.GetMeshNodeDataContext(nodePath, bindContent: false, subPath: $"{nameof(MeshNode.Content).ToCamelCase()}/{nameof(Thread.Composer).ToCamelCase()}") + : LayoutAreaReference.GetMeshNodeDataContext(nodePath, bindContent: true); + } + + /// + /// Horizontal row of the three selector pickers sized to share width (flex: 1 1 0), kept on + /// ONE line (nowrap) and bottom-aligned so the chat footer fits pickers + chips + Send on a + /// single bottom row. Each picker shrinks rather than wrapping; min-width keeps them usable. + /// + private static UiControl BuildSelectorRow(LayoutAreaHost host, string dataId, string boundContext) + { + var row = Controls.Stack + .WithOrientation(Orientation.Horizontal) + .WithStyle("gap: 6px; flex-wrap: nowrap; align-items: flex-end; width: 100%;"); + // 🚨 ALL three pickers are the generic attribute-driven MeshNodePickerControl, whose query + // runs SERVER-SIDE on THIS layout hub. The per-partition agent registry query + // (namespace:{user}/Agent|{space}/Agent|Agent) must NEVER be issued here: this hub is not the + // chatting user, so per-user RLS strips the {user}/{space} namespaces (empty dropdown) and the + // cross-partition subscribe storms the portal (the 2026-06-17 atioz wedge). Per-partition agent + // selection is driven from the GUI instead — the /agent slash command (ThreadChatView.OpenPicker + // → hub.GetQuery) runs in the user's Blazor circuit under the user's identity. + foreach (var prop in SelectorProperties()) + row = row.WithView( + Controls.Stack.WithStyle("flex: 1 1 0; min-width: 70px; max-width: 220px;") + .WithView(host.Hub.ServiceProvider.MapToToggleableControl( + prop, dataId, canEdit: true, host, isToggleable: false, boundDataContext: boundContext))); + return row; + } + + private static IEnumerable SelectorProperties() => + SelectorPropertyNames + .Select(typeof(ThreadComposer).GetProperty) + .Where(p => p is not null)!; + + /// + /// Builds JUST the harness — no label, just the combobox — + /// data-bound to the composer's field via the node-bound + /// . Constructed directly from the property's [MeshNode] + /// attribute (the same query/layout/open/default the standard editor would build), bypassing + /// MapToToggleableControl so no "Harness" label row is rendered. The picker mutates the + /// harness pointer, which writes straight back to the composer on the node stream. + /// + private static UiControl BuildHarnessPicker(LayoutAreaHost host, string boundContext) + { + var harnessProp = typeof(ThreadComposer).GetProperty(nameof(ThreadComposer.Harness))!; + var meshNodeAttr = harnessProp.GetCustomAttribute()!; + var nodeNamespace = host.Hub.Address.ToString(); + var picker = new MeshNodePickerControl( + new JsonPointerReference(nameof(ThreadComposer.Harness).ToCamelCase()!)) + { + Queries = MeshNodeAttribute.ResolveQueries(meshNodeAttr.Queries, nodeNamespace, nodeNamespace), + Layout = meshNodeAttr.Layout, + Open = meshNodeAttr.Open, + DefaultToFirst = meshNodeAttr.DefaultToFirst, + DataContext = boundContext + }; + return Controls.Stack + .WithStyle("min-width: 90px; max-width: 220px;") + .WithView(picker); + } + + /// + /// Send button — one-shot read of the composer straight off the node stream (ONE source of + /// truth), then the canonical . Identity is + /// resolved at CLICK time from the click delivery's (hub/system + /// principals filtered) — never captured at render time, where the ambient context can be the + /// hub itself. + /// + private static UiControl BuildSendButton() + => Controls.Button("Send") + .WithAppearance(Appearance.Accent) + .WithClickAction(ctx => + { + var host = ctx.Host; + var logger = Logger(host); + var user = ResolveUser(host.Hub.ServiceProvider.GetService()); + host.Workspace.GetMeshNodeStream() + .Select(n => ThreadComposerNodeType.ComposerOf(n, host.Hub.JsonSerializerOptions, logger)) + .Where(c => c is not null) + .Take(1) + .Subscribe( + edited => Send(host, edited, user, logger), + ex => logger?.LogWarning(ex, "[ThreadComposer] Send: composer read failed")); + return Task.CompletedTask; + }); + + /// + /// The submit pipeline: thread under {MainNodeOf(ContextPath) ?? user}/_Thread/…, + /// composer copied onto the thread, then ONE composer-node write that empties the draft and + /// stamps = the created thread's path (the side + /// panel observes the composer node, opens the thread, and clears the signal). + /// + private static void Send(LayoutAreaHost host, ThreadComposer? edited, string? user, ILogger? logger) + { + if (string.IsNullOrWhiteSpace(edited?.MessageContent)) + return; + + var contextPath = string.IsNullOrEmpty(edited!.ContextPath) ? null : edited.ContextPath; + var ns = contextPath is null ? user : ThreadNodeType.MainNodeOf(contextPath); + if (string.IsNullOrEmpty(ns)) + { + logger?.LogWarning( + "[ThreadComposer] Send ignored — no resolvable namespace (no user identity and no context)"); + return; + } + + host.Hub.StartThread( + namespacePath: ns!, + userText: edited.MessageContent!, + agentName: edited.AgentName, + modelName: edited.ModelName, + harness: edited.Harness, + contextPath: contextPath ?? ns, + attachments: edited.Attachments, + createdBy: user, + composer: edited, + onCreated: node => host.Workspace.GetMeshNodeStream() + .Update(n => + { + var c = ThreadComposerNodeType.ComposerOf(n, host.Hub.JsonSerializerOptions, logger); + if (n.Content is not null && c is null) + return n; // unreadable → leave alone, never clobber + return ThreadComposerNodeType.WithComposer( + n, + (c ?? new ThreadComposer()) with + { + MessageContent = null, + Attachments = null, + OpenThreadPath = node.Path + }, + host.Hub.JsonSerializerOptions, logger); + }) + .Subscribe( + _ => { }, + ex => logger?.LogWarning(ex, + "[ThreadComposer] post-send clear/navigate-stamp failed for {Thread}", node.Path)), + onError: err => logger?.LogWarning("[ThreadComposer] StartThread failed: {Error}", err)); + } + + /// + /// The submitting user's identity, filtered the same way as every other resolver + /// (system-security and hub principals are NOT users). AsyncLocal context first — + /// at click time it carries the click delivery's . + /// + private static string? ResolveUser(AccessService? access) + { + if (access is null) return null; + foreach (var candidate in new[] { access.Context?.ObjectId, access.CircuitContext?.ObjectId }) + { + if (!string.IsNullOrEmpty(candidate) + && candidate != WellKnownUsers.System + && !AccessService.LooksLikeHubPrincipal(candidate)) + return candidate; + } + return null; + } +} diff --git a/src/MeshWeaver.AI/ThreadExecution.cs b/src/MeshWeaver.AI/ThreadExecution.cs index b27ff26e8..5a8480af9 100644 --- a/src/MeshWeaver.AI/ThreadExecution.cs +++ b/src/MeshWeaver.AI/ThreadExecution.cs @@ -1,6 +1,8 @@ -using System.Collections.Concurrent; +using System.Collections.Concurrent; using System.Collections.Immutable; using System.Reactive.Linq; +using System.Reactive.Subjects; +using System.Reactive.Threading.Tasks; using System.Text; using System.Text.Json; using MeshWeaver.AI.Plugins; @@ -9,6 +11,7 @@ using MeshWeaver.Layout; using MeshWeaver.Mesh; using MeshWeaver.Mesh.Services; +using MeshWeaver.Mesh.Threading; using MeshWeaver.Messaging; using Microsoft.Extensions.AI; using Microsoft.Extensions.DependencyInjection; @@ -21,410 +24,632 @@ namespace MeshWeaver.AI; /// Handlers for thread message execution: submit, stream, cancel. /// Registered on the Thread hub via ThreadNodeType. /// -public static class ThreadExecution +internal static class ThreadExecution { + // Cell-stream patches are not free — at this cap a 30s response generates + // ~300 patches instead of token-rate. 100ms is below the perceptual threshold + // for "live typing" while keeping patch volume bounded. + private static readonly TimeSpan StreamingSampleInterval = TimeSpan.FromMilliseconds(100); + + private sealed record StreamingSnapshot( + string Text, + ImmutableList ToolCalls, + ImmutableList NodeChanges); + /// - /// Registers thread execution handlers on a hub configuration. - /// Includes a startup recovery check for stale executing cells from crashed sessions. + /// Single canonical write path for ThreadMessage cells: opens a short-lived + /// remote synchronization stream, applies to the cell's + /// content, and disposes. Constructs a placeholder MeshNode when the sync + /// handshake hasn't delivered the initial state — the patch routes via StreamId + /// regardless of local cache freshness, so the cell hub applies it correctly. + /// + /// Use this for one-off cell updates from outside the streaming loop + /// (recovery, "Allocating agent…" placeholders). Writes via + /// IMeshNodeStreamCache.Update — the same shared handle the + /// GUI subscribers read from, so the patch is observed in order. /// + internal static void UpdateResponseCell( + IMessageHub hub, + string responsePath, + string threadPath, + string responseMsgId, + string mainEntity, + Func mutate, + ILogger? logger) + { + hub.GetMeshNodeStream(responsePath).Update(node => + { + var current = node.ContentAs(hub.JsonSerializerOptions, logger); + // Existing node whose content can't be recovered → leave it alone, NEVER clobber. + if (node?.Content is not null && current is null) + return node; + current ??= new ThreadMessage + { + Role = "assistant", + Text = "", + Type = ThreadMessageType.AgentResponse, + Status = ThreadMessageStatus.Streaming + }; + var updated = mutate(current); + if (updated.Status == ThreadMessageStatus.Streaming + && updated.Text.Length < current.Text.Length) + updated = updated with { Text = current.Text }; + return node != null + ? node with { Content = updated } + : new MeshNode(responseMsgId, threadPath) + { + NodeType = ThreadMessageNodeType.NodeType, + MainNode = mainEntity, + Content = updated + }; + }).Subscribe( + _ => { }, + ex => logger?.LogWarning(ex, + "[UpdateResponseCell] cache.Update failed for {Path}", responsePath)); + } + /// - /// Stores completion callbacks keyed by thread path. - /// Used to route ExecutionCompleted responses from the _Exec hub - /// back to the original client delivery on the thread hub. - /// Safe: thread hub + _Exec hub always run on the same grain/process. + /// Registers thread execution handlers on a hub configuration. + /// Includes a startup recovery check for stale executing cells from crashed sessions. /// - private static readonly ConcurrentDictionary> CompletionCallbacks = new(); - private static readonly ConcurrentDictionary ExecutionCancellations = new(); - private static readonly ConcurrentDictionary AgentCache = new(); - public static MessageHubConfiguration AddThreadExecution(this MessageHubConfiguration configuration) + internal static MessageHubConfiguration AddThreadExecution(this MessageHubConfiguration configuration) => configuration - .WithHandler(HandleSubmitMessage) - .WithHandler(ThreadSubmission.HandleAppendUserMessage) - .WithHandler(ThreadSubmission.HandleResubmitUserMessage) - .WithHandler(ThreadSubmission.HandleRecordSubmissionFailure) - .WithHandler(HandleCancelStream) + // No verb-shaped triggers — every thread state mutation rides + // workspace.GetMeshNodeStream().Update(...) on a control-plane + // field, observed by an owning-hub watcher. See + // RequestViaStreamUpdate.md and ActivityControlPlane.md. + // + // Round dispatch (Idle → StartingExecution): InstallSubmissionWatcher + // claims directly on the thread hub's action block. The _Exec + // hosted hub (InstallExecutionHub) subscribes to the parent's + // stream and continues with Step B + C on its own action block. + // + // Resubmit / delete-from / submission-failure are now done inline + // by HubThreadExtensions — single stream.Update per operation, no + // intent fields, no per-operation watchers. + .WithHandler( + MeshWeaver.AI.Delegation.DelegationHandlers.HandleHeartbeatTick) + .WithHandler( + MeshWeaver.AI.Delegation.DelegationHandlers.HandleCancelDelegationSubThread) .WithInitialization(SetThreadHubIdentity) - .WithInitialization(RecoverStaleExecutingThread) - .WithInitialization(WatchForExecution) - .WithInitialization(InstallSubmissionWatcher); + .WithInitialization(InitializeThreadLifecycle) + .WithInitialization(InstallCancellationWatcher) + .WithInitialization(InstallExecutionHub) + .WithInitialization(InstallSubmissionWatcher) + .WithInitialization(InstallHeartbeatTicker); /// - /// Installs the continuous server-side watcher that ingests queued user messages - /// into new rounds and dispatches agent execution. See . + /// Eagerly creates the _Exec hosted hub at thread hub init time and + /// installs its round watcher. The watcher subscribes to the parent thread + /// node's stream via the shared ; on the + /// first emission per claim with Status == StartingExecution + /// ( + /// on ExecutionStartedAt), invokes + /// to drain pending + /// into Messages, allocate the response cell, transition to + /// Executing, and start agent streaming. + /// + /// Eager creation matters: the parent thread hub flips + /// Status → StartingExecution as soon as the submission watcher + /// claims; if _Exec isn't running yet, the resulting transition + /// emission has no subscriber and the round stalls. /// - private static Task InstallSubmissionWatcher(IMessageHub hub, CancellationToken ct) + private static void InstallExecutionHub(IMessageHub threadHub) { - var sub = ThreadSubmission.InstallServerWatcher(hub); - // Dispose with the hub lifetime. - hub.RegisterForDisposal(sub); - return Task.CompletedTask; + threadHub.GetHostedHub( + new Address($"{threadHub.Address}/_Exec"), + config => config.WithInitialization(InstallExecRoundWatcher), + HostedHubCreation.Always); } /// - /// Cancels the active execution on — used by the explicit - /// user "Stop" button. Do NOT call this automatically when queued user messages arrive - /// during execution: the Anthropic Messages API does not support mid-stream injection, - /// and cancelling during a tool_use produces orphaned tool_use blocks that require a - /// synthetic error tool_result to recover. The correct pattern for queued input is - /// "wait for the round to complete, then dispatch a fresh round with all queued - /// messages in history". See ThreadSubmissionServer.InstallServerWatcher. - /// Idempotent — repeated calls during the same round are no-ops. + /// _Exec hosted hub's round watcher. Subscribes to the parent thread node's + /// stream via the shared and fires + /// for each + /// Idle → StartingExecution transition. DistinctUntilChanged + /// on (project first, then dedupe, then + /// filter) ensures the watcher fires ONLY on the transition itself — + /// other field updates (PendingUserMessages, Messages, etc.) that + /// arrive while Status remains StartingExecution don't re-trigger dispatch. + /// + /// Sets the round's from + /// before dispatching so downstream + /// CreateNodeRequest calls (user cells, response cell) carry the + /// right user identity. /// - internal static void RequestSafeCancellation(string threadPath) + private static void InstallExecRoundWatcher(IMessageHub execHub) { - if (ExecutionCancellations.TryGetValue(threadPath, out var cts) && !cts.IsCancellationRequested) - { - try { cts.Cancel(); } catch { /* already disposed */ } - } + var parentHub = execHub.Configuration.ParentHub + ?? throw new InvalidOperationException( + "_Exec hosted hub has no ParentHub — cannot resolve thread path"); + var logger = execHub.ServiceProvider.GetService>(); + var threadPath = parentHub.Address.Path; + + var accessService = execHub.ServiceProvider.GetService(); + + // Self-healing: this watcher dispatches each StartingExecution round. If its + // stream FAULTS it must not die silently — a dead watcher means a claimed + // round never dispatches (Status stuck at StartingExecution, IsExecuting + // forever): the live-path "observer dies" deadlock. On fault, re-establish. + IDisposable? sub = null; + var disposed = false; + // 🚨 Observe the PARENT thread hub's AUTHORITATIVE own MeshNode stream — + // NOT the cross-hub IMeshNodeStreamCache. The cache opens a remote + // subscription to the owning grain and, on Orleans, replays/reorders: + // it interleaves a STALE claim snapshot (StartingExecution, empty + // PendingUserMessages) AFTER the committed Executing state. Because + // DispatchAfterClaim plans the round from the emitted node, a stale + // Pending=0 snapshot makes PlanNextRound return null → it ROLLS the + // claim back to Idle, racing and reverting the in-flight commit. The + // SubmissionWatcher re-claims, the cache replays stale again, and the + // round live-locks (Resubmit_AfterExecution_DoesNotDeadlock hang). + // parentHub.GetWorkspace().GetMeshNodeStream() is the same in-order, + // typed-Content own stream the SubmissionWatcher reads, so the claim → + // Executing transition is observed exactly once, in order, with the + // real Pending — DistinctUntilChanged(Status) then fires the dispatch a + // single time and never sees a phantom re-claim. (Using the THREAD + // hub's workspace here, never the _Exec child's, also matches + // feedback_synced_query_thread_hub.md.) + void Establish() => sub = parentHub.GetWorkspace().GetMeshNodeStream() + // Pair each emission with its current Status so DistinctUntilChanged + // dedupes on the Status field only — concurrent field updates that + // happen while Status stays StartingExecution must NOT re-fire. + .Select(n => new { Node = n, Status = (n?.Content as MeshThread)?.Status }) + .DistinctUntilChanged(x => x.Status) + .Where(x => x.Status == ThreadExecutionStatus.StartingExecution) + .Select(x => x.Node) + .Subscribe( + node => + { + if (node?.Content is not MeshThread thread) + { + logger?.LogWarning( + "[ExecRoundWatcher] thread node has no MeshThread content for {ThreadPath}", + threadPath); + return; + } + + // 🚨 Thread execution ALWAYS runs under the thread owner's + // identity. The cache stream's emission scheduler doesn't + // carry the originating user's AsyncLocal — without this + // scope, every read/write inside DispatchAfterClaim + // (drain pending, allocate response cell, stream LLM + // output) would be stamped with the cache identity, and + // the owning hub's RLS would deny. The access check that + // gated the dispatch already happened (the user with no + // access to the thread couldn't have flipped Status to + // StartingExecution). + using (MeshWeaver.Mesh.Security.AccessContextScope.FromNode(node, accessService, logger)) + { + logger?.LogDebug( + "[ExecRoundWatcher] access context set: {User} for {ThreadPath}", + thread.CreatedBy ?? "(system fallback)", threadPath); + + ThreadSubmissionServer.DispatchAfterClaim(parentHub, node, logger, + onFailure: () => + { + logger?.LogWarning( + "[ExecRoundWatcher] DispatchAfterClaim failed for {ThreadPath} — rolling Status back to Idle", + threadPath); + parentHub.GetWorkspace().GetMeshNodeStream().Update(n => + { + var t = n.ContentAs(parentHub.JsonSerializerOptions, logger); + // Existing node whose content can't be recovered → leave it alone, NEVER clobber. + if (n.Content is not null && t is null) + return n; + t ??= new MeshThread(); + return t.Status == ThreadExecutionStatus.StartingExecution + ? n with { Content = t with { Status = ThreadExecutionStatus.Idle, ExecutionStartedAt = null } } + : n; + }).Subscribe( + _ => { }, + ex => logger?.LogWarning(ex, + "[ExecRoundWatcher] rollback Update failed for {ThreadPath}", threadPath)); + }); + } + }, + ex => + { + logger?.LogWarning(ex, + "[ExecRoundWatcher] stream errored for {ThreadPath} — re-establishing", threadPath); + if (!disposed) + System.Reactive.Linq.Observable.Timer(TimeSpan.FromSeconds(1)) + .Subscribe(_ => Establish()); + }); + + Establish(); + execHub.RegisterForDisposal(_ => { disposed = true; sub?.Dispose(); }); } /// - /// Sets the thread hub's access context to the thread creator's identity. - /// Without this, the hub's default identity is its own address path, - /// causing "Access denied" when reading child message nodes. + /// Installs the periodic + /// emitter on the PARENT THREAD hub. Every HeartbeatInterval posts a + /// tick to self; + /// walks chat.ActiveDelegationPaths (cached on this hub via + /// parentHub.Set<AgentChatClient>), reads each sub-thread's + /// MeshNode via the process-wide cache, and posts + /// for + /// any sub-thread whose is older + /// than its . Replaces the hard + /// 5-min watchdog inside ExecuteDelegationAsync. + /// + /// The tick handler short-circuits when ActiveDelegationPaths + /// is empty — negligible cost when no delegations are in flight. /// - private static Task SetThreadHubIdentity(IMessageHub hub, CancellationToken ct) + private static void InstallHeartbeatTicker(IMessageHub threadHub) { - hub.GetWorkspace().GetStream(new MeshNodeReference())?.Take(1).Subscribe(node => - { - if (node.Value?.Content is MeshThread { CreatedBy: { Length: > 0 } createdBy }) - { - var accessService = hub.ServiceProvider.GetService(); - accessService?.SetContext(new AccessContext { ObjectId = createdBy, Name = createdBy }); - } - }); - return Task.CompletedTask; + var sub = System.Reactive.Linq.Observable + .Interval(MeshWeaver.AI.Delegation.DelegationHandlers.HeartbeatInterval) + .Subscribe(_ => threadHub.Post(new MeshWeaver.AI.Delegation.HeartbeatTick())); + threadHub.RegisterForDisposal(_ => sub.Dispose()); } /// - /// On hub startup, check if this Thread was left in IsExecuting=true state (crashed/restarted). - /// If stale: mark the active response message as "*Cancelled*", clear execution state, - /// and mark all ActiveProgress entries as completed. Fully non-blocking — no await. - /// Each child thread's own hub recovery handles its own cancellation recursively. + /// Installs the continuous server-side watcher that ingests queued user messages + /// into new rounds and dispatches agent execution. See . /// - private static Task RecoverStaleExecutingThread(IMessageHub hub, CancellationToken ct) + private static void InstallSubmissionWatcher(IMessageHub hub) { - var logger = hub.ServiceProvider.GetService>(); - var workspace = hub.GetWorkspace(); - var threadPath = hub.Address.Path; + var sub = ThreadSubmission.InstallServerWatcher(hub); + // Dispose with the hub lifetime. + hub.RegisterForDisposal(sub); + } - // Read the thread node from the workspace stream (already loaded on hub init) - workspace.GetStream()?.Take(1).Subscribe(nodes => - { - var threadNode = nodes?.FirstOrDefault(n => n.Path == threadPath); - if (threadNode?.Content is not Thread { IsExecuting: true } thread) - return; - // Don't recover fresh executions — WatchForExecution handles them. - // Only recover truly stale ones (started > 2 minutes ago or no timestamp). - if (thread.ExecutionStartedAt is { } startedAt && - (DateTime.UtcNow - startedAt).TotalMinutes < 2) - { - logger?.LogInformation("[ThreadExec] Recovery: skipping fresh execution on {ThreadPath} (started {StartedAt})", threadPath, startedAt); + /// + /// Sets the thread hub's access context to the thread creator's identity. + /// Without this, the hub's default identity is its own address path, + /// causing "Access denied" when reading child message nodes. + /// + /// Resolution order: + /// 1. (set by callers that explicitly stamp it) + /// 2. (filled by the CreateNodeRequest handler + /// from the requester's AccessContext) — covers the common case where the + /// caller didn't pass createdBy to BuildThreadNode and the + /// thread content's CreatedBy is null. + /// + private static void SetThreadHubIdentity(IMessageHub hub) + { + // One-shot read of the OWN thread node via GetDataRequest (posted to self) — + // true request/response, no SubscribeRequest+immediate-unsubscribe. + hub.GetMeshNode(hub.Address.ToString()).Subscribe(node => + { + if (node is null) return; + var createdBy = (node.Content as MeshThread)?.CreatedBy; + if (string.IsNullOrEmpty(createdBy)) + createdBy = node.CreatedBy; + if (string.IsNullOrEmpty(createdBy)) return; - } - - logger?.LogInformation("[ThreadExec] Recovery: stale execution on {ThreadPath}, activeMsg={ActiveMsg}", - threadPath, thread.ActiveMessageId); - - // Cancel pending tool calls on the active response message. - // For delegation tool calls, check if the sub-thread actually completed. - if (!string.IsNullOrEmpty(thread.ActiveMessageId)) - { - var responsePath = $"{threadPath}/{thread.ActiveMessageId}"; - - // Mark all pending tool calls as cancelled — no query needed. - // Sub-thread recovery happens independently on their own hub init. - var updatedToolCalls = thread.StreamingToolCalls? - .Select(tc => tc.Result != null - ? tc - : tc with { Result = "Cancelled (server restarted)", IsSuccess = false }) - .ToImmutableList(); - - hub.Post(new UpdateThreadMessageContent - { - Text = "*Cancelled (server restarted)*", - ToolCalls = updatedToolCalls - }, o => o.WithTarget(new Address(responsePath))); - } - - // Clear thread execution state - workspace.UpdateMeshNode(node => - { - var t = node.Content as Thread ?? new Thread(); - var cancelledAt = DateTime.UtcNow; - return node with - { - LastModified = cancelledAt, - Content = t with - { - IsExecuting = false, - ExecutionStatus = null, - ActiveMessageId = null, - TokensUsed = 0, - ExecutionStartedAt = null, - StreamingText = null, - StreamingToolCalls = null - } - }; - }); - - logger?.LogInformation("[ThreadExec] Recovery: cleared stale execution on {ThreadPath}", threadPath); + var accessService = hub.ServiceProvider.GetService(); + var owner = new AccessContext { ObjectId = createdBy, Name = createdBy }; + // 🚨 OWNER-INJECTION: stamp the thread OWNER as BOTH the live Context AND the + // CircuitContext. CircuitContext is the one that CARRIES FORWARD across Rx hops + // (the deferred sync-write continuations where the AsyncLocal Context is wiped) — + // SetContext alone was lost on the hop, so the owner-side data-source sync write + // posted UpdateStreamRequest with a NULL AccessContext and the never-null guard + // failed it closed (the cold-start submit deadlock: pending never landed, the + // watcher saw pending=0 forever, no round dispatched). The thread owner is the + // standing identity for EVERY operation on this thread hub. See OwnerInjection.md. + accessService?.SetContext(owner); + accessService?.SetCircuitContext(owner); }); - - return Task.CompletedTask; } /// - /// On hub startup, check if this Thread has a PendingUserMessage. - /// If so, create message cells and start execution automatically. - /// This enables thread creation + execution in a single CreateNodeRequest. - /// - /// - /// Watches the workspace stream for IsExecuting=true with ActiveMessageId. - /// When detected, starts execution on the _Exec hosted hub. - /// This is the ONLY trigger for execution — state-driven, not command-driven. - /// GUI sets IsExecuting=true via SubmitMessageRequest → execution starts automatically. - /// - /// - /// Watches for auto-execute threads (created with BuildThreadWithMessages). - /// These threads have PendingUserMessage set at creation time (not via HandleSubmitMessage). - /// Creates message cells and starts execution on hub startup. - /// HandleSubmitMessage handles all client-initiated execution directly. + /// Clean wake-up state machine. On hub activation, read the OWN node's FIRST + /// stream emission (the loaded persisted state, correctly ordered on this + /// hub's action block vs any subsequent writes) and drive any non-terminal + /// state to a valid one exactly once. This replaces the old late + /// GetMeshNode round-trip whose response could land AFTER later writes + /// and clobber Status → Idle (the check_inbox phantom-drain + /// flake). + /// + /// Branches (after honoring any pending cancel first): + /// + /// Executing (with a response cell) → resume the same + /// cell by re-launching the streaming loop directly + /// () while + /// Status STAYS Executing. 🚨 Never re-enter + /// StartingExecution from Executing — that inverse of the + /// commit edge is the re-dispatch ping-pong. + /// Executing without a response cell → nothing to resume; reset + /// to Idle so the submission watcher can claim pending input. + /// StartingExecution → no write; the _Exec round watcher + /// fires on its own first emission. + /// Idle / Cancelled (+ pending) → no write; the + /// submission watcher claims. + /// Done → terminal; leave it. + /// + /// Each child thread's own hub runs the same recovery recursively. /// - private static Task WatchForExecution(IMessageHub hub, CancellationToken ct) + private static void InitializeThreadLifecycle(IMessageHub hub) { var logger = hub.ServiceProvider.GetService>(); + var cache = hub.ServiceProvider.GetRequiredService(); var workspace = hub.GetWorkspace(); + var accessService = hub.ServiceProvider.GetService(); var threadPath = hub.Address.Path; - // Only check on startup (Take(1)) — HandleSubmitMessage handles runtime execution. - workspace.GetStream(new MeshNodeReference())?.Take(1).Subscribe(node => - { - if (node.Value?.Content is not MeshThread { PendingUserMessage: not null } thread) - return; - - // Only auto-execute threads created with BuildThreadWithMessages - if (!thread.IsExecuting || thread.ActiveMessageId == null) - return; - - var responseMsgId = thread.ActiveMessageId; - var responsePath = $"{threadPath}/{responseMsgId}"; - var activeIdx = thread.Messages.IndexOf(responseMsgId); - var userMsgId = activeIdx > 0 ? thread.Messages[activeIdx - 1] : null; - // MainNode for child cells = the thread's own MainNode (content node). - var mainEntity = node.Value?.MainNode ?? thread.PendingContextPath ?? threadPath; - - logger?.LogInformation("[ThreadExec] Auto-execute: {ThreadPath}, activeMsg={ActiveMsg}", - threadPath, responseMsgId); - - var accessService = hub.ServiceProvider.GetService(); - if (!string.IsNullOrEmpty(thread.CreatedBy)) - accessService?.SetContext(new AccessContext { ObjectId = thread.CreatedBy, Name = thread.CreatedBy }); - - var userCtx = !string.IsNullOrEmpty(thread.CreatedBy) - ? new AccessContext { ObjectId = thread.CreatedBy, Name = thread.CreatedBy } - : null; - - void StartExecution() - { - hub.Post(new UpdateThreadMessageContent { Text = "Allocating agent..." }, - o => o.WithTarget(new Address(responsePath))); - - var executionHub = hub.GetHostedHub( - new Address($"{hub.Address}/_Exec"), - config => config.WithHandler(ExecuteMessageAsync), - HostedHubCreation.Always); - - executionHub!.Post(new SubmitMessageRequest + // Self-healing recovery. The prior one-shot Take(1).Timeout(15s) SILENTLY + // GAVE UP if the loaded-state emission was missed or arrived late (the + // dropped-patch subscribe-handshake race, amplified under load) — leaving a + // stale Executing thread stuck forever. That is the sub-thread cold-load + // "deadlock": not a lock, but a missed observation the recovery never + // retried. We instead wait for the first real thread emission however long + // it takes, and RE-ESTABLISH the observation if it faults before we read & + // drive the loaded state — no observer may die before the thread reaches a + // valid state. Driving an already-valid state is a no-op write (SetCurrent + // skips equal), so re-establishing is cheap and idempotent. + IDisposable? sub = null; + // Terminal guard for the self-healing re-establish below. Set by the hub's + // disposal hook (bottom of this method). Without it, a SYNCHRONOUS + // Subscribe-time fault after teardown — the hub's Autofac scope is gone, so + // GetMeshNodeStream().Subscribe() resolves a service off a disposed scope and + // throws ObjectDisposedException straight out of Subscribe — would recurse + // onError → Establish → onError until the stack overflows (the Orleans-shard + // SIGABRT). The re-establish must stop when the hub is gone AND must hop off + // the synchronous stack. + var disposed = false; + // Idempotency for the resume path: re-launch an interrupted round AT MOST + // once per ActiveMessageId. The observation is self-healing (re-establishes + // on fault), so without this a re-read of the same Executing state would + // re-launch the streaming loop repeatedly. Captured across re-establishes. + string? resumedRound = null; + void Establish() => sub = workspace.GetMeshNodeStream() + .Where(n => n?.Content is MeshThread) + .Take(1) + .Subscribe( + node => { - ThreadPath = threadPath, - UserMessageText = thread.PendingUserMessage ?? "", - UserMessageId = userMsgId, - ResponseMessageId = responseMsgId, - ResponsePath = responsePath, - AgentName = thread.PendingAgentName, - ModelName = thread.PendingModelName, - ContextPath = thread.PendingContextPath ?? thread.CreatedBy, - Attachments = thread.PendingAttachments - }, o => userCtx != null ? o.WithAccessContext(userCtx) : o); - } + if (node?.Content is not MeshThread thread) + return; + + // 🚨 Forcibly run EVERY recovery write below under the thread OWNER's identity + // (mirrors ExecRoundWatcher's FromNode scope). The reset / cancel / resume writes + // drive an UpdateStreamRequest from the sync hub on a context-less re-establish + // continuation; without this they post a null AccessContext → the never-null guard + // fails them → a DeliveryFailure storm that faults the observation → it re-establishes + // in a tight loop ("[ThreadExec] Init observation faulted for Thread — re-establishing"). + using var recoveryScope = MeshWeaver.Mesh.Security.AccessContextScope.FromNode(node, accessService, logger); + + // (1) Honor a cancel that was requested before the hub died, + // before looking at Status. + if (thread.RequestedStatus == ThreadExecutionStatus.Cancelled + && thread.Status != ThreadExecutionStatus.Cancelled) + { + logger?.LogInformation( + "[ThreadExec] Init: honoring pending cancel on {ThreadPath}", threadPath); + HonorPendingCancelOnWake(workspace, cache, node, thread, threadPath, logger); + return; + } - // Create cells, then start execution - var meshService = hub.ServiceProvider.GetRequiredService(); - meshService.CreateNode(new MeshNode(responseMsgId, threadPath) - { - NodeType = ThreadMessageNodeType.NodeType, MainNode = mainEntity, - Content = new ThreadMessage - { - Role = "assistant", Text = "", Timestamp = DateTime.UtcNow, - Type = ThreadMessageType.AgentResponse, - AgentName = thread.PendingAgentName, ModelName = thread.PendingModelName - } - }).Subscribe(_ => StartExecution(), - error => + switch (thread.Status) + { + case ThreadExecutionStatus.Executing + when !string.IsNullOrEmpty(thread.ActiveMessageId): + // Interrupted mid-round — the in-flight Task.Run is gone. + // 🚨 We STAY Executing and re-launch the streaming loop + // directly. We must NOT write Executing→StartingExecution: + // that is the exact inverse of the _Exec commit edge + // (StartingExecution→Executing), and because BOTH this + // recovery observer and the exec round watcher are + // self-healing, the two volley under load — the re-dispatch + // ping-pong (Resubmit/cold-load flake). Resume re-runs the + // round into its EXISTING response cell while Status stays + // Executing; idempotent per ActiveMessageId so a self-heal + // re-read can't double-launch. + if (resumedRound == thread.ActiveMessageId) + break; + resumedRound = thread.ActiveMessageId; + logger?.LogInformation( + "[ThreadExec] Init: resuming interrupted round on {ThreadPath}, activeMsg={ActiveMsg} (stay Executing)", + threadPath, thread.ActiveMessageId); + ThreadSubmissionServer.ResumeInterruptedRound(hub, node, logger); + break; + + case ThreadExecutionStatus.Executing: + // Executing but no response cell to resume — fall back + // to Idle so the submission watcher can claim pending. + logger?.LogInformation( + "[ThreadExec] Init: Executing with no ActiveMessageId on {ThreadPath} — resetting to Idle", + threadPath); + workspace.GetMeshNodeStream().Update(n => + n.Content is MeshThread t && t.Status == ThreadExecutionStatus.Executing + ? n with + { + LastModified = DateTime.UtcNow, + Content = t with + { + Status = ThreadExecutionStatus.Idle, + ExecutionStatus = null, + ActiveMessageId = null, + ExecutionStartedAt = null, + StreamingText = null, + StreamingToolCalls = null, + } + } + : n) + .Subscribe(_ => { }, ex => logger?.LogWarning(ex, + "[ThreadExec] Init reset: stream.Update failed for {ThreadPath}", threadPath)); + break; + + default: + // StartingExecution / Idle / Cancelled / Done — no + // write; the relevant watcher (or terminal state) + // already covers it. + break; + } + }, + ex => { - logger?.LogDebug("[ThreadExec] Response cell creation error: {Error}", error.Message); - StartExecution(); + // Observer died before we read & drove the loaded state — RESTART. + // (User directive: any observer dying before the thread reaches a + // terminal/valid state must restart the watcher.) Without this a + // faulted observation left the stale thread stuck forever. + // + // 🚨 Two guards make this self-heal safe — mirroring the sanctioned + // SubscribeWithReEstablish pattern (disposed-terminal + scheduled, + // never-synchronous re-establish): + // • `disposed` stops re-establishing once the hub is torn down — + // the fault is then permanent (scope gone), so retrying is futile. + // • the 1 s Timer hops the re-establish OFF the synchronous error + // stack. A Subscribe-time fault re-entering Establish inline would + // recurse to a stack overflow; deferring also lets the disposal + // hook set `disposed` past the teardown window. + if (disposed) + return; + logger?.LogWarning(ex, + "[ThreadExec] Init observation faulted for {ThreadPath} — re-establishing recovery", + threadPath); + System.Reactive.Linq.Observable.Timer(TimeSpan.FromSeconds(1)) + .Subscribe(_ => { if (!disposed) Establish(); }); }); - if (userMsgId != null) - { - meshService.CreateNode(new MeshNode(userMsgId, threadPath) + Establish(); + + // Guarantee-terminal watchdog. Belt-and-suspenders for the case where the + // initial recovery fired but the resumed round still never reached a + // terminal state — a missed StartingExecution dispatch, an observer that + // died mid-round, a child whose completion the parent never saw. If the + // thread node goes SILENT (no emission = no progress) for the grace period + // while still IsExecuting, the round is wedged: force it to Idle so + // IsExecuting clears and the user can resubmit. Two false-positive guards: + // • Throttle resets on EVERY node emission, so a healthy streaming round + // (which bumps LastModified continuously) never trips it. + // • A thread legitimately waiting on a child delegation is silent by + // design — that staleness is the HeartbeatTicker's job (it cancels the + // stale sub-thread), so we skip threads with an unfinished delegation. + var watchdog = workspace.GetMeshNodeStream() + .Throttle(StuckGracePeriod) + .Select(n => n?.Content as MeshThread) + .Where(t => t is { IsExecuting: true } && !HasUnfinishedDelegation(t)) + .Subscribe( + _ => { - NodeType = ThreadMessageNodeType.NodeType, MainNode = mainEntity, - Content = new ThreadMessage - { - Role = "user", Text = thread.PendingUserMessage, Timestamp = DateTime.UtcNow, - Type = ThreadMessageType.ExecutedInput, CreatedBy = thread.CreatedBy - } - }).Subscribe(_ => { }, - error => logger?.LogDebug("[ThreadExec] User cell creation error: {Error}", error.Message)); - } - }); - - return Task.CompletedTask; + logger?.LogWarning( + "[ThreadExec] Init watchdog: {ThreadPath} wedged non-terminal with no progress " + + "for {Grace:F0}s — forcing Idle (guarantee terminal).", + threadPath, StuckGracePeriod.TotalSeconds); + workspace.GetMeshNodeStream().Update(node => + node.Content is MeshThread t && t.IsExecuting && !HasUnfinishedDelegation(t) + ? node with + { + LastModified = DateTime.UtcNow, + Content = t with + { + Status = ThreadExecutionStatus.Idle, + ExecutionStatus = null, + ActiveMessageId = null, + ExecutionStartedAt = null, + StreamingText = null, + StreamingToolCalls = null, + } + } + : node) + .Subscribe(_ => { }, ex => logger?.LogWarning(ex, + "[ThreadExec] Init watchdog: force-Idle Update failed for {ThreadPath}", threadPath)); + }, + ex => logger?.LogWarning(ex, + "[ThreadExec] Init watchdog stream faulted for {ThreadPath}", threadPath)); + + hub.RegisterForDisposal(_ => { disposed = true; sub?.Dispose(); watchdog.Dispose(); }); } /// - /// Handles SubmitMessageRequest: updates thread state, responds immediately, - /// then starts execution. - /// - GUI flow (client provides UserMessageId + ResponseMessageId): cells already exist, - /// start execution directly. - /// - Server flow (no IDs provided): set PendingUserMessage so WatchForExecution - /// creates cells and starts execution. + /// Grace period for the guarantee-terminal watchdog. A thread node that emits + /// nothing (no progress) for this long while still + /// is treated as wedged and forced to Idle. Generous so it never races a slow + /// but live round — healthy work bumps LastModified far more often. /// - internal static IMessageDelivery HandleSubmitMessage( - IMessageHub hub, - IMessageDelivery delivery) - { - var request = delivery.Message; - var threadPath = request.ThreadPath; - var logger = hub.ServiceProvider.GetService>(); + private static readonly TimeSpan StuckGracePeriod = TimeSpan.FromSeconds(90); - var clientProvidedCells = request.UserMessageId != null && request.ResponseMessageId != null; - var userMsgId = request.UserMessageId ?? Guid.NewGuid().ToString("N")[..8]; - var responseMsgId = request.ResponseMessageId ?? Guid.NewGuid().ToString("N")[..8]; - var responsePath = $"{threadPath}/{responseMsgId}"; + /// + /// True when the thread carries an in-flight delegate_to_agent tool call + /// (a streaming tool call with a and + /// no yet). Such a thread is silent by + /// design while its child sub-thread runs; the + /// path — not the stuck-watchdog — owns its staleness. + /// + private static bool HasUnfinishedDelegation(MeshThread t) => + t.StreamingToolCalls is { Count: > 0 } + && t.StreamingToolCalls.Any(tc => + tc.Result is null && !string.IsNullOrEmpty(tc.DelegationPath)); - // Update Thread state. Set PendingUserMessage so WatchForExecution - // creates cells when they don't exist (server flow, delegation flow). - hub.GetWorkspace().UpdateMeshNode(node => + /// + /// Wake-up branch for a thread that has a pending RequestedStatus = + /// Cancelled the previous activation never got to honor. Stamps the + /// active response cell (marking + /// any unfinished tool calls) and writes the terminal thread state + /// (Status = Cancelled, RequestedStatus = null, ActiveMessageId = null), + /// leaving PendingUserMessages intact so the submission watcher + /// re-dispatches a fresh round. + /// + private static void HonorPendingCancelOnWake( + IWorkspace workspace, IMeshNodeStreamCache cache, MeshNode node, MeshThread thread, + string threadPath, ILogger? logger) + { + if (!string.IsNullOrEmpty(thread.ActiveMessageId)) { - var thread = node.Content as MeshThread ?? new MeshThread(); - var msgs = thread.Messages; - if (!msgs.Contains(userMsgId)) msgs = msgs.Add(userMsgId); - if (!msgs.Contains(responseMsgId)) msgs = msgs.Add(responseMsgId); - return node with - { - Content = thread with + var responsePath = $"{threadPath}/{thread.ActiveMessageId}"; + var responseMsgId = thread.ActiveMessageId!; + var mainEntity = node.MainNode ?? threadPath; + var cancelledToolCalls = thread.StreamingToolCalls? + .Select(tc => tc.Result != null + ? tc + : tc with { Result = "Cancelled (server restarted)", IsSuccess = false }) + .ToImmutableList() ?? ImmutableList.Empty; + + UpdateResponseCell(workspace.Hub, responsePath, threadPath, responseMsgId, mainEntity, + msg => msg with { - Messages = msgs, - IsExecuting = true, - ActiveMessageId = responseMsgId, - ExecutionStatus = null, - TokensUsed = 0, - ExecutionStartedAt = DateTime.UtcNow, - PendingUserMessage = request.UserMessageText, - PendingAgentName = request.AgentName, - PendingModelName = request.ModelName, - PendingContextPath = request.ContextPath, - PendingAttachments = request.Attachments?.ToImmutableList() - } - }; - }); - - logger?.LogInformation("[ThreadExec] HandleSubmitMessage: state updated for {ThreadPath}, activeMsg={ActiveMsg}, clientCells={ClientCells}", - threadPath, responseMsgId, clientProvidedCells); - - var userCtx = delivery.AccessContext; - // MainNode for child cells = the thread's own MainNode (content node, e.g. "PartnerRe/AIConsulting"). - // Fall back to request.ContextPath, then threadPath. Read from the workspace to get the - // thread node's actual MainNode — this is authoritative, not the client's ContextPath. - var threadNode = hub.GetWorkspace().GetStream(new MeshNodeReference())?.Current?.Value; - var mainEntity = threadNode?.MainNode ?? request.ContextPath ?? threadPath; - - void RespondAndStartExecution() - { - hub.Post(new SubmitMessageResponse { Success = true, Messages = ImmutableList.Create(userMsgId, responseMsgId) }, - o => o.ResponseFor(delivery)); - - hub.Post(new UpdateThreadMessageContent { Text = "Allocating agent..." }, - o => o.WithTarget(new Address(responsePath))); - - var executionHub = hub.GetHostedHub( - new Address($"{hub.Address}/_Exec"), - config => config.WithHandler(ExecuteMessageAsync), - HostedHubCreation.Always); - - executionHub!.Post(new SubmitMessageRequest - { - ThreadPath = threadPath, - UserMessageText = request.UserMessageText, - UserMessageId = userMsgId, - ResponseMessageId = responseMsgId, - ResponsePath = responsePath, - AgentName = request.AgentName, - ModelName = request.ModelName, - ContextPath = request.ContextPath, - Attachments = request.Attachments - }, o => userCtx != null ? o.WithAccessContext(userCtx) : o); + Text = msg.Text ?? "", + ToolCalls = cancelledToolCalls, + Status = ThreadMessageStatus.Cancelled, + CompletedAt = DateTime.UtcNow + }, + logger); } - void RespondWithError(string error) - { - logger?.LogWarning("[ThreadExec] Cell creation failed for {ThreadPath}: {Error}", threadPath, error); - // Clear execution state since we're not starting - hub.GetWorkspace().UpdateMeshNode(node => - { - var t = node.Content as MeshThread ?? new MeshThread(); - return node with { Content = t with { IsExecuting = false, ActiveMessageId = null, ExecutionStartedAt = null } }; - }); - hub.Post(new SubmitMessageResponse { Success = false, Error = error }, - o => o.ResponseFor(delivery)); - } - - if (clientProvidedCells) - { - // GUI flow — cells already exist, respond and start immediately. - RespondAndStartExecution(); - } - else - { - // Server flow — create cells first, then respond and start execution. - // Response cell creation gates execution; user cell is fire-and-forget. - var meshService = hub.ServiceProvider.GetRequiredService(); - - meshService.CreateNode(new MeshNode(userMsgId, threadPath) - { - NodeType = ThreadMessageNodeType.NodeType, MainNode = mainEntity, - Content = new ThreadMessage - { - Role = "user", Text = request.UserMessageText, Timestamp = DateTime.UtcNow, - Type = ThreadMessageType.ExecutedInput, CreatedBy = delivery.AccessContext?.ObjectId - } - }).Subscribe( - _ => logger?.LogDebug("[ThreadExec] User cell created: {Path}", $"{threadPath}/{userMsgId}"), - ex => logger?.LogDebug("[ThreadExec] User cell creation error (may already exist): {Error}", ex.Message)); - - meshService.CreateNode(new MeshNode(responseMsgId, threadPath) - { - NodeType = ThreadMessageNodeType.NodeType, MainNode = mainEntity, - Content = new ThreadMessage + workspace.GetMeshNodeStream().Update(n => + n.Content is MeshThread t && t.Status != ThreadExecutionStatus.Cancelled + ? n with { - Role = "assistant", Text = "", Timestamp = DateTime.UtcNow, - Type = ThreadMessageType.AgentResponse, - AgentName = request.AgentName, ModelName = request.ModelName + LastModified = DateTime.UtcNow, + Content = t with + { + Status = ThreadExecutionStatus.Cancelled, + RequestedStatus = null, + ExecutionStatus = null, + ActiveMessageId = null, + ExecutionStartedAt = null, + StreamingText = null, + StreamingToolCalls = null, + Summary = "Cancelled (server restarted)." + } } - }).Subscribe( - _ => RespondAndStartExecution(), - ex => RespondWithError($"Failed to create response cell: {ex.Message}")); - } - - return delivery.Processed(); + : n) + .Subscribe(_ => { }, ex => logger?.LogWarning(ex, + "[ThreadExec] HonorPendingCancelOnWake: stream.Update failed for {ThreadPath}", threadPath)); } + // WatchForExecution deleted as part of the "one trigger via GetMeshNodeStream" + // unification. The legacy auto-execute hook for BuildThreadWithMessages used + // PendingUserMessage (singular) + Status=Executing pre-set at construction + // time, then competed with the submission watcher by creating cells + + // calling ExecuteMessageAsync directly. BuildThreadWithMessages now seeds + // PendingUserMessages (dict) at Status=Idle and lets InstallServerWatcher + // claim through the standard flow — a single trigger path for every + // thread, no message-type or watcher rivalry. + + // HandleSubmitMessage + HandleSubmitMessageLegacy deleted 2026-05-25. + // HandleStartExecutionOnExec deleted with the trigger removal — _Exec's + // round watcher (InstallExecRoundWatcher) subscribes to the parent thread + // node's stream and fires DispatchAfterClaim on each Idle → StartingExecution + // transition. Public submissions go through ThreadSubmission.Submit → + // ThreadInput.AppendUserInput → the submission watcher; _Exec calls + // ExecuteMessageAsync directly (method, not message). + /// /// Async handler on the _Exec hosted hub. /// Prepares agent and await-streams the response. @@ -439,88 +664,512 @@ void RespondWithError(string error) /// the tool loop instead of relying on Microsoft.Extensions.AI's auto-invocation; /// that's intentionally NOT done here. /// - internal static IMessageDelivery ExecuteMessageAsync( + /// + /// Per-round mutable handle to the response cell currently being streamed + /// into. Stored on the parent thread hub via hub.Set for the duration + /// of a round. The streaming writer reads and + /// on every push. check_inbox (mid-round) + /// appends any in-flight user message directly into + /// (the live accumulator) with a marker, so the agent's continuation renders + /// below the user's interjection in the SAME response cell — Claude-Code style, + /// no separate cells and no output-cell split. is + /// null outside the streaming window. + /// + internal sealed class ActiveResponseSegment(string responseMsgId) + { + public string ResponseMsgId { get; set; } = responseMsgId; + public int TextBaseline { get; set; } + public System.Text.StringBuilder? ResponseText { get; set; } + } + + /// + /// Parameters for a single agent round. Direct-call replacement for the + /// old SubmitMessageRequest wire message — the inputs the agent loop + /// needs to run one round. Not a wire message: ExecuteMessageAsync is + /// invoked as a method, not via Post/handler dispatch. + /// + internal sealed record RoundParams( + string ThreadPath, + string ResponseMessageId, + string? UserMessageId, + string UserMessageText, + string? AgentName, + string? ModelName, + string? Harness, + string? ContextPath, + IReadOnlyList? Attachments); + + /// + /// Runs ONE agent round as a cold observable that completes when the round's terminal + /// Status write settles (OnError when the round faults or its terminal write fails). + /// 🚨 SUBSCRIBE EXACTLY ONCE per round. The pipeline is cold with heavy side effects + /// per subscription — a second Subscribe launches a SECOND round (double pool invoke, + /// double client init, duplicate cell writes). The single call site + /// (ThreadSubmissionServer.CommitRoundAndExecute) guards with + /// didCommitThisEmission; any new caller must provide an equivalent single-fire + /// guarantee. + /// + internal static IObservable ExecuteMessageAsync( IMessageHub hub, - IMessageDelivery delivery) + RoundParams request, + AccessContext? userAccessContext) { - var request = delivery.Message; + // Selections arrive as picked node PATHS ("Harness/MeshWeaver", + // "_Provider/Anthropic/claude-…", "Agent/Coder", "AgenticPension/Agent/Datenextraktion"). + // Models and harnesses match bare REGISTERED ids (the last path segment), so they + // normalize at this boundary. The AGENT does NOT: a space-scoped agent + // ("AgenticPension/Agent/Datenextraktion") collides with a built-in of the same + // last segment when collapsed to the bare id, so it must resolve by FULL PATH. + // AgentChatClient.SetSelectedAgent / SelectAgent match the full path (with a + // bare-id fallback). The cell stamp's display name is normalized to the short + // name where it's written (PushToResponseMessage), so the persisted AgentName + // stays the friendly last segment, not the full path. + request = request with + { + ModelName = SelectionId.IdOf(request.ModelName), + Harness = SelectionId.IdOf(request.Harness) + }; var parentHub = hub.Configuration.ParentHub!; var threadPath = request.ThreadPath; - var responsePath = request.ResponsePath!; - var responseMsgId = responsePath.Split('/').Last(); var logger = parentHub.ServiceProvider.GetRequiredService>(); - var workspace = parentHub.ServiceProvider.GetRequiredService(); + var cache = parentHub.ServiceProvider.GetRequiredService(); + var responseMsgId = request.ResponseMessageId + ?? throw new InvalidOperationException( + $"ExecuteMessageAsync: RoundParams for thread {threadPath} has no ResponseMessageId"); + var responsePath = $"{threadPath}/{responseMsgId}"; - // Helper: push content to response message hub. - // Posts UpdateThreadMessageContent which is handled ON the response grain — - // calls workspace.UpdateMeshNode() locally → sync stream → clients. - void PushToResponseMessage(string text, ImmutableList toolCalls, + // Active response-cell segment for THIS round. The streaming writer + // (PushToResponseMessage) targets segment.ResponseMsgId, NOT the captured + // responseMsgId, so the check_inbox tool can split the output mid-round: + // it freezes the current cell, inserts the interrupting user cells, and + // switches segment.ResponseMsgId to a fresh cell (clearing the shared + // StringBuilder) so subsequent tokens stream into the new cell. Stored on + // the parent hub so InboxTool.CheckInbox can reach it. See A7 in + // ThreadOperations.md. + var segment = new ActiveResponseSegment(responseMsgId); + parentHub.Set(segment); + + // Helper: push content to the response message via IMeshNodeStreamCache. + // 🚨 Same shared handle that the GUI's ThreadMessageBubbleView reads from — + // single upstream subscription process-wide. Replaces the per-_Exec + // workspace.GetRemoteStream that opened a separate handle (writes through + // one were invisible to readers of the other). + var mainEntity = request.ContextPath ?? threadPath; + // Heartbeat write throttle — stamp LastActivityAt on the OWN thread node + // at most once per heartbeatStampInterval. Reads in the closure run on + // _Exec's serialized action block, so a plain DateTime field is safe. + // 1 s matches the heartbeat scanner cadence so a fresh delta is always + // visible to the scanner before its next tick. + var lastActivityStamped = DateTime.MinValue; + var heartbeatStampInterval = TimeSpan.FromSeconds(1); + // Returns the cache.Update IObservable so terminal-status callers can + // AWAIT the write before signalling round completion — without this, + // the test base's quiesce phase trips on the in-flight DataChangeRequest + // Observe callbacks ("9 pending callback(s) after 0.50s" in + // DelegationWriteCountTest). Streaming-chunk callers still + // Subscribe(...) fire-and-forget for perf. + IObservable PushToResponseMessage(string text, ImmutableList toolCalls, ImmutableList updatedNodes, - string? agentName, string? modelName) + string? agentName, string? modelName, + int? inputTokens = null, int? outputTokens = null, int? totalTokens = null, + DateTime? completedAt = null, + ThreadMessageStatus? status = null, + string? summary = null, + string? harness = null) { - logger.LogInformation("[ThreadExec] PUSH_TO_MSG: responsePath={ResponsePath}, textLen={TextLen}, toolCalls={ToolCalls}, updatedNodes={UpdatedNodes}", - responsePath, text.Length, toolCalls.Count, updatedNodes.Count); - parentHub.Post(new UpdateThreadMessageContent + // Re-read the segment's CURRENT target + text baseline on every push so + // writes follow a mid-round check_inbox split to the new cell. The cell + // receives only the accumulated text PAST the baseline (the prior cells' + // text was committed when they were frozen). A stale buffered push whose + // text is shorter than the baseline slices to empty — harmless. + var curResponseMsgId = segment.ResponseMsgId; + var baseline = segment.TextBaseline; + var curResponsePath = $"{threadPath}/{curResponseMsgId}"; + text = text.Length > baseline ? text[baseline..] : string.Empty; + logger.LogDebug("[ThreadExec] PUSH_TO_MSG: responsePath={ResponsePath}, textLen={TextLen}, toolCalls={ToolCalls}, updatedNodes={UpdatedNodes}, status={Status}", + curResponsePath, text.Length, toolCalls.Count, updatedNodes.Count, status?.ToString() ?? "(preserve)"); + + // 🚨 Re-seed the user AccessContext immediately before the cell write. Every caller of + // this helper reaches it via a Reactive continuation — the streaming Sample(100ms) + // callback, the init-phase history/contextNode SelectMany ("Generating response..."), and + // the terminal completion path — each running on a hub-pipeline / ThreadPool thread where + // the per-hub AsyncLocal Context has been flipped to a per-cell impersonated address or + // dropped entirely (worse under a starved 2-core runner). MeshNodeStreamHandle.Update + // captures the AMBIENT context at THIS point and carries it onto the cross-hub + // UpdateStreamRequest; if it's null the owner's PostPipeline fails the write closed + // ("hub=sync/… UpdateStreamRequest … no AccessContext") → the round faults → the cell shows + // "Agent initialization stalled" and IsExecuting never clears (the 2-core CI flake in + // OrleansSubThreadAutoResume / Reentrancy / NodeChangePropagation). A thread cell is ALWAYS + // owned by the thread user, so re-asserting userAccessContext is the correct identity. + // See AccessContextPropagation.md / feedback_access_context_always_set. + if (userAccessContext != null) + parentHub.ServiceProvider.GetService()?.SetContext(userAccessContext); + + // 🚨 Route the cell write through parentHub (the thread hub), NOT + // `hub` (the _Exec hosted hub). _Exec is created with no AddData, so + // hub.GetMeshNodeStream(...) → hub.GetWorkspace() throws + // "Configuration of message hub is inconsistent: AddData was not + // called." parentHub owns the workspace + resolves the same + // process-wide IMeshNodeStreamCache, so the cross-hub patch routes + // through the identical shared handle the GUI reads from. + var updateObs = parentHub.GetMeshNodeStream(curResponsePath).Update(node => + { + var current = node.ContentAs(parentHub.JsonSerializerOptions, logger); + // Existing node whose content can't be recovered → leave it alone, NEVER clobber. + if (node?.Content is not null && current is null) + return node; + current ??= new ThreadMessage + { + Role = "assistant", + Text = "", + Type = ThreadMessageType.AgentResponse, + Status = ThreadMessageStatus.Streaming + }; + // Status: once terminal (Completed/Cancelled/Error), no patch can + // regress to Streaming. Sample(100ms) emits its last buffered value + // on source completion (snapshots.OnCompleted), and that emission's + // callback writes Status=Streaming — if it lands after the final + // success/cancel/error push it would flip the UI back to "still + // running" until the next render. Visible as flickering at the end + // of the response. + var requestedStatus = status ?? current.Status; + var nextStatus = current.Status is ThreadMessageStatus.Completed + or ThreadMessageStatus.Cancelled + or ThreadMessageStatus.Error + && requestedStatus == ThreadMessageStatus.Streaming + ? current.Status + : requestedStatus; + // 🚨 ToolCalls merge: ExecuteDelegationAsync.StampTerminalOnParentToolCall + // writes Result+Status+DelegationPath onto delegation entries via + // cache.Update concurrently with this streaming loop. If we replaced + // ToolCalls wholesale with `toolCalls` (the in-memory toolCallLog), + // the final iteration of this loop would CLOBBER the terminal stamp + // because toolCallLog only carries DelegationPath, never Result. + // Merge by DelegationPath (or Name+CallId match) — keep whichever + // entry has Result populated. The cache's current state wins for + // entries that have already terminated; toolCallLog wins for + // entries still mid-flight. + var mergedToolCalls = MergeToolCallEntries(current.ToolCalls, toolCalls); + // 🚨 Text monotonic-growth guard: streaming + tool-call mid-stream + // writes both flow through this Update path. A tool-call patch + // that doesn't carry the latest text (because the caller built + // its `text` from a stale snapshot of the streamed-so-far buffer + // BEFORE more tokens arrived) would shrink the field — visible + // to UI subscribers as a flicker / regression. While Status is + // terminal-locked above, Text is otherwise free to shrink. Cap: + // while Status is Streaming, only allow grow OR same length. + // Once terminal, the final text from the streaming loop's + // completion is the source of truth — let it through. + var nextText = nextStatus == ThreadMessageStatus.Streaming + && text.Length < current.Text.Length + ? current.Text + : text; + // 🚨 UpdatedNodes accumulate — never replace. Like ToolCalls (merged + // above) and Text (monotonic), node changes must not regress: the + // trailing Sample(100ms) emission fires AFTER the terminal completion + // push (snapshots.OnCompleted flushes the last buffered snapshot), and + // an earlier/empty snapshot would otherwise CLOBBER the aggregated + // changes back to []. Union by path (min VersionBefore / max + // VersionAfter, last Operation wins) so an empty incoming push is a + // no-op and re-pushes of the same node coalesce to one entry — exactly + // what OrleansNodeChangePropagationTest's ContainSingle assertion wants. + var mergedNodes = updatedNodes.IsEmpty + ? current.UpdatedNodes + : AggregateNodeChanges(current.UpdatedNodes.AddRange(updatedNodes)); + var updatedContent = current with + { + Text = nextText, + ToolCalls = mergedToolCalls, + UpdatedNodes = mergedNodes, + // The agent is carried through as a full PATH for resolution; the + // persisted/displayed cell author is the friendly short name (last segment). + AgentName = SelectionId.IdOf(agentName) ?? current.AgentName, + ModelName = modelName ?? current.ModelName, + Harness = harness ?? current.Harness, + InputTokens = inputTokens ?? current.InputTokens, + OutputTokens = outputTokens ?? current.OutputTokens, + TotalTokens = totalTokens ?? current.TotalTokens, + CompletedAt = completedAt ?? current.CompletedAt, + Status = nextStatus, + Summary = summary ?? current.Summary + }; + return node != null + ? node with { Content = updatedContent } + : new MeshNode(curResponseMsgId, threadPath) + { + NodeType = ThreadMessageNodeType.NodeType, + MainNode = mainEntity, + Content = updatedContent + }; + }); + + // Streaming hot-path callers Subscribe(...) fire-and-forget; the + // hot observable lets terminal-status callers await via FirstAsync. + updateObs.Subscribe( + _ => { }, + ex => logger.LogWarning(ex, + "[ThreadExec] cache.Update failed for {Path}", curResponsePath)); + + // Heartbeat stamp on the OWN thread. Throttled to one write per + // heartbeatStampInterval so the streaming hot path (Sample(100ms)) + // doesn't spam thread-node writes — one per interval is plenty + // since the heartbeat scanner runs every 5 s and the timeout is + // 30 s. Single source of "is this sub-thread still alive?" the + // parent's heartbeat scanner reads via cache.GetStream(threadPath). + var now = DateTime.UtcNow; + if (now - lastActivityStamped > heartbeatStampInterval) { - Text = text, - ToolCalls = toolCalls, - UpdatedNodes = updatedNodes, - AgentName = agentName, - ModelName = modelName - }, o => o.WithTarget(new Address(responsePath))); + lastActivityStamped = now; + parentHub.GetWorkspace().GetMeshNodeStream(threadPath).Update(node => + { + if (node?.Content is not MeshThread t) return node!; + return node with { Content = t with { LastActivityAt = now } }; + }).Subscribe( + _ => { }, + ex => logger.LogDebug(ex, + "[ThreadExec] LastActivityAt stamp failed for {Path}", threadPath)); + } + + return updateObs; } - // Helper: update Thread execution state via parentHub workspace. - // parentHub.GetWorkspace().UpdateMeshNode() is a synchronous function — no message needed. - var threadWorkspace = parentHub.GetWorkspace(); - void UpdateThreadExecution(Func mutate) - { - threadWorkspace.UpdateMeshNode(node => + var execLogger = parentHub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.AI.ThreadExecution"); + // Write thread state from THIS HUB (parentHub = thread hub), not via + // the mesh-hub-backed cache. With delta-based PatchDataRequest in + // MeshNodeStreamHandle.UpdateRemote, concurrent writers from different + // mirrors no longer clobber each other — so the cache routing that + // forced writes through the mesh hub (losing caller identity, surfacing + // 'no AccessContext' warnings, sender=mesh) is obsolete. The owning + // per-node hub remains the source of truth. + // 🚨 IObservable surface (no internal Subscribe). Cold — the + // stream.Update side effect runs once per Subscribe. Callers MUST + // Subscribe (the void-style fire-and-forget callsites Subscribe with + // a default `(_ => {}, ex => log)`; terminal-phase callers chain via + // SelectMany / continuation to wait for the commit before signalling + // round completion). Per AsynchronousCalls.md: never bridge to Task, + // always compose into the observable chain. + IObservable UpdateThreadExecution(Func mutate) => + // Re-seed the user AccessContext before the thread-node write — same reason as + // PushToResponseMessage above. All terminal callers (Completed / Cancelled / Error / + // NothingToSend) reach this from a Reactive continuation where the AsyncLocal Context was + // flipped/dropped; the write must carry the thread owner. Observable.Defer so the reseed + // runs at SUBSCRIBE time (when the cold Update is actually invoked), not at the (often + // earlier, differently-threaded) point the observable is constructed. + System.Reactive.Linq.Observable.Defer(() => { - var thread = node.Content as MeshThread ?? new MeshThread(); + if (userAccessContext != null) + parentHub.ServiceProvider.GetService()?.SetContext(userAccessContext); + return parentHub.GetWorkspace().GetMeshNodeStream(threadPath).Update(node => + { + // 🚨 No silent fallback. This Update runs on the parent thread + // hub's own action block; by the time this lambda fires, the + // node MUST be initialized (the hub's data source loaded it + // before processing the patch). A null Content here means the + // hub was processing this write before its node hydrated — + // surface loudly so the load order is fixed at the source. + if (node.Content is not MeshThread thread) + throw new InvalidOperationException( + $"UpdateThreadExecution: thread node {threadPath} has Content of type " + + $"{node.Content?.GetType().Name ?? ""}, not MeshThread. " + + "The hub must be fully initialized before terminal-state writes."); return node with { Content = mutate(thread) }; }); - } + }); + // Set user access context var accessService = parentHub.ServiceProvider.GetService(); - if (delivery.AccessContext != null) - accessService?.SetContext(delivery.AccessContext); - - // Reuse cached agent (skips 3+ seconds of agent initialization on 2nd+ message) - var chatClient = AgentCache.GetOrAdd(threadPath, _ => + if (userAccessContext != null) + accessService?.SetContext(userAccessContext); + + // Reuse cached agent (skips 3+ seconds of agent init on 2nd+ message). + // hub.Get / hub.Set is per-hub instance state — same hub across + // rounds = same cached client. Cache miss = resume after restart: + // load prior user messages from the persisted thread (excluding the + // current submission, which request.UserMessageText already carries), + // construct the client with that history, cache it, and proceed. + var cachedClient = parentHub.Get(); + IObservable clientObs; + if (cachedClient != null) + { + clientObs = Observable.Return(cachedClient); + } + else { - var c = new AgentChatClient(parentHub.ServiceProvider); + // 🚨 New threads start with empty conversation history — no + // bootstrap query needed. The per-round + // below loads + // ALL prior cells (user + assistant) per round, so resume after + // restart still gets full context. Skipping the cache-miss + // bootstrap query takes ~2s off cold-start latency on a brand- + // new thread (formerly: "Loading conversation history..." + // placeholder + 10s-timeout IMeshQueryCore.Query scan). + var c = new AgentChatClient(parentHub.ServiceProvider, priorMessages: null); c.SetThreadId(threadPath); - return c; - }); + parentHub.Set(c); + clientObs = Observable.Return(c); + } - // Subscribe to Initialize: when agents are ready, start the streaming loop - var initSub = chatClient.Initialize(request.ContextPath, request.ModelName) - .Take(1) // first emission = agents ready - .Subscribe(client => - { - logger.LogInformation("[ThreadExec] Agents ready for {ThreadPath}, starting execution", threadPath); + // 🔁 Resume recovery: a crash-resume re-launches an interrupted round into its + // EXISTING response cell and carries NO fresh selection (PendingUserMessages was + // already drained before the interruption, so PlanNextRound had no message to read + // it from). The response cell IS the persisted single source of truth for what + // agent/model/harness that round used — recover the missing selection from it + // rather than from a thread-level Pending* mirror (which no longer exists). Only + // reads the cell when something is actually missing; the normal submit path carries + // the full selection on the drained message and skips this read entirely. + IObservable requestObs; + if (string.IsNullOrEmpty(request.AgentName) + && string.IsNullOrEmpty(request.ModelName) + && string.IsNullOrEmpty(request.Harness)) + { + requestObs = parentHub.GetMeshNodeStream(responsePath) + .Select(n => n?.ContentAs(parentHub.JsonSerializerOptions, logger)) + .Where(m => m is not null) + .Take(1) + .Timeout(TimeSpan.FromSeconds(5)) + .Select(cell => request with + { + AgentName = request.AgentName ?? cell!.AgentName, + ModelName = request.ModelName ?? cell!.ModelName, + Harness = SelectionId.IdOf(request.Harness ?? cell!.Harness) + }) + .Catch(ex => + { + logger.LogWarning(ex, + "[ThreadExec] Resume: could not recover selection from response cell {ResponsePath}; proceeding with defaults", + responsePath); + return Observable.Return(request); + }); + } + else + { + requestObs = Observable.Return(request); + } - // Set context from remote stream + // Composed, returned round observable — completes when the round reaches a terminal + // Status (Completed/Cancelled/Error) and surfaces real faults via OnError. The submission + // watcher Subscribes to this (no fire-and-forget): it owns the subscription + disposal. + return requestObs.SelectMany(recovered => + { + request = recovered; + return clientObs.Take(1).SelectMany(chatClient => + { + // Initialize is sync (binds the chat client to the workspace's shared + // synced agent collection). Wait for the first WhenInitialized + // emission — synchronous when the synced query is warm-cached, async + // on first cold load — before starting the streaming loop. + // + // 🚨 Fail-fast on stall: if the synced agent query never emits + // (root cause of the prod sub-thread deadlock on 2026-05-20 — + // agent subscription on the sub-thread workspace stalled silently), + // surface the failure within 60s. NOT a Timeout(default) fallback + // (that wipes state — see feedback_timeout_wipes_synced_state) — + // an ERROR Timeout that flips the thread back to Idle and clears + // ActiveMessageId so the UI unsticks instead of perpetually + // "executing". 60s is generous; the workspace-cached synced + // query should emit Initial within seconds even on cold start. + chatClient.Initialize(request.ContextPath, request.ModelName); + return chatClient.WhenInitialized + .Take(1) + .Timeout(TimeSpan.FromSeconds(60)) + .SelectMany(client => + { + logger.LogDebug("[ThreadExec] Agents ready for {ThreadPath}, starting execution", threadPath); + + // 🚦 Harness dispatch. A harness is NOT a model provider: Claude Code / + // GitHub Copilot run their OWN CLI library (ClaudeCodeChatClient / + // CopilotChatClient) and must bypass the model-provider factory chain. + // The MeshWeaver harness returns null → keep the agent/model client. + // This is the fix for "harness selected → Azure DeploymentNotFound": + // the round no longer routes a harness through a provider. + var selectedHarness = HarnessNodeType.ResolveHarness(parentHub.ServiceProvider, request.Harness); + // CLI harnesses (Claude Code / Copilot) return their own IChatClient; + // the MeshWeaver harness returns null → use the AgentChatClient (which + // has its own non-IChatClient streaming signature). harnessClient stays + // null for MeshWeaver so the existing agent path runs unchanged. + // 🔎 A harness was SPECIFIED but does not resolve to any registered IHarness — + // say it was NOT FOUND (a stale/renamed id, e.g. an old "Claude Code" path before + // the slug fix, or a CLI harness whose feature flag is off). Fall back to the + // default agent path rather than crashing or silently ignoring it. (Empty harness + // ⇒ no selection ⇒ default; not a "not found".) + if (selectedHarness is null && !string.IsNullOrEmpty(request.Harness)) + logger.LogWarning( + "[ThreadExec] Harness '{Harness}' not found (no registered IHarness with that id) for " + + "{ThreadPath} — falling back to the default agent path", request.Harness, threadPath); + + // 🛡️ A harness that can't build its client (CLI missing, bad config, a bad + // harness id/path) must NOT crash the round or wedge the hub — catch, log, and + // fall back to the default MeshWeaver agent path (harnessClient stays null) so + // the user still gets a response. No retry/resubscribe here ⇒ no storm. + IChatClient? harnessClient = null; + try + { + harnessClient = selectedHarness?.CreateChatClient( + new HarnessExecutionContext(parentHub, null, request.ModelName)); + } + catch (Exception harnessEx) + { + logger.LogError(harnessEx, + "[ThreadExec] Harness '{Harness}' threw building its chat client for {ThreadPath}; " + + "falling back to the default agent path", request.Harness, threadPath); + } + if (harnessClient != null) + logger.LogInformation( + "[ThreadExec] Harness '{Harness}' → {Client} (bypassing provider chain) for {ThreadPath}", + request.Harness, harnessClient.GetType().Name, threadPath); + + // Set context from remote stream — must subscribe (Current is null on cold streams). + // When ContextPath is empty we just set null; otherwise wait for the first emission + // (with a short timeout fallback so a missing/inaccessible node doesn't stall execution) + // before continuing with SetExecutionContext + history load. + IObservable contextNodeObs; if (!string.IsNullOrEmpty(request.ContextPath)) { - var contextStream = workspace.GetRemoteStream( - new Address(request.ContextPath), new MeshNodeReference()); - client.SetContext(new AgentContext - { - Address = new Address(request.ContextPath), - Context = request.ContextPath, - Node = contextStream.Current?.Value - }); + // Read context node via the typed handle (routes cross-hub + // through the shared cache, deserializes Content). 🚨 Use + // parentHub — `hub` is the _Exec hosted hub which has no + // AddData, so hub.GetMeshNodeStream → hub.GetWorkspace() + // throws "AddData was not called" (the throw escaped on the + // WhenInitialized onNext path, wedging every round — + // all thread tests timed out). + contextNodeObs = parentHub.GetMeshNodeStream(request.ContextPath) + .Select(n => (MeshNode?)n) + .Where(v => v != null) + .Take(1) + .Timeout(TimeSpan.FromSeconds(5)) + .Catch(ex => + { + logger.LogWarning(ex, + "[ThreadExec] Failed to load context node {ContextPath}; proceeding with null Node", + request.ContextPath); + return Observable.Return(null); + }); + } + else + { + contextNodeObs = Observable.Return(null); } + return contextNodeObs.SelectMany(contextNode => + { + if (!string.IsNullOrEmpty(request.ContextPath)) + { + client.SetContext(new AgentContext + { + Address = new Address(request.ContextPath), + Context = request.ContextPath, + Node = contextNode + }); + } + if (!string.IsNullOrEmpty(request.AgentName)) client.SetSelectedAgent(request.AgentName); if (request.Attachments is { Count: > 0 }) client.SetAttachments(request.Attachments); - var userAccessContext = delivery.AccessContext; + // userAccessContext already in scope from the method parameter. client.SetExecutionContext(new ThreadExecutionContext { ThreadPath = threadPath, @@ -529,146 +1178,244 @@ void UpdateThreadExecution(Func mutate) UserAccessContext = userAccessContext }); - // Load history via GetDataRequest to each message — fully reactive - PushToResponseMessage("Loading conversation history...", ImmutableList.Empty, - ImmutableList.Empty, request.AgentName, request.ModelName); - - // Load history: subscribe to Thread stream → get Messages → GetDataRequest each → CombineLatest - threadWorkspace.GetStream(new MeshNodeReference())! - .Select(node => (node.Value?.Content as MeshThread)?.Messages ?? ImmutableList.Empty) - .Where(msgs => msgs.Count > 0) + // 🚨 Load FULL prior conversation (user + assistant) per round. + // Real LLMs (Anthropic, OpenAI) don't share session state across + // SubmitMessageRequest deliveries, and Echo (used by tests) + // doesn't track history at all. So if every round only sent the + // new user message, the agent would never see prior turns — + // ChatHistoryTest catches exactly this regression. + return LoadFullConversationHistoryFromMesh(parentHub, threadPath, + excludeUserMessageId: request.UserMessageId, + excludeResponseMessageId: responseMsgId, + logger) .Take(1) - .Subscribe(allMsgIds => - { - // History = all messages EXCEPT the last one (response cell). - // The current user cell IS included — its text comes from the cell itself. - var historyMsgIds = allMsgIds.Count > 1 - ? allMsgIds.Take(allMsgIds.Count - 1).ToImmutableList() - : ImmutableList.Empty; - logger.LogInformation("[ThreadExec] Loading {Count} history messages via GetDataRequest for {ThreadPath}", - historyMsgIds.Count, threadPath); - - // Post GetDataRequest to each message, collect responses - var historySubjects = historyMsgIds.Select(msgId => - { - var subject = new System.Reactive.Subjects.AsyncSubject<(string Id, ThreadMessage? Msg)>(); - logger.LogDebug("[ThreadExec] HISTORY_REQ: posting GetDataRequest for {MsgPath}", - $"{threadPath}/{msgId}"); - var del = parentHub.Post(new GetDataRequest(new MeshNodeReference()), - o => o.WithTarget(new Address($"{threadPath}/{msgId}"))); - if (del != null) - { - logger.LogDebug("[ThreadExec] HISTORY_REQ: posted, delivery={Id} for {MsgId}", del.Id, msgId); - parentHub.RegisterCallback((IMessageDelivery)del, resp => - { - ThreadMessage? tmsg = null; - if (resp is IMessageDelivery gdr) - tmsg = (gdr.Message.Data as MeshNode)?.Content as ThreadMessage; - logger.LogDebug("[ThreadExec] HISTORY_RESP: {MsgId} → role={Role}, textLen={Len}, respType={Type}", - msgId, tmsg?.Role ?? "(null)", tmsg?.Text?.Length ?? -1, resp.Message?.GetType().Name); - subject.OnNext((msgId, tmsg)); - subject.OnCompleted(); - return resp; - }); - } - else - { - logger.LogDebug("[ThreadExec] HISTORY_REQ: Post returned null for {MsgId}", msgId); - subject.OnNext((msgId, null)); - subject.OnCompleted(); - } - return subject.AsObservable(); - }).ToList(); - - // When all responses arrive (or timeout), build history and start streaming - var historyObs = historySubjects.Count > 0 - ? Observable.CombineLatest(historySubjects).Take(1) - .Select(results => - { - var lookup = results.Where(r => r.Msg != null).ToDictionary(r => r.Id, r => r.Msg!); - return historyMsgIds - .Where(id => lookup.ContainsKey(id)) - .Select(id => - { - var msg = lookup[id]; - var role = msg.Role == "user" ? ChatRole.User : ChatRole.Assistant; - var text = msg.Text ?? ""; - - // For assistant messages: prepend tool call summaries so the - // agent knows what it did (tools called, data read, results) - if (role == ChatRole.Assistant && msg.ToolCalls is { Count: > 0 }) - { - var toolSummary = string.Join("\n", msg.ToolCalls.Select(tc => - $"[Tool: {tc.Name}({tc.Arguments ?? ""})" + - (tc.Result != null ? $" → {tc.Result[..Math.Min(500, tc.Result.Length)]}" : "") + - "]")); - text = $"{toolSummary}\n\n{text}"; - } - - return new ChatMessage(role, text); - }) - .ToImmutableList(); - }) - .Timeout(TimeSpan.FromSeconds(10)) - .Catch, Exception>(ex => - { - logger.LogDebug("[ThreadExec] HISTORY_TIMEOUT: {ThreadPath}, {Count} messages requested, error={Error}", - threadPath, historyMsgIds.Count, ex.Message); - return Observable.Return(ImmutableList.Empty); - }) - : Observable.Return(ImmutableList.Empty); - - historyObs.Take(1).Subscribe(chatHistory => + .Timeout(TimeSpan.FromSeconds(5)) + .Catch, Exception>(ex => { - logger.LogInformation("[ThreadExec] Assembled {Count}/{Total} history messages for {ThreadPath}", - chatHistory.Count, historyMsgIds.Count, threadPath); + // 🚨 LOUD log — history-load failure means the agent sees + // truncated context (or nothing) for this round. Continue with + // empty so the round doesn't wedge, but surface the failure so + // CI surfaces the actual cause (per-cell timeout / stream error) + // instead of producing a wrong-content assertion downstream. + logger.LogError(ex, + "[ThreadExec] HISTORY_LOAD_FAILED threadPath={ThreadPath} — proceeding with EMPTY history; agent will see only the new user message", + threadPath); + return Observable.Return>(Array.Empty()); + }) + .SelectMany(history => + { + var chatHistory = history.ToImmutableList(); var toolCallLog = ImmutableList.Empty; var nodeChangeLog = ImmutableList.Empty; + // toolCallLog + nodeChangeLog are mutated from 4 concurrent paths: + // 1. Streaming loop (Task.Run with await foreach over agent updates) + // 2. FCC middleware (ChatClientAgentFactory's .Use(...) callback, + // fires on FCC's invocation thread) + // 3. client.ForwardToolCall (alias for path 2 on test agents that + // bypass FCC) + // 4. client.UpdateDelegationStatus (sub-thread completion callback + // on the sub-thread hub's grain scheduler) + // Without a lock, the read-modify-write idiom + // `toolCallLog = toolCallLog.Select/Add/SetItem(...)` + // suffers (a) lost updates — the flapping "delegations=0/1" + // alternation in OrleansDelegationTest's STREAM log — and + // (b) Index-out-of-range when one thread captures idx via FindIndex + // and a concurrent thread reassigns the list between FindIndex and + // SetItem (the line 167 InvalidOperationException). Repro: + // OrleansDelegationTest.Delegation_ToolCallsAppear_WithDelegationPath + // failed intermittently before this lock. + var logLock = new object(); // responseText is captured after InvokeAsync creates it (see below) StringBuilder? capturedResponseText = null; - client.ForwardNodeChange = entry => { nodeChangeLog = nodeChangeLog.Add(entry); }; + client.ForwardNodeChange = entry => { lock (logLock) { nodeChangeLog = nodeChangeLog.Add(entry); } }; string? currentStatus = null; - client.UpdateDelegationStatus = status => + // Pending Dispatched paths: when the Dispatched event fires BEFORE + // the outer streaming loop processes the corresponding + // FunctionCallContent and adds the bare entry, we queue the path + // here. The streaming loop drains the queue on each add. Without + // this, fast FCC dispatches lose the stamp and the response cell + // ships a bare delegate_to entry (the failing assertion in + // DelegationWriteCountTest). Drained in FIFO order to preserve + // multi-delegation correlation. + var pendingDispatchedPaths = ImmutableQueue.Empty; + // Subscribe to delegation lifecycle events. On Dispatched, stamp + // the path onto the first unmatched delegate_to tool-call entry + // and push the response. Replaces the legacy UpdateDelegationStatus + // callback (which keyed by display-name string). The subscription + // is disposed in the finally block at end of the round. + IDisposable? delegationStampSub = client is AgentChatClient ac + ? ac.Delegations + .Where(evt => evt.Phase == MeshWeaver.AI.Delegation.DelegationLifecycle.Dispatched) + .Subscribe(evt => { - currentStatus = status; - logger.LogInformation("[ThreadExec] DELEGATION_STATUS: threadPath={ThreadPath}, status={Status}, delegationPaths=[{Paths}]", - threadPath, status, string.Join(",", chatClient.DelegationPaths.Select(kv => $"{kv.Key}={kv.Value}"))); - // Push immediately when delegation path becomes available — - // the streaming loop is blocked during tool execution so the - // throttle block never runs. This ensures the parent message - // shows the delegation link while the sub-thread executes. - if (chatClient.DelegationPaths.TryGetValue(status, out var delPath)) + logger.LogInformation( + "[ThreadExec] DELEGATION_DISPATCHED: threadPath={ThreadPath}, subPath={SubPath}, callId={CallId}, toolCallLogSize={LogSize}", + threadPath, evt.SubThreadPath, evt.CallId, toolCallLog.Count); + var stamped = false; + ImmutableList snapshotLog; + ImmutableList snapshotNodes; + string textSnapshot; + lock (logLock) { - // Stamp the path on the first unmatched delegation tool call - var stamped = false; toolCallLog = toolCallLog.Select(e => { if (!stamped && e.Name.StartsWith("delegate_to") && e.DelegationPath == null) { stamped = true; - return e with { DelegationPath = delPath }; + logger.LogInformation("[ThreadExec] DELEGATION_STAMPED: name={Name} delPath={DelPath} callId={CallId}", + e.Name, evt.SubThreadPath, evt.CallId); + return e with { DelegationPath = evt.SubThreadPath }; } return e; }).ToImmutableList(); - // Preserve any previously streamed text - PushToResponseMessage(capturedResponseText?.ToString() ?? "", toolCallLog, nodeChangeLog, - request.AgentName, request.ModelName); + if (!stamped) + { + // FCC fired Dispatched faster than the outer streaming + // loop could process the FunctionCallContent. Queue the + // path; the streaming-loop add will drain it. + pendingDispatchedPaths = pendingDispatchedPaths.Enqueue(evt.SubThreadPath); + } + snapshotLog = toolCallLog; + snapshotNodes = nodeChangeLog; + // StringBuilder.ToString() is NOT thread-safe with concurrent + // Append — guard with logLock (same primitive as every other + // toolCallLog mutation; otherwise mid-walk ToString throws + // ArgumentOutOfRangeException, the original OrleansDelegationTest + // line 167 failure). + textSnapshot = capturedResponseText?.ToString() ?? ""; + } + if (!stamped) + logger.LogInformation("[ThreadExec] DELEGATION_DEFERRED_STAMP: subPath={SubPath} — queued for streaming-loop add (logSize={LogSize}, queueDepth={Q})", + evt.SubThreadPath, snapshotLog.Count, pendingDispatchedPaths.Count()); + // Push immediately so the parent message shows the delegation + // link while the sub-thread executes (streaming loop is blocked + // during tool execution; throttle block never runs). + PushToResponseMessage(textSnapshot, snapshotLog, snapshotNodes, + request.AgentName, request.ModelName); + }) + : null; + // Middleware-side ForwardToolCall: UPDATES the matching bare entry + // (added by the streaming branch's FunctionCallContent path) with the + // result, instead of skipping. Previously this branch skipped the + // result-bearing entry whenever a Result==null entry existed — + // correct for production agents where the streaming branch's + // FunctionResultContent handler later runs SetItem with the same + // Result. But for delegations with our refactored + // ExecuteDelegationAsync (yields text deltas, not FunctionResultContent + // in the output stream) AND for test agents that bypass FCC's + // FRC-in-output behaviour, the streaming branch's SetItem never fires + // → Result stays null forever. Update-instead-of-skip closes that. + // Production agents pay one redundant SetItem (no-op since the data is + // identical); test/delegation agents finally get Result populated. + client.ForwardToolCall = entry => + { + lock (logLock) + { + // SetItem-only — never Add. ForwardToolCall is the LATE + // mirror for an entry the streaming loop's FunctionCallContent + // branch (line 1346) already added. Adding here causes + // duplicates when the mirror fires before the streaming + // branch (FCC implementation dependent — some buffer FCC + // chunks until after tool invocation, some emit them + // synchronously). + // + // Match priority: + // 1) Same DelegationPath — covers ExecuteDelegationAsync's + // StampTerminal mirror after UpdateDelegationStatus + // already stamped DelegationPath on the bare entry. + // 2) Same Name + null Result — the standard "bare entry + // waiting for completion" shape. + var idx = -1; + if (entry.DelegationPath is not null) + idx = toolCallLog.FindIndex(e => e.DelegationPath == entry.DelegationPath); + if (idx < 0) + idx = toolCallLog.FindIndex(e => e.Name == entry.Name && e.Result == null); + if (idx >= 0) + { + var existing = toolCallLog[idx]; + // Merge: incoming carries the late updates (Result, + // Status, terminal Timestamp). Preserve existing's + // CallId + Arguments + DisplayName when the incoming + // entry doesn't carry them — the streaming branch + // had richer call-site detail. Critically, CallId + // must survive the SetItem so the streaming branch's + // CallId-keyed dedupe (alreadyByCallId) catches a + // re-emitted FunctionCallContent. + toolCallLog = toolCallLog.SetItem(idx, entry with + { + DelegationPath = entry.DelegationPath ?? existing.DelegationPath, + CallId = entry.CallId ?? existing.CallId, + Arguments = entry.Arguments ?? existing.Arguments, + DisplayName = entry.DisplayName ?? existing.DisplayName + }); + } + // No-match case: the mirror fired before the streaming + // loop processed the FCC chunk for this tool call. Drop + // the entry — the streaming loop will eventually add a + // bare entry that the FCC FunctionResultContent handler + // will populate (line 1422). Adding here would duplicate. } }; - client.ForwardToolCall = entry => { toolCallLog = toolCallLog.Add(entry); }; var agentDisplayName = request.AgentName ?? "Agent"; - // Build full message list: history (from GetDataRequest) + current message - // chatHistory already includes the current user message (loaded from the cell). - // Only add it if history is empty (delegation sub-thread, text from PendingUserMessage). - var allMessages = chatHistory.Count > 0 + // Build full message list: prior history (loaded above, EXCLUDES the + // current submission's user/response cells) + current user message. + // The system prompt is added by AgentChatClient.GetStreamingResponseAsync + // before forwarding to the inner IChatClient. + // + // RESUME path: UserMessageText is empty and UserMessageId is null — + // the interrupted round's user message is still in history (nothing + // was excluded), so we append NO trailing user message (an empty one + // would be a malformed final turn). The agent simply re-generates a + // response to the existing last user turn. + var allMessages = string.IsNullOrEmpty(request.UserMessageText) ? chatHistory : chatHistory.Add(new ChatMessage(ChatRole.User, request.UserMessageText)); logger.LogInformation("[ThreadExec] Sending {Count} messages to agent ({HistoryCount} history + 1 new): threadPath={ThreadPath}, agent={Agent}", allMessages.Count, chatHistory.Count, threadPath, request.AgentName ?? "(default)"); + // 🚫 Nothing to send: no current user turn AND no prior history. There is + // genuinely nothing for the agent to respond to — finish the round gracefully + // WITHOUT calling the LLM. Calling it here is exactly the storm path: the chat + // client's CreateChatClient throws ("No model selected") and AgentChatClient + // logs it once per agent. Write the terminal state deterministically (response + // cell → Completed, thread → Idle) and complete the round observable so the + // submission watcher sees a settled round. + if (allMessages.Count == 0) + { + logger.LogInformation( + "[ThreadExec] NOTHING_TO_SEND threadPath={ThreadPath} responseId={ResponseId} — finishing round with no LLM call", + threadPath, responseMsgId); + var nothingDone = new System.Reactive.Subjects.AsyncSubject(); + PushToResponseMessage( + "*Nothing to send — no message content.*", + ImmutableList.Empty, ImmutableList.Empty, + request.AgentName, request.ModelName, + completedAt: DateTime.UtcNow, + status: ThreadMessageStatus.Completed, + summary: "Nothing to send.", + harness: request.Harness).Subscribe( + _ => { }, + ex => execLogger?.LogWarning(ex, + "PushToResponseMessage(NothingToSend) failed for {ThreadPath}", threadPath)); + UpdateThreadExecution(t => t.ResetExecution() with { Summary = "Nothing to send." }).Subscribe( + _ => { }, + ex => + { + execLogger?.LogWarning(ex, + "UpdateThreadExecution(NothingToSend): stream.Update failed for {ThreadPath}", threadPath); + nothingDone.OnError(ex); + }, + () => + { + nothingDone.OnNext(System.Reactive.Unit.Default); + nothingDone.OnCompleted(); + }); + return nothingDone; + } + logger.LogInformation("[ThreadExec] STREAMING_START: threadPath={ThreadPath}, responsePath={ResponsePath}", threadPath, responsePath); // Run streaming on thread pool via Task.Run — the grain scheduler @@ -679,45 +1426,197 @@ void UpdateThreadExecution(Func mutate) // DelayDeactivation keeps the grain alive while the thread pool task runs. // BeginAsyncOperation signals the grain keep-alive timer. // After await Task.Run(...), execution returns to the grain scheduler. - var executionCts = new CancellationTokenSource(); - ExecutionCancellations[threadPath] = executionCts; + // + // Reuse the CancellationTokenSource HandleSubmitMessage stored on the + // parent hub. Storing it here would be too late — a Stop click between + // SubmitMessageResponse arriving at the GUI and us reaching this point + // would find a null CTS and silently no-op. If for some reason the slot + // is empty (auto-execute via WatchForExecution doesn't go through + // HandleSubmitMessage), allocate a fresh one as a safety net. + var executionCts = parentHub.Get() + ?? StoreNewCts(parentHub); // Cancel Task.Run when the hub disposes (grain deactivation). // Without this, OnDeactivateAsync waits up to 120s for the Task.Run // that's stuck on an AI API call with no cancellation signal. - hub.RegisterForDisposal(_ => executionCts.Cancel()); + // Guard against a disposal race: the round may already have completed and + // disposed its CTS (or another disposal path raced here), in which case + // Cancel() throws ObjectDisposedException — which, unhandled in + // HandleShutdownCore, faults the whole hub teardown. A cancel of an + // already-finished round is a no-op; swallow it. + hub.RegisterForDisposal(_ => + { + try { executionCts.Cancel(); } + catch (ObjectDisposedException) { /* round already finished + CTS disposed — nothing to cancel */ } + }); + + // 🚦 Cancellation is driven by the DURABLE "cancellation requested" state, PER ROUND + // — not a timing-fragile external watcher. Subscribe to THIS thread's own node stream + // (which replays the current value) and cancel the round's CTS the instant + // RequestedStatus == Cancelled is seen. A cancel requested in the CTS-storage window, + // or re-asserted across a RESUME (where InstallCancellationWatcher's + // (ExecutionStartedAt, HasCts) dedup swallows it — the round stays Executing with + // RequestedStatus=Cancelled forever, the Cancel_WithPendingMessages stuck-round red), + // is honored immediately. Set up SYNCHRONOUSLY here (before the async pool launch) so + // there is no window; disposed with the round in the finally below. cts.Cancel() is + // idempotent, so this is a robust complement to the watcher's sub-thread propagation. + var cancelOnRequestSub = parentHub.GetWorkspace().GetMeshNodeStream() + .Select(n => (n?.Content as MeshThread)?.RequestedStatus) + .Where(rs => rs == ThreadExecutionStatus.Cancelled) + .Take(1) + .Subscribe(_ => + { + try { executionCts.Cancel(); } + catch (ObjectDisposedException) { /* round already finished */ } + }); + + // The live response-text accumulator. Wired into the round's + // ActiveResponseSegment + capturedResponseText BEFORE the first + // push so InboxTool.CheckInbox can split the output cell the moment + // streaming is observable (a check_inbox that races the first + // "Generating response..." emission still sees a non-null + // ResponseText and splits correctly). + var responseText = new StringBuilder(); + capturedResponseText = responseText; + segment.ResponseText = responseText; + // Push progress: generating PushToResponseMessage("Generating response...", ImmutableList.Empty, - ImmutableList.Empty, request.AgentName, request.ModelName); - - _ = Task.Run(async () => + ImmutableList.Empty, request.AgentName, request.ModelName, + status: ThreadMessageStatus.Streaming, harness: request.Harness); + + // 🚦 The streaming round is an async I/O leaf and MUST run on the bounded AI + // I/O pool — NEVER Task.Run, NEVER inline on the hub turn. The pool offloads onto + // the ThreadPool with ConfigureAwait(false)/TaskPoolScheduler discipline (so + // continuations never bounce back to the grain scheduler the way a bare Task.Run's + // can) AND caps concurrent rounds. The thread hub's single turn thus stays FREE to + // answer tool-call responses, delegation callbacks, and — critically — GetData / + // GetPermission for its OWN output cell. A blocked turn here is exactly the + // "harness hangs after submit" wedge (GetDataRequest to {thread}/{cell} pending for + // tens of seconds). The `await foreach` is the only async place; tool calls + cell + // pushes inside run as observable composition. + // 🔁 Delegation nests: a delegating round holds its slot while awaiting a sub-thread + // round (which takes its own slot), so the Ai cap is a runaway-fan-out stop, not a + // fine throttle (see IoPoolOptions.Ai). Unbounded fallback when no registry is wired + // (DI-less tests) — still offloads, just no cap. + // See Doc/Architecture/ControlledIoPooling.md → "Streaming an agent response into a cell". + // Completes when the round's terminal Status write SETTLES: OnNext+OnCompleted when + // the terminal write (success/cancel/error) COMMITS, OnError when the terminal write + // itself FAILS — so the watcher's fault path (which writes the terminal state + // deterministically; see ThreadSubmission.CommitRoundAndExecute's onError) takes over + // instead of the round masquerading as cleanly finished while the node still says + // Executing. AsyncSubject replays its terminal signal to a late subscriber, closing + // the race where the write lands before `.SelectMany(_ => roundCompletion)` subscribes. + var roundCompletion = new System.Reactive.Subjects.AsyncSubject(); + var aiPool = parentHub.ServiceProvider.GetService()?.Get(IoPoolNames.Ai) + ?? IoPool.Unbounded; + // poolCt (the pool's cancellation) is intentionally unused — the round's + // own executionCts.Token (below) is authoritative and is cancelled on hub + // disposal, so cancellation flows through it regardless of the pool token. + return aiPool.Invoke(async poolCt => { + // Re-seed user AccessContext at the task-launch boundary. Inside this lambda we + // run the streaming loop + tool calls + responseStream.Update, all of which + // post to other hubs. Preceded by a chain of Subscribe callbacks + // (Initialize, contextNodeObs, threadWorkspace.GetStream, history loaders) — + // each fires on the upstream hub's pipeline where AsyncLocal Context flips + // to the per-cell hub's impersonated address. Reseed here so every downstream + // post goes out under the user's identity. + if (userAccessContext != null) + parentHub.ServiceProvider.GetService()?.SetContext(userAccessContext); + var ct = executionCts.Token; - var responseText = new StringBuilder(); - capturedResponseText = responseText; + // responseText / capturedResponseText / segment.ResponseText were + // wired just before the first push (above) so a check_inbox + // racing the round start still sees the live accumulator. int? inputTokens = null; int? outputTokens = null; int? totalTokens = null; + // Total-token normalization is the static NormalizeTotal helper (below), + // assigned per terminal path — NOT a local function here. A mutable-capturing + // local function threaded through this ~1400-line method's branches exploded + // Roslyn's nullable-flow/closure analysis: the MeshWeaver.AI ~10-min compile + // cliff (build step 259s → 676s at e30e9b5f1). See NormalizeTotal. + // Actual model the harness reports using (e.g. Claude Code resolves + // "sonnet" → a concrete id). Captured from the response updates so + // the output cell records what really ran, not just what was asked. + string? actualModel = null; + + // No time-limit watchdog. A streaming session blocked on an + // unresponsive AI endpoint, a long-running delegation, or a + // sub-thread doing its own multi-minute work is indistinguishable + // from a "stuck" pipeline from the parent's perspective — and an + // arbitrary deadline that fires `executionCts.Cancel()` would + // tear those down even when something is happening down the tree. + // Manual cancellation via the Stop button (RequestedStatus = + // Cancelled on the thread node, see RequestViaStreamUpdate.md) is + // the only legitimate cancel. + try { - logger.LogInformation("[ThreadExec] STREAMING_LOOP_ENTRY: {Time:HH:mm:ss.fff} threadPath={ThreadPath} (on thread pool)", DateTime.UtcNow, threadPath); // Keep the grain alive during the entire execution — including tool calls // and delegations where the streaming loop is blocked. using var heartbeatSubscription = parentHub.BeginAsyncOperation(); - var lastUpdate = DateTimeOffset.MinValue; - var lastPushedTextLength = 0; + using var snapshots = new Subject(); + using var pushSub = snapshots + .Sample(StreamingSampleInterval) + .Subscribe(s => PushToResponseMessage( + StripSummaryBlock(s.Text), s.ToolCalls, s.NodeChanges, + request.AgentName, request.ModelName, + status: ThreadMessageStatus.Streaming)); var pendingCalls = ImmutableDictionary.Empty; string? lastCallKey = null; - // Pass ALL messages through the official AgentChatClient path - await foreach (var update in client.GetStreamingResponseAsync(allMessages, ct)) + // Diagnostic: log the message + tool set we hand to the chat client. + // The 6 OrleansDelegation* tests fail with toolCalls=0 — this lets us + // see whether the test's fake client sees delegate_to_agent in + // options.Tools (which gates its FunctionCallContent emission). + logger.LogInformation( + "[ThreadExec] STREAM_BEGIN threadPath={ThreadPath} agent={Agent} model={Model} msgs={Msgs}", + threadPath, request.AgentName ?? "(default)", request.ModelName ?? "(default)", + allMessages.Count); + + // Pass ALL messages through the harness's client. MeshWeaver → + // AgentChatClient (2-arg streaming); Claude Code / Copilot → that + // harness's own CLI IChatClient (3-arg). Both yield ChatResponseUpdate. + var responseStream = harnessClient != null + ? harnessClient.GetStreamingResponseAsync(allMessages, options: null, ct) + : client.GetStreamingResponseAsync(allMessages, ct); + // 🚦 ConfigureAwait(false) is MANDATORY: this is the ONLY await in the + // round-streaming lambda, and it drives PushToResponseMessage + the + // terminal-Status write that signals round completion. Without it, each + // MoveNextAsync resumes on whatever scheduler was captured — and the agent + // stream's inner awaits (real LLM I/O, or a fake client's await Task.Delay) + // can complete on a per-node HUB action-block thread. The round body would + // then resume on that single-threaded hub scheduler, which under a 2-core + // runner is busy/parked → the continuation is queued but never pumped → the + // round never completes → the submission watcher never observes completion → + // the whole round is a MISSED OBSERVATION (all threads parked, no app frame). + // ConfigureAwait(false) pins the iteration to the ThreadPool (the IoPool's + // domain) so completion is never gated on a hub scheduler. This is the + // "await only in the IoPool" rule: the streaming await must never capture and + // resume on a hub/grain context. + await foreach (var update in responseStream.ConfigureAwait(false)) { + // Diagnostic: surface every content-kind we see. If FunctionInvokingChatClient + // eats the FunctionCallContent before we see it, this loop only logs TextContent / + // UsageContent — the smoking gun for "toolCalls=0" failures. + if (update.Contents.Count > 0) + { + logger.LogDebug("[ThreadExec] STREAM_UPDATE kinds=[{Kinds}]", + string.Join(",", update.Contents.Select(c => c.GetType().Name))); + } + + // Record the actual model the harness used (last non-empty wins). + if (!string.IsNullOrEmpty(update.ModelId)) + actualModel = update.ModelId; + // Capture function call / delegation activity for execution status foreach (var content in update.Contents) { if (content is FunctionCallContent functionCall) { - logger.LogDebug("[ThreadExec] TOOL_START: {Time:HH:mm:ss.fff} {Name} callId={CallId} args={Args}", - DateTime.UtcNow, functionCall.Name, functionCall.CallId, + logger.LogDebug("[ThreadExec] TOOL_START: {Name} callId={CallId} args={Args}", + functionCall.Name, functionCall.CallId, SerializeArgs(functionCall.Arguments)?[..Math.Min(100, SerializeArgs(functionCall.Arguments)?.Length ?? 0)]); var formatted = ToolStatusFormatter.Format(functionCall); var argsDetail = SerializeArgs(functionCall.Arguments); @@ -730,17 +1629,43 @@ void UpdateThreadExecution(Func mutate) pendingCalls = pendingCalls.SetItem(callKey, functionCall); lastCallKey = callKey; - // Add pending tool call to local log — will be pushed on next throttled update - // Skip if we already have an entry for this callKey (re-emitted content) - if (!isDuplicate) + // Add pending tool call to local log — will be pushed on next throttled update. + // Dedupe by CallId across the entire conversation: FCC can re-emit the same + // FunctionCallContent in turn 2's output stream (history echo), and the + // CallId-keyed `pendingCalls` map gets cleared on FunctionResultContent, + // so the second emission isn't caught by `isDuplicate`. Checking the log + // itself by CallId also dedupes against ChatClientAgentFactory.ExecuteDelegationAsync's + // StampTerminal mirror, which writes the same CallId at terminal. + lock (logLock) { - toolCallLog = toolCallLog.Add(new ToolCallEntry + var callId = functionCall.CallId; + var alreadyByCallId = callId is not null + && toolCallLog.Any(e => e.CallId == callId); + var alreadyPending = toolCallLog.Any(e => e.Name == functionCall.Name && e.Result == null); + if (!isDuplicate && !alreadyPending && !alreadyByCallId) { - Name = functionCall.Name, - DisplayName = formatted, - Arguments = argsDetail, - Timestamp = DateTime.UtcNow - }); + // Drain a pending Dispatched path if Dispatched fired + // before this FunctionCallContent reached the loop — + // the bare entry would otherwise ship without a + // DelegationPath (DelegationWriteCountTest failure + // mode). Drain only for delegate_to* names. + string? stampedPath = null; + if (functionCall.Name.StartsWith("delegate_to") && !pendingDispatchedPaths.IsEmpty) + { + pendingDispatchedPaths = pendingDispatchedPaths.Dequeue(out stampedPath); + logger.LogInformation("[ThreadExec] DELEGATION_DRAINED_STAMP: name={Name} delPath={DelPath} callId={CallId}", + functionCall.Name, stampedPath, callId); + } + toolCallLog = toolCallLog.Add(new ToolCallEntry + { + Name = functionCall.Name, + DisplayName = formatted, + Arguments = argsDetail, + CallId = callId, + DelegationPath = stampedPath, + Timestamp = DateTime.UtcNow + }); + } } } else if (content is UsageContent usage) @@ -784,186 +1709,566 @@ void UpdateThreadExecution(Func mutate) // Replace pending entry with final (has Result + DelegationPath). // Preserve DelegationPath if already stamped by UpdateDelegationStatus. - var idx = toolCallLog.FindIndex(e => e.Name == originalCall.Name && e.Result == null); - var existingDelegationPath = idx >= 0 ? toolCallLog[idx].DelegationPath : null; - var finalEntry = new ToolCallEntry + // FindIndex + SetItem must be atomic — without the lock a concurrent + // Select/.ToImmutableList rebuild from another path (UpdateDelegationStatus + // or middleware ForwardToolCall) can change the list reference between + // FindIndex returning idx and SetItem(idx) consuming it. + // Repro: OrleansDelegationTest's + // `Index was out of range. (Parameter 'index')`. + lock (logLock) { - Name = originalCall.Name, - DisplayName = ToolStatusFormatter.Format(originalCall), - Arguments = SerializeArgs(originalCall.Arguments), - Result = Truncate(resultText), - IsSuccess = isSuccess, - DelegationPath = delegationPath ?? existingDelegationPath, - Timestamp = DateTime.UtcNow - }; - toolCallLog = idx >= 0 ? toolCallLog.SetItem(idx, finalEntry) : toolCallLog.Add(finalEntry); - logger.LogDebug("[ThreadExec] TOOL_DONE: {Time:HH:mm:ss.fff} {Name} callId={CallId} delegation={Delegation} resultLen={ResultLen}", - DateTime.UtcNow, originalCall.Name, originalCall.CallId, delegationPath, - finalEntry.Result?.Length ?? 0); + // Match priority: + // 1) By DelegationPath when we have one — covers the case where + // ChatClientAgentFactory.ExecuteDelegationAsync's StampTerminal + // already populated Result on the matching delegation entry + // (so the name+Result-null check below misses). + // 2) Fall back to name + null Result for the standard + // bare-then-result flow. + var idx = -1; + if (delegationPath is not null) + idx = toolCallLog.FindIndex(e => e.DelegationPath == delegationPath); + if (idx < 0) + idx = toolCallLog.FindIndex(e => e.Name == originalCall.Name && e.Result == null); + var existingDelegationPath = idx >= 0 ? toolCallLog[idx].DelegationPath : null; + logger.LogDebug( + "[ThreadExec] TOOL_RESULT_REPLACE: name={Name} callId={CallId} idx={Idx} " + + "existingDelegationPath={ExistingDelegationPath} extractedPath={ExtractedPath} logSize={LogSize}", + originalCall.Name, originalCall.CallId, idx, + existingDelegationPath ?? "(null)", delegationPath ?? "(null)", toolCallLog.Count); + var finalEntry = new ToolCallEntry + { + Name = originalCall.Name, + DisplayName = ToolStatusFormatter.Format(originalCall), + Arguments = SerializeArgs(originalCall.Arguments), + Result = Truncate(resultText), + IsSuccess = isSuccess, + DelegationPath = delegationPath ?? existingDelegationPath, + CallId = originalCall.CallId, + Timestamp = DateTime.UtcNow + }; + toolCallLog = idx >= 0 ? toolCallLog.SetItem(idx, finalEntry) : toolCallLog.Add(finalEntry); + logger.LogDebug("[ThreadExec] TOOL_DONE: {Time:HH:mm:ss.fff} {Name} callId={CallId} delegation={Delegation} resultLen={ResultLen}", + DateTime.UtcNow, originalCall.Name, originalCall.CallId, delegationPath, + finalEntry.Result?.Length ?? 0); + } } currentStatus = null; // Tool call completed } } - if (!string.IsNullOrEmpty(update.Text)) - responseText.Append(update.Text); - - // Push streaming content at ~1/3sec — reduced frequency to avoid - // overloading the grain scheduler (messages expire if queue backs up). - // Push as a TEXT DELTA: we send only the new characters since the last - // push (tracked by lastPushedTextLength). The response cell appends it, - // so we never ship the whole growing string every tick. - if (DateTimeOffset.UtcNow - lastUpdate > TimeSpan.FromMilliseconds(3000)) + // Stamp delegation paths on any unmatched delegation tool calls. + // Same lock as every other toolCallLog mutation site — otherwise this + // rebuild silently overwrites an in-flight FunctionResultContent + // SetItem and the completed entry's Result/IsSuccess fields are lost. + // Also guards responseText.Append + ToString from the StringBuilder + // chunk-walk race with UpdateDelegationStatus on the sub-thread. + ImmutableList snapshotLog; + ImmutableList snapshotNodes; + string textSnapshot; + lock (logLock) { - // Stamp delegation paths on any unmatched delegation tool calls - var pathValues = chatClient.DelegationPaths.Values.ToList(); + if (!string.IsNullOrEmpty(update.Text)) + responseText.Append(update.Text); + + // ActiveDelegationPaths is maintained by AgentChatClient.EmitDelegationEvent + // (Dispatched adds, Terminal removes). Order is non-deterministic for an + // unordered set; the assumption that pathValues[idx] aligns with the + // i-th unmatched delegate_to entry holds only because each Dispatched + // event also fires the same stamp via the Delegations subscription + // installed below — this fallback covers the streaming-loop edge case + // where a delegation lands between Dispatched and the next streaming + // emission, but the subscription is the authoritative stamper. + var pathValues = chatClient.ActiveDelegationPaths.ToList(); var pathIdx = 0; toolCallLog = toolCallLog.Select(e => - { - if (e.Name.StartsWith("delegate_to") && e.DelegationPath == null && pathIdx < pathValues.Count) - return e with { DelegationPath = pathValues[pathIdx++] }; - return e; - }).ToImmutableList(); - - var delta = responseText.Length > lastPushedTextLength - ? responseText.ToString(lastPushedTextLength, responseText.Length - lastPushedTextLength) - : null; - // First push replaces the "Generating response…" placeholder; subsequent - // pushes append deltas only. - var isFirstPush = lastPushedTextLength == 0; - lastPushedTextLength = responseText.Length; - parentHub.Post(new UpdateThreadMessageContent - { - Text = isFirstPush ? responseText.ToString() : null, - TextDelta = isFirstPush ? null : delta, - ToolCalls = toolCallLog, - UpdatedNodes = nodeChangeLog, - AgentName = request.AgentName, - ModelName = request.ModelName - }, o => o.WithTarget(new Address(responsePath))); - lastUpdate = DateTimeOffset.UtcNow; + e.Name.StartsWith("delegate_to") && e.DelegationPath == null && pathIdx < pathValues.Count + ? e with { DelegationPath = pathValues[pathIdx++] } + : e).ToImmutableList(); + snapshotLog = toolCallLog; + snapshotNodes = nodeChangeLog; + textSnapshot = responseText.ToString(); } + + snapshots.OnNext(new StreamingSnapshot( + textSnapshot, snapshotLog, snapshotNodes)); } - // Final update — aggregate node changes (merges sub-thread changes with min/max versions), + snapshots.OnCompleted(); + // Capture a final consistent snapshot under the same lock that + // guarded every prior Append/ToString — UpdateDelegationStatus + // can still fire after the await foreach exits if a sub-thread + // completes during the trailing iteration. + string finalText; + int finalTextLen; + ImmutableList finalToolCalls; + ImmutableList finalNodeChanges; + lock (logLock) + { + finalText = responseText.ToString(); + finalTextLen = responseText.Length; + finalToolCalls = toolCallLog; + finalNodeChanges = nodeChangeLog; + } // include token usage + completion timestamp so the cell can show duration / tokens. - var aggregatedChanges = AggregateNodeChanges(nodeChangeLog); - if (totalTokens is null && (inputTokens.HasValue || outputTokens.HasValue)) - totalTokens = (inputTokens ?? 0) + (outputTokens ?? 0); + var aggregatedChanges = AggregateNodeChanges(finalNodeChanges); + totalTokens = NormalizeTotal(totalTokens, inputTokens, outputTokens); logger.LogInformation("[ThreadExec] EXECUTION_COMPLETE: {Time:HH:mm:ss.fff} threadPath={ThreadPath}, responseLength={Length}, toolCalls={ToolCalls}, tokens={In}/{Out}/{Total}", - DateTime.UtcNow, threadPath, responseText.Length, toolCallLog.Count, + DateTime.UtcNow, threadPath, finalTextLen, finalToolCalls.Count, inputTokens, outputTokens, totalTokens); - var finalText = responseText.ToString(); - parentHub.Post(new UpdateThreadMessageContent + // Empty stream + no tool calls = silent agent failure + // (e.g. underlying API returned nothing). Surface so the + // user sees a real terminal state instead of a blank cell. + if (string.IsNullOrEmpty(finalText) && finalToolCalls.IsEmpty) + finalText = "*Agent returned no response — streaming completed with zero tokens.*"; + + // Dedicated summary: parse ... the agent + // is instructed to emit at end-of-response (system prompt + // boilerplate). If present, that inner text is the tool- + // call result returned to a delegating parent; the marker + // block is also stripped from finalText so the user sees a + // clean response. If the marker is absent (agent forgot, or + // an external chat client), summaryText falls back to + // finalText. No extra LLM round-trip — same single + // streaming foreach drives both. Streaming-time pushes + // already strip the in-flight block via + // StripSummaryBlock so the user never sees the markers. + var summaryText = finalText; + var summaryMatch = System.Text.RegularExpressions.Regex.Match( + finalText, + @"(?[\s\S]*?)", + System.Text.RegularExpressions.RegexOptions.IgnoreCase); + if (summaryMatch.Success) { - Text = finalText, - ToolCalls = toolCallLog, - UpdatedNodes = aggregatedChanges, - AgentName = request.AgentName, - ModelName = request.ModelName, - InputTokens = inputTokens, - OutputTokens = outputTokens, - TotalTokens = totalTokens, - CompletedAt = DateTime.UtcNow - }, o => o.WithTarget(new Address(responsePath))); - // Clear streaming state - UpdateThreadExecution(t => t with + summaryText = summaryMatch.Groups["inner"].Value.Trim(); + finalText = (finalText[..summaryMatch.Index] + finalText[(summaryMatch.Index + summaryMatch.Length)..]).TrimEnd(); + finalTextLen = finalText.Length; + } + + // 🚨 Subscribe to actually fire the cold cache.Update write. + // Single push: writes Text=finalText, Summary=summaryText, + // Status=Completed atomically to the response cell. + PushToResponseMessage(finalText, finalToolCalls, aggregatedChanges, + request.AgentName, actualModel ?? request.ModelName, + inputTokens: inputTokens, outputTokens: outputTokens, + totalTokens: totalTokens, completedAt: DateTime.UtcNow, + status: ThreadMessageStatus.Completed, + summary: summaryText, harness: request.Harness).Subscribe( + _ => { }, + ex => execLogger?.LogWarning(ex, + "PushToResponseMessage(Completed) failed for {ThreadPath}", threadPath)); + // Clear streaming state AND publish the dedicated Summary + // in the SAME stream.Update cycle as the Status → Idle + // flip. Single emission → the parent's reactive subscriber + // (DelegationTool) sees both Summary and Idle atomically, + // never reads a stale empty Summary in an interleaving. + // Token usage is NOT stored on the thread — record it onto the per-model + // TokenUsage satellite ({threadPath}/_Usage/{model}); all cost tracking lives + // outside the Thread node now. + // 🚨 The satellite write is an INDEPENDENT subscribed side effect — it must NOT + // gate the terminal Status write or the round-completion gate. The satellite is a + // SEPARATE node; the GUI chip and the token tests WAIT for it (a Where(...).Timeout + // read), so it can land shortly AFTER the round shows terminal. Chaining it BEFORE + // the Idle flip (via SelectMany) delayed the terminal write on token-reporting + // rounds — up to RecordUsage's 15s cap — and gated roundCompletion on that write, + // so a slow satellite create+accumulate under load could push the round past the + // delegation tests' terminal-status timeouts. RecordUsage is fail-open (never + // errors) and a guaranteed no-op on zero-token rounds, so this fire-and-subscribe + // never touches the round on the no-usage path. + TokenUsageNodeType.RecordUsage(parentHub, threadPath, + AgentPickerProjection.PartitionOf(threadPath), + actualModel ?? request.ModelName, inputTokens, outputTokens, execLogger) + .Subscribe( + _ => { }, + ex => execLogger?.LogWarning(ex, + "RecordUsage(Completed) failed for {ThreadPath}", threadPath)); + UpdateThreadExecution(t => t.ResetExecution() with { - IsExecuting = false, ExecutionStatus = null, ActiveMessageId = null, - ExecutionStartedAt = null, StreamingText = null, StreamingToolCalls = null, - PendingUserMessage = null, PendingAgentName = null, PendingModelName = null, - PendingContextPath = null, PendingAttachments = null - }); + Summary = summaryText + }).Subscribe( + _ => { }, + ex => + { + execLogger?.LogWarning(ex, + "UpdateThreadExecution(Idle/Completed): stream.Update failed for {ThreadPath}", + threadPath); + // The terminal write FAILED — the node may still say Executing. Fault + // the gate so the watcher's onError writes the terminal state + // deterministically (no reliance on the stuck-round watchdog). + roundCompletion.OnError(ex); + }, + () => + { + roundCompletion.OnNext(System.Reactive.Unit.Default); + roundCompletion.OnCompleted(); + }); // Notify parent via SubmitMessageResponse so delegation callback resolves. // Must post on the _Exec hub (hub) — the SubmitMessageResponse handler // is registered there and forwards to the thread hub via ResponseFor. NotifyParentCompletion(parentHub, threadPath, finalText, true, aggregatedChanges); + EmitCompletionNotification(parentHub, threadPath, finalText, request.AgentName); } catch (OperationCanceledException) { logger.LogInformation("[ThreadExec] CANCELLED: {Time:HH:mm:ss.fff} threadPath={ThreadPath}", DateTime.UtcNow, threadPath); - var cancelText = (responseText.ToString() + "\n\n*Cancelled*").Trim(); - PushToResponseMessage(cancelText, toolCallLog, nodeChangeLog, request.AgentName, request.ModelName); + // ToString must be under logLock — UpdateDelegationStatus + // (sub-thread callback) can still Append concurrently after + // the try body exits. + string cancelText; + ImmutableList cancelToolCalls; + ImmutableList cancelNodeChanges; + lock (logLock) + { + cancelText = responseText.ToString(); + cancelToolCalls = toolCallLog; + cancelNodeChanges = nodeChangeLog; + } + // 🚨 Subscribe to fire the cold cache.Update — same reason as + // the Completed branch above. + // Record tokens consumed BEFORE the cancel — the streaming loop + // already aggregated any UsageContent seen prior to the + // OperationCanceledException — so the cell + thread reflect what + // the round actually cost. + totalTokens = NormalizeTotal(totalTokens, inputTokens, outputTokens); + PushToResponseMessage(cancelText, cancelToolCalls, cancelNodeChanges, + request.AgentName, request.ModelName, + inputTokens: inputTokens, outputTokens: outputTokens, + totalTokens: totalTokens, + completedAt: DateTime.UtcNow, + status: ThreadMessageStatus.Cancelled).Subscribe( + _ => { }, + ex => execLogger?.LogWarning(ex, + "PushToResponseMessage(Cancelled) failed for {ThreadPath}", threadPath)); + // Summary invariant: every Idle write must carry a Summary. + // Cancelled path has no agent-emitted block, so + // Summary defaults to the cancellation context's accumulated + // text — same as the user-visible response cell Text. + var cancelSummary = string.IsNullOrEmpty(cancelText) + ? "Cancelled before completion." + : cancelText; + // Terminal Cancelled (not Idle): a distinct, visible status. + // Clear the cancel request now that it's achieved; leave + // PendingUserMessages intact so the submission watcher + // re-dispatches a fresh round from Cancelled+pending. + // A cancelled round still cost tokens — record them on the satellite as an + // INDEPENDENT subscribed side effect (see Completed branch: NOT chained before + // the terminal write; the reader WAITS for the satellite, so it may land just + // after the terminal Cancelled status). Fail-open + no-op on zero tokens. + TokenUsageNodeType.RecordUsage(parentHub, threadPath, + AgentPickerProjection.PartitionOf(threadPath), + request.ModelName, inputTokens, outputTokens, execLogger) + .Subscribe( + _ => { }, + ex => execLogger?.LogWarning(ex, + "RecordUsage(Cancelled) failed for {ThreadPath}", threadPath)); UpdateThreadExecution(t => t with { - IsExecuting = false, ExecutionStatus = null, ActiveMessageId = null, - ExecutionStartedAt = null, StreamingText = null, StreamingToolCalls = null - }); - NotifyParentCompletion(parentHub, threadPath, cancelText, false, nodeChangeLog); + Status = ThreadExecutionStatus.Cancelled, RequestedStatus = null, + ExecutionStatus = null, ActiveMessageId = null, + ExecutionStartedAt = null, StreamingText = null, StreamingToolCalls = null, + Summary = cancelSummary + }).Subscribe( + _ => { }, + ex => + { + execLogger?.LogWarning(ex, + "UpdateThreadExecution(Idle/Cancelled): stream.Update failed for {ThreadPath}", + threadPath); + // Terminal write failed → fault the gate (see Completed branch). + roundCompletion.OnError(ex); + }, + () => + { + roundCompletion.OnNext(System.Reactive.Unit.Default); + roundCompletion.OnCompleted(); + }); + NotifyParentCompletion(parentHub, threadPath, cancelText, false, cancelNodeChanges); + EmitCompletionNotification(parentHub, threadPath, "Cancelled", request.AgentName); } catch (Exception ex) { logger.LogError(ex, "[ThreadExec] ERROR: {Time:HH:mm:ss.fff} threadPath={ThreadPath}", DateTime.UtcNow, threadPath); - var errorText = (responseText.ToString() + $"\n\n*Error: {ex.Message}*").Trim(); - PushToResponseMessage(errorText, toolCallLog, nodeChangeLog, request.AgentName, request.ModelName); - UpdateThreadExecution(t => t with + // Same lock-guarded snapshot as the cancellation path. + string errorTextBase; + ImmutableList errorToolCalls; + ImmutableList errorNodeChanges; + lock (logLock) { - IsExecuting = false, ExecutionStatus = null, ActiveMessageId = null, - ExecutionStartedAt = null, StreamingText = null, StreamingToolCalls = null - }); - NotifyParentCompletion(parentHub, threadPath, errorText, false, nodeChangeLog); + errorTextBase = responseText.ToString(); + errorToolCalls = toolCallLog; + errorNodeChanges = nodeChangeLog; + } + // A CLI harness that isn't logged in (e.g. Claude Code "Not logged in · Please + // run /login") surfaces as AuthRequiredException — render an actionable "/login" + // affordance instead of the cryptic "exit code 1" the SDK throws. + var errorText = (ex is AuthRequiredException authEx + ? errorTextBase + "\n\n" + authEx.ToMarkdown() + : errorTextBase + $"\n\n*Error: {ex.Message}*").Trim(); + // 🚨 NO await on hub-touching observables in src/. Subscribe- + // continuation: push the error cell, then flip Idle, then notify. + // (Previous `.ToTask()` bridge would deadlock the action block — + // forbidden per feedback_no_totask_in_src.md / AsynchronousCalls.md.) + // Record tokens consumed before the fault (same rationale as the + // Cancelled branch) so an errored round still reports its cost. + totalTokens = NormalizeTotal(totalTokens, inputTokens, outputTokens); + var pushErrorObs = PushToResponseMessage(errorText, errorToolCalls, errorNodeChanges, + request.AgentName, request.ModelName, + inputTokens: inputTokens, outputTokens: outputTokens, + totalTokens: totalTokens, + completedAt: DateTime.UtcNow, + status: ThreadMessageStatus.Error) + .Timeout(TimeSpan.FromSeconds(10)); + var errorTextLocal = errorText; + var errorNodeChangesLocal = errorNodeChanges; + pushErrorObs.Subscribe( + _ => { }, + pushEx => + { + execLogger?.LogWarning(pushEx, + "PushToResponseMessage(Error) failed for {ThreadPath}", threadPath); + // The error-cell push faulted, so the inner Idle write never runs — + // fault the gate so the watcher's onError writes the terminal state. + roundCompletion.OnError(pushEx); + }, + () => + { + // Summary invariant for the Error path — non-empty. + var errorSummary = string.IsNullOrEmpty(errorTextLocal) + ? $"Error: {ex.Message}" + : errorTextLocal; + // Tokens burned before the fault — record on the satellite as an + // INDEPENDENT subscribed side effect (see Completed branch: NOT chained + // before the terminal Idle write; the reader WAITS for the satellite). + // Fail-open + no-op on zero tokens. + TokenUsageNodeType.RecordUsage(parentHub, threadPath, + AgentPickerProjection.PartitionOf(threadPath), + request.ModelName, inputTokens, outputTokens, execLogger) + .Subscribe( + _ => { }, + recEx => execLogger?.LogWarning(recEx, + "RecordUsage(Error) failed for {ThreadPath}", threadPath)); + UpdateThreadExecution(t => t with + { + Status = ThreadExecutionStatus.Idle, ExecutionStatus = null, ActiveMessageId = null, + ExecutionStartedAt = null, StreamingText = null, StreamingToolCalls = null, + Summary = errorSummary + }).Subscribe( + _ => { }, + updEx => + { + execLogger?.LogWarning(updEx, + "UpdateThreadExecution(Idle/Error): stream.Update failed for {ThreadPath}", + threadPath); + // Terminal write failed → fault the gate (see Completed branch). + roundCompletion.OnError(updEx); + }, + () => + { + NotifyParentCompletion(parentHub, threadPath, errorTextLocal, false, errorNodeChangesLocal); + EmitCompletionNotification(parentHub, threadPath, errorTextLocal, request.AgentName); + roundCompletion.OnNext(System.Reactive.Unit.Default); + roundCompletion.OnCompleted(); + }); + }); } finally { - ExecutionCancellations.TryRemove(threadPath, out _); + delegationStampSub?.Dispose(); + cancelOnRequestSub.Dispose(); + // Dispose the per-round CLI harness client (Claude Code / Copilot). + // The cached AgentChatClient (MeshWeaver path) is never disposed + // here — it's reused across rounds. + if (harnessClient is IDisposable sd) sd.Dispose(); + else if (harnessClient is IAsyncDisposable sad) _ = sad.DisposeAsync(); + // Detach the accumulator so a check_inbox call between + // rounds can't split on a dead StringBuilder (the guard + // also requires IsExecuting + matching ActiveMessageId). + segment.ResponseText = null; + parentHub.Set(null!); executionCts.Dispose(); + // No per-_Exec stream handle to dispose — writes went through + // IMeshNodeStreamCache.Update, whose upstream handle is owned + // by the cache and outlives this round. } + return System.Reactive.Unit.Default; + }) + // Gate completion on the terminal Status write LANDING (roundCompletion fires from + // each terminal path's UpdateThreadExecution), then surface real faults to the caller + // (the submission watcher) via OnError instead of swallowing them. Disposal-race + // exceptions during teardown stay swallowed. + .SelectMany(_ => roundCompletion) + .Catch(streamingEx => + { + var disposalRace = streamingEx is ObjectDisposedException + || (streamingEx is InvalidOperationException ioe + && ioe.Message.Contains("disposed", StringComparison.OrdinalIgnoreCase)); + if (disposalRace) + return Observable.Empty(); + logger.LogError(streamingEx, + "[ThreadExec] streaming round faulted for {ThreadPath}", threadPath); + return Observable.Throw(streamingEx); + }); + }); // end of LoadFullConversationHistory.SelectMany + }); // end of contextNodeObs.SelectMany + }) + // Agent-init stall/error: recover (unstick the UI + stamp the cell) and COMPLETE + // the round without faulting the chain — the watcher treats an init-stall as a + // settled (Idle) round, exactly as the prior void method did (it never told the + // watcher). A Timeout/init error short-circuits the SelectMany above and lands here. + .Catch(ex => + { + // 🚨 Agent-init stalled or errored — surface and unstick the UI. + // Without this, IsExecuting stays true forever and the user sees + // a perpetually-"executing" thread (prod symptom 2026-05-20). + // Flips Status → Idle, clears ActiveMessageId, marks the response + // cell as Error, and notifies parent (delegation tool watchdog + // already handles the sub-thread side via the cancel + // propagation in ChatClientAgentFactory.ExecuteDelegationAsync). + logger.LogError(ex, + "[ThreadExec] Initialize failed / stalled for {ThreadPath} — flipping thread to Idle", + threadPath); + + // Re-seed the user AccessContext before the unstick writes. This recovery path runs + // on the Catch continuation thread where the AsyncLocal is gone; its cross-hub + // stream.Update + UpdateResponseCell would otherwise post UpdateStreamRequest with no + // AccessContext and fail closed — leaving the thread stuck Executing (the very state + // this handler exists to clear). + if (userAccessContext != null) + accessService?.SetContext(userAccessContext); + + parentHub.GetWorkspace().GetMeshNodeStream().Update(node => + { + if (node?.Content is not MeshThread t) return node!; + return node with + { + LastModified = DateTime.UtcNow, + Content = t with + { + Status = ThreadExecutionStatus.Idle, + ExecutionStatus = null, + ActiveMessageId = null, + ExecutionStartedAt = null, + StreamingText = null, + StreamingToolCalls = null + } + }; + }).Subscribe(_ => { }, ex2 => logger.LogWarning(ex2, + "[ThreadExec] Init-stall unstick: stream.Update failed for {ThreadPath}", + threadPath)); + + // If the in-flight round has a response cell, stamp it with + // the error so the bubble shows something instead of an empty + // "Allocating agent..." placeholder. + var responsePath = $"{threadPath}/{responseMsgId}"; + UpdateResponseCell(parentHub, responsePath, threadPath, responseMsgId, + mainEntity: threadPath, + msg => msg with + { + Text = (msg.Text ?? string.Empty) + + $"\n\n*Agent initialization stalled: {ex.Message}*", + Status = ThreadMessageStatus.Error, + CompletedAt = DateTime.UtcNow + }, + logger); + + NotifyParentCompletion(parentHub, threadPath, + $"Agent initialization stalled: {ex.Message}", success: false); + return Observable.Empty(); }); - }); // end of historyObs.Subscribe - }); // end of threadStream.Subscribe (Messages) - }, // end of Initialize().Subscribe onNext - ex => logger.LogError(ex, "[ThreadExec] Initialize failed for {ThreadPath}", threadPath)); - - // Register subscription for disposal - workspace.AddDisposable(initSub); - - return delivery.Processed(); + }); // end of clientObs.SelectMany + }); // end of requestObs.SelectMany (resume-selection recovery) } /// - /// - /// Push streaming content via DataChangeRequest to the response message hub. - /// One-way message — the response hub updates its own local workspace. - /// No remote stream sync, no amplification storm. - /// - /// - /// Notifies the parent thread that this child thread's execution completed. - /// The parent's delegation tool handler resolves its TaskCompletionSource. - /// Only posts if this thread IS a child (path has a parent response message segment). + /// Logging-only completion stub. The SubmitMessageResponse callback shape + /// was deleted 2026-05-25; parent threads now observe sub-thread completion + /// via the response cell's stream (Status flips to Completed/Cancelled/Error + /// via PushToResponseMessage). Kept as a method for the existing 4 callsites + /// — the delegation tool's reactive completion observation will replace the + /// callsites in the next refactor pass. /// private static void NotifyParentCompletion( IMessageHub hub, string threadPath, string responseText, bool success, ImmutableList? updatedNodes = null) { var logger = hub.ServiceProvider.GetRequiredService>(); - var status = success ? SubmitMessageStatus.ExecutionCompleted - : SubmitMessageStatus.ExecutionFailed; - logger.LogInformation("[ThreadExec] NOTIFY_PARENT: threadPath={ThreadPath}, status={Status}, textLen={TextLen}", - threadPath, status, responseText.Length); - - // Invoke the completion callback registered by HandleSubmitMessage. - // This posts a SubmitMessageResponse(ExecutionCompleted) via ResponseFor(originalDelivery) - // on the thread hub, which routes back to the client's RegisterCallback. - if (CompletionCallbacks.TryGetValue(threadPath, out var callback)) - { - callback(new SubmitMessageResponse - { - Success = success, - Status = status, - ResponseText = Truncate(responseText, 500), - UpdatedNodes = updatedNodes - }); - } - else + logger.LogInformation( + "[ThreadExec] NOTIFY_PARENT: threadPath={ThreadPath}, success={Success}, textLen={TextLen}, updatedNodes={UpdatedNodes}", + threadPath, success, responseText.Length, updatedNodes?.Count ?? 0); + } + + /// + /// Posts a satellite under the thread on + /// successful round completion. The notification stores in the + /// notifications table (satellite routing via + /// ) and shows up + /// in the user's bell — clicking it navigates to the thread. + /// Fire-and-forget; failures are logged but don't fail the round. + /// + private static void EmitCompletionNotification( + IMessageHub hub, string threadPath, string responseText, string? agentName) + { + var logger = hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.AI.ThreadExecution"); + var meshService = hub.ServiceProvider.GetService(); + if (meshService == null) { - logger.LogWarning("[ThreadExec] No completion callback for {ThreadPath}", threadPath); + logger?.LogDebug("[ThreadExec] EmitCompletionNotification: no IMeshService — skipping"); + return; } + + var threadName = (hub.GetWorkspace().GetStream(new MeshNodeReference()) + as ISynchronizationStream)?.Current?.Value?.Name ?? "Thread"; + var preview = Truncate(responseText, 120) ?? ""; + + NotificationService.CreateNotification( + meshService, + mainNodePath: threadPath, + title: $"\"{threadName}\" is ready", + message: preview, + type: NotificationType.General, + targetNodePath: threadPath, + // agentName arrives as a full PATH (resolution form); store the friendly short name. + createdBy: SelectionId.IdOf(agentName) ?? "agent", + icon: "/static/NodeTypeIcons/chat.svg") + .Subscribe( + _ => { }, + ex => logger?.LogWarning(ex, + "[ThreadExec] Failed to create completion notification for {ThreadPath}", + threadPath)); } /// - /// Aggregates node change entries: for the same path, takes min(VersionBefore) and max(VersionAfter). - /// This merges changes from the current thread and any delegation sub-threads. + /// Strips an in-flight or completed <summary>...</summary> + /// block from the agent's response so the user never sees the marker + /// tags. Removes ANY closed block, then trims a trailing open <summary> + /// (and everything after) so chunks mid-stream don't leak the partial + /// inner text into the visible Text. Always returns a trimmed string. /// + private static string StripSummaryBlock(string text) + { + if (string.IsNullOrEmpty(text)) return text; + var stripped = System.Text.RegularExpressions.Regex.Replace( + text, @"[\s\S]*?", "", + System.Text.RegularExpressions.RegexOptions.IgnoreCase); + var openIdx = stripped.LastIndexOf("", StringComparison.OrdinalIgnoreCase); + if (openIdx >= 0) + stripped = stripped[..openIdx]; + return stripped.TrimEnd(); + } + + /// + /// Normalizes a round's total token count: providers vary — some report a total, others + /// only in/out. Returns when present, else in+out (when either is + /// present), else null. STATIC on purpose — NOT a local function inside ExecuteMessageAsync: + /// a mutable-capturing local function threaded through that ~1400-line reactive method's + /// branches exploded Roslyn's nullable-flow/closure analysis and was the MeshWeaver.AI + /// ~10-minute compile cliff (build step 259s → 676s at commit e30e9b5f1). + /// + internal static int? NormalizeTotal(int? total, int? inputTokens, int? outputTokens) + => total ?? ((inputTokens.HasValue || outputTokens.HasValue) + ? (inputTokens ?? 0) + (outputTokens ?? 0) + : (int?)null); + internal static ImmutableList AggregateNodeChanges(ImmutableList entries) { if (entries.Count <= 1) return entries; @@ -1012,6 +2317,13 @@ internal static ImmutableList AggregateNodeChanges(ImmutableLis } } + private static CancellationTokenSource StoreNewCts(IMessageHub hub) + { + var cts = new CancellationTokenSource(); + hub.Set(cts); + return cts; + } + private static string? Truncate(string? value, int maxLength = 500) { if (value == null || value.Length <= maxLength) @@ -1024,6 +2336,78 @@ internal static ImmutableList AggregateNodeChanges(ImmutableLis /// Handles DelegationResult objects directly (no ToString → JSON round-trip). /// Falls back to JSON parsing for serialized results, then plain toString. /// + /// + /// Merge the in-memory toolCallLog with the cell's current persisted + /// ToolCalls. Both sources can write the same logical entry: + /// + /// Streaming loop appends bare entries (Result=null) on FCC FunctionCallContent. + /// writes a + /// terminal entry (Result+Status+DelegationPath) through cache.Update. + /// + /// Without merge, the streaming loop's final write would CLOBBER the cache's + /// terminal stamp. Pair entries by + /// when both have one; else by +positional + /// match. Prefer whichever has set + /// (terminal beats in-flight). Order: follow toolCallLog (in-stream order). + /// + private static ImmutableList MergeToolCallEntries( + ImmutableList current, ImmutableList incoming) + { + if (current.IsEmpty) return incoming; + if (incoming.IsEmpty) return current; + var consumedCurrent = new bool[current.Count]; + var builder = ImmutableList.CreateBuilder(); + foreach (var inc in incoming) + { + var idx = -1; + if (inc.DelegationPath is { } dp) + { + for (var i = 0; i < current.Count; i++) + { + if (!consumedCurrent[i] && current[i].DelegationPath == dp) + { idx = i; break; } + } + } + if (idx < 0) + { + for (var i = 0; i < current.Count; i++) + { + if (!consumedCurrent[i] && current[i].Name == inc.Name && current[i].Result != null && inc.Result == null) + { idx = i; break; } + } + } + if (idx >= 0) + { + consumedCurrent[idx] = true; + var cur = current[idx]; + // Prefer the side that's "further along" — Result populated + + // terminal Status. Field-by-field: keep cur.Result when inc.Result + // is null; keep cur.Status when it's terminal and inc is Streaming. + var preferred = inc.Result is null && cur.Result is not null ? cur : inc; + builder.Add(preferred with + { + DelegationPath = preferred.DelegationPath ?? cur.DelegationPath ?? inc.DelegationPath, + Result = preferred.Result ?? cur.Result ?? inc.Result, + Status = cur.Status != ToolCallStatus.Streaming && inc.Status == ToolCallStatus.Streaming + ? cur.Status : preferred.Status + }); + } + else + { + builder.Add(inc); + } + } + // Append any cell-only entries that incoming didn't carry (e.g. the + // terminal stamp may have landed but the streaming loop's snapshot was + // taken before the next FCC chunk re-appended the bare entry). + for (var i = 0; i < current.Count; i++) + { + if (!consumedCurrent[i] && current[i].DelegationPath is not null) + builder.Add(current[i]); + } + return builder.ToImmutable(); + } + private static (string? ResultText, string? DelegationPath, bool IsSuccess) ExtractToolResult(object? result) { if (result is null) @@ -1074,41 +2458,287 @@ private static (string? ResultText, string? DelegationPath, bool IsSuccess) Extr return (text, null, isSuccess); } - private static IMessageDelivery HandleCancelStream( - IMessageHub hub, IMessageDelivery delivery) + /// + /// Stream-update cancellation: clients set + /// = Cancelled on the thread node via + /// workspace.GetMeshNodeStream(threadPath).Update(...). The watcher + /// below observes the OWN thread node, treats every transition to + /// "RequestedStatus == Cancelled while executing" as a cancel signal, and + /// propagates that request onto every active delegation sub-thread. The round's OWN CTS + /// is cancelled by its per-round RequestedStatus self-cancel (see ExecuteMessageAsync), + /// not here; this watcher only handles sub-thread propagation and the claim-window No-CTS + /// fallback. + /// + /// Dedup is by : each round + /// has a distinct start timestamp, so DistinctUntilChanged on it acts + /// at most once per round. After the CTS is cancelled the streaming loop's + /// catch writes the terminal Status = Cancelled, RequestedStatus = null, + /// at which point the IsExecuting filter stops matching. A subsequent + /// round (new ExecutionStartedAt) re-arms the watcher. + /// + /// No-CTS fallback. If the request lands during the claim window + /// (StartingExecution, before the streaming loop stored its CTS) there + /// is no loop to write the terminal status — the watcher writes it directly, + /// guarded so it only fires while still StartingExecution. + /// + private static void InstallCancellationWatcher(IMessageHub hub) { var logger = hub.ServiceProvider.GetService>(); var threadPath = hub.Address.Path; - // Read Thread.StreamingToolCalls from workspace (runs on grain scheduler — safe). - // Find active delegation sub-threads and propagate cancel via Post (fire-and-forget). - hub.GetWorkspace().UpdateMeshNode(node => - { - var thread = node.Content as MeshThread; - if (thread?.StreamingToolCalls is { Count: > 0 }) - { - foreach (var tc in thread.StreamingToolCalls.Where( - tc => !string.IsNullOrEmpty(tc.DelegationPath) && tc.Result == null)) + var sub = hub.GetWorkspace().GetMeshNodeStream() + .Where(n => n?.Content is MeshThread t + && t.RequestedStatus == ThreadExecutionStatus.Cancelled + && t.IsExecuting) + // Dedup per round — distinct ExecutionStartedAt, so the cancel is handled at most + // once per round. The round's OWN cancellation is now driven by its per-round + // RequestedStatus self-cancel (ExecuteMessageAsync), which replays the current value + // and so is robust across the CTS-storage window AND a resume — so this watcher no + // longer needs to re-arm on CTS availability (the old (ExecutionStartedAt, HasCts) + // key). It only (a) propagates the cancel to sub-threads and (b) covers the pure + // claim-window case (Status stuck at StartingExecution) via the No-CTS fallback below. + .DistinctUntilChanged(n => ((MeshThread)n!.Content!).ExecutionStartedAt) + .Subscribe( + node => { - logger?.LogInformation("[ThreadExec] Propagating cancel to sub-thread {SubThread}", tc.DelegationPath); - hub.Post(new CancelThreadStreamRequest { ThreadPath = tc.DelegationPath! }, - o => o.WithTarget(new Address(tc.DelegationPath!))); - } - } - return node; // No state change needed - }); + var thread = (MeshThread)node!.Content!; + + // Propagate to every active delegation sub-thread via the + // canonical IMeshNodeStreamCache. The sub-thread is a + // non-own path; routing through the cache keeps a single + // shared handle for every reader (the sub-thread's own + // cancel watcher) and avoids opening an ad-hoc remote + // stream that subsequent readers wouldn't see. + // + // 🚨 Discover sub-thread paths from TWO sources: + // (a) thread.StreamingToolCalls — persisted via the + // streaming-loop throttle. STALE when the loop is + // blocked inside a delegate_to_agent call (which is + // exactly when we most need to cancel). + // (b) AgentChatClient.DelegationPaths — live in-memory + // registry on the parent's chat client, written + // synchronously by ExecuteDelegationAsync when each + // sub-thread is dispatched. Always current. + // Union of both: never miss a hung sub-thread whose path + // hasn't yet been throttle-persisted into (a). + // (Repro: SubThreadHangRepro.HungSubThread_UserCancel* + // demonstrated (a) alone fails to settle the sub-thread.) + var subPaths = ImmutableHashSet.Empty; + if (thread.StreamingToolCalls is { Count: > 0 }) + { + foreach (var tc in thread.StreamingToolCalls.Where( + tc => !string.IsNullOrEmpty(tc.DelegationPath) && tc.Result == null)) + subPaths = subPaths.Add(tc.DelegationPath!); + } + var chat = hub.Get(); + if (chat is not null) + { + foreach (var subPath in chat.ActiveDelegationPaths) + if (!string.IsNullOrEmpty(subPath)) + subPaths = subPaths.Add(subPath); + } - // Cancel own execution via CancellationTokenSource (streaming runs on thread pool) - if (ExecutionCancellations.TryGetValue(threadPath, out var cts)) - { - logger?.LogInformation("[ThreadExec] Cancelling own execution for {ThreadPath}", threadPath); - cts.Cancel(); - } + foreach (var subPath in subPaths) + { + logger?.LogInformation( + "[ThreadExec] Propagating cancel to sub-thread {SubThread}", subPath); + hub.GetWorkspace().GetMeshNodeStream(subPath).Update( + curr => curr?.Content is MeshThread sub + ? curr with { Content = sub with { RequestedStatus = ThreadExecutionStatus.Cancelled } } + : curr!) + .Subscribe(_ => { }, ex => logger?.LogWarning(ex, + "[ThreadExec] Cancel propagation failed for {SubThread}", subPath)); + } - // Post response so parent can await confirmation - hub.Post(new CancelThreadStreamResponse { ThreadPath = threadPath }, - o => o.ResponseFor(delivery)); + // The round's OWN CTS is cancelled by its per-round RequestedStatus + // self-cancel (ExecuteMessageAsync) — robust across the CTS-storage window and + // a resume — so no cts.Cancel() is needed here. The ONLY case the self-cancel + // can't cover is a cancel that lands in the CLAIM window (StartingExecution, + // before the round and its self-cancel exist): there is no streaming loop to + // write the terminal status, so write it directly — guarded to fire only while + // still StartingExecution with the cancel still requested, so we never clobber a + // round that has since reached Executing (its loop owns the terminal write). + if (hub.Get() is null) + { + logger?.LogDebug( + "[ThreadExec] Cancel: no CTS for {ThreadPath} (claim window) — writing terminal Cancelled directly", + threadPath); + hub.GetWorkspace().GetMeshNodeStream().Update( + curr => curr?.Content is MeshThread t + && t.Status == ThreadExecutionStatus.StartingExecution + && t.RequestedStatus == ThreadExecutionStatus.Cancelled + ? curr with + { + Content = t with + { + Status = ThreadExecutionStatus.Cancelled, + RequestedStatus = null, + ActiveMessageId = null, + ExecutionStartedAt = null, + ExecutionStatus = null, + } + } + : curr!) + .Subscribe(_ => { }, ex => logger?.LogWarning(ex, + "[ThreadExec] No-CTS cancel fallback failed for {ThreadPath}", threadPath)); + } + }, + ex => logger?.LogWarning(ex, + "[ThreadExec] Cancellation watcher faulted for {ThreadPath}", threadPath)); + + hub.RegisterForDisposal(sub); + } + + /// + /// Loads ALL prior ThreadMessage cells (both user and assistant) for the + /// thread, excluding the current submission's user cell and any cell with + /// empty text (e.g. the just-created in-flight response cell). Ordered by + /// timestamp. Used per-round to give the agent full conversation context — + /// without this, every round only sees the new user message and tests like + /// ChatHistoryTest see "I received 2 messages" forever. + /// + internal static IObservable> LoadFullConversationHistoryFromMesh( + IMessageHub hub, string threadPath, string? excludeUserMessageId, string? excludeResponseMessageId, + ILogger logger, TimeSpan? cellTimeout = null) + { + var perCellTimeout = cellTimeout ?? TimeSpan.FromSeconds(5); + // 🚨 Read thread + each cell from IMeshNodeStreamCache — the hot, shared, + // path-keyed Replay(1) handle every consumer subscribes to. The same cache + // that the per-node hub's writes flow through, so reads here observe the + // exact post-write state without going through IMeshQueryCore (which lags). + // Walk the thread's Messages property for the cell IDs (authoritative ordered + // list of cells in this thread). + return hub.GetMeshNodeStream(threadPath) + .Take(1) + .Timeout(TimeSpan.FromSeconds(10)) + .Select(threadNode => threadNode.Content as MeshThread) + .Where(t => t != null) + .SelectMany(thread => + { + var cellIds = thread!.Messages + .Where(id => id != excludeUserMessageId && id != excludeResponseMessageId) + .ToList(); + if (cellIds.Count == 0) + return Observable.Return>(Array.Empty()); + + // 🚨 Fan out cell reads in PARALLEL via CombineLatest — each + // `cache.GetStream(path)` is its own hot Replay(1), so they all + // subscribe / receive content concurrently. The serial `.Concat()` + // shape was waiting up to N × budget when cells were cold. + // Per-cell semantics: wait for content with text (cache may emit a + // pre-text shell first), Take(1), then Timeout(perCellTimeout). On + // per-cell failure → emit a sentinel null so CombineLatest still fires + // — the projector filters nulls and the caller decides what to do + // (warn + proceed with partial / throw if zero loaded). + var cellLookups = cellIds + .Select(id => + hub.GetMeshNodeStream($"{threadPath}/{id}") + .Where(n => n.Content is ThreadMessage m && !string.IsNullOrEmpty(m.Text)) + .Take(1) + .Timeout(perCellTimeout) + .Select(n => (MeshNode?)n) + .Catch(ex => + { + logger.LogWarning(ex, + "[ThreadExec] HISTORY_CELL_DROP threadPath={ThreadPath} cellId={CellId} — cell unreadable within budget; will be omitted", + threadPath, id); + return Observable.Return(null); + })) + .ToList(); + + return Observable.CombineLatest(cellLookups) + .Take(1) + .Select(nodes => + { + var messages = nodes + .Where(n => n is not null) + .Select(n => (ThreadMessage)n!.Content!) + .OrderBy(m => m.Timestamp) + .Select(m => + { + var role = string.Equals(m.Role, "user", StringComparison.OrdinalIgnoreCase) + ? ChatRole.User + : ChatRole.Assistant; + return new ChatMessage(role, m.Text); + }) + .ToList(); + + // 🚨 Hard failure if every cell dropped despite expecting some: + // submitting an EMPTY history when the thread has prior turns + // would silently corrupt the agent's context (ChatHistoryTest + // would assert "5 messages" instead of "4"). Surface as a + // TimeoutException so the round fails loud instead of producing + // a misleading assertion downstream. + if (cellIds.Count > 0 && messages.Count == 0) + throw new TimeoutException( + $"LoadFullConversationHistoryFromMesh: expected {cellIds.Count} prior cells " + + $"for {threadPath} but ALL timed out / lacked text. Refusing to submit empty history."); + + if (messages.Count < cellIds.Count) + logger.LogWarning( + "[ThreadExec] HISTORY_PARTIAL threadPath={ThreadPath} loaded={Loaded}/{Expected} — proceeding with partial history", + threadPath, messages.Count, cellIds.Count); + + return (IReadOnlyList)messages; + }); + }); + } - return delivery.Processed(); + /// + /// Loads prior user-message ThreadMessage cells for + /// by walking the live thread's Messages list and resolving each cell + /// via GetMeshNodeStream (per-node hub) — the authoritative live read + /// path. Filters to user-role cells, excludes + /// (the current submission, whose text already comes via + /// request.UserMessageText), and orders by timestamp. Called only on + /// AgentChatClient cache miss (post-restart resume). The returned list is fed + /// straight into the AgentChatClient constructor. + /// + private static IObservable> LoadPriorUserMessagesFromMesh( + IMessageHub hub, string threadPath, string? excludeMessageId, ILogger logger) + { + return hub.GetMeshNodeStream(threadPath) + .Take(1) + .Timeout(TimeSpan.FromSeconds(10)) + .Select(threadNode => threadNode.Content as MeshThread) + .Where(t => t != null) + .SelectMany(thread => + { + var cellIds = thread!.Messages + .Where(id => excludeMessageId == null || id != excludeMessageId) + .ToList(); + if (cellIds.Count == 0) + return Observable.Return>(Array.Empty()); + + // 🚨 Subscribe-all-upfront via Observable.CombineLatest — N + // per-cell synchronization streams subscribe simultaneously when + // the consumer subscribes, so the N hub activations are + // concurrent (≈ max(t_i)) instead of serial Σ(t_i) via .Concat(). + // Catch returns sentinel null so a single slow cell doesn't strand + // the load. See AsynchronousCalls.md → "Subscribe-all-upfront cell loading". + var cellLookups = cellIds.Select(id => + hub.GetMeshNodeStream($"{threadPath}/{id}") + .Take(1) + .Timeout(TimeSpan.FromSeconds(5)) + .Select(n => (MeshNode?)n) + .Catch(_ => Observable.Return(null))); + + return Observable.CombineLatest(cellLookups) + .Take(1) + .Select(nodes => (IReadOnlyList)nodes + .Where(n => n != null) + .Select(n => n!.Content as ThreadMessage) + .Where(m => m != null && string.Equals(m.Role, "user", StringComparison.OrdinalIgnoreCase)) + .OrderBy(m => m!.Timestamp) + .Cast() + .ToList()); + }) + .Catch, Exception>(ex => + { + logger.LogWarning(ex, + "[ThreadExec] LoadPriorUserMessages: failed/timed out for {ThreadPath} — empty history", + threadPath); + return Observable.Return>(Array.Empty()); + }); } } diff --git a/src/MeshWeaver.AI/ThreadFlow.cs b/src/MeshWeaver.AI/ThreadFlow.cs new file mode 100644 index 000000000..356a93707 --- /dev/null +++ b/src/MeshWeaver.AI/ThreadFlow.cs @@ -0,0 +1,157 @@ +using System.Reactive.Linq; +using MeshWeaver.Data; +using MeshWeaver.Mesh; +using MeshWeaver.Messaging; +using MeshThread = MeshWeaver.AI.Thread; + +namespace MeshWeaver.AI; + +/// +/// Reactive read-side primitives for observing a thread chat flow from any +/// CLIENT — Blazor side panel, MCP clients, tests. Writes go through the +/// canonical surface +/// (workspace.SubmitMessage, workspace.StartThread, etc.); +/// this class only exposes the matching read-side primitives: +/// +/// workspace.GetMeshNodeStream(path) — the cache-routed, +/// write-coherent thread stream +/// — convenience composition of +/// + a wait +/// for the round to land +/// +/// +/// All methods return . Per +/// Doc/Architecture/AsynchronousCalls.md: no Task<T> on the +/// public surface, no async/await in mesh-reachable code. Tests +/// bridge to Task at their edge via +/// .FirstAsync().ToTask(ct). +/// +/// Always invoked on the CLIENT workspace — the per-thread hub's own +/// workspace would block on the active submission handler. +/// +public static class ThreadFlow +{ + /// + /// Live observable of the at + /// . Subscribes via the cache-routed + /// workspace.GetMeshNodeStream(path) primitive (same handle every + /// reader/writer shares — write-coherent with the GUI and other clients). + /// Filters out empty / null-content emissions so subscribers only see + /// real thread state. + /// + public static IObservable ObserveThread( + IMessageHub client, string threadPath) => + client.GetWorkspace().GetMeshNodeStream(threadPath) + .Select(n => n.Content as MeshThread) + .Where(t => t != null) + .Select(t => t!); + + /// + /// Live observable of — the message + /// id list as the thread evolves. Equivalent to the GUI's data-binding + /// subscription. Stays subscribed. + /// + public static IObservable> ObserveMessages( + IMessageHub client, string threadPath) => + ObserveThread(client, threadPath) + .Select(t => (IReadOnlyList)t.Messages); + + /// + /// Live read of the at + /// {threadPath}/{msgId}. Subscribes to the message satellite's + /// remote stream and waits until holds + /// (default: Text non-empty) then completes. + /// + public static IObservable ReadMessage( + IMessageHub client, string threadPath, string msgId, + Func? predicate = null, + TimeSpan? timeout = null) + { + predicate ??= m => !string.IsNullOrEmpty(m.Text); + timeout ??= TimeSpan.FromSeconds(15); + return client.GetWorkspace().GetMeshNodeStream($"{threadPath}/{msgId}") + .Select(n => n.Content as ThreadMessage) + .Where(m => m != null && predicate(m)) + .Take(1) + .Timeout(timeout.Value) + .Select(m => m!); + } + + /// + /// Waits for the thread to satisfy , then + /// completes with the matching thread snapshot. Same remote-stream + /// primitive as . + /// + public static IObservable ReadThread( + IMessageHub client, string threadPath, + Func? predicate = null, + TimeSpan? timeout = null) + { + predicate ??= _ => true; + timeout ??= TimeSpan.FromSeconds(30); + return ObserveThread(client, threadPath) + .Where(t => predicate(t)) + .Take(1) + .Timeout(timeout.Value); + } + + /// + /// Emits each completed agent reply as the thread runs — one per round. This is the single + /// read-side abstraction for delivering an agent's answer anywhere (Teams reply sender, email reply + /// sender, GUI): observe the thread the canonical way, and each time a round finishes + /// (!IsExecuting) read the new last message at {threadPath}/{msgId} once it is a + /// Completed assistant message with text. Emits the message id alongside so callers can + /// de-duplicate / mark-delivered. Stays subscribed (ongoing, unlike one-shot ). + /// + public static IObservable<(string MessageId, ThreadMessage Message)> ObserveResponses( + IMessageHub client, string threadPath, TimeSpan? readTimeout = null) => + ObserveThread(client, threadPath) + .Where(t => !t.IsExecuting && t.Messages.Count > 0) + .Select(t => t.Messages[^1]) + .DistinctUntilChanged() + .SelectMany(msgId => ReadMessage(client, threadPath, msgId, + m => m.Status == ThreadMessageStatus.Completed + && string.Equals(m.Role, "assistant", StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(m.Text), + readTimeout ?? TimeSpan.FromSeconds(30)) + .Select(m => (msgId, m)) + .Catch(Observable.Empty<(string MessageId, ThreadMessage Message)>())); + + /// + /// Submits via the GUI path then emits exactly once when the resulting + /// round completes — the response message id (last entry of + /// Thread.Messages after IsExecuting flips back to false + /// AND Messages.Count grew past the pre-submit baseline). + /// + /// Reactive end-to-end. Captures baseline from the thread stream + /// once (.Take(1)), fires Submit inside SelectMany, then + /// waits on the SAME stream for the next Idle frame whose count grew. + /// The whole chain is one observable — no Task, no await, + /// no scheduler bridge. Works for first-submit (baseline=0, count→2) + /// AND subsequent submits on an existing thread (baseline=N, count→N+2). + /// + public static IObservable SubmitAndWait( + IMessageHub client, string threadPath, string userText, + string? contextPath = null, string? agentName = null, string? modelName = null, + TimeSpan? timeout = null) => Observable.Defer(() => + { + timeout ??= TimeSpan.FromSeconds(30); + var thread = ObserveThread(client, threadPath); + + return thread + .Select(t => t.Messages.Count) + .Take(1) + .Timeout(timeout.Value) + .SelectMany(baseline => + { + client.SubmitMessage( + threadPath, userText, + agentName: agentName, modelName: modelName, contextPath: contextPath); + return thread + .Where(t => !t.IsExecuting && t.Messages.Count > baseline) + .Select(t => t.Messages[^1]) + .Take(1) + .Timeout(timeout.Value); + }); + }); +} diff --git a/src/MeshWeaver.AI/ThreadInput.cs b/src/MeshWeaver.AI/ThreadInput.cs index cea3b6943..4ac1de2c1 100644 --- a/src/MeshWeaver.AI/ThreadInput.cs +++ b/src/MeshWeaver.AI/ThreadInput.cs @@ -2,6 +2,8 @@ using MeshWeaver.Graph; using MeshWeaver.Mesh; using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using MeshThread = MeshWeaver.AI.Thread; namespace MeshWeaver.AI; @@ -15,7 +17,7 @@ namespace MeshWeaver.AI; /// watcher creates the satellite cell and dispatches the next round. /// /// This replaces the legacy two-message dance (CreateNodeRequest + -/// AppendUserMessageRequest), eliminating the duplicate-dispatch races caused +/// ThreadInput.AppendUserInput), eliminating the duplicate-dispatch races caused /// by interleaved fire-and-forget posts. /// public static class ThreadInput @@ -32,7 +34,8 @@ public static ThreadMessage CreateUserMessage( string? agentName = null, string? modelName = null, string? contextPath = null, - IReadOnlyList? attachments = null) => + IReadOnlyList? attachments = null, + string? harness = null) => new() { Role = "user", @@ -41,18 +44,74 @@ public static ThreadMessage CreateUserMessage( CreatedBy = createdBy, AgentName = agentName, ModelName = modelName, + Harness = harness, ContextPath = contextPath, Attachments = attachments, Timestamp = DateTime.UtcNow, - Type = ThreadMessageType.ExecutedInput + Type = ThreadMessageType.ExecutedInput, + // User cells don't have a streaming lifecycle — Submitted on creation. + // The "queued vs ingested" indicator is derived at the thread level + // (UserMessageIds minus IngestedMessageIds) so the UI can render + // queued user cells with a "waiting in queue" treatment without + // needing per-cell mutations on dispatch. + Status = ThreadMessageStatus.Submitted }; /// - /// Atomically appends a user message to via a - /// single workspace.UpdateMeshNode on the thread's MeshNode. Returns - /// the generated message id. The server-side submission watcher creates the - /// satellite cell from and - /// dispatches the next round. + /// Pure: applies a user input to a thread's state — adds to + /// (preserving submission order), stashes the + /// message in (the queue the inbox drains), and + /// keeps the thread's authoritative for the round's sticky + /// selection (agent / model / harness): when the message carries one, it is folded into + /// the composer. The round reads the selection from (the + /// single source of truth), NOT from a thread-level Pending* mirror nor from the + /// per-message copy. Shared by (the SubmitMessage + /// path, which carries explicit selection params) and HubThreadExtensions.SubmitComposer + /// (which already built the message FROM the composer) so both paths leave the composer + /// current. + /// + public static MeshThread ApplyUserInput(MeshThread thread, string msgId, ThreadMessage message) + { + var userIds = thread.UserMessageIds.Contains(msgId) + ? thread.UserMessageIds + : thread.UserMessageIds.Add(msgId); + var composer = (thread.Composer ?? new ThreadComposer()) with + { + AgentName = message.AgentName ?? thread.Composer?.AgentName, + ModelName = message.ModelName ?? thread.Composer?.ModelName, + Harness = message.Harness ?? thread.Composer?.Harness + }; + return thread with + { + UserMessageIds = userIds, + PendingUserMessages = thread.PendingUserMessages.SetItem(msgId, message), + Composer = composer + }; + } + + /// + /// Appends a user message into on + /// . Returns the generated message id. + /// + /// Inbox pattern. This call only writes the pending dict + + /// on the thread node. It does NOT + /// materialise a user satellite cell and does NOT add the id to + /// — both happen later, at ingestion time, + /// when the inbox drains the queue. See + /// ThreadSubmissionServer.DispatchRound (round-start ingestion) + /// and check_inbox (mid-stream ingestion). + /// + /// GUI binding. While an entry sits in + /// the chat view renders it as a + /// "queued" / "not yet submitted" cell from the dictionary directly. The + /// transition to a materialised cell in + /// (the "submitted" / "picked up by inbox" state) happens when the inbox + /// drains. + /// + /// Pending-not-empty + IsExecuting=false wakes the submission + /// watcher, which dispatches a new round. While + /// IsExecuting=true, the running round's check_inbox tool + /// drains the queue into its current response cell. /// public static string AppendUserInput( IWorkspace workspace, @@ -65,42 +124,49 @@ public static string AppendUserInput( ArgumentNullException.ThrowIfNull(message); var msgId = NewId(); + var logger = workspace.Hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.AI.ThreadInput"); + // Promoted from Debug → Info so the delegation-test diagnostics show + // the submit chain (entry → update lambda → OnNext) at default log level. + logger?.LogInformation( + "[AppendUserInput] ENTRY workspace.Hub={Hub} threadPath={ThreadPath} msgId={MsgId} agent={Agent} model={Model}", + workspace.Hub.Address, threadPath, msgId, + message.AgentName ?? "(null)", message.ModelName ?? "(null)"); - // Append the message to PendingUserMessages + UserMessageIds only. - // - // We deliberately do NOT add to Thread.Messages here — the GUI renders one - // LayoutAreaControl per id in Messages, and rendering a control before its - // satellite ThreadMessage node has been created on the hub triggers - // "Cannot access a disposed object" + spurious area-stream errors. The - // server-side submission watcher creates the satellite cell first via - // IMeshService.CreateNode and only after CreateNode confirms success does - // it add the id into Messages (in the same atomic update that flips - // IsExecuting=true alongside the response cell id). + // Single atomic update on the thread node: add to UserMessageIds (preserves + // submission order for ingestion) and PendingUserMessages (the queue the + // inbox drains). NOT Messages — Messages is the materialised list updated + // by the inbox at ingestion time. Updates Pending* hints for the next + // round's dispatch context. // - // Using the no-address overload (FirstOrDefault) avoids a pre-existing - // path-vs-id key mismatch in the address-aware overload. This call expects - // to run on the thread's own hub (e.g., from the AppendUserMessageRequest - // handler) where there's exactly one node in the collection. - workspace.UpdateMeshNode(node => + // Writes via the caller's own workspace handle — sender is THIS hub + // (portal / thread / wherever AppendUserInput is called from), and the + // AsyncLocal AccessContext stamps the caller's identity. With delta- + // based PatchDataRequest in MeshNodeStreamHandle.UpdateRemote, concurrent + // writes from different mirrors no longer clobber each other, so we no + // longer need to funnel through the mesh-hub cache (which would erase + // the caller's identity and surface as 'no AccessContext' warnings). + workspace.GetMeshNodeStream(threadPath).Update(node => { - var thread = node.Content as MeshThread ?? new MeshThread(); - var userIds = thread.UserMessageIds.Contains(msgId) - ? thread.UserMessageIds - : thread.UserMessageIds.Add(msgId); - var pending = thread.PendingUserMessages.SetItem(msgId, message); - return node with - { - Content = thread with - { - UserMessageIds = userIds, - PendingUserMessages = pending, - PendingAgentName = message.AgentName ?? thread.PendingAgentName, - PendingModelName = message.ModelName ?? thread.PendingModelName, - PendingContextPath = message.ContextPath ?? thread.PendingContextPath, - PendingAttachments = message.Attachments ?? thread.PendingAttachments - } - }; - }); + logger?.LogInformation( + "[AppendUserInput] UPDATE_LAMBDA invoked for {ThreadPath} (node.Path={NodePath} contentType={ContentType})", + threadPath, node.Path ?? "(null)", + node.Content?.GetType().Name ?? "(null)"); + var thread = node.ContentAs(workspace.Hub.JsonSerializerOptions, logger); + // Existing node whose content can't be recovered → leave it alone, NEVER clobber. + if (node.Content is not null && thread is null) + return node; + thread ??= new MeshThread(); + return node with { Content = ApplyUserInput(thread, msgId, message) }; + }).Subscribe( + updated => logger?.LogInformation( + "[AppendUserInput] ON_NEXT for {ThreadPath} msgId={MsgId} — userIds={UserIds} pending={Pending}", + threadPath, msgId, + (updated.Content as MeshThread)?.UserMessageIds.Count ?? -1, + (updated.Content as MeshThread)?.PendingUserMessages.Count ?? -1), + ex => logger?.LogWarning(ex, + "[AppendUserInput] UpdateMeshNode FAILED for thread {ThreadPath} message {MessageId}", + threadPath, msgId)); return msgId; } diff --git a/src/MeshWeaver.AI/ThreadLayoutAreas.cs b/src/MeshWeaver.AI/ThreadLayoutAreas.cs index 81ac18aa1..274b47bdb 100644 --- a/src/MeshWeaver.AI/ThreadLayoutAreas.cs +++ b/src/MeshWeaver.AI/ThreadLayoutAreas.cs @@ -1,4 +1,4 @@ -using System.Collections.Immutable; +using System.Collections.Immutable; using System.Reactive.Linq; using MeshWeaver.Application.Styles; using MeshWeaver.Data; @@ -8,6 +8,7 @@ using MeshWeaver.Layout; using MeshWeaver.Layout.Composition; using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; using MeshWeaver.Mesh.Services; using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; @@ -33,11 +34,13 @@ public static class ThreadLayoutAreas /// public static MessageHubConfiguration AddThreadLayoutAreas(this MessageHubConfiguration configuration) => configuration - .WithHandler(ThreadMessageHandlers.HandleResubmitMessage) - .WithHandler(ThreadMessageHandlers.HandleDeleteFromMessage) + // Legacy ThreadSubmission.ApplyResubmit / ThreadSubmission.ApplyDeleteFromMessage handlers + // removed — click actions now call ThreadSubmission.ApplyResubmit / + // ApplyDeleteFromMessage directly. See RequestViaStreamUpdate.md. .AddDefaultMeshMenu() .AddNodeMenuItems("SidePanel", SidePanelMenuProvider) .AddNodeMenuItems(DelegationsMenuProvider) + .AddNodeMenuItems(ChangesMenuProvider) .AddLayout(layout => layout .WithDefaultArea(ThreadNodeType.ThreadArea) .WithView(ThreadNodeType.ThreadArea, ThreadView) @@ -46,31 +49,51 @@ public static MessageHubConfiguration AddThreadLayoutAreas(this MessageHubConfig .WithView(ThreadNodeType.StreamingArea, StreamingView) .WithView(ThreadNodeType.HistoryArea, HistoryView) .WithView(ThreadNodeType.HeaderArea, HeaderView) + .WithView(ThreadNodeType.ChangesArea, ChangesAreaView) + // The thread's own composer selectors — binds Thread.Composer (the composer + // copied onto the thread at creation), same wiring as the composer node's area. + .WithView(ThreadComposerView.SelectorsArea, ThreadComposerView.ComposerSelectors) .WithView(MeshNodeLayoutAreas.ThumbnailArea, Thumbnail) .WithView(MeshNodeLayoutAreas.ThreadsArea, ThreadsCatalog)); /// - /// Side panel menu items (New Chat, History, Full Screen). + /// Side panel menu items (New Chat, History, Full Screen). Static set — emitted once. /// - private static async IAsyncEnumerable SidePanelMenuProvider( + private static IObservable> SidePanelMenuProvider( + LayoutAreaHost host, RenderingContext ctx) + => Observable.Return>( + [ + new("New Chat", "new-chat", Order: 0), + new("History", "history", Order: 10), + new("Full Screen", "fullscreen", Order: 20), + ]); + + /// + /// Main menu item: Delegations (sub-thread history). + /// + private static IObservable> DelegationsMenuProvider( LayoutAreaHost host, RenderingContext ctx) { - await Task.CompletedTask; - yield return new("New Chat", "new-chat", Order: 0); - yield return new("History", "history", Order: 10); - yield return new("Full Screen", "fullscreen", Order: 20); + var hubPath = host.Hub.Address.ToString(); + return Observable.Return>( + [ + new("Delegations", ThreadNodeType.HistoryArea, Order: 12, + Href: MeshNodeLayoutAreas.BuildUrl(hubPath, ThreadNodeType.HistoryArea)), + ]); } /// - /// Main menu item: Delegations (sub-thread history). + /// Main menu item: Changes (aggregated node modifications + bulk revert). /// - private static async IAsyncEnumerable DelegationsMenuProvider( + private static IObservable> ChangesMenuProvider( LayoutAreaHost host, RenderingContext ctx) { - await Task.CompletedTask; var hubPath = host.Hub.Address.ToString(); - yield return new("Delegations", ThreadNodeType.HistoryArea, Order: 12, - Href: MeshNodeLayoutAreas.BuildUrl(hubPath, ThreadNodeType.HistoryArea)); + return Observable.Return>( + [ + new("Changes", ThreadNodeType.ChangesArea, Order: 13, + Href: MeshNodeLayoutAreas.BuildUrl(hubPath, ThreadNodeType.ChangesArea)), + ]); } private static string GetContextDisplayName(string path) @@ -100,7 +123,9 @@ public static UiControl ThreadsCatalog(LayoutAreaHost host, RenderingContext _) .WithIconStart(FluentIcons.Add()) .WithNavigateToHref(createUrl))) .WithView(Controls.MeshSearch - .WithHiddenQuery($"namespace:{hubPath}/{ThreadNodeType.ThreadPartition} nodeType:{ThreadNodeType.NodeType} sort:lastModified-desc") + // -content.status:Done hides finished threads from the catalog; + // user can type `content.status:Done` in the search box to surface them. + .WithHiddenQuery($"namespace:{hubPath}/{ThreadNodeType.ThreadPartition} nodeType:{ThreadNodeType.NodeType} -content.status:Done sort:lastModified-desc") .WithPlaceholder("Search threads...") .WithRenderMode(MeshSearchRenderMode.Flat) .WithMaxColumns(3)); @@ -125,13 +150,10 @@ public static UiControl ThreadsCatalog(LayoutAreaHost host, RenderingContext _) { var hubPath = host.Hub.Address.ToString(); - // Node stream — drives the observable title and chat control context - var stream = host.Workspace.GetStream(); - + // OWN MeshNode stream — drives the observable title and chat control context. // Push ThreadViewModel to data section — contains all thread state for the Blazor view. - var vmStream = stream!.Select(nodes => + var vmStream = host.Workspace.GetMeshNodeStream().Select(node => { - var node = nodes!.First(n => n.Path == hubPath); var threadContent = node?.Content as MeshThread; var contextPath = node?.MainNode != node?.Path ? node?.MainNode : hubPath; var contextDisplayName = node?.MainNode != node?.Path @@ -146,46 +168,137 @@ public static UiControl ThreadsCatalog(LayoutAreaHost host, RenderingContext _) ExecutionStatus = threadContent?.ExecutionStatus, StreamingText = threadContent?.StreamingText, StreamingToolCalls = threadContent?.StreamingToolCalls, - TokensUsed = threadContent?.TokensUsed ?? 0, + PendingMessageTexts = ExtractPendingTexts(threadContent), ExecutionStartedAt = threadContent?.ExecutionStartedAt, + CreatedBy = node?.CreatedBy, }; }); host.RegisterForDisposal(vmStream.DistinctUntilChanged().Subscribe(vm => host.UpdateData(ThreadDataKey, vm))); - // Push title to data section — data-bound, no observable control rebuild - var titleStream = stream!.Select(nodes => - { - var node = nodes!.First(n => n.Path == hubPath); - return GetThreadTitle(node); - }).DistinctUntilChanged(); + // Push title to data section — data-bound, no observable control rebuild. + var titleStream = host.Workspace.GetMeshNodeStream() + .Select(GetThreadTitle) + .DistinctUntilChanged(); host.RegisterForDisposal(titleStream.Subscribe(title => host.UpdateData("title", title))); - // Push context link HTML to data section + // Push description to data section — read-only viewers see it under the title + // (users with Update permission get an inline auto-saving editor instead). Empty + // string when unset so the read-only Html view renders nothing. + host.RegisterForDisposal(host.Workspace.GetMeshNodeStream() + .Select(node => node?.Description ?? string.Empty) + .DistinctUntilChanged() + .Subscribe(desc => host.UpdateData("description", + string.IsNullOrWhiteSpace(desc) + ? string.Empty + : "
    " + + System.Web.HttpUtility.HtmlEncode(desc) + "
    "))); + + // Push context link HTML to data section — pill-shaped breadcrumb chip. host.RegisterForDisposal(vmStream.DistinctUntilChanged().Subscribe(vm => { if (!string.IsNullOrEmpty(vm.InitialContext)) { var displayName = System.Web.HttpUtility.HtmlEncode(vm.InitialContextDisplayName ?? vm.InitialContext); host.UpdateData("contextLink", - $"" + - $"← {displayName}"); + ""); } })); - // Header: chat icon + context link + h1 title (hidden in side panel via CSS) + // Push subtitle (message count) to the data section — reactive. + host.RegisterForDisposal(vmStream + .Select(vm => vm.Messages?.Count ?? 0) + .DistinctUntilChanged() + .Subscribe(count => + { + var label = count switch + { + 0 => "No messages yet", + 1 => "1 message", + _ => $"{count} messages" + }; + host.UpdateData("subtitle", + "
    " + + "" + + $"{label}" + + "
    " + + ""); + })); + + // Hero header: gradient surface, big glowing chat icon, title + live subtitle. var header = Controls.Stack .WithClass("thread-full-header") .WithWidth("100%") - .WithStyle("padding: 16px 24px 24px 24px; margin-bottom: 24px; border-bottom: 1px solid var(--neutral-stroke-rest);") + .WithStyle( + "padding: 24px 28px 22px 28px; margin-bottom: 20px; " + + "border-radius: 14px; gap: 10px; " + + "background: linear-gradient(135deg, " + + "color-mix(in srgb, var(--accent-fill-rest) 8%, var(--neutral-layer-1)), " + + "var(--neutral-layer-1) 70%); " + + "border: 1px solid color-mix(in srgb, var(--accent-fill-rest) 18%, var(--neutral-stroke-rest)); " + + "box-shadow: 0 6px 24px -8px color-mix(in srgb, var(--accent-fill-rest) 25%, transparent);") .WithView(Controls.Html(new JsonPointerReference(LayoutAreaReference.GetDataPointer("contextLink")))) .WithView(Controls.Stack .WithOrientation(Orientation.Horizontal) - .WithStyle("align-items: center; gap: 16px;") + .WithStyle("align-items: center; gap: 18px; flex: 1; min-width: 0;") .WithView(Controls.Html( - "\"\"")) - .WithView(Controls.Html(new JsonPointerReference(LayoutAreaReference.GetDataPointer("title"))) - .WithStyle("margin: 0; font-size: 2rem; font-weight: bold;"))); + "
    " + + "\"\"" + + "
    ")) + .WithView(Controls.Stack + .WithStyle("gap: 4px; min-width: 0; flex: 1;") + // Title + description: inline auto-saving editors for users with Update + // permission; read-only display otherwise. Bound DIRECTLY to MeshNode.Name / + // MeshNode.Description (fields-mode node-bound DataContext) — one source of + // truth, no /data replica, no save subscription. See Doc/GUI/DataBinding. + .WithView((h, _) => h.Hub.GetEffectivePermissions(hubPath) + .Select(p => p.HasFlag(Permission.Update)) + .DistinctUntilChanged() + .Select(canEdit => (UiControl?)BuildTitleEditor(hubPath, canEdit))) + .WithView(Controls.Html(new JsonPointerReference(LayoutAreaReference.GetDataPointer("subtitle"))))) + // Mark Done / Reopen toggle — reactive. Hidden while the + // thread is executing (MarkThreadDone's CAS guard would + // refuse anyway, but hiding the button is cleaner UX). + .WithView((h, _) => h.Workspace.GetMeshNodeStream() + .Select(node => + { + var t = node?.Content as MeshThread; + if (t is null || t.IsExecuting) return (UiControl?)null; + var isDone = t.Status == ThreadExecutionStatus.Done; + var label = isDone ? "Reopen" : "Mark Done"; + var icon = isDone + ? FluentIcons.ArrowUndo(IconSize.Size16) + : FluentIcons.Checkmark(IconSize.Size16); + return (UiControl?)Controls.Button(label) + .WithAppearance(isDone ? Appearance.Neutral : Appearance.Accent) + .WithIconStart(icon) + .WithClickAction(_ => + h.Hub.MarkThreadDone(hubPath, !isDone)); + }))); // Static container — never rebuilt return Controls.Stack @@ -205,11 +318,10 @@ public static UiControl ThreadsCatalog(LayoutAreaHost host, RenderingContext _) public static UiControl? ThreadChatView(LayoutAreaHost host, RenderingContext _) { var hubPath = host.Hub.Address.ToString(); - var stream = host.Workspace.GetStream(); + var ownNodeStream = host.Workspace.GetMeshNodeStream(); - var vmStream = stream!.Select(nodes => + var vmStream = ownNodeStream.Select(node => { - var node = nodes!.First(n => n.Path == hubPath); var threadContent = node?.Content as MeshThread; var contextPath = node?.MainNode != node?.Path ? node?.MainNode : hubPath; var contextDisplayName = node?.MainNode != node?.Path @@ -224,18 +336,17 @@ public static UiControl ThreadsCatalog(LayoutAreaHost host, RenderingContext _) ExecutionStatus = threadContent?.ExecutionStatus, StreamingText = threadContent?.StreamingText, StreamingToolCalls = threadContent?.StreamingToolCalls, - TokensUsed = threadContent?.TokensUsed ?? 0, + PendingMessageTexts = ExtractPendingTexts(threadContent), ExecutionStartedAt = threadContent?.ExecutionStartedAt, + CreatedBy = node?.CreatedBy, }; }); host.RegisterForDisposal(vmStream.DistinctUntilChanged().Subscribe(vm => host.UpdateData(ThreadDataKey, vm))); - // Push title to data section so the side panel header can read it - var titleStream = stream!.Select(nodes => - { - var node = nodes!.First(n => n.Path == hubPath); - return GetThreadTitle(node); - }).DistinctUntilChanged(); + // Push title to data section so the side panel header can read it. + var titleStream = ownNodeStream + .Select(GetThreadTitle) + .DistinctUntilChanged(); host.RegisterForDisposal(titleStream.Subscribe(title => host.UpdateData("title", title))); return new ThreadChatControl() @@ -252,12 +363,9 @@ public static UiControl ThreadsCatalog(LayoutAreaHost host, RenderingContext _) public static IObservable ThreadProgressView(LayoutAreaHost host, RenderingContext _) { var hubPath = host.Hub.Address.ToString(); - var stream = host.Workspace.GetStream(); - - return stream! - .Select(nodes => + return host.Workspace.GetMeshNodeStream() + .Select(node => { - var node = nodes!.FirstOrDefault(n => n.Path == hubPath); var thread = node?.Content as MeshThread; if (thread == null) return (UiControl?)null; @@ -286,12 +394,13 @@ public static UiControl ThreadsCatalog(LayoutAreaHost host, RenderingContext _) public static IObservable StreamingView(LayoutAreaHost host, RenderingContext _) { var hubPath = host.Hub.Address.ToString(); - var stream = host.Workspace.GetStream(); + var logger = host.Hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.AI.StreamingView"); + logger?.LogDebug("[StreamingView] SUBSCRIBE hub={Hub}", hubPath); - return stream! - .Select(nodes => + return host.Workspace.GetMeshNodeStream() + .Select(node => { - var node = nodes!.FirstOrDefault(n => n.Path == hubPath); var thread = node?.Content as MeshThread; return (IsExecuting: thread?.IsExecuting ?? false, thread?.ActiveMessageId); }) @@ -299,9 +408,15 @@ public static UiControl ThreadsCatalog(LayoutAreaHost host, RenderingContext _) .Select(state => { if (!state.IsExecuting || string.IsNullOrEmpty(state.ActiveMessageId)) + { + logger?.LogDebug("[StreamingView] EMIT_NULL hub={Hub} isExec={IsExec} activeMsg={Msg}", + hubPath, state.IsExecuting, state.ActiveMessageId); return (UiControl?)null; + } var responsePath = $"{hubPath}/{state.ActiveMessageId}"; + logger?.LogDebug("[StreamingView] EMIT_CONTROL hub={Hub} responsePath={Path}", + hubPath, responsePath); return (UiControl?)new LayoutAreaControl(responsePath, new LayoutAreaReference(ThreadMessageNodeType.OverviewArea)); }); @@ -320,28 +435,15 @@ public static UiControl ThreadsCatalog(LayoutAreaHost host, RenderingContext _) return Observable.Return(Controls.Html("

    Query service not available.

    ")); } - // Get the node from the workspace stream - var nodeStream = host.Workspace.GetStream()?.Select(nodes => nodes ?? Array.Empty()) - ?? Observable.Return(Array.Empty()); - - // Query for child Thread nodes (delegations) - var childrenStream = Observable.FromAsync(async () => - { - try - { - return await meshQuery.QueryAsync($"namespace:{hubPath} nodeType:{ThreadNodeType.NodeType}").ToListAsync() as IReadOnlyList; - } - catch - { - return Array.Empty() as IReadOnlyList; - } - }); + // Live observable of child Thread nodes (delegations) — auto-updates on add/remove. + var childrenStream = meshQuery.Query( + MeshQueryRequest.FromQuery($"namespace:{hubPath} nodeType:{ThreadNodeType.NodeType}")) + .Select(change => (IReadOnlyList)change.Items) + .Catch, Exception>(_ => Observable.Return((IReadOnlyList)Array.Empty())); - return nodeStream.CombineLatest(childrenStream, (nodes, children) => - { - var node = nodes.FirstOrDefault(n => n.Path == hubPath); - return BuildHistoryView(host, node, hubPath, children ?? Array.Empty()); - }); + return host.Workspace.GetMeshNodeStream() + .CombineLatest(childrenStream, (node, children) => + BuildHistoryView(host, node, hubPath, children ?? Array.Empty())); } private static UiControl BuildHistoryView(LayoutAreaHost _, MeshNode? node, string threadPath, IReadOnlyList delegations) @@ -411,67 +513,47 @@ private static UiControl BuildDelegationCard(MeshNode delegationNode) /// /// Renders a compact thumbnail for thread nodes in catalogs. - /// Shows title, last activity time, and message preview. - /// Queries child ThreadMessage nodes for message count and preview. + /// Shows title, last activity, and message count synchronously from the + /// thread node alone — does NOT subscribe to any cell streams. The text + /// preview is delegated to a pointing at + /// the last cell's "Streaming" area, so the cell hub streams its own + /// few-lines preview lazily on the Blazor side. Catalog rendering with N + /// threads no longer pays N × M cell-load round-trips. /// public static IObservable Thumbnail(LayoutAreaHost host, RenderingContext _) { var hubPath = host.Hub.Address.ToString(); - var meshQuery = host.Hub.ServiceProvider.GetService(); - - var nodeStream = host.Workspace.GetStream()?.Select(nodes => nodes ?? Array.Empty()) - ?? Observable.Return(Array.Empty()); - - // Query for child ThreadMessage nodes - var messagesStream = Observable.FromAsync(async () => - { - if (meshQuery == null) - return Array.Empty() as IReadOnlyList; - - try - { - return await meshQuery.QueryAsync( - $"namespace:{hubPath} nodeType:{ThreadMessageNodeType.NodeType} sort:Timestamp-asc" - ).ToListAsync() as IReadOnlyList; - } - catch - { - return Array.Empty() as IReadOnlyList; - } - }); - - return nodeStream.CombineLatest(messagesStream, (nodes, messageNodes) => - { - var node = nodes.FirstOrDefault(n => n.Path == hubPath); - return BuildThumbnail(node, hubPath, messageNodes ?? Array.Empty()); - }); + return host.Workspace.GetMeshNodeStream() + .Select(node => BuildThumbnail(node, hubPath)); } - private static UiControl BuildThumbnail(MeshNode? node, string hubPath, IReadOnlyList messageNodes) + private static UiControl BuildThumbnail(MeshNode? node, string hubPath) { - var content = node?.Content as MeshThread; + var thread = node?.Content as MeshThread; + var cellIds = thread?.Messages ?? ImmutableList.Empty; var title = node?.Name ?? "Thread"; var lastActivity = node?.LastModified.ToString("g") ?? ""; - // Extract messages from child nodes - var messages = messageNodes - .Select(n => n.Content as ThreadMessage) - .Where(m => m != null && m.Type != ThreadMessageType.EditingPrompt) - .OrderBy(m => m!.Timestamp) - .ToList(); - - // Fall back to legacy inline messages - var messageCount = messages.Count; - - // Get preview from last message - var preview = ""; - var lastMessage = messages.LastOrDefault(); - if (lastMessage != null) + // Preview is a lazy embedded layout area pointing at the last cell's + // compact Streaming view (last 3 lines + tool-call chips). The cell + // hub only activates when the catalog tile becomes visible. When the + // thread has no cells, fall back to a static "No messages yet" line. + UiControl previewView = cellIds.Count > 0 + ? new LayoutAreaControl( + $"{hubPath}/{cellIds[^1]}", + new LayoutAreaReference("Streaming")) + .WithSpinnerType(SpinnerType.Skeleton) + .WithStyle("margin: 8px 0 0 0; max-height: 60px; overflow: hidden;") + : Controls.Html( + "

    No messages yet.

    "); + + var countLabel = cellIds.Count switch { - preview = lastMessage.Text.Length > 60 - ? lastMessage.Text[..57] + "..." - : lastMessage.Text; - } + 0 => "", + 1 => "1 message", + _ => $"{cellIds.Count} messages" + }; return Controls.Stack .WithStyle("padding: 16px; background: var(--neutral-layer-card-container); border: 1px solid var(--neutral-stroke-rest); border-radius: 8px;") @@ -481,14 +563,62 @@ private static UiControl BuildThumbnail(MeshNode? node, string hubPath, IReadOnl .WithView(Controls.Icon(FluentIcons.Chat(IconSize.Size24)).WithStyle("color: var(--accent-fill-rest);")) .WithView(Controls.Stack .WithView(Controls.Html($"{System.Web.HttpUtility.HtmlEncode(title)}")) - .WithView(Controls.Html($"{lastActivity}")))) - .WithView(!string.IsNullOrEmpty(preview) - ? Controls.Html($"

    {System.Web.HttpUtility.HtmlEncode(preview)}

    ") - : Controls.Html($"

    {messageCount} messages

    ")) + .WithView(Controls.Html( + $"" + + $"{lastActivity}" + + (string.IsNullOrEmpty(countLabel) ? "" : $" · {countLabel}") + + "")))) + .WithView(previewView) .WithView(new NavLinkControl("", null, $"/{hubPath}/{ThreadNodeType.ThreadArea}")); } + /// + /// The title + description block in the thread hero header. + /// For users with both render as inline, + /// auto-saving editors bound DIRECTLY to the thread node's + /// and (fields-mode node-bound DataContext) — + /// one source of truth, no /data replica, no save subscription. Editability + /// is gated, not just the write: read-only viewers get the gradient HTML title plus + /// the description (pushed to the data section), the description hidden when unset. + /// + private static UiControl BuildTitleEditor(string nodePath, bool canEdit) + { + if (!canEdit) + return Controls.Stack + .WithStyle("gap: 4px; min-width: 0;") + .WithView(Controls.Html(new JsonPointerReference(LayoutAreaReference.GetDataPointer("title"))) + .WithStyle("margin: 0; font-size: 1.85rem; font-weight: 600; " + + "letter-spacing: -0.01em; line-height: 1.15; " + + "background: linear-gradient(135deg, var(--neutral-foreground-rest), " + + "color-mix(in srgb, var(--accent-fill-rest) 80%, var(--neutral-foreground-rest))); " + + "-webkit-background-clip: text; background-clip: text; " + + "-webkit-text-fill-color: transparent; color: transparent;")) + .WithView(Controls.Html(new JsonPointerReference(LayoutAreaReference.GetDataPointer("description")))); + + var fieldsContext = LayoutAreaReference.GetMeshNodeDataContext(nodePath, bindContent: false); + + var titleField = new TextFieldControl(new JsonPointerReference(nameof(MeshNode.Name))) + { + Immediate = true, + Placeholder = "Untitled thread", + DataContext = fieldsContext + }.WithStyle("font-size: 1.5rem; font-weight: 600; letter-spacing: -0.01em;") + .WithClass("thread-title-field"); + + var descriptionField = new TextAreaControl(new JsonPointerReference(nameof(MeshNode.Description))) + { + Immediate = true, + Placeholder = "Add a description…", + DataContext = fieldsContext + }.WithRows(2).WithClass("thread-description-field"); + + return Controls.Stack + .WithStyle("gap: 6px; min-width: 0;") + .WithView(titleField) + .WithView(descriptionField); + } + /// /// Gets the thread title from node name or falls back to default. /// @@ -514,7 +644,8 @@ private static UiControl ThreadsView(LayoutAreaHost host, RenderingContext _) { var nodePath = host.Hub.Address.ToString(); return Controls.MeshSearch - .WithHiddenQuery($"nodeType:Thread namespace:{nodePath}/{ThreadNodeType.ThreadPartition}") + // -content.status:Done hides finished threads from the node-scoped Threads view. + .WithHiddenQuery($"nodeType:Thread namespace:{nodePath}/{ThreadNodeType.ThreadPartition} -content.status:Done") .WithNamespace(nodePath) .WithRenderMode(MeshSearchRenderMode.Flat) .WithCreateNodeType("Thread"); @@ -556,6 +687,95 @@ private static UiControl ThreadsView(LayoutAreaHost host, RenderingContext _) return initial.Concat(aggregated); } + /// + /// Full-page Changes view. Reuses the header's + /// aggregation + grid, plus a + /// "Revert All" bulk action that posts one + /// RollbackNodeRequest per entry sequentially (order-sensitive to + /// avoid parent-deleted-before-child issues). + /// + public static IObservable ChangesAreaView(LayoutAreaHost host, RenderingContext _) + { + var threadPath = host.Hub.Address.ToString(); + var stream = host.Workspace.GetStream(new MeshNodeReference()); + if (stream is null) + return Observable.Return(BuildChangesEmpty()); + + var aggregated = stream + .Select(change => (change.Value?.Content as MeshThread)?.Messages ?? ImmutableList.Empty) + .DistinctUntilChanged(ids => string.Join("|", ids)) + .Select(ids => ids.IsEmpty + ? Observable.Return(ImmutableList.Empty) + : CollectUpdatedNodes(host.Hub, threadPath, ids)) + .Switch(); + + return aggregated.Select(updates => BuildChangesPage(host.Hub, threadPath, updates)); + } + + private static UiControl BuildChangesEmpty() + => Controls.Html( + "
    " + + "

    No node changes recorded for this thread.

    "); + + private static UiControl BuildChangesPage( + IMessageHub hub, string threadPath, ImmutableList updates) + { + var container = Controls.Stack + .WithWidth("100%") + .WithStyle("padding:24px; gap:16px;"); + + // Header row: title + count + Revert All button. + var headerStyle = "display:flex; align-items:center; gap:12px;"; + var titleHtml = + "

    Changes

    " + + $"" + + $"{updates.Count} node{(updates.Count == 1 ? "" : "s")} modified"; + + var headerRow = Controls.Stack + .WithOrientation(Orientation.Horizontal) + .WithStyle(headerStyle) + .WithView(Controls.Html( + $"
    {titleHtml}
    ")); + + // Revert All button — only enabled when there's something to revert. + var revertable = updates.Where(e => e.VersionBefore.HasValue).ToImmutableList(); + if (revertable.Count > 0) + { + headerRow = headerRow.WithView(Controls.Button($"Revert all ({revertable.Count})") + .WithAppearance(Appearance.Neutral) + .WithIconStart(FluentIcons.ArrowUndo(IconSize.Size16)) + .WithClickAction(_ => RevertAllChanges(hub, revertable))); + } + container = container.WithView(headerRow); + + if (updates.IsEmpty) + { + container = container.WithView(BuildChangesEmpty()); + return container; + } + + // Reuse the same git-like grid as the header summary — single source of truth + // for path / version chips / Diff / per-row Restore links. + container = container.WithView(Controls.Html(BuildModifiedNodesHtml(updates, threadPath))); + return container; + } + + /// + /// Posts for every entry with a + /// , in sequence. Sequential + /// (not parallel) so dependent ordering (parent-before-child for creates, + /// child-before-parent for deletes) stays predictable. Fire-and-forget per + /// request; failures are independent. + /// + private static void RevertAllChanges(IMessageHub hub, ImmutableList entries) + { + foreach (var entry in entries) + { + if (!entry.VersionBefore.HasValue) continue; + hub.Post(new RollbackNodeRequest(entry.Path, entry.VersionBefore.Value)); + } + } + /// /// Walks , requests each satellite ThreadMessage via /// GetDataRequest (Post + RegisterCallback wrapped as an Observable), accumulates @@ -578,15 +798,21 @@ private static IObservable> CollectUpdatedNodes( } else { - hub.RegisterCallback((IMessageDelivery)del, resp => - { - var msg = resp is IMessageDelivery gdr - ? (gdr.Message.Data as MeshNode)?.Content as ThreadMessage - : null; - subject.OnNext(msg?.UpdatedNodes ?? ImmutableList.Empty); - subject.OnCompleted(); - return resp; - }); + hub.Observe((IMessageDelivery)del) + .Subscribe( + resp => + { + var msg = resp.Message is GetDataResponse gdr + ? (gdr.Data as MeshNode)?.Content as ThreadMessage + : null; + subject.OnNext(msg?.UpdatedNodes ?? ImmutableList.Empty); + subject.OnCompleted(); + }, + _ => + { + subject.OnNext(ImmutableList.Empty); + subject.OnCompleted(); + }); } return subject.AsObservable(); }).ToList(); @@ -816,4 +1042,31 @@ static string Shorten(string path, string? prefix) => sb.Append(""); return sb.ToString(); } + + /// + /// Reads the still-pending user-message texts from . + /// Pending = id is in AND in + /// . The check_inbox tool + /// drains this queue mid-stream — once drained, the texts disappear from + /// this list (they remain visible as user cells in the conversation). + /// + internal static IReadOnlyList ExtractPendingTexts(MeshThread? thread) + { + if (thread is null || thread.PendingUserMessages.IsEmpty) + return Array.Empty(); + + var result = new List(thread.PendingUserMessages.Count); + foreach (var id in thread.UserMessageIds) + { + if (thread.PendingUserMessages.TryGetValue(id, out var msg)) + result.Add(msg.Text); + } + // Catch any pending entries not in UserMessageIds (defensive — shouldn't + // happen via the normal AppendUserInput path, but don't silently drop). + foreach (var (id, msg) in thread.PendingUserMessages) + if (!thread.UserMessageIds.Contains(id)) + result.Add(msg.Text); + + return result; + } } diff --git a/src/MeshWeaver.AI/ThreadMessageHandlers.cs b/src/MeshWeaver.AI/ThreadMessageHandlers.cs deleted file mode 100644 index c34ee739f..000000000 --- a/src/MeshWeaver.AI/ThreadMessageHandlers.cs +++ /dev/null @@ -1,63 +0,0 @@ -using System.Collections.Immutable; -using MeshWeaver.Data; -using MeshWeaver.Graph; -using MeshWeaver.Layout; -using MeshWeaver.Mesh; -using MeshWeaver.Mesh.Services; -using MeshWeaver.Messaging; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; -using MeshThread = MeshWeaver.AI.Thread; - -namespace MeshWeaver.AI; - -/// -/// Handlers for thread message operations: resubmit (re-evaluate) and delete. -/// Factored out from ThreadLayoutAreas for clarity. -/// All handlers are synchronous (no await) and use workspace.UpdateMeshNode -/// in the handler body (grain scheduler) or hub.Post (safe from any thread). -/// -public static class ThreadMessageHandlers -{ - /// - /// Handles DeleteFromMessageRequest — truncates Messages from the given message onwards. - /// - internal static IMessageDelivery HandleDeleteFromMessage( - IMessageHub hub, - IMessageDelivery delivery) - { - var request = delivery.Message; - hub.GetWorkspace().UpdateMeshNode(node => - { - var thread = node.Content as MeshThread ?? new MeshThread(); - var msgIndex = thread.Messages.IndexOf(request.MessageId); - if (msgIndex < 0) return node; - return node with - { - Content = thread with { Messages = thread.Messages.Take(msgIndex).ToImmutableList() } - }; - }); - return delivery.Processed(); - } - - /// - /// Handles ResubmitMessageRequest — truncates the thread after the user message id, - /// drops it from IngestedMessageIds, optionally updates its text. The server-side - /// watcher in ThreadSubmission re-dispatches a new round. - /// Thin shim over to keep one code path. - /// - internal static IMessageDelivery HandleResubmitMessage( - IMessageHub hub, - IMessageDelivery delivery) - { - var request = delivery.Message; - ThreadSubmission.ApplyResubmit( - hub, - request.ThreadPath, - request.MessageId, - newUserText: request.UserMessageText, - agentName: null, - modelName: null); - return delivery.Processed(); - } -} diff --git a/src/MeshWeaver.AI/ThreadMessageLayoutAreas.cs b/src/MeshWeaver.AI/ThreadMessageLayoutAreas.cs index 66b72033b..a391c1e65 100644 --- a/src/MeshWeaver.AI/ThreadMessageLayoutAreas.cs +++ b/src/MeshWeaver.AI/ThreadMessageLayoutAreas.cs @@ -1,4 +1,4 @@ -using System.Reactive.Linq; +using System.Reactive.Linq; using Humanizer; using MeshWeaver.Application.Styles; using MeshWeaver.Data; @@ -27,7 +27,6 @@ public static class ThreadMessageLayoutAreas /// public static MessageHubConfiguration AddThreadMessageViews(this MessageHubConfiguration configuration) => configuration - .WithHandler(HandleUpdateContent) .AddLayout(layout => layout .WithDefaultArea(ThreadMessageNodeType.OverviewArea) .WithView(ThreadMessageNodeType.OverviewArea, Overview) @@ -37,8 +36,6 @@ public static MessageHubConfiguration AddThreadMessageViews(this MessageHubConfi .WithView(MeshNodeLayoutAreas.MetadataArea, MeshNodeLayoutAreas.Metadata) .WithView(MeshNodeLayoutAreas.ThumbnailArea, Thumbnail)); - private const string MessageDataKey = "msg"; - /// /// Compact streaming view for parent thread consumption. /// Shows last 3 lines of text + tool call chips + delegation links. @@ -104,51 +101,16 @@ public static MessageHubConfiguration AddThreadMessageViews(this MessageHubConfi }); } - /// - /// Handles content updates from thread execution. - /// Runs ON the response message grain — updates local workspace → sync stream → clients. - /// - private static IMessageDelivery HandleUpdateContent( - IMessageHub hub, IMessageDelivery delivery) - { - var msg = delivery.Message; - var logger = hub.ServiceProvider.GetService() - ?.CreateLogger("MeshWeaver.AI.MsgLayout"); - logger?.LogInformation("[MsgLayout] HANDLE_UPDATE: hub={Hub}, textLen={TextLen}, toolCalls={ToolCalls}", - hub.Address, msg.Text?.Length ?? -1, msg.ToolCalls?.Count ?? -1); - hub.GetWorkspace().UpdateMeshNode(node => - { - var current = node.Content as ThreadMessage ?? new ThreadMessage { Role = "assistant", Text = "" }; - // Prefer incremental append (TextDelta). Full Text replacement is only used - // for final/terminal writes (completion, error, cancel markers). - var newText = msg.Text ?? (msg.TextDelta is { Length: > 0 } d - ? (current.Text ?? "") + d - : current.Text); - return node with - { - Content = current with - { - Text = newText, - ToolCalls = msg.ToolCalls ?? current.ToolCalls, - UpdatedNodes = msg.UpdatedNodes ?? current.UpdatedNodes, - AgentName = msg.AgentName ?? current.AgentName, - ModelName = msg.ModelName ?? current.ModelName, - InputTokens = msg.InputTokens ?? current.InputTokens, - OutputTokens = msg.OutputTokens ?? current.OutputTokens, - TotalTokens = msg.TotalTokens ?? current.TotalTokens, - CompletedAt = msg.CompletedAt ?? current.CompletedAt - } - }; - }); - return delivery.Processed(); - } - /// /// Renders the Overview area for a ThreadMessage node. - /// Emits control once from first node emission. Text and tool calls are data-bound - /// via JsonPointerReference — updates flow through host.UpdateData, no control rebuilds. - /// Editing is handled purely on the Blazor UI side (ThreadMessageBubbleView toggles - /// between readonly and Edit LayoutArea). + /// Ships a path-bound — the Blazor view + /// subscribes to workspace.GetRemoteStream<MeshNode, MeshNodeReference>(NodePath, ...) + /// directly and renders Text / ToolCalls / UpdatedNodes from the live message. + /// No layout data section, no chain, no + /// per-node republish. See Doc/Architecture/ThreadExecutionStreaming.md. + /// + /// Editing branch (EditingPrompt) still uses the legacy snapshot path — it needs + /// the editor pre-populated with current text and that's a one-shot read. /// public static IObservable Overview(LayoutAreaHost host, RenderingContext _) { @@ -156,67 +118,29 @@ private static IMessageDelivery HandleUpdateContent( var lastSlash = hubPath.LastIndexOf('/'); var threadPath = lastSlash > 0 ? hubPath[..lastSlash] : hubPath; var messageId = lastSlash > 0 ? hubPath[(lastSlash + 1)..] : hubPath; + var nodePath = $"{threadPath}/{messageId}"; - // Subscribe to the MeshNodeReference sync stream — receives updates from - // responseStream.Update() / PatchDataChangeRequest. Map to ThreadMessageViewModel - // and push to data section. The bubble binds to the view model via JsonPointerReference. var syncStream = host.Workspace.GetStream(new MeshNodeReference()); - var msgLogger = host.Hub.ServiceProvider.GetService() ?.CreateLogger("MeshWeaver.AI.MsgLayout"); - host.SubscribeToDataStream(MessageDataKey, syncStream! - .Select(change => - { - var msg = change.Value?.Content as ThreadMessage; - msgLogger?.LogInformation("[MsgLayout] STREAM_EMIT: hub={Hub}, hasContent={HasContent}, textLen={TextLen}", - host.Hub.Address, msg != null, msg?.Text?.Length ?? -1); - return msg; - }) - .Where(m => m != null) - .Select(m => (ThreadMessageViewModel.FromMessage(m!) with - { - Text = ConvertReferencesToLinks(m!.Text ?? "") - })) - .DistinctUntilChanged() - .Select(vm => - { - msgLogger?.LogInformation("[MsgLayout] DATA_PUSH: hub={Hub}, textLen={TextLen}, toolCalls={ToolCalls}", - host.Hub.Address, ((ThreadMessageViewModel)vm).Text.Length, ((ThreadMessageViewModel)vm).ToolCalls.Count); - return (object)vm; - })); - - // Emit control once — role/author are static, text/toolCalls are data-bound. - // Try current value first (synchronous) — the stream may have already replayed - // before this subscription started. Fall back to observable for lazy activation. - var currentMsg = syncStream!.Current?.Value?.Content as ThreadMessage; - if (currentMsg != null) - { - msgLogger?.LogInformation("[MsgLayout] CONTROL_EMIT_SYNC: hub={Hub}, role={Role}, textLen={TextLen}", - hubPath, currentMsg.Role, currentMsg.Text?.Length ?? 0); - var control = currentMsg.Type == ThreadMessageType.EditingPrompt - ? BuildEditingOverview(host, currentMsg, threadPath, messageId) - : BuildMessageOverview(host, currentMsg, threadPath, messageId); - return Observable.Return((UiControl?)control); - } + // Always wait for the first stream emission — never read the snapshot synchronously. + // On cold workspaces / un-initialized remote streams it's null and we'd ship an + // empty overview to the user. msgLogger?.LogInformation("[MsgLayout] CONTROL_EMIT_ASYNC: hub={Hub}, waiting for first emission", hubPath); - return syncStream + return syncStream! .Select(change => change.Value?.Content as ThreadMessage) .Where(m => m != null) .Take(1) - .Select(msg => - { - if (msg!.Type == ThreadMessageType.EditingPrompt) - return (UiControl?)BuildEditingOverview(host, msg, threadPath, messageId); - - return (UiControl?)BuildMessageOverview(host, msg, threadPath, messageId); - }); + .Select(msg => msg!.Type == ThreadMessageType.EditingPrompt + ? (UiControl?)BuildEditingOverview(host, msg, threadPath, messageId) + : (UiControl?)BuildMessageOverview(host, msg, threadPath, messageId, nodePath)); } /// /// Renders the Edit area for a ThreadMessage node. /// Shows a MarkdownEditorControl with the current text and a Submit button. - /// Submit posts ResubmitMessageRequest (re-executes with edited text). + /// Submit posts ThreadSubmission.ApplyResubmit (re-executes with edited text). /// Cancel/Done is handled by the Blazor bubble (local isEditing toggle). /// public static IObservable EditArea(LayoutAreaHost host, RenderingContext _) @@ -260,13 +184,14 @@ private static IMessageDelivery HandleUpdateContent( NodeType = ThreadMessageNodeType.NodeType, MainNode = threadPath, Content = new ThreadMessage { Role = "assistant", Text = "", Timestamp = DateTime.UtcNow, Type = ThreadMessageType.AgentResponse } }), o => o.WithTarget(new Address(threadPath))); - actx.Hub.Post(new ResubmitMessageRequest - { - ThreadPath = threadPath, - MessageId = messageId, - UserMessageText = editedText ?? msg.Text ?? "", - OutputMessageId = outId - }, o => o.WithTarget(new Address(threadPath))); + // Stream-update: ApplyResubmit truncates Messages after + // messageId, drops it from IngestedMessageIds, and stamps + // the new user text — server watcher dispatches the next + // round. No bespoke ThreadSubmission.ApplyResubmit needed. See + // RequestViaStreamUpdate.md. + actx.Hub.ResubmitMessage( + threadPath, messageId, + newUserText: editedText ?? msg.Text ?? ""); }); })); @@ -281,25 +206,21 @@ private static IMessageDelivery HandleUpdateContent( } /// - /// Builds the Overview for messages. Role/author are static from the initial message. - /// Text and tool calls are data-bound via JsonPointerReference. + /// Builds the Overview for messages. Bubble is path-bound — the Blazor view + /// subscribes to the message-node remote stream directly and pulls Role, + /// AuthorName, Text, ToolCalls, UpdatedNodes, ModelName, Timestamp from there. + /// The snapshot is only used for click-handler defaults + /// (Resubmit text fallback) where a one-shot read at render time is acceptable. /// private static UiControl BuildMessageOverview( - LayoutAreaHost host, ThreadMessage msg, string threadPath, string messageId) + LayoutAreaHost host, ThreadMessage msg, string threadPath, string messageId, string nodePath) { var isUser = msg.Role.Equals("user", StringComparison.OrdinalIgnoreCase); - var authorName = msg.AuthorName ?? (isUser ? "You" : msg.AgentName ?? "Assistant"); - // Bind to ThreadMessageViewModel in data section - var dataPointer = LayoutAreaReference.GetDataPointer(MessageDataKey); + // Path-bound bubble — view subscribes to nodePath via GetRemoteStream. + // No JsonPointerReference, no layout data section, no per-chunk republish. var bubble = new ThreadMessageBubbleControl() - .WithRole(msg.Role) - .WithAuthorName(authorName) - .WithModelName(msg.ModelName) - .WithTimestamp(msg.Timestamp) - .WithText(new JsonPointerReference($"{dataPointer}/text")) - .WithToolCalls(new JsonPointerReference($"{dataPointer}/toolCalls")) - .WithUpdatedNodes(new JsonPointerReference($"{dataPointer}/updatedNodes")) + .WithNodePath(nodePath) .WithThreadPath(threadPath) .WithMessageId(messageId); @@ -333,13 +254,9 @@ private static UiControl BuildMessageOverview( } }), o => o.WithTarget(new Address(threadPath))); - host.Hub.Post(new ResubmitMessageRequest - { - ThreadPath = threadPath, - MessageId = messageId, - UserMessageText = msg.Text, - OutputMessageId = outId - }, o => o.WithTarget(new Address(threadPath))); + // Canonical resubmit via the IWorkspace extension surface. + host.Hub.ResubmitMessage( + threadPath, messageId, newUserText: msg.Text); })); } @@ -350,11 +267,8 @@ private static UiControl BuildMessageOverview( .WithLabel("Delete from here") .WithClickAction(_ => { - host.Hub.Post(new DeleteFromMessageRequest - { - ThreadPath = threadPath, - MessageId = messageId - }, o => o.WithTarget(new Address(threadPath))); + // Stream-update delete — see RequestViaStreamUpdate.md. + host.Hub.DeleteFromMessage(threadPath, messageId); })); var container = Controls.Stack @@ -377,12 +291,12 @@ private static UiControl BuildMessageOverview( .Select(m => ( Started: m!.Timestamp, Completed: m.CompletedAt, - Model: m.ModelName, + Harness: m.Harness, In: m.InputTokens, Out: m.OutputTokens, Total: m.TotalTokens)) .DistinctUntilChanged() - .Select(meta => BuildAssistantMetaRow(meta.Started, meta.Completed, meta.Model, + .Select(meta => BuildAssistantMetaRow(meta.Started, meta.Completed, meta.Harness, meta.In, meta.Out, meta.Total)); }); } @@ -415,18 +329,18 @@ private static string FormatDurationHms(TimeSpan d) /// /// Builds the muted one-line metadata row shown below an assistant cell: - /// HH:mm:ss · model · 1.8s · 1,247 in / 392 out (1,639 total). Returns - /// null when there's nothing to show (e.g. response still streaming and no model - /// yet known). + /// Claude Code · HH:mm:ss · 1.8s · 1,247 in / 392 out (1,639 total). Leads + /// with the harness (the headline identity); model is intentionally dropped. + /// Returns null when there's nothing to show (e.g. response still streaming). /// private static UiControl? BuildAssistantMetaRow( - DateTime started, DateTime? completed, string? model, + DateTime started, DateTime? completed, string? harness, int? input, int? output, int? total) { var parts = new List(); + if (!string.IsNullOrEmpty(harness)) + parts.Add(System.Web.HttpUtility.HtmlEncode(harness)); parts.Add(started.ToLocalTime().ToString("HH:mm:ss")); - if (!string.IsNullOrEmpty(model)) - parts.Add(System.Web.HttpUtility.HtmlEncode(model)); if (completed.HasValue) { parts.Add(FormatDurationHms(completed.Value - started)); @@ -525,11 +439,8 @@ private static UiControl BuildEditingOverview( .WithAppearance(Appearance.Neutral) .WithClickAction(_ => { - host.Hub.Post(new DeleteFromMessageRequest - { - ThreadPath = threadPath, - MessageId = messageId - }, o => o.WithTarget(new Address(threadPath))); + // Stream-update delete — see RequestViaStreamUpdate.md. + host.Hub.DeleteFromMessage(threadPath, messageId); })) .WithView(Controls.Button("Submit") .WithAppearance(Appearance.Accent) @@ -544,13 +455,10 @@ private static UiControl BuildEditingOverview( NodeType = ThreadMessageNodeType.NodeType, MainNode = threadPath, Content = new ThreadMessage { Role = "assistant", Text = "", Timestamp = DateTime.UtcNow, Type = ThreadMessageType.AgentResponse } }), o => o.WithTarget(new Address(threadPath))); - actx.Hub.Post(new ResubmitMessageRequest - { - ThreadPath = threadPath, - MessageId = messageId, - UserMessageText = editedText ?? msg.Text ?? "", - OutputMessageId = outId - }, o => o.WithTarget(new Address(threadPath))); + // Canonical resubmit via the IWorkspace extension surface. + actx.Hub.ResubmitMessage( + threadPath, messageId, + newUserText: editedText ?? msg.Text ?? ""); }); })); @@ -717,16 +625,8 @@ private static UiControl BuildAgentResponseView(ThreadMessage message) /// public static IObservable Thumbnail(LayoutAreaHost host, RenderingContext _) { - var hubPath = host.Hub.Address.ToString(); - - var nodeStream = host.Workspace.GetStream()?.Select(nodes => nodes ?? Array.Empty()) - ?? Observable.Return(Array.Empty()); - - return nodeStream.Select(nodes => - { - var node = nodes.FirstOrDefault(n => n.Path == hubPath); - return BuildThumbnail(node); - }); + return host.Workspace.GetMeshNodeStream() + .Select(node => BuildThumbnail(node)); } private static UiControl BuildThumbnail(MeshNode? node) diff --git a/src/MeshWeaver.AI/ThreadMessageNodeType.cs b/src/MeshWeaver.AI/ThreadMessageNodeType.cs index a13696ffb..de2cf0935 100644 --- a/src/MeshWeaver.AI/ThreadMessageNodeType.cs +++ b/src/MeshWeaver.AI/ThreadMessageNodeType.cs @@ -1,12 +1,17 @@ -using System.Collections.Immutable; +using System.Collections.Immutable; +using MeshWeaver.Messaging; using MeshWeaver.Graph; +using MeshWeaver.Graph.Security; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Mesh.Services; +using Microsoft.Extensions.DependencyInjection; namespace MeshWeaver.AI; /// /// Constants, configuration, and MeshNode definition for ThreadMessage node types. /// ThreadMessage nodes are child nodes of Thread nodes containing individual messages. -/// Each ThreadMessage hub manages its own persistence exclusively via AddMeshDataSource — +/// Each ThreadMessage hub manages its own persistence exclusively via AddMeshDataSource — /// no external code should access ThreadMessage persistence via IMeshService or IMeshQuery. /// public static class ThreadMessageNodeType @@ -34,12 +39,29 @@ public static TBuilder AddThreadMessageType(this TBuilder builder) whe { builder.AddMeshNodes(CreateMeshNode()); builder.AddAutocompleteExcludedTypes(NodeType); + // Public-read on the ThreadMessage NodeType HOST hub — shared type + // metadata (layout definitions, schema). Per-message data access is + // gated by RLS on the message's mainNode/path. Without this, per- + // instance ThreadMessage hubs can't subscribe to their type's + // MeshNodeReference at activation. Same rule as Agent / User / + // Markdown / etc. + builder.ConfigureNodeTypeAccess(a => a.WithPublicRead(NodeType)); + // Per-instance access: ThreadMessage is a satellite of its containing + // Thread → which is itself a satellite of the conversation's MainNode. + // SatelliteAccessRule's MainNode delegation chains correctly when the + // message's MainNode points at the Thread's MainNode (set at creation). + builder.ConfigureServices(services => + { + services.AddSingleton(sp => + new SatelliteAccessRule(NodeType, sp.GetRequiredService())); + return services; + }); return builder; } /// /// Creates a MeshNode definition for the ThreadMessage node type. - /// HubConfiguration includes AddMeshDataSource — the hub owns persistence exclusively. + /// HubConfiguration includes AddMeshDataSource — the hub owns persistence exclusively. /// public static MeshNode CreateMeshNode() => new(NodeType) { @@ -47,7 +69,6 @@ public static TBuilder AddThreadMessageType(this TBuilder builder) whe Icon = "/static/NodeTypeIcons/message.svg", IsSatelliteType = true, ExcludeFromContext = ImmutableHashSet.Create("search", "create"), - AssemblyLocation = typeof(ThreadMessageNodeType).Assembly.Location, HubConfiguration = config => config .AddThreadMessageViews() .AddMeshDataSource(source => source.WithContentType()) diff --git a/src/MeshWeaver.AI/ThreadNodeType.cs b/src/MeshWeaver.AI/ThreadNodeType.cs index c1e5e979e..32dbb7331 100644 --- a/src/MeshWeaver.AI/ThreadNodeType.cs +++ b/src/MeshWeaver.AI/ThreadNodeType.cs @@ -1,10 +1,14 @@ using System.Collections.Immutable; +using System.Reactive.Linq; using System.Text.RegularExpressions; using MeshWeaver.Data; using MeshWeaver.Graph; +using MeshWeaver.Layout; +using MeshWeaver.Mesh.Security; using MeshWeaver.Mesh.Services; using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; namespace MeshWeaver.AI; @@ -32,6 +36,21 @@ public static class ThreadNodeType ///     public const string ThreadPartition = "_Thread"; + /// + /// The MAIN NODE of a path: everything before the first /_Thread segment — i.e. + /// "what the chat is about". rbuergi/_Thread/x → rbuergi; + /// acme/Project/X/_Thread/y/ThreadComposer → acme/Project/X; a path with no + /// _Thread returns unchanged. Stripping at the FIRST _Thread collapses any + /// accidental nesting (a thread/composer under another thread) back to the true main node, + /// so the per-(node,user) ThreadComposer never nests. + /// + public static string MainNodeOf(string path) + { + if (string.IsNullOrEmpty(path)) return path; + var idx = path.IndexOf($"/{ThreadPartition}", StringComparison.OrdinalIgnoreCase); + return idx >= 0 ? path[..idx] : path; + } + /// /// Layout area for thread content and message history (default). /// @@ -61,6 +80,14 @@ public static class ThreadNodeType /// public const string HeaderArea = "Header"; + /// + /// Layout area showing the aggregated list of nodes modified across every + /// message of the thread, with version-before / version-after, Diff link, + /// per-row Revert, and a bulk "Revert All" action. Surfaced through the + /// node menu (Changes). + /// + public const string ChangesArea = "Changes"; + /// /// Generates a human-readable speaking ID from message text. /// Takes the first few words, lowercases, replaces non-alphanumeric with hyphens, @@ -88,9 +115,10 @@ public static string GenerateSpeakingId(string messageText) /// The namespace/context path (e.g., "User/Roland") /// First message text — used for name and speaking ID /// User ID who creates the thread - public static MeshNode BuildThreadNode(string contextPath, string messageText, string? createdBy = null) + public static MeshNode BuildThreadNode(string contextPath, string messageText, string? createdBy = null, + string? speakingId = null) { - var speakingId = GenerateSpeakingId(messageText); + speakingId ??= GenerateSpeakingId(messageText); // Add _Thread partition for top-level threads. Sub-threads (delegations) // live directly under the parent response message — no nested _Thread. var ns = string.IsNullOrEmpty(contextPath) @@ -113,8 +141,20 @@ public static MeshNode BuildThreadNode(string contextPath, string messageText, s } /// - /// Builds a thread node with pre-populated messages and pending execution. - /// When the thread grain activates, it auto-starts execution — no separate SubmitMessageRequest needed. + /// Builds a thread node pre-seeded with a user message in + /// . The thread starts at + /// ; when the per-thread hub + /// activates, InstallServerWatcher sees the pending entry and + /// drives the standard claim → DispatchRound → execute flow. No + /// pre-allocated satellite cells, no ActiveMessageId, no + /// auto-execute trigger competing with the submission watcher. + /// + /// The returned UserMsgId is the key used in + /// PendingUserMessages. ResponseMsgId is intentionally + /// the empty string for back-compat: + /// allocates the real response cell id when the watcher claims the + /// round. Callers that need the response id after dispatch should + /// subscribe to . /// public static (MeshNode Thread, string UserMsgId, string ResponseMsgId) BuildThreadWithMessages( string contextPath, string messageText, @@ -134,7 +174,19 @@ public static (MeshNode Thread, string UserMsgId, string ResponseMsgId) BuildThr : messageText; var userMsgId = Guid.NewGuid().ToString("N")[..8]; - var responseMsgId = Guid.NewGuid().ToString("N")[..8]; + var userMessage = new ThreadMessage + { + Role = "user", + Text = messageText, + CreatedBy = createdBy, + AgentName = agentName, + ModelName = modelName, + ContextPath = contextPath, + Attachments = attachments, + Timestamp = DateTime.UtcNow, + Type = ThreadMessageType.ExecutedInput, + Status = ThreadMessageStatus.Submitted + }; var threadNode = new MeshNode(speakingId, ns) { @@ -145,19 +197,20 @@ public static (MeshNode Thread, string UserMsgId, string ResponseMsgId) BuildThr Content = new Thread { CreatedBy = createdBy, - Messages = ImmutableList.Create(userMsgId, responseMsgId), - IsExecuting = true, - ActiveMessageId = responseMsgId, - ExecutionStartedAt = DateTime.UtcNow, - PendingUserMessage = messageText, - PendingAgentName = agentName, - PendingModelName = modelName, - PendingContextPath = contextPath, - PendingAttachments = attachments + Status = ThreadExecutionStatus.Idle, + UserMessageIds = ImmutableList.Create(userMsgId), + // The pending ThreadMessage carries the round's selection + // (agent/model/context/attachments) — no thread-level Pending* mirror. + PendingUserMessages = ImmutableDictionary.Empty + .Add(userMsgId, userMessage) } }; - return (threadNode, userMsgId, responseMsgId); + // ResponseMsgId is now allocated by DispatchAfterClaim, not here. Return + // empty string for back-compat — call sites that wanted the id pre-emptively + // (e.g. for parent tool-call tracking) should read Thread.ActiveMessageId + // from the stream after the submission watcher claims. + return (threadNode, userMsgId, ""); } /// @@ -181,6 +234,27 @@ public static TBuilder AddThreadType(this TBuilder builder, { builder.AddMeshNodes(CreateMeshNode(hubConfiguration)); builder.AddAutocompleteExcludedTypes(NodeType); + // Public-read on the Thread NodeType HOST hub (address = "Thread") — + // grants any authenticated user Read on the type's shared metadata + // (layout definitions, schema). This is the type DEFINITION, not the + // per-instance thread data — instance access is gated by RLS on the + // node's mainNode/path separately. Without this, per-instance Thread + // hubs can't subscribe to their type's MeshNodeReference at activation, + // surfacing as "Access denied: user '' lacks + // Read permission on 'Thread'" and the chat view never loads. Matches + // Agent, User, Code, Markdown, etc. + builder.ConfigureNodeTypeAccess(a => a.WithPublicRead(NodeType)); + // Per-instance access: Thread is a satellite — Read requires Read on + // the conversation's MainNode, Create/Update/Delete require Update + // on the MainNode. Matches Comment / Activity / TrackedChange. + builder.ConfigureServices(services => + { + services.AddSingleton(sp => + new MeshWeaver.Graph.Security.SatelliteAccessRule( + NodeType, + sp.GetRequiredService())); + return services; + }); return builder; } @@ -195,11 +269,71 @@ public static MeshNode CreateMeshNode( Icon = DefaultIcon, IsSatelliteType = true, ExcludeFromContext = ImmutableHashSet.Create("search"), - AssemblyLocation = typeof(ThreadNodeType).Assembly.Location, - HubConfiguration = config => config - .AddThreadLayoutAreas() - .AddThreadExecution() - .AddMeshDataSource(source => source - .WithContentType()) + // The NodeType carries the GUI-create protocol: creating a Thread from the "+" + // (anywhere) opens the new-chat composer (the per-user ThreadComposer / new-thread + // view) and creates nothing up front — the thread is created on submit, NOT via + // the generic Create form. Injected as BuildCreate so the generic CreateLayoutArea + // delegates to us regardless of which "+" routed here. + Content = new MeshWeaver.Graph.Configuration.NodeTypeDefinition + { + BuildCreate = (host, ns) => + { + // Composer node, owned per (main node, user). The MAIN NODE is the path + // before /_Thread (what the chat is ABOUT); the user is the signed-in + // identity. For the user's own partition this is the per-user default + // {user}/_Thread/ThreadComposer (seeded at onboarding); for any other node it's + // {node}/_Thread/{user}/ThreadComposer. Always under _Thread with ThreadComposer as the + // leaf — NEVER the bare {node}/_Thread/ThreadComposer, which doesn't read back. + var mainNode = MainNodeOf(ns); + var accessSvc = host.Hub.ServiceProvider.GetService(); + var user = accessSvc?.Context?.ObjectId ?? accessSvc?.CircuitContext?.ObjectId; + var chatInputPath = + string.IsNullOrEmpty(user) || string.Equals(mainNode, user, StringComparison.OrdinalIgnoreCase) + ? ThreadComposerNodeType.PathFor(string.IsNullOrEmpty(user) ? mainNode : user) + : ThreadComposerNodeType.PathForNode(mainNode, user); + var overviewHref = $"/{chatInputPath}"; + var meshService = host.Hub.ServiceProvider.GetRequiredService(); + var logger = host.Hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.AI.ThreadCreate"); + var node = MeshNode.FromPath(chatInputPath) with + { + Name = "New Chat", + NodeType = ThreadComposerNodeType.NodeType, + MainNode = mainNode, + State = MeshNodeState.Active, + }; + // Create then navigate. A real failure (anything other than "already exists") + // leaves the composer overview empty — surface it in the log instead of + // silently swallowing; we still navigate so an existing composer opens. + return meshService.CreateNode(node) + .Take(1) + .Select(_ => (UiControl?)new RedirectControl(overviewHref)) + .Catch(ex => + { + logger?.LogWarning(ex, + "[ThreadCreate] ThreadComposer create failed at {Path}; navigating to overview anyway", + chatInputPath); + return Observable.Return(new RedirectControl(overviewHref)); + }); + } + }, + // Register AI types DIRECTLY on the per-thread hub config — not just + // via ConfigureDefaultNodeHub. The polymorphic resolver discriminator + // is picked from the SENDING hub's TypeRegistry; if Thread NodeType's + // HubConfiguration runs in isolation (a test or host that didn't wire + // ConfigureDefaultNodeHub), unregistered types fall back to FullName + // on the wire and the receiver (whose registry has the short name) + // can't resolve $type → DeliveryFailure on every response. + // See Doc/Architecture/DebuggingMessageFlow.md "Watch for FQN vs + // short-name mismatches". + HubConfiguration = config => + { + config.TypeRegistry.AddAITypes(); + return config + .AddThreadLayoutAreas() + .AddThreadExecution() + .AddMeshDataSource(source => source + .WithContentType()); + } }; } diff --git a/src/MeshWeaver.AI/ThreadStateTransitions.cs b/src/MeshWeaver.AI/ThreadStateTransitions.cs new file mode 100644 index 000000000..bdbe9d9a8 --- /dev/null +++ b/src/MeshWeaver.AI/ThreadStateTransitions.cs @@ -0,0 +1,127 @@ +using System; +using System.Collections.Immutable; +using MeshWeaver.Mesh; +using Microsoft.Extensions.Logging; +using MeshThread = MeshWeaver.AI.Thread; + +namespace MeshWeaver.AI; + +/// +/// The thread-lifecycle state machine expressed as ONE authoritative table of +/// legal (from → to) edges, applied +/// through inside every +/// GetMeshNodeStream(...).Update lambda that changes +/// . +/// +/// Two invariants this enforces by construction: +/// +/// We always start from the lambda parameter. The Update lambda +/// runs on the owning hub's single-threaded action block against the LATEST +/// state, so the from-state we validate (current.Status) is never +/// stale — no transition is decided from a snapshot read earlier. +/// 🚨 From you may only +/// reach (complete) or +/// (cancel), or stay +/// Executing. You may NEVER write +/// from Executing — +/// that inverse of the commit edge (StartingExecution→Executing) is the +/// re-dispatch ping-pong: the exec round watcher commits forward while a +/// self-healing recovery observer flips it back, and the two volley under +/// load. To continue or resume an interrupted round you STAY Executing and +/// re-launch (). +/// The edge is deliberately ABSENT from , so +/// refuses it. +/// +/// +/// Legal edges (self-edges — payload-only updates that keep Status — are +/// always allowed): +/// +/// Idle → StartingExecution submission watcher claims a round +/// Cancelled → StartingExecution re-claim queued input after a stop +/// StartingExecution → Executing exec round watcher commits + launches +/// StartingExecution → Idle rollback: claim found nothing +/// StartingExecution → Cancelled cancel during claim +/// Executing → Idle round complete +/// Executing → Cancelled cancel during execution +/// Idle → Cancelled honor a cancel requested while idle +/// Idle → Done user marks the thread done +/// Cancelled → Done user marks the thread done +/// Done → Idle a new submission reopens the thread +/// +/// +public static class ThreadStateTransitions +{ + // Immutable, initialised once, never written at runtime — a constant lookup, + // not a cache (allowed static readonly under the No-Static-State rule). + private static readonly ImmutableHashSet<(ThreadExecutionStatus From, ThreadExecutionStatus To)> Legal = + [ + (ThreadExecutionStatus.Idle, ThreadExecutionStatus.StartingExecution), + (ThreadExecutionStatus.Cancelled, ThreadExecutionStatus.StartingExecution), + (ThreadExecutionStatus.StartingExecution, ThreadExecutionStatus.Executing), + (ThreadExecutionStatus.StartingExecution, ThreadExecutionStatus.Idle), + (ThreadExecutionStatus.StartingExecution, ThreadExecutionStatus.Cancelled), + (ThreadExecutionStatus.Executing, ThreadExecutionStatus.Idle), + (ThreadExecutionStatus.Executing, ThreadExecutionStatus.Cancelled), + (ThreadExecutionStatus.Idle, ThreadExecutionStatus.Cancelled), + (ThreadExecutionStatus.Idle, ThreadExecutionStatus.Done), + (ThreadExecutionStatus.Cancelled, ThreadExecutionStatus.Done), + (ThreadExecutionStatus.Done, ThreadExecutionStatus.Idle), + ]; + + /// + /// True when → is a legal + /// lifecycle edge. Self-edges (from == to) are always legal — they are + /// payload-only updates that keep . + /// 🚨 Executing → StartingExecution is deliberately NOT legal. + /// + public static bool CanTransition(ThreadExecutionStatus from, ThreadExecutionStatus to) + => from == to || Legal.Contains((from, to)); + + /// + /// True iff the thread is in an execution phase + /// ( or + /// ). Mirrors + /// for use on a bare status value. + /// + public static bool IsExecuting(this ThreadExecutionStatus status) + => status is ThreadExecutionStatus.StartingExecution or ThreadExecutionStatus.Executing; + + /// + /// Apply to the content of + /// (the value the Update lambda received), + /// but commit the result ONLY IF the implied + /// edge is legal. An illegal edge — above all + /// → + /// — is REFUSED: the + /// node is returned unchanged (and logged), so the engine stays in its current + /// valid state instead of oscillating. Bumps + /// on a real change. + /// + /// Usage — the from-state is read from the lambda parameter, never a + /// stale snapshot: + /// + /// stream.Update(node => node.Transition(t => t with { Status = ThreadExecutionStatus.Idle, ... }, logger)); + /// + /// + public static MeshNode Transition( + this MeshNode current, Func mutate, ILogger? logger = null) + { + ArgumentNullException.ThrowIfNull(mutate); + if (current.Content is not MeshThread thread) + return current; + + var next = mutate(thread); + if (!CanTransition(thread.Status, next.Status)) + { + logger?.LogWarning( + "[ThreadState] Refused illegal transition {From}->{To} on {Path} — staying put.", + thread.Status, next.Status, current.Path); + return current; + } + + if (ReferenceEquals(next, thread)) + return current; + + return current with { LastModified = DateTime.UtcNow, Content = next }; + } +} diff --git a/src/MeshWeaver.AI/ThreadSubmission.cs b/src/MeshWeaver.AI/ThreadSubmission.cs index f7fd6a6bc..79f5dcbdf 100644 --- a/src/MeshWeaver.AI/ThreadSubmission.cs +++ b/src/MeshWeaver.AI/ThreadSubmission.cs @@ -24,10 +24,10 @@ namespace MeshWeaver.AI; /// batched ingestion keeps one output cell per round. /// - Pure helpers and /// are the unit-testable core. -/// - Hard rule: no await, no IMeshService.QueryAsync, no ObserveQuery, no client +/// - Hard rule: no await, no IMeshService.QueryAsync, no Query, no client /// SubmitMessageRequest. Only Hub.Post + RegisterCallback + workspace stream writes. /// -public static class ThreadSubmission +internal static class ThreadSubmission { // ═════════════════════════════════════════════════════════════════════ // Pure helpers — unit-test surface @@ -52,484 +52,577 @@ public static ImmutableList FindUnprocessedUserMessages(MeshThread threa /// /// Returns the next round to dispatch given the current thread state. - /// Returns null when the thread is currently executing or has no queued user messages. + /// Returns null when the thread is currently executing or has nothing + /// queued. + /// + /// Inbox semantics. Every entry in + /// is ingested into a single + /// round — the inbox drains the whole queue at once, all drained ids move + /// into , and exactly one response cell + /// is allocated for the round. Multiple inputs share one response cell; + /// the agent treats the drained list as a multi-message turn. /// public static RoundDispatch? PlanNextRound(MeshThread thread) { - if (thread.IsExecuting) return null; - var unprocessed = FindUnprocessedUserMessages(thread); - if (unprocessed.IsEmpty) return null; - - var responseMessageId = Guid.NewGuid().ToString("N")[..8]; + // Allow planning when the thread is idle / cancelled (a stopped round + // re-dispatches like Idle if input is queued) OR has just been claimed + // by InstallServerWatcher (Status==StartingExecution). Reject the active + // phase (Executing) — it owns the in-flight round. + if (thread.Status is not (ThreadExecutionStatus.Idle + or ThreadExecutionStatus.Cancelled + or ThreadExecutionStatus.StartingExecution)) + return null; + if (thread.PendingUserMessages.IsEmpty) return null; + + var ids = ComputeDrainIds(thread); + if (ids.IsEmpty) return null; + + // 🚨 Deterministic per-round response cell id — NOT a fresh Guid. + // The submission claim Status oscillates (StartingExecution → rollback → + // Idle → re-claim, and Executing → StartingExecution resume bounce), so the + // _Exec round watcher fires DispatchAfterClaim several times for ONE logical + // round. A fresh Guid each call minted a NEW response cell per fire → + // duplicate cells. Deriving the id from the round's drained user ids (+ their + // Timestamp/Text) makes every re-dispatch of the SAME logical round resolve + // to the SAME cell (idempotent create/commit), while a genuinely new round + // (next turn / resubmit — different drained ids, or the same id with a fresh + // resubmit Timestamp) gets a distinct cell. + // + // 🚨 The id MUST NOT depend on Messages.Count. Under rapid concurrent submits + // several DispatchRound calls for the SAME logical round run before any of + // their commits settle; each prior commit appends its response cell to + // Messages, so a Count-keyed id reads a DIFFERENT count per call → a DIFFERENT + // id → a DISTINCT cell PER call (the 4-cells-for-3-messages dispatch STORM + // that wedged RapidSubmits_PileUpAndAllIngest: the thread never reaches a + // terminal state). The drained id set + per-message Timestamp already + // identifies the round uniquely, so Count adds no distinguishing power — only + // the churn that breaks idempotency. + var responseMessageId = DeriveDeterministicResponseId(ids, thread.PendingUserMessages); + // 🎯 The round's selection (agent / model / harness / context / attachments) comes from the + // LAST DRAINED MESSAGE. Each user message captures the composer's selection at the moment it + // was Sent, so the message is self-describing: a later /agent /model /harness pick (or a + // dropdown change) never rewrites the selection of an already-queued message, and a multi- + // message drain runs under the LAST message's selection (its Text is also this turn's input). + // There is NO thread-level Pending* mirror. + // + // The live composer (Thread.Composer — the data-bound selectors source of truth) is the + // FALLBACK only: used per field when the drained message carries no explicit value (e.g. a + // programmatic submit that didn't stamp the selection). Reading message-first keeps the + // delegation/sub-thread flow correct — a sub-thread message carries its OWN agent, not the + // parent composer's. + var composer = thread.Composer; + var lastDrained = ids + .Select(id => thread.PendingUserMessages.TryGetValue(id, out var m) ? m : null) + .LastOrDefault(m => m is not null); return new RoundDispatch( - unprocessed, + ids, responseMessageId, - thread.PendingAgentName, - thread.PendingModelName, - thread.PendingContextPath, - thread.PendingAttachments); + lastDrained?.AgentName ?? composer?.AgentName, + lastDrained?.ModelName ?? composer?.ModelName, + lastDrained?.Harness ?? composer?.Harness, + lastDrained?.ContextPath ?? composer?.ContextPath, + lastDrained?.Attachments); } // ═════════════════════════════════════════════════════════════════════ - // Client-side API — invoked from Blazor click handlers (void, non-blocking) + // Server-side API — invoked from thread hub initialization // ═════════════════════════════════════════════════════════════════════ /// - /// Submits a user message into an existing thread. Posts a single - /// to the thread hub — the handler - /// runs locally (one atomic - /// workspace.UpdateMeshNode), and the server watcher then creates the - /// satellite cell and dispatches the round. No separate CreateNodeRequest from - /// the client — that was the duplicate-dispatch source in the legacy flow. + /// The full drain set for one inbox round: every pending entry, ordered by + /// UserMessageIds (submission order); orphan pending entries not yet in + /// UserMessageIds are appended at the end (defensive, shouldn't happen). + /// Shared by and the round-commit staleness + /// check in CommitRoundAndExecute — both MUST compute the identical + /// sequence for the same thread state. /// - public static void Submit(SubmitContext ctx) + internal static ImmutableList ComputeDrainIds(MeshThread thread) { - if (string.IsNullOrEmpty(ctx.ThreadPath)) - { - ctx.OnError?.Invoke("Submit requires ThreadPath. Use CreateThreadAndSubmit for new threads."); - return; - } - - var delivery = ctx.Hub.Post( - new AppendUserMessageRequest - { - ThreadPath = ctx.ThreadPath!, - UserMessageId = Guid.NewGuid().ToString("N")[..8], // ignored by handler — kept for back-compat shape - UserText = ctx.UserText, - AgentName = ctx.AgentName, - ModelName = ctx.ModelName, - ContextPath = ctx.ContextPath, - Attachments = ctx.Attachments - }, - o => o.WithTarget(new Address(ctx.ThreadPath!))); - - if (delivery == null) - { - ctx.OnError?.Invoke("Hub.Post returned null"); - return; - } - - ctx.Hub.RegisterCallback((IMessageDelivery)delivery, response => - { - if (response is IMessageDelivery { Message.Success: false } fail) - ctx.OnError?.Invoke($"Submit failed: {fail.Message.Error ?? "unknown"}"); - return response; - }); + var idsBuilder = ImmutableList.CreateBuilder(); + foreach (var id in thread.UserMessageIds) + if (thread.PendingUserMessages.ContainsKey(id) && !idsBuilder.Contains(id)) + idsBuilder.Add(id); + foreach (var id in thread.PendingUserMessages.Keys) + if (!idsBuilder.Contains(id)) + idsBuilder.Add(id); + return idsBuilder.ToImmutable(); } /// - /// Creates a new thread node, then submits the first user message via - /// on the new thread. - /// fires when the thread is confirmed. + /// Installs a continuous subscription on the thread hub's workspace. + /// Whenever the thread is idle and has unprocessed user messages, opens a new round. /// - public static void CreateThreadAndSubmit(SubmitContext ctx) - { - if (string.IsNullOrEmpty(ctx.Namespace)) - { - ctx.OnError?.Invoke("CreateThreadAndSubmit requires Namespace."); - return; - } + public static IDisposable InstallServerWatcher(IMessageHub threadHub) + => ThreadSubmissionServer.InstallServerWatcher(threadHub); - var threadNode = ThreadNodeType.BuildThreadNode(ctx.Namespace!, ctx.UserText, ctx.CreatedBy); - var fallbackPath = threadNode.Path!; + /// + /// Stable 8-hex-char response cell id for a round, derived from the drained + /// user-message ids and each drained message's Timestamp + Text. Deterministic so + /// repeated dispatches of the SAME logical round (status oscillation, and the + /// rapid-submit concurrent re-dispatch) reuse one cell; distinct across rounds + /// because the drained ids — or a resubmit's fresh Timestamp on the same id — + /// differ. Deliberately INDEPENDENT of Messages.Count: that count changes as + /// concurrent dispatches append their cells, and keying on it splits one logical + /// round into many cells (the dispatch storm). See PlanNextRound. + /// + internal static string DeriveDeterministicResponseId( + IReadOnlyList ids, + IReadOnlyDictionary pending) + { + // Include each drained pending message's Timestamp + Text in the key. A + // resubmit re-adds the SAME user id with a fresh Timestamp (DateTime.UtcNow) + // and new text (ResubmitMessage), so the Timestamp makes each round's cell + // distinct (Resubmit_*_NewRoundCreated / Resubmit_*_DoesNotDeadlock). The + // value is fixed on the pending message (not recomputed per dispatch), so it + // stays stable across one round's re-dispatch oscillation AND across the + // concurrent dispatches a rapid-submit burst produces. + var content = string.Join("|", ids.Select(id => + pending.TryGetValue(id, out var m) ? $"{m.Timestamp.Ticks}:{m.Text}" : id)); + var key = string.Join(",", ids) + "|" + content; + var hash = System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(key)); + return Convert.ToHexString(hash, 0, 4).ToLowerInvariant(); + } +} - var delivery = ctx.Hub.Post( - new CreateNodeRequest(threadNode), - o => o.WithTarget(new Address(ctx.Namespace!))); +// ═════════════════════════════════════════════════════════════════════════════ +// Legacy client-side API DELETED 2026-05-27 — use HubThreadExtensions instead: +// +// hub.StartThread(...) replaces ThreadSubmission.CreateThreadAndSubmit +// hub.SubmitMessage(...) replaces ThreadSubmission.Submit +// hub.ResubmitMessage(...) replaces ThreadSubmission.Resubmit / ApplyResubmit +// hub.DeleteFromMessage(...) replaces ThreadSubmission.ApplyDeleteFromMessage +// hub.MarkThreadDone(...) replaces ThreadSubmission.MarkThreadDone +// hub.RecordSubmissionFailure(...) replaces ThreadSubmission.ApplyRecordSubmissionFailure +// +// SubmitContext and ResubmitContext parameter-bag records also deleted. +// ═════════════════════════════════════════════════════════════════════════════ - if (delivery == null) - { - ctx.OnError?.Invoke("Hub.Post returned null"); - return; - } +/// +/// One execution round to dispatch. contains exactly one +/// id (per — one user message per round, +/// one response cell per round, Claude-Code-style turn structure). The collection +/// shape is kept for back-compat with downstream code that already iterates it. +/// +internal sealed record RoundDispatch( + ImmutableList UserMessageIds, + string ResponseMessageId, + string? AgentName, + string? ModelName, + string? Harness, + string? ContextPath, + IReadOnlyList? Attachments); - ctx.Hub.RegisterCallback((IMessageDelivery)delivery, response => - { - if (response is not IMessageDelivery { Message.Success: true } cnr) +/// +/// Server-side watcher: reactively dispatches an execution round whenever the thread +/// has unprocessed user messages and isn't already running. Pure observable composition +/// via : +/// +/// +/// Source: workspace.GetMeshNodeStream(). +/// DistinctUntilChanged on a fingerprint of +/// (IsExecuting, Messages.Count, IngestedMessageIds.Count, PendingUserMessages.Count) +/// so the same dispatchable state cannot fire twice. +/// Where: not currently executing AND has at least one +/// unprocessed user id or pending message. +/// SelectMany: each dispatchable emission produces a single +/// observable that creates satellite cells, commits +/// the round to the thread node, and posts to the _Exec hub. +/// +/// +/// No Throttle, no reentrancy flag, no scheduler-hop identity workarounds — +/// the source observable is the thread's own MeshNode stream and the chain runs in +/// the hub's natural scheduler. The previous imperative implementation (200 lines with +/// a dispatching flag + 50 ms Throttle + AsyncLocal fallbacks) is gone. +/// +internal static class ThreadSubmissionServer +{ + /// + /// Subscribes to the thread's OWN node stream and, whenever it observes + /// Status == Idle with non-empty , + /// runs the atomic claim (Status: Idle → StartingExecution) directly + /// against the same stream. The _Exec hosted hub's round watcher + /// observes the resulting transition (via the shared + /// ) and continues with Step B + C + /// (drain pending into , allocate response + /// cell, flip Status → Executing, stream). + /// + /// Single-flight is guaranteed by the atomic claim Update: the + /// hub's action block serialises concurrent emissions, so the first + /// lambda that sees Status == Idle flips it; every other lambda + /// re-reads Status != Idle inside the predicate and bails. No + /// in-memory Interlocked gate, no separate intent field, no + /// cross-hub trigger Post. + /// + public static IDisposable InstallServerWatcher(IMessageHub threadHub) + { + var logger = threadHub.ServiceProvider.GetService>(); + var threadPath = threadHub.Address.Path; + var accessService = threadHub.ServiceProvider.GetService(); + // 🚨 Identity: every write this watcher performs — the ReconcileUserMessageIds own-write in + // .Do below AND the claim Update — runs under the thread OWNER's identity via + // AccessContextScope.FromNode, the SAME pattern InstallExecRoundWatcher uses. Even an OWN + // write is NOT identity-free: UpdateOwn drives the data-source SynchronizationStream.Update, + // which posts a (non-exempt, User-posting) UpdateStreamRequest from the sync/ hub. This + // watcher fires on a scheduler that does NOT carry the originating user's AsyncLocal; in + // prod/Orleans (no persistent circuit context on the grain) that ambient is null, so without + // the re-stamp the UpdateStreamRequest is posted with NO AccessContext and the never-null + // PostPipeline guard fails it → a DeliveryFailure storm the submit never recovers from + // ("thread disappears on submit"). The access check that gated the dispatch already happened + // (a user with no thread access could not have queued the pending message), so the round + // inherits the trust the submit already verified. + // Self-healing: this watcher drains pending input into the next round (the + // resubmit / follow-up-message path). If its stream FAULTS it must NOT die + // silently — a dead watcher means the resubmit is never claimed and the + // thread parks forever (the live-path "observer dies" deadlock behind + // Resubmit_AfterExecution_DoesNotDeadlock). On fault, re-establish after a + // short delay. Mirrors ThreadExecution.InitializeThreadLifecycle and + // ActivityControlPlaneExtensions.WatchControlPlane. + var serial = new System.Reactive.Disposables.SerialDisposable(); + var disposed = false; + void Establish() => serial.Disposable = threadHub.GetWorkspace().GetMeshNodeStream() + // 🚨 React ONLY to control-plane changes — Status, and the pending / ingested / + // user-message id sets. A running round emits hundreds of StreamingText / Messages + // updates; without this filter the watcher re-ran ReconcileUserMessageIds (an + // own-write self-heal) AND re-evaluated dispatch on every one of them — i.e. it + // reacted to, and could write state during, an in-flight round. The fingerprint + // excludes all streaming fields, so an Executing round produces NO fingerprint + // change and the watcher stays dormant until a real submit / ingest / dispatch / + // terminal transition. (Dispatch is still additionally guarded by NeedsDispatch = + // Idle/Cancelled only — this just stops the watcher waking at all mid-round.) + .DistinctUntilChanged(SubmissionFingerprint) + .Do(n => { - var err = (response as IMessageDelivery)?.Message.Error ?? "unknown"; - ctx.OnError?.Invoke($"Thread creation failed: {err}"); - return response; - } - - var createdNode = cnr.Message.Node ?? threadNode; - var createdPath = createdNode.Path ?? fallbackPath; - ctx.OnThreadCreated?.Invoke(createdNode); - - var append = ctx.Hub.Post( - new AppendUserMessageRequest + if (n?.Content is MeshThread t) { - ThreadPath = createdPath, - UserMessageId = Guid.NewGuid().ToString("N")[..8], - UserText = ctx.UserText, - AgentName = ctx.AgentName, - ModelName = ctx.ModelName, - ContextPath = ctx.ContextPath, - Attachments = ctx.Attachments + logger?.LogDebug( + "[SubmissionWatcher] OBSERVE {ThreadPath} status={Status} pending={Pending} ingested={Ingested} userIds={UserIds}", + threadPath, t.Status, t.PendingUserMessages.Count, + t.IngestedMessageIds.Count, t.UserMessageIds.Count); + } + // 🚨 Survive the rapid-submit storm. Each cross-mirror SubmitMessage patches + // the UserMessageIds ARRAY off its own (stale) base; RFC 7396 REPLACES arrays, + // so concurrent submits clobber each other's ids — the thread settles with + // UserMessageIds shorter than the work actually queued (the RapidSubmits / + // Cancel_WithMultiplePending reds). The dict-keyed PendingUserMessages and the + // IngestedMessageIds are merge-safe and authoritative, so the OWNER reconciles + // the derived list back to a superset via an OWN write (serialised, no clobber). + // Idempotent: once UserMessageIds ⊇ pending ∪ ingested the recomputed node is + // byte-identical and the stream's value-equality check dedupes it — no loop. + // 🚨 Run the own-write under the thread OWNER's identity (see the identity note + // above) so its UpdateStreamRequest carries a non-null AccessContext. + using (MeshWeaver.Mesh.Security.AccessContextScope.FromNode(n, accessService, logger)) + ReconcileUserMessageIds(threadHub, n, logger); + }) + .Where(NeedsDispatch) + .Subscribe( + triggerNode => + { + logger?.LogDebug("[SubmissionWatcher] DISPATCH_TRIGGERED for {ThreadPath}", threadPath); + var workspace = threadHub.GetWorkspace(); + // 🚨 Re-stamp the thread OWNER's identity for the claim own-write (see the + // identity note at the top of this method). The .Update().Subscribe() below + // posts the UpdateStreamRequest SYNCHRONOUSLY on this thread, so capturing the + // owner here is sufficient for it to ride the post. + using var dispatchScope = MeshWeaver.Mesh.Security.AccessContextScope.FromNode(triggerNode, accessService, logger); + workspace.GetMeshNodeStream().Update(node => + { + var t = node.Content as MeshThread; + if (t is null + || t.Status is not (ThreadExecutionStatus.Idle or ThreadExecutionStatus.Cancelled) + || t.PendingUserMessages.IsEmpty) + { + logger?.LogDebug( + "[SubmissionWatcher] CLAIM_SKIPPED {ThreadPath} status={Status} pending={Pending} (re-check inside lambda)", + threadPath, t?.Status, t?.PendingUserMessages.Count); + return node; // already running or no longer pending + } + logger?.LogInformation( + "[SubmissionWatcher] CLAIMED: {ThreadPath} pending={Pending} → Status=StartingExecution", + threadPath, t.PendingUserMessages.Count); + // 🚨 The claim lambda MUST be deterministic on its input — concurrent + // emissions can result in multiple lambdas running with the same + // pre-update snapshot; if the resulting node is byte-identical, the + // downstream SynchronizationStream.SetCurrent's value-equality check + // dedupes the second commit (no second emission, no second dispatch). + // Don't stamp DateTime.UtcNow here — DispatchRound sets + // ExecutionStartedAt as part of its Executing-state commit, and that + // path runs serially on the action block via the _Exec round watcher. + return node with + { + Content = t with + { + Status = ThreadExecutionStatus.StartingExecution + } + }; + }).Subscribe( + _ => { /* _Exec's InstallExecRoundWatcher sees Status=StartingExecution and dispatches */ }, + ex => logger?.LogWarning(ex, + "[SubmissionWatcher] claim Update failed for {ThreadPath}", threadPath)); }, - o => o.WithTarget(new Address(createdPath))); - - if (append != null) - { - ctx.Hub.RegisterCallback((IMessageDelivery)append, appendResp => + ex => { - if (appendResp is IMessageDelivery { Message.Success: false } fail) - ctx.OnError?.Invoke($"Append after thread create failed: {fail.Message.Error ?? "unknown"}"); - return appendResp; + logger?.LogWarning(ex, + "[SubmissionWatcher] stream errored for {ThreadPath} — re-establishing", + threadPath); + if (!disposed) + System.Reactive.Linq.Observable.Timer(TimeSpan.FromSeconds(1)) + .Subscribe(_ => Establish()); }); - } - return response; + Establish(); + return System.Reactive.Disposables.Disposable.Create(() => + { + disposed = true; + serial.Dispose(); }); } /// - /// Resubmits an existing user message: truncates Messages and IngestedMessageIds - /// after the replayed id, optionally updating the user cell text. The server watcher - /// creates a new output cell. + /// Predicate equivalent: the thread is idle and has pending work. Used by + /// the submission watcher to filter dispatchable emissions. The lambda + /// inside Update re-checks the same condition so concurrent + /// emissions still single-flight. /// - public static void Resubmit(ResubmitContext ctx) + private static bool NeedsDispatch(MeshNode? node) { - if (string.IsNullOrEmpty(ctx.ThreadPath) || string.IsNullOrEmpty(ctx.UserMessageIdToReplay)) - { - ctx.OnError?.Invoke("Resubmit requires ThreadPath and UserMessageIdToReplay."); - return; - } - - var delivery = ctx.Hub.Post( - new ResubmitUserMessageRequest - { - ThreadPath = ctx.ThreadPath, - UserMessageId = ctx.UserMessageIdToReplay, - NewUserText = ctx.NewUserText, - AgentName = ctx.AgentName, - ModelName = ctx.ModelName - }, - o => o.WithTarget(new Address(ctx.ThreadPath))); - - if (delivery == null) - { - ctx.OnError?.Invoke("Hub.Post returned null"); - return; - } - - ctx.Hub.RegisterCallback((IMessageDelivery)delivery, response => - { - if (response is IMessageDelivery { Message.Success: false } fail) - ctx.OnError?.Invoke($"Resubmit failed: {fail.Message.Error ?? "unknown"}"); - return response; - }); + if (node?.Content is not MeshThread t) return false; + // Idle OR Cancelled (a stopped round re-dispatches like Idle) with + // queued input → claim a fresh round. + return t.Status is ThreadExecutionStatus.Idle or ThreadExecutionStatus.Cancelled + && t.PendingUserMessages.Count > 0; } - // ═════════════════════════════════════════════════════════════════════ - // Server-side API — invoked from thread hub initialization - // ═════════════════════════════════════════════════════════════════════ - - /// - /// Installs a continuous subscription on the thread hub's workspace. - /// Whenever the thread is idle and has unprocessed user messages, opens a new round - /// (creates output cell, updates Messages/Ingested/IsExecuting/Active/Pending*, posts to _Exec). - /// - public static IDisposable InstallServerWatcher(IMessageHub threadHub) - => ThreadSubmissionServer.InstallServerWatcher(threadHub); - - // ═════════════════════════════════════════════════════════════════════ - // Server-side handlers for client requests - // ═════════════════════════════════════════════════════════════════════ - /// - /// Thread-hub handler kept as a back-compat shim: re-routes legacy - /// through the new - /// path. New callers should write directly to the thread's MeshNode via ThreadInput - /// instead of posting this request. + /// The submission watcher's reaction key: ONLY the control-plane fields it acts on — + /// and the pending / ingested / user-message id sets. + /// Deliberately EXCLUDES StreamingText, StreamingToolCalls, Messages content, Summary, + /// ExecutionStartedAt, etc. — the high-churn fields a round mutates while Executing. Used + /// with DistinctUntilChanged so the watcher wakes only on a genuine submit / ingest / + /// dispatch / terminal transition, never on streaming churn during an in-flight round. /// - public static IMessageDelivery HandleAppendUserMessage( - IMessageHub hub, - IMessageDelivery delivery) + private static string SubmissionFingerprint(MeshNode? node) { - var req = delivery.Message; - try - { - var msg = ThreadInput.CreateUserMessage( - req.UserText, - createdBy: delivery.AccessContext?.ObjectId, - authorName: null, - agentName: req.AgentName, - modelName: req.ModelName, - contextPath: req.ContextPath, - attachments: req.Attachments); - // Note: this shim ignores req.UserMessageId — the new flow allocates its own. - // Tests + the legacy client posted the id eagerly; the new flow only uses - // server-allocated ids so we don't honour the request's id here. - ThreadInput.AppendUserInput(hub.GetWorkspace(), req.ThreadPath, msg); - hub.Post(new AppendUserMessageResponse { Success = true }, o => o.ResponseFor(delivery)); - } - catch (Exception ex) - { - hub.Post(new AppendUserMessageResponse { Success = false, Error = ex.Message }, o => o.ResponseFor(delivery)); - } - return delivery.Processed(); + if (node?.Content is not MeshThread t) return "∅"; + static string Join(IEnumerable ids) => + string.Join(",", ids.OrderBy(x => x, StringComparer.Ordinal)); + return string.Concat( + t.Status.ToString(), "|", + Join(t.PendingUserMessages.Keys), "|", + Join(t.IngestedMessageIds), "|", + Join(t.UserMessageIds)); } /// - /// Thread-hub handler: records a failed submission. Creates an error response cell - /// (role=assistant, Text=ErrorMessage, marked as AgentResponse), registers the user - /// message id on the thread if not already there, and marks it as ingested. - /// The UI sees the natural chat flow: user message followed by an error reply. + /// Owner-side self-heal for two derived-id invariants that an own-hub write can break + /// under load. Runs as an OWN write on the thread hub (serialised by the action block — + /// no clobber), idempotent and self-terminating (when nothing is missing the write is + /// skipped, and a reconciled node is byte-identical to the next observation so the stream + /// dedupes it). It NEVER touches , so it can + /// only ever ADD to the derived id arrays — it cannot re-queue work and therefore cannot + /// re-dispatch or storm. + /// + /// (a) UserMessageIds ⊇ pending ∪ ingested. Cross-mirror submits patch the + /// UserMessageIds array off a stale base and RFC 7396 array-replace drops concurrent + /// additions; the keyed dict survives, so the owner reconstructs the list. + /// + /// (b) IngestedMessageIds ⊇ (UserMessageIds ∩ Messages). + /// 's CommitRoundAndExecute (and 's + /// drain) add a user id to Messages AND IngestedMessageIds in ONE atomic own + /// write — so a user message whose satellite cell is in Messages yet is neither + /// pending nor ingested was materialised but lost its ingested mark to a non-atomic + /// own-hub write under 2-core load (the Cancel_WithPendingMessages / RapidSubmits + /// CI reds, where the thread settles with pending=0, ingested=[u1] but the cell exists). + /// The message WAS processed (its cell is rendered), so re-mark it ingested. STOPGAP: + /// the LogWarning below makes the next CI hit self-diagnosing so the non-atomic-write root + /// cause in the framework write path can be pinned and fixed; this restores the user-visible + /// invariant (no acknowledged message left un-ingested) in the meantime. /// - public static IMessageDelivery HandleRecordSubmissionFailure( - IMessageHub hub, - IMessageDelivery delivery) + private static void ReconcileUserMessageIds(IMessageHub threadHub, MeshNode? node, ILogger? logger) { - var req = delivery.Message; - var errorResponseId = Guid.NewGuid().ToString("N")[..8]; + if (node?.Content is not MeshThread t) return; + var have = t.UserMessageIds.ToImmutableHashSet(); + var userIdsMissing = t.PendingUserMessages.Keys + .Concat(t.IngestedMessageIds) + .Where(id => !have.Contains(id)) + .Distinct() + .ToImmutableList(); - // Create the error response cell at {threadPath}/{errorResponseId}. - var errorCell = new MeshNode(errorResponseId, req.ThreadPath) - { - NodeType = ThreadMessageNodeType.NodeType, - MainNode = req.ThreadPath, - Content = new ThreadMessage - { - Role = "assistant", - Text = $"**Submission failed:** {req.ErrorMessage}", - Timestamp = DateTime.UtcNow, - Type = ThreadMessageType.AgentResponse - } - }; - hub.Post(new CreateNodeRequest(errorCell), o => o.WithTarget(hub.Address)); + // (b) materialised-but-not-ingested user messages (see remarks). + var ingestedSet = t.IngestedMessageIds.ToImmutableHashSet(); + var pendingKeys = t.PendingUserMessages.Keys.ToImmutableHashSet(); + var messageSet = t.Messages.ToImmutableHashSet(); + var ingestedMissing = t.UserMessageIds + .Where(id => messageSet.Contains(id) && !ingestedSet.Contains(id) && !pendingKeys.Contains(id)) + .Distinct() + .ToImmutableList(); + + if (userIdsMissing.IsEmpty && ingestedMissing.IsEmpty) return; + if (!ingestedMissing.IsEmpty) + logger?.LogWarning( + "[SubmissionWatcher] lost-message invariant restored for {ThreadPath}: {Ids} were in " + + "Messages+UserMessageIds but neither ingested nor pending — re-marking ingested " + + "(non-atomic own-hub write under load; root cause under investigation)", + threadHub.Address.Path, string.Join(",", ingestedMissing)); - // Update thread state: link user message (if missing) + error response + mark ingested. - hub.GetWorkspace().UpdateMeshNode(node => + threadHub.GetWorkspace().GetMeshNodeStream().Update(n => { - var t = node.Content as MeshThread ?? new MeshThread(); - var msgs = t.Messages; - if (!msgs.Contains(req.UserMessageId)) msgs = msgs.Add(req.UserMessageId); - if (!msgs.Contains(errorResponseId)) msgs = msgs.Add(errorResponseId); - var userIds = t.UserMessageIds.Contains(req.UserMessageId) - ? t.UserMessageIds - : t.UserMessageIds.Add(req.UserMessageId); - var ingested = t.IngestedMessageIds.Contains(req.UserMessageId) - ? t.IngestedMessageIds - : t.IngestedMessageIds.Add(req.UserMessageId); - return node with + // Re-derive inside the lambda from the CURRENT node — never the stale snapshot + // captured above — so the write reflects the latest merged state. + if (n.Content is not MeshThread cur) return n; + var curHave = cur.UserMessageIds.ToImmutableHashSet(); + var addUser = cur.PendingUserMessages.Keys + .Concat(cur.IngestedMessageIds) + .Where(id => !curHave.Contains(id)) + .Distinct() + .ToImmutableList(); + var curIngested = cur.IngestedMessageIds.ToImmutableHashSet(); + var curPending = cur.PendingUserMessages.Keys.ToImmutableHashSet(); + var curMessages = cur.Messages.ToImmutableHashSet(); + var addIngested = cur.UserMessageIds + .Where(id => curMessages.Contains(id) && !curIngested.Contains(id) && !curPending.Contains(id)) + .Distinct() + .ToImmutableList(); + if (addUser.IsEmpty && addIngested.IsEmpty) return n; // raced to reconciled — byte-identical, dedupes + return n with { - Content = t with + Content = cur with { - Messages = msgs, - UserMessageIds = userIds, - IngestedMessageIds = ingested, - // Clear any pending text for this message so the watcher doesn't dispatch it again. - PendingUserMessage = null + UserMessageIds = cur.UserMessageIds.AddRange(addUser), + IngestedMessageIds = cur.IngestedMessageIds.AddRange(addIngested) } }; - }); - - hub.Post(new AppendUserMessageResponse { Success = true }, o => o.ResponseFor(delivery)); - return delivery.Processed(); + }).Subscribe( + _ => { }, + ex => logger?.LogWarning(ex, + "[SubmissionWatcher] derived-id reconcile failed for {ThreadPath}", + threadHub.Address.Path)); } - /// - /// Thread-hub handler: truncates the thread after the replayed user message id, - /// drops it from IngestedMessageIds, optionally updates its text, and resets the - /// executing flags. Watcher re-dispatches. - /// - public static IMessageDelivery HandleResubmitUserMessage( - IMessageHub hub, - IMessageDelivery delivery) - { - var req = delivery.Message; - ApplyResubmit(hub, req.ThreadPath, req.UserMessageId, req.NewUserText, req.AgentName, req.ModelName); - hub.Post(new AppendUserMessageResponse { Success = true }, o => o.ResponseFor(delivery)); - return delivery.Processed(); - } /// - /// Truncates the thread after , drops it from - /// IngestedMessageIds so the watcher re-dispatches a new round, and optionally - /// updates the user cell text. Shared by - /// and the legacy shim. + /// Step B + Step C of the round, called from the _Exec hub's + /// round watcher after observing the parent thread's Status + /// transition to . + /// Drains all pending entries into , + /// materialises user satellite cells, allocates a single response cell, + /// transitions → + /// , and invokes + /// ExecuteMessageAsync directly on _Exec for streaming. /// - public static void ApplyResubmit( - IMessageHub hub, - string threadPath, - string userMessageId, - string? newUserText, - string? agentName, - string? modelName) + internal static void DispatchAfterClaim( + IMessageHub hub, MeshNode threadNode, ILogger? logger, + Action? onFailure = null) { - // Optionally update the user cell text. - if (!string.IsNullOrEmpty(newUserText)) + var thread = threadNode.Content as MeshThread; + if (thread is null) { - var updatedCell = new MeshNode(userMessageId, threadPath) - { - NodeType = ThreadMessageNodeType.NodeType, - Content = new ThreadMessage - { - Role = "user", - Text = newUserText, - Timestamp = DateTime.UtcNow, - Type = ThreadMessageType.ExecutedInput - } - }; - hub.Post(new UpdateNodeRequest(updatedCell), o => o.WithTarget(hub.Address)); + logger?.LogWarning( + "[DispatchAfterClaim] thread node has no MeshThread content for {Path}", + hub.Address.Path); + onFailure?.Invoke(); + return; } - - hub.GetWorkspace().UpdateMeshNode(node => + var dispatch = ThreadSubmission.PlanNextRound(thread); + if (dispatch is null) { - var t = node.Content as MeshThread ?? new MeshThread(); - var idx = t.Messages.IndexOf(userMessageId); - if (idx < 0) return node; - - var keep = t.Messages.Take(idx + 1).ToImmutableList(); - var trimmedUserIds = t.UserMessageIds.Where(uid => keep.Contains(uid)).ToImmutableList(); - var ingested = t.IngestedMessageIds.Remove(userMessageId); - return node with + // RESUME: an interrupted Executing round (InitializeThreadLifecycle + // re-entered StartingExecution) has no NEW pending input but still + // owns a response cell. Re-dispatch into that SAME cell rather than + // rolling back — the user's question already streamed a partial + // answer; we resume generating it. + if (thread.Status == ThreadExecutionStatus.StartingExecution + && !string.IsNullOrEmpty(thread.ActiveMessageId)) { - Content = t with - { - Messages = keep, - UserMessageIds = trimmedUserIds, - IngestedMessageIds = ingested, - IsExecuting = false, - ActiveMessageId = null, - ExecutionStartedAt = null, - PendingUserMessage = newUserText ?? t.PendingUserMessage, - PendingAgentName = agentName ?? t.PendingAgentName, - PendingModelName = modelName ?? t.PendingModelName - } - }; - }); - } -} - -/// -/// Input for a client-side submission (existing or new thread). -/// -public sealed record SubmitContext -{ - public required IMessageHub Hub { get; init; } - /// Target thread path. Null for . - public string? ThreadPath { get; init; } - /// Parent namespace for new thread creation. Required for . - public string? Namespace { get; init; } - public required string UserText { get; init; } - public string? AgentName { get; init; } - public string? ModelName { get; init; } - public string? ContextPath { get; init; } - public IReadOnlyList? Attachments { get; init; } - public string? CreatedBy { get; init; } - public string? AuthorName { get; init; } + logger?.LogInformation( + "[DispatchAfterClaim] resuming interrupted round {ResponseId} for {Path}", + thread.ActiveMessageId, hub.Address.Path); + // No selection here: a resume has no pending message to read it from. The + // round's selection (agent/model/harness/context) is the persisted response + // cell's — ExecuteMessageAsync recovers it from the existing cell on resume. + var resumeDispatch = new RoundDispatch( + ImmutableList.Empty, + thread.ActiveMessageId!, + AgentName: null, + ModelName: null, + Harness: null, + ContextPath: null, + Attachments: null); + DispatchRound(hub, threadNode, resumeDispatch, logger, onFailure, isResume: true); + return; + } - /// - /// Called exactly once if the submit fails (post returned null, timeout, permission denied). - /// Never invoked after a successful submit. - /// - public Action? OnError { get; init; } + logger?.LogDebug( + "[DispatchAfterClaim] nothing to dispatch (post-claim race?) for {Path} — rolling status back to Idle", + hub.Address.Path); + // Roll the claim back so the next watcher tick can re-trigger. + // Rollback writes the thread node — `hub` here is parentHub (the + // thread hub), so its own GetMeshNodeStream is the OWN handle. + hub.GetWorkspace().GetMeshNodeStream().Update(n => + { + var t = n.ContentAs(hub.JsonSerializerOptions, logger); + // Existing node whose content can't be recovered → leave it alone, NEVER clobber. + if (n.Content is not null && t is null) + return n; + t ??= new MeshThread(); + // 🚨 Roll back ONLY a stuck StartingExecution claim that found nothing + // to dispatch. NEVER roll back an Executing round. The claim Status + // oscillates and the _Exec round watcher can fire DispatchAfterClaim + // more than once for one logical round; a duplicate fire reaches here + // with dispatch==null AFTER the real commit already flipped + // StartingExecution→Executing and drained PendingUserMessages. Blindly + // forcing Idle there un-did the RUNNING round, so the next watcher tick + // saw Idle + (still/again) pending and re-claimed the SAME input into a + // fresh round — the re-dispatch loop (hundreds of response-cell creates, + // round never settles: Resubmit_AfterExecution_DoesNotDeadlock under the + // full Orleans sequence). Invariant: whenever pending exists the watcher + // requests execution start, and once started we never silently undo it. + return t.Status == ThreadExecutionStatus.StartingExecution + ? n with { Content = t with { Status = ThreadExecutionStatus.Idle, ExecutionStartedAt = null } } + : n; + }).Subscribe( + _ => { }, + ex => logger?.LogWarning(ex, + "[DispatchAfterClaim] rollback Update failed for {Path}", hub.Address.Path)); + return; + } + DispatchRound(hub, threadNode, dispatch, logger, onFailure); + } /// - /// Called exactly once for when the - /// thread node is confirmed. The caller typically navigates here. + /// Re-launch an interrupted round that is ALREADY + /// — the cold-load / self-heal recovery case driven by + /// InitializeThreadLifecycle. The in-flight streaming Task is gone (the + /// hub re-activated), so we re-run the round into its EXISTING response cell. + /// + /// 🚨 This STAYS Executing. We never re-enter + /// StartingExecution from Executing — that inverse of the commit + /// edge (StartingExecution → Executing) is the re-dispatch ping-pong: + /// the exec round watcher commits StartingExecution→Executing while recovery + /// (self-healing, re-reading the node) flips Executing→StartingExecution, and + /// the two volley under load. Recovery now writes NO status; it calls this + /// directly. The caller (InitializeThreadLifecycle) guarantees a single + /// invocation per , and + /// CommitRoundAndExecute's single-fire guard covers re-emissions. /// - public Action? OnThreadCreated { get; init; } -} - -/// -/// Input for a resubmission (truncate + re-ingest). -/// -public sealed record ResubmitContext -{ - public required IMessageHub Hub { get; init; } - public required string ThreadPath { get; init; } - public required string UserMessageIdToReplay { get; init; } - /// New text for the user cell. Null means reuse the existing cell text. - public string? NewUserText { get; init; } - public string? AgentName { get; init; } - public string? ModelName { get; init; } - public Action? OnError { get; init; } -} - -/// -/// One execution round to dispatch. Includes every unprocessed user message id -/// (batched ingestion) and the newly allocated output cell id. -/// -public sealed record RoundDispatch( - ImmutableList UserMessageIds, - string ResponseMessageId, - string? AgentName, - string? ModelName, - string? ContextPath, - IReadOnlyList? Attachments); - -/// -/// Server-side watcher: observes thread state changes and dispatches execution rounds. -/// Installed once on thread hub initialization. Non-blocking; uses only Post + RegisterCallback -/// and workspace stream subscriptions. -/// -internal static class ThreadSubmissionServer -{ - public static IDisposable InstallServerWatcher(IMessageHub threadHub) + internal static void ResumeInterruptedRound( + IMessageHub threadHub, MeshNode threadNode, ILogger? logger) { - var logger = threadHub.ServiceProvider.GetService>(); - var workspace = threadHub.GetWorkspace(); - var threadPath = threadHub.Address.Path; - - // Reentrancy guard: 0=idle, 1=dispatching. - // Held until IsExecuting=true is observed back through the same stream, so a - // re-emission triggered by our own response-cell write or PendingUserMessages - // patch can't double-dispatch. - var dispatching = 0; - - // Subscribe to this thread's own MeshNode (via MeshNodeReference) instead of the - // collection-wide stream — fewer wakeups, and the patches we observe are exactly - // the writes against this thread. - // - // Throttle by a small window so a burst of rapid AppendUserMessageRequest patches - // (user submits 3 messages in quick succession, or the GUI batches submits) coalesce - // into a SINGLE dispatch with all the queued user ids in one round / one response - // cell. Without throttling each patch individually wins the reentrancy guard and - // produces one round per submit. - var sub = workspace.GetStream(new MeshNodeReference()) - ?.Throttle(TimeSpan.FromMilliseconds(50)) - ?.Subscribe(change => - { - var threadNode = change.Value; - if (threadNode?.Content is not MeshThread thread) return; - - // IsExecuting=true is visible — we held the guard waiting for this commit. - if (thread.IsExecuting && dispatching == 1) - { - Interlocked.Exchange(ref dispatching, 0); - return; - } - if (thread.IsExecuting) return; - - if (Interlocked.CompareExchange(ref dispatching, 1, 0) != 0) - return; - - var releaseGuard = true; - try - { - var dispatch = ThreadSubmission.PlanNextRound(thread); - if (dispatch is null) return; - - // Hold the guard. It will be released when we observe IsExecuting=true - // back on this same stream above (or on hard failure inside DispatchRound). - releaseGuard = false; - DispatchRound(threadHub, threadNode, dispatch, logger, - onFailure: () => Interlocked.Exchange(ref dispatching, 0)); - } - catch (Exception ex) - { - logger?.LogWarning(ex, "[ThreadSubmission] Server watcher iteration failed for {ThreadPath}", threadPath); - } - finally - { - if (releaseGuard) Interlocked.Exchange(ref dispatching, 0); - } - }); + if (threadNode.Content is not MeshThread thread + || string.IsNullOrEmpty(thread.ActiveMessageId)) + return; - return sub ?? System.Reactive.Disposables.Disposable.Empty; + // No selection here: a resume has no pending message to read it from. The round's + // selection (agent/model/harness/context) is the persisted response cell's — + // ExecuteMessageAsync recovers it from the existing cell on resume. + var resumeDispatch = new RoundDispatch( + ImmutableList.Empty, + thread.ActiveMessageId!, + AgentName: null, + ModelName: null, + Harness: null, + ContextPath: null, + Attachments: null); + DispatchRound(threadHub, threadNode, resumeDispatch, logger, onFailure: null, isResume: true); } /// @@ -546,38 +639,374 @@ private static void DispatchRound( MeshNode threadNode, RoundDispatch dispatch, ILogger? logger, - Action? onFailure = null) + Action? onFailure = null, + bool isResume = false) { var threadPath = hub.Address.Path; var responseMsgId = dispatch.ResponseMessageId; var responsePath = $"{threadPath}/{responseMsgId}"; - var thread = threadNode.Content as MeshThread ?? new MeshThread(); + // Read-side recovery: top-of-method read (not a write-back), so no clobber + // guard — recover a degraded JsonElement, preserve the existing new-on-absent fallback. + var thread = threadNode.ContentAs(hub.JsonSerializerOptions, logger) ?? new MeshThread(); var mainEntity = threadNode.MainNode ?? dispatch.ContextPath ?? threadPath; var accessService = hub.ServiceProvider.GetService(); - var userCtx = accessService?.Context ?? accessService?.CircuitContext; - if (userCtx is null && !string.IsNullOrEmpty(thread.CreatedBy)) + var asyncLocalCtx = accessService?.Context; + var circuitCtx = accessService?.CircuitContext; + + // The AsyncLocal at this point may be the THREAD HUB's own address — the + // watcher fires on a Throttle timer scheduler and captures whatever + // ExecutionContext was active at the time `Subscribe` was called (hub init, + // when SetContext hadn't yet propagated). Treat hub-as-user as no-identity + // and fall through to the wrapping MeshNode.CreatedBy (set by the + // CreateNodeRequest handler from the requester's AccessContext). + var hubAsUserMatch = asyncLocalCtx?.ObjectId is { } id + && (string.Equals(id, threadPath, StringComparison.Ordinal) + || string.Equals(id, hub.Address.ToFullString(), StringComparison.Ordinal)); + var userCtx = hubAsUserMatch ? null : (asyncLocalCtx ?? circuitCtx); + + var fellBackToCreatedBy = false; + // Resolution: thread content's CreatedBy → wrapping node's CreatedBy → null. + var resolvedCreatedBy = !string.IsNullOrEmpty(thread.CreatedBy) + ? thread.CreatedBy + : threadNode.CreatedBy; + if (userCtx is null && !string.IsNullOrEmpty(resolvedCreatedBy)) { - userCtx = new AccessContext { ObjectId = thread.CreatedBy, Name = thread.CreatedBy }; + userCtx = new AccessContext { ObjectId = resolvedCreatedBy, Name = resolvedCreatedBy }; + fellBackToCreatedBy = true; } + // Identity-trace at the dispatch boundary. The watcher callback runs after + // Throttle(50ms) on a timer scheduler — AsyncLocal context from the original + // delivery is gone here, so we expect asyncLocal=null and fall back to either + // the persistent circuit context (Blazor) or thread.CreatedBy (Orleans). + logger?.LogInformation( + "[ThreadSubmission] DispatchRound identity thread={ThreadPath} responseId={ResponseId} " + + "asyncLocal={AsyncLocal} hubAsUserMatch={HubAsUser} circuit={Circuit} threadCreatedBy={ThreadCreatedBy} " + + "nodeCreatedBy={NodeCreatedBy} fallbackToCreatedBy={FallbackToCreatedBy} effective={Effective}", + threadPath, responseMsgId, + asyncLocalCtx?.ObjectId ?? "(null)", + hubAsUserMatch, + circuitCtx?.ObjectId ?? "(null)", + thread.CreatedBy ?? "(null)", + threadNode.CreatedBy ?? "(null)", + fellBackToCreatedBy, + userCtx?.ObjectId ?? "(null)"); + var meshService = hub.ServiceProvider.GetRequiredService(); // Step 0: materialize user satellite cells from PendingUserMessages. - // Only ids present in dispatch.UserMessageIds AND PendingUserMessages need creation - // here — legacy paths (PendingUserMessage string) create cells elsewhere. + // dispatch.UserMessageIds is the full set the inbox drains this round + // (PlanNextRound returns every entry). Each cell will be created below + // and committed to Messages atomically with the response cell. var pendingForRound = dispatch.UserMessageIds .Where(id => thread.PendingUserMessages.ContainsKey(id)) .Select(id => (Id: id, Msg: thread.PendingUserMessages[id])) .ToImmutableList(); - var combinedUserText = pendingForRound.Count > 0 - ? string.Join("\n\n---\n\n", pendingForRound.Select(p => p.Msg.Text)) - : (thread.PendingUserMessage ?? ""); + // The "current" user input fed to the agent is the LAST drained message — + // earlier drained messages already exist as user cells in Messages and + // load via LoadFullConversationHistory (with the last one excluded via + // SubmitMessageRequest.UserMessageId). Multi-message round: agent sees + // history's user cells consecutively, then this last one as the + // current turn. Empty on resume (no pending message) — the existing user + // cell is already in history and the !isResume guard below lets resume proceed. + var roundUserText = pendingForRound.Count > 0 + ? pendingForRound[^1].Msg.Text + : ""; + + // 🚫 Never launch a round with nothing to send. A whitespace-only round — a slash command + // whose text was cut, or a stray empty submission — has no user content; running it reaches + // CreateChatClient with no input and storms "No model selected", and the empty round never + // settles (the wedge). The upstream StartThread/SubmitMessage guards already avoid SEEDING an + // empty round; this is the watcher-level backstop so a round NEVER launches on empty. Resumes + // legitimately carry no fresh pending text (their user cell is already in Messages), so guard + // FRESH dispatches only. Roll back any StartingExecution claim so the thread settles to Idle + // instead of parking at StartingExecution. + if (!isResume && string.IsNullOrWhiteSpace(roundUserText)) + { + logger?.LogInformation( + "[ThreadSubmission] DispatchRound NOTHING_TO_RUN thread={ThreadPath} responseId={ResponseId} — skipping launch (no user content)", + threadPath, responseMsgId); + onFailure?.Invoke(); + return; + } + + // Step 2 + 3: commit the round to the thread state (one atomic + // UpdateMeshNode) and start agent streaming. Shared by the fresh-dispatch + // path (after the response cell is created) and the resume path (cell + // already exists). On a fresh round dispatch.UserMessageIds drains the + // pending queue into Messages; on resume it is empty (the round's user + // cells were ingested before the interruption) so the Add/AddRange/Remove + // steps are all no-ops and only the StartingExecution → Executing flip + + // ActiveMessageId re-stamp take effect. + void CommitRoundAndExecute() + { + // The IsExecuting check is the idempotency guard — every other watcher + // emission in this round skips, so this body runs exactly once per round. + // + // Subscribe is mandatory: cache.Update returns a cold + // IObservable; the side effect only runs on + // Subscribe. The downstream UpdateResponseCell + + // ExecuteMessageAsync chain off the Subscribe(onNext) + // so they only fire after the round commit is persisted. + // DispatchRound runs in parentHub context (hub = thread + // hub). Write through THIS hub's own node stream so + // sender = thread hub, AccessContext flows from the + // caller's identity. + // 🚨 Single-fire guard for the side effect, NOT just the state mutation. + // The Status check below makes the UpdateMeshNode lambda a no-op on every + // watcher re-emission after the first — but no-op Updates still call + // OnNext (see feedback_setcurrent_skips_noops: UpdateRemote completes + // inline with OnNext(current)). So without this flag the Subscribe(onNext) + // body re-runs ExecuteMessageAsync on EVERY thread-node change during the + // round (response-cell alloc, heartbeat stamp, streaming writes) → the + // 6×-duplicate-execution bug (OrleansNodeChangePropagation: the Create + // node-change lands in a later duplicate round's nodeChangeLog while an + // earlier round's completion writes the cell → UpdatedNodes=[]; also the + // AutoExecute text-empty + delegation-timeout reds). Only the emission + // that actually performed StartingExecution→Executing launches execution. + var didCommitThisEmission = false; + hub.GetWorkspace().GetMeshNodeStream(threadPath).Update(node => + { + var t = node.ContentAs(hub.JsonSerializerOptions, logger); + // Existing node whose content can't be recovered → leave it alone, NEVER clobber. + if (node.Content is not null && t is null) + return node; + t ??= new MeshThread(); + // Decide whether to LAUNCH from the lambda parameter's CURRENT status: + // • fresh claim: StartingExecution → Executing (the normal commit edge). + // • resume: the round is ALREADY Executing (cold-load / self-heal + // recovery re-launches the interrupted round). We STAY + // Executing — we must NEVER write Executing→StartingExecution + // (that inverse of the commit edge is the re-dispatch + // ping-pong). The cell already exists; this emission only + // re-launches the streaming loop. + // Anything else is an out-of-band state change — drop the commit. + // Reset-then-set so an optimistic-concurrency retry of this lambda + // reflects the FINAL decision: only a genuine launch leaves the flag true. + var canLaunch = isResume + // Resume launches from EITHER a still-claimed round + // (StartingExecution → Executing, the resume-from-claim path via + // DispatchAfterClaim) OR an already-Executing round (cold-load / + // self-heal recovery re-launch — stays Executing). Both are + // forward edges; only Executing→StartingExecution is forbidden, + // and recovery no longer writes it. + ? (t.Status == ThreadExecutionStatus.StartingExecution + || t.Status == ThreadExecutionStatus.Executing) + && t.ActiveMessageId == responseMsgId + : t.Status == ThreadExecutionStatus.StartingExecution; + if (!canLaunch) { didCommitThisEmission = false; return node; } + // NOTE (do NOT add a SequenceEqual staleness bail here): bailing + // when the CURRENT drain set differs from this dispatch's plan + // sounds safe but parks the claim forever when pending changed + // after the latest plan — no in-flight dispatch matches the new + // state and nothing re-fires the watcher (validated empirically: + // the bail variant froze Cancel_With*Pending at StartingExecution + // in 5/6 class runs). A stale dispatch that commits still drains + // only ids present in BOTH the plan and current pending; leftover + // entries re-dispatch when the round settles. + didCommitThisEmission = true; + + // 🚨 Compute the drain set from the CURRENT node — NEVER the pre-claim + // snapshot. dispatch.UserMessageIds / pendingForRound were captured back in + // DispatchAfterClaim, BEFORE this commit; the claim races pending-adds (a + // follow-up submit, or a cancel→re-dispatch) that change PendingUserMessages + // in the window between the plan and this commit. The fix: drain the PLANNED + // ids that are STILL pending right now, and key ingested / Messages / + // PendingUserMessages off THAT same set, so the three mutate consistently in + // one atomic write — we ingest exactly what we remove from pending. A planned + // id no longer pending is skipped (it was drained elsewhere — no ghost-ingest + // of an un-removed id, which left the thread Executing with the entry still + // pending → the Cancel_With*Pending stuck-round). A pending id NOT in this + // plan stays queued and re-dispatches when the round settles. Resume carries + // an empty plan, so drainIds is empty and only the status flip applies. + var drainIds = dispatch.UserMessageIds + .Where(uid => t.PendingUserMessages.ContainsKey(uid)) + .ToImmutableList(); + + // User ids in dispatch order, then the response id last. + // Contains check covers the resubmit case where u1 was already in + // Messages from a prior round — ApplyResubmit removed u1 from + // IngestedMessageIds (so the watcher re-dispatches it) but kept it + // in Messages, so a blind AddRange would duplicate it. Symmetric + // Contains check on responseMsgId catches resume (the cell is + // already in Messages) and DispatchRound retries. + var msgs = t.Messages; + foreach (var uid in drainIds) + if (!msgs.Contains(uid)) msgs = msgs.Add(uid); + if (!msgs.Contains(responseMsgId)) msgs = msgs.Add(responseMsgId); + + var ingested = t.IngestedMessageIds.AddRange( + drainIds.Where(uid => !t.IngestedMessageIds.Contains(uid))); + + // Restore the invariant UserMessageIds ⊇ IngestedMessageIds. A concurrent + // cross-hub SubmitMessage can drop an id from the UserMessageIds *array* + // (the owner merges field-by-field via RFC 7396, which REPLACES arrays — two + // rapid submits off the same stale base lose one id), while the dict-keyed + // PendingUserMessages this round drains from keeps it. So a settled thread + // can end up Idle with ingested=3 but UserMessageIds.Count=2 — the + // RapidSubmits_PileUpAndAllIngest failure (the thread is otherwise correct; + // only the derived list is short). The owner is authoritative for that list, + // so re-add any ingested id missing from it. + var userIds = t.UserMessageIds; + foreach (var uid in ingested) + if (!userIds.Contains(uid)) userIds = userIds.Add(uid); + + // Drop the entries we actually drained this commit — their satellites now + // exist and their ids are now in Messages + IngestedMessageIds. Keyed off + // drainIds (computed above from the CURRENT pending), so we remove exactly + // what we ingested — never a stale pendingForRound id that a concurrent + // path already consumed, and never leaving an ingested id still pending. + var pending = t.PendingUserMessages; + foreach (var uid in drainIds) + pending = pending.Remove(uid); + + return node with + { + Content = t with + { + Messages = msgs, + UserMessageIds = userIds, + IngestedMessageIds = ingested, + Status = ThreadExecutionStatus.Executing, + // ActiveMessageId is the canonical handle — + // full response path derives as {threadPath}/{ActiveMessageId}. + ActiveMessageId = responseMsgId, + ExecutionStartedAt = DateTime.UtcNow, + ExecutionStatus = null, + // The round's selection rides on RoundParams (from the drained + // ThreadMessage) — no thread-level Pending* mirror to write. + PendingUserMessages = pending + } + }; + }).Subscribe( + _ => + { + // Only the emission that flipped StartingExecution→Executing + // launches the round. Every other (no-op) emission's OnNext is a + // duplicate and must not re-enter ExecuteMessageAsync. + if (!didCommitThisEmission) return; + + ThreadExecution.UpdateResponseCell( + hub, responsePath, threadPath, responseMsgId, mainEntity, + msg => msg with { Text = "Allocating agent...", Status = ThreadMessageStatus.Streaming }, + logger); + + // Step 3: direct method call on _Exec hosted hub. + // No SubmitMessageRequest post — the agent loop runs as + // composed observables; only the LLM streaming is async. + var executionHub = hub.GetHostedHub( + new Address($"{hub.Address}/_Exec"), + config => config, + HostedHubCreation.Always); + + // 🚨 ExecuteMessageAsync now returns a COLD IObservable — the round runs + // ONLY on Subscribe, completes when the terminal Status write lands, and faults + // via OnError. Subscribe here (no fire-and-forget) and own the subscription on + // the thread-hub workspace (the round's natural lifetime owner, matching the + // disposal the method used to register internally). + var execSub = ThreadExecution.ExecuteMessageAsync(executionHub!, + new ThreadExecution.RoundParams( + ThreadPath: threadPath, + ResponseMessageId: responseMsgId, + UserMessageId: dispatch.UserMessageIds.LastOrDefault(), + UserMessageText: roundUserText, + AgentName: dispatch.AgentName, + ModelName: dispatch.ModelName, + Harness: dispatch.Harness, + ContextPath: dispatch.ContextPath, + Attachments: dispatch.Attachments), + userCtx) + .Subscribe( + _ => { }, + ex => + { + logger?.LogWarning(ex, + "[ThreadSubmission] Agent round faulted for {ResponseMsgId} on {ThreadPath}", + responseMsgId, threadPath); + // 🚨 An escaped fault means the round's OWN terminal handling did + // NOT run (or its terminal write failed — the gate faults on that + // too). Without a terminal write here the node stays Executing + // forever: onFailure only rolls back a StartingExecution claim (it + // must never undo a running round), so the fault path would + // reintroduce the stuck-Executing wedge this refactor kills. + // Write the terminal state deterministically: response cell → + // Error, thread → Idle — guarded on (Executing, THIS round's + // ActiveMessageId) so a newer round is never clobbered. + ThreadExecution.UpdateResponseCell( + hub, responsePath, threadPath, responseMsgId, mainEntity, + msg => msg with + { + Text = string.IsNullOrEmpty(msg.Text) + ? $"*Error: {ex.Message}*" + : $"{msg.Text}\n\n*Error: {ex.Message}*", + Status = ThreadMessageStatus.Error, + CompletedAt = DateTime.UtcNow + }, + logger); + hub.GetWorkspace().GetMeshNodeStream(threadPath).Update(n => + { + var t = n.ContentAs(hub.JsonSerializerOptions, logger); + // Existing node whose content can't be recovered → leave it alone. + if (n.Content is not null && t is null) + return n; + t ??= new MeshThread(); + return t.Status == ThreadExecutionStatus.Executing + && t.ActiveMessageId == responseMsgId + ? n with + { + Content = t with + { + Status = ThreadExecutionStatus.Idle, + ActiveMessageId = null, + ExecutionStartedAt = null, + ExecutionStatus = null, + Summary = $"Error: {ex.Message}" + } + } + : n; + }).Subscribe( + _ => { }, + termEx => logger?.LogError(termEx, + "[ThreadSubmission] terminal-state write after faulted round FAILED for {ThreadPath} — node may be stuck Executing", + threadPath)); + onFailure?.Invoke(); + }); + hub.GetWorkspace().AddDisposable(execSub); + }, + ex => + { + logger?.LogWarning(ex, + "[ThreadSubmission] Round commit UpdateMeshNode failed for {ResponseMsgId} on {ThreadPath}", + responseMsgId, threadPath); + onFailure?.Invoke(); + }); + } void AfterUserCellsReady() { + if (isResume) + { + // Resume into the EXISTING response cell — no CreateNodeRequest. + // Reset its streaming state (clear the partial text + tool calls + // left by the interrupted round) and commit directly. + ThreadExecution.UpdateResponseCell( + hub, responsePath, threadPath, responseMsgId, mainEntity, + msg => msg with + { + Text = "", + ToolCalls = ImmutableList.Empty, + Status = ThreadMessageStatus.Streaming, + CompletedAt = null + }, + logger); + CommitRoundAndExecute(); + return; + } + // Step 1: create the assistant output cell (CreateNodeRequest → RegisterCallback). + // Status=Streaming until the streaming loop transitions it to Completed/Cancelled/Error. var responseCell = new MeshNode(responseMsgId, threadPath) { NodeType = ThreadMessageNodeType.NodeType, @@ -589,7 +1018,8 @@ void AfterUserCellsReady() Timestamp = DateTime.UtcNow, Type = ThreadMessageType.AgentResponse, AgentName = dispatch.AgentName, - ModelName = dispatch.ModelName + ModelName = dispatch.ModelName, + Status = ThreadMessageStatus.Streaming } }; @@ -607,96 +1037,54 @@ void AfterUserCellsReady() return; } - hub.RegisterCallback((IMessageDelivery)createDelivery, response => - { - if (response is not IMessageDelivery { Message.Success: true }) - { - var err = (response as IMessageDelivery)?.Message.Error ?? "unknown"; - logger?.LogWarning("[ThreadSubmission] Response cell creation failed for {ResponseMsgId} on {ThreadPath}: {Error}", - responseMsgId, threadPath, err); - onFailure?.Invoke(); - return response; - } - - // Step 2: commit the round to the thread state (one atomic UpdateMeshNode). - // Both the user satellite cells (created above in the materialization step) - // and the response satellite cell (just confirmed in the CreateNodeRequest - // callback above) exist on the hub now. Only NOW do we add their ids into - // Messages — the GUI iterates Messages to render LayoutAreaControls, so - // every id it sees has a backing satellite. - // - // The IsExecuting check is the idempotency guard — every other watcher - // emission in this round skips, so this body runs exactly once per round. - hub.GetWorkspace().UpdateMeshNode(node => - { - var t = node.Content as MeshThread ?? new MeshThread(); - if (t.IsExecuting) return node; - - // User ids in dispatch order, then the response id last. - // Contains check covers the resubmit case where u1 was already in - // Messages from a prior round — ApplyResubmit removed u1 from - // IngestedMessageIds (so the watcher re-dispatches it) but kept it - // in Messages, so a blind AddRange would duplicate it. - var msgs = t.Messages; - foreach (var uid in dispatch.UserMessageIds) - if (!msgs.Contains(uid)) msgs = msgs.Add(uid); - msgs = msgs.Add(responseMsgId); - - var ingested = t.IngestedMessageIds.AddRange( - dispatch.UserMessageIds.Where(uid => !t.IngestedMessageIds.Contains(uid))); - - // Drop consumed PendingUserMessages entries — their satellites now exist - // and their ids are now in Messages. - var pending = t.PendingUserMessages; - foreach (var (uid, _) in pendingForRound) - pending = pending.Remove(uid); - - return node with + hub.Observe((IMessageDelivery)createDelivery) + // The delivery observable can emit more than once for the same request + // (Forwarded intermediate delivery + actual CreateNodeResponse, or stream + // re-replay on resubscribe). Take exactly the first terminal response — + // without this guard the commit step below ran 6× per Resubmit, each + // appending the same responseMsgId to Thread.Messages. + .Where(r => r.Message is CreateNodeResponse) + .Take(1) + .Subscribe( + response => { - Content = t with + if (response.Message is not CreateNodeResponse { Success: true }) { - Messages = msgs, - IngestedMessageIds = ingested, - IsExecuting = true, - ActiveMessageId = responseMsgId, - ExecutionStartedAt = DateTime.UtcNow, - TokensUsed = 0, - ExecutionStatus = null, - PendingUserMessage = null, - PendingUserMessages = pending, - PendingContextPath = dispatch.ContextPath, - PendingAttachments = dispatch.Attachments?.ToImmutableList() + var err = (response.Message as CreateNodeResponse)?.Error ?? "unknown"; + // 🚨 "Already exists" is EXPECTED, not a failure. The response + // cell id is deterministic per claim, and the claim Status + // oscillates (rollback→re-claim, Executing→StartingExecution + // resume bounce), so the _Exec round watcher fires DispatchRound + // several times for ONE round. The first creates the cell; the + // siblings hit "Node already exists at path". Rolling back to Idle + // on that re-triggered the claim → re-dispatch → already-exists + // loop and WEDGED the round — the Resubmit_*_DoesNotDeadlock hangs. + // Instead proceed to CommitRoundAndExecute (single-fire guarded): + // the cell is present, so the round commits exactly once and the + // oscillation terminates. + if (err.Contains("already exists", StringComparison.OrdinalIgnoreCase)) + { + CommitRoundAndExecute(); + return; + } + logger?.LogWarning("[ThreadSubmission] Response cell creation failed for {ResponseMsgId} on {ThreadPath}: {Error}", + responseMsgId, threadPath, err); + onFailure?.Invoke(); + return; } - }; - }); - hub.Post( - new UpdateThreadMessageContent { Text = "Allocating agent..." }, - o => o.WithTarget(new Address(responsePath))); - - // Step 3: post to _Exec hosted hub — actual agent streaming runs there. - var executionHub = hub.GetHostedHub( - new Address($"{hub.Address}/_Exec"), - config => config.WithHandler(ThreadExecution.ExecuteMessageAsync), - HostedHubCreation.Always); - - executionHub!.Post( - new SubmitMessageRequest - { - ThreadPath = threadPath, - UserMessageText = combinedUserText, - UserMessageId = dispatch.UserMessageIds.LastOrDefault(), - ResponseMessageId = responseMsgId, - ResponsePath = responsePath, - AgentName = dispatch.AgentName, - ModelName = dispatch.ModelName, - ContextPath = dispatch.ContextPath, - Attachments = dispatch.Attachments + // Both the user satellite cells (created above in the materialization + // step) and the response satellite cell (just confirmed) exist on the + // hub now. Only NOW do we add their ids into Messages — the GUI iterates + // Messages to render LayoutAreaControls, so every id it sees has a + // backing satellite. + CommitRoundAndExecute(); }, - o => userCtx != null ? o.WithAccessContext(userCtx) : o); - - return response; - }); + ex => + { + logger?.LogWarning(ex, "[ThreadSubmission] Response cell creation failed for {ResponseMsgId} on {ThreadPath}", responseMsgId, threadPath); + onFailure?.Invoke(); + }); } if (pendingForRound.Count == 0) @@ -708,6 +1096,14 @@ void AfterUserCellsReady() // Materialize satellite cells in parallel, then proceed. We swallow per-cell errors // (cell may already exist from a prior crashed attempt — that's recoverable) and only // wait for one notification per cell before continuing. + // + // Each CreateNodeRequest is posted via hub.Observe with explicit + // o.WithAccessContext(userCtx) so the cell is created under the user's identity + // (resolved from thread.CreatedBy / MeshNode.CreatedBy by DispatchRound). The + // AsyncLocal at this watcher-callback boundary may still be the thread hub's + // own address (Throttle scheduler hop), so meshService.CreateNode's + // CaptureContext() would otherwise stamp deliveries with hub-as-user — leading + // to "Node created at .../ by " instead of "by ". var creationStreams = pendingForRound.Select(p => { var cell = new MeshNode(p.Id, threadPath) @@ -716,7 +1112,10 @@ void AfterUserCellsReady() MainNode = mainEntity, Content = p.Msg }; - return meshService.CreateNode(cell) + return hub.Observe(new CreateNodeRequest(cell), + o => userCtx != null + ? o.WithAccessContext(userCtx).WithTarget(hub.Address) + : o.WithTarget(hub.Address)) .Take(1) .Select(_ => true) .Catch(ex => diff --git a/src/MeshWeaver.AI/ThreadViewModel.cs b/src/MeshWeaver.AI/ThreadViewModel.cs index 239106da7..3c2301b67 100644 --- a/src/MeshWeaver.AI/ThreadViewModel.cs +++ b/src/MeshWeaver.AI/ThreadViewModel.cs @@ -40,12 +40,27 @@ public record ThreadViewModel /// Streaming tool calls from the active response. public ImmutableList? StreamingToolCalls { get; init; } - /// Total tokens used in the current execution. - public int TokensUsed { get; init; } + /// + /// Texts of user messages currently in + /// — submitted via but not yet + /// drained into . Rendered inline at the end + /// of the chat history as "queued" cells so the user sees their submission + /// immediately, even before round dispatch materialises the satellite cell. + /// Disappears for an id once the inbox drain promotes it into Messages. + /// Order: submission order. + /// + public IReadOnlyList PendingMessageTexts { get; init; } = []; /// When the current execution started (for elapsed time display). public DateTime? ExecutionStartedAt { get; init; } + /// + /// Identity (ObjectId / email) of the user who created the thread, from + /// . The chat view shows the input + edit ops + /// only when this matches the current user — other users' threads are read-only. + /// + public string? CreatedBy { get; init; } + public virtual bool Equals(ThreadViewModel? other) { if (other is null) return false; @@ -58,10 +73,11 @@ public virtual bool Equals(ThreadViewModel? other) && IsExecuting == other.IsExecuting && ExecutionStatus == other.ExecutionStatus && StreamingText == other.StreamingText - && TokensUsed == other.TokensUsed && ExecutionStartedAt == other.ExecutionStartedAt + && CreatedBy == other.CreatedBy && Messages.SequenceEqual(other.Messages) - && (StreamingToolCalls ?? []).SequenceEqual(other.StreamingToolCalls ?? []); + && (StreamingToolCalls ?? []).SequenceEqual(other.StreamingToolCalls ?? []) + && PendingMessageTexts.SequenceEqual(other.PendingMessageTexts); } public override int GetHashCode() @@ -75,13 +91,15 @@ public override int GetHashCode() hash.Add(IsExecuting); hash.Add(ExecutionStatus); hash.Add(StreamingText); - hash.Add(TokensUsed); hash.Add(ExecutionStartedAt); + hash.Add(CreatedBy); foreach (var msg in Messages) hash.Add(msg); if (StreamingToolCalls != null) foreach (var tc in StreamingToolCalls) hash.Add(tc); + foreach (var txt in PendingMessageTexts) + hash.Add(txt); return hash.ToHashCode(); } } diff --git a/src/MeshWeaver.AI/Threading/IThreadManager.cs b/src/MeshWeaver.AI/Threading/IThreadManager.cs deleted file mode 100644 index d727e2282..000000000 --- a/src/MeshWeaver.AI/Threading/IThreadManager.cs +++ /dev/null @@ -1,89 +0,0 @@ -using Microsoft.Extensions.AI; - -namespace MeshWeaver.AI.Threading; - -/// -/// Interface for managing chat threads across all AI providers. -/// Provides a unified abstraction for thread lifecycle management. -/// -public interface IThreadManager -{ - /// - /// Gets an existing thread or creates a new one if it doesn't exist. - /// - /// The unique identifier for the thread - /// Optional scope identifier (e.g., mesh node address) - /// Cancellation token - /// The thread record - Task GetOrCreateThreadAsync(string threadId, string? scope = null, CancellationToken ct = default); - - /// - /// Adds a message to a thread. - /// - /// The thread to add the message to - /// The message to add - /// Cancellation token - Task AddMessageAsync(string threadId, ChatMessage message, CancellationToken ct = default); - - /// - /// Adds multiple messages to a thread. - /// - /// The thread to add messages to - /// The messages to add - /// Cancellation token - Task AddMessagesAsync(string threadId, IEnumerable messages, CancellationToken ct = default); - - /// - /// Gets all messages in a thread. - /// - /// The thread to get messages from - /// Cancellation token - /// List of messages in the thread - Task> GetMessagesAsync(string threadId, CancellationToken ct = default); - - /// - /// Clears all messages from a thread while keeping the thread itself. - /// - /// The thread to clear - /// Cancellation token - Task ClearThreadAsync(string threadId, CancellationToken ct = default); - - /// - /// Lists all threads in a given scope. - /// - /// The scope to list threads for (null for all threads) - /// Cancellation token - /// List of thread IDs - Task> ListThreadsAsync(string? scope = null, CancellationToken ct = default); - - /// - /// Deletes a thread and all its messages. - /// - /// The thread to delete - /// Cancellation token - Task DeleteThreadAsync(string threadId, CancellationToken ct = default); - - /// - /// Gets a thread by ID, or null if it doesn't exist. - /// - /// The thread ID to look up - /// Cancellation token - /// The thread if found, null otherwise - Task GetThreadAsync(string threadId, CancellationToken ct = default); - - /// - /// Updates the title of a thread. - /// - /// The thread to update - /// The new title - /// Cancellation token - Task UpdateTitleAsync(string threadId, string title, CancellationToken ct = default); - - /// - /// Gets the most recent thread in a scope. - /// - /// The scope to search in (null for all threads) - /// Cancellation token - /// The most recent thread, or null if none exist - Task GetMostRecentThreadAsync(string? scope = null, CancellationToken ct = default); -} diff --git a/src/MeshWeaver.AI/Threading/InMemoryThreadManager.cs b/src/MeshWeaver.AI/Threading/InMemoryThreadManager.cs deleted file mode 100644 index 74f7b240d..000000000 --- a/src/MeshWeaver.AI/Threading/InMemoryThreadManager.cs +++ /dev/null @@ -1,235 +0,0 @@ -using System.Collections.Concurrent; -using Microsoft.Extensions.AI; -using MeshWeaver.Messaging; - -namespace MeshWeaver.AI.Threading; - -/// -/// In-memory implementation of IThreadManager. -/// Stores threads and messages in memory with per-user isolation. -/// -public class InMemoryThreadManager : IThreadManager -{ - private readonly AccessService _accessService; - - // Per-user storage: userId -> (threadId -> thread) - private readonly ConcurrentDictionary> _userThreads = new(); - // Per-user message storage: userId -> (threadId -> messages) - private readonly ConcurrentDictionary>> _userMessages = new(); - - public InMemoryThreadManager(AccessService accessService) - { - _accessService = accessService; - } - - private string GetCurrentUserId() - { - var context = _accessService.Context ?? _accessService.CircuitContext; - return context?.ObjectId ?? "anonymous"; - } - - private ConcurrentDictionary GetUserThreads() - { - var userId = GetCurrentUserId(); - return _userThreads.GetOrAdd(userId, _ => new ConcurrentDictionary()); - } - - private ConcurrentDictionary> GetUserMessages() - { - var userId = GetCurrentUserId(); - return _userMessages.GetOrAdd(userId, _ => new ConcurrentDictionary>()); - } - - public Task GetOrCreateThreadAsync(string threadId, string? scope = null, CancellationToken ct = default) - { - var threads = GetUserThreads(); - var messages = GetUserMessages(); - - var thread = threads.GetOrAdd(threadId, _ => - { - messages.TryAdd(threadId, new List()); - return ChatThread.Create(threadId, scope); - }); - - return Task.FromResult(thread); - } - - public Task AddMessageAsync(string threadId, ChatMessage message, CancellationToken ct = default) - { - var threads = GetUserThreads(); - var messages = GetUserMessages(); - - // Ensure thread exists - threads.AddOrUpdate(threadId, - _ => - { - messages.TryAdd(threadId, new List()); - return ChatThread.Create(threadId); - }, - (_, existing) => existing.WithActivity()); - - // Add message - if (messages.TryGetValue(threadId, out var messageList)) - { - lock (messageList) - { - messageList.Add(message); - } - } - - // Auto-title from first user message - if (threads.TryGetValue(threadId, out var thread) && thread.Title == null) - { - if (message.Role == ChatRole.User && !string.IsNullOrWhiteSpace(message.Text)) - { - var title = message.Text.Length > 50 ? message.Text[..50] + "..." : message.Text; - threads[threadId] = thread.WithTitle(title); - } - } - - return Task.CompletedTask; - } - - public Task AddMessagesAsync(string threadId, IEnumerable messagesToAdd, CancellationToken ct = default) - { - var threads = GetUserThreads(); - var messages = GetUserMessages(); - - // Ensure thread exists - threads.AddOrUpdate(threadId, - _ => - { - messages.TryAdd(threadId, new List()); - return ChatThread.Create(threadId); - }, - (_, existing) => existing.WithActivity()); - - // Add messages - if (messages.TryGetValue(threadId, out var messageList)) - { - lock (messageList) - { - messageList.AddRange(messagesToAdd); - } - } - - // Auto-title from first user message - if (threads.TryGetValue(threadId, out var thread) && thread.Title == null) - { - var firstUserMessage = messagesToAdd.FirstOrDefault(m => - m.Role == ChatRole.User && !string.IsNullOrWhiteSpace(m.Text)); - if (firstUserMessage != null) - { - var title = firstUserMessage.Text.Length > 50 - ? firstUserMessage.Text[..50] + "..." - : firstUserMessage.Text; - threads[threadId] = thread.WithTitle(title); - } - } - - return Task.CompletedTask; - } - - public Task> GetMessagesAsync(string threadId, CancellationToken ct = default) - { - var messages = GetUserMessages(); - - if (messages.TryGetValue(threadId, out var messageList)) - { - lock (messageList) - { - return Task.FromResult>(messageList.ToList()); - } - } - - return Task.FromResult>(Array.Empty()); - } - - public Task ClearThreadAsync(string threadId, CancellationToken ct = default) - { - var threads = GetUserThreads(); - var messages = GetUserMessages(); - - if (messages.TryGetValue(threadId, out var messageList)) - { - lock (messageList) - { - messageList.Clear(); - } - } - - // Update thread activity - if (threads.TryGetValue(threadId, out var thread)) - { - threads[threadId] = thread.WithActivity(); - } - - return Task.CompletedTask; - } - - public Task> ListThreadsAsync(string? scope = null, CancellationToken ct = default) - { - var threads = GetUserThreads(); - - var result = threads.Values - .Where(t => scope == null || t.Scope == scope) - .OrderByDescending(t => t.LastActivityAt) - .ToList(); - - return Task.FromResult>(result); - } - - public Task DeleteThreadAsync(string threadId, CancellationToken ct = default) - { - var threads = GetUserThreads(); - var messages = GetUserMessages(); - - threads.TryRemove(threadId, out _); - messages.TryRemove(threadId, out _); - - return Task.CompletedTask; - } - - public Task GetThreadAsync(string threadId, CancellationToken ct = default) - { - var threads = GetUserThreads(); - - threads.TryGetValue(threadId, out var thread); - return Task.FromResult(thread); - } - - public Task UpdateTitleAsync(string threadId, string title, CancellationToken ct = default) - { - var threads = GetUserThreads(); - - if (threads.TryGetValue(threadId, out var thread)) - { - threads[threadId] = thread.WithTitle(title); - } - - return Task.CompletedTask; - } - - public Task GetMostRecentThreadAsync(string? scope = null, CancellationToken ct = default) - { - var threads = GetUserThreads(); - - var mostRecent = threads.Values - .Where(t => scope == null || t.Scope == scope) - .OrderByDescending(t => t.LastActivityAt) - .FirstOrDefault(); - - return Task.FromResult(mostRecent); - } - - /// - /// Clears all threads and messages for the current user. - /// - public void ClearAll() - { - var threads = GetUserThreads(); - var messages = GetUserMessages(); - threads.Clear(); - messages.Clear(); - } -} diff --git a/src/MeshWeaver.AI/Threading/MeshDataSourceThreadManager.cs b/src/MeshWeaver.AI/Threading/MeshDataSourceThreadManager.cs deleted file mode 100644 index e71e9db0a..000000000 --- a/src/MeshWeaver.AI/Threading/MeshDataSourceThreadManager.cs +++ /dev/null @@ -1,229 +0,0 @@ -using System.Text.Json; -using MeshWeaver.Mesh; -using MeshWeaver.Mesh.Services; -using MeshWeaver.Messaging; -using MeshWeaver.ShortGuid; -using Microsoft.Extensions.AI; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Logging; - -namespace MeshWeaver.AI.Threading; - -/// -/// Thread manager that persists chats as MeshNode hierarchies. -/// -/// Storage structure: -/// - Thread: MeshNode with nodeType="Thread" -/// - Messages: Child MeshNodes with nodeType="ThreadMessage" -/// -public class MeshDataSourceThreadManager : IThreadManager -{ - private readonly AccessService _accessService; - private readonly IMessageHub _hub; - private readonly ILogger? _logger; - private readonly IMeshService _nodeFactory; - private readonly IMeshService _meshQuery; - - internal MeshDataSourceThreadManager( - AccessService accessService, - IMessageHub hub, - ILogger? logger = null) - { - _accessService = accessService; - _hub = hub; - _logger = logger; - _nodeFactory = hub.ServiceProvider.GetRequiredService(); - _meshQuery = hub.ServiceProvider.GetRequiredService(); - } - - private string GetUserId() => _accessService.Context?.ObjectId ?? "anonymous"; - - public async Task GetOrCreateThreadAsync(string threadId, string? scope = null, CancellationToken ct = default) - { - var existing = await GetThreadAsync(threadId, ct); - if (existing != null) - return existing; - - var thread = ChatThread.Create(threadId, scope); - - var threadNode = new MeshNode(threadId) - { - NodeType = "Thread", - Name = thread.Title ?? threadId, - Content = new ChatThreadMetadata - { - Id = thread.Id, - Scope = thread.Scope, - Title = thread.Title, - CreatedAt = thread.CreatedAt, - LastActivityAt = thread.LastActivityAt, - ProviderId = thread.ProviderId - } - }; - await _nodeFactory.CreateNodeAsync(threadNode, ct); - - return thread; - } - - public async Task AddMessageAsync(string threadId, ChatMessage message, CancellationToken ct = default) - { - var messageId = Guid.NewGuid().AsString(); - var messagePath = $"{threadId}/{messageId}"; - - var messageType = message.Role == ChatRole.User - ? ThreadMessageType.ExecutedInput - : ThreadMessageType.AgentResponse; - - var threadMessage = new ThreadMessage - { - Role = message.Role.Value, - AuthorName = message.AuthorName, - Text = message.Text ?? string.Empty, - Timestamp = DateTime.UtcNow, - Type = messageType - }; - - var messageNode = new MeshNode(messagePath) - { - NodeType = ThreadMessageNodeType.NodeType, - MainNode = threadId, - Content = threadMessage - }; - - await _nodeFactory.CreateNodeAsync(messageNode, ct); - _logger?.LogDebug("Saved message {MessageId} as child node: {Path}", messageId, messagePath); - - // Auto-title from first user message - if (message.Role == ChatRole.User && !string.IsNullOrWhiteSpace(message.Text)) - { - var thread = await GetThreadAsync(threadId, ct); - if (thread?.Title == null) - { - var title = message.Text.Length > 50 ? message.Text[..50] + "..." : message.Text; - await UpdateTitleAsync(threadId, title, ct); - } - } - } - - public async Task AddMessagesAsync(string threadId, IEnumerable messages, CancellationToken ct = default) - { - foreach (var message in messages) - await AddMessageAsync(threadId, message, ct); - } - - public async Task> GetMessagesAsync(string threadId, CancellationToken ct = default) - { - try - { - var messageNodes = await _meshQuery.QueryAsync( - $"namespace:{threadId} nodeType:{ThreadMessageNodeType.NodeType}" - ).ToListAsync(ct); - - return messageNodes - .Select(n => n.Content as ThreadMessage) - .Where(m => m != null && m.Type != ThreadMessageType.EditingPrompt) - .OrderBy(m => m!.Timestamp) - .Select(m => new ChatMessage(new ChatRole(m!.Role), m.Text) - { - AuthorName = m.AuthorName - }) - .ToList(); - } - catch (Exception ex) - { - _logger?.LogDebug(ex, "Failed to load messages for thread: {Path}", threadId); - return []; - } - } - - public async Task ClearThreadAsync(string threadId, CancellationToken ct = default) - { - await _nodeFactory.DeleteNodeAsync(threadId, ct: ct); - _logger?.LogInformation("Cleared thread {ThreadId}", threadId); - } - - public async Task> ListThreadsAsync(string? scope = null, CancellationToken ct = default) - { - var queryString = "nodeType:Thread"; - if (!string.IsNullOrEmpty(scope)) - queryString += $" parent:{scope}"; - - var threadNodes = await _meshQuery.QueryAsync(queryString).ToListAsync(ct); - - return threadNodes - .Select(n => n.Content as ChatThreadMetadata) - .Where(m => m != null) - .Select(m => m!.ToThread()) - .OrderByDescending(t => t.LastActivityAt) - .ToList(); - } - - public async Task DeleteThreadAsync(string threadId, CancellationToken ct = default) - { - await _nodeFactory.DeleteNodeAsync(threadId, ct: ct); - _logger?.LogInformation("Deleted thread {ThreadId}", threadId); - } - - public async Task GetThreadAsync(string threadId, CancellationToken ct = default) - { - try - { - var node = await _meshQuery.QueryAsync($"path:{threadId}") - .FirstOrDefaultAsync(ct); - - if (node?.Content is ChatThreadMetadata metadata) - return metadata.ToThread(); - - return null; - } - catch - { - return null; - } - } - - public async Task UpdateTitleAsync(string threadId, string title, CancellationToken ct = default) - { - var node = await _meshQuery.QueryAsync($"path:{threadId}") - .FirstOrDefaultAsync(ct); - - if (node != null) - { - var updated = node with { Name = title }; - if (updated.Content is ChatThreadMetadata meta) - updated = updated with { Content = meta with { Title = title } }; - _hub.Post(new UpdateNodeRequest(updated)); - } - } - - public async Task GetMostRecentThreadAsync(string? scope = null, CancellationToken ct = default) - { - var threads = await ListThreadsAsync(scope, ct); - return threads.FirstOrDefault(); - } -} - -/// -/// Persisted chat thread metadata. -/// -public record ChatThreadMetadata -{ - public required string Id { get; init; } - public string? Scope { get; init; } - public string? Title { get; init; } - public DateTime CreatedAt { get; init; } - public DateTime LastActivityAt { get; init; } - public string? ProviderId { get; init; } - - public ChatThread ToThread() => new(Id, Scope, Title, CreatedAt, LastActivityAt, ProviderId); - - public static ChatThreadMetadata FromThread(ChatThread thread) => new() - { - Id = thread.Id, - Scope = thread.Scope, - Title = thread.Title, - CreatedAt = thread.CreatedAt, - LastActivityAt = thread.LastActivityAt, - ProviderId = thread.ProviderId - }; -} diff --git a/src/MeshWeaver.AI/TokenUsage.cs b/src/MeshWeaver.AI/TokenUsage.cs new file mode 100644 index 000000000..e049ec942 --- /dev/null +++ b/src/MeshWeaver.AI/TokenUsage.cs @@ -0,0 +1,177 @@ +using System.Reactive.Linq; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Graph.Security; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.AI; + +/// +/// Per-(thread, model) token usage — the record of how many input/output tokens ONE model +/// consumed in ONE thread. Stored as a SATELLITE MeshNode at {threadPath}/_Usage/{modelKey} +/// (keyed by model) and accumulated across the thread's rounds. +/// +/// This is the SINGLE SOURCE OF TRUTH for token/cost reporting: the +/// node itself carries NO token state — all cost tracking lives here, outside the thread. Cost is +/// NOT stored; it is derived on read from the configured model prices (), +/// so a price change re-prices historical usage. + are +/// denormalized onto the content so usage is queryable nodeType:TokenUsage across the mesh — +/// by thread (the satellite's parent) AND by model, and rolled up per user / per space. +/// +public record TokenUsage +{ + /// ObjectId of the user who owns the thread (per-user usage roll-up). Null if unknown. + public string? UserId { get; init; } + + /// Path of the thread this usage belongs to (equals the satellite node's MainNode). + public string? ThreadId { get; init; } + + /// The bare model id (e.g. claude-opus-4-8) — the satellite's key dimension. + public string Model { get; init; } = string.Empty; + + /// Cumulative input (prompt) tokens for this model in this thread. + public long InputTokens { get; init; } + + /// Cumulative output (completion) tokens for this model in this thread. + public long OutputTokens { get; init; } + + /// Returns a copy with the given round's counts added. + public TokenUsage Add(long inputTokens, long outputTokens) + => this with { InputTokens = InputTokens + inputTokens, OutputTokens = OutputTokens + outputTokens }; +} + +/// +/// The satellite NodeType. Like Activity / Comment, it is a +/// system-generated satellite — excluded from search and create contexts, with access delegated to +/// the MainNode (the thread) via (Read needs Read on the thread; +/// Create/Update need Update on the thread). +/// +public static class TokenUsageNodeType +{ + public const string NodeType = "TokenUsage"; + + /// The satellite sub-namespace under a thread — usage lives at {threadPath}/_Usage/{modelKey}. + public const string SatelliteSegment = "_Usage"; + + public static TBuilder AddTokenUsageType(this TBuilder builder) where TBuilder : MeshBuilder + { + builder.AddMeshNodes(CreateMeshNode()); + builder.AddAutocompleteExcludedTypes(NodeType); + builder.ConfigureServices(services => + { + services.AddSingleton(sp => + new SatelliteAccessRule(NodeType, sp.GetRequiredService())); + return services; + }); + return builder; + } + + public static MeshNode CreateMeshNode() => new(NodeType) + { + Name = "Token Usage", + IsSatelliteType = true, + ExcludeFromContext = new HashSet { "search", "create" }, + HubConfiguration = config => config + .AddMeshDataSource(source => source + .WithContentType()) + }; + + /// + /// Records ONE round's token usage onto the per-model satellite at + /// {threadPath}/_Usage/{modelKey}, ACCUMULATING input/output across the thread's rounds + /// (keyed by model). A no-token round is a no-op. Returns an that + /// completes when the satellite is persisted (fail-open: it never errors). The caller subscribes + /// it as an INDEPENDENT side effect — it MUST NOT be chained before the round's terminal status + /// write (that delayed the terminal write and gated round-completion on a slow satellite write). + /// The satellite is a SEPARATE node; the GUI chip and the token tests WAIT for it (a + /// Where(...).Timeout read), so it can land shortly AFTER the terminal status. + /// + /// Two NON-poisoning phases (rounds run serially per thread, so this read-modify-write is + /// race-free): (1) create-only EnsureExists via (a mesh-targeted + /// CreateNodeRequest — never a point GetMeshNodeStream read of an absent node, which would trip the + /// MeshNodeStreamCache storm breaker); then (2) accumulate via the OWNER's authoritative + /// GetMeshNodeStream(path).Update on the now-existing node, which reads the live current + /// value and adds this round's tokens (exact across rounds, unlike a lagged CQRS query read). + /// + public static IObservable RecordUsage( + IMessageHub hub, string threadPath, string? userId, + string? modelId, int? inputTokens, int? outputTokens, ILogger? logger = null) + { + long inTok = inputTokens ?? 0; + long outTok = outputTokens ?? 0; + if (inTok == 0 && outTok == 0) + return Observable.Return(System.Reactive.Unit.Default); // no-token round → no-op + + var model = string.IsNullOrWhiteSpace(modelId) ? "(unknown)" : modelId!; + var key = new string(model.Select(c => char.IsLetterOrDigit(c) ? c : '_').ToArray()); + var ns = $"{threadPath}/{SatelliteSegment}"; + var usagePath = $"{ns}/{key}"; + + var meshService = hub.ServiceProvider.GetService(); + if (meshService is null) + return Observable.Return(System.Reactive.Unit.Default); + + // 🚨 Two NON-poisoning phases. The OLD code read the (first-round-ABSENT) satellite via a point + // GetMeshNodeStream(usagePath) and created it with an UNTARGETED CreateOrUpdateNodeRequest — both + // bugs (since 616b4e27f): + // • the point-read of an absent node opens a SubscribeRequest to a non-existent owner → NotFound + // → trips the MeshNodeStreamCache STORM BREAKER (2s+ backoff), which then fast-fails EVERY + // reader of usagePath (the GUI ThreadTokenChip AND the token tests' WaitForUsage) for the + // whole window — "No node found at …/_Usage/…". (MeshNodeStreamCache.cs storm breaker / + // project_aisettings_create_storm_fix / feedback_optional_node_query_not_access.) + // • the untargeted CreateOrUpdateNodeRequest never reaches HandleCreateOrUpdateNodeRequest (it + // lives on the MESH hub — IMeshService.CreateNode targets hub.GetMeshHub().Address), so from + // this per-node thread hub the satellite was never created at all. + // Phase 1: EnsureExists via meshService.CreateNode of a ZERO-token satellite — CREATE-ONLY, so an + // existing satellite (round 2+) is left untouched (CreateNode throws NodeAlreadyExists → caught + // → continue). meshService.CreateNode posts a CreateNodeRequest TARGETED at the mesh hub and is + // NOT a point-read, so it neither mis-routes nor poisons. It guarantees the node + its owning + // per-node hub exist before the accumulate. + // Phase 2: accumulate via the OWNER's authoritative stream.Update — the node now exists, so the + // read-modify-write reads the LIVE current value and adds this round's tokens. Race-free (rounds + // are serial per thread) and EXACT across rounds (the cumulative invariant), unlike a lagged + // CQRS query read which could miss a prior round's write. + var freshNode = new MeshNode(key, ns) + { + Name = model, + NodeType = NodeType, + State = MeshNodeState.Active, + MainNode = threadPath, + Content = new TokenUsage { UserId = userId, ThreadId = threadPath, Model = model }, + }; + + return meshService.CreateNode(freshNode) + .Select(_ => true) + // Already exists (every round after the first for this model) → keep going to the accumulate. + // A DIFFERENT failure (e.g. RLS) must NOT fall through to Phase 2: .Update on a node that was + // never created would re-open the absent-node point-access this fix exists to avoid. Rethrow + // so the terminal Catch fails the usage write open without touching the stream. + .Catch((Exception ex) => + ex is InvalidOperationException + && ex.Message.Contains("already exists", StringComparison.OrdinalIgnoreCase) + ? Observable.Return(false) + : Observable.Throw(ex)) + .SelectMany(_ => hub.GetWorkspace().GetMeshNodeStream(usagePath) + .Update(node => + { + var cur = node.ContentAs(hub.JsonSerializerOptions, logger) + ?? new TokenUsage { UserId = userId, ThreadId = threadPath, Model = model }; + return node with { Content = cur.Add(inTok, outTok) }; + })) + .Select(_ => System.Reactive.Unit.Default) + // Subscribed as an INDEPENDENT side effect (NOT chained before the terminal status write), + // so it can never block the round. Still cap + fail open as basic hygiene: a wedged create + // or accumulate resolves to a no-op rather than leaking a live subscription. + .Timeout(TimeSpan.FromSeconds(15), Observable.Return(System.Reactive.Unit.Default)) + .Catch((Exception ex) => + { + logger?.LogWarning(ex, "[TokenUsage] RecordUsage failed for {Path}", usagePath); + return Observable.Return(System.Reactive.Unit.Default); + }); + } +} diff --git a/src/MeshWeaver.AI/ToolStatusFormatter.cs b/src/MeshWeaver.AI/ToolStatusFormatter.cs index 1ac3f6086..c15a46bf3 100644 --- a/src/MeshWeaver.AI/ToolStatusFormatter.cs +++ b/src/MeshWeaver.AI/ToolStatusFormatter.cs @@ -46,9 +46,27 @@ private static string FormatDelegation(IDictionary? args) // Strip "Agent/" prefix for cleaner display if (agent != null && agent.Contains('/')) agent = agent.Split('/').Last(); + + var task = GetArg(args, "task"); + var taskSummary = SummarizeTask(task); + + if (!string.IsNullOrEmpty(taskSummary)) + return $"{taskSummary} ({agent ?? "Agent"})"; + return $"Delegating to {agent ?? "Agent"}..."; } + private static string SummarizeTask(string? task) + { + if (string.IsNullOrWhiteSpace(task)) + return string.Empty; + var firstLine = task.Split('\n', 2)[0].Trim(); + const int maxLen = 40; + if (firstLine.Length > maxLen) + firstLine = firstLine[..(maxLen - 1)] + "…"; + return firstLine; + } + private static string FormatArg(string template, IDictionary? args, string key) { var value = GetArg(args, key); @@ -77,4 +95,20 @@ private static string Truncate(string value) return value; return value[..(MaxArgLength - 3)] + "..."; } + + /// + /// Returns the last lines of . + /// Used as the live progress preview on a delegation + /// : the parent's watcher writes this + /// projection on every sub-thread emission so the GUI shows a bounded, + /// most-recent view of sub-agent output without unbounded growth. + /// + public static string LastNLines(string? text, int n) + { + if (string.IsNullOrEmpty(text)) return string.Empty; + if (n <= 0) return string.Empty; + var lines = text.Split('\n'); + if (lines.Length <= n) return text; + return string.Join('\n', lines[(lines.Length - n)..]); + } } diff --git a/src/MeshWeaver.AI/UpdateThreadMessageContent.cs b/src/MeshWeaver.AI/UpdateThreadMessageContent.cs deleted file mode 100644 index f038f9536..000000000 --- a/src/MeshWeaver.AI/UpdateThreadMessageContent.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System.Collections.Immutable; -using MeshWeaver.Layout; - -namespace MeshWeaver.AI; - -/// -/// Message posted to the response message hub to update content during streaming. -/// Handled locally on the grain — updates workspace → sync stream → clients. -/// -public record UpdateThreadMessageContent -{ - /// - /// Incremental text chunk to APPEND to the current message Text. The preferred shape - /// for streaming — each chunk just carries the new bytes since the previous update. - /// - public string? TextDelta { get; init; } - - /// - /// Full text replacement. Only set for final-state writes (completion text, error text, - /// cancel markers). Streaming should use instead. - /// - public string? Text { get; init; } - - public ImmutableList? ToolCalls { get; init; } - public ImmutableList? UpdatedNodes { get; init; } - public string? AgentName { get; init; } - public string? ModelName { get; init; } - - /// Token usage from the model provider. Set on the final update of a round. - public int? InputTokens { get; init; } - public int? OutputTokens { get; init; } - public int? TotalTokens { get; init; } - - /// Wall-clock completion timestamp. Set on the final update of a round. - public DateTime? CompletedAt { get; init; } -} diff --git a/src/MeshWeaver.Blazor.AI/McpCompletionProvider.cs b/src/MeshWeaver.Blazor.AI/McpCompletionProvider.cs index e1c51c948..9a6a595e6 100644 --- a/src/MeshWeaver.Blazor.AI/McpCompletionProvider.cs +++ b/src/MeshWeaver.Blazor.AI/McpCompletionProvider.cs @@ -1,3 +1,5 @@ +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; using MeshWeaver.Mesh.Services; using Microsoft.Extensions.Logging; @@ -25,32 +27,27 @@ public McpCompletionProvider(IMeshService meshQuery, ILoggerMaximum number of suggestions to return /// Cancellation token /// List of path suggestions - public async Task> GetSuggestionsAsync( + public Task> GetSuggestionsAsync( string prefix, int limit = 100, CancellationToken ct = default) { logger.LogDebug("Getting autocomplete suggestions for prefix={Prefix}", prefix); - var suggestions = new List(); - - try - { - await foreach (var item in meshQuery.AutocompleteAsync( - basePath: "", - prefix: prefix, - mode: AutocompleteMode.RelevanceFirst, - limit: limit, - ct: ct)) + // Compose the Autocomplete observable — never await-foreach a hub query (deadlock). + // This is the MCP/SDK boundary, the one sanctioned single-point Task bridge + // (.FirstAsync().ToTask(); see AsynchronousCalls.md). The first snapshot of + // suggestions is projected to paths. + return meshQuery + .Autocomplete(basePath: "", prefix: prefix, mode: AutocompleteMode.RelevanceFirst, limit: limit) + .TakeLast(1) + .Select(rows => (IReadOnlyList)rows.Select(r => r.Path).ToList()) + .Catch((Exception ex) => { - suggestions.Add(item.Path); - } - } - catch (Exception ex) - { - logger.LogWarning(ex, "Error getting autocomplete suggestions for prefix={Prefix}", prefix); - } - - return suggestions; + logger.LogWarning(ex, "Error getting autocomplete suggestions for prefix={Prefix}", prefix); + return Observable.Return>([]); + }) + .FirstAsync() + .ToTask(ct); } } diff --git a/src/MeshWeaver.Blazor.AI/McpExtensions.cs b/src/MeshWeaver.Blazor.AI/McpExtensions.cs index 52492bcaa..c00e72e16 100644 --- a/src/MeshWeaver.Blazor.AI/McpExtensions.cs +++ b/src/MeshWeaver.Blazor.AI/McpExtensions.cs @@ -26,12 +26,38 @@ public static TBuilder AddMcp(this TBuilder builder) where TBuilder : /// /// Adds MCP server services to the service collection. /// Registers tools from McpMeshPlugin and resources from McpResources. + /// Binds to the Mcp configuration + /// section (Aspire AppHost wires the portal's own external endpoint + /// into Mcp__BaseUrl at deployment time, no per-environment + /// source patches needed). /// + /// + /// Connect-time guidance the MCP client (Claude Code / Copilot) receives in the instructions + /// field of the initialize response — so NOTHING is synced to disk: the mesh is the workspace, search + /// is vector-indexed, and skills are mesh nodes found + read on demand. The full tool + query reference + /// is the tools-reference resource (the same embedded ToolsReference the in-portal agents use). + /// + public const string ServerInstructions = + "You are connected to the MeshWeaver mesh through this MCP server — the mesh IS your workspace " + + "(not a local file tree). Use these tools to read and modify content.\n\n" + + "Everything in the mesh is vector-indexed: retrieve anything with `search` (free-text routes to the " + + "semantic index) — you do not need exact paths.\n\n" + + "Skills are reusable capabilities stored as `nodeType:Skill` nodes. When a request matches a specific " + + "operation, find the relevant skill with `search nodeType:Skill`, then read it with `get` to follow " + + "its instructions. Read each skill only once.\n\n" + + "Read the `tools-reference` resource for the full tool + query-syntax reference."; + public static IServiceCollection AddMeshMcp(this IServiceCollection services) { - services.AddMcpServer() + services.AddMcpServer(options => options.ServerInstructions = ServerInstructions) .WithHttpTransport() - .WithToolsFromAssembly(typeof(McpMeshPlugin).Assembly); + .WithToolsFromAssembly(typeof(McpMeshPlugin).Assembly) + .WithResourcesFromAssembly(typeof(McpResources).Assembly); + + // BindConfiguration resolves IConfiguration from DI at options + // construction — works wherever the standard ASP.NET host is running, + // no caller-side IConfiguration parameter needed. + services.AddOptions().BindConfiguration("Mcp"); return services; } diff --git a/src/MeshWeaver.Blazor.AI/McpMeshPlugin.cs b/src/MeshWeaver.Blazor.AI/McpMeshPlugin.cs index 5fcd3d332..f0891df7e 100644 --- a/src/MeshWeaver.Blazor.AI/McpMeshPlugin.cs +++ b/src/MeshWeaver.Blazor.AI/McpMeshPlugin.cs @@ -1,86 +1,660 @@ using System.ComponentModel; +using System.Net; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using System.Text.Json; using MeshWeaver.AI; +using MeshWeaver.AI.Plugins; +using MeshWeaver.Data.Completion; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Mesh.Services.LanguageServer; using MeshWeaver.Messaging; +using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using ModelContextProtocol.Protocol; using ModelContextProtocol.Server; namespace MeshWeaver.Blazor.AI; /// /// MCP wrapper exposing mesh operations as MCP tools. -/// Thin wrapper over MeshOperations with MCP attributes and URL-based NavigateTo. +/// Thin wrapper over with MCP attributes and +/// URL-based NavigateTo. +/// +/// +/// Session hub: on construction, the plugin materialises a session-scoped +/// hub at portal/mcp-{callerId}-{mcpSessionId} — exactly mirroring the +/// Blazor PortalApplication pattern. Portal-typed addresses are skipped +/// from Orleans grain resolution (RoutingGrain) and the sub-hub is +/// registered with the routing service so responses (e.g. kernel +/// SubmitCodeRequest ack) route back correctly. Inlines the same +/// RouteAddressToHostedHub("kernel", ...) rule so in-session kernel +/// execution stays local. +/// +/// +/// Each authenticated caller × MCP session id gets its own hub; idle +/// hubs dispose when the MCP connection ends. /// [McpServerToolType] public class McpMeshPlugin { private readonly MeshOperations ops; + private readonly IMessageHub rootHub; + private readonly IMessageHub sessionHub; private readonly ILogger logger; private readonly string baseUrl; public McpMeshPlugin( IMessageHub hub, - IOptions? config = null) + IOptions? config = null, + IHttpContextAccessor? httpContextAccessor = null) { - ops = new MeshOperations(hub); logger = hub.ServiceProvider.GetRequiredService>(); - baseUrl = config?.Value.BaseUrl ?? "http://localhost:5000"; + // Resolve the UI base URL in priority order. No hard-coded URLs: + // 1. Configured McpConfiguration.BaseUrl — Aspire AppHost passes the + // portal's external HTTPS endpoint via env var (Mcp__BaseUrl) so + // the deployment topology owns this; no per-environment patching + // of source. Same mechanism in prod / test / local. + // 2. Current HTTP request's scheme + host — the live answer when MCP + // is invoked over the same portal that serves the UI; correct for + // any port Aspire allocates dynamically without any config. + // 3. Empty (signals "no base URL resolvable") — surfaces in the URL + // string returned by GetBaseUrl / NavigateTo so the caller sees + // a clearly-broken value instead of a quietly-wrong localhost + // one. Better to fail loud than to ship the wrong URL. + var requestUrl = httpContextAccessor?.HttpContext?.Request is { } req + ? $"{req.Scheme}://{req.Host.Value}".TrimEnd('/') + : null; + baseUrl = config?.Value.BaseUrl + ?? requestUrl + ?? string.Empty; + rootHub = hub; + + // The session hub IS the MCP-side actor: registered with the routing + // service so responses route back via the standard portal/* path. + // No per-call kernel hub — execute_script flows through the Code hub, + // which creates an Activity MeshNode whose hub hosts the kernel + // (see ActivityNodeType.HubConfiguration + AddKernelSubHubHandlers). + // Replies route through the standard MeshNode chain to portal/mcp-… + // — same routing every other MCP tool already uses (Get, Search, …). + // SessionHubResolver is shared with the REST endpoint module so both + // transports get identical routing semantics. + sessionHub = SessionHubResolver.ResolveSessionHub(hub, httpContextAccessor?.HttpContext, "mcp", logger); + + ops = new MeshOperations(sessionHub); } - [McpServerTool] - [Description("Retrieves a node from the mesh by path. Supports @ prefix shorthand, /* for children, and Unified Path prefixes (path/schema:, path/model:).")] + [McpServerTool(Title = "Get a node or attached resource", ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description(@"Retrieves a node or a resource attached to a node by path. Returns JSON for nodes/data/schemas, or raw file bytes (JSON-escaped) for content-collection files. + +Path shapes: + • `@Node/Path` — the MeshNode itself (metadata + Content) + • `@Node/Path/*` — immediate children of the node + • `@Node/Path/data/` — node Content as structured JSON (whole model) + • `@Node/Path/data/Type/id` — one entity from the node's data collection + • `@Node/Path/schema/` — JSON Schema of the node's Content type + • `@Node/Path/schema/Type` — schema for a specific type + • `@Node/Path/model/` — full data model with all registered types + • `@Node/Path/layoutAreas/` — list of layout areas on the node + • `@Node/Path/area/Name` — that layout area's rendered payload + • `@Node/Path/content/file.ext` — file from the 'content' collection + • `@Node/Path/content/subfolder/file.ext` — file from a nested path + • `@Node/Path/{collection}/file.ext` — file from a NAMED collection (e.g. 'Files/', 'assets/') + • `@Node/Path/collection/` — list of collection configs on the node + • `@Node/Path/collection/name1,name2` — specific collection configs +Legacy colon form `path/prefix:value` still works for backward compatibility.")] public Task Get( - [Description("Path to data (e.g., @graph/org1, @Agent/*, @Cornerstone/schema:, @Cornerstone/schema:TypeName, @Cornerstone/model:)")] string path) - => ops.Get(path); + [Description(@"Path to data. Examples: + @graph/org1 (node) + @Agent/* (children) + @Systemorph/FutuRe/EuropeRe/content/LargeClaims.xlsx (file from 'content' collection) + @Doc/Architecture/content/icon.svg (file) + @Cornerstone/schema/TypeName (schema) + @Cornerstone/model/ (full model)")] string path) + => ops.Get(path).FirstAsync().ToTask(); + + [McpServerTool(Title = "Upload a file into a content collection", Destructive = false, Idempotent = true, OpenWorld = false)] + [Description(@"Uploads raw file bytes into a node's content collection — the write-side mirror of `Get` for content-collection files. Use this to attach images, documents, or any binary asset to a node (e.g. an organisation logo or an Excel input file for a script). + +Path shapes (must include a collection segment + filename): + • `@Node/Path/content/file.ext` — write into the default 'content' collection + • `@Node/Path/content/subfolder/file.ext` — nested path within the collection + • `@Node/Path/{collection}/file.ext` — write into a named collection (e.g. 'Files/', 'assets/') + +The target collection must exist on the node and be editable (`IsEditable=true`). Returns JSON like +`{""status"":""Uploaded"",""path"":""Systemorph/content/logo.png"",""bytes"":4958}` on success, or an `Error: …` string otherwise.")] + public Task Upload( + [Description(@"Target path including collection + filename, e.g. '@Systemorph/content/logo.png' or '@Doc/Architecture/content/diagrams/flow.svg'. The path is parsed as {nodePath}/{collection}/{filePath}.")] string path, + [Description("File content as base64-encoded bytes (no data:URI prefix; just the raw base64 payload).")] string base64Content) + { + if (string.IsNullOrEmpty(base64Content)) + return Task.FromResult("Error: base64Content is required."); + byte[] bytes; + try { bytes = Convert.FromBase64String(base64Content); } + catch (FormatException ex) { return Task.FromResult($"Error: invalid base64 content: {ex.Message}"); } + return ops.Upload(path, bytes).FirstAsync().ToTask(); + } + + [McpServerTool(Title = "Search the mesh", ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description(@"Searches the mesh using GitHub-style query syntax. Returns {count, limit, truncated, results:[{path,name,nodeType}]} — when 'truncated' is true there are more matches than returned; narrow the query or raise 'limit'. - [McpServerTool] - [Description("Searches the mesh using GitHub-style query syntax. Returns up to 50 matching nodes.")] +Query terms (space-separated, all case-insensitive): + • field filters: nodeType:Agent, name:Acme, name:*sales* (wildcards), -status:Archived (negation), price:>100 + • location: namespace:Doc (immediate children), namespace:Doc scope:descendants (recursive), path:Doc/Architecture (exact) + • scope values: descendants | ancestors | hierarchy | subtree | ancestorsandself + • sorting/projection: sort:name, sort:lastModified-desc, select:name,nodeType,icon + • free text terms ('laptop pricing') run semantic/vector search when available +Full reference: read the 'tools-reference' MCP resource.")] public Task Search( [Description("Query string (e.g., 'nodeType:Agent', 'laptop', 'path:ACME scope:descendants', 'name:*sales*')")] string query, - [Description("Base path to search from (e.g., @graph). Empty for all.")] string? basePath = null) - => ops.Search(query, basePath); + [Description("Base path to search from (e.g., @graph). Empty for all.")] string? basePath = null, + [Description("Maximum number of results to return. Default 50, max 200.")] int limit = 50) + => ops.Search(query, basePath, limit).FirstAsync().ToTask(); + + [McpServerTool(Title = "Autocomplete an @-reference", ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description(@"Autocomplete a partial @-reference using the REAL in-portal autocomplete engine — DISTINCT from `search`. Where `search` runs one GitHub-style query, autocomplete blends multiple ranked sources (current-node providers, partition list + drill-down, and a global fan-out), deduped and priority-sorted — the same suggestions the chat input shows as you type `@`. Use it to resolve a name/path fragment the user typed into a concrete `insertText`/path before Get / NavigateTo. Returns {count, results:[{label, insertText, kind, category, description}]}. + +The leading `@` is required (this is @-reference autocomplete, not free-text search): + - '@' -> top suggestions for the current context + - '@/' -> the partition list + - '@/ACME/art' -> drill-down under a partition + - '@MyFile' -> current-node + global matches for 'MyFile'")] + public Task Autocomplete( + [Description("The partial @-reference to complete (leading @ required), e.g. '@', '@/', '@MyFile', '@/ACME/art'.")] string query, + [Description("The current context namespace used to rank nearby results (e.g. 'ACME/Project'). Empty for global only.")] string? context = null) + { + var orchestrator = rootHub.ServiceProvider.GetService(); + if (orchestrator is null) + return Task.FromResult("{\"count\":0,\"results\":[],\"note\":\"autocomplete engine unavailable\"}"); - [McpServerTool] - [Description("Creates a new node in the mesh. Pass a JSON MeshNode object with id, namespace, name, nodeType, and content fields.")] + var q = string.IsNullOrWhiteSpace(query) ? "@" : (query.StartsWith('@') ? query : "@" + query); + + // The orchestrator emits ranked batches progressively and completes when all producers finish. + // Bound it with a timer (autocomplete is interactive — take whatever arrived), sort, project to JSON. + return orchestrator.GetCompletions(q, context) + .TakeUntil(Observable.Timer(TimeSpan.FromSeconds(8))) + .ToList() + .Select(batches => + { + var results = batches + .OrderByDescending(b => b.CategoryPriority) + .SelectMany(b => b.Items.Select(i => new + { + label = i.Label, + insertText = i.InsertText, + kind = i.Kind.ToString(), + category = b.Category, + description = i.Description, + })) + .Take(50) + .ToList(); + return JsonSerializer.Serialize(new { count = results.Count, results }); + }) + .Catch(ex => Observable.Return( + JsonSerializer.Serialize(new { count = 0, results = Array.Empty(), error = ex.Message }))) + .FirstAsync() + .ToTask(); + } + + [McpServerTool(Title = "Search content chunks", ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description(@"Semantic search over INDEXED content chunks — the chunk-level companion to `Search`. Where node `Search` resolves a content hit up to its Document node and drops the chunk position, this returns the matching chunks WITH their (collectionPath, filePath, chunkIndex) so you can read the exact window or step through neighbours with `get_chunk`. Use it to FIND relevant passages and gather context; for whole-document reads (e.g. table extraction) use `Get` on the Document. + +Chunking model: each indexed file is split into 1000-char windows with 150-char overlap, numbered by a 0-based chunk_index per file. + +Returns `{count, results:[{documentPath, collectionPath, filePath, chunkIndex, rank, snippet}]}` — `rank` is the 0-based best-first relevance order. When content indexing isn't enabled in this host, returns a `{count:0, message:…}` envelope rather than erroring.")] + public Task SearchChunks( + [Description("Free-text query, matched semantically against indexed chunk text.")] string query, + [Description("Node path to anchor the search at — this path AND each ancestor prefix are searched (e.g. '@ACME/Reports'). Required: with no scope there is no collection to search and an empty result with a hint is returned.")] string? scope = null, + [Description("Maximum number of chunk hits to return (1-200, default 20). Not deduped by file.")] int limit = 20) + { + var scopePath = string.IsNullOrWhiteSpace(scope) ? null : MeshOperations.ResolvePath(scope); + return ChunkNavigation.SearchChunks(sessionHub.ServiceProvider, query, scopePath, limit) + .FirstAsync().ToTask(); + } + + [McpServerTool(Title = "Read a content chunk by index", ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description(@"Reads ONE indexed content chunk by its 0-based index within a file, with prev/next links to step through the file's chunk sequence. Use after `search_chunks` (which gives the collectionPath/filePath/chunkIndex of a hit) to read the full 1000-char window and walk to adjacent chunks. + +Returns `{found, collectionPath, filePath, chunkIndex, text, prevIndex, nextIndex, totalChunks}` — `prevIndex` is null at index 0, `nextIndex` is null at the last chunk. A null chunk (out of range or file not indexed) returns `{found:false, totalChunks, message:…}` carrying the valid range.")] + public Task GetChunk( + [Description("The content collection path the chunk belongs to (the 'collectionPath' from a search_chunks hit).")] string collectionPath, + [Description("The file path within the collection (the 'filePath' from a search_chunks hit).")] string filePath, + [Description("0-based chunk index within the file (the 'chunkIndex' from a hit, or a prevIndex/nextIndex to step).")] int chunkIndex) + => ChunkNavigation.GetChunk(sessionHub.ServiceProvider, MeshOperations.ResolvePath(collectionPath), filePath, chunkIndex) + .FirstAsync().ToTask(); + + [McpServerTool(Title = "Create a node", Destructive = false, Idempotent = false, OpenWorld = false)] + [Description(@"Creates a new node in the mesh. Pass a JSON MeshNode object. Required fields — validated up-front with a descriptive error before anything is written: + • id — the node's own slug, NO slashes (e.g. ""PricingTool""). The parent path goes in 'namespace'; the node's path is derived as {namespace}/{id}. + • namespace — full parent path (e.g. ""ACME/Projects""). Omit only for partition roots. + • name — human-readable display title (shown as the page heading). + • nodeType — the type definition that gives the node shape and views (e.g. ""Markdown"", ""Code"", ""Organization""). Discover types with search 'nodeType:NodeType'. +Recommended: 'icon' as an inline SVG starting with Create( [Description("JSON MeshNode object to create (e.g., {\"id\": \"NewOrg\", \"namespace\": \"ACME\", \"name\": \"New Org\", \"nodeType\": \"Organization\", \"content\": {}})")] string node) - => ops.Create(node); + => ops.Create(node).FirstAsync().ToTask(); - [McpServerTool] - [Description("Updates existing nodes in the mesh. Pass a JSON array of complete MeshNode objects. Always Get before Update — the entire node is replaced, not merged.")] + [McpServerTool(Title = "Replace nodes (full update)", Destructive = true, Idempotent = true, OpenWorld = false)] + [Description("Updates existing nodes in the mesh. Pass a JSON array of complete MeshNode objects. Always Get before Update — the entire node is replaced, not merged; a node missing 'nodeType' or 'content' is rejected with a descriptive error before anything is written. For small changes prefer Patch (field-level) or edit_content (text-level).")] public Task Update( [Description("JSON array of MeshNode objects with all fields (get existing node first, modify, then pass here)")] string nodes) - => ops.Update(nodes); + => ops.Update(nodes).FirstAsync().ToTask(); + + [McpServerTool(Title = "Patch node fields", Destructive = true, Idempotent = true, OpenWorld = false)] + [Description("Partial update of a single node. Only the keys present in 'fields' are changed; omitted keys preserve existing values. 'content' deep-merges (RFC 7396): the nested keys you send are updated, the ones you omit are kept, and a null member deletes just that key — so you can change a single content field (e.g. {\"content\":{\"logo\":\"…\"}}) without resending the rest. Setting the whole 'content' to null is rejected. Prefer this over Update for small edits like icon/name/category; for edits inside a long Markdown body or source file prefer edit_content.")] + public Task Patch( + [Description("Path to the node (e.g., @User/rbuergi/my-node)")] string path, + [Description("JSON object with ONLY the fields to change. Examples: {\"icon\": \"...\"}, {\"name\": \"New Name\"}, {\"content\":{\"logo\":\"https://…\"}} (deep-merges into existing content).")] string fields) + => ops.Patch(path, fields).FirstAsync().ToTask(); - [McpServerTool] - [Description("Deletes one or more nodes from the mesh by path.")] + [McpServerTool(Title = "Anchored text edit", Destructive = true, Idempotent = false, OpenWorld = false)] + [Description(@"Anchored text edit on a node's content (Markdown body or Code source). Replaces oldText with newText — pass just the snippet to change plus enough surrounding context to make it unique, instead of re-sending the whole document. Fails with a descriptive error when the text isn't found or isn't unique. Preferred over patch for any edit inside a long document or source file (cheaper, and immune to truncation corrupting the rest of the content).")] + public Task EditContent( + [Description("Path to the node (e.g., @User/rbuergi/my-doc or @ACME/Story/Source/Story.cs)")] string path, + [Description("The exact text to replace — copy it verbatim from get, including whitespace and line breaks. Must match exactly once (or set replaceAll).")] string oldText, + [Description("The replacement text.")] string newText, + [Description("Replace every occurrence instead of requiring a unique match. Default: false.")] bool replaceAll = false) + => ops.EditContent(path, oldText, newText, replaceAll).FirstAsync().ToTask(); + + [McpServerTool(Title = "Delete nodes (recursive)", Destructive = true, Idempotent = true, OpenWorld = false)] + [Description("Deletes one or more nodes from the mesh by path. Recursive: deleting a parent removes all descendants. To remove a subtree, just pass the root path — children do not need to be enumerated.")] public Task Delete( [Description("JSON array of path strings to delete (e.g., [\"ACME/OldProject\", \"ACME/ArchivedTask\"])")] string paths) - => ops.Delete(paths); + => ops.Delete(paths).FirstAsync().ToTask(); + + [McpServerTool(Title = "Move a subtree", Destructive = true, Idempotent = false, OpenWorld = false)] + [Description("Moves a node and its descendants to a new path. Equivalent to the Move menu item. Requires Delete on the source namespace and Create on the target. Source and target are full paths (namespace + id), e.g. 'OrgA/Child' -> 'OrgB/Child'.")] + public Task Move( + [Description("Current path of the node (e.g., @OrgA/Child)")] string sourcePath, + [Description("New path for the node (e.g., @OrgB/Child)")] string targetPath) + => ops.Move(sourcePath, targetPath).FirstAsync().ToTask(); + + [McpServerTool(Title = "Copy a subtree", Destructive = false, Idempotent = false, OpenWorld = false)] + [Description("Copies a node and all its descendants to a target namespace. Equivalent to the Copy menu item. Source ids are preserved; paths are rewritten under the target namespace.")] + public Task Copy( + [Description("Current path of the node to copy (e.g., @OrgA/Child)")] string sourcePath, + [Description("Target namespace to copy under (e.g., @OrgB)")] string targetNamespace, + [Description("Overwrite existing nodes at the target. Default: false.")] bool force = false) + => ops.Copy(sourcePath, targetNamespace, force).FirstAsync().ToTask(); - [McpServerTool] - [Description("Returns a URL to view a node in the MeshWeaver UI. Use this to provide links for users to open in their browser.")] + [McpServerTool(Title = "Browser URL for a node", ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description("Returns a URL to view a node in the MeshWeaver UI. The URL shape is `{baseUrl}/{path}` — the mesh path is appended directly to the base URL with no intermediate segment (no `/node/`) and without URL-escaping the path separators. Use this when you want to give a user a link to open in their browser. Call with an empty path to get the base URL on its own.")] public string NavigateTo( - [Description("Path to navigate to (e.g., @graph/org1)")] string path) + [Description("Path to navigate to (e.g., @Systemorph/FutuRe/EuropeRe). Leading `@` is stripped. Empty returns the base URL.")] string? path = null) { logger.LogInformation("MCP NavigateTo called with path={Path}", path); + if (string.IsNullOrWhiteSpace(path)) + return baseUrl.TrimEnd('/'); + var resolvedPath = MeshOperations.ResolvePath(path); - return $"{baseUrl}/node/{Uri.EscapeDataString(resolvedPath)}"; + return $"{baseUrl.TrimEnd('/')}/{resolvedPath.TrimStart('/')}"; } - [McpServerTool] + [McpServerTool(Title = "NodeType compile status", ReadOnly = true, Idempotent = true, OpenWorld = false)] [Description("Returns compilation diagnostics for a NodeType (or any instance of one). Status is 'Ok' when the type compiled cleanly, 'Error' with details when it failed, 'Compiling' while a compile is in progress (with elapsedMs), or 'Unknown' when no compile has happened yet. Use after creating/updating a NodeType to verify it actually compiles — a NodeType that doesn't compile is not 'done'.")] public Task GetDiagnostics( [Description("Path to a NodeType (e.g., @Systemorph/SocialMedia/Profile) or to any instance of one")] string path) - => ops.GetDiagnostics(path); + => ops.GetDiagnostics(path).FirstAsync().ToTask(); - [McpServerTool] - [Description("Recycles the hub at the given path by posting DisposeRequest. Forces a fresh hub initialization on the next access — use after fixing a broken NodeType, after editing the `sources` list, or whenever a grain is stuck in a cached bad state. Returns {status:'Recycled', path}. Wait ~100ms before the next access so the grain teardown completes.")] + [McpServerTool(Title = "Recycle a node's hub", Destructive = true, Idempotent = true, OpenWorld = false)] + [Description("Recycles the hub at the given path by posting DisposeRequest. Forces a fresh hub initialization on the next access — use after fixing a broken NodeType, after editing the `sources` list, or whenever a grain is stuck in a cached bad state. Requires Update permission on the target node. Returns {status:'Recycled', path}. Wait ~100ms before the next access so the grain teardown completes.")] public Task Recycle( [Description("Path to the node (e.g., @Systemorph/SocialMedia/Profile). Use the NodeType path to recycle the whole type; use an instance path to recycle just that instance's hub.")] string path) - => ops.Recycle(path); + => ops.Recycle(path).FirstAsync().ToTask(); + + [McpServerTool(Title = "Compile a NodeType", Idempotent = true, OpenWorld = false)] + [Description(@"Compiles a NodeType and waits for the result inline. Flips the NodeType's `compilationStatus` to `Pending` via the canonical remote-stream `Update` (no PatchDataRequest, no Update permission required), then subscribes to the NodeType's MeshNode stream and waits up to 60s for the framework's CompileWatcher to settle the status to `Ok` or `Error`. + +Returns a structured result: + • `{status:'Ok', path, activityPath, message:'Compile SUCCEEDED.'}` — assembly cached, ready to use. + • `{status:'Error', path, error, activityPath, message:'Compile FAILED ...'}` — `error` carries the Roslyn diagnostics inline. + • `{status:'Pending', path, message:'... did not settle within deadline'}` — fallback only on timeout; `get @nodeTypePath` to poll. + +For the full source-discovery + matched-Code-paths + Roslyn trace, `get @` after the call returns.")] + public Task Compile( + [Description("Path to the NodeType (e.g., @User/me/MyType or @Systemorph/SocialMedia/Profile). Must point at a NodeType definition node, not an instance.")] string path) + => ops.Compile(path).FirstAsync().ToTask(); + + [McpServerTool(Title = "Speculative Roslyn check", ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description(@"PRE-FLIGHT CHECK before committing a source change to a NodeType. Runs Roslyn against the NodeType's current source set with ONE source file substituted by `proposedCode`, returns all diagnostics (errors + warnings). No emit, no Recycle, no side effects — purely speculative. + +Use this in the Coder edit loop: edit a Source/*.cs file in your head → `lsp_check_node` → if diagnostics, fix → repeat → only then `patch` + `compile`. Eliminates the costly blind-patch / Compile / fix cycle. + +Returns `{ok: true, diagnostics: []}` when the substituted source compiles cleanly, or `{ok: false, diagnostics: [{id, severity, message, sourcePath?, line?, character?}, ...]}` when it doesn't. Severity is one of `Hidden|Info|Warning|Error`. Positions are 0-based.")] + public Task LspCheckNode( + [Description("Path to the NodeType (e.g., @ACME/Story).")] string nodeTypePath, + [Description("Path of the Source Code node being edited (e.g., @ACME/Story/Source/StoryTypes.cs). If not in the current source set, the proposed code is added as a new file.")] string sourcePath, + [Description("The proposed full source text for that file.")] string proposedCode) + { + var lang = rootHub.ServiceProvider.GetRequiredService(); + return lang.CheckSpeculative( + MeshOperations.ResolvePath(nodeTypePath), + MeshOperations.ResolvePath(sourcePath), + proposedCode ?? string.Empty) + .Select(diagnostics => FormatDiagnosticsJson(diagnostics, sessionHub.JsonSerializerOptions)) + .FirstAsync().ToTask(); + } + + [McpServerTool(Title = "Roslyn diagnostics for a NodeType", ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description(@"Returns Roslyn diagnostics from the NodeType's CURRENT cached compilation — distinct from `GetDiagnostics` which only reports compile status (Ok/Error/Compiling). This enumerates every diagnostic in the compilation (errors + warnings + info) with source location, so you can see exactly what's wrong without re-compiling. + +Returns `{ok: true|false, diagnostics: [...]}` — same shape as `lsp_check_node`. Empty `diagnostics` plus `ok:true` means clean.")] + public Task LspDiagnosticsForNode( + [Description("Path to the NodeType (e.g., @ACME/Story).")] string nodeTypePath) + { + var lang = rootHub.ServiceProvider.GetRequiredService(); + return lang.GetDiagnostics(MeshOperations.ResolvePath(nodeTypePath)) + .Select(diagnostics => FormatDiagnosticsJson(diagnostics, sessionHub.JsonSerializerOptions)) + .FirstAsync().ToTask(); + } + + // lsp_hover_for_node / lsp_completions_for_node were removed from the MCP surface + // (2026-06-11 tool-surface compaction): position-based hover/completions are + // IDE-interaction shapes — an agent driving JSON tool calls reads the source via + // `get` and runs `lsp_check_node` for the pre-flight loop. IMeshLanguageService + // keeps both capabilities for first-party UI use. + + /// + /// Shared diagnostic JSON shape for the lsp_check_node + lsp_diagnostics_for_node tools. + /// ok is true when the diagnostic list has no Error-severity entries — warnings + /// alone don't fail the check (mirrors how a regular compile succeeds with warnings). + /// + private static string FormatDiagnosticsJson(IReadOnlyList diagnostics, JsonSerializerOptions options) + { + var anyErrors = diagnostics.Any(d => d.Severity == DiagnosticSeverity.Error); + return JsonSerializer.Serialize( + new + { + ok = !anyErrors, + diagnostics = diagnostics.Select(d => new + { + id = d.Id, + severity = d.Severity.ToString(), + message = d.Message, + sourcePath = d.Location?.SourcePath, + line = d.Location?.Range.Start.Line, + character = d.Location?.Range.Start.Character, + }).ToArray() + }, + options); + } + + [McpServerTool(Title = "Execute a Code node", Destructive = true, Idempotent = false, OpenWorld = false)] + [Description("Runs an executable Code node's C# through the kernel (Microsoft.DotNet.Interactive) and returns stdout / return value / errors. The target node must have `CodeConfiguration.IsExecutable == true`. Blocks until the kernel signals completion (side-effects — e.g. mesh.CreateNode calls inside the script — have happened by the time this returns). Use to run import/test scripts from MCP without needing a UI click.")] + public Task ExecuteScript( + [Description("Path to an executable Code node (e.g., @Systemorph/FutuRe/EuropeRe/AcmeSubmission2025/Script/ImportLargeClaims). Must be `IsExecutable=true`.")] string path, + [Description("Timeout in seconds. Default 120.")] int timeoutSeconds = 120) + => ops.ExecuteScript(path, timeoutSeconds).FirstAsync().ToTask(); + + [McpServerTool(Title = "Start an agent thread", Destructive = false, Idempotent = false, OpenWorld = false)] + [Description(@"Starts a new agent conversation thread and submits the first message — the server-side agent executes it asynchronously. This is the ONLY way to launch a thread from MCP: do NOT hand-assemble Thread nodes with `create`/`patch` — the submission protocol (pending-message draining, response-cell allocation) is owned by the thread hub, and bypassing it leaves a wedged thread. + +Returns `{status:'Started', threadPath}` as soon as the thread node exists; the agent keeps working after this returns. Observe progress and the result with `get` on the threadPath: `content.status` is 'Idle' when the round finished and `content.summary` carries the result digest. The full transcript is queryable via `search 'path:{threadPath} scope:descendants nodeType:ThreadMessage select:text,role,timestamp'`.")] + public Task StartThread( + [Description("Namespace to create the thread under (e.g. 'rbuergi' or 'ACME/Projects'). The thread lives at {namespace}/_Thread/{id}.")] string namespacePath, + [Description("The first user message — the task for the agent. Write it self-contained: the agent has not seen this conversation.")] string message, + [Description("Agent to run the thread (e.g. 'Assistant', 'Coder', 'Researcher'). Default: the platform's default agent.")] string? agentName = null, + [Description("Optional node path the agent should treat as its working context (relative @-paths in the conversation resolve against it).")] string? contextPath = null) + { + if (string.IsNullOrWhiteSpace(namespacePath)) + return Task.FromResult("Error: namespacePath is required — the thread is created under it."); + if (string.IsNullOrWhiteSpace(message)) + return Task.FromResult("Error: message is required — it is the task the agent executes."); + + // MCP boundary adapter: bridge the extension's one-shot callbacks to the Task the + // MCP surface requires (same pattern as InboxTool). Bounded so a lost callback + // can't hang the MCP call forever. + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + cts.Token.Register(() => tcs.TrySetResult( + "Error: thread creation did not confirm within 30s. Verify the namespace exists and check with search 'nodeType:Thread namespace:" + + MeshOperations.ResolvePath(namespacePath) + "/_Thread'.")); + + sessionHub.StartThread( + MeshOperations.ResolvePath(namespacePath), + message, + agentName: agentName, + contextPath: contextPath != null ? MeshOperations.ResolvePath(contextPath) : null, + onCreated: node => tcs.TrySetResult(JsonSerializer.Serialize( + new + { + status = "Started", + threadPath = node.Path, + agentName, + hint = "The agent executes asynchronously. get the threadPath to observe: content.status == 'Idle' means the round finished; content.summary is the result digest." + }, + sessionHub.JsonSerializerOptions)), + onError: err => tcs.TrySetResult($"Error starting thread: {err}")); + + return tcs.Task.ContinueWith(t => { cts.Dispose(); return t.Result; }); + } + + [McpServerTool(Title = "Send a message to a thread", Destructive = false, Idempotent = false, OpenWorld = false)] + [Description(@"Queues a follow-up user message on an existing agent thread. If the thread is idle, the submission watcher dispatches a new round immediately; if the agent is mid-round, the message is delivered the next time it checks its inbox. Use `start_thread` to create a new thread; use `get` on the thread path to read status and results.")] + public Task SubmitMessage( + [Description("Path of the thread (e.g. 'rbuergi/_Thread/fix-login-bug-3f9a') — as returned by start_thread or found via search 'nodeType:Thread'.")] string threadPath, + [Description("The user message text.")] string message, + [Description("Optional agent override for this round. Default: the thread's current agent.")] string? agentName = null) + { + if (string.IsNullOrWhiteSpace(threadPath)) + return Task.FromResult("Error: threadPath is required. Use start_thread to create a new thread."); + if (string.IsNullOrWhiteSpace(message)) + return Task.FromResult("Error: message is required."); + + var resolvedPath = MeshOperations.ResolvePath(threadPath); + var tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + sessionHub.SubmitMessage( + resolvedPath, + message, + agentName: agentName, + onError: err => tcs.TrySetResult($"Error submitting to {resolvedPath}: {err}")); + + // SubmitMessage is fire-and-forget with an error-only callback; give a short grace + // window for a synchronous failure (bad path, unreadable thread) to surface, then + // report queued. The write itself is confirmed via the thread node's stream. + return Task.WhenAny(tcs.Task, Task.Delay(TimeSpan.FromSeconds(2))) + .ContinueWith(_ => tcs.Task.IsCompleted + ? tcs.Task.Result + : JsonSerializer.Serialize( + new + { + status = "Submitted", + threadPath = resolvedPath, + hint = "Message queued. get the threadPath to observe: content.status == 'Idle' means the round finished." + }, + sessionHub.JsonSerializerOptions)); + } + + [McpServerTool(Title = "Mirror a subtree to/from a remote portal", Destructive = true, Idempotent = true)] + [Description(@"Mirror a subtree between THIS instance and a remote MeshWeaver portal. `direction='push'` copies local → remote (promote dev content to prod, stage a snapshot for review); `direction='pull'` copies remote → local (seed dev with prod data). + +Authentication — prefer a named remote profile configured server-side (`Mirror:Remotes:{name}` with `BaseUrl` + `Token` in the host configuration) and pass the profile name as `remote`: the token then never enters the model context or transcripts. Passing a base URL as `remote` also works when a profile with a matching BaseUrl is configured. Supplying `remoteToken` inline is a discouraged fallback for ad-hoc one-offs. + +Returns a JSON summary: `{status, direction, sourcePath, targetPath, nodesImported, nodesSkipped, nodesRemoved, partitionsImported, elapsedMs}`. With `dryRun=true` returns `{status:'DryRun', nodesScanned, paths:[...]}` so you can preview before writing. + +Network: this instance must have outbound HTTPS reach to the remote. Prod can't reach localhost — for prod→local you need a tunnel (Cloudflare / ngrok).")] + public Task Mirror( + [Description("'push' (local → remote) or 'pull' (remote → local).")] string direction, + [Description("Named remote profile (configured under Mirror:Remotes:{name}) — preferred — or a base URL like 'https://memex.meshweaver.cloud'.")] string remote, + [Description("Path whose subtree to mirror: a local path for push (e.g. 'rbuergi/Story'), a remote path for pull (e.g. 'Doc/Architecture').")] string sourcePath, + [Description("Optional destination path to write under. Defaults to sourcePath.")] string? targetPath = null, + [Description("If true, delete destination nodes that don't exist at the source (DESTRUCTIVE).")] bool removeMissing = false, + [Description("If true, only enumerate what would be touched without writing.")] bool dryRun = false, + [Description("ApiToken for the remote (mw_…). Discouraged — prefer a configured remote profile so the secret stays server-side.")] string? remoteToken = null) + { + var dir = direction?.Trim().ToLowerInvariant() switch + { + "push" => "Push", + "pull" => "Pull", + _ => null + }; + if (dir is null) + return Task.FromResult("Error: direction must be 'push' (local → remote) or 'pull' (remote → local)."); + + var (resolvedUrl, resolvedToken, error) = ResolveRemote(remote, remoteToken); + if (error != null) + return Task.FromResult(error); + + return PostMirror(new MirrorRequest + { + RemoteBaseUrl = resolvedUrl!, + RemoteToken = resolvedToken!, + SourcePath = sourcePath, + TargetPath = targetPath, + Direction = dir, + RemoveMissing = removeMissing, + DryRun = dryRun, + }); + } + + /// + /// Resolves the mirror destination + credential. Profiles live in host configuration under + /// Mirror:Remotes:{name} (BaseUrl + Token) so the ApiToken stays + /// server-side — tool arguments flow through the model context and transcripts, which is + /// no place for a credential. An explicitly passed token always wins (ad-hoc escape hatch). + /// + private (string? BaseUrl, string? Token, string? Error) ResolveRemote(string remote, string? explicitToken) + { + if (string.IsNullOrWhiteSpace(remote)) + return (null, null, + "Error: 'remote' is required — a profile name configured under Mirror:Remotes, or a base URL."); + + var config = rootHub.ServiceProvider.GetService(); + var isUrl = remote.StartsWith("http", StringComparison.OrdinalIgnoreCase); + + if (!isUrl) + { + var section = config?.GetSection($"Mirror:Remotes:{remote}"); + var url = section?["BaseUrl"]; + if (string.IsNullOrEmpty(url)) + return (null, null, + $"Error: no remote profile '{remote}' is configured (expected Mirror:Remotes:{remote}:BaseUrl " + + "in the host configuration). Configure the profile server-side, or pass the base URL directly."); + var token = !string.IsNullOrEmpty(explicitToken) ? explicitToken : section?["Token"]; + if (string.IsNullOrEmpty(token)) + return (null, null, + $"Error: remote profile '{remote}' has no Token configured (Mirror:Remotes:{remote}:Token)."); + return (url.TrimEnd('/'), token, null); + } + + // URL given: still prefer a configured profile whose BaseUrl matches, so the token stays server-side. + var trimmed = remote.TrimEnd('/'); + var match = config?.GetSection("Mirror:Remotes").GetChildren() + .FirstOrDefault(c => string.Equals(c["BaseUrl"]?.TrimEnd('/'), trimmed, StringComparison.OrdinalIgnoreCase)); + var resolvedToken = !string.IsNullOrEmpty(explicitToken) ? explicitToken : match?["Token"]; + if (string.IsNullOrEmpty(resolvedToken)) + return (null, null, + $"Error: no token available for '{remote}'. Configure a profile under Mirror:Remotes with this " + + "BaseUrl + Token (preferred — the secret never enters the model context), or pass remoteToken explicitly."); + return (trimmed, resolvedToken, null); + } + + /// + /// Shared MCP-tool body: posts the mirror request at the mesh hub via + /// the standard request/response pattern (`hub.Observe`) and serialises + /// the response. The handler is registered by + /// MirrorHubExtensions.AddMirrorHandler on the mesh hub + /// (wired into every AddPersistence-enabled host). + /// + private Task PostMirror(MirrorRequest request) => + sessionHub.Observe(request, o => o.WithTarget(new Address("mesh"))) + .Catch((Exception ex) => + { + logger.LogError(ex, "Mirror failed for {Source} {Direction} {Url}", + request.SourcePath, request.Direction, request.RemoteBaseUrl); + return Observable.Return((IMessageDelivery)null!); + }) + .Select(d => d?.Message ?? new MirrorResult + { + Status = "Error", + Direction = request.Direction, + SourcePath = request.SourcePath, + TargetPath = request.TargetPath ?? request.SourcePath, + Error = "No response from mirror handler — is the mesh hub reachable and AddPersistence configured?", + }) + .Select(r => JsonSerializer.Serialize(r, sessionHub.JsonSerializerOptions)) + .FirstAsync().ToTask(); + + [McpServerTool(Title = "Render a layout area (MCP-UI)", ReadOnly = true, Idempotent = true, OpenWorld = false)] + [Description(@"Returns an interactive rendering of a layout area as an MCP-UI embedded resource. Hosts that support MCP-UI (Claude.ai web/desktop, ChatGPT Apps) render this inline as an iframe widget; text-only hosts see the URL as a fallback. + +Use this when the user would benefit from seeing the live view — charts, grids, dashboards, triangles — rather than a JSON dump. For plain data inspection keep using `Get`. + +Examples: + RenderArea('@Systemorph/FutuRe/EuropeRe/AcmeSubmission2025', 'Triangle') + RenderArea('Northwind', 'SalesByCategory')")] + public CallToolResult RenderArea( + [Description("Path to the node hosting the layout area (e.g., @Systemorph/FutuRe/EuropeRe/AcmeSubmission2025). Leading `@` is stripped.")] string path, + [Description("Layout area name on that node (e.g., 'Triangle', 'Overview', 'Dashboard').")] string areaName) + { + if (string.IsNullOrWhiteSpace(path)) + return ErrorResult("Error: path is required."); + if (string.IsNullOrWhiteSpace(areaName)) + return ErrorResult("Error: areaName is required."); + + var resolvedPath = MeshOperations.ResolvePath(path).TrimStart('/'); + var areaUrl = $"{baseUrl.TrimEnd('/')}/{resolvedPath}/{Uri.EscapeDataString(areaName).Replace("%2F", "/")}"; + var resourceUri = $"ui://mesh/{resolvedPath}/{areaName}"; + + logger.LogInformation("MCP RenderArea path={Path} areaName={Area} url={Url}", resolvedPath, areaName, areaUrl); + + var iframeHtml = BuildIframeHtml(areaUrl, areaName); + + return new CallToolResult + { + Content = + [ + new EmbeddedResourceBlock + { + Resource = new TextResourceContents + { + Uri = resourceUri, + MimeType = "text/html", + Text = iframeHtml, + }, + }, + new ResourceLinkBlock + { + Uri = areaUrl, + Name = areaName, + Title = $"{areaName} — {resolvedPath}", + MimeType = "text/html", + }, + new TextContentBlock + { + Text = $"Open in browser: {areaUrl}", + }, + ], + }; + } + + private static CallToolResult ErrorResult(string message) => new() + { + IsError = true, + Content = [new TextContentBlock { Text = message }], + }; + + private static string BuildIframeHtml(string areaUrl, string areaName) + { + var src = WebUtility.HtmlEncode(areaUrl); + var title = WebUtility.HtmlEncode(areaName); + return $$""" + + + + + {{title}} + + + + + + + """; + } } /// @@ -91,5 +665,5 @@ public class McpConfiguration /// /// Base URL for the MeshWeaver UI. Used for generating NavigateTo URLs. /// - public string BaseUrl { get; set; } = "http://localhost:5000"; + public string BaseUrl { get; set; } = string.Empty; } diff --git a/src/MeshWeaver.Blazor.AI/McpResources.cs b/src/MeshWeaver.Blazor.AI/McpResources.cs new file mode 100644 index 000000000..755e769ed --- /dev/null +++ b/src/MeshWeaver.Blazor.AI/McpResources.cs @@ -0,0 +1,38 @@ +using System.ComponentModel; +using MeshWeaver.AI; +using ModelContextProtocol.Server; + +namespace MeshWeaver.Blazor.AI; + +/// +/// MCP resources: reference documentation an MCP client reads once instead of +/// rediscovering syntax through trial-and-error tool calls. Served from the same +/// embedded ToolsReference document the in-portal agents get @@-included into +/// their instructions, so both surfaces stay in sync by construction. +/// +[McpServerResourceType] +public class McpResources +{ + [McpServerResource(UriTemplate = "meshweaver://reference/tools", Name = "tools-reference", + Title = "MeshWeaver tools reference", MimeType = "text/markdown")] + [Description("Complete reference for the mesh tools: @-path resolution rules, GitHub-style Search query syntax, MeshNode schema for create/update, unified path prefixes (data/, schema/, content/, area/), content collections, satellite namespaces, and icon rules.")] + public static string ToolsReference() + { + var assembly = typeof(BuiltInAgentProvider).Assembly; + const string resourceName = "MeshWeaver.AI.Data.Agent.ToolsReference.md"; + using var stream = assembly.GetManifestResourceStream(resourceName); + if (stream == null) + return $"(embedded resource {resourceName} not found)"; + using var reader = new StreamReader(stream); + var content = reader.ReadToEnd(); + + // Strip the YAML frontmatter — provider metadata, not reference content. + if (content.StartsWith("---")) + { + var end = content.IndexOf("\n---", 3, StringComparison.Ordinal); + if (end > 0) + content = content[(end + 4)..].TrimStart('\r', '\n'); + } + return content; + } +} diff --git a/src/MeshWeaver.Blazor.AI/MeshWeaver.Blazor.AI.csproj b/src/MeshWeaver.Blazor.AI/MeshWeaver.Blazor.AI.csproj index 7c021fd53..f8cffc93f 100644 --- a/src/MeshWeaver.Blazor.AI/MeshWeaver.Blazor.AI.csproj +++ b/src/MeshWeaver.Blazor.AI/MeshWeaver.Blazor.AI.csproj @@ -1,6 +1,7 @@ {8a2c4b5d-6e7f-8901-2345-6789abcdef01} + $(NoWarn);NU1510 @@ -8,7 +9,9 @@ + + diff --git a/src/MeshWeaver.Blazor.AI/SessionHubResolver.cs b/src/MeshWeaver.Blazor.AI/SessionHubResolver.cs new file mode 100644 index 000000000..98f89abfc --- /dev/null +++ b/src/MeshWeaver.Blazor.AI/SessionHubResolver.cs @@ -0,0 +1,106 @@ +using System.Security.Claims; +using MeshWeaver.Data; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.Blazor.AI; + +/// +/// Resolves a per-caller hosted hub at portal/{prefix}-{sessionId} for +/// transports that expose mesh operations to external clients (MCP, REST). +/// +/// +/// Both transports must share this helper so their routing semantics stay +/// identical — same address shape, same +/// + wiring, same fallback to the +/// root hub when no session is resolvable. +/// +/// +public static class SessionHubResolver +{ + /// + /// Materialises (or reuses) a hosted hub for the calling user × protocol session + /// at address portal/{prefix}-{sessionId}. Falls back to + /// if no caller / session can be derived from . + /// + /// Portal-level hub from which to host the child. + /// Current HTTP context (claims + Mcp-Session-Id header). + /// Transport label used in the address segment: "mcp", "api", … + /// Diagnostic sink. + public static IMessageHub ResolveSessionHub( + IMessageHub rootHub, + HttpContext? ctx, + string prefix, + ILogger logger) + { + var sessionId = ResolveSessionId(ctx); + if (sessionId is null) + { + logger.LogWarning( + "No {Prefix} session id resolvable from request — falling back to root hub. " + + "Some routing rules (kernel dispatch, etc.) will not fire.", + prefix); + return rootHub; + } + + var routingService = rootHub.ServiceProvider.GetRequiredService(); + var address = AddressExtensions.CreatePortalAddress($"{prefix}-{sessionId}"); + logger.LogInformation("Materialising {Prefix} session hub at {Address}", prefix, address); + + // AddData() ensures the session hub has its own IWorkspace so MeshOperations.Compile + // can subscribe to the NodeType MeshNode stream and Update its compilationStatus + // through the canonical write path. RegisterStream wires routing so every response + // (Get / Search / Patch / ExecuteScript / …) lands back here. + return rootHub.GetHostedHub( + address, + sessionConfig => sessionConfig + .AddData() + .WithInitialization(hub => + hub.RegisterForDisposal(routingService.RegisterStream(hub))), + HostedHubCreation.Always) + ?? throw new InvalidOperationException( + $"Failed to materialise {prefix} session hub at {address}."); + } + + /// + /// Derives a stable session identifier from by combining + /// the authenticated caller id with the optional Mcp-Session-Id header. + /// Returns null when neither is present. + /// + public static string? ResolveSessionId(HttpContext? ctx) + { + if (ctx is null) return null; + + // Prefer the standard MCP protocol header. REST callers can set it too + // for stable per-connection session scoping; otherwise the caller id alone + // identifies the session. + var protocolSession = ctx.Request.Headers["Mcp-Session-Id"].FirstOrDefault(); + + var callerId = ctx.User?.FindFirst("oid")?.Value + ?? ctx.User?.FindFirst(ClaimTypes.NameIdentifier)?.Value + ?? ctx.User?.FindFirst(ClaimTypes.Email)?.Value + ?? ctx.User?.Identity?.Name; + + if (!string.IsNullOrEmpty(callerId) && !string.IsNullOrEmpty(protocolSession)) + return $"{Sanitize(callerId)}-{Sanitize(protocolSession)}"; + if (!string.IsNullOrEmpty(callerId)) + return Sanitize(callerId); + if (!string.IsNullOrEmpty(protocolSession)) + return $"anon-{Sanitize(protocolSession)}"; + return null; + } + + /// + /// Sanitises a free-form id into a safe address segment: letters, digits, '-', '_'. + /// Everything else is replaced with '-' so hosted-hub grain-key lookup stays well-formed. + /// + public static string Sanitize(string s) + { + var chars = s.Select(c => char.IsLetterOrDigit(c) || c == '-' || c == '_' ? c : '-').ToArray(); + return new string(chars); + } +} diff --git a/src/MeshWeaver.Blazor.Graph/BlazorGraphExtensions.cs b/src/MeshWeaver.Blazor.Graph/BlazorGraphExtensions.cs index 95dcc68f7..7bad29e50 100644 --- a/src/MeshWeaver.Blazor.Graph/BlazorGraphExtensions.cs +++ b/src/MeshWeaver.Blazor.Graph/BlazorGraphExtensions.cs @@ -17,11 +17,13 @@ public static class BlazorGraphExtensions public static MessageHubConfiguration AddGraphViews(this MessageHubConfiguration configuration) { return configuration - .WithTypes(typeof(MeshNodeEditorControl), typeof(MeshNodeThumbnailControl), typeof(MeshNodeCardControl)) + .WithTypes(typeof(MeshNodeEditorControl), typeof(MeshNodeThumbnailControl), typeof(MeshNodeCardControl), + typeof(MeshNodeContentEditorControl)) .AddViews(registry => registry .WithView() .WithView() - .WithView()) + .WithView() + .WithView()) .AddMeshNavigation(); // Enable @ autocomplete in markdown editors } } diff --git a/src/MeshWeaver.Blazor.Graph/MeshNodeEditorView.razor b/src/MeshWeaver.Blazor.Graph/MeshNodeEditorView.razor index ea6551a43..56961b090 100644 --- a/src/MeshWeaver.Blazor.Graph/MeshNodeEditorView.razor +++ b/src/MeshWeaver.Blazor.Graph/MeshNodeEditorView.razor @@ -17,31 +17,12 @@ {

    Metadata

    - -
    - - / - -
    -
    - +
    - -
    - - @(_isSaving ? "Saving..." : "Save Metadata") - -
    - - @if (!string.IsNullOrEmpty(_metadataMessage)) - { - - @_metadataMessage - - }
    @@ -76,24 +57,10 @@ Content editing is only supported for Story and Article node types. } +
    - @if (_nodeType == "story" || _nodeType == "article") - { -
    - - @(_isSaving ? "Saving..." : "Save Content") - -
    - - @if (!string.IsNullOrEmpty(_contentMessage)) - { - - @_contentMessage - - } - } +
    + Changes are saved automatically.
    } diff --git a/src/MeshWeaver.Blazor.Graph/MeshNodeEditorView.razor.cs b/src/MeshWeaver.Blazor.Graph/MeshNodeEditorView.razor.cs index c8b91e2ca..c3e7e4bc3 100644 --- a/src/MeshWeaver.Blazor.Graph/MeshNodeEditorView.razor.cs +++ b/src/MeshWeaver.Blazor.Graph/MeshNodeEditorView.razor.cs @@ -1,91 +1,62 @@ -using MeshWeaver.Blazor.Components.Monaco; -using MeshWeaver.Graph; +using System.Reactive.Linq; +using MeshWeaver.Blazor.Components.Monaco; +using MeshWeaver.Data; using MeshWeaver.Mesh; using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; using Microsoft.AspNetCore.Components; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; namespace MeshWeaver.Blazor.Graph; -public partial class MeshNodeEditorView +public partial class MeshNodeEditorView : IDisposable { private MonacoEditorView? _monacoEditor; private MeshNode? _node; private bool _isLoading = true; - private bool _isSaving; - // Metadata fields - private string _parentPath = string.Empty; - private string _lastSegment = string.Empty; - private string _originalPath = string.Empty; + // Bound fields — refreshed from the live stream and pushed back through it. private string _name = string.Empty; private string? _nodeType; - - // Content field private string _contentText = string.Empty; - // Messages - private string? _metadataMessage; - private bool _metadataSuccess; - private string? _contentMessage; - private bool _contentSuccess; + // Suppress stream-echo refresh while the user is mid-typing in the content area. + private bool _userIsEditingContent; + + // Backend editor: owns one long-lived subscription to the node's MeshNode stream + // and writes back through the same stream. No save buttons, no AwaitResponse — + // every change streams immediately. + private IMeshNodeEditor? _editor; + private IDisposable? _streamSub; protected override void BindData() { base.BindData(); - _ = LoadNodeAsync(); + _editor = new MeshNodeEditor(Hub, ViewModel.NodePath); + _streamSub = _editor.Node.Subscribe(node => + { + _node = node; + ApplyNodeToFields(node); + _isLoading = false; + InvokeAsync(StateHasChanged); + }, ex => + { + Logger.LogError(ex, "Error streaming node at path {Path}", ViewModel.NodePath); + _isLoading = false; + InvokeAsync(StateHasChanged); + }); } - private async Task LoadNodeAsync() + private void ApplyNodeToFields(MeshNode node) { - _isLoading = true; - StateHasChanged(); + _name = node.Name ?? string.Empty; + _nodeType = node.NodeType?.ToLowerInvariant(); - try - { - var meshQuery = Hub.ServiceProvider.GetService(); - if (meshQuery == null) - { - Logger.LogError("IMeshService not available"); - return; - } - - var path = ViewModel.NodePath; - _originalPath = path; - _node = await meshQuery.QueryAsync($"path:{path}").FirstOrDefaultAsync(); - - if (_node != null) - { - // Parse path into parent and last segment - var segments = path.Split('/', StringSplitOptions.RemoveEmptyEntries); - if (segments.Length > 1) - { - _parentPath = string.Join("/", segments.Take(segments.Length - 1)); - _lastSegment = segments[^1]; - } - else - { - _parentPath = ""; - _lastSegment = path; - } - - _name = _node.Name ?? string.Empty; - _nodeType = _node.NodeType?.ToLowerInvariant(); - - // Load content based on node type - LoadContent(); - } - } - catch (Exception ex) - { - Logger.LogError(ex, "Error loading node at path {Path}", ViewModel.NodePath); - } - finally - { - _isLoading = false; - StateHasChanged(); - } + // Don't clobber the user's in-flight edits with the round-trip echo from the + // stream — only refresh content text from the stream when the user isn't typing. + if (!_userIsEditingContent) + LoadContent(); } private void LoadContent() @@ -96,7 +67,7 @@ private void LoadContent() return; } - // Handle Story content using reflection (to avoid circular dependency with Graph.Domain) + // Reflect into Story.Text to avoid circular dependency with Graph.Domain. if (_nodeType == "story") { var textProperty = _node.Content.GetType().GetProperty("Text"); @@ -110,147 +81,41 @@ private void LoadContent() _contentText = string.Empty; } - private void OnContentChanged(string value) + private void OnNameChanged(string newName) { - _contentText = value; + if (_name == newName) return; + _name = newName; + // Push name change through the stream — echo refreshes _node. + _editor?.Update(node => node with { Name = newName }); } - private async Task SaveMetadataAsync() + private void OnContentChanged(string value) { - if (_node == null) return; - - _isSaving = true; - _metadataMessage = null; - StateHasChanged(); - - try - { - // Calculate new path - var newPath = string.IsNullOrEmpty(_parentPath) - ? _lastSegment - : $"{_parentPath}/{_lastSegment}"; - - // Check if path changed - var pathChanged = !newPath.Equals(_originalPath, StringComparison.OrdinalIgnoreCase); - - if (pathChanged) - { - // Move the node to new path - Logger.LogInformation("Moving node from {OldPath} to {NewPath}", _originalPath, newPath); - var moveResponse = await Hub.AwaitResponse( - new MoveNodeRequest(_originalPath, newPath), - o => o.WithTarget(Hub.Address)); - if (!moveResponse.Message.Success) - throw new InvalidOperationException(moveResponse.Message.Error); - _node = moveResponse.Message.Node - ?? throw new InvalidOperationException("Move succeeded but returned no node"); - _originalPath = newPath; - } - - // Update metadata - var updatedNode = MeshNode.FromPath(_node.Path) with - { - Name = _name, - NodeType = _node.NodeType, - Icon = _node.Icon, - Order = _node.Order, - Content = _node.Content, - AssemblyLocation = _node.AssemblyLocation, - HubConfiguration = _node.HubConfiguration, - GlobalServiceConfigurations = _node.GlobalServiceConfigurations - }; - - var updateResponse = await Hub.AwaitResponse( - new UpdateNodeRequest(updatedNode), - o => o.WithTarget(Hub.Address)); - if (!updateResponse.Message.Success) - throw new InvalidOperationException(updateResponse.Message.Error); - _node = updateResponse.Message.Node; - - _metadataMessage = pathChanged - ? $"Metadata saved. Node moved to {newPath}" - : "Metadata saved successfully"; - _metadataSuccess = true; - } - catch (Exception ex) - { - Logger.LogError(ex, "Error saving metadata"); - _metadataMessage = $"Error: {ex.Message}"; - _metadataSuccess = false; - } - finally - { - _isSaving = false; - StateHasChanged(); - } + if (_contentText == value) return; + _userIsEditingContent = true; + _contentText = value; + _editor?.Update(node => node with { Content = WithText(node.Content, value) }); } - private async Task SaveContentAsync() + private object? WithText(object? currentContent, string newText) { - if (_node == null) return; - - _isSaving = true; - _contentMessage = null; - StateHasChanged(); - - try - { - // Update content based on node type - object? newContent = _node.Content; - - if (_nodeType == "story" && _node.Content != null) - { - // Use reflection to update Story.Text (avoid circular dependency with Graph.Domain) - var textProperty = _node.Content.GetType().GetProperty("Text"); - if (textProperty != null) - { - // Create a new instance with updated Text using record's with expression via reflection - // Since Story is a record, we can use the $ method - var cloneMethod = _node.Content.GetType().GetMethod("$"); - if (cloneMethod != null) - { - var cloned = cloneMethod.Invoke(_node.Content, null); - if (cloned != null) - { - textProperty.SetValue(cloned, _contentText); - newContent = cloned; - } - } - } - } - var updatedNode = MeshNode.FromPath(_node.Path) with - { - Name = _node.Name, - NodeType = _node.NodeType, - Icon = _node.Icon, - Order = _node.Order, - Content = newContent, - AssemblyLocation = _node.AssemblyLocation, - HubConfiguration = _node.HubConfiguration, - GlobalServiceConfigurations = _node.GlobalServiceConfigurations - }; - - var updateResponse = await Hub.AwaitResponse( - new UpdateNodeRequest(updatedNode), - o => o.WithTarget(Hub.Address)); - if (!updateResponse.Message.Success) - throw new InvalidOperationException(updateResponse.Message.Error); - _node = updateResponse.Message.Node; - - _contentMessage = "Content saved successfully"; - _contentSuccess = true; - } - catch (Exception ex) - { - Logger.LogError(ex, "Error saving content"); - _contentMessage = $"Error: {ex.Message}"; - _contentSuccess = false; - } - finally - { - _isSaving = false; - StateHasChanged(); - } + if (_nodeType != "story" || currentContent == null) + return currentContent; + + var textProperty = currentContent.GetType().GetProperty("Text"); + if (textProperty == null) + return currentContent; + + // Records: use the compiler-generated $ method so we don't lose any + // fields the editor doesn't surface. + var cloneMethod = currentContent.GetType().GetMethod("$"); + if (cloneMethod == null) + return currentContent; + + var cloned = cloneMethod.Invoke(currentContent, null); + if (cloned == null) return currentContent; + textProperty.SetValue(cloned, newText); + return cloned; } private CompletionProviderConfig GetArticleCompletionProvider() @@ -305,4 +170,10 @@ private CompletionProviderConfig GetArticleCompletionProvider() ] }; } + + public void Dispose() + { + _streamSub?.Dispose(); + _editor?.Dispose(); + } } diff --git a/src/MeshWeaver.Blazor.Portal/Chat/Chat.razor b/src/MeshWeaver.Blazor.Portal/Chat/Chat.razor index fe79078e7..7d3968afe 100644 --- a/src/MeshWeaver.Blazor.Portal/Chat/Chat.razor +++ b/src/MeshWeaver.Blazor.Portal/Chat/Chat.razor @@ -1,8 +1,48 @@ @page "/chat" +@using MeshWeaver.AI +@using MeshWeaver.Data @using MeshWeaver.Layout +@using MeshWeaver.Mesh.Security +@using MeshWeaver.Messaging +@inject AccessService AccessService - +@* The standalone /chat composer is the per-user ThreadComposer node's default ("") + layout area — databinds message text + harness/agent/model to {user}/_Thread/ThreadComposer + (see ThreadComposerView). Falls back to a direct ThreadChatControl while the user + identity is still resolving. *@ +@{ var chatInputArea = GetThreadComposerLayoutArea(); } +@if (chatInputArea is not null) +{ +
    + +
    +} +else +{ + +} @code { - private ThreadChatControl chatControl = new ThreadChatControl(); + private readonly ThreadChatControl chatControl = new(); + + private LayoutAreaControl? GetThreadComposerLayoutArea() + { + // Durable circuit user first; skip a leaked system-security / hub principal on the + // AsyncLocal Context (which would point the composer at a non-existent + // system-security/_Thread/ThreadComposer). Mirrors ThreadSidePanelContent.ResolveUserHome. + string? userHome = null; + foreach (var candidate in new[] { AccessService.CircuitContext?.ObjectId, AccessService.Context?.ObjectId }) + { + if (!string.IsNullOrEmpty(candidate) + && candidate != WellKnownUsers.System + && !AccessService.LooksLikeHubPrincipal(candidate)) + { + userHome = candidate; + break; + } + } + return string.IsNullOrEmpty(userHome) + ? null + : new LayoutAreaControl(ThreadComposerNodeType.PathFor(userHome), new LayoutAreaReference(string.Empty)); + } } diff --git a/src/MeshWeaver.Blazor.Portal/Chat/ChatDelegationView.razor b/src/MeshWeaver.Blazor.Portal/Chat/ChatDelegationView.razor index 4e45cbd96..09fb6f988 100644 --- a/src/MeshWeaver.Blazor.Portal/Chat/ChatDelegationView.razor +++ b/src/MeshWeaver.Blazor.Portal/Chat/ChatDelegationView.razor @@ -9,9 +9,24 @@
    + @{ + var taskSummary = DelegationContent.TaskSummary; + var hasTask = !string.IsNullOrEmpty(taskSummary); + } @if (DelegationContent.RequiresUserFeedback) { - Requesting feedback before delegating to @@@DelegationContent.TargetAgent + if (hasTask) + { + Requesting feedback: @taskSummary (@DelegationContent.TargetAgent) + } + else + { + Requesting feedback before delegating to @@@DelegationContent.TargetAgent + } + } + else if (hasTask) + { + @taskSummary (@DelegationContent.TargetAgent) } else { diff --git a/src/MeshWeaver.Blazor.Portal/Chat/ChatHistorySelector.razor b/src/MeshWeaver.Blazor.Portal/Chat/ChatHistorySelector.razor index d9b42aabf..4eb4dca56 100644 --- a/src/MeshWeaver.Blazor.Portal/Chat/ChatHistorySelector.razor +++ b/src/MeshWeaver.Blazor.Portal/Chat/ChatHistorySelector.razor @@ -1,3 +1,5 @@ +@using System.Reactive.Linq +@using System.Reactive.Threading.Tasks @using MeshWeaver.AI @using MeshWeaver.Data @using MeshWeaver.Graph.Configuration @@ -197,9 +199,9 @@ private record ThreadItem(string Path, MeshNode Node, MeshThread Content); private IReadOnlyList Chats = []; - protected override async Task OnInitializedAsync() + protected override void OnInitialized() { - await LoadConversations(); + LoadConversations(); } private string GetCurrentUserId() @@ -222,25 +224,32 @@ return path; } - private async Task LoadConversations() + private IDisposable? _conversationSubscription; + + private void LoadConversations() { + _conversationSubscription?.Dispose(); try { var userId = GetCurrentUserId(); - // Query threads by node type and created by user var query = $"nodeType:{ThreadNodeType.NodeType} createdBy:{userId}"; - var threadNodes = await MeshQuery.QueryAsync(query).ToListAsync(); - - // Convert to ThreadItem list, filtering by scope if specified - var items = threadNodes - .OrderByDescending(n => n.LastModified) - .Select(n => new ThreadItem(n.Path ?? n.Id, n, n.Content as MeshThread ?? new MeshThread())) - .Where(c => Scope == null || GetPathScope(c.Path) == Scope) - .ToList(); - - Chats = items; - StateHasChanged(); + _conversationSubscription = MeshQuery + .Query(MeshQueryRequest.FromQuery(query)) + .Subscribe(change => + { + var items = change.Items + .OrderByDescending(n => n.LastModified) + .Select(n => new ThreadItem(n.Path ?? n.Id, n, n.Content as MeshThread ?? new MeshThread())) + .Where(c => Scope == null || GetPathScope(c.Path) == Scope) + .ToList(); + + InvokeAsync(() => + { + Chats = items; + StateHasChanged(); + }); + }); } catch (Exception ex) { @@ -323,29 +332,29 @@ await OnClose.InvokeAsync(); } - private async Task DeleteConversation(string chatPath) + private void DeleteConversation(string chatPath) { - try - { - // Delete via DeleteNodeRequest - var request = new DeleteNodeRequest(chatPath); - await Hub.AwaitResponse(request, o => o.WithTarget(new Address(chatPath))); - - // Remove from local list - Chats = Chats.Where(c => c.Path != chatPath).ToList(); + // Subscribe — no await on hub round-trip. Continuation updates state on the dispatcher. + var request = new DeleteNodeRequest(chatPath); + Hub.Observe(request, o => o.WithTarget(new Address(chatPath))) + .FirstAsync() + .Subscribe( + _ => InvokeAsync(() => OnConversationDeletedAsync(chatPath)), + ex => Console.WriteLine($"Error deleting conversation: {ex.Message}")); + } - // If this was the selected conversation, start a new one - if (SelectedConversationId == chatPath) - { - await StartNewConversation(); - } + private async Task OnConversationDeletedAsync(string chatPath) + { + // Remove from local list + Chats = Chats.Where(c => c.Path != chatPath).ToList(); - StateHasChanged(); - } - catch (Exception ex) + // If this was the selected conversation, start a new one + if (SelectedConversationId == chatPath) { - Console.WriteLine($"Error deleting conversation: {ex.Message}"); + await StartNewConversation(); } + + StateHasChanged(); } private string FormatDate(DateTime date) @@ -367,8 +376,11 @@ } } - public async Task RefreshConversations() + public Task RefreshConversations() { - await LoadConversations(); + LoadConversations(); + return Task.CompletedTask; } + + public void Dispose() => _conversationSubscription?.Dispose(); } diff --git a/src/MeshWeaver.Blazor.Portal/Chat/ChatMessageItem.razor b/src/MeshWeaver.Blazor.Portal/Chat/ChatMessageItem.razor index b078aa587..51aaef2bc 100644 --- a/src/MeshWeaver.Blazor.Portal/Chat/ChatMessageItem.razor +++ b/src/MeshWeaver.Blazor.Portal/Chat/ChatMessageItem.razor @@ -14,7 +14,7 @@ @Message.Text
    } -else if (Message.Role != ChatRole.User) +else if (Message.Role != ChatRole.System) { foreach (var content in Message.Contents) { diff --git a/src/MeshWeaver.Blazor.Portal/Chat/MeshNodeAutocomplete.razor b/src/MeshWeaver.Blazor.Portal/Chat/MeshNodeAutocomplete.razor index 658eeafdb..6e91a0351 100644 --- a/src/MeshWeaver.Blazor.Portal/Chat/MeshNodeAutocomplete.razor +++ b/src/MeshWeaver.Blazor.Portal/Chat/MeshNodeAutocomplete.razor @@ -1,12 +1,14 @@ +@using System.Reactive.Linq @using MeshWeaver.Mesh @using MeshWeaver.Mesh.Services @using MeshWeaver.Messaging +@using MeshWeaver.Reactive @using Microsoft.Extensions.DependencyInjection > OnSearchAsync(string searchText) + private IObservable> OnSearch(string searchText) { var meshQuery = Hub.ServiceProvider.GetService(); if (meshQuery == null) - return Enumerable.Empty(); + return Observable.Return>([]); - try - { - List suggestions; - - if (Options.Queries is { Length: > 0 }) - { - // Query mode: run provided queries, append user text, merge results - var userText = (searchText ?? "").Trim(); - var tasks = Options.Queries.Select(async baseQuery => - { - var fullQuery = string.IsNullOrEmpty(userText) - ? baseQuery - : $"{baseQuery} {userText}"; - try - { - return await meshQuery.QueryAsync(fullQuery) - .Select(n => new QuerySuggestion(n.Path, n.Name ?? n.Id, n.NodeType, 1.0, n.Icon)) - .ToListAsync(); - } - catch { return new List(); } - }); - var all = await Task.WhenAll(tasks); - suggestions = all.SelectMany(x => x) - .GroupBy(s => s.Path, StringComparer.OrdinalIgnoreCase) - .Select(g => g.First()) - .ToList(); - } - else - { - // Autocomplete mode (default) - suggestions = await meshQuery.AutocompleteAsync( + IObservable> source = Options.Queries is { Length: > 0 } + ? QueryAutocompleteHelper.LoadSuggestions(meshQuery, Options.Queries, searchText) + : meshQuery.Autocomplete( basePath: Options.BasePath, prefix: searchText ?? "", mode: AutocompleteMode.RelevanceFirst, - limit: Options.Limit * 3 - ).ToListAsync(); - } + limit: Options.Limit * 3) + .Select(snapshot => (IReadOnlyList)snapshot + .Select(r => new QuerySuggestion(r.Path, r.Name ?? "", r.NodeType, r.Score, r.Icon)) + .ToList()); - // If filtering by creatable type, post-filter results - if (!string.IsNullOrEmpty(EffectiveCreatableTypeFilter)) - suggestions = await FilterByCreatableTypeAsync(suggestions, EffectiveCreatableTypeFilter); + var limited = source.Select(list => (IReadOnlyList)list.Take(Options.Limit).ToArray()); - return suggestions.Take(Options.Limit); - } - catch - { - return Enumerable.Empty(); - } + if (string.IsNullOrEmpty(EffectiveCreatableTypeFilter)) + return limited; + + // Filter by creatable type — fully reactive: per-suggestion CanCreate + // observable folded through ScanTopN(int.MaxValue) into an incremental + // snapshot. No await, no FirstAsync, no ToTask. + return source.SelectMany(items => + FilterSuggestionsByCreatableType(items, EffectiveCreatableTypeFilter)); } - /// - /// Filters suggestions to only include nodes that can create the specified type. - /// - private async Task> FilterByCreatableTypeAsync( - List suggestions, - string nodeTypePath) + private IObservable> FilterSuggestionsByCreatableType( + IReadOnlyList items, string nodeTypePath) { - var nodeTypeService = Hub.ServiceProvider.GetService(); - if (nodeTypeService == null) - { - return suggestions; - } - - var result = new List(); - - foreach (var suggestion in suggestions) - { - try - { - // Check if this node can create the specified type - var canCreate = await CanCreateTypeAtPathAsync(nodeTypeService, suggestion.Path, nodeTypePath); - if (canCreate) - { - result.Add(suggestion); - } - } - catch - { - // On error, skip this suggestion - } - } - - return result; + var provider = Hub.ServiceProvider.GetService(); + if (provider == null) + return Observable.Return(items); + + // For each candidate suggestion, ask the provider whether nodeTypePath + // is creatable at that path. Reactive: each per-suggestion observable + // emits once with the verdict; merged + folded via ScanTopN. + return items.ToObservable() + .SelectMany(s => CanCreateTypeAtPath(provider, s.Path, nodeTypePath) + .Where(can => can) + .Select(_ => s)) + .ScanTopN(int.MaxValue, _bySuggestionOrder); } + private static readonly IComparer _bySuggestionOrder = + Comparer.Create((a, b) => + string.Compare(a.Path, b.Path, StringComparison.OrdinalIgnoreCase)); + /// - /// Checks if a specific node type can be created at the given path. + /// Reactive check: does the provider include + /// among the types creatable at ? Single + /// emission, no Task bridge. /// - private static async Task CanCreateTypeAtPathAsync( - INodeTypeService nodeTypeService, + private static IObservable CanCreateTypeAtPath( + ICreatableTypesProvider provider, string nodePath, string nodeTypePath) - { - await foreach (var creatableType in nodeTypeService.GetCreatableTypesAsync(nodePath)) - { - if (creatableType.NodeTypePath.Equals(nodeTypePath, StringComparison.OrdinalIgnoreCase)) - { - return true; - } - } - return false; - } + => provider.GetCreatableTypes(nodePath, parentNode: null) + .Take(1) + .Select(types => types.Any(t => + t.NodeTypePath.Equals(nodeTypePath, StringComparison.OrdinalIgnoreCase))); } diff --git a/src/MeshWeaver.Blazor.Portal/Chat/QueryAutocompleteHelper.cs b/src/MeshWeaver.Blazor.Portal/Chat/QueryAutocompleteHelper.cs new file mode 100644 index 000000000..91c5c6b5d --- /dev/null +++ b/src/MeshWeaver.Blazor.Portal/Chat/QueryAutocompleteHelper.cs @@ -0,0 +1,48 @@ +using System.Collections.Immutable; +using System.Reactive.Linq; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; + +namespace MeshWeaver.Blazor.Portal.Chat; + +/// +/// Helper for : composes per-query +/// streams into a single deduped +/// suggestion list. Pure reactive — caller subscribes to the observable. +/// +internal static class QueryAutocompleteHelper +{ + /// + /// Returns a folded, path-deduped list of + /// for the given queries + user search text. Emits a fresh snapshot + /// whenever a new path arrives. Per-path dedup via + /// Scan. + /// + public static IObservable> LoadSuggestions( + IMeshService meshQuery, + IReadOnlyList queries, + string? searchText) + { + var userText = (searchText ?? "").Trim(); + var observables = queries.Select(baseQuery => + { + var fullQuery = string.IsNullOrEmpty(userText) + ? baseQuery + : $"{baseQuery} {userText}"; + return meshQuery + .Query(MeshQueryRequest.FromQuery(fullQuery)) + .Take(1) + .Catch, Exception>( + _ => Observable.Empty>()); + }); + + return Observable.Merge(observables) + .SelectMany(change => change.Items) + .Select(n => new QuerySuggestion(n.Path, n.Name ?? n.Id, n.NodeType, 1.0, n.Icon)) + .Scan( + ImmutableDictionary.Empty + .WithComparers(StringComparer.OrdinalIgnoreCase), + (acc, s) => acc.ContainsKey(s.Path) ? acc : acc.Add(s.Path, s)) + .Select(acc => (IReadOnlyList)acc.Values.ToArray()); + } +} diff --git a/src/MeshWeaver.Blazor.Portal/Chat/SimpleDropdown.razor b/src/MeshWeaver.Blazor.Portal/Chat/SimpleDropdown.razor index 9a794fbf5..035eeaa49 100644 --- a/src/MeshWeaver.Blazor.Portal/Chat/SimpleDropdown.razor +++ b/src/MeshWeaver.Blazor.Portal/Chat/SimpleDropdown.razor @@ -90,7 +90,7 @@ public IEnumerable? Items { get; set; } [Parameter] - public Func>>? OnSearch { get; set; } + public Func>>? OnSearch { get; set; } [Parameter] public Func? GetDisplayText { get; set; } @@ -223,37 +223,57 @@ } } - private async Task Toggle() + private void Toggle() { isOpen = !isOpen; if (isOpen) { searchText = ""; highlightedIndex = 0; - await LoadItems(); + LoadItems(); } } - private async Task LoadItems() + private IDisposable? _searchSub; + + private void LoadItems() { if (OnSearch != null) { - // Async search mode + // Reactive search mode: subscribe to OnSearch's observable, + // update displayItems on every emission. No await, no Task — + // per Doc/Architecture/BlazorDataBinding.md. + _searchSub?.Dispose(); isLoading = true; + displayItems = new List(); + highlightedIndex = 0; StateHasChanged(); try { - var items = await OnSearch(searchText); - displayItems = items.Take(20).ToList(); - highlightedIndex = 0; + _searchSub = OnSearch(searchText).Subscribe( + items => + { + displayItems = items.Take(20).ToList(); + highlightedIndex = 0; + isLoading = false; + InvokeAsync(StateHasChanged); + }, + _ => + { + displayItems = new List(); + isLoading = false; + InvokeAsync(StateHasChanged); + }, + () => + { + isLoading = false; + InvokeAsync(StateHasChanged); + }); } catch { displayItems = new List(); - } - finally - { isLoading = false; } } @@ -264,9 +284,9 @@ } } - private async Task OnSearchChanged() + private void OnSearchChanged() { - await LoadItems(); + LoadItems(); } private async Task OnKeyDown(KeyboardEventArgs e) @@ -312,6 +332,8 @@ public void Dispose() { + _searchSub?.Dispose(); + try { JS.InvokeVoidAsync("eval", $"window['dropdown_{componentId}_cleanup']?.();"); diff --git a/src/MeshWeaver.Blazor.Portal/Chat/SimpleDropdown.razor.css b/src/MeshWeaver.Blazor.Portal/Chat/SimpleDropdown.razor.css index 3382a9358..71c702bea 100644 --- a/src/MeshWeaver.Blazor.Portal/Chat/SimpleDropdown.razor.css +++ b/src/MeshWeaver.Blazor.Portal/Chat/SimpleDropdown.razor.css @@ -62,7 +62,10 @@ border: 1px solid var(--neutral-stroke-rest); border-radius: 4px; box-shadow: 0 -4px 16px rgba(0, 0, 0, 0.15); - z-index: 1000; + /* Floating-popup tier (matches Monaco's overflow widgets in standard-page-layout.css) so the + chat agent/model/harness dropdown opens ABOVE the menu bar (z-index 1100 in + PortalLayoutBase) instead of being covered by it. */ + z-index: 10000; max-height: 320px; display: flex; flex-direction: column; diff --git a/src/MeshWeaver.Blazor.Portal/Chat/ThreadChatView.razor b/src/MeshWeaver.Blazor.Portal/Chat/ThreadChatView.razor index 20b63200b..d9db74646 100644 --- a/src/MeshWeaver.Blazor.Portal/Chat/ThreadChatView.razor +++ b/src/MeshWeaver.Blazor.Portal/Chat/ThreadChatView.razor @@ -1,14 +1,15 @@ @inherits BlazorView +@using MeshWeaver.AI +@using MeshWeaver.AI.Connect +@using MeshWeaver.Data @using MeshWeaver.Layout @using MeshWeaver.Blazor.Components @using MeshWeaver.Blazor.Components.Monaco -@using FluentOrientation = Microsoft.FluentUI.AspNetCore.Components.Orientation -@using FluentVerticalAlignment = Microsoft.FluentUI.AspNetCore.Components.VerticalAlignment +@using MeshWeaver.Messaging @using FluentAppearance = Microsoft.FluentUI.AspNetCore.Components.Appearance -@using FluentJustifyContent = Microsoft.FluentUI.AspNetCore.Components.JustifyContent @inject NavigationManager NavigationManager -
    +
    @if (viewMode == ChatViewMode.ResumeThreads) {
    @@ -39,25 +40,193 @@ else { } + @{ + var pendingTexts = ThreadViewModel?.PendingMessageTexts ?? Array.Empty(); + var hasAnyMessage = ThreadMessages.Count > 0 || pendingTexts.Count > 0; + } + @* Chat history rendered inline per message — one + IMeshNodeStreamCache subscription per visible id is opened by + SyncMessageSubscriptions; this template just reads from the + messageStates dictionary. No per-message LayoutAreaView server + round-trip; the bubble is the chat's own DOM. *@ @if (ThreadMessages.Count > 0) { @foreach (var msgId in ThreadMessages) { - var cell = GetMessageCell(msgId); - @if (cell != null) + var state = GetMessageState(msgId); + var isUser = state?.Role?.Equals("user", StringComparison.OrdinalIgnoreCase) ?? false; + var bubbleRoleClass = isUser ? "thread-msg-user" : "thread-msg-assistant"; + +
    + @if (state is null && IsMissing(msgId)) { - + @* Satellite cell never delivered an emission within the + probe window — either deleted, never materialised, or + denied by RLS. Surface as "Missing message" instead + of an indefinite skeleton so the user knows the chat + continued past this point. *@ +
    — message missing —
    +
    id: @msgId
    } + else if (state is null) + { +
    +
    + } + else + { +
    @state.AuthorName
    + @* Assistant meta row: Harness · time · duration · in/out tokens. + Model name intentionally dropped — the harness is the headline + identity; for Claude Code / Copilot the model is implied. *@ + @if (!isUser && (!string.IsNullOrEmpty(state.Harness) || state.Timestamp.HasValue)) + { +
    + @if (!string.IsNullOrEmpty(state.Harness)) + { + @state.Harness + } + @if (state.Timestamp.HasValue) + { + @state.Timestamp.Value.ToString("HH:mm:ss") + } + @* Stream duration: + • Status == Streaming → live (ticker-driven, "0:12") + • CompletedAt set → frozen final ("1:23") + • Otherwise (legacy bubbles loaded without Status/CompletedAt) → omit. *@ + @if (state.Status == "Streaming" && state.Timestamp.HasValue) + { + @FormatElapsed(state.Timestamp) + } + else if (state.Timestamp.HasValue && state.CompletedAt.HasValue) + { + @FormatElapsed(state.Timestamp, state.CompletedAt) + } + @if (state.InputTokens.HasValue || state.OutputTokens.HasValue) + { + @($"{state.InputTokens ?? 0:N0} in / {state.OutputTokens ?? 0:N0} out") + } +
    + } + + @if (IsEditing(msgId)) + { + +
    + +
    + } + else + { + @if (!string.IsNullOrEmpty(state.Text)) + { + + } + @if (isUser && !IsReadOnlyThread) + { +
    + + + +
    + } + } + + @if (state.ToolCalls is { Count: > 0 }) + { +
    + @foreach (var call in state.ToolCalls) + { + var isDelegation = !string.IsNullOrEmpty(call.DelegationPath); + var isPending = call.Result == null; + var summary = FormatToolCallSummary(call); + + @if (isDelegation) + { + @* Inline link only — sub-thread progress + status lives in the + runtime panel below the chat (anchored near the input). Keeps + the bubble compact and avoids the duplicate "tool call + embedded + streaming area" the user flagged. *@ + var delHeader = GetDelegationHeader(call.DelegationPath); + var title = delHeader?.Title ?? summary; + var icon = delHeader?.Icon ?? "/static/NodeTypeIcons/chat.svg"; + var stateIcon = isPending + ? "↗" + : (call.IsSuccess ? "✓" : "✗"); + var stateClass = isPending + ? "thread-msg-tool-delegation-pending" + : (call.IsSuccess ? "thread-msg-tool-delegation-ok" : "thread-msg-tool-delegation-error"); + + @stateIcon + + @title + @summary + + } + else + { + var display = FormatToolCallDisplay(call); + var change = display.IsNodeModifying ? FindChange(state.UpdatedNodes, display.Path) : null; + +
    + + @(call.IsSuccess ? "✓" : "✗") + @display.Verb + @if (!string.IsNullOrEmpty(display.Path)) + { +   + @display.Path + } + @if (change is not null) + { + @if (change.VersionBefore.HasValue && change.VersionAfter.HasValue) + { + Diff + } + @if (change.VersionBefore.HasValue) + { + Revert v@(change.VersionBefore) + } + } + + @if (!string.IsNullOrEmpty(call.Arguments)) + { + 
    @call.Arguments
    + } + @if (!string.IsNullOrEmpty(call.Result)) + { + 
    @call.Result
    + } +
    + } + } +
    + } + } +
    } } - else if (isCreatingThread) + @* Pending user messages (PendingMessageTexts) and running sub-threads + moved OUT of the scrollable history into the runtime panel + immediately above the input — see `.thread-runtime-panel` below. + Keeps the chat history clean (only persisted messages) and pins + in-flight state next to the input where the user is acting. *@ + @if (!hasAnyMessage && isCreatingThread) {

    Creating thread...

    } - else if (string.IsNullOrEmpty(threadPath)) + else if (!hasAnyMessage && string.IsNullOrEmpty(threadPath)) {
    @@ -68,6 +237,13 @@ else
    } + @* ─── Bottom bar ─────────────────────────────────────────────────── + Everything pinned to the bottom of the screen, top-down: execution + status ("where we are"), the runtime panel, then the input footer + (harness selector + editor + send button). Grouped in one flex + column so the chat history above scrolls and this block stays put. *@ +
    + @if (ThreadViewModel?.IsExecuting == true) { @@ -75,10 +251,20 @@ else
    ✹ @(ThreadViewModel.ExecutionStatus ?? "Generating response...") - + @* Elapsed-time chip — driven by the 1 s ticker installed in OnInitialized. + ExecutionStartedAt is null until the round flips Status → Executing, so + the chip auto-hides for the brief StartingExecution window. *@ + @if (ThreadViewModel.ExecutionStartedAt is { } started) + { + @FormatElapsed(started) + } + @if (!IsReadOnlyThread) + { + + }
    @if (!string.IsNullOrEmpty(ThreadViewModel.StreamingText)) { @@ -95,17 +281,97 @@ else }
    } + @* Pending-user-message rendering moved INTO the chat history (above) + so the user sees their submission immediately — including before + round dispatch starts streaming. The exec-bar no longer duplicates + the queue display. *@
    } - + @* ─── Runtime panel ───────────────────────────────────────────────── + Pinned just above the chat input. Two sections, top-down: + (1) Running sub-threads — live cards for every delegation whose + sub-thread currently has IsExecuting=true. Title, icon, + ExecutionStatus, and a truncated preview of StreamingText. + Drops out the moment the sub-thread settles. + (2) Scheduled messages — entries the user typed while a round is + in flight. They sit BELOW the running section so the visual + ordering matches the lifecycle: "in flight" above, + "waiting to be picked up" below. When the round completes + and the inbox drains, they get promoted to real messages in + the history. *@ + @{ + var runningSubThreads = GetRunningSubThreads().ToList(); + var runtimeScheduled = ThreadViewModel?.PendingMessageTexts ?? Array.Empty(); + } + @if (runningSubThreads.Count > 0 || runtimeScheduled.Count > 0) + { +
    + @if (runningSubThreads.Count > 0) + { +
    +
    Running sub-threads
    + @foreach (var (path, header) in runningSubThreads) + { + var subIcon = header.Icon ?? "/static/NodeTypeIcons/chat.svg"; + var subTitle = header.Title ?? path; + var preview = !string.IsNullOrWhiteSpace(header.StreamingText) + ? TruncateText(header.StreamingText.Trim(), 200) + : (header.ExecutionStatus ?? "Working…"); + + +
    +
    + @subTitle + @* Elapsed-time chip for the sub-thread — null StartedAt + means the sub-thread is in StartingExecution and + hasn't flipped LastActivityAt yet; chip auto-hides. *@ + @if (header.StartedAt is { } subStarted) + { + @FormatElapsed(subStarted) + } +
    +
    @preview
    +
    + +     + } +
    + } + @if (runtimeScheduled.Count > 0) + { +
    +
    + Scheduled — waiting for current round to finish +
    + @foreach (var pendingText in runtimeScheduled) + { +
    + + @pendingText +
    + } +
    + } +
    + } + + + @* Input footer — only for editable threads. Other users' threads are + read-only (threads are editable only by their owner). *@ + @if (!IsReadOnlyThread) + {
    -
    +
    @if (showSubmissionProgress) { -
    @@ -113,128 +379,237 @@ else
    Creating conversation and redirecting
    + @if (!string.IsNullOrWhiteSpace(lastSubmittedText)) + { + @* Echo the just-submitted message under the progress so it's visible during create+redirect. *@ +
    +
    You
    +
    @lastSubmittedText
    +
    + } + } + @{ var inputDimmed = showSubmissionProgress; } +
    + + @* Thin, READ-ONLY thread status — what this thread uses (harness · agent · model) + plus a token-usage chip, fully data-bound to the composer selections + (boundHarness/boundAgentPath/boundModelPath). The chip reads the per-model TokenUsage + satellites ({threadPath}/_Usage/*). Selection itself is via the /harness, /agent and + /model slash-commands — this row only displays. *@ + @if (!string.IsNullOrEmpty(boundHarness) || !string.IsNullOrEmpty(boundModelPath) || !string.IsNullOrEmpty(boundAgentPath)) + { +
    + @if (!string.IsNullOrEmpty(boundHarness)) + { + ⬡ @LastSegment(boundHarness) + } + @if (!string.IsNullOrEmpty(boundAgentPath)) + { + 🤖 @LastSegment(boundAgentPath) + } + @if (!string.IsNullOrEmpty(boundModelPath)) + { + ✦ @LastSegment(boundModelPath) + } + · + +
    + } + + @* GENERIC node picker — every node-pick command (/agent, /model, /harness, and any + module-defined one) pops THIS one menu. The command supplied the query + title; the + host queried the mesh (OpenPicker) and listed the nodes; clicking one writes its PATH + to the command's composer field. No per-command widget. *@ + @if (pendingPicker is { } picker) + { +
    +
    + @picker.Title + +
    +
    + @for (var i = 0; i < pickerNodes.Count; i++) + { + var node = pickerNodes[i]; + var isActive = i == _pickerHighlight; + + } + @if (pickerNodes.Count == 0) + { +
    Nothing to pick.
    + } +
    +
    } - else + + @* Inline harness LOGIN (Connect) — driven by a harness's /login command. Reuses + ConnectSessionManager (the same flow Settings → Models uses), surfaced in the chat so the + user never leaves the conversation to authenticate a CLI harness. *@ + @if (_connectProvider is not null) { - - - - @* Monaco editor for markdown input with completion *@ -
    + @* Uses the shared widget's responsive max-height (min(440px, calc(100vh-180px))) so it fits + below the side-panel header; the inner list scrolls if the login content exceeds it. *@ +
    +
    + Log in to @_connectHarnessLabel + +
    + @* Composed with the framework's FluentStack + FluentAnchor / FluentTextField / FluentButton + rather than hand-rolled flex divs + a raw — one vertical stack, framework-themed. *@ + + @switch (_connectStatus) + { + case ConnectStatus.Connected: +
    ✓ Connected — you can chat now.
    + break; + + case ConnectStatus.Error err: +
    @err.Reason
    + Try again + break; + + case ConnectStatus.Connecting c when !string.IsNullOrEmpty(c.Challenge.VerificationUrl): + @* A compact accent link instead of the raw OAuth URL — the full URL wraps to + ~8 lines and shoves the paste field + Submit off the bottom of the widget. *@ +
    1  Open the authorization page and approve:
    + Open authorization page ↗ + @if (c.Challenge.RequiresPastedCode) + { +
    2  Paste the code Claude shows you, then submit:
    + + @(_connectBusy ? "Verifying…" : "Submit code") + } + else if (!string.IsNullOrEmpty(c.Challenge.UserCode)) + { +
    Enter this code in the browser: @c.Challenge.UserCode
    +
    Waiting for authorization…
    + } + break; + + default: +
    Starting login…
    + break; + } +     +
    + } + + @* Monaco editor for markdown input with completion. Stays visible during + execution (Claude.ai pattern) — typing while a round is in flight is + allowed; it queues via PendingUserMessages and dispatches when the round + completes. Esc cancels the in-flight round; the typed text is preserved. + HIDDEN while a CLI-harness login (Connect) widget is up, so the user pastes the + code into the widget's field above — not into this chat box — and the widget's + Submit isn't pushed below the fold. *@ + @if (_connectProvider is null) + { +
    + CompletionCallback="@GetCompletions" />
    - -
    - - -
    @agent.Name
    - @if (!string.IsNullOrEmpty(agent.Path) && agent.Path != agent.Name) - { -
    @agent.Path
    - } - @if (!string.IsNullOrEmpty(agent.Description)) - { -
    @TruncateText(agent.Description, 80)
    - } -     -     + } - @if (availableModels.Count > 0) - { - - -
    @model.Name
    -     -     - } + @* Attachments split: the navigation-context chip(s) sit inline in the compact bottom + row (next to the harness combobox + Send); any extra @@-reference attachments get + their own section above the row so the row never overflows. *@ + @{ + var contextChips = attachments.Where(a => a.IsContext).ToList(); + var referenceChips = attachments.Where(a => !a.IsContext).ToList(); + } + @if (referenceChips.Count > 0) + { +
    + +
    + } - @if (!submissionHandler.IsInputEnabled) - { - - - - } - else + @* Compact bottom row: context chip(s) + Send. Harness/agent/model are chosen via the + /harness /agent /model slash-commands (which write the same composer node) — the in-footer + MeshNodePicker comboboxes were removed: eagerly resolving a picked agent whose model is not + wired threw out of the embedded area and tore the whole composer down (the "choosing + Assistant crashes the chat" defect). Slash-command selection records the choice on the + composer without building the agent, so it can never crash the footer. *@ + @* Hidden while the login (Connect) widget is up — only its paste-field + Submit are interactive. *@ + @if (_connectProvider is null) + { +
    + @if (contextChips.Count > 0) { - - - +
    + +
    } +
    + @if (!submissionHandler.IsInputEnabled) + { + + + + } + else + { + + + + } +
    } + @if (!string.IsNullOrEmpty(lastCommandStatus)) + { + @lastCommandStatus + } +
    @* /thread-chat-input-content *@
    + } +
    @* /thread-chat-bottom *@ }
    - - diff --git a/src/MeshWeaver.Blazor.Portal/Chat/ThreadChatView.razor.cs b/src/MeshWeaver.Blazor.Portal/Chat/ThreadChatView.razor.cs index 417a6fa7f..e2bb83e52 100644 --- a/src/MeshWeaver.Blazor.Portal/Chat/ThreadChatView.razor.cs +++ b/src/MeshWeaver.Blazor.Portal/Chat/ThreadChatView.razor.cs @@ -1,6 +1,7 @@ using System.Reactive.Linq; using System.Text.Json; using MeshWeaver.AI; +using MeshWeaver.AI.Parsing; using MeshWeaver.Blazor.Components; using MeshWeaver.Blazor.Components.Monaco; using MeshWeaver.Blazor.Portal.SidePanel; @@ -9,11 +10,15 @@ using MeshWeaver.Graph; using MeshWeaver.Layout; using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Security; using MeshWeaver.Mesh.Services; using MeshWeaver.Messaging; +using MeshWeaver.Reactive; using Microsoft.AspNetCore.Components; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using Microsoft.JSInterop; namespace MeshWeaver.Blazor.Portal.Chat; @@ -26,8 +31,37 @@ public partial class ThreadChatView : BlazorViewStateless — single instance reused per submission.
     + private static readonly ChatPreParser ChatParser = new(); + + /// + /// Most recent command-result message for the breadcrumb / status row. + /// Cleared on the next submission. + /// + private string? lastCommandStatus; + private bool lastCommandStatusIsError; + + /// + /// The node-pick request the most recent command asked us to render (null hides the picker), + /// plus the mesh nodes it resolved. Driven by the command's NodePickerRequest via OpenPicker. + /// + private NodePickerRequest? pendingPicker; + private IReadOnlyList pickerNodes = []; + + // Keyboard navigation of the command picker (the /agent etc. node list). The list is a focusable + // widget; ↑/↓ move _pickerHighlight, Enter commits, Escape dismisses. _focusPickerOnRender moves + // focus from the Monaco editor (where the command was typed) onto the widget when it opens, so the + // arrow keys reach the widget instead of being swallowed by Monaco. + private int _pickerHighlight; + private ElementReference _pickerWidget; + private bool _focusPickerOnRender; private bool _isDisposed; + private IDisposable? _navContextSubscription; + private NavigationContext? _currentNavContext; private IDisposable? agentSubscription; private readonly string _instanceId = Guid.NewGuid().ToString("N")[..8]; @@ -36,6 +70,35 @@ public partial class ThreadChatView : BlazorView + /// Single live stream for the thread MeshNode — serves every read AND + /// every write the chat performs (cancel, update sticky agent/model, + /// append pending message). Held as a field so we don't re-open a fresh + /// per click; we resolve the cache once and call Update(threadPath, fn) + /// on each write. + /// + /// This mirrors the canonical [DataBinding] pattern: all reads + + /// writes go through IMeshNodeStreamCache so the patch is observed + /// by every reader on the same path. No per-view upstream subscription. + /// + private IMeshNodeStreamCache? _cache; + + private IMeshNodeStreamCache? EnsureCache() + { + if (_cache is not null) return _cache; + try + { + _cache = Hub.ServiceProvider.GetRequiredService(); + } + catch (Exception ex) + { + Logger.LogWarning(ex, + "[ThreadChat:{InstanceId}] Failed to resolve IMeshNodeStreamCache; thread writes will fail", + _instanceId); + } + return _cache; + } + private ThreadViewModel? _threadViewModel; private ThreadViewModel? ThreadViewModel @@ -57,8 +120,26 @@ private ThreadViewModel? ThreadViewModel { threadPath = value.ThreadPath ?? threadPath; initialContext = value.InitialContext ?? initialContext; + + // Inside a thread the composer lives ON the thread node (Thread.Composer) — + // bind the embedded selectors area + the selection projection to the THREAD + // path. The thread node is the node we're rendering, so it is guaranteed + // present (no maybe-absent read, no lazy-create/stamp machinery). + if (!string.IsNullOrEmpty(value.ThreadPath) && _templatePath != value.ThreadPath) + { + _templatePath = value.ThreadPath; + OpenComposerProjection(value.ThreadPath); + } } + // Open per-message cache subscriptions AFTER threadPath is set — + // DataBind invokes the value converter BEFORE this setter runs, so + // calling SyncMessageSubscriptions from inside the converter sees + // an empty threadPath on the first emission and bails. Result was + // 9 skeleton bubbles forever when the upstream didn't push a + // second time. Call here, where threadPath is guaranteed current. + SyncMessageSubscriptions(value?.Messages ?? []); + // If messages changed, force re-render and release submission handler if (!Equals(old, value)) { @@ -87,6 +168,15 @@ private ThreadViewModel? ThreadViewModel } private IReadOnlyList ThreadMessages => ThreadViewModel?.Messages ?? []; + /// + /// True when the thread has no materialised messages AND nothing queued — the brand-new / + /// empty-thread state. Drives the full-height composer (the "start a conversation" landing): + /// the input box fills the view instead of sitting as a thin bar at the bottom. + /// + private bool HasNoMessages => + ThreadMessages.Count == 0 + && (ThreadViewModel?.PendingMessageTexts?.Count ?? 0) == 0; + // Input state private MonacoEditorView? monacoEditor; private ElementReference messagesContainer; @@ -102,6 +192,10 @@ private ThreadViewModel? ThreadViewModel // pendingCells removed — GUI creates real nodes, LayoutAreaView renders them directly. private bool showSubmissionProgress; + // The just-submitted user text, shown UNDER the progress panel while the new thread is created + + // redirected to, so the user sees their message immediately instead of a blank composer. + private string? lastSubmittedText; + // Unified attachments (context + @references) private readonly List attachments = new(); private const string placeholderText = "Type a message... Use @ to reference nodes"; @@ -112,14 +206,58 @@ private ThreadViewModel? ThreadViewModel // Resume threads state private MeshSearchControl? resumeSearchControl; - // Agent/model selection - private AgentDisplayInfo? selectedAgentInfo; - private ModelInfo? selectedModelInfo; + // Agent/model lists — fed by AgentPickerProjection; consumed by the /agent and /model + // slash commands + @-reference agent detection. The visible harness/agent/model SELECTION + // is 100% data-bound to the composer node (ThreadComposerView.SelectorsArea, embedded in + // the footer) — there is no imperative selectedAgent/selectedModel/selectedHarness state, + // no sticky restore, no resolve/rebuild machinery. The current selection is projected + // one-way from the composer node into the bound* fields below. private IReadOnlyList agentDisplayInfos = []; - private IReadOnlyList availableModels = []; - private readonly Dictionary agentModelPreferences = new(); - private IEnumerable ChatClientFactories => Hub.ServiceProvider.GetServices(); + /// + /// Current selection, projected one-way from the bound composer state (). + /// Each holds the picked node's PATH and flows through submit/resubmit UN-resolved — the + /// execution boundary normalizes paths to ids (SelectionId.IdOf). The data-bound pickers write + /// the composer themselves; the /agent /model commands and @-agent references write it via + /// WriteComposerSelection. + /// + private string? boundHarness; + private string? boundAgentPath; + private string? boundModelPath; + private IDisposable? composerSubscription; + private IDisposable? composerDefaultsSubscription; + + // ─── Composer binding target ─── + // Out of a thread: the per-user singleton composer NODE {userHome}/_Thread/ThreadComposer. + // Inside a thread: the THREAD path — the composer is embedded on the thread content + // (Thread.Composer) and the thread hub serves the same data-bound Selectors area. + private string? _userHome; + private string? _templatePath; + + /// The Id (last path segment) the execution pipeline matches on, from a picked node path. + private static string? LastSegment(string? path) => + string.IsNullOrEmpty(path) ? path : path.Split('/')[^1]; + + /// Compact token count for the thin thread status row (1234 → "1.2k"). + private static string FormatTokens(int tokens) => + tokens >= 1000 ? $"{tokens / 1000.0:0.#}k" : tokens.ToString(); + + // (Removed DefaultModelId / ModelTier:Standard — defaults are no longer hardcoded. The default + // model is the Order=-1 model resolved by AgentPickerProjection.ObserveDefaultComposer and + // written onto the composer; submit sends the bound selection, never an invented fallback.) + + /// + /// True when viewing an existing thread created by another user. Threads are + /// editable only by their owner, so in this case the chat input and all + /// thread-modifying ops (stop, edit, resubmit, delete) are hidden — the thread + /// renders read-only. The new-thread composer (no threadPath) is always + /// editable, as are the current user's own threads. + /// + private bool IsReadOnlyThread => + !string.IsNullOrEmpty(threadPath) + && !string.IsNullOrEmpty(_userHome) + && !string.IsNullOrEmpty(ThreadViewModel?.CreatedBy) + && !string.Equals(ThreadViewModel.CreatedBy, _userHome, StringComparison.OrdinalIgnoreCase); protected override void OnInitialized() { @@ -132,36 +270,79 @@ protected override void OnInitialized() // Subscribe to side panel menu actions SidePanelState.OnActionRequested += OnSidePanelAction; - // Track navigation changes via NavigationService — no query, no await. - NavigationService.OnNavigationContextChanged += OnNavigationContextChanged; + // 1-second ticker for elapsed-time chips on the exec bar, sub-thread cards, + // and per-bubble streaming chips. Only fires StateHasChanged when something's + // executing — silent on idle threads so we don't burn render cycles when + // there's nothing to update. + elapsedTicker = System.Reactive.Linq.Observable + .Interval(TimeSpan.FromSeconds(1)) + .Subscribe(_ => + { + if (_isDisposed) return; + var anyExecuting = + ThreadViewModel?.IsExecuting == true + || delegationHeaders.Values.Any(h => h.IsExecuting) + || messageStates.Values.Any(s => s.Status == "Streaming"); + if (anyExecuting) + InvokeAsync(StateHasChanged); + }); + + // Track navigation changes — subscribe to the reactive NavigationContext stream. + _navContextSubscription = NavigationService.NavigationContext + .Subscribe(ctx => { _currentNavContext = ctx; OnNavigationContextChanged(ctx); }); // Set initial title UpdateSidePanelTitle(); + // Resolve the composer binding: the per-user singleton {userHome}/_Thread/ThreadComposer + // in compose mode (EnsureComposer creates it with defaults if absent, then projects); + // inside a thread the composer is EMBEDDED on the thread node (Thread.Composer) — bind + // the thread path directly (the node we're rendering is guaranteed present). + var accessSvc = Hub.ServiceProvider.GetService(); + _userHome = ResolveUserHome(accessSvc); + if (!string.IsNullOrEmpty(threadPath)) + { + _templatePath = threadPath; + OpenComposerProjection(threadPath); + } + else if (!string.IsNullOrEmpty(_userHome)) + { + _templatePath = MeshWeaver.AI.ThreadComposerNodeType.PathFor(_userHome); + EnsureComposer(); + } + // Seed initial context attachment from NavigationService (already resolved, no query). if (string.IsNullOrEmpty(initialContext)) { - var ctx = NavigationService.Context; + var ctx = _currentNavContext; if (ctx is not null && !string.IsNullOrEmpty(ctx.PrimaryPath) && ctx.Path != "chat") { - initialContext = ctx.PrimaryPath; - attachments.Add(new AttachmentInfo(ctx.PrimaryPath, ctx.Node?.Name ?? ctx.Node?.Id, IsContext: true)); + var normalized = NormalizeContextPath(ctx.PrimaryPath); + initialContext = normalized; + // normalized is "" for a reserved route partition (login, …) — no context chip. + if (!string.IsNullOrEmpty(normalized) && !attachments.Any(a => a.IsContext && a.Path == normalized)) + attachments.Add(new AttachmentInfo(normalized, ctx.Node?.Name ?? ctx.Node?.Id, IsContext: true)); } } else { // ViewModel.InitialContext passed the raw path (e.g., side panel with ctx.PrimaryPath). // Look up the display name via GetDataRequest + RegisterCallback — never await. - var capturedContext = initialContext; - attachments.Add(new AttachmentInfo(capturedContext, null, IsContext: true)); - RequestDisplayName(capturedContext, name => InvokeAsync(() => + var capturedContext = NormalizeContextPath(initialContext); + initialContext = capturedContext; + // capturedContext is "" for a reserved route partition (login, …) — no context chip / read. + if (!string.IsNullOrEmpty(capturedContext) && !attachments.Any(a => a.IsContext && a.Path == capturedContext)) { - if (_isDisposed) return; - var idx = attachments.FindIndex(a => a.IsContext && a.Path == capturedContext); - if (idx >= 0) - attachments[idx] = attachments[idx] with { DisplayName = name }; - StateHasChanged(); - })); + attachments.Add(new AttachmentInfo(capturedContext, null, IsContext: true)); + RequestDisplayName(capturedContext, name => InvokeAsync(() => + { + if (_isDisposed) return; + var idx = attachments.FindIndex(a => a.IsContext && a.Path == capturedContext); + if (idx >= 0) + attachments[idx] = attachments[idx] with { DisplayName = name }; + StateHasChanged(); + })); + } } try @@ -177,9 +358,202 @@ protected override void OnInitialized() Logger.LogDebug("[ThreadChat:{InstanceId}] OnInitialized completed", _instanceId); } + // ─── Per-user chat template (_ThreadTemplate) ───────────────────────────── + + /// + /// The signed-in user's partition — the partition that owns + /// {user}/_Thread/ThreadComposer and the namespace a submitted thread is created + /// under. Prefer (the durable per-circuit + /// identity); (AsyncLocal) is only a fallback and + /// is filtered for a leaked system-security / hub principal. Trusting + /// Context first pointed the composer at system-security/_Thread/ThreadComposer + /// and would have created threads under the wrong partition. + /// + private static string? ResolveUserHome(AccessService? accessSvc) + { + if (accessSvc is null) return null; + foreach (var candidate in new[] { accessSvc.CircuitContext?.ObjectId, accessSvc.Context?.ObjectId }) + { + if (!string.IsNullOrEmpty(candidate) + && candidate != WellKnownUsers.System + && !AccessService.LooksLikeHubPrincipal(candidate)) + return candidate; + } + return null; + } + + /// + /// Robust composer bring-up, run every time the chat opens / rebinds (). + /// The data-bound selectors area (embedded in the footer) needs the composer node to EXIST, so we + /// reliably create it with default content if absent via MeshQuery.CreateNode (never clobbers + /// — rejects an existing node), then open the one-way selection projection into + /// boundHarness/boundAgentPath/boundModelPath ONLY after the node is confirmed present. Fully + /// reactive (Subscribe, never await); we never read GetMeshNodeStream on a not-yet-present node + /// (that NotFound-storms the partition hub). Bad-data tolerant via ContentAs. + /// + private void EnsureComposer() + { + composerSubscription?.Dispose(); + composerDefaultsSubscription?.Dispose(); + boundHarness = boundAgentPath = boundModelPath = null; + if (string.IsNullOrEmpty(_templatePath)) + return; + var path = _templatePath; + + // Resolve the default composer selection BY ORDER — the Order=-1 (lowest-order) agent / model / + // harness from the live registries, never a hardcoded name/id (AgentPickerProjection.ObserveDefaultComposer). + // Take(1) + a short timeout so a brand-new composer seeds promptly; on timeout/empty we seed with + // empty selections (no invented fallback) — the picker still defaults to the Order=-1 item. + var picker = AgentPickerProjection.DerivePickerContext(_currentNavContext, initialContext); + composerDefaultsSubscription = AgentPickerProjection + .ObserveDefaultComposer(Hub, _userHome, AgentPickerProjection.PartitionOf(initialContext), + picker.ContextPath, picker.NodeTypePath) + .Take(1) + .Timeout(TimeSpan.FromSeconds(5)) + .Catch(_ => System.Reactive.Linq.Observable.Return(new MeshWeaver.AI.ThreadComposer())) + .Subscribe(defaults => InvokeAsync(() => CreateComposerWithDefaults(path, defaults))); + } + /// - /// Resolves the display name of a node at the given path via GetDataRequest. - /// Purely Post + RegisterCallback — no query, no await. + /// Creates the composer node with the order-resolved (heals users who + /// predate the onboarding seed), then fills any EMPTY selection + opens the live projection. + /// CreateNode registers a routable node (unlike GetMeshNodeStream(path).Update, which only patches + /// an EXISTING node and otherwise NotFound-storms the partition hub — that wedged the portal); + /// NodeAlreadyExists is benign (node present), so both paths proceed to fill + project. + /// + private void CreateComposerWithDefaults(string path, MeshWeaver.AI.ThreadComposer defaults) + { + if (_isDisposed || _templatePath != path) + return; + MeshQuery.CreateNode(MeshWeaver.Mesh.MeshNode.FromPath(path) with + { + NodeType = MeshWeaver.AI.ThreadComposerNodeType.NodeType, + Name = "Chat Input", + Content = defaults + }) + .Subscribe( + _ => InvokeAsync(() => FillDefaultsAndProject(path, defaults)), + ex => InvokeAsync(() => + { + Logger.LogDebug(ex, + "[ThreadChat:{InstanceId}] ensure composer node (benign if already exists) {Path}", _instanceId, path); + FillDefaultsAndProject(path, defaults); + })); + } + + /// + /// Fills EMPTY selection fields on the (now-present) composer node with + /// — coalesce only, NEVER clobbering a value the user already set — then opens the live projection. + /// One idempotent Update (skips the write when nothing's empty), on an existing node (no storm). + /// + private void FillDefaultsAndProject(string path, MeshWeaver.AI.ThreadComposer defaults) + { + if (_isDisposed || _templatePath != path) + return; + Hub.GetMeshNodeStream(path).Update(node => + { + var c = node.ContentAs(Hub.JsonSerializerOptions, Logger); + if (c is null) return node; // unreadable → leave alone, never clobber + var filled = c with + { + Harness = string.IsNullOrEmpty(c.Harness) ? defaults.Harness : c.Harness, + // Default (or RE-default) the agent: empty OR a background-generator agent → the + // conversational default. The latter migrates composers a pre-sort:order picker + // stamped with the first (utility) agent — the "ThreadNamer pre-selected" symptom. + AgentName = NeedsAgentDefault(c.AgentName) ? defaults.AgentName : c.AgentName, + ModelName = string.IsNullOrEmpty(c.ModelName) ? defaults.ModelName : c.ModelName, + }; + return filled == c ? node : node with { Content = filled }; + }).Subscribe( + _ => { }, + ex => Logger.LogDebug(ex, + "[ThreadChat:{InstanceId}] composer default-fill failed for {Path}", _instanceId, path)); + + OpenComposerProjection(path); + } + + /// + /// True when the composer's AgentName should be replaced with the conversational default: + /// it's empty, or it's a background-GENERATOR agent (modelTier:utility — ThreadNamer, + /// NodeInitializer, DescriptionWriter) that must never be the chat's selected agent (they emit + /// structured "Name:/Id:/Svg:" output, not conversation). A pre-sort:order picker could + /// default-to-first onto one of these and persist it; this clears that. + /// + private static bool NeedsAgentDefault(string? agentName) + { + if (string.IsNullOrEmpty(agentName)) return true; + var seg = agentName.Contains('/') ? agentName[(agentName.LastIndexOf('/') + 1)..] : agentName; + return seg is "ThreadNamer" or "NodeInitializer" or "DescriptionWriter"; + } + + /// + /// Opens the live one-way projection of the composer selection into bound* — called ONLY after the + /// node is confirmed present (created or already-exists; a thread node is the node being rendered), + /// so the read never hits a missing node. + /// discriminates by NodeType: a ThreadComposer node's own content, or the thread's embedded + /// Thread.Composer. + /// + private void OpenComposerProjection(string path) + { + if (_isDisposed || _templatePath != path) + return; + composerSubscription?.Dispose(); + composerSubscription = Hub.GetMeshNodeStream(path) + .Select(n => MeshWeaver.AI.ThreadComposerNodeType.ComposerOf(n, Hub.JsonSerializerOptions, Logger)) + .Where(c => c is not null) + .Subscribe( + c => InvokeAsync(() => + { + if (_isDisposed || _templatePath != path) return; + boundHarness = c!.Harness; + boundAgentPath = c.AgentName; + boundModelPath = c.ModelName; + StateHasChanged(); + }), + ex => Logger.LogDebug(ex, + "[ThreadChat:{InstanceId}] composer projection errored for {Path}", _instanceId, path)); + StateHasChanged(); + } + + /// + /// Writes a harness/agent/model selection onto the bound composer state — the imperative entry + /// point for the /agent and /model slash-commands and @-agent references (the visual pickers + /// write the node themselves). Values are node PATHS (or bare ids — execution normalizes); a + /// null arg leaves that field untouched. Targets the composer node out of a thread and the + /// thread's embedded Thread.Composer inside one. Bad-data tolerant: an unreadable node + /// is left alone, never clobbered. + /// + private void WriteComposerSelection(string? harness = null, string? agentPath = null, string? modelName = null) + { + if (string.IsNullOrEmpty(_templatePath)) + return; + Hub.GetMeshNodeStream(_templatePath).Update(node => + { + var existing = MeshWeaver.AI.ThreadComposerNodeType.ComposerOf(node, Hub.JsonSerializerOptions, Logger); + if (node?.Content is not null && existing is null) + return node!; + var updated = (existing ?? new MeshWeaver.AI.ThreadComposer()) with + { + Harness = harness ?? existing?.Harness, + AgentName = agentPath ?? existing?.AgentName, + ModelName = modelName ?? existing?.ModelName + }; + return MeshWeaver.AI.ThreadComposerNodeType.WithComposer( + node!, updated, Hub.JsonSerializerOptions, Logger); + }).Subscribe( + _ => { }, + ex => Logger.LogDebug(ex, + "[ThreadChat:{InstanceId}] composer selection write failed for {Path}", _instanceId, _templatePath)); + } + + /// + /// Resolves the display name of a node at the given path via the + /// timeout-bounded Hub.GetMeshNode helper. Bounded by an internal + /// 5 s deadline — for missing or unroutable paths the helper returns + /// null (instead of leaving a hub callback dangling indefinitely, which + /// is what the prior direct-Post + Observe shape did and what surfaced + /// in prod 2026-05-24 as the chat-page SSR hang when a satellite at the + /// requested path didn't exist). /// private void RequestDisplayName(string path, Action onResult) { @@ -191,35 +565,23 @@ private void RequestDisplayName(string path, Action onResult) try { - var delivery = Hub.Post(new GetDataRequest(new MeshNodeReference()), - o => o.WithTarget(new Address(path))); - - if (delivery == null) - { - onResult(null); - return; - } - - Hub.RegisterCallback((IMessageDelivery)delivery, response => - { - try - { - if (response is IMessageDelivery gdr && gdr.Message.Data is MeshNode node) - onResult(node.Name ?? node.Id); - else + Hub.GetMeshNode(path, TimeSpan.FromSeconds(5)) + .Subscribe( + node => + { + if (_isDisposed) return; + onResult(node?.Name ?? node?.Id); + }, + ex => + { + if (!_isDisposed) + Logger.LogDebug(ex, "Error reading display name for {Path}", path); onResult(null); - } - catch (Exception ex) when (!_isDisposed) - { - Logger.LogDebug(ex, "Error reading display name for {Path}", path); - onResult(null); - } - return response; - }); + }); } catch (Exception ex) when (!_isDisposed) { - Logger.LogDebug(ex, "Error posting GetDataRequest for {Path}", path); + Logger.LogDebug(ex, "Error reading display name for {Path}", path); onResult(null); } } @@ -233,28 +595,14 @@ protected override void BindData() private void InitializeAgentAndModelSelections() { - // Load available models from DI-registered factories - var factories = ChatClientFactories.ToList(); - Logger.LogInformation("[ThreadChat:{InstanceId}] IChatClientFactory instances resolved: {Count}", _instanceId, factories.Count); - - availableModels = factories - .OrderBy(f => f.Order) - .SelectMany(f => f.Models.Select(m => new ModelInfo - { - Name = m, - Provider = f.Name, - Order = f.Order - })) - .ToList(); - - Logger.LogInformation("[ThreadChat:{InstanceId}] Available models ({Count}): [{Models}]", - _instanceId, availableModels.Count, string.Join(", ", availableModels.Select(m => $"{m.Name} ({m.Provider})"))); - - // Subscribe to agent MeshNodes reactively + // The chat view loads NO model/harness lists: /agent, /model and /harness are generic + // node-pick commands that query the mesh on demand (OpenPicker). The only subscription + // kept is the agent snapshot, used for @-reference agent detection. SubscribeToAgentNodes(); } - // Merged agent nodes from multiple reactive queries, keyed by path + // Agent nodes from the reactive query, keyed by node path — used ONLY for @-reference + // agent detection (the /agent, /model, /harness commands query the mesh on demand). private readonly Dictionary _agentsByPath = new(); private void SubscribeToAgentNodes() @@ -262,105 +610,53 @@ private void SubscribeToAgentNodes() agentSubscription?.Dispose(); _agentsByPath.Clear(); - var subscriptions = new List(); - - // Query 1: Agents from the Agent namespace - var agentNsRequest = MeshQueryRequest.FromQuery("namespace:Agent nodeType:Agent"); - subscriptions.Add(MeshQuery.ObserveQuery(agentNsRequest) - .Subscribe(change => InvokeAsync(() => OnAgentQueryChange(change)))); - - // Query 2: Agents along the current context path's ancestor chain - if (!string.IsNullOrEmpty(initialContext)) + var workspace = Hub.ServiceProvider.GetService(); + if (workspace == null) { - var contextRequest = MeshQueryRequest.FromQuery($"namespace:{initialContext} nodeType:Agent scope:selfAndAncestors"); - subscriptions.Add(MeshQuery.ObserveQuery(contextRequest) - .Subscribe(change => InvokeAsync(() => OnAgentQueryChange(change)))); + Logger.LogWarning("[ThreadChat:{InstanceId}] No IWorkspace — synced agent/model query skipped", + _instanceId); + return; } - agentSubscription = new System.Reactive.Disposables.CompositeDisposable(subscriptions); + // Agent snapshot subscription — kept ONLY for @-reference agent detection + // (OnCompletionItemAccepted decides whether an @-ref is an agent). The /agent, + // /model and /harness commands no longer load lists here: they go through the + // GENERIC node picker (OpenPicker), which queries the mesh on demand. + agentSubscription = AgentPickerProjection + .ObserveAgents(Hub, _userHome, AgentPickerProjection.PartitionOf(initialContext)) + .Subscribe(agents => InvokeAsync(() => OnAgentList(agents))); } - private void OnAgentQueryChange(QueryResultChange change) + /// + /// Receives the full path-keyed snapshot from + /// + /// and forks each node into agent / model bucket. Snapshot semantics + /// are simple — every emission IS the complete current set, so we + /// rebuild from scratch each time (no delta tracking, no flashing + /// empty between queries' Initial events). + /// + private void OnAgentList(IReadOnlyList agents) { - if (change.ChangeType == QueryChangeType.Initial || - change.ChangeType == QueryChangeType.Reset || - change.ChangeType == QueryChangeType.Added || - change.ChangeType == QueryChangeType.Updated) - { - foreach (var node in change.Items) - { - var info = ToAgentDisplayInfo(node); - if (info != null && node.Path != null) - _agentsByPath[node.Path] = info; - } - } - else if (change.ChangeType == QueryChangeType.Removed) - { - foreach (var node in change.Items) - { - if (node.Path != null) - _agentsByPath.Remove(node.Path); - } - } + // Drop background-generator agents (modelTier:utility — ThreadNamer, NodeInitializer, + // DescriptionWriter) from every CONVERSATIONAL surface: the /agent picker, @-reference + // selection, and the inline agent widget. They're invoked programmatically and emit + // structured "Name:/Id:/Svg:" output, so e.g. ThreadNamer must never answer a user's + // "hi". The projection itself keeps them (the generators resolve them); we filter here, + // at the chat UI, so generators are unaffected. + var conversational = agents.Where(a => !AgentPickerProjection.IsUtilityAgent(a)).ToList(); - agentDisplayInfos = _agentsByPath.Values - .OrderBy(a => a.Order) - .ThenBy(a => a.Name) - .ToList(); + Logger.LogDebug("[ThreadChat:{InstanceId}] Agents received: count={Count} (conversational={Conv})", + _instanceId, agents.Count, conversational.Count); - // Preserve current selection if still valid, otherwise select first - if (selectedAgentInfo != null && - agentDisplayInfos.Any(a => a.Path == selectedAgentInfo.Path)) - { - selectedAgentInfo = agentDisplayInfos.First(a => a.Path == selectedAgentInfo.Path); - } - else - { - selectedAgentInfo = agentDisplayInfos.FirstOrDefault(); - } - - // Set model from agent's preferred model - if (selectedAgentInfo != null) - selectedModelInfo = GetPreferredModelInfoForAgent(selectedAgentInfo.Name); - else - selectedModelInfo = availableModels.FirstOrDefault(); + _agentsByPath.Clear(); + foreach (var a in conversational) + if (!string.IsNullOrEmpty(a.Path)) + _agentsByPath[a.Path] = a; + agentDisplayInfos = conversational; StateHasChanged(); } - private static AgentDisplayInfo? ToAgentDisplayInfo(MeshNode node) - { - if (node.Content is not AgentConfiguration config) - return null; - return new AgentDisplayInfo - { - Name = config.DisplayName ?? config.Id, - Path = node.Path, - Description = config.Description ?? "", - GroupName = config.GroupName, - Order = config.Order, - Icon = config.Icon, - CustomIconSvg = config.CustomIconSvg, - AgentConfiguration = config - }; - } - - private ModelInfo? GetPreferredModelInfoForAgent(string agentName) - { - if (agentModelPreferences.TryGetValue(agentName, out var preferredModelName)) - return availableModels.FirstOrDefault(m => m.Name == preferredModelName); - - var agentConfig = agentDisplayInfos.FirstOrDefault(a => a.Name == agentName)?.AgentConfiguration; - if (!string.IsNullOrEmpty(agentConfig?.PreferredModel)) - { - var configuredModel = availableModels.FirstOrDefault(m => m.Name == agentConfig.PreferredModel); - if (configuredModel != null) - return configuredModel; - } - - return availableModels.FirstOrDefault(); - } - private void SendMessage() { if (_isDisposed) @@ -374,6 +670,10 @@ private void SendMessage() private void SubmitMessageCore() { + // 🔬 Track submit → thread-created → first-message-visible timings. + // Logs at Information level under channel `ChatPerf` so you can grep + // a single resource log for "[ChatPerf]" and see step-by-step elapsed. + var perfSw = System.Diagnostics.Stopwatch.StartNew(); try { if (_isDisposed) @@ -382,6 +682,41 @@ private void SubmitMessageCore() // Use MessageText (updated via Monaco ValueChanged binding) — no blocking Monaco read. var userMessageText = MessageText; + // Slash-skill interception: parse leading "/word args" via ChatPreParser. If it resolves to + // a nodeType:Skill, run its action and short-circuit (don't post to the agent). + if (!string.IsNullOrWhiteSpace(userMessageText)) + { + var parsed = ChatParser.Parse(userMessageText); + if (parsed.Command != null) + { + // Under a CLI harness (Claude Code / Copilot) FORWARD slash commands 1:1 to the + // harness as the message — with two exceptions that MeshWeaver MUST own: + // • /harness — the runtime switch (so the user is never stuck in a CLI harness); + // • the harness's OWN Commands (/login, /logout) — these drive the inline Connect + // flow (`claude setup-token` + store token). They CANNOT be forwarded: the CLI + // can't interactively authenticate from a piped prompt, so forwarding /login + // just yields an endless "Not logged in" loop. + // Under the MeshWeaver harness, intercept every slash-skill as before. + var harness = ActiveHarness(); + var isRuntimeSwitch = string.Equals(parsed.Command.Name, "harness", StringComparison.OrdinalIgnoreCase); + var isHarnessOwnedCommand = harness?.Commands.Any( + c => string.Equals(c.Name, parsed.Command.Name, StringComparison.OrdinalIgnoreCase)) == true; + if (harness is null || isRuntimeSwitch || isHarnessOwnedCommand) + { + _ = HandleSlashCommandAsync(parsed.Command); + // Clear the input + bail — submissionHandler.TryBeginSubmit + // hasn't been called yet, so no need to release. + MessageText = null; + if (monacoEditor != null) + _ = ClearMonacoAsync(); + StateHasChanged(); + return; + } + // else: CLI harness + a command the harness does NOT own → fall through, so the raw + // "/command" text is submitted to the harness as the message (forwarded 1:1). + } + } + // Attempt to begin submission — rejects empty text and concurrent submissions if (!submissionHandler.TryBeginSubmit(userMessageText)) return; @@ -402,72 +737,683 @@ private void SubmitMessageCore() var isCompact = ViewModel.HideEmptyState; var capturedAttachments = attachments.Select(a => a.Path).ToList(); - var ns = !string.IsNullOrEmpty(NavigationService.CurrentNamespace) - ? NavigationService.CurrentNamespace - : !string.IsNullOrEmpty(initialContext) - ? initialContext + // A thread must live in a REAL partition (a space or the user's home), NEVER a rogue/reserved + // ROUTE partition (login, welcome, …): that partition has no write policy, so StartThread there + // is denied → onError → no thread → the side-panel chat tears down (the "login" symptom). Strip + // a reserved nav-namespace / context and fall back to the user's own namespace. + var navNs = MeshWeaver.AI.AgentPickerProjection.IsReservedPartition(NavigationService.CurrentNamespace) + ? null : NavigationService.CurrentNamespace; + var safeContext = MeshWeaver.AI.AgentPickerProjection.IsReservedPartition(initialContext) + ? null : initialContext; + var ns = !string.IsNullOrEmpty(navNs) + ? navNs + : !string.IsNullOrEmpty(safeContext) + ? safeContext : !string.IsNullOrEmpty(createdBy) - ? $"User/{createdBy}" - : "User"; + ? createdBy + : ""; + + Action onError = err => InvokeAsync(() => + { + if (_isDisposed) return; + Logger.LogWarning("[ThreadChat:{InstanceId}] Submit failed: {Error}", _instanceId, err); + showSubmissionProgress = false; + lastSubmittedText = null; + submissionHandler.ForceRelease(); + StateHasChanged(); + }); - var ctx = new SubmitContext + if (string.IsNullOrEmpty(threadPath)) { - Hub = Hub, - ThreadPath = string.IsNullOrEmpty(threadPath) ? null : threadPath, - Namespace = ns, - UserText = userMessageText!, - AgentName = selectedAgentInfo?.Name, - ModelName = selectedModelInfo?.Name, - ContextPath = initialContext, - Attachments = capturedAttachments, - CreatedBy = createdBy, - AuthorName = authorName, - OnError = err => InvokeAsync(() => + showSubmissionProgress = true; // step 2: show progress in the composer immediately on submit + lastSubmittedText = userMessageText; // ...and echo the submitted message under the progress + Logger.LogInformation("[Chat] Creating thread + submitting message"); + // Selections flow as the picked node PATHS — execution normalizes to ids + // at its boundary (SelectionId.IdOf). The composer snapshot is copied onto + // the created thread (Thread.Composer) so the in-thread selectors continue + // the same selection. + Hub.StartThread( + namespacePath: ns, + userText: userMessageText!, + agentName: boundAgentPath, + modelName: boundModelPath, + contextPath: safeContext, + attachments: capturedAttachments, + createdBy: createdBy, + authorName: authorName, + harness: boundHarness ?? Harnesses.MeshWeaver, + composer: new MeshWeaver.AI.ThreadComposer + { + Harness = boundHarness, + AgentName = boundAgentPath, + ModelName = boundModelPath, + ContextPath = safeContext + }, + onCreated: node => + { + var path = node.Path; + if (string.IsNullOrEmpty(path)) { onError("Thread created with no path"); return; } + // 🚨 Redirect ONLY once the thread node is actually READABLE on its own stream. + // Navigating on the bare CreateNode ack races the thread's per-node hub + // activation: the target page subscribes to a not-yet-ready node, the render + // throws, and the whole side panel blanks (the "blackout"). Subscribing here is + // "redirect in .Subscribe() when the thread is written" — no miss. + Hub.GetWorkspace().GetMeshNodeStream(path) + .Where(n => n is not null) + .Take(1) + .Timeout(TimeSpan.FromSeconds(10)) + .Subscribe( + ready => InvokeAsync(() => + { + if (_isDisposed) return; + Logger.LogInformation( + "[Chat] Thread created+readable path={Path} elapsed={Ms}ms", + path, perfSw.ElapsedMilliseconds); + threadPath = path; + threadName = ready!.Name; + UpdateSidePanelTitle(); + if (isCompact) + NavigationManager.NavigateTo($"/{path}"); + else + SidePanelState.SetContentPath(path); + showSubmissionProgress = false; + lastSubmittedText = null; + StateHasChanged(); + }), + ex => onError(ex.Message)); + }, + onError: onError); + } + else + { + Logger.LogInformation("[Chat] Submitting to thread {Thread}", threadPath); + // Drain through the thread's embedded composer (Thread.Composer): the + // harness/agent/model selection comes from the composer the selectors area + // is data-bound to, the typed text + live context/attachments are passed + // explicitly, and the composer empties itself in the same atomic update. + Hub.SubmitComposer( + threadPath: threadPath, + userText: userMessageText!, + contextPath: safeContext, + attachments: capturedAttachments, + createdBy: createdBy, + authorName: authorName, + onError: onError); + } + + // Claude-Code-style queue: input stays enabled so the user can keep typing while + // previous submissions are being processed by the thread. The server watcher + // batches unprocessed user messages into a single round. + submissionHandler.ForceRelease(); + StateHasChanged(); + } + catch (Exception ex) + { + Logger.LogError(ex, "[ThreadChat:{InstanceId}] SubmitMessageCore failed", _instanceId); + } + } + + /// + /// Dispatches a parsed leading "/word args" — harness-owned commands (/login, /logout) first, then + /// a nodeType:Skill resolved by slash word (). Updates + /// for the breadcrumb. No await on hub calls — skill actions are + /// in-process GUI logic (open a picker, load the content window) or reactive subscriptions. + /// + private async Task HandleSlashCommandAsync(ParsedCommand parsedCommand) + { + // Harness-owned commands take priority: when a non-MeshWeaver harness is active, its own + // slash-commands (/login, /logout) route to the harness itself — NOT to MeshWeaver's + // /agent /model node-pickers. /harness and everything else fall through below. + if (TryHandleHarnessCommand(parsedCommand)) + { + await InvokeAsync(StateHasChanged); + return; + } + + // Otherwise resolve a nodeType:Skill by slash word and run its action (Pick → combobox, + // OpenContent → content window, …). Skills are declarative mesh nodes (the built-in + // /agent /model /harness + any Space/NodeType/user-defined one) — there is no C# registry. + ResolveSkillNodeAndRun(parsedCommand); + await InvokeAsync(StateHasChanged); + } + + /// + /// Resolves a nodeType:Skill mesh node by its slash word and runs its action. Built-in + /// skills (/agent, /model, /harness — Pick behaviours shipped by + /// ) AND any Space/NodeType/user-defined skill + /// resolve here, with namespace inheritance (). + /// Reactive: queries the mesh once, then runs the matched skill or reports "unknown command". + /// + private void ResolveSkillNodeAndRun(ParsedCommand parsed) + { + var workspace = Hub.ServiceProvider.GetService(); + if (workspace is null) + { + ShowSkillStatus($"Unknown command: /{parsed.Name}", true); + return; + } + + var queries = MeshWeaver.AI.SkillNodeType.SkillQueries(initialContext, _userHome); + AgentPickerProjection.ObserveSnapshot(workspace, Hub, $"skills|{initialContext}|{_userHome}", queries) + .Take(1) + .Timeout(TimeSpan.FromSeconds(5)) + .Subscribe( + snapshot => InvokeAsync(() => + { + var match = MeshWeaver.AI.SkillNodeType.ProjectSkills(snapshot, Hub.JsonSerializerOptions) + .FirstOrDefault(s => string.Equals(s.Id, parsed.Name, StringComparison.OrdinalIgnoreCase)); + if (match is null) + { + ShowSkillStatus($"Unknown command: /{parsed.Name}", true); + return; + } + RunSkill(match, parsed); + }), + _ => InvokeAsync(() => ShowSkillStatus($"Unknown command: /{parsed.Name}", true))); + } + + /// + /// Runs a resolved skill's action: Pick → combobox (write the pick to the composer); + /// OpenContent → load into the content window; instruction/Connect skills have no inline + /// chat behaviour (mounted to the CLI harnesses / advertised to the agent). + /// + private void RunSkill(MeshWeaver.AI.SkillInfo skill, ParsedCommand parsed) + { + var term = parsed.Arguments.Length == 0 ? null : LastSegment(parsed.RawArguments.Trim()); + var action = skill.Definition.Action; + switch (action?.Kind) + { + case MeshWeaver.AI.SkillActionKind.Pick: + var picker = skill.ToPickerRequest(term); + if (picker is not null) + { + lastCommandStatus = null; + lastCommandStatusIsError = false; + OpenPicker(picker); + } + break; + case MeshWeaver.AI.SkillActionKind.OpenContent: + // Load the manual/document into the content window (side panel) and make it visible. + var path = string.IsNullOrEmpty(action.ContentPath) ? skill.Path : action.ContentPath; + if (!string.IsNullOrEmpty(path)) + { + SidePanelState.SetTitle(skill.Name ?? skill.Id); + SidePanelState.OpenWithContent(path); + } + ShowSkillStatus(skill.Name ?? skill.Id, false); + break; + default: + ShowSkillStatus(skill.Description ?? $"/{skill.Id}", false); + break; + } + } + + /// Surface a status / error line under the chat input (replaces the old command status callback). + private void ShowSkillStatus(string message, bool isError) + { + pendingPicker = null; + pickerNodes = []; + lastCommandStatus = message; + lastCommandStatusIsError = isError; + } + + // ─── Harness-owned commands + inline Connect (login) ─── + // When a non-MeshWeaver harness is active, ITS slash-commands (/login, /logout) route to the + // harness, not to MeshWeaver's node-pickers. /login drives the per-user Connect flow inline + // (auth URL + paste-code), reusing ConnectSessionManager — the same flow Settings → Models uses. + private MeshWeaver.AI.Connect.ConnectProvider? _connectProvider; + private MeshWeaver.AI.IHarness? _connectHarness; + private string? _connectHarnessLabel; + private MeshWeaver.AI.Connect.ConnectStatus? _connectStatus; + private string _connectCode = ""; + private bool _connectBusy; + private IDisposable? _connectSub; + + /// The active non-MeshWeaver harness (resolved from the composer's harness selection), or + /// null when MeshWeaver is active. Drives harness-owned command dispatch + autocomplete. + private MeshWeaver.AI.IHarness? ActiveHarness() + { + var harness = MeshWeaver.AI.HarnessNodeType.ResolveHarness(Hub.ServiceProvider, boundHarness); + return harness is null + || string.Equals(harness.Id, MeshWeaver.AI.Harnesses.MeshWeaver, StringComparison.OrdinalIgnoreCase) + ? null : harness; + } + + /// + /// Routes a parsed slash-command to the active harness when the harness owns a command of that + /// name (/login, /logout). Returns true when handled (so the caller short-circuits the MeshWeaver + /// command path). /harness, /help and everything else fall through. + /// + private bool TryHandleHarnessCommand(ParsedCommand parsed) + { + var harness = ActiveHarness(); + var cmd = harness?.Commands.FirstOrDefault( + c => string.Equals(c.Name, parsed.Name, StringComparison.OrdinalIgnoreCase)); + if (harness is null || cmd is null) + return false; + switch (cmd.Kind) + { + case MeshWeaver.AI.HarnessCommandKind.Connect: StartHarnessConnect(harness); break; + case MeshWeaver.AI.HarnessCommandKind.Disconnect: DisconnectHarness(harness); break; + } + return true; + } + + /// + /// Begins the harness's per-user login (Connect) and renders the challenge inline (replacing the + /// node picker). Reactive: subscribe ConnectSessionManager.StartConnect; each ConnectStatus drives + /// the widget. For Claude (paste-code) the user pastes the code via ; + /// Copilot (device-flow) auto-polls to Connected. + /// + private void StartHarnessConnect(MeshWeaver.AI.IHarness harness) + { + var provider = harness.AuthProvider; + if (provider is null) + return; + var sessionManager = Hub.ServiceProvider.GetService(); + if (sessionManager is null || !sessionManager.Supports(provider.Value)) + { + lastCommandStatus = "Login is not available in this deployment."; + lastCommandStatusIsError = true; + return; + } + if (string.IsNullOrEmpty(_userHome)) + { + lastCommandStatus = "No user identity for login."; + lastCommandStatusIsError = true; + return; + } + // Mutually exclusive with the node picker. + pendingPicker = null; + pickerNodes = []; + lastCommandStatus = null; + _connectHarness = harness; + _connectProvider = provider; + _connectHarnessLabel = harness.Definition.DisplayName ?? harness.Id; + _connectStatus = null; // "Starting login…" until the first emission + _connectCode = ""; + _connectBusy = false; + _connectSub?.Dispose(); + var ownerPath = _userHome!; + var configDir = ResolveHarnessConfigDir(provider.Value); + _connectSub = sessionManager.StartConnect(ownerPath, provider.Value, configDir) + .Subscribe( + status => InvokeAsync(() => { if (_isDisposed) return; - Logger.LogWarning("[ThreadChat:{InstanceId}] Submit failed: {Error}", _instanceId, err); - showSubmissionProgress = false; - submissionHandler.ForceRelease(); + _connectStatus = status; + _connectBusy = false; StateHasChanged(); + // Login finished — auto-dismiss the widget (briefly showing "✓ Connected") so the + // user doesn't have to click ✕. + if (status is MeshWeaver.AI.Connect.ConnectStatus.Connected) + ScheduleConnectAutoClose(); }), - OnThreadCreated = node => InvokeAsync(() => + ex => InvokeAsync(() => { if (_isDisposed) return; - threadPath = node.Path; - threadName = node.Name; - UpdateSidePanelTitle(); - if (isCompact && !string.IsNullOrEmpty(node.Path)) - { - NavigationManager.NavigateTo($"/{node.Path}"); - } - else if (!string.IsNullOrEmpty(node.Path)) - { - SidePanelState.SetContentPath(node.Path); - } - showSubmissionProgress = false; + _connectStatus = new MeshWeaver.AI.Connect.ConnectStatus.Error(ex.Message); + _connectBusy = false; StateHasChanged(); - }) - }; + })); + StateHasChanged(); + } - if (string.IsNullOrEmpty(threadPath)) + /// Submits the pasted code for a Claude paste-code login and drives it to completion. + private void SubmitConnectCode() + { + if (_connectProvider is not { } provider || string.IsNullOrWhiteSpace(_connectCode) || string.IsNullOrEmpty(_userHome)) + return; + var sessionManager = Hub.ServiceProvider.GetService(); + if (sessionManager is null) + return; + var ownerPath = _userHome!; + var code = _connectCode.Trim(); + _connectBusy = true; + _connectSub?.Dispose(); + _connectSub = sessionManager.SubmitCode(ownerPath, provider, code) + .Subscribe( + status => InvokeAsync(() => + { + if (_isDisposed) return; + _connectStatus = status; + _connectBusy = false; + StateHasChanged(); + // Login finished — auto-dismiss the widget (briefly showing "✓ Connected") so the + // user doesn't have to click ✕. + if (status is MeshWeaver.AI.Connect.ConnectStatus.Connected) + ScheduleConnectAutoClose(); + }), + ex => InvokeAsync(() => + { + if (_isDisposed) return; + _connectStatus = new MeshWeaver.AI.Connect.ConnectStatus.Error(ex.Message); + _connectBusy = false; + StateHasChanged(); + })); + StateHasChanged(); + } + + /// Enter in the paste-code field submits the code. + private void OnConnectInputKeyDown(Microsoft.AspNetCore.Components.Web.KeyboardEventArgs e) + { + if (e.Key == "Enter") + SubmitConnectCode(); + } + + /// Dismiss the Connect widget and tear down any live login session. + private void CancelConnect() + { + if (_connectProvider is { } p && !string.IsNullOrEmpty(_userHome)) + Hub.ServiceProvider.GetService()?.Cancel(_userHome!, p); + _connectSub?.Dispose(); + _connectSub = null; + _connectProvider = null; + _connectHarness = null; + _connectStatus = null; + _connectCode = ""; + _connectBusy = false; + StateHasChanged(); + } + + /// Reactively dismiss the Connect widget ~1.2s after a successful login (no Task.Delay) — + /// shows "✓ Connected" briefly, then closes so the user never has to click ✕. The timer lives in + /// so a dispose / new connect cancels a pending auto-close. + private void ScheduleConnectAutoClose() + { + _connectSub?.Dispose(); + _connectSub = System.Reactive.Linq.Observable.Timer(TimeSpan.FromSeconds(1.2)) + .Subscribe(_ => InvokeAsync(() => { - showSubmissionProgress = isCompact; - ThreadSubmission.CreateThreadAndSubmit(ctx); + if (_isDisposed) return; + CloseConnectWidget(); + })); + } + + /// Clears the Connect widget UI state WITHOUT cancelling the session — used after a + /// successful login (the session already completed and stored the token); mirrors + /// minus the sessionManager.Cancel call. + private void CloseConnectWidget() + { + _connectSub?.Dispose(); + _connectSub = null; + _connectProvider = null; + _connectHarness = null; + _connectStatus = null; + _connectCode = ""; + _connectBusy = false; + StateHasChanged(); + } + + /// + /// Logs the user out of a CLI harness: forgets the stored per-user subscription token (the + /// ModelProvider node the harness reads) AND clears the CLI's own cached credentials in the + /// per-user config dir, so the next round is genuinely logged out. + /// + private void DisconnectHarness(MeshWeaver.AI.IHarness harness) + { + var provider = harness.AuthProvider; + if (provider is null || string.IsNullOrEmpty(_userHome)) + return; + Hub.ServiceProvider.GetService()?.Cancel(_userHome!, provider.Value); + + var providerPath = $"{MeshWeaver.AI.ModelProviderNodeType.UserNamespacePath(_userHome!)}/{harness.Id}"; + MeshQuery.DeleteNode(providerPath).Subscribe( + _ => { }, + ex => Logger.LogDebug(ex, "[ThreadChat:{InstanceId}] logout: delete provider node {Path} failed", + _instanceId, providerPath)); + + // Clear the CLI's own cached credentials on the shared volume (best-effort, server-side). + var configDir = ResolveHarnessConfigDir(provider.Value); + if (!string.IsNullOrEmpty(configDir)) + { + try + { + var creds = System.IO.Path.Combine(configDir, ".credentials.json"); + if (System.IO.File.Exists(creds)) System.IO.File.Delete(creds); } - else + catch (Exception ex) { - ThreadSubmission.Submit(ctx); + Logger.LogDebug(ex, "[ThreadChat:{InstanceId}] logout: clear credentials failed", _instanceId); } + } + + lastCommandStatus = $"Logged out of {harness.Definition.DisplayName ?? harness.Id}."; + lastCommandStatusIsError = false; + } + + /// + /// The per-user CLI config dir the login writes to — IDENTICAL to the dir the harness reads + /// ({ClaudeCode:ConfigDirRoot}/{userId}/.claude), so a completed login authenticates the + /// harness's CLI. Null for Copilot (device-flow uses the SDK's own auth state, not a creds file). + /// + private string? ResolveHarnessConfigDir(MeshWeaver.AI.Connect.ConnectProvider provider) + { + if (provider != MeshWeaver.AI.Connect.ConnectProvider.ClaudeCode || string.IsNullOrEmpty(_userHome)) + return null; + var root = Configuration["ClaudeCode:ConfigDirRoot"]?.TrimEnd('/', '\\'); + return string.IsNullOrEmpty(root) ? null : $"{root}/{_userHome}/.claude"; + } + + private void DismissWidget() + { + pendingPicker = null; + pickerNodes = []; + StateHasChanged(); + } + + /// + /// When the command picker has just opened, move keyboard focus from the Monaco editor onto the + /// picker widget so ↑/↓/Enter/Escape operate the list (Monaco would otherwise swallow the arrows). + /// + protected override async Task OnAfterRenderAsync(bool firstRender) + { + await base.OnAfterRenderAsync(firstRender); + if (_focusPickerOnRender && pendingPicker is not null) + { + _focusPickerOnRender = false; + try { await _pickerWidget.FocusAsync(); } + catch { /* widget may not be in the DOM yet — harmless */ } + } + } + + /// + /// Generic node picker for any : queries the mesh for the + /// command's node query and either auto-selects an exact name match (when the user typed an + /// argument, e.g. /model gpt-4o) or shows the picker list. One render path serves every + /// node-pick command — agent, model, harness, and any module-defined one. + /// + private void OpenPicker(NodePickerRequest picker) + { + var workspace = Hub.ServiceProvider.GetService(); + if (workspace == null) + return; + + // 🚨 Capture the circuit USER NOW, on the circuit thread where CircuitContext is live. + // The picker query subscribes later (RunPicker → ObserveSnapshot), in the agent/model + // branch on the NavigationContext→InvokeAsync hop where the ambient AccessContext may be + // cleared or leaked. Running the synced query context-null makes WrapWithPerUserRls BYPASS + // → the combobox shows agents/models the user has no Read on (wrong access rights). + // RunPicker re-establishes THIS captured user at its subscribe so RLS filters correctly. + // Prefer the durable CircuitContext; reject a leaked system/hub principal (not a real user). + var accessService = Hub.ServiceProvider.GetService(); + var ambientCtx = accessService?.CircuitContext ?? accessService?.Context; + var pickerUser = ambientCtx is not null + && !string.IsNullOrEmpty(ambientCtx.ObjectId) + && ambientCtx.ObjectId != WellKnownUsers.System + && !AccessService.LooksLikeHubPrincipal(ambientCtx.ObjectId) + ? ambientCtx : null; + + var field = picker.ComposerField; + var isAgent = string.Equals(field, nameof(MeshWeaver.AI.ThreadComposer.AgentName), StringComparison.OrdinalIgnoreCase); + var isModel = string.Equals(field, nameof(MeshWeaver.AI.ThreadComposer.ModelName), StringComparison.OrdinalIgnoreCase); + + // Harness / custom Command pickers carry no context-scoped union — run them + // straight off their declared query (no navigation read needed). + if (!isAgent && !isModel) + { + RunPicker(workspace, picker, new[] { picker.Query }, + $"picker|{field}|{picker.Query}", accessService, pickerUser); + return; + } + + // Agent and model pickers must surface NOT ONLY the built-in catalog but also the ones declared + // in the CURRENT context's namespace (+ ancestors) AND the context node's NodeType namespace + // (+ ancestors) — the SAME context-scoped query union AgentChatClient resolves agents/models + // from at execution time. AgentPickerProjection.BuildAgentQueries / BuildModelQueries is the + // single source of truth for that union (built-in + path:{current} ancestors + + // namespace:{nodeType} selfAndAncestors); inlining the single global `namespace:Agent` query + // here is exactly why a Space's own agent/model never appeared in the picker. + // + // 🚨 TIMING-SAFE: the navigation context resolves ASYNCHRONOUSLY, so the stale + // `_currentNavContext` field (and the seeded `initialContext`) are frequently NULL in the + // floating side-panel chat — collapsing the union to the built-in query only (the atioz + // "Space agent missing from /agent" bug). We instead read the LATEST RESOLVED context off + // NavigationService.NavigationContext (ReplaySubject(1) → the last value replays + // immediately). Take(1) after a short Timeout so a still-resolving context can't hang the + // picker; the timeout/error branch falls back to the seeded initialContext (never null → + // never block). AgentPickerProjection.DerivePickerContext is the single source of truth for + // turning that resolved context into (currentPath, nodeTypePath). + NavigationService.NavigationContext + .Select(ctx => AgentPickerProjection.DerivePickerContext(ctx, initialContext)) + .Take(1) + .Timeout(TimeSpan.FromSeconds(2)) + .Catch(_ => + System.Reactive.Linq.Observable.Return( + AgentPickerProjection.DerivePickerContext(null, initialContext))) + .Subscribe(pc => InvokeAsync(() => + { + if (_isDisposed) return; + var queries = isAgent + ? AgentPickerProjection.BuildAgentQueries(_userHome, AgentPickerProjection.PartitionOf(pc.ContextPath)) + : AgentPickerProjection.BuildModelQueries(pc.ContextPath, pc.NodeTypePath, userPath: _userHome); + var cacheKey = $"picker|{field}|{pc.ContextPath}|{pc.NodeTypePath}|{string.Join("|", queries)}"; + RunPicker(workspace, picker, queries, cacheKey, accessService, pickerUser); + })); + } + + /// + /// Runs the resolved picker query union: snapshots the mesh once, orders by the node's + /// universal Order field (then Name), auto-selects an exact term match or shows the list. + /// Shared by the context-scoped agent/model branch and the declared-query harness/custom branch. + /// + private void RunPicker(IWorkspace workspace, NodePickerRequest picker, string[] queries, string cacheKey, + AccessService? accessService, AccessContext? pickerUser) + { + // 🚨 Run the picker query under the CAPTURED circuit user, re-established HERE at the + // subscribe via Observable.Using(SwitchAccessContext). ObserveSnapshot subscribes on the + // NavigationContext→InvokeAsync hop where the ambient AccessContext may be cleared/leaked; + // running the synced query context-null makes WrapWithPerUserRls BYPASS → the combobox + // surfaces agents/models the user has no Read on (wrong access rights). The scope flows + // through ObserveSnapshot's IIoPool hops (the pool carries the AsyncLocal), so RLS filters + // the picker to exactly what this user can read. + Observable.Using( + () => pickerUser is not null && accessService is not null + ? accessService.SwitchAccessContext(pickerUser) + : (IDisposable)System.Reactive.Disposables.Disposable.Empty, + _ => AgentPickerProjection.ObserveSnapshot(workspace, Hub, cacheKey, queries)) + .Take(1) + .Subscribe(snapshot => InvokeAsync(() => + { + if (_isDisposed) return; + // Order by the node's universal Order field (then Name) so the picker leads with the + // catalog head — Assistant (order:-1) for agents, the flagship model for models. This + // is NOT command-specific logic: it's the generic "order nodes by Order" every picker + // wants. (The query's `sort:order` is lost when the synced-query snapshot re-buckets by + // path into a dict, so the order must be re-applied here on the node data.) + var nodes = snapshot.Where(n => !string.IsNullOrEmpty(n.Path)) + .OrderBy(n => n.Order ?? 0) + .ThenBy(n => n.Name ?? n.Id, StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (!string.IsNullOrEmpty(picker.SearchTerm)) + { + // Exact name/last-segment match → switch immediately, no visible picker. + var exact = nodes.FirstOrDefault(n => PickerNodeMatches(n, picker.SearchTerm, exact: true)); + if (exact != null) + { + SelectFromPicker(picker, exact); + return; + } + // Otherwise pre-filter the list to the term. + nodes = nodes.Where(n => PickerNodeMatches(n, picker.SearchTerm!, exact: false)).ToList(); + } + + pendingPicker = picker; + pickerNodes = nodes; + _pickerHighlight = 0; + _focusPickerOnRender = true; // move focus off Monaco onto the widget so ↑/↓ reach it + lastCommandStatus = null; + StateHasChanged(); + })); + } - // Claude-Code-style queue: input stays enabled so the user can keep typing while - // previous submissions are being processed by the thread. The server watcher - // batches unprocessed user messages into a single round. - submissionHandler.ForceRelease(); - StateHasChanged(); + /// + /// Keyboard navigation of the command picker list: ↑/↓ move the highlight (wrapping), Enter commits + /// the highlighted node, Escape dismisses. Fires on the focused widget (see ), + /// so the arrow keys are handled here instead of by the Monaco editor. + /// + private void OnPickerKeyDown(Microsoft.AspNetCore.Components.Web.KeyboardEventArgs e) + { + if (pendingPicker is null || pickerNodes.Count == 0) + return; + + switch (e.Key) + { + case "ArrowDown": + _pickerHighlight = (_pickerHighlight + 1) % pickerNodes.Count; + StateHasChanged(); + break; + case "ArrowUp": + _pickerHighlight = (_pickerHighlight - 1 + pickerNodes.Count) % pickerNodes.Count; + StateHasChanged(); + break; + case "Enter": + if (_pickerHighlight >= 0 && _pickerHighlight < pickerNodes.Count) + SelectFromPicker(pendingPicker, pickerNodes[_pickerHighlight]); + break; + case "Escape": + DismissWidget(); + break; } - catch (Exception ex) + } + + private static bool PickerNodeMatches(MeshNode node, string term, bool exact) + { + var name = node.Name ?? node.Id ?? ""; + var seg = LastSegment(node.Path) ?? ""; + return exact + ? name.Equals(term, StringComparison.OrdinalIgnoreCase) || seg.Equals(term, StringComparison.OrdinalIgnoreCase) + : name.Contains(term, StringComparison.OrdinalIgnoreCase) || seg.Contains(term, StringComparison.OrdinalIgnoreCase); + } + + /// Writes the selected node's PATH onto the picker's composer field and dismisses. + private void SelectFromPicker(NodePickerRequest picker, MeshNode node) + { + WriteComposerSelection(picker.ComposerField, node.Path); + lastCommandStatus = $"{picker.Title}: {node.Name ?? node.Id}"; + lastCommandStatusIsError = false; + pendingPicker = null; + pickerNodes = []; + StateHasChanged(); + } + + /// + /// Generic composer-field writer — maps a camelCase ThreadComposer field name to the + /// typed write. This is the ONE place that knows the composer's selectable fields; commands + /// stay generic (they only name the field). + /// + private void WriteComposerSelection(string field, string? path) + { + switch (field) { - Logger.LogError(ex, "[ThreadChat:{InstanceId}] SubmitMessageCore failed", _instanceId); + case "harness": WriteComposerSelection(harness: path); break; + case "agentName": WriteComposerSelection(agentPath: path); break; + case "modelName": WriteComposerSelection(modelName: path); break; + default: + Logger.LogWarning("[ThreadChat:{InstanceId}] pick command targeted unknown composer field '{Field}'", + _instanceId, field); + break; } } @@ -515,31 +1461,66 @@ private void CancelSubmission() StateHasChanged(); } + /// + /// Esc on the input cancels the in-flight round (Claude.ai pattern). + /// The typed text is preserved in MessageText so the user can re-send + /// after the cancel completes — or just hit Send to queue it as the + /// next round (PendingUserMessages on the thread). + /// + private void OnInputKeyDown(Microsoft.AspNetCore.Components.Web.KeyboardEventArgs e) + { + if (e.Key == "Escape" && ThreadViewModel?.IsExecuting == true && !isCancelling) + { + CancelExecution(); + } + } + private void CancelExecution() { if (string.IsNullOrEmpty(threadPath) || isCancelling) return; - isCancelling = true; - StateHasChanged(); - - var delivery = Hub.Post(new CancelThreadStreamRequest { ThreadPath = threadPath }, - o => o.WithTarget(new Address(threadPath))); - - if (delivery != null) - { - Hub.RegisterCallback(delivery, (response, _) => - { - isCancelling = false; - InvokeAsync(StateHasChanged); - return Task.FromResult(response); - }, CancellationToken.None); - } - else + var cache = EnsureCache(); + if (cache is null || string.IsNullOrEmpty(threadPath)) { isCancelling = false; StateHasChanged(); + return; } + + isCancelling = true; + StateHasChanged(); + + // Stream-update cancellation: set RequestedStatus = Cancelled on the + // thread node through the process-wide cache. The thread hub's cancel + // watcher cancels the CTS and propagates to every active delegation + // sub-thread. The button clears once IsExecuting flips false via the + // live thread stream (which every other reader is subscribed to on + // the same shared cache handle). + Hub.GetMeshNodeStream(threadPath) + .Update(curr => curr?.Content is MeshWeaver.AI.Thread t + ? curr with { Content = t with { RequestedStatus = MeshWeaver.AI.ThreadExecutionStatus.Cancelled } } + : curr!) + .Subscribe( + updated => + { + if ((updated?.Content as MeshWeaver.AI.Thread)?.RequestedStatus is null) + { + Logger.LogWarning( + "[ThreadChat:{InstanceId}] Cancel stream.Update returned a node WITHOUT RequestedStatus set for {Thread}", + _instanceId, threadPath); + } + isCancelling = false; + InvokeAsync(StateHasChanged); + }, + ex => + { + Logger.LogWarning(ex, + "[ThreadChat:{InstanceId}] Cancel stream.Update failed for {Thread}", + _instanceId, threadPath); + isCancelling = false; + InvokeAsync(StateHasChanged); + }); } /// @@ -552,7 +1533,7 @@ private void OnNavigationContextChanged(NavigationContext? ctx) if (_isDisposed) return; if (ctx is null || string.IsNullOrEmpty(ctx.PrimaryPath) || ctx.Path == "chat") return; - var newPath = ctx.PrimaryPath; + var newPath = NormalizeContextPath(ctx.PrimaryPath); if (newPath == initialContext) return; var name = ctx.Node?.Name ?? ctx.Node?.Id; @@ -567,11 +1548,42 @@ private void OnNavigationContextChanged(NavigationContext? ctx) initialContext = newPath; attachments.RemoveAll(a => a.IsContext); - attachments.Insert(0, new AttachmentInfo(newPath, name, IsContext: true)); + // newPath is "" when the context is a reserved route partition (login, …) — clear the + // context chip rather than pinning an empty/rogue one. + if (!string.IsNullOrEmpty(newPath)) + attachments.Insert(0, new AttachmentInfo(newPath, name, IsContext: true)); StateHasChanged(); }); } + /// + /// Normalizes a node path by stripping any satellite-partition suffix + /// (segments starting with _ such as _Thread, _Comment, + /// _Access, _Activity, _Approval, _Tracking). + /// Returns everything before the first such segment; returns the path + /// unchanged when no satellite segment is present. + /// + private static string NormalizeContextPath(string path) + { + if (string.IsNullOrEmpty(path)) + return path; + + var normalized = path; + var segments = path.Split('/'); + for (var i = 0; i < segments.Length; i++) + { + if (segments[i].StartsWith('_')) + { + normalized = string.Join('/', segments, 0, i); + break; + } + } + // A rogue/reserved ROUTE partition (login, welcome, settings, …) is NOT a real node — never use + // it as a chat context. Reading it sends a GetDataRequest to a hub that never opens its init + // gates (DataContextInit/MeshNodeInit) and the read hangs >30s. Treat a reserved context as none. + return MeshWeaver.AI.AgentPickerProjection.IsReservedPartition(normalized) ? "" : normalized; + } + private void OnMessageTextChanged(string value) { MessageText = value; @@ -588,7 +1600,7 @@ private void OnCompletionItemAccepted(string path) // Check if this path matches a known agent — select it instead of adding a chip if (_agentsByPath.TryGetValue(path, out var agentInfo)) { - OnAgentChanged(agentInfo); + WriteComposerSelection(agentPath: agentInfo.Path); return; } @@ -628,7 +1640,7 @@ private async Task UpdateExtractedReferencesAsync() if (_agentsByPath.TryGetValue(refPath, out var agentInfo)) { - OnAgentChanged(agentInfo); + WriteComposerSelection(agentPath: agentInfo.Path); // Only remove agent references from text if (!string.IsNullOrEmpty(updatedText)) { @@ -691,38 +1703,6 @@ private void OnChipClicked(string path) NavigationManager.NavigateTo($"/{path}"); } - private void OnAgentChanged(AgentDisplayInfo? newAgent) - { - if (newAgent == null || newAgent.Name == selectedAgentInfo?.Name) - return; - - selectedAgentInfo = newAgent; - - // Update model to agent's preferred model - var preferredModel = GetPreferredModelInfoForAgent(newAgent.Name); - if (preferredModel != null) - { - selectedModelInfo = preferredModel; - } - - StateHasChanged(); - } - - private void OnModelChanged(ModelInfo? newModel) - { - if (newModel?.Name == selectedModelInfo?.Name || newModel == null) - return; - - selectedModelInfo = newModel; - - if (selectedAgentInfo != null) - { - agentModelPreferences[selectedAgentInfo.Name] = newModel.Name; - } - - StateHasChanged(); - } - // --- Side panel title and action handling --- private void UpdateSidePanelTitle() @@ -742,6 +1722,7 @@ private void OnSidePanelAction(string action) switch (action) { case "New": + viewMode = ChatViewMode.Chat; SidePanelState.SetContentPath(null); break; case "Resume": @@ -761,7 +1742,7 @@ private Task SwitchToResumeModeAsync() { var accessService = Hub.ServiceProvider.GetService(); var userId = accessService?.Context?.ObjectId ?? accessService?.CircuitContext?.ObjectId; - ns = !string.IsNullOrEmpty(userId) ? $"User/{userId}" : null; + ns = userId; } var hiddenQuery = string.IsNullOrEmpty(ns) @@ -803,120 +1784,152 @@ private CompletionProviderConfig GetCompletionConfig() { return new CompletionProviderConfig { - TriggerCharacters = ["@"], + // "@" → node/agent references; "/" → slash-commands (handled by GetCommandCompletions). + TriggerCharacters = ["@", "/"], Items = [] }; } - private CancellationTokenSource? _completionCts; + private const int CompletionTopN = 50; + + // Sort by SortKey ascending — AutocompleteToCompletion encodes priority into a + // numeric prefix that puts higher-priority items first. + private static readonly IComparer CompletionBySortKey = + Comparer.Create((a, b) => + string.Compare(a.SortKey ?? "", b.SortKey ?? "", StringComparison.Ordinal)); /// - /// Main completion handler — delegates to IChatCompletionOrchestrator. - /// Returns the first batch immediately; streams remaining batches in the background - /// and pushes progressive updates to the Monaco widget. + /// True while a completion stream is in flight. Drives the chat input's loading + /// indicator: SetCompletionsInflight(true) on subscription, false when + /// the orchestrator's emits OnCompleted. /// - private async Task GetCompletionsAsync(string query) - { - if (string.IsNullOrWhiteSpace(query) || !query.StartsWith("@")) - return []; + private bool _isCompletingInflight; - // Cancel any previous streaming request - _completionCts?.Cancel(); - _completionCts = new CancellationTokenSource(); - var ct = _completionCts.Token; + /// True while a chat-completion stream has subscribers but hasn't yet completed. + public bool IsCompletingInflight => _isCompletingInflight; - try - { - var currentAddress = NavigationService.CurrentNamespace ?? initialContext ?? ""; - - var allItems = new List(); - var isFirst = true; + /// + /// Streams top-N completion snapshots from . + /// The orchestrator yields batches as providers finish (fast local first, remote later); + /// each item flows through , which folds it + /// into a sorted snapshot. Monaco subscribes once per query and pushes each snapshot to + /// the suggest widget — pure reactive, no Task, no await, no IAsyncEnumerable bridge. + /// + /// The stream is wrapped in Defer + Finally so we know when it + /// starts and when it completes (all providers done). That toggles + /// which drives the chat-input spinner via + /// StateHasChanged. + /// + /// DistinctUntilChanged over the snapshot prevents redundant push-to-JS + /// when a producer finishes without changing the visible top-N (e.g., a partition + /// fan-out yields items that all rank below the existing top-N). + /// + private IObservable> GetCompletions(string query) + { + // Slash-commands: route straight to the command catalog (nodeType:Command + the registry), + // bypassing the @-oriented node-search orchestrator so a "/" query lists ONLY commands. + if (query?.StartsWith("/") == true) + return GetCommandCompletions(query); - await foreach (var batch in CompletionOrchestrator.GetCompletionsAsync(query, currentAddress, ct)) - { - foreach (var item in batch.Items) - { - allItems.Add(AutocompleteToCompletion(item, batch.Category, batch.CategoryPriority)); - } + if (string.IsNullOrWhiteSpace(query) || !query.StartsWith("@")) + return Observable.Return>(Array.Empty()); - if (isFirst) - { - isFirst = false; - // Return first batch immediately; collect remaining in background - var firstResults = allItems.ToArray(); - _ = CollectRemainingBatchesAsync(query, currentAddress, allItems, ct); - return firstResults; - } - } + var currentAddress = NavigationService.CurrentNamespace ?? initialContext ?? ""; - return allItems.ToArray(); - } - catch (OperationCanceledException) - { - return []; - } - catch (Exception ex) + return Observable.Defer(() => { - Logger.LogError(ex, "Error getting completions for query: {Query}", query); - return []; - } + SetCompletionsInflight(true); + return CompletionOrchestrator.GetCompletions(query, currentAddress) + .SelectMany(batch => batch.Items + .Select(item => AutocompleteToCompletion(item, batch.Category, batch.CategoryPriority))) + .ScanTopN(CompletionTopN, CompletionBySortKey) + .DistinctUntilChanged(SnapshotKey) + .Catch, Exception>(ex => + { + Logger.LogError(ex, "Error streaming completions for query: {Query}", query); + return Observable.Return>(Array.Empty()); + }) + .Finally(() => SetCompletionsInflight(false)); + }); } /// - /// Continues collecting batches from the orchestrator after the first batch was returned. - /// Pushes progressive updates to the Monaco widget via PushCompletionUpdateAsync. + /// Slash-skill completions: lists nodeType:Skill nodes (built-ins imported to PG plus any + /// Space/NodeType/user skill via namespace inheritance), straight from + /// — NOT through the @-oriented + /// node-search orchestrator. Monaco filters the list by the typed "/word". /// - private async Task CollectRemainingBatchesAsync( - string query, - string currentAddress, - List allItems, - CancellationToken ct) + private IObservable> GetCommandCompletions(string query) { - try - { - // Start a new streaming call to get all batches (including any we already have). - // Deduplication below ensures we only push genuinely new items. - await foreach (var batch in CompletionOrchestrator.GetCompletionsAsync(query, currentAddress, ct)) + // When a non-MeshWeaver harness is active, IT is the authority for the slash-command list: + // show its own commands (/login, /logout) plus /harness (to switch back), NOT MeshWeaver's + // /agent /model. Monaco filters the list by the typed "/word". + var harness = ActiveHarness(); + if (harness is not null) + return Observable.Return(BuildHarnessCommandCompletions(harness)); + + // Construct the provider directly against the chat hub's service provider — it self-resolves + // its deps (IWorkspace + IMessageHub for the nodeType:Skill catalog). Resolving via + // GetServices() does NOT work here: the provider is only registered in + // the Agents-application hub's container (ConfigureAgentsApplication), never on the chat hub — + // so the enumerable lookup returned null and typing "/" showed nothing. + var provider = new MeshWeaver.AI.Completion.SkillAutocompleteProvider(Hub.ServiceProvider); + + return provider.GetItems(query, initialContext) + .Select(items => (IReadOnlyList)items + .Select(i => AutocompleteToCompletion(i, "Commands", 2000)) + .OrderBy(c => c.SortKey, StringComparer.Ordinal) + .ToList()) + .Catch, Exception>(ex => { - var hadNew = false; - foreach (var item in batch.Items) - { - var completionItem = AutocompleteToCompletion(item, batch.Category, batch.CategoryPriority); - // Deduplicate by InsertText - if (!allItems.Any(existing => - string.Equals(existing.InsertText, completionItem.InsertText, StringComparison.OrdinalIgnoreCase))) - { - allItems.Add(completionItem); - hadNew = true; - } - } + Logger.LogDebug(ex, "[ThreadChat:{InstanceId}] command completions failed", _instanceId); + return Observable.Return>(Array.Empty()); + }); + } - // Push updated list to Monaco if we got new items. Fire-and-forget by design — - // this runs inside the streaming completion loop and must not block; errors are - // non-fatal (debug-logged). Discard silences CS4014. - if (hadNew && monacoEditor != null) - { - _ = InvokeAsync(async () => - { - try - { - await monacoEditor.PushCompletionUpdateAsync(allItems.ToArray()); - } - catch (Exception ex) - { - Logger.LogDebug(ex, "[ThreadChat] Failed to push completion update"); - } - }); - } - } - } - catch (OperationCanceledException) { /* expected when user types more */ } - catch (Exception ex) - { - Logger.LogDebug(ex, "[ThreadChat] Background completion collection failed"); - } + /// + /// Builds the slash-command completion list for an active non-MeshWeaver harness: the harness's + /// OWN commands (the harness is the autocomplete authority) plus /harness so the user can + /// still switch back. No mesh query — the list is the harness's declared . + /// + private IReadOnlyList BuildHarnessCommandCompletions(MeshWeaver.AI.IHarness harness) + { + CompletionItem Item(string name, string description) => + AutocompleteToCompletion( + new Data.Completion.AutocompleteItem( + Label: $"/{name}", InsertText: $"/{name} ", Description: description, + Category: "Commands", Priority: 2000, Kind: Data.Completion.AutocompleteKind.Command), + "Commands", 2000); + + var items = harness.Commands.Select(c => Item(c.Name, c.Description)).ToList(); + // /harness stays available (it falls through to the node-pick path) so a CLI harness isn't a + // one-way door — the user can switch runtime back to MeshWeaver or another harness. + items.Add(Item("harness", "Switch the harness (runtime)")); + return items.OrderBy(c => c.SortKey, StringComparer.Ordinal).ToList(); + } + + /// + /// Toggles the chat-input's "loading" flag and re-renders. Idempotent — only + /// fires StateHasChanged when the value actually changes (the + /// -equivalent is a manual DistinctUntilChanged + /// guard at the sink). + /// + private void SetCompletionsInflight(bool inflight) + { + if (_isCompletingInflight == inflight) return; + _isCompletingInflight = inflight; + if (!_isDisposed) + InvokeAsync(StateHasChanged); } + /// + /// Stable key for a completion-snapshot. Two consecutive snapshots collapse to a + /// single push when their items (and their order) are identical — saves redundant + /// JS-interop pushes when a producer finishes without changing the visible top-N. + /// + private static string SnapshotKey(IReadOnlyList items) => + string.Join('', items.Select(i => i.SortKey ?? i.InsertText ?? i.Label ?? "")); + private static CompletionItem AutocompleteToCompletion( Data.Completion.AutocompleteItem item, string category, int categoryPriority) { @@ -944,8 +1957,10 @@ private static CompletionItem AutocompleteToCompletion( } /// - /// Converts the data-bound ThreadViewModel to a message ID list. - /// GetStream<object> deserializes the ThreadViewModel (has $type), so we get the typed object. + /// Converts the data-bound ThreadViewModel to a message ID list. Also + /// syncs per-message cache subscriptions so the inline bubble render in + /// 's Razor template gets live ThreadMessage + /// content via . /// private ThreadViewModel? ConvertThreadViewModel(object? value, ThreadViewModel? _) { @@ -958,20 +1973,471 @@ private static CompletionItem AutocompleteToCompletion( }; Logger.LogDebug("[ThreadChat:{InstanceId}] ConvertThreadViewModel: input={InputType}, msgs={MsgCount}", _instanceId, value?.GetType().Name ?? "null", result?.Messages?.Count ?? 0); + // SyncMessageSubscriptions runs in the property setter (below) — calling + // here is too early: threadPath is set by the setter AFTER conversion. return result; } + // ─── Inline bubble subscriptions ────────────────────────────────────── + // Per-message live state, keyed by message id. Populated by + // SyncMessageSubscriptions opening one IMeshNodeStreamCache subscription + // per visible id. Razor template iterates ThreadMessages and renders + // each bubble inline using messageStates[id]. + private record MessageBubbleState( + string Role, + string AuthorName, + string? ModelName, + DateTime? Timestamp, + string? Text, + IReadOnlyList? ToolCalls, + IReadOnlyList? UpdatedNodes, + string? Status = null, + DateTime? CompletedAt = null, + string? Harness = null, + int? InputTokens = null, + int? OutputTokens = null); + + private readonly Dictionary messageStates = new(); + private readonly Dictionary messageSubs = new(); + private readonly HashSet editingMessages = new(); + /// Message ids whose satellite cell did NOT emit within the cache + /// settle window — surfaced as "Missing message" in the bubble instead of + /// the loading skeleton. A deleted-by-someone-else or never-materialised + /// satellite would otherwise leave the bubble stuck on a skeleton forever + /// and (in prod 2026-05-24) hang any code path that does a GetDataRequest + /// on the same path. + private readonly HashSet missingMessages = new(); + private readonly Dictionary missingProbes = new(); + + /// Live state for a delegated sub-thread. is the + /// sub-thread MeshNode's Name (ThreadNamer-generated or user-edited); + /// is the node's Icon property; + /// drives the runtime panel — running sub-threads show a live row, completed + /// ones drop out. + + /// feed the inline progress preview ("Calling search_nodes…" / first 120 chars + /// of the streaming response). drives the elapsed-time + /// chip; null on a freshly-created sub-thread before its first + /// StartingExecution → Executing flip. + private record DelegationHeader( + string? Title, + string? Icon, + bool IsExecuting, + string? ExecutionStatus, + string? StreamingText, + DateTime? StartedAt); + + /// delegationPath → live header (Title + Icon) so the chip can show + /// the sub-thread's actual name instead of just the agent's name. Populated by + /// subscriptions opened from . + private readonly Dictionary delegationHeaders = new(); + private readonly Dictionary delegationSubs = new(); + + private void SyncMessageSubscriptions(IReadOnlyList messageIds) + { + if (_isDisposed || string.IsNullOrEmpty(threadPath)) + return; + + IMeshNodeStreamCache? cache; + try + { + cache = Hub.ServiceProvider.GetRequiredService(); + } + catch (Exception ex) + { + Logger.LogWarning(ex, + "[ThreadChat:{InstanceId}] IMeshNodeStreamCache unavailable — bubbles will not update live", + _instanceId); + return; + } + var accessService = Hub.ServiceProvider.GetService(); + + var idSet = messageIds.ToHashSet(StringComparer.Ordinal); + var stale = messageSubs.Keys.Where(id => !idSet.Contains(id)).ToList(); + foreach (var id in stale) + { + messageSubs[id].Dispose(); + messageSubs.Remove(id); + messageStates.Remove(id); + editingMessages.Remove(id); + if (missingProbes.Remove(id, out var probe)) probe.Dispose(); + missingMessages.Remove(id); + } + + foreach (var id in messageIds) + { + if (messageSubs.ContainsKey(id)) continue; + var nodePath = $"{threadPath}/{id}"; + + // 🚨 Subscribe INSIDE the ImpersonateAsSystem scope. + // `cache.GetStream` returns a cold IObservable whose RLS gate + // resolves at Subscribe-time (not at GetStream-time). The + // framework carries the AccessContext captured at Subscribe + // through to the gate via CarryAccessContext. If we subscribe + // after the using block closes, the gate sees the Blazor + // circuit's identity in a sync-emission scope and silently + // rejects every emission — symptom: empty skeleton bars. + // Same pattern UserActivityLayoutAreas.cs:42-67 documents. + var capturedId = id; + using (accessService?.ImpersonateAsSystem()) + { + var stream = Hub.GetMeshNodeStream(nodePath); + messageSubs[id] = stream + .Where(n => n?.Content is not null) + .Subscribe( + n => + { + // Real content arrived — drop any "missing" mark and the + // probe; the bubble will render normally. + if (missingProbes.Remove(capturedId, out var probe)) probe.Dispose(); + if (missingMessages.Remove(capturedId)) + InvokeAsync(StateHasChanged); + UpdateMessageState(capturedId, n); + }, + ex => + { + // 🚨 CRITICAL: handle errors here. The cache surfaces + // missing satellites as OnError(DeliveryFailureException) + // — without this handler the exception is unhandled and + // crashes the Blazor circuit (the user-visible + // "still crashing / stuck on progress screen" symptom + // in prod 2026-05-24). Mark the bubble as missing and + // re-render. Reproduced by + // test/MeshWeaver.Threading.Test/MissingSatelliteTest. + Logger.LogDebug(ex, + "[ThreadChat:{InstanceId}] cache.GetStream errored for {NodePath} — marking message as missing", + _instanceId, nodePath); + InvokeAsync(() => + { + if (_isDisposed) return; + if (missingProbes.Remove(capturedId, out var probe)) probe.Dispose(); + if (missingMessages.Add(capturedId)) + StateHasChanged(); + }); + }); + } + + // Missing-message probe — backup for the case where the cache stream + // neither emits content nor errors within the deadline (cold-observable + // starvation; not the path the OnError above catches). Surfaces the + // bubble as "missing" so the GUI never gets stuck on an indefinite + // skeleton. + var probeDelay = TimeSpan.FromSeconds(5); + missingProbes[id] = System.Reactive.Linq.Observable + .Timer(probeDelay) + .Subscribe(_ => InvokeAsync(() => + { + if (_isDisposed) return; + if (!messageStates.ContainsKey(capturedId) && missingMessages.Add(capturedId)) + StateHasChanged(); + })); + } + } + + private void UpdateMessageState(string id, MeshNode node) + { + if (_isDisposed) return; + var je = ToJsonElement(node.Content!, Hub.JsonSerializerOptions); + + var role = je.TryGetProperty("role", out var roleProp) && roleProp.ValueKind == JsonValueKind.String + ? roleProp.GetString() ?? "user" : "user"; + var explicitAuthor = je.TryGetProperty("authorName", out var aProp) && aProp.ValueKind == JsonValueKind.String + ? aProp.GetString() : null; + var agentName = je.TryGetProperty("agentName", out var agProp) && agProp.ValueKind == JsonValueKind.String + ? agProp.GetString() : null; + var author = explicitAuthor + ?? (role.Equals("user", StringComparison.OrdinalIgnoreCase) + ? "You" : (agentName ?? "Assistant")); + var modelName = je.TryGetProperty("modelName", out var mProp) && mProp.ValueKind == JsonValueKind.String + ? mProp.GetString() : null; + DateTime? timestamp = je.TryGetProperty("timestamp", out var tsProp) && tsProp.ValueKind == JsonValueKind.String + && DateTime.TryParse(tsProp.GetString(), out var parsed) ? parsed : null; + var text = je.TryGetProperty("text", out var textProp) && textProp.ValueKind == JsonValueKind.String + ? textProp.GetString() : null; + IReadOnlyList? toolCalls = je.TryGetProperty("toolCalls", out var tcProp) + && tcProp.ValueKind == JsonValueKind.Array + ? tcProp.Deserialize>(Hub.JsonSerializerOptions) : null; + IReadOnlyList? updatedNodes = je.TryGetProperty("updatedNodes", out var unProp) + && unProp.ValueKind == JsonValueKind.Array + ? unProp.Deserialize>(Hub.JsonSerializerOptions) : null; + // Status + CompletedAt drive the per-bubble duration chip — "1:23" while + // Streaming (live ticker), "1:23 ✓" once Completed (frozen final value). + var status = je.TryGetProperty("status", out var stProp) && stProp.ValueKind == JsonValueKind.String + ? stProp.GetString() : null; + DateTime? completedAt = je.TryGetProperty("completedAt", out var caProp) + && caProp.ValueKind == JsonValueKind.String + && DateTime.TryParse(caProp.GetString(), out var ca) ? ca : null; + // Harness + token usage drive the assistant meta line "Harness · time · N in / M out". + var harness = je.TryGetProperty("harness", out var hProp) && hProp.ValueKind == JsonValueKind.String + ? hProp.GetString() : null; + int? inputTokens = je.TryGetProperty("inputTokens", out var itProp) && itProp.ValueKind == JsonValueKind.Number + ? itProp.GetInt32() : null; + int? outputTokens = je.TryGetProperty("outputTokens", out var otProp) && otProp.ValueKind == JsonValueKind.Number + ? otProp.GetInt32() : null; + + var newState = new MessageBubbleState(role, author, modelName, timestamp, text, toolCalls, updatedNodes, status, completedAt, harness, inputTokens, outputTokens); + var prev = messageStates.GetValueOrDefault(id); + if (Equals(prev, newState)) return; + + messageStates[id] = newState; + SyncDelegationSubscriptions(); + InvokeAsync(StateHasChanged); + } + /// - /// Creates a LayoutAreaControl pointing to a ThreadMessage node's Overview layout area. + /// Opens (and tears down) per-sub-thread MeshNode subscriptions so the + /// delegation chips can render the sub-thread's actual Name + Icon instead + /// of just the agent name. Called from + /// whenever a bubble's ToolCalls list changes — picks up newly-emitted + /// DelegationPaths and drops subscriptions for paths no longer referenced + /// by any current bubble. /// - private LayoutAreaControl? GetMessageCell(string msgId) + private void SyncDelegationSubscriptions() { - if (string.IsNullOrEmpty(threadPath)) - return null; - return new LayoutAreaControl( - $"{threadPath}/{msgId}", - new LayoutAreaReference(ThreadMessageNodeType.OverviewArea)) - .WithSpinnerType(SpinnerType.Skeleton); + if (_isDisposed) return; + + var activePaths = messageStates.Values + .SelectMany(s => s.ToolCalls ?? (IReadOnlyList)[]) + .Select(c => c.DelegationPath) + .Where(p => !string.IsNullOrEmpty(p)) + .Cast() + .ToHashSet(StringComparer.Ordinal); + + // Drop subscriptions for delegations no longer in any bubble's tool calls + // (e.g. message edited / deleted-from-here truncated the tail). + var stale = delegationSubs.Keys.Where(p => !activePaths.Contains(p)).ToList(); + foreach (var p in stale) + { + delegationSubs[p].Dispose(); + delegationSubs.Remove(p); + delegationHeaders.Remove(p); + } + + if (activePaths.Count == 0) return; + + IMeshNodeStreamCache? cache; + try + { + cache = Hub.ServiceProvider.GetRequiredService(); + } + catch + { + // No cache available (minimal test fixture) — chips fall back to + // the agent-name summary, which is fine. + return; + } + var accessService = Hub.ServiceProvider.GetService(); + + foreach (var path in activePaths) + { + if (delegationSubs.ContainsKey(path)) continue; + + // Same ImpersonateAsSystem pattern as SyncMessageSubscriptions — + // the cache's RLS gate resolves at Subscribe-time; without the + // system scope the gate sees the Blazor circuit's identity and + // can deny silently. Parent-thread access was already gated when + // the user opened this chat; the chip read piggybacks on that. + using (accessService?.ImpersonateAsSystem()) + { + var stream = Hub.GetMeshNodeStream(path); + delegationSubs[path] = stream + .Where(n => n is not null) + .Subscribe( + n => UpdateDelegationHeader(path, n), + ex => Logger.LogDebug(ex, + "[ThreadChat:{InstanceId}] delegation cache.GetStream errored for {Path} — chip falls back to agent-name summary", + _instanceId, path)); + } + } + } + + private void UpdateDelegationHeader(string path, MeshNode node) + { + if (_isDisposed) return; + // Parse the Thread content for live execution state. The Title/Icon come + // from the MeshNode envelope; IsExecuting + ExecutionStatus + StreamingText + // live inside Content (MeshThread record). We use JsonElement parsing so + // this view doesn't have to take a hard dependency on MeshWeaver.AI types + // — same shape UpdateMessageState uses for ThreadMessage. + bool isExecuting = false; + string? executionStatus = null; + string? streamingText = null; + DateTime? startedAt = null; + if (node.Content is not null) + { + var je = ToJsonElement(node.Content, Hub.JsonSerializerOptions); + // MeshThread.IsExecuting is a computed `[JsonIgnore]` property — read + // Status directly and recompute (StartingExecution / Executing → + // executing). Order matches Thread.cs Status enum: Idle=0, + // StartingExecution=1, Executing=2, Cancelled=3, Done=4 — so 3 + // (Cancelled) is NOT executing. + if (je.TryGetProperty("status", out var statusProp)) + { + var s = statusProp.ValueKind == JsonValueKind.String + ? statusProp.GetString() + : statusProp.ValueKind == JsonValueKind.Number + ? statusProp.GetInt32().ToString() + : null; + isExecuting = s is "StartingExecution" or "Executing" or "1" or "2"; + } + if (je.TryGetProperty("executionStatus", out var esProp) && esProp.ValueKind == JsonValueKind.String) + executionStatus = esProp.GetString(); + if (je.TryGetProperty("streamingText", out var stProp) && stProp.ValueKind == JsonValueKind.String) + streamingText = stProp.GetString(); + // Drives the elapsed-time chip on the sub-thread card. + if (je.TryGetProperty("executionStartedAt", out var startProp) + && startProp.ValueKind == JsonValueKind.String + && startProp.TryGetDateTime(out var parsed)) + startedAt = parsed; + } + + var header = new DelegationHeader( + Title: string.IsNullOrEmpty(node.Name) ? null : node.Name, + Icon: string.IsNullOrEmpty(node.Icon) ? null : node.Icon, + IsExecuting: isExecuting, + ExecutionStatus: executionStatus, + StreamingText: streamingText, + StartedAt: startedAt); + var prev = delegationHeaders.GetValueOrDefault(path); + if (Equals(prev, header)) return; + delegationHeaders[path] = header; + InvokeAsync(StateHasChanged); + } + + private DelegationHeader? GetDelegationHeader(string? path) => + string.IsNullOrEmpty(path) ? null : delegationHeaders.GetValueOrDefault(path); + + /// + /// Enumerates all currently-executing sub-threads launched from this thread's + /// bubbles — drives the runtime panel below the chat. Deduplicated by path + /// (an agent can re-delegate to the same sub-thread; we render one card per + /// distinct path). Order: first appearance in tool-call traversal, so the + /// list is stable across re-renders. + /// + private IEnumerable<(string Path, DelegationHeader Header)> GetRunningSubThreads() + { + var seen = new HashSet(StringComparer.Ordinal); + foreach (var state in messageStates.Values) + { + if (state.ToolCalls is null) continue; + foreach (var call in state.ToolCalls) + { + if (string.IsNullOrEmpty(call.DelegationPath)) continue; + if (!seen.Add(call.DelegationPath)) continue; + var header = delegationHeaders.GetValueOrDefault(call.DelegationPath); + if (header is { IsExecuting: true }) + yield return (call.DelegationPath, header); + } + } + } + + // Use the hub's options (camelCase property naming) so the field-name + // lookups below ("role", "text", "agentName" …) match what the wire + // serializer produced. With default options the serialiser emits "Role" + // / "Text" and every TryGetProperty miss falls through to defaults — + // symptom: every bubble labelled "You" with no message text. + private static JsonElement ToJsonElement(object content, JsonSerializerOptions options) + => content is JsonElement je ? je + : JsonSerializer.SerializeToElement(content, options); + + private MessageBubbleState? GetMessageState(string id) => messageStates.GetValueOrDefault(id); + + private bool IsMissing(string id) => missingMessages.Contains(id); + + private bool IsEditing(string id) => editingMessages.Contains(id); + + private void StartEdit(string id) + { + editingMessages.Add(id); + StateHasChanged(); + } + + private void CancelEdit(string id) + { + editingMessages.Remove(id); + StateHasChanged(); + } + + private void ResubmitMessage(string id) + { + var state = GetMessageState(id); + if (state == null || string.IsNullOrEmpty(threadPath)) return; + var outId = Guid.NewGuid().ToString("N")[..8]; + Hub.Post(new CreateNodeRequest(new MeshNode(outId, threadPath) + { + NodeType = ThreadMessageNodeType.NodeType, + MainNode = threadPath, + Content = new ThreadMessage + { + Role = "assistant", + Text = "", + Timestamp = DateTime.UtcNow, + Type = ThreadMessageType.AgentResponse + } + }), o => o.WithTarget(new Address(threadPath))); + // Picked node PATHS flow through — execution normalizes to ids at its boundary. + Hub.ResubmitMessage(threadPath, id, newUserText: state.Text ?? "", + agentName: boundAgentPath, modelName: boundModelPath, harness: boundHarness); + } + + private void DeleteFromMessage(string id) + { + if (string.IsNullOrEmpty(threadPath)) return; + Hub.DeleteFromMessage(threadPath, id); + } + + // ─── Tool-call display helpers ──────────────────────────────────────── + + private readonly record struct ToolCallDisplay(string Verb, string? Path, bool IsNodeModifying); + + private static string FormatToolCallSummary(ToolCallEntry call) + { + var d = FormatToolCallDisplay(call); + return d.Path is null ? d.Verb : $"{d.Verb} {d.Path}"; + } + + private static ToolCallDisplay FormatToolCallDisplay(ToolCallEntry call) + { + if (!string.IsNullOrEmpty(call.DelegationPath)) + { + var name = call.DisplayName ?? call.Name; + if (name.Contains("Delegating to ")) + name = name.Replace("Delegating to ", "").TrimEnd('.', ' '); + return new ToolCallDisplay(name, null, false); + } + var rawArgs = call.Arguments ?? ""; + string? path = null; + foreach (var line in rawArgs.Split('\n')) + { + var trimmed = line.Trim(); + if (trimmed.StartsWith("path:", StringComparison.OrdinalIgnoreCase)) { path = trimmed["path:".Length..].Trim(); break; } + if (trimmed.StartsWith("url:", StringComparison.OrdinalIgnoreCase)) { path = trimmed["url:".Length..].Trim(); break; } + if (trimmed.StartsWith("query:", StringComparison.OrdinalIgnoreCase)) { path = trimmed["query:".Length..].Trim(); break; } + } + if (string.IsNullOrEmpty(path)) + path = rawArgs.Split('\n').FirstOrDefault()?.Trim(); + if (!string.IsNullOrEmpty(path) && path.StartsWith('@')) + path = path[1..].TrimStart('/'); + + return call.Name switch + { + "Get" or "get_node" => new ToolCallDisplay("Reading", path, false), + "Search" or "search_nodes" => new ToolCallDisplay("Searching", path, false), + "Create" or "create_node" => new ToolCallDisplay("Created", path, true), + "Update" or "update_node" => new ToolCallDisplay("Updated", path, true), + "Patch" or "patch_node" => new ToolCallDisplay("Patched", path, true), + "Delete" or "delete_node" => new ToolCallDisplay("Deleted", path, true), + "NavigateTo" or "navigate_to" => new ToolCallDisplay("Navigating to", path, false), + "SearchWeb" => new ToolCallDisplay("Searching web for", path, false), + "FetchWebPage" => new ToolCallDisplay("Fetching", path, false), + "store_plan" => new ToolCallDisplay("Stored plan", null, false), + _ => new ToolCallDisplay(call.DisplayName ?? call.Name, path, false) + }; + } + + private static NodeChangeEntry? FindChange(IReadOnlyList? updatedNodes, string? path) + { + if (string.IsNullOrEmpty(path) || updatedNodes is null) return null; + return updatedNodes.FirstOrDefault(n => string.Equals(n.Path, path, StringComparison.Ordinal)); } /// @@ -997,14 +2463,53 @@ private static string TruncateText(string text, int maxLength) return text[..maxLength] + "..."; } + /// + /// Compact elapsed-time formatter for "how long has this been running". + /// Returns "0:12" for < 1 h, "1:23:45" for >= 1 h. + /// Negative or null clamps to "0:00" so a clock-skew anomaly + /// doesn't render -3:14. + /// + private static string FormatElapsed(DateTime? startedAt, DateTime? endedAt = null) + { + if (startedAt is null) return "0:00"; + var end = endedAt ?? DateTime.UtcNow; + var span = end - startedAt.Value; + if (span < TimeSpan.Zero) span = TimeSpan.Zero; + return span.TotalHours >= 1 + ? $"{(int)span.TotalHours}:{span.Minutes:D2}:{span.Seconds:D2}" + : $"{span.Minutes}:{span.Seconds:D2}"; + } + + /// + /// 1-second ticker that drives the elapsed-time chips' re-render. Subscribed + /// in ; disposed in . + /// Only triggers StateHasChanged when something is + /// executing (own thread or a sub-thread) — silent otherwise so an idle + /// thread view doesn't burn render cycles every second. + /// + private IDisposable? elapsedTicker; + public override ValueTask DisposeAsync() { if (!_isDisposed) { _isDisposed = true; - NavigationService.OnNavigationContextChanged -= OnNavigationContextChanged; + elapsedTicker?.Dispose(); + _connectSub?.Dispose(); + _navContextSubscription?.Dispose(); + composerSubscription?.Dispose(); + composerDefaultsSubscription?.Dispose(); agentSubscription?.Dispose(); submissionHandler.Dispose(); + foreach (var sub in messageSubs.Values) sub.Dispose(); + messageSubs.Clear(); + messageStates.Clear(); + foreach (var sub in missingProbes.Values) sub.Dispose(); + missingProbes.Clear(); + missingMessages.Clear(); + foreach (var sub in delegationSubs.Values) sub.Dispose(); + delegationSubs.Clear(); + delegationHeaders.Clear(); SidePanelState.OnActionRequested -= OnSidePanelAction; } diff --git a/src/MeshWeaver.Blazor.Portal/Chat/ThreadChatView.razor.css b/src/MeshWeaver.Blazor.Portal/Chat/ThreadChatView.razor.css index e56ce37d3..ed5972890 100644 --- a/src/MeshWeaver.Blazor.Portal/Chat/ThreadChatView.razor.css +++ b/src/MeshWeaver.Blazor.Portal/Chat/ThreadChatView.razor.css @@ -17,6 +17,27 @@ width: 100%; } +/* No-messages (new-chat) landing. The composer is height:100% of its slot a priori; the + Thread view controls how big that slot is by collapsing the empty message history and + letting the bottom block flex-fill. The fill propagates down the chain (bottom → footer → + input-area → input-content → editor) so the input field grows to occupy the space rather + than sitting as a thin bar. With messages present this class is absent, so the history + takes flex:1 and the composer reverts to its natural bottom-bar height. */ +.thread-chat-container.no-messages .thread-chat-messages { + flex: 0 1 auto; + min-height: 0; +} +.thread-chat-container.no-messages .thread-chat-bottom, +.thread-chat-container.no-messages .thread-chat-footer, +.thread-chat-container.no-messages .thread-chat-input-area, +.thread-chat-container.no-messages .thread-chat-input-content, +.thread-chat-container.no-messages .input-container { + flex: 1 1 auto; + min-height: 0; + display: flex; + flex-direction: column; +} + .thread-chat-messages { flex: 1; min-height: 0; @@ -24,11 +45,35 @@ width: 100%; overflow-y: auto; overflow-x: hidden; - padding: 16px; + padding: 20px 24px; + /* Flex column so the bubble's `align-self: flex-end` (user) + puts user bubbles flush right while assistant stays left. */ + display: flex; + flex-direction: column; +} + +/* Loading skeleton lines — shown when a message's satellite cell hasn't + delivered content yet through the IMeshNodeStreamCache subscription. */ +.thread-msg-skeleton-line { + height: 13px; + border-radius: 6px; + background: var(--neutral-stroke-rest); + margin: 4px 0; + width: 60%; + animation: thread-msg-skeleton-pulse 1.4s ease-in-out infinite; +} +@@keyframes thread-msg-skeleton-pulse { + 0%, 100% { opacity: 0.35; } + 50% { opacity: 0.15; } } -/* Execution status bar above chat input */ +/* Execution status bar above chat input. `flex: 0 1 auto` + `min-height: 0` + lets it shrink (and scroll internally) when the bottom bar runs out of room, + so it yields space to the always-visible input footer rather than pushing it + off the bottom. */ .thread-exec-bar { + flex: 0 1 auto; + min-height: 0; display: flex; flex-direction: column; gap: 6px; @@ -82,6 +127,17 @@ text-overflow: ellipsis; white-space: nowrap; } +.thread-exec-elapsed { + font-family: ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 0.78rem; + font-variant-numeric: tabular-nums; + color: var(--neutral-foreground-hint); + padding: 2px 8px; + border: 1px solid var(--neutral-stroke-rest); + border-radius: 10px; + background: var(--neutral-layer-1); + flex: 0 0 auto; +} .thread-exec-cancel { display: inline-flex; align-items: center; @@ -99,6 +155,82 @@ background: var(--neutral-layer-4); } +/* Pending-inbox section inside the executing-progress strip. + Shows messages the user typed during the in-flight turn + that the agent has NOT yet pulled via check_inbox. */ +.thread-exec-pending-inbox { + display: flex; + flex-direction: column; + gap: 4px; + padding: 8px 10px; + margin-top: 6px; + border: 1px dashed color-mix(in srgb, var(--accent-fill-rest) 50%, transparent); + border-radius: 6px; + background: color-mix(in srgb, var(--accent-fill-rest) 4%, transparent); +} +.thread-exec-pending-inbox-header { + display: flex; + align-items: center; + gap: 6px; + font-size: 0.74rem; + color: var(--accent-fill-rest); + font-weight: 600; +} +.thread-exec-pending-inbox-item { + font-size: 0.78rem; + color: var(--neutral-foreground-rest); + padding: 2px 6px; + border-left: 2px solid color-mix(in srgb, var(--accent-fill-rest) 60%, transparent); + background: var(--neutral-layer-1); + border-radius: 0 4px 4px 0; + line-height: 1.35; + white-space: pre-wrap; +} + +/* In-chat queued user message — rendered inline after the materialised + message cells, showing the user their just-submitted text immediately + (before round dispatch creates the satellite cell). Pulse animation + conveys "waiting to be picked up". */ +.thread-chat-pending-message { + display: flex; + align-items: flex-start; + gap: 8px; + padding: 10px 14px; + margin: 4px 0; + border-radius: 8px; + background: color-mix(in srgb, var(--accent-fill-rest) 6%, transparent); + border: 1px dashed color-mix(in srgb, var(--accent-fill-rest) 35%, transparent); + color: var(--neutral-foreground-rest); + animation: thread-pending-pulse 1.6s ease-in-out infinite; +} +.thread-chat-pending-message-text { + flex: 1; + line-height: 1.4; + white-space: pre-wrap; + word-break: break-word; +} + +/* Bottom bar: groups the execution status ("where we are"), the runtime panel, + and the input footer (harness selector + editor + send) into one block pinned + to the bottom of the screen. The chat history above (flex:1) scrolls under it. + + `flex: 0 1 auto` + `min-height: 0` (NOT `flex-shrink: 0`): the block must be + ALLOWED to shrink so the container's height bounds it. Otherwise, when a thread + is running with sub-threads, the exec-bar (≤200px) + runtime-panel (≤38vh) + + input together exceed the side-panel height; with the block unshrinkable the + surplus overflows the `overflow:hidden` container and clips its LAST child — + the input footer — off the bottom. Letting the block shrink hands the surplus + to the shrinkable status panels below (which scroll internally) while the + footer stays flex-shrink:0 and fully visible. */ +.thread-chat-bottom { + flex: 0 1 auto; + min-height: 0; + display: flex; + flex-direction: column; + min-width: 0; + max-width: 100%; +} + /* Footer: progress + input pinned at bottom */ .thread-chat-footer { flex-shrink: 0; @@ -117,6 +249,72 @@ overflow: visible; min-width: 0; max-width: 100%; + /* Position context so the "Allocating agent…" overlay can sit on top of + the dimmed input content without changing the box's outer height. */ + position: relative; +} + +.thread-chat-input-area.is-allocating .thread-allocating-panel { + /* Float the banner over the (still-rendered) input content so the box + keeps its natural height. The dialog used to swap the entire input + subtree for a small spinner panel and snap back when the thread + was ready, which the user noticed as a layout jump. */ + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + margin: 0; + z-index: 2; + background: color-mix(in srgb, var(--neutral-layer-card-container) 92%, transparent); + backdrop-filter: blur(2px); + border-radius: 8px; +} + +.thread-chat-input-content { + transition: opacity 120ms ease; + /* Anchor the slash-command picker as a floating popup (see .thread-chat-widget) so opening it + does NOT push the Monaco editor's DOM down + back up — which re-laid-out the editor and made + the composer appear to vanish on selection. */ + position: relative; +} + +.thread-chat-input-content.dimmed { + opacity: 0.35; + pointer-events: none; + user-select: none; +} + +/* Thin, read-only thread status — harness · agent · model · tokens used. + Muted + small so it reads as metadata above the editor, not a control. */ +.thread-chat-status-row { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 10px; + padding: 0 2px 6px 2px; + font-size: 11px; + line-height: 1.4; + color: var(--neutral-foreground-hint); + min-width: 0; +} + +.thread-chat-status-item { + display: inline-flex; + align-items: center; + gap: 4px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: 180px; +} + +.thread-chat-status-sep { + color: var(--neutral-stroke-rest); +} + +.thread-chat-status-tokens { + font-variant-numeric: tabular-nums; } .input-container { @@ -137,6 +335,265 @@ min-width: 0; } +/* Compact harness combobox in the bottom row — JUST the picker (no label). + Agent + model are chosen via the /agent and /model slash-commands, so the + row stays small enough to fit harness + context chip + Send on one line. */ +.thread-chat-harness { + display: flex; + align-items: center; + min-width: 0; + max-width: 220px; +} + +/* Navigation-context chip(s) inline in the bottom row, between the harness and + Send. The collection wraps its own chips; cap its share so a long context + name truncates rather than pushing Send off the row. */ +.thread-chat-context { + display: flex; + align-items: center; + min-width: 0; +} +.thread-chat-context ::deep .reference-chip-collection { + margin: 0; + flex-wrap: nowrap; +} + +/* Legacy selectors container — kept for any remaining consumer; the chat + footer now uses .thread-chat-harness above. */ +.thread-chat-selectors { + display: flex; + align-items: center; + gap: 6px; + padding: 4px 2px 6px 2px; + min-width: 0; +} + +/* Each dropdown shares the row equally and shrinks below content width so + Agent + Model always fit on a single line. SimpleDropdown's inner + `.simple-dropdown-text` already has text-overflow: ellipsis, so long + names ("Description Writer", "claude-sonnet-4-6") truncate cleanly + when the chat dock narrows. Max-width caps the spread on wide layouts + so the row doesn't look stretched. */ +.thread-chat-selectors ::deep .simple-dropdown { + flex: 1 1 0; + min-width: 0; + max-width: 240px; +} + +.thread-chat-selectors .simple-dropdown-item-path, +.thread-chat-selectors .simple-dropdown-item-description { + font-size: 0.78rem; + color: var(--neutral-foreground-hint); +} + +/* Icon / name layout inside dropdown items. Explicit color: in dark mode + the inner inherits transparent text from FluentIcon's wrapper + if we don't pin it — that produced "icon visible, name invisible" + chips that the user noticed. */ +.thread-chat-selectors ::deep .dropdown-icon-name { + display: inline-flex; + align-items: center; + gap: 6px; + color: var(--neutral-foreground-rest); +} + +.thread-chat-selectors ::deep .dropdown-icon-name span { + color: var(--neutral-foreground-rest); + font-weight: 500; +} + +.thread-chat-selectors ::deep .dropdown-icon-name svg { + width: 14px; + height: 14px; + flex: 0 0 auto; + color: var(--neutral-foreground-hint); +} + +/* Inside the SELECTED dropdown button — same layout, but the span/icon + inherit the button's foreground (which Fluent's button styling + already sets correctly for accent / neutral surfaces). */ +.thread-chat-selectors ::deep .simple-dropdown-button .dropdown-icon-name, +.thread-chat-selectors ::deep .simple-dropdown-button .dropdown-icon-name span, +.thread-chat-selectors ::deep .simple-dropdown-button .dropdown-icon-name svg { + color: inherit; +} + +/* Confirmation / error message from a slash-command (e.g. "Switched + agent → Worker"). Trails after the dropdowns; cleared on next + submission. */ +.thread-chat-status-msg { + font-size: 0.78rem; + color: var(--neutral-foreground-hint); + font-style: italic; +} + +.thread-chat-status-msg.error { + color: var(--error-rest, #c0392b); + font-style: normal; +} + +/* Inline picker widget rendered above the editor when /agent or /model + is invoked without an argument. Theme-aware via Fluent design tokens + so dark mode + high-contrast read correctly. */ +.thread-chat-widget { + /* Floating popup ABOVE the input (a command palette popping up), anchored to + .thread-chat-input-content — does not push the Monaco DOM. Floating-popup tier (10000) so it + opens ABOVE the menu bar (z-index 1100 in PortalLayoutBase) instead of being covered by it. */ + position: absolute; + bottom: calc(100% + 6px); + left: 0; + right: 0; + z-index: 10000; + border: 1px solid var(--neutral-stroke-rest); + border-radius: 8px; + background: var(--neutral-layer-1); + overflow: hidden; + /* Responsive cap: never taller than the space between the side-panel header and the input, + so the popup (agent/model picker AND the CLI-login dialog) can't overrun the "New Thread" + header at the top of a short side panel. calc(100vh - 180px) ≈ viewport minus header+input+gap; + capped at 440px so it stays compact on a tall window. The inner list scrolls past that. */ + max-height: min(440px, calc(100vh - 180px)); + display: flex; + flex-direction: column; + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.18); +} + +.thread-chat-widget-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 8px 10px; + font-size: 0.85rem; + font-weight: 600; + color: var(--neutral-foreground-rest); + background: var(--neutral-layer-2); + border-bottom: 1px solid var(--neutral-stroke-rest); +} + +.thread-chat-widget-list { + overflow-y: auto; + padding: 4px; + display: flex; + flex-direction: column; + gap: 2px; +} + +.thread-chat-widget-group { + font-size: 0.68rem; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--neutral-foreground-hint); + padding: 8px 8px 4px 8px; + font-weight: 600; +} + +/* Reset native button defaults — without `appearance: none` the chip-style + buttons fell back to the browser's grey-on-grey button rendering and + ignored the dark-mode tokens. */ +.thread-chat-widget-item { + appearance: none; + -webkit-appearance: none; + text-align: left; + border: 1px solid transparent; + background: transparent; + cursor: pointer; + border-radius: 6px; + padding: 8px 10px; + display: flex; + flex-direction: row; + align-items: center; + gap: 8px; + color: var(--neutral-foreground-rest); + font: inherit; + transition: background 80ms ease, border-color 80ms ease; +} + +.thread-chat-widget-item-icon { + flex: 0 0 auto; + width: 18px; + height: 18px; + display: inline-flex; + align-items: center; + justify-content: center; + color: var(--neutral-foreground-hint); +} + +.thread-chat-widget-item-icon :deep(svg), +.thread-chat-widget-item-icon img, +.thread-chat-widget-item-icon svg { + width: 18px; + height: 18px; +} + +.thread-chat-widget-item-text { + display: flex; + flex-direction: column; + gap: 2px; + min-width: 0; +} + +.thread-chat-widget-item:hover, +.thread-chat-widget-item:focus-visible { + background: var(--neutral-fill-stealth-hover); + border-color: var(--neutral-stroke-rest); + outline: none; +} + +.thread-chat-widget-item.selected { + background: var(--accent-fill-rest); + color: var(--foreground-on-accent-rest); + border-color: var(--accent-stroke-control-rest); +} + +/* Keyboard-highlighted row (↑/↓). Accent box so the Enter target is obvious — distinct from hover. */ +.thread-chat-widget-item--active { + background: var(--neutral-fill-stealth-hover); + border-color: var(--accent-fill-rest); + outline: 1px solid var(--accent-fill-rest); + outline-offset: -1px; +} + +/* The picker widget takes focus when opened so arrow keys reach it; don't draw a focus ring on the + container itself (the highlighted row is the affordance). */ +.thread-chat-widget:focus, +.thread-chat-widget:focus-visible { + outline: none; +} + +.thread-chat-widget-item-name { + font-weight: 600; + font-size: 0.9rem; + color: inherit; +} + +.thread-chat-widget-item-path, +.thread-chat-widget-item-desc { + font-size: 0.78rem; + color: var(--neutral-foreground-hint); +} + +.thread-chat-widget-item.selected .thread-chat-widget-item-path, +.thread-chat-widget-item.selected .thread-chat-widget-item-desc { + color: var(--foreground-on-accent-rest); + opacity: 0.85; +} + +.thread-chat-widget-empty { + padding: 12px; + text-align: center; + color: var(--neutral-foreground-hint); + font-size: 0.85rem; +} + +/* Attachment chips moved below the send button — they're inputs to the + next message, not state about the current one, so they belong at the + bottom of the dialog where the user is actively composing. */ +.thread-chat-attachments { + padding-top: 8px; + margin-top: 4px; + border-top: 1px dashed var(--neutral-stroke-rest); +} + .loading-overlay { display: flex; flex-direction: column; @@ -286,3 +743,529 @@ color: var(--neutral-foreground-hint); margin-top: 2px; } + +/* Allocating-thread / message-pending styles — moved out of the inline + + @code { - [Parameter] - public GlobalState Content { get; set; } = default!; + [Parameter] public GlobalState Content { get; set; } = default!; + + private IReadOnlyList notifications = Array.Empty(); + private bool markedOnOpen; + private IDisposable? subscription; + + protected override void OnInitialized() + { + subscription = MeshQuery.Query( + MeshQueryRequest.FromQuery("nodeType:Notification sort:CreatedAt-desc")) + .Subscribe(change => + { + notifications = (IReadOnlyList?)change.Items?.ToList() ?? Array.Empty(); + // Opening the panel counts as seeing the notifications: flip the + // displayed set to read so the bell badge clears. Gate on the first + // populated load so notifications that arrive live while the panel + // is open still surface as unread. + if (!markedOnOpen && notifications.Count > 0) + { + markedOnOpen = true; + MarkAllRead(); + } + InvokeAsync(StateHasChanged); + }); + } + + private void OnNotificationClick(MeshNode node, Notification notif, string? targetPath) + { + if (!notif.IsRead) + MarkRead(node, notif); + if (!string.IsNullOrEmpty(targetPath)) + NavigationManager.NavigateTo($"/{targetPath}"); + } + + private void MarkRead(MeshNode node, Notification notif) + { + // stream.Update is the canonical mutation primitive. Cross-hub Updates + // for a scalar property like IsRead are race-safe — no satellite-list + // staleness concerns. + StreamCache.Update(node.Path, n => + { + if (n.Content is not Notification current || current.IsRead) return n; + return n with { Content = current with { IsRead = true } }; + }, Hub.JsonSerializerOptions).Subscribe( + _ => { }, + ex => Hub.ServiceProvider + .GetService() + ?.CreateLogger("MeshWeaver.Blazor.Portal.NotificationCenterPanel") + .LogWarning(ex, "Mark-read failed for {Path}", node.Path)); + } + + private void MarkAllRead() + { + var updated = new List(notifications.Count); + var changed = false; + foreach (var n in notifications) + { + if (n.Content is Notification notif && !notif.IsRead) + { + MarkRead(n, notif); // durable cross-hub write + updated.Add(n with { Content = notif with { IsRead = true } }); // optimistic + changed = true; + } + else + { + updated.Add(n); + } + } + if (changed) + { + notifications = updated; + InvokeAsync(StateHasChanged); + } + } + + private static string FormatTime(DateTimeOffset ts) + { + var delta = DateTimeOffset.UtcNow - ts; + if (delta.TotalSeconds < 60) return "just now"; + if (delta.TotalMinutes < 60) return $"{(int)delta.TotalMinutes}m ago"; + if (delta.TotalHours < 24) return $"{(int)delta.TotalHours}h ago"; + if (delta.TotalDays < 7) return $"{(int)delta.TotalDays}d ago"; + return ts.LocalDateTime.ToString("MMM d, HH:mm"); + } + + private static Icon DefaultIconFor(NotificationType type) => type switch + { + NotificationType.ApprovalRequired => new Icons.Regular.Size20.ShieldQuestion(), + NotificationType.ApprovalGiven => new Icons.Regular.Size20.ShieldCheckmark(), + NotificationType.ApprovalRejected => new Icons.Regular.Size20.ShieldDismiss(), + _ => new Icons.Regular.Size20.Info() + }; + + public void Dispose() => subscription?.Dispose(); } diff --git a/src/MeshWeaver.Blazor.Portal/Components/PortalErrorModal.razor b/src/MeshWeaver.Blazor.Portal/Components/PortalErrorModal.razor new file mode 100644 index 000000000..5df03f9ba --- /dev/null +++ b/src/MeshWeaver.Blazor.Portal/Components/PortalErrorModal.razor @@ -0,0 +1,57 @@ +@* Mounted ONCE in PortalLayoutBase (next to ). Subscribes to the + circuit-scoped PortalErrorSink and shows ONE modal at a time with an OK button; a failure + burst is drained sequentially, never stacked into a wall of dialogs. Mirrors the + IDialogService usage in NotificationCenter. *@ +@implements IDisposable +@inject IDialogService DialogService +@inject MeshWeaver.Blazor.Infrastructure.PortalErrorSink ErrorSink + +@code { + private IDisposable? sub; + private readonly object gate = new(); + private System.Collections.Immutable.ImmutableQueue pending = + System.Collections.Immutable.ImmutableQueue.Empty; + private bool showing; + + protected override void OnInitialized() => + sub = ErrorSink.Errors.Subscribe(msg => + { + bool start; + lock (gate) // tiny sync critical section, NOT an async gate + { + pending = pending.Enqueue(msg); + start = !showing; + if (start) showing = true; + } + if (start) + _ = InvokeAsync(DrainAsync); // marshal onto the renderer + }); + + // One modal at a time, OK-gated — the same await-the-dialog shape NotificationCenter uses. + private async Task DrainAsync() + { + while (true) + { + string msg; + lock (gate) + { + if (pending.IsEmpty) { showing = false; return; } + pending = pending.Dequeue(out msg); + } + + var dialog = await DialogService.ShowMessageBoxAsync(new DialogParameters + { + Content = new MessageBoxContent + { + Title = "Something went wrong", + Message = msg, + }, + PrimaryAction = "OK", + SecondaryAction = null, + }); + await dialog.Result; // block until the user clicks OK / dismisses + } + } + + public void Dispose() => sub?.Dispose(); +} diff --git a/src/MeshWeaver.Blazor.Portal/Components/SearchBar.razor.cs b/src/MeshWeaver.Blazor.Portal/Components/SearchBar.razor.cs index 2cf89d247..9c271d524 100644 --- a/src/MeshWeaver.Blazor.Portal/Components/SearchBar.razor.cs +++ b/src/MeshWeaver.Blazor.Portal/Components/SearchBar.razor.cs @@ -1,13 +1,16 @@ +using System.Reactive.Linq; +using System.Reactive.Subjects; using MeshWeaver.Mesh.Services; using MeshWeaver.Messaging; using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components.Web; +using Microsoft.Extensions.DependencyInjection; using Microsoft.FluentUI.AspNetCore.Components; using Microsoft.JSInterop; namespace MeshWeaver.Blazor.Portal.Components; -public partial class SearchBar : IAsyncDisposable +public partial class SearchBar : IDisposable { private const string SearchPlaceholder = "Search the mesh... (e.g. nodeType:Story status:Open)"; private const int MaxResults = 10; @@ -27,7 +30,15 @@ public partial class SearchBar : IAsyncDisposable [Inject] public required IJSRuntime JSRuntime { get; set; } - private SearchHub? _searchHub; + private IMeshService _meshService = default!; + + // Typed search terms flow through this subject; the reactive pipeline in + // OnInitialized debounces, switches to the latest term's suggestion stream, + // and binds the WHOLE collection per emission. No channels, no await foreach. + private readonly Subject _terms = new(); + private IDisposable? _searchSubscription; + private IDisposable? _defaultsSubscription; + private ElementReference inputRef; private int _inputKey; private string? searchTerm; @@ -35,16 +46,52 @@ public partial class SearchBar : IAsyncDisposable private bool showDropdown; private int highlightedIndex = -1; private bool isLoading; - private string? _lastSearchedTerm; - private bool _isFirstKeystroke = true; - private CancellationTokenSource? debounceCts; + + /// One progressive snapshot of the suggestion set; + /// marks the frame after the source has quieted (drops the progress bar). + private readonly record struct SearchFrame(IReadOnlyList Suggestions, bool Settled); protected override void OnInitialized() { KeyCodeService.RegisterListener(OnKeyDownAsync); - _searchHub = new SearchHub(Hub); + _meshService = Hub.ServiceProvider.GetRequiredService(); + + // Debounce keystrokes, switch to the latest term's live suggestion stream, + // and bind the entire collection on every emission. MeshSearch.Suggestions + // re-emits as each source converges (progressive CombineLatest inside the + // mesh query surface), so partial results render immediately and re-order + // by score as more sources return. Switch() cancels the prior search. + _searchSubscription = _terms + .Throttle(TimeSpan.FromMilliseconds(250)) + .DistinctUntilChanged() + .Select(term => MeshSearch + .Suggestions(_meshService, term, NavigationService?.CurrentNamespace, MaxResults) + // Run the search once and derive two signals from it: + // • progressive — bind each NON-EMPTY snapshot as sources converge + // (Settled:false keeps the progress bar on). Skipping the leading + // all-empty frame means a refinement search doesn't blank the + // current results before the new ones land. + // • settle — a 700ms-quiet throttle binds the final snapshot and + // drops the progress bar, including the genuinely-empty (no + // results) case so the dropdown clears instead of spinning forever. + .Publish(shared => shared + .Where(list => list.Count > 0) + .Select(list => new SearchFrame(list, Settled: false)) + .Merge(shared + .Throttle(TimeSpan.FromMilliseconds(700)) + .Select(list => new SearchFrame(list, Settled: true))))) + .Switch() + .Subscribe(OnFrame); } + private void OnFrame(SearchFrame frame) => InvokeAsync(() => + { + suggestions = frame.Suggestions.ToArray(); + if (frame.Settled) + isLoading = false; + StateHasChanged(); + }); + public async Task OnKeyDownAsync(FluentKeyCodeEventArgs? args) { if (args is not null && args.Key == KeyCode.Slash) @@ -59,115 +106,27 @@ public async Task OnKeyDownAsync(FluentKeyCodeEventArgs? args) } /// - /// Captures input value and fires search — completely decoupled from rendering. - /// The native input is uncontrolled (no value binding), so Blazor never - /// pushes values back to the DOM. Re-renders from search results cannot - /// interfere with typing. + /// Captures input value and feeds the reactive pipeline — decoupled from + /// rendering. The native input is uncontrolled (no value binding), so Blazor + /// never pushes values back to the DOM and re-renders from search results + /// cannot interfere with typing. /// private void OnInput(ChangeEventArgs e) { searchTerm = e.Value?.ToString(); highlightedIndex = -1; - debounceCts?.Cancel(); - if (string.IsNullOrWhiteSpace(searchTerm)) { suggestions = []; showDropdown = false; isLoading = false; - _isFirstKeystroke = true; return; } isLoading = true; showDropdown = true; - - // Clear stale results when the query diverges (e.g. start changed) - if (!IsRefinement(searchTerm.Trim(), _lastSearchedTerm)) - suggestions = []; - - var cts = new CancellationTokenSource(); - debounceCts = cts; - _ = DebounceAndSearchAsync(searchTerm.Trim(), cts.Token); - } - - private async Task DebounceAndSearchAsync(string input, CancellationToken ct) - { - try - { - // Show loading dropdown immediately - await InvokeAsync(StateHasChanged); - - if (!_isFirstKeystroke) - { - await Task.Delay(300, ct); - if (ct.IsCancellationRequested) return; - } - _isFirstKeystroke = false; - - await StreamSearchResultsAsync(input, ct); - } - catch (OperationCanceledException) - { - // Expected when debounce cancels - } - } - - private async Task StreamSearchResultsAsync(string input, CancellationToken ct) - { - if (_searchHub == null) - { - isLoading = false; - return; - } - - try - { - var results = new List(); - var contextPath = NavigationService?.CurrentNamespace; - var firstBatchRendered = false; - - await foreach (var suggestion in _searchHub.SearchAsync(input, contextPath, MaxResults, ct)) - { - if (ct.IsCancellationRequested) break; - - var idx = results.FindIndex(s => s.Score < suggestion.Score); - if (idx < 0) - results.Add(suggestion); - else - results.Insert(idx, suggestion); - - if (results.Count > MaxResults) - results.RemoveAt(results.Count - 1); - - // Render once when the first results arrive so the user sees - // suggestions + progress bar (isLoading is still true). - if (!firstBatchRendered) - { - firstBatchRendered = true; - suggestions = results.ToArray(); - await InvokeAsync(StateHasChanged); - } - } - - if (!ct.IsCancellationRequested) - { - _lastSearchedTerm = input; - suggestions = results.ToArray(); - isLoading = false; - await InvokeAsync(StateHasChanged); - } - } - catch (OperationCanceledException) - { - // Expected - } - catch - { - isLoading = false; - suggestions = []; - } + _terms.OnNext(searchTerm.Trim()); } private void HandleKeyDown(KeyboardEventArgs e) @@ -262,30 +221,13 @@ private void NavigateToSuggestion(QuerySuggestion suggestion) private void ClearSearch() { searchTerm = null; - _lastSearchedTerm = null; suggestions = []; showDropdown = false; highlightedIndex = -1; isLoading = false; - _isFirstKeystroke = true; - debounceCts?.Cancel(); _inputKey++; // forces Blazor to recreate the , clearing its DOM value } - /// - /// Returns true if the new query is a refinement of the previous one - /// (i.e. starts with the same prefix). Stale results are kept visible - /// while the refined search runs. Returns false when the start diverges, - /// triggering an immediate clear of the dropdown. - /// - private static bool IsRefinement(string current, string? previous) - { - if (string.IsNullOrEmpty(previous)) - return false; - return current.StartsWith(previous, StringComparison.OrdinalIgnoreCase) - || previous.StartsWith(current, StringComparison.OrdinalIgnoreCase); - } - private void OnFocus() { if (suggestions.Length > 0) @@ -294,30 +236,22 @@ private void OnFocus() return; } - if (string.IsNullOrWhiteSpace(searchTerm) && _searchHub != null) + if (string.IsNullOrWhiteSpace(searchTerm)) { isLoading = true; showDropdown = true; - _ = LoadDefaultSuggestionsAsync(); - } - } - - private async Task LoadDefaultSuggestionsAsync() - { - try - { - var contextPath = NavigationService?.CurrentNamespace; - var results = new List(); - await foreach (var s in _searchHub!.SearchAsync(null, contextPath, MaxResults, CancellationToken.None)) - results.Add(s); - suggestions = results.ToArray(); - isLoading = false; - await InvokeAsync(StateHasChanged); - } - catch - { - isLoading = false; - suggestions = []; + _defaultsSubscription?.Dispose(); + // Empty box on focus: bind the recently-accessed default set (whole + // collection per emission, same progressive surface). + _defaultsSubscription = MeshSearch + .Suggestions(_meshService, null, NavigationService?.CurrentNamespace, MaxResults) + .Subscribe(list => InvokeAsync(() => + { + suggestions = list.ToArray(); + if (list.Count > 0) + isLoading = false; + StateHasChanged(); + })); } } @@ -344,11 +278,11 @@ private static string GetNodeTypeDisplay(string? nodeType) return lastSlash >= 0 ? nodeType[(lastSlash + 1)..] : nodeType; } - public ValueTask DisposeAsync() + public void Dispose() { KeyCodeService.UnregisterListener(OnKeyDownAsync, OnKeyDownAsync); - debounceCts?.Cancel(); - debounceCts?.Dispose(); - return ValueTask.CompletedTask; + _searchSubscription?.Dispose(); + _defaultsSubscription?.Dispose(); + _terms.Dispose(); } } diff --git a/src/MeshWeaver.Blazor.Portal/Components/SearchHub.cs b/src/MeshWeaver.Blazor.Portal/Components/SearchHub.cs deleted file mode 100644 index 698e9938a..000000000 --- a/src/MeshWeaver.Blazor.Portal/Components/SearchHub.cs +++ /dev/null @@ -1,213 +0,0 @@ -using System.Collections.Concurrent; -using System.Threading.Channels; -using MeshWeaver.Mesh; -using MeshWeaver.Mesh.Services; -using MeshWeaver.Messaging; -using Microsoft.Extensions.DependencyInjection; - -namespace MeshWeaver.Blazor.Portal.Components; - -/// -/// Hosted hub that executes search queries off the Blazor UI thread. -/// The handler runs on the hub's own scheduler, streaming results -/// back to the caller via a Channel stored in a side-table (not in the message, -/// because the delivery pipeline touches serialization). -/// -internal sealed class SearchHub -{ - /// - /// Fetch more candidates than displayed so relevance scoring can surface - /// the best matches even if the DB returns them in a different order. - /// - private const int CandidatePoolSize = 50; - - private readonly IMessageHub _hub; - - /// - /// Side-table mapping correlation id -> channel + cancellation. - /// Kept out of the message to avoid serialization of non-serializable types. - /// - private static readonly ConcurrentDictionary Pending = new(); - - public SearchHub(IMessageHub parentHub) - { - _hub = parentHub.GetHostedHub( - new Address($"{parentHub.Address}/_Search"), - config => config.WithHandler(ExecuteSearchAsync)); - } - - /// - /// Posts a search request to the hosted hub and returns an async enumerable - /// that yields results as the hub handler streams them back via Channel. - /// The caller's thread is free between yields. - /// - public IAsyncEnumerable SearchAsync( - string? input, string? contextPath, int maxResults, CancellationToken ct) - { - var id = Guid.NewGuid().ToString("N"); - var channel = Channel.CreateUnbounded(); - Pending[id] = new PendingSearch(channel.Writer, ct); - - _hub.Post(new SearchRequest(id, input?.Trim(), contextPath, maxResults)); - - return ReadAndCleanup(id, channel.Reader, ct); - } - - private static async IAsyncEnumerable ReadAndCleanup( - string id, - ChannelReader reader, - [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct) - { - try - { - await foreach (var item in reader.ReadAllAsync(ct)) - yield return item; - } - finally - { - Pending.TryRemove(id, out _); - } - } - - /// - /// Runs on the hosted hub's scheduler — not the Blazor UI thread. - /// Looks up the Channel and CancellationToken from the side-table. - /// - private static async Task ExecuteSearchAsync( - IMessageHub hub, IMessageDelivery delivery, CancellationToken hubCt) - { - var req = delivery.Message; - - if (!Pending.TryGetValue(req.Id, out var pending)) - return delivery.Processed(); - - using var linked = CancellationTokenSource.CreateLinkedTokenSource(pending.Ct, hubCt); - var ct = linked.Token; - - try - { - var meshService = hub.ServiceProvider.GetRequiredService(); - - if (string.IsNullOrEmpty(req.Input)) - { - // Empty input: show recently accessed items ordered by last access time - var query = $"source:accessed scope:descendants is:main sort:LastModified-desc context:search limit:{req.MaxResults}"; - await foreach (var obj in meshService.QueryAsync( - new MeshQueryRequest { Query = query }, ct)) - { - if (obj is MeshNode n) - { - await pending.Writer.WriteAsync(new QuerySuggestion( - n.Path ?? "", - n.Name ?? n.Id ?? "", - n.NodeType, - 0, - n.Icon), ct); - } - } - } - else if (req.Input.StartsWith('@')) - { - var afterAt = req.Input[1..]; - var lastSlash = afterAt.LastIndexOf('/'); - var basePath = lastSlash >= 0 ? afterAt[..lastSlash] : ""; - var prefix = lastSlash >= 0 ? afterAt[(lastSlash + 1)..] : afterAt; - - await foreach (var s in meshService.AutocompleteAsync( - basePath, prefix, AutocompleteMode.RelevanceFirst, req.MaxResults, - req.ContextPath, context: "search", ct)) - await pending.Writer.WriteAsync(s, ct); - } - else - { - await ExecuteTextSearchAsync(meshService, req, pending, ct); - } - } - catch (OperationCanceledException) { } - finally - { - pending.Writer.Complete(); - } - - return delivery.Processed(); - } - - /// - /// Free-text search: fetches a wider candidate pool from QueryAsync, - /// scores each result by where the search terms match (name > nodeType > path > content), - /// adds proximity boost, then streams results ordered by score. - /// - private static async Task ExecuteTextSearchAsync( - IMeshService meshService, SearchRequest req, PendingSearch pending, CancellationToken ct) - { - // Fetch a wider pool so scoring can surface the best matches - var query = $"*{req.Input}* scope:descendants context:search is:main limit:{CandidatePoolSize}"; - var candidates = new List(); - - await foreach (var obj in meshService.QueryAsync( - new MeshQueryRequest { Query = query }, ct)) - { - if (obj is MeshNode n) - { - var score = ComputeRelevanceScore(n, req.Input!, req.ContextPath); - candidates.Add(new QuerySuggestion( - n.Path ?? "", - n.Name ?? n.Id ?? "", - n.NodeType, - score, - n.Icon)); - } - } - - // Sort by score descending and stream top results - foreach (var s in candidates.OrderByDescending(c => c.Score).Take(req.MaxResults)) - await pending.Writer.WriteAsync(s, ct); - } - - /// - /// Scores a MeshNode by how well it matches the search input. - /// Name matches score highest — this is the "goodness of match" measure. - /// Uses the same tier structure as autocomplete scoring. - /// - private static double ComputeRelevanceScore(MeshNode node, string searchInput, string? contextPath) - { - var name = node.Name ?? ""; - var terms = searchInput.Split(' ', StringSplitOptions.RemoveEmptyEntries); - - double totalScore = 0; - var scoredTerms = 0; - - foreach (var rawTerm in terms) - { - var term = rawTerm.Trim('*'); - if (string.IsNullOrEmpty(term)) continue; - - scoredTerms++; - if (name.StartsWith(term, StringComparison.OrdinalIgnoreCase)) - totalScore += 100; - else if (name.Contains(term, StringComparison.OrdinalIgnoreCase)) - totalScore += 80; - else if ((node.Path ?? "").Contains(term, StringComparison.OrdinalIgnoreCase)) - totalScore += 20; - else if ((node.NodeType ?? "").Contains(term, StringComparison.OrdinalIgnoreCase)) - totalScore += 10; - else - totalScore += 1; // matched in content/description - } - - // Normalize so multi-word queries don't get inflated scores - var score = scoredTerms > 0 ? totalScore / scoredTerms : 1; - - // Proximity boost: nodes closer to the user's current context rank higher - score += PathProximity.ComputeBoost(contextPath, node.Path); - - return score; - } - - /// - /// Serialization-safe message — only contains plain data types. - /// - private record SearchRequest(string Id, string? Input, string? ContextPath, int MaxResults); - - private record PendingSearch(ChannelWriter Writer, CancellationToken Ct); -} diff --git a/src/MeshWeaver.Blazor.Portal/Components/UserProfile.razor.cs b/src/MeshWeaver.Blazor.Portal/Components/UserProfile.razor.cs index b4d4406cf..a659ec914 100644 --- a/src/MeshWeaver.Blazor.Portal/Components/UserProfile.razor.cs +++ b/src/MeshWeaver.Blazor.Portal/Components/UserProfile.razor.cs @@ -1,8 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; using System.Security.Claims; +using MeshWeaver.Blazor.Infrastructure; using MeshWeaver.Blazor.Portal.Authentication; +using MeshWeaver.Mesh; using MeshWeaver.Messaging; using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components.Authorization; @@ -24,6 +28,9 @@ public partial class UserProfile : ComponentBase [Inject] public required AccessService AccessService { get; init; } + [Inject] + public required PortalApplication PortalApp { get; init; } + [CascadingParameter] public required Task AuthenticationState { get; set; } @@ -57,8 +64,13 @@ protected override async Task OnParametersSetAsync() username = name; initials = GetInitials(name); - // Check if the user has PlatformAdmin role - isPlatformAdmin = AccessService.Context?.Roles?.Contains("PlatformAdmin") == true; + // Canonical platform-admin check: admin on the Admin partition + // (hub.IsGlobalAdmin). Wait for the positive within a short window — the + // synced AccessAssignment query emits an empty seed first. + isPlatformAdmin = await PortalApp.Hub.IsGlobalAdmin() + .Where(x => x).Take(1) + .Timeout(TimeSpan.FromSeconds(5), Observable.Return(false)) + .FirstAsync().ToTask(); } } diff --git a/src/MeshWeaver.Blazor.Portal/Infrastructure/VUserHelper.cs b/src/MeshWeaver.Blazor.Portal/Infrastructure/VUserHelper.cs index 03469a3a7..2abe7816d 100644 --- a/src/MeshWeaver.Blazor.Portal/Infrastructure/VUserHelper.cs +++ b/src/MeshWeaver.Blazor.Portal/Infrastructure/VUserHelper.cs @@ -1,7 +1,7 @@ +using System.Reactive.Linq; using MeshWeaver.Blazor.Infrastructure; using MeshWeaver.Mesh; using MeshWeaver.Mesh.Security; -using MeshWeaver.Mesh.Services; using MeshWeaver.Messaging; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -10,41 +10,65 @@ namespace MeshWeaver.Blazor.Portal.Infrastructure; /// /// Helper for managing VUser (virtual/anonymous user) nodes. -/// Uses unprotected storage for existence checks and ImpersonateAsHub for creation. /// public static class VUserHelper { /// - /// Ensures a VUser node exists for the given virtual user ID. - /// Uses unprotected storage read for existence check (no security overhead), - /// and ImpersonateAsHub for creation (VUserAccessRule allows portal namespace). + /// Ensures a VUser node exists for the given virtual user ID. Posts a + /// with skip-on-exists semantics — the + /// handler rejects with + /// when the node is already there, which we treat as success. No + /// existence query, no race. /// - public static async Task EnsureVUserNodeAsync(PortalApplication portalApp, string virtualUserId, ILogger? logger = null) + public static void EnsureVUserNode(PortalApplication portalApp, string virtualUserId, ILogger? logger = null) { var hub = portalApp.Hub; - var persistence = hub.ServiceProvider.GetRequiredService(); var path = $"VUser/{virtualUserId}"; - - if (await persistence.ExistsAsync(path)) - return; - var accessService = hub.ServiceProvider.GetRequiredService(); - using (accessService.ImpersonateAsHub(hub)) + + var userNode = new MeshNode(virtualUserId, "VUser") { - var userNode = new MeshNode(virtualUserId, "VUser") - { - Name = "Guest", - NodeType = "VUser", - State = MeshNodeState.Active, - Content = new AccessObject - { - IsVirtual = true - } - }; + Name = "Guest", + NodeType = "VUser", + State = MeshNodeState.Active, + Content = new AccessObject { IsVirtual = true } + }; - var meshService = hub.ServiceProvider.GetRequiredService(); - await meshService.CreateNodeAsync(userNode, CancellationToken.None); - logger?.LogDebug("VirtualUser: Created VUser node {Path}", path); + // 🚨 CreateNodeRequest must target the MESH hub — that's where + // WithNodeOperationHandlers registers the handler. PortalApp.Hub is a + // hosted hub at `portal/{userId}` (or `portal/anonymous`) which has no + // CreateNodeRequest handler, so a bare Observe without a target sends + // the request to portal/anonymous and prod surfaces + // "No handler found for message type CreateNodeRequest in portal/anonymous" + // — the page-open crash a real user just hit on the sub-thread URL. + var meshHub = hub.GetMeshHub(); + + // Provisioning a guest VUser node is an infrastructure write, so it runs as + // the well-known system identity — NOT ImpersonateAsHub(hub). For an + // anonymous session `hub` is `portal/anonymous`, a hub-shaped principal; + // RestoreUserContextOnEmission's leak-guard rejects hub-shaped principals + // ("SetContext: hub-shaped principal … must never happen") and logged an + // Error on every anonymous request. `system-security` is a real principal + // with Permission.All, so it passes the guard. Subscribe stays INSIDE the + // scope so the emission-side context is system, not the leaked hub identity. + using (accessService.ImpersonateAsSystem()) + { + hub.Observe( + new CreateNodeRequest(userNode), + o => o.WithTarget(meshHub.Address)) + .FirstAsync() + .Subscribe( + delivery => + { + var resp = delivery.Message; + if (resp.Success) + logger?.LogDebug("VirtualUser: Created VUser node {Path}", path); + else if (resp.RejectionReason == NodeCreationRejectionReason.NodeAlreadyExists) + logger?.LogDebug("VirtualUser: VUser node {Path} already exists", path); + else + logger?.LogWarning("VirtualUser: Failed to create VUser node {Path}: {Error}", path, resp.Error); + }, + ex => logger?.LogWarning(ex, "VirtualUser: Failed to ensure VUser node {Path}", path)); } } } diff --git a/src/MeshWeaver.Blazor.Portal/Layout/PortalLayoutBase.razor b/src/MeshWeaver.Blazor.Portal/Layout/PortalLayoutBase.razor index 05203803d..81018ff6d 100644 --- a/src/MeshWeaver.Blazor.Portal/Layout/PortalLayoutBase.razor +++ b/src/MeshWeaver.Blazor.Portal/Layout/PortalLayoutBase.razor @@ -1,4 +1,5 @@ @using Orientation = Microsoft.FluentUI.AspNetCore.Components.Orientation +@using Microsoft.AspNetCore.Components.Authorization @inherits LayoutComponentBase @inject IDialogService DialogService @@ -21,100 +22,111 @@
    - @* Node menu (Cube icon) — per-node operations *@ - - @if (!string.IsNullOrEmpty(CurrentNodeName)) - { -
    @CurrentNodeName
    -
    - } - @foreach (var menuItem in FlattenMenuItems(GetNodeMenuItems())) - { - var item = menuItem; - @if (item.Area == "_separator") - { -
    - } - else - { - - @if (!string.IsNullOrEmpty(item.Icon)) + @* Authenticated-only menus — Node/Mesh/Settings/Chat require a hub + workspace, which isn't available to anonymous users. Wrapping in + AuthorizeView prevents both render-time and click-time errors. *@ + + + @* Node menu (Cube icon) — per-node operations *@ + + @if (!string.IsNullOrEmpty(CurrentNodeName)) + { +
    @CurrentNodeName
    +
    + } + @foreach (var menuItem in FlattenMenuItems(GetNodeMenuItems())) + { + var item = menuItem; + @if (item.Area == "_separator") { - if (IsEmoji(item.Icon)) - { - @item.Icon - } - else - { - - } +
    } - @item.Label -     - } - } -     - - - + else + { + + @if (!string.IsNullOrEmpty(item.Icon)) + { + if (IsEmoji(item.Icon)) + { + @item.Icon + } + else + { + + } + } + @item.Label + + } + } + + + + - @* Mesh menu — mesh-level operations (Create, Import, Export) *@ - - @if (!string.IsNullOrEmpty(CurrentNodeName)) - { -
    @CurrentNodeName
    -
    - } - @foreach (var menuItem in FlattenMenuItems(GetMeshMenuItems())) - { - var item = menuItem; - @if (item.Area == "_separator") - { -
    - } - else - { - - @if (!string.IsNullOrEmpty(item.Icon)) + @* Mesh menu — mesh-level operations (Create, Import, Export) *@ + + @if (!string.IsNullOrEmpty(CurrentNodeName)) + { +
    @CurrentNodeName
    +
    + } + @foreach (var menuItem in FlattenMenuItems(GetMeshMenuItems())) + { + var item = menuItem; + @if (item.Area == "_separator") + { +
    + } + else { - if (IsEmoji(item.Icon)) - { - @item.Icon - } - else - { - - } + + @if (!string.IsNullOrEmpty(item.Icon)) + { + if (IsEmoji(item.Icon)) + { + @item.Icon + } + else + { + + } + } + @item.Label + } - @item.Label -     - } - } -     - - - + } + + + + - @* Settings — direct navigation *@ - - - + @* Settings — direct navigation *@ + + + + + @HeaderLinks
    -
    - - - -
    + + +
    + + + +
    +     +     @@ -140,6 +152,7 @@ + @@ -149,8 +162,8 @@ When closed, inline styles collapse the second pane to zero. *@ + Class="@(IsSidePanelVisible ? "body-splitter" : "body-splitter panel-collapsed")" + BarSize="@(IsSidePanelVisible ? "16" : "0")">
    diff --git a/src/MeshWeaver.Blazor.Portal/Layout/PortalLayoutBase.razor.cs b/src/MeshWeaver.Blazor.Portal/Layout/PortalLayoutBase.razor.cs index 877027861..6306a6244 100644 --- a/src/MeshWeaver.Blazor.Portal/Layout/PortalLayoutBase.razor.cs +++ b/src/MeshWeaver.Blazor.Portal/Layout/PortalLayoutBase.razor.cs @@ -1,4 +1,6 @@ -using MeshWeaver.AI; +using System.Reactive.Linq; +using System.Reactive.Threading.Tasks; +using MeshWeaver.AI; using MeshWeaver.Blazor.Portal.Resize; using MeshWeaver.Blazor.Portal.SidePanel; using MeshWeaver.Blazor.Services; @@ -10,6 +12,7 @@ using MeshWeaver.Mesh.Services; using MeshWeaver.Messaging; using Microsoft.AspNetCore.Components; +using Microsoft.AspNetCore.Components.Authorization; using Microsoft.JSInterop; namespace MeshWeaver.Blazor.Portal.Layout; @@ -23,6 +26,15 @@ public partial class PortalLayoutBase : LayoutComponentBase, IDisposable [Inject] protected INavigationService NavigationService { get; set; } = null!; [Inject] protected IMenuItemsProvider MenuItemsProvider { get; set; } = null!; [Inject] protected IPathResolver PathResolver { get; set; } = null!; + [Inject] protected AccessService AccessService { get; set; } = null!; + + [CascadingParameter] + protected Task? AuthStateTask { get; set; } + + // Tracks whether the current circuit's user is signed in. Side panel content + // (ThreadChatView / LayoutAreaView) accesses the workspace and throws for + // anonymous users — so we hide it when not authenticated. + private bool isAuthenticated; // Splitter pane sizes - default 3:1 ratio (75% main, 25% side panel) private string MainPaneSize => SidePanelState.Width.HasValue ? $"{100 - SidePanelState.Width.Value}%" : "75%"; @@ -70,12 +82,15 @@ public partial class PortalLayoutBase : LayoutComponentBase, IDisposable private IJSObjectReference? jsModule; private DotNetObjectReference? dotNetRef; + private IDisposable? _navContextSubscription; + protected override void OnInitialized() { base.OnInitialized(); SidePanelState.OnStateChanged += OnSidePanelStateChanged; NavigationService.SidePanelNavigationRequested += OnSidePanelNavigation; - NavigationService.OnNavigationContextChanged += OnNavigationContextChanged; + _navContextSubscription = NavigationService.NavigationContext + .Subscribe(OnNavigationContextChanged); _nodeMenuSubscription = MenuItemsProvider.GetMenu(NodeMenuContext).Subscribe(items => { _nodeMenuItems = items; @@ -91,10 +106,26 @@ protected override void OnInitialized() protected override async Task OnInitializedAsync() { await base.OnInitializedAsync(); - await NavigationService.InitializeAsync(); - // Only resolve side panel content if already visible — defer until opened otherwise - if (SidePanelState.IsVisible) - await ResolveSidePanelContentAsync(); + // Synchronous (no await): Initialize() only wires Rx subscriptions; a Task + // awaited in OnInitializedAsync would deadlock the circuit's sync-context. + NavigationService.Initialize(); + + // Snapshot auth state. If the user signed out (or arrived anonymous) with a + // previously-persisted IsVisible=true, force the panel closed before any + // child component subscribes to a workspace it can't access. + if (AuthStateTask is not null) + { + var authState = await AuthStateTask; + isAuthenticated = authState.User?.Identity?.IsAuthenticated == true; + } + if (!isAuthenticated && SidePanelState.IsVisible) + { + SidePanelState.SetVisible(false); + } + + // Only resolve side panel content if visible AND authenticated. + if (isAuthenticated && SidePanelState.IsVisible) + ResolveSidePanelContent(); } /// @@ -105,7 +136,7 @@ private string? CurrentNodeName { get { - var node = NavigationService.Context?.Node; + var node = _currentNavContext?.Node; return node?.Name ?? node?.Id; } } @@ -142,6 +173,20 @@ private void NavigateToSettings() NavigationManager.NavigateTo(url); } + /// + /// Navigates to the current user's Activity dashboard — the canonical + /// "all my threads" surface (Latest Threads section already filters out + /// Done threads by default; type content.status:Done in the search + /// box to surface them). + /// + private void NavigateToThreads() + { + var userId = AccessService?.Context?.ObjectId; + if (string.IsNullOrEmpty(userId)) + return; + NavigationManager.NavigateTo($"/User/{userId}/Activity"); + } + /// /// Handles a click on a dynamic menu item. /// Uses Href for absolute navigation when set, otherwise constructs URL from Area. @@ -259,8 +304,13 @@ private async Task RestoreSidePanelStateAsync() var saved = await jsModule!.InvokeAsync("loadSidePanelState"); if (saved != null) { + // Anonymous circuits must never restore a visible panel — workspace + // access fails for them and the panel children throw on render. + if (!isAuthenticated && saved.IsVisible) + saved = saved with { IsVisible = false }; SidePanelState.State = saved; - await ResolveSidePanelContentAsync(); + if (isAuthenticated) + ResolveSidePanelContent(); StateHasChanged(); } } @@ -294,7 +344,7 @@ private void OnSidePanelStateChanged() { InvokeAsync(async () => { - await ResolveSidePanelContentAsync(); + ResolveSidePanelContent(); await SaveSidePanelStateAsync(); StateHasChanged(); @@ -312,10 +362,22 @@ private void OnSidePanelStateChanged() }); } - private void OnNavigationContextChanged(NavigationContext? _) + private NavigationContext? _currentNavContext; + + private void OnNavigationContextChanged(NavigationContext? ctx) { - // Context changed (user navigated) — invalidate cached side panel control - // so next render picks up the new context path + _currentNavContext = ctx; + + // A thread lives in EITHER the main view OR the side panel, never both. + // When navigation puts a thread full-screen in the main view, close the + // side panel (mirrors MoveToMainPanel, which does the same explicitly). + if (ctx?.Node != null + && ThreadNodeType.IsThreadNodeType(ctx.Node.NodeType) + && SidePanelState.IsVisible) + { + SidePanelState.SetVisible(false); + } + InvokeAsync(StateHasChanged); } @@ -343,12 +405,13 @@ protected void CloseMobileNavMenu() } - public bool IsSidePanelVisible => SidePanelState.IsVisible; + // Side panel is gated on auth — anonymous users see neither toggle nor pane. + public bool IsSidePanelVisible => isAuthenticated && SidePanelState.IsVisible; protected SidePanelPosition SidePanelPositionValue => SidePanelState.Position; public async Task ToggleSidePanel() { - var context = NavigationService.Context; + var context = _currentNavContext; // Check if viewing a thread full-screen by checking the node's NodeType if (context?.Node != null && ThreadNodeType.IsThreadNodeType(context.Node.NodeType)) @@ -396,7 +459,7 @@ private async Task EnsureJsModuleAsync() // OnInitialized, re-seeding the context attachment chip) when navigation moves // to a different node. Without this, the ThreadChatView stays stuck on the // InitialContext it was first rendered with. - private string sidePanelContentKey => $"newchat-{NavigationService.Context?.PrimaryPath ?? string.Empty}"; + private string sidePanelContentKey => $"newchat-{_currentNavContext?.PrimaryPath ?? string.Empty}"; private ThreadChatControl? _cachedSidePanelControl; private string? _cachedContentPath; private string? _cachedContextPath; @@ -409,7 +472,7 @@ private async Task EnsureJsModuleAsync() /// If the content path points to a node that no longer exists (e.g. deleted thread), /// the path resolves to a parent with satellite segments as remainder — detect and clear. /// - private async Task ResolveSidePanelContentAsync() + private void ResolveSidePanelContent() { var contentPath = SidePanelState.ContentPath; if (contentPath == resolvedSidePanelPath) @@ -423,31 +486,34 @@ private async Task ResolveSidePanelContentAsync() return; } - var resolution = await PathResolver.ResolvePathAsync(contentPath); - if (resolution == null) + // Reactive — Subscribe, never await on PathResolver chain (deadlock surface; + // see Doc/Architecture/AsynchronousCalls.md). + PathResolver.ResolvePath(contentPath).Subscribe(resolution => { - // Node doesn't exist at all — clear stale content path - sidePanelViewModel = null; - SidePanelState.SetContentPath(null); - resolvedSidePanelPath = null; - return; - } + if (resolution == null) + { + sidePanelViewModel = null; + SidePanelState.SetContentPath(null); + resolvedSidePanelPath = null; + InvokeAsync(StateHasChanged); + return; + } - // If the resolved prefix doesn't match the content path, it means the node - // no longer exists and resolution fell back to a parent (e.g. _Thread/id became - // remainder on the parent hub → invalid area). Clear the stale path. - if (!string.Equals(resolution.Prefix, contentPath, StringComparison.OrdinalIgnoreCase) - && !string.IsNullOrEmpty(resolution.Remainder)) - { - sidePanelViewModel = null; - SidePanelState.SetContentPath(null); - resolvedSidePanelPath = null; - return; - } + if (!string.Equals(resolution.Prefix, contentPath, StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrEmpty(resolution.Remainder)) + { + sidePanelViewModel = null; + SidePanelState.SetContentPath(null); + resolvedSidePanelPath = null; + InvokeAsync(StateHasChanged); + return; + } - var (area, id) = ParseSidePanelRemainder(resolution.Remainder); - var reference = new LayoutAreaReference(area) { Id = id ?? "" }; - sidePanelViewModel = Controls.LayoutArea((Address)resolution.Prefix, reference); + var (area, id) = ParseSidePanelRemainder(resolution.Remainder); + var reference = new LayoutAreaReference(area) { Id = id ?? "" }; + sidePanelViewModel = Controls.LayoutArea((Address)resolution.Prefix, reference); + InvokeAsync(StateHasChanged); + }); } private static (string? Area, string? Id) ParseSidePanelRemainder(string? remainder) @@ -462,7 +528,7 @@ private static (string? Area, string? Id) ParseSidePanelRemainder(string? remain private ThreadChatControl GetSidePanelControl() { - var context = NavigationService.Context; + var context = _currentNavContext; var contextPath = context?.PrimaryPath; var contentPath = SidePanelState.ContentPath ?? string.Empty; @@ -486,7 +552,7 @@ public void Dispose() { SidePanelState.OnStateChanged -= OnSidePanelStateChanged; NavigationService.SidePanelNavigationRequested -= OnSidePanelNavigation; - NavigationService.OnNavigationContextChanged -= OnNavigationContextChanged; + _navContextSubscription?.Dispose(); _nodeMenuSubscription?.Dispose(); _meshMenuSubscription?.Dispose(); dotNetRef?.Dispose(); @@ -512,3 +578,5 @@ protected static bool IsEmoji(string? value) return true; } } + + diff --git a/src/MeshWeaver.Blazor.Portal/Layout/PortalLayoutBase.razor.css b/src/MeshWeaver.Blazor.Portal/Layout/PortalLayoutBase.razor.css index 6e9706099..4842dd016 100644 --- a/src/MeshWeaver.Blazor.Portal/Layout/PortalLayoutBase.razor.css +++ b/src/MeshWeaver.Blazor.Portal/Layout/PortalLayoutBase.razor.css @@ -31,7 +31,12 @@ height: 100vh; width: 100vw; display: grid; - grid-template-columns: auto 1fr; + /* minmax(0, 1fr): the main column must be allowed to SHRINK below its + min-content size. A bare 1fr keeps a min-content floor, so the + splitter's pixel-sized panes (or wide content) force the whole grid + wider than the viewport and push the `auto` nav column off the left + edge — the "broken left" after toggling the side panel. */ + grid-template-columns: auto minmax(0, 1fr); grid-template-rows: auto auto 1fr; grid-template-areas: "icon head" @@ -40,13 +45,14 @@ background-color: var(--fill-color); color: var(--neutral-foreground-rest); transition: grid-template-columns 0.3s ease; + overflow: hidden; } /* When side panel is visible on desktop, main area contains the splitter */ ::deep.layout.panel-visible.panel-position-right, ::deep.layout.panel-visible.panel-position-left, ::deep.layout.panel-visible.panel-position-bottom { - grid-template-columns: auto 1fr; + grid-template-columns: auto minmax(0, 1fr); grid-template-areas: "icon head" "nav messagebar" @@ -214,7 +220,7 @@ max-width: 100vw; } - ::deep .body-splitter fluent-multi-splitter-pane { + ::deep .body-splitter .fluent-multi-splitter-pane { overflow-x: clip !important; overflow-y: auto !important; } @@ -249,7 +255,7 @@ height: 100vh; width: 100vw; display: grid; - grid-template-columns: auto 1fr; + grid-template-columns: auto minmax(0, 1fr); grid-template-rows: auto auto auto 1fr; grid-template-areas: "icon head" @@ -258,6 +264,7 @@ "main main"; background-color: var(--fill-color); color: var(--neutral-foreground-rest); + overflow: hidden; } /* Mobile bottom position */ @@ -361,14 +368,14 @@ /* Desktop layout - Side panel with FluentMultiSplitter */ @media (min-width: 768px) { /* Style the splitter panes */ - ::deep .body-splitter fluent-multi-splitter-pane { + ::deep .body-splitter .fluent-multi-splitter-pane { overflow: hidden; display: flex; flex-direction: column; } /* Ensure pane content wrapper fills the pane and can shrink */ - ::deep .body-splitter fluent-multi-splitter-pane > div { + ::deep .body-splitter .fluent-multi-splitter-pane > div { width: 100%; height: 100%; min-width: 0; diff --git a/src/MeshWeaver.Blazor.Portal/MeshWeaver.Blazor.Portal.csproj b/src/MeshWeaver.Blazor.Portal/MeshWeaver.Blazor.Portal.csproj index 355e1b977..ee1871756 100644 --- a/src/MeshWeaver.Blazor.Portal/MeshWeaver.Blazor.Portal.csproj +++ b/src/MeshWeaver.Blazor.Portal/MeshWeaver.Blazor.Portal.csproj @@ -2,10 +2,14 @@ {8b7e3c9a-5f2d-4e1b-a8c3-6d9f0e2b1a4c} _content/MeshWeaver.Blazor.Portal + $(NoWarn);NU1510 + + + diff --git a/src/MeshWeaver.Blazor.Portal/Pages/CreateNode.razor b/src/MeshWeaver.Blazor.Portal/Pages/CreateNode.razor index 4db47ce16..96b9dd83e 100644 --- a/src/MeshWeaver.Blazor.Portal/Pages/CreateNode.razor +++ b/src/MeshWeaver.Blazor.Portal/Pages/CreateNode.razor @@ -1,6 +1,9 @@ @page "/create" +@using System.Reactive.Linq @using System.Text.RegularExpressions @using MeshWeaver.Blazor.Portal.Chat +@using MeshWeaver.Data +@using MeshWeaver.Graph @using MeshWeaver.Mesh @using MeshWeaver.Mesh.Services @using MeshWeaver.Mesh.Security @@ -87,7 +90,7 @@ @* Action buttons *@
    @if (isCreating) { @@ -193,7 +196,6 @@ try { var meshConfig = Hub.ServiceProvider.GetService(); - var nodeTypeService = Hub.ServiceProvider.GetService(); var defaultNs = NamespaceParam ?? ParentParam ?? NavigationService.CurrentNamespace ?? ""; @@ -217,7 +219,7 @@ // Pre-select type: from ?type= param, or default to Markdown var defaultType = TypeParam ?? "Markdown"; - await PreSelectAsync(defaultType, v => selectedTypeItem = v); + PreSelect(defaultType, v => selectedTypeItem = v); // --- Location picker: queries for namespaces --- var nsQueries = !string.IsNullOrEmpty(NamespaceQueriesParam) @@ -236,7 +238,7 @@ // Pre-select location from default namespace if (!string.IsNullOrEmpty(defaultNs)) - await PreSelectAsync(defaultNs, v => selectedLocation = v); + PreSelect(defaultNs, v => selectedLocation = v); } catch (Exception ex) { @@ -260,20 +262,20 @@ selectedLocation = location; } - private async Task PreSelectAsync(string path, Action setter) + private void PreSelect(string path, Action setter) { - try - { - var node = await MeshService.QueryAsync($"path:{path}").FirstOrDefaultAsync(); - if (node != null) - setter(new QuerySuggestion(node.Path, node.Name ?? node.Id, node.NodeType, 1.0, node.Icon)); - else - setter(new QuerySuggestion(path, path, null, 1.0)); - } - catch - { - setter(new QuerySuggestion(path, path, null, 1.0)); - } + // Reactive — Subscribe, never await on a hub round-trip (100% deadlock; + // see Doc/Architecture/AsynchronousCalls.md). + Hub.GetMeshNode(path, TimeSpan.FromSeconds(5)) + .Catch(_ => Observable.Return(null)) + .Subscribe(node => + { + var suggestion = node != null + ? new QuerySuggestion(node.Path, node.Name ?? node.Id, node.NodeType, 1.0, node.Icon) + : new QuerySuggestion(path, path, null, 1.0); + setter(suggestion); + InvokeAsync(StateHasChanged); + }); } private string BackHref @@ -285,7 +287,8 @@ } } - private async Task CreateNodeAsync() + // Sync — the body only Subscribes; never awaits a hub round-trip. + private void CreateNodeAction() { if (selectedTypeItem == null || string.IsNullOrWhiteSpace(nodeName) || selectedLocation == null) return; @@ -318,17 +321,28 @@ DesiredId = id }; - await MeshService.CreateNodeAsync(node); - NavigationService.NavigateTo($"/{nodePath}/Edit"); + // Subscribe to CreateNode (IObservable) — await on hub-backed + // writes deadlocks the hub pump (see AsynchronousCalls.md). + MeshService.CreateNode(node).Subscribe( + _ => + { + isCreating = false; + NavigationService.NavigateTo($"/{nodePath}/Edit"); + }, + ex => + { + errorMessage = ex.Message.Contains("Access denied") || ex.Message.Contains("Unauthorized") + ? "You do not have permission to create nodes in this namespace." + : $"Failed to create node: {ex.Message}"; + isCreating = false; + _ = InvokeAsync(StateHasChanged); + }); } catch (Exception ex) { errorMessage = ex.Message.Contains("Access denied") || ex.Message.Contains("Unauthorized") ? "You do not have permission to create nodes in this namespace." : $"Failed to create node: {ex.Message}"; - } - finally - { isCreating = false; } } diff --git a/src/MeshWeaver.Blazor.Portal/PortalNodeType.cs b/src/MeshWeaver.Blazor.Portal/PortalNodeType.cs index e03971d6f..51e5055ae 100644 --- a/src/MeshWeaver.Blazor.Portal/PortalNodeType.cs +++ b/src/MeshWeaver.Blazor.Portal/PortalNodeType.cs @@ -1,4 +1,5 @@ -using MeshWeaver.AI; +using MeshWeaver.AI; +using MeshWeaver.Messaging; using MeshWeaver.ContentCollections; using MeshWeaver.Graph.Security; using MeshWeaver.Mesh; @@ -25,7 +26,7 @@ public static TBuilder AddPortalType(this TBuilder builder) where TBui builder.ConfigureServices(services => { services.AddSingleton(sp => - new SatelliteAccessRule(NodeType, sp.GetService() ?? new NullSecurityService())); + new SatelliteAccessRule(NodeType, sp.GetRequiredService())); return services; }); return builder; @@ -41,7 +42,6 @@ public static TBuilder AddPortalType(this TBuilder builder) where TBui Name = "Portal Session", IsSatelliteType = true, ExcludeFromContext = new HashSet { "search", "create" }, - AssemblyLocation = typeof(PortalNodeType).Assembly.Location, HubConfiguration = config => { config.TypeRegistry.AddAITypes(); diff --git a/src/MeshWeaver.Blazor.Portal/SidePanel/SidePanel.razor.cs b/src/MeshWeaver.Blazor.Portal/SidePanel/SidePanel.razor.cs index 590914bfd..cc1453aea 100644 --- a/src/MeshWeaver.Blazor.Portal/SidePanel/SidePanel.razor.cs +++ b/src/MeshWeaver.Blazor.Portal/SidePanel/SidePanel.razor.cs @@ -42,6 +42,13 @@ private void OnStateChanged() private void OnNewThread() { + // Clear the active thread so the new-chat composer renders. This must happen + // here (the always-mounted panel), not only via RequestAction: when a thread + // is displayed the panel body is a LayoutAreaView, so no ThreadChatView is + // subscribed to OnActionRequested and the click would otherwise do nothing + // ("clicking + keeps me on the thread"). + SidePanelState.SetContentPath(null); + // Notify a mounted composer to reset its view mode (e.g. Resume → Chat). SidePanelState.RequestAction("New"); } diff --git a/src/MeshWeaver.Blazor.Radzen/RadzenPivotGridView.razor.cs b/src/MeshWeaver.Blazor.Radzen/RadzenPivotGridView.razor.cs index 91e1e5276..d90b8ceac 100644 --- a/src/MeshWeaver.Blazor.Radzen/RadzenPivotGridView.razor.cs +++ b/src/MeshWeaver.Blazor.Radzen/RadzenPivotGridView.razor.cs @@ -6,6 +6,8 @@ namespace MeshWeaver.Blazor.Radzen; public partial class RadzenPivotGridView : RadzenViewBase { + [Inject] private DynamicTypeGenerator TypeGenerator { get; set; } = null!; + private PivotConfiguration? Configuration { get; set; } private object? RawData { get; set; } private Type? DataItemType { get; set; } @@ -38,7 +40,7 @@ protected override void BindData() // Generate a type based on the configuration using DynamicTypeGenerator var properties = GetPropertiesFromConfiguration(Configuration); - DataItemType = DynamicTypeGenerator.GenerateType(properties); + DataItemType = TypeGenerator.GenerateType(properties); // Deserialize to the generated type var listType = typeof(List<>).MakeGenericType(DataItemType); diff --git a/src/MeshWeaver.Blazor.Radzen/RadzenServiceExtensions.cs b/src/MeshWeaver.Blazor.Radzen/RadzenServiceExtensions.cs index 4e3f8af85..4575523a6 100644 --- a/src/MeshWeaver.Blazor.Radzen/RadzenServiceExtensions.cs +++ b/src/MeshWeaver.Blazor.Radzen/RadzenServiceExtensions.cs @@ -13,6 +13,9 @@ public static class RadzenServiceExtensions public static IServiceCollection AddRadzenServices(this IServiceCollection services) { services.AddRadzenComponents(); + // Instance type-generator (memoization cache lives and dies with this + // ServiceProvider — no process-wide static cache, see NoStaticState.md). + services.AddSingleton(); return services; } } diff --git a/src/MeshWeaver.Blazor/BlazorView.razor.cs b/src/MeshWeaver.Blazor/BlazorView.razor.cs index fa113ac92..f6ba7a69a 100644 --- a/src/MeshWeaver.Blazor/BlazorView.razor.cs +++ b/src/MeshWeaver.Blazor/BlazorView.razor.cs @@ -7,6 +7,7 @@ using MeshWeaver.Layout; using MeshWeaver.Layout.Client; using MeshWeaver.Layout.DataGrid; +using MeshWeaver.Mesh; using MeshWeaver.Messaging; using Microsoft.AspNetCore.Components; using Microsoft.Extensions.Logging; @@ -22,6 +23,10 @@ public class BlazorView : ComponentBase, IAsyncDisposable [Inject] protected ILogger Logger { get; set; } = null!; [Inject] protected PortalApplication PortalApplication { get; set; } = null!; [Inject] protected IJSRuntime JSRuntime { get; set; } = null!; + // The circuit's AccessService — CircuitAccessHandler sets the clicking user's identity on it. Used to + // stamp user-driven messages (ClickedEvent) so they don't lose identity crossing the sync-stream + // boundary (Stream.Hub is the sync hub, whose AccessService has no user context). + [Inject] protected AccessService AccessService { get; set; } = null!; protected IMessageHub Hub => PortalApplication.Hub; [Parameter] public required TViewModel ViewModel { get; set; } @@ -64,7 +69,7 @@ protected virtual void BindDataAfterParameterReset() private bool _viewDisposed; /// True after has been entered. Subscription callbacks - /// can check this to avoid invoking on a dead renderer. + /// can check this to avoid invoking StateHasChanged on a dead renderer.
     protected bool IsViewDisposed => _viewDisposed; public virtual ValueTask DisposeAsync() @@ -108,7 +113,43 @@ protected void DataBind( { try { - if (Model is not null && !reference.Pointer.StartsWith('/')) + // Node-bound DataContext: read the field straight off the node stream (the + // process-wide IMeshNodeStreamCache) instead of a /data replica. ONE source of + // truth — see LayoutAreaReference.MeshNodePrefix and Doc/GUI/DataBinding. + if (LayoutAreaReference.TryParseMeshNodeDataContext(DataContext) is { } meshNode + && !reference.Pointer.StartsWith('/')) + { + bindings.Add(MeshNodeBindingExtensions + .Bind(Hub, meshNode.NodePath, meshNode.BindContent, meshNode.SubPath, reference) + .Subscribe(v => + { + if (_viewDisposed) return; + try + { + InvokeAsync(() => + { + if (_viewDisposed) return; + try + { + setter(Hub.ConvertSingle(v, conversion, defaultValue)); + RequestStateChange(); + } + catch (ObjectDisposedException) { /* renderer gone */ } + catch (Exception ex) + { + Logger.LogError(ex, "Error setting node-bound property value in Area {area}", Area); + } + }); + } + catch (ObjectDisposedException) { /* renderer gone */ } + catch (Exception ex) + { + Logger.LogError(ex, "Error scheduling node-bound property update in Area {area}", Area); + } + }, + ex => Logger.LogError(ex, "Node-bound binding faulted for '{pointer}' in Area {area}", reference.Pointer, Area))); + } + else if (Model is not null && !reference.Pointer.StartsWith('/')) { var convertedValue = Hub.ConvertSingle(Model.GetValueFromModel(reference), conversion, defaultValue); setter(convertedValue); @@ -145,6 +186,43 @@ protected void DataBind( { Logger.LogError(ex, "Error scheduling bound property update in Area {area}", Area); } + }, + // 🚨 A bound stream that FAULTS must be SURFACED, never swallowed: a + // Subscribe with no onError leaves the fault unobserved, the property + // never gets a value, and the control spins forever — the "gui is just + // hanging" symptom (atioz 2026-06-21, when the data stream OnError'd from + // the AccessContext storm). Mirror the node-bound branch above: log it, + // then render the DEFAULT value on the UI thread so the control draws + // (empty/zeroed) instead of hanging. ObjectDisposedException is a benign + // teardown artifact (navigation / component swap) — Debug, not surfaced. + ex => + { + if (_viewDisposed || ex is ObjectDisposedException) + { + Logger.LogDebug(ex, "Suppressed teardown error binding '{pointer}' in Area {area}", reference.Pointer, Area); + return; + } + Logger.LogWarning(ex, + "Data binding for '{pointer}' in Area {area} faulted — rendering default so the control does not hang", + reference.Pointer, Area); + try + { + InvokeAsync(() => + { + if (_viewDisposed) return; + try + { + setter(Hub.ConvertSingle(null, conversion, defaultValue)); + RequestStateChange(); + } + catch (ObjectDisposedException) { /* renderer gone */ } + catch (Exception inner) + { + Logger.LogError(inner, "Error setting default after binding fault in Area {area}", Area); + } + }); + } + catch (ObjectDisposedException) { /* renderer gone */ } } ) ); @@ -221,6 +299,16 @@ protected string SubArea(string area) protected virtual void UpdatePointer(object? value, JsonPointerReference reference) { + // Node-bound DataContext: write the edited field straight back to the node stream + // (per-field read-modify-write through IMeshNodeStreamCache). No /data replica, no + // server-side save subscription — see LayoutAreaReference.MeshNodePrefix. + if (LayoutAreaReference.TryParseMeshNodeDataContext(DataContext) is { } meshNode + && !reference.Pointer.StartsWith('/')) + { + MeshNodeBindingExtensions.Write(Hub, Logger, meshNode.NodePath, meshNode.BindContent, meshNode.SubPath, reference, value); + return; + } + if(Stream is null) throw new InvalidOperationException("Stream must be set before updating pointers."); Stream.UpdatePointer(value, DataContext ?? "/", reference, Model); @@ -229,6 +317,13 @@ protected virtual void UpdatePointer(object? value, JsonPointerReference referen protected virtual void BindData() { + // ViewModel is declared `required` but Blazor's parameter pipeline can + // still feed null through transient binding races — most reliably during + // thread-launch / chat-side-panel re-render where the upstream Stream is + // being torn down and a new ViewModel hasn't landed yet. Without this + // guard the user sees a NullReferenceException at the .Id access below. + if (ViewModel is null) return; + DataBind(ViewModel.Id, x => x.Id); DataBind(ViewModel.Class, x => x.Class); DataBind(ViewModel.Style, x => x.Style); @@ -246,7 +341,17 @@ protected virtual void OnClick() { if(Stream is null) throw new InvalidOperationException("Stream must be set before sending click events."); - Stream.Hub.Post(new ClickedEvent(Area, Stream.StreamId), o => o.WithTarget(Stream.Owner)); + // ClickedEvent is a USER-driven message — it must carry the clicking user's identity. Stream.Hub + // is the sync hub, whose AccessService has no user context (the circuit's AccessService holds it), + // so a bare Post lands context-less → PostPipeline fails closed → the click's downstream write is + // denied and the synced area blanks. Stamp the circuit user's AccessContext explicitly; it then + // travels with the delivery to the owning hub (Phase 2 rule 1 keeps an explicit context). + // See Doc/Architecture/AccessContextPropagation.md → "sync stream protocol vs user-data". + var userContext = AccessService?.Context ?? AccessService?.CircuitContext; + Stream.Hub.Post(new ClickedEvent(Area, Stream.StreamId), o => + userContext is not null + ? o.WithTarget(Stream.Owner).WithAccessContext(userContext) + : o.WithTarget(Stream.Owner)); } diff --git a/src/MeshWeaver.Blazor/BlazorViewRegistry.cs b/src/MeshWeaver.Blazor/BlazorViewRegistry.cs index c96d4f37b..b1a5a8329 100644 --- a/src/MeshWeaver.Blazor/BlazorViewRegistry.cs +++ b/src/MeshWeaver.Blazor/BlazorViewRegistry.cs @@ -17,7 +17,6 @@ using MeshWeaver.Markdown.Export.Configuration; using MeshWeaver.Mesh; using MeshWeaver.Messaging; -using Microsoft.DotNet.Interactive.Formatting; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using static MeshWeaver.Layout.Client.LayoutClientConfiguration; @@ -125,7 +124,7 @@ ItemTemplateControl itemTemplate MeshNodeCardControl card => StandardView(card, stream, area), AppearanceControl appearance => StandardView(appearance, stream, area), ThreadMessageBubbleControl bubble => StandardView(bubble, stream, area), - _ => DelegateToDotnetInteractive(instance, stream, area), + _ => FallbackHtml(instance, stream, area), }; } catch (Exception ex) @@ -177,14 +176,13 @@ private static ViewDescriptor MapSkinnedView(UiControl control, ISynchronization } - private static ViewDescriptor DelegateToDotnetInteractive( + private static ViewDescriptor FallbackHtml( object instance, ISynchronizationStream? stream, string area ) { - var mimeType = Formatter.GetPreferredMimeTypesFor(instance.GetType()).FirstOrDefault() ?? "text/html"; - var output = Controls.Html(instance.ToDisplayString(mimeType)); + var output = Controls.Html(System.Net.WebUtility.HtmlEncode(instance.ToString() ?? string.Empty)); return new ViewDescriptor( typeof(HtmlView), ImmutableDictionary diff --git a/src/MeshWeaver.Blazor/Components/ChatSubmissionHandler.cs b/src/MeshWeaver.Blazor/Components/ChatSubmissionHandler.cs index 80a0b653d..01fe1b307 100644 --- a/src/MeshWeaver.Blazor/Components/ChatSubmissionHandler.cs +++ b/src/MeshWeaver.Blazor/Components/ChatSubmissionHandler.cs @@ -15,11 +15,8 @@ public enum SubmissionState WaitingForResponse } - private readonly TimeSpan _timeout; private readonly TimeSpan _dedupWindow; private readonly Func _now; - private readonly Func _scheduleTimeout; - private IDisposable? _timeoutDisposable; private bool _disposed; private DateTime? _lastAcceptedAt; @@ -43,19 +40,10 @@ public enum SubmissionState ///
    public int SubmissionCount { get; private set; } - /// - /// Creates a ChatSubmissionHandler with a configurable timeout. - /// - /// Timeout after which a stuck submission auto-releases. Default 30s. - /// Optional scheduler for testing. If null, uses Task.Delay. public ChatSubmissionHandler( - TimeSpan? timeout = null, - Func? scheduleTimeout = null, TimeSpan? dedupWindow = null, Func? now = null) { - _timeout = timeout ?? TimeSpan.FromSeconds(30); - _scheduleTimeout = scheduleTimeout ?? DefaultScheduleTimeout; _dedupWindow = dedupWindow ?? TimeSpan.FromMilliseconds(500); _now = now ?? (() => DateTime.UtcNow); } @@ -100,7 +88,6 @@ public bool TryBeginSubmit(string? text) /// /// Transitions from Submitting to WaitingForResponse after the message has been posted. - /// Starts the timeout timer. /// public void OnMessagePosted() { @@ -108,7 +95,6 @@ public void OnMessagePosted() return; State = SubmissionState.WaitingForResponse; - StartTimeout(); } /// @@ -117,7 +103,6 @@ public void OnMessagePosted() /// public void OnResponseAppeared() { - CancelTimeout(); State = SubmissionState.Idle; } @@ -126,47 +111,11 @@ public void OnResponseAppeared() ///     public void ForceRelease() { - CancelTimeout(); State = SubmissionState.Idle; } - private void StartTimeout() - { - CancelTimeout(); - _timeoutDisposable = _scheduleTimeout(_timeout, OnTimeout); - } - - private void OnTimeout() - { - if (State != SubmissionState.Idle) - { - State = SubmissionState.Idle; - } - } - - private void CancelTimeout() - { - _timeoutDisposable?.Dispose(); - _timeoutDisposable = null; - } - - private static IDisposable DefaultScheduleTimeout(TimeSpan delay, Action callback) - { - var cts = new CancellationTokenSource(); - _ = Task.Delay(delay, cts.Token).ContinueWith(t => - { - if (!t.IsCanceled) - callback(); - }, TaskScheduler.Default); - return cts; - } - public void Dispose() { - if (_disposed) - return; - _disposed = true; - CancelTimeout(); } } diff --git a/src/MeshWeaver.Blazor/Components/CollaborativeMarkdownView.razor.cs b/src/MeshWeaver.Blazor/Components/CollaborativeMarkdownView.razor.cs index 831dfb96c..e1a732669 100644 --- a/src/MeshWeaver.Blazor/Components/CollaborativeMarkdownView.razor.cs +++ b/src/MeshWeaver.Blazor/Components/CollaborativeMarkdownView.razor.cs @@ -53,17 +53,41 @@ public partial class CollaborativeMarkdownView private bool _showPageCommentInput; private string _pageCommentText = ""; - // Interactive markdown (kernel execution) + // Interactive markdown (kernel execution). The "kernel address" is now a + // per-view Activity MeshNode path (`{owner}/_Activity/markdown-{kernelId}`) + // — its hub hosts the kernel handlers via ActivityNodeType.HubConfiguration. private readonly string _kernelId = Guid.NewGuid().AsString(); private Address? _kernelAddress; - private Address KernelAddress => _kernelAddress ??= AddressExtensions.CreateKernelAddress(_kernelId); + private Address KernelAddress => _kernelAddress ??= ResolveActivityAddress(); private bool _codeSubmitted; private IReadOnlyCollection? _codeSubmissions; + private Address ResolveActivityAddress() + { + var ownerPath = BoundNodePath ?? Stream?.Owner?.Path; + var activityNamespace = string.IsNullOrEmpty(ownerPath) + ? "_Activity" + : $"{ownerPath}/_Activity"; + return new Address($"{activityNamespace}/markdown-{_kernelId}"); + } + // Parsed data private string? _processedHtml; private List Annotations = new(); + // Memoize the last successful render so ProcessContent is idempotent — repeated + // calls with the same (RawContent, CurrentViewMode) skip the Markdig parse entirely. + // Saves ~1 ms per redundant call on a medium-sized doc. + private string? _lastRenderedContent; + private string? _lastRenderedMode; + + // Reads + writes go through IMeshNodeStreamCache — process-wide shared + // handle per path. SaveContentAsync calls _cache.Update(BoundNodePath, fn) + // to push edits through the same handle the read subscription is on. The + // cache stays alive for the process; this view just holds a reference to + // call Update later. See Doc/GUI/ItemTemplateMeshNodeStreamBinding. + private IMeshNodeStreamCache? _cache; + // Comment data cache (markerId -> Comment), populated by mesh query subscription private Dictionary commentNodes = new(); // Comment path cache (markerId -> MeshNode path), for resolve/delete operations @@ -107,19 +131,19 @@ protected override void BindData() var accessService = Hub.ServiceProvider.GetService(); CurrentAuthor = (accessService?.Context ?? accessService?.CircuitContext)?.Name ?? ""; - // Subscribe to workspace stream for reactive content updates. - // When workspace.UpdateMeshNode() injects comment markers, the stream - // pushes the updated node and the view re-renders with the new content. - if (!string.IsNullOrEmpty(BoundHubAddress)) + // Long-standing subscription to the per-node stream via the process-wide + // IMeshNodeStreamCache. Hold the cache reference so SaveContentAsync can + // call _cache.Update(BoundNodePath, fn) — writes go through the same + // shared handle the read subscription is on, so every reader observes + // the patch in order. See Doc/GUI/ItemTemplateMeshNodeStreamBinding. + if (!string.IsNullOrEmpty(BoundNodePath)) { - var workspace = Hub.GetWorkspace(); - var remoteStream = workspace.GetRemoteStream(new Address(BoundHubAddress)); - if (remoteStream != null) + try { - AddBinding(remoteStream - .Select(nodes => nodes?.FirstOrDefault(n => n.Path == BoundNodePath)) - .Where(n => n != null) - .Select(n => MarkdownOverviewLayoutArea.GetMarkdownContent(n)) + _cache = Hub.ServiceProvider.GetRequiredService(); + AddBinding(Hub.GetMeshNodeStream(BoundNodePath) + .Where(node => node is not null) + .Select(node => MarkdownOverviewLayoutArea.GetMarkdownContent(node)) .DistinctUntilChanged() .Subscribe(content => { @@ -131,9 +155,10 @@ protected override void BindData() } })); } - else + catch { - // Fallback: one-time bind from ViewModel + // Cache service unavailable — fall back to one-time bind from + // the ViewModel. No live updates in this mode. DataBind(ViewModel.Value, x => x.RawContent, defaultValue: ""); } } @@ -160,7 +185,7 @@ private void SubscribeToCommentStatuses() var query = MeshQueryRequest.FromQuery( $"namespace:{BoundNodePath}/_Comment nodeType:Comment"); - AddBinding(meshQuery.ObserveQuery(query) + AddBinding(meshQuery.Query(query) .Scan(new List(), (list, change) => { if (change.ChangeType == QueryChangeType.Initial || change.ChangeType == QueryChangeType.Reset) @@ -221,11 +246,14 @@ protected override async Task OnAfterRenderAsync(bool firstRender) await jsModule.InvokeVoidAsync("enableCommentSelection", containerRef, dotNetRef); } - // Submit code to kernel — the mesh routing rule creates the hub on demand. + // Submit code to the per-view Activity hub (which hosts the kernel). if (!_codeSubmitted && _codeSubmissions is { Count: > 0 }) { _codeSubmitted = true; - MarkdownViewLogic.SubmitCode(Hub, KernelAddress, _codeSubmissions); + var meshService = Hub.ServiceProvider.GetRequiredService(); + var ownerPath = BoundNodePath ?? Stream?.Owner?.Path; + MarkdownViewLogic.CreateActivityAndSubmit( + Hub, meshService, KernelAddress, ownerPath, _kernelId, _codeSubmissions); } } @@ -235,9 +263,19 @@ private void ProcessContent() { _processedHtml = ""; Annotations = new(); + _lastRenderedContent = ""; + _lastRenderedMode = CurrentViewMode; return; } + // Skip the parse when nothing observable to the rendered HTML changed. + // Filter changes route through OnCommentFilterChanged → StateHasChanged + // (no re-parse), but other lifecycle events can still call here. + if (RawContent == _lastRenderedContent + && CurrentViewMode == _lastRenderedMode + && _processedHtml != null) + return; + // Extract annotations for the side panel Annotations = AnnotationParser.ExtractAnnotations(RawContent); @@ -255,6 +293,9 @@ private void ProcessContent() // Replace kernel address placeholder with actual kernel address if (_processedHtml != null && _codeSubmissions is { Count: > 0 }) _processedHtml = MarkdownViewLogic.ReplaceKernelPlaceholder(_processedHtml, KernelAddress); + + _lastRenderedContent = RawContent; + _lastRenderedMode = CurrentViewMode; } /// @@ -300,11 +341,11 @@ private void OnViewModeChanged(string mode) StateHasChanged(); } - // Comment filter + // Comment filter only affects the FilteredAnnotations computed property + // (side panel) — the rendered HTML is independent of it, so no re-parse. private void OnCommentFilterChanged(string filter) { CurrentCommentFilter = filter; - ProcessContent(); StateHasChanged(); } @@ -367,11 +408,19 @@ private Task PostContentUpdateAsync(string newContent) try { - var workspace = Hub.ServiceProvider.GetRequiredService(); - workspace.UpdateMeshNode( - node => node with { Content = new MarkdownContent { Content = newContent } }, - new Address(BoundHubAddress), - BoundNodePath); + // Push the edit through the process-wide IMeshNodeStreamCache.Update. + // The cache routes the write through the SAME shared handle the read + // subscription is on, so the echo flows back to this view's Subscribe + // and re-renders without an extra read. Other GUIs watching the same + // path see the patch through their own subscriptions on the same handle. + if (_cache == null || string.IsNullOrEmpty(BoundNodePath)) return Task.FromResult(false); + Hub.GetMeshNodeStream(BoundNodePath).Update(current => + current with { Content = new MarkdownContent { Content = newContent } }) + .Subscribe( + _ => { }, + ex => Hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.Blazor.CollaborativeMarkdownView") + .LogWarning(ex, "Content save failed for {Path}", BoundNodePath)); return Task.FromResult(true); } catch @@ -455,15 +504,21 @@ private void SubmitSelectionComment() if (delivery != null) { - Hub.RegisterCallback(delivery, response => - { - if (!response.Message.Success) - { - var logger = Hub.ServiceProvider.GetService()?.CreateLogger("MeshWeaver.Blazor.CollaborativeMarkdownView"); - logger?.LogWarning("[SubmitComment] FAILED: {Error}", response.Message.Error); - } - return response; - }); + Hub.Observe(delivery) + .Subscribe( + response => + { + if (response.Message is CreateCommentResponse { Success: false } resp) + { + var logger = Hub.ServiceProvider.GetService()?.CreateLogger("MeshWeaver.Blazor.CollaborativeMarkdownView"); + logger?.LogWarning("[SubmitComment] FAILED: {Error}", resp.Error); + } + }, + ex => + { + var logger = Hub.ServiceProvider.GetService()?.CreateLogger("MeshWeaver.Blazor.CollaborativeMarkdownView"); + logger?.LogWarning(ex, "[SubmitComment] FAILED"); + }); } } @@ -501,15 +556,21 @@ private void SubmitPageComment() if (delivery != null) { - Hub.RegisterCallback(delivery, response => - { - if (!response.Message.Success) - { - var logger = Hub.ServiceProvider.GetService()?.CreateLogger("MeshWeaver.Blazor.CollaborativeMarkdownView"); - logger?.LogWarning("[SubmitPageComment] FAILED: {Error}", response.Message.Error); - } - return response; - }); + Hub.Observe(delivery) + .Subscribe( + response => + { + if (response.Message is CreateCommentResponse { Success: false } resp) + { + var logger = Hub.ServiceProvider.GetService()?.CreateLogger("MeshWeaver.Blazor.CollaborativeMarkdownView"); + logger?.LogWarning("[SubmitPageComment] FAILED: {Error}", resp.Error); + } + }, + ex => + { + var logger = Hub.ServiceProvider.GetService()?.CreateLogger("MeshWeaver.Blazor.CollaborativeMarkdownView"); + logger?.LogWarning(ex, "[SubmitPageComment] FAILED"); + }); } } @@ -520,27 +581,31 @@ private void SubmitPageComment() private bool IsResolved(string markerId) => commentNodes.TryGetValue(markerId, out var c) && c.Status == CommentStatus.Resolved; - private async Task ResolveComment(string markerId) + private void ResolveComment(string markerId) { - if (!commentPaths.TryGetValue(markerId, out var path) || string.IsNullOrEmpty(BoundHubAddress)) + if (!commentPaths.TryGetValue(markerId, out var path)) return; - var meshQuery = Hub.ServiceProvider.GetService(); - if (meshQuery == null) return; - var node = await meshQuery.QueryAsync($"path:{path}").FirstOrDefaultAsync(); - if (node?.Content is Comment comment) + // Write through the shared cache — the lambda fires against the live + // MeshNode the cache holds, no separate Read → Post round-trip needed. + Hub.GetMeshNodeStream(path).Update(n => { - var updated = node with { Content = comment with { Status = CommentStatus.Resolved } }; - Hub.Post(new UpdateNodeRequest(updated), o => o.WithTarget(new Address(BoundHubAddress))); - } + if (n.Content is not Comment c) return n; + return n with { Content = c with { Status = CommentStatus.Resolved } }; + }).Subscribe( + _ => { }, + ex => Hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.Blazor.CollaborativeMarkdownView") + .LogWarning(ex, "Comment resolve failed for {Path}", path)); } - private async Task DeleteComment(string markerId) + private void DeleteComment(string markerId) { if (!commentPaths.TryGetValue(markerId, out var path)) return; var meshQuery = Hub.ServiceProvider.GetService(); - if (meshQuery == null) return; - await meshQuery.DeleteNodeAsync(path); + meshQuery?.DeleteNode(path).Subscribe( + _ => { }, + _ => { }); } private static string Truncate(string text, int maxLength) => diff --git a/src/MeshWeaver.Blazor/Components/CompileProgressIndicator.razor b/src/MeshWeaver.Blazor/Components/CompileProgressIndicator.razor new file mode 100644 index 000000000..f418e8d9d --- /dev/null +++ b/src/MeshWeaver.Blazor/Components/CompileProgressIndicator.razor @@ -0,0 +1,42 @@ +@* Code-behind moved to CompileProgressIndicator.razor.cs *@ + +@if (CompilingPath is not null) +{ +
    + + + Compiling @CompilingPath + @if (!string.IsNullOrEmpty(ProgressMessage)) + { + — @ProgressMessage + } + @if (Seconds > 0) + { + (@Seconds s) + } + … + +
    +} +else if (ErrorPath is not null) +{ + @* Surface a failed compile so a layout area that won't load tells the user + WHY instead of showing an indefinite blank/spinner. *@ +
    + + + Compilation failed for @ErrorPath + @if (!string.IsNullOrEmpty(CompileError)) + { + — @CompileError + } + +
    +} +else if (StreamError is not null) +{ +
    + + Compile status unavailable — @StreamError +
    +} diff --git a/src/MeshWeaver.Blazor/Components/CompileProgressIndicator.razor.cs b/src/MeshWeaver.Blazor/Components/CompileProgressIndicator.razor.cs new file mode 100644 index 000000000..7153fc886 --- /dev/null +++ b/src/MeshWeaver.Blazor/Components/CompileProgressIndicator.razor.cs @@ -0,0 +1,179 @@ +using System.Reactive.Linq; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using MeshWeaver.Messaging; +using Microsoft.AspNetCore.Components; +using Microsoft.Extensions.DependencyInjection; + +namespace MeshWeaver.Blazor.Components; + +/// +/// Code-behind for . When a +/// NodeTypePath is provided, subscribes via +/// IMeshNodeStreamCache.GetStream on that path and surfaces +/// progress whenever NodeTypeDefinition.CompilationStatus = Compiling. +/// Without a path, falls back to the global synced +/// nodeType:NodeType query. +/// +/// The single-path mode follows the canonical cache pattern: one +/// upstream subscription per path, shared across every GUI watching the +/// same NodeType. See Doc/GUI/ItemTemplateMeshNodeStreamBinding. +/// +public partial class CompileProgressIndicator : IDisposable +{ + [Inject] private IMessageHub Hub { get; set; } = default!; + + /// + /// Optional: restrict the indicator to a specific NodeType path. When set, + /// the indicator only surfaces compilation progress for that exact path. + /// When null, it watches the synced nodeType:NodeType query and + /// reports the first NodeType whose + /// is + /// . + /// + [Parameter] public string? NodeTypePath { get; set; } + + private string? CompilingPath; + private string? ProgressMessage; + private string? ErrorPath; + private string? CompileError; + private string? StreamError; + private int Seconds; + private IDisposable? _statusSub; + private IDisposable? _activitySub; + private IDisposable? _tickSub; + + /// + /// Snapshot of what the watched NodeType(s) are doing — drives the render + /// branches: an in-flight compile, a terminal failure, or nothing. + /// + private sealed record CompileState( + string? Path, CompilationStatus? Status, string? ActivityPath, string? Error); + + protected override void OnInitialized() + { + // Surface BOTH an in-flight compile (spinner + live activity message) + // AND a terminal failure (the CompilationError text). The activity path + // is written onto the NodeType at compile start + // (NodeTypeCompileActivityHandler → LastCompilationActivityPath), so it is + // live throughout the compile — we follow it to surface real progress. + // 🚨 Errors are surfaced, never swallowed: a stuck/blank layout area must + // tell the user WHY it isn't loading (compile failed) instead of showing + // an indefinite spinner with no message. + IObservable compileObs; + if (!string.IsNullOrEmpty(NodeTypePath)) + { + // Single-NodeType mode: subscribe to that node's live stream + // through IMeshNodeStreamCache — process-wide shared handle, + // joined by every GUI watching the same NodeType. + compileObs = Hub.GetMeshNodeStream(NodeTypePath) + .Select(n => n?.Content is NodeTypeDefinition def + && def.CompilationStatus is CompilationStatus.Compiling or CompilationStatus.Error + ? new CompileState(NodeTypePath, def.CompilationStatus, + def.LastCompilationActivityPath, def.CompilationError) + : new CompileState(null, null, null, null)); + } + else + { + // Global mode: synced NodeType query — replaces the old 1-second + // INodeTypeService.GetCompilingPaths poll. Query is the right + // primitive here (set of nodes); single-node cache wouldn't apply. + // Prefer an in-flight compile; fall back to a failed one so a layout + // area that won't load surfaces the compile error rather than a blank. + var workspace = Hub.GetWorkspace(); + compileObs = workspace.GetQuery("nodetypes-compiling", "nodeType:NodeType") + .Select(snapshot => + { + var node = snapshot.FirstOrDefault(n => n.Content is NodeTypeDefinition d + && d.CompilationStatus == CompilationStatus.Compiling) + ?? snapshot.FirstOrDefault(n => n.Content is NodeTypeDefinition d + && d.CompilationStatus == CompilationStatus.Error); + return node?.Content is NodeTypeDefinition def + ? new CompileState(node.Path, def.CompilationStatus, + def.LastCompilationActivityPath, def.CompilationError) + : new CompileState(null, null, null, null); + }); + } + + _statusSub = compileObs + .DistinctUntilChanged() + .Subscribe( + state => + { + var compiling = state.Status == CompilationStatus.Compiling; + CompilingPath = compiling ? state.Path : null; + ErrorPath = state.Status == CompilationStatus.Error ? state.Path : null; + CompileError = state.Status == CompilationStatus.Error ? state.Error : null; + ProgressMessage = null; + StreamError = null; + Seconds = 0; + StartOrStopTicker(compiling); + SubscribeToActivity(state.ActivityPath, compiling); + InvokeAsync(StateHasChanged); + }, + // 🚨 Surface, don't swallow. A faulted status stream means we can + // no longer report compile state — say so instead of going blank. + ex => + { + StreamError = ex.Message; + CompilingPath = null; + ErrorPath = null; + InvokeAsync(StateHasChanged); + }); + } + + /// + /// Follow the compile and surface its live message + /// tail ("starting Roslyn", "Roslyn produced assembly", "Release created") so + /// the user sees real progress, not just a blind spinner. Best-effort: a + /// missing/slow activity simply leaves the generic "Compiling …" text. + /// + private void SubscribeToActivity(string? activityPath, bool active) + { + _activitySub?.Dispose(); + _activitySub = null; + if (!active || string.IsNullOrEmpty(activityPath)) return; + + _activitySub = Hub.GetMeshNodeStream(activityPath) + .Select(n => (n?.Content as ActivityLog)?.Messages is { Count: > 0 } msgs + ? msgs[^1].Message + : null) + .DistinctUntilChanged() + .Subscribe( + msg => + { + ProgressMessage = msg; + InvokeAsync(StateHasChanged); + }, + // Activity progress is best-effort, but still surface the fault as + // the progress tail rather than swallowing it silently. + ex => + { + ProgressMessage = $"(compile progress unavailable: {ex.Message})"; + InvokeAsync(StateHasChanged); + }); + } + + private void StartOrStopTicker(bool active) + { + _tickSub?.Dispose(); + _tickSub = null; + if (!active) return; + _tickSub = Observable.Interval(TimeSpan.FromSeconds(1)) + .Subscribe(_ => + { + Seconds++; + InvokeAsync(StateHasChanged); + }); + } + + public void Dispose() + { + _statusSub?.Dispose(); + _activitySub?.Dispose(); + _tickSub?.Dispose(); + } +} diff --git a/src/MeshWeaver.Blazor/Components/CompileProgressIndicator.razor.css b/src/MeshWeaver.Blazor/Components/CompileProgressIndicator.razor.css new file mode 100644 index 000000000..31e8bfefa --- /dev/null +++ b/src/MeshWeaver.Blazor/Components/CompileProgressIndicator.razor.css @@ -0,0 +1,33 @@ +.compile-progress { + display: flex; + align-items: center; + gap: 12px; + padding: 12px 16px; + margin: 8px 0; + background: var(--neutral-layer-2); + border: 1px solid var(--neutral-stroke-rest); + border-radius: 6px; + color: var(--neutral-foreground-hint); + font-size: 0.9rem; + font-family: var(--body-font); +} + +.compile-progress strong { + color: var(--neutral-foreground-rest); + font-weight: 600; +} + +.compile-progress.compile-error { + background: var(--neutral-layer-2); + border-color: var(--error, #d13438); + color: var(--error, #d13438); +} + +.compile-progress.compile-error strong { + color: var(--error, #d13438); +} + +.compile-error-icon { + font-size: 1.1rem; + line-height: 1; +} diff --git a/src/MeshWeaver.Blazor/Components/DispatchView.razor b/src/MeshWeaver.Blazor/Components/DispatchView.razor index 5066bafb5..d5243c2d8 100644 --- a/src/MeshWeaver.Blazor/Components/DispatchView.razor +++ b/src/MeshWeaver.Blazor/Components/DispatchView.razor @@ -12,18 +12,35 @@ else if(ViewDescriptor != null) { - @((RenderFragment)(builder => - { - builder.OpenComponent(0, ViewDescriptor.Type); - var index = 0; + @* Every UiControl is rendered through here, so this is the ONE place to contain a + render-time exception. A control that throws inside BuildRenderTime (e.g. an enum + property bound to an unparseable literal in DataGridView.RenderPropertyColumn) would + otherwise escape the render, tear down the whole Blazor circuit, and hang the page + with no feedback (atioz 2026-06-21). The ErrorBoundary catches it, SURFACES the + message in the control's own slot, and lets the rest of the page render. *@ + + + @((RenderFragment)(builder => + { + builder.OpenComponent(0, ViewDescriptor.Type); + var index = 0; - foreach (var parameter in ViewDescriptor.Parameters) - { - builder.AddAttribute(++index, parameter.Key, parameter.Value); - } + foreach (var parameter in ViewDescriptor.Parameters) + { + builder.AddAttribute(++index, parameter.Key, parameter.Value); + } - builder.CloseComponent(); - })) + builder.CloseComponent(); + })) + + +
    + This view failed to render. +
    @renderError.Message
    +
    +     +     } diff --git a/src/MeshWeaver.Blazor/Components/EditFormView.razor.cs b/src/MeshWeaver.Blazor/Components/EditFormView.razor.cs index 1edc44b1c..5ae62e101 100644 --- a/src/MeshWeaver.Blazor/Components/EditFormView.razor.cs +++ b/src/MeshWeaver.Blazor/Components/EditFormView.razor.cs @@ -1,4 +1,5 @@ -using System.Text.Json; +using System.Reactive.Linq; +using System.Text.Json; using MeshWeaver.Data; using MeshWeaver.Layout.Client; using Microsoft.AspNetCore.Components.Forms; @@ -20,22 +21,31 @@ protected override void BindData() ); } - private async void Submit(EditContext context) + private void Submit(EditContext context) { - if(Stream is null) + if (Stream is null) throw new InvalidOperationException("Stream must be set before submitting the form."); - var log = await Stream.SubmitModel(model!); - if(log.Status == ActivityStatus.Succeeded) - { - Log = null; - ShowSuccess(); - Reset(); - } - else + + // Subscribe — do NOT bridge to Task / await. The hub round-trip stays + // observable end-to-end (see Doc/Architecture/AsynchronousCalls.md). + Stream.SubmitModel(model!).Subscribe(log => { - Log = log; - ShowError(); - } + InvokeAsync(() => + { + if (log.Status == ActivityStatus.Succeeded) + { + Log = null; + ShowSuccess(); + Reset(); + } + else + { + Log = log; + ShowError(); + } + StateHasChanged(); + }); + }); } private ModelParameter Convert(JsonElement jsonObject) diff --git a/src/MeshWeaver.Blazor/Components/ExportDocumentView.razor b/src/MeshWeaver.Blazor/Components/ExportDocumentView.razor index 2a2284f20..ae64c5bff 100644 --- a/src/MeshWeaver.Blazor/Components/ExportDocumentView.razor +++ b/src/MeshWeaver.Blazor/Components/ExportDocumentView.razor @@ -1,8 +1,14 @@ +@using System.Reactive.Linq +@using System.Reactive.Threading.Tasks +@using System.Text.Json +@using MeshWeaver.Data @using MeshWeaver.Layout @using MeshWeaver.Markdown.Export.Configuration @using MeshWeaver.Markdown.Export.Messaging @using MeshWeaver.Messaging @using MeshWeaver.Mesh +@using MeshWeaver.Mesh.Services +@using Microsoft.Extensions.DependencyInjection @using Microsoft.JSInterop @inherits BlazorView @@ -145,7 +151,7 @@ } } - private async Task ExportAsync() + private void ExportAsync() { if (string.IsNullOrEmpty(SourcePath)) return; @@ -154,38 +160,80 @@ _done = false; StateHasChanged(); - try + var options = new DocumentExportOptions { - var options = new DocumentExportOptions - { - Format = _format, - Title = string.IsNullOrWhiteSpace(_title) ? null : _title, - BrandNodePath = string.IsNullOrWhiteSpace(_brandNodePath) ? null : _brandNodePath, - CoverPage = _coverPage, - TableOfContents = _toc, - PageBreakBeforeH1 = _pbH1, - PageBreakBeforeH2 = _pbH2, - IncludeChildren = _includeChildren, - PageBreakBetweenChildren = _pbChildren, - HeaderOverride = string.IsNullOrWhiteSpace(_header) ? null : _header, - FooterOverride = string.IsNullOrWhiteSpace(_footer) ? null : _footer - }; - - var response = await Hub.AwaitResponse( - new ExportDocumentRequest(SourcePath, options), - o => o.WithTarget(new MeshWeaver.Messaging.Address(SourcePath))); + Format = _format, + Title = string.IsNullOrWhiteSpace(_title) ? null : _title, + BrandNodePath = string.IsNullOrWhiteSpace(_brandNodePath) ? null : _brandNodePath, + CoverPage = _coverPage, + TableOfContents = _toc, + PageBreakBeforeH1 = _pbH1, + PageBreakBeforeH2 = _pbH2, + IncludeChildren = _includeChildren, + PageBreakBetweenChildren = _pbChildren, + HeaderOverride = string.IsNullOrWhiteSpace(_header) ? null : _header, + FooterOverride = string.IsNullOrWhiteSpace(_footer) ? null : _footer + }; - if (response?.Message is { } result && result.Content.Length > 0) + // Two-step subscription per the just-start dispatch contract: + // 1. Observe the ExportDocumentResponse — handler returns immediately + // with the activity path; no wait-for-terminal in the hub handler. + // 2. Subscribe to the activity stream, project to ActivityLog, + // filter on terminal status, take(1), deserialize the rendered + // bytes from ActivityLog.ReturnValue, trigger the browser download. + // Pure observable composition — no `await` on hub-touching streams. + Hub.Observe( + new ExportDocumentRequest(SourcePath, options), + o => o.WithTarget(new MeshWeaver.Messaging.Address(SourcePath))) + .Take(1) + .SelectMany(dispatch => { - using var ms = new MemoryStream(result.Content); - using var streamRef = new DotNetStreamReference(ms); - await JSRuntime.InvokeVoidAsync("meshweaverDownloadFileFromStream", result.FileName, streamRef); + var msg = dispatch.Message; + if (!string.IsNullOrEmpty(msg.Error)) + return Observable.Throw(new InvalidOperationException(msg.Error)); + if (string.IsNullOrEmpty(msg.ActivityPath)) + return Observable.Throw( + new InvalidOperationException("Export handler returned no activity path")); - _done = true; + // Subscribe to the activity node via IMeshNodeStreamCache — + // process-wide shared handle; the export progress GUI elsewhere + // joins the same upstream. See Doc/GUI/ItemTemplateMeshNodeStreamBinding. + var cache = Hub.ServiceProvider.GetRequiredService(); + return cache.GetStream(msg.ActivityPath, Hub.JsonSerializerOptions) + .Select(n => n?.Content as ActivityLog) + .Where(log => log is not null && log.Status != ActivityStatus.Running) + .Take(1) + .Select(log => log!); + }) + .Subscribe( + terminal => _ = InvokeAsync(() => OnExportTerminalAsync(terminal)), + ex => _ = InvokeAsync(() => OnExportErrorAsync(ex.Message))); + } + + private async Task OnExportTerminalAsync(ActivityLog terminal) + { + try + { + if (terminal.Status == ActivityStatus.Succeeded + && terminal.ReturnValue is { } el) + { + var rendered = el.Deserialize(Hub.JsonSerializerOptions); + if (rendered is { } r && r.Content.Length > 0) + { + using var ms = new MemoryStream(r.Content); + using var streamRef = new DotNetStreamReference(ms); + await JSRuntime.InvokeVoidAsync("meshweaverDownloadFileFromStream", r.FileName, streamRef); + _done = true; + } + else + { + _error = "Export script returned no content."; + } } else { - _error = response?.Message?.Error ?? "Empty response from the export handler."; + var lastError = terminal.Messages.LastOrDefault(m => m.LogLevel >= Microsoft.Extensions.Logging.LogLevel.Warning)?.Message; + _error = lastError ?? $"Export {terminal.Status}"; } } catch (Exception ex) @@ -199,6 +247,13 @@ } } + private void OnExportErrorAsync(string message) + { + _error = $"Export failed: {message}"; + _isExporting = false; + StateHasChanged(); + } + private void Reset() { _done = false; diff --git a/src/MeshWeaver.Blazor/Components/LayoutAreaView.razor b/src/MeshWeaver.Blazor/Components/LayoutAreaView.razor index a40a29a98..e4c951dfb 100644 --- a/src/MeshWeaver.Blazor/Components/LayoutAreaView.razor +++ b/src/MeshWeaver.Blazor/Components/LayoutAreaView.razor @@ -1,8 +1,25 @@ @using MeshWeaver.Layout @inherits BlazorView -
    - +
    + @* Surface NodeType compile progress when the stream hasn't emitted yet so the + user sees "Compiling (Ns)…" instead of an indefinite spinner. *@ + @if (!IsContentLoaded) + { + + } + @* Guard: Blazor's parameter pipeline transiently feeds a null ViewModel (navigation / stream + tear-down — see SetParametersAsync). The render path must mirror that guard, else NamedArea + (which reads ViewModel.SpinnerType) NREs and tears down the whole circuit. A re-render arrives + once ViewModel repopulates. + Do NOT also guard on Area: a null Area is the LEGITIMATE bare-node-URL case — it means + "render the node's default area". The server resolves it and stores the indirection under + the empty key, and NamedArea.GetArea() normalizes null to "". Guarding on Area blanked every + plain node page portal-wide (regression 7359ba42f, fixed 2026-06-12). *@ + @if (ViewModel is not null) + { + + }
    @* Render dialog if one is active *@ diff --git a/src/MeshWeaver.Blazor/Components/LayoutAreaView.razor.cs b/src/MeshWeaver.Blazor/Components/LayoutAreaView.razor.cs index 6517344c6..d91517660 100644 --- a/src/MeshWeaver.Blazor/Components/LayoutAreaView.razor.cs +++ b/src/MeshWeaver.Blazor/Components/LayoutAreaView.razor.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using MeshWeaver.Data; using MeshWeaver.Layout; +using MeshWeaver.Layout.Composition; using MeshWeaver.Mesh; using MeshWeaver.Blazor.Services; using MeshWeaver.Messaging; @@ -26,17 +27,26 @@ public partial class LayoutAreaView public override async Task SetParametersAsync(ParameterView parameters) { await base.SetParametersAsync(parameters); + // ViewModel is declared `required` but Blazor's parameter pipeline still + // feeds null through transient binding races (navigation, stream tear-down). + // Mirror the guard in BlazorView.BindData — re-renders will arrive with a + // populated ViewModel once the upstream parameter resolves. + if (ViewModel is null) return; BindViewModel(); + var hadStream = AreaStream is not null; if (AreaStream is not null && (!AreaStream.Reference.Equals(ViewModel.Reference) || !AreaStream.Owner.Equals(Address))) { - Logger.LogDebug("LayoutAreaView disposing stale stream for {Address}/{Reference} (parameters changed)", - ViewModel.Address, ViewModel.Reference); + Logger.LogDebug("[LAV] DISPOSE_STALE area={Area} addr={Address} ref={Ref} (parameters changed)", + Area, ViewModel.Address, ViewModel.Reference); AreaStream.Dispose(); AreaStream = null; } + Logger.LogDebug("[LAV] SET_PARAMS area={Area} addr={Address} ref={Ref} preRender={PreRender} hadStream={HadStream} keepStream={KeepStream}", + Area, Address, ViewModel.Reference, !IsNotPreRender, hadStream, AreaStream is not null); + // Only bind stream when already in interactive mode (not during prerender) // try/catch: during navigation, old circuit's hub may already be disposing // while Blazor still re-renders components before their DisposeAsync runs @@ -51,6 +61,7 @@ public override async Task SetParametersAsync(ParameterView parameters) private void BindViewModel() { + if (ViewModel is null) return; DataBind(ViewModel.ProgressMessage, x => x.progressMessage); DataBind(ViewModel.ShowProgress, x => x.showProgress); DataBind(ViewModel.Reference.Layout ?? ViewModel.Reference.Area, x => x.Area); @@ -88,6 +99,7 @@ public override async ValueTask DisposeAsync() MenuStream?.Dispose(); NodeMenuStream?.Dispose(); MeshMenuStream?.Dispose(); + ProgressStream?.Dispose(); } else { @@ -99,6 +111,7 @@ public override async ValueTask DisposeAsync() MenuStream = null; NodeMenuStream = null; MeshMenuStream = null; + ProgressStream = null; await base.DisposeAsync(); } @@ -109,11 +122,13 @@ public override async ValueTask DisposeAsync() MenuStream?.Dispose(); NodeMenuStream?.Dispose(); MeshMenuStream?.Dispose(); + ProgressStream?.Dispose(); AreaStream = null; DialogStream = null; MenuStream = null; NodeMenuStream = null; MeshMenuStream = null; + ProgressStream = null; } // Must match NodeMenuItemsExtensions.NodeMenuContext / MeshMenuContext — duplicated here to avoid @@ -125,13 +140,29 @@ private void BindStream() { if (AreaStream is null) { - - Logger.LogDebug("Acquiring stream for {Owner} and {Reference}", Address!, ViewModel.Reference); - AreaStream = Address!.Equals(Workspace.Hub.Address) + var isLocal = Address!.Equals(Workspace.Hub.Address); + Logger.LogDebug("[LAV] BIND_STREAM area={Area} addr={Address} ref={Ref} mode={Mode}", + Area, Address, ViewModel.Reference, isLocal ? "local" : "remote"); + AreaStream = isLocal ? Workspace.GetStream(ViewModel.Reference)!.Reduce(new JsonPointerReference("/")) : Workspace.GetRemoteStream(Address!, ViewModel.Reference); + if (AreaStream is null) + Logger.LogWarning("[LAV] BIND_STREAM_NULL area={Area} addr={Address} ref={Ref} — GetRemoteStream returned null", + Area, Address, ViewModel.Reference); DialogStream = SetupDialogAreaMonitoring(AreaStream!); DialogStream?.RegisterForDisposal(DialogStream.DistinctUntilChanged().Subscribe(el => OnDialogStreamChanged(el.Value))); + + // Phase-aware loading label: bind the server-side progress item + // ({ message, progress } in the EntityStore "data" collection — seeded + // "Building layout…" by LayoutAreaHost.BuildInitialization, advanced by + // the framework milestones / host.UpdateProgress, cleared on first + // render). Riding the SAME area stream means the static "Subscribing…" + // seed is replaced by live phases as soon as the first frame lands — + // pure display, derived only from data already on the stream. + ProgressStream = AreaStream!.Reduce( + new JsonPointerReference(LayoutAreaReference.GetDataPointer(LayoutAreaHost.ProgressDataId))); + ProgressStream?.RegisterForDisposal(ProgressStream.DistinctUntilChanged() + .Subscribe(el => OnProgressStreamChanged(el.Value))); if (Top) { MenuStream = SetupMenuAreaMonitoring(AreaStream!, MenuControl.MenuArea); @@ -150,6 +181,39 @@ private void BindStream() private ISynchronizationStream? MenuStream { get; set; } private ISynchronizationStream? NodeMenuStream { get; set; } private ISynchronizationStream? MeshMenuStream { get; set; } + private ISynchronizationStream? ProgressStream { get; set; } + + /// + /// Applies a server-side progress-item change ({ message, progress }) to the + /// loading label. Non-empty message → show it (the area is still assembling: + /// "Building layout…", "Initializing data sources…", "Rendering…", or a + /// view-pushed phase via host.UpdateProgress). Empty message = content + /// rendered — NamedAreaView hides the spinner via RootControl, so nothing to + /// do. Pure display: derived exclusively from data already on the stream. + /// + private void OnProgressStreamChanged(JsonElement progressData) + { + try + { + if (!IsNotPreRender) + return; + if (progressData.ValueKind != JsonValueKind.Object) + return; + var message = progressData.TryGetProperty("message", out var m) + && m.ValueKind == JsonValueKind.String + ? m.GetString() + : null; + if (string.IsNullOrEmpty(message) || message == progressMessage) + return; + progressMessage = message; + showProgress = true; + InvokeAsync(StateHasChanged); + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Error processing loading-progress stream change for area {Area}", Area); + } + } private ISynchronizationStream? SetupDialogAreaMonitoring(ISynchronizationStream areaStream) { @@ -227,11 +291,11 @@ protected override async Task OnAfterRenderAsync(bool firstRender) if (firstRender) { IsNotPreRender = true; + Logger.LogDebug("[LAV] FIRST_RENDER area={Area} addr={Address} ref={Ref} hasStream={HasStream}", + Area, Address, ViewModel?.Reference, AreaStream != null); // If we're now rendered and we don't have a stream yet, bind it if (AreaStream == null) { - Logger.LogDebug("LayoutAreaView first interactive render — binding stream for {Area} ({Address}/{Reference})", - Area, Address, ViewModel?.Reference); BindStream(); StateHasChanged(); } diff --git a/src/MeshWeaver.Blazor/Components/LayoutAreaView.razor.css b/src/MeshWeaver.Blazor/Components/LayoutAreaView.razor.css index e1282cfc1..51dd848e5 100644 --- a/src/MeshWeaver.Blazor/Components/LayoutAreaView.razor.css +++ b/src/MeshWeaver.Blazor/Components/LayoutAreaView.razor.css @@ -11,3 +11,18 @@ display: flex; flex-direction: column; } + +/* Default: a top-level (page) area fills the full available height. + The container already fills (.body-content is a definite-height flex column + and .layout-area-container is flex:1), but the single control rendered inside + would otherwise take only its natural height — so page backgrounds "stop in the + middle", and fixed-height leaves (code view, pdf, editors) don't reach the + bottom. Stretching the top-level content gives it the full height to fill or + scroll within. Only applies to top-level pages (Top=true) and only once the + content has loaded, so embedded areas and the loading spinner are untouched. + Authors who set an explicit height still win: flex-basis:auto reads it, and + flex-grow merely lets it reach the bottom. */ +.layout-area-container.top-area.content-loaded > * { + flex: 1 1 auto; + min-height: 0; +} diff --git a/src/MeshWeaver.Blazor/Components/MarkdownEditorView.razor b/src/MeshWeaver.Blazor/Components/MarkdownEditorView.razor index ad9d5ed27..3edf57db0 100644 --- a/src/MeshWeaver.Blazor/Components/MarkdownEditorView.razor +++ b/src/MeshWeaver.Blazor/Components/MarkdownEditorView.razor @@ -1,14 +1,22 @@ +@using System.Reactive.Disposables +@using System.Reactive.Linq @using MeshWeaver.Blazor @using MeshWeaver.Blazor.Components.Monaco @using MeshWeaver.Data @using MeshWeaver.Data.Completion +@using MeshWeaver.Data.Serialization @using MeshWeaver.Layout.Client @using MeshWeaver.Markdown @using MeshWeaver.Markdown.Collaboration +@using MeshWeaver.Mesh +@using MeshWeaver.Mesh.Services @using MeshWeaver.Messaging @using Microsoft.Extensions.DependencyInjection @using Microsoft.Extensions.Logging @using MarkdownAnnotationType = MeshWeaver.Markdown.Collaboration.AnnotationType +@using TrackedChange = MeshWeaver.Markdown.Collaboration.TrackedChange +@using TrackedChangeType = MeshWeaver.Markdown.Collaboration.TrackedChangeType +@using TrackedChangeStatus = MeshWeaver.Markdown.Collaboration.TrackedChangeStatus @namespace MeshWeaver.Blazor.Components @inherits BlazorView @@ -37,7 +45,7 @@ ShowBorder="true" CodeEditMode="true" CompletionProvider="@GetCompletionConfig()" - AsyncCompletionCallback="@GetCompletionsAsync" + CompletionCallback="@GetCompletions" ValueChanged="@OnValueChanged" OnEditorReady="@OnEditorReady" OnCommentRequested="@OnCommentFromEditor" /> @@ -147,6 +155,13 @@ // Data pointer for two-way binding private JsonPointerReference? ValuePointer; + // Writes go through the process-wide IMeshNodeStreamCache. + // AutoSaveContentAsync calls _cache.Update(BoundAutoSaveAddress, fn) so + // every reader on that path (this editor + any preview view + any other + // GUI) sees the patch through their own subscription on the same shared + // upstream handle. See Doc/GUI/ItemTemplateMeshNodeStreamBinding. + private IMeshNodeStreamCache? _cache; + protected override void BindData() { base.BindData(); @@ -158,11 +173,18 @@ autoSaveHandler = new AutoSaveHandler(AutoSaveThrottleInterval, AutoSaveContent); AddBinding(autoSaveHandler); - // Bind Value with reactive subscription - if (Stream != null && ValuePointer != null) + // Bind Value with reactive subscription. A node-bound DataContext (/$meshNode/…) must read + // straight off the node stream via MeshNodeBindingExtensions (BindNodeBoundValue) — the SAME seam + // every form control inherits through BlazorView.DataBind. Binding the layout-area Stream with a node-bound + // pointer hands the Base64Url node-path segment to JsonSerializer.Deserialize inside + // LayoutExtensions.GetStream, which throws ("'Q' is an invalid start of a value") and tears + // down the whole Blazor circuit. + if (ValuePointer != null) { - AddBinding(Stream - .DataBind(ValuePointer, DataContext) + var valueStream = BindNodeBoundValue(ValuePointer) + ?? Stream?.DataBind(ValuePointer, DataContext); + if (valueStream != null) + AddBinding(valueStream .Subscribe(value => { if (value != null && value != AnnotatedContent) @@ -200,6 +222,22 @@ // Auto-save configuration (direct assignment, not bindable) BoundAutoSaveAddress = ViewModel.AutoSaveAddress; + + // Resolve the process-wide IMeshNodeStreamCache so AutoSaveContentAsync + // can call _cache.Update(BoundAutoSaveAddress, fn) on every save without + // re-resolving. Resolves to the same instance every Blazor view sees + // (singleton on the mesh hub's service provider). + if (!string.IsNullOrEmpty(BoundAutoSaveAddress)) + { + try + { + _cache = Hub.ServiceProvider.GetRequiredService(); + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Failed to resolve IMeshNodeStreamCache for auto-save at {Address}", BoundAutoSaveAddress); + } + } BoundNodePath = ViewModel.NodePath; // Resolve current user for annotation metadata @@ -253,8 +291,31 @@ await UpdateContent(newContent); } + /// + /// Node-bound read seam for the editor's single Value pointer. Returns a live, string-converted + /// stream of the field off the node stream when the DataContext is node-bound (via + /// ), or null so the caller falls back to the + /// layout-area Stream.DataBind. Routing a node-bound pointer through the layout Stream + /// instead crashes in LayoutExtensions.GetStream<T> ("'Q' is an invalid start of a value"). + /// + private IObservable? BindNodeBoundValue(JsonPointerReference pointer) + { + if (LayoutAreaReference.TryParseMeshNodeDataContext(DataContext) is not { } meshNode + || pointer.Pointer.StartsWith('/')) + return null; + return MeshNodeBindingExtensions + .Bind(Hub, meshNode.NodePath, meshNode.BindContent, meshNode.SubPath, pointer) + .Select(v => Hub.ConvertSingle(v, null)); + } + private string? GetInitialValue() { + // Node-bound contexts are seeded by the reactive subscription above (MeshNodeBindingExtensions + // replays the node's current field value on subscribe); a synchronous Stream.GetDataBoundValue + // would crash on the Base64Url node-path pointer segment, just like the reactive path did. + if (ViewModel.Value is JsonPointerReference pointer && MeshNodeBindingExtensions.IsNodeBound(DataContext, pointer)) + return null; + if (Stream == null || ViewModel.Value == null) return null; @@ -270,38 +331,58 @@ }; } - private async Task GetCompletionsAsync(string query) + /// + /// Streams completions through the local hub. Posts a single AutocompleteRequest; + /// the handler in AgentsApplicationExtensions ScanTopN-folds provider results and + /// posts a fresh AutocompleteResponse for each new snapshot. Each response is + /// projected into a top-N CompletionItem array and pushed downstream so Monaco + /// streams the suggest widget as items arrive. + /// + private IObservable> GetCompletions(string query) { if (Hub == null || string.IsNullOrWhiteSpace(query)) - return []; + return Observable.Return>(Array.Empty()); - try + return Observable.Create>(observer => { - var request = new AutocompleteRequest(query, BoundNodePath); - var response = await Hub.AwaitResponse( - request, - o => o.WithTarget(Hub.Address), - default); - - if (response.Message is not AutocompleteResponse autocompleteResponse) - return []; - - return autocompleteResponse.Items - .Select(item => new CompletionItem + try + { + var request = new AutocompleteRequest(query, BoundNodePath); + var delivery = Hub.Post(request, o => o.WithTarget(Hub.Address)); + if (delivery == null) { - Label = item.Label, - InsertText = item.InsertText, - Description = item.Description, - Category = item.Category, - Kind = MapKind(item.Kind) - }) - .ToArray(); - } - catch (Exception ex) - { - Logger.LogWarning(ex, "Failed to get autocomplete suggestions for query: {Query}", query); - return []; - } + observer.OnError(new InvalidOperationException("Autocomplete: Hub.Post returned null.")); + observer.OnCompleted(); + return Disposable.Empty; + } + Hub.Observe(delivery) + .Subscribe( + response => + { + if (response.Message is AutocompleteResponse ar) + { + observer.OnNext(ar.Items + .Select(item => new CompletionItem + { + Label = item.Label, + InsertText = item.InsertText, + Description = item.Description, + Category = item.Category, + Kind = MapKind(item.Kind) + }) + .ToArray()); + } + }, + ex => Logger.LogWarning(ex, "Autocomplete failed for query: {Query}", query)); + } + catch (Exception ex) + { + Logger.LogWarning(ex, "Failed to get autocomplete suggestions for query: {Query}", query); + observer.OnNext(Array.Empty()); + observer.OnCompleted(); + } + return Disposable.Empty; + }); } private static CompletionItemKind MapKind(AutocompleteKind kind) => kind switch @@ -419,14 +500,19 @@ try { - // Sync the full MeshNode via the remote stream and patch Content only — - // preserves Name/Icon/Description and avoids the key-mapping failure that - // DataChangeRequest + partial MeshNode triggers on the hosting hub. - var workspace = Hub.ServiceProvider.GetRequiredService(); - workspace.UpdateMeshNode( - node => node with { Content = new MarkdownContent { Content = content } }, - new Address(BoundAutoSaveAddress), - BoundNodePath); + // Push the edit through the process-wide IMeshNodeStreamCache.Update. + // The cache routes the write through the SAME shared MeshNodeStreamHandle + // every reader on this path is subscribed to, so the echo flows back + // through their subscriptions in order. No view-local stream handle + // needed — the cache owns the upstream lifetime. + if (_cache == null) return Task.FromResult(false); + _cache.Update(BoundAutoSaveAddress, + current => current with { Content = new MarkdownContent { Content = content } }, + Hub.JsonSerializerOptions) + .Subscribe( + _ => { }, + ex => Logger.LogWarning(ex, + "Failed to auto-save content to {Address}", BoundAutoSaveAddress)); Logger.LogDebug("Auto-saved content to {Address} for node {Path}", BoundAutoSaveAddress, BoundNodePath); return Task.FromResult(true); } diff --git a/src/MeshWeaver.Blazor/Components/MarkdownHtmlRenderer.cs b/src/MeshWeaver.Blazor/Components/MarkdownHtmlRenderer.cs index 787e101bb..fba419b96 100644 --- a/src/MeshWeaver.Blazor/Components/MarkdownHtmlRenderer.cs +++ b/src/MeshWeaver.Blazor/Components/MarkdownHtmlRenderer.cs @@ -107,7 +107,11 @@ private void RenderNodes(RenderTreeBuilder builder, IEnumerable nodes) case { Name: "div" } when node.GetAttributeValue("class", "").Contains("mermaid"): builder.OpenComponent(1); builder.AddAttribute(2, nameof(Mermaid.Mode), _mode); - builder.AddAttribute(3, nameof(Mermaid.Diagram), node.InnerHtml); + // DeEntitize(InnerText), not InnerHtml: the diagram body is HTML-escaped + // at render time (ExecutableCodeBlockRenderer) so '<' in stereotypes / + // inheritance survives. The Mermaid component sets pre.textContent, which + // needs the decoded literal source — entities must be resolved here. + builder.AddAttribute(3, nameof(Mermaid.Diagram), HtmlEntity.DeEntitize(node.InnerText)); builder.CloseComponent(); break; case { Name: "pre" } when node.ChildNodes.Any(n => n.Name == "code"): diff --git a/src/MeshWeaver.Blazor/Components/MarkdownView.razor.cs b/src/MeshWeaver.Blazor/Components/MarkdownView.razor.cs index 338a9ac9a..58f91f5c4 100644 --- a/src/MeshWeaver.Blazor/Components/MarkdownView.razor.cs +++ b/src/MeshWeaver.Blazor/Components/MarkdownView.razor.cs @@ -1,8 +1,10 @@ using MeshWeaver.Kernel; using MeshWeaver.Markdown; +using MeshWeaver.Mesh.Services; using MeshWeaver.Messaging; using MeshWeaver.ShortGuid; using Microsoft.AspNetCore.Components.Rendering; +using Microsoft.Extensions.DependencyInjection; namespace MeshWeaver.Blazor.Components; @@ -12,17 +14,34 @@ public partial class MarkdownView private object? HtmlRaw { get; set; } private object? CodeSubmissionsRaw { get; set; } private object? ShowReferencesRaw { get; set; } + private object? NodePathRaw { get; set; } private string? Html { get; set; } private IReadOnlyList? CodeSubmissions { get; set; } public bool ShowReferencesSection { get; set; } = true; + // Per-view kernel session id. Used as the Activity MeshNode id below. private readonly string _kernelId = Guid.NewGuid().AsString(); private Address? _kernelAddress; - private Address KernelAddress => _kernelAddress ??= AddressExtensions.CreateKernelAddress(_kernelId); + + // The "kernel address" is now the per-view Activity path + // (`{owner}/_Activity/markdown-{kernelId}`). The Activity hub hosts the + // kernel handlers via `ActivityNodeType.HubConfiguration` + + // `AddKernelSubHubHandlers`. Replaces the legacy `kernel/*` standalone + // hub addressing — replies route through the standard MeshNode chain. + private Address KernelAddress => _kernelAddress ??= ResolveActivityAddress(); private bool _codeSubmitted; + private Address ResolveActivityAddress() + { + var ownerPath = Stream?.Owner?.Path; + var activityNamespace = string.IsNullOrEmpty(ownerPath) + ? "_Activity" + : $"{ownerPath}/_Activity"; + return new Address($"{activityNamespace}/markdown-{_kernelId}"); + } + protected override void BindData() { base.BindData(); @@ -30,15 +49,21 @@ protected override void BindData() DataBind(ViewModel.Html, x => x.HtmlRaw); DataBind(ViewModel.CodeSubmissions, x => x.CodeSubmissionsRaw); DataBind(ViewModel.ShowReferences, x => x.ShowReferencesRaw); + DataBind(ViewModel.NodePath, x => x.NodePathRaw); var markdown = MarkdownViewLogic.CoerceString(MarkdownRaw); Html = MarkdownViewLogic.CoerceString(HtmlRaw); CodeSubmissions = MarkdownViewLogic.CoerceCodeSubmissions(CodeSubmissionsRaw, Hub.JsonSerializerOptions); ShowReferencesSection = MarkdownViewLogic.CoerceBool(ShowReferencesRaw, defaultValue: true); + // Explicit NodePath (set by the producing control) wins over the bound stream's owner. + // Relative @@-embeds resolve against this path; child controls whose stream owner is + // not the authoring node (e.g. a Space's body inside the Overview) rely on it. + var nodePath = MarkdownViewLogic.CoerceString(NodePathRaw) ?? Stream?.Owner?.ToString(); + if (Html is null && !string.IsNullOrEmpty(markdown)) { - var result = MarkdownViewLogic.Render(markdown, Stream?.Owner, Stream?.Owner?.ToString()); + var result = MarkdownViewLogic.Render(markdown, Stream?.Owner, nodePath); Html = result.Html; CodeSubmissions ??= result.CodeSubmissions; } @@ -48,7 +73,7 @@ protected override void BindData() && !string.IsNullOrEmpty(markdown)) { CodeSubmissions = MarkdownViewLogic.ExtractCodeSubmissions( - markdown, Stream?.Owner, Stream?.Owner?.ToString()); + markdown, Stream?.Owner, nodePath); } if (Html is not null && CodeSubmissions is { Count: > 0 }) @@ -62,7 +87,10 @@ protected override async Task OnAfterRenderAsync(bool firstRender) if (firstRender && !_codeSubmitted && CodeSubmissions is { Count: > 0 }) { _codeSubmitted = true; - MarkdownViewLogic.SubmitCode(Hub, KernelAddress, CodeSubmissions); + var meshService = Hub.ServiceProvider.GetRequiredService(); + var ownerPath = Stream?.Owner?.Path; + MarkdownViewLogic.CreateActivityAndSubmit( + Hub, meshService, KernelAddress, ownerPath, _kernelId, CodeSubmissions); } } diff --git a/src/MeshWeaver.Blazor/Components/MenuItemView.razor.css b/src/MeshWeaver.Blazor/Components/MenuItemView.razor.css index 922d26568..8eb9d5c24 100644 --- a/src/MeshWeaver.Blazor/Components/MenuItemView.razor.css +++ b/src/MeshWeaver.Blazor/Components/MenuItemView.razor.css @@ -21,3 +21,9 @@ ::deep .dropdown-menu .menu-item-link:active { background-color: var(--neutral-fill-stealth-active); } + +::deep .dropdown-menu .menu-item-link.active { + background-color: var(--neutral-fill-stealth-hover); + color: var(--accent-foreground-rest); + font-weight: 600; +} diff --git a/src/MeshWeaver.Blazor/Components/MeshNodeCardView.razor b/src/MeshWeaver.Blazor/Components/MeshNodeCardView.razor index 1c1ed4517..a4e6e5f1f 100644 --- a/src/MeshWeaver.Blazor/Components/MeshNodeCardView.razor +++ b/src/MeshWeaver.Blazor/Components/MeshNodeCardView.razor @@ -25,6 +25,14 @@ else private string Initial => !string.IsNullOrEmpty(Title) ? Title[0].ToString().ToUpper() : "?"; + /// + /// Hard cap on the title so a very long name can never overflow the card (a backstop to the + /// CSS single-line ellipsis, which can't kick in when a parent doesn't constrain width). + /// + private string TruncatedTitle => Title?.Length > 64 + ? Title[..63].TrimEnd() + "…" + : Title ?? ""; + private string TruncatedDescription => Description?.Length > 100 ? Description[..97] + "..." : Description ?? ""; @@ -63,7 +71,7 @@ else
    @Initial
    }
    -
    @Title
    +
    @TruncatedTitle
    @if (!string.IsNullOrEmpty(Description)) {
    @TruncatedDescription
    diff --git a/src/MeshWeaver.Blazor/Components/MeshNodeCollectionView.razor.cs b/src/MeshWeaver.Blazor/Components/MeshNodeCollectionView.razor.cs index f6a4916fa..9bb2f6e1f 100644 --- a/src/MeshWeaver.Blazor/Components/MeshNodeCollectionView.razor.cs +++ b/src/MeshWeaver.Blazor/Components/MeshNodeCollectionView.razor.cs @@ -1,3 +1,4 @@ +using System.Reactive.Linq; using System.Text.Json; using System.Text.Json.Nodes; using MeshWeaver.Data; @@ -14,62 +15,79 @@ public partial class MeshNodeCollectionView : BlazorView _items = []; private bool _isLoading = true; + private readonly List _subscriptions = new(); protected override void BindData() { base.BindData(); - _ = LoadItemsAsync(); + LoadItems(); } - private async Task LoadItemsAsync() + private void LoadItems() { + // Tear down any prior live subscriptions before re-binding. + foreach (var s in _subscriptions) s.Dispose(); + _subscriptions.Clear(); + _isLoading = true; - await InvokeAsync(StateHasChanged); + _ = InvokeAsync(StateHasChanged); - try - { - var queries = ViewModel?.Queries ?? []; - if (queries.Length == 0) - { - _items = []; - return; - } - - var tasks = queries.Select(async q => - { - try - { - return await MeshQuery.QueryAsync(q).ToListAsync(); - } - catch - { - return new List(); - } - }); - - var results = await Task.WhenAll(tasks); - _items = results - .SelectMany(r => r) - .GroupBy(n => n.Path) - .Select(g => g.First()) - .ToList(); - } - catch + var queries = ViewModel?.Queries ?? []; + if (queries.Length == 0) { _items = []; + _isLoading = false; + _ = InvokeAsync(StateHasChanged); + return; } - finally + + // Per-query live subscription. The view aggregates the latest snapshots across queries + // (same dedup-by-Path semantics as before) but stays live: any change to the matching + // sets refreshes the view via the Subscribe callback. + var perQueryResults = new Dictionary>(); + foreach (var q in queries) { - _isLoading = false; - await InvokeAsync(StateHasChanged); + var query = q; + var sub = MeshQuery.Query(MeshQueryRequest.FromQuery(query)) + .Subscribe( + change => + { + perQueryResults[query] = MergeQueryChange( + perQueryResults.GetValueOrDefault(query, Array.Empty()), + change); + _items = perQueryResults.Values + .SelectMany(r => r) + .GroupBy(n => n.Path) + .Select(g => g.First()) + .ToList(); + _isLoading = false; + _ = InvokeAsync(StateHasChanged); + }, + _ => { }); + _subscriptions.Add(sub); } } - private async Task DeleteItem(string nodePath) + private static IReadOnlyList MergeQueryChange(IReadOnlyList current, + QueryResultChange change) => change.ChangeType switch + { + QueryChangeType.Initial or QueryChangeType.Reset => change.Items, + QueryChangeType.Added => current.Concat(change.Items).ToList(), + QueryChangeType.Updated => current + .Select(n => change.Items.FirstOrDefault(c => c.Path == n.Path) ?? n) + .ToList(), + QueryChangeType.Removed => current + .Where(n => !change.Items.Any(r => r.Path == n.Path)) + .ToList(), + _ => current + }; + + private void DeleteItem(string nodePath) { var nodeFactory = Hub!.ServiceProvider.GetRequiredService(); - await nodeFactory.DeleteNodeAsync(nodePath); - await LoadItemsAsync(); + nodeFactory.DeleteNode(nodePath).Subscribe( + (bool _) => LoadItems(), + (Exception _) => { }); } private void NavigateToItem(string nodePath) => NavigationManager.NavigateTo($"/{nodePath}"); @@ -118,10 +136,10 @@ private async Task DeleteItem(string nodePath) } /// - /// Removes a sub-entry (role or group) from a node's content and persists the change. - /// Uses the same DataChangeRequest pattern as OverviewLayoutArea.SetupAutoSave. + /// Removes a sub-entry (role or group) from a node's content and persists the change by writing + /// straight back to the node stream (the canonical GetMeshNodeStream(path).Update(...) path). /// - private async Task RemoveSubEntry(MeshNode node, int index) + private void RemoveSubEntry(MeshNode node, int index) { if (node.Content is not JsonElement json) return; @@ -130,7 +148,6 @@ private async Task RemoveSubEntry(MeshNode node, int index) if (jsonObj == null) return; - // Determine which array to modify string? arrayProp = null; if (jsonObj["roles"] is JsonArray) arrayProp = "roles"; else if (jsonObj["groups"] is JsonArray) arrayProp = "groups"; @@ -144,11 +161,9 @@ private async Task RemoveSubEntry(MeshNode node, int index) arr.RemoveAt(index); - // Build updated node with modified content var updatedContent = JsonSerializer.Deserialize(jsonObj.ToJsonString()); var updatedNode = node with { Content = updatedContent }; - // Persist via DataChangeRequest targeting the node's hub (namespace address) if (!string.IsNullOrEmpty(node.Namespace)) { var targetAddress = new Address(node.Namespace); @@ -157,7 +172,7 @@ private async Task RemoveSubEntry(MeshNode node, int index) o => o.WithTarget(targetAddress)); } - await LoadItemsAsync(); + LoadItems(); } private record SubEntry(int Index, string Label, bool IsDenied); diff --git a/src/MeshWeaver.Blazor/Components/MeshNodeContentEditorView.razor b/src/MeshWeaver.Blazor/Components/MeshNodeContentEditorView.razor new file mode 100644 index 000000000..7763c00ad --- /dev/null +++ b/src/MeshWeaver.Blazor/Components/MeshNodeContentEditorView.razor @@ -0,0 +1,40 @@ +@inherits BlazorView +@using MeshWeaver.Graph +@using Microsoft.FluentUI.AspNetCore.Components + +@if (!_loaded) +{ + +} +else if (Fields.Count == 0) +{ + No editable settings. +} +else +{ +
    + @foreach (var f in Fields) + { +
    + @if (f.Kind == MeshNodeEditorFieldKind.Bool) + { + + } + else + { + @f.Label + + } +
    + } +
    +} diff --git a/src/MeshWeaver.Blazor/Components/MeshNodeContentEditorView.razor.cs b/src/MeshWeaver.Blazor/Components/MeshNodeContentEditorView.razor.cs new file mode 100644 index 000000000..c6f54cbc6 --- /dev/null +++ b/src/MeshWeaver.Blazor/Components/MeshNodeContentEditorView.razor.cs @@ -0,0 +1,124 @@ +using System.Reactive.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using MeshWeaver.Graph; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using Microsoft.Extensions.Logging; + +namespace MeshWeaver.Blazor.Components; + +/// +/// Code-behind for — the GUI-client, cache-bound editor +/// for . +/// +/// This is the CORRECT data-binding shape (see Doc/GUI/DataBinding "The Golden Rule"): +/// reads come straight from Hub.GetMeshNodeStream(NodePath) (the process-wide +/// IMeshNodeStreamCache) and every edit writes back through +/// GetMeshNodeStream(NodePath).Update(...) as a per-field read-modify-write patch. There is +/// NO server-side /data replica of the node and NO debounced save subscription +/// (SetupAutoSave) — one source of truth, the node stream itself. The fields to render are +/// declared on the control (computed on the backend), so the view needs no client type registry. +/// +public partial class MeshNodeContentEditorView +{ + private string NodePath { get; set; } = string.Empty; + private bool CanEdit { get; set; } = true; + private IReadOnlyList Fields { get; set; } = Array.Empty(); + private bool _loaded; + private string? _focusedKey; + + private readonly Dictionary _text = new(); + private readonly Dictionary _bool = new(); + + protected override void BindData() + { + base.BindData(); + NodePath = ViewModel.NodePath; + CanEdit = ViewModel.CanEdit; + Fields = ViewModel.Fields ?? (IReadOnlyList)Array.Empty(); + if (string.IsNullOrEmpty(NodePath)) return; + + // Bind DIRECTLY to the node stream — reads stay live with the node, no replica. + AddBinding(Hub.GetMeshNodeStream(NodePath) + .Where(n => n is not null) + .Subscribe(node => + { + if (IsViewDisposed) return; + LoadValues(node!); + _loaded = true; + InvokeAsync(StateHasChanged); + })); + } + + private void LoadValues(MeshNode node) + { + var obj = ToJsonObject(node.Content); + foreach (var f in Fields) + { + // Don't clobber the field the user is actively editing with an echoed emission. + if (f.Key == _focusedKey) continue; + var value = obj is null ? null : obj[f.Key]; + if (f.Kind == MeshNodeEditorFieldKind.Bool) + _bool[f.Key] = value is JsonValue jb && jb.TryGetValue(out var b) && b; + else + _text[f.Key] = value is JsonValue js ? js.ToString() : value?.ToString(); + } + } + + private string? TextOf(MeshNodeEditorField f) => _text.GetValueOrDefault(f.Key); + private bool BoolOf(MeshNodeEditorField f) => _bool.GetValueOrDefault(f.Key); + + private void OnFocus(MeshNodeEditorField f) => _focusedKey = f.Key; + private void OnBlur(MeshNodeEditorField f) + { + if (_focusedKey == f.Key) _focusedKey = null; + } + + private void OnTextChanged(MeshNodeEditorField f, string? value) + { + _text[f.Key] = value; + Persist(f.Key, value is null ? null : JsonValue.Create(value)); + } + + private void OnBoolChanged(MeshNodeEditorField f, bool value) + { + _bool[f.Key] = value; + Persist(f.Key, JsonValue.Create(value)); + } + + /// + /// Per-field read-modify-write straight to the node via the cache: re-read the latest content + /// inside the lambda and set ONLY this field, so concurrent writers / hidden fields + /// (e.g. the sync operation's LastSyncCommitSha) are never clobbered. + /// + private void Persist(string key, JsonNode? value) + { + if (!CanEdit || string.IsNullOrEmpty(NodePath)) return; + var opts = Hub.JsonSerializerOptions; + Hub.GetMeshNodeStream(NodePath) + .Update(node => + { + var obj = ToJsonObject(node.Content) ?? new JsonObject(); + obj[key] = value is null ? null : JsonNode.Parse(value.ToJsonString()); + return node with { Content = JsonSerializer.SerializeToElement(obj, opts) }; + }) + .Subscribe(_ => { }, ex => Logger.LogWarning(ex, + "MeshNodeContentEditor: persist failed for {Path} field {Key}", NodePath, key)); + } + + private JsonObject? ToJsonObject(object? content) + { + if (content is null) return null; + try + { + return content is JsonElement je + ? JsonNode.Parse(je.GetRawText()) as JsonObject + : JsonSerializer.SerializeToNode(content, Hub.JsonSerializerOptions) as JsonObject; + } + catch + { + return null; + } + } +} diff --git a/src/MeshWeaver.Blazor/Components/MeshNodeErrorCardView.razor b/src/MeshWeaver.Blazor/Components/MeshNodeErrorCardView.razor new file mode 100644 index 000000000..c5d82c2db --- /dev/null +++ b/src/MeshWeaver.Blazor/Components/MeshNodeErrorCardView.razor @@ -0,0 +1,59 @@ +@using MeshWeaver.Mesh +@using Microsoft.FluentUI.AspNetCore.Components + +@if (Error is not null) +{ + +
    +
    @Error.Message
    +
    + @Error.Code + @Error.Path +
    + @if (!string.IsNullOrEmpty(Error.Diagnostic)) + { + + + 
    @Error.Diagnostic
    +     +     + } +
    +     +} + +@code +{ + /// + /// Structured error from a MeshNode read/write failure. Render via + /// <MeshNodeErrorCardView Error="@error" /> from any catch + /// block that handles : + /// + /// try { ... } + /// catch (MeshNodeStreamException ex) { error = ex.Error; } + /// + /// + [Parameter, EditorRequired] public MeshNodeError? Error { get; set; } + + private MessageIntent Intent => Error?.Code switch + { + MeshNodeErrorCode.AccessDenied => MessageIntent.Warning, + MeshNodeErrorCode.NotFound => MessageIntent.Info, + MeshNodeErrorCode.Validation => MessageIntent.Warning, + MeshNodeErrorCode.OwnerUnreachable => MessageIntent.Error, + MeshNodeErrorCode.Deserialization => MessageIntent.Error, + MeshNodeErrorCode.Conflict => MessageIntent.Warning, + _ => MessageIntent.Error, + }; + + private string Title => Error?.Code switch + { + MeshNodeErrorCode.AccessDenied => "Access denied", + MeshNodeErrorCode.NotFound => "Not found", + MeshNodeErrorCode.Validation => "Validation failed", + MeshNodeErrorCode.OwnerUnreachable => "Owner unreachable", + MeshNodeErrorCode.Deserialization => "Content could not be deserialized", + MeshNodeErrorCode.Conflict => "Concurrent update conflict", + _ => "MeshNode operation failed", + }; +} diff --git a/src/MeshWeaver.Blazor/Components/MeshNodePickerView.razor b/src/MeshWeaver.Blazor/Components/MeshNodePickerView.razor index 05f47bd8a..3dcb3cefe 100644 --- a/src/MeshWeaver.Blazor/Components/MeshNodePickerView.razor +++ b/src/MeshWeaver.Blazor/Components/MeshNodePickerView.razor @@ -2,7 +2,7 @@ @using MeshWeaver.Mesh @inherits FormComponentBase -
    +
    @if (Label != null) { @Label @@ -15,9 +15,12 @@ @{ var selInitial = GetInitial(_selectedNode); } @RenderAvatar(_selectedNode, selInitial)
    -
    @(_selectedNode.Name ?? _selectedNode.Id)
    -
    @(_selectedNode.NodeType ?? "")
    +
    @(_selectedNode.Name ?? _selectedNode.Id)
    + @if (!string.IsNullOrEmpty(_selectedNode.Category)) + { + @_selectedNode.Category + }
    +
    + } + else if (_expandedRowGroups.Contains(groupKey) + && BoundMaxRows.HasValue + && leaves.Count > BoundMaxRows.Value * (BoundMaxColumns ?? 3)) + { +
    + +
    + } + } + }; + + /// + /// Renders a sub-namespace as a modern folder tile. The PRIMARY click navigates + /// to the folder's default page /{folderPath} (empty area) so drilling + /// expands the URL; when a drill-down area is set, a secondary "Drill down" anchor + /// re-roots the Search/catalog area at /{folderPath}/{BoundDrillDownArea}. + /// + private RenderFragment RenderFolderTile(NamespaceTreeFolder folder, string countLabel, bool showCounts) => __builder => + { + var folderPath = folder.Path; + var drillArea = BoundDrillDownArea; + + // Avoid nested anchors: the tile body is a full-cover primary anchor; the + // secondary "Drill down" anchor is a SIBLING positioned over the top-right. +
    + + + + + + @folder.Name + @if (showCounts) + { + @countLabel + } + + + @if (!string.IsNullOrEmpty(drillArea)) + { + + @DrillDownIcon + + } +
    + }; + + // ----- Graph navigator rendering (MeshSearchRenderMode.GraphNavigator) ----- + + /// Small "open ↗" glyph for the secondary "open the node's page" affordance. + private static readonly MarkupString OpenIcon = (MarkupString)( + "" + + ""); + + /// Folder glyph for a sub-namespace drill link. + private static readonly MarkupString FolderIcon = (MarkupString)( + "" + + ""); + + /// + /// Renders the navigator for the Search layout area on a mesh node: the ancestor breadcrumb rail + /// ABOVE, an "include documents" toggle, the mesh NODES at this level as cards (TOP — a node that + /// also contains content gets a "drill in" symbol), and the sub-NAMESPACES (no node of their own) + /// as drill-down links (BOTTOM — visually distinct from node cards). Clicking a namespace, an + /// ancestor, or a node's drill symbol re-roots the navigator at that path and recomputes. + /// + private RenderFragment RenderGraphNavigator() => __builder => + { + if (_navLoading && _navModel is null) + { + @RenderTreeLoading() + return; + } + + var model = _navModel; + var navArea = NavArea; + var nodes = NavNodes; + var namespaces = NavNamespaces; + +
    + + + + + @* TOP: the mesh nodes at this level. *@ + @if (nodes.Count > 0) + { +
    +
    Nodes
    +
    + @foreach (var item in nodes) + { + @RenderNavNode(item) + } +
    +
    + } + + @* BOTTOM: sub-namespaces to drill into — distinct link styling, not node cards. *@ + @if (namespaces.Count > 0) + { +
    +
    Namespaces
    +
    + @foreach (var ns in namespaces) + { + @* A pure namespace has no node — redirect to the search control scoped to + it, never to /{nsPath}/Search (a node area on a non-existent node). *@ + + @FolderIcon + @ns.Name + @ns.Count + + + } +
    +
    + } + + @if (nodes.Count == 0 && namespaces.Count == 0 && BoundShowEmptyMessage) + { +
    + + Nothing below this node. + +
    + } +
    + }; + + /// + /// One node at the current level (TOP). The card opens the node; when the node ALSO has content + /// inside its namespace () a "drill in" symbol re-roots the + /// navigator at that node. DisableNavigation stops the inner card from emitting its own + /// anchor (no nesting). + /// + private RenderFragment RenderNavNode(GraphNavNode item) => __builder => + { + var node = item.Node; + var navArea = NavArea; + var cardControl = MeshNodeCardControl.FromNode(node, node.Path, BoundItemArea, true); +
    + + + + @if (item.HasChildren) + { + + @DrillDownIcon + + } +
    + }; } diff --git a/src/MeshWeaver.Blazor/Components/MeshSearchView.razor.cs b/src/MeshWeaver.Blazor/Components/MeshSearchView.razor.cs index 2bc59dd46..0e08cc7ab 100644 --- a/src/MeshWeaver.Blazor/Components/MeshSearchView.razor.cs +++ b/src/MeshWeaver.Blazor/Components/MeshSearchView.razor.cs @@ -1,4 +1,5 @@ -using System.Text.Json; +using System.Collections.Immutable; +using System.Text.Json; using System.Text.RegularExpressions; using System.Reactive.Linq; using Microsoft.AspNetCore.Components; @@ -9,6 +10,7 @@ using MeshWeaver.Layout.Catalog; using MeshWeaver.Mesh; using MeshWeaver.Mesh.Services; +using MeshWeaver.Reactive; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -125,7 +127,29 @@ private bool BoundReactiveMode return false; } } - private MeshSearchRenderMode BoundRenderMode => ViewModel?.RenderMode is MeshSearchRenderMode mode ? mode : MeshSearchRenderMode.Grouped; + private MeshSearchRenderMode BoundRenderMode + { + get + { + switch (ViewModel?.RenderMode) + { + case MeshSearchRenderMode mode: + return mode; + case string s when Enum.TryParse(s, true, out var parsed): + return parsed; + // Controls round-trip through the synchronization stream as JSON; + // enums serialize as strings (EnumMemberJsonStringEnumConverter). + case JsonElement { ValueKind: JsonValueKind.String } je + when Enum.TryParse(je.GetString(), true, out var fromJson): + return fromJson; + case JsonElement { ValueKind: JsonValueKind.Number } jn + when jn.TryGetInt32(out var n) && Enum.IsDefined(typeof(MeshSearchRenderMode), n): + return (MeshSearchRenderMode)n; + default: + return MeshSearchRenderMode.Grouped; + } + } + } // Config objects private SectionConfig? BoundSections => ViewModel?.Sections; @@ -219,7 +243,12 @@ protected override void OnParametersSet() _currentValue = BoundVisibleQuery; if (!IsPrecomputedMode) { - _ = LoadResultsAsync(); + if (IsNamespaceTreeMode) + LoadTreeSearch(); + else if (IsGraphNavigatorMode) + RunGraphNavigatorSearch(); // browse when empty, vector search when text typed + else + LoadResults(); } } @@ -228,7 +257,12 @@ protected override void OnParametersSet() _lastBoundHiddenQuery = BoundHiddenQuery; if (!IsPrecomputedMode) { - _ = LoadResultsAsync(); + if (IsNamespaceTreeMode) + ResetTree(); + else if (IsGraphNavigatorMode) + ResetGraphNavigator(); // re-rooted → recompute above + below + else + LoadResults(); } } @@ -267,7 +301,25 @@ protected override async Task OnAfterRenderAsync(bool firstRender) return; } - // Reactive mode: subscribe to ObserveQuery for live updates + // Namespace tree mode: lazily load the first level only. + if (IsNamespaceTreeMode) + { + InitializeTree(); + return; + } + + // Graph navigator: load the next level below + ancestors above. When the + // box already carries search text (e.g. ?q=), also run the semantic query + // so the initial render shows results instead of the browse view. + if (IsGraphNavigatorMode) + { + InitializeGraphNavigator(); + if (!string.IsNullOrWhiteSpace(_currentValue)) + LoadResults(); + return; + } + + // Reactive mode: subscribe to Query for live updates if (BoundReactiveMode) { SubscribeToReactiveUpdates(); @@ -275,7 +327,7 @@ protected override async Task OnAfterRenderAsync(bool firstRender) } // Client-side query mode (one-shot) - await LoadResultsAsync(); + LoadResults(); StateHasChanged(); } } @@ -285,7 +337,12 @@ private Task OnValueChanged(string value) _currentValue = value; if (BoundLiveSearch && !IsPrecomputedMode) { - _ = LoadResultsAsync(); + if (IsNamespaceTreeMode) + LoadTreeSearch(); + else if (IsGraphNavigatorMode) + RunGraphNavigatorSearch(); + else + LoadResults(); } return Task.CompletedTask; } @@ -299,7 +356,12 @@ private async Task HandleSearch() if (!IsPrecomputedMode) { - await LoadResultsAsync(); + if (IsNamespaceTreeMode) + LoadTreeSearch(); + else if (IsGraphNavigatorMode) + RunGraphNavigatorSearch(); + else + LoadResults(); } // Update URL if we're on the search page so the URL is shareable @@ -324,43 +386,59 @@ private void UpdateSearchUrl() Navigation.NavigateTo(url, replace: true); } - private async Task LoadResultsAsync() + private void LoadResults() { _isLoading = true; StateHasChanged(); - try - { - var query = BuildFullQuery(); - _nodes = await MeshQuery.QueryAsync(query).ToListAsync(); + var query = BuildFullQuery(); + // Subscribe to Query so the result set stays live as data changes. + // _reactiveSubscription holds the active subscription (set up in SubscribeToReactiveUpdates). + _reactiveSubscription?.Dispose(); + _reactiveSubscription = MeshQuery.Query(MeshQueryRequest.FromQuery(query)) + .Subscribe( + change => + { + var current = (IReadOnlyList)_nodes; + var merged = change.ChangeType switch + { + QueryChangeType.Initial or QueryChangeType.Reset => change.Items, + QueryChangeType.Added => current.Concat(change.Items).ToList(), + QueryChangeType.Updated => current + .Select(n => change.Items.FirstOrDefault(c => c.Path == n.Path) ?? n) + .ToList(), + QueryChangeType.Removed => current + .Where(n => !change.Items.Any(r => r.Path == n.Path)) + .ToList(), + _ => current + }; - // Exclude base path node if configured - if (BoundExcludeBasePath && !string.IsNullOrEmpty(BoundNamespace)) - { - var basePath = BoundNamespace.Trim('/'); - _nodes = _nodes.Where(n => n.Path != basePath).ToList(); - } + if (BoundExcludeBasePath && !string.IsNullOrEmpty(BoundNamespace)) + { + var basePath = BoundNamespace.Trim('/'); + merged = merged.Where(n => n.Path != basePath).ToList(); + } - // Compute groups locally - if (ViewModel != null) - { - _computedGroups = ProcessResults(_nodes); - InitializeCollapsedState(_computedGroups); - } - } - catch (Exception ex) - { - var logger = Hub.ServiceProvider.GetService() - ?.CreateLogger("MeshWeaver.MeshSearchView"); - logger?.LogWarning(ex, "MeshSearchView query failed: {Query}", BuildFullQuery()); - _nodes = new List(); - _computedGroups = null; - } - finally - { - _isLoading = false; - StateHasChanged(); - } + _nodes = merged.ToList(); + + if (ViewModel != null) + { + _computedGroups = ProcessResults(_nodes); + InitializeCollapsedState(_computedGroups); + } + _isLoading = false; + InvokeAsync(StateHasChanged); + }, + ex => + { + var logger = Hub.ServiceProvider.GetService() + ?.CreateLogger("MeshWeaver.MeshSearchView"); + logger?.LogWarning(ex, "MeshSearchView query failed: {Query}", BuildFullQuery()); + _nodes = new List(); + _computedGroups = null; + _isLoading = false; + InvokeAsync(StateHasChanged); + }); } private void SubscribeToReactiveUpdates() @@ -374,7 +452,7 @@ private void SubscribeToReactiveUpdates() // Individual cards handle their own content updates via LayoutAreaView. var knownPaths = new HashSet(); - _reactiveSubscription = MeshQuery.ObserveQuery(request) + _reactiveSubscription = MeshQuery.Query(request) .Subscribe(change => { // Compute updated path set without touching _nodes yet. @@ -470,7 +548,17 @@ private GroupedSearchResult ProcessResults(List nodes) groupByProperty = "NodeType"; var groups = sortedNodes - .GroupBy(n => GetPropertyValue(n, groupByProperty) ?? "") + // When grouping by Category, fall back to NodeType for nodes that don't + // carry an explicit category so they still bucket meaningfully rather + // than collapsing into a single empty-label group. + .GroupBy(n => + { + var val = GetPropertyValue(n, groupByProperty); + if (!string.IsNullOrEmpty(val)) return val; + if (groupByProperty.Equals("Category", StringComparison.OrdinalIgnoreCase)) + return n.NodeType?.Split('/').LastOrDefault() ?? ""; + return ""; + }) .Select(g => { var groupKey = g.Key; @@ -615,59 +703,61 @@ private string BuildFullQuery() return string.Join(" ", parts); } - private async Task GetCompletionsAsync(string query) + private const int CompletionLimit = 20; + + // Higher score = better. Sort descending. + private static readonly IComparer CompletionByScore = + Comparer.Create((a, b) => b.Score.CompareTo(a.Score)); + + /// + /// Returns a stream of top-N completion snapshots for . + /// Monaco subscribes per query and pushes each fresh snapshot to the suggest widget + /// as it arrives. No Task, no await, no component-level state. + /// + private IObservable> GetCompletions(string query) { if (string.IsNullOrWhiteSpace(query)) - return []; + return Observable.Return>(Array.Empty()); - try - { - // Parse the query to split into basePath and prefix for AutocompleteAsync - var text = query.TrimStart('@'); - string basePath; - string namePrefix; + // Parse the query to split into basePath and prefix for AutocompleteAsync + var text = query.TrimStart('@'); + string basePath; + string namePrefix; - if (text.EndsWith("/")) + if (text.EndsWith("/")) + { + basePath = text.TrimEnd('/'); + namePrefix = ""; + } + else + { + var lastSlash = text.LastIndexOf('/'); + if (lastSlash >= 0) { - // User typed @path/ — get children of that path - basePath = text.TrimEnd('/'); - namePrefix = ""; + basePath = text[..lastSlash]; + namePrefix = text[(lastSlash + 1)..]; } else { - // Split into path and name parts: "ACME/Mark" → basePath="ACME", namePrefix="Mark" - var lastSlash = text.LastIndexOf('/'); - if (lastSlash >= 0) - { - basePath = text[..lastSlash]; - namePrefix = text[(lastSlash + 1)..]; - } - else - { - basePath = BoundNamespace ?? ""; - namePrefix = text; - } + basePath = BoundNamespace ?? ""; + namePrefix = text; } - - var suggestions = await MeshQuery - .AutocompleteAsync(basePath, namePrefix, AutocompleteMode.RelevanceFirst, 20, BoundNamespace) - .ToArrayAsync(); - - return suggestions.Select((s, i) => new CompletionItem - { - Label = s.Name, - InsertText = $"@{s.Path}/", - Description = s.NodeType ?? "", - Path = s.Path, - Category = s.NodeType ?? "Nodes", - IconUrl = s.Icon, - SortKey = (99999 - Math.Clamp((int)s.Score, 0, 99999)).ToString("D5") - }).ToArray(); - } - catch - { - return []; } + + return MeshQuery + .Autocomplete(basePath, namePrefix, AutocompleteMode.RelevanceFirst, CompletionLimit, BoundNamespace) + .Select(snapshot => (IReadOnlyList)snapshot + .Select(s => new CompletionItem + { + Label = s.Name ?? s.Path, + InsertText = $"@{s.Path}/", + Description = s.NodeType ?? "", + Path = s.Path, + Category = s.NodeType ?? "Nodes", + IconUrl = s.Icon, + SortKey = (99999 - Math.Clamp((int)s.Score, 0, 99999)).ToString("D5") + }) + .ToArray()); } private IReadOnlyList GetGroups() @@ -697,6 +787,26 @@ private string? BoundItemArea } } + /// + /// The re-rooted "Drill down" area (e.g. "Search"). When non-empty, each card + /// and folder shows a secondary anchor to /{path}/{BoundDrillDownArea}; + /// the primary click still opens the node's default page /{path}. + /// Empty/unset = no drill-down affordance (opt-in). + /// + private string? BoundDrillDownArea + { + get + { + if (ViewModel?.DrillDownArea is string s) return string.IsNullOrWhiteSpace(s) ? null : s; + if (ViewModel?.DrillDownArea is JsonElement je && je.ValueKind == JsonValueKind.String) + { + var v = je.GetString(); + return string.IsNullOrWhiteSpace(v) ? null : v; + } + return null; + } + } + private int? BoundMaxColumns { get @@ -799,6 +909,490 @@ private void HandleCreateClick() private MeshNodeCardControl GetCardControl(MeshNode node) => MeshNodeCardControl.FromNode(node, node.Path, BoundItemArea, BoundDisableNavigation); + #region Namespace tree mode (MeshSearchRenderMode.NamespaceTree) + + /// One lazily-loaded namespace level: direct children resolved into folders/leaves. + private sealed record TreeLevel(bool IsLoading, ImmutableList Items) + { + public static readonly TreeLevel Loading = new(true, ImmutableList.Empty); + } + + /// Direct-children count probes cap out here; the badge shows "99+" at the cap. + private const int FolderCountProbeCap = 100; + + private ImmutableDictionary _treeLevels = + ImmutableDictionary.Empty.WithComparers(StringComparer.OrdinalIgnoreCase); + private ImmutableHashSet _expandedFolders = + ImmutableHashSet.Empty.WithComparer(StringComparer.OrdinalIgnoreCase); + private ImmutableDictionary _treeLevelSubscriptions = + ImmutableDictionary.Empty.WithComparers(StringComparer.OrdinalIgnoreCase); + private ImmutableList _treeProbeSubscriptions = ImmutableList.Empty; + private IDisposable? _treeSearchSubscription; + private ImmutableList? _treeSearchItems; + private bool _treeSearchLoading; + + private bool IsNamespaceTreeMode => + BoundRenderMode == MeshSearchRenderMode.NamespaceTree && !IsPrecomputedMode; + + private bool TreeHasSearchText => !string.IsNullOrWhiteSpace(_currentValue); + + /// The catalog root — the namespace: of the hidden query (fallback: Namespace property). + private string TreeRootNamespace + { + get + { + var match = Regex.Match(BoundHiddenQuery, @"(?:^|\s)namespace:(\S+)"); + if (match.Success) + return match.Groups[1].Value.Trim('/'); + return BoundNamespace.Trim('/'); + } + } + + private ILogger? TreeLogger => + Hub.ServiceProvider.GetService()?.CreateLogger("MeshWeaver.MeshSearchView"); + + /// + /// Per-level query: the hidden query's filters with namespace: forced to + /// and any scope: stripped (levels are always direct children). + /// + private string BuildTreeLevelQuery(string ns) + { + var query = Regex.Replace(BoundHiddenQuery, @"(?:^|\s)scope:\S+", " "); + query = Regex.IsMatch(query, @"(?:^|\s)namespace:\S+") + ? Regex.Replace(query, @"(?:^|\s)namespace:\S+", $" namespace:{ns}") + : $"namespace:{ns} {query}"; + return Regex.Replace(query, @"\s+", " ").Trim(); + } + + /// + /// Subtree search query for typed text: same filters, namespace: at the root, + /// scope:descendants, plus the user's search terms. + /// + private string BuildTreeSearchQuery() + => Regex.Replace($"{BuildTreeLevelQuery(TreeRootNamespace)} scope:descendants {_currentValue.Trim()}", @"\s+", " ").Trim(); + + private void InitializeTree() + { + LoadTreeLevel(TreeRootNamespace); + if (TreeHasSearchText) + LoadTreeSearch(); + } + + /// Disposes all tree subscriptions and reloads from the (possibly changed) root. + private void ResetTree() + { + DisposeTreeSubscriptions(); + _treeLevels = _treeLevels.Clear(); + _expandedFolders = _expandedFolders.Clear(); + _treeSearchItems = null; + InitializeTree(); + } + + private void DisposeTreeSubscriptions() + { + foreach (var subscription in _treeLevelSubscriptions.Values) + subscription.Dispose(); + _treeLevelSubscriptions = _treeLevelSubscriptions.Clear(); + foreach (var probe in _treeProbeSubscriptions) + probe.Dispose(); + _treeProbeSubscriptions = ImmutableList.Empty; + _treeSearchSubscription?.Dispose(); + _treeSearchSubscription = null; + } + + /// + /// Subscribes the live direct-children query for one namespace level. Each + /// structural emission re-probes child counts and rebuilds the level's items. + /// + private void LoadTreeLevel(string ns) + { + if (_treeLevelSubscriptions.TryGetValue(ns, out var existing)) + existing.Dispose(); + + _treeLevels = _treeLevels.SetItem(ns, TreeLevel.Loading); + + // Live per-emission snapshot of the level's direct children, keyed by path. + var levelNodes = ImmutableDictionary.Empty + .WithComparers(StringComparer.OrdinalIgnoreCase); + + var subscription = MeshQuery + .Query(MeshQueryRequest.FromQuery(BuildTreeLevelQuery(ns))) + .Subscribe( + change => + { + switch (change.ChangeType) + { + case QueryChangeType.Initial: + case QueryChangeType.Reset: + levelNodes = change.Items + .Where(n => !string.IsNullOrEmpty(n.Path)) + .Aggregate( + ImmutableDictionary.Empty + .WithComparers(StringComparer.OrdinalIgnoreCase), + (map, n) => map.SetItem(n.Path, n)); + break; + case QueryChangeType.Added: + case QueryChangeType.Updated: + foreach (var n in change.Items.Where(n => !string.IsNullOrEmpty(n.Path))) + levelNodes = levelNodes.SetItem(n.Path, n); + break; + case QueryChangeType.Removed: + foreach (var n in change.Items.Where(n => !string.IsNullOrEmpty(n.Path))) + levelNodes = levelNodes.Remove(n.Path); + break; + default: + return; + } + ProbeTreeChildCounts(ns, levelNodes.Values.ToImmutableList()); + }, + ex => + { + TreeLogger?.LogWarning(ex, "Namespace tree level query failed for {Namespace}", ns); + InvokeAsync(() => + { + _treeLevels = _treeLevels.SetItem( + ns, new TreeLevel(false, ImmutableList.Empty)); + StateHasChanged(); + }); + }); + + _treeLevelSubscriptions = _treeLevelSubscriptions.SetItem(ns, subscription); + } + + /// + /// Fires one direct-children existence/count probe per child node and resolves + /// the level once all probes answered. Subscribe-all-upfront (CombineLatest) + /// per AsynchronousCalls.md; each probe Catches to a 0-count sentinel so one + /// failing probe degrades a single folder to a leaf instead of wedging the level. + /// + private void ProbeTreeChildCounts(string ns, ImmutableList nodes) + { + if (nodes.Count == 0) + { + InvokeAsync(() => + { + _treeLevels = _treeLevels.SetItem( + ns, new TreeLevel(false, ImmutableList.Empty)); + StateHasChanged(); + }); + return; + } + + var probes = nodes + .Where(n => !string.IsNullOrEmpty(n.Path)) + .Select(n => + { + var path = n.Path; + return MeshQuery + .Query(MeshQueryRequest.FromQuery( + $"{BuildTreeLevelQuery(path)} limit:{FolderCountProbeCap}")) + .Where(c => c.ChangeType is QueryChangeType.Initial or QueryChangeType.Reset) + .Take(1) + .Select(c => (Path: path, Count: c.Items.Count)) + .Timeout(TimeSpan.FromSeconds(10)) + .Catch((Exception ex) => + { + TreeLogger?.LogWarning(ex, "Child-count probe failed for {Path}", path); + return Observable.Return((Path: path, Count: 0)); + }); + }) + .ToArray(); + + var probeSubscription = Observable.CombineLatest(probes) + .Take(1) + .Subscribe(counts => InvokeAsync(() => + { + var countMap = counts.ToImmutableDictionary( + c => c.Path, c => c.Count, StringComparer.OrdinalIgnoreCase); + _treeLevels = _treeLevels.SetItem( + ns, new TreeLevel(false, NamespaceTreeBuilder.BuildLevel(ns, nodes, countMap))); + StateHasChanged(); + })); + _treeProbeSubscriptions = _treeProbeSubscriptions.Add(probeSubscription); + } + + private void OnTreeFolderHeaderClick(string folderPath, bool lazy) + { + if (lazy) + ToggleTreeFolder(folderPath); + else + ToggleSearchTreeFolder(folderPath); + } + + private void ToggleTreeFolder(string folderPath) + { + if (_expandedFolders.Contains(folderPath)) + { + _expandedFolders = _expandedFolders.Remove(folderPath); + } + else + { + _expandedFolders = _expandedFolders.Add(folderPath); + if (!_treeLevels.ContainsKey(folderPath)) + LoadTreeLevel(folderPath); + } + StateHasChanged(); + } + + /// + /// Typed-text mode: one live subtree query; results are relativised to the root + /// and grouped into nested namespace sections via . + /// Clearing the text returns to the lazily-loaded browse levels (still cached). + /// + private void LoadTreeSearch() + { + _treeSearchSubscription?.Dispose(); + _treeSearchSubscription = null; + + if (!TreeHasSearchText) + { + _treeSearchItems = null; + _treeSearchLoading = false; + StateHasChanged(); + return; + } + + _treeSearchLoading = true; + StateHasChanged(); + + var root = TreeRootNamespace; + var resultNodes = ImmutableDictionary.Empty + .WithComparers(StringComparer.OrdinalIgnoreCase); + + _treeSearchSubscription = MeshQuery + .Query(MeshQueryRequest.FromQuery(BuildTreeSearchQuery())) + .Subscribe( + change => + { + switch (change.ChangeType) + { + case QueryChangeType.Initial: + case QueryChangeType.Reset: + resultNodes = change.Items + .Where(n => !string.IsNullOrEmpty(n.Path)) + .Aggregate( + ImmutableDictionary.Empty + .WithComparers(StringComparer.OrdinalIgnoreCase), + (map, n) => map.SetItem(n.Path, n)); + break; + case QueryChangeType.Added: + case QueryChangeType.Updated: + foreach (var n in change.Items.Where(n => !string.IsNullOrEmpty(n.Path))) + resultNodes = resultNodes.SetItem(n.Path, n); + break; + case QueryChangeType.Removed: + foreach (var n in change.Items.Where(n => !string.IsNullOrEmpty(n.Path))) + resultNodes = resultNodes.Remove(n.Path); + break; + default: + return; + } + var items = NamespaceTreeBuilder.Build(root, resultNodes.Values.ToImmutableList()); + InvokeAsync(() => + { + _treeSearchItems = items; + _treeSearchLoading = false; + StateHasChanged(); + }); + }, + ex => + { + TreeLogger?.LogWarning(ex, "Namespace tree search failed: {Query}", BuildTreeSearchQuery()); + InvokeAsync(() => + { + _treeSearchItems = ImmutableList.Empty; + _treeSearchLoading = false; + StateHasChanged(); + }); + }); + } + + private static string FormatTreeCount(int count) + => count >= FolderCountProbeCap ? $"{FolderCountProbeCap - 1}+" : count.ToString(); + + /// Search-mode folders default to expanded; the toggle reuses _collapsedGroups. + private bool IsSearchTreeFolderExpanded(string folderPath) + => !_collapsedGroups.Contains($"tree:{folderPath}"); + + private void ToggleSearchTreeFolder(string folderPath) + => ToggleGroup($"tree:{folderPath}"); + + private bool TreeRootIsLoading => + !_treeLevels.TryGetValue(TreeRootNamespace, out var level) || level.IsLoading; + + private bool TreeRootIsEmpty => + _treeLevels.TryGetValue(TreeRootNamespace, out var level) + && !level.IsLoading + && level.Items.Count == 0; + + #endregion + + #region Graph navigator mode (MeshSearchRenderMode.GraphNavigator) + + private GraphNavigatorModel? _navModel; + private bool _navLoading; + private bool _includeDocuments; + private IDisposable? _navSubscription; + + private bool IsGraphNavigatorMode => + BoundRenderMode == MeshSearchRenderMode.GraphNavigator && !IsPrecomputedMode; + + /// The node the navigator is centered on — the hidden query's namespace: (fallback: Namespace). + private string NavRootPath => TreeRootNamespace; + + /// Re-root target area on click — keeps the navigator. Falls back to the Search area. + private string NavArea => string.IsNullOrEmpty(BoundDrillDownArea) + ? MeshNodeLayoutAreas.SearchArea + : BoundDrillDownArea!; + + /// + /// A pure sub-namespace () has NO node of its own, so it cannot + /// host a node-page Search area: routing to /{nsPath}/Search would render a layout area on + /// a phantom node and NotFound-storm its hub. Instead, a namespace click redirects to the global + /// search control (/search) scoped to that namespace — carrying the navigator's own hidden + /// query (filters + the "Include documents" toggle state via ) so + /// the search shows exactly what the navigator would have if it could re-root there. + /// + private string BuildNamespaceSearchHref(string nsPath) => + $"/search?ns={Uri.EscapeDataString(nsPath)}&hq={Uri.EscapeDataString(BuildNavBelowQuery(nsPath))}"; + + /// + /// Below = the current node's whole subtree (one scope:descendants query — no N+1 probes). + /// The builder surfaces the immediate level from it: real nodes here (cards on top) + pure + /// sub-namespaces (drill links at the bottom). Documents (indexed content) are excluded unless + /// the user ticks "Include documents" — and ONLY here, in the node Search area; top-level/global + /// search keeps documents. When included, Document nodes ({collection}/_Documents/{slug}) + /// join the subtree, so content shows up as a navigable namespace too. + /// + private string BuildNavBelowQuery(string root) + { + var q = $"{BuildTreeLevelQuery(root)} scope:descendants"; + if (!_includeDocuments) + q += " -nodeType:Document"; + return Regex.Replace(q, @"\s+", " ").Trim(); + } + + /// Above = the ancestor chain INCLUDING self, so the builder can pull out the current node + /// and order the rail. Real ancestors only — empty namespace segments are never nodes. + private static string BuildNavAboveQuery(string root) => + $"path:{root} scope:ancestorsandself is:main"; + + /// + /// The navigator browses the graph when the box is empty; a non-empty box switches to a + /// real subtree query (see ) so typed text flows + /// through the standard MeshQuery.Query surface and its bare-text tokens hit the Postgres + /// HNSW vector intercept. The navigator level is therefore always shown unfiltered — there + /// is no client-side narrowing here anymore. + /// + private bool NavBrowsing => IsGraphNavigatorMode && string.IsNullOrWhiteSpace(_currentValue); + + /// Mesh nodes at the current level (rendered only in browse mode). + private IReadOnlyList NavNodes => _navModel?.Nodes ?? ImmutableList.Empty; + + /// Sub-namespace drill links at the current level (rendered only in browse mode). + private IReadOnlyList NavNamespaces => _navModel?.Namespaces ?? ImmutableList.Empty; + + /// + /// GraphNavigator search: browse the graph when the box is empty, run a real query when the + /// user types. Routing typed text through (MeshQuery.Query) means a + /// query like namespace:{node} scope:subtree … laptop reaches + /// PostgreSqlMeshQuery.QueryAsync's vector intercept — semantic HNSW cosine search over + /// the node's subtree — exactly like does for NamespaceTree mode. + /// Clearing the box drops the query and returns to the (still-loaded) navigator browse view. + /// + private void RunGraphNavigatorSearch() + { + if (string.IsNullOrWhiteSpace(_currentValue)) + { + _reactiveSubscription?.Dispose(); + _reactiveSubscription = null; + _isLoading = false; + StateHasChanged(); + } + else + { + LoadResults(); + } + } + + private void ToggleIncludeDocuments(ChangeEventArgs e) + { + var include = e.Value is bool b ? b : bool.TryParse(e.Value?.ToString(), out var parsed) && parsed; + if (_includeDocuments == include) return; + _includeDocuments = include; + ResetGraphNavigator(); + } + + private void InitializeGraphNavigator() + { + _navSubscription?.Dispose(); + _navLoading = true; + StateHasChanged(); + + var root = NavRootPath; + + // hub.GetQuery is the canonical synced-query surface (delegates to workspace → the shared + // IMeshNodeStreamCache): live, deduped, all-Initial gated, provider-fanned, injects hub + // JsonSerializerOptions. The below id keys on the doc toggle so each variant caches separately. + var below = Hub.GetQuery($"nav-below:{root}:{_includeDocuments}", BuildNavBelowQuery(root)); + var above = string.IsNullOrEmpty(root) + ? Observable.Return>(Array.Empty()) + : Hub.GetQuery($"nav-above:{root}", BuildNavAboveQuery(root)); + + _navSubscription = below + .CombineLatest(above, (b, a) => (Below: b, Above: a)) + .Subscribe( + t => + { + var aboveList = (t.Above ?? Enumerable.Empty()).ToList(); + var belowList = (t.Below ?? Enumerable.Empty()).ToList(); + var current = aboveList.FirstOrDefault(n => + string.Equals(n.Path?.Trim('/'), root, StringComparison.OrdinalIgnoreCase)); + var model = GraphNavigatorBuilder.Build(root, aboveList, belowList, current); + InvokeAsync(() => + { + _navModel = model; + _navLoading = false; + StateHasChanged(); + }); + }, + ex => + { + TreeLogger?.LogWarning(ex, "Graph navigator query failed for {Root}", root); + InvokeAsync(() => + { + _navModel = GraphNavigatorBuilder.Build( + root, Array.Empty(), Array.Empty()); + _navLoading = false; + StateHasChanged(); + }); + }); + } + + private void ResetGraphNavigator() + { + _navSubscription?.Dispose(); + _navSubscription = null; + _navModel = null; + InitializeGraphNavigator(); + } + + /// Current node display name — the node when present, else the last path segment. + private string NavCurrentName + { + get + { + var current = _navModel?.Current; + if (!string.IsNullOrEmpty(current?.Name)) return current!.Name!; + var root = NavRootPath; + if (string.IsNullOrEmpty(root)) return "Mesh"; + var slash = root.LastIndexOf('/'); + return slash < 0 ? root : root[(slash + 1)..]; + } + } + + #endregion + private void ToggleSearchOptions() { _showSearchOptions = !_showSearchOptions; @@ -808,16 +1402,22 @@ private void ToggleSearchOptions() } } - private async Task ApplySearchOptions() + private Task ApplySearchOptions() { _overriddenHiddenQuery = _editableHiddenQuery; _showSearchOptions = false; - await LoadResultsAsync(); + if (IsNamespaceTreeMode) + ResetTree(); + else + LoadResults(); UpdateSearchUrl(); + return Task.CompletedTask; } public void Dispose() { _reactiveSubscription?.Dispose(); + DisposeTreeSubscriptions(); + _navSubscription?.Dispose(); } } diff --git a/src/MeshWeaver.Blazor/Components/MeshSearchView.razor.css b/src/MeshWeaver.Blazor/Components/MeshSearchView.razor.css index 8e8121972..3d1ae70aa 100644 --- a/src/MeshWeaver.Blazor/Components/MeshSearchView.razor.css +++ b/src/MeshWeaver.Blazor/Components/MeshSearchView.razor.css @@ -327,6 +327,365 @@ font-size: 0.85rem; } +/* Card wrapper hosting the absolute "Drill down" affordance (top-right). */ +.mesh-search-card-wrap { + position: relative; + width: 100%; +} + +/* Secondary "Drill down" / "Search this node" affordance — subtle until hover. */ +.mesh-search-drilldown { + position: absolute; + top: 6px; + right: 6px; + z-index: 2; + display: inline-flex; + align-items: center; + justify-content: center; + width: 24px; + height: 24px; + border-radius: 6px; + color: var(--neutral-foreground-hint, #888); + background: var(--neutral-layer-2, rgba(0, 0, 0, 0.04)); + border: 1px solid transparent; + text-decoration: none; + opacity: 0; + transition: opacity 0.15s, color 0.15s, background 0.15s, border-color 0.15s; +} + +.mesh-search-card-wrap:hover .mesh-search-drilldown, +.mesh-search-drilldown:focus-visible { + opacity: 1; +} + +.mesh-search-drilldown:hover { + color: var(--accent-foreground-rest, #0078d4); + background: var(--neutral-layer-1, #fff); + border-color: var(--neutral-stroke-rest, #d1d1d1); +} + +/* Modern folder TILE for namespace sub-folders (browse mode). The outer element + only positions the secondary drill-down anchor; the link is the clickable body. */ +.mesh-search-folder-tile { + position: relative; + width: 100%; +} + +.mesh-search-folder-tile-link { + display: flex; + align-items: center; + gap: 12px; + width: 100%; + box-sizing: border-box; + padding: 12px 14px; + min-height: 60px; + border-radius: 10px; + background: var(--neutral-layer-1, #fff); + border: 1px solid var(--neutral-stroke-rest, #e0e0e0); + color: var(--neutral-foreground-rest); + text-decoration: none; + cursor: pointer; + transition: box-shadow 0.15s, border-color 0.15s, transform 0.15s, background 0.15s; +} + +.mesh-search-folder-tile-link:hover { + border-color: var(--accent-fill-rest, #0078d4); + box-shadow: var(--elevation-shadow-card-active); + transform: translateY(-1px); + background: var(--neutral-layer-2, #fafafa); +} + +.mesh-search-folder-tile-icon { + display: flex; + align-items: center; + justify-content: center; + width: 40px; + height: 40px; + min-width: 40px; + border-radius: 8px; + background: var(--neutral-fill-secondary-rest, #f0f0f0); + color: var(--accent-foreground-rest, #0078d4); + flex-shrink: 0; +} + +.mesh-search-folder-tile-text { + display: flex; + align-items: center; + gap: 8px; + min-width: 0; + flex: 1; +} + +.mesh-search-folder-tile-name { + font-weight: 600; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.mesh-search-folder-tile-count { + font-size: 0.75rem; + font-weight: 600; + line-height: 1; + padding: 3px 8px; + border-radius: 10px; + background: var(--neutral-fill-secondary-rest, #e8e8e8); + color: var(--neutral-foreground-hint, #666); + flex-shrink: 0; +} + +/* The drill-down anchor sitting on a folder tile reveals on tile hover. */ +.mesh-search-folder-tile:hover .mesh-search-folder-drilldown, +.mesh-search-folder-drilldown:focus-visible { + opacity: 1; +} + +/* Namespace tree (MeshSearchRenderMode.NamespaceTree) */ +.mesh-search-tree-section { + margin-bottom: 12px; +} + +.mesh-search-tree-folder-glyph { + display: flex; + align-items: center; + color: var(--accent-foreground-rest, #0078d4); +} + +.mesh-search-tree-count { + font-size: 0.75rem; + font-weight: 600; + line-height: 1; + padding: 3px 8px; + border-radius: 10px; + background: var(--neutral-fill-secondary-rest, #e8e8e8); + color: var(--neutral-foreground-hint, #666); +} + +.mesh-search-tree-open { + margin-left: auto; + color: var(--neutral-foreground-hint, #666); + text-decoration: none; + font-size: 0.9rem; + padding: 0 4px; + border-radius: 4px; + opacity: 0.5; + transition: opacity 0.15s, color 0.15s; +} + +.mesh-search-section-header:hover .mesh-search-tree-open { + opacity: 1; +} + +.mesh-search-tree-open:hover { + color: var(--accent-foreground-rest, #0078d4); +} + +.mesh-search-tree-children { + margin: 4px 0 8px 11px; + padding: 4px 0 4px 16px; + border-left: 2px solid var(--neutral-stroke-rest, #e0e0e0); +} + +.mesh-search-tree-empty { + color: var(--neutral-foreground-hint, #888); + font-size: 0.85rem; + font-style: italic; + padding: 4px 0; +} + +.mesh-search-tree-loading { + display: flex; + flex-direction: column; + gap: 8px; + padding: 8px 0; + max-width: 480px; +} + +/* ===== Graph navigator (MeshSearchRenderMode.GraphNavigator) ===== */ +.mesh-nav { + display: flex; + flex-direction: column; + gap: 16px; + width: 100%; +} + +/* Above: ancestor breadcrumb rail (root-most → current). */ +.mesh-nav-rail { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 4px; + padding: 8px 12px; + border-radius: 10px; + background: var(--neutral-layer-2, rgba(0, 0, 0, 0.03)); + font-size: 0.9rem; +} + +.mesh-nav-crumb { + color: var(--accent-foreground-rest, #0078d4); + text-decoration: none; + padding: 3px 8px; + border-radius: 6px; + white-space: nowrap; + transition: background 0.15s, color 0.15s; +} + +.mesh-nav-crumb:hover { + background: var(--neutral-layer-1, #fff); +} + +.mesh-nav-crumb-current { + font-weight: 600; + color: var(--neutral-foreground-rest); + cursor: default; +} + +.mesh-nav-sep { + color: var(--neutral-foreground-hint, #999); + user-select: none; +} + +.mesh-nav-open { + display: inline-flex; + align-items: center; + margin-left: 2px; + color: var(--neutral-foreground-hint, #888); + text-decoration: none; + padding: 2px; + border-radius: 4px; + opacity: 0.6; + transition: opacity 0.15s, color 0.15s; +} + +.mesh-nav-open:hover { + opacity: 1; + color: var(--accent-foreground-rest, #0078d4); +} + +/* Include-documents toggle (node Search area only). */ +.mesh-nav-doctoggle { + display: inline-flex; + align-items: center; + gap: 6px; + font-size: 0.85rem; + color: var(--neutral-foreground-hint, #777); + cursor: pointer; + user-select: none; + width: fit-content; +} + +.mesh-nav-doctoggle input { + cursor: pointer; + margin: 0; +} + +/* A section (Nodes on top, Namespaces at the bottom). */ +.mesh-nav-section { + display: flex; + flex-direction: column; + gap: 8px; + width: 100%; +} + +.mesh-nav-section-label { + font-size: 0.78rem; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--neutral-foreground-hint, #777); +} + +.mesh-nav-empty { + padding: 24px 0; +} + +/* TOP: a node card. The body opens the node; a drill symbol (containers only) re-roots here. */ +.mesh-nav-card-wrap { + position: relative; + width: 100%; +} + +.mesh-nav-card-link { + display: block; + width: 100%; + text-decoration: none; + color: inherit; + border-radius: 10px; + transition: transform 0.15s, box-shadow 0.15s; +} + +.mesh-nav-card-link:hover { + transform: translateY(-1px); + box-shadow: var(--elevation-shadow-card-active); +} + +/* "Drill in" symbol on a node that also contains content — reuses .mesh-search-drilldown base. */ +.mesh-nav-card-wrap:hover .mesh-nav-card-drill, +.mesh-nav-card-drill:focus-visible { + opacity: 1; +} + +/* BOTTOM: sub-namespaces as drill-down LINKS — deliberately NOT card-styled, to read as "navigate". + Responsive grid that caps at 3 columns: 1 on narrow, 2 on medium, 3 on wide (each column ≥240px, + and ≥1/3 width so it never exceeds three). Same column-capping trick as the node card grid. */ +.mesh-nav-namespaces { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(max(33.333% - 6px, 240px), 1fr)); + gap: 4px 8px; + width: 100%; +} + +.mesh-nav-namespace { + display: flex; + align-items: center; + gap: 10px; + padding: 8px 10px; + border-radius: 8px; + text-decoration: none; + color: var(--neutral-foreground-rest); + border: 1px solid transparent; + transition: background 0.15s, border-color 0.15s; +} + +.mesh-nav-namespace:hover { + background: var(--neutral-layer-2, rgba(0, 0, 0, 0.04)); + border-color: var(--neutral-stroke-rest, #e0e0e0); +} + +.mesh-nav-namespace-icon { + display: inline-flex; + align-items: center; + color: var(--accent-foreground-rest, #0078d4); + flex-shrink: 0; +} + +.mesh-nav-namespace-name { + flex: 1; + min-width: 0; + font-weight: 500; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.mesh-nav-namespace-count { + font-size: 0.75rem; + font-weight: 600; + line-height: 1; + padding: 3px 8px; + border-radius: 10px; + background: var(--neutral-fill-secondary-rest, #e8e8e8); + color: var(--neutral-foreground-hint, #666); + flex-shrink: 0; +} + +.mesh-nav-namespace-chevron { + color: var(--neutral-foreground-hint, #999); + font-size: 1.1rem; + line-height: 1; + flex-shrink: 0; +} + /* Skeleton loading placeholders */ .mesh-search-skeleton-avatar { width: 48px; diff --git a/src/MeshWeaver.Blazor/Components/Monaco/CodeEditorView.razor b/src/MeshWeaver.Blazor/Components/Monaco/CodeEditorView.razor index 0bce7264c..7e8d742f4 100644 --- a/src/MeshWeaver.Blazor/Components/Monaco/CodeEditorView.razor +++ b/src/MeshWeaver.Blazor/Components/Monaco/CodeEditorView.razor @@ -2,20 +2,25 @@ @using MeshWeaver.Layout.Client @using MeshWeaver.Data @using MeshWeaver.Blazor.Components.Monaco +@using MeshWeaver.Mesh.Services.LanguageServer @using System.Text.Json @using System.Reactive.Linq @implements IDisposable +@inject IServiceProvider Services + ValueChanged="@OnValueChanged" + DiagnosticsCallback="@diagnosticsCallback" + DiagnosticSourcePath="@diagnosticSourcePath" /> @code { private MonacoEditorView? editorRef; @@ -28,6 +33,12 @@ private bool BoundReadonly = false; private string BoundPlaceholder = ""; + // Live-diagnostics wiring — populated by BindData when ViewModel.LanguageServer is set. + // Re-resolved on each OnParametersSet so a swap of CodeEditorControl (e.g., editing a + // different Code node in the same component lifetime) picks up the new paths. + private Func>>? diagnosticsCallback; + private string? diagnosticSourcePath; + // Store the data pointer for two-way binding private JsonPointerReference? ValuePointer; @@ -110,6 +121,55 @@ BoundHeight = ResolveValue(ViewModel.Height) ?? "300px"; BoundReadonly = ResolveValue(ViewModel.Readonly) ?? false; BoundPlaceholder = ResolveValue(ViewModel.Placeholder) ?? ""; + + // LSP opt-in: when the control specifies LanguageServer, resolve IMeshLanguageService + // from DI and build a closure that feeds the editor's current text into CheckSpeculative + // on each debounced change. DiagnosticSourcePath filters to this file so we don't + // surface diagnostics from sibling sources as squiggles in the wrong editor. + var lspConfig = ViewModel.LanguageServer; + if (lspConfig != null) + { + var languageService = Services.GetService(typeof(IMeshLanguageService)) as IMeshLanguageService; + if (languageService != null) + { + var nodeTypePath = lspConfig.NodeTypePath; + var sourcePath = lspConfig.SourcePath; + diagnosticsCallback = text => + languageService.CheckSpeculative(nodeTypePath, sourcePath, text ?? string.Empty); + diagnosticSourcePath = sourcePath; + } + else + { + diagnosticsCallback = null; + diagnosticSourcePath = null; + } + } + else if (ViewModel.Diagnostics is { Count: > 0 } staticDiags) + { + // Static, pre-computed markers (the compile-error "error overlay"): convert the + // flat Layout-level CodeEditorDiagnostic shape into the DiagnosticInfo the marker + // pipeline expects, and feed them as a constant observable. The JS enableDiagnostics + // fires an initial diagnostics pass on load, so the squiggles appear immediately + // even though the (read-only) content never changes — no live language-server call. + var infos = (IReadOnlyList)staticDiags + .Select(d => new DiagnosticInfo( + d.Code ?? string.Empty, + (DiagnosticSeverity)d.Severity, + d.Message, + new SourceLocation( + string.Empty, + new SourceRange( + new SourcePosition(d.StartLine, d.StartCharacter), + new SourcePosition(d.EndLine, d.EndCharacter))))) + .ToList(); + diagnosticsCallback = _ => Observable.Return(infos); + diagnosticSourcePath = null; // already this file's diagnostics — no filtering + } + else + { + diagnosticsCallback = null; + diagnosticSourcePath = null; + } } private string? GetEffectiveDataContext() diff --git a/src/MeshWeaver.Blazor/Components/Monaco/MarkdownMonacoEditor.razor b/src/MeshWeaver.Blazor/Components/Monaco/MarkdownMonacoEditor.razor index 44c687638..2c010b689 100644 --- a/src/MeshWeaver.Blazor/Components/Monaco/MarkdownMonacoEditor.razor +++ b/src/MeshWeaver.Blazor/Components/Monaco/MarkdownMonacoEditor.razor @@ -1,5 +1,8 @@ +@using System.Reactive.Linq +@using MeshWeaver.Mesh @using MeshWeaver.Mesh.Services @using MeshWeaver.Messaging +@using MeshWeaver.Reactive @inject IMeshService? MeshQuery @inject IMessageHub Hub @@ -16,7 +19,7 @@ ShowBorder="@ShowBorder" CodeEditMode="@CodeEditMode" CompletionProvider="@DefaultCompletionProvider" - AsyncCompletionCallback="@GetCompletionsAsync" /> + CompletionCallback="@GetCompletions" /> @code { private MonacoEditorView? editorRef; @@ -55,7 +58,7 @@ /// Optional custom completion callback. If not provided, uses default UCR autocomplete. /// [Parameter] - public Func>? CustomCompletionCallback { get; set; } + public Func>>? CustomCompletionCallback { get; set; } private static readonly CompletionProviderConfig DefaultCompletionProvider = new() { @@ -63,101 +66,62 @@ Items = [] }; - private async Task GetCompletionsAsync(string query) + private const int CompletionLimit = 20; + + // Higher score = better. Sort descending. + private static readonly IComparer CompletionByScore = + Comparer.Create((a, b) => b.Score.CompareTo(a.Score)); + + /// + /// Streams top-N UCR completion snapshots for the current query. Monaco subscribes + /// once per query and pushes each fresh snapshot into the suggest widget as it + /// arrives. No Task, no await. + /// + private IObservable> GetCompletions(string query) { - // Use custom callback if provided if (CustomCompletionCallback != null) - { - return await CustomCompletionCallback(query); - } + return CustomCompletionCallback(query); - // Default UCR autocomplete using IMeshService if (MeshQuery == null || string.IsNullOrWhiteSpace(query)) - return []; - - try - { - if (query.StartsWith("@")) - { - var reference = query[1..]; - return await GetReferenceCompletionsAsync(reference); - } - - return await SearchNodesAsync(query); - } - catch - { - return []; - } - } - - private async Task GetReferenceCompletionsAsync(string reference) - { - if (string.IsNullOrWhiteSpace(reference)) - { - return await GetTopLevelNodesAsync(); - } + return Observable.Return>(Array.Empty()); - if (reference.EndsWith("/")) + if (query.StartsWith("@")) { - var basePath = reference.TrimEnd('/'); - return await GetChildNodesAsync(basePath); + var reference = query[1..]; + if (string.IsNullOrWhiteSpace(reference)) + return BuildPathCompletions("", "", "Addresses"); + if (reference.EndsWith("/")) + return BuildPathCompletions(reference.TrimEnd('/'), "", ""); + return BuildPathCompletions("", reference, "Addresses"); } - return await GetNodesMatchingPrefixAsync(reference); - } - - private async Task GetTopLevelNodesAsync() - { - var suggestions = await MeshQuery!.AutocompleteAsync("", "", 20).ToArrayAsync(); - return suggestions.Select(s => new CompletionItem - { - Label = $"{s.Path}/", - InsertText = $"@{s.Path}/", - Description = s.NodeType ?? s.Name, - Detail = s.Name, - Category = "Addresses" - }).ToArray(); - } - - private async Task GetChildNodesAsync(string basePath) - { - var suggestions = await MeshQuery!.AutocompleteAsync(basePath, "", 20).ToArrayAsync(); - return suggestions.Select(s => new CompletionItem - { - Label = $"{s.Path}/", - InsertText = $"@{s.Path}/", - Description = s.NodeType ?? "", - Detail = s.Name, - Category = "" - }).ToArray(); - } - - private async Task GetNodesMatchingPrefixAsync(string prefix) - { - var suggestions = await MeshQuery!.AutocompleteAsync("", prefix, 20).ToArrayAsync(); - return suggestions.Select(s => new CompletionItem - { - Label = $"{s.Path}/", - InsertText = $"@{s.Path}/", - Description = s.NodeType ?? s.Name, - Detail = s.Name, - Category = "Addresses" - }).ToArray(); + return MeshQuery + .Autocomplete("", query, AutocompleteMode.PathFirst, CompletionLimit) + .Select(snapshot => (IReadOnlyList)snapshot + .Select(s => new CompletionItem + { + Label = s.Path, + InsertText = s.Path, + Description = s.NodeType ?? "", + Detail = s.Name, + Category = "" + }) + .ToArray()); } - private async Task SearchNodesAsync(string query) - { - var suggestions = await MeshQuery!.AutocompleteAsync("", query, 20).ToArrayAsync(); - return suggestions.Select(s => new CompletionItem - { - Label = s.Path, - InsertText = s.Path, - Description = s.NodeType ?? "", - Detail = s.Name, - Category = "" - }).ToArray(); - } + private IObservable> BuildPathCompletions(string basePath, string prefix, string category) + => MeshQuery! + .Autocomplete(basePath, prefix, AutocompleteMode.PathFirst, CompletionLimit) + .Select(snapshot => (IReadOnlyList)snapshot + .Select(s => new CompletionItem + { + Label = $"{s.Path}/", + InsertText = $"@{s.Path}/", + Description = s.NodeType ?? (string.IsNullOrEmpty(category) ? "" : s.Name), + Detail = s.Name, + Category = category + }) + .ToArray()); // Expose methods from the underlying editor public async ValueTask GetValueAsync() diff --git a/src/MeshWeaver.Blazor/Components/Monaco/MonacoEditorView.razor b/src/MeshWeaver.Blazor/Components/Monaco/MonacoEditorView.razor index 3dd503220..207b5bb10 100644 --- a/src/MeshWeaver.Blazor/Components/Monaco/MonacoEditorView.razor +++ b/src/MeshWeaver.Blazor/Components/Monaco/MonacoEditorView.razor @@ -1,14 +1,23 @@ @using Microsoft.FluentUI.AspNetCore.Components @using Microsoft.Extensions.Logging +@using System.Reactive.Linq @inject IJSRuntime JSRuntime @inject ILogger Logger
    + @* 🚨 Do NOT wire OnDidChangeModelContent (or any other BlazorMonaco editor-event + callback) here. BlazorMonaco registers those through its ASYNC SetEventListeners + in OnAfterRenderAsync(firstRender); when the editor is mid-teardown/recreate that + awaited JS interop throws JSException "Couldn't find the editor with id …" straight + out of the render lifecycle, which tears down the whole Blazor circuit (the chat + input "Connection disconnected" blocker). We wire onDidChangeModelContent + SYNCHRONOUSLY on the editor instance in our own JS module (initEditor) instead and + push the value to HandleContentChanged — no async event-listener race, no + editor.GetValue() round-trip. See Doc/Architecture/AsynchronousCalls.md. *@
    @@ -63,11 +72,35 @@ public CompletionProviderConfig? CompletionProvider { get; set; } /// - /// Optional async callback for server-side completion with fuzzy scoring. - /// When provided, this will be called instead of using static CompletionProvider items. + /// Optional reactive callback for server-side completion with fuzzy scoring. + /// When provided, this is invoked instead of using static + /// items. The callback returns an that emits a fresh + /// snapshot of the top-N completions whenever a new item arrives — the editor + /// pushes each snapshot to the suggest widget so the UI never blocks waiting for + /// the full result set. /// [Parameter] - public Func>? AsyncCompletionCallback { get; set; } + public Func>>? CompletionCallback { get; set; } + + /// + /// Optional reactive callback that supplies live diagnostics (squiggles) for the + /// current editor content. Called with the editor's full text on every debounced + /// content change. Each emission is converted to Monaco markers and rendered via + /// monaco.editor.setModelMarkers. The callback typically wraps + /// IMeshLanguageService.CheckSpeculative — see Stage-3 LSP wiring. + /// filters emissions to diagnostics whose + /// location matches the file currently in the editor; null = show all. + /// + [Parameter] + public Func>>? DiagnosticsCallback { get; set; } + + /// + /// When is set, restrict surfaced diagnostics to + /// those whose SourcePath matches this value. Suppresses noise from sibling + /// source files in the same compilation. Null = show all diagnostics regardless. + /// + [Parameter] + public string? DiagnosticSourcePath { get; set; } /// /// Show line numbers in the editor. Default is false (chat-style input). @@ -185,6 +218,13 @@ // Register completion provider if configured await TryRegisterCompletionProvider(); + + // Enable LSP-style live diagnostics if the consumer wired a callback. + // JS side debounces onDidChangeModelContent and invokes back to RequestDiagnostics. + if (DiagnosticsCallback != null) + { + await jsModule.InvokeVoidAsync("enableDiagnostics", EditorId, dotNetRef); + } } // Notify parent that editor is fully ready @@ -197,7 +237,7 @@ return; // Check if async mode should be used - var useAsync = AsyncCompletionCallback != null; + var useAsync = CompletionCallback != null; var currentItemCount = CompletionProvider?.Items?.Count ?? 0; // For async mode, we always register (the callback handles everything) @@ -261,17 +301,23 @@ lastCompletionItemCount = currentItemCount; } - private async Task OnContentChanged(ModelContentChangedEvent e) + /// + /// Content-change callback, invoked SYNCHRONOUSLY from our JS module's + /// onDidChangeModelContent handler (see MonacoEditorView.razor.js → initEditor) + /// with the editor's current text. Replaces the BlazorMonaco OnDidChangeModelContent + /// Razor callback whose async SetEventListeners registration raced the editor + /// lifecycle and tore down the circuit. JS already has the value, so there is no + /// editor.GetValue() round-trip here. + /// + [JSInvokable] + public async Task HandleContentChanged(string? value) { - if (editor != null) + value ??= string.Empty; + if (value != Value) { - var value = await editor.GetValue(); - if (value != Value) - { - Value = value; - lastSetValue = value; // Track user-typed value - await ValueChanged.InvokeAsync(value); - } + Value = value; + lastSetValue = value; // Track user-typed value + await ValueChanged.InvokeAsync(value); } } @@ -304,37 +350,116 @@ await OnCompletionAccepted.InvokeAsync(path); } + // Latest snapshot from the active completion observable. Updated by the Subscribe + // handler in GetAsyncCompletions; returned synchronously to JS on subsequent invokes. + private CompletionItem[] _currentCompletions = []; + private string _currentCompletionsQuery = ""; + private IDisposable? _activeCompletionSub; + /// /// JSInvokable method called from JavaScript to get async completions. + /// Subscribes to the configured observable callback; returns the current snapshot + /// synchronously and pushes subsequent snapshots via + /// so the suggest widget streams in results as they arrive. /// [JSInvokable] - public async Task GetAsyncCompletions(string query) + public Task GetAsyncCompletions(string query) { - if (AsyncCompletionCallback == null) + if (CompletionCallback == null) + return Task.FromResult(Array.Empty()); + + if (!string.Equals(_currentCompletionsQuery, query, StringComparison.Ordinal)) { - return []; + _currentCompletionsQuery = query; + _activeCompletionSub?.Dispose(); + _currentCompletions = []; + + try + { + _activeCompletionSub = CompletionCallback(query).Subscribe( + snapshot => + { + var arr = snapshot is CompletionItem[] a ? a : snapshot.ToArray(); + _currentCompletions = arr; + // Fire-and-forget: order is preserved by JS event loop. We can't + // await here — Subscribe handlers must stay synchronous. + if (jsModule != null) + _ = PushCompletionUpdateAsync(arr); + }, + ex => Logger.LogError(ex, "Error getting async completions for query: {Query}", query)); + } + catch (Exception ex) + { + Logger.LogError(ex, "Error subscribing to async completions for query: {Query}", query); + } } + return Task.FromResult(MapCompletionsToJs(_currentCompletions)); + } + + private static object[] MapCompletionsToJs(IReadOnlyList items) + => items.Select(i => (object)new + { + label = i.Label, + insertText = i.InsertText ?? i.Label, + description = i.Description ?? "", + detail = i.Detail ?? "", + category = i.Category ?? "", + path = i.Path ?? "", + iconUrl = i.IconUrl ?? "", + sortKey = i.SortKey ?? "" + }).ToArray(); + + // Latest active subscription to the diagnostics callback. Disposed on the next + // RequestDiagnostics call (each keystroke window starts a fresh subscription) and + // on component disposal so we never leak an observable across editor lifecycles. + private IDisposable? _activeDiagnosticsSub; + + /// + /// JSInvokable: the JS module's debounced onDidChangeModelContent listener calls + /// this with the current editor text. We invoke the consumer's + /// , take the first emission, optionally filter + /// to , and push results to JS as Monaco markers. + /// + [JSInvokable] + public Task RequestDiagnostics(string currentText) + { + if (DiagnosticsCallback == null || jsModule == null) + return Task.CompletedTask; + + _activeDiagnosticsSub?.Dispose(); try { - var items = await AsyncCompletionCallback(query); - return items.Select(i => new - { - label = i.Label, - insertText = i.InsertText ?? i.Label, - description = i.Description ?? "", - detail = i.Detail ?? "", - category = i.Category ?? "", - path = i.Path ?? "", - iconUrl = i.IconUrl ?? "", - sortKey = i.SortKey ?? "" - }).ToArray(); + _activeDiagnosticsSub = DiagnosticsCallback(currentText ?? string.Empty) + .Take(1) + .Subscribe( + diagnostics => + { + var filtered = DiagnosticSourcePath is null + ? diagnostics + : diagnostics.Where(d => d.Location?.SourcePath == DiagnosticSourcePath).ToList(); + var payload = filtered.Select(d => new + { + severity = (int)d.Severity, + message = d.Message, + id = d.Id, + startLine = d.Location?.Range.Start.Line ?? 0, + startCharacter = d.Location?.Range.Start.Character ?? 0, + endLine = d.Location?.Range.End.Line ?? 0, + endCharacter = d.Location?.Range.End.Character ?? 0, + }).ToArray(); + // Fire-and-forget — JS event-loop preserves order; we don't block the + // Subscribe handler. Errors during push surface in the JS console. + if (jsModule != null) + _ = jsModule.InvokeVoidAsync("pushDiagnostics", EditorId, payload).AsTask(); + }, + ex => Logger.LogError(ex, "Diagnostics callback failed for editor {EditorId}", EditorId)); } catch (Exception ex) { - Logger.LogError(ex, "Error getting async completions for query: {Query}", query); - return []; + Logger.LogError(ex, "Error subscribing to diagnostics callback for editor {EditorId}", EditorId); } + return Task.CompletedTask; } /// @@ -632,6 +757,8 @@ public async ValueTask DisposeAsync() { + _activeDiagnosticsSub?.Dispose(); + _activeCompletionSub?.Dispose(); if (jsModule != null) { try diff --git a/src/MeshWeaver.Blazor/Components/Monaco/MonacoEditorView.razor.js b/src/MeshWeaver.Blazor/Components/Monaco/MonacoEditorView.razor.js index 62cc85251..a0f926573 100644 --- a/src/MeshWeaver.Blazor/Components/Monaco/MonacoEditorView.razor.js +++ b/src/MeshWeaver.Blazor/Components/Monaco/MonacoEditorView.razor.js @@ -267,10 +267,22 @@ export function initEditor(editorId, placeholder, dotNetRef, codeEditMode = fals monaco.editor.setTheme(monacoTheme); } if (editorInstance) { - // Handle content changes for placeholder + // Handle content changes for placeholder AND push the value back to C#. + // 🚨 This is the SYNC content-change wiring that replaces BlazorMonaco's + // OnDidChangeModelContent Razor callback: that callback is registered via + // BlazorMonaco's async SetEventListeners in OnAfterRenderAsync, which throws + // "Couldn't find the editor with id …" (tearing down the circuit) when the + // editor is mid-teardown. Wiring the listener directly on the editor instance + // here avoids that race. The invoke is best-effort — a disposed dotNetRef + // rejects, which we swallow so a teardown can never surface an unhandled error. editorInstance.onDidChangeModelContent(() => { const value = editorInstance.getValue(); updatePlaceholderVisibility(editorId, !value); + const st = editorState.get(editorId); + if (st?.dotNetRef) { + st.dotNetRef.invokeMethodAsync('HandleContentChanged', value) + .catch(err => console.debug('HandleContentChanged failed (editor disposed?):', err)); + } }); // Handle Enter key - in code edit mode, Enter inserts newline; in chat mode, Enter submits @@ -692,6 +704,81 @@ export function pushCompletionUpdate(editorId, items) { } } +// ============================================================================= +// LSP-style live diagnostics (Stage-3) +// ============================================================================= +// Subscribes to debounced onDidChangeModelContent and calls back to .NET with the +// current model text; the .NET side (RequestDiagnostics) invokes the consumer's +// IObservable, then pushes results here via pushDiagnostics. + +const DIAGNOSTICS_DEBOUNCE_MS = 300; +const DIAGNOSTICS_MARKER_OWNER = 'meshweaver-lsp'; + +export function enableDiagnostics(editorId, dotNetRef) { + const state = editorState.get(editorId); + if (!state || !state.editorInstance) return; + if (state._diagnosticsEnabled) return; // idempotent + state._diagnosticsEnabled = true; + state._diagnosticsDotNetRef = dotNetRef; + + const requestNow = () => { + const model = state.editorInstance?.getModel(); + if (!model) return; + dotNetRef.invokeMethodAsync('RequestDiagnostics', model.getValue()).catch(err => { + console.warn('LSP RequestDiagnostics failed:', err); + }); + }; + + const scheduleRequest = () => { + if (state._diagnosticsTimer) clearTimeout(state._diagnosticsTimer); + state._diagnosticsTimer = setTimeout(requestNow, DIAGNOSTICS_DEBOUNCE_MS); + }; + + // Initial fetch — current saved text gets a diagnostics pass before the user types. + requestNow(); + + // Subsequent edits — debounced so we don't fire on every keystroke. + state._diagnosticsChangeDisposable = state.editorInstance.onDidChangeModelContent(() => { + scheduleRequest(); + }); +} + +// Maps an LSP DiagnosticSeverity (0..3) to a Monaco MarkerSeverity (Hint=1, Info=2, Warning=4, Error=8). +function lspSeverityToMonaco(lspSeverity) { + switch (lspSeverity) { + case 3: return 8; // Error + case 2: return 4; // Warning + case 1: return 2; // Info + case 0: return 1; // Hidden / Hint + default: return 2; + } +} + +// Push a fresh diagnostics snapshot from .NET into Monaco's marker layer for this editor. +// Items shape (matches RequestDiagnostics's anonymous payload): +// { severity, message, id, startLine, startCharacter, endLine, endCharacter } (LSP 0-based) +export function pushDiagnostics(editorId, items) { + const state = editorState.get(editorId); + if (!state) return; + const editorInstance = state.editorInstance; + if (!editorInstance) return; + const model = editorInstance.getModel(); + if (!model) return; + + const markers = (items || []).map(d => ({ + // Monaco is 1-based; LSP is 0-based — add 1 to each coordinate. + startLineNumber: (d.startLine | 0) + 1, + startColumn: (d.startCharacter | 0) + 1, + endLineNumber: (d.endLine | 0) + 1, + endColumn: (d.endCharacter | 0) + 1, + message: d.message || '', + code: d.id || '', + severity: lspSeverityToMonaco(d.severity | 0), + source: 'Roslyn', + })); + monaco.editor.setModelMarkers(model, DIAGNOSTICS_MARKER_OWNER, markers); +} + // Set cursor position to end of content export function setCursorToEnd(editorId) { const editorInstance = editorState.get(editorId)?.editorInstance; @@ -968,6 +1055,19 @@ export function dispose(editorId) { if (state.completionDisposable) { state.completionDisposable.dispose(); } + if (state._diagnosticsChangeDisposable) { + state._diagnosticsChangeDisposable.dispose(); + } + if (state._diagnosticsTimer) { + clearTimeout(state._diagnosticsTimer); + } + const editorInstance = state.editorInstance; + if (editorInstance) { + const model = editorInstance.getModel(); + if (model) { + monaco.editor.setModelMarkers(model, DIAGNOSTICS_MARKER_OWNER, []); + } + } editorState.delete(editorId); } } diff --git a/src/MeshWeaver.Blazor/Components/Monaco/NotebookEditorView.razor b/src/MeshWeaver.Blazor/Components/Monaco/NotebookEditorView.razor index 2821970d7..3b41a666a 100644 --- a/src/MeshWeaver.Blazor/Components/Monaco/NotebookEditorView.razor +++ b/src/MeshWeaver.Blazor/Components/Monaco/NotebookEditorView.razor @@ -7,6 +7,7 @@ @using System.Collections.Immutable @using System.Text.Json @using System.Reactive.Linq +@using System.Reactive.Threading.Tasks @using FluentAppearance = Microsoft.FluentUI.AspNetCore.Components.Appearance @using FluentOrientation = Microsoft.FluentUI.AspNetCore.Components.Orientation @inject IJSRuntime JSRuntime @@ -274,10 +275,13 @@ } } - private async Task RunCell(int index) + private void RunCell(int index, Action? onComplete = null) { if (index < 0 || index >= cellControls.Count) + { + onComplete?.Invoke(); return; + } var cellControl = cellControls[index]; var skin = cellControl.Skins.OfType().FirstOrDefault(); @@ -285,84 +289,81 @@ var cellId = skin?.CellId as string ?? ""; if (cellType != NotebookCellType.Code) + { + onComplete?.Invoke(); return; + } executingCellIds.Add(cellId); StateHasChanged(); - try - { - var content = cellControl.Content as string ?? ""; - var language = skin?.Language as string ?? "csharp"; - var output = await ExecuteCodeAsync(content, language); - executionCounter++; - - cellControls[index] = cellControl - .WithOutput(output) - .WithExecutionCount(executionCounter) - .WithIsExecuting(false); - } - catch (Exception ex) - { - cellControls[index] = cellControl - .WithOutput(new NotebookCellOutput - { - Error = ex.Message, - IsSuccess = false - }) - .WithIsExecuting(false); - } - finally + var content = cellControl.Content as string ?? ""; + var language = skin?.Language as string ?? "csharp"; + + // Subscribe — no await on hub round-trip. Update state via InvokeAsync. + ExecuteCode(content, language).Subscribe(output => { - executingCellIds.Remove(cellId); - NotifyCellsChanged(); - } + InvokeAsync(() => + { + executionCounter++; + cellControls[index] = cellControl + .WithOutput(output) + .WithExecutionCount(executionCounter) + .WithIsExecuting(false); + executingCellIds.Remove(cellId); + NotifyCellsChanged(); + onComplete?.Invoke(); + }); + }); } - private async Task RunAndAddBelow(int index) + private void RunAndAddBelow(int index) { - await RunCell(index); - AddCellBelow(index, NotebookCellType.Code); + RunCell(index, () => AddCellBelow(index, NotebookCellType.Code)); } - private async Task RunAndMoveNext(int index) + private void RunAndMoveNext(int index) { - await RunCell(index); - if (index < cellControls.Count - 1) + RunCell(index, () => { - selectedCellIndex = index + 1; - StateHasChanged(); - } - else - { - AddCellBelow(index, NotebookCellType.Code); - } + if (index < cellControls.Count - 1) + { + selectedCellIndex = index + 1; + StateHasChanged(); + } + else + { + AddCellBelow(index, NotebookCellType.Code); + } + }); } - private async Task RunAllCells() + private void RunAllCells() { isRunningAll = true; StateHasChanged(); + RunNextCellInSequence(0); + } - try + private void RunNextCellInSequence(int startIndex) + { + for (int i = startIndex; i < cellControls.Count; i++) { - for (int i = 0; i < cellControls.Count; i++) + var skin = cellControls[i].Skins.OfType().FirstOrDefault(); + var cellType = skin?.CellType as NotebookCellType? ?? NotebookCellType.Code; + if (cellType == NotebookCellType.Code) { - var skin = cellControls[i].Skins.OfType().FirstOrDefault(); - var cellType = skin?.CellType as NotebookCellType? ?? NotebookCellType.Code; - if (cellType == NotebookCellType.Code) - { - selectedCellIndex = i; - StateHasChanged(); - await RunCell(i); - } + selectedCellIndex = i; + StateHasChanged(); + var nextIndex = i + 1; + RunCell(i, () => RunNextCellInSequence(nextIndex)); + return; // RunCell continuation drives the rest } } - finally - { - isRunningAll = false; - StateHasChanged(); - } + + // No more code cells — done. + isRunningAll = false; + StateHasChanged(); } private void ClearAllOutputs() @@ -377,42 +378,30 @@ NotifyCellsChanged(); } - private async Task ExecuteCodeAsync(string code, string language) + private IObservable ExecuteCode(string code, string language) { if (Hub == null || BoundKernelAddress == null) { - return new NotebookCellOutput + return Observable.Return(new NotebookCellOutput { Error = "No kernel available for execution. Please configure KernelAddress.", IsSuccess = false - }; + }); } - try - { - var request = new SubmitCodeRequest(code); - // Post request and wait for response - var response = await Hub.AwaitResponse(request, o => o.WithTarget(BoundKernelAddress)); - - if (response is NotebookCellOutput output) + var request = new SubmitCodeRequest(code); + return Hub.Observe(request, o => o.WithTarget(BoundKernelAddress)) + .FirstAsync() + .Select(d => new NotebookCellOutput { - return output; - } - - return new NotebookCellOutput - { - Value = response, - IsSuccess = true - }; - } - catch (Exception ex) - { - return new NotebookCellOutput + Value = d.Message?.SubmissionId, + IsSuccess = d.Message?.Success ?? false + }) + .Catch(ex => Observable.Return(new NotebookCellOutput { Error = ex.Message, IsSuccess = false - }; - } + })); } private void OnGlobalKeyDown(KeyboardEventArgs e) diff --git a/src/MeshWeaver.Blazor/Components/NamedAreaView.razor.cs b/src/MeshWeaver.Blazor/Components/NamedAreaView.razor.cs index c99ff794c..e177fc460 100644 --- a/src/MeshWeaver.Blazor/Components/NamedAreaView.razor.cs +++ b/src/MeshWeaver.Blazor/Components/NamedAreaView.razor.cs @@ -1,4 +1,8 @@ -using MeshWeaver.Layout; +using MeshWeaver.Data; +using MeshWeaver.Graph; +using MeshWeaver.Layout; +using MeshWeaver.Mesh; +using MeshWeaver.Messaging; using Microsoft.AspNetCore.Components; using Microsoft.Extensions.Logging; @@ -37,8 +41,15 @@ protected override void BindData() if (Stream is null) return; - // When area is empty, GetControlStream returns a NamedAreaControl pointing to the default area - var controlStream = Stream.GetControlStream(AreaToBeRendered); + // When area is empty, GetControlStream returns a NamedAreaControl pointing to the default area. + // Bounded, throttled, fully-reactive retry: a transiently unaddressable area (per-node hub + // still bootstrapping, NodeType mid-compile, activity node not yet routable) self-heals within + // a few backoff steps; an inexistent address gives up after AreaStreamRetry.DefaultMaxRetries + // instead of resubscribing forever (the NotFound message storm that wedged the partition). + // A CompilationInProgress NACK is NOT retried — it falls straight through to the error handler + // below, which swaps to the Progress view at once. + var controlStream = Stream.GetControlStream(AreaToBeRendered) + .RetryAreaWithBackoff(AreaErrorClassifier.ShouldRetryArea); AddBinding(controlStream .Subscribe( @@ -80,7 +91,96 @@ protected override void BindData() Logger.LogDebug(error, "Suppressed teardown error in control stream for area {Area}", AreaToBeRendered); return; } - Logger.LogError(error, "Error in control stream for area {Area}", AreaToBeRendered); + + // Routing-grain NACK: the target hub cannot activate because its NodeType + // is mid-compile. Swap RootControl to a LayoutAreaControl pointing at the + // NodeType hub's "Progress" area — Blazor renders that as a fresh + // LayoutAreaView whose stream binds to the NodeType's own MeshNode stream + // (status lines) and, transitively, the compile activity's progress. + // When the compile settles and the original area becomes addressable, the + // user re-navigates / re-mounts and the regular path resumes. Detected + // strictly on the typed Failure.ErrorType — no string sniff. + if (AreaErrorClassifier.TryGetCompilationInProgressNodeType(error) is { } nodeTypePath) + { + Logger.LogInformation( + "NACK CompilationInProgress for area {Area} on NodeType {NodeType} — swapping to Progress", + AreaToBeRendered, nodeTypePath); + try + { + InvokeAsync(() => + { + if (IsViewDisposed) return; + try + { + RootControl = new LayoutAreaControl( + new Address(nodeTypePath), + new LayoutAreaReference(NodeTypeLayoutAreas.ProgressArea)); + RequestStateChange(); + } + catch (ObjectDisposedException) { /* renderer gone */ } + }); + } + catch (ObjectDisposedException) { /* renderer gone */ } + return; + } + + // Transient hub/network failures (request timeouts, undeliverable + // routing, dropped circuit) are usually self-healing — the upstream + // hub finishes initialising, the security pipeline emits its first + // value, the per-node hub comes back online — and a subsequent + // navigation will resubscribe. Don't render the framework-internal + // "No response received in hub …" markdown to the user (hostile + // and unactionable). Log at Warning, leave the previous RootControl + // in place so the GUI doesn't flicker between "loading" and "error", + // and let the next BindData (route change, parameter change) restart + // the subscription naturally. Auto-retrying from this handler used + // to be tempting but caused a feedback loop: each retry recreated + // a subscription that emitted null first, the success handler reset + // the counter, the next failure re-armed retries, and the GUI looped + // forever consuming circuit bandwidth. + if (AreaErrorClassifier.IsTransientHubFailure(error)) + { + // By the time a transient failure reaches here the bounded reactive + // retry (RetryAreaWithBackoff, applied to controlStream above) is + // ALREADY exhausted — the upstream had AreaStreamRetry.DefaultMaxRetries + // attempts over ~8s of backoff to come online and didn't. Give up and + // report, rather than spin forever (the inexistent-address storm) or + // silently keep a stale render that never updates. The earlier + // "keep previous, no retry" behaviour is now the retry's job. + Logger.LogWarning(error, + "Area {Area} unavailable after {Retries} reactive retries — reporting failure. Hub={Message}", + AreaToBeRendered, AreaStreamRetry.DefaultMaxRetries, error.Message); + try + { + InvokeAsync(() => + { + if (IsViewDisposed) return; + try + { + RootControl = new MarkdownControl( + $"**Area unavailable.** The view at `{AreaToBeRendered}` did not become " + + $"addressable after {AreaStreamRetry.DefaultMaxRetries} retries — it may still be " + + "initialising or its NodeType may be compiling. Reload to retry."); + RequestStateChange(); + } + catch (ObjectDisposedException) { /* renderer gone */ } + }); + } + catch (ObjectDisposedException) { /* renderer gone */ } + return; + } + + // Access denied / validation failures / not-found are user-action + // outcomes (the user lacks the right; the input was invalid; the + // node was deleted) — not engineering errors. Log them as Warning + // so production log dashboards don't page on every "user clicked + // a thing they couldn't do". Real errors (NullReferenceException, + // IO failures, runtime crashes) still land at Error level. + if (AreaErrorClassifier.IsExpectedUserActionFailure(error)) + Logger.LogWarning(error, "Expected user-action failure in control stream for area {Area}: {Message}", + AreaToBeRendered, error.Message); + else + Logger.LogError(error, "Error in control stream for area {Area}", AreaToBeRendered); try { InvokeAsync(() => @@ -104,5 +204,4 @@ protected override void BindData() ); } - } diff --git a/src/MeshWeaver.Blazor/Components/NavItemView.cs b/src/MeshWeaver.Blazor/Components/NavItemView.cs index 9b57324ce..6b594004f 100644 --- a/src/MeshWeaver.Blazor/Components/NavItemView.cs +++ b/src/MeshWeaver.Blazor/Components/NavItemView.cs @@ -13,6 +13,7 @@ public abstract class NavItemView : BlazorView /// Cascading parameter indicating whether this component is rendered inside a dropdown menu. @@ -30,6 +31,8 @@ protected override void BindData() DataBind(ViewModel.Title, x => x.Title); DataBind(ViewModel.Url, x => x.Href); DataBind(ViewModel.Icon, x => x.Icon); + if (ViewModel is NavLinkControl link) + DataBind(link.IsActive, x => x.IsActive); } } diff --git a/src/MeshWeaver.Blazor/Components/NavLink.razor b/src/MeshWeaver.Blazor/Components/NavLink.razor index 2d691ed0b..e49209705 100644 --- a/src/MeshWeaver.Blazor/Components/NavLink.razor +++ b/src/MeshWeaver.Blazor/Components/NavLink.razor @@ -2,7 +2,7 @@ @if (IsInMenuContext) { - + @if (Icon != null) { @@ -12,6 +12,7 @@ } else { - @Title + @Title } diff --git a/src/MeshWeaver.Blazor/Components/NavMenuView.razor.css b/src/MeshWeaver.Blazor/Components/NavMenuView.razor.css index acc5f9f81..cacf8b8cf 100644 --- a/src/MeshWeaver.Blazor/Components/NavMenuView.razor.css +++ b/src/MeshWeaver.Blazor/Components/NavMenuView.razor.css @@ -1,3 +1,18 @@ +/* Fill the parent (e.g. a Splitter pane) and let the menu list scroll + internally instead of clipping when there are many items. */ +.navmenu { + display: flex; + flex-direction: column; + height: 100%; + min-height: 0; +} + +.sitenav { + flex: 1 1 auto; + min-height: 0; + overflow-y: auto; +} + .navbar-toggler { background-color: rgba(255, 255, 255, 0.1); } @@ -45,6 +60,27 @@ color: white; } +/* Highlight the currently selected NavLink (set via NavLinkControl.WithIsActive) + so users see which menu item corresponds to the visible content. */ +.sitenav ::deep .fluent-nav-link.active, +.sitenav ::deep a.fluent-nav-link.active { + background-color: var(--neutral-fill-stealth-hover); + color: var(--accent-foreground-rest); + font-weight: 600; + position: relative; +} + +.sitenav ::deep .fluent-nav-link.active::before { + content: ""; + position: absolute; + width: 3px; + height: 60%; + top: 20%; + left: 0; + background-color: var(--accent-fill-rest); + border-radius: 2px; +} + .nav-item ::deep a:hover { background-color: rgba(255,255,255,0.1); color: white; diff --git a/src/MeshWeaver.Blazor/Components/NavigationProgressBar.razor b/src/MeshWeaver.Blazor/Components/NavigationProgressBar.razor new file mode 100644 index 000000000..0149778fa --- /dev/null +++ b/src/MeshWeaver.Blazor/Components/NavigationProgressBar.razor @@ -0,0 +1,40 @@ +@using MeshWeaver.Mesh.Services +@using Microsoft.FluentUI.AspNetCore.Components + +@* + Central "what is the page doing right now" indicator. Renders a spinner + PLUS the NavigationStatus.Message — never a spinner without a label. This + component is what enforces the "no endless spinner" invariant at the UI. +*@ +
    + +

    @Status.Message

    + @if (!string.IsNullOrWhiteSpace(Status.Detail)) + { +

    @Status.Detail

    + } +
    + +@code { + /// + /// The status to render. Must never be null — parent is responsible for + /// supplying a current value (BehaviorSubject semantics guarantee this + /// for INavigationService.Status). + /// + [Parameter, EditorRequired] public NavigationStatus Status { get; set; } = null!; + + /// + /// When true, render a small inline strip (used as an overlay on top of + /// prerendered content) instead of a centered full-page spinner. + /// + [Parameter] public bool Compact { get; set; } + + /// + /// Optional extra CSS class to apply to the container. + /// + [Parameter] public string? CssClass { get; set; } + + private string RingStyle => Compact + ? "width: 20px; height: 20px;" + : "width: 40px; height: 40px;"; +} diff --git a/src/MeshWeaver.Blazor/Components/NavigationProgressBar.razor.css b/src/MeshWeaver.Blazor/Components/NavigationProgressBar.razor.css new file mode 100644 index 000000000..9518e6a32 --- /dev/null +++ b/src/MeshWeaver.Blazor/Components/NavigationProgressBar.razor.css @@ -0,0 +1,40 @@ +.nav-progress { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 12px; +} + +.nav-progress:not(.compact) { + min-height: 100vh; + min-height: 100dvh; + flex: 1; +} + +.nav-progress-message { + margin: 0; + color: var(--neutral-foreground-rest); + font-size: var(--type-ramp-base-font-size); +} + +.nav-progress-detail { + margin: 0; + color: var(--neutral-foreground-hint); + font-size: var(--type-ramp-minus-1-font-size); +} + +.nav-progress.overlay { + position: sticky; + top: 0; + z-index: 10; + flex-direction: row; + padding: 8px 16px; + background-color: var(--neutral-layer-2); + border-bottom: 1px solid var(--neutral-stroke-divider-rest); + min-height: unset; +} + +.nav-progress.overlay .nav-progress-message { + font-size: var(--type-ramp-minus-1-font-size); +} diff --git a/src/MeshWeaver.Blazor/Components/NodeExportView.razor b/src/MeshWeaver.Blazor/Components/NodeExportView.razor index d2b8d86e7..7068d13d3 100644 --- a/src/MeshWeaver.Blazor/Components/NodeExportView.razor +++ b/src/MeshWeaver.Blazor/Components/NodeExportView.razor @@ -1,3 +1,5 @@ +@using System.Reactive.Linq +@using System.Reactive.Threading.Tasks @using MeshWeaver.Mesh @using MeshWeaver.Mesh.Services @using Microsoft.Extensions.DependencyInjection @@ -113,67 +115,15 @@ ExcludedSatellites.Add(satelliteType); } - private async Task ExportAndDownloadAsync() + private void ExportAndDownloadAsync() { - if (string.IsNullOrEmpty(SourcePath)) return; - - IsExporting = true; - ExportError = null; - ExportDone = false; - ProgressMessage = "Exporting nodes..."; + // IMeshExportService was deleted in the persistence-cull (2026-05-12). + // Export will be rewired via per-node CreateNodeRequest fan-out + ZIP + // assembly as a follow-up. + ExportError = "Export is being rewired (persistence-cull migration, 2026-05-12). " + + "The low-level IMeshExportService surface was removed; no replacement wired up yet."; + IsExporting = false; StateHasChanged(); - - string? tempDir = null; - try - { - var exportService = Hub.ServiceProvider.GetRequiredService(); - - tempDir = Path.Combine(Path.GetTempPath(), $"meshexport_{Guid.NewGuid():N}"); - - IReadOnlySet? excluded = ExcludedSatellites.Count > 0 - ? ExcludedSatellites - : null; - - var result = await exportService.ExportToDirectoryAsync(SourcePath, tempDir, excluded); - - if (!result.Success) - { - ExportError = result.Error; - IsExporting = false; - StateHasChanged(); - return; - } - - NodesExported = result.NodesExported; - ProgressMessage = $"Zipping {NodesExported} nodes..."; - StateHasChanged(); - - // Create ZIP in memory - using var memoryStream = new MemoryStream(); - ZipFile.CreateFromDirectory(tempDir, memoryStream, CompressionLevel.Optimal, includeBaseDirectory: false); - memoryStream.Position = 0; - - // Trigger browser download via JS interop - var nodeId = SourcePath.Split('/').LastOrDefault() ?? "export"; - var fileName = $"{nodeId}_{DateTime.UtcNow:yyyyMMdd_HHmmss}.zip"; - - using var streamRef = new DotNetStreamReference(memoryStream); - await JSRuntime.InvokeVoidAsync("meshweaverDownloadFileFromStream", fileName, streamRef); - - IsExporting = false; - ExportDone = true; - } - catch (Exception ex) - { - ExportError = $"Export failed: {ex.Message}"; - IsExporting = false; - } - finally - { - if (tempDir != null && Directory.Exists(tempDir)) - try { Directory.Delete(tempDir, recursive: true); } catch { } - StateHasChanged(); - } } private void Reset() diff --git a/src/MeshWeaver.Blazor/Components/NodeImportView.razor b/src/MeshWeaver.Blazor/Components/NodeImportView.razor index 76dc98f59..e0459f65c 100644 --- a/src/MeshWeaver.Blazor/Components/NodeImportView.razor +++ b/src/MeshWeaver.Blazor/Components/NodeImportView.razor @@ -43,6 +43,7 @@ Mode="InputFileMode.Stream" Multiple="@IsFileMode" Accept="@AcceptFilter" + MaximumFileCount="1000" MaximumFileSize="@(500 * 1024 * 1024)" OnFileUploaded="@OnFileUploadedAsync" OnCompleted="@OnUploadCompletedAsync" /> @@ -170,30 +171,12 @@ StateHasChanged(); } - // Call import service - var importService = Hub.ServiceProvider.GetService(); - if (importService == null) - { - ErrorMessage = "Import service is not available."; - IsImporting = false; - return; - } - - ImportResult = await importService.ImportNodesAsync( - tempDir, - TargetPath, - ForceReimport, - removeMissing: RemoveMissing, - onProgress: (nodes, partitions, currentPath) => - { - ProgressPercent = Math.Min(60 + (nodes + partitions) % 40, 99); - ProgressMessage = $"Importing... {nodes} nodes ({currentPath})"; - InvokeAsync(StateHasChanged); - }, - ct: CancellationToken.None); - - ProgressPercent = 100; - ProgressMessage = "Done!"; + // IMeshImportService was deleted in the persistence-cull (2026-05-12). + // Import will be rewired as per-node CreateNodeRequest fan-out as a follow-up. + ErrorMessage = "Import is being rewired (persistence-cull migration, 2026-05-12). " + + "The low-level IMeshImportService surface was removed; no replacement wired up yet."; + IsImporting = false; + return; } catch (InvalidDataException) { diff --git a/src/MeshWeaver.Blazor/Components/PathBasedLayoutArea.razor b/src/MeshWeaver.Blazor/Components/PathBasedLayoutArea.razor index 2f10db466..68e6787d4 100644 --- a/src/MeshWeaver.Blazor/Components/PathBasedLayoutArea.razor +++ b/src/MeshWeaver.Blazor/Components/PathBasedLayoutArea.razor @@ -1,3 +1,5 @@ +@using System.Reactive.Linq +@using System.Reactive.Threading.Tasks @using MeshWeaver.Layout @using MeshWeaver.Mesh.Services @inject IPathResolver PathResolver @@ -35,7 +37,7 @@ else private string? DiagInfo { get; set; } private LayoutAreaControl? ResolvedViewModel { get; set; } - protected override async Task OnParametersSetAsync() + protected override void OnParametersSet() { if (string.IsNullOrEmpty(Path)) { @@ -48,73 +50,54 @@ else ErrorMessage = null; ResolvedViewModel = null; - try + if (PathResolver == null) { - if (PathResolver == null) - { - ErrorMessage = "PathResolver service not available"; - return; - } + ErrorMessage = "PathResolver service not available"; + IsLoading = false; + return; + } - var resolution = await PathResolver.ResolvePathAsync(Path); - if (resolution != null) + // Reactive — Subscribe, never await on the resolver chain (deadlock surface; + // see Doc/Architecture/AsynchronousCalls.md). + PathResolver.ResolvePath(Path) + .Catch(ex => { - var (area, id) = ParseRemainder(resolution.Remainder); - ResolvedViewModel = new LayoutAreaControl( - (Messaging.Address)resolution.Prefix, - new Data.LayoutAreaReference(area) { Id = id } - ) - { - ShowProgress = true, - SpinnerType = Layout.SpinnerType.Skeleton, - ProgressMessage = ShortName - }; - DiagInfo = $"Resolved: {resolution.Prefix}" + - (area != null ? $" area={area}" : "") + - (id != null ? $" id={id}" : ""); - } - else + ErrorMessage = $"Error resolving path: {ex.Message}"; + return Observable.Return(null); + }) + .Subscribe(resolution => { - ErrorMessage = $"Path not found: {Path}"; - } - } - catch (Exception ex) - { - ErrorMessage = $"Error resolving path: {ex.Message}"; - } - finally - { - IsLoading = false; - await InvokeAsync(StateHasChanged); - } + if (resolution != null) + { + var (area, id) = ParseRemainder(resolution.Remainder); + ResolvedViewModel = new LayoutAreaControl( + (Messaging.Address)resolution.Prefix, + new Data.LayoutAreaReference(area) { Id = id } + ) + { + ShowProgress = true, + SpinnerType = Layout.SpinnerType.Skeleton, + ProgressMessage = ShortName + }; + DiagInfo = $"Resolved: {resolution.Prefix}" + + (area != null ? $" area={area}" : "") + + (id != null ? $" id={id}" : ""); + } + else if (ErrorMessage == null) + { + ErrorMessage = $"Path not found: {Path}"; + } + IsLoading = false; + InvokeAsync(StateHasChanged); + }); } + // Keyword-aware: "area/Search" → area "Search", "data/Type/id" → area "$Data" id "Type/id", + // "Search" → area "Search". The naive split-on-first-'/' mistook the reserved keyword for the + // area name, so every keyword-form @@-embed rendered a non-existent area. Shared with + // NavigationService so embeds and /node/area/Name navigation never drift. private static (string? Area, string? Id) ParseRemainder(string? remainder) - { - if (string.IsNullOrEmpty(remainder)) - return (null, null); - - // First, check for query string - it becomes the Id - string? queryString = null; - var questionIndex = remainder.IndexOf('?'); - if (questionIndex >= 0) - { - queryString = remainder.Substring(questionIndex); // Keep the ? prefix - remainder = remainder.Substring(0, questionIndex); - } + => MeshWeaver.Markdown.LayoutAreaMarkdownParser.ParseAreaAndId(remainder); +} - // Now handle area/id from the path portion - var slashIndex = remainder.IndexOf('/'); - if (slashIndex >= 0) - { - var area = remainder.Substring(0, slashIndex); - var pathId = remainder.Substring(slashIndex + 1); - // Combine path id with query string if both exist - var id = string.IsNullOrEmpty(pathId) ? queryString : (pathId + (queryString ?? "")); - return (area, id); - } - // No slash - remainder is the area, query string is the id - return (remainder, queryString); - } -} diff --git a/src/MeshWeaver.Blazor/Components/RedirectToLogin.razor b/src/MeshWeaver.Blazor/Components/RedirectToLogin.razor new file mode 100644 index 000000000..5b7cc7c96 --- /dev/null +++ b/src/MeshWeaver.Blazor/Components/RedirectToLogin.razor @@ -0,0 +1,20 @@ +@* Sends a LOGGED-OUT visitor to the unified /login page, preserving where they came from + (returnUrl) so login bounces them straight back to the page they tried to open. + + Rendered from ApplicationPage's AccessDenied branch under : + a logged-OUT user who hits a node they can't see is taken to login (logging in may grant + access) instead of a dead "Access Denied" page; a logged-IN user who simply lacks access + still gets Access Denied — re-authenticating wouldn't help. + + /login and the returnUrl parameter are the AuthenticationOptions defaults (the unified login + page the portal serves). forceLoad does a full navigation so any wedged interactive circuit + on the protected page is dropped. *@ +@inject NavigationManager Navigation +@code { + protected override void OnInitialized() + { + // Navigation.Uri is the absolute URL the visitor tried to open — "where they came from". + var returnUrl = Uri.EscapeDataString(Navigation.Uri); + Navigation.NavigateTo($"/login?returnUrl={returnUrl}", forceLoad: true); + } +} diff --git a/src/MeshWeaver.Blazor/Components/SearchBoxView.razor b/src/MeshWeaver.Blazor/Components/SearchBoxView.razor index 0f133f2cf..f11a1ab68 100644 --- a/src/MeshWeaver.Blazor/Components/SearchBoxView.razor +++ b/src/MeshWeaver.Blazor/Components/SearchBoxView.razor @@ -4,6 +4,8 @@ @using MeshWeaver.Data @using MeshWeaver.Mesh @using MeshWeaver.Mesh.Services +@using MeshWeaver.Reactive +@using System.Reactive.Linq @using System.Text.Json @inject NavigationManager NavigationManager @@ -19,7 +21,7 @@ ShowBorder="true" CodeEditMode="false" CompletionProvider="@SearchCompletionConfig" - AsyncCompletionCallback="@GetCompletionsAsync" /> + CompletionCallback="@GetCompletions" />