diff --git a/.github/workflows/docs-mcp-health.yml b/.github/workflows/docs-mcp-health.yml new file mode 100644 index 00000000..85ed33c9 --- /dev/null +++ b/.github/workflows/docs-mcp-health.yml @@ -0,0 +1,23 @@ +name: Docs MCP Production Health + +on: + schedule: + - cron: "*/15 * * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: docs-mcp-production-health + cancel-in-progress: true + +jobs: + mcp-handshake: + runs-on: ubuntu-latest + timeout-minutes: 3 + steps: + - uses: actions/checkout@v4 + + - name: Initialize MCP and list tools + run: node docs-mcp/smoke-test.mjs https://docs-mcp.genlayer.com/sse diff --git a/.github/workflows/docs-mcp.yml b/.github/workflows/docs-mcp.yml index 3af203f9..da639d30 100644 --- a/.github/workflows/docs-mcp.yml +++ b/.github/workflows/docs-mcp.yml @@ -1,6 +1,10 @@ -name: Build & Deploy Docs MCP Server +name: Build & Publish Docs MCP Server on: + pull_request: + paths: + - "docs-mcp/**" + - ".github/workflows/docs-mcp.yml" push: branches: [main] paths: @@ -13,17 +17,25 @@ permissions: contents: read packages: write +concurrency: + group: docs-mcp-image + cancel-in-progress: false + env: IMAGE: ghcr.io/${{ github.repository }}/docs-mcp-server - DOCS_MCP_URL: https://docs-mcp.genlayer.com jobs: build-and-push: runs-on: ubuntu-latest + timeout-minutes: 20 steps: - uses: actions/checkout@v4 + - name: Test MCP handshake canary + run: node --test docs-mcp/smoke-test.test.mjs + - name: Log in to GitHub Container Registry + if: github.event_name == 'push' uses: docker/login-action@v3 with: registry: ghcr.io @@ -34,27 +46,7 @@ jobs: uses: docker/build-push-action@v6 with: context: ./docs-mcp - push: true + push: ${{ github.event_name == 'push' }} tags: | ${{ env.IMAGE }}:latest ${{ env.IMAGE }}:sha-${{ github.sha }} - - refresh-index: - runs-on: ubuntu-latest - needs: build-and-push - if: always() - steps: - - name: Trigger docs reindex - run: | - # Trigger a refresh of both libraries via the web UI's scrape endpoint. - # Uses formMode=refresh to only re-scrape changed pages. - # Fails silently if server is not yet deployed. - curl -sf -X POST "${{ env.DOCS_MCP_URL }}/web/jobs/scrape" \ - -H "Content-Type: application/x-www-form-urlencoded" \ - -d "formMode=refresh&library=genlayer-docs&url=https://docs.genlayer.com&scope=hostname&scrapeMode=fetch&excludePattern=/full-documentation\.txt/" \ - || echo "Docs refresh trigger failed (server may not be deployed yet)" - - curl -sf -X POST "${{ env.DOCS_MCP_URL }}/web/jobs/scrape" \ - -H "Content-Type: application/x-www-form-urlencoded" \ - -d "formMode=refresh&library=genlayer-sdk&url=https://sdk.genlayer.com/main/&scope=hostname&scrapeMode=fetch" \ - || echo "SDK refresh trigger failed (server may not be deployed yet)" diff --git a/docs-mcp/OPERATIONS.md b/docs-mcp/OPERATIONS.md new file mode 100644 index 00000000..fea2b05e --- /dev/null +++ b/docs-mcp/OPERATIONS.md @@ -0,0 +1,59 @@ +# Docs MCP operations + +The Docs MCP service has three separately owned parts: + +- this repository builds and publishes the server image; +- `genlayerlabs/devexp-apps-workload` deploys the image to Kubernetes through + ArgoCD; +- `genlayerlabs/skills` exposes the public SSE endpoint from the + `genlayer-dev` plugin. + +Publishing an image is not a deployment. Production must reference an immutable +`sha-` image tag (or digest) in the workload repository. A deployment is +complete only after the new Kubernetes rollout succeeds and +`smoke-test.mjs` completes an MCP `initialize` and `tools/list` exchange against +the public endpoint. + +## Health check + +Run the protocol-level canary: + +```bash +node docs-mcp/smoke-test.mjs https://docs-mcp.genlayer.com/sse +``` + +An HTTP-only `/healthz` check is insufficient. The production nginx sidecar can +answer that route even when the MCP container is unavailable. + +## Incident triage + +Check both the pod and the MCP container before restarting anything: + +```bash +kubectl get pods -n studio-prd -l app=docs-mcp-server +kubectl describe pod -n studio-prd -l app=docs-mcp-server +kubectl logs -n studio-prd -l app=docs-mcp-server -c docs-mcp-server --tail=200 +kubectl logs -n studio-prd -l app=docs-mcp-server -c docs-mcp-server \ + --previous --tail=200 +kubectl get events -n studio-prd --sort-by=.lastTimestamp +``` + +Inspect PVC capacity and index size when logs mention SQLite, disk, migration, +or verification errors. `documents.db` is derived data; the entrypoint removes +an unusable database and rebuilds it from the configured docs sources. + +## Required deployment guardrails + +The workload repository should: + +1. pin an immutable image tag or digest; +2. use a rolling deployment with at least two serving replicas; +3. move indexing into a separate Job that writes a new index before rollout; +4. make readiness and ingress health depend on the MCP process, not nginx; +5. wait for rollout completion and run the public protocol canary; +6. roll back automatically when the canary fails; +7. alert the owning channel when the scheduled canary fails. + +Do not restore the removed `POST /web/jobs/scrape` workflow call. That route is +not exposed by the pinned `docs-mcp-server mcp` runtime. Refresh the index with a +dedicated indexing Job instead. diff --git a/docs-mcp/entrypoint.sh b/docs-mcp/entrypoint.sh index cdd561d7..5832b3d4 100644 --- a/docs-mcp/entrypoint.sh +++ b/docs-mcp/entrypoint.sh @@ -30,12 +30,28 @@ index_is_usable() { docs-mcp-server --quiet find-version genlayer-sdk --store-path "$STORE_PATH" >/dev/null 2>&1 } +explain_index_failure() { + echo "Index diagnostics:" + docs-mcp-server find-version genlayer-docs --store-path "$STORE_PATH" || true + docs-mcp-server find-version genlayer-sdk --store-path "$STORE_PATH" || true +} + +reset_index() { + echo "Removing unusable derived index files before rebuilding..." + rm -f \ + "$STORE_PATH/documents.db" \ + "$STORE_PATH/documents.db-shm" \ + "$STORE_PATH/documents.db-wal" + index_docs +} + if [ ! -f "$STORE_PATH/documents.db" ]; then echo "No index found at $STORE_PATH/documents.db" index_docs elif ! index_is_usable; then - echo "Existing index at $STORE_PATH/documents.db is not usable; rebuilding..." - index_docs + echo "Existing index at $STORE_PATH/documents.db is not usable." + explain_index_failure + reset_index else echo "Usable index found at $STORE_PATH/documents.db" fi @@ -43,6 +59,7 @@ fi echo "Verifying index..." if ! index_is_usable; then echo "Index verification failed." >&2 + explain_index_failure >&2 exit 1 fi echo "Index verification complete." diff --git a/docs-mcp/smoke-test.mjs b/docs-mcp/smoke-test.mjs new file mode 100644 index 00000000..c7bf8be5 --- /dev/null +++ b/docs-mcp/smoke-test.mjs @@ -0,0 +1,252 @@ +#!/usr/bin/env node + +import { pathToFileURL } from "node:url"; + +const DEFAULT_TIMEOUT_MS = 15_000; +const DEFAULT_EXPECTED_TOOL = "search_docs"; + +function parsePositiveInteger(value, fallback) { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +async function* parseSse(stream) { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + + try { + while (true) { + const { done, value } = await reader.read(); + buffer += decoder.decode(value, { stream: !done }); + + let separator = buffer.match(/\r?\n\r?\n/); + while (separator?.index !== undefined) { + const block = buffer.slice(0, separator.index); + buffer = buffer.slice(separator.index + separator[0].length); + + let event = "message"; + const data = []; + + for (const line of block.split(/\r?\n/)) { + if (line.startsWith("event:")) { + event = line.slice("event:".length).trim(); + } else if (line.startsWith("data:")) { + data.push(line.slice("data:".length).trimStart()); + } + } + + if (data.length > 0) { + yield { event, data: data.join("\n") }; + } + + separator = buffer.match(/\r?\n\r?\n/); + } + + if (done) { + return; + } + } + } finally { + await reader.cancel().catch(() => {}); + reader.releaseLock(); + } +} + +async function waitForEvent(events, predicate, description) { + while (true) { + const { done, value } = await events.next(); + if (done) { + throw new Error(`SSE stream ended before ${description}`); + } + if (predicate(value)) { + return value; + } + } +} + +async function waitForJsonRpcResponse(events, id) { + const event = await waitForEvent( + events, + (candidate) => { + if (candidate.event !== "message") { + return false; + } + try { + return JSON.parse(candidate.data).id === id; + } catch { + return false; + } + }, + `JSON-RPC response ${id}`, + ); + + const payload = JSON.parse(event.data); + if (payload.error) { + throw new Error( + `JSON-RPC response ${id} failed: ${JSON.stringify(payload.error)}`, + ); + } + return payload; +} + +async function responseError(response) { + const body = (await response.text()).trim(); + return `HTTP ${response.status}${body ? `: ${body.slice(0, 500)}` : ""}`; +} + +async function postJson(url, payload, signal) { + const response = await fetch(url, { + method: "POST", + headers: { + Accept: "application/json, text/event-stream", + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), + signal, + }); + + if (!response.ok) { + throw new Error(`MCP message endpoint returned ${await responseError(response)}`); + } +} + +export async function runSmokeTest({ + endpoint, + expectedTool = DEFAULT_EXPECTED_TOOL, + timeoutMs = DEFAULT_TIMEOUT_MS, +} = {}) { + if (!endpoint) { + throw new Error("An MCP SSE endpoint is required"); + } + + const sseUrl = new URL(endpoint); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + let events; + + try { + const response = await fetch(sseUrl, { + headers: { Accept: "text/event-stream" }, + signal: controller.signal, + }); + + if (!response.ok) { + throw new Error(`MCP SSE endpoint returned ${await responseError(response)}`); + } + if (!response.body) { + throw new Error("MCP SSE endpoint returned no response body"); + } + + events = parseSse(response.body); + const endpointEvent = await waitForEvent( + events, + (candidate) => candidate.event === "endpoint", + "the MCP message endpoint", + ); + const messagesUrl = new URL(endpointEvent.data, sseUrl); + + const initializeResponse = waitForJsonRpcResponse(events, 1); + await postJson( + messagesUrl, + { + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2024-11-05", + capabilities: {}, + clientInfo: { + name: "genlayer-docs-mcp-smoke-test", + version: "1.0.0", + }, + }, + }, + controller.signal, + ); + const initialize = await initializeResponse; + + await postJson( + messagesUrl, + { + jsonrpc: "2.0", + method: "notifications/initialized", + params: {}, + }, + controller.signal, + ); + + const toolsResponse = waitForJsonRpcResponse(events, 2); + await postJson( + messagesUrl, + { + jsonrpc: "2.0", + id: 2, + method: "tools/list", + params: {}, + }, + controller.signal, + ); + const tools = (await toolsResponse).result?.tools; + + if (!Array.isArray(tools)) { + throw new Error("MCP tools/list response did not contain a tools array"); + } + if (expectedTool && !tools.some((tool) => tool.name === expectedTool)) { + throw new Error( + `MCP tool ${JSON.stringify(expectedTool)} was not advertised; received: ${tools + .map((tool) => tool.name) + .join(", ")}`, + ); + } + + return { + serverInfo: initialize.result?.serverInfo ?? null, + toolNames: tools.map((tool) => tool.name), + }; + } catch (error) { + if (controller.signal.aborted) { + throw new Error(`MCP smoke test timed out after ${timeoutMs}ms`, { + cause: error, + }); + } + throw error; + } finally { + clearTimeout(timeout); + controller.abort(); + if (events) { + await events.return().catch(() => {}); + } + } +} + +async function main() { + const endpoint = + process.argv[2] ?? + process.env.DOCS_MCP_ENDPOINT ?? + "https://docs-mcp.genlayer.com/sse"; + const timeoutMs = parsePositiveInteger( + process.env.DOCS_MCP_TIMEOUT_MS, + DEFAULT_TIMEOUT_MS, + ); + const expectedTool = + process.env.DOCS_MCP_EXPECTED_TOOL ?? DEFAULT_EXPECTED_TOOL; + + const result = await runSmokeTest({ endpoint, expectedTool, timeoutMs }); + const server = result.serverInfo + ? `${result.serverInfo.name ?? "unknown"}@${result.serverInfo.version ?? "unknown"}` + : "unknown"; + console.log( + `MCP smoke test passed: ${server}; tools: ${result.toolNames.join(", ")}`, + ); +} + +const isMain = + process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; + +if (isMain) { + main().catch((error) => { + console.error(`MCP smoke test failed: ${error.message}`); + process.exitCode = 1; + }); +} diff --git a/docs-mcp/smoke-test.test.mjs b/docs-mcp/smoke-test.test.mjs new file mode 100644 index 00000000..e70cf800 --- /dev/null +++ b/docs-mcp/smoke-test.test.mjs @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { afterEach, test } from "node:test"; + +import { runSmokeTest } from "./smoke-test.mjs"; + +const servers = []; + +afterEach(async () => { + await Promise.all( + servers.splice(0).map( + (server) => + new Promise((resolve) => { + server.close(resolve); + server.closeAllConnections(); + }), + ), + ); +}); + +async function listen(handler) { + const server = createServer(handler); + servers.push(server); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + return `http://127.0.0.1:${address.port}`; +} + +async function readJson(request) { + const chunks = []; + for await (const chunk of request) { + chunks.push(chunk); + } + return JSON.parse(Buffer.concat(chunks).toString("utf8")); +} + +test("completes an MCP initialize and tools/list handshake", async () => { + let sseResponse; + + const baseUrl = await listen(async (request, response) => { + const url = new URL(request.url, "http://localhost"); + + if (request.method === "GET" && url.pathname === "/sse") { + sseResponse = response; + response.writeHead(200, { + "Cache-Control": "no-cache", + "Content-Type": "text/event-stream", + }); + response.write("event: endpoint\ndata: /messages?sessionId=test\n\n"); + return; + } + + if (request.method === "POST" && url.pathname === "/messages") { + const payload = await readJson(request); + response.writeHead(202).end(); + + if (payload.method === "initialize") { + sseResponse.write( + `event: message\ndata: ${JSON.stringify({ + jsonrpc: "2.0", + id: payload.id, + result: { + protocolVersion: "2024-11-05", + capabilities: { tools: {} }, + serverInfo: { name: "fixture", version: "1.0.0" }, + }, + })}\n\n`, + ); + } else if (payload.method === "tools/list") { + sseResponse.write( + `event: message\ndata: ${JSON.stringify({ + jsonrpc: "2.0", + id: payload.id, + result: { tools: [{ name: "search_docs" }] }, + })}\n\n`, + ); + } + return; + } + + response.writeHead(404).end(); + }); + + const result = await runSmokeTest({ + endpoint: `${baseUrl}/sse`, + timeoutMs: 2_000, + }); + + assert.deepEqual(result.serverInfo, { name: "fixture", version: "1.0.0" }); + assert.deepEqual(result.toolNames, ["search_docs"]); +}); + +test("reports an upstream SSE failure", async () => { + const baseUrl = await listen((_request, response) => { + response.writeHead(502, { "Content-Type": "text/plain" }); + response.end("bad gateway"); + }); + + await assert.rejects( + runSmokeTest({ + endpoint: `${baseUrl}/sse`, + timeoutMs: 2_000, + }), + /MCP SSE endpoint returned HTTP 502: bad gateway/, + ); +});