Fix Docs MCP deployment reliability

.github/workflows/docs-mcp-health.yml

@@ -0,0 +1,23 @@
+name: Docs MCP Production Health
+
+on:
+  schedule:
+    - cron: "*/15 * * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: docs-mcp-production-health
+  cancel-in-progress: true
+
+jobs:
+  mcp-handshake:
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Initialize MCP and list tools
+        run: node docs-mcp/smoke-test.mjs https://docs-mcp.genlayer.com/sse

.github/workflows/docs-mcp.yml

@@ -1,6 +1,10 @@
-name: Build & Deploy Docs MCP Server
+name: Build & Publish Docs MCP Server
 
 on:
+  pull_request:
+    paths:
+      - "docs-mcp/**"
+      - ".github/workflows/docs-mcp.yml"
   push:
     branches: [main]
     paths:
@@ -13,17 +17,25 @@ permissions:
   contents: read
   packages: write
 
+concurrency:
+  group: docs-mcp-image
+  cancel-in-progress: false
+
 env:
   IMAGE: ghcr.io/${{ github.repository }}/docs-mcp-server
-  DOCS_MCP_URL: https://docs-mcp.genlayer.com
 
 jobs:
   build-and-push:
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v4
 
+      - name: Test MCP handshake canary
+        run: node --test docs-mcp/smoke-test.test.mjs
+
       - name: Log in to GitHub Container Registry
+        if: github.event_name == 'push'
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
@@ -34,27 +46,7 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: ./docs-mcp
-          push: true
+          push: ${{ github.event_name == 'push' }}
           tags: |
             ${{ env.IMAGE }}:latest
             ${{ env.IMAGE }}:sha-${{ github.sha }}
-
-  refresh-index:
-    runs-on: ubuntu-latest
-    needs: build-and-push
-    if: always()
-    steps:
-      - name: Trigger docs reindex
-        run: |
-          # Trigger a refresh of both libraries via the web UI's scrape endpoint.
-          # Uses formMode=refresh to only re-scrape changed pages.
-          # Fails silently if server is not yet deployed.
-          curl -sf -X POST "${{ env.DOCS_MCP_URL }}/web/jobs/scrape" \
-            -H "Content-Type: application/x-www-form-urlencoded" \
-            -d "formMode=refresh&library=genlayer-docs&url=https://docs.genlayer.com&scope=hostname&scrapeMode=fetch&excludePattern=/full-documentation\.txt/" \
-            || echo "Docs refresh trigger failed (server may not be deployed yet)"
-
-          curl -sf -X POST "${{ env.DOCS_MCP_URL }}/web/jobs/scrape" \
-            -H "Content-Type: application/x-www-form-urlencoded" \
-            -d "formMode=refresh&library=genlayer-sdk&url=https://sdk.genlayer.com/main/&scope=hostname&scrapeMode=fetch" \
-            || echo "SDK refresh trigger failed (server may not be deployed yet)"

docs-mcp/OPERATIONS.md

@@ -0,0 +1,59 @@
+# Docs MCP operations
+
+The Docs MCP service has three separately owned parts:
+
+- this repository builds and publishes the server image;
+- `genlayerlabs/devexp-apps-workload` deploys the image to Kubernetes through
+  ArgoCD;
+- `genlayerlabs/skills` exposes the public SSE endpoint from the
+  `genlayer-dev` plugin.
+
+Publishing an image is not a deployment. Production must reference an immutable
+`sha-<commit>` image tag (or digest) in the workload repository. A deployment is
+complete only after the new Kubernetes rollout succeeds and
+`smoke-test.mjs` completes an MCP `initialize` and `tools/list` exchange against
+the public endpoint.
+
+## Health check
+
+Run the protocol-level canary:
+
+```bash
+node docs-mcp/smoke-test.mjs https://docs-mcp.genlayer.com/sse
+```
+
+An HTTP-only `/healthz` check is insufficient. The production nginx sidecar can
+answer that route even when the MCP container is unavailable.
+
+## Incident triage
+
+Check both the pod and the MCP container before restarting anything:
+
+```bash
+kubectl get pods -n studio-prd -l app=docs-mcp-server
+kubectl describe pod -n studio-prd -l app=docs-mcp-server
+kubectl logs -n studio-prd -l app=docs-mcp-server -c docs-mcp-server --tail=200
+kubectl logs -n studio-prd -l app=docs-mcp-server -c docs-mcp-server \
+  --previous --tail=200
+kubectl get events -n studio-prd --sort-by=.lastTimestamp
+```
+
+Inspect PVC capacity and index size when logs mention SQLite, disk, migration,
+or verification errors. `documents.db` is derived data; the entrypoint removes
+an unusable database and rebuilds it from the configured docs sources.
+
+## Required deployment guardrails
+
+The workload repository should:
+
+1. pin an immutable image tag or digest;
+2. use a rolling deployment with at least two serving replicas;
+3. move indexing into a separate Job that writes a new index before rollout;
+4. make readiness and ingress health depend on the MCP process, not nginx;
+5. wait for rollout completion and run the public protocol canary;
+6. roll back automatically when the canary fails;
+7. alert the owning channel when the scheduled canary fails.
+
+Do not restore the removed `POST /web/jobs/scrape` workflow call. That route is
+not exposed by the pinned `docs-mcp-server mcp` runtime. Refresh the index with a
+dedicated indexing Job instead.

docs-mcp/entrypoint.sh

@@ -30,19 +30,36 @@ index_is_usable() {
     docs-mcp-server --quiet find-version genlayer-sdk --store-path "$STORE_PATH" >/dev/null 2>&1
 }
 
+explain_index_failure() {
+  echo "Index diagnostics:"
+  docs-mcp-server find-version genlayer-docs --store-path "$STORE_PATH" || true
+  docs-mcp-server find-version genlayer-sdk --store-path "$STORE_PATH" || true
+}
+
+reset_index() {
+  echo "Removing unusable derived index files before rebuilding..."
+  rm -f \
+    "$STORE_PATH/documents.db" \
+    "$STORE_PATH/documents.db-shm" \
+    "$STORE_PATH/documents.db-wal"
+  index_docs
+}
+
 if [ ! -f "$STORE_PATH/documents.db" ]; then
   echo "No index found at $STORE_PATH/documents.db"
   index_docs
 elif ! index_is_usable; then
-  echo "Existing index at $STORE_PATH/documents.db is not usable; rebuilding..."
-  index_docs
+  echo "Existing index at $STORE_PATH/documents.db is not usable."
+  explain_index_failure
+  reset_index
 else
   echo "Usable index found at $STORE_PATH/documents.db"
 fi
 
 echo "Verifying index..."
 if ! index_is_usable; then
   echo "Index verification failed." >&2
+  explain_index_failure >&2
   exit 1
 fi
 echo "Index verification complete."