From 69b7d9da3810ce1f7673e164dc0bf4fcefce619f Mon Sep 17 00:00:00 2001 From: lucarlig Date: Fri, 21 Aug 2026 09:40:02 +0100 Subject: [PATCH 1/3] docs: document stateless dual-protocol support Signed-off-by: lucarlig --- _context/wiki/architecture.md | 2 +- _context/wiki/index.md | 2 +- _context/wiki/mcp-capability-allocation.md | 161 +++++++++++++-------- _context/wiki/preferences.md | 9 +- _context/wiki/project.md | 12 +- _context/wiki/testing.md | 2 +- 6 files changed, 114 insertions(+), 74 deletions(-) diff --git a/_context/wiki/architecture.md b/_context/wiki/architecture.md index 2f1570ed..18a40873 100644 --- a/_context/wiki/architecture.md +++ b/_context/wiki/architecture.md @@ -234,7 +234,7 @@ Changing a load-bearing choice requires updating more than one file: | Change | Required follow-through | | --- | --- | -| Downstream MCP version | Coordinate with control plane; update protocol tests and examples; keep legacy traffic on control-plane routes. | +| Downstream MCP version | Coordinate with the control plane and update the `2026-07-28`/`2025-11-25` compatibility matrix, protocol tests, examples, and front-door routing. During migration the current legacy route split may remain, but the target sends both supported Streamable HTTP versions to the stateless Rust dataplane. | | Backend namespace / prefix contract | Update merge logic, split logic, tests, docs, and control-plane integration if client-facing surface moves. | | Session state moves external | Update `SessionManager`, cleanup behavior, load-balancing docs, and failure-mode tests. | | Config transport changes | Keep `UserConfigStore` as the boundary; update adapter tests. | diff --git a/_context/wiki/index.md b/_context/wiki/index.md index 62221e46..03f9c974 100644 --- a/_context/wiki/index.md +++ b/_context/wiki/index.md @@ -25,7 +25,7 @@ then follow only the links that are relevant. - **Repo**: `contextforge-data-plane` — the Rust dataplane for ContextForge. - **Core invariant**: this crate is pure routing logic. No IAM, UI, or metrics storage. -- **Protocol target**: MCP `2026-07-28` over Streamable HTTP. Legacy SSE paths are being removed. +- **Protocol target**: MCP `2026-07-28` and `2025-11-25` over Streamable HTTP. Both use stateless request handling; `initialize` remains supported but does not establish dataplane session state. Legacy SSE paths are being removed. - **Status convention**: project, architecture, routing, and operations pages describe the current implementation. The page under **Upcoming** describes the tentative ContextForge 2.0 target and migration roadmap. diff --git a/_context/wiki/mcp-capability-allocation.md b/_context/wiki/mcp-capability-allocation.md index 2cfc0610..7fa7cdd2 100644 --- a/_context/wiki/mcp-capability-allocation.md +++ b/_context/wiki/mcp-capability-allocation.md @@ -11,16 +11,19 @@ move control-plane responsibilities into this repository. ## Vision and Constraints -- ContextForge as a product supports modern MCP `2026-07-28` and legacy MCP - `2025-11-25` and older. Streamable HTTP is the preferred transport. -- The fast Rust dataplane accepts only modern MCP `2026-07-28` downstream - traffic. Legacy downstream clients stay on the Python slow path. -- The control plane and both dataplanes may connect to modern or legacy - upstream MCP servers using the protocol and transport appropriate to each - server. -- Legacy upstream session handling is best effort. The target request path - does not depend on a durable MCP session between ContextForge and an upstream - server. +- ContextForge supports MCP `2026-07-28` and `2025-11-25` over Streamable HTTP + on both the client-facing and backend-facing sides of the fast Rust + dataplane. +- Same-version client/backend paths are supported directly. Cross-version + `2026-07-28` → `2025-11-25` and `2025-11-25` → `2026-07-28` adaptation is + best effort. +- All request/response handling is stateless for both versions. The target + request path does not depend on an MCP session, session affinity, or a + retained backend transport. +- `initialize` remains supported for compatibility, but it is a stateless + request: the dataplane generates its response from effective configuration + and does not use it to establish state required by later requests. +- Legacy SSE transport is not part of the dataplane target. - Fan-out and other one-to-many MCP work is limited to the control plane. The slow and fast dataplanes generate discovery, capability, and list responses from control-plane-authored effective configuration. @@ -29,72 +32,98 @@ move control-plane responsibilities into this repository. reads but is never the source of truth. - MCP subscriptions and notifications remain Phase 4 work. +## Stateless Protocol Compatibility + +[IBM/mcp-context-forge issue #6327](https://github.com/IBM/mcp-context-forge/issues/6327) +tracks the first targeted-operation slice for `tools/call`. The issue calls the +incoming/client-facing side “upstream” and the selected backend-facing side +“downstream”; this wiki uses the explicit names below. + +| Incoming client | Selected backend | Target behavior | +| --- | --- | --- | +| `2026-07-28` | `2026-07-28` | Supported directly as one stateless request. | +| `2026-07-28` | `2025-11-25` | Best-effort protocol adaptation within one stateless request. | +| `2025-11-25` | `2026-07-28` | Best-effort protocol adaptation within one stateless request. | +| `2025-11-25` | `2025-11-25` | Supported directly as one stateless request. | + +For every row, the dataplane authenticates and authorizes the request, reads +the principal-bound effective configuration, validates that the requested +object is visible and permitted, resolves exactly one backend, adapts the +protocol when necessary, and closes the request-scoped backend connection after +the response. A client may call `initialize`, but later operations neither +require nor reuse state created by it. + +“Best effort” never permits hidden session state. If a semantic difference or +backend requirement cannot be handled within the current request, the +dataplane returns an explicit error instead of creating affinity or retaining a +backend transport for a later request. + +For the initial `tools/call` slice, issue #6327 assumes that the selected +backend needs neither application authentication nor mTLS and that its server +certificate chains to the system CA. Those are issue-scope assumptions, not a +change to the dataplane's broader transport-security model. + ## Target End State -The front door uses route and protocol metadata to split management, legacy -MCP, and modern MCP traffic. PostgreSQL remains the durable management store; -the shared runtime store carries compiled configuration to both dataplanes. +The front door separates management traffic from supported Streamable HTTP MCP +traffic. It does not split the two supported MCP versions in the target end +state: both go to the fast Rust dataplane. PostgreSQL remains the durable +management store; the shared runtime store carries compiled configuration to +the dataplane. ```mermaid flowchart TB subgraph Clients[Traffic] direction LR AdminClient([Admin or User]) - LegacyClient([Legacy MCP Client]) - ModernClient([Modern MCP Client]) + CompatClient([MCP 2025-11-25 Client]) + ModernClient([MCP 2026-07-28 Client]) end - FrontDoor[Load Balancer and Header-Based Router] + FrontDoor[Load Balancer and Router] subgraph ContextForge[ContextForge 2.0] direction LR - subgraph PythonPlane[Python Control and Slow Plane] - direction TB - Admin[Admin and Management API] - Slow[Slow Path MCP Dataplane] - end + Admin[Python Admin and Management API] Fast[Fast Rust MCP Dataplane] end Postgres[(PostgreSQL Management State)] RuntimeStore[(Shared Effective Configuration)] - Upstreams[Modern and Legacy MCP Servers] + Upstreams[MCP 2026-07-28 and 2025-11-25 Servers] AdminClient -->|Management API| FrontDoor - LegacyClient -->|Legacy MCP| FrontDoor - ModernClient -->|Modern MCP 2026-07-28| FrontDoor + CompatClient -->|Streamable HTTP MCP 2025-11-25| FrontDoor + ModernClient -->|Streamable HTTP MCP 2026-07-28| FrontDoor FrontDoor -->|Management routes| Admin - FrontDoor -->|Legacy MCP routes| Slow - FrontDoor -->|Modern MCP routes| Fast + FrontDoor -->|Both supported MCP versions| Fast Admin -->|Persist administrative state| Postgres Admin -->|Publish effective configuration| RuntimeStore - RuntimeStore -->|Read shared configuration| Slow RuntimeStore -->|Read-only configuration| Fast Admin -->|Discover catalogs and poll liveness| Upstreams - Slow -->|Targeted MCP calls| Upstreams Fast -->|Targeted MCP calls| Upstreams ``` -The preferred end state is for both dataplanes to consume the same compiled -configuration. Redis is the current fast-path store and the preferred shared -implementation. During the Python migration, the slow path may instead read -shared Redis or PostgreSQL state. It must not rely on process memory alone when -multiple slow-path instances are deployed. +Redis is the current fast-path store and the preferred shared implementation. +During migration, the Python slow path may consume the same compiled +configuration from Redis or PostgreSQL, but it is not the permanent home for +either supported Streamable HTTP version. It must not rely on process memory +alone when multiple slow-path instances are deployed. ## Component Responsibilities | Component | Target responsibility | | --- | --- | -| Front door | Route management APIs to the control plane, legacy MCP to the slow path, and modern `2026-07-28` Streamable HTTP MCP to the fast path. | +| Front door | Route management APIs to the control plane and both supported Streamable HTTP MCP versions to the fast Rust dataplane. A temporary slow-path split may remain during migration. | | Admin and management API | Manage the virtual-server lifecycle and upstream assignments; connect to heterogeneous upstreams; retrieve and page through capabilities, tools, resources, prompts, completions, and other catalogs; normalize and persist them; let administrators select exposed objects and rules; compile effective runtime configuration; poll upstream liveness and changes. | | PostgreSQL | Persist administrative source data such as virtual servers, upstream definitions, normalized catalogs, selections, and policies. It is not on the fast request path. | | Configuration synchronization | Publish effective configuration one way from the control plane to externally shared state. Both dataplanes should consume the same shape where practical. | -| Slow path MCP dataplane | Handle modern and legacy downstream protocols during the Python transition; remain the legacy path after modern traffic moves to Rust. Read effective configuration from shared state and generate aggregate responses without live upstream fan-out. | -| Fast Rust MCP dataplane | Handle modern downstream MCP efficiently. Read effective configuration, serve aggregate responses locally, and route a targeted method to exactly one selected modern or legacy upstream. It does not own IAM, UI, management APIs, or durable metrics storage. | -| Upstream MCP servers | May use modern or legacy MCP. Legacy upstream sessions are best effort; the architecture does not require durable upstream session affinity. | +| Slow path MCP dataplane | Temporary migration path only. Read effective configuration from shared state and generate aggregate responses without live backend fan-out while methods move to Rust. | +| Fast Rust MCP dataplane | Handle `2026-07-28` and `2025-11-25` Streamable HTTP requests statelessly. Read effective configuration, serve aggregate and `initialize` responses locally, and route a targeted method to exactly one selected backend. Cross-version adaptation is best effort. It does not own IAM, UI, management APIs, or durable metrics storage. | +| Backend MCP servers | May use `2026-07-28` or `2025-11-25`, independently of the incoming client version. Connections and any required negotiation are request-scoped and leave no reusable session; the architecture does not require backend session affinity. | ## Administrative State and Effective Configuration @@ -144,9 +173,9 @@ authorization context. | --- | --- | | Virtual-server creation and upstream assignment | Control plane persists management state and connects to assigned upstreams. | | Upstream discovery, initialization where required, catalog pagination, capability aggregation, filtering, and liveness polling | Control plane only; this is the intentional fan-out boundary. | -| Modern downstream `server/discover` and effective capabilities | After per-request authorization, the fast dataplane generates the response from the principal-bound effective configuration. Legacy initialization remains on the slow path. | +| `server/discover`, `initialize`, and effective capabilities | After per-request authorization, the fast dataplane generates the response from principal-bound effective configuration. `initialize` is supported for compatibility but creates no state that later requests depend on. | | `tools/list`, `resources/list`, `prompts/list`, resource-template listing, and similar aggregate methods | After method-scope and compiled-RBAC enforcement, the slow or fast dataplane generates the visible response from principal-bound effective configuration with no live upstream fan-out. | -| `tools/call`, `resources/read`, `prompts/get`, completion, and similar targeted methods | Dataplane resolves the effective entry under the trusted authorization key, applies default-deny scope and object policy, and calls exactly one selected upstream only when authorized. | +| `tools/call`, `resources/read`, `prompts/get`, completion, and similar targeted methods | Dataplane resolves the effective entry under the trusted authorization key, applies default-deny scope and object policy, adapts between the incoming and backend protocol versions when necessary, and calls exactly one selected backend only when authorized. The backend connection is request-scoped and leaves no reusable session. | | Plugins for trusted aggregate responses | Prefer policy compiled by the control plane; avoid mandatory per-request plugin calls for a response already produced from trusted effective configuration. | | Plugins for targeted calls | May run on the fast path when request or response inspection is required. Exact hook allocation remains an implementation decision. | | Subscriptions, server notifications, and downstream list-change notifications | Deferred to Phase 4 because their state and delivery model do not fit the request/response simplification. | @@ -156,14 +185,16 @@ authorization context. | Phase | Scope | | --- | --- | | **1. Separate the Python control and slow planes** | Establish a clear boundary inside the current Python component. The control plane writes effective configuration per user, team, or other principal to shared state; the slow dataplane reads it and responds accordingly. Slow-path paging may be deferred or skipped if the fast-path migration progresses quickly. Plugins that belong on the fast path need not be duplicated in the slow path. | -| **2. Offload targeted calls to the fast dataplane** | Make the slow and fast dataplanes follow the same configuration-driven pattern. Initially send only targeted operations such as `tools/call`, `resources/read`, `prompts/get`, and completion to the fast dataplane. | -| **3. Offload all request/response MCP methods to the fast dataplane** | Serve modern discovery, capabilities, aggregate lists, and targeted calls correctly from the fast dataplane. Aggregate responses come from effective configuration; targeted calls reach exactly one upstream. | +| **2. Offload targeted calls to the fast dataplane** | Make the slow and fast dataplanes follow the same configuration-driven pattern. Initially send targeted operations such as `tools/call`, `resources/read`, `prompts/get`, and completion to the fast dataplane. For each operation, support both same-version `2026-07-28`/`2025-11-25` paths and attempt both cross-version paths on a best-effort basis, always without reusable session state. The `tools/call` slice is tracked by [#6327](https://github.com/IBM/mcp-context-forge/issues/6327). | +| **3. Offload all request/response MCP methods to the fast dataplane** | Serve discovery, stateless `initialize`, capabilities, aggregate lists, and targeted calls for both supported protocol versions from the fast dataplane. Aggregate responses come from effective configuration; targeted calls reach exactly one backend. | | **4. Implement subscriptions and notifications** | Add the state, routing, and delivery model for upstream subscriptions, resource notifications, and list-change notifications after the request/response architecture is complete. | ## Phase 3 Reference Flows The examples below use tools, but the same ownership applies to resources, prompts, completions, and other aggregate or targeted request/response methods. +“Supported MCP client” and “supported MCP server” mean either `2026-07-28` or +`2025-11-25`; when the two sides differ, adaptation is best effort. ### 1. Create a Virtual Server and Select Capabilities @@ -174,8 +205,8 @@ sequenceDiagram participant UI as Admin UI or API participant CP as Control Plane participant DB as Control Plane DB - participant MCP1 as Modern MCP Server - participant MCP2 as Legacy MCP Server + participant MCP1 as MCP 2026-07-28 Server + participant MCP2 as MCP 2025-11-25 Server participant Store as Shared Config Store (Redis) participant DP as Fast Rust Dataplane @@ -187,10 +218,10 @@ sequenceDiagram UI->>CP: Update backend associations CP->>DB: Store backend associations - par Inspect modern upstream + par Inspect 2026-07-28 backend CP->>MCP1: Discover capabilities and retrieve catalogs MCP1-->>CP: Capabilities and catalog - and Inspect legacy upstream + and Inspect 2025-11-25 backend CP->>MCP2: Initialize or discover and retrieve catalogs MCP2-->>CP: Capabilities and catalog end @@ -217,19 +248,19 @@ sequenceDiagram Note over CP,DP: Snapshot carries compiled scopes, RBAC and visible objects ``` -### 2. Discover the Server and List Tools +### 2. Initialize or Discover the Server and List Tools ```mermaid sequenceDiagram autonumber - participant Client as Modern MCP Client + participant Client as Supported MCP Client participant Ingress participant DP as Fast Rust Dataplane participant Cache as Local Cache participant Store as Shared Config Store (Redis) - Client->>Ingress: server/discover - Ingress->>DP: Forward modern MCP request + Client->>Ingress: initialize or server/discover + Ingress->>DP: Forward supported MCP request DP->>DP: Verify JWT, metadata and server route DP->>DP: Derive authorization key from trusted context DP->>Cache: Get snapshot by authorization key @@ -247,13 +278,13 @@ sequenceDiagram DP->>DP: Enforce discovery scope and compiled RBAC alt Snapshot mapped and authorized - DP-->>Client: Server identity and visible capabilities + DP-->>Client: Version-appropriate identity and visible capabilities else Missing, unmapped or denied DP-->>Client: Authorization error without catalog details end - Client->>Ingress: tools/list - Ingress->>DP: Forward modern MCP request + Client->>Ingress: tools/list as independent request + Ingress->>DP: Forward supported MCP request DP->>DP: Reverify and derive authorization key DP->>DP: Enforce tools/list scope and compiled RBAC alt Snapshot mapped and authorized @@ -266,6 +297,7 @@ sequenceDiagram Note over DP,Store: The shared store distributes compiled state Note over DP: No live upstream call for discovery or aggregate lists + Note over Client,DP: initialize does not create required session state Note over Client,DP: Client-supplied identity or routing metadata is untrusted ``` @@ -274,35 +306,42 @@ sequenceDiagram ```mermaid sequenceDiagram autonumber - participant Client as Modern MCP Client + participant Client as Supported MCP Client participant Ingress participant DP as Fast Rust Dataplane participant Cache as Local Cache participant CPEX as Policy and CPEX - participant MCP as Selected Modern or Legacy MCP Server + participant MCP as Selected Supported MCP Server Client->>Ingress: tools/call name inc - Ingress->>DP: Forward modern MCP request + Ingress->>DP: Forward supported MCP request DP->>DP: Verify JWT, metadata and server route DP->>DP: Derive authorization key from trusted context DP->>Cache: Resolve inc under authorization key - Cache-->>DP: Backend mapping and policy or missing + Cache-->>DP: Backend mapping, protocol version and policy or missing DP->>DP: Enforce tools/call scope and compiled RBAC alt Tool mapped and authorized DP->>CPEX: Run pre-call policy CPEX-->>DP: Allow or modify request + DP->>DP: Adapt client protocol to backend protocol + opt Backend negotiation is required + DP->>MCP: Request-scoped initialize + MCP-->>DP: Initialize result + end DP->>MCP: tools/call name inc MCP-->>DP: Tool result + DP->>MCP: Close request-scoped connection DP->>CPEX: Run post-call policy CPEX-->>DP: Allow or modify result - DP-->>Client: Return tool result directly + DP-->>Client: Return version-appropriate tool result else Missing, unmapped or denied DP-->>Client: Authorization error with no upstream call end - Note over DP,MCP: Exactly one upstream is called - Note over DP,MCP: No durable upstream MCP session is required + Note over DP,MCP: Exactly one backend is called + Note over DP,MCP: Client and backend versions are independently 2026-07-28 or 2025-11-25 + Note over DP,MCP: No durable backend MCP session is required Note over DP: Control Plane, DB and Redis are not on this result path Note over Client,DP: Client-supplied identity or backend selection is untrusted ``` @@ -312,12 +351,12 @@ sequenceDiagram ```mermaid sequenceDiagram autonumber - participant MCP as Modern or Legacy MCP Server + participant MCP as Supported MCP Server participant CP as Control Plane Reconciler participant DB as Control Plane DB participant Store as Shared Config Store (Redis) participant DP as Fast Rust Dataplane - participant Client as Modern MCP Client + participant Client as Supported MCP Client CP->>MCP: Poll liveness and refresh discovery and lists MCP-->>CP: Updated catalog diff --git a/_context/wiki/preferences.md b/_context/wiki/preferences.md index 55fc3b75..55c374e7 100644 --- a/_context/wiki/preferences.md +++ b/_context/wiki/preferences.md @@ -50,14 +50,15 @@ CI additionally runs `cargo shear --check-test-targets --deny-warnings --locked` - The dataplane is pure routing logic. **No IAM, UI, or metrics-storage concerns.** - Config access goes through `UserConfigStore` only — never push Redis details into routing code. - The backend prefix naming contract must not change without updating merge logic, split logic, and tests. -- Legacy SSE transport and old `initialize`/session behavior are being **removed** — do not build on temporary shims. +- Legacy SSE transport and stateful session behavior are being **removed**. `initialize` remains supported as a stateless compatibility method; do not use it to create affinity, persist client state, or retain backend transports between requests. - Prefer the right architecture over backward compatibility; this project has no external users yet. ## Protocol target -- All new behavior targets MCP protocol version **`2026-07-28`** over **Streamable HTTP**. -- New tests and examples use `server/discover`, per-request client metadata, and the `2026-07-28` version. -- Do not add new compatibility for older MCP protocol versions. +- The dataplane target supports MCP **`2026-07-28`** and **`2025-11-25`** over **Streamable HTTP**. +- Every request is independent for both versions. Do not require `Mcp-Session-Id`, session affinity, or a previously retained backend transport. +- Retain `initialize` for clients that use it, but generate its response from effective configuration and do not treat it as session establishment. `2026-07-28` tests and examples should continue to exercise `server/discover` and per-request client metadata. +- Protocol-sensitive tests cover both same-version paths and the best-effort cross-version paths (`2026-07-28` → `2025-11-25` and the reverse). Do not add SSE or versions earlier than `2025-11-25` without a separate architecture decision. ## AI interaction preferences diff --git a/_context/wiki/project.md b/_context/wiki/project.md index 4128f61d..d234dc8a 100644 --- a/_context/wiki/project.md +++ b/_context/wiki/project.md @@ -54,7 +54,7 @@ flowchart LR ## Goals and objectives - Provide a **production-grade, low-latency routing layer** between MCP clients and backend MCP servers. -- Target **MCP protocol version `2026-07-28`** over Streamable HTTP as the sole downstream contract. +- Support MCP `2026-07-28` and `2025-11-25` over Streamable HTTP as stateless downstream contracts. - Enforce a clean **dataplane/control-plane boundary** — no IAM, UI, or metrics storage logic in this repo. - Keep config access behind the **`UserConfigStore` abstraction** (backed by Redis/MessagePack). - Remain in the right architectural shape during early development, prioritising correctness over backward compatibility. @@ -63,7 +63,7 @@ flowchart LR - **Platform teams** — deploy and operate the gateway as infrastructure. - **AI application developers** — use the gateway as the MCP proxy layer for their applications. -- **Internal contributors** — engineers evolving the dataplane toward the `2026-07-28` protocol target. +- **Internal contributors** — engineers evolving the dataplane toward stateless `2026-07-28` and `2025-11-25` protocol support. ## Key modules and architecture @@ -94,9 +94,9 @@ Architecture context lives in the wiki. Key pages: ## Active work (near-term) -- **Protocol migration**: replacing all remaining legacy MCP paths (SSE transport, `initialize`/session shims) with `2026-07-28` equivalents over Streamable HTTP. -- Legacy SSE transport and old session behavior are **being removed**, not maintained. Do not build new behavior on temporary shims. -- New tests and examples should use `server/discover`, per-request client metadata, and protocol version `2026-07-28`. +- **Protocol migration**: support same-version `2026-07-28` and `2025-11-25` paths over Streamable HTTP, provide best-effort translation in either cross-version direction, and replace stateful session paths with request-scoped handling. +- Legacy SSE transport and session affinity are **being removed**. `initialize` is retained as a stateless compatibility request and must not create persistent dataplane or backend session state. +- Protocol-sensitive tests must cover the two direct and two best-effort cross-version combinations. Modern examples should continue to use `server/discover` and per-request client metadata; compatibility examples may use `initialize` without relying on later session reuse. ## Control-Plane Integration Contract @@ -105,7 +105,7 @@ Architecture context lives in the wiki. Key pages: | Agreement | Value today | | --- | --- | | Client-facing route | `/servers/{virtual_host_id}/mcp`. Front door rewrites modern MCP `2026-07-28` Streamable HTTP traffic to `/contextforge-rs/servers/{virtual_host_id}/mcp` on the dataplane. | -| Protocol compatibility | Dataplane target is MCP `2026-07-28` only. Control plane serves older versions, legacy session init, and SSE on its own routes. | +| Protocol compatibility | Today the dataplane route accepts MCP `2026-07-28`; the control plane serves `2025-11-25`, session-based initialization, and SSE on its own routes. The target moves both supported Streamable HTTP versions to stateless dataplane handling, with cross-version adaptation on a best-effort basis. | | Unknown virtual host | `404` with body `{"detail":"Server not found"}`, matching the control-plane response shape. | | Token issuer and audience | `iss = mcpgateway`, `aud = mcpgateway-api`. | | Claims shape | `sub`, `jti`, `iss`, `aud`, `exp`, and `user` required. `token_use`, `iat`, `teams`, `scopes`, and `user.full_name` optional. Dataplane routes on `sub` only. | diff --git a/_context/wiki/testing.md b/_context/wiki/testing.md index 65d7644a..3669e7df 100644 --- a/_context/wiki/testing.md +++ b/_context/wiki/testing.md @@ -19,7 +19,7 @@ cargo nextest run --locked --workspace Use `cargo test` when nextest is unavailable. For wiki changes, also run `mdbook build _context/wiki` and `mdbook test _context/wiki`. -Protocol tests and fixtures should target MCP `2026-07-28`, use `server/discover`, and include the required per-request client metadata. Do not add new dataplane coverage for older protocol versions, legacy session initialization, or SSE; those paths belong in control-plane tests. Existing legacy-shaped tests are migration inventory and should be replaced as the modern implementation lands. +Protocol-sensitive tests and fixtures must cover MCP `2026-07-28` and `2025-11-25` in all four incoming-client/selected-backend combinations. The same-version paths are supported directly; the two cross-version paths are best effort and tests must cover both successful adaptation and explicit failure for semantics that cannot be translated without state. Every case must prove request independence: no required `Mcp-Session-Id`, session affinity, or retained backend transport. Keep `2026-07-28` coverage for `server/discover` and required per-request client metadata, and retain `initialize` coverage as a stateless compatibility request. SSE remains outside the dataplane contract. ## In-Repo Integration Tests From cb8558abfccf18d224cc01a2c4364267199dfb1b Mon Sep 17 00:00:00 2001 From: lucarlig Date: Fri, 21 Aug 2026 09:56:59 +0100 Subject: [PATCH 2/3] docs: clarify ContextForge plane terminology Signed-off-by: lucarlig --- _context/wiki/SUMMARY.md | 2 +- _context/wiki/architecture.md | 4 +- _context/wiki/config.md | 4 +- _context/wiki/deployment.md | 7 +- _context/wiki/getting-started.md | 8 +- _context/wiki/index.md | 16 +-- _context/wiki/mcp-capability-allocation.md | 150 +++++++++++---------- _context/wiki/performance.md | 12 +- _context/wiki/preferences.md | 8 +- _context/wiki/project.md | 132 +++++++++++------- _context/wiki/security.md | 15 ++- _context/wiki/testing.md | 51 ++++--- 12 files changed, 244 insertions(+), 165 deletions(-) diff --git a/_context/wiki/SUMMARY.md b/_context/wiki/SUMMARY.md index 477a1e61..8f90c35b 100644 --- a/_context/wiki/SUMMARY.md +++ b/_context/wiki/SUMMARY.md @@ -4,7 +4,7 @@ # The Project -- [What is ContextForge Data Plane?](project.md) +- [What is the ContextForge External Dataplane?](project.md) - [Getting Started](getting-started.md) # Architecture diff --git a/_context/wiki/architecture.md b/_context/wiki/architecture.md index 18a40873..c9146b29 100644 --- a/_context/wiki/architecture.md +++ b/_context/wiki/architecture.md @@ -184,7 +184,7 @@ Flow checkpoints — each must exist before the next dependency runs: | Checkpoint | Fact established | Next dependency | | --- | --- | --- | -| Listener | Request reached Rust dataplane over TCP/TLS. | Metrics, tracing, nested routing. | +| Listener | Request reached the ContextForge external dataplane over TCP/TLS. | Metrics, tracing, nested routing. | | Path extraction | Inner path matched `/servers/{virtual_host_id}/mcp`. | MCP handlers can resolve a `VirtualHost`. | | Claims validation | Bearer token accepted; `ContextForgeClaims` exists. | Config lookup can use `claims.sub`. | | User config lookup | `UserConfig` exists for the authenticated subject. | Virtual host check can run. | @@ -234,7 +234,7 @@ Changing a load-bearing choice requires updating more than one file: | Change | Required follow-through | | --- | --- | -| Downstream MCP version | Coordinate with the control plane and update the `2026-07-28`/`2025-11-25` compatibility matrix, protocol tests, examples, and front-door routing. During migration the current legacy route split may remain, but the target sends both supported Streamable HTTP versions to the stateless Rust dataplane. | +| Downstream MCP version | Coordinate with the ContextForge control plane and built-in dataplane; update the `2026-07-28`/`2025-11-25` compatibility matrix, protocol tests, examples, and front-door routing. The ContextForge built-in dataplane handles both stateful and stateless traffic; the ContextForge external dataplane handles both supported Streamable HTTP versions statelessly. | | Backend namespace / prefix contract | Update merge logic, split logic, tests, docs, and control-plane integration if client-facing surface moves. | | Session state moves external | Update `SessionManager`, cleanup behavior, load-balancing docs, and failure-mode tests. | | Config transport changes | Keep `UserConfigStore` as the boundary; update adapter tests. | diff --git a/_context/wiki/config.md b/_context/wiki/config.md index 33a55bcf..a9f79880 100644 --- a/_context/wiki/config.md +++ b/_context/wiki/config.md @@ -212,7 +212,7 @@ Start lightweight dependencies: docker compose -f docker/docker-compose-local.yaml up -d redis gateway-one gateway-two ``` -Register payload-marker configuration before starting the data plane: +Register payload-marker configuration before starting the ContextForge external dataplane: ```bash docker compose -f docker/docker-compose-local.yaml exec -T redis \ @@ -395,7 +395,7 @@ cargo run --release --bin contextforge-data-plane -- \ ## Known Telemetry Gaps -Tracked upstream, not yet implemented in the dataplane: +Tracked upstream, not yet implemented in the ContextForge external dataplane: | Gap | Issue | | --- | --- | diff --git a/_context/wiki/deployment.md b/_context/wiki/deployment.md index 97ccf65e..876b3026 100644 --- a/_context/wiki/deployment.md +++ b/_context/wiki/deployment.md @@ -7,7 +7,7 @@ ## Checklist -1. Front door routes only `/contextforge-rs` to the dataplane. +1. Front door routes only `/contextforge-rs` to the ContextForge external dataplane. 2. JWT verification key/secret matches the control plane's signing material; clients use control-plane API tokens whose `sub` matches the published user-config key. 3. Redis reachable; TLS/mTLS across trust zones; write access restricted to the control plane; `DATAPLANE_PUBLISHER=true` on the control plane. 4. Upstream connection mode matches backend URL schemes. @@ -23,8 +23,9 @@ ## nginx Front-Door Routing Reference `docker/nginx.conf` split: -- `location ^~ /contextforge-rs` → proxies to the gateway. -- All other traffic (UI, management, SSE, legacy MCP) → control-plane. +- `location ^~ /contextforge-rs` → proxies to the ContextForge external dataplane. +- UI and management traffic → ContextForge control plane. +- Other MCP routes, including stateful and legacy/SSE compatibility routes → ContextForge built-in dataplane. - Upstream retries on `error timeout http_502/503/504`: 2 tries, 10-second window. Non-idempotent MCP `POST` bodies are not re-sent after they reached an upstream — only connection-stage failures retry. ## Session Affinity And Failover diff --git a/_context/wiki/getting-started.md b/_context/wiki/getting-started.md index 5e1a8af6..31bf95dd 100644 --- a/_context/wiki/getting-started.md +++ b/_context/wiki/getting-started.md @@ -4,7 +4,7 @@ ```bash make docker-prod # build contextforge-data-plane:latest from docker/Dockerfile -make compose-up # start nginx, control-plane, redis, postgres, dataplane, fast_time_server +make compose-up # start nginx, Python control/built-in components, Redis, Postgres, external dataplane, fast_time_server ``` Wait for `register_fast_time` to finish, then allow ~60s config propagation: @@ -20,14 +20,16 @@ docker compose -f docker/docker-compose.yml logs -f register_fast_time | Bearer token | `GET http://localhost:8080/contextforge-rs/admin/tokens/admin@example.com` | | fast_time_server virtual host id | `b8e3f1a2c4d5e6f7a1b2c3d4e5f6a7b8` | -> **Critical**: `/contextforge-rs` prefix → dataplane. Without it → control-plane (you'll get `{"detail":"..."}` from mcpgateway, not a dataplane response). +> **Critical**: `/contextforge-rs` prefix → ContextForge external dataplane. +> Without it, MCP routes reach the ContextForge built-in dataplane (you'll get +> `{"detail":"..."}` from mcpgateway, not an external-dataplane response). Teardown: `make compose-down` (stops containers; volumes kept). ## cf-integration Harness (full end-to-end) ```bash -scripts/cf-integration.sh up # checkout control-plane, pull dataplane image, start full stack +scripts/cf-integration.sh up # checkout Python control/built-in repo, pull external-dataplane image, start full stack scripts/cf-integration.sh probe # smoke: 401 check → initialize → tools/list → tools/call scripts/cf-integration.sh test-all # all lanes: live-mcp, live-rbac, live-protocol scripts/cf-integration.sh down diff --git a/_context/wiki/index.md b/_context/wiki/index.md index 03f9c974..d16c93a6 100644 --- a/_context/wiki/index.md +++ b/_context/wiki/index.md @@ -1,4 +1,4 @@ -# ContextForge Data Plane — Wiki +# ContextForge External Dataplane — Wiki This wiki captures durable project context and working preferences. Check this index at the start of a task to decide whether deeper context is needed, @@ -16,19 +16,19 @@ then follow only the links that are relevant. | [mcp-capability-allocation.md](mcp-capability-allocation.md) | Tentative ContextForge 2.0 target topology, ownership, state model, Phase 1-4 roadmap, and Phase 3 flows | | [failure-modes.md](failure-modes.md) | HTTP/MCP/routing/backend/plugin failure table — exact HTTP codes and JSON-RPC errors | | [config.md](config.md) | Key CLI flags, JWT claims, UserConfig shape, plugin config, telemetry debugging, startup validation, local observability stack | -| [deployment.md](deployment.md) | Deployment checklist, health endpoint caveat, nginx routing, TLS choices, session affinity, Redis availability, image pinning | -| [security.md](security.md) | Trust boundaries, control-plane/dataplane authentication split, Origin/Host validation, transport security, secrets handling | -| [performance.md](performance.md) | Dataplane-only load testing (Goose), full-stack Locust runs, benchmark settings, control-plane baseline | +| [deployment.md](deployment.md) | External-dataplane deployment checklist, health endpoint caveat, nginx routing, TLS choices, session affinity, Redis availability, image pinning | +| [security.md](security.md) | Trust boundaries among the control plane, built-in dataplane, and external dataplane; Origin/Host validation; transport security; secrets handling | +| [performance.md](performance.md) | External-dataplane-only load testing (Goose), full-stack Locust runs, benchmark settings, built-in-dataplane baseline | | [testing.md](testing.md) | Workspace checks, in-repo integration tests, full-stack harness lanes, settings, and control-plane baseline | ## Quick orientation -- **Repo**: `contextforge-data-plane` — the Rust dataplane for ContextForge. -- **Core invariant**: this crate is pure routing logic. No IAM, UI, or metrics storage. -- **Protocol target**: MCP `2026-07-28` and `2025-11-25` over Streamable HTTP. Both use stateless request handling; `initialize` remains supported but does not establish dataplane session state. Legacy SSE paths are being removed. +- **Repo**: `contextforge-data-plane` — the Rust ContextForge external dataplane. +- **Core invariant**: the ContextForge external dataplane is pure routing logic. No IAM, UI, or metrics storage. +- **Protocol target**: the ContextForge external dataplane supports MCP `2026-07-28` and `2025-11-25` over Streamable HTTP. Both use stateless request handling; `initialize` remains supported but does not establish external-dataplane session state. Legacy SSE paths are being removed from the external dataplane. - **Status convention**: project, architecture, routing, and operations pages describe the current implementation. The page under **Upcoming** describes the tentative ContextForge 2.0 target and migration roadmap. - **Architecture context**: [architecture.md](architecture.md) — read before touching the hot path. Full wiki index above. - **Validation gate**: `cargo fmt` + `cargo clippy` + `cargo nextest` + `cargo deny` must be clean; CI also runs `cargo shear`. See [preferences.md](preferences.md) for by-change-type requirements. -- **System topology**: `client → nginx → [dataplane | control-plane]`; config flows from control-plane via `dataplane_publisher.py` → Redis → dataplane. See [project.md § System topology](project.md#system-topology). +- **System topology**: `client → nginx → [ContextForge built-in dataplane | ContextForge external dataplane | ContextForge control plane]`; external-dataplane config flows from the control plane via `dataplane_publisher.py` → Redis → external dataplane. See [project.md § System topology](project.md#system-topology). diff --git a/_context/wiki/mcp-capability-allocation.md b/_context/wiki/mcp-capability-allocation.md index 7fa7cdd2..c5fe1cf5 100644 --- a/_context/wiki/mcp-capability-allocation.md +++ b/_context/wiki/mcp-capability-allocation.md @@ -5,31 +5,35 @@ > implementation. See [Architecture](architecture.md) and > [MCP Routing Semantics](routing.md) for current behavior. -This is a product-wide view because the Rust dataplane boundary depends on -work owned by the external Python control plane and slow dataplane. It does not -move control-plane responsibilities into this repository. +This is a product-wide view because the ContextForge external dataplane +boundary depends on work owned by the ContextForge control plane and built-in +dataplane in the Python `IBM/mcp-context-forge` repository. It does not move +control-plane or built-in-dataplane responsibilities into this repository. See +[Project terminology](project.md#terminology) for the canonical component +names. ## Vision and Constraints - ContextForge supports MCP `2026-07-28` and `2025-11-25` over Streamable HTTP - on both the client-facing and backend-facing sides of the fast Rust - dataplane. + on both the client-facing and backend-facing sides of the ContextForge + external dataplane. - Same-version client/backend paths are supported directly. Cross-version `2026-07-28` → `2025-11-25` and `2025-11-25` → `2026-07-28` adaptation is best effort. -- All request/response handling is stateless for both versions. The target - request path does not depend on an MCP session, session affinity, or a - retained backend transport. +- All external-dataplane request/response handling is stateless for both + versions. The target request path does not depend on an MCP session, session + affinity, or a retained backend transport. - `initialize` remains supported for compatibility, but it is a stateless - request: the dataplane generates its response from effective configuration - and does not use it to establish state required by later requests. -- Legacy SSE transport is not part of the dataplane target. + request: the external dataplane generates its response from effective + configuration and does not use it to establish state required by later + requests. +- Legacy SSE transport is not part of the external-dataplane target. - Fan-out and other one-to-many MCP work is limited to the control plane. The - slow and fast dataplanes generate discovery, capability, and list responses - from control-plane-authored effective configuration. -- Effective configuration flows one way from the control plane to the - dataplanes through externally shared state. A process-local cache may speed - reads but is never the source of truth. + built-in and external dataplanes generate discovery, capability, and list + responses from control-plane-authored effective configuration. +- Effective configuration flows one way from the control plane to the built-in + and external dataplanes through externally shared state. A process-local + cache may speed reads but is never the source of truth. - MCP subscriptions and notifications remain Phase 4 work. ## Stateless Protocol Compatibility @@ -46,7 +50,7 @@ incoming/client-facing side “upstream” and the selected backend-facing side | `2025-11-25` | `2026-07-28` | Best-effort protocol adaptation within one stateless request. | | `2025-11-25` | `2025-11-25` | Supported directly as one stateless request. | -For every row, the dataplane authenticates and authorizes the request, reads +For every row, the external dataplane authenticates and authorizes the request, reads the principal-bound effective configuration, validates that the requested object is visible and permitted, resolves exactly one backend, adapts the protocol when necessary, and closes the request-scoped backend connection after @@ -55,21 +59,22 @@ require nor reuse state created by it. “Best effort” never permits hidden session state. If a semantic difference or backend requirement cannot be handled within the current request, the -dataplane returns an explicit error instead of creating affinity or retaining a -backend transport for a later request. +external dataplane returns an explicit error instead of creating affinity or +retaining a backend transport for a later request. For the initial `tools/call` slice, issue #6327 assumes that the selected backend needs neither application authentication nor mTLS and that its server certificate chains to the system CA. Those are issue-scope assumptions, not a -change to the dataplane's broader transport-security model. +change to the external dataplane's broader transport-security model. ## Target End State -The front door separates management traffic from supported Streamable HTTP MCP -traffic. It does not split the two supported MCP versions in the target end -state: both go to the fast Rust dataplane. PostgreSQL remains the durable -management store; the shared runtime store carries compiled configuration to -the dataplane. +The front door separates management traffic from MCP traffic and chooses the +built-in or external dataplane by deployment route and session model, not only +by protocol version. The built-in dataplane can handle either supported version in stateful +or stateless mode. The external dataplane can handle either supported version +only in stateless mode. PostgreSQL remains the durable management store; the +shared runtime store carries compiled configuration to both dataplanes. ```mermaid flowchart TB @@ -84,8 +89,12 @@ flowchart TB subgraph ContextForge[ContextForge 2.0] direction LR - Admin[Python Admin and Management API] - Fast[Fast Rust MCP Dataplane] + subgraph PythonRepo[IBM mcp-context-forge Python Repository] + direction TB + Control[ContextForge Control Plane] + Builtin[ContextForge Built-In Dataplane] + end + External[ContextForge External Dataplane - Rust] end Postgres[(PostgreSQL Management State)] @@ -96,33 +105,36 @@ flowchart TB CompatClient -->|Streamable HTTP MCP 2025-11-25| FrontDoor ModernClient -->|Streamable HTTP MCP 2026-07-28| FrontDoor - FrontDoor -->|Management routes| Admin - FrontDoor -->|Both supported MCP versions| Fast + FrontDoor -->|Management routes| Control + FrontDoor -->|Stateful or built-in MCP routes| Builtin + FrontDoor -->|Stateless external MCP routes| External - Admin -->|Persist administrative state| Postgres - Admin -->|Publish effective configuration| RuntimeStore - RuntimeStore -->|Read-only configuration| Fast + Control -->|Persist administrative state| Postgres + Control -->|Publish effective configuration| RuntimeStore + RuntimeStore -->|Read shared configuration| Builtin + RuntimeStore -->|Read-only configuration| External - Admin -->|Discover catalogs and poll liveness| Upstreams - Fast -->|Targeted MCP calls| Upstreams + Control -->|Discover catalogs and poll liveness| Upstreams + Builtin -->|Stateful or stateless MCP calls| Upstreams + External -->|Stateless targeted MCP calls| Upstreams ``` -Redis is the current fast-path store and the preferred shared implementation. -During migration, the Python slow path may consume the same compiled -configuration from Redis or PostgreSQL, but it is not the permanent home for -either supported Streamable HTTP version. It must not rely on process memory -alone when multiple slow-path instances are deployed. +Redis is the current external-dataplane configuration store and the preferred +shared implementation. The built-in dataplane may consume the same compiled +configuration from Redis or PostgreSQL. When multiple built-in-dataplane +instances are deployed, stateful MCP behavior requires an explicit shared-state +or affinity design; stateless behavior must not rely on process memory. ## Component Responsibilities | Component | Target responsibility | | --- | --- | -| Front door | Route management APIs to the control plane and both supported Streamable HTTP MCP versions to the fast Rust dataplane. A temporary slow-path split may remain during migration. | -| Admin and management API | Manage the virtual-server lifecycle and upstream assignments; connect to heterogeneous upstreams; retrieve and page through capabilities, tools, resources, prompts, completions, and other catalogs; normalize and persist them; let administrators select exposed objects and rules; compile effective runtime configuration; poll upstream liveness and changes. | -| PostgreSQL | Persist administrative source data such as virtual servers, upstream definitions, normalized catalogs, selections, and policies. It is not on the fast request path. | -| Configuration synchronization | Publish effective configuration one way from the control plane to externally shared state. Both dataplanes should consume the same shape where practical. | -| Slow path MCP dataplane | Temporary migration path only. Read effective configuration from shared state and generate aggregate responses without live backend fan-out while methods move to Rust. | -| Fast Rust MCP dataplane | Handle `2026-07-28` and `2025-11-25` Streamable HTTP requests statelessly. Read effective configuration, serve aggregate and `initialize` responses locally, and route a targeted method to exactly one selected backend. Cross-version adaptation is best effort. It does not own IAM, UI, management APIs, or durable metrics storage. | +| Front door | Route management APIs to the ContextForge control plane. Route MCP to the built-in dataplane when the built-in route or stateful behavior is required, and to the external dataplane when the configured stateless external route is selected. Protocol version alone does not identify the component. | +| ContextForge control plane | Manage the virtual-server lifecycle and upstream assignments; connect to heterogeneous upstreams; retrieve and page through capabilities, tools, resources, prompts, completions, and other catalogs; normalize and persist them; let administrators select exposed objects and rules; compile effective runtime configuration; poll upstream liveness and changes. | +| PostgreSQL | Persist administrative source data such as virtual servers, upstream definitions, normalized catalogs, selections, and policies. It is not on the external-dataplane request path. | +| Configuration synchronization | Publish effective configuration one way from the control plane to externally shared state. The built-in and external dataplanes should consume the same shape where practical. | +| ContextForge built-in dataplane | Handle `2026-07-28` and `2025-11-25` MCP requests in Python, including stateful and stateless behavior. It is the MCP request path shipped in the same repository as the control plane, not the control plane itself. | +| ContextForge external dataplane | Handle `2026-07-28` and `2025-11-25` Streamable HTTP requests statelessly in Rust. Read effective configuration, serve aggregate and `initialize` responses locally, and route a targeted method to exactly one selected backend. Cross-version adaptation is best effort. It does not own IAM, UI, management APIs, or durable metrics storage. | | Backend MCP servers | May use `2026-07-28` or `2025-11-25`, independently of the incoming client version. Connections and any required negotiation are request-scoped and leave no reusable session; the architecture does not require backend session affinity. | ## Administrative State and Effective Configuration @@ -132,13 +144,13 @@ The control plane owns two distinct forms of state: | State | Contents | Owner and consumers | | --- | --- | --- | | Administrative source state | Virtual servers, upstream registrations, raw and normalized catalogs, exposure selections, policies, and liveness. | Written by the control plane to PostgreSQL; used by management workflows and reconciliation. | -| Effective runtime configuration | Effective server identity and capabilities, visible tools/resources/prompts/completions, downstream paging material, backend resolution, required scopes/roles, and applicable runtime policy for a tenant or isolation domain, user, team, or other principal. | Compiled and published by the control plane; read by slow and fast dataplanes. | +| Effective runtime configuration | Effective server identity and capabilities, visible tools/resources/prompts/completions, downstream paging material, backend resolution, required scopes/roles, and applicable runtime policy for a tenant or isolation domain, user, team, or other principal. | Compiled and published by the control plane; read by the built-in and external dataplanes. | The control plane must exhaust upstream pagination while reconciling catalogs. -The compiled snapshot must contain enough information for either dataplane to -produce downstream paging without contacting every upstream. Publication must -be atomic or revisioned so a dataplane never combines partial catalog and -policy state. +The compiled snapshot must contain enough information for either the built-in +or external dataplane to produce downstream paging without contacting every +upstream. Publication must be atomic or revisioned so neither the built-in nor +external dataplane combines partial catalog and policy state. ## Target Authorization Invariants @@ -147,8 +159,8 @@ catalog precomputation. A cached snapshot is data, not an authorization grant. Every downstream request must independently establish and enforce its trusted authorization context. -- The dataplane derives the authorization key only from verified JWT claims - and the validated server route. MCP params and client metadata must not +- The external dataplane derives the authorization key only from verified JWT + claims and the validated server route. MCP params and client metadata must not supply or override a principal, team, tenant, virtual server, backend, or cache key. - Snapshot and cache partitions include the applicable trust or tenant @@ -156,16 +168,16 @@ authorization context. server, and configuration revision. Entries must never be reused across authorization contexts. - The control plane maps verified identity attributes to an effective - principal and compiles its visible objects and RBAC policy. The dataplane - enforces required token scopes or roles and the compiled policy on every - discovery, list, and targeted operation. + principal and compiles its visible objects and RBAC policy. The built-in and + external dataplanes enforce required token scopes or roles and the compiled + policy on every discovery, list, and targeted operation. - Missing, unmapped, ambiguous, expired, or unauthorized snapshots and objects are denied by default. A targeted denial makes no upstream call, and errors must not disclose another principal's catalog or backend mapping. - The exact tenant/team claim mapping and token-scope-to-RBAC rules are a cross-repository contract that the control plane, publisher, schemas, - dataplane, and integration tests must define together. The current coarse - `sub`-only implementation is not the Phase 3 target. + external dataplane, and integration tests must define together. The current + coarse `sub`-only implementation is not the Phase 3 target. ## MCP Work Allocation @@ -173,20 +185,20 @@ authorization context. | --- | --- | | Virtual-server creation and upstream assignment | Control plane persists management state and connects to assigned upstreams. | | Upstream discovery, initialization where required, catalog pagination, capability aggregation, filtering, and liveness polling | Control plane only; this is the intentional fan-out boundary. | -| `server/discover`, `initialize`, and effective capabilities | After per-request authorization, the fast dataplane generates the response from principal-bound effective configuration. `initialize` is supported for compatibility but creates no state that later requests depend on. | -| `tools/list`, `resources/list`, `prompts/list`, resource-template listing, and similar aggregate methods | After method-scope and compiled-RBAC enforcement, the slow or fast dataplane generates the visible response from principal-bound effective configuration with no live upstream fan-out. | -| `tools/call`, `resources/read`, `prompts/get`, completion, and similar targeted methods | Dataplane resolves the effective entry under the trusted authorization key, applies default-deny scope and object policy, adapts between the incoming and backend protocol versions when necessary, and calls exactly one selected backend only when authorized. The backend connection is request-scoped and leaves no reusable session. | +| `server/discover`, `initialize`, and effective capabilities | After per-request authorization, the built-in or external dataplane generates the response from principal-bound effective configuration. The built-in dataplane may support a stateful flow; the external dataplane treats `initialize` as stateless compatibility and creates no state required by later requests. | +| `tools/list`, `resources/list`, `prompts/list`, resource-template listing, and similar aggregate methods | After method-scope and compiled-RBAC enforcement, the built-in or external dataplane generates the visible response from principal-bound effective configuration with no live upstream fan-out. | +| `tools/call`, `resources/read`, `prompts/get`, completion, and similar targeted methods | The built-in or external dataplane resolves the effective entry under the trusted authorization key, applies default-deny scope and object policy, and calls exactly one selected backend only when authorized. The external dataplane adapts protocol versions when necessary and leaves no reusable session; the built-in dataplane may use its stateful or stateless execution model. | | Plugins for trusted aggregate responses | Prefer policy compiled by the control plane; avoid mandatory per-request plugin calls for a response already produced from trusted effective configuration. | -| Plugins for targeted calls | May run on the fast path when request or response inspection is required. Exact hook allocation remains an implementation decision. | +| Plugins for targeted calls | May run on the external-dataplane request path when request or response inspection is required. Exact hook allocation remains an implementation decision. | | Subscriptions, server notifications, and downstream list-change notifications | Deferred to Phase 4 because their state and delivery model do not fit the request/response simplification. | ## Delivery Roadmap | Phase | Scope | | --- | --- | -| **1. Separate the Python control and slow planes** | Establish a clear boundary inside the current Python component. The control plane writes effective configuration per user, team, or other principal to shared state; the slow dataplane reads it and responds accordingly. Slow-path paging may be deferred or skipped if the fast-path migration progresses quickly. Plugins that belong on the fast path need not be duplicated in the slow path. | -| **2. Offload targeted calls to the fast dataplane** | Make the slow and fast dataplanes follow the same configuration-driven pattern. Initially send targeted operations such as `tools/call`, `resources/read`, `prompts/get`, and completion to the fast dataplane. For each operation, support both same-version `2026-07-28`/`2025-11-25` paths and attempt both cross-version paths on a best-effort basis, always without reusable session state. The `tools/call` slice is tracked by [#6327](https://github.com/IBM/mcp-context-forge/issues/6327). | -| **3. Offload all request/response MCP methods to the fast dataplane** | Serve discovery, stateless `initialize`, capabilities, aggregate lists, and targeted calls for both supported protocol versions from the fast dataplane. Aggregate responses come from effective configuration; targeted calls reach exactly one backend. | +| **1. Separate control-plane and built-in-dataplane responsibilities** | Establish a clear boundary between the ContextForge control plane and built-in dataplane inside the Python repository. The control plane writes effective configuration per user, team, or other principal to shared state; the built-in dataplane reads it and handles MCP requests. | +| **2. Route targeted calls through the external dataplane** | Make the built-in and external dataplanes follow the same configuration-driven contract. Send selected targeted operations such as `tools/call`, `resources/read`, `prompts/get`, and completion to the external dataplane. For each operation, support both same-version `2026-07-28`/`2025-11-25` paths and attempt both cross-version paths on a best-effort basis, always without reusable session state. The `tools/call` slice is tracked by [#6327](https://github.com/IBM/mcp-context-forge/issues/6327). | +| **3. Route all stateless request/response MCP methods through the external dataplane** | Serve discovery, stateless `initialize`, capabilities, aggregate lists, and targeted calls for both supported protocol versions from the external dataplane. Aggregate responses come from effective configuration; targeted calls reach exactly one backend. The built-in dataplane continues to support both stateful and stateless behavior. | | **4. Implement subscriptions and notifications** | Add the state, routing, and delivery model for upstream subscriptions, resource notifications, and list-change notifications after the request/response architecture is complete. | ## Phase 3 Reference Flows @@ -203,12 +215,12 @@ sequenceDiagram autonumber actor User participant UI as Admin UI or API - participant CP as Control Plane + participant CP as ContextForge Control Plane participant DB as Control Plane DB participant MCP1 as MCP 2026-07-28 Server participant MCP2 as MCP 2025-11-25 Server participant Store as Shared Config Store (Redis) - participant DP as Fast Rust Dataplane + participant DP as ContextForge External Dataplane User->>UI: Create virtual server UI->>CP: Submit virtual server @@ -255,7 +267,7 @@ sequenceDiagram autonumber participant Client as Supported MCP Client participant Ingress - participant DP as Fast Rust Dataplane + participant DP as ContextForge External Dataplane participant Cache as Local Cache participant Store as Shared Config Store (Redis) @@ -308,7 +320,7 @@ sequenceDiagram autonumber participant Client as Supported MCP Client participant Ingress - participant DP as Fast Rust Dataplane + participant DP as ContextForge External Dataplane participant Cache as Local Cache participant CPEX as Policy and CPEX participant MCP as Selected Supported MCP Server @@ -355,7 +367,7 @@ sequenceDiagram participant CP as Control Plane Reconciler participant DB as Control Plane DB participant Store as Shared Config Store (Redis) - participant DP as Fast Rust Dataplane + participant DP as ContextForge External Dataplane participant Client as Supported MCP Client CP->>MCP: Poll liveness and refresh discovery and lists diff --git a/_context/wiki/performance.md b/_context/wiki/performance.md index d494afb6..330c6f87 100644 --- a/_context/wiki/performance.md +++ b/_context/wiki/performance.md @@ -2,12 +2,12 @@ ## Two Load Paths -- **Dataplane-only:** `contextforge-load-test` measures the Rust dataplane in isolation. -- **Full-stack:** `cf-integration` harness measures the full nginx → control-plane → dataplane path with Locust. +- **External-dataplane-only:** `contextforge-load-test` measures the ContextForge external dataplane in isolation. +- **Full-stack:** `cf-integration` measures the nginx → external dataplane → backend request path with Locust while the ContextForge control plane publishes configuration. Use the first to profile gateway changes; use the second to measure what users see. -## Dataplane-Only (Goose) +## External-Dataplane-Only (Goose) `crates/contextforge-load-test` is a [Goose](https://book.goose.rs/)-based driver that speaks full streamable HTTP MCP. Start the local stack and seed user config first (see [getting-started.md](getting-started.md)), then: @@ -51,9 +51,11 @@ Restore both to `60` before measuring throughput — fast publish + per-request | `CF_DATAPLANE_PUBLISHER_INTERVAL_SECONDS` | `2` (fast config publish) | `60` (upstream default) | | `CF_DATAPLANE_USER_CONFIG_CACHE_EXPIRY_SECONDS` | `0` (cache disabled) | `60` (upstream default) | -## Control-Plane Baseline +## Built-In-Dataplane Baseline -Compare against the stack without the dataplane: +Compare against the stock Python repository, where MCP traffic uses the +ContextForge built-in dataplane and the ContextForge external dataplane is +absent: ```bash scripts/cf-integration.sh down # free shared ports diff --git a/_context/wiki/preferences.md b/_context/wiki/preferences.md index 55c374e7..49e88342 100644 --- a/_context/wiki/preferences.md +++ b/_context/wiki/preferences.md @@ -25,7 +25,7 @@ CI additionally runs `cargo shear --check-test-targets --deny-warnings --locked` ## Code style - **Idiomatic Rust** — no unnecessary clones, heap allocations, `Arc`, or `Mutex` unless justified by the design. -- Most product behavior lives in `contextforge-data-plane-lib`. Do not let dataplane logic accumulate in the binary crate. +- Most ContextForge external-dataplane behavior lives in `contextforge-data-plane-lib`. Do not let external-dataplane logic accumulate in the binary crate. - Typed errors — propagate errors rather than swallowing them silently. - Keep change size minimal. Every changed line must trace directly to the task at hand. @@ -33,7 +33,7 @@ CI additionally runs `cargo shear --check-test-targets --deny-warnings --locked` - Use `tracing` for all log output. - **Prefer message-embedded fields**: `level!("method_name - event field = {val} other_field = {other}")`. - Do **not** use structured field syntax (`, field = val`) for dataplane logs. + Do **not** use structured field syntax (`, field = val`) for ContextForge external-dataplane logs. - Keep method/event prefixes stable and reuse the same field names and order for related events. - `warn!` is for unexpected conditions that need operator attention. Expected user/config misses → `debug!` or `info!`. - **Never log**: tokens, authorization headers, secrets, Redis key/value bytes, full `UserConfig`, or backend credentials. @@ -47,7 +47,7 @@ CI additionally runs `cargo shear --check-test-targets --deny-warnings --locked` ## Architectural rules (non-negotiable) -- The dataplane is pure routing logic. **No IAM, UI, or metrics-storage concerns.** +- The ContextForge external dataplane is pure routing logic. **No IAM, UI, or metrics-storage concerns.** - Config access goes through `UserConfigStore` only — never push Redis details into routing code. - The backend prefix naming contract must not change without updating merge logic, split logic, and tests. - Legacy SSE transport and stateful session behavior are being **removed**. `initialize` remains supported as a stateless compatibility method; do not use it to create affinity, persist client state, or retain backend transports between requests. @@ -55,7 +55,7 @@ CI additionally runs `cargo shear --check-test-targets --deny-warnings --locked` ## Protocol target -- The dataplane target supports MCP **`2026-07-28`** and **`2025-11-25`** over **Streamable HTTP**. +- The ContextForge external-dataplane target supports MCP **`2026-07-28`** and **`2025-11-25`** over **Streamable HTTP**. - Every request is independent for both versions. Do not require `Mcp-Session-Id`, session affinity, or a previously retained backend transport. - Retain `initialize` for clients that use it, but generate its response from effective configuration and do not treat it as session establishment. `2026-07-28` tests and examples should continue to exercise `server/discover` and per-request client metadata. - Protocol-sensitive tests cover both same-version paths and the best-effort cross-version paths (`2026-07-28` → `2025-11-25` and the reverse). Do not add SSE or versions earlier than `2025-11-25` without a separate architecture decision. diff --git a/_context/wiki/project.md b/_context/wiki/project.md index d234dc8a..c0884b68 100644 --- a/_context/wiki/project.md +++ b/_context/wiki/project.md @@ -6,26 +6,58 @@ ## What this project is -`contextforge-data-plane` is a Rust-based MCP (Model Context Protocol) gateway — the **dataplane** component of ContextForge. It acts as a scalable, secure proxy layer that routes AI tool calls from MCP clients to one or more backend MCP servers. +`contextforge-data-plane` is the Rust-based **ContextForge external dataplane**. +It is a scalable, separately deployable MCP (Model Context Protocol) gateway +that routes AI tool calls from MCP clients to backend MCP servers. -It is paired with the external ContextForge control plane at [`IBM/mcp-context-forge`](https://github.com/IBM/mcp-context-forge). The two components have a strict division of responsibility: +The [`IBM/mcp-context-forge`](https://github.com/IBM/mcp-context-forge) +Python repository contains two different product components: the ContextForge +control plane and the ContextForge built-in dataplane. This Rust repository is +the third component: | Layer | Owns today | | --- | --- | -| **This repo (dataplane)** | Request routing, auth enforcement, backend fan-out, session ownership | -| **Control plane** | IAM, UI, metrics storage, legacy MCP client compatibility | - -The dataplane must never take on control-plane concerns. +| **ContextForge control plane** (Python) | IAM, UI, management APIs, durable administrative state, policy/catalog compilation, metrics storage, and external-dataplane configuration publishing. | +| **ContextForge built-in dataplane** (Python) | MCP request handling shipped in the same repository as the control plane. Supports `2026-07-28` and `2025-11-25`, including stateful and stateless behavior. | +| **ContextForge external dataplane** (Rust, this repo) | Separately deployed MCP request routing and authorization enforcement. The target supports both protocol versions without session state; cross-version adaptation is best effort. | + +The ContextForge external dataplane must never take on control-plane concerns. + +## Terminology + +Use the full component names in product-wide architecture and deployment +documentation: + +- **ContextForge control plane** means the Python management plane in + `IBM/mcp-context-forge`. It owns administrative workflows and publishes + effective runtime configuration; it is not the name for every process or MCP + route in that repository. +- **ContextForge built-in dataplane** means the Python MCP request path in the + same `IBM/mcp-context-forge` repository. “Built-in” describes where it ships, + not a legacy-only or slow-path role. It handles the old and new protocol + versions and can serve stateful or stateless clients. +- **ContextForge external dataplane** means this independently deployable Rust + repository. “External” means external to the Python repository/deployment, + not untrusted or third-party. Its target request path is stateless for both + supported protocol versions. +- **Stateful** means later MCP requests can depend on session context established + by `initialize` or a session identifier. **Stateless** means every request is + independently authenticated, authorized, resolved, and completed without + reusable MCP session state. + +Always use one of the three canonical names. Do not use unqualified +“dataplane,” “local dataplane,” “slow dataplane,” or “fast dataplane” as a +product component name. ```mermaid flowchart LR - C(["MCP Client\nprotocol 2026-07-28\nStreamable HTTP"]) + C(["MCP Client\nold/new · stateful/stateless"]) subgraph Infra["Infrastructure"] N["nginx\nTLS termination\nrouting fan-out"] end - subgraph DP["ContextForge Data Plane (this repo)"] + subgraph EDP["ContextForge External Dataplane (Rust, this repo)"] direction TB MW["Middleware stack\nvirtual host · JWT · session · user config"] RT["MCP Routing\nfan-out · prefix namespace\nlist merge · capability merge"] @@ -33,21 +65,25 @@ flowchart LR MW --> RT --> PL end - subgraph CP["Control Plane (IBM/mcp-context-forge)"] + subgraph PythonRepo["IBM/mcp-context-forge (Python repo)"] direction TB - IAM["IAM · UI\nmetrics storage"] + CP["ContextForge control plane\nIAM · UI · management"] + BDP["ContextForge built-in dataplane\nold/new · stateful/stateless"] PUB["dataplane_publisher.py\nwrites UserConfig to Redis"] + CP --> PUB end R[("Redis\nUserConfig store\nMessagePack")] BE["Backend MCP Servers"] C --> N - N -->|"/contextforge-rs/*"| DP - N -->|"UI / IAM / legacy MCP / SSE"| CP - CP --> R - DP -->|"read-only UserConfig"| R - DP -->|"MCP calls"| BE + N -->|"external route - currently 2026-07-28"| EDP + N -->|"UI / IAM / management"| CP + N -->|"built-in MCP routes"| BDP + PUB --> R + EDP -->|"read-only UserConfig"| R + EDP -->|"MCP calls"| BE + BDP -->|"MCP calls"| BE ``` @@ -55,7 +91,7 @@ flowchart LR - Provide a **production-grade, low-latency routing layer** between MCP clients and backend MCP servers. - Support MCP `2026-07-28` and `2025-11-25` over Streamable HTTP as stateless downstream contracts. -- Enforce a clean **dataplane/control-plane boundary** — no IAM, UI, or metrics storage logic in this repo. +- Enforce a clean **ContextForge external dataplane/control plane boundary** — no IAM, UI, or metrics storage logic in this repo. - Keep config access behind the **`UserConfigStore` abstraction** (backed by Redis/MessagePack). - Remain in the right architectural shape during early development, prioritising correctness over backward compatibility. @@ -63,7 +99,7 @@ flowchart LR - **Platform teams** — deploy and operate the gateway as infrastructure. - **AI application developers** — use the gateway as the MCP proxy layer for their applications. -- **Internal contributors** — engineers evolving the dataplane toward stateless `2026-07-28` and `2025-11-25` protocol support. +- **Internal contributors** — engineers evolving the ContextForge external dataplane toward stateless `2026-07-28` and `2025-11-25` protocol support. ## Key modules and architecture @@ -81,8 +117,8 @@ Architecture context lives in the wiki. Key pages: | Crate | Purpose | | --- | --- | -| `contextforge-data-plane-lib` | All dataplane behavior: routing, middleware, sessions, transports. Almost everything goes here. | -| `contextforge-data-plane` (binary) | Process shell only: CLI flags, logging, runtime shape. No dataplane logic. | +| `contextforge-data-plane-lib` | All ContextForge external-dataplane behavior: routing, middleware, sessions, transports. Almost everything goes here. | +| `contextforge-data-plane` (binary) | Process shell only: CLI flags, logging, runtime shape. No ContextForge external-dataplane logic. | | `contextforge-data-plane-apis` | Shared config shapes (`UserConfig`, `User`, plugin config). Regenerate JSON schemas after any change: `cargo run -p contextforge-data-plane-apis`. | | `contextforge-data-plane-cpex` | Plugin integration (CPEX hook factories). | | `contextforge-load-test` | Performance harness: end-to-end MCP traffic driver. | @@ -95,26 +131,26 @@ Architecture context lives in the wiki. Key pages: ## Active work (near-term) - **Protocol migration**: support same-version `2026-07-28` and `2025-11-25` paths over Streamable HTTP, provide best-effort translation in either cross-version direction, and replace stateful session paths with request-scoped handling. -- Legacy SSE transport and session affinity are **being removed**. `initialize` is retained as a stateless compatibility request and must not create persistent dataplane or backend session state. +- Legacy SSE transport and session affinity are **being removed** from the ContextForge external dataplane. `initialize` is retained as a stateless compatibility request and must not create persistent external-dataplane or backend session state. - Protocol-sensitive tests must cover the two direct and two best-effort cross-version combinations. Modern examples should continue to use `server/discover` and per-request client metadata; compatibility examples may use `initialize` without relying on later session reuse. -## Control-Plane Integration Contract +## ContextForge Integration Contract > **Provisional.** No formal contract has been stipulated yet. This section documents the current de-facto integration surface with [IBM/mcp-context-forge](https://github.com/IBM/mcp-context-forge). Any row may change while the project is early; when a proper contract is agreed, update this section to track it. | Agreement | Value today | | --- | --- | -| Client-facing route | `/servers/{virtual_host_id}/mcp`. Front door rewrites modern MCP `2026-07-28` Streamable HTTP traffic to `/contextforge-rs/servers/{virtual_host_id}/mcp` on the dataplane. | -| Protocol compatibility | Today the dataplane route accepts MCP `2026-07-28`; the control plane serves `2025-11-25`, session-based initialization, and SSE on its own routes. The target moves both supported Streamable HTTP versions to stateless dataplane handling, with cross-version adaptation on a best-effort basis. | +| Client-facing route | `/servers/{virtual_host_id}/mcp`. Front door rewrites modern MCP `2026-07-28` Streamable HTTP traffic to `/contextforge-rs/servers/{virtual_host_id}/mcp` on the ContextForge external dataplane. | +| Protocol compatibility | Today the external-dataplane route accepts MCP `2026-07-28`; the built-in dataplane handles `2026-07-28` and `2025-11-25`, including stateful and stateless behavior and legacy SSE compatibility. The external-dataplane target handles both supported Streamable HTTP versions statelessly, with cross-version adaptation on a best-effort basis. | | Unknown virtual host | `404` with body `{"detail":"Server not found"}`, matching the control-plane response shape. | | Token issuer and audience | `iss = mcpgateway`, `aud = mcpgateway-api`. | -| Claims shape | `sub`, `jti`, `iss`, `aud`, `exp`, and `user` required. `token_use`, `iat`, `teams`, `scopes`, and `user.full_name` optional. Dataplane routes on `sub` only. | +| Claims shape | `sub`, `jti`, `iss`, `aud`, `exp`, and `user` required. `token_use`, `iat`, `teams`, `scopes`, and `user.full_name` optional. The ContextForge external dataplane routes on `sub` only. | | User config Redis key | `MessagePack(User::new(jwt_subject))` — key type plus subject, not the raw subject string. | | User config Redis value | `MessagePack(UserConfig)`. JSON schema at `schemas/user_config.json`. | | User key Redis schema | `schemas/user.json`. | | Plugin config key | `ContextForgeGatewayRuntimePluginConfig`, JSON or MessagePack, `version: 1` with a `cpex` section. | -**Coordination rule:** changing any row above is a cross-repo change. The dataplane, the control-plane publisher (`dataplane_publisher.py`), and the `cf-integration` harness all need updating together. +**Coordination rule:** changing any row above is a cross-repo change. The external dataplane, the control-plane publisher (`dataplane_publisher.py`), and the `cf-integration` harness all need updating together. Regenerate both schemas after any struct change to `UserConfig`, `VirtualHost`, `BackendMCPGateway`, or the `User` key type: ```bash @@ -123,45 +159,51 @@ cargo run -p contextforge-data-plane-apis ## System topology (current) -All external traffic enters through **nginx**, which fans out to either the dataplane or the control plane: +All external traffic enters through **nginx**, which routes management traffic +to the control plane and MCP traffic to either the built-in or external +dataplane: ```mermaid flowchart LR client(["client"]) --> nginx["nginx"] - nginx --> dataplane["data-plane"] - nginx --> controlplane["control-plane"] - dataplane --> redis["redis"] - controlplane --> redis - controlplane --> postgres["postgres\n(via pgbouncer)"] - dataplane --> fastts["fast_time_server"] + nginx --> external["external dataplane\nRust · this repo"] + nginx --> builtin["built-in dataplane\nPython repo"] + nginx --> control["control plane\nPython repo"] + external --> redis["redis"] + control --> redis + control --> postgres["postgres\n(via pgbouncer)"] + external --> fastts["fast_time_server"] ``` -### How the control plane publishes config to the dataplane +### How the control plane publishes config to the external dataplane -The control plane and dataplane do **not** communicate over HTTP. Config is exchanged exclusively through Redis: +The control plane and external dataplane do **not** communicate over HTTP. +Config is exchanged exclusively through Redis: -1. The control plane runs **`dataplane_publisher.py`** — a publisher script that writes dataplane configuration (user config, backend definitions, etc.) into Redis. -2. The dataplane reads that config from Redis via the **`UserConfigStore`** abstraction (MessagePack-encoded `UserConfig`). +1. The control plane runs **`dataplane_publisher.py`** — a publisher script that writes external-dataplane configuration (user config, backend definitions, etc.) into Redis. +2. The external dataplane reads that config from Redis via the **`UserConfigStore`** abstraction (MessagePack-encoded `UserConfig`). This means: -- The dataplane is a **pure reader** of Redis config. It never writes back to the control-plane's Redis keys. -- The control plane is the **sole writer** of dataplane config; the dataplane has no direct dependency on the control-plane process at runtime. -- Config changes from the control plane are picked up by the dataplane through normal cache refresh / Redis reads — no restart or direct RPC required. +- The external dataplane is a **pure reader** of Redis config. It never writes back to the control plane's Redis keys. +- The control plane is the **sole writer** of external-dataplane config; the external dataplane has no direct dependency on the control-plane process at runtime. +- Config changes from the control plane are picked up by the external dataplane through normal cache refresh / Redis reads — no restart or direct RPC required. ### Per-component responsibilities | Component | Role | Persistence | | --- | --- | --- | | **nginx** | TLS termination, routing fan-out | — | -| **dataplane** (`contextforge-data-plane`) | MCP routing, auth enforcement, fan-out to backends | Redis (read-only for config) | -| **control-plane** (`IBM/mcp-context-forge`) | IAM, UI, metrics, legacy MCP clients, config publishing | Redis (write) + PostgreSQL (via pgbouncer) | +| **ContextForge external dataplane** (`contextforge-data-plane`) | MCP routing, auth enforcement, and backend calls; current session-backed paths are migration state, while the target is stateless | Redis (read-only for config) | +| **ContextForge built-in dataplane** (`IBM/mcp-context-forge`) | Python MCP request handling for old/new protocols and stateful/stateless clients | Python repository runtime state and stores | +| **ContextForge control plane** (`IBM/mcp-context-forge`) | IAM, UI, management APIs, metrics, and external-dataplane config publishing | Redis (write) + PostgreSQL (via pgbouncer) | | **redis** | Runtime config store, inter-component pub/sub channel | In-memory + persistence | | **postgres** (via pgbouncer) | Control-plane relational store | Durable | -| **fast_time_server** | High-resolution time source used by the dataplane | — | +| **fast_time_server** | High-resolution time source used by the ContextForge external dataplane | — | ## External dependencies and integration points -- **Redis** — runtime config store (MessagePack-encoded `UserConfig`). Populated by `dataplane_publisher.py` on the control plane; read by the dataplane via `UserConfigStore`. -- **Control plane** (`IBM/mcp-context-forge`) — owns legacy MCP client routes and publishes dataplane config via `dataplane_publisher.py`. Does not route through this dataplane at runtime. -- **fast_time_server** — high-resolution time source consumed by the dataplane. +- **Redis** — runtime config store (MessagePack-encoded `UserConfig`). Populated by `dataplane_publisher.py` on the control plane; read by the external dataplane via `UserConfigStore`. +- **ContextForge control plane** (`IBM/mcp-context-forge`) — owns management workflows and publishes external-dataplane config via `dataplane_publisher.py`. +- **ContextForge built-in dataplane** (`IBM/mcp-context-forge`) — owns the Python repository's MCP request paths, including old/new and stateful/stateless handling. Requests sent there do not route through the external dataplane. +- **fast_time_server** — high-resolution time source consumed by the ContextForge external dataplane. - **Tokio + Axum** — fixed async runtime and web framework. diff --git a/_context/wiki/security.md b/_context/wiki/security.md index bf0ac535..38a5b72b 100644 --- a/_context/wiki/security.md +++ b/_context/wiki/security.md @@ -6,7 +6,7 @@ | --- | --- | --- | | Downstream client | Untrusted. Every request must present a valid bearer JWT; session id alone grants nothing without matching principal state. | `claims_layer`, validators, and principal-scoped backend session keys. | | JWT verification material | Trust anchor. The RSA public key or HMAC secret in process config decides which tokens are accepted. | Process config; loaded at startup. | -| Redis | Control-plane trust boundary. Whoever can write Redis controls routing (`UserConfig`) and, when runtime plugins are enabled, which registered hooks execute (`ContextForgeGatewayRuntimePluginConfig`). | Redis TLS/mTLS connection modes; the dataplane never writes user config in production builds. | +| Redis | Control-plane trust boundary. Whoever can write Redis controls routing (`UserConfig`) and, when runtime plugins are enabled, which registered hooks execute (`ContextForgeGatewayRuntimePluginConfig`). | Redis TLS/mTLS connection modes; the external dataplane never writes user config in production builds. | | Backend MCP servers | Trusted per configured URL. The gateway forwards caller traffic to them and merges their responses. | `UserConfig` backend URLs plus the upstream connection mode. | | Plugins | Fully trusted code. Hooks run in-process and can read and mutate tool payloads. | Compiled-in factories only; Redis config activates registered factories, it cannot load new code. | @@ -14,14 +14,15 @@ | Plane | Current responsibility | | --- | --- | -| Control plane | Owns login/SSO, users, teams, IAM, API-token issuance and revocation, and legacy routes. `dataplane_publisher.py` writes visibility-filtered `UserConfig` snapshots to Redis by user email. | -| Data plane | Has no IAM or user database. It verifies modern MCP bearer JWTs locally, loads `UserConfig` by `sub`, and requires the requested virtual host to exist. No runtime control-plane call occurs. | +| ContextForge control plane | Owns login/SSO, users, teams, IAM, API-token issuance and revocation, and external-dataplane configuration publication. `dataplane_publisher.py` writes visibility-filtered `UserConfig` snapshots to Redis by user email. | +| ContextForge built-in dataplane | Owns the Python repository's MCP request routes, including old/new protocol and stateful/stateless behavior. | +| ContextForge external dataplane | Has no IAM or user database. It currently verifies modern MCP bearer JWTs locally, loads `UserConfig` by `sub`, and requires the requested virtual host to exist. No runtime control-plane call occurs. | -Request path: control-plane API token (`sub` = email) → Origin check → +External-dataplane request path: control-plane API token (`sub` = email) → Origin check → `claims_layer` → Redis config lookup → virtual-host check → RMCP Host check → MCP routing. Browser/login session tokens are management-plane credentials, not the -dataplane contract. +external-dataplane contract. - JWT validation accepts `RS256/384/512` or `HS256/384/512` and requires a valid signature, `iss=mcpgateway`, `aud=mcpgateway-api`, and `exp`. `jti` and `user` @@ -35,8 +36,8 @@ dataplane contract. target requires principal- and isolation-bound snapshots, per-request scope and compiled-RBAC enforcement, and default denial for missing or unauthorized entries. See [Target Authorization Invariants](mcp-capability-allocation.md#target-authorization-invariants). -- Dataplane requests do not consult the control-plane token blocklist. Revoked - tokens pass JWT validation until `exp` or signing-key rotation/restart. +- External-dataplane requests do not consult the control-plane token blocklist. + Revoked tokens pass JWT validation until `exp` or signing-key rotation/restart. Removing a subject's config eventually blocks all its tokens after publisher and cache expiry. diff --git a/_context/wiki/testing.md b/_context/wiki/testing.md index 3669e7df..607b9e63 100644 --- a/_context/wiki/testing.md +++ b/_context/wiki/testing.md @@ -4,7 +4,7 @@ - **Workspace checks** — code compiles and unit behavior holds. - **In-repo integration tests** — MCP routing against mock backends. -- **`cf-integration` harness** — full control-plane-to-dataplane path end to end. +- **`cf-integration` harness** — full control-plane publication and external-dataplane request path end to end. - **Load and benchmark** — see [Performance](performance.md). ## Workspace Validation @@ -19,7 +19,7 @@ cargo nextest run --locked --workspace Use `cargo test` when nextest is unavailable. For wiki changes, also run `mdbook build _context/wiki` and `mdbook test _context/wiki`. -Protocol-sensitive tests and fixtures must cover MCP `2026-07-28` and `2025-11-25` in all four incoming-client/selected-backend combinations. The same-version paths are supported directly; the two cross-version paths are best effort and tests must cover both successful adaptation and explicit failure for semantics that cannot be translated without state. Every case must prove request independence: no required `Mcp-Session-Id`, session affinity, or retained backend transport. Keep `2026-07-28` coverage for `server/discover` and required per-request client metadata, and retain `initialize` coverage as a stateless compatibility request. SSE remains outside the dataplane contract. +Protocol-sensitive tests and fixtures must cover MCP `2026-07-28` and `2025-11-25` in all four incoming-client/selected-backend combinations. The same-version paths are supported directly; the two cross-version paths are best effort and tests must cover both successful adaptation and explicit failure for semantics that cannot be translated without state. Every case must prove request independence: no required `Mcp-Session-Id`, session affinity, or retained backend transport. Keep `2026-07-28` coverage for `server/discover` and required per-request client metadata, and retain `initialize` coverage as a stateless compatibility request. SSE remains outside the external-dataplane contract. ## In-Repo Integration Tests @@ -38,17 +38,17 @@ These run in `cargo nextest run` with no Docker dependencies. `.github/workflows/mcp_conformance.yml` runs the pinned official conformance suite `0.2.0-alpha.11` with `--requirements 2026-07-28`. Its small live path is -official runner → nginx → checked-out dataplane → fixture proxy → official -fixture, with the published `latest` control plane registering and publishing -the fixture through Redis. The backend-only proxy rewrites `Host` to +official runner → nginx → checked-out external dataplane → fixture proxy → +official fixture, with the published `latest` Python image's control plane +registering and publishing the fixture through Redis. The backend-only proxy rewrites `Host` to `localhost:3000`, which the official fixture's DNS-rebinding protection -requires, while leaving dataplane header protections unchanged. The control -plane uses ephemeral SQLite, so PostgreSQL is unnecessary. The harness lives +requires, while leaving external-dataplane header protections unchanged. The +control plane uses ephemeral SQLite, so PostgreSQL is unnecessary. The harness lives in `tests/conformance/`. Because this conformance CLI cannot set a bearer header, nginx adds an ephemeral control-plane token when one is absent; there is no auth proxy or -repository-owned JavaScript. A route probe prevents control-plane fallback. +repository-owned JavaScript. A route probe prevents built-in-dataplane fallback. Counts and the official fixture log appear directly in the Actions log, and `expected-failures.yml` guards the current baseline. The job does not retain a separate conformance artifact. `upstream-fixture-failures.yml` records the @@ -59,7 +59,14 @@ entries, and writes the same comparison to the job summary. ## Full-Stack Integration Harness -[`cf-integration`](https://github.com/contextforge-org/contextforge-dev-tools) wires the external ContextForge control plane to this dataplane the way production intends: the stock upstream Compose stack, plus exactly two intentional differences — nginx routes only `/servers/{virtual_host_id}/mcp` to the dataplane (as `/contextforge-rs/servers/{virtual_host_id}/mcp`), and the control plane runs with `DATAPLANE_PUBLISHER=true` so virtual server configs reach the dataplane through Redis. +[`cf-integration`](https://github.com/contextforge-org/contextforge-dev-tools) +wires the ContextForge control plane, built-in dataplane, and this ContextForge +external dataplane together. The stock Python Compose stack contains both the +control plane and built-in dataplane. The harness adds two intentional +differences: nginx routes the selected `/servers/{virtual_host_id}/mcp` path to +the external dataplane as `/contextforge-rs/servers/{virtual_host_id}/mcp`, and +the control plane runs with `DATAPLANE_PUBLISHER=true` so virtual-server config +reaches the external dataplane through Redis. ### Quick Start @@ -67,7 +74,12 @@ entries, and writes the same comparison to the job summary. scripts/cf-integration.sh up ``` -This checks out the control plane under `.integration/mcp-context-forge`, pulls the published dataplane image, and starts the combined stack plus a local MCP counter backend. The admin UI is at `http://localhost:8080/admin` (`admin@example.com` / `changeme`). A Fast Time backend is auto-registered as a fixed virtual server, so the commands below work with no manual UI step. +This checks out the Python control-plane/built-in-dataplane repository under +`.integration/mcp-context-forge`, pulls the published external-dataplane image, +and starts the combined stack plus a local MCP counter backend. The admin UI is +at `http://localhost:8080/admin` (`admin@example.com` / `changeme`). A Fast Time +backend is auto-registered as a fixed virtual server, so the commands below +work with no manual UI step. ### Route Probe @@ -75,7 +87,7 @@ This checks out the control plane under `.integration/mcp-context-forge`, pulls scripts/cf-integration.sh probe ``` -Verifies the public nginx-to-dataplane route end to end: a 401 negative check, `initialize`, session reuse, `tools/list`, and `tools/call`. +Verifies the public nginx-to-external-dataplane route end to end: a 401 negative check, `initialize`, session reuse, `tools/list`, and `tools/call`. ### Full Test Runs @@ -86,11 +98,18 @@ Verifies the public nginx-to-dataplane route end to end: a 401 negative check, ` | `scripts/cf-integration.sh test-all-up` | Start or update the stack, then `test-all` without the load lane. | | `scripts/cf-integration.sh test-all-up-load` | Start or update the stack, then `test-all` with the load lane. | -Individual lanes: `live-mcp`, `live-rbac`, `live-protocol`, and `live-all`. `live-mcp` is the green lane: the full MCP protocol end-to-end suite passes against this harness. Remaining failures in other lanes measure known dataplane feature gaps; the harness `reports/` directory keeps the current classification. +Individual lanes: `live-mcp`, `live-rbac`, `live-protocol`, and `live-all`. +`live-mcp` is the green lane: the full MCP protocol end-to-end suite passes +against this harness. Remaining failures in other lanes measure known +external-dataplane feature gaps; the harness `reports/` directory keeps the +current classification. -### Control-Plane Baseline +### Built-In-Dataplane Baseline -To separate dataplane regressions from upstream behavior, the harness can run the stock control-plane-only stack (no dataplane, no nginx split, no publisher): +To separate external-dataplane regressions from Python behavior, the harness +can run the stock `IBM/mcp-context-forge` stack. MCP traffic then uses the +ContextForge built-in dataplane; the external dataplane, nginx split, and +publisher are absent: ```bash scripts/cf-integration.sh down # frees the shared host ports @@ -103,7 +122,7 @@ Individual steps: `controlplane-up`, `controlplane-live-core`, `controlplane-liv | Variable | Purpose | | --- | --- | -| `CF_DATAPLANE_IMAGE` / `CF_DATAPLANE_VERSION` | Which published dataplane image the stack runs. | -| `CF_CONTROLPLANE_IMAGE` / `CF_CONTROLPLANE_REF` | Which control-plane image and git ref to use. | +| `CF_DATAPLANE_IMAGE` / `CF_DATAPLANE_VERSION` | Which published external-dataplane image the stack runs. | +| `CF_CONTROLPLANE_IMAGE` / `CF_CONTROLPLANE_REF` | Which `IBM/mcp-context-forge` Python image and git ref to use for the control plane and built-in dataplane. | | `NGINX_PORT` | Public front-door port (default `8080`). | | `CF_TEST_LOG_DIR` | Where `test-all` writes timestamped logs. | From ba67695033411f2825efc39aab235ebd1e5d9224 Mon Sep 17 00:00:00 2001 From: lucarlig Date: Fri, 21 Aug 2026 10:24:14 +0100 Subject: [PATCH 3/3] fix: resolve unused async handler lints Signed-off-by: lucarlig --- .../src/handle.rs | 10 ++++--- .../tests/support/paginating_mock.rs | 10 ++++--- .../tests/support/plugin.rs | 26 +++++++++++-------- .../tests/support/plugin_gateway.rs | 26 +++++++++++-------- .../plugins/cpex-secrets-detection/src/lib.rs | 10 +++---- 5 files changed, 47 insertions(+), 35 deletions(-) diff --git a/crates/contextforge-data-plane-cpex/src/handle.rs b/crates/contextforge-data-plane-cpex/src/handle.rs index eae5f182..b80b0844 100644 --- a/crates/contextforge-data-plane-cpex/src/handle.rs +++ b/crates/contextforge-data-plane-cpex/src/handle.rs @@ -510,12 +510,12 @@ mod tests { } impl HookHandler for TestPlugin { - async fn handle( + fn handle( &self, payload: &MessagePayload, _extensions: &Extensions, ctx: &mut PluginContext, - ) -> PluginResult { + ) -> impl std::future::Future> { let is_post = payload.message.role == Role::Tool; let mut observations = self.observations.lock().expect("observations lock poisoned"); if is_post { @@ -529,7 +529,7 @@ mod tests { } drop(observations); - if is_post { + let result = if is_post { match self.post_behavior { PostBehavior::Allow => PluginResult::allow(), PostBehavior::Rewrite => PluginResult::modify_payload(payload.clone()), @@ -604,7 +604,9 @@ mod tests { PluginResult::allow() }, } - } + }; + + std::future::ready(result) } } diff --git a/crates/contextforge-data-plane-lib/tests/support/paginating_mock.rs b/crates/contextforge-data-plane-lib/tests/support/paginating_mock.rs index 0e8b2066..7981ed93 100755 --- a/crates/contextforge-data-plane-lib/tests/support/paginating_mock.rs +++ b/crates/contextforge-data-plane-lib/tests/support/paginating_mock.rs @@ -36,17 +36,19 @@ impl ServerHandler for PaginatingServer { .with_protocol_version(ProtocolVersion::V_2024_11_05) } - async fn list_tools( + fn list_tools( &self, request: Option, _: RequestContext, - ) -> Result { - if request.as_ref().and_then(|r| r.cursor.as_deref()) == Some(PAGE2_CURSOR) { + ) -> impl std::future::Future> { + let result = if request.as_ref().and_then(|r| r.cursor.as_deref()) == Some(PAGE2_CURSOR) { Ok(ListToolsResult::with_all_items(Self::page2_tools())) } else { let mut result = ListToolsResult::with_all_items(Self::page1_tools()); result.next_cursor = Some(PAGE2_CURSOR.to_owned()); Ok(result) - } + }; + + std::future::ready(result) } } diff --git a/crates/contextforge-data-plane-lib/tests/support/plugin.rs b/crates/contextforge-data-plane-lib/tests/support/plugin.rs index 788bfe9f..383409af 100644 --- a/crates/contextforge-data-plane-lib/tests/support/plugin.rs +++ b/crates/contextforge-data-plane-lib/tests/support/plugin.rs @@ -154,12 +154,12 @@ impl Plugin for TestPlugin { } impl HookHandler for TestPlugin { - async fn handle( + fn handle( &self, payload: &MessagePayload, _extensions: &Extensions, ctx: &mut PluginContext, - ) -> PluginResult { + ) -> impl std::future::Future> { let is_post = payload.message.role == Role::Tool; let mut observations = self.observations.lock().expect("observations lock poisoned"); if is_post { @@ -180,7 +180,7 @@ impl HookHandler for TestPlugin { } drop(observations); - if is_post { + let result = if is_post { match self.post_behavior { PostBehavior::Allow => PluginResult::allow(), PostBehavior::Rewrite => { @@ -190,7 +190,7 @@ impl HookHandler for TestPlugin { modified.message.content.iter_mut().find(|part| matches!(part, ContentPart::ToolResult { .. })) { if !is_tool_result_content(&content.content) { - return PluginResult::allow(); + return std::future::ready(PluginResult::allow()); } content.content = serde_json::to_value(CallToolResult::success(vec![ContentBlock::text( format!("post:{result_text}"), @@ -205,7 +205,7 @@ impl HookHandler for TestPlugin { modified.message.content.iter_mut().find(|part| matches!(part, ContentPart::ToolResult { .. })) { if !is_tool_result_content(&content.content) { - return PluginResult::allow(); + return std::future::ready(PluginResult::allow()); } content.content = json!("raw-post"); } @@ -220,7 +220,7 @@ impl HookHandler for TestPlugin { { progress.message = progress.message.map(|message| format!("plugin:{message}")); content.content = serde_json::to_value(progress).expect("progress serializes"); - return PluginResult::modify_payload(modified); + return std::future::ready(PluginResult::modify_payload(modified)); } PluginResult::allow() }, @@ -279,7 +279,9 @@ impl HookHandler for TestPlugin { PluginResult::allow() }, } - } + }; + + std::future::ready(result) } } @@ -492,17 +494,19 @@ impl Plugin for PromptTestPlugin { } impl HookHandler for PromptTestPlugin { - async fn handle( + fn handle( &self, payload: &MessagePayload, _extensions: &Extensions, ctx: &mut PluginContext, - ) -> PluginResult { - if payload.message.get_prompt_results().is_empty() { + ) -> impl std::future::Future> { + let result = if payload.message.get_prompt_results().is_empty() { self.handle_pre(payload, ctx) } else { self.handle_post(payload, ctx) - } + }; + + std::future::ready(result) } } diff --git a/crates/contextforge-data-plane-lib/tests/support/plugin_gateway.rs b/crates/contextforge-data-plane-lib/tests/support/plugin_gateway.rs index a118e25a..89cbd4a0 100644 --- a/crates/contextforge-data-plane-lib/tests/support/plugin_gateway.rs +++ b/crates/contextforge-data-plane-lib/tests/support/plugin_gateway.rs @@ -59,20 +59,22 @@ struct TestBackend { } impl ServerHandler for TestBackend { - async fn initialize( + fn initialize( &self, _request: InitializeRequestParams, _cx: RequestContext, - ) -> Result { - Ok(InitializeResult::new(ServerCapabilities::builder().enable_tools().enable_prompts().build()) - .with_server_info(Implementation::new("test-backend", "0.1.0"))) + ) -> impl std::future::Future> { + std::future::ready(Ok(InitializeResult::new( + ServerCapabilities::builder().enable_tools().enable_prompts().build(), + ) + .with_server_info(Implementation::new("test-backend", "0.1.0")))) } - async fn get_prompt( + fn get_prompt( &self, request: GetPromptRequestParams, _cx: RequestContext, - ) -> Result { + ) -> impl std::future::Future> { self.state .prompts .lock() @@ -86,8 +88,8 @@ impl ServerHandler for TestBackend { .and_then(|arguments| arguments.get("topic")) .and_then(Value::as_str) .unwrap_or("nothing"); - if request.name == "review_bundle" { - return Ok(GetPromptResult::new(vec![ + let result = if request.name == "review_bundle" { + Ok(GetPromptResult::new(vec![ PromptMessage::new_text(Role::User, format!("review of {topic}")), PromptMessage::new( Role::User, @@ -95,10 +97,12 @@ impl ServerHandler for TestBackend { ), PromptMessage::new(Role::Assistant, ContentBlock::image(BACKEND_PROMPT_IMAGE, "image/png")), ]) - .into()); - } + .into()) + } else { + Ok(GetPromptResult::new(vec![PromptMessage::new_text(Role::User, format!("review of {topic}"))]).into()) + }; - Ok(GetPromptResult::new(vec![PromptMessage::new_text(Role::User, format!("review of {topic}"))]).into()) + std::future::ready(result) } async fn call_tool( diff --git a/crates/plugins/cpex-secrets-detection/src/lib.rs b/crates/plugins/cpex-secrets-detection/src/lib.rs index 09f0e26e..ba74c68d 100644 --- a/crates/plugins/cpex-secrets-detection/src/lib.rs +++ b/crates/plugins/cpex-secrets-detection/src/lib.rs @@ -80,12 +80,12 @@ impl Plugin for StageHandler { } impl HookHandler for StageHandler { - async fn handle( + fn handle( &self, payload: &MessagePayload, extensions: &Extensions, _ctx: &mut PluginContext, - ) -> PluginResult { + ) -> impl std::future::Future> { let scan = self.scan_payload(payload); if self.core.should_block(scan.count) { @@ -93,20 +93,20 @@ impl HookHandler for StageHandler { let mut result = PluginResult::deny(violation); result.modified_payload = scan.modified_payload.or_else(|| Some(payload.clone())); attach_metrics(&mut result, extensions, scan.count, &scan.findings, DetectionOutcome::Blocked); - return result; + return std::future::ready(result); } if let Some(modified_payload) = scan.modified_payload { let mut result = PluginResult::modify_payload(modified_payload); attach_metrics(&mut result, extensions, scan.count, &scan.findings, DetectionOutcome::Masked); - return result; + return std::future::ready(result); } let mut result = PluginResult::allow(); if scan.count > 0 { attach_metrics(&mut result, extensions, scan.count, &scan.findings, DetectionOutcome::None); } - result + std::future::ready(result) } }