From 349ea6c837376f79f922325e121499dfb683243c Mon Sep 17 00:00:00 2001 From: ava kim Date: Fri, 28 Aug 2026 12:48:58 +1000 Subject: [PATCH 1/8] docs: reposition UpdAPI as API freshness observatory --- README.md | 572 ++++++++++++++++++++++++------------------------------ 1 file changed, 257 insertions(+), 315 deletions(-) diff --git a/README.md b/README.md index bb0f644..a9efc51 100644 --- a/README.md +++ b/README.md @@ -1,442 +1,384 @@ +# UpdAPI -# UpdAPI πŸ”§ +> **A living benchmark of how quickly frontier coding agents adapt to real API changes.** -### "Update your knowledge base with the latest **API** resources" -### A free, lightweight tool to streamline the discovery of API documentation, policies, and community resources and enhancing LLMs with accurate, relevant context +UpdAPI tracks API evolution over time and turns verified, timestamped changes into reproducible evaluation cases for frontier coding models and agents. - ![updAPI](https://github.com/user-attachments/assets/7a67269e-5ce0-480d-95d8-cd7dfe79ca87) +The long-term goal is a public **API Freshness Observatory**: an independently reproducible view of how well AI coding systems keep pace with the software ecosystem they are increasingly asked to build. -[![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) -[![Contributions Welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](#contributing) -[![Build Status](https://img.shields.io/badge/status-under_construction-orange.svg)](#status) -[![Open Issues](https://img.shields.io/github/issues/in-c0/updapi)](https://github.com/in-c0/updapi/issues) +> **Direction reset β€” August 2026.** The original UpdAPI index is no longer the product. It is becoming part of the measurement infrastructure. --- -> Like the project? Please give it a Star so it can reach more people >>>>> [![Star on GitHub](https://img.shields.io/badge/⭐-Star_on_GitHub-blue?style=flat)](https://github.com/UpdAPI/updAPI/stargazers) +## Why UpdAPI changed direction -> ⚠️ **Under Construction** -> This project is in the early stages of development and may not function as intended yet. Contributions, feedback, and ideas are highly welcome! +UpdAPI was originally created to address a common failure mode in LLM-assisted programming: models frequently relied on outdated API knowledge, producing deprecated methods, invalid endpoints, and code based on old documentation. -## πŸ“‹ Links to Public API DOCS +That environment has changed substantially. -`api-docs-urls.csv` contains a **centralized collection of popular APIs** with **links to their official documentation and associated policies**. It includes tools to scrape, preprocess, and update the dataset for better usability and retrieval. +Modern coding systems are no longer just static language models answering from training data. Frontier agents can search the web, inspect repositories and installed packages, retrieve official documentation, execute code, observe failures, and iteratively repair their implementations at inference time. -api-docs-urls.csv: +A useful signal of this shift came from Cursor in August 2026, when it removed its `@Docs` documentation-indexing feature. Cursor explained that agents had become good enough at locating and reading documentation themselves that maintaining a separate documentation index was no longer necessary. -| API Name | Official Documentation URL | Privacy Policy URL | Terms of Service URL | Rate Limiting Policy URL | Changelog/Release Notes URL | Security Policy URL | Developer Community/Forum URL | -|-----------------------|-----------------------------------------------------|------------------------------------|-------------------------------|----------------------------------------|------------------------------------------|-----------------------------------|------------------------------------------| -| OpenAI API | [Documentation](https://platform.openai.com/docs) | [Privacy](https://openai.com/privacy) | [Terms](https://openai.com/terms) | [Rate Limits](https://platform.openai.com/docs/guides/rate-limits) | [Changelog](https://platform.openai.com/docs/release-notes) | [Security](https://openai.com/security) | [Community](https://community.openai.com/) | -... +Source: [Cursor staff explanation, 5 August 2026](https://forum.cursor.com/t/cursor-3-14-2-sunsetting-docs/167423/7) -> ⚠️ **The URLs are auto-generated and require manual verification** -> They are now re-verified weekly β€” see [Dataset freshness](#-dataset-freshness) below. +This changes the interesting question. +The original question was: -## πŸ”„ Dataset freshness +> **How do we give an LLM current API documentation?** -A table of URLs is only worth what its URLs are worth, and URLs rot. Documentation -sites reorganise, whole doc platforms migrate (`api.slack.com` β†’ `docs.slack.dev`, -`stripe.com/docs` β†’ `docs.stripe.com`), and pages are retired. So the dataset -re-verifies itself. +UpdAPI now asks: -```bash -npm run check-links # probe every URL in every column, write reports -npm run check-links:fix # also rewrite rows whose URL now redirects elsewhere -``` +> **Do frontier coding agents still suffer from stale API knowledge β€” and if so, how quickly do they adapt when APIs change?** -[`.github/workflows/link-freshness.yml`](.github/workflows/link-freshness.yml) -runs this every Monday and opens a pull request with whatever it can repair. +That is an empirical question, and it should be measured continuously rather than assumed. -**What it does and does not repair.** A URL that now redirects somewhere else has -a known-good replacement β€” the place it redirects to β€” so those rows are rewritten -automatically. A dead URL does not; inventing a plausible replacement would put a -fabrication in the dataset, so 404s are reported for a human to resolve rather -than guessed at. `403` and `429` are recorded as `blocked`, never as dead: being -refused by a bot-detecting CDN says nothing about whether the page exists, and -treating a Cloudflare challenge as rot would silently delete good rows. +--- -### Outputs +## What UpdAPI is becoming -| file | what it holds | -|---|---| -| `datasets/link-health.json` | full snapshot β€” per-column tallies, every non-OK finding, `/llms.txt` availability per host | -| `datasets/link-health-history.csv` | one row per column per run, appended forever | +UpdAPI has three connected outputs. -The history file is the point. A single snapshot says the dataset is stale today; -the series says how fast documentation rots, which is a question nobody seems to -have measured. +### 1. API Evolution Dataset -### Two caveats on reading the numbers +A timestamped, provenance-rich record of real API and SDK changes: -**Per-column statistics cover structurally complete rows only.** Most rows in this -dataset omit fields rather than leaving them empty, which shifts every later value -one column to the left β€” so a privacy-policy URL can end up sitting in the -documentation column. Counting those rows would make "documentation URLs are N% -healthy" partly a statement about privacy policies. The checker reports the -alignment problem separately under `alignment` rather than folding it into rot. +- newly introduced APIs +- deprecated APIs +- removed APIs +- renamed methods and endpoints +- signature and schema changes +- SDK and major-version migrations +- behavioural changes +- documentation changes +- rate-limit, policy, and other developer-relevant changes -**`ok` is a strict test.** It means the stored URL returned 200 at exactly that -address. A URL that resolves only after a redirect is counted as `moved`, not -`ok`, because for a dataset whose product *is* the URL, pointing at a redirect -stub is a defect even when a human following it lands somewhere fine. +Each accepted change event should be traceable to authoritative evidence and, where practical, include executable before/after fixtures. ---- +### 2. Frontier API Freshness Benchmark -## 🧭 Column alignment +Verified API-change events become evaluation cases for contemporary coding systems. -Rot was not this dataset's worst defect. `utils/verify_and_clean_csv.py` used to -**drop** a URL it judged broken rather than blanking it: +The benchmark evaluates the workflow developers actually use: -```python -for url in urls: - if is_broken_url(url): - print(f"URL for {api_name} is invalid: {url}") - else: - cleaned_row.append(url) # broken URLs simply vanish +```text +task + β†’ agent + β†’ search / retrieval / repository inspection + β†’ implementation + β†’ execution + β†’ diagnosis / recovery + β†’ verified result ``` -Dropping shortens the row, so every later value slides one column left and gets -filed under the wrong heading. `Microsoft Graph People API` ended up serving its -privacy statement as its documentation URL. Those rows were not stale β€” they were -**mislabelled**, which is worse: a consumer gets a confident wrong answer with -nothing to indicate a problem. - -It compounded, too, because `is_broken_url` returned `True` on *any* request -exception with a five-second timeout. A slow host, a TLS hiccup or a 403 from a -bot-detecting CDN permanently deleted a good URL β€” and shifted everything after it. +The primary object of study is therefore the **coding agent as a system**, not merely a naked model completion. -Both bugs are fixed: broken URLs are blanked in place, rows are padded to the full -width, and only a confirmed `404`/`410` counts as broken. +Controlled variants can still isolate the contribution of individual capabilities, for example: -### Repairing what was already damaged +- model without external retrieval +- agent with its normal tools +- agent with web search disabled +- agent with authoritative current documentation supplied +- agent with a retrieval layer such as MCP/RAG +- agent with execution and repair enabled -```bash -npm run realign # dry run β€” report what would move -npm run realign:apply # write the repaired CSV -npm run verify-alignment # prove no shifted rows remain -``` +This lets us measure whether extra documentation infrastructure still provides meaningful value rather than presuming that it does. -Dropping preserves order, so the surviving URLs are still in their original -relative sequence, just compressed leftward. Repair is therefore an -**order-preserving assignment** of the observed URLs back onto the column slots, -solved exactly with dynamic programming and scored on what each URL looks like. -A URL that contradicts every column still free is left blank and recorded in -`datasets/realignment.json` rather than filed somewhere wrong β€” the entire point -being that a mislabelled URL is worse than a missing one. +### 3. Public API Freshness Observatory -The first pass moved **1,089 URLs across 369 rows**. +Results should ultimately be published as a continuously updated, drill-downable benchmark rather than buried in repository artifacts. -### On trusting the verification +A public view could compare systems on metrics such as: -`verify-alignment` runs a **negative control before it reports anything**: it -deliberately re-breaks a copy of the data the way the old cleaner did and requires -the detector to catch it. A detector that always returns zero also reports zero, -so a clean result only means something once the control has fired. +| Metric | Question | +|---|---| +| **Verified task success** | Does the final implementation actually work against the target API/version? | +| **Stale API error rate** | How often does the system use an obsolete interface? | +| **API knowledge lag** | How long after a real API change until the system reliably succeeds? | +| **Recovery rate** | When initially wrong, can the agent diagnose and repair the failure? | +| **Retrieval lift** | How much do web/docs/RAG tools improve verified success? | +| **Reliability** | Does the system succeed repeatedly, not merely once? | +| **Cost / latency** | What does successful recovery cost in time, tokens, and tool calls? | -``` -negative control (data deliberately re-shifted): 147 rows detected -api-docs-urls.csv: 0 rows detected -``` +The headline should remain grounded in verified execution, with secondary metrics explaining *why* systems differ. --- -## πŸ”Œ MCP server +## The key idea: temporal evaluation -The index is most useful to a coding agent, so it is exposed over the -[Model Context Protocol](https://modelcontextprotocol.io). Any MCP client β€” -Claude Code, Claude Desktop, Cursor, VS Code, Windsurf, Zed β€” can query it. - -```bash -npm install -npm run mcp # stdio server -``` +Static API benchmarks decay quickly. Once their cases are old, they become increasingly likely to appear in training data, examples, benchmark-specific optimizations, or model memory. -Client configuration: +UpdAPI instead aims to continuously capture **new, timestamped API changes** and turn them into temporal holdouts. -```json -{ - "mcpServers": { - "updapi": { - "command": "node", - "args": ["/absolute/path/to/updAPI/mcp/server.mjs"] - } - } -} -``` +A simplified lifecycle: -### Tools +```text +OBSERVE + official changelogs, release notes, SDK releases, specs, docs -| tool | what it answers | -|---|---| -| `search_apis` | "which entries match *stripe*?" β€” name search, returns the documentation URL | -| `get_api_resources` | "give me everything indexed for this API" β€” docs, privacy, terms, rate limits, release notes, security, community | -| `index_health` | "how much should I trust the answers above?" β€” freshness and per-column health | +VERIFY + confirm the change against authoritative sources and/or executable behaviour -**Every URL is returned with its freshness.** Each resource carries when it was -last verified and what the verifier saw (`ok`, `moved` with the redirect target, -`dead`, `blocked`). Handing a model a link without saying how old the claim is -invites it to trust a 404 β€” worse than returning nothing, because a tool result -reads as authoritative. When no `link-health.json` is present the server reports -`"verified": "unknown"` rather than implying the data is current. +FREEZE + store provenance, timestamps, before/after states, fixtures, and expected outcome -The `Rate Limiting Policy`, `Terms of Service` and `Security Policy` columns are -the ones worth reaching for. General web search handles "where are the Stripe API -docs" perfectly well; it does much worse on policy pages, which rank below -marketing pages and are exactly where a model will confidently invent a number. +EVALUATE + run the same case across coding systems under defined tool conditions -To verify the server end-to-end after a change: +REPEAT + rerun over time to measure adoption and reliability -```bash -node tools/verify-mcp.mjs +PUBLISH + leaderboard + per-case evidence + methodology + reproducible artifacts ``` ---- +This makes **time-to-adoption** measurable. -## πŸ›  Adding More APIs to the Dataset +Example: -### **Option 1: Manually Add to `api-docs-urls.csv`** -You can manually add new entries to `api-docs-urls.csv` with the following format: -```csv -API_Name,Official_Documentation_URL,Privacy_Policy_URL,Terms_of_Service_URL,Rate_Limiting_Policy_URL,Changelog_Release_Notes_URL,Security_Policy_URL,Developer_Community_Forum_URL -Example API,https://example.com/docs,https://example.com/privacy,https://example.com/tos,https://example.com/rate-limits,https://example.com/changelog,https://example.com/security,https://example.com/community -``` +```text +API change published: 2026-08-01 -### **Option 2: Combine Multiple CSV Files** -If you have additional entries in separate CSV files, use the provided Python utility script to merge them into the main dataset. +2026-08-01 Agent A: fail Agent B: fail Agent C: fail +2026-08-03 Agent A: pass Agent B: fail Agent C: fail +2026-08-07 Agent A: pass Agent B: pass Agent C: fail +2026-08-12 Agent A: pass Agent B: pass Agent C: pass +``` -#### Combine CSV Files -1. Ensure you have Python installed. -2. Run the script: - ```bash - python utils/combine_csv.py new_entries.csv api-docs-urls.csv combined_dataset.csv - ``` -3. Replace the existing `api-docs-urls.csv` with the new `combined_dataset.csv`. +Rather than asking whether a model has memorised an old benchmark, UpdAPI can observe how quickly coding systems absorb or recover from changes occurring in the live software ecosystem. --- -## What can we do with the API URLs? - -**Use Case 1:** - You can use the scrapers (fast-scraper.js or accurate-scraper.js) to extract content from API docs and enhance your LLM to provide specific and accurate answers about APIs +## Benchmark principles -**Workflow Example:** -1. Retrieve relevant snippets with a custom script / Query the vector database for a user question -2. Generate Answers with an LLM: Pass the retrieved snippets as context to the LLM (e.g., GPT-4 or LLaMA-2) +UpdAPI should optimize for measurement quality before leaderboard scale. - ```python - from transformers import AutoModelForCausalLM, AutoTokenizer - from faiss import read_index +### Verify outcomes, not prose - # Load vector index - index = read_index('vector_index.faiss') +Whenever practical, a benchmark case should end in an executable or otherwise deterministic assertion. "The answer sounds current" is not sufficient evidence. - # Query embeddings - user_query = "What are the rate limits for the OpenAI API?" - query_embedding = model.encode(user_query) - _, indices = index.search(np.array([query_embedding]), k=5) +### Preserve provenance - # Retrieve relevant chunks - context = " ".join([documents[i] for i in indices[0]]) +Every change event should retain: - # Use an LLM to answer - model = AutoModelForCausalLM.from_pretrained('gpt-4') - tokenizer = AutoTokenizer.from_pretrained('gpt-4') +- canonical API / package identity +- before and after versions +- first observed / published timestamps +- authoritative source URLs +- archived evidence where licensing permits +- change classification +- expected modern usage +- verification method +- benchmark-case version - prompt = f"Context: {context}\nQuestion: {user_query}\nAnswer:" - inputs = tokenizer(prompt, return_tensors='pt') - outputs = model.generate(**inputs, max_new_tokens=200) - print(tokenizer.decode(outputs[0], skip_special_tokens=True)) - ``` +### Separate discovery from evaluation -**Use Case 2:** -Maintain offline copies of API documentation for scenarios where internet access is unavailable or restricted. Offline access ensures reliability and speed when querying API documentation. +The system that discovers a change must not silently decide that its interpretation is ground truth. High-value cases need independent verification or deterministic evidence. -**How?** -- Use the scrapers to generate offline copies of the documentation in JSON, HTML, or Markdown formats. -- Serve these copies locally or integrate them into a lightweight desktop or web application. +### Measure agents as deployed +A coding agent's web search, repository inspection, terminal, package manager, and repair loop are part of the product developers use. Default-agent evaluation should preserve those capabilities and record them explicitly. -**Use Case 3:** -API documentation changes frequently, and outdated information can lead to bugs or misconfigurations. Automating change detection ensures your knowledge base remains up-to-date. +### Include controlled ablations -**How?** -- Compare the current version of a page with its previously saved version. -- Use hashing (e.g., MD5) or diff-checking tools to detect changes in content. +Agent-native evaluation should be complemented by controlled conditions when they answer a useful causal question, especially whether current documentation, web search, or RAG materially improves outcomes. ---- +### Treat reliability as first-class -## πŸš€ How to Use the Scrapers +One lucky pass should not imply robust freshness. Repeated trials should be used where system nondeterminism can materially affect conclusions. +### Keep benchmark versions immutable +Published results must identify the benchmark version, case set, runner configuration, model/agent version, tool permissions, date, and scoring rules. Methodology changes create a new benchmark version rather than silently rewriting history. -### Check Python Version -**Recommended Python Versions**: Python >=3.7 and <3.10 +### Defend against contamination - 1. Check your Python version: - ```bash - python --version - ``` - 2. If your Python version is incompatible, you can: - - Install a compatible version (e.g., Python 3.9). - - Use a virtual environment: - ```bash - python3.9 -m venv venv - source venv/bin/activate # Or venv\Scripts\activate on Windows - pip install -r requirements.txt - ``` - 3. Alternatively, use Conda to install PyTorch and its dependencies: - ```bash - conda install pytorch torchvision torchaudio pytorch-cuda=11.8 -c pytorch -c nvidia - ``` +Recent change events are the strongest holdouts. Some evaluation cases may need an embargoed/private evaluation window before their full fixtures are released publicly. Public reproducibility and temporal integrity should both be preserved by releasing cases after their live evaluation window where appropriate. +--- +## Initial scope -We provide two scraping tools to suit different needs: -- **`fast-scraper.js`**: A lightweight Cheerio-based scraper for fast retrieval of static content. -- **`accurate-scraper.js`**: A Playwright-based scraper for handling JavaScript-loaded pages and more dynamic content. +Start small and high-confidence rather than attempting to monitor every API on the internet. +The first corpus should focus on approximately 10–20 fast-moving, developer-relevant ecosystems with authoritative version/changelog evidence and practical executable verification. Candidate families include: -### **1. `fast-scraper.js` (Cheerio-Based)** -- **Purpose**: For quickly scraping static API documentation pages. -- **Strengths**: - - Lightweight and fast. - - Suitable for pages without JavaScript content. -- **Limitations**: - - Does not handle JavaScript-loaded content. +- major AI model/provider SDKs +- AI application SDKs +- major web frameworks +- cloud/edge developer platforms +- payments +- databases and hosted data platforms +- high-usage npm/PyPI libraries with meaningful API evolution -#### Run the Script -1. Install dependencies: - ```bash - npm install - ``` -2. Run the script: - ```bash - node fast-scraper.js - ``` -3. Results will be saved in `scraped_data_fast.json`. +Selection should be based on measurable change frequency, developer relevance, verifiability, and benchmark diversityβ€”not brand prestige alone. --- -### **2. `accurate-scraper.js` (Playwright-Based)** -- **Purpose**: For scraping API documentation pages that rely on JavaScript for rendering. -- **Strengths**: - - Handles dynamic content and JavaScript-loaded pages. - - More accurate for modern, interactive documentation sites. -- **Limitations**: - - Slower compared to `fast-scraper.js`. - -#### Run the Script -1. Install Playwright: - ```bash - npm install playwright - ``` -2. Run the script: - ```bash - node accurate-scraper.js - ``` -3. Results will be saved in `scraped_data_accurate.json`. +## Change-event data model ---- +The durable asset is not a flat list of documentation URLs. It is a corpus of verified change events. + +Illustrative shape: + +```json +{ + "id": "vendor.package.2026-08-01.signature-change", + "ecosystem": "npm", + "package": "example-sdk", + "version_before": "3.8.0", + "version_after": "4.0.0", + "published_at": "2026-08-01T00:00:00Z", + "first_observed_at": "2026-08-01T03:14:00Z", + "change": { + "type": "signature_change", + "old": "client.create(name, options)", + "new": "client.create({ name, ...options })" + }, + "sources": [ + { + "type": "official_changelog", + "url": "https://example.com/changelog" + } + ], + "verification": { + "kind": "executable_test", + "fixture": "cases/vendor-package-signature-change/" + } +} +``` +The production schema will be versioned and stricter than this example. See [`docs/BENCHMARK_SPEC.md`](docs/BENCHMARK_SPEC.md). +--- -## πŸ’‘ How to Contribute +## What happens to the existing UpdAPI index? -> For first time contributors, I recommend you to check out https://github.com/firstcontributions/first-contributions and https://www.youtube.com/watch?v=YaToH3s_-nQ +It stays useful, but its role changes. -Contributions are welcome! Here's how you can contribute: +`api-docs-urls.csv`, the URL verifier, alignment repair tooling, scraper experiments, and MCP server are now **legacy/source-acquisition infrastructure**. They can help discover authoritative resources, establish provenance, and study documentation movement over time. +They are no longer the core product thesis. -1. **Add API Entries**: - - Add new API entries directly to `api-docs-urls.csv` or via pull request. - - Ensure URLs point to the **current version** of the documentation and policies. - -2. **Verify API Entries**: - - Is the URL up-to-date? - - Is the URL root-level for the relevant page? (`api.com/docs/`, not `api.com/docs/nested`) - - Is the API doc public and does it comply with "robots.txt"? - - Does the URL provide all the expected information (changelogs, rate limits, etc) ? - - Is there any dynamically loaded page content that the scraper is able to extract? - -3. **Improve Scrapers**: - - Enhance `fast-scraper.js` or `accurate-scraper.js` for better performance and compatibility. - - Add features like advanced error handling or field-specific scraping. - -4. **Submit Pull Requests**: - - Fork the repository. - - Create a new branch for your changes. - - Submit a pull request for review. +In particular, UpdAPI should **not** spend substantial effort manually expanding a generic documentation directory merely so an agent can find documentation it could already locate itself. + +Existing commands remain useful locally: -If you're using the scripts, first install dependencies: ```bash npm install -pip install -r requirements.txt +npm test +npm run check-links +npm run realign +npm run verify-alignment +npm run mcp ``` -This installs everything listed in package.json and requirements.txt +--- -### πŸš€ Roadmap Features -- πŸ” **Search & Browse:** Easily find APIs by keyword or category (e.g., "Machine Learning APIs," "Finance APIs") -- πŸ“„ **Latest API Metadata Retrieval:** Retrieve up-to-date API endpoints and parameters, directly from official documentation. -- πŸ›  **VS Code Integration:** Use the lightweight UpdAPI extension to search and retrieve APIs directly from your terminal. +## Execution model ---- +UpdAPI does **not** rely on GitHub Actions for benchmark operation or dataset maintenance. -## πŸ“œ License +For now, collection and evaluation runners should be executable locally and in explicitly controlled compute environments. The architecture should remain portable so the same jobs can later be scheduled and supervised by persistent agent infrastructure such as XUXI without changing benchmark semantics. -This repository is licensed under the [MIT License](LICENSE). +The benchmark must record the environment in which each run occurred; scheduler choice must never become hidden methodology. --- -## Status +## Roadmap -### πŸ›  Current Phase: -- **Under Construction:** We’re building the core MVP features and testing functionality. +### Phase 0 β€” Methodology lock -### πŸ“Œ Known Issues: -- Limited API support. -- Some features may not work as expected. -- ~~Most rows are missing columns, so values shift left and land under the wrong - heading.~~ **Fixed** β€” the cause was `utils/verify_and_clean_csv.py` dropping - broken URLs instead of blanking them; see [Column alignment](#-column-alignment). +- define versioned change-event schema +- define benchmark-case schema +- define runner/run-manifest schema +- define evidence and acceptance requirements +- define scoring and repeated-run policy +- define temporal holdout / publication policy +- create a tiny hand-verified gold set -Check the [Open Issues](https://github.com/in-c0/updapi/issues) for more details. +### Phase 1 β€” First living corpus + +- capture 30–50 high-confidence change events across several ecosystems +- create executable fixtures for the strongest cases +- validate event ingestion and provenance +- establish deterministic baseline checks independent of any LLM + +### Phase 2 β€” Agent benchmark + +- implement pluggable agent runners +- run at least three materially different frontier coding systems +- record complete run manifests and artifacts +- add default-agent and controlled retrieval conditions +- calculate reliability and time-to-adoption metrics + +### Phase 3 β€” Public observatory + +- publish methodology and benchmark versions +- publish model/agent leaderboard +- provide per-case drill-down with evidence +- expose downloadable result data +- chart API freshness longitudinally +- rerun new frontier systems and new API-change cohorts continuously + +### Phase 4 β€” Longitudinal research asset + +- study API-change characteristics and documentation evolution over time +- quantify which classes of changes cause the most agent failures +- measure how agent tooling changes knowledge lag +- publish reproducible analyses and benchmark reports --- -## Roadmap +## What UpdAPI is *not* -### βœ… MVP Goals -- Basic search and browse functionality. -- JSON exports for select APIs. -- Direct links to official API documentation. -- Weekly automated link verification ([`link-freshness.yml`](.github/workflows/link-freshness.yml)). -- MCP server, so coding agents can query the index directly. +UpdAPI is not trying to become: -### πŸ”œ Future Enhancements -- IDE integrations (e.g., VS Code plugin). -- API update notifications via email/webhooks. -- Support for more APIs. +- another generic API directory +- another documentation search engine +- another RAG wrapper over public docs +- another MCP directory +- a benchmark designed to prove that UpdAPI itself improves scores + +The benchmark should remain useful even if its eventual conclusion is that frontier coding agents have almost completely solved stale API knowledge. + +That result would itself be valuable. --- -## Community +## Related work and signals + +The problem is active, and UpdAPI should explicitly build on rather than ignore prior work. + +- Cursor's removal of `@Docs` is a concrete product signal that general-purpose agents are increasingly capable of finding their own documentation: [Cursor Community Forum](https://forum.cursor.com/t/cursor-3-14-2-sunsetting-docs/167423/7) +- SotaDocs publishes a benchmark focused on API correctness/freshness and documentation assistance: [SotaDocs Benchmarks](https://sotadocs.com/benchmarks/) +- Academic work on API evolution, deprecated API use, and temporal coding evaluation should be tracked in the benchmark methodology and related-work notes as the corpus matures. -- [Discussions](https://github.com/in-c0/updapi/discussions) -- [Bug Reports](https://github.com/in-c0/updapi/issues) +UpdAPI's intended differentiation is a **continuously refreshed, timestamped, independently reproducible benchmark built from real API evolution and evaluated on full coding-agent systems**. --- -## ❀️ Acknowledgments +## Contributing -We thank all API providers for publishing robust documentation and fostering developer-friendly ecosystems. Your contributions make projects like this possible! -Special thanks to: +The highest-value contributions are no longer bulk additions of API names. -- [Crawlee](https://github.com/apify/crawlee): A powerful web scraping and crawling library that simplifies the extraction of structured data from websites. -- [OpenAPI](https://github.com/APIs-guru/openapi-directory): For setting the standard in API specifications and enabling better interoperability and accessibility. +Useful contributions include: +- verified recent API change events +- authoritative provenance sources +- executable before/after fixtures +- deterministic validators +- agent-runner adapters +- benchmark methodology review +- contamination and leakage analysis +- reproducibility improvements +- public visualization and result exploration + +A small number of excellent cases is more valuable than thousands of weakly verified rows. + +--- -## Questions? +## License -Send emails to support@updapi.com +This repository is licensed under the [Apache License 2.0](LICENSE). From 235e7732b34f268d092f4778693b0e0c4383b56b Mon Sep 17 00:00:00 2001 From: ava kim Date: Fri, 28 Aug 2026 12:50:27 +1000 Subject: [PATCH 2/8] docs: define living API freshness benchmark specification --- docs/BENCHMARK_SPEC.md | 605 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 605 insertions(+) create mode 100644 docs/BENCHMARK_SPEC.md diff --git a/docs/BENCHMARK_SPEC.md b/docs/BENCHMARK_SPEC.md new file mode 100644 index 0000000..5619195 --- /dev/null +++ b/docs/BENCHMARK_SPEC.md @@ -0,0 +1,605 @@ +# UpdAPI Benchmark Specification + +**Status:** design seed for UpdAPI's 2026 direction reset +**Scope:** living API-evolution dataset + frontier coding-agent freshness benchmark +**Normative intent:** this document defines the measurement contract. Implementations may evolve, but published benchmark versions must remain reproducible. + +--- + +## 1. Research objective + +UpdAPI measures how reliably and how quickly contemporary coding systems adapt to real API and SDK changes. + +The primary research question is: + +> **When an external software interface changes, can a frontier coding agent produce a verified working implementation using the tools it normally has available?** + +Secondary questions include: + +1. Which categories of API evolution remain difficult for frontier agents? +2. How much does web search improve freshness? +3. How much does authoritative current documentation improve freshness beyond normal agent tooling? +4. Does RAG/MCP still provide measurable benefit for public APIs? +5. How often does execution/retry recover from stale initial knowledge? +6. How does performance change as an API change ages? +7. How do cost, latency, and tool use trade off against verified correctness? +8. Are systems robust across repeated trials, or do they succeed only intermittently? + +UpdAPI is explicitly allowed to discover that stale API knowledge has become a negligible problem. The benchmark is not designed to justify an UpdAPI retrieval product. + +--- + +## 2. Unit of evidence: the change event + +The fundamental dataset unit is a **verified API change event**, not an API-document URL and not an LLM prompt. + +A change event represents a bounded transition between two externally meaningful interface states. + +Examples: + +- method renamed +- endpoint removed +- request parameter renamed +- positional arguments replaced by an options object +- return shape changed +- authentication method changed +- SDK namespace moved +- deprecated interface becomes invalid +- newly released feature creates a better/current implementation path +- documented behavior changes in a way that affects executable code + +A URL move with no developer-visible interface consequence is historical metadata, not automatically a benchmark-worthy change event. + +--- + +## 3. Evidence hierarchy + +Every accepted event must have enough evidence to establish ground truth independently of a model's interpretation. + +Preferred evidence, strongest first: + +1. **Executable behavior** against version-pinned software or a safely testable public API. +2. **Versioned source/specification** such as an OpenAPI diff, SDK source diff, tagged release, or package type definitions. +3. **Official release notes/changelog/migration guide.** +4. **Official current documentation** with a trustworthy historical counterpart. +5. Maintainer statements or issue discussions, used only when the interface state cannot be established more directly. + +Third-party tutorials, search snippets, generated summaries, and model claims are discovery aids only. They cannot by themselves define benchmark ground truth. + +--- + +## 4. Change-event lifecycle + +```text +discovered + ↓ +evidence_collected + ↓ +verified + ↓ +case_authored + ↓ +case_validated + ↓ +embargoed/live_holdout (optional) + ↓ +published + ↓ +retired (only if no longer runnable/relevant) +``` + +Transitions must be auditable. Rejected candidates should retain a minimal rejection reason where useful so repeated rediscovery does not create needless work. + +### 4.1 Discovery + +A candidate may be discovered from changelogs, release feeds, package registries, repository diffs, specifications, documentation diffs, or manual observation. + +Discovery is intentionally permissive. + +### 4.2 Verification + +Verification is intentionally strict. A candidate does not become benchmark ground truth merely because an official changelog sentence appears to describe a change. + +Where practical, the verifier should demonstrate both sides: + +- old usage succeeds under the old target state +- old usage fails or is obsolete under the new state +- new usage succeeds under the new state + +### 4.3 Case validation + +A benchmark case must be tested independently of any frontier agent. The harness itself must prove that its validator can distinguish the intended correct and stale behaviors. + +A useful negative control is required for deterministic executable cases: deliberately provide the known-stale implementation and prove that validation rejects it. + +--- + +## 5. Versioned data contracts + +Schemas should live under `schemas/` and carry explicit schema versions once implementation begins. + +The following fields describe the minimum semantic contract. + +### 5.1 `change_event` + +```json +{ + "schema_version": "0.1", + "id": "provider.package.2026-08-01.signature-change", + "provider": "provider", + "ecosystem": "npm", + "package": "package", + "interface": "Client.create", + "version_before": "3.8.0", + "version_after": "4.0.0", + "published_at": "2026-08-01T00:00:00Z", + "first_observed_at": "2026-08-01T03:14:00Z", + "change": { + "type": "signature_change", + "summary": "create() moved from positional arguments to an options object", + "old": "client.create(name, options)", + "new": "client.create({ name, ...options })" + }, + "sources": [ + { + "kind": "official_changelog", + "url": "https://example.com/changelog", + "retrieved_at": "2026-08-01T03:14:00Z" + } + ], + "verification": { + "kind": "executable_fixture", + "fixture": "cases/provider-package-signature-change" + } +} +``` + +### Required semantic properties + +- IDs are stable once published. +- `published_at` refers to the best-supported public availability time of the changed interface, not the time UpdAPI noticed it. +- `first_observed_at` records UpdAPI observation and must never be backdated. +- old/new representations are explanatory, not substitutes for evidence. +- source records preserve retrieval time. +- material corrections create a revision record rather than silently rewriting published history. + +### 5.2 `benchmark_case` + +A benchmark case binds a verified event to a concrete developer task. + +Minimum fields: + +```json +{ + "schema_version": "0.1", + "case_id": "case-provider-package-create-v4", + "event_id": "provider.package.2026-08-01.signature-change", + "case_version": 1, + "task": "Implement ...", + "workspace_fixture": "fixtures/...", + "target_environment": { + "runtime": "node", + "runtime_version": "22.x", + "dependencies": { + "package": "4.0.0" + } + }, + "validator": { + "kind": "command", + "command": "npm test" + }, + "controls": { + "known_stale_fixture": "controls/stale/...", + "known_current_fixture": "controls/current/..." + } +} +``` + +The task should not gratuitously reveal the changed syntax. It should reproduce a plausible developer objective for which stale API knowledge can affect implementation. + +### 5.3 `run_manifest` + +Every result must be attributable to the exact system and environment that produced it. + +Minimum fields: + +- benchmark version +- case ID/version +- run ID +- timestamp +- agent/product name and version +- underlying model name/version when exposed +- reasoning/effort setting when exposed +- tool availability +- web access policy +- repository/filesystem access policy +- execution access +- retrieval/MCP configuration +- starting workspace hash +- dependency lock/hash +- environment/runtime/container identity +- timeout/turn/token budgets +- number of retries allowed by the product/harness +- geographic/provider endpoint when relevant +- measured wall-clock duration +- token/tool usage and estimated cost when available + +Unknown product internals must be recorded as unknown rather than inferred. + +### 5.4 `run_result` + +Minimum fields: + +```json +{ + "run_id": "...", + "case_id": "...", + "verified_success": true, + "validator_exit_code": 0, + "initial_stale_use": false, + "recovered": false, + "tool_trace_summary": { + "web_search": 2, + "docs_fetch": 1, + "shell": 5 + }, + "duration_ms": 93241, + "artifacts": ["final.patch", "validator.log"] +} +``` + +Full trajectories may contain sensitive provider/tool information and should be stored/released according to licensing and product terms. The public scoring record must remain sufficient to reproduce the outcome where possible. + +--- + +## 6. Change taxonomy + +Initial taxonomy: + +### Surface changes + +- `endpoint_rename` +- `method_rename` +- `namespace_move` +- `signature_change` +- `parameter_add` +- `parameter_remove` +- `parameter_rename` +- `parameter_type_change` +- `return_shape_change` +- `authentication_change` +- `configuration_change` + +### Lifecycle changes + +- `new_api` +- `deprecation` +- `removal` +- `default_change` +- `version_migration` + +### Behavioral changes + +- `semantic_behavior_change` +- `error_behavior_change` +- `rate_limit_change` +- `capability_change` + +Taxonomy labels may be multi-valued when a migration spans several mechanisms, but one primary label should identify the dominant failure mode being evaluated. + +--- + +## 7. Evaluation conditions + +### 7.1 Primary condition: product-realistic agent + +The headline leaderboard should evaluate each coding agent substantially as developers normally use it. + +If the product normally has: + +- web search +- shell access +- repository inspection +- package installation +- documentation tools +- automatic retries + +those capabilities belong in the primary condition and must be recorded. + +We are testing the usefulness of the deployed system, not artificially reproducing a 2024-era chat completion. + +### 7.2 Controlled ablations + +Ablations answer narrower causal questions and should not replace the primary condition. + +Canonical labels: + +- `agent_default` +- `no_web` +- `no_external_retrieval` +- `authoritative_docs_supplied` +- `updapi_mcp` +- `no_execution` + +Not every agent/product will support every ablation without distortion. Unsupported conditions are omitted rather than simulated inaccurately. + +--- + +## 8. Scoring + +### 8.1 Primary metric: Verified Task Success + +Binary per attempt: + +```text +1 = deterministic validator accepts the final state +0 = validator rejects it, run errors, or budget expires +``` + +The validatorβ€”not an LLM judgeβ€”should determine correctness whenever executable verification is possible. + +For cases that cannot be executed safely or deterministically, scoring must use an explicit case-specific rubric with human validation before public use. + +### 8.2 Reliability + +For stochastic agents, report the distribution across repeated attempts rather than only best-of-N. + +For public v1, target **five attempts per case/system/condition** where economics permit. During development, three attempts may be used to diagnose harness behavior but should be labeled pre-release. + +Recommended reported values: + +- pass rate +- pass^5: probability/observed fraction represented by success on all five attempts at the aggregate task level where appropriate +- 95% confidence intervals for aggregate comparisons + +Never use hidden retries to transform a failed attempt into a nominal first-attempt success. Agent-native retries inside one product run are part of the trajectory; harness-level reruns are separate attempts. + +### 8.3 Stale API Error Rate + +A run counts as a stale-API error only when evidence in the produced implementation/trajectory shows use of the pre-change or otherwise obsolete interface relevant to the case. + +General coding mistakes are not stale-API errors. + +This metric requires a case-specific classifier or deterministic marker and must not be guessed from failure alone. + +### 8.4 Recovery Rate + +Among runs that demonstrably enter a stale state, measure the fraction that subsequently produce a verified current implementation before the run ends. + +This captures the value of execution, search, diagnostics, and repair loops. + +### 8.5 Retrieval Lift + +For compatible controlled conditions: + +```text +retrieval_lift = success_rate(with_retrieval) - success_rate(control) +``` + +Always show absolute rates alongside the delta. + +### 8.6 API Knowledge Lag + +Knowledge lag is a longitudinal system/event statistic, not a one-shot score. + +Provisional definition: + +> elapsed time between `published_at` and the earliest evaluation window in which the system reaches the benchmark's reliability threshold on that event, subject to confirmation by a subsequent evaluation window. + +The reliability threshold and confirmation rule must be fixed per benchmark version before public reporting. Until the sampling cadence is dense enough, report observations as intervals (for example, `2–5 days`) rather than fabricated point precision. + +### 8.7 Cost and latency + +Where observable, record: + +- wall-clock time +- model tokens +- search/tool calls +- provider-reported cost +- benchmark-run infrastructure cost + +Costs should be normalized carefully and never mixed with unknown estimates as if they were equivalent. + +--- + +## 9. Temporal integrity and contamination + +API freshness benchmarks are unusually vulnerable to aging into the training distribution. + +UpdAPI uses several defenses. + +### 9.1 Recent-event cohorts + +Continuously introduce newly verified change events. Report results by event age, for example: + +- 0–7 days +- 8–30 days +- 31–90 days +- 91–365 days +- historical + +Exact buckets may change before v1 but must be versioned afterward. + +### 9.2 Embargoed live holdouts + +For high-value cases, metadata may be retained privately for a bounded live-evaluation period before full case fixtures are published. + +After the holdout window, release enough information for independent reproduction unless licensing/security constraints prohibit it. + +### 9.3 No secret benchmark-specific hints + +The benchmark harness must not give one system private knowledge of the target change unless that condition explicitly evaluates supplied documentation/retrieval. + +### 9.4 Immutable published cohorts + +Once a cohort is published, its membership and ground truth are frozen. Corrections are versioned and disclosed. + +--- + +## 10. Reproducibility + +A benchmark result is publishable only when UpdAPI can answer: + +1. What exact case was run? +2. What interface/version constituted ground truth? +3. What evidence established that ground truth? +4. What exact agent/model/product version ran? +5. What tools could it use? +6. What environment did it start from? +7. What validator decided success? +8. What artifacts prove the claimed result? +9. Can the case be rerun later within reasonable external-service constraints? + +Published tables without this provenance are marketing charts, not benchmark results. + +--- + +## 11. Benchmark artifact layout + +Proposed repository shape: + +```text +schemas/ + change-event.schema.json + benchmark-case.schema.json + run-manifest.schema.json + run-result.schema.json + +events/ + //.json + +cases/ + / + case.json + README.md + fixture/ + controls/ + stale/ + current/ + validator/ + +runs/ + /// + manifest.json + result.json + artifacts/ + +docs/ + BENCHMARK_SPEC.md + METHODOLOGY.md + RELATED_WORK.md +``` + +Large generated artifacts should not automatically be committed to Git. Storage strategy should preserve hashes and provenance without turning the repository into an unbounded binary archive. + +--- + +## 12. MVP acceptance gate + +The first credible benchmark release should not be declared complete until all of the following are true. + +### Dataset + +- at least **30 verified change events** +- at least **5 distinct ecosystems/provider families** +- at least **20 executable benchmark cases** +- each executable case has stale and current controls +- validators reject their stale negative control and accept their current positive control +- all published events have authoritative provenance + +### Runners + +- at least **3 materially distinct frontier coding systems** +- runner adapters produce the same normalized manifest/result contract +- tool capabilities are recorded, not assumed +- local rerun command exists +- no dependency on GitHub Actions + +### Evaluation + +- development runs demonstrate repeatability +- public v1 policy fixes trial count before results are revealed +- primary leaderboard uses verified task success +- failures are categorized without conflating general coding errors with stale API errors +- at least one controlled retrieval experiment is included + +### Publication + +- methodology is public +- benchmark version is immutable +- per-system configuration is public to the extent allowed by provider products +- downloadable machine-readable results exist +- per-case evidence is inspectable after any holdout period +- methodology/version history is maintained + +--- + +## 13. Initial implementation sequence + +Implementation should proceed in this order unless evidence justifies a change: + +1. Add JSON Schemas for event/case/run/result. +2. Author 3–5 gold change events manually. +3. Build validators and stale/current controls for those cases. +4. Build a local benchmark runner that runs a **human-authored fixed implementation** first; prove scoring works without any LLM. +5. Add one coding-agent adapter. +6. Add normalized artifact capture. +7. Add two more agent adapters. +8. Add repeated-trial orchestration. +9. Add change discovery/ingestion automation only after the ground-truth workflow is trustworthy. +10. Build public visualization only after real benchmark results exist. + +The sequence deliberately puts **ground truth before automation and visualization**. + +--- + +## 14. Execution and scheduler policy + +GitHub Actions is not part of the UpdAPI operating architecture. + +All collection, validation, and benchmark jobs must have explicit local/portable entry points. They may run under a workstation process, container, server, or later a persistent supervisory system such as XUXI, but the scheduler must be separable from benchmark semantics. + +A future autonomous operator may: + +- notice new releases +- propose candidate events +- gather evidence +- run deterministic verification +- schedule benchmark cohorts +- retry infrastructure failures +- publish validated results + +It must not silently promote uncertain discovery into benchmark ground truth or rewrite published history. + +--- + +## 15. Public-product direction + +The public interface should behave more like an independent measurement site than a documentation service. + +Likely top-level views: + +- **Leaderboard** β€” agent freshness, reliability, recovery, cost, latency +- **Live changes** β€” newest verified API-evolution cohort +- **Agents** β€” per-system drill-down and tool configuration +- **APIs / ecosystems** β€” failure rates and change types by ecosystem +- **Change explorer** β€” before/after evidence and benchmark case +- **Trends** β€” knowledge-lag and stale-error trajectories over time +- **Methodology** β€” benchmark versions, scoring, caveats, corrections +- **Data** β€” downloadable machine-readable events and results + +Visual polish matters, but no public dashboard should precede credible measured data. + +--- + +## 16. Independence principle + +UpdAPI should not optimize the benchmark to demonstrate that UpdAPI retrieval improves agent performance. + +If `updapi_mcp` produces no statistically meaningful lift over normal agent web search, publish that result. + +If frontier agents converge on near-perfect freshness, publish that result. + +If one vendor consistently adapts faster than another, publish the evidence and methodology regardless of provider. + +The benchmark's durable asset is **trust**. From 0f06bc6d39dabbda8b452537cb7abd2317649470 Mon Sep 17 00:00:00 2001 From: ava kim Date: Fri, 28 Aug 2026 12:50:33 +1000 Subject: [PATCH 3/8] chore: retire GitHub Actions freshness workflow --- .github/workflows/link-freshness.yml | 88 ---------------------------- 1 file changed, 88 deletions(-) delete mode 100644 .github/workflows/link-freshness.yml diff --git a/.github/workflows/link-freshness.yml b/.github/workflows/link-freshness.yml deleted file mode 100644 index 40c24ca..0000000 --- a/.github/workflows/link-freshness.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Link freshness - -# The dataset is a table of URLs, and URLs rot. This re-probes every URL every -# week, appends the result to datasets/link-health-history.csv, and opens a PR -# with the corrections it can make safely (redirects only β€” a dead URL has no -# known replacement, so it is reported rather than guessed at). - -on: - schedule: - # Mondays 00:20 UTC β€” off the hour, where scheduled-job contention is worst. - - cron: '20 0 * * 1' - workflow_dispatch: - -permissions: - contents: write - pull-requests: write - -concurrency: - group: link-freshness - cancel-in-progress: false - -jobs: - check: - runs-on: ubuntu-latest - timeout-minutes: 60 - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-node@v4 - with: - node-version: '22' - - # Only the two CSV libraries β€” installing the full manifest would pull - # crawlee and playwright browsers for a job that makes plain fetch calls. - - name: Install check dependencies - run: npm install --no-save --no-audit --no-fund csv-parse csv-stringify - - - name: Probe every URL and apply redirect corrections - run: node tools/check-links.mjs --fix | tee check-links.out - - - name: Summarise - run: | - { - echo '## Link freshness' - echo - echo '```' - tail -n 20 check-links.out - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - - - name: Upload full report - uses: actions/upload-artifact@v4 - with: - name: link-health-${{ github.run_number }} - path: datasets/link-health.json - retention-days: 90 - - - name: Open a pull request with the corrections - uses: peter-evans/create-pull-request@v6 - with: - branch: automated/link-freshness - # Recreate rather than accumulate: each run supersedes the last, so a - # missed week never leaves a stale correction sitting in an open PR. - delete-branch: true - commit-message: 'chore(data): weekly link freshness pass' - title: 'chore(data): weekly link freshness pass' - add-paths: | - api-docs-urls.csv - datasets/link-health.json - datasets/link-health-history.csv - body: | - Automated by `.github/workflows/link-freshness.yml`. - - `tools/check-links.mjs` probed every URL in every column, followed - redirects, and rewrote the rows whose URL now resolves somewhere else. - - **Dead URLs are deliberately not touched.** A 404 has no known - replacement and inventing one would put a fabrication in the dataset; - they are listed in `datasets/link-health.json` under `findings` with - `"outcome": "dead"` for a human to resolve. - - `403` and `429` are recorded as `blocked`, not `dead` β€” being refused - by a bot-detecting CDN says nothing about whether the page exists, so - those never drive a correction either. - - Per-run totals are appended to `datasets/link-health-history.csv`. - The series is the interesting output; a single snapshot is not. From 3e83d359e149a851f5afc4b96662ddf836406e60 Mon Sep 17 00:00:00 2001 From: ava kim Date: Fri, 28 Aug 2026 12:50:42 +1000 Subject: [PATCH 4/8] chore: align package metadata with new direction --- package.json | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 6b4c7ad..00789ba 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "updapi-remote", "version": "1.0.0", "type": "module", - "description": "![updAPI](https://github.com/user-attachments/assets/7a67269e-5ce0-480d-95d8-cd7dfe79ca87)", + "description": "Living API evolution dataset and frontier coding-agent freshness benchmark", "main": "index.js", "scripts": { "test": "mocha", @@ -13,9 +13,15 @@ "realign:apply": "node tools/realign-columns.mjs --apply", "verify-alignment": "node tools/verify-alignment.mjs" }, - "keywords": [], + "keywords": [ + "api-evolution", + "benchmark", + "coding-agents", + "llm-evaluation", + "api-freshness" + ], "author": "", - "license": "ISC", + "license": "Apache-2.0", "dependencies": { "@modelcontextprotocol/sdk": "^1.12.0", "cheerio": "^1.0.0", From 5a183a856b313160ac2e1746be3ed4f66c33d69d Mon Sep 17 00:00:00 2001 From: ava kim Date: Fri, 28 Aug 2026 12:53:22 +1000 Subject: [PATCH 5/8] chore: remove GitHub Actions CI workflow --- .github/workflows/ci.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 8e9952e..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: CI - -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - test: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: '18' - - - name: Install dependencies - run: npm ci - - - name: Run tests - run: npm test From a782d52bb92a145124f0f5aa1cc72bd5ff225491 Mon Sep 17 00:00:00 2001 From: AvaKim Date: Fri, 28 Aug 2026 14:29:50 +1000 Subject: [PATCH 6/8] bench v0 core: schemas, 5 gold events, 3 discriminating cases, LLM-free harness Implements steps 1-4 of the #24 sequence on the direction-reset branch: - schemas/ 0.1.0: change-event, benchmark-case, run-manifest, run-result (JSON Schema 2020-12; taxonomy extends spec section 6 with runtime_requirement_change, also written back into the spec) - events/: 5 hand-verified change events with registry-timestamped provenance (ai@7 removals x2, MCP 2026-07-28 serverInfo->_meta, openai@7 Node-22 floor, anthropic 0.119 Files/Skills GA); re-derive any registry claim via `npm run bench:evidence -- ` - cases/: executable fixtures with pinned deps + lockfiles, stale/current controls, and deterministic validators; expected_stale_failure recorded from observation, never guessed - tools/bench/run-controls.mjs: proves every validator REJECTS its stale control and ACCEPTS its current control (spec section 4.3 made executable); restores fixtures byte-identically (hash-checked); refuses validators that cannot fail - tools/bench/validate-data.mjs: ajv + cross-file invariants; --root hook so its own negative control can prove it rejects a broken tree - mocha suites with negative controls for BOTH tools; npm test 35/35 Empirical findings encoded in the cases (observed, not assumed): - ai@7 ToolExecutionOptions is generic over CONTEXT with no default - not a plain rename; a name-swap migration still fails to compile - ai@7's provider registry returns a compat Proxy around registered v2-spec models (its own warning says "v2 specification compatibility mode"; reading specificationVersion through it violates the JS proxy invariant on frozen targets), so the validator asserts modelId preservation rather than object identity - MCP GA v2's modern era is opt-in on BOTH ends: McpServer over a raw transport still serves legacy (server/discover -> -32601) and the Client defaults to legacy negotiation; the 2026-07-28 wire lives on createMcpHandler, so the case serves a modern-only strict endpoint ({ legacy: 'reject' }) fully in-process via injected fetch; the stale 2025-pattern client hard-fails at connect with JSON-RPC -32022 No GitHub Actions anywhere; every entry point is local (npm run bench:*). --- README.md | 19 + .../case-mcp-serverinfo-discover-v2/README.md | 39 ++ .../case-mcp-serverinfo-discover-v2/case.json | 31 ++ .../controls/current/src/solution.mjs | 19 + .../controls/stale/src/solution.mjs | 15 + .../fixture/package-lock.json | 170 ++++++ .../fixture/package.json | 10 + .../fixture/src/server-harness.mjs | 21 + .../fixture/src/solution.mjs | 4 + cases/case-mcp-serverinfo-discover-v2/task.md | 25 + .../validator/validate.mjs | 51 ++ .../README.md | 23 + .../case.json | 30 ++ .../controls/current/src/solution.mjs | 7 + .../controls/stale/src/solution.mjs | 9 + .../fixture/package-lock.json | 123 +++++ .../fixture/package.json | 9 + .../fixture/src/solution.mjs | 2 + .../fixture/src/stub-model.mjs | 14 + .../case-vercel-ai-custom-provider-v7/task.md | 14 + .../validator/validate.mjs | 57 ++ .../README.md | 20 + .../case.json | 31 ++ .../controls/current/src/solution.ts | 16 + .../controls/stale/src/solution.ts | 15 + .../fixture/package-lock.json | 501 ++++++++++++++++++ .../fixture/package.json | 12 + .../fixture/src/audit.ts | 7 + .../fixture/src/solution.ts | 2 + .../fixture/tsconfig.json | 12 + .../task.md | 21 + .../validator/validate.mjs | 33 ++ docs/BENCHMARK_SPEC.md | 5 + ...pic-ai-sdk.2026-08-19.files-skills-ga.json | 46 ++ ...026-07-27.serverinfo-into-result-meta.json | 47 ++ ...penai.2026-07-27.node22-runtime-floor.json | 40 ++ ...5.remove-experimental-custom-provider.json | 46 ++ ...026-06-25.remove-toolcalloptions-type.json | 46 ++ package-lock.json | 6 +- package.json | 7 + schemas/benchmark-case.schema.json | 103 ++++ schemas/change-event.schema.json | 192 +++++++ schemas/run-manifest.schema.json | 121 +++++ schemas/run-result.schema.json | 58 ++ test/bench-data.test.cjs | 68 +++ test/bench-harness.test.cjs | 97 ++++ tools/bench/capture-registry-evidence.mjs | 37 ++ tools/bench/run-controls.mjs | 209 ++++++++ tools/bench/validate-data.mjs | 121 +++++ 49 files changed, 2610 insertions(+), 1 deletion(-) create mode 100644 cases/case-mcp-serverinfo-discover-v2/README.md create mode 100644 cases/case-mcp-serverinfo-discover-v2/case.json create mode 100644 cases/case-mcp-serverinfo-discover-v2/controls/current/src/solution.mjs create mode 100644 cases/case-mcp-serverinfo-discover-v2/controls/stale/src/solution.mjs create mode 100644 cases/case-mcp-serverinfo-discover-v2/fixture/package-lock.json create mode 100644 cases/case-mcp-serverinfo-discover-v2/fixture/package.json create mode 100644 cases/case-mcp-serverinfo-discover-v2/fixture/src/server-harness.mjs create mode 100644 cases/case-mcp-serverinfo-discover-v2/fixture/src/solution.mjs create mode 100644 cases/case-mcp-serverinfo-discover-v2/task.md create mode 100644 cases/case-mcp-serverinfo-discover-v2/validator/validate.mjs create mode 100644 cases/case-vercel-ai-custom-provider-v7/README.md create mode 100644 cases/case-vercel-ai-custom-provider-v7/case.json create mode 100644 cases/case-vercel-ai-custom-provider-v7/controls/current/src/solution.mjs create mode 100644 cases/case-vercel-ai-custom-provider-v7/controls/stale/src/solution.mjs create mode 100644 cases/case-vercel-ai-custom-provider-v7/fixture/package-lock.json create mode 100644 cases/case-vercel-ai-custom-provider-v7/fixture/package.json create mode 100644 cases/case-vercel-ai-custom-provider-v7/fixture/src/solution.mjs create mode 100644 cases/case-vercel-ai-custom-provider-v7/fixture/src/stub-model.mjs create mode 100644 cases/case-vercel-ai-custom-provider-v7/task.md create mode 100644 cases/case-vercel-ai-custom-provider-v7/validator/validate.mjs create mode 100644 cases/case-vercel-ai-tool-execution-options-v7/README.md create mode 100644 cases/case-vercel-ai-tool-execution-options-v7/case.json create mode 100644 cases/case-vercel-ai-tool-execution-options-v7/controls/current/src/solution.ts create mode 100644 cases/case-vercel-ai-tool-execution-options-v7/controls/stale/src/solution.ts create mode 100644 cases/case-vercel-ai-tool-execution-options-v7/fixture/package-lock.json create mode 100644 cases/case-vercel-ai-tool-execution-options-v7/fixture/package.json create mode 100644 cases/case-vercel-ai-tool-execution-options-v7/fixture/src/audit.ts create mode 100644 cases/case-vercel-ai-tool-execution-options-v7/fixture/src/solution.ts create mode 100644 cases/case-vercel-ai-tool-execution-options-v7/fixture/tsconfig.json create mode 100644 cases/case-vercel-ai-tool-execution-options-v7/task.md create mode 100644 cases/case-vercel-ai-tool-execution-options-v7/validator/validate.mjs create mode 100644 events/anthropic/anthropic-ai-sdk/anthropic.anthropic-ai-sdk.2026-08-19.files-skills-ga.json create mode 100644 events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json create mode 100644 events/openai/openai/openai.openai.2026-07-27.node22-runtime-floor.json create mode 100644 events/vercel/ai/vercel.ai.2026-06-25.remove-experimental-custom-provider.json create mode 100644 events/vercel/ai/vercel.ai.2026-06-25.remove-toolcalloptions-type.json create mode 100644 schemas/benchmark-case.schema.json create mode 100644 schemas/change-event.schema.json create mode 100644 schemas/run-manifest.schema.json create mode 100644 schemas/run-result.schema.json create mode 100644 test/bench-data.test.cjs create mode 100644 test/bench-harness.test.cjs create mode 100644 tools/bench/capture-registry-evidence.mjs create mode 100644 tools/bench/run-controls.mjs create mode 100644 tools/bench/validate-data.mjs diff --git a/README.md b/README.md index a9efc51..ff54ca9 100644 --- a/README.md +++ b/README.md @@ -284,6 +284,25 @@ For now, collection and evaluation runners should be executable locally and in e The benchmark must record the environment in which each run occurred; scheduler choice must never become hidden methodology. +### Running the benchmark tooling (v0) + +Everything runs locally; there is no CI dependency. + +```bash +npm install +npm run bench:validate # schemas + dataset cross-reference checks +npm run bench:controls # prove every case validator REJECTS its known-stale + # control and ACCEPTS its known-current control (no LLM involved) +npm test # mocha suite, including negative controls that prove + # the validator and the harness can themselves fail +npm run bench:evidence -- # re-derive registry evidence for an event +``` + +`bench:controls` is BENCHMARK_SPEC section 4.3 made executable: a case whose +validator cannot reject its stale control is refused, because a detector that +cannot fail is not evidence. Fixtures are mutated in place during a control +run and restored byte-identically afterwards. + --- ## Roadmap diff --git a/cases/case-mcp-serverinfo-discover-v2/README.md b/cases/case-mcp-serverinfo-discover-v2/README.md new file mode 100644 index 0000000..45ec5b6 --- /dev/null +++ b/cases/case-mcp-serverinfo-discover-v2/README.md @@ -0,0 +1,39 @@ +# case-mcp-serverinfo-discover-v2 + +**Event:** [`modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta`](../../events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json) β€” +the final MCP 2026-07-28 protocol revision moved server identity out of the +result body into result `_meta['io.modelcontextprotocol/serverInfo']`, carried +by the GA v2 SDK packages. + +**What the case measures.** Protocol-era adaptation with an executable local +oracle: a real client against a real **modern-only strict** endpoint +(`createMcpHandler(factory, { legacy: 'reject' })`), served entirely +in-process through an injected fetch β€” no sockets, no credentials. The task is +a plausible developer objective: connect and report the server's identity. + +**Empirical grounding** (observed against the GA 2.0.0 packages, not taken +from documentation): + +- The high-level `McpServer` over a raw in-memory transport still serves the + **legacy** era: a default client connects via `initialize`, receives + `serverInfo` in the result body, and `server/discover` answers `-32601`. + The modern 2026-07-28 wire lives on the `createMcpHandler` HTTP surface. +- The v2 `Client` **defaults to legacy negotiation**; current code must opt in + (`versionNegotiation: { mode: 'auto' }` or a pinned revision). +- Stale code (the idiomatic 2025 pattern: default client + `initialize`) + against this endpoint **hard-fails at connect** with JSON-RPC `-32022` + `Unsupported protocol version: 2025-11-25` (`supported: ['2026-07-28']`) β€” + precisely the failure mode the SDK release notes warn about for modern-only + peers. +- The migration guide's "graceful anonymous identity" statement applies to a + different layer (identity-read within an established modern connection). + The two sources are consistent once separated by layer; this fixture is the + ground truth for the layer it exercises. + +**Validator:** `node validator/validate.mjs` from this directory β€” behavioural: +the solution must return the identity the fixture server actually declares. + +**Controls:** `controls/stale` (default client, 2025 handshake) must be +rejected; `controls/current` (opt-in negotiation) must pass. Run +`npm run bench:controls -- --case case-mcp-serverinfo-discover-v2` from the +repo root to prove both directions. diff --git a/cases/case-mcp-serverinfo-discover-v2/case.json b/cases/case-mcp-serverinfo-discover-v2/case.json new file mode 100644 index 0000000..d6c3d40 --- /dev/null +++ b/cases/case-mcp-serverinfo-discover-v2/case.json @@ -0,0 +1,31 @@ +{ + "schema_version": "0.1.0", + "case_id": "case-mcp-serverinfo-discover-v2", + "event_id": "modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta", + "case_version": 1, + "task": "The workspace pins @modelcontextprotocol/client and @modelcontextprotocol/server at 2.0.0. src/server-harness.mjs serves a local, current-generation MCP endpoint entirely in-process and exports its URL plus a fetch implementation that reaches it. Implement getServerIdentity({ url, fetch }) in src/solution.mjs: connect an MCP client to that endpoint (route all HTTP through the provided fetch), obtain the server's declared identity from the connection exchange, close cleanly, and return { name, version }. No sockets, no network, no credentials.", + "workspace_fixture": "fixture", + "solution_path": "src/solution.mjs", + "target_environment": { + "runtime": "node", + "runtime_version": ">=22", + "dependencies": { + "@modelcontextprotocol/client": "2.0.0", + "@modelcontextprotocol/server": "2.0.0" + } + }, + "setup": { + "install_command": "npm install --no-audit --no-fund" + }, + "validator": { + "kind": "command", + "command": "node validator/validate.mjs", + "timeout_ms": 90000 + }, + "controls": { + "known_stale_fixture": "controls/stale", + "known_current_fixture": "controls/current" + }, + "expected_stale_failure": "Hard connect failure, observed: connect() rejects with SdkHttpError carrying JSON-RPC error -32022 'Unsupported protocol version: 2025-11-25' with data {supported: ['2026-07-28'], requested: '2025-11-25'} - the plain 2025 initialize handshake (the v2 client's DEFAULT; version negotiation is opt-in) against the modern-only strict endpoint. No identity is ever readable on this path.", + "notes": "Empirically grounded design, all observed against the GA 2.0.0 packages: (1) the high-level McpServer over a raw in-memory transport still serves the LEGACY era - a default client connects via initialize and receives serverInfo in the result body, and server/discover answers -32601 - so the modern 2026-07-28 wire lives on the createMcpHandler HTTP surface; (2) this fixture's endpoint uses legacy:'reject' (modern-only strict), the deployment posture the release notes' hard-connect-failure warning describes, served fully in-process via an injected fetch; (3) the stale control is the idiomatic 2025 pattern (default client), current code opts into versionNegotiation mode 'auto'. The sources' apparent disagreement (migration guide: graceful anonymous identity; release notes: hard connect failure) resolves by layer: identity-read within an established modern connection degrades gracefully, era mismatch against a strict endpoint hard-fails. This case exercises the era mismatch, with the identity read verified on the passing path." +} diff --git a/cases/case-mcp-serverinfo-discover-v2/controls/current/src/solution.mjs b/cases/case-mcp-serverinfo-discover-v2/controls/current/src/solution.mjs new file mode 100644 index 0000000..120632a --- /dev/null +++ b/cases/case-mcp-serverinfo-discover-v2/controls/current/src/solution.mjs @@ -0,0 +1,19 @@ +// KNOWN-CURRENT CONTROL - connects the post-2026-07-28 way. +// Version negotiation is opt-in on the v2 client (default stays legacy), so +// current code enables it; `auto` probes with server/discover and adopts the +// modern era on definitive evidence. Identity then comes from the negotiated +// connection (the SDK reads it from result _meta under +// 'io.modelcontextprotocol/serverInfo'; getServerVersion() is the helper). +import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client'; + +export async function getServerIdentity({ url, fetch }) { + const client = new Client( + { name: 'updapi-case-client', version: '1.0.0' }, + { versionNegotiation: { mode: 'auto' } } + ); + const transport = new StreamableHTTPClientTransport(new URL(url), { fetch }); + await client.connect(transport); + const info = client.getServerVersion(); + await client.close(); + return { name: info.name, version: info.version }; +} diff --git a/cases/case-mcp-serverinfo-discover-v2/controls/stale/src/solution.mjs b/cases/case-mcp-serverinfo-discover-v2/controls/stale/src/solution.mjs new file mode 100644 index 0000000..a2527ff --- /dev/null +++ b/cases/case-mcp-serverinfo-discover-v2/controls/stale/src/solution.mjs @@ -0,0 +1,15 @@ +// KNOWN-STALE CONTROL - connects the pre-2026-07-28 way. +// This is exactly the idiomatic 2025 client pattern: a default Client (no +// version negotiation) whose connect() runs the plain `initialize` handshake. +// Against a modern-only strict endpoint that handshake is rejected, so the +// connection hard-fails before any identity can be read. +import { Client, StreamableHTTPClientTransport } from '@modelcontextprotocol/client'; + +export async function getServerIdentity({ url, fetch }) { + const client = new Client({ name: 'updapi-case-client', version: '1.0.0' }); + const transport = new StreamableHTTPClientTransport(new URL(url), { fetch }); + await client.connect(transport); + const info = client.getServerVersion(); + await client.close(); + return { name: info.name, version: info.version }; +} diff --git a/cases/case-mcp-serverinfo-discover-v2/fixture/package-lock.json b/cases/case-mcp-serverinfo-discover-v2/fixture/package-lock.json new file mode 100644 index 0000000..8d53672 --- /dev/null +++ b/cases/case-mcp-serverinfo-discover-v2/fixture/package-lock.json @@ -0,0 +1,170 @@ +{ + "name": "case-mcp-serverinfo-discover-v2-fixture", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "case-mcp-serverinfo-discover-v2-fixture", + "dependencies": { + "@modelcontextprotocol/client": "2.0.0", + "@modelcontextprotocol/server": "2.0.0" + } + }, + "node_modules/@modelcontextprotocol/client": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/client/-/client-2.0.0.tgz", + "integrity": "sha512-8f1OghQ2rjzIOfqgUCP+8GiUWqRs89njoWLNqAe8kWmDePv3s1fZXseej+QXemssEuuOvLLmLO/kqM3IQHtISw==", + "license": "MIT", + "dependencies": { + "@modelcontextprotocol/core": "2.0.0", + "cross-spawn": "^7.0.5", + "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", + "jose": "^6.1.3", + "pkce-challenge": "^5.0.0", + "zod": "^4.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@modelcontextprotocol/core": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/core/-/core-2.0.0.tgz", + "integrity": "sha512-pJCEwGG7Lfr/+PQp9ZTwKXNeO5wzbfKL7H3MYpCorM4oFBoQrdjnBgEoqG+RjhsvS1FKrDbKux+M1HhlnGWqcA==", + "license": "MIT", + "dependencies": { + "zod": "^4.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@modelcontextprotocol/server": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/server/-/server-2.0.0.tgz", + "integrity": "sha512-YhHWdHfpFMQfd0prsEnxKeS3Qz3ytIGmsS0sth4KDjnacIT7hxk6hXHkJ9KysxlkvTM+WZAtQbbcUhdoP4Hvtw==", + "license": "MIT", + "dependencies": { + "@modelcontextprotocol/core": "2.0.0", + "zod": "^4.2.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/eventsource": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", + "integrity": "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.0.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/jose": { + "version": "6.2.10", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.10.tgz", + "integrity": "sha512-iiW7J9qRFlGxvCOIBDBDxFePQSn7ZMAnrYGhrrOo6siO/MIqwfyilLR27pkfDgUk+raLuzADS8A3S/KLBisc0g==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/pkce-challenge": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/pkce-challenge/-/pkce-challenge-5.0.1.tgz", + "integrity": "sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==", + "license": "MIT", + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/cases/case-mcp-serverinfo-discover-v2/fixture/package.json b/cases/case-mcp-serverinfo-discover-v2/fixture/package.json new file mode 100644 index 0000000..dcf1800 --- /dev/null +++ b/cases/case-mcp-serverinfo-discover-v2/fixture/package.json @@ -0,0 +1,10 @@ +{ + "name": "case-mcp-serverinfo-discover-v2-fixture", + "private": true, + "type": "module", + "description": "Benchmark workspace. See ../task.md for the objective.", + "dependencies": { + "@modelcontextprotocol/client": "2.0.0", + "@modelcontextprotocol/server": "2.0.0" + } +} diff --git a/cases/case-mcp-serverinfo-discover-v2/fixture/src/server-harness.mjs b/cases/case-mcp-serverinfo-discover-v2/fixture/src/server-harness.mjs new file mode 100644 index 0000000..b45c5d2 --- /dev/null +++ b/cases/case-mcp-serverinfo-discover-v2/fixture/src/server-harness.mjs @@ -0,0 +1,21 @@ +// Fixture infrastructure (not part of the task): a local, modern-only MCP +// endpoint served entirely in-process. +// +// createMcpHandler serves the 2026-07-28 protocol revision from a per-request +// server factory; `legacy: 'reject'` makes the endpoint modern-only strict, so +// 2025-era traffic (the plain `initialize` handshake) is rejected rather than +// served. `handler.fetch` is the web-standard face, which means the whole +// exchange can run through an injected fetch with no sockets and no network. +import { createMcpHandler, McpServer } from '@modelcontextprotocol/server'; + +export const SERVER_NAME = 'updapi-fixture-server'; +export const SERVER_VERSION = '1.2.3'; +export const SERVER_URL = 'http://updapi-fixture.internal/mcp'; + +const handler = createMcpHandler( + () => new McpServer({ name: SERVER_NAME, version: SERVER_VERSION }), + { legacy: 'reject' } +); + +/** In-process fetch implementation routing every request to the endpoint. */ +export const serverFetch = (input, init) => handler.fetch(new Request(input, init)); diff --git a/cases/case-mcp-serverinfo-discover-v2/fixture/src/solution.mjs b/cases/case-mcp-serverinfo-discover-v2/fixture/src/solution.mjs new file mode 100644 index 0000000..dfd14cf --- /dev/null +++ b/cases/case-mcp-serverinfo-discover-v2/fixture/src/solution.mjs @@ -0,0 +1,4 @@ +// TASK: see ../../task.md. Implement and export getServerIdentity here. +export async function getServerIdentity({ url, fetch }) { + throw new Error('not implemented'); +} diff --git a/cases/case-mcp-serverinfo-discover-v2/task.md b/cases/case-mcp-serverinfo-discover-v2/task.md new file mode 100644 index 0000000..3bec70e --- /dev/null +++ b/cases/case-mcp-serverinfo-discover-v2/task.md @@ -0,0 +1,25 @@ +# Task + +This workspace pins `@modelcontextprotocol/client@2.0.0` and +`@modelcontextprotocol/server@2.0.0` (see `fixture/package.json`). + +`fixture/src/server-harness.mjs` serves a local, current-generation MCP +endpoint entirely in-process, and exports: + +- `SERVER_URL` β€” the endpoint URL +- `serverFetch` β€” a fetch implementation that reaches that endpoint + +Nothing may touch a socket or the network; route all HTTP through the +provided fetch. + +Implement `fixture/src/solution.mjs`: + +```js +export async function getServerIdentity({ url, fetch }) { ... } +``` + +It must connect an MCP client to the endpoint, obtain the server's declared +identity from the connection exchange, close cleanly, and return +`{ name, version }`. + +Acceptance: `node validator/validate.mjs` (run from the case directory) exits 0. diff --git a/cases/case-mcp-serverinfo-discover-v2/validator/validate.mjs b/cases/case-mcp-serverinfo-discover-v2/validator/validate.mjs new file mode 100644 index 0000000..82bce7b --- /dev/null +++ b/cases/case-mcp-serverinfo-discover-v2/validator/validate.mjs @@ -0,0 +1,51 @@ +// Deterministic validator for case-mcp-serverinfo-discover-v2. +// Exit 0 = verified success; non-zero = failure. No LLM involvement. +// The oracle is behavioural: the solution must return the identity the +// fixture server actually declares, over a real (in-process) MCP exchange. +const failures = []; +const ok = (cond, label, detail) => { + if (cond) console.log(`PASS ${label}`); + else { console.log(`FAIL ${label}${detail ? ' - ' + detail : ''}`); failures.push(label); } +}; + +const { SERVER_NAME, SERVER_VERSION, SERVER_URL, serverFetch } = + await import(new URL('../fixture/src/server-harness.mjs', import.meta.url)); + +let solution = null; +try { + solution = await import(new URL('../fixture/src/solution.mjs', import.meta.url)); +} catch (err) { + console.log(`FAIL solution-loads - ${err.constructor.name}: ${String(err.message).split('\n')[0]}`); + process.exit(1); +} +console.log('PASS solution-loads'); +ok(typeof solution.getServerIdentity === 'function', 'exports-getServerIdentity'); + +if (typeof solution.getServerIdentity === 'function') { + let identity = null; + let callError = null; + const guard = new Promise((_, reject) => + setTimeout(() => reject(new Error('timed out after 45s')), 45000).unref?.()); + try { + identity = await Promise.race([ + solution.getServerIdentity({ url: SERVER_URL, fetch: serverFetch }), + guard + ]); + } catch (err) { + callError = err; + } + ok(callError === null, 'connects-and-returns', + callError && `${callError.name ?? 'Error'}: ${String(callError.message).split('\n')[0]}`); + if (callError === null) { + ok(identity != null && typeof identity === 'object', 'identity-is-object', `got ${typeof identity}`); + ok(identity?.name === SERVER_NAME, 'identity-name', `got ${JSON.stringify(identity?.name)}`); + ok(identity?.version === SERVER_VERSION, 'identity-version', `got ${JSON.stringify(identity?.version)}`); + } +} + +if (failures.length > 0) { + console.log(`RESULT FAIL (${failures.length} failed)`); + process.exit(1); +} +console.log('RESULT PASS'); +process.exit(0); diff --git a/cases/case-vercel-ai-custom-provider-v7/README.md b/cases/case-vercel-ai-custom-provider-v7/README.md new file mode 100644 index 0000000..b9c57bc --- /dev/null +++ b/cases/case-vercel-ai-custom-provider-v7/README.md @@ -0,0 +1,23 @@ +# case-vercel-ai-custom-provider-v7 + +**Event:** [`vercel.ai.2026-06-25.remove-experimental-custom-provider`](../../events/vercel/ai/vercel.ai.2026-06-25.remove-experimental-custom-provider.json) β€” +AI SDK 7 removed the deprecated `experimental_customProvider` export; the +stable replacement is `customProvider`. + +**What the case measures.** A developer objective ("expose this stub model as +a provider") whose implementation differs by exactly one piece of API +knowledge. Code written with pre-7.0 knowledge imports a named export that no +longer exists, and fails at module load; current code passes a behavioural +validator: the resolved model must preserve the stub's `modelId` (ai@7's +registry may hand back a compatibility wrapper, so object identity is +deliberately not asserted) and unknown ids must be rejected +(`AI_NoSuchModelError`). + +**Validator:** `node validator/validate.mjs` from this directory. Runtime +behaviour, not compilation: the fixture pins `ai@7.0.0` and the validator +imports the solution and exercises the provider. + +**Controls:** `controls/stale` (v6-style `experimental_customProvider`) must be +rejected; `controls/current` (`customProvider`) must be accepted. Run +`npm run bench:controls -- --case case-vercel-ai-custom-provider-v7` from the +repo root to prove both directions. diff --git a/cases/case-vercel-ai-custom-provider-v7/case.json b/cases/case-vercel-ai-custom-provider-v7/case.json new file mode 100644 index 0000000..c4e84c6 --- /dev/null +++ b/cases/case-vercel-ai-custom-provider-v7/case.json @@ -0,0 +1,30 @@ +{ + "schema_version": "0.1.0", + "case_id": "case-vercel-ai-custom-provider-v7", + "event_id": "vercel.ai.2026-06-25.remove-experimental-custom-provider", + "case_version": 1, + "task": "The workspace pins ai@7.0.0. Using the ai package's provider-construction API, expose the stub model exported by src/stub-model.mjs as a provider under the language model id 'fast', and export the provider object as `provider` from src/solution.mjs. Do not call any network API. The deterministic validator must pass (it checks that resolving 'fast' yields the stub's model identity and that unknown ids are rejected).", + "workspace_fixture": "fixture", + "solution_path": "src/solution.mjs", + "target_environment": { + "runtime": "node", + "runtime_version": ">=22", + "dependencies": { + "ai": "7.0.0" + } + }, + "setup": { + "install_command": "npm install --no-audit --no-fund" + }, + "validator": { + "kind": "command", + "command": "node validator/validate.mjs", + "timeout_ms": 180000 + }, + "controls": { + "known_stale_fixture": "controls/stale", + "known_current_fixture": "controls/current" + }, + "expected_stale_failure": "Module load fails: ai@7 no longer provides the named export 'experimental_customProvider' (ESM named-export resolution error). Recorded from observation by the control harness.", + "notes": "The task deliberately says 'provider-construction API' without naming the symbol; stale knowledge reaches for the removed experimental_ alias, current knowledge uses the stable name." +} diff --git a/cases/case-vercel-ai-custom-provider-v7/controls/current/src/solution.mjs b/cases/case-vercel-ai-custom-provider-v7/controls/current/src/solution.mjs new file mode 100644 index 0000000..e8258c0 --- /dev/null +++ b/cases/case-vercel-ai-custom-provider-v7/controls/current/src/solution.mjs @@ -0,0 +1,7 @@ +// KNOWN-CURRENT CONTROL - implements the task the post-change (ai v7) way. +import { customProvider } from 'ai'; +import { stubFastModel } from './stub-model.mjs'; + +export const provider = customProvider({ + languageModels: { fast: stubFastModel } +}); diff --git a/cases/case-vercel-ai-custom-provider-v7/controls/stale/src/solution.mjs b/cases/case-vercel-ai-custom-provider-v7/controls/stale/src/solution.mjs new file mode 100644 index 0000000..a2b6134 --- /dev/null +++ b/cases/case-vercel-ai-custom-provider-v7/controls/stale/src/solution.mjs @@ -0,0 +1,9 @@ +// KNOWN-STALE CONTROL - implements the task the pre-change (ai v6) way. +// ai@7 removed the deprecated experimental_customProvider export, so loading +// this module against the pinned ai@7.0.0 must fail. +import { experimental_customProvider } from 'ai'; +import { stubFastModel } from './stub-model.mjs'; + +export const provider = experimental_customProvider({ + languageModels: { fast: stubFastModel } +}); diff --git a/cases/case-vercel-ai-custom-provider-v7/fixture/package-lock.json b/cases/case-vercel-ai-custom-provider-v7/fixture/package-lock.json new file mode 100644 index 0000000..502015d --- /dev/null +++ b/cases/case-vercel-ai-custom-provider-v7/fixture/package-lock.json @@ -0,0 +1,123 @@ +{ + "name": "case-vercel-ai-custom-provider-v7-fixture", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "case-vercel-ai-custom-provider-v7-fixture", + "dependencies": { + "ai": "7.0.0" + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-4.0.0.tgz", + "integrity": "sha512-rcKukspbM4h511ot2E8TsPl7rXjRK1zHKrMCP7w4+XF55UKqQHaDzo2kKbGv5rp8Bjb1yQatIHJZE1E2yrOOMw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.0", + "@ai-sdk/provider-utils": "5.0.0", + "@vercel/oidc": "3.2.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.0.tgz", + "integrity": "sha512-fr9Gs89prDWiuox/T+kCA+i2cJkHpxU5S+tr4megjTzRC27ZsvFhwjU/+XrqqMbvBUlfmXxTOYWy8ng45dsjIg==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.0.tgz", + "integrity": "sha512-zj66M02jc6ASYwIgWZowsooDUwaVngeNZQ3H10GwcPMZ+KR6gHMhcUuKl6tkai+JPXTKDyHY1pnszuxRtw2D4A==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.0", + "@standard-schema/spec": "^1.1.0", + "@workflow/serde": "4.1.0", + "eventsource-parser": "^3.0.8" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@vercel/oidc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/@workflow/serde": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0.tgz", + "integrity": "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==", + "license": "Apache-2.0" + }, + "node_modules/ai": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/ai/-/ai-7.0.0.tgz", + "integrity": "sha512-hncs+jamJh8r36K6G8xky7oF4Ai/RLU5TF85FMzI2vElyMJGGnLoHihpdmuDiuY2BsktDWHevKaJM1l0VcRLGw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "4.0.0", + "@ai-sdk/provider": "4.0.0", + "@ai-sdk/provider-utils": "5.0.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/cases/case-vercel-ai-custom-provider-v7/fixture/package.json b/cases/case-vercel-ai-custom-provider-v7/fixture/package.json new file mode 100644 index 0000000..43791a8 --- /dev/null +++ b/cases/case-vercel-ai-custom-provider-v7/fixture/package.json @@ -0,0 +1,9 @@ +{ + "name": "case-vercel-ai-custom-provider-v7-fixture", + "private": true, + "type": "module", + "description": "Benchmark workspace. See ../task.md for the objective.", + "dependencies": { + "ai": "7.0.0" + } +} diff --git a/cases/case-vercel-ai-custom-provider-v7/fixture/src/solution.mjs b/cases/case-vercel-ai-custom-provider-v7/fixture/src/solution.mjs new file mode 100644 index 0000000..68d77f8 --- /dev/null +++ b/cases/case-vercel-ai-custom-provider-v7/fixture/src/solution.mjs @@ -0,0 +1,2 @@ +// TASK: see ../../task.md. Export `provider` from this file. +export const provider = null; diff --git a/cases/case-vercel-ai-custom-provider-v7/fixture/src/stub-model.mjs b/cases/case-vercel-ai-custom-provider-v7/fixture/src/stub-model.mjs new file mode 100644 index 0000000..9b9d255 --- /dev/null +++ b/cases/case-vercel-ai-custom-provider-v7/fixture/src/stub-model.mjs @@ -0,0 +1,14 @@ +// An inert stand-in for a language model instance. The provider registry +// validates the declared specification version before returning a model +// (observed against ai@7.0.0: resolving a model without one fails with +// "Unsupported model version undefined ... only supports models that +// implement specification version \"v2\""), so the stub declares exactly +// what the registry demands and nothing else. The registry may hand back a +// compatibility wrapper rather than this exact object; the validator checks +// modelId preservation. No method here is ever invoked. +export const stubFastModel = Object.freeze({ + specificationVersion: 'v2', + provider: 'acme-internal', + modelId: 'acme-fast-internal', + marker: 'updapi-stub-language-model' +}); diff --git a/cases/case-vercel-ai-custom-provider-v7/task.md b/cases/case-vercel-ai-custom-provider-v7/task.md new file mode 100644 index 0000000..a9873cf --- /dev/null +++ b/cases/case-vercel-ai-custom-provider-v7/task.md @@ -0,0 +1,14 @@ +# Task + +This workspace pins `ai@7.0.0` (see `fixture/package.json`). + +`fixture/src/stub-model.mjs` exports `stubFastModel`, an inert stand-in language +model instance used by this project's tests. Nothing may call a network. + +Implement `fixture/src/solution.mjs` so that it exports a `provider` object, +built with the `ai` package's provider-construction API, in which the language +model id `fast` resolves to the registered `stubFastModel` (the validator +checks the resolved model carries the stub's `modelId`, and that unknown ids +are rejected). + +Acceptance: `node validator/validate.mjs` (run from the case directory) exits 0. diff --git a/cases/case-vercel-ai-custom-provider-v7/validator/validate.mjs b/cases/case-vercel-ai-custom-provider-v7/validator/validate.mjs new file mode 100644 index 0000000..989c236 --- /dev/null +++ b/cases/case-vercel-ai-custom-provider-v7/validator/validate.mjs @@ -0,0 +1,57 @@ +// Deterministic validator for case-vercel-ai-custom-provider-v7. +// Exit 0 = verified success; non-zero = failure. No LLM involvement. +const failures = []; +const ok = (cond, label, detail) => { + if (cond) console.log(`PASS ${label}`); + else { console.log(`FAIL ${label}${detail ? ' - ' + detail : ''}`); failures.push(label); } +}; + +let solution = null; +let loadError = null; +try { + solution = await import(new URL('../fixture/src/solution.mjs', import.meta.url)); +} catch (err) { + loadError = err; +} + +if (loadError) { + // A workspace whose solution cannot even load has not implemented the task. + console.log(`FAIL solution-loads - ${loadError.constructor.name}: ${String(loadError.message).split('\n')[0]}`); + process.exit(1); +} +console.log('PASS solution-loads'); + +const { stubFastModel } = await import(new URL('../fixture/src/stub-model.mjs', import.meta.url)); +const provider = solution.provider; + +ok(provider != null && typeof provider === 'object', 'provider-exported', `got ${typeof provider}`); +ok(typeof provider?.languageModel === 'function', 'provider-has-languageModel'); + +if (typeof provider?.languageModel === 'function') { + let resolved = null; + let resolveError = null; + try { resolved = provider.languageModel('fast'); } catch (err) { resolveError = err; } + ok(resolveError === null, 'fast-resolves', resolveError && resolveError.message); + // ai@7's registry may wrap a registered model in a compatibility shim + // (observed: a v2-spec stub comes back wrapped, with an SDK warning), so + // the behavioural contract is modelId preservation, not object identity. + ok(resolved != null && resolved.modelId === stubFastModel.modelId, 'fast-preserves-modelId', + `expected ${stubFastModel.modelId}, got ${resolved && resolved.modelId}`); + // Do not read specificationVersion off the resolved model: ai@7's compat + // wrapper is a Proxy that rewrites it ('v2' -> 'v4'), which violates the JS + // proxy invariant against the frozen stub target and throws TypeError. + if (resolved) console.log(`INFO resolved model: modelId=${resolved.modelId} wrapped=${resolved !== stubFastModel}`); + + let unknownThrew = false; + let unknownErrName = ''; + try { provider.languageModel('model-that-does-not-exist'); } + catch (err) { unknownThrew = true; unknownErrName = err?.name ?? ''; } + ok(unknownThrew, 'unknown-id-throws', 'provider without fallback must not silently resolve unknown ids'); + if (unknownThrew) console.log(`INFO unknown-id error name: ${unknownErrName}`); +} + +if (failures.length > 0) { + console.log(`RESULT FAIL (${failures.length} failed)`); + process.exit(1); +} +console.log('RESULT PASS'); diff --git a/cases/case-vercel-ai-tool-execution-options-v7/README.md b/cases/case-vercel-ai-tool-execution-options-v7/README.md new file mode 100644 index 0000000..474a0ff --- /dev/null +++ b/cases/case-vercel-ai-tool-execution-options-v7/README.md @@ -0,0 +1,20 @@ +# case-vercel-ai-tool-execution-options-v7 + +**Event:** [`vercel.ai.2026-06-25.remove-toolcalloptions-type`](../../events/vercel/ai/vercel.ai.2026-06-25.remove-toolcalloptions-type.json) β€” +AI SDK 7 removed the deprecated `ToolCallOptions` type; typed code must use +`ToolExecutionOptions`. + +**What the case measures.** The type surface of the same ai@7.0.0 release that +`case-vercel-ai-custom-provider-v7` measures on the runtime surface. A removed +exported *type* surfaces at compile time by nature, so this is deliberately +the one compile-gated case among the first three: the pinned TypeScript +compiler in strict mode is the deterministic oracle, and that is the honest +oracle for this failure mode rather than a convenience shortcut. + +**Validator:** `node validator/validate.mjs` from this directory (resolves the +fixture-pinned compiler and runs `tsc --noEmit` strict). + +**Controls:** `controls/stale` (imports `ToolCallOptions`) must fail to +compile; `controls/current` (imports `ToolExecutionOptions`) must compile +clean. Run `npm run bench:controls -- --case case-vercel-ai-tool-execution-options-v7` +from the repo root to prove both directions. diff --git a/cases/case-vercel-ai-tool-execution-options-v7/case.json b/cases/case-vercel-ai-tool-execution-options-v7/case.json new file mode 100644 index 0000000..1c478b7 --- /dev/null +++ b/cases/case-vercel-ai-tool-execution-options-v7/case.json @@ -0,0 +1,31 @@ +{ + "schema_version": "0.1.0", + "case_id": "case-vercel-ai-tool-execution-options-v7", + "event_id": "vercel.ai.2026-06-25.remove-toolcalloptions-type", + "case_version": 1, + "task": "The workspace pins ai@7.0.0 and a current TypeScript. Implement makeAuditedExecute in src/solution.ts: given a tool execute function (input, options) => output and an audit log (src/audit.ts), return a wrapped function with the same signature that records { toolCallId, at } before delegating. Type the options parameter with the type the ai package exports for the second argument of a tool's execute function. Acceptance is strict compilation by the pinned TypeScript.", + "workspace_fixture": "fixture", + "solution_path": "src/solution.ts", + "target_environment": { + "runtime": "node", + "runtime_version": ">=22", + "dependencies": { + "ai": "7.0.0", + "typescript": "7.0.2" + } + }, + "setup": { + "install_command": "npm install --no-audit --no-fund" + }, + "validator": { + "kind": "command", + "command": "node validator/validate.mjs", + "timeout_ms": 120000 + }, + "controls": { + "known_stale_fixture": "controls/stale", + "known_current_fixture": "controls/current" + }, + "expected_stale_failure": "TypeScript cannot resolve the removed exported type (TS2305-family error at the import site). Recorded from observation by the control harness.", + "notes": "Deliberately the one compile-gated case among the first three: a removed exported TYPE surfaces at compile time by nature, so the pinned compiler in strict mode IS the deterministic oracle. The other two launch cases use runtime/behavioural validators (see issue #24 reviewer constraint)." +} diff --git a/cases/case-vercel-ai-tool-execution-options-v7/controls/current/src/solution.ts b/cases/case-vercel-ai-tool-execution-options-v7/controls/current/src/solution.ts new file mode 100644 index 0000000..0b4f17a --- /dev/null +++ b/cases/case-vercel-ai-tool-execution-options-v7/controls/current/src/solution.ts @@ -0,0 +1,16 @@ +// KNOWN-CURRENT CONTROL - types the execute options the post-change (ai v7) way. +// Note the type did not merely get renamed: at ai@7.0.0 ToolExecutionOptions is +// generic over an execution CONTEXT with no default type argument (observed via +// the pinned tsc: "TS2314 ... requires 1 type argument(s)"). +import type { ToolExecutionOptions } from 'ai'; +import type { AuditLog } from './audit.js'; + +export function makeAuditedExecute( + execute: (input: Input, options: ToolExecutionOptions) => PromiseLike | Output, + log: AuditLog +): (input: Input, options: ToolExecutionOptions) => Promise { + return async (input, options) => { + log.push({ toolCallId: options.toolCallId, at: Date.now() }); + return execute(input, options); + }; +} diff --git a/cases/case-vercel-ai-tool-execution-options-v7/controls/stale/src/solution.ts b/cases/case-vercel-ai-tool-execution-options-v7/controls/stale/src/solution.ts new file mode 100644 index 0000000..10ecfbe --- /dev/null +++ b/cases/case-vercel-ai-tool-execution-options-v7/controls/stale/src/solution.ts @@ -0,0 +1,15 @@ +// KNOWN-STALE CONTROL - types the execute options the pre-change (ai v6) way. +// ai@7 removed the deprecated ToolCallOptions type, so strict compilation +// against the pinned ai@7.0.0 must fail at this import. +import type { ToolCallOptions } from 'ai'; +import type { AuditLog } from './audit.js'; + +export function makeAuditedExecute( + execute: (input: Input, options: ToolCallOptions) => PromiseLike | Output, + log: AuditLog +): (input: Input, options: ToolCallOptions) => Promise { + return async (input, options) => { + log.push({ toolCallId: options.toolCallId, at: Date.now() }); + return execute(input, options); + }; +} diff --git a/cases/case-vercel-ai-tool-execution-options-v7/fixture/package-lock.json b/cases/case-vercel-ai-tool-execution-options-v7/fixture/package-lock.json new file mode 100644 index 0000000..a75d6f5 --- /dev/null +++ b/cases/case-vercel-ai-tool-execution-options-v7/fixture/package-lock.json @@ -0,0 +1,501 @@ +{ + "name": "case-vercel-ai-tool-execution-options-v7-fixture", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "case-vercel-ai-tool-execution-options-v7-fixture", + "dependencies": { + "ai": "7.0.0" + }, + "devDependencies": { + "typescript": "7.0.2" + } + }, + "node_modules/@ai-sdk/gateway": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/gateway/-/gateway-4.0.0.tgz", + "integrity": "sha512-rcKukspbM4h511ot2E8TsPl7rXjRK1zHKrMCP7w4+XF55UKqQHaDzo2kKbGv5rp8Bjb1yQatIHJZE1E2yrOOMw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.0", + "@ai-sdk/provider-utils": "5.0.0", + "@vercel/oidc": "3.2.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-4.0.0.tgz", + "integrity": "sha512-fr9Gs89prDWiuox/T+kCA+i2cJkHpxU5S+tr4megjTzRC27ZsvFhwjU/+XrqqMbvBUlfmXxTOYWy8ng45dsjIg==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-5.0.0.tgz", + "integrity": "sha512-zj66M02jc6ASYwIgWZowsooDUwaVngeNZQ3H10GwcPMZ+KR6gHMhcUuKl6tkai+JPXTKDyHY1pnszuxRtw2D4A==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "4.0.0", + "@standard-schema/spec": "^1.1.0", + "@workflow/serde": "4.1.0", + "eventsource-parser": "^3.0.8" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@typescript/typescript-aix-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", + "integrity": "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", + "integrity": "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", + "integrity": "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", + "integrity": "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", + "integrity": "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", + "integrity": "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", + "integrity": "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-loong64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", + "integrity": "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-mips64el": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", + "integrity": "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", + "integrity": "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-riscv64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", + "integrity": "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-s390x": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", + "integrity": "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", + "integrity": "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", + "integrity": "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", + "integrity": "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", + "integrity": "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", + "integrity": "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-sunos-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", + "integrity": "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", + "integrity": "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", + "integrity": "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@vercel/oidc": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.2.0.tgz", + "integrity": "sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20" + } + }, + "node_modules/@workflow/serde": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@workflow/serde/-/serde-4.1.0.tgz", + "integrity": "sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==", + "license": "Apache-2.0" + }, + "node_modules/ai": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/ai/-/ai-7.0.0.tgz", + "integrity": "sha512-hncs+jamJh8r36K6G8xky7oF4Ai/RLU5TF85FMzI2vElyMJGGnLoHihpdmuDiuY2BsktDWHevKaJM1l0VcRLGw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/gateway": "4.0.0", + "@ai-sdk/provider": "4.0.0", + "@ai-sdk/provider-utils": "5.0.0" + }, + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "zod": "^3.25.76 || ^4.1.8" + } + }, + "node_modules/eventsource-parser": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", + "integrity": "sha512-EKN1vKAMcZ8MlYMpaNuxN6R9yakzH6uajHcHVTqWJzvu5pWw9DyhbP35HH8MVBQ+dZjAfDxk+A8NiR9KWaXiyQ==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/typescript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", + "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc" + }, + "engines": { + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.0.2", + "@typescript/typescript-darwin-arm64": "7.0.2", + "@typescript/typescript-darwin-x64": "7.0.2", + "@typescript/typescript-freebsd-arm64": "7.0.2", + "@typescript/typescript-freebsd-x64": "7.0.2", + "@typescript/typescript-linux-arm": "7.0.2", + "@typescript/typescript-linux-arm64": "7.0.2", + "@typescript/typescript-linux-loong64": "7.0.2", + "@typescript/typescript-linux-mips64el": "7.0.2", + "@typescript/typescript-linux-ppc64": "7.0.2", + "@typescript/typescript-linux-riscv64": "7.0.2", + "@typescript/typescript-linux-s390x": "7.0.2", + "@typescript/typescript-linux-x64": "7.0.2", + "@typescript/typescript-netbsd-arm64": "7.0.2", + "@typescript/typescript-netbsd-x64": "7.0.2", + "@typescript/typescript-openbsd-arm64": "7.0.2", + "@typescript/typescript-openbsd-x64": "7.0.2", + "@typescript/typescript-sunos-x64": "7.0.2", + "@typescript/typescript-win32-arm64": "7.0.2", + "@typescript/typescript-win32-x64": "7.0.2" + } + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "peer": true, + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/cases/case-vercel-ai-tool-execution-options-v7/fixture/package.json b/cases/case-vercel-ai-tool-execution-options-v7/fixture/package.json new file mode 100644 index 0000000..4a50446 --- /dev/null +++ b/cases/case-vercel-ai-tool-execution-options-v7/fixture/package.json @@ -0,0 +1,12 @@ +{ + "name": "case-vercel-ai-tool-execution-options-v7-fixture", + "private": true, + "type": "module", + "description": "Benchmark workspace. See ../task.md for the objective.", + "dependencies": { + "ai": "7.0.0" + }, + "devDependencies": { + "typescript": "7.0.2" + } +} diff --git a/cases/case-vercel-ai-tool-execution-options-v7/fixture/src/audit.ts b/cases/case-vercel-ai-tool-execution-options-v7/fixture/src/audit.ts new file mode 100644 index 0000000..e5cd642 --- /dev/null +++ b/cases/case-vercel-ai-tool-execution-options-v7/fixture/src/audit.ts @@ -0,0 +1,7 @@ +// Audit log shape used by this project's tooling instrumentation. +export interface AuditEntry { + toolCallId: string; + at: number; +} + +export type AuditLog = AuditEntry[]; diff --git a/cases/case-vercel-ai-tool-execution-options-v7/fixture/src/solution.ts b/cases/case-vercel-ai-tool-execution-options-v7/fixture/src/solution.ts new file mode 100644 index 0000000..c06888e --- /dev/null +++ b/cases/case-vercel-ai-tool-execution-options-v7/fixture/src/solution.ts @@ -0,0 +1,2 @@ +// TASK: see ../../task.md. Implement and export makeAuditedExecute here. +export {}; diff --git a/cases/case-vercel-ai-tool-execution-options-v7/fixture/tsconfig.json b/cases/case-vercel-ai-tool-execution-options-v7/fixture/tsconfig.json new file mode 100644 index 0000000..578c075 --- /dev/null +++ b/cases/case-vercel-ai-tool-execution-options-v7/fixture/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "strict": true, + "noEmit": true, + "target": "es2022", + "module": "nodenext", + "moduleResolution": "nodenext", + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src"] +} diff --git a/cases/case-vercel-ai-tool-execution-options-v7/task.md b/cases/case-vercel-ai-tool-execution-options-v7/task.md new file mode 100644 index 0000000..a364ebf --- /dev/null +++ b/cases/case-vercel-ai-tool-execution-options-v7/task.md @@ -0,0 +1,21 @@ +# Task + +This workspace pins `ai@7.0.0` and TypeScript (see `fixture/package.json`). + +`fixture/src/audit.ts` defines the audit log type used by this project. + +Implement `fixture/src/solution.ts`: + +```ts +export function makeAuditedExecute(execute, log) { ... } +``` + +Given any tool execute function of the form `(input, options) => output` and +an `AuditLog`, it must return a wrapped function with the same signature that +pushes `{ toolCallId: options.toolCallId, at: Date.now() }` to the log before +delegating. Type the `options` parameter with the type the `ai` package +exports for the second argument of a tool's `execute` function; keep the +wrapper fully generic over input and output types. + +Acceptance: `node validator/validate.mjs` (run from the case directory) +exits 0. The validator runs the pinned TypeScript compiler in strict mode. diff --git a/cases/case-vercel-ai-tool-execution-options-v7/validator/validate.mjs b/cases/case-vercel-ai-tool-execution-options-v7/validator/validate.mjs new file mode 100644 index 0000000..d7401cd --- /dev/null +++ b/cases/case-vercel-ai-tool-execution-options-v7/validator/validate.mjs @@ -0,0 +1,33 @@ +// Deterministic validator for case-vercel-ai-tool-execution-options-v7: +// strict compilation by the TypeScript version pinned in the fixture. +// Resolves the pinned compiler's bin from the fixture's own node_modules so +// the validator keeps working across TS major layouts (5.x JS, 7.x native). +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const caseDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const fixtureDir = path.join(caseDir, 'fixture'); +const tsPkgDir = path.join(fixtureDir, 'node_modules', 'typescript'); +const tsPkg = JSON.parse(fs.readFileSync(path.join(tsPkgDir, 'package.json'), 'utf8')); +const binRel = typeof tsPkg.bin === 'string' ? tsPkg.bin : tsPkg.bin?.tsc; +if (!binRel) { + console.log('FAIL tsc-bin-not-found - typescript package exposes no tsc bin'); + process.exit(1); +} +const tscBin = path.join(tsPkgDir, binRel); + +const run = spawnSync(process.execPath, [tscBin, '--noEmit', '-p', path.join(fixtureDir, 'tsconfig.json')], { + encoding: 'utf8', + timeout: 110000 +}); +const output = `${run.stdout ?? ''}${run.stderr ?? ''}`.trim(); +console.log(`INFO typescript ${tsPkg.version}`); +if (output) console.log(output.split(/\r?\n/).slice(0, 20).join('\n')); +if (run.status === 0) { + console.log('RESULT PASS (strict compile clean)'); + process.exit(0); +} +console.log(`RESULT FAIL (tsc exit ${run.status})`); +process.exit(1); diff --git a/docs/BENCHMARK_SPEC.md b/docs/BENCHMARK_SPEC.md index 5619195..c1b2401 100644 --- a/docs/BENCHMARK_SPEC.md +++ b/docs/BENCHMARK_SPEC.md @@ -277,6 +277,11 @@ Initial taxonomy: - `removal` - `default_change` - `version_migration` +- `runtime_requirement_change` β€” a supported-runtime/platform floor change + (e.g. an SDK major dropping a Node.js version) that is not itself an + interface rename. Added in v0 implementation for the openai-node 7.0 Node-22 + floor event; measures ecosystem freshness rather than API-call freshness, so + events of this type are taxonomy-diversity events rather than headline cases. ### Behavioral changes diff --git a/events/anthropic/anthropic-ai-sdk/anthropic.anthropic-ai-sdk.2026-08-19.files-skills-ga.json b/events/anthropic/anthropic-ai-sdk/anthropic.anthropic-ai-sdk.2026-08-19.files-skills-ga.json new file mode 100644 index 0000000..39b027a --- /dev/null +++ b/events/anthropic/anthropic-ai-sdk/anthropic.anthropic-ai-sdk.2026-08-19.files-skills-ga.json @@ -0,0 +1,46 @@ +{ + "schema_version": "0.1.0", + "id": "anthropic.anthropic-ai-sdk.2026-08-19.files-skills-ga", + "provider": "anthropic", + "ecosystem": "npm", + "package": "@anthropic-ai/sdk", + "interface": "client.files / client.skills", + "version_before": "0.118.0", + "version_after": "0.119.0", + "published_at": "2026-08-19T16:56:08.720Z", + "first_observed_at": "2026-08-28T02:53:02Z", + "change": { + "type": "new_api", + "secondary_types": ["capability_change"], + "summary": "@anthropic-ai/sdk 0.119.0 promotes the Files and Skills APIs to GA (non-beta resources) and adds computer-use and browser-use toolsets. Code written against pre-0.119 knowledge reaches these only under beta namespaces or not at all.", + "old": "Files/Skills reachable only under beta namespaces (client.beta.*)", + "new": "GA resources: client.files and client.skills, plus tools/agent-toolset (computer use, browser use)" + }, + "sources": [ + { + "kind": "official_release", + "url": "https://github.com/anthropics/anthropic-sdk-typescript/releases/tag/sdk-v0.119.0", + "retrieved_at": "2026-08-28T03:35:00Z", + "note": "Published 2026-08-19T16:50:57Z: 'Files and Skills APIs are now GA; add computer use and browser use toolsets'." + }, + { + "kind": "package_registry", + "url": "https://registry.npmjs.org/@anthropic-ai/sdk", + "retrieved_at": "2026-08-28T03:44:00Z", + "note": "time['0.119.0'] = 2026-08-19T16:56:08.720Z; 0.118.0 published the same day at 13:34Z." + }, + { + "kind": "source_diff", + "url": "https://unpkg.com/@anthropic-ai/sdk@0.119.0/?meta", + "retrieved_at": "2026-08-28T03:55:00Z", + "note": "Published 0.119.0 tarball contains non-beta resources/files.js and resources/skills/* alongside the beta variants, plus tools/agent-toolset/* including skills and computer-use tooling." + } + ], + "verification": { + "kind": "versioned_source", + "notes": "Verified against the published tarball's file tree (non-beta files/skills resources present at 0.119.0). Candidate for a credential-free compile/type fixture per issue #24 case suggestion 5; a live-API fixture only if credential and cost semantics are clean." + }, + "status": "verified", + "revision": 1, + "notes": "Temporal-cohort value: this event was 9 days old at first observation." +} diff --git a/events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json b/events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json new file mode 100644 index 0000000..8fae18d --- /dev/null +++ b/events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json @@ -0,0 +1,47 @@ +{ + "schema_version": "0.1.0", + "id": "modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta", + "provider": "modelcontextprotocol", + "ecosystem": "npm", + "package": "@modelcontextprotocol/server", + "interface": "DiscoverResult.serverInfo", + "version_before": "2.0.0-beta.5", + "version_after": "2.0.0", + "published_at": "2026-07-27T23:55:22.239Z", + "first_observed_at": "2026-08-28T02:53:02Z", + "change": { + "type": "return_shape_change", + "secondary_types": ["version_migration"], + "summary": "The final MCP 2026-07-28 protocol revision moves server identity out of the DiscoverResult body: servers identify themselves via result _meta under the key io.modelcontextprotocol/serverInfo (exported as SERVER_INFO_META_KEY), DiscoverResult no longer declares a serverInfo field, and the per-request clientInfo envelope field becomes optional. SDK v2 GA (server and client 2.0.0) implements the final wire shape; the 2.0.0 betas implemented the draft shape.", + "old": "const info = discoverResult.serverInfo", + "new": "const info = discoverResult._meta?.['io.modelcontextprotocol/serverInfo']" + }, + "sources": [ + { + "kind": "official_release", + "url": "https://github.com/modelcontextprotocol/typescript-sdk/releases/tag/%40modelcontextprotocol%2Fserver%402.0.0", + "retrieved_at": "2026-08-28T03:35:00Z", + "note": "Breaking type changes: DiscoverResult no longer declares serverInfo; RequestMetaEnvelope's clientInfo is optional. Release notes also describe a hard connect failure mode when a pre-final peer misclassifies a modern-only server as legacy and attempts an initialize handshake." + }, + { + "kind": "migration_guide", + "url": "https://github.com/modelcontextprotocol/typescript-sdk/blob/main/docs/migration/support-2026-07-28.md", + "retrieved_at": "2026-08-28T03:20:00Z", + "note": "Documents SERVER_INFO_META_KEY = 'io.modelcontextprotocol/serverInfo' and that a server stamping no identity is treated as anonymous (getServerVersion() undefined) rather than an error." + }, + { + "kind": "package_registry", + "url": "https://registry.npmjs.org/@modelcontextprotocol/server", + "retrieved_at": "2026-08-28T03:44:00Z", + "note": "time['2.0.0'] = 2026-07-27T23:55:22.239Z after five 2.0.0-beta releases; @modelcontextprotocol/client 2.0.0 published the same minute. The pre-v2 @modelcontextprotocol/sdk package line ends at 1.30.0 (2026-07-27)." + } + ], + "verification": { + "kind": "executable_fixture", + "fixture": "cases/case-mcp-serverinfo-discover-v2", + "notes": "Verified with a local client+server pair on the GA v2 packages over an in-memory transport. The case records the empirically observed failure mode of the stale read path; source claims about hard-vs-graceful failure differ by layer, so the fixture is the ground truth." + }, + "status": "case_validated", + "revision": 1, + "notes": "published_at is the npm publish time (2026-07-27T23:55Z UTC) of the GA packages carrying the final 2026-07-28 protocol revision; the revision itself is dated 2026-07-28." +} diff --git a/events/openai/openai/openai.openai.2026-07-27.node22-runtime-floor.json b/events/openai/openai/openai.openai.2026-07-27.node22-runtime-floor.json new file mode 100644 index 0000000..5f9ed40 --- /dev/null +++ b/events/openai/openai/openai.openai.2026-07-27.node22-runtime-floor.json @@ -0,0 +1,40 @@ +{ + "schema_version": "0.1.0", + "id": "openai.openai.2026-07-27.node22-runtime-floor", + "provider": "openai", + "ecosystem": "npm", + "package": "openai", + "interface": null, + "version_before": "6.x", + "version_after": "7.0.0", + "published_at": "2026-07-27T21:56:56.615Z", + "first_observed_at": "2026-08-28T02:53:02Z", + "change": { + "type": "runtime_requirement_change", + "secondary_types": ["version_migration"], + "summary": "openai-node 7.0.0 raises the supported Node.js floor to 22, dropping Node 20. A dependency bump alone is an incomplete migration for projects pinned to Node 20; runtime/engines/container configuration must move coherently.", + "old": "Node.js 20 supported (openai 6.x)", + "new": "engines: { node: '>=22.0.0' } (openai 7.0.0)" + }, + "sources": [ + { + "kind": "official_release", + "url": "https://github.com/openai/openai-node/releases/tag/v7.0.0", + "retrieved_at": "2026-08-28T03:20:00Z", + "note": "Published 2026-07-27T21:56:10Z; release notes: 'require Node.js 22 and codify version support (#2026)'." + }, + { + "kind": "package_registry", + "url": "https://registry.npmjs.org/openai/7.0.0", + "retrieved_at": "2026-08-28T03:55:00Z", + "note": "The published 7.0.0 manifest's engines field reads node >=22.0.0." + } + ], + "verification": { + "kind": "versioned_source", + "notes": "Ground truth is the engines field of the published 7.0.0 registry manifest (node >=22.0.0), corroborated by the release notes. Candidate for a future executable migration case (environment-gated validator); measures ecosystem freshness more than API-call freshness, so it is a taxonomy-diversity event rather than a headline case." + }, + "status": "verified", + "revision": 1, + "notes": "runtime_requirement_change is a v0 taxonomy extension over BENCHMARK_SPEC.md section 6; flagged for methodology review in PR #25." +} diff --git a/events/vercel/ai/vercel.ai.2026-06-25.remove-experimental-custom-provider.json b/events/vercel/ai/vercel.ai.2026-06-25.remove-experimental-custom-provider.json new file mode 100644 index 0000000..09543fb --- /dev/null +++ b/events/vercel/ai/vercel.ai.2026-06-25.remove-experimental-custom-provider.json @@ -0,0 +1,46 @@ +{ + "schema_version": "0.1.0", + "id": "vercel.ai.2026-06-25.remove-experimental-custom-provider", + "provider": "vercel", + "ecosystem": "npm", + "package": "ai", + "interface": "experimental_customProvider", + "version_before": "6.0.x", + "version_after": "7.0.0", + "published_at": "2026-06-25T12:47:23.880Z", + "first_observed_at": "2026-08-28T02:53:02Z", + "change": { + "type": "removal", + "secondary_types": ["method_rename", "version_migration"], + "summary": "AI SDK 7 removes the deprecated experimental_customProvider export; the stable replacement is customProvider. Stale imports fail at module resolution because the named export no longer exists (the ai package is also ESM-only as of 7.0).", + "old": "import { experimental_customProvider } from 'ai'", + "new": "import { customProvider } from 'ai'" + }, + "sources": [ + { + "kind": "official_release", + "url": "https://github.com/vercel/ai/releases/tag/ai%407.0.0", + "retrieved_at": "2026-08-28T03:20:00Z" + }, + { + "kind": "migration_guide", + "url": "https://github.com/vercel/ai/blob/main/content/docs/08-migration-guides/23-migration-guide-7-0.mdx", + "retrieved_at": "2026-08-28T03:20:00Z", + "note": "States: 'The deprecated experimental_customProvider export has been removed in AI SDK 7. Replace it with customProvider.'" + }, + { + "kind": "package_registry", + "url": "https://registry.npmjs.org/ai", + "retrieved_at": "2026-08-28T03:44:00Z", + "note": "time['7.0.0'] = 2026-06-25T12:47:23.880Z. The 6.0.x line remains separately maintained after the 7.0.0 release." + } + ], + "verification": { + "kind": "executable_fixture", + "fixture": "cases/case-vercel-ai-custom-provider-v7", + "notes": "The case validator imports the solution against pinned ai@7.0.0 and exercises provider behaviour; the known-stale control fails at import because the export is absent." + }, + "status": "case_validated", + "revision": 1, + "notes": "version_before is recorded as 6.0.x: the removal ships with the 7.0.0 major while the 6.0.x line continues to receive releases in parallel, so no single 6.0.x version is 'the last before'." +} diff --git a/events/vercel/ai/vercel.ai.2026-06-25.remove-toolcalloptions-type.json b/events/vercel/ai/vercel.ai.2026-06-25.remove-toolcalloptions-type.json new file mode 100644 index 0000000..9210d7a --- /dev/null +++ b/events/vercel/ai/vercel.ai.2026-06-25.remove-toolcalloptions-type.json @@ -0,0 +1,46 @@ +{ + "schema_version": "0.1.0", + "id": "vercel.ai.2026-06-25.remove-toolcalloptions-type", + "provider": "vercel", + "ecosystem": "npm", + "package": "ai", + "interface": "ToolCallOptions", + "version_before": "6.0.x", + "version_after": "7.0.0", + "published_at": "2026-06-25T12:47:23.880Z", + "first_observed_at": "2026-08-28T02:53:02Z", + "change": { + "type": "removal", + "secondary_types": ["parameter_type_change", "version_migration"], + "summary": "AI SDK 7 removes the deprecated ToolCallOptions type; typed code must use ToolExecutionOptions (the second argument of a tool's execute function). The failure surface is compile-time: TypeScript can no longer resolve the removed exported type.", + "old": "import type { ToolCallOptions } from 'ai'", + "new": "import type { ToolExecutionOptions } from 'ai'" + }, + "sources": [ + { + "kind": "official_release", + "url": "https://github.com/vercel/ai/releases/tag/ai%407.0.0", + "retrieved_at": "2026-08-28T03:20:00Z" + }, + { + "kind": "migration_guide", + "url": "https://github.com/vercel/ai/blob/main/content/docs/08-migration-guides/23-migration-guide-7-0.mdx", + "retrieved_at": "2026-08-28T03:20:00Z", + "note": "States: 'The deprecated ToolCallOptions type has been removed in AI SDK 7. Replace all remaining usages with ToolExecutionOptions.'" + }, + { + "kind": "package_registry", + "url": "https://registry.npmjs.org/ai", + "retrieved_at": "2026-08-28T03:44:00Z", + "note": "time['7.0.0'] = 2026-06-25T12:47:23.880Z." + } + ], + "verification": { + "kind": "executable_fixture", + "fixture": "cases/case-vercel-ai-tool-execution-options-v7", + "notes": "Type removals surface at compile time by nature, so this case's deterministic validator is the pinned TypeScript compiler in strict mode - deliberately the only compile-gated case among the first three (see the reviewer constraint on tsc trivia in issue #24)." + }, + "status": "case_validated", + "revision": 1, + "notes": "Shares the ai@7.0.0 release with vercel.ai.2026-06-25.remove-experimental-custom-provider but exercises the type surface rather than the runtime export surface." +} diff --git a/package-lock.json b/package-lock.json index 5931b1e..1d40730 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,7 @@ "": { "name": "updapi-remote", "version": "1.0.0", - "license": "ISC", + "license": "Apache-2.0", "dependencies": { "@modelcontextprotocol/sdk": "^1.12.0", "cheerio": "^1.0.0", @@ -17,6 +17,10 @@ "mocha": "^11.1.0", "playwright": "^1.49.1", "zod": "^3.23.8" + }, + "devDependencies": { + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1" } }, "node_modules/@apify/consts": { diff --git a/package.json b/package.json index 00789ba..85a3ee0 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,9 @@ "main": "index.js", "scripts": { "test": "mocha", + "bench:validate": "node tools/bench/validate-data.mjs", + "bench:controls": "node tools/bench/run-controls.mjs", + "bench:evidence": "node tools/bench/capture-registry-evidence.mjs", "check-links": "node tools/check-links.mjs", "check-links:fix": "node tools/check-links.mjs --fix", "mcp": "node mcp/server.mjs", @@ -31,5 +34,9 @@ "mocha": "^11.1.0", "playwright": "^1.49.1", "zod": "^3.23.8" + }, + "devDependencies": { + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1" } } diff --git a/schemas/benchmark-case.schema.json b/schemas/benchmark-case.schema.json new file mode 100644 index 0000000..ace8912 --- /dev/null +++ b/schemas/benchmark-case.schema.json @@ -0,0 +1,103 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/in-c0/updAPI/blob/main/schemas/benchmark-case.schema.json", + "title": "UpdAPI benchmark case", + "description": "Binds a verified change event to a concrete developer task with a deterministic validator and stale/current controls. See docs/BENCHMARK_SPEC.md section 5.2. schema_version 0.1.0.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "case_id", + "event_id", + "case_version", + "task", + "workspace_fixture", + "target_environment", + "setup", + "validator", + "controls" + ], + "properties": { + "schema_version": { "const": "0.1.0" }, + "case_id": { + "type": "string", + "pattern": "^case-[a-z0-9][a-z0-9-]*$" + }, + "event_id": { + "type": "string", + "description": "Must reference an existing change event id." + }, + "case_version": { "type": "integer", "minimum": 1 }, + "task": { + "type": "string", + "minLength": 40, + "description": "The developer objective given to an agent. Must reproduce a plausible task for which stale API knowledge can affect implementation; must not gratuitously reveal the changed syntax." + }, + "workspace_fixture": { + "type": "string", + "description": "Case-relative path to the workspace an agent starts from." + }, + "solution_path": { + "type": "string", + "description": "Fixture-relative path of the file(s) the agent is expected to produce/modify. Controls replace exactly these paths." + }, + "target_environment": { + "type": "object", + "additionalProperties": false, + "required": ["runtime", "runtime_version", "dependencies"], + "properties": { + "runtime": { "enum": ["node", "python", "deno", "bun"] }, + "runtime_version": { "type": "string" }, + "dependencies": { + "type": "object", + "description": "Exact pinned versions, matching the fixture manifest.", + "additionalProperties": { "type": "string" } + } + } + }, + "setup": { + "type": "object", + "additionalProperties": false, + "required": ["install_command"], + "properties": { + "install_command": { + "type": "string", + "description": "Command that makes the fixture runnable offline-deterministically (run in the fixture directory)." + } + } + }, + "validator": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "command"], + "properties": { + "kind": { + "enum": ["command"], + "description": "v0 supports command validators: exit 0 = verified success, non-zero = failure. The validator, not an LLM judge, decides." + }, + "command": { "type": "string" }, + "timeout_ms": { "type": "integer", "minimum": 1000, "default": 120000 } + } + }, + "controls": { + "type": "object", + "additionalProperties": false, + "required": ["known_stale_fixture", "known_current_fixture"], + "properties": { + "known_stale_fixture": { + "type": "string", + "description": "Case-relative dir whose files, overlaid on the fixture, implement the task the pre-change way. The validator MUST reject it." + }, + "known_current_fixture": { + "type": "string", + "description": "Case-relative dir whose files, overlaid on the fixture, implement the task the post-change way. The validator MUST accept it." + } + } + }, + "expected_stale_failure": { + "type": "string", + "description": "What the stale control's failure looks like (error class/message family), recorded after observation - never guessed." + }, + "notes": { "type": "string" } + } +} diff --git a/schemas/change-event.schema.json b/schemas/change-event.schema.json new file mode 100644 index 0000000..e83d9ec --- /dev/null +++ b/schemas/change-event.schema.json @@ -0,0 +1,192 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/in-c0/updAPI/blob/main/schemas/change-event.schema.json", + "title": "UpdAPI change event", + "description": "A verified API change event: a bounded transition between two externally meaningful interface states. See docs/BENCHMARK_SPEC.md sections 2-6. schema_version 0.1.0.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "id", + "provider", + "ecosystem", + "package", + "version_before", + "version_after", + "published_at", + "first_observed_at", + "change", + "sources", + "verification", + "status", + "revision" + ], + "properties": { + "schema_version": { "const": "0.1.0" }, + "id": { + "type": "string", + "description": "Stable once published. Format: ..., all lowercase.", + "pattern": "^[a-z0-9][a-z0-9-]*\\.[a-z0-9][a-z0-9.-]*\\.[0-9]{4}-[0-9]{2}-[0-9]{2}\\.[a-z0-9][a-z0-9-]*$" + }, + "provider": { "type": "string", "minLength": 1 }, + "ecosystem": { + "enum": ["npm", "pypi", "cargo", "rubygems", "maven", "go", "http_api", "protocol", "other"] + }, + "package": { + "type": "string", + "minLength": 1, + "description": "Canonical package/registry name (e.g. an npm name including scope), or API product name for http_api events." + }, + "interface": { + "type": ["string", "null"], + "description": "The specific interface affected (symbol, endpoint, field), when the change is narrower than the whole package." + }, + "version_before": { + "type": ["string", "null"], + "description": "Last version exhibiting the old state. null when no meaningful prior version exists (e.g. a brand-new package)." + }, + "version_after": { + "type": "string", + "minLength": 1, + "description": "First version exhibiting the new state." + }, + "published_at": { + "type": "string", + "format": "date-time", + "description": "Best-supported public availability time of the changed interface. NOT the time UpdAPI noticed it." + }, + "first_observed_at": { + "type": "string", + "format": "date-time", + "description": "When UpdAPI first observed the change. Never backdated." + }, + "change": { + "type": "object", + "additionalProperties": false, + "required": ["type", "summary"], + "properties": { + "type": { "$ref": "#/$defs/changeType" }, + "secondary_types": { + "type": "array", + "items": { "$ref": "#/$defs/changeType" }, + "uniqueItems": true + }, + "summary": { "type": "string", "minLength": 10 }, + "old": { + "type": ["string", "null"], + "description": "Explanatory old-state representation. Not a substitute for evidence." + }, + "new": { + "type": ["string", "null"], + "description": "Explanatory new-state representation. Not a substitute for evidence." + } + } + }, + "sources": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "url", "retrieved_at"], + "properties": { + "kind": { + "enum": [ + "official_changelog", + "official_release", + "migration_guide", + "official_docs", + "package_registry", + "source_diff", + "specification", + "maintainer_statement" + ] + }, + "url": { "type": "string", "format": "uri" }, + "retrieved_at": { "type": "string", "format": "date-time" }, + "note": { "type": "string" } + } + } + }, + "verification": { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { + "kind": { + "enum": ["executable_fixture", "versioned_source", "release_evidence"], + "description": "Strongest verification backing this event, per the evidence hierarchy in BENCHMARK_SPEC.md section 3." + }, + "fixture": { + "type": "string", + "description": "Repo-relative case directory (required when kind is executable_fixture)." + }, + "notes": { "type": "string" } + }, + "if": { + "properties": { "kind": { "const": "executable_fixture" } }, + "required": ["kind"] + }, + "then": { + "properties": { "fixture": { "type": "string" } }, + "required": ["fixture"] + } + }, + "status": { + "enum": [ + "discovered", + "evidence_collected", + "verified", + "case_authored", + "case_validated", + "embargoed", + "published", + "retired" + ] + }, + "revision": { "type": "integer", "minimum": 1 }, + "revisions": { + "type": "array", + "description": "Explicit correction history. Material corrections append here; published history is never silently rewritten.", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["revision", "at", "reason"], + "properties": { + "revision": { "type": "integer", "minimum": 2 }, + "at": { "type": "string", "format": "date-time" }, + "reason": { "type": "string", "minLength": 5 } + } + } + }, + "notes": { "type": "string" } + }, + "$defs": { + "changeType": { + "description": "Taxonomy per BENCHMARK_SPEC.md section 6, plus runtime_requirement_change (v0 extension: a supported-runtime floor/platform change, e.g. dropping a Node.js major, that is not itself an interface rename).", + "enum": [ + "endpoint_rename", + "method_rename", + "namespace_move", + "signature_change", + "parameter_add", + "parameter_remove", + "parameter_rename", + "parameter_type_change", + "return_shape_change", + "authentication_change", + "configuration_change", + "new_api", + "deprecation", + "removal", + "default_change", + "version_migration", + "runtime_requirement_change", + "semantic_behavior_change", + "error_behavior_change", + "rate_limit_change", + "capability_change" + ] + } + } +} diff --git a/schemas/run-manifest.schema.json b/schemas/run-manifest.schema.json new file mode 100644 index 0000000..bee4685 --- /dev/null +++ b/schemas/run-manifest.schema.json @@ -0,0 +1,121 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/in-c0/updAPI/blob/main/schemas/run-manifest.schema.json", + "title": "UpdAPI run manifest", + "description": "Attributes one benchmark run to the exact system and environment that produced it. Unknown product internals are recorded as \"unknown\", never inferred. See docs/BENCHMARK_SPEC.md section 5.3. schema_version 0.1.0.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "benchmark_version", + "run_id", + "case_id", + "case_version", + "started_at", + "condition", + "agent", + "tools", + "environment", + "workspace_hash", + "dependency_lock_hash", + "budgets", + "harness_retries_allowed" + ], + "properties": { + "schema_version": { "const": "0.1.0" }, + "benchmark_version": { "type": "string", "minLength": 1 }, + "run_id": { "type": "string", "minLength": 8 }, + "case_id": { "type": "string" }, + "case_version": { "type": "integer", "minimum": 1 }, + "started_at": { "type": "string", "format": "date-time" }, + "condition": { + "description": "Evaluation condition per BENCHMARK_SPEC.md section 7.", + "enum": [ + "agent_default", + "no_web", + "no_external_retrieval", + "authoritative_docs_supplied", + "updapi_mcp", + "no_execution", + "control_stale", + "control_current", + "control_human_fixed" + ] + }, + "agent": { + "type": "object", + "additionalProperties": false, + "required": ["product", "product_version", "model"], + "properties": { + "product": { "type": "string", "description": "e.g. claude-code, codex-cli, cursor, none (control runs)." }, + "product_version": { "type": "string" }, + "model": { "type": "string", "description": "Underlying model id when exposed, else \"unknown\"." }, + "model_version": { "type": "string" }, + "reasoning_setting": { "type": "string", "description": "Effort/reasoning knob when exposed, else \"unknown\"." } + } + }, + "tools": { + "type": "object", + "additionalProperties": false, + "required": ["web_search", "repository_access", "execution", "retrieval_mcp"], + "properties": { + "web_search": { "$ref": "#/$defs/capability" }, + "repository_access": { "$ref": "#/$defs/capability" }, + "execution": { "$ref": "#/$defs/capability" }, + "package_install": { "$ref": "#/$defs/capability" }, + "retrieval_mcp": { "$ref": "#/$defs/capability" }, + "notes": { "type": "string" } + } + }, + "environment": { + "type": "object", + "additionalProperties": false, + "required": ["os", "runtime"], + "properties": { + "os": { "type": "string" }, + "runtime": { "type": "string", "description": "e.g. node 22.12.0" }, + "container": { "type": "string", "description": "Image/id when containerized, else \"none\"." }, + "endpoint_region": { "type": "string" } + } + }, + "workspace_hash": { + "type": "string", + "description": "Content hash of the starting workspace (fixture) the run began from." + }, + "dependency_lock_hash": { + "type": "string", + "description": "Hash of the dependency lockfile in effect, or \"none\"." + }, + "budgets": { + "type": "object", + "additionalProperties": false, + "properties": { + "timeout_ms": { "type": ["integer", "null"] }, + "max_turns": { "type": ["integer", "null"] }, + "max_tokens": { "type": ["integer", "null"] } + } + }, + "harness_retries_allowed": { + "type": "integer", + "minimum": 0, + "description": "Harness-level reruns permitted. Agent-native retries inside one product run are part of the trajectory, not counted here." + }, + "duration_ms": { "type": ["integer", "null"] }, + "usage": { + "type": "object", + "additionalProperties": false, + "properties": { + "tokens_in": { "type": ["integer", "null"] }, + "tokens_out": { "type": ["integer", "null"] }, + "tool_calls": { "type": ["integer", "null"] }, + "cost_usd_estimate": { "type": ["number", "null"] } + } + }, + "notes": { "type": "string" } + }, + "$defs": { + "capability": { + "enum": ["allowed", "denied", "unavailable", "unknown"] + } + } +} diff --git a/schemas/run-result.schema.json b/schemas/run-result.schema.json new file mode 100644 index 0000000..351b2ad --- /dev/null +++ b/schemas/run-result.schema.json @@ -0,0 +1,58 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/in-c0/updAPI/blob/main/schemas/run-result.schema.json", + "title": "UpdAPI run result", + "description": "Outcome of one benchmark run. verified_success is decided by the case validator, never an LLM judge. initial_stale_use/recovered are null unless case-specific evidence classifies them. See docs/BENCHMARK_SPEC.md sections 5.4 and 8. schema_version 0.1.0.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "run_id", + "case_id", + "verified_success", + "validator_exit_code", + "initial_stale_use", + "recovered", + "duration_ms", + "artifacts" + ], + "properties": { + "schema_version": { "const": "0.1.0" }, + "run_id": { "type": "string", "minLength": 8 }, + "case_id": { "type": "string" }, + "verified_success": { "type": "boolean" }, + "validator_exit_code": { "type": ["integer", "null"] }, + "initial_stale_use": { + "type": ["boolean", "null"], + "description": "true only when evidence in the implementation/trajectory shows use of the obsolete interface relevant to THIS case. null = not classified. Never inferred from generic failure." + }, + "recovered": { + "type": ["boolean", "null"], + "description": "Among runs that demonstrably entered a stale state: did the run end in verified success? null when initial_stale_use is null/false." + }, + "failure_class": { + "type": ["string", "null"], + "enum": [ + "stale_api_use", + "general_coding_error", + "environment_error", + "budget_exhausted", + "validator_error", + "unknown", + null + ] + }, + "tool_trace_summary": { + "type": "object", + "additionalProperties": { "type": "integer" }, + "description": "Counts by tool family (web_search, docs_fetch, shell, edit, ...) when observable." + }, + "duration_ms": { "type": ["integer", "null"] }, + "artifacts": { + "type": "array", + "items": { "type": "string" }, + "description": "Run-relative paths: final patch, validator log, trajectory summary, etc." + }, + "notes": { "type": "string" } + } +} diff --git a/test/bench-data.test.cjs b/test/bench-data.test.cjs new file mode 100644 index 0000000..5531a83 --- /dev/null +++ b/test/bench-data.test.cjs @@ -0,0 +1,68 @@ +// Dataset validation: the real tree must validate, and β€” the part that makes +// the first assertion meaningful β€” a deliberately broken tree must be REJECTED. +// A validator that cannot fail is not evidence (see verify-alignment.mjs for +// the same discipline on the legacy dataset). +const { execFileSync } = require('node:child_process'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +const repoRoot = path.resolve(__dirname, '..'); +const tool = path.join(repoRoot, 'tools', 'bench', 'validate-data.mjs'); + +function runValidator(args) { + try { + const stdout = execFileSync(process.execPath, [tool, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }); + return { code: 0, output: stdout }; + } catch (err) { + return { code: err.status, output: `${err.stdout ?? ''}${err.stderr ?? ''}` }; + } +} + +describe('bench dataset validation', function () { + this.timeout(30000); + + it('accepts the committed events and cases', function () { + const r = runValidator([]); + assert.strictEqual(r.code, 0, `expected exit 0, got ${r.code}:\n${r.output}`); + assert.match(r.output, /bench:validate OK/); + }); + + it('negative control: rejects a tree with an invalid event and a dangling case reference', function () { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'updapi-validate-nc-')); + try { + // Real schemas, broken data. + fs.cpSync(path.join(repoRoot, 'schemas'), path.join(root, 'schemas'), { recursive: true }); + const eventsDir = path.join(root, 'events', 'x', 'y'); + fs.mkdirSync(eventsDir, { recursive: true }); + const goodEvent = JSON.parse(fs.readFileSync( + path.join(repoRoot, 'events', 'openai', 'openai', 'openai.openai.2026-07-27.node22-runtime-floor.json'), 'utf8')); + const broken = { ...goodEvent, change: { ...goodEvent.change, type: 'not-a-real-change-type' } }; + fs.writeFileSync(path.join(eventsDir, `${broken.id}.json`), JSON.stringify(broken)); + + const caseDir = path.join(root, 'cases', 'case-dangling'); + fs.mkdirSync(caseDir, { recursive: true }); + fs.writeFileSync(path.join(caseDir, 'case.json'), JSON.stringify({ + schema_version: '0.1.0', + case_id: 'case-dangling', + event_id: 'nobody.nothing.2026-01-01.missing', + case_version: 1, + task: 'A task string long enough to satisfy the schema minimum length requirement.', + workspace_fixture: 'fixture', + target_environment: { runtime: 'node', runtime_version: '>=22', dependencies: {} }, + setup: { install_command: 'true' }, + validator: { kind: 'command', command: 'node -e "process.exit(0)"' }, + controls: { known_stale_fixture: 'controls/stale', known_current_fixture: 'controls/current' } + })); + + const r = runValidator(['--root', root]); + assert.notStrictEqual(r.code, 0, `validator accepted a broken tree:\n${r.output}`); + assert.match(r.output, /not-a-real-change-type|must be equal to one of the allowed values/); + assert.match(r.output, /does not match any event/); + assert.match(r.output, /workspace_fixture missing/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/test/bench-harness.test.cjs b/test/bench-harness.test.cjs new file mode 100644 index 0000000..45dd28b --- /dev/null +++ b/test/bench-harness.test.cjs @@ -0,0 +1,97 @@ +// Control-harness selftest. The harness's whole job is to refuse validators +// that cannot discriminate, so the first assertion here is the negative +// control: a case whose validator passes BOTH controls must make the harness +// exit non-zero. Then a genuinely discriminating case must pass. +const { execFileSync } = require('node:child_process'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); + +const repoRoot = path.resolve(__dirname, '..'); +const harness = path.join(repoRoot, 'tools', 'bench', 'run-controls.mjs'); + +function runHarness(casesDir, extraArgs = []) { + try { + const stdout = execFileSync(process.execPath, [harness, '--json', ...extraArgs], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env, UPDAPI_CASES_DIR: casesDir } + }); + return { code: 0, output: stdout }; + } catch (err) { + return { code: err.status, output: `${err.stdout ?? ''}${err.stderr ?? ''}` }; + } +} + +function writeSyntheticCase(root, { discriminating }) { + const caseDir = path.join(root, 'case-synthetic'); + fs.mkdirSync(path.join(caseDir, 'fixture'), { recursive: true }); + fs.mkdirSync(path.join(caseDir, 'controls', 'stale'), { recursive: true }); + fs.mkdirSync(path.join(caseDir, 'controls', 'current'), { recursive: true }); + fs.mkdirSync(path.join(caseDir, 'validator'), { recursive: true }); + + fs.writeFileSync(path.join(caseDir, 'fixture', 'impl.txt'), 'STUB\n'); + fs.writeFileSync(path.join(caseDir, 'controls', 'stale', 'impl.txt'), 'OLD\n'); + fs.writeFileSync(path.join(caseDir, 'controls', 'current', 'impl.txt'), 'NEW\n'); + + const validator = discriminating + ? `const fs = require('node:fs'); + const path = require('node:path'); + const v = fs.readFileSync(path.join(__dirname, '..', 'fixture', 'impl.txt'), 'utf8').trim(); + process.exit(v === 'NEW' ? 0 : 1);` + : `process.exit(0); // accepts everything - must be refused by the harness`; + fs.writeFileSync(path.join(caseDir, 'validator', 'check.cjs'), validator); + + fs.writeFileSync(path.join(caseDir, 'case.json'), JSON.stringify({ + schema_version: '0.1.0', + case_id: 'case-synthetic', + event_id: 'synthetic.synthetic.2026-01-01.selftest', + case_version: 1, + task: 'Synthetic selftest case exercising the harness overlay/restore/expectation machinery.', + workspace_fixture: 'fixture', + solution_path: 'impl.txt', + target_environment: { runtime: 'node', runtime_version: '>=22', dependencies: {} }, + setup: { install_command: 'node -e "process.exit(0)"' }, + validator: { kind: 'command', command: 'node validator/check.cjs', timeout_ms: 30000 }, + controls: { known_stale_fixture: 'controls/stale', known_current_fixture: 'controls/current' } + }, null, 2)); + return caseDir; +} + +describe('bench control harness', function () { + this.timeout(60000); + + it('negative control: refuses a validator that passes both controls', function () { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'updapi-harness-nc-')); + try { + writeSyntheticCase(root, { discriminating: false }); + const r = runHarness(root); + assert.notStrictEqual(r.code, 0, `harness accepted a non-discriminating validator:\n${r.output}`); + const report = JSON.parse(r.output); + assert.strictEqual(report.all_discriminate, false); + assert.strictEqual(report.cases[0].controls.stale.as_expected, false, 'stale control should be flagged'); + assert.strictEqual(report.cases[0].controls.current.as_expected, true); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + + it('accepts a discriminating case and restores the fixture byte-identically', function () { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'updapi-harness-ok-')); + try { + const caseDir = writeSyntheticCase(root, { discriminating: true }); + const before = fs.readFileSync(path.join(caseDir, 'fixture', 'impl.txt'), 'utf8'); + const r = runHarness(root); + assert.strictEqual(r.code, 0, `expected exit 0:\n${r.output}`); + const report = JSON.parse(r.output); + assert.strictEqual(report.all_discriminate, true); + assert.strictEqual(report.cases[0].verdict, 'DISCRIMINATES'); + assert.strictEqual(report.cases[0].fixture_restored, true); + const after = fs.readFileSync(path.join(caseDir, 'fixture', 'impl.txt'), 'utf8'); + assert.strictEqual(after, before, 'fixture must be restored after the run'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/tools/bench/capture-registry-evidence.mjs b/tools/bench/capture-registry-evidence.mjs new file mode 100644 index 0000000..02cba23 --- /dev/null +++ b/tools/bench/capture-registry-evidence.mjs @@ -0,0 +1,37 @@ +// Captures registry evidence for a change event, reproducibly: +// node tools/bench/capture-registry-evidence.mjs +// Prints the fields an event's package_registry source is built from, stamped +// with retrieval time, so a reviewer can re-derive published_at and the +// runtime-floor evidence without trusting the event author. + +const [pkg, version] = process.argv.slice(2); +if (!pkg || !version) { + console.error('usage: node tools/bench/capture-registry-evidence.mjs '); + process.exit(2); +} + +const enc = pkg.startsWith('@') ? pkg.replace('/', '%2f') : pkg; +const docUrl = `https://registry.npmjs.org/${enc}`; +const res = await fetch(docUrl); +if (!res.ok) { + console.error(`registry returned ${res.status} for ${docUrl}`); + process.exit(1); +} +const doc = await res.json(); +const time = doc.time?.[version]; +const manifest = doc.versions?.[version]; +if (!time || !manifest) { + console.error(`version ${version} not found for ${pkg} (known: ${Object.keys(doc.time ?? {}).filter((v) => /^\d/.test(v)).slice(-8).join(', ')})`); + process.exit(1); +} +console.log(JSON.stringify({ + package: pkg, + version, + published_at: time, + latest: doc['dist-tags']?.latest ?? null, + engines: manifest.engines ?? null, + types: manifest.types ?? manifest.typings ?? null, + dist_integrity: manifest.dist?.integrity ?? null, + registry_url: docUrl, + retrieved_at: new Date().toISOString() +}, null, 2)); diff --git a/tools/bench/run-controls.mjs b/tools/bench/run-controls.mjs new file mode 100644 index 0000000..e11e382 --- /dev/null +++ b/tools/bench/run-controls.mjs @@ -0,0 +1,209 @@ +// UpdAPI control harness: proves each case's validator DISCRIMINATES. +// +// For every case (cases/*/case.json), overlays the known-stale control onto the +// fixture and requires the validator to REJECT it, then overlays the +// known-current control and requires the validator to ACCEPT it. No LLM is +// involved anywhere; this is BENCHMARK_SPEC.md section 4.3 made executable. +// +// A validator that cannot fail its stale control is not evidence of anything, +// so this harness exits non-zero unless every case shows BOTH outcomes. +// +// Usage: +// node tools/bench/run-controls.mjs [--case ] [--json] [--skip-install] +// +// The fixture is mutated in place during a run and restored afterwards +// (including on crashes); a run leaves the working tree byte-identical. + +import { spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); +const casesRoot = path.join(repoRoot, 'cases'); + +const args = process.argv.slice(2); +const wantJson = args.includes('--json'); +const skipInstall = args.includes('--skip-install'); +const caseFilter = args.includes('--case') ? args[args.indexOf('--case') + 1] : null; +// Test hook: lets the harness selftest point at a synthetic cases directory. +const casesDir = process.env.UPDAPI_CASES_DIR ? path.resolve(process.env.UPDAPI_CASES_DIR) : casesRoot; + +const log = (line) => { if (!wantJson) console.log(line); }; + +function listCases() { + if (!fs.existsSync(casesDir)) return []; + return fs.readdirSync(casesDir, { withFileTypes: true }) + .filter((d) => d.isDirectory() && fs.existsSync(path.join(casesDir, d.name, 'case.json'))) + .map((d) => d.name) + .sort(); +} + +function* walkFiles(dir) { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const p = path.join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name === 'node_modules') continue; + yield* walkFiles(p); + } else if (entry.isFile()) { + yield p; + } + } +} + +function sha256(buf) { + return createHash('sha256').update(buf).digest('hex'); +} + +// Overlay every file under controlDir into fixtureDir (same relative paths). +// Returns a restore() that puts the fixture back exactly as it was. +function overlay(controlDir, fixtureDir) { + const saved = []; + for (const src of walkFiles(controlDir)) { + const rel = path.relative(controlDir, src); + const dest = path.join(fixtureDir, rel); + const existedBefore = fs.existsSync(dest); + saved.push({ + dest, + existedBefore, + original: existedBefore ? fs.readFileSync(dest) : null + }); + fs.mkdirSync(path.dirname(dest), { recursive: true }); + fs.copyFileSync(src, dest); + } + return function restore() { + for (const s of saved) { + if (s.existedBefore) fs.writeFileSync(s.dest, s.original); + else fs.rmSync(s.dest, { force: true }); + } + }; +} + +function runCommand(command, cwd, timeoutMs) { + return new Promise((resolve) => { + const started = Date.now(); + const child = spawn(command, { cwd, shell: true, windowsHide: true }); + let out = ''; + let timedOut = false; + const cap = (chunk) => { out += chunk.toString(); if (out.length > 200000) out = out.slice(-200000); }; + child.stdout.on('data', cap); + child.stderr.on('data', cap); + const timer = setTimeout(() => { timedOut = true; child.kill(); }, timeoutMs); + child.on('close', (code) => { + clearTimeout(timer); + resolve({ exitCode: timedOut ? null : code, timedOut, output: out, durationMs: Date.now() - started }); + }); + child.on('error', (err) => { + clearTimeout(timer); + resolve({ exitCode: null, timedOut: false, output: String(err), durationMs: Date.now() - started }); + }); + }); +} + +function hashDir(dir) { + const h = createHash('sha256'); + const files = [...walkFiles(dir)].sort(); + for (const f of files) { + h.update(path.relative(dir, f).replaceAll('\\', '/')); + h.update('\0'); + h.update(fs.readFileSync(f)); + h.update('\0'); + } + return h.digest('hex'); +} + +async function runCase(caseId) { + const caseDir = path.join(casesDir, caseId); + const spec = JSON.parse(fs.readFileSync(path.join(caseDir, 'case.json'), 'utf8')); + const fixtureDir = path.join(caseDir, spec.workspace_fixture); + const timeoutMs = spec.validator.timeout_ms ?? 120000; + + const result = { + case_id: spec.case_id, + event_id: spec.event_id, + install: 'skipped', + controls: {}, + fixture_restored: null, + verdict: null + }; + + // Deterministic setup: install once per fixture when absent. + if (!skipInstall && spec.setup?.install_command && fs.existsSync(path.join(fixtureDir, 'package.json')) + && !fs.existsSync(path.join(fixtureDir, 'node_modules'))) { + log(` installing fixture deps: ${spec.setup.install_command}`); + const inst = await runCommand(spec.setup.install_command, fixtureDir, 600000); + result.install = inst.exitCode === 0 ? 'ok' : 'failed'; + if (inst.exitCode !== 0) { + result.verdict = 'INSTALL_FAILED'; + log(` INSTALL FAILED (exit ${inst.exitCode})\n${inst.output.slice(-2000)}`); + return result; + } + } else if (fs.existsSync(path.join(fixtureDir, 'node_modules'))) { + result.install = 'cached'; + } + + const preHash = hashDir(fixtureDir); + + const expectations = [ + { name: 'stale', dir: spec.controls.known_stale_fixture, expect: 'fail' }, + { name: 'current', dir: spec.controls.known_current_fixture, expect: 'pass' } + ]; + + for (const { name, dir, expect } of expectations) { + const controlDir = path.join(caseDir, dir); + const restore = overlay(controlDir, fixtureDir); + let run; + try { + run = await runCommand(spec.validator.command, caseDir, timeoutMs); + } finally { + restore(); + } + const passed = run.exitCode === 0; + const asExpected = expect === 'pass' ? passed : !passed; + result.controls[name] = { + expected: expect, + exit_code: run.exitCode, + timed_out: run.timedOut, + as_expected: asExpected, + duration_ms: run.durationMs, + output_tail: run.output.split(/\r?\n/).filter(Boolean).slice(-12) + }; + log(` control:${name} expected=${expect} exit=${run.exitCode}${run.timedOut ? ' (TIMEOUT)' : ''} -> ${asExpected ? 'AS EXPECTED' : 'VIOLATION'}`); + if (!wantJson) for (const l of result.controls[name].output_tail) log(` | ${l}`); + } + + const postHash = hashDir(fixtureDir); + result.fixture_restored = preHash === postHash; + if (!result.fixture_restored) log(' WARNING: fixture hash changed across the run (restore incomplete)'); + + const discriminates = result.controls.stale?.as_expected && result.controls.current?.as_expected; + result.verdict = discriminates && result.fixture_restored ? 'DISCRIMINATES' : 'FAILED'; + return result; +} + +const all = listCases(); +const selected = caseFilter ? all.filter((c) => c === caseFilter) : all; +if (selected.length === 0) { + console.error(caseFilter ? `no case named ${caseFilter} under ${casesDir}` : `no cases found under ${casesDir}`); + process.exit(2); +} + +const results = []; +for (const caseId of selected) { + log(`case ${caseId}`); + results.push(await runCase(caseId)); +} + +const failed = results.filter((r) => r.verdict !== 'DISCRIMINATES'); +if (wantJson) { + console.log(JSON.stringify({ ran_at: new Date().toISOString(), cases: results, all_discriminate: failed.length === 0 }, null, 2)); +} else { + console.log(''); + console.log('summary:'); + for (const r of results) console.log(` ${r.verdict === 'DISCRIMINATES' ? 'OK ' : 'FAIL'} ${r.case_id} (stale exit ${r.controls.stale?.exit_code ?? '-'}, current exit ${r.controls.current?.exit_code ?? '-'})`); + console.log(failed.length === 0 + ? `all ${results.length} case validator(s) discriminate their controls` + : `${failed.length} of ${results.length} case(s) FAILED to discriminate`); +} +process.exit(failed.length === 0 ? 0 : 1); diff --git a/tools/bench/validate-data.mjs b/tools/bench/validate-data.mjs new file mode 100644 index 0000000..ec944cd --- /dev/null +++ b/tools/bench/validate-data.mjs @@ -0,0 +1,121 @@ +// Validates the benchmark dataset against its versioned schemas, plus the +// cross-file invariants a JSON Schema cannot see: +// - every case references an existing event, and vice versa for +// executable_fixture verification +// - referenced fixture/control/solution paths exist +// - ids are unique +// Exit 0 only when everything holds. Used by `npm run bench:validate` and the +// mocha suite; deliberately dependency-light (ajv + ajv-formats only). + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import Ajv2020 from 'ajv/dist/2020.js'; +import addFormats from 'ajv-formats'; + +const argv = process.argv.slice(2); +const rootFlag = argv.indexOf('--root'); +// --root : validate a different tree (the mocha negative control uses this +// to prove the validator can actually reject a broken dataset). +const repoRoot = rootFlag >= 0 + ? path.resolve(argv[rootFlag + 1]) + : path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); +const problems = []; +const note = (file, msg) => problems.push({ file: path.relative(repoRoot, file).replaceAll('\\', '/'), msg }); + +const ajv = new Ajv2020.default({ allErrors: true, strict: true }); +addFormats.default(ajv); + +const loadSchema = (name) => { + const p = path.join(repoRoot, 'schemas', name); + return ajv.compile(JSON.parse(fs.readFileSync(p, 'utf8'))); +}; + +const validateEvent = loadSchema('change-event.schema.json'); +const validateCase = loadSchema('benchmark-case.schema.json'); +// Compiled for their own syntax validity even though v0 has no run data yet. +loadSchema('run-manifest.schema.json'); +loadSchema('run-result.schema.json'); + +function* jsonFilesUnder(dir) { + if (!fs.existsSync(dir)) return; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const p = path.join(dir, entry.name); + if (entry.isDirectory()) yield* jsonFilesUnder(p); + else if (entry.isFile() && entry.name.endsWith('.json')) yield p; + } +} + +// ---- events ---- +const events = new Map(); +for (const file of jsonFilesUnder(path.join(repoRoot, 'events'))) { + let data; + try { data = JSON.parse(fs.readFileSync(file, 'utf8')); } + catch (err) { note(file, `unparseable JSON: ${err.message}`); continue; } + if (!validateEvent(data)) { + for (const e of validateEvent.errors) note(file, `schema: ${e.instancePath || '/'} ${e.message}`); + continue; + } + if (events.has(data.id)) note(file, `duplicate event id ${data.id} (also in ${events.get(data.id).file})`); + events.set(data.id, { file, data }); + const expectedBasename = `${data.id}.json`; + if (path.basename(file) !== expectedBasename) note(file, `filename should be ${expectedBasename}`); +} + +// ---- cases ---- +const casesRoot = path.join(repoRoot, 'cases'); +const cases = new Map(); +if (fs.existsSync(casesRoot)) { + for (const dirent of fs.readdirSync(casesRoot, { withFileTypes: true })) { + if (!dirent.isDirectory()) continue; + const caseDir = path.join(casesRoot, dirent.name); + const caseFile = path.join(caseDir, 'case.json'); + if (!fs.existsSync(caseFile)) { note(caseDir, 'missing case.json'); continue; } + let data; + try { data = JSON.parse(fs.readFileSync(caseFile, 'utf8')); } + catch (err) { note(caseFile, `unparseable JSON: ${err.message}`); continue; } + if (!validateCase(data)) { + for (const e of validateCase.errors) note(caseFile, `schema: ${e.instancePath || '/'} ${e.message}`); + continue; + } + if (data.case_id !== dirent.name) note(caseFile, `case_id ${data.case_id} != directory name ${dirent.name}`); + if (cases.has(data.case_id)) note(caseFile, `duplicate case id ${data.case_id}`); + cases.set(data.case_id, { file: caseFile, data }); + + if (!events.has(data.event_id)) note(caseFile, `event_id ${data.event_id} does not match any event`); + + const fixtureDir = path.join(caseDir, data.workspace_fixture); + if (!fs.existsSync(fixtureDir)) note(caseFile, `workspace_fixture missing: ${data.workspace_fixture}`); + if (data.solution_path && !fs.existsSync(path.join(fixtureDir, data.solution_path))) { + note(caseFile, `solution_path missing in fixture: ${data.solution_path}`); + } + for (const key of ['known_stale_fixture', 'known_current_fixture']) { + const p = path.join(caseDir, data.controls[key]); + if (!fs.existsSync(p)) { note(caseFile, `${key} missing: ${data.controls[key]}`); continue; } + if (data.solution_path && !fs.existsSync(path.join(p, data.solution_path))) { + note(caseFile, `${key} does not provide solution_path ${data.solution_path}`); + } + } + } +} + +// ---- event -> case back-references ---- +for (const { file, data } of events.values()) { + if (data.verification.kind !== 'executable_fixture') continue; + const fixture = data.verification.fixture; // e.g. cases/case-x + const caseName = path.basename(fixture); + const entry = cases.get(caseName); + if (!entry) { note(file, `verification.fixture ${fixture} does not match any case directory`); continue; } + if (entry.data.event_id !== data.id) { + note(file, `verification.fixture case ${caseName} references event ${entry.data.event_id}, not ${data.id}`); + } +} + +// ---- report ---- +if (problems.length === 0) { + console.log(`bench:validate OK - ${events.size} event(s), ${cases.size} case(s), 4 schemas compile, all cross-references hold`); + process.exit(0); +} +console.error(`bench:validate FAILED - ${problems.length} problem(s):`); +for (const p of problems) console.error(` ${p.file}: ${p.msg}`); +process.exit(1); From ed44116463438f99355df1930c5adfe746862b09 Mon Sep 17 00:00:00 2001 From: AvaKim Date: Fri, 28 Aug 2026 14:34:38 +1000 Subject: [PATCH 7/8] bench: apply reviewer guardrails (round 1) From the Auto Run review round: change-event schema gains optional effective_at + verified_at (verified_at populated on all 5 events); spec 8.6 gains the system-adoption-lag framing (the unit under test is agent+model+tools, so the manifest must attribute which component moved a longitudinal metric); the MCP fixture's server-harness narration is slimmed so the workspace file reads as inspectable deployed config rather than an answer sheet, with the boundary decision documented in-file. Re-verified: bench:validate OK, MCP controls still discriminate. --- .../fixture/src/server-harness.mjs | 18 ++++++++++-------- docs/BENCHMARK_SPEC.md | 7 ++++++- ...opic-ai-sdk.2026-08-19.files-skills-ga.json | 1 + ...2026-07-27.serverinfo-into-result-meta.json | 1 + ...openai.2026-07-27.node22-runtime-floor.json | 1 + ...25.remove-experimental-custom-provider.json | 1 + ...2026-06-25.remove-toolcalloptions-type.json | 1 + schemas/change-event.schema.json | 10 ++++++++++ 8 files changed, 31 insertions(+), 9 deletions(-) diff --git a/cases/case-mcp-serverinfo-discover-v2/fixture/src/server-harness.mjs b/cases/case-mcp-serverinfo-discover-v2/fixture/src/server-harness.mjs index b45c5d2..f8af2e9 100644 --- a/cases/case-mcp-serverinfo-discover-v2/fixture/src/server-harness.mjs +++ b/cases/case-mcp-serverinfo-discover-v2/fixture/src/server-harness.mjs @@ -1,11 +1,7 @@ -// Fixture infrastructure (not part of the task): a local, modern-only MCP -// endpoint served entirely in-process. -// -// createMcpHandler serves the 2026-07-28 protocol revision from a per-request -// server factory; `legacy: 'reject'` makes the endpoint modern-only strict, so -// 2025-era traffic (the plain `initialize` handshake) is rejected rather than -// served. `handler.fetch` is the web-standard face, which means the whole -// exchange can run through an injected fetch with no sockets and no network. +// Fixture infrastructure (not part of the task): the project's MCP endpoint, +// served entirely in-process through an injected fetch - no sockets, no +// network. Treat this file as the deployed server's configuration: it can be +// inspected, not modified. import { createMcpHandler, McpServer } from '@modelcontextprotocol/server'; export const SERVER_NAME = 'updapi-fixture-server'; @@ -19,3 +15,9 @@ const handler = createMcpHandler( /** In-process fetch implementation routing every request to the endpoint. */ export const serverFetch = (input, init) => handler.fetch(new Request(input, init)); + +// Workspace-boundary note for benchmark runs: this file is legitimately +// readable infrastructure (a developer can inspect their own server config), +// so discovering the endpoint's posture from here or from its error responses +// is fair diagnosis work. What it does not contain is the client-side +// migration answer. diff --git a/docs/BENCHMARK_SPEC.md b/docs/BENCHMARK_SPEC.md index c1b2401..ced9345 100644 --- a/docs/BENCHMARK_SPEC.md +++ b/docs/BENCHMARK_SPEC.md @@ -383,9 +383,14 @@ retrieval_lift = success_rate(with_retrieval) - success_rate(control) Always show absolute rates alongside the delta. -### 8.6 API Knowledge Lag +### 8.6 API Knowledge Lag (system adoption lag) Knowledge lag is a longitudinal system/event statistic, not a one-shot score. +Because the unit under test is the deployed system (agent + model + tools), +not isolated model weights, the concept is equally correctly called **system +adoption lag**; a change in the metric can come from a model update, a tool +change, or a product change, and the run manifest must carry enough +provenance to attribute which. Provisional definition: diff --git a/events/anthropic/anthropic-ai-sdk/anthropic.anthropic-ai-sdk.2026-08-19.files-skills-ga.json b/events/anthropic/anthropic-ai-sdk/anthropic.anthropic-ai-sdk.2026-08-19.files-skills-ga.json index 39b027a..7e364d0 100644 --- a/events/anthropic/anthropic-ai-sdk/anthropic.anthropic-ai-sdk.2026-08-19.files-skills-ga.json +++ b/events/anthropic/anthropic-ai-sdk/anthropic.anthropic-ai-sdk.2026-08-19.files-skills-ga.json @@ -40,6 +40,7 @@ "kind": "versioned_source", "notes": "Verified against the published tarball's file tree (non-beta files/skills resources present at 0.119.0). Candidate for a credential-free compile/type fixture per issue #24 case suggestion 5; a live-API fixture only if credential and cost semantics are clean." }, + "verified_at": "2026-08-28T03:55:00Z", "status": "verified", "revision": 1, "notes": "Temporal-cohort value: this event was 9 days old at first observation." diff --git a/events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json b/events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json index 8fae18d..db5081a 100644 --- a/events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json +++ b/events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json @@ -41,6 +41,7 @@ "fixture": "cases/case-mcp-serverinfo-discover-v2", "notes": "Verified with a local client+server pair on the GA v2 packages over an in-memory transport. The case records the empirically observed failure mode of the stale read path; source claims about hard-vs-graceful failure differ by layer, so the fixture is the ground truth." }, + "verified_at": "2026-08-28T05:10:00Z", "status": "case_validated", "revision": 1, "notes": "published_at is the npm publish time (2026-07-27T23:55Z UTC) of the GA packages carrying the final 2026-07-28 protocol revision; the revision itself is dated 2026-07-28." diff --git a/events/openai/openai/openai.openai.2026-07-27.node22-runtime-floor.json b/events/openai/openai/openai.openai.2026-07-27.node22-runtime-floor.json index 5f9ed40..f95c137 100644 --- a/events/openai/openai/openai.openai.2026-07-27.node22-runtime-floor.json +++ b/events/openai/openai/openai.openai.2026-07-27.node22-runtime-floor.json @@ -34,6 +34,7 @@ "kind": "versioned_source", "notes": "Ground truth is the engines field of the published 7.0.0 registry manifest (node >=22.0.0), corroborated by the release notes. Candidate for a future executable migration case (environment-gated validator); measures ecosystem freshness more than API-call freshness, so it is a taxonomy-diversity event rather than a headline case." }, + "verified_at": "2026-08-28T03:55:00Z", "status": "verified", "revision": 1, "notes": "runtime_requirement_change is a v0 taxonomy extension over BENCHMARK_SPEC.md section 6; flagged for methodology review in PR #25." diff --git a/events/vercel/ai/vercel.ai.2026-06-25.remove-experimental-custom-provider.json b/events/vercel/ai/vercel.ai.2026-06-25.remove-experimental-custom-provider.json index 09543fb..8ee4e2c 100644 --- a/events/vercel/ai/vercel.ai.2026-06-25.remove-experimental-custom-provider.json +++ b/events/vercel/ai/vercel.ai.2026-06-25.remove-experimental-custom-provider.json @@ -40,6 +40,7 @@ "fixture": "cases/case-vercel-ai-custom-provider-v7", "notes": "The case validator imports the solution against pinned ai@7.0.0 and exercises provider behaviour; the known-stale control fails at import because the export is absent." }, + "verified_at": "2026-08-28T05:10:00Z", "status": "case_validated", "revision": 1, "notes": "version_before is recorded as 6.0.x: the removal ships with the 7.0.0 major while the 6.0.x line continues to receive releases in parallel, so no single 6.0.x version is 'the last before'." diff --git a/events/vercel/ai/vercel.ai.2026-06-25.remove-toolcalloptions-type.json b/events/vercel/ai/vercel.ai.2026-06-25.remove-toolcalloptions-type.json index 9210d7a..6b33153 100644 --- a/events/vercel/ai/vercel.ai.2026-06-25.remove-toolcalloptions-type.json +++ b/events/vercel/ai/vercel.ai.2026-06-25.remove-toolcalloptions-type.json @@ -40,6 +40,7 @@ "fixture": "cases/case-vercel-ai-tool-execution-options-v7", "notes": "Type removals surface at compile time by nature, so this case's deterministic validator is the pinned TypeScript compiler in strict mode - deliberately the only compile-gated case among the first three (see the reviewer constraint on tsc trivia in issue #24)." }, + "verified_at": "2026-08-28T05:10:00Z", "status": "case_validated", "revision": 1, "notes": "Shares the ai@7.0.0 release with vercel.ai.2026-06-25.remove-experimental-custom-provider but exercises the type surface rather than the runtime export surface." diff --git a/schemas/change-event.schema.json b/schemas/change-event.schema.json index e83d9ec..bfb0fea 100644 --- a/schemas/change-event.schema.json +++ b/schemas/change-event.schema.json @@ -60,6 +60,16 @@ "format": "date-time", "description": "When UpdAPI first observed the change. Never backdated." }, + "effective_at": { + "type": ["string", "null"], + "format": "date-time", + "description": "When the change takes effect for consumers, when that differs from published_at (e.g. an announced deprecation with a future sunset date). Omit or null when identical to published_at." + }, + "verified_at": { + "type": ["string", "null"], + "format": "date-time", + "description": "When UpdAPI verification (section 4.2) completed for the current revision. Expected present once status reaches verified." + }, "change": { "type": "object", "additionalProperties": false, From cdeaeeab4138214ac887eabd470fafe850234922 Mon Sep 17 00:00:00 2001 From: AvaKim Date: Fri, 28 Aug 2026 14:44:01 +1000 Subject: [PATCH 8/8] bench: round-2 review fixes - event split/rebind + temporal integrity Blocker 1 (causal binding): the strict-endpoint MCP case discriminates protocol-era negotiation, not the serverInfo field move, so it is rebound to a new event modelcontextprotocol.client.2026-07-27.opt-in-version- negotiation (version_migration; executable verification) and renamed case-mcp-serverinfo-discover-v2 -> case-mcp-modern-era-negotiation-v2. The serverinfo-into-result-meta event is retained as verified (versioned_source: the SERVER_INFO_META_KEY constant and typings in the published packages plus the migration guide, behaviourally corroborated on the negotiation case's passing path) and awaits its own clean discriminating oracle. Blocker 2 (temporal integrity): verified_at values were future-dated approximations; now machine-anchored - the three case_validated events carry the all-green control-run completion time (2026-08-28T04:28:36Z, recorded before commit a782d52 at 04:29:50Z), and the two versioned-source events carry fresh `npm run bench:evidence` re-verification retrieval times (04:41:04Z). validate-data now enforces: no future-dated timestamp anywhere except effective_at (documented 5-min skew tolerance), published_at <= first_observed_at <= verified_at where the fields exist, and verified_at required from status verified onward. The mocha negative-control tree gains two new mutants proving the future-dated and missing-verified_at rejections actually fire. Re-verified green: bench:validate (6 events / 3 cases, cross-refs hold), bench:controls (all 3 cases discriminate), npm test 35/35. --- .../README.md | 20 ++++--- .../case.json | 4 +- .../controls/current/src/solution.mjs | 0 .../controls/stale/src/solution.mjs | 0 .../fixture/package-lock.json | 0 .../fixture/package.json | 0 .../fixture/src/server-harness.mjs | 0 .../fixture/src/solution.mjs | 0 .../task.md | 0 .../validator/validate.mjs | 0 ...pic-ai-sdk.2026-08-19.files-skills-ga.json | 4 +- ...2026-07-27.opt-in-version-negotiation.json | 53 +++++++++++++++++++ ...026-07-27.serverinfo-into-result-meta.json | 9 ++-- ...penai.2026-07-27.node22-runtime-floor.json | 4 +- ...5.remove-experimental-custom-provider.json | 2 +- ...026-06-25.remove-toolcalloptions-type.json | 2 +- test/bench-data.test.cjs | 16 ++++++ tools/bench/validate-data.mjs | 28 ++++++++++ 18 files changed, 123 insertions(+), 19 deletions(-) rename cases/{case-mcp-serverinfo-discover-v2 => case-mcp-modern-era-negotiation-v2}/README.md (64%) rename cases/{case-mcp-serverinfo-discover-v2 => case-mcp-modern-era-negotiation-v2}/case.json (95%) rename cases/{case-mcp-serverinfo-discover-v2 => case-mcp-modern-era-negotiation-v2}/controls/current/src/solution.mjs (100%) rename cases/{case-mcp-serverinfo-discover-v2 => case-mcp-modern-era-negotiation-v2}/controls/stale/src/solution.mjs (100%) rename cases/{case-mcp-serverinfo-discover-v2 => case-mcp-modern-era-negotiation-v2}/fixture/package-lock.json (100%) rename cases/{case-mcp-serverinfo-discover-v2 => case-mcp-modern-era-negotiation-v2}/fixture/package.json (100%) rename cases/{case-mcp-serverinfo-discover-v2 => case-mcp-modern-era-negotiation-v2}/fixture/src/server-harness.mjs (100%) rename cases/{case-mcp-serverinfo-discover-v2 => case-mcp-modern-era-negotiation-v2}/fixture/src/solution.mjs (100%) rename cases/{case-mcp-serverinfo-discover-v2 => case-mcp-modern-era-negotiation-v2}/task.md (100%) rename cases/{case-mcp-serverinfo-discover-v2 => case-mcp-modern-era-negotiation-v2}/validator/validate.mjs (100%) create mode 100644 events/modelcontextprotocol/client/modelcontextprotocol.client.2026-07-27.opt-in-version-negotiation.json diff --git a/cases/case-mcp-serverinfo-discover-v2/README.md b/cases/case-mcp-modern-era-negotiation-v2/README.md similarity index 64% rename from cases/case-mcp-serverinfo-discover-v2/README.md rename to cases/case-mcp-modern-era-negotiation-v2/README.md index 45ec5b6..3aa595a 100644 --- a/cases/case-mcp-serverinfo-discover-v2/README.md +++ b/cases/case-mcp-modern-era-negotiation-v2/README.md @@ -1,9 +1,17 @@ -# case-mcp-serverinfo-discover-v2 +# case-mcp-modern-era-negotiation-v2 -**Event:** [`modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta`](../../events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json) β€” -the final MCP 2026-07-28 protocol revision moved server identity out of the -result body into result `_meta['io.modelcontextprotocol/serverInfo']`, carried -by the GA v2 SDK packages. +**Event:** [`modelcontextprotocol.client.2026-07-27.opt-in-version-negotiation`](../../events/modelcontextprotocol/client/modelcontextprotocol.client.2026-07-27.opt-in-version-negotiation.json) β€” +entering the MCP 2026-07-28 protocol era is opt-in on the GA v2 client +(`ClientOptions.versionNegotiation`; the default remains the plain 2025 +`initialize` handshake), and modern-only strict endpoints reject that default +handshake outright. + +*(Renamed from `case-mcp-serverinfo-discover-v2` and rebound in PR #25 round 1: +the fixture discriminates protocol-era negotiation, which is a distinct causal +intervention from the serverInfo field relocation. The +[`serverinfo-into-result-meta`](../../events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json) +event is retained as a verified historical event awaiting its own clean +discriminating oracle.)* **What the case measures.** Protocol-era adaptation with an executable local oracle: a real client against a real **modern-only strict** endpoint @@ -35,5 +43,5 @@ the solution must return the identity the fixture server actually declares. **Controls:** `controls/stale` (default client, 2025 handshake) must be rejected; `controls/current` (opt-in negotiation) must pass. Run -`npm run bench:controls -- --case case-mcp-serverinfo-discover-v2` from the +`npm run bench:controls -- --case case-mcp-modern-era-negotiation-v2` from the repo root to prove both directions. diff --git a/cases/case-mcp-serverinfo-discover-v2/case.json b/cases/case-mcp-modern-era-negotiation-v2/case.json similarity index 95% rename from cases/case-mcp-serverinfo-discover-v2/case.json rename to cases/case-mcp-modern-era-negotiation-v2/case.json index d6c3d40..189f881 100644 --- a/cases/case-mcp-serverinfo-discover-v2/case.json +++ b/cases/case-mcp-modern-era-negotiation-v2/case.json @@ -1,7 +1,7 @@ { "schema_version": "0.1.0", - "case_id": "case-mcp-serverinfo-discover-v2", - "event_id": "modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta", + "case_id": "case-mcp-modern-era-negotiation-v2", + "event_id": "modelcontextprotocol.client.2026-07-27.opt-in-version-negotiation", "case_version": 1, "task": "The workspace pins @modelcontextprotocol/client and @modelcontextprotocol/server at 2.0.0. src/server-harness.mjs serves a local, current-generation MCP endpoint entirely in-process and exports its URL plus a fetch implementation that reaches it. Implement getServerIdentity({ url, fetch }) in src/solution.mjs: connect an MCP client to that endpoint (route all HTTP through the provided fetch), obtain the server's declared identity from the connection exchange, close cleanly, and return { name, version }. No sockets, no network, no credentials.", "workspace_fixture": "fixture", diff --git a/cases/case-mcp-serverinfo-discover-v2/controls/current/src/solution.mjs b/cases/case-mcp-modern-era-negotiation-v2/controls/current/src/solution.mjs similarity index 100% rename from cases/case-mcp-serverinfo-discover-v2/controls/current/src/solution.mjs rename to cases/case-mcp-modern-era-negotiation-v2/controls/current/src/solution.mjs diff --git a/cases/case-mcp-serverinfo-discover-v2/controls/stale/src/solution.mjs b/cases/case-mcp-modern-era-negotiation-v2/controls/stale/src/solution.mjs similarity index 100% rename from cases/case-mcp-serverinfo-discover-v2/controls/stale/src/solution.mjs rename to cases/case-mcp-modern-era-negotiation-v2/controls/stale/src/solution.mjs diff --git a/cases/case-mcp-serverinfo-discover-v2/fixture/package-lock.json b/cases/case-mcp-modern-era-negotiation-v2/fixture/package-lock.json similarity index 100% rename from cases/case-mcp-serverinfo-discover-v2/fixture/package-lock.json rename to cases/case-mcp-modern-era-negotiation-v2/fixture/package-lock.json diff --git a/cases/case-mcp-serverinfo-discover-v2/fixture/package.json b/cases/case-mcp-modern-era-negotiation-v2/fixture/package.json similarity index 100% rename from cases/case-mcp-serverinfo-discover-v2/fixture/package.json rename to cases/case-mcp-modern-era-negotiation-v2/fixture/package.json diff --git a/cases/case-mcp-serverinfo-discover-v2/fixture/src/server-harness.mjs b/cases/case-mcp-modern-era-negotiation-v2/fixture/src/server-harness.mjs similarity index 100% rename from cases/case-mcp-serverinfo-discover-v2/fixture/src/server-harness.mjs rename to cases/case-mcp-modern-era-negotiation-v2/fixture/src/server-harness.mjs diff --git a/cases/case-mcp-serverinfo-discover-v2/fixture/src/solution.mjs b/cases/case-mcp-modern-era-negotiation-v2/fixture/src/solution.mjs similarity index 100% rename from cases/case-mcp-serverinfo-discover-v2/fixture/src/solution.mjs rename to cases/case-mcp-modern-era-negotiation-v2/fixture/src/solution.mjs diff --git a/cases/case-mcp-serverinfo-discover-v2/task.md b/cases/case-mcp-modern-era-negotiation-v2/task.md similarity index 100% rename from cases/case-mcp-serverinfo-discover-v2/task.md rename to cases/case-mcp-modern-era-negotiation-v2/task.md diff --git a/cases/case-mcp-serverinfo-discover-v2/validator/validate.mjs b/cases/case-mcp-modern-era-negotiation-v2/validator/validate.mjs similarity index 100% rename from cases/case-mcp-serverinfo-discover-v2/validator/validate.mjs rename to cases/case-mcp-modern-era-negotiation-v2/validator/validate.mjs diff --git a/events/anthropic/anthropic-ai-sdk/anthropic.anthropic-ai-sdk.2026-08-19.files-skills-ga.json b/events/anthropic/anthropic-ai-sdk/anthropic.anthropic-ai-sdk.2026-08-19.files-skills-ga.json index 7e364d0..247c88f 100644 --- a/events/anthropic/anthropic-ai-sdk/anthropic.anthropic-ai-sdk.2026-08-19.files-skills-ga.json +++ b/events/anthropic/anthropic-ai-sdk/anthropic.anthropic-ai-sdk.2026-08-19.files-skills-ga.json @@ -38,9 +38,9 @@ ], "verification": { "kind": "versioned_source", - "notes": "Verified against the published tarball's file tree (non-beta files/skills resources present at 0.119.0). Candidate for a credential-free compile/type fixture per issue #24 case suggestion 5; a live-API fixture only if credential and cost semantics are clean." + "notes": "Verified against the published tarball's file tree (non-beta files/skills resources present at 0.119.0). verified_at is the retrieval time of a `npm run bench:evidence -- @anthropic-ai/sdk 0.119.0` re-verification run against the registry manifest. Candidate for a credential-free compile/type fixture per issue #24 case suggestion 5; a live-API fixture only if credential and cost semantics are clean." }, - "verified_at": "2026-08-28T03:55:00Z", + "verified_at": "2026-08-28T04:41:04Z", "status": "verified", "revision": 1, "notes": "Temporal-cohort value: this event was 9 days old at first observation." diff --git a/events/modelcontextprotocol/client/modelcontextprotocol.client.2026-07-27.opt-in-version-negotiation.json b/events/modelcontextprotocol/client/modelcontextprotocol.client.2026-07-27.opt-in-version-negotiation.json new file mode 100644 index 0000000..f77f159 --- /dev/null +++ b/events/modelcontextprotocol/client/modelcontextprotocol.client.2026-07-27.opt-in-version-negotiation.json @@ -0,0 +1,53 @@ +{ + "schema_version": "0.1.0", + "id": "modelcontextprotocol.client.2026-07-27.opt-in-version-negotiation", + "provider": "modelcontextprotocol", + "ecosystem": "npm", + "package": "@modelcontextprotocol/client", + "interface": "Client.connect / ClientOptions.versionNegotiation", + "version_before": null, + "version_after": "2.0.0", + "published_at": "2026-07-27T23:55:22.113Z", + "first_observed_at": "2026-08-28T04:22:27Z", + "verified_at": "2026-08-28T04:28:36Z", + "change": { + "type": "version_migration", + "secondary_types": ["configuration_change", "error_behavior_change"], + "summary": "Entering the MCP 2026-07-28 protocol era is opt-in on the GA v2 client: ClientOptions.versionNegotiation (mode 'auto' probes with server/discover; { pin } is strict) with the DEFAULT remaining the plain 2025 initialize handshake. Against a modern-only strict endpoint (createMcpHandler with legacy:'reject' - a supported deployment posture) the default handshake is rejected with JSON-RPC -32022 Unsupported protocol version, a hard connect failure. Code written with 2025-era MCP knowledge therefore fails to connect to modern-only servers at all.", + "old": "await new Client(info).connect(transport) // plain 2025 initialize handshake (still the default)", + "new": "await new Client(info, { versionNegotiation: { mode: 'auto' } }).connect(transport)" + }, + "sources": [ + { + "kind": "official_release", + "url": "https://github.com/modelcontextprotocol/typescript-sdk/releases/tag/%40modelcontextprotocol%2Fserver%402.0.0", + "retrieved_at": "2026-08-28T03:35:00Z", + "note": "Release notes describe the probe/negotiation behaviour and the hard connect failure a pre-final peer hits against a modern-only server." + }, + { + "kind": "migration_guide", + "url": "https://github.com/modelcontextprotocol/typescript-sdk/blob/main/docs/migration/support-2026-07-28.md", + "retrieved_at": "2026-08-28T03:20:00Z" + }, + { + "kind": "source_diff", + "url": "https://unpkg.com/@modelcontextprotocol/client@2.0.0/dist/index.d.mts", + "retrieved_at": "2026-08-28T04:22:00Z", + "note": "Published typings document VersionNegotiationOptions: default 'legacy' ('the default is legacy: absent, connect() runs the plain 2025 sequence'), 'auto' probe semantics, and { pin } strictness." + }, + { + "kind": "package_registry", + "url": "https://registry.npmjs.org/@modelcontextprotocol/client", + "retrieved_at": "2026-08-28T03:44:00Z", + "note": "time['2.0.0'] = 2026-07-27T23:55:22.113Z." + } + ], + "verification": { + "kind": "executable_fixture", + "fixture": "cases/case-mcp-modern-era-negotiation-v2", + "notes": "Executable both-sides verification on the GA packages: the default-client control is rejected by a modern-only strict endpoint with -32022 (requested 2025-11-25, supported ['2026-07-28']) while the versionNegotiation:'auto' control connects, negotiates the modern era, and reads the server identity. Additionally observed: over a raw in-memory transport the default pair still connects legacy-style and McpServer answers server/discover with -32601, so the era mechanism is endpoint-surface-specific." + }, + "status": "case_validated", + "revision": 1, + "notes": "version_before is null because @modelcontextprotocol/client is a new package introduced by the v2 split: the pre-change state is the 2025-era handshake as implemented by any pre-2026-07-28 client, including @modelcontextprotocol/sdk <= 1.30.0. Split from modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta after methodology review (PR #25 round 1): the two changes shipped together, but era negotiation and the serverInfo field relocation are distinct causal interventions, and the executable case discriminates the former." +} diff --git a/events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json b/events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json index db5081a..e2da0bf 100644 --- a/events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json +++ b/events/modelcontextprotocol/server/modelcontextprotocol.server.2026-07-27.serverinfo-into-result-meta.json @@ -37,12 +37,11 @@ } ], "verification": { - "kind": "executable_fixture", - "fixture": "cases/case-mcp-serverinfo-discover-v2", - "notes": "Verified with a local client+server pair on the GA v2 packages over an in-memory transport. The case records the empirically observed failure mode of the stale read path; source claims about hard-vs-graceful failure differ by layer, so the fixture is the ground truth." + "kind": "versioned_source", + "notes": "Verified against the published GA packages: both @modelcontextprotocol/client@2.0.0 and /server@2.0.0 export SERVER_INFO_META_KEY = 'io.modelcontextprotocol/serverInfo' (observed via package import), DiscoverResult in the published typings declares no serverInfo body field, and the migration guide documents the _meta location with graceful anonymous fallback. Corroborated behaviourally: on a negotiated modern connection (case-mcp-modern-era-negotiation-v2 passing path) the SDK-derived server identity is readable. No dedicated executable case yet - a clean discriminating oracle for the field relocation alone (stale body-read vs current _meta-read on an established modern connection) is future work; see PR #25 round-1 review." }, - "verified_at": "2026-08-28T05:10:00Z", - "status": "case_validated", + "verified_at": "2026-08-28T04:21:22Z", + "status": "verified", "revision": 1, "notes": "published_at is the npm publish time (2026-07-27T23:55Z UTC) of the GA packages carrying the final 2026-07-28 protocol revision; the revision itself is dated 2026-07-28." } diff --git a/events/openai/openai/openai.openai.2026-07-27.node22-runtime-floor.json b/events/openai/openai/openai.openai.2026-07-27.node22-runtime-floor.json index f95c137..0b98245 100644 --- a/events/openai/openai/openai.openai.2026-07-27.node22-runtime-floor.json +++ b/events/openai/openai/openai.openai.2026-07-27.node22-runtime-floor.json @@ -32,9 +32,9 @@ ], "verification": { "kind": "versioned_source", - "notes": "Ground truth is the engines field of the published 7.0.0 registry manifest (node >=22.0.0), corroborated by the release notes. Candidate for a future executable migration case (environment-gated validator); measures ecosystem freshness more than API-call freshness, so it is a taxonomy-diversity event rather than a headline case." + "notes": "Ground truth is the engines field of the published 7.0.0 registry manifest (node >=22.0.0), corroborated by the release notes. verified_at is the retrieval time of a `npm run bench:evidence -- openai 7.0.0` run, which re-derives this claim (engines observed: node >=22.0.0). Candidate for a future executable migration case (environment-gated validator); measures ecosystem freshness more than API-call freshness, so it is a taxonomy-diversity event rather than a headline case." }, - "verified_at": "2026-08-28T03:55:00Z", + "verified_at": "2026-08-28T04:41:04Z", "status": "verified", "revision": 1, "notes": "runtime_requirement_change is a v0 taxonomy extension over BENCHMARK_SPEC.md section 6; flagged for methodology review in PR #25." diff --git a/events/vercel/ai/vercel.ai.2026-06-25.remove-experimental-custom-provider.json b/events/vercel/ai/vercel.ai.2026-06-25.remove-experimental-custom-provider.json index 8ee4e2c..263d8a2 100644 --- a/events/vercel/ai/vercel.ai.2026-06-25.remove-experimental-custom-provider.json +++ b/events/vercel/ai/vercel.ai.2026-06-25.remove-experimental-custom-provider.json @@ -40,7 +40,7 @@ "fixture": "cases/case-vercel-ai-custom-provider-v7", "notes": "The case validator imports the solution against pinned ai@7.0.0 and exercises provider behaviour; the known-stale control fails at import because the export is absent." }, - "verified_at": "2026-08-28T05:10:00Z", + "verified_at": "2026-08-28T04:28:36Z", "status": "case_validated", "revision": 1, "notes": "version_before is recorded as 6.0.x: the removal ships with the 7.0.0 major while the 6.0.x line continues to receive releases in parallel, so no single 6.0.x version is 'the last before'." diff --git a/events/vercel/ai/vercel.ai.2026-06-25.remove-toolcalloptions-type.json b/events/vercel/ai/vercel.ai.2026-06-25.remove-toolcalloptions-type.json index 6b33153..7bb6c17 100644 --- a/events/vercel/ai/vercel.ai.2026-06-25.remove-toolcalloptions-type.json +++ b/events/vercel/ai/vercel.ai.2026-06-25.remove-toolcalloptions-type.json @@ -40,7 +40,7 @@ "fixture": "cases/case-vercel-ai-tool-execution-options-v7", "notes": "Type removals surface at compile time by nature, so this case's deterministic validator is the pinned TypeScript compiler in strict mode - deliberately the only compile-gated case among the first three (see the reviewer constraint on tsc trivia in issue #24)." }, - "verified_at": "2026-08-28T05:10:00Z", + "verified_at": "2026-08-28T04:28:36Z", "status": "case_validated", "revision": 1, "notes": "Shares the ai@7.0.0 release with vercel.ai.2026-06-25.remove-experimental-custom-provider but exercises the type surface rather than the runtime export surface." diff --git a/test/bench-data.test.cjs b/test/bench-data.test.cjs index 5531a83..ca808ca 100644 --- a/test/bench-data.test.cjs +++ b/test/bench-data.test.cjs @@ -41,6 +41,20 @@ describe('bench dataset validation', function () { const broken = { ...goodEvent, change: { ...goodEvent.change, type: 'not-a-real-change-type' } }; fs.writeFileSync(path.join(eventsDir, `${broken.id}.json`), JSON.stringify(broken)); + // Second broken event: claims verified status while carrying no verified_at. + const broken2 = { ...goodEvent, id: 'openai.openai.2026-07-27.no-verified-at', status: 'verified' }; + delete broken2.verified_at; + fs.writeFileSync(path.join(eventsDir, `${broken2.id}.json`), JSON.stringify(broken2)); + + // Third broken event: schema-valid but future-dated verified_at, so the + // temporal-integrity check (not the schema) must be what rejects it. + const broken3 = { + ...goodEvent, + id: 'openai.openai.2026-07-27.future-verified-at', + verified_at: new Date(Date.now() + 86_400_000).toISOString() + }; + fs.writeFileSync(path.join(eventsDir, `${broken3.id}.json`), JSON.stringify(broken3)); + const caseDir = path.join(root, 'cases', 'case-dangling'); fs.mkdirSync(caseDir, { recursive: true }); fs.writeFileSync(path.join(caseDir, 'case.json'), JSON.stringify({ @@ -61,6 +75,8 @@ describe('bench dataset validation', function () { assert.match(r.output, /not-a-real-change-type|must be equal to one of the allowed values/); assert.match(r.output, /does not match any event/); assert.match(r.output, /workspace_fixture missing/); + assert.match(r.output, /verified_at is future-dated/); + assert.match(r.output, /status verified requires verified_at/); } finally { fs.rmSync(root, { recursive: true, force: true }); } diff --git a/tools/bench/validate-data.mjs b/tools/bench/validate-data.mjs index ec944cd..03f560b 100644 --- a/tools/bench/validate-data.mjs +++ b/tools/bench/validate-data.mjs @@ -60,6 +60,34 @@ for (const file of jsonFilesUnder(path.join(repoRoot, 'events'))) { events.set(data.id, { file, data }); const expectedBasename = `${data.id}.json`; if (path.basename(file) !== expectedBasename) note(file, `filename should be ${expectedBasename}`); + + // Temporal integrity (PR #25 round-1 gate): nothing may be future-dated + // except effective_at (announced future changes), and where the fields + // exist they must satisfy published_at <= first_observed_at <= verified_at. + const skewMs = 5 * 60 * 1000; // documented clock-skew tolerance + const nowMs = Date.now(); + const parseAt = (label, value) => { + if (value == null) return null; + const t = Date.parse(value); + if (Number.isNaN(t)) { note(file, `${label} is not a parseable date-time: ${value}`); return null; } + return t; + }; + const noFuture = (label, value) => { + const t = parseAt(label, value); + if (t !== null && t > nowMs + skewMs) note(file, `${label} is future-dated: ${value}`); + return t; + }; + const pub = noFuture('published_at', data.published_at); + const obs = noFuture('first_observed_at', data.first_observed_at); + const ver = noFuture('verified_at', data.verified_at); + parseAt('effective_at', data.effective_at); // may be future by design + (data.sources ?? []).forEach((s, i) => noFuture(`sources[${i}].retrieved_at`, s.retrieved_at)); + (data.revisions ?? []).forEach((r, i) => noFuture(`revisions[${i}].at`, r.at)); + if (pub !== null && obs !== null && pub > obs + skewMs) note(file, 'published_at is after first_observed_at'); + if (obs !== null && ver !== null && obs > ver + skewMs) note(file, 'first_observed_at is after verified_at'); + if (['verified', 'case_authored', 'case_validated', 'embargoed', 'published'].includes(data.status) && !data.verified_at) { + note(file, `status ${data.status} requires verified_at`); + } } // ---- cases ----