From d53572d9f1ba30100b7ea2afbbdd7845b9b2d8bd Mon Sep 17 00:00:00 2001 From: colwill <62014134+colwill@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:14:38 +0100 Subject: [PATCH 1/2] feat: add parse cache, metric folding, mcp commands, node king matching to int instead of strings, parallel graph constructor via par.rs --- .github/workflows/ccc-pages.yaml | 99 ---- AGENTS.md | 3 +- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 434 +-------------- docs/Advanced.md | 261 +++++++++ docs/CLI.md | 400 ++++++++++++++ docs/DEPS.md | 274 ---------- docs/EXTENSION.md | 113 ---- docs/EXTERNALS.md | 230 -------- docs/LANGUAGES.md | 63 --- docs/MCP.md | 58 -- docs/PERF.md | 31 -- docs/PIPELINES.md | 76 --- docs/STREAM.md | 33 -- docs/TESTING.md | 64 --- extensions/vscode/README.md | 277 ++-------- extensions/vscode/package.json | 40 +- extensions/vscode/src/commands.ts | 10 - extensions/vscode/src/config.ts | 15 +- extensions/vscode/src/extension.ts | 5 +- extensions/vscode/src/server.ts | 4 + extensions/vscode/src/session.ts | 5 - src/changes.rs | 251 ++++++--- src/coverage.rs | 147 +++-- src/deps.rs | 3 +- src/extract.rs | 320 +++++++---- src/html.rs | 830 +---------------------------- src/insights.rs | 568 +++++++++++--------- src/inspect.rs | 75 ++- src/lib.rs | 4 + src/main.rs | 153 +++--- src/model.rs | 48 +- src/par.rs | 50 ++ src/parse_cache.rs | 178 +++++++ src/prompts.rs | 8 +- src/scan.rs | 11 +- src/search.rs | 641 ++++++++++++++++++++++ src/serve.rs | 299 +++++------ src/telemetry.rs | 16 +- src/violations.rs | 636 ++++++++++++++++++++++ 41 files changed, 3434 insertions(+), 3303 deletions(-) delete mode 100644 .github/workflows/ccc-pages.yaml create mode 100644 docs/Advanced.md create mode 100644 docs/CLI.md delete mode 100644 docs/DEPS.md delete mode 100644 docs/EXTENSION.md delete mode 100644 docs/EXTERNALS.md delete mode 100644 docs/LANGUAGES.md delete mode 100644 docs/MCP.md delete mode 100644 docs/PERF.md delete mode 100644 docs/PIPELINES.md delete mode 100644 docs/STREAM.md delete mode 100644 docs/TESTING.md create mode 100644 src/par.rs create mode 100644 src/parse_cache.rs create mode 100644 src/search.rs create mode 100644 src/violations.rs diff --git a/.github/workflows/ccc-pages.yaml b/.github/workflows/ccc-pages.yaml deleted file mode 100644 index 9f673fb..0000000 --- a/.github/workflows/ccc-pages.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: Publish insights to GitHub Pages - -# Builds the `ccc insights` page as a static site. -# One-time setup: Settings -> Pages -> Build and deployment -> Source: -# "GitHub Actions". The site then appears at -# https://.github.io// - -on: - push: - branches: [main] - workflow_dispatch: - -permissions: - contents: read - pages: write - id-token: write - -# never let two deploys race; let a newer commit win -concurrency: - group: pages - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # REQUIRED: the 'test triggers' tab diffs against a base ref, and tags - # are needed to pick one. A shallow clone has neither. - fetch-depth: 0 - - - name: Install Rust - uses: dtolnay/rust-toolchain@stable - - - name: Cache cargo build - uses: Swatinem/rust-cache@v2 - - - name: Build ccc - run: cargo build --release --locked - - - name: Pick alt base for test triggers - id: base - run: | - # use ref history and take the first that - # actually moved some functions, so the page always demonstrates what - # a real working tree looks like against origin/main. - # - # `ccc insights` is the source of truth here: a docs-only commit - # changes files without changing any function, and would still leave - # the tab empty. - candidates=( - "$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || true)" - "HEAD~5" "HEAD~20" "HEAD~50" - "$(git rev-list --max-parents=0 HEAD | tail -1)" - ) - chosen="" - for ref in "${candidates[@]}"; do - [ -z "$ref" ] && continue - git rev-parse --verify --quiet "${ref}^{commit}" >/dev/null || continue - n="$(./target/release/ccc insights . --base "$ref" \ - | jq '.test_triggers.counts.changed_functions // 0')" - echo " $ref -> ${n} changed function(s)" - if [ "${n:-0}" -gt 0 ]; then chosen="$ref"; break; fi - done - echo "ref=${chosen}" >> "$GITHUB_OUTPUT" - if [ -n "$chosen" ]; then - echo "Test triggers will diff against $chosen" - else - echo "Nothing moved against any candidate; using the default base" - fi - - - name: Generate the insights page - run: | - if [ -n "${{ steps.base.outputs.ref }}" ]; then - ./target/release/ccc insights . \ - --html public/index.html \ - --base "${{ steps.base.outputs.ref }}" - else - ./target/release/ccc insights . --html public/index.html - fi - ls -lh public/index.html - - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - with: - path: public - - deploy: - needs: build - runs-on: ubuntu-latest - environment: - name: github-pages - url: ${{ steps.deploy.outputs.page_url }} - steps: - - name: Deploy to GitHub Pages - id: deploy - uses: actions/deploy-pages@v4 diff --git a/AGENTS.md b/AGENTS.md index b48844d..09fc0e6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,8 +13,7 @@ to pick the new one up. - bash/grep shouldn't be used for understanding the project - IF `ccc` tool calls are unable to find a term your are searching for; stop the session and respond with `CCC: unable to find in ccc using calls: [calls]` - Every interaction: use `ccc` tool calls to gather information about the source of this project. -- All thinking, navigation, and questions about the codebase go through the MCP server tools: (index, find, references, dependencies, vulnerabilities, deps, security, file, notes, changes, prompts, pr_summary, test_triggers, test_targets, lints, hot, services, refresh) -- When I ask to *see* the analysis, call `insights` - it opens the insights UI in my browser +- All thinking, navigation, and questions about the codebase go through the MCP server tools: (index, find, references, dependencies, vulnerabilities, deps, security, file, notes, changes, prompts, pr_summary, test_triggers, test_targets, lints, hot, services, search, violations, refresh) - ccc is the only writer in this project. Every change goes through its edit tools, never through your own Edit/Write/MultiEdit/NotebookEdit tools, `sed -i`, or shell redirection: - a symbol: take the handle a `find` / `references` answer ends with and pass it to `edit_rename`, diff --git a/Cargo.lock b/Cargo.lock index 91031e0..49115d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -378,7 +378,7 @@ checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" [[package]] name = "codecache" -version = "1.7.0" +version = "1.8.0" dependencies = [ "age", "age-core", diff --git a/Cargo.toml b/Cargo.toml index e88a74a..1b1e6f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "codecache" -version = "1.7.0" +version = "1.8.0" edition = "2021" rust-version = "1.77" description = "Code Change Capture (ccc) generates an agent friendly local source map" diff --git a/README.md b/README.md index 7929eb7..8b072be 100644 --- a/README.md +++ b/README.md @@ -2,436 +2,52 @@ [![Release Code Change Capture](https://github.com/colwill/ccc/actions/workflows/ccc-release.yaml/badge.svg)](https://github.com/colwill/ccc/actions/workflows/ccc-release.yaml) -

Code Change Capture Visualiser Example

# Code Change Capture (`ccc`) -What is becoming obvious is that understanding the changes being made by LLMs/Agents to projects is an increasingly complex problem. +The `ccc` static analyser maps your project into memory in under half a second. +It supports C99, C++20 (except modules), C#, Rust, Go, Python, Zig, Odin, TypeScript, JavaScript and Protobuf. -## Table of content +Before you commit, it shows everything a change touches: the functions, tests and services it reaches, the contracts between services, dependencies and vulnerabilities. -
-Expand contents +With some configuration of `map.json`; **ccc**'s call tracing can bridge the gap between development and testing by choosing the **minimal** required tests (interop, unit, contract & smoke) for any given change. -- [The human contract](#the-human-contract) -- [Why does this project matter](#why-does-this-project-matter) -- [Quick-Start](#quick-start) -- [Acknowledgement](#acknowledgement) -- [Edits go through ccc](#edits-go-through-ccc) -- [Stats for heavy model usage](#stats-for-heavy-model-usage) -- [Usage](#usage) -- [Insights](#insights) -- [Architecture Visualiser](#architecture-visualiser) -- [Extension](#extension) -- [Replays and Secrets](#replays-and-secrets) -- [Dependency Map](#dependencymap) -- [Cross-Repository Calls](#externals) -- [Agents.md](#agentsmd) -- [Test Triggers](#testing) -- [Pipelines (CICD)](#pipelines) -- [Performance](#performance) -- [MCP Server for Agents](#mcp) -- [Token Stream](#stream) +This aims to help teams make changes faster, improve the developer experience and eliminate the long wait times for CI and CT. -
+## Extra Controls for Engineers -## The human contract +Because the static analyser provides an overwhelming amount of data; I've added support for MCP. With this, agents work differently. -The "human contract" as I've started to call it, centres around the idea that if you ask someone to review your work, code or idea that you are effectively asking them for help. If that work was not created by you then asking a human to review it; is in violation of the human contract. +- **Controls** - All read and write operations happen in-memory before committing to disk +- **Controls** - All source changes are atomic and revertable in a single call +- **Controls** - All operations can be replayed for PR or code-review sessions at any time +- **Controls** - All replays are encrypted by default for repo owners [runccc teams](https://teams.runccc.dev) +- **Info Sec** - Search can be restricted to allow/block list +- **Info Sec** - Bash/terminal use is disabled +- **Audit** - Violations are tracked by ccc and reported on for the engineer -This **was** widely understood and agreed on by software engineers across industry; you create a PR and ask some colleagues to review it, they didn't have to but they did because they would ask of you the same - ending the cycle. This is now being lost. Some are happy, some are not. Overall it seems to be to our detriment. - -The human contract and conversation is the bulk of value in the PR review process, human thinking, edge cases, maintainability, how a product or consumer would be affected by a change etc. Slapping an agent onto your PR as-is invalidates the human contract and can cause significant frustration when engineers are reading through a change that is mostly written by an agent. It's hard to not consider this wasted time as an engineer. - -Now, PRs are generated and reviewed by agents with little-to-no oversight, however; the engineers are still responsible for the code they push in work. This creates a dynamic that tends towards laziness and frustration from both AI usage on both ends and an under-appreciation of what a technical craftsman brings to a project. - -Let's keep the human contract and make it easier to honour. - - -## Why does this project matter? - -`Code Change Capture` shows everything a change by you or an agent touches (before you commit it!) visually in your IDE, in your pipeline or anywhere else you need to see it. This includes; functions, tests, services and cross-service contracts it reaches. - -`ccc` is a static analyser. It looks at your project (the working dir) and generates an in-memory cache in under half a second. This cache is read by LLMs/agents over MCP for all discovery, decision making, understanding and edits to the project. All changes made by the LLMs/Agent are tracked and can be pushed alongside a PR for the team to review with *less effort*. - -The analyser can also determine what tests are going to be triggered by changes, where there are testing gaps for unit, integration, smoke etc. It also has access to a change stream from the current session which captures what an engineer has asked, what the agent viewed, reasoned about, changed or discarded in response to that ask. - -The change stream is played back to the engineers in a PR or in their own IDE's for rapid review and understanding of the why, where and how of a PR. - -Two side-effects of ccc: -1. It can drop token usage between 5-25% (ymmv) -2. It adds a layer of security by constraining agents to MCP tool calls (cannot use bash, sed etc). - - -In summary: ccc reduces the *burden of the human contract when LLMs are involved* by providing a tool to capture and visualise changes for humans - whilst enabling LLMs/Agents to make the changes they need to with significant oversight by the human in charge. - - -## Quick-Start - -1. **Install** - - a. Latest github release (Linux / macOS / Windows) - - ```sh - curl -fsSL https://raw.githubusercontent.com/colwill/ccc/main/install.sh | bash - ``` - b. or build from source - ```sh - cargo build --release && ./target/release/ccc -- install - ``` - -2. **Initialise `ccc init` to generate the basic `.ccc/map.json`** - - a. (recommended) edit dependency map `.ccc/map.json` to include service locations and cross-service calls - -3. **Start local MCP `ccc run`** - - a. (recommended) visit `http://127.0.0.1:6767/insights` for Insights UI - -4. **Register MCP server with tooling** - - a. claude: `claude mcp add --transport http ccc http://127.0.0.1:6767/mcp` - - b. copilot: `copilot mcp add --transport http ccc http://127.0.0.1:6767/mcp` - -5. **Instruct your model to use the MCP tool `ccc`** - -6. **Work as usual** - - -## Acknowledgement - -**ccc** stands on the shoulders of [Tree-Sitter](https://github.com/tree-sitter/tree-sitter). It uses tree-sitter to scan projects with multiple langauge types and facilitates building the **ccc** code map in memory quickly. - -It is designed to give engineers a always-fresh index of a project, the latest changes, how those changes impact tests or other branches (compare working branch against any other branch). In addition it also provides language models a local MCP server for an always up-to-date map of your codebase, dependencies, call-graph and cross-service calls. - -Supports: `C99`, `C++ (20 except modules)`, `C#`, `Rust`, `Go`, `Python`, `Zig`, `Odin`, `TypeScript`, `Protobuf` -& `JavaScript` - see [`LANGUAGES.md`](docs/LANGUAGES.md) for what each one resolves. - -## Edits go through ccc - -Agents don't only read the map - they change the code through it. A `find` or `references` answer ends with a -handle, and the `edit_*` MCP tools take that handle. One call renames a symbol across the project, deletes a dead -function, or adds code beside a definition. The agent never opens each file or edits it line by line. - -- **One call per change, not one per site.** `edit_rename` follows a name through every file the map ties to it, - including lines the map does not index, such as a `ccc:skip` body. It lists every same-named identifier it left - alone, with the reason, so nothing changes silently. -- **Safe by construction.** Every write stays inside the project root: no `..`, no symlinks, never `.git`. Each - change is staged and shown as a diff first. On apply, every file is checked against what it held when staged, - and then every file is written or none is. The map is rescanned before the call returns, and `edit_revert` - takes the change back. -- **ccc is the only writer.** The MCP server tells agents to make every change through these tools, never through - their own Edit/Write tools or `sed -i`. That way every change is confined, previewed and revertible, and - [`ccc prompts`](docs/MCP.md#editing) credits it to the request that asked for it. - -## Stats for heavy model usage - -Project-wide rename, dead-code delete, new functions beside an old one are a single confined, previewed and revertible atomic call that costs a fraction of the tokens read-then-edit does with an extra layer of safety ([more info](#edits-go-through-ccc)) - -Napkin math. Assumptions: 300-line source files at ~12 tokens a line, and an agent that must read a file before it -edits it, as Claude Code does. - -| Task | Agent's own tools | Through ccc | Saving | -|---|---|---|---| -| Rename a function used at 20 sites in 8 files | grep + 8 reads + 20 edits ≈ 33k tokens, 29 calls | `references` + `edit_rename` + `edit_apply` ≈ 2.3k tokens, 3 calls | ~14x tokens, ~10x calls | -| Delete a dead 50-line function | grep + read + edit ≈ 4.4k tokens, 3 calls | `references` + `edit_delete` ≈ 1k tokens, 2 calls | ~4x tokens | -| Add a 25-line function beside an existing one | grep + read + edit ≈ 4.2k tokens, 3 calls | `find` + `edit_insert` ≈ 0.9k tokens, 2 calls | ~4.5x tokens | -| Change a few lines inside one function | read + edit | read + `edit_text` | about even | - -Every call saved is also a model turn saved, and each turn re-reads the whole conversation. On a 60k-token -session, the rename's 26 fewer turns skip roughly 1.5M cached input tokens and 26 round trips of latency. - -## Usage +## Install ```sh -ccc run # runs in-memory map and MCP server -ccc run --vis # opens the architecture visualiser at `:6767/vis` -ccc init # Generate basic `.ccc/map.json` -ccc changes [PATH] --telemetry # Changes vs base ref (services to test for CT) -ccc tokenize # Encode in-memory map of project into tokens.bin + tokens.json -ccc deps [PATH] # Dependency delta (for CI as JSON) -ccc prompts [PATH] # Which requests produced specific changes (JSON) -ccc insights [PATH] --html # The insights analysis as JSON (call graph, triggers, lints) -ccc sast [PATH] # Security findings, defaults to non-zero on a high finding -ccc audit [PATH] # Resolve lockfiles and check against the OSV advisory db -ccc install [--dir] # Install the ccc binary onto your PATH (Linux) -ccc scan [PATH] --dir= --tokens # Parse tree and report map (legacy) +curl -fsSL https://raw.githubusercontent.com/colwill/ccc/main/install.sh | bash ``` -### ccc run - -`ccc run` builds the project map in memory and makes it queryable by MCP via tool calls and viewable by the local web UI @ `:6767/insights`. - -## ccc run --vis (Architecture Visualiser) - -`ccc run --vis` serves a 2D map of the project at `http://127.0.0.1:6767/vis` and opens it in your -browser. It follows the C4 model from the whole system down to the code, then goes one level -further: the logic of a single function, drawn as a node graph. - -| level | boxes | arrows | -|---|---|---| -| Context | the system; peer repos from `externals` in `.ccc/map.json`; endpoints nothing in the map answers | `ccc:calls` / `ccc:serves` crossings, by transport | -| Containers | services from `.ccc/map.json`, or top-level directories without one | resolved and declared calls between services | -| Components | the files in one container; other containers it touches sit outside the boundary | file-to-file calls | -| Code | the functions in one file, each with an input pin and one output pin per function it calls | resolved calls; functions in other files as stand-ins | -| Logic | one function: entry, calls, branches, switch/match/try arms, loops, closures, returns and throws | execution order | - -Double-click (or Enter) drills down a level, Escape goes back up, and the breadcrumb jumps to any -level. Every view has its own URL, so the browser's back button works and views can be -bookmarked. Drag to rearrange; positions are remembered per view. The search box (`/`) finds -functions and files and opens them directly. The page reloads by itself when the map changes. - -Every level is built from the same language-agnostic map the MCP tools read, so all supported -languages are drawn the same way. Like the rest of ccc it reads syntax, not behaviour: an arrow -is a call the resolver found evidence for, and a branch is a node in the syntax tree, not a path -anything was seen to take. Calls that reach nothing in the project (the standard library, a -dependency) are folded into grey pills, and the toolbar can show them in full or hide them. - -Along the foot of the page runs a player for what agents did through ccc - the same in a browser -as in the editor. Each ask plays as a story: the agent's plan, every step it took and how it -ended, told in subtitles and read aloud, with the bar cut into chapters and a rail beside the -canvas listing them: - -- **Chapters.** One for each part of the work, named for what the agent said its steps were for - - the intent on its edit calls, the why on its reads. The chapter on show opens on its steps in - the rail; pick a chapter or a step to go there, or **All asks** to play another ask. -- **The lines a step wrote** sit inside the nodes they fall in, what it added green and what it - took away struck through. A change to a file the map does not draw - a stylesheet, a README - - shows on a card over the canvas instead. -- **A definition at a time.** An edit is split into the definitions it touches - imports first, - then what it adds, then what it changes, then what it removes - so a replay builds the change up - one function, type or constant at a time, in the order the agent staged it. -- **Inspects too.** What an agent reads lands as an inspect of what it read, so you can see what - it looked at before it wrote anything: `find`, `references`, `file` and `dependencies` calls as - ccc answers them, and its own reads - Claude Code's `Read`, a `cat`, `sed -n` or `grep` of a - file in its shell, a subagent's too - from its transcript, as it writes them. -- **Live.** While **Live** is lit each change is shown as it lands (`L` turns it on and off); - stepping, scrubbing or picking a chapter stops on that step. Space plays and pauses, `←` and `→` - step, `c` turns the subtitles off and `m` the voice. -- **At your pace.** 1× keeps the time between steps as it was; 1.5× and 2× hurry it, ½× and ¼× - stretch it, and the voice reads at the same speed. Following runs behind by however many steps - are still waiting. -- **Big views in groups.** A view of 50 or more components or functions is laid out a group at a - time -- **Granular changes.** On in the player's settings: while a step is on show, what it did not touch - fades back, and a big scene draws only what is wired to it, laid out together to fit the screen. - Click the canvas to bring the rest back. - -Every ccc server on a project shares its steps through `.ccc/timeline.jsonl` (kept out of git), so -the visualiser shows an agent's work whichever server the agent talks to. - -The data behind the page is plain JSON, with or without `--vis`: +Or build it from source: ```sh -curl -s localhost:6767/vis.json # context, containers, components -curl -s 'localhost:6767/vis/code?file=src/scan.rs' # one file's functions and calls -curl -s 'localhost:6767/vis/flow?file=src/scan.rs&line=76' # one function's logic +cargo build --release && ./target/release/ccc install ``` -## Extension - -[`extensions/vscode`](extensions/vscode) is an editor client for the same analysis. It runs -`ccc run` in the background for each workspace folder and reads it over loopback HTTP, so nothing -leaves the machine and no configuration is needed to get started. - -It also serves the visualiser component of the extension, so if you want the visualiser in VSCode you can just -click the RUN button in the activity bar and see the visualiser in action. - -See ['EXTENSION.md'](docs/EXTENSION.md) for more information. - -

Code Change Capture Visualiser Example

- -## Replays and Secrets - -ccc can keep the story of a branch - each ask, what the agent read and changed, and its narration - -as a **replay** beside the branch, under `refs/ccc/replay/`, so a reviewer can play back how -a change was made. A replay never lives in a branch's files, so a merge cannot carry one into the -default branch. - -**Recording is opt-in, per repository.** Nothing is recorded until someone says yes: `ccc run` asks -once on a terminal, the VS Code extension asks once per folder (`ccc: Session Recording…` asks -again), and the answer is kept in `git config ccc.replay`, so both agree. - -```sh -ccc replay enable # record this repository's agent sessions -ccc replay disable # stop - replays already saved stay where they are -ccc replay status # where recording stands, and who can read the remote -``` - -A replay carries your prompts and the code each step touched. That makes it ideal for a team on a -**private** repository - everyone who can read the code can replay how it was written - and a leak -on a public one. So: - -- **No replay goes to a public remote** - unless it is encrypted to your team (below). Before pushing - one, ccc asks the remote - signed out - whether it answers. If it does, the replay stays on your - machine. To allow it - your company's own - instance, say, where every repository is readable company-wide - add this to - [`.ccc/map.json`](#dependencymap), where the whole team sees it in review: - - ```json - { "replays": { "allow_public_remote": true } } - ``` - -- **Secrets are redacted first.** Before a replay is committed, anything that looks like a - credential - in a prompt, the agent's words, a diff or a file it touched - is replaced with - `[redacted: …]`, and a narration line that said one stays on your machine. - -**Team-encrypted replays.** A team on [runccc](https://teams.runccc.dev) can encrypt every replay to -its project's key, so only the team opens one - wherever the repository lives, public remotes -included. The project page gives the snippet for `.ccc/map.json`: - -```json -{ "replays": { "encrypt": { "project": "prj_some_random_key" } } } -``` - -```sh -ccc login # sign in - approve the code in your browser -ccc whoami # who you are, your teams, and the project this repository encrypts to -ccc logout # revoke the token and forget it here -``` - -In VS Code there is nothing to run: turning recording on in a repository that encrypts its replays -signs you in, the page opening in your browser. - -Each save fetches the project's public key and encrypts every replay. Nothing in `refs/ccc/` reads in the clear. - -Team members need to be in the same team in teams.runccc.dev to view shared repo replays. - -Where `encrypt` is set, a replay that cannot be encrypted - not signed in, not on the team, not paid -for, runccc out of reach with no key fetched in the last day - is not saved (the push itself goes ahead with a not explaining why the replay wasn't saved). A replay a teammate's older ccc pushed in the clear is warned of by -`ccc replay status` and the visualiser, and is encrypted - its history started over - the next time -its branch is pushed. `service` beside `project`, or `RUNCCC_URL`, points ccc at another key -service. The token and cached keys live in `~/.config/ccc/runccc/`, readable by you alone. - -**Secret warnings.** The same scanner warns before a secret is committed or pushed, whether or not -you record replays. It knows provider token shapes (GitHub, AWS, Slack, Stripe, Google, OpenAI, -Anthropic and more), private keys, JSON Web Tokens, passwords in URLs, and high-entropy values given -to credential-shaped names - in any file, `.env`, YAML and JSON as well as code: - -- `security` and `ccc sast` report it as `secret-in-change` -- `changes` and `pr_summary` open with a warning -- the pre-push hook `ccc run` installs says so as you push - it never blocks the push, and - `git config ccc.secrets false` turns it off -- VS Code marks the line and pops up a message when one first appears - -A line meant to carry one - a test fixture - can say so with `ccc:allow-secret` (`pragma: allowlist -secret` and `gitleaks:allow` are read too). - -## DependencyMap - -The `.ccc/map.json` file is used to hint to ccc where to find dependencies, for example services that call each other or share common functionality. - -```json -{ - "services": { - "auth": ["apps/auth/**"], - "billing": ["apps/billing/**", "libs/money/**"], - "gateway": ["apps/gateway/**"] - }, - "relatives": { - "gateway": ["auth"] // gateway calls auth over HTTP, so declare it! - }, - "externals": { - "billing": { "repo": "acme/billing", "lang": "go", "path": "../billing" } - } -} -``` - -`relatives` are the declared relationships: service to service here, or service to a peer under `externals`. - -## Externals - -Calls do not stop at your project. `externals` names peer repositories - a sibling checkout, another -corner of a monorepo, or a private repo you only have a published surface for - and `ccc:serves` / -`ccc:calls` comments name the key both ends agree on: - -```rust -// gateway (rust) -fn checkout(cart: &[Item]) { - // ccc:calls grpc billing.v1.Charge - client.charge(total) -} -``` - -```go -// billing (go), another repository -// ccc:serves grpc billing.v1.Charge -func Charge(account string, amount int) error { ... } -``` - -Matching keys become real edges of the service graph, with a file and line at each end, whatever -language each side is written in. Publish a surface for others to consume with `ccc init`. - -See [EXTERNALS.md](docs/EXTERNALS.md). - -## Skipping code - -A `ccc:skip` comment withdraws code from the analysis, in whatever comment syntax the file's -language uses. Placement decides the scope: - -- **At the very top of a file** - the whole file is skipped. -- **Directly above a function** (attribute and decorator lines may sit between, a blank line may - not) or **inside its body** - just that function is skipped: it is not measured, not ranked, and - calls to it are not resolved. -- **Anywhere else at file level** - the whole file is skipped. - -```rust -// ccc:skip generated - do not analyse -``` - -Trailing prose after the marker is allowed, so a skip can say why. A marker inside backticks is -documentation quoting one, not a directive, so a comment that explains `ccc:skip` withdraws -nothing. - -To see everything regardless, pass `--ignore-skip` to any command - `ccc run --ignore-skip`, -`ccc changes --ignore-skip`, ... - and every `ccc:skip` reads as an ordinary comment: the code it -marks is mapped, searched, measured and edited like the rest. In the VS Code extension the same -switch is the `ccc.server.ignoreSkip` setting. Without it, a `find` or `references` that misses -also searches the skipped code as text and lists what it finds there, marked as not indexed, so -a miss never hides code you skipped. - -## AGENTS.md - -#### Note: To use the file-system map (instead of in-memory map); you run `ccc scan --dir` and then add a block to your AGENTS.md file to scan the `.ccc` directory instead. - -(recommended) For those using `ccc run`; add the following block to an AGENTS.md file at the root of your project - agents that read an [`AGENTS.md`](https://agents.md) at the repo root pick this up automatically e.g. Copilot, Claude, Cursor etc. - -```md -# AGENTS.md - -This repo has a Code Change Capture (ccc) map - a generated in-memory code map served over MCP at `http://127.0.0.1:6767/mcp`. Use it -as the entry point for everything you do here. - -- no bash, grep or sed usage for exploring the project -- Every interaction: use `ccc` tool calls to gather information about the source of this project. -- All thinking, navigation, and questions about the codebase go through the MCP server tools: (index, find, references, dependencies, vulnerabilities, deps, security, file, notes, changes, prompts, pr_summary, test_triggers, test_targets, lints, hot, services, refresh) -- When I ask to *see* the analysis, call `insights` - it opens the insights UI in my browser (off under `ccc run --no-html`) -- ccc is the only writer: make every change through its edit tools - `edit_rename`, `edit_replace`, `edit_delete`, `edit_insert` or `edit_text`, then `edit_apply` - never your own Edit/Write tools, `sed -i` or shell redirection. -- `edit_apply` rescans the map before it answers; call `refresh` only after a change made outside ccc. - -``` - -Because the agent loads `AGENTS.md` at the start of a session, this wires the code map into every interaction: -reasoning and answers come from `ccc`'s map, and every change goes back through `ccc`, so it stays inside the -project, is previewed before it is written, and can be reverted. - -## Testing - -See [TESTING.md](docs/TESTING.md) - -## Pipelines - -See [PIPELINES.md](docs/PIPELINES.md) +## Choose how to use it -## Performance +### 1. CLI -See [PERF.md](docs/PERF.md) +Run ccc from a terminal or a pipeline. Serve the map to your agents, see what a branch changed, and gate CI on tests, dependencies and security findings. -## MCP +**[CLI.md](docs/CLI.md)** has one section per command: the command, its options, and what it is for. -See [MCP.md](docs/MCP.md) +### 2. Advanced -## Stream +Watch and review the work. Explore the project in the architecture visualiser, get hints in VS Code, and replay how an agent made a change. -See [STREAM.md](docs/STREAM.md) +**[Advanced.md](docs/Advanced.md)** has one section each for the visualiser, IDEs and replays. diff --git a/docs/Advanced.md b/docs/Advanced.md new file mode 100644 index 0000000..e8730a1 --- /dev/null +++ b/docs/Advanced.md @@ -0,0 +1,261 @@ + +

Code Change Capture Visualiser Example

+ +# Advanced + +Ways to see and review what changed, beyond the command line: + +- [Visualisation](#visualisation): the change visualiser, and the player that replays agent work inside it +- [IDEs](#ides): the VS Code extension, and wiring agents to ccc +- [Replays](#replays): keeping the story of a branch beside it, privately + +--- + +## Visualisation + +`ccc run --vis` serves a 2D map of the project at `http://127.0.0.1:6767/vis` and opens it in your browser. The VS Code extension shows the same page in its panel. It follows the C4 model from the whole system down to the code, then goes one level further: the logic of a single function, drawn as a node graph. + +```sh +ccc run --vis +``` + +### Levels + +| Level | Boxes | Arrows | +|---|---|---| +| Context | the system; peer repositories from `externals` in `.ccc/map.json`; endpoints nothing in the map answers | `ccc:calls` / `ccc:serves` crossings, by transport | +| Containers | services from `.ccc/map.json`, or top-level directories without one | resolved and declared calls between services | +| Components | the files in one container; other containers it touches sit outside the boundary | file-to-file calls | +| Code | the functions in one file, each with an input pin and one output pin per function it calls | resolved calls; functions in other files as stand-ins | +| Logic | one function: entry, calls, branches, switch/match/try arms, loops, closures, returns and throws | execution order | + +Every level is built from the same map the MCP tools read, so every supported language is drawn the same way. Like the rest of ccc, it reads syntax, not behaviour. An arrow is a call the resolver found evidence for, and a branch is a node in the syntax tree, not a path anything was seen to take. Calls that reach nothing in the project (the standard library, a dependency) are folded into grey pills; the toolbar shows them in full or hides them. + +### Getting around + +| Key or action | What it does | +|---|---| +| Double-click or `Enter` | Drill down a level. | +| `Escape` | Go back up a level. | +| Breadcrumb | Jump to any level. | +| `/` | Search for functions and files, and open them directly. | +| Drag | Rearrange boxes. Positions are remembered per view. | + +Every view has its own URL, so the browser's back button works and views can be bookmarked. The page reloads by itself when the map changes. A view of 50 or more components or functions is laid out a group at a time. + +### The player + +A player along the foot of the page replays what agents did through ccc. Each ask plays as a story: the agent's plan, every step it took and how it ended. The story is told in subtitles and read aloud, and the bar is cut into chapters, listed in a rail beside the canvas. + +- **Chapters.** One per part of the work, named after what the agent said its steps were for: the intent on its edit calls, the reason on its reads. Pick a chapter or a step to go there, or **All asks** to play another ask. +- **The lines a step wrote** appear inside the nodes they fall in: additions in green, removals struck through. A change to a file the map doesn't draw, such as a stylesheet or a README, shows on a card over the canvas. +- **One definition at a time.** An edit is split into the definitions it touches: imports first, then what it adds, then what it changes, then what it removes. A replay builds the change up in the order the agent staged it. +- **Reads too.** What an agent reads appears as an inspect, so you can see what it looked at before it wrote anything. That covers ccc's `find`, `references`, `file` and `dependencies` answers, and the agent's own reads from its transcript: Claude Code's `Read`, or a `cat`, `sed -n` or `grep` in its shell, a subagent's included. +- **Live.** While **Live** is lit, each change is shown as it lands. Stepping, scrubbing or picking a chapter stops on that step. +- **At your pace.** 1× keeps the original time between steps. 1.5× and 2× speed it up, ½× and ¼× slow it down, and the voice reads at the same speed. +- **Granular changes.** Turn this on in the player's settings. While a step is on show, everything it didn't touch fades back, and a big scene draws only what is wired to the step, laid out to fit the screen. Click the canvas to bring the rest back. + +| Key | What it does | +|---|---| +| `Space` | Play or pause. | +| `←` / `→` | Step back or forward. | +| `L` | Turn **Live** on or off. | +| `c` | Turn subtitles off or on. | +| `m` | Turn the voice off or on. | + +The voice is a local model, downloaded once per machine (see [`ccc voice`](CLI.md#ccc-voice)). Every ccc server on a project shares its steps through `.ccc/timeline.jsonl`, which is kept out of git. So the visualiser shows an agent's work whichever server the agent talks to. + +### The data behind it + +The page is built from plain JSON, which is served with or without `--vis`: + +```sh +curl -s localhost:6767/vis.json # context, containers, components +curl -s 'localhost:6767/vis/code?file=src/scan.rs' # one file's functions and calls +curl -s 'localhost:6767/vis/flow?file=src/scan.rs&line=76' # one function's logic +``` + +--- + +## IDEs + +### VS Code + +[`extensions/vscode`](../extensions/vscode) puts ccc's analysis on the code it applies to. It runs its own `ccc` analyser in the background for each workspace folder and reads it over loopback HTTP, so nothing leaves your machine and nothing needs configuring to start. When no `ccc` binary is found, it downloads the release for your platform. + +**Install.** `cargo build --release` packages the extension next to the binary: + +```sh +cargo build --release # -> dist/ccc-codecache.vsix +code --install-extension dist/ccc-codecache.vsix +``` + +To build only the extension: `cd extensions/vscode && npm install && npm run package`. It isn't published to the Marketplace. + +**Hints.** Each changed function carries a clickable CodeLens: + +| Hint | What it means | +|---|---| +| `✓ 3 tests` | The function changed and tests cover it. Opens them, nearest call hop first. | +| `✗ no smoke test` | Nothing covers the change. The kind of test worth writing is named: smoke, integration, contract, perf or load. | +| `→ billing` | This line calls into another service. Opens the callee. | +| `← gateway` | Another service calls this function. Opens the callers. | +| `▲ 37 callers` | A hot path: among the most called, most complex or widest fan-out. | +| `↻ cycle of 3` | The function is in a call cycle. | +| `↑ calls billing in acme/billing` | The call leaves the repository. Opens the handler when that peer is checked out locally. | +| `? billing.v1.Charge unanswered` | A `ccc:calls` whose key nothing serves: a typo at one end, or a peer missing from `externals`. | + +Test, coverage and service-boundary hints are based on the diff. A file identical to the base ref has none, by design. Hot paths and cycles come from the call graph alone, so they show on any file. Every function also shows its complexity as a circled number from 1 to 10 between its name and signature, coloured from grey to red as it climbs. + +**The panel.** Click the **Code Change Capture** mark in the activity bar. It has three views: + +- **Triggers:** the tests your changes make necessary (**Run these**), the changed functions nothing covers (**No test covers**), and the command that runs exactly that set (**Commands**, click to run it in a terminal). The badge is the number of tests worth running before you push. +- **Complexity:** every measured function grouped by band, worst first, filterable by name, parameter count and band. +- **Visualiser:** the [visualiser](#visualisation) above, with the timeline of edit steps. While an agent edits through ccc, it flies to each change as it lands. A staged change is drawn as **Proposed** before anything is written, next to the intent the agent gave for it. + +Everything runs against the working tree, so uncommitted and untracked files count. Hints reflect the last saved state; they fade while a file has unsaved edits and refresh on save. + +The extension's own page, [`extensions/vscode/README.md`](../extensions/vscode/README.md), lists its commands. + +**Settings:** + +| Group | Settings | +|---|---| +| Analyser | `ccc.binaryPath` (your own `ccc`), `ccc.autoInstall` (`false` stops the download), `ccc.server.autoStart`, `ccc.server.address`, `ccc.server.port` (`0` picks a free port; a fixed one collides across windows), `ccc.server.watchIntervalSec` (`0` lets the extension drive rescans), `ccc.server.startupTimeoutMs`, `ccc.server.extraArgs` (extra `ccc run` options, for example `--auto`), `ccc.server.ignoreSkip` (the same as `--ignore-skip`) | +| Hints | `ccc.enable`, `ccc.baseRef`, `ccc.hints.codeLens`, a toggle per kind (`ccc.hints.testTriggers`, `untested`, `outbound`, `inbound`, `hotPaths`), `ccc.hints.includeTestFiles`, `ccc.hints.crossServiceMode` (`auto`, `always`, `off`), `ccc.hints.minEvidence` (`evidence`, `any`), `ccc.untested.showUncoveredTargets`, `ccc.untested.minPriority` | +| Complexity | `ccc.complexity.enabled`, `ccc.complexity.minScore` | +| Decorations | `ccc.decorations.style` (`badge+gutter`, `badge`, `gutter`), `badgeMaxLength`, `dimWhenDirty`, `overviewRuler` | +| Refresh | `ccc.refresh.onSave`, `onWindowFocus`, `intervalSec` (off by default), `debounceMs` | +| Search | `ccc.search.mode` (`none`, `allow`, `block`), `ccc.search.urls` and `ccc.search.endpoints`: what the `search` tool may reach, for example only your company's docs and portals | +| Logging | `ccc.trace` (`off`, `messages`, `verbose`) | + +`ccc.hints.codeLens: false` with `ccc.decorations.style: badge+gutter` shows end-of-line badges instead of lenses. + +**How the hints are worked out.** The analysis is static. Knowing what it did and didn't establish is the difference between a useful hint and a misleading one. + +- **Coverage comes from the call graph, not from running anything.** A test is linked to a change because the graph connects them. A test that reaches code through dynamic dispatch, a table of function pointers or a framework is invisible, so `✗ no test` means "nothing reaches it", not "nothing runs it". +- **A call is tied to a definition only with evidence:** the receiver's declared type, the same file, the same package, an import, or a qualifier naming the defining file. Where that evidence fits more than one definition, nothing is claimed, and a Rust test can never mark a TypeScript function covered. In untyped languages, a bare call to a name defined exactly once still counts, and its hover says it was matched on the name alone. `ccc.hints.minEvidence: evidence` (the default) hides those. +- **Cross-service hints depend on `.ccc/map.json`.** With a `services` block, a hint sits on the exact call line. Without one, ccc groups by top-level directory and labels the result a module. In a flat project, `ccc.hints.crossServiceMode: auto` hides these hints rather than calling every import a service call. A declared edge with no call site is normal: that's what an HTTP, RPC or queue dependency looks like to static analysis. +- **"Hot" means central to the call graph, not measured.** Only the top 25 of each ranking are marked, so a function without a flame isn't necessarily cold. +- **Lists are capped:** 25 covering tests, 60 ranked targets, 60 call sites and 100 symbols per edge, and 8 names per triggered test. Hovers say when a list is at its cap. + +**Troubleshooting.** The status bar entry says what's wrong: + +- **ccc with an error icon:** the analyser didn't start. Run **ccc: Show Log**. +- **ccc with a warning icon:** git couldn't resolve a base ref, on a shallow clone or a repository with none of the default branches. Set one with **ccc: Select Git Base Ref...**. +- **No hints in one file:** either nothing in it changed against the base ref, or it isn't in the map (ignored by git, or in a language ccc doesn't parse). The status bar tooltip says which. To widen the diff, point `ccc.baseRef` at an older ref or turn on `ccc.untested.showUncoveredTargets`. +- **Everything looks like a cross-service call:** the project has no `.ccc/map.json`, so add a `services` block. +- **"could not find the ccc binary":** the download failed, or `ccc.autoInstall` is off. **Retry Install** tries again; otherwise build `ccc` and set `ccc.binaryPath`. +- **The analyser keeps restarting:** it backs off and gives up after five failures in five minutes. **ccc: Show Log** has the last 20 lines of each crash. + +**Working on the extension:** + +```sh +cd extensions/vscode +npm install +npm run watch # esbuild in watch mode, then F5 for an Extension Development Host +npm run check # type-check +npm run package # -> ../../dist/ccc-codecache.vsix +``` + +`src/model.ts` turns an `/insights.json` payload into hints without importing `vscode`, so it can be tested against a saved payload without an extension host. + +### Agents + +The extension's analyser runs on a free port so windows never clash. Each time it starts, it writes that port to `.mcp.json` (Claude Code) and `.vscode/mcp.json` (Copilot), so both find the server without being told. Both files are generated; keep them out of git. An agent session that was open when the analyser restarted still has the old port, so reload it. + +Without the extension, register a `ccc run` yourself: + +```sh +claude mcp add --transport http ccc http://127.0.0.1:6767/mcp +copilot mcp add --transport http ccc http://127.0.0.1:6767/mcp +``` + +Then add this block to an `AGENTS.md` at the repository root. Claude Code, Copilot, Cursor and other agents that read [`AGENTS.md`](https://agents.md) pick it up at the start of every session: + +```md +# AGENTS.md + +This repo has a Code Change Capture (ccc) map, served over MCP. Use it as the entry point for everything you do here. + +- no bash, grep or sed usage for exploring the project +- Every interaction: use `ccc` tool calls to gather information about the source of this project. +- All thinking, navigation, and questions about the codebase go through the MCP server tools: (index, find, references, dependencies, vulnerabilities, deps, security, file, notes, changes, prompts, pr_summary, test_triggers, test_targets, lints, hot, services, search, violations, refresh) +- ccc is the only writer: make every change through its edit tools - `edit_rename`, `edit_replace`, `edit_delete`, `edit_insert` or `edit_text`, then `edit_apply` - never your own Edit/Write tools, `sed -i` or shell redirection. +- `edit_apply` rescans the map before it answers; call `refresh` only after a change made outside ccc. +``` + +With that in place, the agent's answers come from ccc's map, and every change goes back through ccc. That keeps each change inside the project, previewed before it is written, and revertible. + +--- + +## Replays + +ccc can keep the story of a branch as a **replay** beside it, under `refs/ccc/replay/`: each ask, what the agent read and changed, and its narration. A reviewer can then play back how a change was made in the [visualiser](#visualisation). A replay never lives in a branch's files, so a merge can't carry one into the default branch. + +### Turning recording on + +Recording is opt-in, per repository: nothing is recorded or pushed until someone turns it on. Neither `ccc run` nor the VS Code extension asks, records or pushes anything by itself. Turn it on with `ccc replay enable`, or **ccc: Session Recording…** in VS Code. The answer is kept in `git config ccc.replay`, so both agree. + +```sh +ccc replay enable # record this repository's agent sessions +ccc replay disable # stop; replays already saved stay where they are +ccc replay status # whether recording is on, and who can read the remote +``` + +Once recording is on, the pre-push hook saves each branch's replay as you push it. [CLI.md](CLI.md#ccc-replay) has every replay command and option. + +### Keeping them private + +A replay carries your prompts and the code each step touched. On a private repository, that's ideal: everyone who can read the code can replay how it was written. On a public one, it's a leak. So: + +- **No replay goes to a public remote** unless it is encrypted to your team (below). Before pushing one, ccc checks, signed out, whether the remote answers. If it does, the replay stays on your machine. To override the default setting which disallows pushing replays to public repositories see the section at the bottom. + +- **Secrets are redacted first.** Before a replay is committed, anything that looks like a credential is replaced with `[redacted: …]`, whether it is in a prompt, the agent's words, a diff or a file it touched. A narration line that read one out stays on your machine. + +### Team-encrypted replays + +A team on [runccc](https://teams.runccc.dev) can encrypt every replay to its project's key, so only the team can open one, wherever the repository lives, public remotes included. The project page gives the snippet for `.ccc/map.json`: + +```json +{ "replays": { "encrypt": { "project": "prj_some_random_key" } } } +``` + +```sh +ccc login # sign in: approve the code in your browser +ccc whoami # who you are, your teams, and the project this repository encrypts to +ccc logout # revoke the token and forget it here +``` + +In VS Code there's nothing to run. Turning recording on in a repository that encrypts its replays signs you in, opening the approval page in your browser. Team members must be in the same runccc team to view each other's replays. + +Each save fetches the project's public key and encrypts every replay, so nothing in `refs/ccc/` is readable in the clear. Where `encrypt` is set, a replay that can't be encrypted isn't saved. That happens when you're not signed in, not on the team, or not on a paid plan, or when runccc is out of reach and no key was fetched in the last day. The push itself goes ahead, with a note explaining why the replay wasn't saved. + +A replay a teammate's older ccc pushed in the clear is flagged by `ccc replay status` and the visualiser. It is encrypted, with its history started over, the next time its branch is pushed. Set `service` next to `project`, or `RUNCCC_URL`, to point ccc at another key service. The token and cached keys live in `~/.config/ccc/runccc/`, readable only by you. + +### Secret warnings + +The same scanner warns before a secret is committed or pushed, whether or not you record replays. It knows: + +- provider token formats (GitHub, AWS, Slack, Stripe, Google, OpenAI, Anthropic and more) +- private keys and JSON Web Tokens +- passwords in URLs +- high-entropy values assigned to credential-like names + +It checks every file, including `.env`, YAML and JSON as well as code. You'll see the warning in several places: + +- `ccc sast` and the `security` MCP tool report it as `secret-in-change` +- `changes` and `pr_summary` open with a warning +- the pre-push hook says so as you push, once `ccc replay enable` or `ccc replay install` has installed it. It never blocks the push, and `git config ccc.secrets false` turns it off +- VS Code marks the line and shows a message the first time it appears + +A line meant to carry one, such as a test fixture, can say so with `ccc:allow-secret`. `pragma: allowlist secret` and `gitleaks:allow` are recognised too. + +## PROCEED WITH CAUTION - this will publish your agent conversations and/or transcriptions + +**DO NOT DO THIS: This makes your replays and transcriptions/conversations with the agent public.** This causes an override of the *default* ccc behaviour (*default* behaviour ***prevents*** replays being pushed to public remotes). Adding the below snippet to `.ccc/map.json` will push replays and their transcriptions **(including potentially sensitive content)** to the public repo: + + ```json + // DO NOT DO THIS ON PUBLIC FACING REPOSITORIES + { "replays": { "allow_public_remote": true } } + ``` \ No newline at end of file diff --git a/docs/CLI.md b/docs/CLI.md new file mode 100644 index 0000000..ad3e214 --- /dev/null +++ b/docs/CLI.md @@ -0,0 +1,400 @@ +# ccc CLI + +Each section covers a ccc command: what it is for, how to call it, and every option it takes. Commands that take a `PATH` treat it as the project root, and it defaults to the current directory. + +## Quick start + +```sh +ccc init # write a starter .ccc/map.json +ccc run # serve the map on 127.0.0.1:6767 +claude mcp add --transport http ccc http://127.0.0.1:6767/mcp # Claude Code +copilot mcp add --transport http ccc http://127.0.0.1:6767/mcp # Copilot +``` + +Then tell your agent to use the `ccc` MCP tools. [Advanced.md](Advanced.md#agents) has an `AGENTS.md` block that does this. + +## Commands + +| Command | For | +|---|---| +| [`ccc install`](#ccc-install) | Putting the binary on your `PATH` | +| [`ccc init`](#ccc-init) | Setting up `.ccc/`: the service map, and the surface other repositories read | +| [`ccc run`](#ccc-run) | Serving the map to agents (MCP) and people (the visualiser) | +| [`ccc changes`](#ccc-changes) | What a branch changed, function by function, and which services it puts at risk | +| [`ccc insights`](#ccc-insights) | The full analysis as JSON, including which tests to run for the branch | +| [`ccc deps`](#ccc-deps) | What a branch did to the dependency tree | +| [`ccc audit`](#ccc-audit) | Known vulnerabilities in your locked dependencies | +| [`ccc sast`](#ccc-sast) | Security risks in your own source | +| [`ccc prompts`](#ccc-prompts) | Which agent request produced each change | +| [`ccc pr-summary`](#ccc-pr-summary) | A pull request description built from those requests | +| [`ccc violations`](#ccc-violations) | Where agents broke ccc's rules, read from their transcripts | +| [`ccc replay`](#ccc-replay) | Recording and saving replays of agent sessions | +| [`ccc voice`](#ccc-voice) | The narration voice replays are read with | +| [`ccc login`](#ccc-login) | Signing in to runccc, for team-encrypted replays | +| [`ccc logout`](#ccc-logout) | Signing out of runccc | +| [`ccc whoami`](#ccc-whoami) | Who is signed in, and where this repository's replays are encrypted to | +| [`ccc scan`](#ccc-scan) | Writing the map to disk as markdown | +| [`ccc tokenize`](#ccc-tokenize) | Writing the map as a pre-encoded token stream | + +## Options every command takes + +| Option | What it does | +|---|---| +| `--ignore-skip` | Treat every `ccc:skip` comment as an ordinary comment, so the code it marks is mapped, searched, measured and edited like the rest. | +| `--no-parse-cache` | Parse every file again instead of reusing the parse kept from an earlier run. | +| `-h`, `--help` | Show the command's help. | +| `-V`, `--version` | Print the version (top level only: `ccc --version`). | + +A `ccc:skip` comment removes code from the analysis. Placed directly above a function, or inside its body, it skips that function. Anywhere else at file level, including the top of the file, it skips the whole file. + +Every command reuses parses from earlier runs. Each file's parse is stored under a hash of its language and its text, so a file that hasn't changed is read back instead of parsed. A copy, a rename or another checkout of the same file reuses the same entry. The parses live in your user cache directory (`~/.cache/ccc/parses` on Linux, `~/Library/Caches/ccc/parses` on macOS, `%LOCALAPPDATA%\ccc\cache\parses` on Windows), never in the project. Each build of `ccc` keeps its own, and only the four most recently used builds are kept. Set `CCC_CACHE_DIR` to put them somewhere else, for example a directory your CI caches between runs. + +--- + +## ccc install + +Copies the running `ccc` binary into a directory on your `PATH`. Linux only; the default directory is `~/.local/bin`. + +```sh +ccc install [DIR] [--force] +``` + +| Option | Default | What it does | +|---|---|---| +| `DIR` / `--dir DIR` | `~/.local/bin` | Directory to install into. Positional or as a flag, not both. | +| `--force` | off | Overwrite a `ccc` already in that directory. | + +## ccc init + +Sets up `.ccc/` from one parse of the project: + +- `.ccc/map.json`: a starter service map, for you to edit. An existing one is kept, never overwritten. +- `.ccc/surface.json`: the endpoints this repository serves and calls. Other repositories read it through `externals` in their own map. + +```sh +ccc init [PATH] [--name NAME] [--repo OWNER/REPO] [-o FILE] +``` + +| Option | Default | What it does | +|---|---|---| +| `--name NAME` | the directory | The name other repositories know this one by. | +| `--repo OWNER/REPO` | none | Recorded in the surface, for display in the repositories that read it. | +| `-o`, `--out FILE` | `.ccc/surface.json` | Write the surface here instead. `-` prints it to stdout. | + +## ccc run + +Builds the map in memory and serves it over HTTP to agents and people: + +- the MCP endpoint at `/mcp` +- REST endpoints (`/find`, `/references`, `/dependencies`, ...) +- the architecture visualiser at `/vis` + +The map rescans itself as files change. `ccc run` never records or pushes [replays](Advanced.md#replays), and never installs a git hook: recording starts only after `ccc replay enable`. Also available as `ccc serve`. + +```sh +ccc run [PATH] [OPTIONS] +``` + +| Option | Default | What it does | +|---|---|---| +| `--addr ADDR` | `127.0.0.1` | Address to bind. Think twice before widening it. | +| `--port PORT` | `6767` | Port to listen on. `0` picks a free port and prints it. | +| `--watch-interval SECS` | `2` | Seconds between checks for changed files. `0` turns watching off. | +| `--no-watch` | off | Don't watch files; rescan only on `POST /refresh`. | +| `--no-html` | off | Serve only the agent endpoints, with no visualiser. | +| `--vis` | off | Open the [architecture visualiser](Advanced.md#visualisation) in your browser. Can't be combined with `--no-html`. | +| `--auto` | off | Ask every agent that connects to annotate the cross-service calls it comes across: `ccc:serves` above a handler, `ccc:calls` beside a call. | +| `--search-mode none\|allow\|block` | `none` | What the `search` MCP tool may reach: `allow` only URLs matching a `--search-url`, `block` everything except them, `none` any https URL. | +| `--search-url PATTERN` | none | A URL pattern (`https://docs.corp/**`) or host pattern (`*.corp.example`) the mode applies to. Repeatable. | +| `--search-endpoint TEMPLATE` | none | A search page `search` sends queries to, with `{query}` where the words go (`https://wiki.corp/search?q={query}`). Repeatable. | +| `--html`, `--deps` | | Accepted so old scripts keep working. They do nothing. | + +## ccc changes + +Reports what the branch changed at function level and what that means for services and tests. Built for pipelines, so it prints JSON by default. + +It: + +- diffs the branch against its base and maps each changed hunk to the function it falls in +- for each changed function, lists the tests that call it directly and the other services that call it +- lists `services_to_test`: the changed services and every service that calls them, directly or through others, using `.ccc/map.json`, gRPC schemas and matched `ccc:calls`/`ccc:serves` comments +- lists changed functions no test calls, calls it couldn't resolve, and changed files that belong to no service +- always includes the dependency delta and newly added secrets + +```sh +ccc changes [PATH] [OPTIONS] +``` + +| Option | Default | What it does | +|---|---|---| +| `--base REF` | merge-base with `origin/main`, `main`, `origin/master` or `master` (the first that exists) | Ref to diff against. | +| `--service NAME=GLOB` | none | Define or extend a service inline, on top of `.ccc/map.json`. Repeatable. | +| `--worktree` | off | Include uncommitted edits and untracked files. CI wants the committed view; a local run usually wants this. | +| `--format json\|text` | `json` | Output format. | +| `--fail-untested` | off | Exit non-zero when a changed function has no test that calls it directly. | +| `--fail-introduced` | off | Exit non-zero when the branch introduces a dependency advisory the base didn't have, or the delta couldn't be checked. | +| `--prompts` | off | Name the agent request behind each change (see [`ccc prompts`](#ccc-prompts)). | +| `--telemetry` | off | Print only what the branch did to the OpenTelemetry metrics the code defines. | +| `--markdown` | off | Print the dependency delta as markdown for an agent. With `--telemetry`, print the telemetry changes instead. | +| `--html FILE` | none | Also write a single-file HTML view of the report. | +| `--from REPORT.json` | none | With `--html`: render the HTML from a saved JSON report instead of analysing. Needs no git. | +| `--deps` | | Accepted so old scripts keep working. The dependency delta is always included. | + +`changes` lists only tests that call a changed function directly. For the tests a change reaches through the call graph, and a command to run them, use [`ccc insights`](#ccc-insights). + +## ccc insights + +Runs the whole analysis once and prints it as JSON. The output covers: + +- the shape of the call graph: hot paths, the deepest call chains, call cycles +- `test_triggers`: the tests the branch's changes put at risk, found through the call graph, with a runnable command per language under `test_triggers.commands` +- `test_targets`: where a missing test would cost the most, and which kind to write +- lints + +It is the same data `ccc run` serves at `/insights.json`. + +```sh +ccc insights [PATH] [--base REF] +``` + +| Option | Default | What it does | +|---|---|---| +| `--base REF` | same as `ccc changes` | Ref the test-trigger diff runs against. | + +## ccc deps + +Reports what the branch did to the dependency tree: which packages entered, left or changed version, and any advisory that came with them. + +```sh +ccc deps [PATH] [OPTIONS] +``` + +| Option | Default | What it does | +|---|---|---| +| `--base REF` | same as `ccc changes` | Ref to diff against. | +| `--worktree` | off | Include uncommitted edits and untracked files. | +| `--format json\|text` | `json` | Output format. | +| `--markdown` | off | Print the delta as markdown for an agent. | +| `--fail-introduced` | off | Exit non-zero when the branch introduces an advisory the base didn't have, or the delta couldn't be checked. | + +## ccc audit + +Resolves the exact versions in your lockfiles and checks them against the OSV advisory database. It exits non-zero when a runtime dependency has an advisory, so CI can gate on it. + +```sh +ccc audit [PATH] [--offline] [--dev] [--format text|json] +``` + +| Option | Default | What it does | +|---|---|---| +| `--offline` | off | Resolve only. Never contact the advisory database. | +| `--dev` | off | Let advisories on dev- and build-only dependencies fail the run too. | +| `--format text\|json` | `text` | Output format. | + +## ccc sast + +Scans this project's own source for security risks, such as: + +- credentials committed to source +- certificate checks turned off +- commands or SQL statements built from a value +- broken hashes + +Only a high-severity finding fails the run, so a low-severity one (a checksum using a weak hash, say) doesn't break CI. + +```sh +ccc sast [PATH] [--include-tests] [--min-severity LEVEL] [--format text|json] +``` + +| Option | Default | What it does | +|---|---|---| +| `--include-tests` | off | Scan test files too. They usually hold fixtures, not leaks. | +| `--min-severity low\|medium\|high` | `low` | Hide findings below this level in the text output. JSON always carries every finding. | +| `--format text\|json` | `text` | Output format. | + +## ccc prompts + +Links each change on the branch to the Claude Code or Copilot request that produced it, using the agents' local transcripts. + +```sh +ccc prompts [PATH] [OPTIONS] +``` + +| Option | Default | What it does | +|---|---|---| +| `--base REF` | same as `ccc changes` | Ref to diff against. | +| `--worktree` | off | Include uncommitted edits and untracked files. | +| `--agent claude\|copilot` | both | Read only this agent's transcripts. | +| `--session ID` | all | Only this session. | +| `--model NAME` | none | The session's model name, as the caller reports it. Requires `--session`. | +| `--since DAYS` | all | Only requests from the last `DAYS` days. | +| `--record` | off | Append the collected requests to `.ccc/prompts.jsonl`. | +| `--format json\|text` | `json` | Output format. | + +## ccc pr-summary + +Prints markdown for a pull request description. It uses the same attribution as `ccc prompts`, written as prose a reviewer can read from start to finish. + +```sh +ccc pr-summary [PATH] [OPTIONS] +``` + +| Option | Default | What it does | +|---|---|---| +| `--base REF` | same as `ccc changes` | Ref to diff against. | +| `--worktree` | off | Include uncommitted edits and untracked files. | +| `--agent claude\|copilot` | both | Read only this agent's transcripts. | +| `--session ID` | all | Only this session. | +| `--model NAME` | none | The session's model name. Requires `--session`. | +| `--since DAYS` | all | Only requests from the last `DAYS` days. | + +## ccc violations + +Reads the agents' transcripts for this project (Claude Code, subagents included) and reports every tool call that broke ccc's rules. + +| Rule | What it catches | Use instead | +|---|---|---| +| `writes-outside-ccc` | Edit/Write tools, `sed -i`, `perl -i`, `>`/`>>`, `tee`, `cp`/`mv`/`rm` or `git apply` on project files | ccc's `edit_*` tools | +| `shell-exploration` | Read/Grep/Glob, or `cat`, `grep`, `rg`, `find`, `sed -n`, `head`, `ls` on the project | `index`, `find`, `references`, `file` | +| `web-outside-search` | WebFetch, WebSearch, or `curl`/`wget` to anywhere but this machine | `search` | +| `blocked-url` | URLs the `search` tool refused | nothing: your rules refuse them | + +```sh +ccc violations [PATH] [OPTIONS] +``` + +| Option | Default | What it does | +|---|---|---| +| `--since DAYS` | all | Only calls from the last `DAYS` days. | +| `--session ID` | all | Only this session. | +| `--rule RULE` | every rule | Only this rule. Repeatable. | +| `--format text\|json` | `text` | Output format. | +| `--fail` | off | Exit non-zero when any violation is found, for a CI or pre-push gate. | + +Copilot's chat log keeps no record of tool calls, so only Claude Code sessions are checked. A path or URL held in a shell variable can't be checked, and requests to this machine (ccc's own server) don't count as web access. + +## ccc replay + +Manages replays: the story of what agents did on a branch, kept under `refs/ccc/replay/`. They sit beside your branches, never in them, so a merge can't carry one into the default branch. Recording is opt-in per repository. [Advanced.md](Advanced.md#replays) covers how replays are kept private. + +### ccc replay enable + +Turns recording on for this repository. From then on, agent sessions are saved as replays and pushed with their branches. Anything that looks like a secret is removed first, and nothing goes to a remote anyone can read unless `.ccc/map.json` allows it. + +```sh +ccc replay enable [PATH] +``` + +### ccc replay disable + +Turns recording off. Replays already saved stay where they are. + +```sh +ccc replay disable [PATH] +``` + +### ccc replay status + +Shows whether recording is on, and who could read a pushed replay. + +```sh +ccc replay status [PATH] [--json] +``` + +| Option | Default | What it does | +|---|---|---| +| `--json` | off | Print the status as JSON. | + +### ccc replay save + +Saves the branch's new steps to its replay ref and pushes it beside the branch. The pre-push hook runs this for you; run it yourself to save without pushing the branch. + +```sh +ccc replay save [PATH] [OPTIONS] +``` + +| Option | Default | What it does | +|---|---|---| +| `--branch NAME` | the checked-out branch | Branch to save. | +| `--remote NAME` | the branch's upstream remote, else `origin` | Where to push it. | +| `--no-push` | off | Write the ref locally and push nothing. | +| `--default-branch` | off | Save a replay for the default branch too, which normally keeps none. | +| `--quiet` | off | Print nothing when nothing was saved. | + +### ccc replay install + +Installs the pre-push hook. It checks each push for secrets, and saves the replay when recording is on. `ccc replay enable` installs it too. + +```sh +ccc replay install [PATH] +``` + +## ccc voice + +Manages the narration voice the visualiser reads replays aloud with. It is downloaded once and shared by every `ccc` on the machine. The visualiser downloads it the first time it needs it if you haven't already. + +```sh +ccc voice status # downloaded or not, and where it is kept +ccc voice download # download it now +ccc voice remove # remove it, and every line it has read, from this machine +``` + +## ccc login + +Signs in to runccc, the key service team-encrypted replays go through. It prints a code; approve it in your browser. + +```sh +ccc login [PATH] [--json] +``` + +| Option | Default | What it does | +|---|---|---| +| `PATH` | `.` | The repository whose `.ccc/map.json` names the key service. runccc's own is used when it names none. | +| `--json` | off | Print the code and the result as JSON lines, for the VS Code extension to read. | + +## ccc logout + +Signs out of runccc on this machine: the token is revoked, and the keys it fetched are forgotten. + +```sh +ccc logout [PATH] +``` + +## ccc whoami + +Shows who is signed in to runccc, their teams, and the project this repository's replays are encrypted to. It exits non-zero when it can't find out who is signed in. + +```sh +ccc whoami [PATH] +``` + +## ccc scan + +Parses the project and reports what the map holds. The map is always built in memory. `--dir` also writes it as markdown, for tools that read a map from disk instead of over MCP. + +```sh +ccc scan [PATH] [--dir[=DIR]] [--tokens] [--encoding NAME] +``` + +| Option | Default | What it does | +|---|---|---| +| `--dir[=DIR]` | not written | Write the markdown map. A bare `--dir` writes to `PATH/.ccc`, and `--dir=DIR` writes to `DIR`, relative to `PATH`. The `=` is required. | +| `--tokens` | off | Also write the token stream (see [`ccc tokenize`](#ccc-tokenize)). Requires `--dir`. | +| `--encoding NAME` | `o200k_base` | The vocabulary `--tokens` encodes with. `cl100k_base` is also supported. | + +## ccc tokenize + +Builds the map and encodes it as a pre-encoded token stream, `tokens.bin` plus a `tokens.json` index, for models that share an OpenAI tiktoken vocabulary. The token IDs are approximate and not for Anthropic models: give Claude the markdown map or the MCP server instead. + +```sh +ccc tokenize [PATH] [--dir DIR] [--encoding NAME] +``` + +| Option | Default | What it does | +|---|---|---| +| `--dir DIR` | `.ccc` | Where to write the stream, relative to `PATH`. | +| `--encoding NAME` | `o200k_base` | The tiktoken vocabulary. `cl100k_base` is also supported. | + +--- + +Two commands are hidden from `--help`: `ccc check` is deprecated (regenerate a committed map with `ccc scan --dir` instead), and `ccc hook pre-push` is what the git hook runs, not something to call by hand. diff --git a/docs/DEPS.md b/docs/DEPS.md deleted file mode 100644 index 1689c0a..0000000 --- a/docs/DEPS.md +++ /dev/null @@ -1,274 +0,0 @@ -# Dependency changes - -`ccc audit` answers *what do we depend on right now, and is any of it vulnerable*. It never looks at -git. This document specifies the other half — *what did this branch do to our dependencies* — carried -in the `ccc changes` report by default, with JSON output and `--markdown` for agents. - -The question worth answering is not "did `Cargo.lock` change". Today a lockfile edit already shows up -in `changes` as a touched path with no functions and no detail, which is true and useless. The -question is: **which packages entered the tree, which left, which moved version, and did this branch -introduce an advisory that was not there before.** - -## Scope - -In, for v1: - -- added / removed / version-changed packages between a base ref and the head side, per ecosystem -- direct↔transitive and dev↔runtime transitions, which change whether a package ships -- advisories **introduced** and **resolved** by the change, not the standing set -- manifest-vs-lockfile drift: a manifest edited without its lockfile being regenerated -- `ccc changes`, `ccc deps`, `ccc run` + `/deps.json`, and the `deps` MCP tool - -Out, for v1: - -- per-entrypoint reachability (`go list -deps ./cmd/x` style). ccc resolves whole-module closures - from lockfiles; narrowing them to what one binary reaches needs a build, not a parse. -- license and provenance deltas -- the editor surface. The payload below is shaped for it — see *Editor* at the end — but no marks - are drawn in v1. - -`ccc audit` is unchanged: it stays the current-state command, with no `--base`. - -## Command surface - -The delta hangs off `changes`, not `audit`, because `changes` is the command that already owns a base -ref, `--worktree`, and the report every other surface consumes. - -``` -ccc changes [PATH] [--base ] [--worktree] [--fail-introduced] [--markdown] -ccc deps [PATH] [--base ] [--worktree] [--fail-introduced] [--markdown] [--format text|json] -ccc run [PATH] -``` - -**Nothing here is opt-in any more.** `ccc changes` computes the delta and puts it in the report; -`ccc run` answers `/deps.json` and advertises the `deps` MCP tool. That is affordable because of -the short-circuit below: a branch that touched no manifest or lockfile returns before a git blob is -read or a packet is sent, which is most branches. `--deps` is still accepted on both commands so the -pipelines and editor integrations that spell it out keep working, and on `changes` it still reads as -"the dependency section" beside `--markdown` — but it turns nothing on. - -`ccc deps` is the same delta with none of the change set around it: it prints the `deps` object -`changes` nests, so a CI step that only gates on dependencies pipes it straight in without a -`jq .deps` in the middle. `--base` and `--worktree` resolve identically on both — one shared -`changes::deps_report`, which `/deps.json` already answers from — so the two commands cannot disagree -about what the branch did. - -The one caller that still says no is `insights`, which builds a change set on every file-watch tick -and does not draw the delta. That is why `ChangesOptions.deps` survives as an internal knob even -though no CLI path sets it to false. - -| flag | effect | -|---|---| -| `--deps` | accepted and ignored on `changes` and `run`; the delta is computed and served either way | -| `--markdown` | render one section as markdown for an agent: the dependency delta, or the metric delta with `--telemetry` | -| `--fail-introduced` | exit non-zero when the change introduces an advisory; needs the network, so it fails loudly if OSV was unreachable rather than passing quietly | -| `--offline` | *not* added here — the delta resolves without OSV whenever `assess()` cannot reach the network, and says so, exactly as `audit` does | - -`--worktree` picks the head side, and nothing else changes: without it the head side is `HEAD` (the -committed view CI wants), with it the head side is the working tree. This matters more here than -elsewhere — reading lockfiles off disk in the committed view would report a dirty `Cargo.lock` as if -it were committed, so **both sides go through the same text-based resolver** rather than the disk -walk `audit::resolve` does today. - -## The refactor it needs - -`audit::resolve(root)` walks the filesystem, reads each lockfile, and dispatches on the basename to -parsers that are already pure text (`parse_toml_lock(&text, &rel, eco, &direct)` and friends). Only -the *input* layer is disk-bound: `resolve`'s walk, and the four `direct_*(dir)` helpers that read a -sibling manifest to decide which packages are direct. - -Introduce a source, and everything else is reused as-is: - -```rust -// where a resolution reads its lockfiles and manifests from -pub trait Source { - // every lockfile and manifest path this source holds, relative to the root - fn inputs(&self) -> Vec; - fn read(&self, rel: &str) -> Option; -} - -pub struct DiskSource<'a> { pub root: &'a Path } -// a committed tree, read with `git show :` -pub struct GitSource<'a> { pub root: &'a Path, pub sha: &'a str } - -pub fn resolve_with(src: &dyn Source) -> AuditReport; -pub fn resolve(root: &Path) -> AuditReport { // unchanged signature, unchanged behaviour - resolve_with(&DiskSource { root }) -} -``` - -`direct_cargo` / `direct_npm` / `direct_go` / `direct_python` take `(&dyn Source, dir)` instead of -`dir` and read through it. `GitSource::inputs()` is one `git ls-tree -r --name-only ` filtered -to `LOCK_NAMES ∪ MANIFEST_NAMES`; `read` is one `git show`. `locate()` gets the same treatment so -head-side manifest lines resolve against the head side, not the disk. - -This is the whole of the change to `audit.rs`. Nothing in the parsers, the OSV client, the CVSS -banding or the `Package`/`Finding` model moves. - -## Algorithm - -1. **Short-circuit.** `changes` already runs `git diff --name-status -z -M `. Filter that - result to basenames in `LOCK_NAMES ∪ MANIFEST_NAMES`. Empty → emit an empty `deps` section with - `changed: false` and stop. No blob is read, no package is parsed, no packet is sent. A branch that - touched no manifest costs nothing, which is most branches. -2. **Resolve both sides.** Base = `resolve_with(GitSource { sha: base_sha })`, reusing the merge-base - `resolve_base()` already computed. Head = `GitSource { sha: head_sha }`, or `DiskSource` under - `--worktree`. -3. **Follow renames.** `parse_name_status` reads `-M` rename records but flattens them — it emits - `renamed(new)` and `deleted(old)` as separate rows and drops the pairing, which is all its current - callers need. The deps pass needs the pairing, to map a moved lockfile's base path to its head - path so `deps/Cargo.lock` → `crates/x/Cargo.lock` is not a wholesale remove + add. Either return - the old path alongside the row, or leave that parser alone and run one narrow - `git diff --name-status -z -M -- ` for this pass. The second is smaller and - keeps a hot, well-tested parser untouched. -4. **Key and diff.** Key on `(ecosystem, lockfile, name)` → the set of versions under that key. Not - `(ecosystem, name)`: npm legitimately holds several versions of one package in one lockfile, and a - monorepo holds one package at different versions in different lockfiles. Diffing version *sets* - keeps both honest. -5. **Classify** each key (see below). -6. **Assess the delta only.** Query OSV for the versions that exist on exactly one side — added and - removed versions — not the full closure. A branch that bumps one package costs one bounded query, - and the standing set is `audit`'s job. `introduced` = advisories on head-only versions; - `resolved` = advisories on base-only versions that no head version carries. -7. **Locate** each change against the head manifests, reusing the `Location` attribution, so a - transitive bump points at the direct dependency whose line a person can actually edit. - -## Classification - -| kind | when | why it matters | -|---|---|---| -| `added` | key absent at base | new code in the tree | -| `removed` | key absent at head | attack surface gone; may resolve advisories | -| `upgraded` / `downgraded` | one version in, one out | the ordinary bump; direction per ecosystem semver, `unknown` when unparseable rather than guessed | -| `versions-changed` | many-in / many-out | npm's multi-version reality, reported with both lists rather than flattened into a fake bump | -| `promoted` / `demoted` | `direct` flipped, version same | a transitive package a manifest now names, or stopped naming | -| `now-ships` / `no-longer-ships` | `dev` flipped, version same | a dev-only package moved into the runtime closure. A dev advisory that never shipped now does | - -Version ordering is per-ecosystem and deliberately conservative: semver where the ecosystem -guarantees it, and `unknown` where it does not (Go pseudo-versions, PyPI epochs, npm prerelease -tags). An `unknown` direction is still reported as a change — it just does not claim which way. - -## Report shape - -`ChangesReport` gains one optional field. `ccc changes` always fills it; the option remains because -`insights` asks for the change set without it, and an absent key is honest where a null one would -not be. `schema` stays `ccc-changes/1` — additive optional fields do not bump it. - -```rust -#[serde(skip_serializing_if = "Option::is_none")] -pub deps: Option, -``` - -```rust -pub struct DepsReport { - pub schema: &'static str, // "ccc-deps/1", for /deps.json served standalone - pub base_sha: String, - pub head_sha: String, - // false when no manifest or lockfile was touched: everything below is empty - pub changed: bool, - // the base side had no lockfile at all - the whole closure reads as `added` - pub baseline: bool, - pub changes: Vec, - // advisories this change brings in, and ones it clears - pub introduced: Vec, - pub resolved: Vec, - // manifests edited without their lockfile being regenerated, and lockfiles - // that could not be read at one end - pub drift: Vec, - pub assessed: bool, - pub error: Option, - pub counts: DepsCounts, -} - -pub struct DepChange { - pub kind: DepChangeKind, - pub ecosystem: Ecosystem, - pub name: String, - pub lockfile: String, - pub from: Vec, // versions at base - pub to: Vec, // versions at head - pub direct: bool, // as of the head side - pub dev: bool, - // head manifest lines to draw this on; empty for a removed package nothing declares - pub locations: Vec, -} - -pub struct DepsCounts { - pub added: usize, - pub removed: usize, - pub upgraded: usize, - pub downgraded: usize, - pub other: usize, - pub introduced: usize, - pub resolved: usize, -} -``` - -Text format, appended to the `changes` text output, in the voice `print_audit_text` already uses: - -``` -dependencies: 3 changed (2 added, 1 upgraded) against origin/main - + tokio 1.40.0 cargo Cargo.lock direct - + tokio-util 0.7.11 cargo Cargo.lock transitive, via tokio - ~ serde 1.0.203 -> 1.0.210 cargo Cargo.lock direct - -introduced (1): - RUSTSEC-2024-0011 high tokio-util 0.7.11 - fixed in 0.7.12 - Cargo.toml:24 - via tokio -``` - -A branch that changed nothing prints one line: `dependencies: unchanged against origin/main`. - -## Serving it - -`ccc run` answers `GET /deps.json?base=` and advertises the `deps` MCP tool, with no flag to -turn either on. The delta short-circuits on a branch that touched no manifest, so an agent that asks -about a project which never moved a lockfile gets one line back and the server reads nothing. - -Cache it the way `Analysis` is cached — keyed on `(map generation ts, base)` — not the way -`audit_report` is, which is keyed on generation alone. The base ref is a parameter here, and two -agents asking about two refs must not evict each other into a re-query. - -The MCP tool description follows the `vulnerabilities` one in tone: say what it answers, say what it -does not. It answers *what this branch did to the dependency tree*; it does not answer *what we -depend on* (that is `vulnerabilities`) and it cannot see a change that was never committed unless the -server was started against a working tree. - -## Failure modes - -Every one of these is reported, never fatal, matching `audit`'s standing rule — a resolution that -half-worked must never read as a clean result. - -| situation | behaviour | -|---|---| -| base ref missing / shallow clone | the existing `resolve_base` error, unchanged: `changes` cannot run at all without a base | -| git not on PATH | `deps` present, `changed: false`, `error` set | -| no lockfile at base (new project, first lock) | `baseline: true`; every package reads as `added` and the counts say so, rather than a triumphant "1,204 dependencies added" with no context | -| lockfile deleted at head | its packages read as `removed`, and `drift` notes the manifest that now pins nothing | -| manifest edited, lockfile untouched | `drift` entry: *"Cargo.toml changed but Cargo.lock did not — the declared range moved and the pinned version did not"* | -| OSV unreachable | `assessed: false`, `error` set, `changes` still complete. `--fail-introduced` exits non-zero, because "we could not check" is not "it is fine" | -| unpinned requirements.txt | already an `Unresolved` in `audit`; carried through to `drift` unchanged | - -## Tests - -Mirroring the existing split in `changes.rs` and `audit.rs`: - -- unit, on the classifier alone: synthetic `Vec` pairs covering every row of the - classification table, plus the npm multi-version case and the unparseable-version case -- unit, on `GitSource`: a lockfile read at a sha resolves to the same packages `DiskSource` gets from - the same content -- end-to-end, in the style of `changes_end_to_end_git`: init a repo, commit a `Cargo.toml` + - `Cargo.lock`, branch, bump one dependency, and assert one `upgraded` change and no others — then - the same with `--worktree` and the bump uncommitted -- the short-circuit: a branch that touches only `.rs` files produces `changed: false` and reads no - blob (assert via a source that panics on `read`) -- `baseline`, drift, and the OSV-unreachable path, each asserting the report is still well-formed - -## Editor - -Deferred, but the payload above is already the right shape for it: `DepChange.locations` carries -manifest lines exactly as `Finding.locations` does, so `VulnerabilityMarks` in the extension can draw -"upgraded on this branch" or "advisory introduced here" on a manifest line with no new plumbing. It -needs a second payload alongside `vulnerabilities`, and nothing else — the spawned server now -answers `/deps.json` without being asked. Nothing in this spec should be designed around that; it is -listed so v1 does not close the door on it. diff --git a/docs/EXTENSION.md b/docs/EXTENSION.md deleted file mode 100644 index 2b6e207..0000000 --- a/docs/EXTENSION.md +++ /dev/null @@ -1,113 +0,0 @@ -# Extension - -[`extensions/vscode`](extensions/vscode) is an editor client for the same analysis. It runs -`ccc serve` in the background for each workspace folder and reads it over loopback HTTP, so nothing -leaves the machine and no configuration is needed to get started. - -## Install - -`cargo build` packages the extension alongside the binary: - -```sh -cargo build --release # -> dist/ccc-codecache.vsix -code --install-extension dist/ccc-codecache.vsix -``` - -The packaging step is best-effort: it is skipped without `npm`, under `CI`, or with `CCC_SKIP_VSIX` -set, and never fails the Rust build. To build the extension on its own, `cd extensions/vscode` and -run `npm run package` (or `npm run watch` and press F5 for an Extension Development Host). - -## Using the extension - -Open a file with work in progress. Each changed function carries a CodeLens above it, and every lens -is clickable: - -| lens | what it means | -|---|---| -| `3 tests` | Tests cover this change - opens them, nearest call hop first. | -| `no smoke test` | Nothing covers this change, and a smoke test is the kind worth writing. | -| `calls billing` | This call crosses a service boundary - opens the handler, in a peer checkout if that is where it lives. | -| `called by gateway` | Another service calls this function - opens the callers. | -| `37 callers`, `cycle of 3` | A hot path, or a call cycle. From the call graph, so these show on files nobody has touched. | -| `billing.v1.Charge unanswered` | A `ccc:calls` whose key nothing serves - a typo at one end, or a peer missing from `externals`. | - -Every function the analyser parsed also carries its complexity as a filled circled number between -its name and its signature - `fn parse ❸ (s: &str)`. It is a cyclomatic-style count (one path, plus -one per decision point and loop) banded onto 1-10, and the colour runs grey, plain, green, blue, -purple, yellow, brown, amber, orange, red as it climbs. Unlike the hints above it is not diff-driven: -it describes the code as written, so it shows on files nobody has touched. Hover it for the raw -count, the branches and the loop depth behind the band. The `⚠` (no test covers this) and `🔥` -(hot path) verdicts sit inline in the same spot, right after the band, rather than out in the -gutter; the other hint kinds keep their gutter icons. `ccc.complexity.minScore` raises the floor if -you only want to see the functions worth a second look, and `ccc.complexity.enabled` turns it off; -the ten colours are contributed theme colours, so a theme or a `workbench.colorCustomizations` entry -can restyle any of them. - -Click the **Code Change Capture** mark in the activity bar to open the panel, and again to close it. It has -three views. **Triggers** is the tests your changes invoke: a triggered test usually lives in a -different file from the change that triggered it, so this is the only place that shows the whole set -- **Run these** (click to open; the tooltip carries how many call hops it sits from the change and -why), **No test covers**, and **Commands** - the suggested command for running exactly that set, -click to run it in a terminal. The badge is the number of tests worth running before you push. - -**Complexity** is every measured function grouped by band, worst first, with the count per band on -the group row. The title-bar buttons filter it: by name (substring), by parameter count (niladic, -monadic, dyadic, variadic - pick several), and by band (a 1-10 range). Test functions are measured -but hidden by default; the beaker button shows them, and the clear button appears whenever any -filter is active. The view's subtitle always says how many of the measured functions you are -looking at, so a filtered list can never pass itself off as the whole map. - -**Visualiser** is the [architecture visualiser](../README.md#architecture-visualiser) - the system, -its containers, components, code and the logic of each function - with the timeline of ccc's edit -steps along its foot. While an agent edits through ccc's tools, it flies to each change as it lands -and marks it, and any step can be replayed before and after. A staged change is drawn as -**Proposed** - the code as it would be - before anything is written, beside the one-line intent the -agent gave for it. Changes saved by hand land on the timeline too, marked with a person. Each edit -plays a definition at a time and an agent's reads land as inspects - through ccc or with its own -tools - one ask at a time, at the pace the speed button sets - -it is the same page as in a browser, so it behaves the same there (see the -[README](../README.md#architecture-visualiser)). A change -that only moves whitespace or wraps lines is not marked unless ¶ is on. Its title-bar buttons open the same -page as a tab beside the editor (`ccc: Open Architecture Visualiser`), where a long function has -room, or in a browser. "Open in editor" in its panels opens the file at the line. The page talks to -the analyser through the extension, so it works the same when the workspace is remote. - -The status bar entry on the right is the summary - counts, the base ref being compared, and, when a -file has no marks at all, which of the two reasons applies: nothing in it changed, or it is not in -the ccc map. - -Everything runs against the **working tree**, so untracked and uncommitted files count. Hints reflect -the last *saved* state, since the analyser reads files rather than editor buffers; they fade while a -file is dirty and refresh on save. - -Coverage and boundary hints are diff-driven, so a file identical to the base ref has no changed -functions and therefore no hints - that is the design, not a fault. Hot paths come from the call -graph alone and appear regardless. - -## Worth knowing - -- **Session recording is asked, never assumed.** The first time the analyser starts for a folder - that is a git repository, a message asks whether its agent sessions may be recorded as - [replays](../README.md#replays-and-secrets). Put it aside and it asks again next time; answer and - it is kept in `git config ccc.replay`, shared with `ccc run`. `ccc: Session Recording…` asks again. - The message says who can read the remote - a public one gets no replay unless `.ccc/map.json` - allows it. Where `.ccc/map.json` encrypts replays to a team on Runccc Teams and nobody on this - machine has signed in, the answer is **Sign in and record**: the approval page opens in your - browser, its code shown in VS Code to check it against, and replays are saved from the next push. -- **A line that looks like a secret is marked.** Anything the branch adds that looks like a - credential - committed, uncommitted or untracked, in any file - gets a warning in the Problems - view, and a message the first time it appears. Mark a line `ccc:allow-secret` if it is meant to - be there. -- Cross-service hints need a `services` block in [`.ccc/map.json`](#dependencymap); cross-repository - hints need [`externals` and `ccc:` comments](#externals). With no map, ccc groups by directory and - the hints still mean something; where even that degenerates to one unit per file, it stays quiet - rather than calling every import a service call. -- Coverage is matched through the static call graph by name - not by running anything. A same-named - function elsewhere can produce a false positive, and a test that reaches code only through a - framework is invisible. -- Useful settings: `ccc.baseRef` (what to diff against), `ccc.binaryPath`, `ccc.hints.crossServiceMode`, - and `ccc.hints.codeLens` - set that to `false` with `ccc.decorations.style` as `badge+gutter` for - end-of-line badges instead of lenses. Commands are under **ccc:** in the palette. - -Full details, every setting, and the troubleshooting list are in -[`extensions/vscode/README.md`](extensions/vscode/README.md). \ No newline at end of file diff --git a/docs/EXTERNALS.md b/docs/EXTERNALS.md deleted file mode 100644 index 6873f7b..0000000 --- a/docs/EXTERNALS.md +++ /dev/null @@ -1,230 +0,0 @@ -# Cross-repository calls - -A project's calls do not stop at its own checkout. A gateway calls a billing service that lives in -another repository, written in another language, behind an HTTP or gRPC hop. No parser can follow -that call: there is no symbol to resolve, and the code on the other side is not even present. - -`ccc` closes the gap with two pieces that fit together: - -- **`externals` in `.ccc/map.json`** names the peer repositories. -- **`ccc:serves` / `ccc:calls` comments** name the key both ends agree on. - -Matching keys become real edges of the service graph, with a file and line at each end. For gRPC -the key needs no comment at all: the `.proto` schema supplies it -([gRPC without comments](#grpc-without-comments)). - -## The hints - -One spelling, in whatever comment syntax the language already uses. Nothing to install, nothing to -import, and no effect on the build: - -```rust -// ccc:serves grpc billing.v1.Charge -pub fn charge(account: &str, amount: u64) -> Result { … } -``` - -```go -func Checkout(cart []Item) error { - // ccc:calls grpc billing.v1.Charge - return client.Charge(ctx, req) -} -``` - -```python -# ccc:serves rest POST /v1/refund -@app.route("/v1/refund", methods=["POST"]) -def refund(): - ... -``` - -The grammar is `ccc: [transport] `: - -| part | values | -|---|---| -| directive | `serves` (or `provides`, `handles`) · `calls` (or `consumes`, `uses`) | -| transport | `grpc` `rest` `http` `https` `graphql` `queue` `event` `webhook` `ffi` `cli` `soap` `websocket` `ws` `sql` `rpc` — optional | -| key | everything after that, verbatim: `billing.v1.Charge`, `POST /v1/refund`, `orders.created` | - -The **key is the whole mechanism**. Both ends must write the same string; matching ignores case and -surrounding whitespace and nothing else. A `ccc:calls` whose key nothing answers is reported rather -than dropped — that is a typo at one end, or a repository nobody configured. - -Placement follows the convention each directive already implies. A directive **above a definition** -belongs to it, which is where `ccc:serves` naturally sits in a handler's doc block; decorators and -attributes in between are stepped over. A directive **inside a body** belongs to the enclosing -function, which is where `ccc:calls` naturally sits, next to the call it describes. A directive more -than ten lines above any definition is treated as file-level rather than being dragged onto whatever -function happens to appear later. - -## Naming the peers - -```json -{ - "services": { - "gateway": ["gateway/**"], - "shared": ["shared/**"] - }, - "relatives": { "gateway": ["billing"] }, - "externals": { - "billing": { - "repo": "acme/billing", - "lang": "go", - "path": "../billing" - }, - "ledger": { - "repo": "acme/ledger", - "surface": "https://artifacts.internal/ledger/surface.json", - "auth": "env:CCC_TOKEN" - } - } -} -``` - -An external is a service like any other: `relatives` may name it, and edges end at it. -A name cannot be both a service and an external — it is either code in this repo or code in another one. - -| field | meaning | -|---|---| -| `path` | A directory to parse: a sibling checkout, or another corner of a monorepo. Relative paths resolve against the repo root. | -| `surface` | A file, a directory containing `surface.json` (or the older `ccc-surface.json`, still read), or an `http(s)` URL, holding a surface published with `ccc init`. | -| `auth` | `env:VARIABLE` — the variable holding a bearer token for a private URL. Only this form is accepted; a literal token in a file that belongs in git is a mistake, not a feature. | -| `repo` | `owner/repo`, for display. | -| `lang` | The peer's language, for display when no surface is reachable. | - -`path` wins when the directory is really there — a checkout is the freshest view of a peer, and in a -monorepo it is the only one. `surface` is the fallback, and the only option for a repository you -cannot or should not clone. - -**A peer that cannot be reached is reported, never fatal.** A wrong token, an unreachable host or a -missing directory shows up as `resolved: false` with the reason attached, and the rest of the -analysis runs exactly as before. - -## Publishing a surface - -```sh -ccc init --name billing --repo acme/billing # -> .ccc/surface.json -ccc init --name billing -o - # stdout, for a CI artifact -``` - -A surface is only what a repository publishes and consumes — no bodies, no call graph, no private -symbols: - -```json -{ - "schema": "ccc-surface/1", - "name": "billing", - "generated": "20260812-18-47-02", - "repo": "acme/billing", - "languages": ["go"], - "provides": [ - { "key": "billing.v1.Charge", "transport": "grpc", - "function": "Charge", "file": "svc/charge.go", "line": 4 } - ], - "consumes": [ - { "key": "ledger.v1.Write", "transport": "grpc", - "function": "Refund", "file": "svc/charge.go", "line": 11 } - ] -} -``` - -It is typically a few hundred bytes. Consuming one costs no clone, no toolchain for that repo's -language, and no access to its source — which is what makes a private peer workable in the first -place. Publish it from CI on merge: - -```yaml -- run: ccc init --name billing --repo ${{ github.repository }} -- uses: actions/upload-artifact@v4 - with: { name: ccc-surface, path: .ccc/surface.json } -``` - -`consumes` is what lets a repository learn that something out there calls **in** to it — the one -direction no amount of local analysis could ever discover. - -## What you get - -`ccc changes` and `/insights.json` gain: - -- `externals[]` — each peer, how it was reached, and whether it resolved -- `crossings[]` — every matched pair, with the call site here and the handler there -- `edges[]` — crossings folded into the ordinary service graph, `via: "annotation"` - -``` -gateway -> billing [declared, detected] 2 call site(s) - gateway/checkout.rs:9 cancel -> POST /v1/refund (svc/charge.go:9) [rest, other repo] - gateway/checkout.rs:4 checkout -> billing.v1.Charge (svc/charge.go:4) [grpc, other repo] - -## external repos (1) -billing (acme/billing) via path ../peer-go - 2 provided, 1 consumed - -## unanswered keys (1) -gateway/checkout.rs:13 dangling calls 'nobody.v1.Answers' [grpc] - nothing serves this key -``` - -Because a crossing is an ordinary edge, everything downstream follows for free: a change in a peer -pulls the calling service into `services_to_test`, and `contract-test` targets rank the boundary -others depend on. - -The VS Code extension shows a crossing as a CodeLens above the call — `↑ calls billing in -acme/billing` — and opens the handler when that peer is checked out locally, or says where it lives -when it is not. - -## gRPC without comments - -An rpc is the rare call that crosses languages on purpose, and the schema every side was generated from already names it. - -Each rpc is keyed by its wire name, `acme.billing.v1.Billing/CreateInvoice`, -and code is tied to it by evidence — never by a name alone, because the name changes with the -generator: `CreateInvoice` in Go, `createInvoice` in TypeScript, `create_invoice` in Rust. - -| side | tied when the name matches (case and `_` ignored) and | evidence | -|---|---|---| -| handler | a parameter is the rpc's request message | `request-type` | -| handler | its owning type is named after the service, and the file imports the generated code | `service-owner` | -| caller | the receiver is the service's generated stub (`BillingClient`, `BillingStub`, …) | `stub-type` | -| caller | the file imports the generated code (`billing_pb2_grpc`, `…/billing/v1`, `billing_connect`) | `generated-import` | - -Where the evidence fits two rpcs equally, nothing is linked. Generated files themselves -(`*.pb.go`, `*_pb2_grpc.py`, `*_connect.ts`, `*Grpc.cs`, …) are never read as handlers or callers. - -What the links feed: - -- **`/references`** — looking up an rpc by any spelling (`create_invoice`, `Billing.CreateInvoice`, - the wire name) adds an `rpcs` block: the schema, its handlers and its callers, in every language. -- **Coverage** — a Python test through the stub covers the Go handler (evidence `rpc`). -- **The call graph** — client → rpc in the schema → handler. -- **`ccc changes`** — `rpc` edges from each caller's service to each handler's, and from both to the - service holding the schema, so a changed `.proto` reaches everything generated from it. -- **Surfaces** — `ccc init` publishes handlers under `provides` and calls under `consumes`, marked - `"via": "rpc"`, so a peer repository links to them exactly as it would to a `ccc:` comment. - -### Schemas kept elsewhere - -Schemas inside the project are found by the scan. When they live in a shared protos repository, a -buf module or a vendored directory, name them under `contracts` in `.ccc/map.json`: - -```json -{ - "contracts": ["../protos/acme/**/*.proto", "third_party/proto"] -} -``` - -Each entry is a file, a directory (every `.proto` under it) or a glob, relative to the repository -root and free to leave it. Ignore files are not consulted — vendored schemas are routinely ignored, -and naming them here is the opt-in. A peer checkout named by `path` is linked through its own -`contracts` and this repository's, since the schema is often in neither. - -A surface published before rpc links existed lacks `"rpc_endpoints": true`. A `path` peer whose -published surface is that old is parsed instead; a `surface`-only peer needs to re-run `ccc init`. - -An rpc call that nothing in view serves is not reported as an unanswered key: that is usually a -third-party API rather than a typo. - -## What this does and does not establish - -`ccc:serves` and `ccc:calls` are **statements by an author**, not inferences. That is exactly why -they can cross a process boundary when nothing else can — and exactly why they are only as accurate -as the comments. ccc reports them faithfully; it cannot check them. - -A crossing carries `via: "annotation"`, ranked above every inferred evidence kind, because a human -stated it. A `declared` dep with no detected calls remains the right shape for a link nobody has -annotated yet. diff --git a/docs/LANGUAGES.md b/docs/LANGUAGES.md deleted file mode 100644 index cd135e0..0000000 --- a/docs/LANGUAGES.md +++ /dev/null @@ -1,63 +0,0 @@ -# Language support - -What grammars/languages the `ccc` static analyser supports - -## Status - -Every language ccc can analyse. - -| | grammar | detect | funcs | consts | calls | imports | types | modules | metrics | x-file | tests | -|---|---|---|---|---|---|---|---|---|---|---|---| -| C | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | n/a | ✅ | ✅ | ✅ | -| C++ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| C# | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| Go | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| JavaScript | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | n/a | ✅ | ✅ | ✅ | -| Odin | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| Protobuf | ✅ | ✅ | ✅ | ✅ | n/a | ✅ | ✅ | ✅ | n/a | ✅ | ✅ | -| Python | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | n/a | ✅ | ✅ | ✅ | -| Rust | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| TypeScript | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| TSX | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| Zig | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | n/a | ✅ | ✅ | ✅ | - -✅ - done - -n/a - the language has no such concept - - -### Extension mapping - -| language | extensions | -|---|---| -| C | `.c` | -| C++ | `.cpp`, `.cc`, `.cxx`, `.c++`, `.hpp`, `.hh`, `.hxx`, `.h++`, `.h` | -| C# | `.cs`, `.csx` | -| Go | `.go` | -| JavaScript | `.js`, `.jsx`, `.mjs`, `.cjs` | -| Odin | `.odin` | -| Protobuf | `.proto` | -| Python | `.py`, `.pyi` | -| Rust | `.rs` | -| TypeScript | `.ts`, `.mts`, `.cts` | -| TSX | `.tsx` | -| Zig | `.zig` | - -Anything else is skipped rather than guessed at. `.h` is the one contested extension due to -me not wanting to handle the cases where we're walking a C++ vs C header so it defaults to C++. - -Please create a PR to change this if you want to solve that specific case. - -## Approximations - -Protobuf is a schema, so it is mapped onto the same shapes: a -`message` is a struct, a `service` an interface, each `rpc` a method of its -service taking its request message and returning its response (`stream` is -dropped), and enum values are constants. It has no calls or bodies, so the -calls and metrics columns do not apply. Code in other languages is linked to -each rpc through its generated stubs - see [gRPC without comments](EXTERNALS.md#grpc-without-comments). - -One deliberate approximation is worth naming: Zig's `errdefer` counts as a -guard even though it only runs on the error path. Reading a correctly written -cleanup as a leak is the worse of the two errors, and the heuristic is -name-matched with no data flow regardless. diff --git a/docs/MCP.md b/docs/MCP.md deleted file mode 100644 index 79baa19..0000000 --- a/docs/MCP.md +++ /dev/null @@ -1,58 +0,0 @@ -# MCP - -`ccc run` parses the project into an **in-memory copy of the map** and -serves it over local HTTP, so AI agents query the code map directly instead -of reading `.ccc` files from disk. A file watcher rescans automatically when -source changes (default: every 2s; `--no-watch` to disable): - -```sh -ccc run # http://127.0.0.1:6767 (MCP endpoint at /mcp -- insights at /insights) -``` - -```sh -curl -s localhost:6767/find?q=charge # symbol search (file:line + docs) -curl -s localhost:6767/references?symbol=charge # definitions + every call site -curl -s localhost:6767/dependencies?file=src/render.rs # file-level impact -curl -s -X POST localhost:6767/refresh # force an immediate rescan -``` - -## Editing - -ccc is the only writer an agent uses on a project it serves - the server's instructions rule out the agent's own -Edit/Write tools, `sed -i` and shell redirection. A `find` or `references` answer ends with a handle, and the edit -tools take it. Every edit is staged into a changeset and shown as a diff before anything is written: - -```text -find {"query": "hello"} -> 2 result(s) ... handle: `r1` (2 site(s)) -edit_rename {"result": "r1", "to": "hi"} -> `hello_all` -> `hi_all`, `hello_world` -> `hi_world` + diff, changeset `c2-...` -edit_apply {"changeset": "c2-..."} -> applied changeset `c2-...`, verified, map rescanned -``` - -| Tool | What it does | -|---|---| -| `edit_rename` | renames each identifier a handle points at. A name with a single definition is followed through every file the map ties to it, including lines the map skips. Same-named sites it left alone are listed with the reason, under a handle that pulls them in when passed back. Comments, strings and module path segments never change | -| `edit_replace` | replaces the token, the line, or the whole definition at each site | -| `edit_delete` | removes a definition with its doc comments and attributes, a call that is a statement on its own, a single-name import, or a note. Refuses while anything still names what it removes | -| `edit_insert` | adds code before or after the one site a handle names, at that site's indentation | -| `edit_text` | every other write: an exact-text replacement, a whole file, a removed file | -| `edit_apply` | writes a changeset - every file or none - and rescans the map | -| `edit_discard` | drops a pending changeset | -| `edit_revert` | puts back an applied changeset, provided none of its files changed since | - -What every write is held to: - -- **Confined.** The path stays inside the project root: no `..`, no symlinked directory or file on the way, never - `.git`, and under `.ccc` only `map.json` and `surface.json`. -- **Checked.** Every file must still hold what it held when the change was staged. If one moved, nothing is - written and the changeset stays pending. -- **Atomic.** Each file is written to a temp file beside it and renamed into place. If any rename fails, the files - already swapped are put back. -- **Agents only.** The edit tools refuse any request that carries an `Origin` header. A browser page (even a - `file://` report) can read the map but never write through it. -- **Attributed.** Each applied changeset is appended to `.ccc/edits.jsonl`, which is kept out of git. `ccc prompts` - reads it to credit a ccc edit to the request behind it, as strongly as it credits an `Edit` or `Write`. -- **Shared.** Every step - staged, applied, reverted, and each inspect an agent made - is appended to - `.ccc/timeline.jsonl`, also kept out of git, so the visualiser of any ccc server on the project shows it. - An inspect is a read tool's answer, or a file the agent read with its own tools, which the server - finds in Claude Code's transcripts (subagents included) every couple of seconds. - diff --git a/docs/PERF.md b/docs/PERF.md deleted file mode 100644 index eb3101e..0000000 --- a/docs/PERF.md +++ /dev/null @@ -1,31 +0,0 @@ -# Performance - -Same model, repo, prompt (*"generate an architecture diagram of this project"*), 5 runs each. - -## Human Acceptability - -Visual verification of each output to determine how human readable the results are. If they aren't readable then they aren't useful as architecture diagrams for humans. - -| Variant | Accepted rounds | **Rate** | -|---|---|---:| -| A (ccc MCP) | 2, 3, 4, 5 | **80%** (4/5) | -| B | 4 only | 20% (1/5) | -| C (frugal) | 5 only | 20% (1/5) | - -## Cost and Human Acceptability Conclusion - -Human accepted outputs measured by tokens used - -| Variant | Total spent (5 runs) | Accepted | **Tokens per valid output** | vs A | -|---|---:|---:|---:|---:| -| A (ccc MCP) | 5,415,904 | 4 | **1,353,976** | — | -| B | 7,726,770 | 1 | 7,726,770 | **5.71x** | -| C (frugal) | 4,481,014 | 1 | 4,481,014 | **3.31x** | - -Requiring both objective correctness *and* human acceptance: - -| Variant | Passes both | Tokens per pass | vs A | -|---|---:|---:|---:| -| A | 2 | 2,707,952 | — | -| B | 1 | 7,726,770 | 2.85x | -| C | 1 | 4,481,014 | 1.65x | \ No newline at end of file diff --git a/docs/PIPELINES.md b/docs/PIPELINES.md deleted file mode 100644 index e90fab7..0000000 --- a/docs/PIPELINES.md +++ /dev/null @@ -1,76 +0,0 @@ -# Pipelines - -The command `ccc changes` can output data to inform a pipeline about **what changed and what needs testing**. It diffs -the branch against a base ref (the merge-base with `origin/main` by default), -maps the diff down to function granularity, groups files into named *services* -(from `.ccc/map.json`), and detects cross-service call edges - so when -Service A calls Service B and B changes, both land in the test set: - -```sh -ccc init # scaffold .ccc/map.json from your top-level dirs, and the surface -ccc changes # one line of JSON: services_to_test, edges, untested, ... -ccc changes --format text # human-readable summary -ccc changes --fail-untested # gate: exit 1 when changed functions lack test references -ccc changes --worktree # include uncommitted edits and untracked files in the diff -``` - -```jsonc -{ - "schema": "ccc-changes/1", - "services_to_test": ["billing","gateway"], - "edges": - [ - { - "from": "gateway", - "to": "billing", - "declared": false, - "symbols": - [ - { - "symbol": "charge", - "file": "gateway/src/main.rs", - "line": 1, - "via": "receiver-type", - "kind": "call" - } - ] - } - ], - "changed_functions": - [ - { - "file": "billing/src/charge.rs", - "function": "charge", - "lines": [2,4], - "tested": true, - "tested_by": ["test_charge","TestCharge"], - "called_from": ["gateway"] - } - ], - "untested": - [ - { - "file": "billing/src/charge.rs", - "function": "fee", - "lines": [6,7], - "tested": false, - "tested_by": [], - "called_from": [] - } - ], - "...":"..." -} -``` - -`tested_by` names the test functions that call a changed function, so a review -can tell "covered by one smoke test" from "covered by twelve" without running -anything. Test functions are recognised by path (`tests/`, `*_test.go`, -`*.spec.ts`, ...), by Rust `mod tests`, by name (`test_charge`, `TestCharge`, -`BenchmarkCharge`), and - for jest/mocha/vitest suites, whose tests are -anonymous callbacks - by their label, reported as `it("charges a fee")`. It -lists direct callers only, is matched on the bare function name like the rest of -`changes`'s resolution, and is capped at 25 entries. `tested` can be true with an -empty `tested_by` when the only reference came from test-file top level rather -than from inside a named test. - - diff --git a/docs/STREAM.md b/docs/STREAM.md deleted file mode 100644 index 57048d0..0000000 --- a/docs/STREAM.md +++ /dev/null @@ -1,33 +0,0 @@ - -# Token Stream (stuffing pre-encoded cache into an LLM) - -> **Token stream is not compatible with Anthropic models.** These are **approximate** [tiktoken](https://github.com/openai/tiktoken) -> IDs (an OpenAI vocabulary). Which can be used with DeepSeek V4-Pro etc. -> Use it for a downstream model that shares the OpenAI vocab, or for rough size estimates. -> If using Claude, use the `.ccc` markdown as context. -> For exact Claude token counts, use Anthropic's `count_tokens` endpoint. -> `tokens.json` carries this caveat inline (`approximate: true` + a `note`). - -`ccc tokenize` (or `ccc scan --dir --tokens`) encodes the whole map with a -pretrained tiktoken vocabulary (`o200k_base` by default, `--encoding cl100k_base` -also supported) and writes: - -``` -.ccc/ -├── tokens.bin # little-endian u32 token IDs for every cache file, concatenated -└── tokens.json # index: encoding, layout, and per-file {offset, len} in tokens -``` - -Consumers load raw tokens with **no re-tokenization** - read `tokens.bin` as a -`u32` slice and index into it via `tokens.json`. The [`TokenCache`](src/tokenize.rs) -loader does exactly this and every `tokenize` run verifies the persisted stream -decodes back to the byte-identical corpus: - -```rust -let cache = codecache::TokenCache::load(project_root)?; -let ids: &[u32] = cache.file("src-main.rs.md").unwrap(); // raw tokens, ready to use -let text = cache.decode(ids)?; // optional: back to markdown -``` - -Token artifacts are derived, so a `ccc scan --dir` clears them; re-run with -`--tokens` (or `ccc tokenize`) to refresh. diff --git a/docs/TESTING.md b/docs/TESTING.md deleted file mode 100644 index e27e185..0000000 --- a/docs/TESTING.md +++ /dev/null @@ -1,64 +0,0 @@ -# Test triggers - -### Test triggers show what tests are called due to your changes. - -It diffs the branch against its base (the merge-base with `origin/main` by default) -**including uncommitted edits and untracked files**, maps the changed functions -onto the call graph, and reports: - -- **Run these tests.** A change does not only invalidate the tests that name the - changed function: any test exercising something *upstream* runs through it. - So the impacted set is the changed functions plus everything that transitively - calls them, and a test triggers if it references any member. `distance` is how - many call hops away the test landed - `direct` (0) first, since those are the - most likely to fail. -- **A runnable command per language**, so a CI job can paste it: - `cargo test -- ` · `go test -run '^(A|B)$' ./pkg` · `pytest -k "a or b"` - · `npx jest -t "…"` · `ctest -R '…'`. Each carries a caveat where the - selector is imprecise. When the trigger set covers 80%+ of the suite the tab - says so - at that share a long name filter is slower and more fragile than - just running everything. -- **Missing coverage.** Changed functions no test reaches, each with the kind of - test the signals justify. A CI gate can fail on this list. - -The same data is at `/insights.json` under `test_triggers`, so a pipeline can -consume it directly. `ccc changes --worktree` applies the same working-tree diff -on the command line. - -```sh -curl -s localhost:6767/insights.json | jq -r '.test_triggers.commands[].command' -curl -s localhost:6767/insights.json | jq '.test_triggers.counts' -``` - -**What it cannot know.** Tests are matched to changes by name through the call -graph, so this is the set *worth running* - not proof that running it covers the -change. A test that exercises code without naming it, or reaches it through -dynamic dispatch, is invisible. Outside a git repo, or without a base ref, the -tab says why rather than rendering an empty list that reads as "nothing to run". - -**Test targets** scores each kind and takes the strongest, so a recursive AST -walker with 31 call-outs is an *integration* target while a function with three -nested loops is a *performance* one: - -| kind | chosen when | -|---|---| -| `smoke-test` | Nothing stronger applies - typically an entry point nothing calls. | -| `integration-test` | It orchestrates others: `call-outs x4 + call depth x3 + complexity`. | -| `contract-test` | Callers in a **different service** - the boundary others depend on (`25` per calling service). | -| `perf-test` | `loop depth² x10` (nested iteration is superlinear), plus call depth when recursive. | -| `load-test` | Call sites in the **top decile** *and* it loops or acquires resources. | - -Language semantics sharpen the advice rather than the kind: a `Result`/`error` -return asks for the error path, an `Option` for the empty case, an acquired -resource for release on both paths, and an untyped language is told that a -contract test is the only thing pinning its argument shapes. - -Coverage here means **a test mentions the function by name** - not that its -behaviour is asserted, and not that the recommended *kind* of test exists. The -default filter shows only functions no test mentions at all. - -**Read this before trusting the findings.** ccc's map is a tree-sitter symbol -and call index: there is no type inference, no data flow, and no runtime -profile. So the flame view is a *static* call tree, not a sampled profile; -"hot" means structurally central, not frequently executed; and the language -rules are heuristics diff --git a/extensions/vscode/README.md b/extensions/vscode/README.md index e4ebdea..a5690e3 100644 --- a/extensions/vscode/README.md +++ b/extensions/vscode/README.md @@ -1,262 +1,73 @@ -# Code Change Capture (ccc) for VS Code - -Inline hints from the [`ccc`](../../README.md) static analyser, on the code they apply to. -Open a file with work in progress and each changed function tells you whether tests cover it, -what test to add if nothing does, and where it crosses a service boundary. - -The extension runs `ccc serve` in the background for each workspace folder and reads the analysis -over loopback HTTP. Nothing leaves your machine. - -## What it shows - -| | | -|---|---| -| `✓ 3 tests` | The function changed and tests cover it. The hover links straight to each test, with how many call hops away it sits and why the analyser thinks it applies. | -| `✗ no smoke test` | The function changed and nothing covers it. The kind named is the one to write — smoke, integration, contract, perf or load — and the hover quotes the analyser's reasoning and the signals that made it rank. Unranked functions read plain `✗ no test`. | -| `→ billing` | This line calls into another service. The hover links to the callee's definition and says how the call was resolved — by receiver type, by import, or, less certainly, by name alone. | -| `← gateway` | Another service calls this function. The hover links to each calling function. | -| `▲ 37 callers` | A hot path: the function is among the most called, the most complex, or the widest fan-out, or it heads the deepest call chain. | -| `↻ cycle of 3` | The function is in a call cycle. The hover links to the other members. | -| `↑ calls billing in acme/billing` | This call leaves the repository. Clicking opens the handler when that peer is checked out locally, and says where it lives when it is not. | -| `? billing.v1.Charge unanswered` | A `ccc:calls` whose key nothing serves — a typo at one end, or a peer missing from `externals`. | - -Hints render as **CodeLens lines above the function or the call**, so they never compete with the -code for horizontal space, and each one is clickable: a test lens opens the test, a boundary lens -opens the handler, an untested lens explains what to add. Several lenses share one row, so a changed -function that also calls out reads `✓ 2 tests | ↑ calls billing`. - -A gutter icon marks the same line, and the hover on the function name carries the full detail. Set -`ccc.hints.codeLens` to `false` and `ccc.decorations.style` to `badge+gutter` for end-of-line badges -instead. - -Untracked and uncommitted files are included — the analysis runs against the working tree, not the -last commit. - -**The first four hints are diff-driven; hot paths are not.** If a file is identical to the base ref, -it has no changed functions and therefore no coverage or cross-service hints — that is the design, -not a fault. Hot paths come from the call graph alone, so they appear on untouched files too, and -the status bar tooltip always says which of the two situations you are looking at. - -## The test-triggers panel +

Run ccc

-A triggered test almost always lives in a *different* file from the change that triggered it, so no -decoration can show you the whole set. Click the **ccc** mark in the activity bar — the banded disc — -to open the panel, and click it again to close it. It lists: +# Code Change Capture (ccc) for VS Code -- **Run these** — every test your working tree makes necessary, with how many call hops it sits from - the change. Click to open it. -- **No test covers** — changed functions nothing exercises, each with the kind of test to add. -- **Commands** — the analyser's suggested command for running exactly that set. Click to run it in a - terminal. +See what every change touches before you commit it, watch an agent's edits land on a live map of your code, and replay how a branch was built. -The panel badge is the number of tests worth running before you push. +The extension runs the `ccc` analyser in the background for each workspace folder. Nothing leaves your machine. -## Requirements +## See the impact of your changes -**The `ccc` binary.** The extension installs it for you. When it activates it looks for a working -`ccc` — `ccc.binaryPath`, your `PATH`, `/target/release/ccc`, `/target/debug/ccc`, -then a copy it installed earlier — and, finding none, downloads the release matching your platform -into its own storage before the first analyser starts. Nothing on your `PATH` is modified, and -`ccc.autoInstall: false` turns the download off. To manage the binary yourself, install it with -`./install.sh` from the repo root, or `cargo build --release` and point `ccc.binaryPath` at -`target/release/ccc`; a binary you provide is always preferred to the one the extension installs. +Save a file and each function you changed shows what it affects, right above its signature: -**A git repo with a resolvable base ref**, for the coverage hints. The analyser diffs against the -first of `origin/main`, `main`, `origin/master`, `master` that exists. On a shallow clone or a repo -with none of those, hints 1 and 2 go quiet and the status bar says why; **ccc: Select Git Base Ref…** -sets `ccc.baseRef` for the folder. +| Hint | What it tells you | +|---|---| +| `✓ 3 tests` | Tests reach this change. Click to open them, nearest first. | +| `✗ no smoke test` | Nothing tests it, and which kind of test to write. | +| `→ billing` · `← gateway` | It calls into another service, or another service calls it. | +| `↑ calls billing in acme/billing` | It calls into another repository. Click to open the handler there. | +| `▲ 37 callers` · `↻ cycle of 3` | A hot path or a call cycle. These show on every file, changed or not. | -**`externals` and `ccc:` comments**, for cross-repository hints — see -[EXTERNALS.md](../../docs/EXTERNALS.md). A call annotated `// ccc:calls grpc billing.v1.Charge` -whose key is served by a peer repository becomes a lens that opens the handler over there. +The **ccc** panel in the activity bar lists every test your changes put at risk, with a command that runs exactly those tests. It also ranks every function by complexity, worst first, and a circled number from 1 to 10 next to each function's name shows its complexity in the editor. -**A `services` block in `.ccc/map.json`**, for the in-repo cross-boundary hints: +## Visualiser -```json -{ - "services": { "gateway": ["gateway/**"], "billing": ["billing/**", "libs/money/**"] }, - "deps": { "gateway": ["billing"] } -} -``` +

The visualiser drawing one function's logic, with an agent's edit marked in place

-Without it the analyser has no notion of a boundary, and falls back to grouping by directory. See -*Understanding the hints* below for what the extension does in that case. +A map of your project you can drill into: from the system and its services, through files and functions, down to the logic inside a single function. While an agent edits through ccc, the visualiser moves to each change as it lands and draws it in place, with added lines in green and removed ones struck through. Open it in the panel, in a tab beside your code, or in a browser. -## Install +## Replays -`cargo build` at the repo root packages the extension too, so the analyser and the client that talks -to it stay in step: +

A replay of an agent session, with its steps listed beside the map

-```sh -cargo build --release # -> dist/ccc-codecache.vsix -code --install-extension dist/ccc-codecache.vsix -``` +Play back how a change was made: what you asked, what the agent read, and every edit it made, split into chapters, shown in subtitles and read aloud. Replays are saved beside your branch when you push, so reviewers can watch them too. -That step is best-effort: it is skipped, with a `cargo:warning`, when `npm` is missing, when `CI` is -set, or when `CCC_SKIP_VSIX` is set — a broken extension build never fails the Rust build. It also -only re-runs when something under `extensions/vscode` actually changed, so ordinary Rust rebuilds -stay fast. +- **Recording is off until you turn it on** for a repository, with **ccc: Session Recording…**. The extension never records or pushes a replay by itself. +- **Secrets are removed** before anything is saved. +- **Public remotes are protected.** A replay never goes to a remote anyone can read, unless it's encrypted for your team or `.ccc/map.json` allows it. -To build the vsix on its own: +## Get started -```sh -cd extensions/vscode -npm install -npm run package # -> ../../dist/ccc-codecache.vsix -``` +1. Install the extension. It uses the `ccc` on your machine, or downloads the right release if there isn't one. +2. Open a folder that's a git repository and change some code. +3. Save. Hints appear on the functions you changed. -This is not published to the Marketplace. +For hints about services and other repositories, describe your services in `.ccc/map.json`. Run `ccc init` to create a starter file. ## Commands | Command | What it does | |---|---| -| `ccc: Refresh Hints` | Rescan and re-analyse now, bypassing every cache. | -| `ccc: Restart Analyser` | Kill and respawn the analyser process. | -| `ccc: Stop Analyser` | Shut the analyser down for this window. | -| `ccc: Toggle Inline Hints` | Turn all decorations on or off. | -| `ccc: Show Log` | Open the output channel, including the analyser's own stderr. | -| `ccc: Open Insights UI in Browser` | Open the analyser's full insights page. | -| `ccc: Select Git Base Ref…` | Pick the ref to diff against, from your branches. | -| `ccc: Copy Test Command for Changes` | Copy the analyser's suggested command for running exactly the triggered tests. | +| `ccc: Refresh Hints` | Analyse again now. | +| `ccc: Open Architecture Visualiser` | Open the visualiser in a tab beside your code. | +| `ccc: Copy Test Command for Changes` | Copy the command that runs exactly the tests your changes put at risk. | +| `ccc: Select Git Base Ref...` | Choose the branch your changes are compared against. | +| `ccc: Session Recording…` | Turn replay recording on or off for this repository. | +| `ccc: Toggle Inline Hints` | Show or hide all hints. | +| `ccc: Restart Analyser` / `ccc: Stop Analyser` | Restart or stop the background analyser. | +| `ccc: Show Log` | Open the extension's log, including the analyser's own output. | ## Settings -**Server** — `ccc.binaryPath`, `ccc.server.autoStart`, `ccc.server.address`, `ccc.server.port` -(`0` picks a free port; a fixed port collides across windows), `ccc.server.watchIntervalSec` -(`0` starts the analyser with `--no-watch` and lets the extension drive rescans), -`ccc.server.startupTimeoutMs`, `ccc.server.extraArgs`. - -**Hints** — `ccc.enable`, `ccc.baseRef`, `ccc.hints.codeLens`, and one toggle per hint type: -`ccc.hints.testTriggers`, `ccc.hints.untested`, `ccc.hints.outbound`, `ccc.hints.inbound`, -`ccc.hints.hotPaths`. -`ccc.hints.includeTestFiles` also decorates changed test code. -`ccc.hints.crossServiceMode` (`auto` / `always` / `off`) and `ccc.hints.minEvidence` -(`evidence` / `any`) are explained below. -`ccc.untested.showUncoveredTargets` hints high-priority uncovered functions even where there is no -diff, and `ccc.untested.minPriority` sets the floor. - -**Decorations** — `ccc.decorations.style` (`badge+gutter` / `badge` / `gutter`), -`badgeMaxLength`, `dimWhenDirty`, `overviewRuler`. - -**Refresh** — `ccc.refresh.onSave`, `onWindowFocus`, `intervalSec` (off by default), `debounceMs`. - -**Logging** — `ccc.trace` (`off` / `messages` / `verbose`). - -## Understanding the hints - -This is the section worth reading. The analysis is static, and knowing what it did and did not -establish is the difference between a useful hint and a misleading one. - -**Coverage is matched through the call graph, not by running anything.** A test is linked to a -change because the graph connects the test to the changed function. A test that exercises code -without naming it — through dynamic dispatch, a table of function pointers, a framework — is -invisible, so `✗ no test` means "nothing reaches it", not "nothing runs it". - -The link is to a *definition*, not to a name. A call is tied to the function it names only when -something beyond the name agrees: the receiver's declared type, the same file, the same package -scope, an import, or a qualifier naming the defining file — and where that evidence fits more than -one definition, nothing is claimed at all. A call that leaves the project (`std::fs::write`) covers -nothing, and a candidate must be in the caller's runtime family, so a Rust test can never mark a -TypeScript function covered. One weak tie survives: in an untyped language, a bare call to a name -with exactly one definition in the project. It counts, and the hover says it was matched on the name -alone. Every listed test carries its own file and line, so the link opens the test that was matched -rather than one that happens to share its name. - -**Test functions are recognised, not counted.** A changed function inside an inline `mod tests` or a -`tests/` directory is shown as test code, not as covered — coverage of a test by itself is not a -useful claim. - -**Cross-boundary hints depend on configuration.** With a `services` block the hint sits on the exact -call line and says how the call was resolved. Without one, the analyser groups by top-level -directory and the extension labels the result *module* rather than *service*, and marks the calling -function rather than the call, because that is all the data supports. When the grouping degenerates -to one unit per file — a flat project with no sub-directories — every import would look like a -service call, so `ccc.hints.crossServiceMode: auto` suppresses these hints entirely. Set it to -`always` to see them anyway. - -`ccc.hints.minEvidence: evidence` (the default) drops calls resolved by name alone, which is the -untyped-language fallback and the main source of false positives. Set it to `any` to see them, with -the hover marking them `⚠ matched by name only`. - -An edge that is declared in `.ccc/map.json` but has no detected call site is normal, not a bug — -that is what an HTTP, RPC or queue dependency looks like to a static analyser. - -**Hints reflect the last saved state.** The analyser reads the filesystem, not your editor buffer, -so while a file has unsaved edits no refresh can describe what is on screen. Rather than hide the -hints or let them look authoritative, they fade, the status bar shows `$(circle-outline)`, and each -hover says so. Save, and they come back. VS Code moves the decorations along with your edits, so -their positions stay right even while the analysis behind them is stale. - -**"Hot" means structurally central, not measured.** The ranking comes from the shape of the call -graph — how many functions call this one, how complex it is, how wide it fans out — and nothing is -executed or profiled. A function called once from a tight loop is not hot by this measure, and one -called from thirty places that never runs in production is. The analyser ships that caveat itself -and the hover quotes it. Only the top 25 of each view are marked, so the absence of a flame is not a -claim that a function is cold. - -**Lists are capped by the analyser**, at 25 covering tests, 60 ranked targets, 60 call sites per -edge, 100 symbols per edge, and 8 names per triggered test. Hovers say when a list is at its cap. - -## Troubleshooting - -**No hints anywhere.** Check the status bar. `$(error) ccc` means the analyser did not start — run -`ccc: Show Log`. `$(beaker) ccc $(warning)` means git could not resolve a base ref, and the tooltip -carries the analyser's own explanation. - -**No hints in one file.** Two different causes, and the status bar tooltip tells you which: - -- *"Nothing in this file changed against `origin/main`"* — the file is mapped and understood, it - simply has no changed functions and no hot path. Point `ccc.baseRef` at an older ref to widen the - diff, or turn on `ccc.untested.showUncoveredTargets`. -- *"This file is not in the ccc map"* — it is ignored by git, or written in a language `ccc` does - not parse. - -**Everything looks like a cross-service call.** Your project has no `.ccc/map.json`, so boundaries -were inferred from directories. Add a `services` block. - -**"could not find the ccc binary".** The download failed, or `ccc.autoInstall` is off and nothing is -installed. The message lists every path that was searched; **Retry Install** tries the download -again, and `ccc: Show Log` says why the last one failed. Failing that, build it -(`cargo build --release`) or set `ccc.binaryPath`. - -**The analyser keeps restarting.** It is restarted with backoff and gives up after five failures in -five minutes. `ccc: Show Log` has the last 20 lines of its stderr for each crash. - -**Seeing the raw data.** `ccc: Open Insights UI in Browser` opens the analyser's own page over the -same data the hints are built from. - -## Performance - -One analyser process per workspace folder per window, started lazily the first time you open a file -in that folder. Activation itself only probes for the binary — a single `ccc --version` — and costs -a download once, when there is no ccc to find. It runs with `--no-watch` by default and rescans on save and on window focus, so it -is idle between edits. Two windows on the same folder run two analysers, each on its own free port; -a `ccc serve` you started yourself is neither used nor disturbed. - -## Known limitations - -- No Problems-panel entries, by design — hints stay on the code they describe. -- A boundary crossing is only as accurate as the `ccc:` comments: ccc reports what an author stated - and cannot check it. -- The extension does not run tests; it tells you which to run and can copy the command. -- Unsaved edits are invisible to the analysis. -- Multi-root workspaces spawn one analyser per folder. -- Only `file:` documents are decorated, so the SCM diff editor stays clean. - -## Development - -```sh -npm install -npm run watch # esbuild in watch mode -# then F5 -> Extension Development Host -npm run check # tsc --noEmit -npm run package # -> ccc-codecache.vsix -``` +- `ccc.baseRef`: the ref to compare against. Defaults to the first of `origin/main`, `main`, `origin/master` and `master` that exists. +- `ccc.hints.*`: turn each kind of hint on or off. `ccc.hints.codeLens: false` with `ccc.decorations.style: badge+gutter` shows end-of-line badges instead of lenses. +- `ccc.complexity.enabled` and `ccc.complexity.minScore`: the complexity numbers, and the lowest one worth showing. +- `ccc.search.mode`, `ccc.search.urls` and `ccc.search.endpoints`: what the analyser's `search` tool may reach on the web. For example, only your company's docs and portals. +- `ccc.binaryPath` and `ccc.autoInstall`: use your own `ccc`, or stop the extension downloading one. +- `ccc.server.*`: how the analyser runs. `ccc.server.extraArgs` passes extra `ccc run` options. -`src/model.ts` holds the whole payload-to-hints mapping and imports nothing from `vscode`, so it can -be exercised against a saved `/insights.json` without an extension host. +How the hints are worked out, every setting in detail, and troubleshooting are in the [Advanced guide](../../docs/Advanced.md#ides). ## License -MIT, the same as the parent repo. +MIT diff --git a/extensions/vscode/package.json b/extensions/vscode/package.json index a8fa9dc..8a8e2ad 100644 --- a/extensions/vscode/package.json +++ b/extensions/vscode/package.json @@ -1,8 +1,8 @@ { "name": "run-ccc", "displayName": "Code Change Capture (ccc)", - "description": "ccc static analyser", - "version": "1.7.0", + "description": "See what every change touches, watch agents edit on a live map of your code, and replay how a branch was built.", + "version": "1.8.0", "publisher": "colwill", "license": "MIT", "icon": "media/ccc.png", @@ -124,11 +124,6 @@ "title": "Show Log", "category": "ccc" }, - { - "command": "ccc.openInsights", - "title": "Open Insights UI in Browser", - "category": "ccc" - }, { "command": "ccc.openVisualiser", "title": "Open Architecture Visualiser", @@ -290,10 +285,6 @@ "command": "ccc.stopServer", "when": "ccc.active" }, - { - "command": "ccc.openInsights", - "when": "ccc.active" - }, { "command": "ccc.copyTestCommand", "when": "ccc.active" @@ -406,6 +397,33 @@ "default": false, "description": "Read every `ccc:skip` comment as an ordinary comment (`--ignore-skip`), so the code it marks is mapped, searched and edited like the rest." }, + "ccc.search.mode": { + "type": "string", + "enum": ["none", "allow", "block"], + "enumDescriptions": [ + "No rules: the `search` tool may fetch any https URL.", + "Only URLs matching an entry in `ccc.search.urls` may be fetched.", + "URLs matching an entry in `ccc.search.urls` are refused; everything else may be fetched." + ], + "default": "none", + "markdownDescription": "Which URLs the analyser's `search` MCP tool may reach. Requests are https only, whatever the mode." + }, + "ccc.search.urls": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "markdownDescription": "The URL list `#ccc.search.mode#` applies to. An entry with a scheme is a URL pattern (`https://docs.corp/**`); one without is a host pattern (`*.corp.example`). Matching ignores case." + }, + "ccc.search.endpoints": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "markdownDescription": "Search pages the `search` tool sends queries to, with `{query}` where the words go, e.g. `https://wiki.corp/search?q={query}`. Each request still passes `#ccc.search.mode#`." + }, "ccc.hints.testTriggers": { "type": "boolean", "default": true, diff --git a/extensions/vscode/src/commands.ts b/extensions/vscode/src/commands.ts index 0dcd840..0485daf 100644 --- a/extensions/vscode/src/commands.ts +++ b/extensions/vscode/src/commands.ts @@ -38,16 +38,6 @@ export function registerCommands(host: CommandHost): vscode.Disposable[] { host.render(); }), - vscode.commands.registerCommand('ccc.openInsights', async () => { - const session = host.activeSession(); - const url = session?.insightsUrl; - if (!url) { - void vscode.window.showWarningMessage('ccc: the analyser is not running yet.'); - return; - } - await vscode.env.openExternal(vscode.Uri.parse(url)); - }), - vscode.commands.registerCommand('ccc.openLocation', async (arg: unknown) => { const target = readLocation(arg); const session = host.activeSession(); diff --git a/extensions/vscode/src/config.ts b/extensions/vscode/src/config.ts index 0b9369d..ce0749d 100644 --- a/extensions/vscode/src/config.ts +++ b/extensions/vscode/src/config.ts @@ -4,6 +4,7 @@ import type { TraceLevel } from './log'; export type CrossServiceMode = 'auto' | 'always' | 'off'; export type MinEvidence = 'any' | 'evidence'; export type DecorationStyle = 'badge+gutter' | 'badge' | 'gutter'; +export type SearchMode = 'none' | 'allow' | 'block'; export interface Cfg { enable: boolean; @@ -53,6 +54,12 @@ export interface Cfg { intervalSec: number; debounceMs: number; }; + // what the analyser's `search` tool may reach - `none` any https url, `allow` only the urls listed, `block` all but them + search: { + mode: SearchMode; + urls: string[]; + endpoints: string[]; + }; trace: TraceLevel; } @@ -105,6 +112,11 @@ export function readConfig(scope?: vscode.ConfigurationScope): Cfg { intervalSec: clampInt(c.get('refresh.intervalSec', 0), 0, 86400, 0), debounceMs: clampInt(c.get('refresh.debounceMs', 400), 0, 60000, 400), }, + search: { + mode: c.get('search.mode', 'none'), + urls: c.get('search.urls', []).filter((u) => typeof u === 'string'), + endpoints: c.get('search.endpoints', []).filter((e) => typeof e === 'string'), + }, trace: c.get('trace', 'off'), }; } @@ -117,7 +129,8 @@ export function needsServerRestart(a: Cfg, b: Cfg): boolean { a.server.port !== b.server.port || a.server.watchIntervalSec !== b.server.watchIntervalSec || a.server.ignoreSkip !== b.server.ignoreSkip || - a.server.extraArgs.join('\u0000') !== b.server.extraArgs.join('\u0000') + a.server.extraArgs.join('\u0000') !== b.server.extraArgs.join('\u0000') || + JSON.stringify(a.search) !== JSON.stringify(b.search) ); } diff --git a/extensions/vscode/src/extension.ts b/extensions/vscode/src/extension.ts index 1e44e20..8a31fd0 100644 --- a/extensions/vscode/src/extension.ts +++ b/extensions/vscode/src/extension.ts @@ -291,7 +291,6 @@ class Extension implements CommandHost { try { await session.ensureStarted(); await vscode.commands.executeCommand('setContext', 'ccc.active', true); - this.askRecording(folder); } catch (err) { this.reportStartFailure(folder, err); } @@ -500,8 +499,8 @@ class Extension implements CommandHost { this.secrets.clear(); } - // Whether this folder's agent sessions may be recorded - asked once a window, - // and only while nobody has answered; the command asks it again on purpose. + // Whether this folder's agent sessions may be recorded - asked only when the + // Session Recording command is run, never on its own. private askRecording(folder: vscode.WorkspaceFolder, again = false): void { const key = folder.uri.toString(); if (!again && this.recordingAsked.has(key)) return; diff --git a/extensions/vscode/src/server.ts b/extensions/vscode/src/server.ts index a5d753f..634a36d 100644 --- a/extensions/vscode/src/server.ts +++ b/extensions/vscode/src/server.ts @@ -126,6 +126,10 @@ export class ServerProcess implements vscode.Disposable { ? ['--no-watch'] : ['--watch-interval', String(this.cfg.server.watchIntervalSec)]), ...(this.cfg.server.ignoreSkip ? ['--ignore-skip'] : []), + // only when set, so a ccc older than `search` still starts + ...(this.cfg.search.mode !== 'none' ? ['--search-mode', this.cfg.search.mode] : []), + ...this.cfg.search.urls.flatMap((u) => ['--search-url', u]), + ...this.cfg.search.endpoints.flatMap((e) => ['--search-endpoint', e]), ...this.cfg.server.extraArgs, ]; this.log.info(`[${this.label}] spawning: ${bin} serve ${args.join(' ')}`); diff --git a/extensions/vscode/src/session.ts b/extensions/vscode/src/session.ts index c3a2165..1b2e699 100644 --- a/extensions/vscode/src/session.ts +++ b/extensions/vscode/src/session.ts @@ -269,11 +269,6 @@ export class WorkspaceSession implements vscode.Disposable { return this.client.references(symbol); } - get insightsUrl(): string | undefined { - const address = this.server.address; - return address ? `${address.base}/insights` : undefined; - } - get visualiserUrl(): string | undefined { const address = this.server.address; return address ? `${address.base}/vis` : undefined; diff --git a/src/changes.rs b/src/changes.rs index 117d4b9..b08b25d 100644 --- a/src/changes.rs +++ b/src/changes.rs @@ -8,6 +8,7 @@ use crate::contracts::ContractIndex; use crate::coverage; use crate::extract::BDD_REGISTRARS; use crate::model::{Boundary, FileCache}; +use crate::par; use crate::scan; use anyhow::{anyhow, bail, Context, Result}; use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; @@ -277,8 +278,16 @@ pub struct ChangesReport { // Analyze `root` and build the changes report, parsing the tree first. pub fn changes(root: &Path, root_label: &str, opts: &ChangesOptions) -> Result { let files = scan::collect_files(root)?; - let caches = scan::build_caches(root, &files); - changes_with_caches(root, root_label, opts, &caches) + // the git half needs nothing from the map, so it runs while the map is parsed + let (caches, prepared) = std::thread::scope(|s| { + let prepared = s.spawn(|| prepare(root, opts)); + let caches = scan::build_caches(root, &files); + (caches, prepared.join().unwrap_or_else(|e| std::panic::resume_unwind(e))) + }); + let report = assemble(root, root_label, opts, &caches, prepared?, None); + // a large map takes a while to free, and the caller only wants the report + std::thread::spawn(move || drop(caches)); + report } // Same analysis against an already-parsed map, so a caller that holds one @@ -289,6 +298,30 @@ pub fn changes_with_caches( opts: &ChangesOptions, caches: &[FileCache], ) -> Result { + assemble(root, root_label, opts, caches, prepare(root, opts)?, None) +} + +// What a change report needs from git and the disk and nothing from the parsed +// map - the diff and what it did to dependencies, metrics, secrets and which +// request made it - so a caller can have it read while the map is parsed. +pub(crate) struct Prepared { + config: ChangesConfig, + matchers: Vec<(String, GlobSet)>, + service_names: Vec, + base_label: String, + base_sha: String, + head_sha: String, + statuses: Vec<(String, String)>, + hunks: BTreeMap>, + uncommitted: BTreeSet, + deps: Option, + telemetry: crate::telemetry::TelemetryReport, + secrets: crate::secrets::Report, + turns: Vec, + attribution: crate::prompts::Attribution, +} + +pub(crate) fn prepare(root: &Path, opts: &ChangesOptions) -> Result { let mut config = ChangesConfig::load(root)?; for (name, glob) in &opts.service_flags { config @@ -328,11 +361,6 @@ pub fn changes_with_caches( } let matchers = build_matchers(&config.services)?; let service_names: Vec = config.services.keys().cloned().collect(); - // the rpcs every schema declares, and the code in any language tied to them - let schemas = crate::contracts::load_schemas(root, &config.contracts); - let contracts = ContractIndex::build(caches, &schemas); - let externals = crate::externals::resolve_all(root, &config.externals, &contracts.schemas); - let (base_label, base_sha) = resolve_base(root, opts.base.as_deref())?; let head_sha = git(root, &["rev-parse", "HEAD"])?.trim().to_string(); // With `worktree`, omitting the second ref makes git diff the base against @@ -378,65 +406,140 @@ pub fn changes_with_caches( BTreeSet::new() }; - // What the branch did to the dependency tree - let deps = opts.deps.then(|| { - crate::deps::analyse( - root, - &crate::deps::DepsOptions { - base_sha: &base_sha, - head_sha: &head_sha, - worktree: opts.worktree, - touched: &statuses, - offline: false, - }, + // These four share nothing and each waits on git or the disk, so they run + // side by side: the slowest sets the pace rather than their sum. + let (deps, telemetry, secrets, (turns, attribution)) = std::thread::scope(|s| { + // what the branch did to the dependency tree + let deps = s.spawn(|| { + opts.deps.then(|| { + crate::deps::analyse( + root, + &crate::deps::DepsOptions { + base_sha: &base_sha, + head_sha: &head_sha, + worktree: opts.worktree, + touched: &statuses, + offline: false, + }, + ) + }) + }); + // what the branch did to the metrics the source emits + let telemetry = s.spawn(|| { + crate::telemetry::analyse( + root, + &crate::telemetry::TelemetryOptions { + base_sha: &base_sha, + head_sha: &head_sha, + worktree: opts.worktree, + }, + ) + }); + // what the branch adds that looks like a credential, while it can still be taken out + let secrets = s.spawn(|| crate::secrets::against(root, &base_label, &base_sha, opts.worktree)); + // which request produced which change + let prompts = if opts.prompts { + let (turns, _) = crate::prompts::collect(root, &crate::prompts::PromptsOptions::default()); + let times = write_times(root, &base_sha, &hunks, &uncommitted); + let attribution = crate::prompts::attribute(root, &turns, &hunks, ×); + (turns, attribution) + } else { + (Vec::new(), crate::prompts::Attribution::default()) + }; + ( + deps.join().unwrap_or_else(|e| std::panic::resume_unwind(e)), + telemetry.join().unwrap_or_else(|e| std::panic::resume_unwind(e)), + secrets.join().unwrap_or_else(|e| std::panic::resume_unwind(e)), + prompts, ) }); - // What the branch did to the metrics the source emits - let telemetry = crate::telemetry::analyse( - root, - &crate::telemetry::TelemetryOptions { - base_sha: &base_sha, - head_sha: &head_sha, - worktree: opts.worktree, - }, - ); - - // What the branch adds that looks like a credential, while it can still be taken out - let secrets = crate::secrets::against(root, &base_label, &base_sha, opts.worktree); + Ok(Prepared { + config, + matchers, + service_names, + base_label, + base_sha, + head_sha, + statuses, + hunks, + uncommitted, + deps, + telemetry, + secrets, + turns, + attribution, + }) +} - // Which request produced which change - let (turns, attribution) = if opts.prompts { - let (turns, _) = crate::prompts::collect(root, &crate::prompts::PromptsOptions::default()); - let times = write_times(root, &base_sha, &hunks, &uncommitted); - let attribution = crate::prompts::attribute(root, &turns, &hunks, ×); - (turns, attribution) - } else { - (Vec::new(), crate::prompts::Attribution::default()) +// The half of a change report that reads the parsed map. `built` hands in a +// contract index and coverage relation the caller already has, so `insights` +// does not build them twice. +pub(crate) fn assemble( + root: &Path, + root_label: &str, + opts: &ChangesOptions, + caches: &[FileCache], + prepared: Prepared, + built: Option<(&ContractIndex, &coverage::CoverageIndex)>, +) -> Result { + let _ = opts; + let Prepared { + config, + matchers, + service_names, + base_label, + base_sha, + head_sha, + statuses, + hunks, + uncommitted, + deps, + telemetry, + secrets, + turns, + attribution, + } = prepared; + // the rpcs every schema declares, and the code in any language tied to them + let own_contracts; + let contracts = match built { + Some((contracts, _)) => contracts, + None => { + own_contracts = ContractIndex::build(caches, &crate::contracts::load_schemas(root, &config.contracts)); + &own_contracts + } }; + let externals = crate::externals::resolve_all(root, &config.externals, &contracts.schemas); - let idx = build_indexes(root, caches, &matchers); - // Which tests reach which definitions. One relation, shared with - // `insights`, so the two reports cannot disagree about what is covered. let project_ids: BTreeSet = manifest_identities(root).into_iter().map(|(id, _)| id).collect(); - let cov = coverage::build(caches, &project_ids, &contracts); - - // cross-service edges + per-symbol caller map - let (mut edges, mut symbol_callers, mut unresolved_calls) = detect_edges(&idx, &config.relatives); - merge_rpc_edges( - &mut edges, - &mut symbol_callers, - &mut unresolved_calls, - caches, - &matchers, - &contracts, - ); - - // boundary crossings: the calls that leave the process, which the call - // graph cannot see - named by an author, or derived from a schema - let crossings = detect_crossings(caches, &matchers, &externals, &contracts); - merge_crossings(&mut edges, &crossings); + // Coverage and the service edges each read the map and not each other, so + // they are built side by side. + let (own_cov, edges, symbol_callers, unresolved_calls, crossings) = std::thread::scope(|s| { + // which tests reach which definitions - one relation, shared with + // `insights`, so the two reports cannot disagree about what is covered + let own_cov = s.spawn(|| built.is_none().then(|| coverage::build(caches, &project_ids, contracts))); + // cross-service edges + per-symbol caller map + // one service and no relatives leaves nothing to cross into - the case with no `.ccc/map.json` + let (mut edges, mut symbol_callers, mut unresolved_calls) = + if matchers.len() < 2 && config.relatives.is_empty() { + (Vec::new(), BTreeMap::new(), Vec::new()) + } else { + detect_edges(&build_indexes(root, caches, &matchers), &config.relatives) + }; + merge_rpc_edges(&mut edges, &mut symbol_callers, &mut unresolved_calls, caches, &matchers, contracts); + // boundary crossings: the calls that leave the process, which the call + // graph cannot see - named by an author, or derived from a schema + let crossings = detect_crossings(caches, &matchers, &externals, contracts); + merge_crossings(&mut edges, &crossings); + let own_cov = own_cov.join().unwrap_or_else(|e| std::panic::resume_unwind(e)); + (own_cov, edges, symbol_callers, unresolved_calls, crossings) + }); + let cov = match (built, &own_cov) { + (Some((_, cov)), _) => cov, + (None, Some(cov)) => cov, + (None, None) => unreachable!("built above when none was handed in"), + }; // changed files -> services let mut changed_files = Vec::new(); @@ -794,15 +897,16 @@ fn build_indexes( } } - // pass 2: imports, calls, type references - for cache in caches { + // pass 2: imports, calls, type references - read only what pass 1 built, so worked out at once per file + let per_file = par::map(caches.len(), |i| { + let cache = &caches[i]; let rel = path_str(&cache.rel_path); let services = assign(matchers, &rel); let typed = cache.language.is_typed(); // which services each imported name could have come from let mut wildcard: BTreeSet = BTreeSet::new(); - let bound = idx.imports.entry(rel.clone()).or_default(); + let mut bound: BTreeMap> = BTreeMap::new(); for imp in &cache.imports { let mut from: BTreeSet = BTreeSet::new(); for seg in module_segments(&imp.module) { @@ -832,13 +936,10 @@ fn build_indexes( } } - if !wildcard.is_empty() { - idx.wildcard_imports.entry(rel.clone()).or_default().extend(wildcard); - } - + let mut calls = Vec::new(); for c in &cache.calls { for s in &services { - idx.calls.push(OwnedCall { + calls.push(OwnedCall { service: s.clone(), file: rel.clone(), line: c.line, @@ -852,6 +953,7 @@ fn build_indexes( // a signature that mentions another service's type depends on it even // if it never calls into it + let mut type_refs = Vec::new(); if typed { for f in &cache.funcs { let mentioned = f @@ -862,7 +964,7 @@ fn build_indexes( .chain(f.owner.iter().cloned()); for t in mentioned.filter(|t| !t.is_empty()) { for s in &services { - idx.type_refs.push(TypeRef { + type_refs.push(TypeRef { service: s.clone(), file: rel.clone(), line: f.line, @@ -872,6 +974,15 @@ fn build_indexes( } } } + (rel, bound, wildcard, calls, type_refs) + }); + for (rel, bound, wildcard, calls, type_refs) in per_file { + if !wildcard.is_empty() { + idx.wildcard_imports.entry(rel.clone()).or_default().extend(wildcard); + } + idx.imports.entry(rel).or_default().extend(bound); + idx.calls.extend(calls); + idx.type_refs.extend(type_refs); } idx } @@ -1156,8 +1267,10 @@ fn detect_edges( } }; - for call in &idx.calls { - match resolve_call(call, idx) { + // every call resolved at once from the indexes alone - recorded below in call order, as before + let resolved = par::map(idx.calls.len(), |k| resolve_call(&idx.calls[k], idx)); + for (call, outcome) in idx.calls.iter().zip(resolved) { + match outcome { Ok((target, via)) => { symbol_callers .entry(call.name.clone()) diff --git a/src/coverage.rs b/src/coverage.rs index 3cc700a..d7361c0 100644 --- a/src/coverage.rs +++ b/src/coverage.rs @@ -4,6 +4,7 @@ use crate::changes::{is_test_fn_name, is_test_path, module_segments, names_proje use crate::contracts::ContractIndex; use crate::extract::TOP_LEVEL; use crate::model::FileCache; +use crate::par; use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; @@ -84,6 +85,9 @@ pub struct TestRef { // A definition: (index into `caches`, index into that file's `funcs`). type DefId = (usize, usize); +// what one test call reaches and the evidence that tied it - `None` when it leaves the project +type Reached = Option<(Vec, Evidence)>; + pub struct CoverageIndex { covering: BTreeMap>, // definitions reached only from a test file's top level. Not a selectable @@ -152,7 +156,6 @@ pub(crate) fn file_aliases(caches: &[FileCache]) -> Vec> { // `insights::build_graph` for the same reason as `file_aliases`. pub(crate) fn imported_names(caches: &[FileCache]) -> Vec>> { let stems: Vec<&str> = caches.iter().map(stem_of).collect(); - let mut imported: Vec>> = vec![BTreeMap::new(); caches.len()]; let mut stem_files: BTreeMap<&str, Vec> = BTreeMap::new(); for (i, s) in stems.iter().enumerate() { stem_files.entry(s).or_default().push(i); @@ -171,7 +174,9 @@ pub(crate) fn imported_names(caches: &[FileCache]) -> Vec>> = par::map(caches.len(), |a| { + let mut own: BTreeMap> = BTreeMap::new(); let files_named = |s: &str| { stem_files .get(s) @@ -181,7 +186,7 @@ pub(crate) fn imported_names(caches: &[FileCache]) -> Vec>() }; - for imp in &c.imports { + for imp in &caches[a].imports { let segs: Vec<&str> = module_segments(&imp.module).collect(); // The last segment names the module the statement actually reaches; // the ones before it are the packages on the way there. Preferring @@ -201,7 +206,7 @@ pub(crate) fn imported_names(caches: &[FileCache]) -> Vec Vec> = par::map(caches.len(), |a| { + imported[a] + .iter() + .filter(|(name, from)| from.iter().any(|&b| imported[b].contains_key(name.as_str()))) + .map(|(name, _)| name.clone()) + .collect() + }); + // the rounds keep their file order, so each still sees what earlier files gained within it for _ in 0..MAX_FACADE_HOPS { let mut grew = false; - for a in 0..caches.len() { - for name in imported[a].keys().cloned().collect::>() { - let hops: BTreeSet = imported[a][&name] + for (a, names) in growing.iter().enumerate() { + for name in names { + let hops: BTreeSet = imported[a][name] .iter() - .filter_map(|&b| imported[b].get(&name)) + .filter_map(|&b| imported[b].get(name)) .flatten() .copied() .filter(|&b| b != a) .collect(); - let reached = imported[a].entry(name).or_default(); + let Some(reached) = imported[a].get_mut(name) else { continue }; let before = reached.len(); reached.extend(hops); grew |= reached.len() != before; @@ -249,6 +264,18 @@ fn stem_of(c: &FileCache) -> &str { c.rel_path.file_stem().and_then(|s| s.to_str()).unwrap_or("") } +// what a qualifier can name and what each file imports from where - built once for the graph and coverage alike +pub(crate) struct Reach { + pub(crate) aliases: Vec>, + pub(crate) imported: Vec>>, +} + +impl Reach { + pub(crate) fn of(caches: &[FileCache]) -> Reach { + Reach { aliases: file_aliases(caches), imported: imported_names(caches) } + } +} + // Build the coverage relation. `project_ids` are the identities declared by // manifests (crate name, go module path, npm name), so an integration test // calling `mycrate::parse` is understood as staying inside the project. @@ -257,8 +284,21 @@ pub fn build( project_ids: &BTreeSet, contracts: &ContractIndex, ) -> CoverageIndex { - let aliases = file_aliases(caches); - let imported = imported_names(caches); + let reach = Reach::of(caches); + let index = build_over(caches, project_ids, contracts, &reach); + // the tables are large and the caller only wants the index + std::thread::spawn(move || drop(reach)); + index +} + +// `build`, through tables the caller already holds +pub(crate) fn build_over( + caches: &[FileCache], + project_ids: &BTreeSet, + contracts: &ContractIndex, + reach: &Reach, +) -> CoverageIndex { + let (aliases, imported) = (&reach.aliases, &reach.imported); let alias_any: BTreeSet<&str> = aliases.iter().flatten().map(String::as_str).collect(); let mut by_name: BTreeMap<&str, Vec> = BTreeMap::new(); @@ -297,50 +337,63 @@ pub fn build( .unwrap_or_default() }) .collect(); + let tables = Tables { + aliases, + alias_any: &alias_any, + imported, + by_name: &by_name, + by_file_name: &by_file_name, + by_owner: &by_owner, + project_types: &project_types, + project_ids, + families: &families, + dirs: &dirs, + }; + // every file's test calls resolved at once - folding them in below is order-free, sets and a count and a minimum + let resolved: Vec> = par::map(caches.len(), |a| { + let c = &caches[a]; + let file_is_test = is_test_path(&path_str(&c.rel_path)); + c.calls + .iter() + .enumerate() + .filter(|(_, call)| file_is_test || call.test_ctx || is_test_fn_name(&call.caller)) + .map(|(ci, call)| { + // An rpc is the one call that crosses runtime families on + // purpose: a python test through the stub exercises the go + // handler behind it. Settled before `resolve`, whose family + // rule would refuse it. + let rpc = contracts + .caller(a, ci) + .map(|link| (contracts.targets(link.contract), Evidence::Rpc)); + let matched = rpc.or_else(|| resolve( + Site { + file: a, + name: call.name.as_str(), + qualifier: call.qualifier.as_deref(), + recv_type: call.recv_type.as_deref(), + typed: c.language.is_typed(), + package_scoped: c.language.package_scoped(), + }, + &tables, + )); + (ci, matched) + }) + .collect() + }); + let mut hits: BTreeMap> = BTreeMap::new(); let mut touched: BTreeSet = BTreeSet::new(); let mut tests: BTreeSet<(usize, &str)> = BTreeSet::new(); let mut external_calls = 0usize; - for (a, c) in caches.iter().enumerate() { + for ((a, c), found) in caches.iter().enumerate().zip(resolved) { let path = path_str(&c.rel_path); - let file_is_test = is_test_path(&path); - for (ci, call) in c.calls.iter().enumerate() { - if !(file_is_test || call.test_ctx || is_test_fn_name(&call.caller)) { - continue; - } + for (ci, matched) in found { + let call = &c.calls[ci]; let selectable = call.caller != TOP_LEVEL; if selectable { tests.insert((a, call.caller.as_str())); } - // An rpc is the one call that crosses runtime families on purpose: - // a python test through the stub exercises the go handler behind - // it. Settled before `resolve`, whose family rule would refuse it. - let rpc = contracts - .caller(a, ci) - .map(|link| (contracts.targets(link.contract), Evidence::Rpc)); - let matched = rpc.or_else(|| resolve( - Site { - file: a, - name: call.name.as_str(), - qualifier: call.qualifier.as_deref(), - recv_type: call.recv_type.as_deref(), - typed: c.language.is_typed(), - package_scoped: c.language.package_scoped(), - }, - &Tables { - aliases: &aliases, - alias_any: &alias_any, - imported: &imported, - by_name: &by_name, - by_file_name: &by_file_name, - by_owner: &by_owner, - project_types: &project_types, - project_ids, - families: &families, - dirs: &dirs, - }, - )); let Some((defs, evidence)) = matched else { external_calls += 1; continue; diff --git a/src/deps.rs b/src/deps.rs index 4a12ed4..5c12d50 100644 --- a/src/deps.rs +++ b/src/deps.rs @@ -798,8 +798,7 @@ pub fn markdown(r: &DepsReport, base: &str) -> String { } if !r.changed { out.push_str( - "\nno manifest or lockfile changed on this branch, so nothing entered, left or moved \ - version. What the project depends on right now is `vulnerabilities`, not this.\n", + "\nno manifest or lockfile changed on this branch.\n", ); return out; } diff --git a/src/extract.rs b/src/extract.rs index 3cf29ac..c7948a4 100644 --- a/src/extract.rs +++ b/src/extract.rs @@ -4,8 +4,10 @@ use crate::languages::Language; use crate::model::{ Annotation, Boundary, CallSite, Const, Func, FuncMetrics, Import, LoopInfo, Note, Ref, ResourceOp, TypeDef, }; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::OnceLock; use tree_sitter::{Node, Parser}; // `CallSite::caller` for a call that sits outside any function @@ -24,6 +26,8 @@ pub fn skips_ignored() -> bool { IGNORE_SKIP.load(Ordering::Relaxed) } +// everything one parse yields - nothing in it depends on the path, so `parse_cache` keys it on content +#[derive(Serialize, Deserialize)] pub struct Extracted { pub consts: Vec, pub funcs: Vec, @@ -69,8 +73,110 @@ enum Scope { Type { name: String, recv: Option }, } +// the roles a node kind plays in the walk, one bit each +const FUNC: u16 = 1; +const CONST: u16 = 1 << 1; +const CALL: u16 = 1 << 2; +const USE: u16 = 1 << 3; +const VARIANT: u16 = 1 << 4; +const IMPORT: u16 = 1 << 5; +const COMMENT: u16 = 1 << 6; +const MODULE: u16 = 1 << 7; +const LOOP: u16 = 1 << 8; +const BRANCH: u16 = 1 << 9; +const GUARD: u16 = 1 << 10; +const DECLARATION: u16 = 1 << 11; + +// local declarations `bind_declaration` reads a type off, in any grammar +const DECLARATION_KINDS: &[&str] = &[ + "let_declaration", // rust + "short_var_declaration", // go `x := ...` + "var_spec", // go `var x T` + "const_spec", + "variable_declarator", // js/ts + "declaration", // c++ + "field_declaration", +]; + +// each node kind's roles by numeric kind id - built once per grammar from the `languages` name lists +struct Kinds { + roles: Vec, + // the label `type_kinds` gives a type definition's kind + type_def: Vec>, + // the field `construct_kinds` names a construction's type by + construct: Vec>, +} + +impl Kinds { + fn of(lang: Language) -> &'static Kinds { + static TABLES: OnceLock> = OnceLock::new(); + let tables = TABLES.get_or_init(|| Language::ALL.iter().map(|&l| Kinds::build(l)).collect()); + &tables[Language::ALL.iter().position(|&l| l == lang).unwrap_or(0)] + } + + fn build(lang: Language) -> Kinds { + let ts = lang.ts_language(); + let n = ts.node_kind_count(); + let lists: [(u16, &[&str]); 12] = [ + (FUNC, lang.func_kinds()), + (CONST, lang.const_kinds()), + (CALL, lang.call_kinds()), + (USE, lang.use_kinds()), + (VARIANT, lang.variant_kinds()), + (IMPORT, lang.import_kinds()), + (COMMENT, lang.comment_kinds()), + (MODULE, lang.module_kinds()), + (LOOP, lang.loop_kinds()), + (BRANCH, lang.branch_kinds()), + (GUARD, lang.guard_kinds()), + (DECLARATION, DECLARATION_KINDS), + ]; + let mut kinds = Kinds { + roles: vec![0; n], + type_def: vec![None; n], + construct: vec![None; n], + }; + for id in 0..n { + let Some(name) = ts.node_kind_for_id(id as u16) else { continue }; + for (role, names) in lists { + if names.contains(&name) { + kinds.roles[id] |= role; + } + } + kinds.type_def[id] = lang.type_kinds().iter().find(|(k, _)| *k == name).map(|(_, v)| *v); + kinds.construct[id] = lang.construct_kinds().iter().find(|(k, _)| *k == name).map(|(_, f)| *f); + } + kinds + } + + // an error node's id lies past the grammar's kinds - no role + fn role(&self, id: u16) -> u16 { + self.roles.get(usize::from(id)).copied().unwrap_or(0) + } + + fn type_def(&self, id: u16) -> Option<&'static str> { + self.type_def.get(usize::from(id)).copied().flatten() + } + + fn construct(&self, id: u16) -> Option<&'static str> { + self.construct.get(usize::from(id)).copied().flatten() + } +} + +// one function's measurements, taken as the walk passes through its body +struct Measure { + // where the definition sits in `Ctx::funcs` + func: usize, + name: String, + // loop and guard depth at the definition - depths count from the function, not the file + loops_above: usize, + guards_above: usize, + metrics: FuncMetrics, +} + struct Ctx<'a> { lang: Language, + kinds: &'static Kinds, src: &'a str, consts: Vec, funcs: Vec, @@ -106,6 +212,11 @@ struct Ctx<'a> { // names of enclosing Python Enum-subclass definitions; members are // const-like definitions typed by the innermost one enum_types: Vec, + // a measurement per enclosing function, innermost last - each node counts toward all of them + measuring: Vec, + // loops and resource guards the walk is inside, counted from the root + loop_depth: usize, + guard_depth: usize, } impl Ctx<'_> { @@ -169,6 +280,7 @@ pub(crate) fn extract_with(lang: Language, src: &str, honour_skips: bool) -> Opt let mut ctx = Ctx { lang, + kinds: Kinds::of(lang), src, consts: Vec::new(), funcs: Vec::new(), @@ -190,6 +302,9 @@ pub(crate) fn extract_with(lang: Language, src: &str, honour_skips: bool) -> Opt test_mod_depth: 0, import_depth: 0, enum_types: Vec::new(), + measuring: Vec::new(), + loop_depth: 0, + guard_depth: 0, }; visit(tree.root_node(), &mut ctx); @@ -273,8 +388,11 @@ pub(crate) fn extract_with(lang: Language, src: &str, honour_skips: bool) -> Opt fn visit(node: Node, ctx: &mut Ctx) { let kind = node.kind(); + let id = node.kind_id(); + let role = ctx.kinds.role(id); let lang = ctx.lang; let mut pushed = 0usize; + let measured = ctx.measuring.len(); // rust unit tests conventionally live in an inline `mod tests`; track it so // call sites inside are flagged as test context for `changes` @@ -287,11 +405,11 @@ fn visit(node: Node, ctx: &mut Ctx) { // BDD suites name their tests with a string, not an identifier // (`test("charge", () => { ... })`) - the callback is the test body, so it // is a test scope whose "name" is that label. - let bdd_label = bdd_test_label(node, ctx); + let bdd_label = bdd_test_label(node, kind, ctx); if test_mod || bdd_label.is_some() { ctx.test_mod_depth += 1; } - let import = lang.import_kinds().contains(&kind); + let import = role & IMPORT != 0; if import { if ctx.import_depth == 0 { extract_import(node, ctx); @@ -322,7 +440,7 @@ fn visit(node: Node, ctx: &mut Ctx) { // dispatch below, because several of their node kinds (a Rust `trait_item`, // a C++ `class_specifier`, a TS `class_declaration`) are also type scopes // and would otherwise be swallowed by that branch - if let Some(kind) = lang.type_kinds().iter().find(|(k, _)| *k == kind).map(|(_, v)| *v) { + if let Some(kind) = ctx.kinds.type_def(id) { if let Some(name) = type_def_name(node, ctx) { ctx.types.push(TypeDef { line: pos(node).0, @@ -333,7 +451,7 @@ fn visit(node: Node, ctx: &mut Ctx) { } // A test scope is a module the language recognises but not one the project // is built out of - if lang.module_kinds().contains(&kind) && ctx.test_mod_depth == 0 { + if role & MODULE != 0 && ctx.test_mod_depth == 0 { // a go `package_clause` has no `name` field; its identifier child is it let name = node .child_by_field_name("name") @@ -349,13 +467,15 @@ fn visit(node: Node, ctx: &mut Ctx) { } } } - bind_declaration(node, ctx); + if role & DECLARATION != 0 { + bind_declaration(node, ctx); + } - if let Some((name, recv)) = type_scope(node, ctx) { + if let Some((name, recv)) = type_scope(node, kind, ctx) { ctx.scope_stack.push(Scope::Type { name, recv }); pushed += 1; - } else if lang.func_kinds().contains(&kind) { - if let Some(func) = extract_func(node, ctx) { + } else if role & FUNC != 0 { + if let Some(mut func) = extract_func(node, ctx) { let owner = func_owner(node, ctx); let entry = (func.line, func.col, func.ret.clone()); match &owner { @@ -369,6 +489,14 @@ fn visit(node: Node, ctx: &mut Ctx) { } } let name = func.name.clone(); + // the body is measured on the walk's way through it, in `measure` + ctx.measuring.push(Measure { + func: ctx.funcs.len(), + name: name.clone(), + loops_above: ctx.loop_depth, + guards_above: ctx.guard_depth, + metrics: std::mem::take(&mut func.metrics), + }); ctx.funcs.push(func); // go has to be special, methods carry their receiver on the definition rather than // nesting inside a type node ugh... @@ -392,7 +520,7 @@ fn visit(node: Node, ctx: &mut Ctx) { ctx.scope_stack.push(Scope::Func(label.clone())); pushed += 1; } - } else if lang.const_kinds().contains(&kind) { + } else if role & CONST != 0 { // python Enum members live in a class body, which const_eligible // rejects - admit them when directly inside an Enum subclass let py_enum_member = @@ -400,7 +528,7 @@ fn visit(node: Node, ctx: &mut Ctx) { if const_eligible(ctx) || py_enum_member { extract_consts(node, ctx); } - } else if lang.variant_kinds().contains(&kind) { + } else if role & VARIANT != 0 { if !ctx.in_function() { extract_variant(node, ctx); } @@ -412,7 +540,7 @@ fn visit(node: Node, ctx: &mut Ctx) { if !ctx.in_function() { extract_odin_enum(node, ctx); } - } else if lang.call_kinds().contains(&kind) { + } else if role & CALL != 0 { if let Some(call) = classify_call(node, ctx) { ctx.calls.push(call); } @@ -421,11 +549,11 @@ fn visit(node: Node, ctx: &mut Ctx) { if let Some(site) = loose_call(node, ctx) { ctx.loose_calls.push(site); } - } else if let Some(&(_, field)) = lang.construct_kinds().iter().find(|(k, _)| *k == kind) { + } else if let Some(field) = ctx.kinds.construct(id) { if let Some(site) = construct_site(node, field, ctx) { ctx.constructs.push(site); } - } else if lang.comment_kinds().contains(&kind) { + } else if role & COMMENT != 0 { maybe_note(node, ctx); maybe_annotations(node, ctx); maybe_skips(node, ctx); @@ -433,15 +561,22 @@ fn visit(node: Node, ctx: &mut Ctx) { // macro bodies (`assert_eq!(charge(1), 31)`) are token trees, not // expressions - approximate the calls inside so `changes` sees them scan_macro_tokens(node, ctx); - } else if lang.use_kinds().contains(&kind) { + } else if role & USE != 0 { maybe_use(node, ctx); } + // a guard covers the node itself - a loop only what it repeats + let guard = role & GUARD != 0; + let looped = role & LOOP != 0; + ctx.guard_depth += usize::from(guard); + measure(node, kind, role, ctx); + ctx.loop_depth += usize::from(looped); + // one type-environment frame per scope, so a `let x: T` in an inner block // cannot leak its binding out to the enclosing function if pushed > 0 { ctx.type_env.push(HashMap::new()); - if lang.func_kinds().contains(&kind) { + if role & FUNC != 0 { bind_signature(node, ctx); } } @@ -453,6 +588,16 @@ fn visit(node: Node, ctx: &mut Ctx) { if pushed > 0 { ctx.type_env.pop(); } + ctx.loop_depth -= usize::from(looped); + ctx.guard_depth -= usize::from(guard); + // the whole body has been walked - its measurements are complete + if ctx.measuring.len() > measured { + if let Some(mut m) = ctx.measuring.pop() { + m.metrics.loops.sort_by_key(|l| l.line); + m.metrics.resources.sort_by_key(|r| r.line); + ctx.funcs[m.func].metrics = m.metrics; + } + } for _ in 0..pushed { ctx.scope_stack.pop(); } @@ -467,12 +612,12 @@ fn visit(node: Node, ctx: &mut Ctx) { } } -fn type_scope(node: Node, ctx: &Ctx) -> Option<(String, Option)> { +fn type_scope(node: Node, kind: &str, ctx: &Ctx) -> Option<(String, Option)> { let name_of = |field: &str| { node.child_by_field_name(field) .map(|n| oneline(text(n, ctx.src))) }; - match (ctx.lang, node.kind()) { + match (ctx.lang, kind) { (Language::Rust, "impl_item") => Some((name_of("type")?, Some("self".to_string()))), (Language::Rust, "trait_item") => Some((name_of("name")?, Some("self".to_string()))), (Language::Python, "class_definition") => { @@ -616,7 +761,12 @@ fn extract_func(node: Node, ctx: &Ctx) -> Option { let (line, col) = pos(name_node); let ret = func_return(node, ctx); let comment = preceding_comment(node, ctx); - let metrics = func_metrics(node, &name, ctx); + // the body's outline - what is inside it `measure` counts on the walk's way through + let metrics = FuncMetrics { + body_lines: node.end_position().row - node.start_position().row + 1, + params: count_params(node, ctx), + ..FuncMetrics::default() + }; let owner = func_owner(node, ctx).map(|(t, _)| normalize_type(&t)); let param_types = param_pairs(node, ctx) .into_iter() @@ -782,20 +932,6 @@ fn bind_signature(node: Node, ctx: &mut Ctx) { // types are recorded - an inferred `let c = make_client()` stays unknown // rather than being guessed at. fn bind_declaration(node: Node, ctx: &mut Ctx) { - let kind = node.kind(); - let interesting = matches!( - kind, - "let_declaration" // rust - | "short_var_declaration" // go `x := ...` - | "var_spec" // go `var x T` - | "const_spec" - | "variable_declarator" // js/ts - | "declaration" // c++ - | "field_declaration" - ); - if !interesting { - return; - } let src = ctx.src; let name_of = |n: Node| oneline(text(n, src)); @@ -908,22 +1044,6 @@ fn constructed_type(value: &str) -> Option { None } -// Measure a function body: size, decision points, loop nesting, and -// acquire/release calls. One extra walk of the definition's subtree, so a -// closure nested in a function is counted into that function too - which is -// what "how big is this body" should mean. -fn func_metrics(node: Node, own_name: &str, ctx: &Ctx) -> FuncMetrics { - let mut m = FuncMetrics { - body_lines: node.end_position().row - node.start_position().row + 1, - params: count_params(node, ctx), - ..FuncMetrics::default() - }; - walk_metrics(node, ctx, own_name, 0, 0, &mut m); - m.loops.sort_by_key(|l| l.line); - m.resources.sort_by_key(|r| r.line); - m -} - fn count_params(node: Node, ctx: &Ctx) -> usize { if ctx.lang == Language::Proto { return usize::from(proto_rpc_types(node, ctx).0.is_some()); @@ -957,46 +1077,39 @@ fn param_list<'a>(node: Node<'a>, ctx: &Ctx) -> Option> { None } -fn walk_metrics( - node: Node, - ctx: &Ctx, - own_name: &str, - loop_depth: usize, - guard_depth: usize, - m: &mut FuncMetrics, -) { - let lang = ctx.lang; - let kind = node.kind(); - m.nodes += 1; - - let is_loop = lang.loop_kinds().contains(&kind); - let depth = if is_loop { - let d = loop_depth + 1; - m.loops.push(LoopInfo { - line: pos(node).0, - kind: loop_label(kind), - depth: d, - trip: literal_trip(node, ctx), - }); - d - } else { - loop_depth - }; - let guard = guard_depth + usize::from(lang.guard_kinds().contains(&kind)); - - if lang.branch_kinds().contains(&kind) { - m.branches += 1; - } - if let Some(op) = resource_op(node, ctx, guard > 0) { - m.resources.push(op); - } - if lang.call_kinds().contains(&kind) && callee_name(node, ctx).as_deref() == Some(own_name) { - m.recursive = true; +// count one node toward every enclosing function, nested closures included - depths relative to each +fn measure(node: Node, kind: &str, role: u16, ctx: &mut Ctx) { + if ctx.measuring.is_empty() { + return; } - - let mut cursor = node.walk(); - for child in node.children(&mut cursor) { - walk_metrics(child, ctx, own_name, depth, guard, m); + // what the node is, worked out once however many functions it counts toward + let looped = (role & LOOP != 0).then(|| (pos(node).0, loop_label(kind), literal_trip(node, ctx))); + let callee = if role & CALL != 0 { callee_name(node, ctx) } else { None }; + let op = resource_op(node, kind, callee.as_deref(), ctx); + let branch = role & BRANCH != 0; + let (loops, guards) = (ctx.loop_depth, ctx.guard_depth); + for m in &mut ctx.measuring { + m.metrics.nodes += 1; + if let Some((line, label, trip)) = &looped { + m.metrics.loops.push(LoopInfo { + line: *line, + kind: label.clone(), + depth: loops - m.loops_above + 1, + trip: *trip, + }); + } + if branch { + m.metrics.branches += 1; + } + if let Some(op) = &op { + m.metrics.resources.push(ResourceOp { + guarded: guards > m.guards_above, + ..op.clone() + }); + } + if callee.as_deref() == Some(m.name.as_str()) { + m.metrics.recursive = true; + } } } @@ -1020,14 +1133,13 @@ pub(crate) fn loop_label(kind: &str) -> String { } } -// A resource acquire/release call, matched on the callee's rightmost name. -// C++ `new`/`delete` are operators rather than calls, so they are matched on -// their own node kinds. -fn resource_op(node: Node, ctx: &Ctx, guarded: bool) -> Option { +// a resource acquire/release by the callee's rightmost name, or c++ `new`/`delete` by kind - `measure` sets `guarded` +fn resource_op(node: Node, kind: &str, callee: Option<&str>, ctx: &Ctx) -> Option { let pairs = ctx.lang.resource_pairs(); - let line = pos(node).0; + let guarded = false; if ctx.lang == Language::Cpp { - match node.kind() { + let line = pos(node).0; + match kind { "new_expression" => { return Some(ResourceOp { line, @@ -1049,16 +1161,11 @@ fn resource_op(node: Node, ctx: &Ctx, guarded: bool) -> Option { _ => {} } } - if !ctx.lang.call_kinds().contains(&node.kind()) { - return None; - } - let name = callee_name(node, ctx)?; + let name = callee?; for (acq, rel) in pairs { - if name == *acq { - return Some(ResourceOp { line, name, pair: acq, acquire: true, guarded }); - } - if name == *rel { - return Some(ResourceOp { line, name, pair: acq, acquire: false, guarded }); + if name == *acq || name == *rel { + let (line, acquire) = (pos(node).0, name == *acq); + return Some(ResourceOp { line, name: name.to_string(), pair: acq, acquire, guarded }); } } None @@ -1161,11 +1268,11 @@ pub(crate) const BDD_REGISTRARS: &[&str] = &[ // `test("charge", () => { ... })` -> `test("charge")`. These callbacks are the // only "test functions" JS/TS suites have - without this they are anonymous and // every call inside them is attributed to ``. -fn bdd_test_label(node: Node, ctx: &Ctx) -> Option { +fn bdd_test_label(node: Node, kind: &str, ctx: &Ctx) -> Option { if !matches!( ctx.lang, Language::JavaScript | Language::TypeScript | Language::Tsx - ) || !matches!(node.kind(), "arrow_function" | "function_expression") + ) || !matches!(kind, "arrow_function" | "function_expression") { return None; } @@ -1773,13 +1880,14 @@ fn maybe_use(node: Node, ctx: &mut Ctx) { return; } if let Some(parent) = node.parent() { + let role = ctx.kinds.role(parent.kind_id()); // the callee of a call is already recorded as a call site - let is_callee = ctx.lang.call_kinds().contains(&parent.kind()) + let is_callee = role & CALL != 0 && parent .child_by_field_name("function") .is_some_and(|f| f.id() == node.id()); // nested paths (`a::b::C`) record only at the outermost node - if is_callee || ctx.lang.use_kinds().contains(&parent.kind()) { + if is_callee || role & USE != 0 { return; } } diff --git a/src/html.rs b/src/html.rs index 71845e2..2206a31 100644 --- a/src/html.rs +++ b/src/html.rs @@ -277,804 +277,6 @@ document.body.addEventListener('htmx:sendError', e => { "####; -// Render the `/insights` page; two modes from one template. -pub fn render_insights_html(root_label: &str, embed: Option<&Value>) -> String { - let data = match embed { - // `<\/` is a legal JSON escape, and defuses any ` serde_json::to_string(v) - .unwrap_or_else(|_| "null".into()) - .replace(" "null".to_string(), - }; - INSIGHTS_TEMPLATE - .replace("__CCC_ROOT__", &esc(root_label)) - .replace("__CCC_EMBED__", &data) -} - -// Write the self-contained page for static hosting, creating the output -// directory if a build step has not made it yet (`public/index.html`). -pub fn write_insights_html(path: &Path, root_label: &str, report: &Value) -> Result<()> { - if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) { - std::fs::create_dir_all(dir) - .with_context(|| format!("creating {}", dir.display()))?; - } - std::fs::write(path, render_insights_html(root_label, Some(report))) - .with_context(|| format!("writing {}", path.display())) -} - -const INSIGHTS_TEMPLATE: &str = r####" - - - - -ccc insights - __CCC_ROOT__ - - - -
- -
-
-

__CCC_ROOT__

- - - -
-

- Everything below is read from the syntax tree - the call graph, per-function - measurements and your service globs. There is no type inference, no data flow - and no runtime profile behind it, so treat the findings as - advisory, not proofs. -

-
- -
- - -
- -
- - served by ccc run · data at /insights.json -
-
- - - - - -"####; - #[cfg(test)] mod tests { use super::*; @@ -1102,38 +304,10 @@ mod tests { assert!(html.contains("/fragment/find")); } - // A statically exported page has to stand on its own: the analysis is - // inlined, and the controls that need a server are gone. - #[test] - fn static_insights_page_embeds_its_data_and_drops_the_server_controls() { - let report = json!({ - "schema": "ccc-insights/v1", - "totals": {"files": 2, "lines": 40}, - // a string that would end the script block if it were not escaped - "root": "", - }); - let html = render_insights_html("demo", Some(&report)); - - // the payload round-trips out of the inline block - let start = html.find(r#"").unwrap(); - let parsed: Value = serde_json::from_str(&rest[open..close]).unwrap(); - assert_eq!(parsed, report); - // and the breakout attempt never appears raw inside it - assert!(!rest[open..close].contains("")); - - // served live, the block is empty and the page fetches instead - let live = render_insights_html("demo", None); - assert!(live.contains(r#""#)); - assert!(live.contains("/insights.json")); - } - - // Both pages render service names + // The page renders service names #[test] fn every_template_labels_the_implicit_root_service() { - for (name, tpl) in [("changes", TEMPLATE), ("insights", INSIGHTS_TEMPLATE)] { + for (name, tpl) in [("changes", TEMPLATE)] { assert!( tpl.contains("const svcLabel"), "{name} template renders service names without svcLabel" diff --git a/src/insights.rs b/src/insights.rs index c0d9412..a965b52 100644 --- a/src/insights.rs +++ b/src/insights.rs @@ -13,6 +13,7 @@ use crate::extract::TOP_LEVEL; use crate::languages::Language; use crate::model::{FileCache, Func, FuncMetrics}; use crate::changes::{self, ChangesConfig}; +use crate::par; use serde_json::{json, Value}; use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::path::Path; @@ -103,6 +104,15 @@ impl<'a> Graph<'a> { // file granularity. A call with no evidence, or with evidence for more than // one target, produces no edge: an absent edge is better than a wrong one. pub(crate) fn build_graph<'a>(caches: &'a [FileCache], contracts: &ContractIndex) -> Graph<'a> { + build_graph_over(caches, contracts, &coverage::Reach::of(caches)) +} + +// `build_graph` through the qualifier and import tables coverage resolves through too +pub(crate) fn build_graph_over<'a>( + caches: &'a [FileCache], + contracts: &ContractIndex, + reach: &coverage::Reach, +) -> Graph<'a> { let mut nodes = Vec::new(); // (file, name) -> every node with that name, in definition order. A name // is not unique within a file: overloads share one, and so does an @@ -160,7 +170,7 @@ pub(crate) fn build_graph<'a>(caches: &'a [FileCache], contracts: &ContractIndex // What a qualifier can name, and what each file imports from where. Built // in `coverage` and shared with it, so a qualifier means the same thing to // the call graph and to the coverage relation derived beside it. - let aliases = coverage::file_aliases(caches); + let (aliases, imported) = (&reach.aliases, &reach.imported); // (owning type, method) -> nodes, for receiver-typed calls let mut by_owner: BTreeMap<(&str, &str), Vec> = BTreeMap::new(); for (pos, NodeId(fi, ki)) in g.nodes.iter().copied().enumerate() { @@ -170,7 +180,6 @@ pub(crate) fn build_graph<'a>(caches: &'a [FileCache], contracts: &ContractIndex by_owner.entry((owner, f.name.as_str())).or_default().push(pos); } } - let imported = coverage::imported_names(caches); // Which definition of `name` in file `fi` a call on `line` belongs to. // With one candidate this is the old behaviour; with several - an overload @@ -210,9 +219,10 @@ pub(crate) fn build_graph<'a>(caches: &'a [FileCache], contracts: &ContractIndex } } - // cross-file edges, with evidence - for (a, c) in caches.iter().enumerate() { - for call in &c.calls { + // cross-file edges, with evidence - resolved per file at once, then added in file and call order so counts hold + let cross: Vec> = par::map(caches.len(), |a| { + let mut edges = Vec::new(); + for call in &caches[a].calls { let Some(from) = owner_of(a, call.caller.as_str(), call.line) else { continue; }; @@ -235,15 +245,14 @@ pub(crate) fn build_graph<'a>(caches: &'a [FileCache], contracts: &ContractIndex // local's type to know which method it means. if let Some(ty) = call.recv_type.as_deref() { if let Some([to]) = by_owner.get(&(ty, call.name.as_str())).map(|v| &v[..]) { - let to = *to; - if to != from && g.out[from].insert(to) { - g.into[to].insert(from); - g.call_sites[to] += 1; + if *to != from { + edges.push((from, *to, true)); } continue; } } let Some(candidates) = candidates else { continue }; + let imports_name = imported[a].get(call.name.as_str()); let evidenced: Vec = candidates .iter() .copied() @@ -253,18 +262,22 @@ pub(crate) fn build_graph<'a>(caches: &'a [FileCache], contracts: &ContractIndex return false; } let by_qualifier = qual.iter().any(|q| aliases[b].contains(*q)); - let by_import = imported[a] - .get(call.name.as_str()) - .is_some_and(|s| s.contains(&b)); + let by_import = imports_name.is_some_and(|s| s.contains(&b)); by_qualifier || by_import }) .collect(); if let [to] = evidenced[..] { - g.out[from].insert(to); - g.into[to].insert(from); - g.call_sites[to] += 1; + edges.push((from, to, false)); } } + edges + }); + for (from, to, typed) in cross.into_iter().flatten() { + // a typed call counts once per caller, an evidenced one at every site + if g.out[from].insert(to) || !typed { + g.into[to].insert(from); + g.call_sites[to] += 1; + } } // rpc edges, the only ones allowed to cross languages: a call through a @@ -392,17 +405,20 @@ fn flame_node( // chain that many callers funnel into is where a change ripples furthest. It is // not an execution frequency - nothing is measured at runtime. fn deepest_chains(g: &Graph, roots: &[usize]) -> Vec { - let mut chains: Vec<(usize, usize, Vec)> = Vec::new(); - for &r in roots { + // each root's walk has its own budget and reads the graph alone - all walked at once, kept in root order + let mut chains: Vec<(usize, usize, Vec)> = par::map(roots.len(), |k| { let mut best = Vec::new(); let mut path = Vec::new(); let mut on_path = BTreeSet::new(); - longest(g, r, &mut path, &mut on_path, &mut best, &mut 20_000); - if best.len() > 1 { + longest(g, roots[k], &mut path, &mut on_path, &mut best, &mut 20_000); + (best.len() > 1).then(|| { let weight = best.iter().map(|&i| g.call_sites[i]).sum(); - chains.push((best.len(), weight, best)); - } - } + (best.len(), weight, best) + }) + }) + .into_iter() + .flatten() + .collect(); chains.sort_by(|a, b| (b.0, b.1).cmp(&(a.0, a.1))); chains.truncate(TOP_N); chains @@ -1092,16 +1108,13 @@ fn test_targets( sites[sites.len() * 9 / 10].max(3) }; - let mut targets: Vec<(usize, bool, Value)> = Vec::new(); - // `i` indexes several parallel structures and the graph accessors, not just - // one slice, so a range loop reads better than zipping them - #[allow(clippy::needless_range_loop)] - for i in 0..n { + // each row reads the graph and coverage alone - all written at once, kept in map order for the ranking + let mut targets: Vec<(usize, bool, Value)> = par::map(n, |i| { if g.is_test(i) { - continue; // do not recommend tests for tests + return None; // do not recommend tests for tests } if g.is_module(i) { - continue; // there is no module scope to call from a test + return None; // there is no module scope to call from a test } let f = g.func(i); let m = &f.metrics; @@ -1140,7 +1153,7 @@ fn test_targets( // a trivial leaf nobody calls is noise, not a gap - unless the branch // just changed it, in which case it is exactly the gap being looked for if !pinned && !covered && cx <= 1 && call_outs == 0 && callers == 0 && m.body_lines <= 3 { - continue; + return None; } let mut why: Vec = Vec::new(); @@ -1267,7 +1280,7 @@ fn test_targets( if m.recursive { score += 6; } - targets.push(( + Some(( score, pinned, json!({ @@ -1299,8 +1312,11 @@ fn test_targets( "recursive": m.recursive, }, }), - )); - } + )) + }) + .into_iter() + .flatten() + .collect(); targets.sort_by(|a, b| b.0.cmp(&a.0)); let total = targets.len(); @@ -1356,15 +1372,20 @@ fn target_id(file: &str, function: &str) -> String { // is ranked by complexity so that if the cap does bite, what it keeps is the // end of the list anyone came here for. fn complexity_table(g: &Graph, ctx: &ServiceCtx) -> Value { - let mut rows: Vec<(usize, Value)> = Vec::new(); - for i in 0..g.nodes.len() { - if g.is_module(i) { - continue; // a file's top level is a frame, not a function anyone wrote - } - let f = g.func(i); - let cx = f.metrics.complexity(); - rows.push(( - cx, + // ranked first with a stable sort, so equals keep map order - only the rows the cap keeps are built + let mut ranked: Vec<(usize, usize)> = (0..g.nodes.len()) + // a file's top level is a frame, not a function anyone wrote + .filter(|&i| !g.is_module(i)) + .map(|i| (g.func(i).metrics.complexity(), i)) + .collect(); + ranked.sort_by(|a, b| b.0.cmp(&a.0)); + let total = ranked.len(); + let truncated = total > MAX_COMPLEXITY_ROWS; + let rows: Vec = ranked + .into_iter() + .take(MAX_COMPLEXITY_ROWS) + .map(|(cx, i)| { + let f = g.func(i); json!({ "id": target_id(&g.file(i), &f.name), "function": f.name, @@ -1384,14 +1405,11 @@ fn complexity_table(g: &Graph, ctx: &ServiceCtx) -> Value { // a test's own complexity is real, but it is not code anyone // ships - the panel defaults to hiding these "test": g.is_test(i), - }), - )); - } - rows.sort_by(|a, b| b.0.cmp(&a.0)); - let total = rows.len(); - let truncated = total > MAX_COMPLEXITY_ROWS; + }) + }) + .collect(); json!({ - "functions": rows.into_iter().take(MAX_COMPLEXITY_ROWS).map(|(_, v)| v).collect::>(), + "functions": rows, "total": total, "truncated": truncated, "note": "`complexity` is cyclomatic-style: one path, plus one per decision point and \ @@ -1416,13 +1434,9 @@ fn arity(params: usize) -> &'static str { // The working-tree view: an engineer's uncommitted edit has to count, and CI // re-runs this against the committed tree anyway. // ccc:skip -fn change_set( - g: &Graph, - root: &Path, - root_label: &str, - base: Option<&str>, -) -> Result { - let opts = changes::ChangesOptions { +// what the change set is asked for: the working tree against `base`, with the request behind each change +fn change_opts(base: Option<&str>) -> changes::ChangesOptions { + changes::ChangesOptions { // `ccc insights --base` passes one, so a page built on the default // branch can diff against a tag instead of against itself base: base.map(str::to_string), @@ -1430,8 +1444,7 @@ fn change_set( worktree: true, prompts: true, deps: false, - }; - changes::changes_with_caches(root, root_label, &opts, g.caches).map_err(|e| format!("{e:#}")) + } } // How a runner selects a single test, per language. Emitted so a CI job can @@ -1726,7 +1739,10 @@ fn test_triggers(report: &changes::ChangesReport, trig: &Triggered, targets: &Va pub fn analyse(root: &Path, root_label: &str, base: Option<&str>) -> anyhow::Result { let files = crate::scan::collect_files(root)?; let caches = crate::scan::build_caches(root, &files); - Ok(insights(&caches, root, root_label, &crate::render::now_ts(), base)) + let report = insights(&caches, root, root_label, &crate::render::now_ts(), base); + // a large map takes a while to free, and the caller only wants the report + std::thread::spawn(move || drop(caches)); + Ok(report) } // Build the whole insights payload for a parsed map. @@ -1740,224 +1756,272 @@ pub fn insights( ) -> Value { let started = Instant::now(); let contracts = ContractIndex::for_root(root, caches); - let g = build_graph(caches, &contracts); - // The coverage relation, built from the same evidence rules as the graph - // and shared by every section that reports what a test reaches. let project_ids: BTreeSet = changes::manifest_identities(root) .into_iter() .map(|(id, _)| id) .collect(); - let cov = coverage::build(caches, &project_ids, &contracts); + let change_opts = change_opts(base); + // The graph, the coverage relation and the git half of the change set read + // nothing of each other's, so they are built side by side. The coverage + // relation follows the same evidence rules as the graph and is shared by + // every section that reports what a test reaches. + let (g, cov, prepared) = std::thread::scope(|s| { + let prepared = s.spawn(|| changes::prepare(root, &change_opts)); + // what qualifiers name and what each file imports, built once for the graph and coverage alike + let reach = coverage::Reach::of(caches); + let (g, cov) = std::thread::scope(|inner| { + let cov = inner.spawn(|| coverage::build_over(caches, &project_ids, &contracts, &reach)); + (build_graph_over(caches, &contracts, &reach), joined(cov)) + }); + // the tables are large, and nothing reads them from here on + std::thread::spawn(move || drop(reach)); + (g, cov, joined(prepared)) + }); let ctx = service_ctx(&g, root, &contracts); let n = g.nodes.len(); - - // The change set is computed once, here, and every consumer refers to this - // one copy: the `changes` section below, the trigger list, and the ranked - // targets that back it. It used to be recomputed inside `test_triggers` - // and restated in its output, which made the same facts arrive twice. - let change_set = change_set(&g, root, root_label, base); - // The trigger walk runs first: its gap list is exactly the set of target - // rows that must survive truncation, since `add` cites them by id alone. - let trig = change_set.as_ref().ok().map(|r| triggered(&g, &cov, r)); - let must_keep: BTreeSet<(String, String)> = trig - .as_ref() - .map(|t| { - t.gaps - .iter() - .map(|(file, function, _)| (file.clone(), function.clone())) - .collect() - }) - .unwrap_or_default(); - let targets = test_targets(&g, &cov, &ctx, &must_keep); - let (changes_section, triggers) = match (&change_set, &trig) { - (Ok(r), Some(t)) => ( - serde_json::to_value(r).unwrap_or(Value::Null), - test_triggers(r, t, &targets), - ), - // no git, no base ref, shallow clone: say why rather than render an - // empty tab that reads as "nothing to run" - _ => { - let reason = change_set.as_ref().err().cloned().unwrap_or_default(); - let why = json!({ - "available": false, - "reason": reason, - "hint": "The change set diffs the branch against its base. In CI, fetch history \ - (actions/checkout with fetch-depth: 0); locally, make sure the branch \ - has an upstream such as origin/main.", - }); - (why.clone(), why) - } - }; + // each file's path as text, written once rather than per sort comparison + let paths: Vec = caches.iter().map(|c| changes::path_str(&c.rel_path)).collect(); + let path = |i: usize| paths[g.node_file(i)].as_str(); // roots: nothing *else* calls them - a self-recursive function with no // other caller is still an entry point. Test functions are roots too, but // they drown out the application tree, so they are ranked last :) let mut roots: Vec = (0..n).filter(|&i| g.is_root(i)).collect(); - roots.sort_by_key(|&i| { + roots.sort_by_cached_key(|&i| { ( g.is_test(i), std::cmp::Reverse(g.out[i].len()), - g.file(i), + path(i), g.func(i).line, ) }); - let mut budget = FLAME_NODES; - let (tree, flame_truncated) = flame(&g, &ctx, &roots, &mut budget); - let (lint_rows, lints_truncated) = lints(&g); - - // One flame graph per service that declares dependencies in map.json - - // those are the services whose call trees are expected to leave their own - // boundary. With no `deps` declared, every service gets one instead, so the - // view is not empty just because nobody filled in the config. - // With `deps` declared, those services are exactly the ones whose call - // trees are expected to leave their own boundary. Without them every real - // service gets one - but not when the grouping degenerated to one unit per - // file, where it would just redraw the same tree once per module. - let flame_keys: Vec = if !ctx.relatives.is_empty() { - ctx.relatives.keys().cloned().collect() - } else if ctx.per_file { - Vec::new() - } else { - ctx.map.keys().cloned().collect() - }; - let mut groups = vec![json!({ - "service": "(whole map)", - "declares": Value::Null, - "roots": tree, - "truncated": flame_truncated, - })]; - for key in flame_keys.iter().take(MAX_FLAME_GROUPS) { - // a service's entry points: called by nobody, or only from outside it. - // Those are the frames a caller in another service actually lands on. - let svc_roots: Vec = (0..n) - .filter(|&i| ctx.of_node(&g, i) == Some(key.as_str())) - .filter(|&i| { - g.into[i] - .iter() - .all(|&c| c == i || ctx.of_node(&g, c) != Some(key.as_str())) - }) - .collect(); - let mut b = FLAME_NODES; - let (t, cut) = flame(&g, &ctx, &svc_roots, &mut b); - groups.push(json!({ - "service": key, - "declares": ctx.relatives.get(key), - "roots": t, - "truncated": cut, - })); - } - let groups_truncated = flame_keys.len() > MAX_FLAME_GROUPS; + // every section from here on only reads the graph and the maps beside it - each gets a thread of its own + std::thread::scope(|s| { + let change_part = s.spawn(|| { + // The change set is computed once, here, and every consumer refers + // to this one copy: the `changes` section below, the trigger list, + // and the ranked targets that back it. It used to be recomputed + // inside `test_triggers` and restated in its output, which made the + // same facts arrive twice. + let change_set = prepared + .and_then(|p| changes::assemble(root, root_label, &change_opts, caches, p, Some((&contracts, &cov)))) + .map_err(|e| format!("{e:#}")); + // The trigger walk runs first: its gap list is exactly the set of + // target rows that must survive truncation, since `add` cites them + // by id alone. + let trig = change_set.as_ref().ok().map(|r| triggered(&g, &cov, r)); + let must_keep: BTreeSet<(String, String)> = trig + .as_ref() + .map(|t| { + t.gaps + .iter() + .map(|(file, function, _)| (file.clone(), function.clone())) + .collect() + }) + .unwrap_or_default(); + let targets = test_targets(&g, &cov, &ctx, &must_keep); + let (changes_section, triggers) = match (&change_set, &trig) { + (Ok(r), Some(t)) => ( + serde_json::to_value(r).unwrap_or(Value::Null), + test_triggers(r, t, &targets), + ), + // no git, no base ref, shallow clone: say why rather than render + // an empty tab that reads as "nothing to run" + _ => { + let reason = change_set.as_ref().err().cloned().unwrap_or_default(); + let why = json!({ + "available": false, + "reason": reason, + "hint": "The change set diffs the branch against its base. In CI, fetch history \ + (actions/checkout with fetch-depth: 0); locally, make sure the branch \ + has an upstream such as origin/main.", + }); + (why.clone(), why) + } + }; + (changes_section, targets, triggers) + }); + let flames = s.spawn(|| { + let mut budget = FLAME_NODES; + let (tree, flame_truncated) = flame(&g, &ctx, &roots, &mut budget); + // One flame graph per service that declares dependencies in + // map.json - those are the services whose call trees are expected + // to leave their own boundary. With no `deps` declared, every + // service gets one instead, so the view is not empty just because + // nobody filled in the config - but not when the grouping + // degenerated to one unit per file, where it would just redraw the + // same tree once per module. + let flame_keys: Vec = if !ctx.relatives.is_empty() { + ctx.relatives.keys().cloned().collect() + } else if ctx.per_file { + Vec::new() + } else { + ctx.map.keys().cloned().collect() + }; + let mut groups = vec![json!({ + "service": "(whole map)", + "declares": Value::Null, + "roots": tree, + "truncated": flame_truncated, + })]; + for key in flame_keys.iter().take(MAX_FLAME_GROUPS) { + // a service's entry points: called by nobody, or only from + // outside it. Those are the frames a caller in another service + // actually lands on. + let svc_roots: Vec = (0..n) + .filter(|&i| ctx.of_node(&g, i) == Some(key.as_str())) + .filter(|&i| { + g.into[i] + .iter() + .all(|&c| c == i || ctx.of_node(&g, c) != Some(key.as_str())) + }) + .collect(); + let mut b = FLAME_NODES; + let (t, cut) = flame(&g, &ctx, &svc_roots, &mut b); + groups.push(json!({ + "service": key, + "declares": ctx.relatives.get(key), + "roots": t, + "truncated": cut, + })); + } + (groups, flame_keys.len() > MAX_FLAME_GROUPS) + }); + let lint_part = s.spawn(|| lints(&g)); + let chains = s.spawn(|| deepest_chains(&g, &roots)); + let cycle_rows = s.spawn(|| cycles(&g)); + let service_map = s.spawn(|| services(&g, &ctx)); + let complexity = s.spawn(|| complexity_table(&g, &ctx)); + + // the rankings, on this thread while the others run + let (most_called, widest, most_complex) = { + let mut by_callers: Vec = (0..n).collect(); + by_callers.sort_by_key(|&i| { + ( + std::cmp::Reverse(g.into[i].len()), + std::cmp::Reverse(g.call_sites[i]), + path(i), + ) + }); + let mut by_fanout: Vec = (0..n).collect(); + by_fanout.sort_by_key(|&i| (std::cmp::Reverse(g.out[i].len()), path(i))); + let mut by_complexity: Vec = (0..n).collect(); + by_complexity.sort_by_key(|&i| (std::cmp::Reverse(g.func(i).metrics.complexity()), path(i))); - let mut by_callers: Vec = (0..n).collect(); - by_callers.sort_by_key(|&i| { - ( - std::cmp::Reverse(g.into[i].len()), - std::cmp::Reverse(g.call_sites[i]), - g.file(i), - ) - }); - let mut by_fanout: Vec = (0..n).collect(); - by_fanout.sort_by_key(|&i| (std::cmp::Reverse(g.out[i].len()), g.file(i))); - let mut by_complexity: Vec = (0..n).collect(); - by_complexity.sort_by_key(|&i| (std::cmp::Reverse(g.func(i).metrics.complexity()), g.file(i))); + let row = |i: usize| { + let f = g.func(i); + json!({ + "name": f.name, + "file": g.file(i), + "line": f.line, + "callers": g.into[i].len(), + "call_sites": g.call_sites[i], + "calls": g.out[i].len(), + "lines": f.metrics.body_lines, + "complexity": f.metrics.complexity(), + "loop_depth": f.metrics.max_loop_depth(), + "recursive": f.metrics.recursive, + "language": g.lang(i).as_str(), + "test": g.is_test(i), + }) + }; + let take = |v: &Vec, pred: &dyn Fn(usize) -> bool| { + v.iter() + .copied() + .filter(|&i| pred(i)) + .take(TOP_N) + .map(row) + .collect::>() + }; + // module frames are callers, not functions, so they carry the + // `most_called` counts without ever being ranked as a row + ( + take(&by_callers, &|i| !g.into[i].is_empty() && !g.is_module(i)), + take(&by_fanout, &|i| !g.out[i].is_empty() && !g.is_module(i)), + take(&by_complexity, &|i| !g.is_test(i) && !g.is_module(i)), + ) + }; - let row = |i: usize| { - let f = g.func(i); + // per-language totals, so the UI can say which rules even apply + let mut langs: BTreeMap<&str, (usize, usize, usize, usize)> = BTreeMap::new(); + for c in caches { + let e = langs.entry(c.language.as_str()).or_default(); + e.0 += 1; + e.1 += c.funcs.len(); + e.2 += c.funcs.iter().map(|f| f.metrics.complexity()).sum::(); + e.3 += c.lines; + } + let total_lines: usize = caches.iter().map(|c| c.lines).sum(); + let edge_count: usize = g.out.iter().map(|s| s.len()).sum(); + + let (changes_section, targets, triggers) = joined(change_part); + let (groups, groups_truncated) = joined(flames); + let (lint_rows, lints_truncated) = joined(lint_part); + let (chains, cycle_rows) = (joined(chains), joined(cycle_rows)); + let (service_map, complexity) = (joined(service_map), joined(complexity)); json!({ - "name": f.name, - "file": g.file(i), - "line": f.line, - "callers": g.into[i].len(), - "call_sites": g.call_sites[i], - "calls": g.out[i].len(), - "lines": f.metrics.body_lines, - "complexity": f.metrics.complexity(), - "loop_depth": f.metrics.max_loop_depth(), - "recursive": f.metrics.recursive, - "language": g.lang(i).as_str(), - "test": g.is_test(i), + "schema": SCHEMA, + "root": root_label, + // when the map this was computed from was parsed + "generated": generated, + // how long this analysis took, measured over the whole build above + "took_ns": started.elapsed().as_nanos() as u64, + "totals": { + "files": caches.len(), + // source lines across every mapped file + "lines": total_lines, + // definitions only - the module frames the graph adds for + // top-level code are nodes, but nobody wrote them as functions + "functions": caches.iter().map(|c| c.funcs.len()).sum::(), + "edges": edge_count, + "roots": roots.len(), + }, + "languages": langs.iter().map(|(name, (files, funcs, cx, lines))| json!({ + "language": name, + "files": files, + "funcs": funcs, + "lines": lines, + "avg_complexity": if *funcs == 0 { 0.0 } else { *cx as f64 / *funcs as f64 }, + })).collect::>(), + "flame": { + // one group per service declaring deps, plus the whole map + "groups": groups, + "groups_truncated": groups_truncated, + "note": "static call tree - `value` counts reachable call sites, not time. \ + Recursion is cut where it repeats. A highlighted frame is one the \ + call reached by leaving its caller's service.", + }, + "hot": { + "most_called": most_called, + "widest": widest, + "most_complex": most_complex, + "deepest_chains": chains, + "cycles": cycle_rows, + "note": "structural, not measured: ranks by call-graph shape, not execution frequency.", + }, + "services": service_map, + "complexity": complexity, + // the one canonical change set; `test_triggers` refers to it + "changes": changes_section, + "test_targets": targets, + "test_triggers": triggers, + "lints": { + "findings": lint_rows, + "truncated": lints_truncated, + "rules": rule_catalogue(), + "note": "syntax-level heuristics with no type or data-flow information. \ + Every finding cites the measurement it came from - verify before acting.", + }, }) - }; - let take = |v: &Vec, pred: &dyn Fn(usize) -> bool| { - v.iter() - .copied() - .filter(|&i| pred(i)) - .take(TOP_N) - .map(row) - .collect::>() - }; - - // per-language totals, so the UI can say which rules even apply - let mut langs: BTreeMap<&str, (usize, usize, usize, usize)> = BTreeMap::new(); - for c in caches { - let e = langs.entry(c.language.as_str()).or_default(); - e.0 += 1; - e.1 += c.funcs.len(); - e.2 += c.funcs.iter().map(|f| f.metrics.complexity()).sum::(); - e.3 += c.lines; - } - let total_lines: usize = caches.iter().map(|c| c.lines).sum(); - - let edge_count: usize = g.out.iter().map(|s| s.len()).sum(); - json!({ - "schema": SCHEMA, - "root": root_label, - // when the map this was computed from was parsed - "generated": generated, - // how long this analysis took, measured over the whole build above - "took_ns": started.elapsed().as_nanos() as u64, - "totals": { - "files": caches.len(), - // source lines across every mapped file - "lines": total_lines, - // definitions only - the module frames the graph adds for top-level - // code are nodes, but nobody wrote them as functions - "functions": caches.iter().map(|c| c.funcs.len()).sum::(), - "edges": edge_count, - "roots": roots.len(), - }, - "languages": langs.iter().map(|(name, (files, funcs, cx, lines))| json!({ - "language": name, - "files": files, - "funcs": funcs, - "lines": lines, - "avg_complexity": if *funcs == 0 { 0.0 } else { *cx as f64 / *funcs as f64 }, - })).collect::>(), - "flame": { - // one group per service declaring deps, plus the whole map - "groups": groups, - "groups_truncated": groups_truncated, - "note": "static call tree - `value` counts reachable call sites, not time. \ - Recursion is cut where it repeats. A highlighted frame is one the \ - call reached by leaving its caller's service.", - }, - "hot": { - // module frames are callers, not functions, so they carry the - // `most_called` counts without ever being ranked as a row - "most_called": take(&by_callers, &|i| !g.into[i].is_empty() && !g.is_module(i)), - "widest": take(&by_fanout, &|i| !g.out[i].is_empty() && !g.is_module(i)), - "most_complex": take(&by_complexity, &|i| !g.is_test(i) && !g.is_module(i)), - "deepest_chains": deepest_chains(&g, &roots), - "cycles": cycles(&g), - "note": "structural, not measured: ranks by call-graph shape, not execution frequency.", - }, - "services": services(&g, &ctx), - "complexity": complexity_table(&g, &ctx), - // the one canonical change set; `test_triggers` refers to it - "changes": changes_section, - "test_targets": targets, - "test_triggers": triggers, - "lints": { - "findings": lint_rows, - "truncated": lints_truncated, - "rules": rule_catalogue(), - "note": "syntax-level heuristics with no type or data-flow information. \ - Every finding cites the measurement it came from - verify before acting.", - }, }) } +// a section's result, or its panic carried on to the caller as if it had run here +fn joined(h: std::thread::ScopedJoinHandle<'_, T>) -> T { + h.join().unwrap_or_else(|e| std::panic::resume_unwind(e)) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/inspect.rs b/src/inspect.rs index 1511154..6fac743 100644 --- a/src/inspect.rs +++ b/src/inspect.rs @@ -20,14 +20,14 @@ const FOUND_SLOW: std::time::Duration = std::time::Duration::from_secs(600); const READ_SITES: usize = 59; // commands that read the files they name and write none -const READERS: &[&str] = &[ +pub(crate) const READERS: &[&str] = &[ "cat", "head", "tail", "sed", "awk", "nl", "less", "more", "bat", "batcat", "view", "grep", "egrep", "fgrep", "rg", "ag", "wc", "cut", "sort", "uniq", "diff", "strings", "xxd", "od", "jq", ]; // readers whose first word is a pattern or a script rather than a file -const PATTERNED: &[&str] = &["sed", "awk", "grep", "egrep", "fgrep", "rg", "ag", "jq"]; +pub(crate) const PATTERNED: &[&str] = &["sed", "awk", "grep", "egrep", "fgrep", "rg", "ag", "jq"]; // words ahead of a command that only change how it runs -const WRAPPERS: &[&str] = &["sudo", "env", "time", "command", "nohup", "exec", "nice"]; +pub(crate) const WRAPPERS: &[&str] = &["sudo", "env", "time", "command", "nohup", "exec", "nice"]; // One file an agent read with its own tools. #[derive(Debug, Clone, PartialEq)] @@ -134,7 +134,7 @@ impl Reads { } // claude's transcripts for this project, each session's subagents beside it -fn transcripts(root: &Path) -> Vec { +pub(crate) fn transcripts(root: &Path) -> Vec { let mut out = Vec::new(); for main in crate::prompts::claude_transcripts(root).0 { out.extend(crate::prompts::jsonl_files(&main.with_extension("").join("subagents"))); @@ -224,7 +224,7 @@ fn read_range(input: &Value) -> Option<(usize, usize)> { } // an absolute path as a path inside the project, relative to it -fn inside(root: &Path, path: &Path) -> Option { +pub(crate) fn inside(root: &Path, path: &Path) -> Option { let mut abs = PathBuf::new(); for c in path.components() { match c { @@ -278,7 +278,7 @@ fn shell_reads(line: &str, cwd: &Path) -> Vec<(PathBuf, Option<(usize, usize)>)> } // `NAME=value` ahead of a command -fn assignment(word: &str) -> bool { +pub(crate) fn assignment(word: &str) -> bool { word.split_once('=') .is_some_and(|(k, _)| !k.is_empty() && k.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')) } @@ -319,11 +319,23 @@ enum Into { Fed, // written with `>` - no read Written, + // `>&2`: a stream joined to another, no file at all + Joined, +} + +// one simple command: its words, the files a `<` feeds it, the files a `>` +// writes, and whether a pipe feeds it rather than the files it names +#[derive(Default)] +pub(crate) struct Part { + pub(crate) words: Vec, + pub(crate) fed: Vec, + pub(crate) written: Vec, + pub(crate) piped: bool, } // a command line on its way to its simple commands struct Split { - commands: Vec<(Vec, Vec)>, + parts: Vec, word: Option, into: Into, } @@ -337,19 +349,20 @@ impl Split { let Some(w) = self.word.take() else { return; }; - let Some((args, fed)) = self.commands.last_mut() else { + let Some(part) = self.parts.last_mut() else { return; }; match std::mem::replace(&mut self.into, Into::Arg) { - Into::Arg => args.push(w), - Into::Fed => fed.push(w), - Into::Written => {} + Into::Arg => part.words.push(w), + Into::Fed => part.fed.push(w), + Into::Written => part.written.push(w), + Into::Joined => {} } } - fn command(&mut self) { + fn command(&mut self, piped: bool) { self.word(); - self.commands.push(Default::default()); + self.parts.push(Part { piped, ..Default::default() }); } } @@ -357,7 +370,16 @@ impl Split { // off and the words a `<` feeds it apart. A heredoc's body is text, not // commands, so the line is read no further than one. fn commands(line: &str) -> Vec<(Vec, Vec)> { - let mut s = Split { commands: vec![Default::default()], word: None, into: Into::Arg }; + parts(line) + .into_iter() + .filter(|p| !p.words.is_empty() || !p.fed.is_empty()) + .map(|p| (p.words, p.fed)) + .collect() +} + +// a command line as its simple commands, with what each writes and whether a pipe feeds it +pub(crate) fn parts(line: &str) -> Vec { + let mut s = Split { parts: vec![Part::default()], word: None, into: Into::Arg }; let mut chars = line.chars().peekable(); while let Some(c) = chars.next() { match c { @@ -379,10 +401,18 @@ fn commands(line: &str) -> Vec<(Vec, Vec)> { } '\\' => chars.next().into_iter().filter(|&n| n != '\n').for_each(|n| s.push(n)), ' ' | '\t' => s.word(), - '\n' | ';' | '|' | '&' | '(' | ')' => s.command(), + // `||` runs the next command after this one; `|` and `|&` feed it + '|' => { + let or = chars.next_if_eq(&'|').is_some(); + if !or { + chars.next_if_eq(&'&'); + } + s.command(!or); + } + '\n' | ';' | '&' | '(' | ')' => s.command(false), '#' if s.word.is_none() => { chars.by_ref().take_while(|&c| c != '\n').for_each(drop); - s.command(); + s.command(false); } '<' if chars.peek() == Some(&'<') => break, '<' => { @@ -395,15 +425,18 @@ fn commands(line: &str) -> Vec<(Vec, Vec)> { s.word = None; } s.word(); - while chars.next_if(|&c| c == '>' || c == '&').is_some() {} - s.into = Into::Written; + let mut joined = false; + while let Some(c) = chars.next_if(|&c| c == '>' || c == '&') { + joined |= c == '&'; + } + s.into = if joined { Into::Joined } else { Into::Written }; } c => s.push(c), } } - s.command(); - s.commands.retain(|(args, fed)| !args.is_empty() || !fed.is_empty()); - s.commands + s.command(false); + s.parts.retain(|p| !p.words.is_empty() || !p.fed.is_empty() || !p.written.is_empty()); + s.parts } #[cfg(test)] diff --git a/src/lib.rs b/src/lib.rs index 7768dff..b50b759 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,8 @@ pub mod instance; pub mod languages; pub mod model; pub mod naming; +mod par; +pub mod parse_cache; pub mod pr_summary; pub mod prompts; pub mod render; @@ -20,12 +22,14 @@ pub mod replay; pub mod runccc; pub mod sast; pub mod scan; +pub mod search; pub mod secrets; pub mod serve; pub mod changes; pub mod contracts; pub mod telemetry; pub mod tokenize; +pub mod violations; pub mod vis; pub mod voice; diff --git a/src/main.rs b/src/main.rs index 0d41dc8..ca64f4b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,6 +17,9 @@ struct Cli { // mapped, searched, measured and edited like the rest - any command #[arg(long, global = true)] ignore_skip: bool, + // parse every file afresh rather than reading back an earlier run's parse - `CCC_CACHE_DIR` moves them + #[arg(long, global = true)] + no_parse_cache: bool, } const CACHE_DIR: &str = ".ccc"; @@ -93,9 +96,7 @@ enum Command { out: Option, }, // surface branch changes to a continuous-testing suite: which services - // changed, who calls them, and what needs testing (JSON by default). - // `surf` was the original name; kept so existing scripts keep working. - #[command(alias = "surf")] + // changed, who calls them, and what needs testing (JSON by default) Changes { #[arg(default_value = ".")] path: PathBuf, @@ -206,9 +207,28 @@ enum Command { #[arg(long)] worktree: bool, }, + // where agents broke ccc's rules - writes around its edit tools, exploring with the shell, the web outside `search` - read from their transcripts + Violations { + #[arg(default_value = ".")] + path: PathBuf, + // only calls from the last DAYS days + #[arg(long, value_name = "DAYS")] + since: Option, + // only this session + #[arg(long, value_name = "ID")] + session: Option, + // only these rules - repeatable + #[arg(long = "rule", value_name = "RULE", value_parser = ["writes-outside-ccc", "shell-exploration", "web-outside-search", "blocked-url"])] + rules: Vec, + #[arg(long, value_enum, default_value_t = OutputFormat::Text)] + format: OutputFormat, + // exit non-zero when any violation is found + #[arg(long)] + fail: bool, + }, // run the code map over HTTP for AI agents and people: REST endpoints // (/find /references /dependencies ...), an MCP endpoint at /mcp, and the - // insights UI at /insights. `serve` was the original name; kept so existing + // architecture visualiser at /vis. `serve` was the original name; kept so existing // scripts and editor integrations keep working #[command(alias = "serve")] Run { @@ -226,7 +246,7 @@ enum Command { // disable file watching (rescan only via POST /refresh) #[arg(long)] no_watch: bool, - // The insights UI is served either way now. Accepted so the scripts and + // The visualiser is served either way. Accepted so the scripts and // editor integrations that pass it keep working, but it turns nothing on #[arg(long)] html: bool, @@ -239,13 +259,24 @@ enum Command { // the system down to one function's logic, served at /vis with the html UI #[arg(long, conflicts_with = "no_html")] vis: bool, + // ask agents to annotate the cross-service calls they come across - `ccc:serves` above a handler, + // `ccc:calls` beside a call - so the service map follows calls no parser can + #[arg(long)] + auto: bool, + // what the `search` tool may reach: `allow` only urls matching a --search-url, `block` all but them, `none` any https url + #[arg(long, default_value = "none", value_parser = ["none", "allow", "block"])] + search_mode: String, + // a url pattern (`https://docs.corp/**`) or host pattern (`*.corp.example`) the mode applies to - repeatable + #[arg(long = "search-url", value_name = "PATTERN")] + search_urls: Vec, + // a search page with `{query}` where the words go (`https://wiki.corp/search?q={query}`) - repeatable + #[arg(long = "search-endpoint", value_name = "TEMPLATE")] + search_endpoints: Vec, }, // analyse the project and emit the insights payload Insights { #[arg(default_value = ".")] path: PathBuf, - #[arg(long, value_name = "FILE")] - html: Option, // base ref for the test-trigger diff #[arg(long)] base: Option, @@ -279,7 +310,7 @@ enum Command { // Replays of what agents did on a branch, kept under `refs/ccc/replay/` // - beside the branches, never in them, so a merge cannot carry one to the // default branch. Recording is opt-in per repository (`ccc replay enable`); - // once on, the pre-push hook `ccc run` installs saves each branch's replay as + // once on, the pre-push hook it installs saves each branch's replay as // it is pushed, with anything that looks like a secret taken out first. Replay { #[command(subcommand)] @@ -411,6 +442,9 @@ fn run() -> Result { let cli = Cli::parse(); // before anything is parsed, so every map this process builds agrees codecache::extract::ignore_skips(cli.ignore_skip); + if !cli.no_parse_cache { + codecache::parse_cache::enable(); + } match cli.command { Command::Scan { path, @@ -711,6 +745,19 @@ fn run() -> Result { print!("{}", codecache::pr_summary::markdown(&report, None, 0)); Ok(ExitCode::SUCCESS) } + Command::Violations { path, since, session, rules, format, fail } => { + let opts = codecache::violations::Options { + since_days: since, + session, + rules: rules.iter().filter_map(|r| codecache::violations::Rule::parse(r)).collect(), + }; + let report = codecache::violations::check(&canonical(&path), &opts); + match format { + OutputFormat::Json => println!("{}", serde_json::to_string(&report)?), + OutputFormat::Text => print!("{}", codecache::violations::text(&report)), + } + Ok(if fail && !report.violations.is_empty() { ExitCode::FAILURE } else { ExitCode::SUCCESS }) + } Command::Run { path, addr, @@ -721,6 +768,10 @@ fn run() -> Result { no_html, deps, vis, + auto, + search_mode, + search_urls, + search_endpoints, } => { let watch = if no_watch || watch_interval == 0 { None @@ -735,13 +786,19 @@ fn run() -> Result { watch, html: !no_html, vis, + auto, + search: codecache::search::Policy::new( + codecache::search::Mode::parse(&search_mode).map_err(|e| anyhow!(e))?, + &search_urls, + &search_endpoints, + ) + .map_err(|e| anyhow!(e))?, }; let root = canonical(&path); - ask_recording(&root); codecache::serve(&root, &opts)?; Ok(ExitCode::SUCCESS) } - Command::Insights { path, html, base } => { + Command::Insights { path, base } => { let root = canonical(&path); let label = root .file_name() @@ -749,13 +806,7 @@ fn run() -> Result { .unwrap_or(".") .to_string(); let report = codecache::insights::analyse(&root, &label, base.as_deref())?; - match html { - Some(file) => { - codecache::html::write_insights_html(&file, &label, &report)?; - println!("wrote {}", file.display()); - } - None => println!("{}", serde_json::to_string(&report)?), - } + println!("{}", serde_json::to_string(&report)?); Ok(ExitCode::SUCCESS) } Command::Sast { path, include_tests, min_severity, format } => { @@ -1388,74 +1439,6 @@ fn dir_on_path(dir: &Path) -> bool { .unwrap_or(false) } -// The question `ccc run` asks once per repository, on a terminal: record -// agent sessions as replays beside the branches? What it says depends on who -// can read the remote, since a replay carries prompts and code. Off a -// terminal - an editor starting it - nothing is asked and nothing recorded; -// the editor asks for itself. -fn ask_recording(root: &Path) { - use codecache::replay::{consent, exposure_here, is_repo, set_consent, Consent}; - use std::io::IsTerminal; - if !is_repo(root) || consent(root) != Consent::Unasked { - return; - } - if !std::io::stdin().is_terminal() || !std::io::stderr().is_terminal() { - eprintln!("ccc: session recording is off for this repository until someone turns it on - `ccc replay enable`"); - return; - } - eprintln!( - "\nccc can record your agent sessions as replays, kept beside this repository's branches\n\ - (refs/ccc) and pushed with them, so the team can watch how each change was made.\n\ - A replay carries your prompts and the code each step touched - anything that looks\n\ - like a secret is redacted first. For a team on a private repository that is ideal:\n\ - everyone who can read the code can replay how it was written." - ); - if let Some(e) = exposure_here(root) { - match e.public { - Some(true) if e.sealed => eprintln!( - "\n{} ({}) answers without signing in, but .ccc/map.json encrypts replays to the team's\n\ - runccc key, so only the team opens them.", - e.remote, e.url - ), - Some(true) if e.allowed => eprintln!( - "\n{} ({}) answers without signing in, and .ccc/map.json allows replays to it.", - e.remote, e.url - ), - Some(true) if e.public_host => eprintln!( - "\n{} ({}) can be read by anyone, without signing in. Replays would stay on this\n\ - machine - make the repository private to share them with your team.", - e.remote, e.url - ), - Some(true) => eprintln!( - "\n{} ({}) answers without signing in, so replays would stay on this machine.\n\ - If {} is your company's internal instance, where every repository is readable\n\ - company-wide, allow it in .ccc/map.json, where the team sees it: {}", - e.remote, - e.url, - e.host, - codecache::replay::ALLOW_PUBLIC - ), - Some(false) => eprintln!("\n{} needs a sign-in to read, so replays reach only people who can read the code.", e.remote), - None => eprintln!("\nccc could not tell whether {} ({}) can be read without signing in.", e.remote, e.url), - } - } - let yes = ask("\nRecord agent sessions for this repository? [y/N] "); - match set_consent(root, yes) { - Ok(_) if yes => eprintln!("ccc: recording on - `ccc replay disable` turns it off\n"), - Ok(_) => eprintln!("ccc: not recording - `ccc replay enable` turns it on later\n"), - Err(e) => eprintln!("ccc: could not keep the answer: {e:#}\n"), - } -} - -// a yes-or-no on the terminal - anything but yes is no -fn ask(question: &str) -> bool { - use std::io::Write; - eprint!("{question}"); - let _ = std::io::stderr().flush(); - let mut line = String::new(); - std::io::stdin().read_line(&mut line).is_ok() && matches!(line.trim().to_ascii_lowercase().as_str(), "y" | "yes") -} - fn canonical(path: &Path) -> PathBuf { path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) } diff --git a/src/model.rs b/src/model.rs index 49d127a..f932453 100644 --- a/src/model.rs +++ b/src/model.rs @@ -1,10 +1,11 @@ //! data model for extracted symbols and a whole-file cache entry. use crate::languages::Language; +use serde::{Deserialize, Serialize}; use std::path::PathBuf; // module/file-level constant or static binding. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Const { pub line: usize, // 1-based pub name: String, @@ -12,14 +13,14 @@ pub struct Const { } // named type definition `struct`/`enum`/`class`/`interface`/`trait`/alias. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct TypeDef { pub line: usize, pub name: String, pub kind: String, // struct | enum | class | interface | trait | alias | union | protocol } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct LoopInfo { pub line: usize, pub kind: String, // for | while | do | loop | comprehension @@ -28,7 +29,7 @@ pub struct LoopInfo { } // a call that acquires or releases a resource, for the leak heuristic. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] pub struct ResourceOp { pub line: usize, pub name: String, @@ -38,8 +39,31 @@ pub struct ResourceOp { pub guarded: bool, } +// a stored `ResourceOp` - its pair must name an acquire a language table holds, or the file is parsed afresh +#[derive(Deserialize)] +struct StoredOp { + line: usize, + name: String, + pair: String, + acquire: bool, + guarded: bool, +} + +impl<'de> Deserialize<'de> for ResourceOp { + fn deserialize>(d: D) -> Result { + let op = StoredOp::deserialize(d)?; + let pair = Language::ALL + .iter() + .flat_map(|l| l.resource_pairs()) + .map(|(acquire, _)| *acquire) + .find(|acquire| *acquire == op.pair) + .ok_or_else(|| serde::de::Error::custom(format!("no language pairs `{}`", op.pair)))?; + Ok(ResourceOp { line: op.line, name: op.name, pair, acquire: op.acquire, guarded: op.guarded }) + } +} + // structural measurements of one function body. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct FuncMetrics { pub body_lines: usize, pub params: usize, @@ -80,7 +104,7 @@ impl FuncMetrics { } // function / method definition. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Func { pub line: usize, // 1-based (position of the name token) pub col: usize, // 1-based @@ -100,7 +124,7 @@ pub struct Func { } // a call site kept in "loose" form: every call in the file, resolved or not. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct CallSite { // nearest enclosing function (`` at file level) pub caller: String, @@ -115,7 +139,7 @@ pub struct CallSite { // resolved call: `caller` (at `call_line`) invokes a function defined in the // same file at (`target_line`, `target_col`). -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Ref { pub caller: String, pub call_line: usize, @@ -126,14 +150,14 @@ pub struct Ref { } // A free-form "note" (TODO/FIXME/NOTE/...). -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Note { pub line: usize, pub text: String, } // Which side of a boundary an annotation describes. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] pub enum Boundary { // this function handles the key: `ccc:serves grpc billing.v1.Charge` Serves, @@ -151,7 +175,7 @@ impl Boundary { } // An author-written hint that a call leaves this process -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Annotation { pub line: usize, pub boundary: Boundary, @@ -165,7 +189,7 @@ pub struct Annotation { } // one import/use/include statement, in loose form -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Import { pub line: usize, pub module: String, diff --git a/src/par.rs b/src/par.rs new file mode 100644 index 0000000..040a76a --- /dev/null +++ b/src/par.rs @@ -0,0 +1,50 @@ +//! Parallelise independent pieces of work - over every core + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread; + +// indices a worker takes at a time - small enough to keep every core busy, large enough not to contend +const CHUNK: usize = 16; + +// `f(i)` for every `i` in `0..n` in order, chunks pulled off a shared counter by every core and the caller +pub(crate) fn map(n: usize, f: impl Fn(usize) -> T + Sync) -> Vec { + let workers = thread::available_parallelism().map_or(1, |p| p.get()).min(n.div_ceil(CHUNK)); + if workers <= 1 { + return (0..n).map(f).collect(); + } + let next = AtomicUsize::new(0); + let work = || { + let mut done = Vec::new(); + loop { + let start = next.fetch_add(CHUNK, Ordering::Relaxed); + if start >= n { + return done; + } + done.push((start, (start..(start + CHUNK).min(n)).map(&f).collect::>())); + } + }; + let mut parts = thread::scope(|s| { + let helpers: Vec<_> = (1..workers).filter_map(|_| thread::Builder::new().spawn_scoped(s, work).ok()).collect(); + let mut parts = work(); + for h in helpers { + parts.extend(h.join().unwrap_or_else(|e| std::panic::resume_unwind(e))); + } + parts + }); + parts.sort_unstable_by_key(|(start, _)| *start); + parts.into_iter().flat_map(|(_, part)| part).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + // the answers come back in index order however the chunks were shared out + #[test] + fn results_keep_their_order() { + for n in [0, 1, CHUNK - 1, CHUNK, CHUNK * 7 + 3, 10_000] { + let squares = map(n, |i| i * i); + assert_eq!(squares, (0..n).map(|i| i * i).collect::>()); + } + } +} diff --git a/src/parse_cache.rs b/src/parse_cache.rs new file mode 100644 index 0000000..baf7b41 --- /dev/null +++ b/src/parse_cache.rs @@ -0,0 +1,178 @@ +//! parses kept by content, so a file unchanged since an earlier run is read back rather than parsed again + +use crate::extract::{self, Extracted}; +use crate::languages::Language; +use sha2::{Digest, Sha256}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::OnceLock; +use std::time::{SystemTime, UNIX_EPOCH}; + +// builds whose parses are kept side by side - the one in development, the editor's, a release +const KEEP_BUILDS: usize = 4; + +// stamped each time a build opens its directory, so the sweep knows which ran last +const USED: &str = "last-used"; + +static ACTIVE: OnceLock> = OnceLock::new(); + +// one build's parses, each under the hash of what was parsed +pub struct ParseCache { + dir: PathBuf, +} + +// keep parses under the user's cache directory - the binary turns this on, a library caller parses afresh +pub fn enable() { + let root = crate::voice::cache_dir().map(|d| d.join("parses")); + let _ = ACTIVE.set(root.and_then(|r| ParseCache::open(&r))); +} + +pub fn active() -> Option<&'static ParseCache> { + ACTIVE.get().and_then(Option::as_ref) +} + +impl ParseCache { + // this build's directory under `root` - `None` when it cannot be made, and every file is parsed + pub fn open(root: &Path) -> Option { + let dir = root.join(build_id()?); + fs::create_dir_all(&dir).ok()?; + sweep(root, &dir); + Some(ParseCache { dir }) + } + + // what `extract::extract` makes of `src`, read back when kept - an entry that will not read is replaced + pub fn extract(&self, lang: Language, src: &str) -> Option { + let honour_skips = !extract::skips_ignored(); + let path = self.dir.join(key(lang, honour_skips, src)); + let kept = fs::read(&path).ok().and_then(|b| serde_json::from_slice::>(&b).ok()); + if let Some(kept) = kept { + return kept; + } + let parsed = extract::extract_with(lang, src, honour_skips); + keep(&path, &parsed); + parsed + } +} + +// the parse's address - everything `extract_with` reads and no path, so a copy or rename reuses it +fn key(lang: Language, honour_skips: bool, src: &str) -> String { + let mut h = Sha256::new(); + h.update(lang.as_str()); + h.update([0, u8::from(honour_skips)]); + h.update(src); + format!("{}.json", hex(&h.finalize())) +} + +// written beside its place and renamed in, so no reader sees half an entry +fn keep(path: &Path, parsed: &Option) { + static SEQ: AtomicUsize = AtomicUsize::new(0); + let Ok(body) = serde_json::to_vec(parsed) else { return }; + let tmp = path.with_extension(format!("{}-{}.tmp", std::process::id(), SEQ.fetch_add(1, Ordering::Relaxed))); + if fs::write(&tmp, body).is_err() || fs::rename(&tmp, path).is_err() { + let _ = fs::remove_file(&tmp); + } +} + +// the version plus the binary's size and mtime - a rebuild without a version bump must not read old parses +fn build_id() -> Option { + let meta = fs::metadata(std::env::current_exe().ok()?).ok()?; + let modified = meta.modified().ok()?.duration_since(UNIX_EPOCH).ok()?.as_nanos(); + let mut h = Sha256::new(); + h.update(meta.len().to_le_bytes()); + h.update(modified.to_le_bytes()); + Some(format!("{}-{}", env!("CARGO_PKG_VERSION"), &hex(&h.finalize())[..16])) +} + +// stamp `current` as the latest build to run, and clear all but the most recently run builds +fn sweep(root: &Path, current: &Path) { + let now = SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |d| d.as_nanos()); + let _ = fs::write(current.join(USED), now.to_string()); + let Ok(entries) = fs::read_dir(root) else { return }; + let mut builds: Vec<(SystemTime, PathBuf)> = entries + .flatten() + .map(|e| e.path()) + .filter(|p| p.is_dir()) + .map(|p| (fs::metadata(p.join(USED)).and_then(|m| m.modified()).unwrap_or(UNIX_EPOCH), p)) + .collect(); + builds.sort_by(|a, b| b.0.cmp(&a.0)); + for (_, old) in builds.into_iter().skip(KEEP_BUILDS) { + if old != current { + let _ = fs::remove_dir_all(old); + } + } +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn root(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("ccc-parses-{tag}-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + dir + } + + fn json(ex: &Option) -> String { + serde_json::to_string(ex).expect("encode") + } + + const SRC: &str = "// TODO: tidy\nfn charge(c: u64) -> u64 {\n for _ in 0..3 { helper(); }\n if c > 1 { charge(c - 1) } else { c }\n}\nfn helper() { let f = File::open(\"x\"); }\n"; + const C_SRC: &str = "#include \nvoid f(void) {\n FILE *x = fopen(\"a\", \"r\");\n while (x) { fclose(x); }\n}\n"; + + // a parse read back is the parse that was kept - resource pairs and measurements included + #[test] + fn a_kept_parse_reads_back_as_the_parse_itself() { + let dir = root("roundtrip"); + let cache = ParseCache::open(&dir).expect("open"); + for (lang, src) in [(Language::Rust, SRC), (Language::C, C_SRC)] { + let fresh = extract::extract(lang, src); + let first = cache.extract(lang, src); + let again = cache.extract(lang, src); + assert_eq!(json(&first), json(&fresh)); + assert_eq!(json(&again), json(&fresh)); + } + let c = cache.extract(Language::C, C_SRC).expect("parse"); + assert_eq!(c.funcs[0].metrics.resources.len(), 2, "{:?}", c.funcs[0].metrics); + assert_eq!(fs::read_dir(&cache.dir).expect("dir").count(), 3, "two entries beside the stamp"); + // the language is part of the address, so a file read as another one never gets this parse + cache.extract(Language::Go, SRC); + assert_eq!(fs::read_dir(&cache.dir).expect("dir").count(), 4); + let _ = fs::remove_dir_all(&dir); + } + + // an entry that will not read is parsed again and replaced, never trusted + #[test] + fn a_broken_entry_is_parsed_again() { + let dir = root("broken"); + let cache = ParseCache::open(&dir).expect("open"); + let path = cache.dir.join(key(Language::Rust, !extract::skips_ignored(), SRC)); + fs::write(&path, b"{\"consts\": [").expect("write"); + let read = cache.extract(Language::Rust, SRC); + assert_eq!(json(&read), json(&extract::extract(Language::Rust, SRC))); + assert!(serde_json::from_slice::>(&fs::read(&path).expect("read")).is_ok()); + let _ = fs::remove_dir_all(&dir); + } + + // old builds' parses are cleared, but never the running build's own + #[test] + fn only_the_most_recent_builds_are_kept() { + let dir = root("sweep"); + for i in 0..KEEP_BUILDS + 2 { + let old = dir.join(format!("0.0.{i}-old")); + fs::create_dir_all(&old).expect("mkdir"); + let stamp = fs::File::create(old.join(USED)).expect("stamp"); + // long ago, so the running build is the latest however coarse the clock + stamp.set_modified(UNIX_EPOCH + std::time::Duration::from_secs(i as u64 + 1)).expect("age"); + } + let cache = ParseCache::open(&dir).expect("open"); + let left: Vec = fs::read_dir(&dir).expect("dir").flatten().map(|e| e.path()).collect(); + assert_eq!(left.len(), KEEP_BUILDS); + assert!(left.contains(&cache.dir)); + let _ = fs::remove_dir_all(&dir); + } +} diff --git a/src/prompts.rs b/src/prompts.rs index b18eef3..6a9b9e5 100644 --- a/src/prompts.rs +++ b/src/prompts.rs @@ -260,9 +260,11 @@ pub(crate) fn claude_slug(root: &Path) -> String { // The slug is only a hint - this is the fact // ccc:skip fn transcript_cwd(path: &Path) -> Option { - let raw = fs::read_to_string(path).ok()?; - for line in raw.lines().take(50) { - let v: Value = serde_json::from_str(line).ok()?; + use std::io::BufRead; + // the cwd is in the first records, and a transcript can run to hundreds of megabytes + let file = fs::File::open(path).ok()?; + for line in std::io::BufReader::new(file).lines().take(50) { + let v: Value = serde_json::from_str(&line.ok()?).ok()?; if let Some(cwd) = v.get("cwd").and_then(|c| c.as_str()) { return Some(cwd.to_string()); } diff --git a/src/scan.rs b/src/scan.rs index 2da76c6..d82c67c 100644 --- a/src/scan.rs +++ b/src/scan.rs @@ -2,7 +2,7 @@ use crate::languages::Language; use crate::model::{Counts, FileCache}; -use crate::{extract, naming, render}; +use crate::{extract, naming, parse_cache, render}; use anyhow::{Context, Result}; use ignore::WalkBuilder; use std::collections::{BTreeMap, BTreeSet, HashMap}; @@ -147,6 +147,9 @@ const PARSE_STACK_BYTES: usize = 8 << 20; // next file off a shared counter so one large file never stalls a whole batch fn parse_all(root: &Path, files: &[PathBuf]) -> Vec { let workers = thread::available_parallelism().map_or(1, |n| n.get()).min(files.len()); + // the largest first, so the file that takes longest starts at once rather than finishing last + let mut files: Vec<&PathBuf> = files.iter().collect(); + files.sort_by_cached_key(|f| std::cmp::Reverse(fs::metadata(f).map_or(0, |m| m.len()))); let next = AtomicUsize::new(0); let drain = || { let mut out = Vec::new(); @@ -201,7 +204,11 @@ fn build_one(root: &Path, path: &Path) -> Option { // written yet; none for a file in no language ccc reads, or one that will not parse pub fn read_one(rel: &Path, src: &str) -> Option { let lang = Language::from_path(rel)?; - let ex = extract::extract(lang, src)?; + // read back when an earlier run already parsed this exact text + let ex = match parse_cache::active() { + Some(kept) => kept.extract(lang, src)?, + None => extract::extract(lang, src)?, + }; let rel = rel.to_path_buf(); Some(FileCache { cache_name: naming::cache_name(&rel), diff --git a/src/search.rs b/src/search.rs new file mode 100644 index 0000000..d742e63 --- /dev/null +++ b/src/search.rs @@ -0,0 +1,641 @@ +//! `search` - web requests for agents, https only and through the user's rules +//! +//! The rules are a mode and a url list. `allow` lets a request through only +//! when an entry matches it, `block` refuses one when an entry matches it, and +//! with no mode any https url goes. Search endpoints are url templates holding +//! `{query}` - an internal wiki's search page, say - and every request, an +//! endpoint's or a fetched page's, redirects included, passes the same rules. + +use globset::{GlobBuilder, GlobMatcher}; +use std::process::Command; + +// what every refusal starts with, so the rules check can find one in a transcript +pub const REFUSED: &str = "search refused:"; +// lines one answer carries before the rest waits behind `offset` +pub const PAGE_LINES: usize = 200; +const FETCH_TIMEOUT_SECS: u64 = 20; +const MAX_BYTES: u64 = 5 << 20; +const MAX_REDIRECTS: usize = 5; +// the lines one endpoint's results add to a search, so one portal cannot crowd out the rest +const ENDPOINT_LINES: usize = 120; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Mode { + #[default] + None, + Allow, + Block, +} + +impl Mode { + pub fn parse(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "" | "none" => Ok(Mode::None), + "allow" => Ok(Mode::Allow), + "block" => Ok(Mode::Block), + other => Err(format!("search mode wants `none`, `allow` or `block`, got '{other}'")), + } + } + + pub fn label(self) -> &'static str { + match self { + Mode::None => "none", + Mode::Allow => "allow", + Mode::Block => "block", + } + } +} + +// one url list entry: a url glob when it names a scheme, a host glob otherwise +#[derive(Clone, Debug)] +struct Rule { + entry: String, + whole_url: bool, + glob: GlobMatcher, +} + +impl Rule { + fn matches(&self, url: &Url) -> bool { + if self.whole_url { + self.glob.is_match(&url.full) + } else { + self.glob.is_match(&url.host) + } + } +} + +#[derive(Clone, Debug, Default)] +pub struct Policy { + pub mode: Mode, + rules: Vec, + pub endpoints: Vec, +} + +impl Policy { + // every entry checked up front, so a typo stops the server rather than quietly allowing or blocking nothing + pub fn new(mode: Mode, urls: &[String], endpoints: &[String]) -> Result { + let mut rules = Vec::new(); + for raw in urls.iter().map(|u| u.trim()).filter(|u| !u.is_empty()) { + let whole_url = raw.contains("://"); + if !whole_url && raw.contains('/') { + return Err(format!("search url '{raw}' is neither a url (`https://host/path/**`) nor a host (`*.corp.example`)")); + } + let glob = GlobBuilder::new(raw) + .case_insensitive(true) + .literal_separator(false) + .build() + .map_err(|e| format!("search url '{raw}' is not a valid pattern: {e}"))? + .compile_matcher(); + rules.push(Rule { entry: raw.to_string(), whole_url, glob }); + } + let mut kept = Vec::new(); + for raw in endpoints.iter().map(|e| e.trim()).filter(|e| !e.is_empty()) { + if !raw.contains("{query}") { + return Err(format!("search endpoint '{raw}' has no `{{query}}` for the words to go in")); + } + parse_url(&raw.replace("{query}", "q")).map_err(|e| format!("search endpoint '{raw}': {e}"))?; + kept.push(raw.to_string()); + } + Ok(Policy { mode, rules, endpoints: kept }) + } + + // one line on what the rules are, for startup and every answer + pub fn describe(&self) -> String { + let rules = match self.mode { + Mode::None => "no mode set, so any https url goes".to_string(), + Mode::Allow => format!("allow - only urls matching one of {} entr(ies) go", self.rules.len()), + Mode::Block => format!("block - urls matching one of {} entr(ies) are refused", self.rules.len()), + }; + format!("{rules}; {} search endpoint(s)", self.endpoints.len()) + } + + // the url as it will be requested, or why the rules refuse it + pub fn check(&self, raw: &str) -> Result { + let url = parse_url(raw).map_err(|e| format!("{REFUSED} {} - {e}", raw.trim()))?; + let hit = self.rules.iter().find(|r| r.matches(&url)); + match (self.mode, hit) { + (Mode::None, _) | (Mode::Allow, Some(_)) | (Mode::Block, None) => Ok(url), + (Mode::Allow, None) => Err(format!("{REFUSED} {} - not on the allow list", url.full)), + (Mode::Block, Some(r)) => Err(format!("{REFUSED} {} - matches block entry `{}`", url.full, r.entry)), + } + } +} + +// an https url, taken apart only as far as the rules need +#[derive(Clone, Debug, PartialEq)] +pub struct Url { + // scheme and host lowercased, a bare host given a `/`, the fragment dropped + pub full: String, + pub host: String, + origin: String, +} + +fn parse_url(raw: &str) -> Result { + let raw = raw.trim(); + if raw.chars().any(|c| c.is_whitespace() || c.is_control()) { + return Err("a url cannot hold spaces or control characters".into()); + } + let (scheme, rest) = raw.split_once("://").ok_or("not an absolute url")?; + if !scheme.eq_ignore_ascii_case("https") { + return Err(format!("only https is allowed, not {scheme}")); + } + let end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + let authority = rest[..end].to_ascii_lowercase(); + if authority.contains('@') { + return Err("credentials in a url are refused".into()); + } + let host = match authority.strip_prefix('[') { + Some(v6) => format!("[{}]", v6.split_once(']').ok_or("an unclosed ipv6 host")?.0), + None => authority.split(':').next().unwrap_or_default().to_string(), + }; + if host.is_empty() { + return Err("no host".into()); + } + let tail = rest[end..].split('#').next().unwrap_or_default(); + let path = if tail.starts_with('/') { tail.to_string() } else { format!("/{tail}") }; + let origin = format!("https://{authority}"); + Ok(Url { full: format!("{origin}{path}"), host, origin }) +} + +// where a link or a redirect points, made absolute against the page it is on +fn resolve(base: &Url, href: &str) -> String { + let href = href.trim(); + if href.contains("://") { + return href.to_string(); + } + if let Some(rest) = href.strip_prefix("//") { + return format!("https://{rest}"); + } + if href.starts_with('/') { + return format!("{}{href}", base.origin); + } + let path = base.full[base.origin.len()..].split(['?', '#']).next().unwrap_or("/"); + let dir = path.rsplit_once('/').map_or("", |(d, _)| d); + format!("{}{dir}/{href}", base.origin) +} + +// the words of a query as a url carries them +fn encode(q: &str) -> String { + q.bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => (b as char).to_string(), + _ => format!("%{b:02X}"), + }) + .collect() +} + +// one answered request +struct Fetched { + url: String, + // the url asked for, when redirects ended somewhere else + from: Option, + status: u16, + content_type: String, + body: Vec, +} + +impl Policy { + // a GET with every redirect followed by hand, so each hop passes the rules too + fn fetch(&self, first: Url) -> Result { + let mut url = first.clone(); + for _ in 0..=MAX_REDIRECTS { + let (status, headers, body) = get(&url.full)?; + let location = headers.iter().find(|(k, _)| k == "location").map(|(_, v)| v.as_str()); + if let (300..=399, Some(next)) = (status, location) { + url = self.check(&resolve(&url, next))?; + continue; + } + let content_type = headers + .iter() + .find(|(k, _)| k == "content-type") + .map(|(_, v)| v.to_ascii_lowercase()) + .unwrap_or_default(); + let from = (url != first).then(|| first.full.clone()); + return Ok(Fetched { url: url.full, from, status, content_type, body }); + } + Err(format!("{} redirected more than {MAX_REDIRECTS} times", first.full)) + } +} + +// one request by curl - https only, redirects not followed, its status and headers read back +// no http client is linked in for this: curl honours the proxy settings a corporate network relies on +fn get(url: &str) -> Result<(u16, Vec<(String, String)>, Vec), String> { + let out = Command::new("curl") + .args(["--silent", "--show-error", "--include", "--proto", "=https", "--proto-redir", "=https"]) + .args(["--max-time", &FETCH_TIMEOUT_SECS.to_string(), "--max-filesize", &MAX_BYTES.to_string()]) + .args(["--user-agent", concat!("ccc-search/", env!("CARGO_PKG_VERSION"))]) + .arg(url) + .output() + .map_err(|e| format!("running curl for {url} (is curl installed?): {e}"))?; + if !out.status.success() { + let why = String::from_utf8_lossy(&out.stderr).trim().to_string(); + return Err(match out.status.code() { + Some(63) => format!("{url} is larger than {} MB", MAX_BYTES >> 20), + _ => format!("fetching {url} failed: {why}"), + }); + } + split_response(&out.stdout).ok_or_else(|| format!("{url} answered with something that is not http")) +} + +// `--include` output as status, headers and body - a proxy's `200 Connection +// established` and any `100 Continue` are the head of what follows, not the answer +fn split_response(raw: &[u8]) -> Option<(u16, Vec<(String, String)>, Vec)> { + let mut rest = raw; + loop { + let end = rest.windows(4).position(|w| w == b"\r\n\r\n").map(|i| (i, i + 4)) + .or_else(|| rest.windows(2).position(|w| w == b"\n\n").map(|i| (i, i + 2)))?; + let head = String::from_utf8_lossy(&rest[..end.0]).to_string(); + let body = &rest[end.1..]; + let mut lines = head.lines(); + let status_line = lines.next()?; + let status: u16 = status_line.split_whitespace().nth(1)?.parse().ok()?; + let interim = (100..200).contains(&status) || status_line.to_ascii_lowercase().contains("connection established"); + if interim && body.starts_with(b"HTTP/") { + rest = body; + continue; + } + let headers = lines + .filter_map(|l| l.split_once(':')) + .map(|(k, v)| (k.trim().to_ascii_lowercase(), v.trim().to_string())) + .collect(); + return Some((status, headers, body.to_vec())); + } +} + +// what a page says, as lines of text - html read down to its words and links +fn readable(page: &Fetched) -> Result, String> { + let ct = page.content_type.as_str(); + let text = String::from_utf8_lossy(&page.body); + let sniffed_html = || { + let head = text.trim_start().get(..15).unwrap_or_default().to_ascii_lowercase(); + head.starts_with("(&page.body) + .ok() + .and_then(|v| serde_json::to_string_pretty(&v).ok()) + .unwrap_or_else(|| text.to_string()) + } else if ct.is_empty() || ct.starts_with("text/") || ct.contains("xml") || ct.contains("markdown") { + if page.body.contains(&0) { + return Err(format!("{} is binary, and search reads text pages only", page.url)); + } + text.to_string() + } else { + return Err(format!("{} is {ct}, and search reads text pages only", page.url)); + }; + Ok(body.lines().map(str::to_string).collect()) +} + +// tags that end a line of text where they open or close +const BLOCKS: &[&str] = &[ + "address", "article", "aside", "blockquote", "br", "dd", "div", "dl", "dt", "footer", "form", "h1", "h2", "h3", "h4", "h5", + "h6", "header", "hr", "main", "nav", "ol", "p", "pre", "section", "table", "tr", "ul", +]; +// tags whose content is never text a reader sees +const HIDDEN: &[&str] = &["script", "style", "noscript", "svg", "template", "head"]; + +// an html page's text, a line per block, its links written `[text](url)` so the agent can fetch the next page +fn html_text(html: &str, base: &Url) -> String { + let lower = html.to_ascii_lowercase(); + let title = lower + .find("')? + s + 1))) + .and_then(|(_, open)| Some((open, lower[open..].find(">().join(" ")) + .unwrap_or_default(); + let mut out = String::new(); + let mut link: Option<(String, usize)> = None; + let mut i = 0; + while i < html.len() { + if !html[i..].starts_with('<') { + let next = html[i..].find('<').map_or(html.len(), |n| i + n); + let raw = decode(&html[i..next]); + let words = raw.split_whitespace().collect::>().join(" "); + // the space between this text and the tag either side of it + if raw.starts_with(char::is_whitespace) && !out.is_empty() && !out.ends_with([' ', '\n']) { + out.push(' '); + } + out.push_str(&words); + if raw.ends_with(char::is_whitespace) && !words.is_empty() { + out.push(' '); + } + i = next; + continue; + } + if lower[i..].starts_with("").map_or(html.len(), |e| i + e + 3); + continue; + } + let end = tag_end(html, i); + let tag = &html[i + 1..end.min(html.len())]; + i = (end + 1).min(html.len()); + let closing = tag.starts_with('/'); + let name: String = tag + .trim_start_matches('/') + .chars() + .take_while(|c| c.is_ascii_alphanumeric()) + .collect::() + .to_ascii_lowercase(); + if HIDDEN.contains(&name.as_str()) && !closing { + i = lower[i..].find(&format!("').map_or(html.len(), |g| close + g + 1) + }); + continue; + } + match name.as_str() { + "li" if !closing => out.push_str("\n- "), + "td" | "th" if !closing => out.push_str(" | "), + "a" if !closing => { + link = attr(tag, "href") + .map(|h| decode(&h)) + .filter(|h| !h.starts_with('#') && !h.to_ascii_lowercase().starts_with("javascript:") && !h.starts_with("mailto:")) + .map(|h| (resolve(base, &h), out.len())); + } + "a" => { + if let Some((href, start)) = link.take() { + let text = out[start..].split_whitespace().collect::>().join(" "); + out.truncate(start); + if text.is_empty() { + out.push_str(&format!("<{href}>")); + } else { + out.push_str(&format!("[{text}]({href})")); + } + } + } + n if BLOCKS.contains(&n) => out.push('\n'), + _ => {} + } + } + let mut lines: Vec = Vec::new(); + if !title.is_empty() { + lines.push(format!("# {title}")); + } + for line in out.lines().map(str::trim) { + // one blank line between blocks, never a run of them + if line.is_empty() && lines.last().is_none_or(|l| l.is_empty()) { + continue; + } + lines.push(line.to_string()); + } + lines.join("\n") +} + +// the `>` that closes the tag opening at `start`, quotes respected +fn tag_end(html: &str, start: usize) -> usize { + let mut quote: Option = None; + for (k, c) in html[start..].char_indices() { + match (quote, c) { + (None, '"' | '\'') => quote = Some(c), + (Some(q), c) if c == q => quote = None, + (None, '>') => return start + k, + _ => {} + } + } + html.len() +} + +// an attribute's value inside a tag, quoted or bare +fn attr(tag: &str, name: &str) -> Option { + let lower = tag.to_ascii_lowercase(); + let mut from = 0; + while let Some(at) = lower[from..].find(name) { + let at = from + at; + from = at + name.len(); + let before_ok = at == 0 || lower[..at].ends_with(char::is_whitespace); + let rest = lower[from..].trim_start(); + if !before_ok || !rest.starts_with('=') { + continue; + } + let value_at = tag.len() - rest.len() + 1; + let value = tag[value_at..].trim_start(); + return Some(match value.chars().next() { + Some(q @ ('"' | '\'')) => value[1..].split(q).next().unwrap_or_default().to_string(), + _ => value.split(|c: char| c.is_whitespace() || c == '>').next().unwrap_or_default().to_string(), + }); + } + None +} + +// the character references a page's text is written with +fn decode(s: &str) -> String { + if !s.contains('&') { + return s.to_string(); + } + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(at) = rest.find('&') { + out.push_str(&rest[..at]); + rest = &rest[at..]; + let Some(semi) = rest.bytes().take(12).position(|b| b == b';') else { + out.push('&'); + rest = &rest[1..]; + continue; + }; + let entity = &rest[1..semi]; + let ch = match entity { + "amp" => Some('&'), + "lt" => Some('<'), + "gt" => Some('>'), + "quot" => Some('"'), + "apos" | "#39" => Some('\''), + "nbsp" => Some(' '), + e if e.starts_with("#x") || e.starts_with("#X") => u32::from_str_radix(&e[2..], 16).ok().and_then(char::from_u32), + e if e.starts_with('#') => e[1..].parse().ok().and_then(char::from_u32), + _ => None, + }; + match ch { + Some(c) => { + out.push(c); + rest = &rest[semi + 1..]; + } + None => { + out.push('&'); + rest = &rest[1..]; + } + } + } + out.push_str(rest); + out +} + +// the `search` tool: a query run against the endpoints, or one page fetched - lines past `offset + PAGE_LINES` wait for the next call +pub fn run(policy: &Policy, query: Option<&str>, url: Option<&str>, endpoint: Option<&str>, offset: usize) -> Result { + let (title, lines) = match (url.map(str::trim).filter(|u| !u.is_empty()), query.map(str::trim).filter(|q| !q.is_empty())) { + (Some(url), _) => { + let page = policy.fetch(policy.check(url)?)?; + let mut lines = vec![about(&page)]; + lines.extend(readable(&page)?); + (format!("# {}", page.url), lines) + } + (None, Some(query)) => (format!("# search \"{query}\""), search_lines(policy, query, endpoint)?), + (None, None) => return Err("pass `query` to search, or `url` to fetch one https page".into()), + }; + let total = lines.len(); + let start = offset.min(total); + let end = (start + PAGE_LINES).min(total); + let mut out = format!("{title}\nrules: {}\n", policy.describe()); + if start > 0 || end < total { + let more = if end < total { format!(", pass offset={end} for the next page") } else { String::new() }; + out.push_str(&format!("(showing lines {}-{end} of {total}{more})\n", start + 1)); + } + out.push('\n'); + out.push_str(&lines[start..end].join("\n")); + out.push('\n'); + Ok(out) +} + +// the status line every fetched page opens with +fn about(page: &Fetched) -> String { + let ct = if page.content_type.is_empty() { "no content type" } else { &page.content_type }; + match &page.from { + Some(from) => format!("status {} - {ct} - redirected from {from}", page.status), + None => format!("status {} - {ct}", page.status), + } +} + +// every endpoint's results for one query, each under its own heading +fn search_lines(policy: &Policy, query: &str, only: Option<&str>) -> Result, String> { + if policy.endpoints.is_empty() { + return Err("no search endpoints are set: add one with `{query}` where the words go - the \ + `ccc.search.endpoints` setting in VS Code, or `ccc run --search-endpoint \ + 'https://wiki.example/search?q={query}'` - or pass `url` to fetch a page" + .into()); + } + let chosen: Vec<&String> = policy + .endpoints + .iter() + .filter(|e| only.is_none_or(|o| e.to_ascii_lowercase().contains(&o.to_ascii_lowercase()))) + .collect(); + if chosen.is_empty() { + return Err(format!("no search endpoint matches '{}' - the endpoints are: {}", only.unwrap_or_default(), policy.endpoints.join(", "))); + } + let mut lines = Vec::new(); + let mut answered = 0; + for template in chosen { + let target = template.replace("{query}", &encode(query)); + lines.push(format!("## {target}")); + match policy.check(&target).and_then(|u| policy.fetch(u)).and_then(|p| Ok((about(&p), readable(&p)?))) { + Ok((status, text)) => { + answered += 1; + lines.push(status); + let cut = text.len().saturating_sub(ENDPOINT_LINES); + lines.extend(text.into_iter().take(ENDPOINT_LINES)); + if cut > 0 { + lines.push(format!("({cut} more line(s) - fetch the page with `url` for all of it)")); + } + } + Err(e) => lines.push(e), + } + lines.push(String::new()); + } + if answered == 0 { + return Err(lines.into_iter().filter(|l| !l.is_empty() && !l.starts_with("## ")).collect::>().join("\n")); + } + Ok(lines) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn policy(mode: Mode, urls: &[&str]) -> Policy { + let urls: Vec = urls.iter().map(|u| u.to_string()).collect(); + Policy::new(mode, &urls, &[]).unwrap() + } + + #[test] + fn only_https_gets_through_whatever_the_mode() { + let open = policy(Mode::None, &[]); + assert!(open.check("https://example.com/a").is_ok()); + for bad in ["http://example.com", "ftp://example.com", "file:///etc/passwd", "example.com", "https://user:pw@example.com/"] { + let err = open.check(bad).unwrap_err(); + assert!(err.starts_with(REFUSED), "{bad}: {err}"); + } + } + + #[test] + fn allow_lets_through_only_what_an_entry_matches() { + let p = policy(Mode::Allow, &["https://docs.corp/**", "*.wiki.corp"]); + assert!(p.check("https://docs.corp/billing/retries?x=1").is_ok()); + assert!(p.check("https://team.wiki.corp/page").is_ok()); + assert!(p.check("https://DOCS.CORP/x").is_ok()); + let err = p.check("https://example.com/").unwrap_err(); + assert!(err.contains("not on the allow list"), "{err}"); + // a host entry never matches a lookalike that merely contains it + assert!(p.check("https://wiki.corp.evil.example/").is_err()); + } + + #[test] + fn block_refuses_what_an_entry_matches_and_names_it() { + let p = policy(Mode::Block, &["*.social.example", "https://example.com/private/**"]); + assert!(p.check("https://example.com/public").is_ok()); + let err = p.check("https://example.com/private/key").unwrap_err(); + assert!(err.contains("https://example.com/private/**"), "{err}"); + assert!(p.check("https://feed.social.example/").is_err()); + } + + #[test] + fn a_bad_entry_or_endpoint_is_an_error_up_front() { + assert!(Policy::new(Mode::Allow, &["docs.corp/path".into()], &[]).is_err()); + assert!(Policy::new(Mode::None, &[], &["https://wiki.corp/search".into()]).is_err()); + assert!(Policy::new(Mode::None, &[], &["http://wiki.corp/search?q={query}".into()]).is_err()); + assert!(Policy::new(Mode::None, &[], &["https://wiki.corp/search?q={query}".into()]).is_ok()); + assert!(Mode::parse("deny").is_err()); + } + + #[test] + fn a_redirect_resolves_against_the_page_it_came_from() { + let base = parse_url("https://docs.corp/a/b/page?x=1").unwrap(); + assert_eq!(resolve(&base, "/root"), "https://docs.corp/root"); + assert_eq!(resolve(&base, "next"), "https://docs.corp/a/b/next"); + assert_eq!(resolve(&base, "//cdn.corp/x"), "https://cdn.corp/x"); + assert_eq!(resolve(&base, "https://other.corp/"), "https://other.corp/"); + } + + #[test] + fn a_query_is_percent_encoded() { + assert_eq!(encode("retry policy & co"), "retry%20policy%20%26%20co"); + } + + #[test] + fn a_response_skips_the_proxy_and_continue_heads() { + let raw = b"HTTP/1.1 200 Connection established\r\n\r\nHTTP/1.1 100 Continue\r\n\r\nHTTP/2 301\r\nLocation: /next\r\nContent-Type: text/html\r\n\r\nbody"; + let (status, headers, body) = split_response(raw).unwrap(); + assert_eq!(status, 301); + assert!(headers.contains(&("location".to_string(), "/next".to_string()))); + assert_eq!(body, b"body"); + } + + #[test] + fn html_reads_as_text_with_its_links() { + let base = parse_url("https://wiki.corp/search?q=x").unwrap(); + let html = "Results & more\ + \ +

Retry policy

Use the retries doc first.

\ +
  • one
  • two A
top"; + let text = html_text(html, &base); + assert!(text.starts_with("# Results & more"), "{text}"); + assert!(text.contains("Retry policy"), "{text}"); + assert!(text.contains("Use [the retries doc](https://wiki.corp/doc/retries) first."), "{text}"); + assert!(text.contains("- one") && text.contains("- two A"), "{text}"); + assert!(!text.contains("var a") && !text.contains("hidden") && !text.contains("p{}"), "{text}"); + assert!(!text.contains("(#top)"), "{text}"); + } + + #[test] + fn nothing_to_do_and_no_endpoints_say_what_to_set() { + let p = policy(Mode::None, &[]); + assert!(run(&p, None, None, None, 0).unwrap_err().contains("pass `query`")); + assert!(run(&p, Some("retries"), None, None, 0).unwrap_err().contains("ccc.search.endpoints")); + } + + #[test] + fn a_refused_url_is_never_fetched() { + let p = policy(Mode::Allow, &["docs.corp"]); + let err = run(&p, None, Some("https://example.com/"), None, 0).unwrap_err(); + assert!(err.starts_with(REFUSED), "{err}"); + } +} diff --git a/src/serve.rs b/src/serve.rs index 507643e..8f19299 100644 --- a/src/serve.rs +++ b/src/serve.rs @@ -1,4 +1,4 @@ -//! `ccc run` local REST/MCP endpoints for AI agents, plus the insights UI. +//! `ccc run` local REST/MCP endpoints for AI agents, plus the architecture visualiser. //! //! On startup the whole project is parsed into an in-memory map (the same //! model `.ccc` is rendered from); every query answers from memory. A watcher @@ -25,10 +25,14 @@ pub struct ServeOptions { pub addr: String, pub port: u16, pub watch: Option, - // serve the human-facing `/insights` UI alongside the agent endpoints + // serve the human-facing visualiser at `/vis` alongside the agent endpoints pub html: bool, - // open the architecture visualiser on start; `/vis` is served with `html` + // open the architecture visualiser on start pub vis: bool, + // ask agents to annotate the cross-service calls they come across with `ccc:serves` / `ccc:calls` + pub auto: bool, + // what the `search` tool may reach + pub search: crate::search::Policy, } impl Default for ServeOptions { @@ -39,6 +43,8 @@ impl Default for ServeOptions { watch: Some(std::time::Duration::from_secs(2)), html: true, vis: false, + auto: false, + search: Default::default(), } } } @@ -56,6 +62,10 @@ struct MapState { facade: Option, watch_secs: Option, html: bool, + // `--auto`: the instructions ask agents to annotate cross-service calls + auto: bool, + // what `search` may reach - the rules `ccc run` was started with + search: crate::search::Policy, // a git work tree, so its steps keep their branch and texts for replays git: bool, origin: String, @@ -493,6 +503,8 @@ impl MapState { facade: cargo_package_name(root), watch_secs: None, html: true, + auto: false, + search: Default::default(), git: crate::replay::is_repo(root), // `--port 0` picks one at runtime origin: { @@ -3050,6 +3062,30 @@ fn mcp_tools() -> Value { ), // the write tools - the only way this project is changed + tool( + "search", + "USE INSTEAD OF WebFetch, WebSearch, curl or wget for anything on the web - internal docs and portals included. `query` runs a search against the endpoints the user configured (search pages with `{query}` where the words go) and returns each one's results as text, links written `[text](url)`; `url` fetches one https page as text - follow a result's link this way. https only. Every request, redirects included, passes the user's rules: in `allow` mode only listed urls are reachable, in `block` mode listed urls are refused, and with no mode any https url goes. A refusal names the rule that refused it - do not reach the url another way: `violations` reports that to the user. Long pages page: pass `offset` for the rest.", + json!({ + "query": {"type": "string", "description": "words to search the configured endpoints for"}, + "url": {"type": "string", "description": "one https page to fetch as text, instead of a query"}, + "endpoint": {"type": "string", "description": "only the endpoints whose template contains this, e.g. `wiki` (optional)"}, + "offset": {"type": "integer", "description": "lines to skip (default 0)"}, + "why": {"type": "string", "description": WHY_DOC}, + }), + &[], + ), + tool( + "violations", + "ANSWERS whether any agent on this project broke ccc's rules, read from the agents' own transcripts (Claude Code, subagents included): writing project files around ccc's edit tools (Edit/Write, `sed -i`, `>` or `tee`, `cp`/`mv`/`rm`), reading or searching the project with Read/Grep/Glob or the shell (`cat`, `grep`, `rg`, `find`, `sed -n`, `ls`) instead of ccc's tools, reaching the web outside `search` (WebFetch, WebSearch, `curl`, `wget`), and urls `search` refused. Each violation carries its time, session, tool, the command or input, the files or urls it touched, and the ccc tool that does the job instead. Narrow with `rule`, `session` or `since` (days); pages with `offset`.", + json!({ + "rule": {"type": "string", "enum": ["writes-outside-ccc", "shell-exploration", "web-outside-search", "blocked-url"], "description": "only this rule (optional)"}, + "session": {"type": "string", "description": "only this session id (optional)"}, + "since": {"type": "integer", "description": "only calls from the last N days (optional)"}, + "limit": {"type": "integer", "description": "violations per page (default 40)"}, + "offset": {"type": "integer", "description": "violations to skip (default 0)"}, + }), + &[], + ), tool( "edit_rename", "CHANGES A NAME EVERYWHERE IT IS THE SAME SYMBOL, in one call instead of a read and an edit per file. Pass the `result` handle a `find` or `references` answer ends with, and `to`. A `references` handle renames that exact name; a `find` handle swaps the matched substring inside each identifier and keeps its case shape (`hello_all` -> `hi_all`, `HelloWorld` -> `HiWorld`). Renaming only the definitions would strand their callers, so each name is followed through every file the map ties to it - definitions, calls, usages, imports, re-exports, and lines the map does not index such as a `ccc:skip` body - provided it has exactly one definition and the lookup was not qualified. Only identifiers change: never a comment, a string, or the module segment of a path. Every same-named identifier it did NOT change is listed with the reason, under a new handle that pulls them in when passed back; comments and strings that spell the name are listed too, and a new name that collides with an existing definition is flagged. Stages a changeset and returns its diff - nothing is written until `edit_apply` (or apply=true).", @@ -3137,14 +3173,6 @@ fn mcp_tools() -> Value { json!({"changeset": {"type": "string", "description": "the id `edit_apply` reported"}}), &["changeset"], ), - - // the one tool aimed at the person rather than the agent - tool( - "insights", - "CALL THIS WHEN THE USER ASKS TO SEE the analysis - show me the insights, open the dashboard, what does this codebase look like. Opens the human-facing insights UI (`/insights`) in their browser: the flame graph of the static call tree, hot paths, the service map, this branch's changes, test triggers and targets, lints and per-language totals - the same analysis pass the other tools read, laid out to be looked at rather than parsed. Returns the URL and the headline totals, so you can talk about the page while they read it. Served by default; only `ccc run --no-html` turns it off, and then the tool says so and the data is still at /insights.json.", - json!({}), - &[], - ), ]; // tell a client which tools write, so it can ask before those and not before the rest let tools: Vec = tools @@ -3166,7 +3194,23 @@ fn mcp_tools() -> Value { json!({ "tools": tools }) } -fn mcp_initialize(params: &Value) -> Value { +// what `--auto` adds to the instructions - agents mark the calls that leave this process, which no parser can follow +const CROSSINGS: &str = "\n\n4. CROSS-SERVICE CALLS - annotate every one you identify. A call that \ + leaves this process - an HTTP or gRPC request to another service, a message published to or \ + consumed from a queue or topic, a webhook, a CLI or FFI hop - is invisible to the map until a \ + comment names it. Whenever you read, write or change one, mark each end that is in this project \ + with a comment in the file's own syntax, staged through the edit tools like any other change: \ + `ccc:serves ` directly above the handler that answers it (attribute and \ + decorator lines may sit between), `ccc:calls ` inside the calling function, \ + beside the call. The key is the contract both ends share, spelled the same on each side - \ + `POST /v1/refund`, `billing.v1.Charge`, `orders.created` - and the transport is one of grpc, \ + rest, http, graphql, queue, event, webhook, websocket, ffi, cli, soap, sql or rpc. Call \ + `services` first: a crossing it already lists needs no comment, and an rpc a `.proto` schema \ + declares is linked without one. Never annotate a call that stays inside this process, and never \ + guess a key the code does not show - a `ccc:calls` nothing serves is reported as unanswered, \ + which is right for a peer outside this repository and wrong for a typo."; + +fn mcp_initialize(params: &Value, auto: bool) -> Value { let asked = params .get("protocolVersion") .and_then(|v| v.as_str()) @@ -3176,7 +3220,7 @@ fn mcp_initialize(params: &Value) -> Value { } else { MCP_LATEST }; - json!({ + let mut init = json!({ "protocolVersion": version, "capabilities": {"tools": {}, "resources": {}}, "serverInfo": { @@ -3250,12 +3294,12 @@ fn mcp_initialize(params: &Value) -> Value { resolve to, runtime ones first, `deps` what this branch did to that dependency tree - \ what entered it, what left, what moved version and which advisories that introduced. \ `security` syntax-level security findings in this \ - project's own code - secrets, disabled TLS, injection surfaces. Edits: \ + project's own code - secrets, disabled TLS, injection surfaces. `search` \ + is the only way to the web, internal docs included, and `violations` \ + reports where agents broke these rules. Edits: \ `edit_rename`, `edit_replace`, `edit_delete`, `edit_insert` and `edit_text` \ stage a changeset, `edit_apply` writes it, `edit_discard` drops it, \ - `edit_revert` undoes it. For a person rather \ - than an agent: `insights` opens the UI over that same analysis in their \ - browser - call it when they ask to *see* the code, not to read about it. \ + `edit_revert` undoes it. \ The analysis tools are \ heuristics over a syntax tree - no type inference, data flow or runtime \ profile - so each result carries its evidence and limits; read those before \ @@ -3263,7 +3307,12 @@ fn mcp_initialize(params: &Value) -> Value { `offset` for the rest.\n\n\ Results are markdown; the same data is JSON over HTTP (/index, /find, \ /references, /dependencies, /file, /notes, /insights.json, /deps.json).", - }) + }); + if auto { + let text = format!("{}{CROSSINGS}", init["instructions"].as_str().unwrap_or_default()); + init["instructions"] = Value::String(text); + } + init } fn mcp_md(text: &str, is_error: bool) -> Value { @@ -4718,7 +4767,7 @@ pub(crate) fn open_in_browser(url: &str) -> Result<(), String> { .map_err(|e| format!("{program}: {e}")) } -// what `insights` opens the page with - tests dispatch every tool, and must never launch a real browser doing it +// what `--vis` opens the visualiser with - tests must never launch a real browser fn launcher() -> fn(&str) -> Result<(), String> { if cfg!(test) { |_| Err("not opened - tests never launch a browser".into()) @@ -4727,48 +4776,6 @@ fn launcher() -> fn(&str) -> Result<(), String> { } } -// open the insights page -fn q_insights(map: &MapState, open: impl Fn(&str) -> Result<(), String>) -> Result { - let url = format!("{}/insights", map.origin); - if !map.html { - return Err(format!( - "the insights UI is disabled by `--no-html`; restart the server as `ccc run` to \ - serve it at {url} (the analysis itself is at {}/insights.json, and the \ - `changes`, `test_triggers`, `test_targets`, `lints`, `hot` and `services` \ - tools read the same pass)", - map.origin - )); - } - let opened = open(&url); - let a = map.analysis(None); - let t = &a["totals"]; - let mut out = format!("# insights\n{url}\n\n"); - out.push_str(match &opened { - Ok(()) => "opened in the user's browser - tell them to look at it.\n", - Err(_) => "could not open a browser here; give the user the URL above.\n", - }); - if let Err(e) = &opened { - out.push_str(&format!("reason: {e}\n")); - } - out.push_str(&format!( - "\n{} file(s), {} line(s), {} function(s), {} call edge(s), {} root(s)\n\ - generated {} in {} ms\n", - jnum(t, "files"), - jnum(t, "lines"), - jnum(t, "functions"), - jnum(t, "edges"), - jnum(t, "roots"), - jstr(&a, "generated"), - a["took_ns"].as_u64().unwrap_or(0) / 1_000_000, - )); - out.push_str( - "\ntabs: flame (static call tree), hot (call-graph shape), services, changes, \ - test triggers, test targets, lints, languages. The page reads /insights.json \ - live and has its own refresh button.\n", - ); - Ok(out) -} - // `browser` - the request carried an Origin header, which agents never send and a page always does fn mcp_tool_call(state: &RwLock, params: &Value, browser: bool) -> Result { let name = params @@ -4795,6 +4802,33 @@ fn mcp_tool_call(state: &RwLock, params: &Value, browser: bool) -> Res }; } + // the web and the transcripts are an agent's to reach, never a page's - and a slow portal must not hold the map + if name == "search" || name == "violations" { + if browser { + return Ok(mcp_md(&format!("error: `{name}` refuses requests from a browser page - only an agent's MCP client may use it"), true)); + } + let (policy, root) = { + let map = state.read().expect("map lock poisoned"); + (map.search.clone(), map.root.clone()) + }; + let num = |k: &str| args.get(k).and_then(|v| v.as_u64()); + return Ok(if name == "search" { + let offset = num("offset").unwrap_or(0) as usize; + match crate::search::run(&policy, arg("query").as_deref(), arg("url").as_deref(), arg("endpoint").as_deref(), offset) { + Ok(text) => mcp_md(&text, false), + Err(e) => mcp_md(&format!("error: {e}"), true), + } + } else { + let opts = crate::violations::Options { + since_days: num("since"), + session: arg("session"), + rules: arg("rule").as_deref().and_then(crate::violations::Rule::parse).into_iter().collect(), + }; + let page = Page::from(&args, 40); + mcp_md(&crate::violations::markdown(&crate::violations::check(&root, &opts), page.offset, page.limit), false) + }); + } + let map = state.read().expect("map lock poisoned"); // `file` returns the rendered markdown by default; `structured: true` swaps // it for spans plus the intra-file call graph. One representation per call - @@ -4962,7 +4996,6 @@ fn mcp_tool_call(state: &RwLock, params: &Value, browser: bool) -> Res Page::from(&args, 25), )) } - "insights" => q_insights(&map, launcher()), _ => return Err((-32602, format!("unknown tool '{name}'"))), }; // an agent's reads are its looks - a page reading for itself is not @@ -5155,7 +5188,7 @@ fn mcp_handle_from(state: &RwLock, msg: &Value, browser: bool) -> Opti } let params = msg.get("params").cloned().unwrap_or(json!({})); let result: Result = match method { - "initialize" => Ok(mcp_initialize(¶ms)), + "initialize" => Ok(mcp_initialize(¶ms, state.read().expect("map lock poisoned").auto)), "ping" => Ok(json!({})), "tools/list" => Ok(mcp_tools()), "tools/call" => mcp_tool_call(state, ¶ms, browser), @@ -5439,7 +5472,6 @@ const ENDPOINTS: &[&str] = &[ "GET /health", "GET /prompts[?base=] (which claude/copilot request produced each change)", "GET /insights.json[?base=] (the whole analysis payload)", - "GET /insights (human UI over the same data; off with --no-html)", "GET /vis.json | /vis/code?file= | /vis/flow?file=&line= (architecture levels)", "GET /vis/events?since= | /vis/event?seq= | /vis/asks (edit timeline: each step the edit tools took, and the ask behind it)", "GET /vis (architecture visualiser; off with --no-html, opened by --vis)", @@ -5564,21 +5596,8 @@ fn route_from(state: &RwLock, method: &str, url: &str, body: &[u8], or let map = state.read().expect("map lock poisoned"); ok((*map.analysis(get("base"))).clone()) } - // human-facing insights UI; on unless `ccc run --no-html` - ("GET", "/insights") => { - let map = state.read().expect("map lock poisoned"); - if !map.html { - return bad( - 404, - "insights UI is disabled by `--no-html`; restart with `ccc run` \ - (the data is at /insights.json either way)", - ); - } - html_ok(crate::html::render_insights_html(&map.root_label, None)) - } - // the architecture visualiser: the page goes with the insights UI and - // `--vis` opens it; the data behind it answers either way, the way - // /insights.json outlives --no-html + // the architecture visualiser: on unless `ccc run --no-html`, and + // `--vis` opens it; the data behind it answers either way ("GET", "/vis") => { let map = state.read().expect("map lock poisoned"); if !map.html { @@ -5788,6 +5807,8 @@ pub fn serve(root: &Path, opts: &ServeOptions) -> Result<()> { let mut map = state.write().expect("map lock poisoned"); map.watch_secs = opts.watch.map(|d| d.as_secs()); map.html = opts.html; + map.auto = opts.auto; + map.search = opts.search.clone(); // only the page's timeline reads the texts kept for changes by hand if !map.html { *map.seen.get_mut().unwrap_or_else(|p| p.into_inner()) = None; @@ -5804,12 +5825,6 @@ pub fn serve(root: &Path, opts: &ServeOptions) -> Result<()> { let pids: Vec = stopped.iter().map(u32::to_string).collect(); println!("stopped {} other ccc instance(s) serving this project (pid {})", stopped.len(), pids.join(", ")); } - // each branch pushed carries its replay with it - match crate::replay::install_hook(root) { - Ok(hook @ (crate::replay::Hook::Installed(_) | crate::replay::Hook::Foreign(_))) => println!("{}", hook.describe()), - Ok(_) => {} - Err(e) => eprintln!("ccc: the replay hook was not installed: {e:#}"), - } let bind = format!("{}:{}", opts.addr, opts.port); let server = tiny_http::Server::http(&bind) .map_err(|e| anyhow::anyhow!("binding {bind}: {e}"))?; @@ -5831,8 +5846,9 @@ pub fn serve(root: &Path, opts: &ServeOptions) -> Result<()> { } println!("listening on http://{addr} (MCP endpoint: http://{addr}/mcp)"); println!("endpoints: {}", ENDPOINTS.join(" | ")); - if opts.html { - println!("insights UI: http://{addr}/insights"); + println!("search: {}", opts.search.describe()); + if opts.html && !opts.vis { + println!("visualiser: http://{addr}/vis"); } if opts.vis { let url = format!("{}/vis", state.read().expect("map lock poisoned").origin); @@ -7073,16 +7089,35 @@ mod tests { #[test] fn mcp_initialize_negotiates_version() { - let known = mcp_initialize(&json!({"protocolVersion": "2024-11-05"})); + let known = mcp_initialize(&json!({"protocolVersion": "2024-11-05"}), false); assert_eq!(known["protocolVersion"], "2024-11-05"); - let unknown = mcp_initialize(&json!({"protocolVersion": "1999-01-01"})); + let unknown = mcp_initialize(&json!({"protocolVersion": "1999-01-01"}), false); assert_eq!(unknown["protocolVersion"], MCP_LATEST); assert_eq!(known["serverInfo"]["name"], "ccc"); } + // `ccc run --auto` asks every agent that connects to mark the calls leaving this process, on top of the usual instructions + #[test] + fn auto_asks_agents_to_annotate_cross_service_calls() { + let text = |auto| mcp_initialize(&json!({}), auto)["instructions"].as_str().unwrap().to_string(); + assert!(!text(false).contains("ccc:serves"), "only --auto asks"); + let asked = text(true); + assert!(asked.starts_with(&text(false)), "the usual instructions stay whole"); + for said in ["CROSS-SERVICE CALLS", "`ccc:serves `", "`ccc:calls `", "`services`"] { + assert!(asked.contains(said), "{said} missing: {asked}"); + } + + let (_dir, state) = scratch("auto", &[("src/a.rs", "pub fn a() {}\n")]); + let init = json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}); + let answer = |state: &RwLock| mcp_handle_from(state, &init, false).unwrap()["result"]["instructions"].clone(); + assert_eq!(answer(&state), text(false)); + state.write().unwrap().auto = true; + assert_eq!(answer(&state), asked); + } + #[test] fn the_instructions_cover_search_editing_and_every_tool() { - let text = mcp_initialize(&json!({}))["instructions"] + let text = mcp_initialize(&json!({}), false)["instructions"] .as_str() .unwrap() .to_string(); @@ -7146,6 +7181,8 @@ mod tests { "lints", "hot", "services", + "search", + "violations", "edit_rename", // the only writers "edit_replace", "edit_delete", @@ -7154,7 +7191,6 @@ mod tests { "edit_apply", "edit_discard", "edit_revert", - "insights", // for user only ] ); // a client can tell the writers from the readers without knowing the names @@ -7369,8 +7405,7 @@ mod tests { assert_eq!(pre.status, 204); } - // The page goes with the insights UI; the data behind it always answers, - // the way /insights.json outlives --no-html. + // The page goes with --no-html; the data behind it always answers. #[test] fn the_visualiser_page_goes_with_the_html_ui_and_its_data_does_not() { let state = RwLock::new(fixture()); @@ -7495,25 +7530,12 @@ mod tests { assert_ne!(route(&state, "GET", "/deps.json", b"").status, 404); } + // there is no insights page; the payload answers whether or not the + // visualiser is served #[test] - fn the_insights_ui_is_served_unless_it_is_refused() { + fn the_insights_payload_answers_with_the_html_ui_off() { let state = RwLock::new(fixture()); - // off only when asked: the error names the flag that turned it off and - // the JSON way in, which stays open either way state.write().unwrap().html = false; - let off = route(&state, "GET", "/insights", b""); - assert_eq!(off.status, 404); - assert!(json_of(&off)["error"].as_str().unwrap().contains("--no-html")); - assert_eq!(route(&state, "GET", "/insights.json", b"").status, 200); - - state.write().unwrap().html = true; - let page = route(&state, "GET", "/insights", b""); - assert_eq!(page.status, 200); - let ReplyBody::Html(body) = &page.body else { - panic!("the UI must be served as HTML"); - }; - assert!(body.contains("ccc insights")); - assert!(body.contains("/insights.json")); let data = route(&state, "GET", "/insights.json", b""); assert_eq!(data.status, 200); @@ -7522,63 +7544,32 @@ mod tests { // the fixture's cross-file call resolves, so the graph is not empty assert!(v["totals"]["functions"].as_u64().unwrap() >= 5); assert!(v["totals"]["edges"].as_u64().unwrap() >= 1); - // every tab the page renders has a section to render from + // every section the extension and the tools read is there for key in ["flame", "hot", "services", "lints", "languages"] { assert!(!v[key].is_null(), "insights payload missing {key}"); } - // findings never ship without the caveat the UI prints beside them + // findings never ship without their caveat assert!(v["lints"]["note"].as_str().unwrap().contains("heuristics")); - // the page reports how long the analysis took, beside when it ran + // the payload says how long the analysis took, beside when it ran assert!(v["took_ns"].as_u64().unwrap() > 0); assert!(!v["generated"].as_str().unwrap().is_empty()); // the flame view is grouped, so per-service trees have somewhere to go assert!(!v["flame"]["groups"].as_array().unwrap().is_empty()); } - // The `insights` tool is the one that acts on the user's machine, so the - // browser call is injected: these assertions must never launch anything. + // a page could aim `search` at an intranet and read the answer, and the + // transcripts hold every prompt - both are an agent's alone #[test] - fn the_insights_tool_hands_the_user_a_reachable_url() { - let mut map = fixture(); - map.origin = "http://127.0.0.1:7788".into(); - - // disabled UI: the error names the flag and the JSON way in, and no - // browser is opened for a page that would 404 - map.html = false; - let err = q_insights(&map, |_| panic!("must not open a browser with --no-html")) - .expect_err("the UI was turned off above"); - assert!(err.contains("--no-html"), "{err}"); - assert!(err.contains("http://127.0.0.1:7788/insights.json"), "{err}"); - - map.html = true; - let asked = Mutex::new(Vec::new()); - let out = q_insights(&map, |url| { - asked.lock().unwrap().push(url.to_string()); - Ok(()) - }) - .expect("the UI is enabled"); - assert_eq!( - asked.into_inner().unwrap(), - vec!["http://127.0.0.1:7788/insights".to_string()] - ); - assert!(out.contains("http://127.0.0.1:7788/insights"), "{out}"); - assert!(out.contains("opened in the user's browser"), "{out}"); - // the headline figures the agent talks about while they read - assert!(out.contains("function(s)"), "{out}"); - assert!(out.contains("call edge(s)"), "{out}"); - - // a headless box has nothing to open: still an answer, with the URL and - // the reason, not an error - let headless = q_insights(&map, |_| Err("xdg-open: not found".into())) - .expect("a missing browser is not a failed tool call"); - assert!(headless.contains("give the user the URL"), "{headless}"); - assert!(headless.contains("xdg-open: not found"), "{headless}"); - assert!(headless.contains("http://127.0.0.1:7788/insights"), "{headless}"); - - // dispatched the way an agent calls it, a test run still opens nothing + fn search_and_violations_refuse_a_browser_page() { let state = RwLock::new(fixture()); - let (err, out) = tool(&state, "insights", json!({})); - assert!(!err && out.contains("tests never launch a browser"), "{out}"); + for (name, args) in [("search", json!({"url": "https://example.com/"})), ("violations", json!({}))] { + let out = mcp_tool_call(&state, &json!({"name": name, "arguments": args}), true).unwrap(); + assert_eq!(out["isError"], true, "{name}"); + assert!(out["content"][0]["text"].as_str().unwrap().contains("browser page"), "{name}"); + } + // an agent asking for plain http is refused before anything is fetched + let out = mcp_tool_call(&state, &json!({"name": "search", "arguments": {"url": "http://example.com/"}}), false).unwrap(); + assert!(out["content"][0]["text"].as_str().unwrap().contains(crate::search::REFUSED)); } #[test] diff --git a/src/telemetry.rs b/src/telemetry.rs index d58a04d..af0ae86 100644 --- a/src/telemetry.rs +++ b/src/telemetry.rs @@ -402,16 +402,22 @@ fn positional(lang: Language) -> Option<(usize, usize)> { pub fn analyse(root: &Path, opts: &TelemetryOptions) -> TelemetryReport { let prefix = git_prefix(root); - let base = match collect(root, Side::Commit(opts.base_sha), &prefix) { - Ok(m) => m, - Err(e) => return TelemetryReport::empty(opts.base_sha, opts.head_sha, Some(e)), - }; let head_side = if opts.worktree { Side::Worktree } else { Side::Commit(opts.head_sha) }; - let head = match collect(root, head_side, &prefix) { + // each side is its own `git grep` over the whole tree, so both run at once + let (base, head) = std::thread::scope(|s| { + let head = s.spawn(|| collect(root, head_side, &prefix)); + let base = collect(root, Side::Commit(opts.base_sha), &prefix); + (base, head.join().unwrap_or_else(|e| std::panic::resume_unwind(e))) + }); + let base = match base { + Ok(m) => m, + Err(e) => return TelemetryReport::empty(opts.base_sha, opts.head_sha, Some(e)), + }; + let head = match head { Ok(m) => m, Err(e) => return TelemetryReport::empty(opts.base_sha, opts.head_sha, Some(e)), }; diff --git a/src/violations.rs b/src/violations.rs new file mode 100644 index 0000000..fd68959 --- /dev/null +++ b/src/violations.rs @@ -0,0 +1,636 @@ +//! Where an agent broke ccc's rules, read from its own transcripts. +//! +//! ccc asks agents to explore the project through its tools, to write it only +//! through its edit tools, and to reach the web only through `search`. Every +//! tool call in a transcript is checked against those rules, and each one that +//! broke one is reported with the call, the files or urls it touched, and the +//! ccc tool that does that job. +//! +//! Claude Code's transcripts only, a session's subagents with it: Copilot's +//! chat log keeps no record of its tool calls. + +use crate::inspect::{assignment, inside, parts, PATTERNED, READERS, WRAPPERS}; +use serde::Serialize; +use serde_json::Value; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Component, Path, PathBuf}; + +pub const SCHEMA: &str = "ccc-violations/1"; +// the most of a command or input one violation quotes +const QUOTE_CAP: usize = 240; +// the files or urls one violation names +const TARGET_CAP: usize = 12; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum Rule { + WritesOutsideCcc, + ShellExploration, + WebOutsideSearch, + BlockedUrl, +} + +impl Rule { + pub const ALL: [Rule; 4] = [Rule::WritesOutsideCcc, Rule::ShellExploration, Rule::WebOutsideSearch, Rule::BlockedUrl]; + + pub fn label(self) -> &'static str { + match self { + Rule::WritesOutsideCcc => "writes-outside-ccc", + Rule::ShellExploration => "shell-exploration", + Rule::WebOutsideSearch => "web-outside-search", + Rule::BlockedUrl => "blocked-url", + } + } + + pub fn parse(s: &str) -> Option { + Rule::ALL.into_iter().find(|r| r.label() == s.trim()) + } + + // what the rule asks of an agent + pub fn says(self) -> &'static str { + match self { + Rule::WritesOutsideCcc => "project files are written only through ccc's edit tools", + Rule::ShellExploration => "the project is read and searched only through ccc's tools", + Rule::WebOutsideSearch => "the web is reached only through ccc's `search`", + Rule::BlockedUrl => "a url the search rules refuse is not to be reached", + } + } + + // the ccc tool that does the job instead + pub fn instead(self) -> &'static str { + match self { + Rule::WritesOutsideCcc => "edit_text, or edit_rename / edit_replace / edit_delete / edit_insert on a find or references handle, then edit_apply", + Rule::ShellExploration => "index, find, references, file, dependencies or notes", + Rule::WebOutsideSearch => "search", + Rule::BlockedUrl => "nothing - the user's search rules refuse it", + } + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct Violation { + pub rule: Rule, + // when the call was made, rfc 3339 + pub at: String, + pub session: String, + // `claude`, or `claude subagent` for a call a subagent made + pub agent: String, + pub tool: String, + // the command line or the input, cut short + pub call: String, + // project-relative paths, or urls + pub targets: Vec, + pub transcript: String, + // the transcript line the call is on, 1-based + pub line: usize, +} + +#[derive(Debug, Serialize)] +pub struct Report { + pub schema: &'static str, + pub root: String, + pub transcripts: usize, + pub sessions: usize, + pub calls: usize, + pub counts: BTreeMap<&'static str, usize>, + pub violations: Vec, + pub note: &'static str, +} + +#[derive(Debug, Default, Clone)] +pub struct Options { + // only calls from the last N days + pub since_days: Option, + pub session: Option, + // only these rules; none means every rule + pub rules: Vec, +} + +const NOTE: &str = "Read from Claude Code's transcripts, subagents included - Copilot's chat log keeps no tool calls. \ + Shell lines are read as written: a path or url held in a variable cannot be checked, and loopback \ + requests (ccc's own server) are not web access."; + +// every transcript of this project, each tool call in it checked against the rules +pub fn check(root: &Path, opts: &Options) -> Report { + let files = crate::inspect::transcripts(root); + let cutoff_ms = opts.since_days.map(|d| chrono::Utc::now().timestamp_millis() - (d as i64) * 86_400_000); + let mut scan = Scan { root, opts, cutoff_ms, calls: 0, sessions: BTreeSet::new(), out: Vec::new() }; + for f in &files { + scan.file(f); + } + scan.out.sort_by(|a, b| (&a.at, &a.transcript, a.line).cmp(&(&b.at, &b.transcript, b.line))); + let mut counts: BTreeMap<&'static str, usize> = Rule::ALL.iter().map(|r| (r.label(), 0)).collect(); + for v in &scan.out { + *counts.entry(v.rule.label()).or_default() += 1; + } + Report { + schema: SCHEMA, + root: crate::changes::path_str(root), + transcripts: files.len(), + sessions: scan.sessions.len(), + calls: scan.calls, + counts, + violations: scan.out, + note: NOTE, + } +} + +struct Scan<'a> { + root: &'a Path, + opts: &'a Options, + cutoff_ms: Option, + calls: usize, + sessions: BTreeSet, + out: Vec, +} + +// where a call was made, kept until its answer is read +struct Pending { + at: String, + session: String, + tool: String, + call: String, + line: usize, +} + +impl Scan<'_> { + fn file(&mut self, path: &Path) { + let Ok(text) = std::fs::read_to_string(path) else { + return; + }; + let transcript = path.to_string_lossy().into_owned(); + let agent = if transcript.contains("/subagents/") { "claude subagent" } else { "claude" }; + // `search` calls waiting for their answers, by call id + let mut searches: BTreeMap = BTreeMap::new(); + for (n, line) in text.lines().enumerate() { + // a tool's answer can run to megabytes - only calls, and the answers to searches, are parsed + let call = line.contains("\"tool_use\"") && line.contains("\"type\":\"assistant\""); + let answer = !searches.is_empty() && line.contains("\"tool_result\"") && searches.keys().any(|id| line.contains(id.as_str())); + if !call && !answer { + continue; + } + let Ok(rec) = serde_json::from_str::(line) else { + continue; + }; + let at = rec["timestamp"].as_str().unwrap_or_default().to_string(); + let session = rec["sessionId"].as_str().unwrap_or_default().to_string(); + if !self.wanted(&at, &session) { + continue; + } + let cwd = rec["cwd"].as_str().map_or_else(|| self.root.to_path_buf(), PathBuf::from); + for b in rec["message"]["content"].as_array().into_iter().flatten() { + match b["type"].as_str() { + Some("tool_use") => { + let tool = b["name"].as_str().unwrap_or_default(); + let input = &b["input"]; + self.calls += 1; + self.sessions.insert(session.clone()); + let quoted = quote(tool, input); + if tool.starts_with("mcp__") && tool.ends_with("__search") { + if let Some(id) = b["id"].as_str() { + let pending = Pending { at: at.clone(), session: session.clone(), tool: tool.to_string(), call: quoted.clone(), line: n + 1 }; + searches.insert(id.to_string(), pending); + } + } + for (rule, targets) in classify(tool, input, &cwd, self.root) { + self.push(rule, &at, &session, agent, tool, "ed, targets, &transcript, n + 1); + } + } + Some("tool_result") => { + let Some(p) = b["tool_use_id"].as_str().and_then(|id| searches.remove(id)) else { + continue; + }; + let refused = refused_urls(&result_text(&b["content"])); + if !refused.is_empty() { + self.push(Rule::BlockedUrl, &p.at, &p.session, agent, &p.tool, &p.call, refused, &transcript, p.line); + } + } + _ => {} + } + } + } + } + + fn wanted(&self, at: &str, session: &str) -> bool { + if self.opts.session.as_deref().is_some_and(|s| s != session) { + return false; + } + let Some(cutoff) = self.cutoff_ms else { + return true; + }; + chrono::DateTime::parse_from_rfc3339(at).is_ok_and(|t| t.timestamp_millis() >= cutoff) + } + + #[allow(clippy::too_many_arguments)] + fn push(&mut self, rule: Rule, at: &str, session: &str, agent: &str, tool: &str, call: &str, mut targets: Vec, transcript: &str, line: usize) { + if !self.opts.rules.is_empty() && !self.opts.rules.contains(&rule) { + return; + } + targets.dedup(); + targets.truncate(TARGET_CAP); + self.out.push(Violation { + rule, + at: at.to_string(), + session: session.to_string(), + agent: agent.to_string(), + tool: tool.to_string(), + call: call.to_string(), + targets, + transcript: transcript.to_string(), + line, + }); + } +} + +// the part of a call worth quoting: a shell line, a path, a url, or the input itself +fn quote(tool: &str, input: &Value) -> String { + let text = ["command", "file_path", "notebook_path", "url", "query", "pattern", "path"] + .iter() + .find_map(|k| input[k].as_str().map(str::to_string)) + .unwrap_or_else(|| input.to_string()); + let text = text.replace('\n', " ⏎ "); + match text.char_indices().nth(QUOTE_CAP) { + Some((cut, _)) => format!("{}…", &text[..cut]), + None if text.is_empty() => tool.to_string(), + None => text, + } +} + +// a tool answer's text, whether it came as a string or as content blocks +fn result_text(content: &Value) -> String { + match content { + Value::String(s) => s.clone(), + Value::Array(blocks) => blocks.iter().filter_map(|b| b["text"].as_str()).collect::>().join("\n"), + _ => String::new(), + } +} + +// the urls a `search` answer says it refused +fn refused_urls(text: &str) -> Vec { + text.match_indices(crate::search::REFUSED) + .filter_map(|(at, m)| text[at + m.len()..].split_whitespace().next()) + .map(str::to_string) + .collect() +} + +// the rules one tool call broke, each with what it touched +fn classify(tool: &str, input: &Value, cwd: &Path, root: &Path) -> Vec<(Rule, Vec)> { + let path = |k: &str| input[k].as_str().and_then(|p| project_path(root, &cwd.join(p))).into_iter().collect::>(); + let one = |rule: Rule, targets: Vec| if targets.is_empty() { Vec::new() } else { vec![(rule, targets)] }; + match tool { + "Edit" | "Write" | "MultiEdit" => one(Rule::WritesOutsideCcc, path("file_path")), + "NotebookEdit" => one(Rule::WritesOutsideCcc, path("notebook_path")), + "Read" => one(Rule::ShellExploration, path("file_path")), + "NotebookRead" => one(Rule::ShellExploration, path("notebook_path")), + // a search with no path searches where the agent stands + "Grep" | "Glob" | "LS" => { + let at = input["path"].as_str().map_or_else(|| cwd.to_path_buf(), |p| cwd.join(p)); + one(Rule::ShellExploration, project_path(root, &at).into_iter().collect()) + } + "WebFetch" => vec![(Rule::WebOutsideSearch, input["url"].as_str().map(str::to_string).into_iter().collect())], + "WebSearch" => vec![(Rule::WebOutsideSearch, input["query"].as_str().map(|q| format!("query: {q}")).into_iter().collect())], + "Bash" => input["command"].as_str().map(|c| shell(c, cwd, root)).unwrap_or_default(), + t if t.starts_with("mcp__") && !t.starts_with("mcp__ccc__") && web_tool(t) => { + vec![(Rule::WebOutsideSearch, input["url"].as_str().map(str::to_string).into_iter().collect())] + } + _ => Vec::new(), + } +} + +// another mcp server's tool that reaches the web +fn web_tool(name: &str) -> bool { + let n = name.to_ascii_lowercase(); + ["fetch", "browse", "browser", "scrape", "crawl", "web", "http"].iter().any(|w| n.contains(w)) +} + +// a path inside the project, relative to it - `.` for the project itself, never one under `.git`, which is git's own +fn project_path(root: &Path, path: &Path) -> Option { + let mut abs = PathBuf::new(); + for c in path.components() { + match c { + Component::ParentDir => { + abs.pop(); + } + Component::CurDir => {} + c => abs.push(c), + } + } + if abs == root { + return Some(".".to_string()); + } + inside(root, &abs).filter(|p| p != ".git" && !p.starts_with(".git/")) +} + +// commands that write the files they name +const WRITERS: &[&str] = &["tee", "rm", "rmdir", "unlink", "touch", "truncate", "shred"]; +// commands that list or search a tree, the current directory when they name none +const WALKERS: &[&str] = &["find", "fd", "fdfind", "ls", "tree", "rg", "ag", "du"]; +// commands that reach the network +const FETCHERS: &[&str] = &["curl", "wget", "http", "https", "xh", "httpie", "aria2c", "lynx", "w3m", "links"]; +// git subcommands that write the working tree +const GIT_WRITES: &[&str] = &["apply", "am", "restore", "mv", "rm"]; + +// the rules one shell line broke - each simple command in it read on its own +fn shell(line: &str, cwd: &Path, root: &Path) -> Vec<(Rule, Vec)> { + let mut cwd = cwd.to_path_buf(); + let mut writes = Vec::new(); + let mut reads = Vec::new(); + let mut web = Vec::new(); + for part in parts(line) { + // a path held in a variable or a substitution names nothing that can be checked + let at = |p: &str, cwd: &Path| { + let full = cwd.join(p); + (!full.to_string_lossy().contains(['$', '`'])).then(|| project_path(root, &full)).flatten() + }; + writes.extend(part.written.iter().filter_map(|w| at(w, &cwd))); + let mut words = part.words.iter().map(String::as_str).skip_while(|w| assignment(w) || WRAPPERS.contains(w)); + let Some(cmd) = words.next() else { + continue; + }; + let args: Vec<&str> = words.collect(); + let name = cmd.rsplit('/').next().unwrap_or(cmd); + let plain: Vec<&str> = args.iter().copied().filter(|a| !a.starts_with('-')).collect(); + match name { + "cd" => { + if let Some(dir) = args.first() { + cwd = cwd.join(dir); + } + } + "sed" if args.iter().any(|a| a.starts_with("-i") || a.starts_with("--in-place")) => { + let given = args.iter().any(|a| matches!(*a, "-e" | "-f" | "--expression" | "--file")); + writes.extend(plain.iter().skip(usize::from(!given)).filter_map(|f| at(f, &cwd))); + } + "perl" if args.iter().any(|a| perl_in_place(a)) => { + let script = args.iter().position(|a| *a == "-e" || *a == "-E").map(|i| i + 1); + let files = args.iter().enumerate().filter(|(i, a)| !a.starts_with('-') && Some(*i) != script); + writes.extend(files.filter_map(|(_, f)| at(f, &cwd))); + } + "cp" | "install" | "ln" => writes.extend(plain.last().and_then(|f| at(f, &cwd))), + "mv" => writes.extend(plain.iter().filter_map(|f| at(f, &cwd))), + "dd" => writes.extend(args.iter().filter_map(|a| a.strip_prefix("of=")).filter_map(|f| at(f, &cwd))), + "patch" => writes.extend(at(".", &cwd)), + "git" => { + let sub = plain.first().copied().unwrap_or_default(); + let reverts = sub == "checkout" && args.contains(&"--"); + if GIT_WRITES.contains(&sub) || reverts { + let named: Vec = plain.iter().skip(1).filter_map(|f| at(f, &cwd)).collect(); + writes.extend(if named.is_empty() { at(".", &cwd).into_iter().collect() } else { named }); + } + } + n if WRITERS.contains(&n) => writes.extend(plain.iter().filter_map(|f| at(f, &cwd))), + n if FETCHERS.contains(&n) => web.extend(args.iter().filter(|a| !a.starts_with('-')).filter(|a| remote_url(a)).map(|a| a.to_string())), + n if READERS.contains(&n) || WALKERS.contains(&n) => { + // a search's first word is its pattern, unless a flag gave it + let given = args.iter().any(|a| matches!(*a, "-e" | "-f" | "--regexp" | "--expression" | "--file")); + let pattern = usize::from(PATTERNED.contains(&n) && !given); + let named: Vec<&str> = match n { + // find's paths come before its expression + "find" => args.iter().copied().take_while(|a| !a.starts_with('-') && !matches!(*a, "(" | "!")).collect(), + _ => plain.iter().copied().filter(|a| !a.bytes().all(|b| b.is_ascii_digit())).skip(pattern).collect(), + }; + let files: Vec = named.iter().copied().chain(part.fed.iter().map(String::as_str)).filter_map(|f| at(f, &cwd)).collect(); + if !files.is_empty() { + reads.extend(files); + } else if named.is_empty() && part.fed.is_empty() && !part.piped && walks(n, &args) { + // nothing named and nothing piped in: it walks where it stands + reads.extend(at(".", &cwd)); + } + } + _ => {} + } + } + let mut out = Vec::new(); + for (rule, targets) in [(Rule::WritesOutsideCcc, writes), (Rule::ShellExploration, reads), (Rule::WebOutsideSearch, web)] { + if !targets.is_empty() { + out.push((rule, targets)); + } + } + out +} + +// a command that searches the tree it stands in when it names no path +fn walks(name: &str, args: &[&str]) -> bool { + WALKERS.contains(&name) + || (matches!(name, "grep" | "egrep" | "fgrep") && args.iter().any(|a| *a == "--recursive" || (a.starts_with('-') && !a.starts_with("--") && (a.contains('r') || a.contains('R'))))) +} + +// `-i`, `-pi`, `-i.bak` - perl editing in place; letters after one that takes an argument are that argument +fn perl_in_place(arg: &str) -> bool { + let Some(flags) = arg.strip_prefix('-').filter(|f| !f.starts_with('-')) else { + return false; + }; + for c in flags.chars() { + match c { + 'i' => return true, + 'e' | 'E' | 'M' | 'm' | 'I' | 'x' | 'C' | 'd' | 'D' => return false, + _ => {} + } + } + false +} + +// a literal url, or bare host, that is not this machine +fn remote_url(word: &str) -> bool { + let rest = match word.split_once("://") { + Some((_, rest)) => rest, + None if word.starts_with("www.") => word, + None => { + // `example.com/path`, which curl reads as a url too - a bare `payer.jar` is a file + let Some((host, _)) = word.split_once('/') else { + return false; + }; + let host = host.split(':').next().unwrap_or_default(); + let labels: Vec<&str> = host.split('.').collect(); + let named = labels.len() > 1 && labels.iter().all(|l| !l.is_empty() && l.chars().all(|c| c.is_ascii_alphanumeric() || c == '-')); + if !named || !labels.last().is_some_and(|tld| tld.len() > 1 && tld.chars().all(|c| c.is_ascii_alphabetic())) { + return false; + } + word + } + }; + let authority = rest.split(['/', '?', '#']).next().unwrap_or_default(); + let host = authority.rsplit('@').next().unwrap_or_default(); + let host = if host.starts_with('[') { host.split(']').next().unwrap_or_default().trim_start_matches('[') } else { host.split(':').next().unwrap_or_default() }; + let host = host.to_ascii_lowercase(); + !(host.is_empty() || host == "localhost" || host.ends_with(".localhost") || host.starts_with("127.") || host == "::1" || host == "0.0.0.0" || host.contains('$')) +} + +// the report as a person reads it on a terminal +pub fn text(r: &Report) -> String { + let mut out = summary(r); + for v in &r.violations { + out.push_str(&format!("{} {} [{}] {} `{}`", short_at(&v.at), short_session(&v.session), v.rule.label(), v.tool, v.call)); + if !v.targets.is_empty() { + out.push_str(&format!(" -> {}", v.targets.join(", "))); + } + out.push_str(&format!("\n {}:{}\n", v.transcript, v.line)); + } + out +} + +// the report for an agent: grouped by rule, each with what to use instead, paged +pub fn markdown(r: &Report, offset: usize, limit: usize) -> String { + let mut out = format!("# violations\n{}", summary(r)); + let total = r.violations.len(); + let start = offset.min(total); + let end = (start + limit.max(1)).min(total); + if start > 0 || end < total { + let more = if end < total { format!(", pass offset={end} for the next page") } else { String::new() }; + out.push_str(&format!("(showing {}-{end} of {total}{more})\n", start + 1)); + } + let mut rule = None; + for v in &r.violations[start..end] { + if rule != Some(v.rule) { + rule = Some(v.rule); + out.push_str(&format!("\n## {} - {}; use {} instead\n", v.rule.label(), v.rule.says(), v.rule.instead())); + } + out.push_str(&format!("- {} {} {} `{}`", short_at(&v.at), short_session(&v.session), v.tool, v.call)); + if !v.targets.is_empty() { + out.push_str(&format!(" -> {}", v.targets.join(", "))); + } + out.push('\n'); + } + out.push_str(&format!("\n{}\n", r.note)); + out +} + +fn summary(r: &Report) -> String { + let counts: Vec = r.counts.iter().map(|(k, n)| format!("{k} {n}")).collect(); + format!( + "{} violation(s) in {} tool call(s) across {} session(s), {} transcript(s)\n{}\n", + r.violations.len(), + r.calls, + r.sessions, + r.transcripts, + counts.join(" | ") + ) +} + +fn short_at(at: &str) -> String { + chrono::DateTime::parse_from_rfc3339(at).map_or_else(|_| at.to_string(), |t| t.format("%Y-%m-%d %H:%M").to_string()) +} + +fn short_session(s: &str) -> &str { + s.get(..8).unwrap_or(s) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn rules(line: &str) -> Vec<(Rule, Vec)> { + shell(line, Path::new("/p"), Path::new("/p")) + } + + fn rule_of(line: &str) -> Vec { + rules(line).into_iter().map(|(r, _)| r).collect() + } + + #[test] + fn a_write_around_the_edit_tools_is_caught_however_it_is_spelled() { + for line in [ + "sed -i 's/a/b/' src/x.rs", + "perl -pi -e 's/a/b/' src/x.rs", + "echo hi > src/x.rs", + "cat /tmp/a >> docs/notes.md", + "printf x | tee src/x.rs", + "cp /tmp/x src/x.rs", + "mv src/a.rs src/b.rs", + "rm -f src/x.rs", + "git apply /tmp/fix.patch", + "git checkout -- src/x.rs", + ] { + assert_eq!(rule_of(line), vec![Rule::WritesOutsideCcc], "{line}"); + } + assert_eq!(rules("sed -i 's/a/b/' src/x.rs")[0].1, vec!["src/x.rs"]); + } + + #[test] + fn writes_outside_the_project_or_to_a_stream_are_not_violations() { + for line in ["echo hi > /tmp/x", "cargo build 2>&1 | tail", "make 2>/dev/null", "cp src/x.rs /tmp/x", "git commit -m 'x'", "echo x > $S/sid", "cd $S && rm out.log"] { + assert!(!rule_of(line).contains(&Rule::WritesOutsideCcc), "{line}"); + } + } + + #[test] + fn exploring_with_the_shell_is_caught_but_filtering_a_pipe_is_not() { + for line in ["cat src/main.rs", "sed -n '1,40p' src/main.rs", "grep -rn insights src", "grep -rn insights", "rg foo", "find . -name '*.rs'", "ls docs", "head -20 README.md"] { + assert_eq!(rule_of(line), vec![Rule::ShellExploration], "{line}"); + } + for line in ["cargo test | grep FAILED", "git log | head", "ls /tmp", "cat /etc/hosts", "echo x | wc -l"] { + assert!(rule_of(line).is_empty(), "{line}"); + } + } + + #[test] + fn the_web_is_flagged_but_loopback_is_ccc_itself() { + assert_eq!(rule_of("curl -s https://example.com/x"), vec![Rule::WebOutsideSearch]); + assert_eq!(rule_of("wget docs.example.org/page"), vec![Rule::WebOutsideSearch]); + for line in ["curl -s http://127.0.0.1:6767/mcp", "curl localhost:3000", "curl -s $BASE/mcp", "curl -s -X POST \"$URL\"", "curl -sL -o payer.jar $JAR", "curl -D hdr.txt $U"] { + assert!(rule_of(line).is_empty(), "{line}"); + } + } + + #[test] + fn a_cd_moves_where_relative_paths_land() { + assert!(rule_of("cd /tmp && cat notes.txt").is_empty()); + assert_eq!(rules("cd src && cat main.rs")[0].1, vec!["src/main.rs"]); + } + + #[test] + fn agent_tools_are_classified_by_what_they_touch() { + let (cwd, root) = (Path::new("/p"), Path::new("/p")); + let c = |tool: &str, input: Value| classify(tool, &input, cwd, root).into_iter().map(|(r, _)| r).collect::>(); + assert_eq!(c("Edit", json!({"file_path": "/p/src/x.rs"})), vec![Rule::WritesOutsideCcc]); + assert!(c("Write", json!({"file_path": "/tmp/scratch.md"})).is_empty()); + assert_eq!(c("Read", json!({"file_path": "/p/README.md"})), vec![Rule::ShellExploration]); + assert_eq!(c("Grep", json!({"pattern": "x"})), vec![Rule::ShellExploration]); + assert_eq!(c("WebFetch", json!({"url": "https://example.com"})), vec![Rule::WebOutsideSearch]); + assert_eq!(c("mcp__fetch__fetch", json!({"url": "https://example.com"})), vec![Rule::WebOutsideSearch]); + assert!(c("mcp__ccc__find", json!({"query": "x"})).is_empty()); + assert!(c("mcp__ccc__search", json!({"url": "https://example.com"})).is_empty()); + } + + #[test] + fn a_transcript_yields_its_violations_and_its_refused_searches() { + let dir = std::env::temp_dir().join(format!("ccc-violations-{}", std::process::id())); + let root = dir.join("proj"); + std::fs::create_dir_all(&root).unwrap(); + let file = dir.join("t.jsonl"); + let call = |id: &str, name: &str, input: Value| { + json!({"type": "assistant", "timestamp": "2026-10-10T12:00:00Z", "sessionId": "s1", "cwd": root, + "message": {"content": [{"type": "tool_use", "id": id, "name": name, "input": input}]}}) + .to_string() + }; + let answer = |id: &str, text: &str| { + json!({"type": "user", "sessionId": "s1", "message": {"content": [{"type": "tool_result", "tool_use_id": id, "content": text}]}}).to_string() + }; + let lines = [ + call("a", "Bash", json!({"command": "sed -i 's/x/y/' src/lib.rs"})), + call("b", "mcp__ccc__search", json!({"url": "https://blocked.example/"})), + answer("b", &format!("error: {} https://blocked.example/ - not on the allow list", crate::search::REFUSED)), + call("c", "mcp__ccc__find", json!({"query": "x"})), + ]; + std::fs::write(&file, lines.join("\n") + "\n").unwrap(); + let mut scan = Scan { root: &root, opts: &Options::default(), cutoff_ms: None, calls: 0, sessions: BTreeSet::new(), out: Vec::new() }; + scan.file(&file); + let found: Vec<(Rule, Vec)> = scan.out.iter().map(|v| (v.rule, v.targets.clone())).collect(); + assert_eq!( + found, + vec![ + (Rule::WritesOutsideCcc, vec!["src/lib.rs".to_string()]), + (Rule::BlockedUrl, vec!["https://blocked.example/".to_string()]), + ] + ); + assert_eq!(scan.calls, 3); + assert_eq!(scan.out[1].line, 2, "a refusal points at the call that asked"); + + // a rule filter and a time window narrow it + let only = Options { rules: vec![Rule::BlockedUrl], ..Default::default() }; + let mut scan = Scan { root: &root, opts: &only, cutoff_ms: Some(i64::MAX), calls: 0, sessions: BTreeSet::new(), out: Vec::new() }; + scan.file(&file); + assert!(scan.out.is_empty()); + std::fs::remove_dir_all(&dir).ok(); + } +} From d382202ee2b2faf2789ad7eb660292ab500d33ca Mon Sep 17 00:00:00 2001 From: colwill <62014134+colwill@users.noreply.github.com> Date: Sat, 10 Oct 2026 21:20:47 +0100 Subject: [PATCH 2/2] feat: observations and violations --- AGENTS.md | 3 + docs/Advanced.md | 5 +- docs/CLI.md | 46 ++- extensions/vscode/src/hover.ts | 3 +- extensions/vscode/src/types.ts | 2 +- src/changes.rs | 14 +- src/coverage.rs | 31 +- src/externals.rs | 7 +- src/insights.rs | 184 ++++++++- src/lib.rs | 1 + src/main.rs | 4 + src/observed.rs | 717 +++++++++++++++++++++++++++++++++ src/serve.rs | 178 +++++--- src/violations.rs | 25 +- 14 files changed, 1145 insertions(+), 75 deletions(-) create mode 100644 src/observed.rs diff --git a/AGENTS.md b/AGENTS.md index 09fc0e6..e077424 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,6 +14,9 @@ to pick the new one up. - IF `ccc` tool calls are unable to find a term your are searching for; stop the session and respond with `CCC: unable to find in ccc using calls: [calls]` - Every interaction: use `ccc` tool calls to gather information about the source of this project. - All thinking, navigation, and questions about the codebase go through the MCP server tools: (index, find, references, dependencies, vulnerabilities, deps, security, file, notes, changes, prompts, pr_summary, test_triggers, test_targets, lints, hot, services, search, violations, refresh) +- The web, internal docs and portals included, goes through `search`: `query` to search the configured endpoints, + `url` to fetch one https page. Never WebFetch, WebSearch, another MCP server's fetch or browser tool, or + `curl`/`wget` - ccc can't track those, and `violations` reports each one. If `search` refuses a URL, leave it unreached. - ccc is the only writer in this project. Every change goes through its edit tools, never through your own Edit/Write/MultiEdit/NotebookEdit tools, `sed -i`, or shell redirection: - a symbol: take the handle a `find` / `references` answer ends with and pass it to `edit_rename`, diff --git a/docs/Advanced.md b/docs/Advanced.md index e8730a1..11c2da6 100644 --- a/docs/Advanced.md +++ b/docs/Advanced.md @@ -133,7 +133,7 @@ The extension's own page, [`extensions/vscode/README.md`](../extensions/vscode/R **How the hints are worked out.** The analysis is static. Knowing what it did and didn't establish is the difference between a useful hint and a misleading one. -- **Coverage comes from the call graph, not from running anything.** A test is linked to a change because the graph connects them. A test that reaches code through dynamic dispatch, a table of function pointers or a framework is invisible, so `✗ no test` means "nothing reaches it", not "nothing runs it". +- **Coverage comes from the call graph, unless a recorded run says more.** A test is linked to a change because the graph connects them. A test that reaches code through dynamic dispatch, a table of function pointers or a framework is invisible to the graph, so without records `✗ no test` means "nothing reaches it", not "nothing runs it". [Observed coverage](CLI.md#observed-coverage) records close that gap: a function a recorded fixture ran counts as tested, and its hover says the run saw it. - **A call is tied to a definition only with evidence:** the receiver's declared type, the same file, the same package, an import, or a qualifier naming the defining file. Where that evidence fits more than one definition, nothing is claimed, and a Rust test can never mark a TypeScript function covered. In untyped languages, a bare call to a name defined exactly once still counts, and its hover says it was matched on the name alone. `ccc.hints.minEvidence: evidence` (the default) hides those. - **Cross-service hints depend on `.ccc/map.json`.** With a `services` block, a hint sits on the exact call line. Without one, ccc groups by top-level directory and labels the result a module. In a flat project, `ccc.hints.crossServiceMode: auto` hides these hints rather than calling every import a service call. A declared edge with no call site is normal: that's what an HTTP, RPC or queue dependency looks like to static analysis. - **"Hot" means central to the call graph, not measured.** Only the top 25 of each ranking are marked, so a function without a flame isn't necessarily cold. @@ -181,7 +181,8 @@ This repo has a Code Change Capture (ccc) map, served over MCP. Use it as the en - no bash, grep or sed usage for exploring the project - Every interaction: use `ccc` tool calls to gather information about the source of this project. - All thinking, navigation, and questions about the codebase go through the MCP server tools: (index, find, references, dependencies, vulnerabilities, deps, security, file, notes, changes, prompts, pr_summary, test_triggers, test_targets, lints, hot, services, search, violations, refresh) -- ccc is the only writer: make every change through its edit tools - `edit_rename`, `edit_replace`, `edit_delete`, `edit_insert` or `edit_text`, then `edit_apply` - never your own Edit/Write tools, `sed -i` or shell redirection. +- The web, internal docs and portals included, goes through `search`: `query` to search the configured endpoints, `url` to fetch one https page. Never WebFetch, WebSearch, another MCP server's fetch or browser tool, or `curl`/`wget` - ccc can't track those, and `violations` reports each one. If `search` refuses a URL, leave it unreached. +- ccc is the only writer: make every change through its edit tools - `edit_rename`, `edit_replace`, `edit_delete`, `edit_insert` or `edit_text`, then `edit_apply` - never your own Edit/Write tools, `sed -i` or shell redirection. `edit_discard` drops a staged changeset and `edit_revert` undoes an applied one. - `edit_apply` rescans the map before it answers; call `refresh` only after a change made outside ccc. ``` diff --git a/docs/CLI.md b/docs/CLI.md index ad3e214..c2170fa 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -42,6 +42,7 @@ Then tell your agent to use the `ccc` MCP tools. [Advanced.md](Advanced.md#agent |---|---| | `--ignore-skip` | Treat every `ccc:skip` comment as an ordinary comment, so the code it marks is mapped, searched, measured and edited like the rest. | | `--no-parse-cache` | Parse every file again instead of reusing the parse kept from an earlier run. | +| `--observed SRC` | Also read [observed coverage](#observed-coverage) records from `SRC`: a file, a directory or a URL. Repeatable. | | `-h`, `--help` | Show the command's help. | | `-V`, `--version` | Print the version (top level only: `ccc --version`). | @@ -116,7 +117,7 @@ Reports what the branch changed at function level and what that means for servic It: - diffs the branch against its base and maps each changed hunk to the function it falls in -- for each changed function, lists the tests that call it directly and the other services that call it +- for each changed function, lists the tests that call it directly, the fixtures a [recorded run](#observed-coverage) saw execute it, and the other services that call it - lists `services_to_test`: the changed services and every service that calls them, directly or through others, using `.ccc/map.json`, gRPC schemas and matched `ccc:calls`/`ccc:serves` comments - lists changed functions no test calls, calls it couldn't resolve, and changed files that belong to no service - always includes the dependency delta and newly added secrets @@ -148,6 +149,7 @@ Runs the whole analysis once and prints it as JSON. The output covers: - the shape of the call graph: hot paths, the deepest call chains, call cycles - `test_triggers`: the tests the branch's changes put at risk, found through the call graph, with a runnable command per language under `test_triggers.commands` +- `test_triggers.observed`: when the project has [observed coverage](#observed-coverage) records, the fixtures a recorded run saw execute the changed functions, as a safe set and a cheapest one - `test_targets`: where a missing test would cost the most, and which kind to write - lints @@ -161,6 +163,44 @@ ccc insights [PATH] [--base REF] |---|---|---| | `--base REF` | same as `ccc changes` | Ref the test-trigger diff runs against. | +### Observed coverage + +The call graph can't see a test that reaches code through another service, a framework or dynamic dispatch. A recorded test run can. Give ccc the records of a run and `changes`, `insights`, the `test_triggers` tool and the VS Code hints use them beside the call graph. A project without records gets the same output as before. + +Records are read from all of these: + +- `observed` in `.ccc/map.json`: a list of files, directories or URLs. A private URL takes its token from an environment variable: `{"from": "https://ci.example.com/artifacts/observed.jsonl", "auth": "env:CI_TOKEN"}`. +- `--observed SRC`, repeatable, on any command. +- `.ccc/observed/`, when neither of the above names anything. Keep it out of git. + +A directory is read for its `*.jsonl` and `*.json` files. A URL is fetched at most once every ten minutes. + +Each line of a record file is one JSON object, `ccc-observed/1`: one fixture, as one service saw it. Spread over lines here for reading: + +```json +{"schema": "ccc-observed/1", + "fixture": {"name": "test_partial_refund", "kind": "contract", "file": "tests/api/test_refunds.py", "language": "python"}, + "run": "nightly-4812", + "service": {"name": "billing", "commit": "3f2a9c1e"}, + "duration_ms": 1840, + "files": [{"path": "internal/billing/fee.go", "functions": [{"name": "(*Server).ApplyFee", "line": 88}]}]} +``` + +- `fixture.name` is what the test runner selects it by. `kind` is `unit`, `contract`, `interop`, `integration` or `smoke`. +- `files[].path` is relative to the repository root. An absolute path is matched by its tail, so a CI checkout's paths work. A relative path the map doesn't hold belongs to another repository, and is counted as unmatched. +- A function is matched by name, then by owner, then by the nearest line, so a recording keeps matching after the code moves. A probe that sees lines rather than functions can send `"lines": [[start, end], ...]` instead. Those are used only while the file is unchanged since `service.commit`. + +With records: + +- A function a fixture ran counts as tested. The fixture is listed first among the tests that cover it, with evidence `observed`. +- `test_triggers.observed` lists the fixtures that ran the branch's changed functions, two ways, each with its own run commands: + - `safe`, the default: every fixture that ran a changed function. + - `minimal`: the cheapest fixtures that still run each changed function once. A fixture costs its recorded duration, weighted by kind: unit 1, contract 2, interop 3, integration 4, smoke 5. The set is the cheapest possible unless `exact` is `false`. +- `test_triggers.observed.unobserved` lists the changed functions no fixture ran. The static `run` list is all there is for those. +- The static `run` list stays as it was. A test written since the recording exists only in the source. + +Agents get the same from the `test_triggers` tool. `select: "minimal"` asks for the cheaper set. + ## ccc deps Reports what the branch did to the dependency tree: which packages entered, left or changed version, and any advisory that came with them. @@ -255,8 +295,8 @@ Reads the agents' transcripts for this project (Claude Code, subagents included) | Rule | What it catches | Use instead | |---|---|---| | `writes-outside-ccc` | Edit/Write tools, `sed -i`, `perl -i`, `>`/`>>`, `tee`, `cp`/`mv`/`rm` or `git apply` on project files | ccc's `edit_*` tools | -| `shell-exploration` | Read/Grep/Glob, or `cat`, `grep`, `rg`, `find`, `sed -n`, `head`, `ls` on the project | `index`, `find`, `references`, `file` | -| `web-outside-search` | WebFetch, WebSearch, or `curl`/`wget` to anywhere but this machine | `search` | +| `shell-exploration` | Grep/Glob, or `grep`, `rg`, `find`, `ls` on the project, or a `cat` of a glob such as `src/*.rs`. Reading a named file, with Read or the shell, doesn't count. | `index`, `find`, `references`, `file` | +| `web-outside-search` | WebFetch, WebSearch, another MCP server's fetch or browser tool, or `curl`/`wget` to anywhere but this machine | `search` | | `blocked-url` | URLs the `search` tool refused | nothing: your rules refuse them | ```sh diff --git a/extensions/vscode/src/hover.ts b/extensions/vscode/src/hover.ts index f050c42..8da03e7 100644 --- a/extensions/vscode/src/hover.ts +++ b/extensions/vscode/src/hover.ts @@ -160,8 +160,9 @@ function testList(tests: TestLink[]): string[] { return out; } -// only the weak tie is worth a word - the strong ones are what "covered by" already implies +// the weak tie and the run-observed one are worth a word - the rest are what "covered by" already implies function evidenceNote(evidence: string | undefined): string { + if (evidence === 'observed') return ' _(a recorded run saw it execute this)_'; return evidence === 'name-only' ? ' _(matched on the name alone)_' : ''; } diff --git a/extensions/vscode/src/types.ts b/extensions/vscode/src/types.ts index 6ab36e8..7c8b278 100644 --- a/extensions/vscode/src/types.ts +++ b/extensions/vscode/src/types.ts @@ -80,7 +80,7 @@ export interface TestedBySite { file: string; line: number; language: string; - // receiver-type | same-file | same-package | import | qualifier | name-only + // observed | receiver-type | same-file | same-package | import | qualifier | rpc | name-only evidence: string; } diff --git a/src/changes.rs b/src/changes.rs index b08b25d..8068c5b 100644 --- a/src/changes.rs +++ b/src/changes.rs @@ -61,6 +61,9 @@ pub struct ChangesConfig { // `.proto` schemas kept outside the project #[serde(default)] pub contracts: Vec, + // where a recorded run's `ccc-observed/1` records are read from - files, directories or urls + #[serde(default)] + pub observed: Vec, // what replays may do - pushing one to a remote anyone can read is allowed // here, in the repository, where the whole team sees it, or not at all #[serde(default)] @@ -518,7 +521,14 @@ pub(crate) fn assemble( let (own_cov, edges, symbol_callers, unresolved_calls, crossings) = std::thread::scope(|s| { // which tests reach which definitions - one relation, shared with // `insights`, so the two reports cannot disagree about what is covered - let own_cov = s.spawn(|| built.is_none().then(|| coverage::build(caches, &project_ids, contracts))); + let own_cov = s.spawn(|| { + built.is_none().then(|| { + let mut cov = coverage::build(caches, &project_ids, contracts); + // what a recorded run saw each test execute, on top of what the source shows + cov.observe(&crate::observed::load(root, &config.observed, caches)); + cov + }) + }); // cross-service edges + per-symbol caller map // one service and no relatives leaves nothing to cross into - the case with no `.ccc/map.json` let (mut edges, mut symbol_callers, mut unresolved_calls) = @@ -582,7 +592,7 @@ pub(crate) fn assemble( // Coverage is looked up by *definition*, not by name: two functions // sharing a name - in another module, or another language - are two // different things, and only one of them is the one that changed. - let covering = cov.covering((fi, ki)); + let covering = cov.tested_by((fi, ki)); let tested = is_test_code || cov.is_covered((fi, ki)); let tested_by: Vec = covering.iter().map(|r| r.site.name.clone()).collect(); diff --git a/src/coverage.rs b/src/coverage.rs index d7361c0..5776449 100644 --- a/src/coverage.rs +++ b/src/coverage.rs @@ -33,6 +33,8 @@ const GENERIC_DIRS: &[&str] = &[ // strongest first, which is the order `tested_by` lists them in. #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] pub enum Evidence { + // a recorded run saw the test execute the definition - see `observed` + Observed, // the receiver's declared type addresses the method exactly ReceiverType, // the definition is in the file the test calls from @@ -54,6 +56,7 @@ pub enum Evidence { impl Evidence { pub fn label(self) -> &'static str { match self { + Evidence::Observed => "observed", Evidence::ReceiverType => "receiver-type", Evidence::SameFile => "same-file", Evidence::SamePackage => "same-package", @@ -83,7 +86,7 @@ pub struct TestRef { } // A definition: (index into `caches`, index into that file's `funcs`). -type DefId = (usize, usize); +pub(crate) type DefId = (usize, usize); // what one test call reaches and the evidence that tied it - `None` when it leaves the project type Reached = Option<(Vec, Evidence)>; @@ -100,6 +103,8 @@ pub struct CoverageIndex { // test calls dropped as leaving the project, reported so the gating is // auditable rather than silent external_calls: usize, + // fixtures a recorded run saw execute each definition - kept apart from `covering` so the call-graph walk stays static + observed: BTreeMap>, } impl CoverageIndex { @@ -108,10 +113,31 @@ impl CoverageIndex { self.covering.get(&def).map(|v| &v[..]).unwrap_or_default() } + // every test tied to a definition - the fixtures a run saw execute it, then the tests the source shows, capped + pub fn tested_by(&self, def: DefId) -> Vec { + let seen = self.observed.get(&def).map(|v| &v[..]).unwrap_or_default(); + let mut refs = seen.to_vec(); + refs.extend(self.covering(def).iter().filter(|r| !seen.iter().any(|s| s.site == r.site)).cloned()); + refs.truncate(MAX_TESTED_BY); + refs + } + // Does any test reference this definition at all? True for a definition // reached only from file-level test setup, which names no test. pub fn is_covered(&self, def: DefId) -> bool { - self.covering.contains_key(&def) || self.touched.contains(&def) + self.covering.contains_key(&def) || self.touched.contains(&def) || self.observed.contains_key(&def) + } + + // a recorded run's fixtures, each one the strongest evidence there is - it ran the definition + pub(crate) fn observe(&mut self, observed: &crate::observed::Observed) { + for (&def, fixtures) in &observed.ran { + let mut refs: Vec = fixtures + .iter() + .map(|&f| TestRef { site: observed.fixtures[f].site(), evidence: Evidence::Observed }) + .collect(); + refs.sort_by(|x, y| (&x.site.file, &x.site.name).cmp(&(&y.site.file, &y.site.name))); + self.observed.insert(def, refs); + } } pub fn total_tests(&self) -> usize { @@ -447,6 +473,7 @@ pub(crate) fn build_over( touched, total_tests: tests.len(), external_calls, + observed: BTreeMap::new(), } } diff --git a/src/externals.rs b/src/externals.rs index 399a4f5..d1eb0ab 100644 --- a/src/externals.rs +++ b/src/externals.rs @@ -331,6 +331,11 @@ fn load_surface(root: &Path, location: &str, auth: Option<&str>) -> Result) -> Result { + fetch_limited(url, auth, MAX_SURFACE_BYTES) +} + +// `fetch` with a size of its own - observed records run far larger than a surface +pub(crate) fn fetch_limited(url: &str, auth: Option<&str>, max_bytes: u64) -> Result { let token = match auth { Some(spec) => Some(read_auth(spec)?), None => None, @@ -344,7 +349,7 @@ fn fetch(url: &str, auth: Option<&str>) -> Result { "--max-time", &FETCH_TIMEOUT_SECS.to_string(), "--max-filesize", - &MAX_SURFACE_BYTES.to_string(), + &max_bytes.to_string(), ]); if let Some(token) = &token { // via a header argument rather than the URL, so the token cannot end diff --git a/src/insights.rs b/src/insights.rs index a965b52..37b922a 100644 --- a/src/insights.rs +++ b/src/insights.rs @@ -8,7 +8,8 @@ //! labelled accordingly; do not present these as proofs. use crate::contracts::ContractIndex; -use crate::coverage::{self, CoverageIndex, TestSite}; +use crate::coverage::{self, CoverageIndex, DefId, TestSite}; +use crate::observed::{Observed, Selection}; use crate::extract::TOP_LEVEL; use crate::languages::Language; use crate::model::{FileCache, Func, FuncMetrics}; @@ -1735,6 +1736,90 @@ fn test_triggers(report: &changes::ChangesReport, trig: &Triggered, targets: &Va }) } +// What a recorded run saw - the fixtures that executed the change, chosen both +// ways so a caller takes the safe set or the cheapest cover without asking +// again. It adds to the static `run` list rather than replacing it: a test +// written since the recording exists only in the source. +fn observed_triggers(report: &changes::ChangesReport, caches: &[FileCache], observed: &Observed) -> Value { + let by_path: BTreeMap = + caches.iter().enumerate().map(|(i, c)| (changes::path_str(&c.rel_path), i)).collect(); + let mut changed: Vec<(DefId, &str)> = Vec::new(); + let mut unobserved = Vec::new(); + // a changed test that a record names is a fixture to rerun whatever else is chosen + let mut rerun: BTreeSet = BTreeSet::new(); + for c in &report.changed_functions { + let Some(&fi) = by_path.get(&c.file) else { continue }; + let Some(ki) = caches[fi].funcs.iter().position(|f| f.name == c.function && f.start_line == c.lines[0]) else { + continue; + }; + rerun.extend(observed.fixture_at(&c.file, &c.function)); + let is_test = changes::is_test_path(&c.file) || caches[fi].funcs[ki].test_ctx || changes::is_test_fn_name(&c.function); + if observed.ran.contains_key(&(fi, ki)) { + changed.push(((fi, ki), c.function.as_str())); + } else if !is_test { + unobserved.push(json!({"file": c.file, "function": c.function, "lines": c.lines})); + } + } + let defs: Vec = changed.iter().map(|(d, _)| *d).collect(); + let chosen = |sel: Selection| -> Value { + let picked: Vec = sel.fixtures.iter().chain(&rerun).copied().collect::>().into_iter().collect(); + let mut by_lang: BTreeMap<&str, Vec> = BTreeMap::new(); + for &f in &picked { + let site = observed.fixtures[f].site(); + by_lang.entry(site.language).or_default().push(site); + } + let commands: Vec = by_lang + .iter() + .filter_map(|(lang, sites)| { + let refs: Vec<&TestSite> = sites.iter().collect(); + run_command(lang, &refs).map(|(command, caveat)| { + json!({"language": lang, "command": command, "selects": sites.len(), "caveat": caveat}) + }) + }) + .collect(); + let fixtures: Vec = picked + .iter() + .map(|&f| { + let fx = &observed.fixtures[f]; + let covers: Vec<&str> = + changed.iter().filter(|(d, _)| observed.ran[d].contains(&f)).map(|(_, name)| *name).take(8).collect(); + json!({ + "fixture": fx.name, + "file": fx.file, + "line": fx.line, + "language": fx.language, + "kind": fx.kind.label(), + "duration_ms": fx.duration_ms, + "cost": fx.cost(), + "runs": fx.runs, + "services": fx.services, + "covers": covers, + // the fixture is itself one of the changed functions + "changed": rerun.contains(&f), + }) + }) + .collect(); + json!({"fixtures": fixtures, "cost": observed.cost_of(&picked), "exact": sel.exact, "commands": commands}) + }; + json!({ + "sources": observed.sources, + "errors": observed.errors, + "records": observed.records, + "bad_lines": observed.bad_lines, + "fixtures": observed.fixtures.len(), + "unmatched_files": observed.unmatched_files.len(), + "unmatched": observed.unmatched, + "default": "safe", + "safe": chosen(observed.safe(&defs)), + "minimal": chosen(observed.minimal(&defs)), + "observed_functions": defs.len(), + // changed functions no recorded fixture ran - the static `run` list is all there is for these + "unobserved": unobserved, + "note": "Fixtures a recorded run saw execute the changed functions. The static `run` list still \ + stands beside it: a test written since the recording exists only in the source.", + }) +} + // Analyse `root` from scratch: walk, parse and build the payload pub fn analyse(root: &Path, root_label: &str, base: Option<&str>) -> anyhow::Result { let files = crate::scan::collect_files(root)?; @@ -1765,8 +1850,11 @@ pub fn insights( // nothing of each other's, so they are built side by side. The coverage // relation follows the same evidence rules as the graph and is shared by // every section that reports what a test reaches. - let (g, cov, prepared) = std::thread::scope(|s| { + // where a recorded run's records live, when the project names any + let sources = ChangesConfig::load(root).map(|c| c.observed).unwrap_or_default(); + let (g, mut cov, prepared, observed) = std::thread::scope(|s| { let prepared = s.spawn(|| changes::prepare(root, &change_opts)); + let observed = s.spawn(|| crate::observed::load(root, &sources, caches)); // what qualifiers name and what each file imports, built once for the graph and coverage alike let reach = coverage::Reach::of(caches); let (g, cov) = std::thread::scope(|inner| { @@ -1775,8 +1863,9 @@ pub fn insights( }); // the tables are large, and nothing reads them from here on std::thread::spawn(move || drop(reach)); - (g, cov, joined(prepared)) + (g, cov, joined(prepared), joined(observed)) }); + cov.observe(&observed); let ctx = service_ctx(&g, root, &contracts); let n = g.nodes.len(); // each file's path as text, written once rather than per sort comparison @@ -1822,10 +1911,14 @@ pub fn insights( .unwrap_or_default(); let targets = test_targets(&g, &cov, &ctx, &must_keep); let (changes_section, triggers) = match (&change_set, &trig) { - (Ok(r), Some(t)) => ( - serde_json::to_value(r).unwrap_or(Value::Null), - test_triggers(r, t, &targets), - ), + (Ok(r), Some(t)) => { + let mut triggers = test_triggers(r, t, &targets); + // only where records were looked for, so a project without them reads as it always has + if observed.configured() { + triggers["observed"] = observed_triggers(r, caches, &observed); + } + (serde_json::to_value(r).unwrap_or(Value::Null), triggers) + } // no git, no base ref, shallow clone: say why rather than render // an empty tab that reads as "nothing to run" _ => { @@ -3033,6 +3126,83 @@ pub fn helper() -> u64 { 1 } drop(dir); } + // A python suite reaches a go function over http, which no call in the + // source shows. A recorded run does, and its fixtures are chosen both ways. + #[test] + fn a_recorded_run_selects_the_fixtures_that_executed_the_change() { + let (dir, _) = map( + "observed", + &[ + ("svc/fee.go", "package svc\nfunc Fee() int { return 30 }\nfunc Spare() int { return 1 }\n"), + ("tests/test_api.py", "def test_refund(client):\n assert client.post('/refund').ok\n\ndef test_soak(client):\n assert client.get('/').ok\n"), + ], + ); + let d = dir.path(); + std::fs::create_dir_all(d.join(".ccc/observed")).unwrap(); + let record = |name: &str, kind: &str, ms: u64| { + format!(r#"{{"schema":"ccc-observed/1","fixture":{{"name":"{name}","kind":"{kind}","file":"tests/test_api.py","language":"python"}},"run":"nightly-7","duration_ms":{ms},"files":[{{"path":"svc/fee.go","functions":[{{"name":"Fee"}}]}}]}}"#) + }; + std::fs::write( + d.join(".ccc/observed/nightly.jsonl"), + [record("test_refund", "integration", 2_000), record("test_soak", "smoke", 90_000)].join("\n"), + ) + .unwrap(); + run_git(d, &["init", "-q", "-b", "main"]); + run_git(d, &["add", "-A"]); + run_git(d, &["-c", "user.email=t@t", "-c", "user.name=t", "-c", "commit.gpgsign=false", + "commit", "-qm", "base"]); + run_git(d, &["checkout", "-q", "-b", "work"]); + std::fs::write(d.join("svc/fee.go"), "package svc\nfunc Fee() int { return 45 }\nfunc Spare() int { return 1 }\n").unwrap(); + + let files = scan::collect_files(d).unwrap(); + let caches = scan::build_caches(d, &files); + let v = insights(&caches, d, "demo", "ts", None); + let o = &v["test_triggers"]["observed"]; + assert_eq!(o["records"], 2, "{o}"); + assert_eq!(o["observed_functions"], 1, "{o}"); + + let picked = |mode: &str| -> Vec<&str> { + o[mode]["fixtures"].as_array().unwrap().iter().map(|f| f["fixture"].as_str().unwrap()).collect() + }; + assert_eq!(picked("safe"), vec!["test_refund", "test_soak"]); + // the integration run covers `Fee` for a fraction of the smoke run's cost + assert_eq!(picked("minimal"), vec!["test_refund"]); + assert_eq!(o["minimal"]["exact"], true); + let cmd = o["minimal"]["commands"][0]["command"].as_str().unwrap(); + assert_eq!(cmd, "pytest -k \"test_refund\""); + + // the change report names the fixtures as what tests `Fee`, strongest evidence first + let fee = v["changes"]["changed_functions"] + .as_array() + .unwrap() + .iter() + .find(|f| f["function"] == "Fee") + .unwrap(); + assert_eq!(fee["tested"], true); + assert_eq!(fee["tested_by_sites"][0]["evidence"], "observed"); + assert_eq!(fee["tested_by_sites"][0]["test"], "test_refund"); + // and no gap is reported for a function a run saw executed + assert!(v["test_triggers"]["add"].as_array().unwrap().is_empty(), "{}", v["test_triggers"]["add"]); + drop(dir); + } + + // Without records the payload is exactly what it was before they existed. + #[test] + fn no_records_means_no_observed_section() { + let (dir, _) = map("unobserved", &[("src/a.rs", "pub fn one() -> u64 { 1 }\n")]); + let d = dir.path(); + run_git(d, &["init", "-q", "-b", "main"]); + run_git(d, &["add", "-A"]); + run_git(d, &["-c", "user.email=t@t", "-c", "user.name=t", "-c", "commit.gpgsign=false", + "commit", "-qm", "base"]); + let files = scan::collect_files(d).unwrap(); + let caches = scan::build_caches(d, &files); + let v = insights(&caches, d, "demo", "ts", None); + assert_eq!(v["test_triggers"]["available"], true); + assert!(v["test_triggers"].get("observed").is_none()); + drop(dir); + } + #[test] fn payload_is_shaped_and_services_fall_back_to_directories() { let (dir, caches) = map("payload", &[ diff --git a/src/lib.rs b/src/lib.rs index b50b759..7566ba4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,7 @@ pub mod instance; pub mod languages; pub mod model; pub mod naming; +pub mod observed; mod par; pub mod parse_cache; pub mod pr_summary; diff --git a/src/main.rs b/src/main.rs index ca64f4b..7dc18a0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,6 +20,9 @@ struct Cli { // parse every file afresh rather than reading back an earlier run's parse - `CCC_CACHE_DIR` moves them #[arg(long, global = true)] no_parse_cache: bool, + // read `ccc-observed/1` records from here too - a file, a directory or a url, repeatable + #[arg(long, global = true, value_name = "SRC")] + observed: Vec, } const CACHE_DIR: &str = ".ccc"; @@ -445,6 +448,7 @@ fn run() -> Result { if !cli.no_parse_cache { codecache::parse_cache::enable(); } + codecache::observed::add_sources(cli.observed.clone()); match cli.command { Command::Scan { path, diff --git a/src/observed.rs b/src/observed.rs new file mode 100644 index 0000000..7453e04 --- /dev/null +++ b/src/observed.rs @@ -0,0 +1,717 @@ +//! What a test run observed: which functions each fixture actually ran, read +//! from `ccc-observed/1` records and tied to the map's own definitions. + +use crate::changes::path_str; +use crate::coverage::{DefId, TestSite}; +use crate::languages::Language; +use crate::model::{FileCache, Func}; +use anyhow::{bail, Context, Result}; +use serde::Deserialize; +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use std::process::Command; +use std::sync::{Arc, Mutex, OnceLock}; +use std::time::{Duration, Instant}; + +pub const SCHEMA: &str = "ccc-observed/1"; +// where a run on this machine leaves its records when nothing else is named +pub const LOCAL_DIR: &str = ".ccc/observed"; +// a fetched record set is reused this long - a nightly run does not change by the minute +const FETCH_TTL: Duration = Duration::from_secs(600); +// the records of a large suite run to hundreds of megabytes +const MAX_RECORD_BYTES: u64 = 512 * 1024 * 1024; +// what a fixture with no recorded duration is taken to cost +const DEFAULT_MS: u64 = 1000; +// search steps the exact cover may take before the greedy one stands in +const EXACT_BUDGET: usize = 200_000; +// malformed lines named in the report - the rest are only counted +const MAX_ERRORS: usize = 20; + +// sources named on the command line, read on top of the project's own +static EXTRA: OnceLock> = OnceLock::new(); +// record sets by url, each with when it was fetched +type Fetched = Mutex)>>; +// fetched urls - a server re-analysing every few seconds must not refetch a nightly run +static FETCHED: OnceLock = OnceLock::new(); + +// read these as well as `.ccc/map.json`'s `observed` - the binary passes `--observed` here +pub fn add_sources(sources: Vec) { + let _ = EXTRA.set(sources); +} + +// what a fixture is, which decides how dear running it is +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Kind { + Unit, + Contract, + Interop, + Integration, + Smoke, +} + +impl Kind { + pub fn label(self) -> &'static str { + match self { + Kind::Unit => "unit", + Kind::Contract => "contract", + Kind::Interop => "interop", + Kind::Integration => "integration", + Kind::Smoke => "smoke", + } + } + + // how much dearer a run of this kind is than a unit test of the same length + fn weight(self) -> u64 { + match self { + Kind::Unit => 1, + Kind::Contract => 2, + Kind::Interop => 3, + Kind::Integration => 4, + Kind::Smoke => 5, + } + } +} + +// one place records are read from, as `.ccc/map.json` names it - a file, a directory or a url +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +pub enum Source { + At(String), + // `auth` is `env:NAME` - the variable holding a bearer token for a private url + Authed { + from: String, + #[serde(default)] + auth: Option, + }, +} + +impl Source { + fn parts(&self) -> (&str, Option<&str>) { + match self { + Source::At(at) => (at, None), + Source::Authed { from, auth } => (from, auth.as_deref()), + } + } +} + +// one line of a record file - one fixture as one service saw it +#[derive(Deserialize)] +struct Record { + schema: String, + fixture: FixtureIn, + #[serde(default)] + run: Option, + #[serde(default)] + service: Option, + #[serde(default)] + duration_ms: Option, + #[serde(default)] + files: Vec, +} + +#[derive(Deserialize)] +struct FixtureIn { + // what the runner selects it by - `TestRefund`, `test_partial_refund`, `Billing.Refunds` + name: String, + kind: Kind, + #[serde(default)] + file: Option, + #[serde(default)] + language: Option, +} + +#[derive(Deserialize)] +struct ServiceIn { + #[serde(default)] + name: Option, + // the commit the service was built from - line numbers hold only against it + #[serde(default)] + commit: Option, +} + +#[derive(Deserialize)] +struct FileIn { + path: String, + #[serde(default)] + functions: Vec, + // ranges of lines that ran, for a probe that sees lines rather than functions + #[serde(default)] + lines: Vec<[usize; 2]>, +} + +#[derive(Deserialize)] +struct FuncIn { + // bare or qualified - `charge`, `Billing.charge`, `(*Server).Charge` + name: String, + #[serde(default)] + owner: Option, + #[serde(default)] + line: Option, +} + +// one fixture across every record that names it +#[derive(Debug, Clone)] +pub struct Fixture { + pub name: String, + pub file: Option, + // where the map defines it - 0 when the map does not hold the fixture + pub line: usize, + pub language: Option, + pub kind: Kind, + // the longest any record gave it + pub duration_ms: Option, + pub runs: BTreeSet, + pub services: BTreeSet, +} + +impl Fixture { + // what running it costs - its length weighted by its kind + pub fn cost(&self) -> u64 { + self.duration_ms.unwrap_or(DEFAULT_MS).max(1) * self.kind.weight() + } + + // the fixture as a test a runner can select + pub fn site(&self) -> TestSite { + let language = self.language.as_deref().unwrap_or_default(); + TestSite { + name: self.name.clone(), + file: self.file.clone().unwrap_or_default(), + line: self.line, + language: Language::ALL.iter().map(|l| l.as_str()).find(|l| *l == language).unwrap_or("other"), + } + } +} + +#[derive(Debug, Default)] +pub struct Observed { + // every place records were looked for + pub sources: Vec, + // sources that could not be read, and the first lines that were not records + pub errors: Vec, + pub bad_lines: usize, + pub records: usize, + pub fixtures: Vec, + // the fixtures that ran each definition + pub ran: BTreeMap>, + // record paths no file in the map holds - another repository's, usually + pub unmatched_files: BTreeSet, + // functions and line ranges a record names that the map no longer has + pub unmatched: usize, +} + +// the fixtures chosen to run, what they cost together, and whether no cheaper cover exists +#[derive(Debug, Clone, Default)] +pub struct Selection { + pub fixtures: Vec, + pub cost: u64, + pub exact: bool, +} + +impl Observed { + // whether any source was looked at - with none, every report reads as it did before records existed + pub fn configured(&self) -> bool { + !self.sources.is_empty() + } + + // the fixture a test function is, when a record names it + pub fn fixture_at(&self, file: &str, name: &str) -> Option { + self.fixtures.iter().position(|f| f.file.as_deref() == Some(file) && f.name == name) + } + + pub fn cost_of(&self, fixtures: &[usize]) -> u64 { + fixtures.iter().map(|&f| self.fixtures[f].cost()).sum() + } + + // every fixture that ran any of `defs` + pub fn safe(&self, defs: &[DefId]) -> Selection { + let chosen: Vec = defs + .iter() + .filter_map(|d| self.ran.get(d)) + .flatten() + .copied() + .collect::>() + .into_iter() + .collect(); + Selection { cost: self.cost_of(&chosen), fixtures: chosen, exact: true } + } + + // the cheapest fixtures that between them run each of `defs` at least once + pub fn minimal(&self, defs: &[DefId]) -> Selection { + let universe: Vec<&BTreeSet> = + defs.iter().filter_map(|d| self.ran.get(d)).filter(|s| !s.is_empty()).collect(); + let candidates: Vec = universe.iter().copied().flatten().copied().collect::>().into_iter().collect(); + // what each candidate covers, as positions in `universe` + let covers: Vec> = candidates + .iter() + .map(|c| (0..universe.len()).filter(|&u| universe[u].contains(c)).collect()) + .collect(); + let costs: Vec = candidates.iter().map(|&c| self.fixtures[c].cost()).collect(); + let exact = (universe.len() <= 128).then(|| exact_cover(&covers, &costs, universe.len())).flatten(); + let found = exact.is_some(); + let picked = exact.unwrap_or_else(|| greedy_cover(&covers, &costs, universe.len())); + let mut chosen: Vec = picked.into_iter().map(|p| candidates[p]).collect(); + chosen.sort_unstable(); + Selection { cost: self.cost_of(&chosen), fixtures: chosen, exact: found } + } +} + +// the cheapest cover, branching on the first element left uncovered - `None` past the step budget +fn exact_cover(covers: &[BTreeSet], costs: &[u64], size: usize) -> Option> { + struct Search<'a> { + masks: &'a [u128], + costs: &'a [u64], + by_elem: &'a [Vec], + full: u128, + best: Option<(u64, Vec)>, + steps: usize, + } + // false once the budget runs out + fn go(s: &mut Search, covered: u128, cost: u64, chosen: &mut Vec) -> bool { + s.steps += 1; + if s.steps > EXACT_BUDGET { + return false; + } + if s.best.as_ref().is_some_and(|(best, _)| cost >= *best) { + return true; + } + if covered == s.full { + s.best = Some((cost, chosen.clone())); + return true; + } + let first = (!covered & s.full).trailing_zeros() as usize; + let (by_elem, masks, costs) = (s.by_elem, s.masks, s.costs); + for &c in &by_elem[first] { + chosen.push(c); + let finished = go(s, covered | masks[c], cost + costs[c], chosen); + chosen.pop(); + if !finished { + return false; + } + } + true + } + let masks: Vec = covers.iter().map(|c| c.iter().fold(0u128, |m, &u| m | 1 << u)).collect(); + let mut by_elem: Vec> = vec![Vec::new(); size]; + for (c, cover) in covers.iter().enumerate() { + for &u in cover { + by_elem[u].push(c); + } + } + // cheapest first, so the first cover found is already a tight bound + for list in &mut by_elem { + list.sort_by_key(|&c| (costs[c], c)); + } + let full = if size == 128 { u128::MAX } else { (1u128 << size) - 1 }; + let mut search = Search { masks: &masks, costs, by_elem: &by_elem, full, best: None, steps: 0 }; + let finished = go(&mut search, 0, 0, &mut Vec::new()); + finished.then(|| search.best.map(|(_, chosen)| chosen).unwrap_or_default()) +} + +// a cover built by taking whichever candidate buys the most uncovered elements per unit of cost - the wider one on a tie +fn greedy_cover(covers: &[BTreeSet], costs: &[u64], size: usize) -> Vec { + let mut left: BTreeSet = (0..size).collect(); + let mut picked = Vec::new(); + while !left.is_empty() { + let best = (0..covers.len()) + .map(|c| (c, covers[c].intersection(&left).count())) + .filter(|&(_, gain)| gain > 0) + .max_by(|&(a, ga), &(b, gb)| { + (ga as u128 * costs[b] as u128) + .cmp(&(gb as u128 * costs[a] as u128)) + .then(ga.cmp(&gb)) + .then(b.cmp(&a)) + }); + let Some((c, _)) = best else { break }; + left.retain(|u| !covers[c].contains(u)); + picked.push(c); + } + picked +} + +// every record the project's sources hold, tied to the map's definitions +pub fn load(root: &Path, configured: &[Source], caches: &[FileCache]) -> Observed { + let mut wanted: Vec<(String, Option)> = configured + .iter() + .map(|s| { + let (at, auth) = s.parts(); + (at.to_string(), auth.map(str::to_string)) + }) + .collect(); + wanted.extend(EXTRA.get().into_iter().flatten().map(|at| (at.clone(), None))); + if wanted.is_empty() && root.join(LOCAL_DIR).is_dir() { + wanted.push((LOCAL_DIR.to_string(), None)); + } + let mut out = Observed::default(); + let mut records = Vec::new(); + for (at, auth) in wanted { + match read(root, &at, auth.as_deref()) { + Ok(texts) => { + for (origin, text) in &texts { + parse(origin, text, &mut records, &mut out); + } + } + Err(e) => out.errors.push(format!("{at}: {e:#}")), + } + out.sources.push(at); + } + out.records = records.len(); + tie(root, caches, &records, &mut out); + out +} + +// the texts one source holds, each with where it came from +fn read(root: &Path, at: &str, auth: Option<&str>) -> Result)>> { + if at.starts_with("https://") || at.starts_with("http://") { + return Ok(vec![(at.to_string(), fetched(at, auth)?)]); + } + let path = root.join(at.trim_start_matches("file://")); + if !path.is_dir() { + return Ok(vec![(path.display().to_string(), Arc::new(read_file(&path)?))]); + } + let mut files: Vec<_> = std::fs::read_dir(&path) + .with_context(|| format!("reading {}", path.display()))? + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.is_file() && p.extension().is_some_and(|x| x == "jsonl" || x == "json")) + .collect(); + files.sort(); + files.iter().map(|p| Ok((p.display().to_string(), Arc::new(read_file(p)?)))).collect() +} + +fn read_file(path: &Path) -> Result { + let meta = std::fs::metadata(path).with_context(|| format!("reading {}", path.display()))?; + if meta.len() > MAX_RECORD_BYTES { + bail!("{} is {} bytes, over the {MAX_RECORD_BYTES}-byte record limit", path.display(), meta.len()); + } + std::fs::read_to_string(path).with_context(|| format!("reading {}", path.display())) +} + +// a url's records, fetched at most once per `FETCH_TTL` +fn fetched(url: &str, auth: Option<&str>) -> Result> { + let cache = FETCHED.get_or_init(Default::default); + if let Some((at, text)) = cache.lock().unwrap_or_else(|p| p.into_inner()).get(url) { + if at.elapsed() < FETCH_TTL { + return Ok(text.clone()); + } + } + let text = Arc::new(crate::externals::fetch_limited(url, auth, MAX_RECORD_BYTES)?); + cache.lock().unwrap_or_else(|p| p.into_inner()).insert(url.to_string(), (Instant::now(), text.clone())); + Ok(text) +} + +// each line of `text` that is a record - the rest are counted and the first few named +fn parse(origin: &str, text: &str, records: &mut Vec, out: &mut Observed) { + for (n, line) in text.lines().enumerate() { + let line = line.trim(); + if line.is_empty() { + continue; + } + let problem = match serde_json::from_str::(line) { + Ok(r) if r.schema == SCHEMA => { + records.push(r); + continue; + } + Ok(r) => format!("schema '{}', expected '{SCHEMA}'", r.schema), + Err(e) => e.to_string(), + }; + out.bad_lines += 1; + if out.errors.len() < MAX_ERRORS { + out.errors.push(format!("{origin}:{}: {problem}", n + 1)); + } + } +} + +// each record's functions as definitions in the map, and its fixture merged with every other record naming it +fn tie(root: &Path, caches: &[FileCache], records: &[Record], out: &mut Observed) { + let paths: Vec = caches.iter().map(|c| path_str(&c.rel_path)).collect(); + let by_path: BTreeMap<&str, usize> = paths.iter().enumerate().map(|(i, p)| (p.as_str(), i)).collect(); + let mut located: BTreeMap<&str, Option> = BTreeMap::new(); + let mut moved: BTreeMap<&str, Option>> = BTreeMap::new(); + let mut keys: BTreeMap<(Option<&str>, Option<&str>, &str), usize> = BTreeMap::new(); + for r in records { + let fx = &r.fixture; + let key = (fx.language.as_deref(), fx.file.as_deref(), fx.name.as_str()); + let at = *keys.entry(key).or_insert_with(|| { + let line = fx + .file + .as_deref() + .and_then(|f| by_path.get(f)) + .and_then(|&i| caches[i].funcs.iter().find(|f| f.name == fx.name)) + .map_or(0, |f| f.line); + out.fixtures.push(Fixture { + name: fx.name.clone(), + file: fx.file.clone(), + line, + language: fx.language.clone(), + kind: fx.kind, + duration_ms: None, + runs: BTreeSet::new(), + services: BTreeSet::new(), + }); + out.fixtures.len() - 1 + }); + let fixture = &mut out.fixtures[at]; + fixture.duration_ms = fixture.duration_ms.max(r.duration_ms); + fixture.runs.extend(r.run.clone()); + fixture.services.extend(r.service.as_ref().and_then(|s| s.name.clone())); + + // the files changed since the recording, where its line numbers no longer hold + let commit = r.service.as_ref().and_then(|s| s.commit.as_deref()); + let changed = commit.map(|c| &*moved.entry(c).or_insert_with(|| changed_since(root, c))); + for file in &r.files { + let Some(fi) = *located.entry(file.path.as_str()).or_insert_with(|| locate(&by_path, &file.path)) else { + out.unmatched_files.insert(file.path.clone()); + continue; + }; + let lines_hold = match changed { + None => true, + Some(Some(since)) => !since.contains(&paths[fi]), + Some(None) => false, + }; + let funcs = &caches[fi].funcs; + let mut hits: BTreeSet = BTreeSet::new(); + for want in &file.functions { + let found = function(funcs, want, lines_hold); + out.unmatched += usize::from(found.is_empty()); + hits.extend(found); + } + // line ranges stand in for functions only when the record names none + if file.functions.is_empty() { + for &[a, b] in &file.lines { + let found: Vec = if lines_hold { [a, b].iter().filter_map(|&l| innermost(funcs, l)).collect() } else { Vec::new() }; + out.unmatched += usize::from(found.is_empty()); + hits.extend(found); + } + } + for k in hits { + out.ran.entry((fi, k)).or_default().insert(at); + } + } + } +} + +// the definitions in one file a record's function names - by name, then owner, then nearest line +fn function(funcs: &[Func], want: &FuncIn, lines_hold: bool) -> Vec { + let (hinted, bare) = split_name(&want.name); + let owner = want.owner.as_deref().map(bare_owner).or(hinted); + let named: Vec = (0..funcs.len()).filter(|&k| funcs[k].name == bare).collect(); + let owns = |k: &usize| funcs[*k].owner.as_deref().map(bare_owner) == owner; + let owned: Vec = if owner.is_some() && named.iter().any(owns) { + named.into_iter().filter(owns).collect() + } else { + named + }; + match (owned.len(), want.line) { + (0, Some(line)) if lines_hold => innermost(funcs, line).into_iter().collect(), + (n, Some(line)) if n > 1 => owned.into_iter().min_by_key(|&k| funcs[k].start_line.abs_diff(line)).into_iter().collect(), + _ => owned, + } +} + +// `Billing.charge`, `acme::Server::charge`, `(*Server).Charge` - the owner a qualified name carries, and the bare name +fn split_name(name: &str) -> (Option<&str>, &str) { + match name.rfind(['.', ':']) { + Some(at) => { + let owner = bare_owner(name[..at].trim_end_matches([':', '.'])); + ((!owner.is_empty()).then_some(owner), &name[at + 1..]) + } + None => (None, name), + } +} + +// `pkg.(*Server)` and `acme::Server` both own as `Server` +fn bare_owner(owner: &str) -> &str { + let last = owner.rsplit(['.', ':']).find(|s| !s.is_empty()).unwrap_or(owner); + last.trim_matches(|c: char| matches!(c, '(' | ')' | '*' | '&')) +} + +// the smallest definition whose span holds `line` +fn innermost(funcs: &[Func], line: usize) -> Option { + (0..funcs.len()) + .filter(|&k| funcs[k].start_line <= line && line <= funcs[k].end_line) + .min_by_key(|&k| funcs[k].end_line - funcs[k].start_line) +} + +// the map's file a record path names - a relative path as written, an absolute one by the tail inside this checkout +fn locate(by_path: &BTreeMap<&str, usize>, path: &str) -> Option { + let norm = path.replace('\\', "/"); + let mut rest = norm.trim_start_matches("./"); + if let Some(&i) = by_path.get(rest) { + return Some(i); + } + // only an absolute path is another checkout of this repository - a relative one that misses is another repository's + if !(norm.starts_with('/') || norm.as_bytes().get(1) == Some(&b':')) { + return None; + } + loop { + rest = rest.split_once('/')?.1; + if let Some(&i) = by_path.get(rest) { + return Some(i); + } + } +} + +// the files that differ between `commit` and the working tree - `None` when git does not know the commit +fn changed_since(root: &Path, commit: &str) -> Option> { + // a record may come from anywhere, so only a hex id ever reaches git's command line + if !(7..=64).contains(&commit.len()) || !commit.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + let out = Command::new("git").args(["diff", "--name-only", "--no-renames", commit, "--"]).current_dir(root).output().ok()?; + out.status.success().then(|| String::from_utf8_lossy(&out.stdout).lines().map(str::to_string).collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::scan; + use std::path::PathBuf; + + // a throwaway directory that cleans up on drop + struct Dir(PathBuf); + impl Drop for Dir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + fn project(tag: &str, files: &[(&str, &str)]) -> (Dir, Vec) { + let dir = Dir(std::env::temp_dir().join(format!("ccc-observed-{tag}-{}", std::process::id()))); + let _ = std::fs::remove_dir_all(&dir.0); + for (name, body) in files { + let to = dir.0.join(name); + std::fs::create_dir_all(to.parent().unwrap()).unwrap(); + std::fs::write(&to, body).unwrap(); + } + let found = scan::collect_files(&dir.0).unwrap(); + let caches = scan::build_caches(&dir.0, &found); + (dir, caches) + } + + fn def(caches: &[FileCache], file: &str, name: &str) -> DefId { + let fi = caches.iter().position(|c| path_str(&c.rel_path) == file).unwrap(); + (fi, caches[fi].funcs.iter().position(|f| f.name == name).unwrap()) + } + + fn observed(root: &Path, caches: &[FileCache], lines: &[&str]) -> Observed { + let mut out = Observed::default(); + let mut records = Vec::new(); + parse("test", &lines.join("\n"), &mut records, &mut out); + out.records = records.len(); + tie(root, caches, &records, &mut out); + out + } + + fn names(o: &Observed, sel: &Selection) -> Vec { + sel.fixtures.iter().map(|&f| o.fixtures[f].name.clone()).collect() + } + + const BILLING: &str = "package billing\n\ + type Server struct{}\n\ + type Ledger struct{}\n\ + func (s *Server) Charge(n int) int { return n + fee() }\n\ + func (l *Ledger) Charge(n int) int { return n }\n\ + func fee() int { return 30 }\n"; + + #[test] + fn a_record_ties_each_function_to_its_definition_by_name_and_owner() { + let (dir, caches) = project("tie", &[("billing/server.go", BILLING)]); + let o = observed(&dir.0, &caches, &[ + r#"{"schema":"ccc-observed/1","fixture":{"name":"TestRefund","kind":"contract","language":"go"},"run":"ci-1","files":[{"path":"billing/server.go","functions":[{"name":"(*Server).Charge"},{"name":"fee"}]}]}"#, + ]); + assert_eq!(o.records, 1); + assert_eq!(o.unmatched, 0); + let ran: BTreeSet = o.ran.keys().copied().collect(); + let server = caches[0].funcs.iter().position(|f| f.name == "Charge" && f.owner.as_deref() == Some("Server")).unwrap(); + assert_eq!(ran, BTreeSet::from([(0, server), def(&caches, "billing/server.go", "fee")]), "the ledger's Charge never ran"); + } + + #[test] + fn a_ci_checkout_path_is_found_by_its_tail_and_another_repos_path_is_not() { + let (dir, caches) = project("paths", &[("billing/server.go", BILLING)]); + let o = observed(&dir.0, &caches, &[ + r#"{"schema":"ccc-observed/1","fixture":{"name":"TestFee","kind":"unit"},"files":[{"path":"/builds/acme/billing/billing/server.go","functions":[{"name":"fee"}]},{"path":"payments/billing/server.go","functions":[{"name":"fee"}]}]}"#, + ]); + assert!(o.ran.contains_key(&def(&caches, "billing/server.go", "fee"))); + assert_eq!(o.unmatched_files, BTreeSet::from(["payments/billing/server.go".to_string()])); + } + + #[test] + fn lines_stand_in_for_functions_only_where_the_code_has_not_moved() { + let (dir, caches) = project("lines", &[("billing/server.go", BILLING)]); + // no commit, so the lines are taken as they are + let o = observed(&dir.0, &caches, &[ + r#"{"schema":"ccc-observed/1","fixture":{"name":"TestFee","kind":"unit"},"files":[{"path":"billing/server.go","lines":[[6,6]]}]}"#, + ]); + assert_eq!(o.ran.keys().copied().collect::>(), vec![def(&caches, "billing/server.go", "fee")]); + // a commit git cannot place might have numbered every line differently + let o = observed(&dir.0, &caches, &[ + r#"{"schema":"ccc-observed/1","fixture":{"name":"TestFee","kind":"unit"},"service":{"commit":"deadbeef"},"files":[{"path":"billing/server.go","lines":[[6,6]]}]}"#, + ]); + assert!(o.ran.is_empty()); + assert_eq!(o.unmatched, 1); + } + + #[test] + fn a_commit_that_is_not_an_object_id_never_reaches_git() { + assert!(changed_since(Path::new("."), "--output=/tmp/x").is_none()); + assert!(changed_since(Path::new("."), "HEAD").is_none()); + } + + #[test] + fn records_of_one_fixture_merge_and_lines_that_are_not_records_are_counted() { + let (dir, caches) = project("merge", &[("billing/server.go", BILLING)]); + let o = observed(&dir.0, &caches, &[ + r#"{"schema":"ccc-observed/1","fixture":{"name":"TestRefund","kind":"integration","language":"go"},"run":"ci-1","service":{"name":"billing"},"duration_ms":900,"files":[{"path":"billing/server.go","functions":[{"name":"fee"}]}]}"#, + r#"{"schema":"ccc-observed/1","fixture":{"name":"TestRefund","kind":"integration","language":"go"},"run":"ci-1","service":{"name":"ledger"},"duration_ms":1200,"files":[]}"#, + r#"{"schema":"ccc-observed/2","fixture":{"name":"TestOther","kind":"unit"}}"#, + "not json", + ]); + assert_eq!(o.records, 2); + assert_eq!(o.bad_lines, 2); + assert_eq!(o.fixtures.len(), 1); + let f = &o.fixtures[0]; + assert_eq!(f.duration_ms, Some(1200)); + assert_eq!(f.services, BTreeSet::from(["billing".to_string(), "ledger".to_string()])); + assert_eq!(f.cost(), 1200 * 4); + } + + #[test] + fn safe_takes_every_fixture_that_ran_the_change_and_minimal_the_cheapest_cover() { + let (dir, caches) = project("select", &[("billing/server.go", BILLING)]); + let fee = def(&caches, "billing/server.go", "fee"); + let charge = caches[0].funcs.iter().position(|f| f.name == "Charge" && f.owner.as_deref() == Some("Server")).unwrap(); + let rec = |name: &str, kind: &str, ms: u64, funcs: &str| { + format!(r#"{{"schema":"ccc-observed/1","fixture":{{"name":"{name}","kind":"{kind}","language":"go"}},"duration_ms":{ms},"files":[{{"path":"billing/server.go","functions":[{funcs}]}}]}}"#) + }; + let lines = [ + // runs both changed functions, but as a smoke test that takes a minute + rec("TestSmoke", "smoke", 60_000, r#"{"name":"fee"},{"name":"(*Server).Charge"}"#), + rec("TestFee", "unit", 50, r#"{"name":"fee"}"#), + rec("TestCharge", "contract", 400, r#"{"name":"(*Server).Charge"}"#), + // ran nothing that changed + rec("TestLedger", "unit", 10, r#"{"name":"(*Ledger).Charge"}"#), + ]; + let o = observed(&dir.0, &caches, &lines.iter().map(String::as_str).collect::>()); + let changed = [fee, (0, charge)]; + + let safe = o.safe(&changed); + assert_eq!(names(&o, &safe), vec!["TestSmoke", "TestFee", "TestCharge"]); + + let minimal = o.minimal(&changed); + assert!(minimal.exact); + assert_eq!(names(&o, &minimal), vec!["TestFee", "TestCharge"]); + assert_eq!(minimal.cost, 50 + 400 * 2); + } + + #[test] + fn the_greedy_cover_still_covers_everything() { + // three elements - one candidate covers two cheaply, the rest one each + let covers: Vec> = vec![BTreeSet::from([0, 1]), BTreeSet::from([1]), BTreeSet::from([2])]; + let picked = greedy_cover(&covers, &[2, 1, 5], 3); + let covered: BTreeSet = picked.iter().flat_map(|&c| covers[c].iter().copied()).collect(); + assert_eq!(covered, BTreeSet::from([0, 1, 2])); + assert_eq!(picked, vec![0, 2]); + assert_eq!(exact_cover(&covers, &[2, 1, 5], 3), Some(vec![0, 2])); + } +} diff --git a/src/serve.rs b/src/serve.rs index 8f19299..88c74a3 100644 --- a/src/serve.rs +++ b/src/serve.rs @@ -2972,8 +2972,7 @@ fn mcp_tools() -> Value { ), tool("refresh", "RESCANS the tree into memory. `edit_apply` and `edit_revert` already do this before they answer; call it after a change made outside ccc - a branch switch, a person editing - when the next answer cannot wait the three seconds the watcher takes to notice.", json!({}), &[]), - // analysis tools. All six are views onto one pass, computed once - // per (map generation, base), paged. + // analysis tools - each pages one pass computed once per map generation and base - pr_summary and a filtered prompts read transcripts instead tool( "changes", "ANSWERS what have I changed on this branch, at function granularity rather than line granularity. Use instead of reading `git diff` by hand when you want to know which functions moved and what they touch. Diffed against a base ref: changed functions with the tests that name them, which services need testing, service edges, and the calls the resolver refused to attribute. Includes uncommitted edits and untracked files by default. This is the change set `test_triggers` refers to.", @@ -3012,9 +3011,10 @@ fn mcp_tools() -> Value { ), tool( "test_triggers", - "CALL BEFORE RUNNING ANY TEST SUITE, and again after editing: it answers which tests your changes actually put at risk, so you run those instead of everything, and which changes no test covers at all. Tests are matched to changed functions through the call graph, so a change deep in the stack still surfaces the tests above it; `distance` is how many call hops away each sits. Returns a runnable command per language.", + "CALL BEFORE RUNNING ANY TEST SUITE, and again after editing: it answers which tests your changes actually put at risk, so you run those instead of everything, and which changes no test covers at all. Tests are matched to changed functions through the call graph, so a change deep in the stack still surfaces the tests above it; `distance` is how many call hops away each sits. Returns a runnable command per language. Where the project has records of a test run (`ccc-observed/1`), it also lists the fixtures that run saw execute the change - across services and languages the call graph cannot follow - as the `safe` set (every one of them, the default) or the `minimal` one (the cheapest that still runs each changed function).", json!({ "base": {"type": "string", "description": "git ref to diff against (default: merge-base with origin/main, main, origin/master or master - first that exists)"}, + "select": {"type": "string", "enum": ["safe", "minimal"], "description": "which observed fixtures to list: `safe` every one that ran the change (default), `minimal` the cheapest that still runs each changed function"}, "limit": {"type": "integer", "description": "triggered tests and gaps per page (default 25, max 500)"}, "offset": {"type": "integer", "description": "triggered tests and gaps to skip (default 0)"}, }), @@ -3061,10 +3061,10 @@ fn mcp_tools() -> Value { &[], ), - // the write tools - the only way this project is changed + // the only way to the web - and where agents broke these rules tool( "search", - "USE INSTEAD OF WebFetch, WebSearch, curl or wget for anything on the web - internal docs and portals included. `query` runs a search against the endpoints the user configured (search pages with `{query}` where the words go) and returns each one's results as text, links written `[text](url)`; `url` fetches one https page as text - follow a result's link this way. https only. Every request, redirects included, passes the user's rules: in `allow` mode only listed urls are reachable, in `block` mode listed urls are refused, and with no mode any https url goes. A refusal names the rule that refused it - do not reach the url another way: `violations` reports that to the user. Long pages page: pass `offset` for the rest.", + "USE INSTEAD OF WebFetch, WebSearch, a browser or fetch tool from another MCP server, curl or wget for anything on the web - internal docs and portals included. `query` runs a search against the endpoints the user configured (search pages with `{query}` where the words go) and returns each one's results as text, links written `[text](url)`; `url` fetches one https page as text - follow a result's link this way. https only. Every request, redirects included, passes the user's rules: in `allow` mode only listed urls are reachable, in `block` mode listed urls are refused, and with no mode any https url goes. A refusal names the rule that refused it - do not reach the url another way: `violations` reports that to the user. Long pages page: pass `offset` for the rest.", json!({ "query": {"type": "string", "description": "words to search the configured endpoints for"}, "url": {"type": "string", "description": "one https page to fetch as text, instead of a query"}, @@ -3076,7 +3076,7 @@ fn mcp_tools() -> Value { ), tool( "violations", - "ANSWERS whether any agent on this project broke ccc's rules, read from the agents' own transcripts (Claude Code, subagents included): writing project files around ccc's edit tools (Edit/Write, `sed -i`, `>` or `tee`, `cp`/`mv`/`rm`), reading or searching the project with Read/Grep/Glob or the shell (`cat`, `grep`, `rg`, `find`, `sed -n`, `ls`) instead of ccc's tools, reaching the web outside `search` (WebFetch, WebSearch, `curl`, `wget`), and urls `search` refused. Each violation carries its time, session, tool, the command or input, the files or urls it touched, and the ccc tool that does the job instead. Narrow with `rule`, `session` or `since` (days); pages with `offset`.", + "ANSWERS whether any agent on this project broke ccc's rules, read from the agents' own transcripts (Claude Code, subagents included): writing project files around ccc's edit tools (Edit/Write, `sed -i`, `>` or `tee`, `cp`/`mv`/`rm`), searching or listing the project with Grep/Glob or the shell (`grep`, `rg`, `find`, `ls`, or a `cat` of a glob) instead of ccc's tools - reading a named file is fine, reaching the web outside `search` (WebFetch, WebSearch, another MCP server's fetch or browser tool, `curl`, `wget`), and urls `search` refused. Each violation carries its time, session, tool, the command or input, the files or urls it touched, and the ccc tool that does the job instead. Narrow with `rule`, `session` or `since` (days); pages with `offset`.", json!({ "rule": {"type": "string", "enum": ["writes-outside-ccc", "shell-exploration", "web-outside-search", "blocked-url"], "description": "only this rule (optional)"}, "session": {"type": "string", "description": "only this session id (optional)"}, @@ -3086,6 +3086,8 @@ fn mcp_tools() -> Value { }), &[], ), + + // the write tools - the only way this project is changed tool( "edit_rename", "CHANGES A NAME EVERYWHERE IT IS THE SAME SYMBOL, in one call instead of a read and an edit per file. Pass the `result` handle a `find` or `references` answer ends with, and `to`. A `references` handle renames that exact name; a `find` handle swaps the matched substring inside each identifier and keeps its case shape (`hello_all` -> `hi_all`, `HelloWorld` -> `HiWorld`). Renaming only the definitions would strand their callers, so each name is followed through every file the map ties to it - definitions, calls, usages, imports, re-exports, and lines the map does not index such as a `ccc:skip` body - provided it has exactly one definition and the lookup was not qualified. Only identifiers change: never a comment, a string, or the module segment of a path. Every same-named identifier it did NOT change is listed with the reason, under a new handle that pulls them in when passed back; comments and strings that spell the name are listed too, and a new name that collides with an existing definition is flagged. Stages a changeset and returns its diff - nothing is written until `edit_apply` (or apply=true).", @@ -3195,7 +3197,7 @@ fn mcp_tools() -> Value { } // what `--auto` adds to the instructions - agents mark the calls that leave this process, which no parser can follow -const CROSSINGS: &str = "\n\n4. CROSS-SERVICE CALLS - annotate every one you identify. A call that \ +const CROSSINGS: &str = "\n\n5. CROSS-SERVICE CALLS - annotate every one you identify. A call that \ leaves this process - an HTTP or gRPC request to another service, a message published to or \ consumed from a queue or topic, a webhook, a CLI or FFI hop - is invisible to the map until a \ comment names it. Whenever you read, write or change one, mark each end that is in this project \ @@ -3272,7 +3274,13 @@ fn mcp_initialize(params: &Value, auto: bool) -> Value { connects, not every character of it: read the real source before writing code \ that depends on its exact text. Never write under `.ccc` except map.json and \ surface.json.\n\n\ - 3. TOOLS. Map: `index` project overview - every file named in path order, \ + 3. THE WEB - `search` is the only way to the web, internal docs and portals \ + included: `query` searches the endpoints the user configured, `url` fetches one \ + https page. Do not use WebFetch, WebSearch, another server's fetch or browser \ + tool, `curl` or `wget` - ccc cannot see those calls, and `violations` reports \ + each one to the user. A url `search` refuses is refused by the user's rules: \ + never reach it another way.\n\n\ + 4. TOOLS. Map: `index` project overview - every file named in path order, \ never collapsed into directory summaries, most projects whole; past ~1000 \ lines the rest waits behind `offset`, and `path` narrows to a subtree. \ `find` symbols by \ @@ -3295,7 +3303,7 @@ fn mcp_initialize(params: &Value, auto: bool) -> Value { what entered it, what left, what moved version and which advisories that introduced. \ `security` syntax-level security findings in this \ project's own code - secrets, disabled TLS, injection surfaces. `search` \ - is the only way to the web, internal docs included, and `violations` \ + is the only way to the web, internal portals included, and `violations` \ reports where agents broke these rules. Edits: \ `edit_rename`, `edit_replace`, `edit_delete`, `edit_insert` and `edit_text` \ stage a changeset, `edit_apply` writes it, `edit_discard` drops it, \ @@ -4305,7 +4313,76 @@ fn md_changes(v: &Value, page: Page) -> String { out } -fn md_triggers(v: &Value, targets: &Value, page: Page) -> String { +// A name filter selecting 80 tests runs to kilobytes, and pasting it is +// worse than running the suite. Never truncate it - a half-copied command +// silently selects the wrong tests - so past the budget, describe it and +// point at where in /insights.json it is served whole. +fn md_commands(cmds: &[Value], whole_at: &str) -> String { + const MAX_COMMAND: usize = 1200; + cmds.iter() + .map(|x| { + let cmd = jstr(x, "command"); + if cmd.len() > MAX_COMMAND { + format!( + "({}: a filter naming {} tests, {} chars - omitted)\n \ + Run the whole suite instead; the exact command is in /insights.json \ + under {whole_at}.\n", + jstr(x, "language"), + jnum(x, "selects"), + cmd.len(), + ) + } else { + format!( + "$ {cmd}\n ({}, selects {}) {}\n", + jstr(x, "language"), + jnum(x, "selects"), + jstr(x, "caveat") + ) + } + }) + .collect() +} + +// what a recorded run saw execute the change, in the selection asked for - safe unless `minimal` +fn md_observed(v: &Value, select: Option<&str>) -> Option<(String, String)> { + let o = v.get("observed")?; + let mode = if select == Some("minimal") { "minimal" } else { "safe" }; + let sel = &o[mode]; + let fixtures = jarr(sel, "fixtures"); + let mut body = format!( + "{} fixture(s), cost {} - a recorded run saw {} of {} changed function(s) execute, from {} record(s) in {} source(s){}\n", + fixtures.len(), + jnum(sel, "cost"), + jnum(o, "observed_functions"), + jnum(&v["counts"], "changed_functions"), + jnum(o, "records"), + jarr(o, "sources").len(), + if mode == "minimal" && !jbool(sel, "exact") { " - found greedily, so not proven the cheapest" } else { "" }, + ); + body.push_str(&md_commands(&jarr(sel, "commands"), &format!("test_triggers.observed.{mode}.commands"))); + for f in &fixtures { + let covers: Vec = jarr(f, "covers").iter().filter_map(|c| c.as_str().map(str::to_string)).collect(); + let place = if jstr(f, "file").is_empty() { "(outside this repo)".to_string() } else { at(f) }; + body.push_str(&format!( + "[{}] {} {} - ran {}{}\n", + jstr(f, "kind"), + place, + jstr(f, "fixture"), + if covers.is_empty() { "nothing that changed".to_string() } else { covers.join(", ") }, + if jbool(f, "changed") { " - and changed itself" } else { "" }, + )); + } + let unobserved = jarr(o, "unobserved").len(); + if unobserved > 0 { + body.push_str(&format!("{unobserved} changed function(s) no recorded fixture ran - the run list above is all there is for them\n")); + } + for e in jarr(o, "errors") { + body.push_str(&format!("! {}\n", e.as_str().unwrap_or_default())); + } + Some((format!("observed - {mode}"), body)) +} + +fn md_triggers(v: &Value, targets: &Value, select: Option<&str>, page: Page) -> String { if let Some(why) = md_unavailable("test triggers", v) { return why; } @@ -4327,35 +4404,7 @@ fn md_triggers(v: &Value, targets: &Value, page: Page) -> String { ); } - // A name filter selecting 80 tests runs to kilobytes, and pasting it is - // worse than running the suite. Never truncate it - a half-copied command - // silently selects the wrong tests - so past the budget, describe it and - // point at the endpoint that serves it whole. - const MAX_COMMAND: usize = 1200; - let cmds: String = jarr(v, "commands") - .iter() - .map(|x| { - let cmd = jstr(x, "command"); - if cmd.len() > MAX_COMMAND { - format!( - "({}: a filter naming {} tests, {} chars - omitted)\n \ - Run the whole suite instead; the exact command is in /insights.json \ - under test_triggers.commands.\n", - jstr(x, "language"), - jnum(x, "selects"), - cmd.len(), - ) - } else { - format!( - "$ {cmd}\n ({}, selects {}) {}\n", - jstr(x, "language"), - jnum(x, "selects"), - jstr(x, "caveat") - ) - } - }) - .collect(); - md_section(&mut out, "commands", &cmds); + md_section(&mut out, "commands", &md_commands(&jarr(v, "commands"), "test_triggers.commands")); let run = jarr(v, "run"); let (window, note) = page.apply(&run); @@ -4377,6 +4426,9 @@ fn md_triggers(v: &Value, targets: &Value, page: Page) -> String { }) .collect(); md_section(&mut out, &format!("run {note}"), &body); + if let Some((title, body)) = md_observed(v, select) { + md_section(&mut out, &title, &body); + } // Each gap is a `target` id into `test_targets`; the recommendation lives // there and is looked up rather than restated. @@ -4955,14 +5007,13 @@ fn mcp_tool_call(state: &RwLock, params: &Value, browser: bool) -> Res } } } - "test_triggers" => { - let a = map.analysis(arg("base").as_deref()); - Ok(md_triggers( - &a["test_triggers"], - &a["test_targets"], - Page::from(&args, 25), - )) - } + "test_triggers" => match arg("select").as_deref() { + Some(s) if !matches!(s, "safe" | "minimal") => Err(format!("select wants `safe` or `minimal`, got '{s}'")), + select => { + let a = map.analysis(arg("base").as_deref()); + Ok(md_triggers(&a["test_triggers"], &a["test_targets"], select, Page::from(&args, 25))) + } + }, "test_targets" => { let a = map.analysis(None); Ok(md_targets( @@ -7432,6 +7483,39 @@ mod tests { assert_eq!(route(&state, "GET", "/vis/flow?file=lib%2Fmoney.rs&line=9999", b"").status, 404); } + // the observed section shows the selection asked for, and says when a cover was only found greedily + #[test] + fn test_triggers_render_the_observed_selection_asked_for() { + let fixture = |name: &str| { + json!({"fixture": name, "file": "tests/test_api.py", "line": 3, "kind": "integration", "covers": ["Fee"], "changed": false}) + }; + let v = json!({ + "available": true, + "counts": {"changed_functions": 2}, + "observed": { + "records": 4, + "sources": [".ccc/observed"], + "observed_functions": 1, + "safe": {"fixtures": [fixture("test_refund"), fixture("test_soak")], "cost": 458000, "exact": true, + "commands": [{"language": "python", "command": "pytest -k \"test_refund or test_soak\"", "selects": 2, "caveat": ""}]}, + "minimal": {"fixtures": [fixture("test_refund")], "cost": 8000, "exact": false, "commands": []}, + "unobserved": [{"file": "svc/a.go", "function": "Spare"}], + "errors": [], + }, + }); + let page = Page::from(&json!({}), 25); + let safe = md_triggers(&v, &json!({}), None, page); + assert!(safe.contains("## observed - safe") && safe.contains("test_soak"), "{safe}"); + assert!(safe.contains("$ pytest -k \"test_refund or test_soak\""), "{safe}"); + assert!(safe.contains("1 changed function(s) no recorded fixture ran"), "{safe}"); + let minimal = md_triggers(&v, &json!({}), Some("minimal"), page); + assert!(minimal.contains("## observed - minimal") && !minimal.contains("test_soak"), "{minimal}"); + assert!(minimal.contains("found greedily"), "{minimal}"); + // a report without records has no observed section at all + let plain = md_triggers(&json!({"available": true}), &json!({}), None, page); + assert!(!plain.contains("observed"), "{plain}"); + } + // Call every analysis tool the way an agent would, and check the two // things that make one usable: it renders as markdown rather than raw // JSON, and a list it cannot fit says how to get the rest. diff --git a/src/violations.rs b/src/violations.rs index fd68959..b63dc97 100644 --- a/src/violations.rs +++ b/src/violations.rs @@ -50,7 +50,7 @@ impl Rule { pub fn says(self) -> &'static str { match self { Rule::WritesOutsideCcc => "project files are written only through ccc's edit tools", - Rule::ShellExploration => "the project is read and searched only through ccc's tools", + Rule::ShellExploration => "the project is searched and listed only through ccc's tools - reading a named file is fine", Rule::WebOutsideSearch => "the web is reached only through ccc's `search`", Rule::BlockedUrl => "a url the search rules refuse is not to be reached", } @@ -280,8 +280,8 @@ fn classify(tool: &str, input: &Value, cwd: &Path, root: &Path) -> Vec<(Rule, Ve match tool { "Edit" | "Write" | "MultiEdit" => one(Rule::WritesOutsideCcc, path("file_path")), "NotebookEdit" => one(Rule::WritesOutsideCcc, path("notebook_path")), - "Read" => one(Rule::ShellExploration, path("file_path")), - "NotebookRead" => one(Rule::ShellExploration, path("notebook_path")), + // reading a named file is not exploring - searching or listing the tree is + "Read" | "NotebookRead" => Vec::new(), // a search with no path searches where the agent stands "Grep" | "Glob" | "LS" => { let at = input["path"].as_str().map_or_else(|| cwd.to_path_buf(), |p| cwd.join(p)); @@ -325,6 +325,8 @@ fn project_path(root: &Path, path: &Path) -> Option { const WRITERS: &[&str] = &["tee", "rm", "rmdir", "unlink", "touch", "truncate", "shred"]; // commands that list or search a tree, the current directory when they name none const WALKERS: &[&str] = &["find", "fd", "fdfind", "ls", "tree", "rg", "ag", "du"]; +// readers that search what they read rather than show it +const SEARCHERS: &[&str] = &["grep", "egrep", "fgrep", "rg", "ag"]; // commands that reach the network const FETCHERS: &[&str] = &["curl", "wget", "http", "https", "xh", "httpie", "aria2c", "lynx", "w3m", "links"]; // git subcommands that write the working tree @@ -388,7 +390,11 @@ fn shell(line: &str, cwd: &Path, root: &Path) -> Vec<(Rule, Vec)> { "find" => args.iter().copied().take_while(|a| !a.starts_with('-') && !matches!(*a, "(" | "!")).collect(), _ => plain.iter().copied().filter(|a| !a.bytes().all(|b| b.is_ascii_digit())).skip(pattern).collect(), }; - let files: Vec = named.iter().copied().chain(part.fed.iter().map(String::as_str)).filter_map(|f| at(f, &cwd)).collect(); + // showing a named file is a read - searching or listing the tree or showing a glob explores it + let explores = SEARCHERS.contains(&n) || WALKERS.contains(&n); + let shown = named.iter().copied().filter(|f| explores || f.contains(['*', '?'])); + let fed = part.fed.iter().map(String::as_str).filter(|_| explores); + let files: Vec = shown.chain(fed).filter_map(|f| at(f, &cwd)).collect(); if !files.is_empty() { reads.extend(files); } else if named.is_empty() && part.fed.is_empty() && !part.piped && walks(n, &args) { @@ -554,11 +560,11 @@ mod tests { } #[test] - fn exploring_with_the_shell_is_caught_but_filtering_a_pipe_is_not() { - for line in ["cat src/main.rs", "sed -n '1,40p' src/main.rs", "grep -rn insights src", "grep -rn insights", "rg foo", "find . -name '*.rs'", "ls docs", "head -20 README.md"] { + fn exploring_with_the_shell_is_caught_but_reading_a_file_or_filtering_a_pipe_is_not() { + for line in ["grep -rn insights src", "grep -rn insights", "grep -n fn src/main.rs", "rg foo", "find . -name '*.rs'", "ls docs", "cat src/*.rs"] { assert_eq!(rule_of(line), vec![Rule::ShellExploration], "{line}"); } - for line in ["cargo test | grep FAILED", "git log | head", "ls /tmp", "cat /etc/hosts", "echo x | wc -l"] { + for line in ["cat src/main.rs", "sed -n '1,40p' src/main.rs", "head -20 README.md", "cargo test | grep FAILED", "git log | head", "ls /tmp", "cat /etc/hosts", "echo x | wc -l"] { assert!(rule_of(line).is_empty(), "{line}"); } } @@ -575,7 +581,7 @@ mod tests { #[test] fn a_cd_moves_where_relative_paths_land() { assert!(rule_of("cd /tmp && cat notes.txt").is_empty()); - assert_eq!(rules("cd src && cat main.rs")[0].1, vec!["src/main.rs"]); + assert_eq!(rules("cd src && grep -n fn main.rs")[0].1, vec!["src/main.rs"]); } #[test] @@ -584,10 +590,11 @@ mod tests { let c = |tool: &str, input: Value| classify(tool, &input, cwd, root).into_iter().map(|(r, _)| r).collect::>(); assert_eq!(c("Edit", json!({"file_path": "/p/src/x.rs"})), vec![Rule::WritesOutsideCcc]); assert!(c("Write", json!({"file_path": "/tmp/scratch.md"})).is_empty()); - assert_eq!(c("Read", json!({"file_path": "/p/README.md"})), vec![Rule::ShellExploration]); + assert!(c("Read", json!({"file_path": "/p/README.md"})).is_empty()); assert_eq!(c("Grep", json!({"pattern": "x"})), vec![Rule::ShellExploration]); assert_eq!(c("WebFetch", json!({"url": "https://example.com"})), vec![Rule::WebOutsideSearch]); assert_eq!(c("mcp__fetch__fetch", json!({"url": "https://example.com"})), vec![Rule::WebOutsideSearch]); + assert_eq!(c("mcp__playwright__browser_navigate", json!({"url": "https://example.com"})), vec![Rule::WebOutsideSearch]); assert!(c("mcp__ccc__find", json!({"query": "x"})).is_empty()); assert!(c("mcp__ccc__search", json!({"url": "https://example.com"})).is_empty()); }