From 93abeec4e3d5d9a80821cdee7372c9f937d1b47d Mon Sep 17 00:00:00 2001 From: ShawnX Date: Thu, 30 Jul 2026 08:29:13 +0800 Subject: [PATCH 1/4] feat: add support for Copilot platform in session analysis and related scripts - Updated asset providers to include "copilot" in agent customization. - Modified CLI options to support "copilot" as a valid platform. - Enhanced contract definitions to recognize "copilot" as a provider. - Updated report quality checks to include Copilot in session scope. - Adjusted report run and task loop scripts to accommodate Copilot. - Implemented new session analysis logic for Copilot, including event normalization and workspace handling. - Added Copilot-specific session analyzer and event processing in session analysis. - Updated help documentation and usage summaries to reflect Copilot integration. - Adjusted tests to validate changes related to Copilot support. --- .github/plugin/marketplace.json | 27 + .github/plugin/plugin.json | 23 + package.json | 1 + scripts/agent-customize/cli.mjs | 5 +- scripts/agent-customize/providers/copilot.mjs | 425 +++++++++++++ scripts/agent-customize/providers/index.mjs | 2 + scripts/agent-lint/cli.mjs | 2 +- scripts/better-harness-cli/registry.mjs | 2 +- .../coding-agent-practices/asset-baseline.mjs | 6 +- .../asset-integrity.mjs | 6 +- scripts/coding-agent-practices/inventory.mjs | 14 +- .../evidence-bundle/agent-customize.mjs | 2 +- .../harness-analysis/evidence-bundle/cli.mjs | 4 +- .../evidence-bundle/contract.mjs | 2 +- scripts/harness-analysis/report-quality.mjs | 2 +- scripts/harness-analysis/report-run.mjs | 6 +- scripts/harness-analysis/task-loop-report.mjs | 2 +- scripts/harness-analysis/task-loop-source.mjs | 7 +- scripts/npm-package/verify-pack.mjs | 5 + scripts/session-analysis.mjs | 11 +- scripts/session-analysis/analyzer.mjs | 19 +- .../lifecycle-demand-signals.mjs | 2 +- .../session-analysis/platforms/copilot.mjs | 570 ++++++++++++++++++ .../session-analysis/selection-profile.mjs | 2 +- .../session-analysis/session-core-facts.mjs | 2 + scripts/session-analysis/usage-summary.mjs | 2 +- .../scripts-refactor-contract/root-help.txt | 4 +- .../session-help.txt | 2 +- test/scripts-refactor-contract.test.mjs | 4 +- 29 files changed, 1118 insertions(+), 43 deletions(-) create mode 100644 .github/plugin/marketplace.json create mode 100644 .github/plugin/plugin.json create mode 100644 scripts/agent-customize/providers/copilot.mjs create mode 100644 scripts/session-analysis/platforms/copilot.mjs diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json new file mode 100644 index 0000000..aad8bda --- /dev/null +++ b/.github/plugin/marketplace.json @@ -0,0 +1,27 @@ +{ + "name": "better-harness", + "owner": { + "name": "Qoder", + "email": "dev@qoder.com" + }, + "metadata": { + "description": "Better Harness plugins for AI delivery readiness.", + "version": "0.3.0" + }, + "plugins": [ + { + "name": "better-harness", + "description": "Build an AI-ready engineering system for safe coding-agent delivery and continuous software improvement.", + "version": "0.3.0", + "source": "./", + "author": { + "name": "Qoder", + "email": "dev@qoder.com" + }, + "homepage": "https://github.com/QoderAI/better-harness", + "repository": "https://github.com/QoderAI/better-harness", + "license": "MIT", + "skills": "./skills/" + } + ] +} diff --git a/.github/plugin/plugin.json b/.github/plugin/plugin.json new file mode 100644 index 0000000..f4d2eba --- /dev/null +++ b/.github/plugin/plugin.json @@ -0,0 +1,23 @@ +{ + "name": "better-harness", + "description": "Build an AI-ready engineering system for safe coding-agent delivery and continuous software improvement.", + "version": "0.3.0", + "author": { + "name": "Qoder", + "email": "dev@qoder.com", + "url": "https://qoder.com/" + }, + "homepage": "https://github.com/QoderAI/better-harness", + "repository": "https://github.com/QoderAI/better-harness", + "license": "MIT", + "keywords": [ + "copilot-plugin", + "better-harness", + "ai-delivery", + "continuous-improvement", + "agent-harness", + "change-confidence" + ], + "category": "Coding", + "skills": "./skills/" +} diff --git a/package.json b/package.json index d0e7e1e..3646dd3 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ ".claude-plugin/", ".codex-plugin/", ".cursor-plugin/", + ".github/plugin/", ".qoder-plugin/", "qwen-extension.json", "AGENTS.md", diff --git a/scripts/agent-customize/cli.mjs b/scripts/agent-customize/cli.mjs index 63b9ada..5605b6d 100644 --- a/scripts/agent-customize/cli.mjs +++ b/scripts/agent-customize/cli.mjs @@ -5,12 +5,12 @@ import { collectAgentCustomizeInventory, filterManageItems, groupManageItems } f function usage() { return [ - "Usage: better-harness agent-customize [inventory|manage] --provider [--workspace ]", + "Usage: better-harness agent-customize [inventory|manage] --provider [--workspace ]", " better-harness agent-customize manage --provider [--tab ] [--query ] [--scope ] [--group-by ]", "", "Collect configured agent-customize inventory for one provider as JSON.", "Provider home overrides: --cursor-home, --qoder-home, --codex-home, --claude-home,", - "--qwen-home, --claude-state, --codex-app-path, --qoder-shared-client-cache-root.", + "--qwen-home, --copilot-home, --claude-state, --codex-app-path, --qoder-shared-client-cache-root.", "", ].join("\n"); } @@ -30,6 +30,7 @@ function summarize(inventory, options) { codexHome: inventory.codexHome, claudeHome: inventory.claudeHome, qwenHome: inventory.qwenHome, + copilotHome: inventory.copilotHome, claudeStatePath: inventory.claudeStatePath, codexAppPath: inventory.codexAppPath, sharedClientCacheRoot: inventory.sharedClientCacheRoot, diff --git a/scripts/agent-customize/providers/copilot.mjs b/scripts/agent-customize/providers/copilot.mjs new file mode 100644 index 0000000..a04542e --- /dev/null +++ b/scripts/agent-customize/providers/copilot.mjs @@ -0,0 +1,425 @@ +import os from "node:os"; +import path from "node:path"; + +import { pathExists } from "../../session-analysis/fs.mjs"; +import { expandHome, normalizeWorkspace } from "../../session-analysis/paths.mjs"; +import { MANAGE_TABS } from "../constants.mjs"; +import { + agentsMarkdownRuleSource, + buildManageCollections, + collectHookItems, + collectMarkdownItems, + collectMcpFromConfig, + collectRuleSources, + collectSkillFiles, + designMarkdownRuleSource, + directoryRuleSource, + evidence, + normalizePluginDisplayName, + pluginMetadataEvidencePath, + readJson, + readText, + sortByName, + titleCase, + uniqueAssetsByRealPath, + workspaceSourceLabel, +} from "../core/items.mjs"; + +// Copilot resolves a plugin manifest from these roots, in this order. +const COPILOT_PLUGIN_MANIFESTS = [ + [".plugin", "plugin.json"], + ["plugin.json"], + [".github", "plugin", "plugin.json"], + [".claude-plugin", "plugin.json"], +]; + +function defaultCopilotHome() { + return process.env.COPILOT_HOME ?? path.join(os.homedir(), ".copilot"); +} + +// Copilot writes `config.json` and `settings.json` as JSONC with whole-line `//` +// comments. Only full-line comments are stripped so string values that contain +// `//` (for example URLs) stay intact. +async function readCopilotJson(filePath) { + const text = await readText(filePath, 512_000); + if (!text) return undefined; + const stripped = text + .split(/\r?\n/u) + .filter((line) => !/^\s*\/\//u.test(line)) + .join("\n"); + try { + return JSON.parse(stripped); + } catch { + return undefined; + } +} + +function copilotInstructionsRuleSource(filePath, scope, sourceLabel, rootForEvidence, precedence) { + return { + type: "file", + filePath, + scope, + sourceLabel, + rootForEvidence, + name: path.basename(filePath), + sourceKind: "copilot-instructions", + precedence, + useHeading: true, + }; +} + +function normalizeSettingsMcp(name, config, scope, sourceLabel, evidencePath, rootForEvidence) { + return { + name, + scope, + sourceLabel, + command: config.command ?? null, + args: config.args ?? [], + url: config.url ?? config.httpUrl ?? null, + evidence: evidence(evidencePath, rootForEvidence), + }; +} + +function flattenSettingsHooks(settingsHooks) { + if (!settingsHooks || typeof settingsHooks !== "object" || Array.isArray(settingsHooks)) return []; + const items = []; + for (const [, definitions] of Object.entries(settingsHooks)) { + if (!Array.isArray(definitions)) continue; + for (const definition of definitions) { + const hooks = Array.isArray(definition?.hooks) ? definition.hooks : [definition]; + for (const hook of hooks) { + const command = hook?.command ?? hook?.bash ?? hook?.powershell; + if (command) items.push({ command, type: hook?.type ?? "command" }); + else if (hook?.url) items.push({ command: hook.url, type: "http" }); + } + } + } + return items; +} + +function normalizeProvidedCopilotRecord(record) { + const name = String(record?.name ?? "").trim(); + const installPath = record?.cache_path ?? record?.cachePath ?? record?.installPath; + if (!name || !installPath) { + return undefined; + } + const marketplaceName = record.marketplace || "_direct"; + return { + id: `${marketplaceName}/${name}`, + name, + marketplaceName, + installPath: path.resolve(expandHome(installPath)), + version: record.version, + enabled: record.enabled !== false, + source: record.source?.source ?? (record.marketplace ? "marketplace" : "direct"), + sources: [record.marketplace ? "marketplace" : "direct"], + installMatch: record.marketplace ? "copilot-marketplace" : "copilot-direct", + installedAt: record.installed_at ?? record.installedAt, + }; +} + +async function readCopilotInstalledPluginState(options = {}) { + if (Array.isArray(options.copilotInstalledPluginRecords) || Array.isArray(options.installedPluginRecords)) { + const records = (options.copilotInstalledPluginRecords ?? options.installedPluginRecords) + .map(normalizeProvidedCopilotRecord) + .filter(Boolean); + return { records, source: "provided", installRecordFiles: [] }; + } + + const copilotHome = path.resolve(expandHome(options.copilotHome ?? options["copilot-home"] ?? defaultCopilotHome())); + const configPath = path.join(copilotHome, "config.json"); + const config = (await readCopilotJson(configPath)) ?? {}; + const entries = Array.isArray(config.installedPlugins) ? config.installedPlugins : []; + const records = entries.map(normalizeProvidedCopilotRecord).filter(Boolean); + return { + records, + source: records.length > 0 ? "copilot-config" : "missing", + installRecordFiles: records.length > 0 ? [configPath] : [], + }; +} + +async function collectCopilotPluginMcpItems(pluginRoot, sourceLabel, manifest) { + const items = []; + if (manifest?.mcpServers && typeof manifest.mcpServers === "object") { + for (const [name, config] of Object.entries(manifest.mcpServers)) { + if (!config || typeof config !== "object") continue; + items.push({ + name, + scope: "plugin", + sourceLabel, + command: config.command ?? null, + args: config.args ?? [], + evidence: evidence(path.join(pluginRoot, "plugin.json"), pluginRoot), + }); + } + } + for (const candidate of [ + path.join(pluginRoot, ".mcp.json"), + path.join(pluginRoot, ".github", "mcp.json"), + ]) { + if (await pathExists(candidate)) { + items.push(...(await collectMcpFromConfig(candidate, "plugin", sourceLabel, pluginRoot))); + } + } + return items; +} + +async function readCopilotPluginManifest(pluginRoot) { + for (const segments of COPILOT_PLUGIN_MANIFESTS) { + const candidate = path.join(pluginRoot, ...segments); + if (await pathExists(candidate)) { + return (await readJson(candidate)) ?? {}; + } + } + return {}; +} + +function manifestPaths(manifest, key, fallback) { + const value = manifest?.[key]; + if (typeof value === "string") return [value]; + if (Array.isArray(value)) return value.filter((entry) => typeof entry === "string"); + return fallback; +} + +async function collectCopilotPlugin(record) { + const pluginRoot = path.resolve(expandHome(record.installPath)); + if (!(await pathExists(pluginRoot))) { + return null; + } + const metadataEvidencePath = await pluginMetadataEvidencePath(pluginRoot, COPILOT_PLUGIN_MANIFESTS); + const manifest = await readCopilotPluginManifest(pluginRoot); + const readme = await readText(path.join(pluginRoot, "README.md"), 6000); + const heading = readme.match(/^#\s+(.+)$/mu)?.[1]?.trim(); + const rawDisplayName = manifest.displayName || heading || titleCase(manifest.name || record.name); + const displayName = normalizePluginDisplayName(rawDisplayName, record.name); + const plugin = { + id: record.id, + copilotPluginName: record.name, + marketplaceName: record.marketplaceName, + rootPath: pluginRoot, + scope: "plugin", + sourceLabel: displayName, + name: manifest.name || record.name, + displayName, + description: manifest.description || "", + publisher: { displayName: titleCase(record.marketplaceName) }, + version: record.version || manifest.version, + installSources: record.sources, + installSource: record.source, + installMatch: record.installMatch, + installedAt: record.installedAt, + enabled: record.enabled, + evidence: evidence(metadataEvidencePath, path.dirname(path.dirname(pluginRoot))), + }; + + const skillRoots = manifestPaths(manifest, "skills", ["skills"]); + const agentRoots = manifestPaths(manifest, "agents", ["agents"]); + const commandRoots = manifestPaths(manifest, "commands", ["commands"]); + + plugin.skills = ( + await Promise.all( + skillRoots.map((root) => collectSkillFiles(path.resolve(pluginRoot, root), "plugin", displayName, pluginRoot)), + ) + ) + .flat() + .sort(sortByName); + plugin.subagents = ( + await Promise.all( + agentRoots.map((root) => + collectMarkdownItems(path.resolve(pluginRoot, root), "subagent", "plugin", displayName, pluginRoot), + ), + ) + ) + .flat() + .sort(sortByName); + plugin.commands = ( + await Promise.all( + commandRoots.map((root) => + collectMarkdownItems(path.resolve(pluginRoot, root), "command", "plugin", displayName, pluginRoot), + ), + ) + ) + .flat() + .sort(sortByName); + plugin.mcpServers = await collectCopilotPluginMcpItems(pluginRoot, displayName, manifest); + plugin.rules = []; + plugin.hooks = await collectHookItems(pluginRoot, "plugin", displayName, pluginRoot); + return plugin; +} + +async function collectCopilotPlugins(records) { + const plugins = []; + for (const record of records) { + const plugin = await collectCopilotPlugin(record); + if (plugin) { + plugins.push(plugin); + } + } + const byId = new Map(); + for (const plugin of plugins.sort(sortByName)) { + if (!byId.has(plugin.id)) { + byId.set(plugin.id, plugin); + } + } + return [...byId.values()].sort(sortByName); +} + +async function collectCopilotUserPrimitives(copilotHome) { + const settingsPath = path.join(copilotHome, "settings.json"); + const settings = (await readCopilotJson(settingsPath)) ?? {}; + const mcps = []; + const mcpConfigPath = path.join(copilotHome, "mcp-config.json"); + if (await pathExists(mcpConfigPath)) { + mcps.push(...((await collectMcpFromConfig(mcpConfigPath, "user", "User", copilotHome)) ?? [])); + } + if (settings.mcpServers && typeof settings.mcpServers === "object") { + for (const [name, config] of Object.entries(settings.mcpServers)) { + if (!config || typeof config !== "object") continue; + if (!mcps.some((mcp) => mcp.name === name)) { + mcps.push(normalizeSettingsMcp(name, config, "user", "User", settingsPath, copilotHome)); + } + } + } + + const hooks = await collectHookItems(copilotHome, "user", "User", copilotHome); + for (const hook of flattenSettingsHooks(settings.hooks)) { + if (!hooks.some((existing) => existing.command === hook.command)) { + hooks.push({ + command: hook.command, + scope: "user", + sourceLabel: "User", + evidence: evidence(settingsPath, copilotHome), + }); + } + } + + const skills = await uniqueAssetsByRealPath([ + ...(await collectSkillFiles(path.join(copilotHome, "skills"), "user", "User", copilotHome)), + ...(await collectSkillFiles( + path.join(path.dirname(copilotHome), ".agents", "skills"), + "user", + "User", + copilotHome, + )), + ]); + + return { + skills: skills.sort(sortByName), + subagents: await collectMarkdownItems(path.join(copilotHome, "agents"), "subagent", "user", "User", copilotHome), + rules: await collectRuleSources([ + copilotInstructionsRuleSource( + path.join(copilotHome, "copilot-instructions.md"), + "user", + "User", + copilotHome, + "before-project-rules", + ), + directoryRuleSource(path.join(copilotHome, "instructions"), "user", "User", copilotHome, { + sourceKind: "copilot-instructions", + precedence: "before-project-rules", + }), + ]), + commands: [], + hooks, + mcps, + }; +} + +async function collectCopilotWorkspacePrimitives(workspace) { + const sourceLabel = await workspaceSourceLabel(workspace); + const github = path.join(workspace, ".github"); + + const skills = await uniqueAssetsByRealPath([ + ...(await collectSkillFiles(path.join(github, "skills"), "project", sourceLabel, workspace)), + ...(await collectSkillFiles(path.join(workspace, ".agents", "skills"), "project", sourceLabel, workspace)), + ]); + + const mcps = []; + for (const candidate of [path.join(workspace, ".mcp.json"), path.join(github, "mcp.json")]) { + if (await pathExists(candidate)) { + for (const mcp of (await collectMcpFromConfig(candidate, "project", sourceLabel, workspace)) ?? []) { + if (!mcps.some((existing) => existing.name === mcp.name)) mcps.push(mcp); + } + } + } + + const settingsPath = path.join(github, "copilot", "settings.json"); + const settings = (await readCopilotJson(settingsPath)) ?? {}; + const hooks = await collectHookItems(github, "project", sourceLabel, workspace); + for (const hook of flattenSettingsHooks(settings.hooks)) { + if (!hooks.some((existing) => existing.command === hook.command)) { + hooks.push({ + command: hook.command, + scope: "project", + sourceLabel, + evidence: evidence(settingsPath, workspace), + }); + } + } + + return { + skills: skills.sort(sortByName), + subagents: await collectMarkdownItems(path.join(github, "agents"), "subagent", "project", sourceLabel, workspace), + rules: await collectRuleSources([ + copilotInstructionsRuleSource( + path.join(github, "copilot-instructions.md"), + "project", + sourceLabel, + workspace, + "before-agents-md", + ), + directoryRuleSource(path.join(github, "instructions"), "project", sourceLabel, workspace, { + sourceKind: "copilot-instructions", + precedence: "before-agents-md", + }), + agentsMarkdownRuleSource(workspace, sourceLabel), + designMarkdownRuleSource(workspace, sourceLabel), + ]), + commands: await collectMarkdownItems(path.join(github, "prompts"), "command", "project", sourceLabel, workspace), + hooks, + mcps, + }; +} + +function emptyPrimitives() { + return { skills: [], subagents: [], rules: [], commands: [], hooks: [], mcps: [] }; +} + +export async function collectCopilotCustomizeInventory(options = {}) { + const copilotHome = path.resolve( + expandHome(options.copilotHome ?? options["copilot-home"] ?? defaultCopilotHome()), + ); + const workspace = normalizeWorkspace(options.workspace ?? process.cwd()); + const includeUserHome = options.includeUserHome !== false; + const installState = includeUserHome + ? await readCopilotInstalledPluginState({ ...options, copilotHome }) + : { records: [], source: "not-authorized", installRecordFiles: [] }; + const [plugins, user, project] = await Promise.all([ + includeUserHome ? collectCopilotPlugins(installState.records ?? []) : [], + includeUserHome ? collectCopilotUserPrimitives(copilotHome) : emptyPrimitives(), + collectCopilotWorkspacePrimitives(workspace), + ]); + return { + generatedAt: new Date().toISOString(), + provider: "copilot", + copilotHome, + workspace, + tabs: MANAGE_TABS, + plugins, + manage: buildManageCollections(plugins, user, project), + diagnostics: { + installedPluginState: installState.source, + installedPluginRecordCount: plugins.length, + installedPluginRecordFiles: installState.installRecordFiles ?? [], + remotePluginInstallMarkersRequired: false, + }, + unsupported: [ + "remote marketplace browse ordering", + "remote marketplace display metadata", + "organization and enterprise managed plugin standards", + "remote organization and enterprise custom agents", + "session-scoped hooks", + "cloud MCP authentication state", + ], + }; +} diff --git a/scripts/agent-customize/providers/index.mjs b/scripts/agent-customize/providers/index.mjs index 768444c..a000baa 100644 --- a/scripts/agent-customize/providers/index.mjs +++ b/scripts/agent-customize/providers/index.mjs @@ -1,5 +1,6 @@ import { collectClaudeCustomizeInventory } from "./claude.mjs"; import { collectCodexCustomizeInventory } from "./codex.mjs"; +import { collectCopilotCustomizeInventory } from "./copilot.mjs"; import { collectCursorCustomizeInventory } from "./cursor.mjs"; import { collectQoderCustomizeInventory } from "./qoder.mjs"; import { collectQwenCustomizeInventory } from "./qwen.mjs"; @@ -10,6 +11,7 @@ export const PROVIDER_COLLECTORS = new Map([ ["codex", collectCodexCustomizeInventory], ["claude", collectClaudeCustomizeInventory], ["qwen", collectQwenCustomizeInventory], + ["copilot", collectCopilotCustomizeInventory], ]); export async function collectProviderInventory(provider, options = {}) { diff --git a/scripts/agent-lint/cli.mjs b/scripts/agent-lint/cli.mjs index 4f728b2..a66279b 100644 --- a/scripts/agent-lint/cli.mjs +++ b/scripts/agent-lint/cli.mjs @@ -167,7 +167,7 @@ function usage() { return [ "Usage: better-harness agent-lint [--workspace ] [--profile agents-md-review|agent-assets-review] [--json|--format markdown]", " better-harness agent-lint --workspace-root --scan-children --profile agents-md-review", - " better-harness agent-lint --profile agent-assets-review --provider [--skill ]", + " better-harness agent-lint --profile agent-assets-review --provider [--skill ]", "", "Parse agent instruction entrypoints and bounded local Markdown references into review evidence.", "", diff --git a/scripts/better-harness-cli/registry.mjs b/scripts/better-harness-cli/registry.mjs index 7aba164..71c892e 100644 --- a/scripts/better-harness-cli/registry.mjs +++ b/scripts/better-harness-cli/registry.mjs @@ -52,7 +52,7 @@ const COMMANDS = [ kind: "direct", audience: "advanced", script: "session-analysis.mjs", - summary: "Collect and normalize Qoder, Codex, Claude, Cursor, and Qwen session evidence.", + summary: "Collect and normalize Qoder, Codex, Claude, Cursor, Qwen, and Copilot session evidence.", subcommands: [ { name: "sources", diff --git a/scripts/coding-agent-practices/asset-baseline.mjs b/scripts/coding-agent-practices/asset-baseline.mjs index fb117cf..9c1c433 100644 --- a/scripts/coding-agent-practices/asset-baseline.mjs +++ b/scripts/coding-agent-practices/asset-baseline.mjs @@ -15,7 +15,7 @@ export const ASSET_BASELINE_SCHEMA_VERSION = 1; export const MAX_BASELINE_FINDINGS = 16; export const MAX_BASELINE_OWNER_ROUTES = 16; -const PROVIDERS = new Set(["qoder", "codex", "claude", "cursor", "qwen"]); +const PROVIDERS = new Set(["qoder", "codex", "claude", "cursor", "qwen", "copilot"]); const SEVERITY_RANK = Object.freeze({ error: 0, warning: 1, advisory: 2 }); const OWNER_KIND_RANK = Object.freeze({ rules: 0, @@ -200,7 +200,7 @@ function available(data) { export async function collectAssetBaseline(options = {}, dependencies = {}) { const provider = options.provider ?? options.platform ?? "qoder"; if (!PROVIDERS.has(provider)) { - throw new Error(`Unsupported provider: ${provider}. Supported providers: qoder, codex, claude, cursor, qwen.`); + throw new Error(`Unsupported provider: ${provider}. Supported providers: qoder, codex, claude, cursor, qwen, copilot.`); } const workspace = normalizeWorkspace(options.workspace ?? "."); const includeUserHome = parseBooleanFlag(options.includeUserHome ?? options["include-user-home"] ?? false); @@ -300,7 +300,7 @@ export function formatAssetBaselineMarkdown(result) { return `${lines.join("\n")}\n`; } -const USAGE = `Usage: better-harness coding-agent-practices asset-baseline [qoder|codex|claude|cursor|qwen] [options] +const USAGE = `Usage: better-harness coding-agent-practices asset-baseline [qoder|codex|claude|cursor|qwen|copilot] [options] Collect one compact, read-only AI evidence envelope from a shared asset snapshot. diff --git a/scripts/coding-agent-practices/asset-integrity.mjs b/scripts/coding-agent-practices/asset-integrity.mjs index 8105f94..a7585d4 100644 --- a/scripts/coding-agent-practices/asset-integrity.mjs +++ b/scripts/coding-agent-practices/asset-integrity.mjs @@ -340,7 +340,7 @@ export function formatAssetIntegrityMarkdown(result) { return `${lines.join("\n")}\n`; } -const USAGE = `Usage: better-harness coding-agent-practices asset-integrity [qoder|codex|claude|cursor|qwen] [options] +const USAGE = `Usage: better-harness coding-agent-practices asset-integrity [qoder|codex|claude|cursor|qwen|copilot] [options] Run a read-only metadata integrity review for Memory titles, enabled Plugins, and Hooks. Memory bodies are never read. @@ -368,8 +368,8 @@ async function runCli(argv) { } const { command, options } = parseArgs(argv); const provider = options.provider ?? options.platform ?? command ?? "qoder"; - if (!["qoder", "codex", "claude", "cursor", "qwen"].includes(provider)) { - throw new Error(`Unsupported provider: ${provider}. Supported providers: qoder, codex, claude, cursor, qwen.`); + if (!["qoder", "codex", "claude", "cursor", "qwen", "copilot"].includes(provider)) { + throw new Error(`Unsupported provider: ${provider}. Supported providers: qoder, codex, claude, cursor, qwen, copilot.`); } const includeUserHome = options.includeUserHome ?? options["include-user-home"] ?? false; const includeMemories = options.includeMemories ?? options["include-memories"] ?? false; diff --git a/scripts/coding-agent-practices/inventory.mjs b/scripts/coding-agent-practices/inventory.mjs index 353ba1e..31ff229 100644 --- a/scripts/coding-agent-practices/inventory.mjs +++ b/scripts/coding-agent-practices/inventory.mjs @@ -379,7 +379,7 @@ async function collectCodexMemories(scope) { } function makeSessionSourceHints(scope) { - if (!["qoder", "codex", "claude", "cursor", "qwen"].includes(scope.platform)) { + if (!["qoder", "codex", "claude", "cursor", "qwen", "copilot"].includes(scope.platform)) { return []; } return [ @@ -448,6 +448,7 @@ function providerScope(options = {}, platform = options.platform ?? "qoder") { claudeHome: options.claudeHome ?? options["claude-home"], claudeStatePath: options.claudeStatePath ?? options["claude-state"] ?? options["claude-state-path"], qwenHome: options.qwenHome ?? options["qwen-home"], + copilotHome: options.copilotHome ?? options["copilot-home"], }; } @@ -536,7 +537,7 @@ function customizeSurface({ provider, group, scope, type, label, basePath, items async function buildConfiguredAssetSurfaces(inventory, scope) { const provider = scope.platform; const projectBase = scope.workspace; - const userBase = inventory.cursorHome ?? inventory.qoderHome ?? inventory.codexHome ?? inventory.claudeHome ?? inventory.qwenHome; + const userBase = inventory.cursorHome ?? inventory.qoderHome ?? inventory.codexHome ?? inventory.claudeHome ?? inventory.qwenHome ?? inventory.copilotHome; const surfaceTypes = [ ["skills", "skills", "Skills"], ["subagents", "agents", "Agents"], @@ -639,6 +640,7 @@ export async function collectProviderInventory(options = {}) { claudeHome: scope.claudeHome, claudeStatePath: scope.claudeStatePath, qwenHome: scope.qwenHome, + copilotHome: scope.copilotHome, includeUserHome: scope.includeUserHome, includeGlobalHooks: scope.includeGlobalHooks, }); @@ -890,12 +892,12 @@ export function formatInventoryMarkdown(result) { return `${lines.join("\n")}\n`; } -const USAGE = `Usage: better-harness coding-agent-practices inventory [qoder|codex|claude|cursor|qwen] [options] +const USAGE = `Usage: better-harness coding-agent-practices inventory [qoder|codex|claude|cursor|qwen|copilot] [options] Inspect configured coding-agent assets and practice evidence for one platform. Options: - --platform Select the platform (default: qoder; may also be the first positional) + --platform Select the platform (default: qoder; may also be the first positional) --workspace Workspace root to inspect (default: current directory) --json Emit JSON (default) --format Output format @@ -917,9 +919,9 @@ async function runCli(argv) { } const { command, options } = parseArgs(argv); const platform = options.platform ?? command ?? "qoder"; - if (!["cursor", "qoder", "codex", "claude", "qwen"].includes(platform)) { + if (!["cursor", "qoder", "codex", "claude", "qwen", "copilot"].includes(platform)) { throw new Error( - `Unsupported platform: ${platform}. Supported platforms: cursor, qoder, codex, claude, qwen.\n\n${USAGE}`, + `Unsupported platform: ${platform}. Supported platforms: cursor, qoder, codex, claude, qwen, copilot.\n\n${USAGE}`, ); } const result = platform === "qoder" diff --git a/scripts/harness-analysis/evidence-bundle/agent-customize.mjs b/scripts/harness-analysis/evidence-bundle/agent-customize.mjs index 737d91d..6eb0eb4 100644 --- a/scripts/harness-analysis/evidence-bundle/agent-customize.mjs +++ b/scripts/harness-analysis/evidence-bundle/agent-customize.mjs @@ -1,7 +1,7 @@ import { collectAssetBaseline } from "../../coding-agent-practices/asset-baseline.mjs"; import { availableLane, unavailableLane } from "./contract.mjs"; -const ASSET_PROVIDERS = new Set(["qoder", "codex", "claude", "cursor", "qwen"]); +const ASSET_PROVIDERS = new Set(["qoder", "codex", "claude", "cursor", "qwen", "copilot"]); export async function collectAgentCustomize(context, options = {}, dependencies = {}) { if (!ASSET_PROVIDERS.has(context.provider)) { diff --git a/scripts/harness-analysis/evidence-bundle/cli.mjs b/scripts/harness-analysis/evidence-bundle/cli.mjs index f176cf9..8b24f51 100644 --- a/scripts/harness-analysis/evidence-bundle/cli.mjs +++ b/scripts/harness-analysis/evidence-bundle/cli.mjs @@ -13,7 +13,7 @@ Harness, and Agent Customize specialists plus the lead analyzer. Options: --workspace Target workspace (required) - --platform qoder, codex, claude, cursor, or qwen (default: qoder) + --platform qoder, codex, claude, cursor, qwen, or copilot (default: qoder) --language Evidence language (default: en) --depth 7-day/3-item or 30-day/5-item review (default: normal) --since Override the frozen window start @@ -33,7 +33,7 @@ const ALLOWED = new Set([ "workspace", "platform", "provider", "language", "depth", "since", "until", "evidence-limit", "include-user-home", "include-memories", "canvas-out", "replace-canvas", "format", "json", "qoder-home", "codex-home", "claude-home", - "cursor-home", "qwen-home", "claude-state", "help", "h", + "cursor-home", "qwen-home", "copilot-home", "claude-state", "help", "h", ]); function assertOptions(command, options) { diff --git a/scripts/harness-analysis/evidence-bundle/contract.mjs b/scripts/harness-analysis/evidence-bundle/contract.mjs index 1b6438f..f395a25 100644 --- a/scripts/harness-analysis/evidence-bundle/contract.mjs +++ b/scripts/harness-analysis/evidence-bundle/contract.mjs @@ -8,7 +8,7 @@ export const EVIDENCE_LANE_NAMES = Object.freeze([ "agentCustomize", ]); -const PROVIDERS = new Set(["qoder", "codex", "claude", "cursor", "qwen"]); +const PROVIDERS = new Set(["qoder", "codex", "claude", "cursor", "qwen", "copilot"]); const DEPTHS = new Set(["quick", "normal"]); function enabled(value) { diff --git a/scripts/harness-analysis/report-quality.mjs b/scripts/harness-analysis/report-quality.mjs index 8758bef..27625e2 100644 --- a/scripts/harness-analysis/report-quality.mjs +++ b/scripts/harness-analysis/report-quality.mjs @@ -132,7 +132,7 @@ const AI_PRACTICE_SCOPE_NEGATIVE_RE = /^\s*(?:(?:not in scope|out of scope|not i const AI_PRACTICE_SECTION_RE = /^(#{2,4})\s+(?:AI Agent Practices|Coding Agent Practices|AI Agent 实践|智能体实践|编码代理实践)(?:\s|$)/im; const AI_PRACTICE_LABEL_RE = /^\s*\*\*(?:AI Agent Practices|Coding Agent Practices|AI Agent 实践|智能体实践|编码代理实践)\s*[:：]?\*\*\s*$/im; const AI_PRACTICE_SURFACE_RE = /\b(?:Rules|Hooks|Skills|Custom Agents|MCP|Plugins|Session Insights|Sessions|DESIGN\.md|Design Tokens?|Design Contract|design-token contract)\b|规则|技能|自定义\s*(?:Agent|智能体)|插件|会话洞察|会话|设计(?:令牌|契约)/i; -const AI_PRACTICE_SESSION_SCOPE_RE = /(?:\.qoder|\.codex|\.claude|\.cursor|\.qwen|qoder|codex|claude|cursor|qwen|session-analysis|session sources|session evidence|会话分析|会话证据)/i; +const AI_PRACTICE_SESSION_SCOPE_RE = /(?:\.qoder|\.codex|\.claude|\.cursor|\.qwen|\.copilot|qoder|codex|claude|cursor|qwen|copilot|session-analysis|session sources|session evidence|会话分析|会话证据)/i; const AI_READINESS_DIMENSION_RE = /\bAI Readiness\b|\bAI Agent Readiness\b|AI\s*(?:就绪度|就绪|准备度)/i; const SESSION_SOURCES_RE = /session-analysis\.mjs\s+sources/i; const SESSION_BOUNDARY_RE = /session-analysis\.mjs\s+(?:sources\s+and\s+)?facets|no enabled roots|no enabled session|no sessions|no session(?: analysis)? evidence|no agent session logs|no execution history|no proof of agent workflow|no-session boundary|source probe failed|session-analysis (?:was )?not run|session sources.*none|qoder\/codex session sources.*none|没有启用.*(?:root|session|根|会话)|没有.*会话|源探测失败/i; diff --git a/scripts/harness-analysis/report-run.mjs b/scripts/harness-analysis/report-run.mjs index 81b5a5c..004ec57 100644 --- a/scripts/harness-analysis/report-run.mjs +++ b/scripts/harness-analysis/report-run.mjs @@ -19,7 +19,7 @@ an explicit Qoder Canvas output is requested. Options: --workspace Target workspace (required) - --platform qoder, codex, claude, cursor, or qwen (default: qoder) + --platform qoder, codex, claude, cursor, qwen, or copilot (default: qoder) --language en or zh-CN (default: en) --since Include sessions at or after the frozen window start --until Include sessions at or before the frozen window end @@ -36,7 +36,7 @@ function clone(value) { function reportPlatform(value = "qoder") { const platform = String(value || "qoder").toLowerCase(); - if (!["qoder", "codex", "claude", "cursor", "qwen"].includes(platform)) { + if (!["qoder", "codex", "claude", "cursor", "qwen", "copilot"].includes(platform)) { throw Object.assign(new Error(`unsupported Harness report platform: ${platform}`), { code: "UNSUPPORTED_REPORT_PLATFORM", }); @@ -51,7 +51,7 @@ function flagEnabled(value) { function assertCliOptions(options) { const allowed = new Set([ "workspace", "platform", "language", "since", "until", "format", "canvas-out", "replace-canvas", "include-global-capabilities", - "qoder-home", "codex-home", "claude-home", "cursor-home", "qwen-home", + "qoder-home", "codex-home", "claude-home", "cursor-home", "qwen-home", "copilot-home", ]); const positional = Array.isArray(options._) ? options._ : []; const unknown = Object.keys(options).filter((key) => key !== "_" && !allowed.has(key)); diff --git a/scripts/harness-analysis/task-loop-report.mjs b/scripts/harness-analysis/task-loop-report.mjs index fcfebc5..f7a2b63 100644 --- a/scripts/harness-analysis/task-loop-report.mjs +++ b/scripts/harness-analysis/task-loop-report.mjs @@ -1509,7 +1509,7 @@ function checkupReportFindings(source) { function usageOutcomeReviewLead(source, locale) { const usage = source?.sessionEvents?.usageEfficiency; if (!isObject(usage) || !isObject(usage.selection) || !isObject(usage.longSessions) || !isObject(usage.outcomeReview)) return null; - const platform = ["qoder", "codex", "claude", "cursor", "qwen"].includes(source?.manifest?.scope?.platform) + const platform = ["qoder", "codex", "claude", "cursor", "qwen", "copilot"].includes(source?.manifest?.scope?.platform) ? source.manifest.scope.platform : "qoder"; const activeCount = Number(usage?.longSessions?.activeCount ?? 0); diff --git a/scripts/harness-analysis/task-loop-source.mjs b/scripts/harness-analysis/task-loop-source.mjs index 3a14975..20c935a 100644 --- a/scripts/harness-analysis/task-loop-source.mjs +++ b/scripts/harness-analysis/task-loop-source.mjs @@ -66,13 +66,13 @@ const REQUIRED_SOFTWARE_FLUENCY_CAPABILITIES = Object.freeze([ const HELP = `Usage: node scripts/harness-analysis/task-loop-source.mjs --workspace --source [options] Create a conservative Agent Work Loop report-source candidate from normalized -Qoder, Codex, Claude, Cursor, or Qwen sessions. It retains privacy-safe episode, change, validation, +Qoder, Codex, Claude, Cursor, Qwen, or Copilot sessions. It retains privacy-safe episode, change, validation, repair-candidate, and explicit host-decision identities. Task understanding, validation relevance, repair, delivery, recovery, and Learning Capture remain unobserved until the prepared source-bound review resolves them. Options: - --platform + --platform Session platform (default: qoder) --workspace Target workspace (required) --source Candidate report.source.json path (required) @@ -813,7 +813,7 @@ export function buildTaskLoopSourceCandidate({ export async function collectAgentLintPracticeEvidence(options = {}) { const provider = options.platform ?? "qoder"; - const assetReviewSupported = ["qoder", "codex", "claude", "cursor", "qwen"].includes(provider); + const assetReviewSupported = ["qoder", "codex", "claude", "cursor", "qwen", "copilot"].includes(provider); const common = { workspace: options.workspace, provider, @@ -822,6 +822,7 @@ export async function collectAgentLintPracticeEvidence(options = {}) { claudeHome: options.claudeHome ?? options["claude-home"], cursorHome: options.cursorHome ?? options["cursor-home"], qwenHome: options.qwenHome ?? options["qwen-home"], + copilotHome: options.copilotHome ?? options["copilot-home"], }; const [instructionReview, assetReview, practiceInventory] = await Promise.all([ runAgentLint({ ...common, profile: "agents-md-review" }), diff --git a/scripts/npm-package/verify-pack.mjs b/scripts/npm-package/verify-pack.mjs index c5229ef..a5b3335 100644 --- a/scripts/npm-package/verify-pack.mjs +++ b/scripts/npm-package/verify-pack.mjs @@ -53,6 +53,8 @@ function verifyReleaseVersionAlignment() { [".claude-plugin/marketplace.json", readJson(".claude-plugin/marketplace.json").plugins?.[0]?.version], [".codex-plugin/plugin.json", readJson(".codex-plugin/plugin.json").version], [".cursor-plugin/plugin.json", readJson(".cursor-plugin/plugin.json").version], + [".github/plugin/plugin.json", readJson(".github/plugin/plugin.json").version], + [".github/plugin/marketplace.json", readJson(".github/plugin/marketplace.json").plugins?.[0]?.version], ["qwen-extension.json", readJson("qwen-extension.json").version], ]; for (const [source, version] of versions) { @@ -137,6 +139,8 @@ const required = [ "package/.codex-plugin/plugin.json", "package/.cursor-plugin/plugin.json", "package/.cursor-plugin/marketplace.json", + "package/.github/plugin/plugin.json", + "package/.github/plugin/marketplace.json", "package/.qoder-plugin/plugin.json", "package/qwen-extension.json", "package/case-studies/factory/model/factory-readiness.md", @@ -317,6 +321,7 @@ const forbiddenBundlePrefixes = [ ".claude-plugin/", ".codex-plugin/", ".cursor-plugin/", + ".github/plugin/", "qwen-extension.json", "test/", "dev/", diff --git a/scripts/session-analysis.mjs b/scripts/session-analysis.mjs index 51adbbc..bd08a58 100644 --- a/scripts/session-analysis.mjs +++ b/scripts/session-analysis.mjs @@ -231,7 +231,14 @@ async function loadPlatform(platform = "qoder") { main: module.main, }; } - throw new Error(`Unsupported platform: ${platform}. Supported platforms: qoder, codex, claude, cursor, qwen.`); + if (platform === "copilot") { + const module = await import("./session-analysis/platforms/copilot.mjs"); + return { + Analyzer: module.CopilotSessionAnalyzer, + main: module.main, + }; + } + throw new Error(`Unsupported platform: ${platform}. Supported platforms: qoder, codex, claude, cursor, qwen, copilot.`); } export async function createAnalyzer(platform = "qoder") { @@ -281,7 +288,7 @@ export async function main(argv = process.argv.slice(2)) { ] : []; process.stdout.write([ - `Usage: session-analysis${command ? ` ${command}` : " "} --platform --workspace [options]`, + `Usage: session-analysis${command ? ` ${command}` : " "} --platform --workspace [options]`, "", "Commands: sources, sessions, facets, insights, facts, file-reads, show, events, claude-facets", ...factsOptions, diff --git a/scripts/session-analysis/analyzer.mjs b/scripts/session-analysis/analyzer.mjs index 79c626f..5732f05 100644 --- a/scripts/session-analysis/analyzer.mjs +++ b/scripts/session-analysis/analyzer.mjs @@ -19,8 +19,8 @@ import { createCodexCliJsonModelClient } from "./codex-json-model.mjs"; export const SESSION_ANALYSIS_HELP = `Usage: better-harness session-analysis [command] [options] -Inspect local Qoder, Codex, Claude, Cursor, or Qwen session evidence. The default -command is sessions and the default platform is qoder. Help exits before +Inspect local Qoder, Codex, Claude, Cursor, Qwen, or Copilot session evidence. The +default command is sessions and the default platform is qoder. Help exits before reading HOME or workspace. Commands: @@ -35,13 +35,15 @@ Commands: events Show normalized events selected with --session-id Options: - --platform + --platform Session host (default: qoder) --workspace Workspace scope (default: current directory) --qoder-home Qoder data root (default: ~/.qoder) --codex-home Codex data root (default: ~/.codex) --claude-home Claude Code data root (default: ~/.claude) --cursor-home Cursor data root (default: ~/.cursor) + --qwen-home Qwen Code data root (default: ~/.qwen) + --copilot-home Copilot CLI data root (default: ~/.copilot) --include-cache Include optional Qoder cache evidence --include-global-capabilities Include optional user-global Qoder evidence @@ -264,7 +266,14 @@ async function loadPlatform(platform = "qoder") { main: module.main, }; } - throw new Error(`Unsupported platform: ${platform}. Supported platforms: qoder, codex, claude, cursor, qwen.`); + if (platform === "copilot") { + const module = await import("./platforms/copilot.mjs"); + return { + Analyzer: module.CopilotSessionAnalyzer, + main: module.main, + }; + } + throw new Error(`Unsupported platform: ${platform}. Supported platforms: qoder, codex, claude, cursor, qwen, copilot.`); } export async function createAnalyzer(platform = "qoder") { @@ -278,7 +287,7 @@ export async function main(argv = process.argv.slice(2), dependencies = {}) { if (command === "claude-facets") { if (options.help === true) { stdout.write([ - "Usage: session-analysis claude-facets --platform --workspace [options]", + "Usage: session-analysis claude-facets --platform --workspace [options]", "", "Options:", " --limit <1-5> Maximum semantic facets (default: 5)", diff --git a/scripts/session-analysis/lifecycle-demand-signals.mjs b/scripts/session-analysis/lifecycle-demand-signals.mjs index f627aec..63500bf 100644 --- a/scripts/session-analysis/lifecycle-demand-signals.mjs +++ b/scripts/session-analysis/lifecycle-demand-signals.mjs @@ -540,7 +540,7 @@ function fingerprint(value) { function safeHost(value) { const host = String(value ?? "").toLowerCase(); - return ["qoder", "codex", "claude", "cursor", "qwen"].includes(host) ? host : "unknown"; + return ["qoder", "codex", "claude", "cursor", "qwen", "copilot"].includes(host) ? host : "unknown"; } function safeEvidenceToken(value, fallback) { diff --git a/scripts/session-analysis/platforms/copilot.mjs b/scripts/session-analysis/platforms/copilot.mjs new file mode 100644 index 0000000..17b6c51 --- /dev/null +++ b/scripts/session-analysis/platforms/copilot.mjs @@ -0,0 +1,570 @@ +#!/usr/bin/env node + +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { SessionAnalyzer } from "../../session-analysis.mjs"; +import { parseArgs, parseBooleanFlag } from "../cli.mjs"; +import { forEachJsonLine, isDirectory, pathExists } from "../fs.mjs"; +import { expandHome, normalizeWorkspace } from "../paths.mjs"; +import { + emitProviderResult, + runProviderAnalysis, + runProviderCommand, +} from "../provider-runner.mjs"; +import { parseResultFacts } from "../result-facts.mjs"; +import { mergeTimeRange, normalizeCliDate, normalizeTimestamp, timestampMillis, withinTimeRange } from "../time.mjs"; + +const TRANSCRIPT_FILE = "events.jsonl"; +const WORKSPACE_FILE = "workspace.yaml"; + +function isWorkspaceMatch(candidate, workspace) { + if (!candidate) return false; + const resolved = normalizeWorkspace(candidate); + return resolved === workspace || resolved.startsWith(`${workspace}${path.sep}`); +} + +/** + * Read the `cwd` binding from a Copilot session's `workspace.yaml`. + * + * The file is a flat `key: value` list, so only the two keys this adapter needs + * are parsed. No YAML dependency is introduced and unknown keys are ignored. + */ +export function parseWorkspaceDescriptor(text) { + const descriptor = { id: null, cwd: null }; + for (const line of String(text ?? "").split(/\r?\n/u)) { + const match = /^(id|cwd):\s*(.+?)\s*$/u.exec(line); + if (!match) continue; + const value = match[2].replace(/^["']|["']$/gu, ""); + descriptor[match[1]] = value.length > 0 ? value : null; + } + return descriptor; +} + +function inferTimestamp(raw) { + return normalizeTimestamp(raw?.timestamp ?? null); +} + +function toolArguments(raw) { + const args = raw?.data?.arguments; + return args && typeof args === "object" && !Array.isArray(args) ? args : {}; +} + +function inferFilePath(toolName, input = {}) { + if (!/(?:read|edit|write|file|notebook|view|create|patch)/iu.test(String(toolName ?? ""))) return null; + return input.file_path ?? input.filePath ?? input.path ?? null; +} + +function inferCommandText(toolName, input = {}) { + if (!/(?:bash|shell|exec|terminal|run|powershell)/iu.test(String(toolName ?? ""))) return null; + return input.command ?? input.cmd ?? null; +} + +function evidenceRef(sourceRef, type) { + return { + kind: sourceRef.kind, + path: sourceRef.path, + line: sourceRef.line ?? null, + seq: null, + type, + }; +} + +function resultText(data) { + const result = data?.result; + if (typeof result === "string") return result; + return result?.content ?? result?.detailedContent ?? ""; +} + +/** + * Normalize one Copilot `events.jsonl` record. + * + * Copilot records a typed lifecycle stream. Event names are taken from observed + * transcripts rather than a published schema, so unrecognized types stay + * explicit `metadata.*` events instead of being dropped or reinterpreted. + */ +function transcriptEvents(raw, sourceRef, options) { + const rawType = String(raw?.type ?? "record"); + const data = raw?.data ?? {}; + const sessionId = data.sessionId ?? sourceRef.sessionId ?? null; + const base = { + sessionId, + timestamp: inferTimestamp(raw), + sourceKind: sourceRef.kind, + planningScope: "workspace", + cwd: data?.context?.cwd ?? sourceRef.cwd ?? null, + isSubagent: raw?.agentId ? true : null, + }; + const events = []; + + if (rawType === "user.message") { + const text = typeof data.content === "string" ? data.content : ""; + events.push({ + ...base, + type: "user", + category: "user", + evidenceRef: evidenceRef(sourceRef, "user"), + summary: text ? `user message (${text.length} chars)` : "user", + contentLength: text.length, + userPrompt: text.length > 0, + ...(options.includeUserText && text ? { userText: text } : {}), + ...(options.includeContent && text ? { content: text } : {}), + }); + return events; + } + + if (rawType === "assistant.message") { + const text = typeof data.content === "string" ? data.content : ""; + const event = { + ...base, + type: "assistant", + category: "assistant", + evidenceRef: evidenceRef(sourceRef, "assistant"), + summary: text ? `assistant message (${text.length} chars)` : "assistant", + contentLength: text.length, + }; + if (text) event.userVisibleAssistantMessage = true; + if (options.includeContent && text) event.content = text; + if (data.model) event.model = data.model; + events.push(event); + return events; + } + + if (rawType === "tool.execution_start") { + const input = toolArguments(raw); + const toolName = data.toolName ?? "unknown-tool"; + const event = { + ...base, + type: "tool.call", + category: "tool", + lifecyclePhase: "request", + toolName, + toolInvocationId: data.toolCallId ?? null, + evidenceRef: evidenceRef(sourceRef, "tool.call"), + summary: `${toolName} request`, + }; + const commandText = inferCommandText(toolName, input); + const filePath = inferFilePath(toolName, input); + if (options.includeCommandText && commandText) event.commandText = commandText; + if (filePath) event.filePath = filePath; + if (/^(?:skill|task)$/iu.test(String(toolName))) { + const skillName = input.skill ?? input.name ?? input.agent_type; + if (skillName) { + event.skillName = String(skillName).split(":").at(-1); + event.skillNames = [event.skillName]; + } + } + events.push(event); + return events; + } + + if (rawType === "tool.execution_complete") { + const success = data.success !== false; + const event = { + ...base, + type: "tool.result", + category: "tool", + lifecyclePhase: "result", + toolInvocationId: data.toolCallId ?? null, + success, + hasError: !success, + evidenceRef: evidenceRef(sourceRef, "tool.result"), + summary: success ? "tool result" : "tool result failed", + }; + const facts = parseResultFacts(String(resultText(data)).slice(-8_192)); + if (facts) event.resultFacts = facts; + events.push(event); + return events; + } + + if (rawType === "hook.start" || rawType === "hook.end") { + const event = { + ...base, + type: rawType === "hook.start" ? "hook.call" : "hook.result", + category: "hook", + lifecyclePhase: rawType === "hook.start" ? "request" : "result", + hookEvent: data.hookType ?? null, + toolInvocationId: data.hookInvocationId ?? null, + evidenceRef: evidenceRef(sourceRef, rawType), + summary: `${data.hookType ?? "hook"} ${rawType === "hook.start" ? "request" : "result"}`, + }; + if (rawType === "hook.end") { + const success = data.success !== false; + event.success = success; + event.hasError = !success; + } + events.push(event); + return events; + } + + if (rawType === "subagent.started" || rawType === "subagent.completed") { + const event = { + ...base, + type: rawType === "subagent.started" ? "subagent.start" : "subagent.stop", + category: "delegation", + lifecyclePhase: rawType === "subagent.started" ? "request" : "result", + toolInvocationId: data.toolCallId ?? null, + subagentName: data.agentName ?? null, + evidenceRef: evidenceRef(sourceRef, rawType), + summary: `${data.agentName ?? "subagent"} ${rawType === "subagent.started" ? "started" : "completed"}`, + }; + if (data.model) event.model = data.model; + if (rawType === "subagent.completed") { + // Copilot reports subagent aggregates only. These are not per-response + // model usage and must not be projected as such. + if (Number.isFinite(Number(data.totalToolCalls))) event.subagentToolCalls = Number(data.totalToolCalls); + if (Number.isFinite(Number(data.totalTokens))) event.subagentTotalTokens = Number(data.totalTokens); + if (Number.isFinite(Number(data.durationMs))) event.subagentDurationMs = Number(data.durationMs); + } + events.push(event); + return events; + } + + if (rawType === "session.plan_changed") { + events.push({ + ...base, + type: "plan.update", + category: "planning", + evidenceRef: evidenceRef(sourceRef, "plan.update"), + summary: "session plan changed", + }); + return events; + } + + if (rawType === "session.compaction_start" || rawType === "session.compaction_complete") { + const event = { + ...base, + type: "context.compaction", + category: "context", + lifecyclePhase: rawType === "session.compaction_start" ? "request" : "result", + evidenceRef: evidenceRef(sourceRef, rawType), + summary: rawType === "session.compaction_start" ? "context compaction started" : "context compaction completed", + }; + if (Number.isFinite(Number(data.preCompactionTokens))) { + event.preCompactionTokens = Number(data.preCompactionTokens); + } + events.push(event); + return events; + } + + if (rawType === "session.permissions_changed" || rawType === "session.mode_changed") { + events.push({ + ...base, + type: "control.change", + category: "control", + evidenceRef: evidenceRef(sourceRef, rawType), + summary: rawType === "session.permissions_changed" ? "session permissions changed" : "session mode changed", + }); + return events; + } + + if (rawType === "external_tool.requested" || rawType === "external_tool.completed") { + events.push({ + ...base, + type: rawType === "external_tool.requested" ? "tool.call" : "tool.result", + category: "tool", + lifecyclePhase: rawType === "external_tool.requested" ? "request" : "result", + toolName: data.toolName ?? "external-tool", + toolInvocationId: data.toolCallId ?? null, + external: true, + evidenceRef: evidenceRef(sourceRef, rawType), + summary: `external tool ${rawType === "external_tool.requested" ? "request" : "result"}`, + }); + return events; + } + + if (rawType === "session.start") { + events.push({ + ...base, + type: "metadata.session-start", + category: "metadata", + model: data.selectedModel ?? null, + evidenceRef: evidenceRef(sourceRef, "metadata.session-start"), + summary: "Copilot session started", + }); + return events; + } + + events.push({ + ...base, + type: `metadata.${rawType}`, + category: "metadata", + evidenceRef: evidenceRef(sourceRef, `metadata.${rawType}`), + summary: rawType, + }); + return events; +} + +async function probeSessionDirectory(sessionDir, workspace) { + const transcriptPath = path.join(sessionDir, TRANSCRIPT_FILE); + const descriptor = parseWorkspaceDescriptor(await readWorkspaceDescriptor(sessionDir)); + const summary = { + sessionId: descriptor.id || path.basename(sessionDir), + transcriptPath, + transcriptAvailable: await pathExists(transcriptPath), + cwd: descriptor.cwd, + records: 0, + firstSeen: null, + lastSeen: null, + workspaceMatch: isWorkspaceMatch(descriptor.cwd, workspace), + }; + if (!summary.transcriptAvailable) { + return summary; + } + + await forEachJsonLine(transcriptPath, (raw) => { + summary.records += 1; + if (raw?.type === "session.start") { + const data = raw?.data ?? {}; + if (data.sessionId) summary.sessionId = data.sessionId; + const cwd = data?.context?.cwd; + if (cwd) { + summary.cwd = summary.cwd ?? cwd; + if (isWorkspaceMatch(cwd, workspace)) summary.workspaceMatch = true; + } + } + mergeTimeRange(summary, inferTimestamp(raw)); + }); + return summary; +} + +/** + * Classify Copilot workspace coverage. + * + * A Copilot session directory can exist and match the workspace while carrying + * no `events.jsonl`. That state stays explicit instead of collapsing into zero + * activity or a clean result. + */ +function buildCopilotSourceCoverage({ scope, roots, matched, inWindow }) { + const root = roots.find((entry) => entry.kind === "copilot-session-jsonl"); + const workspaceSessions = matched.length; + const withTranscript = matched.filter((probe) => probe.transcriptAvailable); + const withoutTranscript = workspaceSessions - withTranscript.length; + const timeUnobserved = withTranscript.filter((probe) => !probe.firstSeen && !probe.lastSeen).length; + const emptyTranscripts = withTranscript.filter((probe) => probe.records === 0).length; + const requestedWindow = scope.sinceTime !== null || scope.untilTime !== null; + + let status = "observed"; + if (!root?.exists || workspaceSessions === 0) { + status = "absent"; + } else if (inWindow.length === 0 && requestedWindow && timeUnobserved === 0 && withTranscript.length > 0) { + status = "out-of-window"; + } else if (withTranscript.length === 0 || inWindow.length === 0) { + status = "unobserved"; + } else if (withoutTranscript > 0 || timeUnobserved > 0 || emptyTranscripts > 0) { + status = "partial"; + } + + return { + status, + transcript: { + sourceAvailable: Boolean(root?.exists), + workspaceSessions, + withTranscript: withTranscript.length, + withoutTranscript, + emptyTranscripts, + timeUnobservedSessions: timeUnobserved, + inWindowSessions: inWindow.length, + }, + usage: { + // Copilot transcripts carry no per-response model usage. Subagent and + // compaction totals are aggregates and are never projected as per-response + // usage. + perResponseUsageObserved: false, + }, + }; +} + +async function readWorkspaceDescriptor(sessionDir) { + const descriptorPath = path.join(sessionDir, WORKSPACE_FILE); + try { + return await readFile(descriptorPath, "utf8"); + } catch { + return ""; + } +} + +function dedupeEvents(events) { + const seen = new Set(); + return events.filter((event) => { + const key = event.toolInvocationId && event.lifecyclePhase + ? `${event.sessionId}:${event.type}:${event.lifecyclePhase}:${event.toolInvocationId}` + : null; + if (!key) return true; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +export class CopilotSessionAnalyzer extends SessionAnalyzer { + currentSessionId() { + return process.env.COPILOT_SESSION_ID ?? null; + } + + async resolveScope(options = {}) { + const since = normalizeCliDate(options.since, false); + const until = normalizeCliDate(options.until, true); + const workspace = normalizeWorkspace(options.workspace); + const home = path.resolve(expandHome( + options.home ?? options.copilotHome ?? options["copilot-home"] ?? process.env.COPILOT_HOME ?? "~/.copilot", + )); + return { + platform: "copilot", + workspace, + home, + since: since.label, + sinceTime: since.time, + until: until.label, + untilTime: until.time, + sessionId: options["session-id"] ?? options.sessionId ?? options._?.[0] ?? null, + includeGlobalCapabilities: parseBooleanFlag(options["include-global-capabilities"] ?? false), + }; + } + + async discoverSourceRoots(scope) { + const sessionStateRoot = path.join(scope.home, "session-state"); + return [ + { + id: "copilot-session-state", + kind: "copilot-session-jsonl", + role: "session-transcript", + path: sessionStateRoot, + optional: false, + enabled: true, + workspaceScoped: false, + coverage: "primary", + exists: await pathExists(sessionStateRoot), + }, + ]; + } + + async discoverSessions(scope, roots) { + const transcriptRoot = roots.find((root) => root.kind === "copilot-session-jsonl"); + let entries = []; + if (transcriptRoot?.exists) { + try { + entries = await readdir(transcriptRoot.path, { withFileTypes: true }); + } catch { + entries = []; + } + } + + const matched = []; + for (const entry of entries) { + const sessionDir = path.join(transcriptRoot.path, entry.name); + if (!entry.isDirectory() && !(await isDirectory(sessionDir))) continue; + const probe = await probeSessionDirectory(sessionDir, scope.workspace); + if (probe?.workspaceMatch) matched.push(probe); + } + + const inWindow = matched + .filter((probe) => probe.transcriptAvailable) + .filter((probe) => { + const timestamp = probe.lastSeen ?? probe.firstSeen; + if ((scope.sinceTime !== null || scope.untilTime !== null) && !timestamp) return false; + return withinTimeRange(timestamp, scope); + }) + .map((probe) => ({ + sessionId: probe.sessionId, + workspace: scope.workspace, + firstSeen: probe.firstSeen, + lastSeen: probe.lastSeen, + sourceKinds: [transcriptRoot.kind], + sourceRefs: [ + { + kind: transcriptRoot.kind, + role: transcriptRoot.role, + path: probe.transcriptPath, + cwd: probe.cwd, + firstSeen: probe.firstSeen, + lastSeen: probe.lastSeen, + }, + ], + })) + .sort((left, right) => (timestampMillis(right.lastSeen) ?? 0) - (timestampMillis(left.lastSeen) ?? 0)); + + scope._copilotSourceCoverage = buildCopilotSourceCoverage({ scope, roots, matched, inWindow }); + return inWindow; + } + + normalizeEvent(raw, sourceRef, options = {}) { + return this.normalizeEvents(raw, sourceRef, options)[0] ?? null; + } + + normalizeEvents(raw, sourceRef, options = {}) { + return transcriptEvents(raw, sourceRef, options); + } + + async readSession(session, scope, options = {}) { + const events = []; + for (const ref of session.sourceRefs ?? []) { + if (!ref.path.endsWith(".jsonl")) continue; + await forEachJsonLine(ref.path, (raw, line) => { + for (const event of this.normalizeEvents( + raw, + { ...ref, sessionId: session.sessionId, line }, + options, + )) { + if (withinTimeRange(event.timestamp, scope)) events.push(event); + } + }); + } + return dedupeEvents(events).sort((left, right) => + (timestampMillis(left.timestamp) ?? 0) - (timestampMillis(right.timestamp) ?? 0) + || Number(left.evidenceRef?.line ?? 0) - Number(right.evidenceRef?.line ?? 0)); + } + + async analysisWarnings(scope, _roots, _sessions) { + const coverage = scope._copilotSourceCoverage; + const warnings = [{ + code: "copilot-per-response-usage-unobserved", + message: "Copilot transcripts do not record per-response model token usage; usage evidence requires the opt-in OpenTelemetry export.", + }]; + if (!coverage || coverage.status === "absent") { + warnings.push({ + code: "copilot-workspace-transcripts-absent", + message: "No Copilot session transcript matched the selected workspace.", + }); + return warnings; + } + if (coverage.transcript.withoutTranscript > 0) { + warnings.push({ + code: "copilot-session-transcript-partial", + message: `Copilot session state matched ${coverage.transcript.workspaceSessions} workspace sessions, and ${coverage.transcript.withoutTranscript} carry no ${TRANSCRIPT_FILE}.`, + }); + } + if (coverage.transcript.timeUnobservedSessions > 0) { + warnings.push({ + code: "copilot-session-timestamps-unobserved", + message: `Copilot event timestamps were unobserved in ${coverage.transcript.timeUnobservedSessions} matched transcripts.`, + }); + } + return warnings; + } + + factsSourceCoverage(scope) { + return scope._copilotSourceCoverage ?? null; + } + + async analyze(options = {}) { + return runProviderAnalysis(this, options, { platform: "copilot", adapterVersion: "copilot-v1" }); + } +} + +export async function main(argv = process.argv.slice(2)) { + const { command = "sessions", options } = parseArgs(argv); + const analyzer = new CopilotSessionAnalyzer(); + const result = await runProviderCommand(analyzer, command, options); + await emitProviderResult({ provider: "Copilot", command, options, result }); + return result; +} + +const isCli = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url); +if (isCli) { + main().catch((error) => { + process.stderr.write(`copilot session-analysis failed: ${error.stack ?? error.message}\n`); + process.exitCode = 1; + }); +} diff --git a/scripts/session-analysis/selection-profile.mjs b/scripts/session-analysis/selection-profile.mjs index 3b62dbb..f421a3b 100644 --- a/scripts/session-analysis/selection-profile.mjs +++ b/scripts/session-analysis/selection-profile.mjs @@ -19,7 +19,7 @@ a declarative session selection plan. Raw prompts, commands, paths, and session identifiers never enter the profile. Options: - --platform + --platform Session platform (default: qoder) --workspace Target workspace (required) --since Exclude earlier sessions diff --git a/scripts/session-analysis/session-core-facts.mjs b/scripts/session-analysis/session-core-facts.mjs index 35cdac4..a90948f 100644 --- a/scripts/session-analysis/session-core-facts.mjs +++ b/scripts/session-analysis/session-core-facts.mjs @@ -21,6 +21,8 @@ export function createFactsRunContext(options = {}, platform = "unknown", provid qoder: process.env.QODER_SESSION_ID, claude: process.env.CLAUDE_SESSION_ID, cursor: process.env.CURSOR_SESSION_ID, + qwen: process.env.QWEN_SESSION_ID, + copilot: process.env.COPILOT_SESSION_ID, })[platform]; const excludedSessionId = String( options["exclude-session-id"] ?? options.excludeSessionId ?? environmentSessionId ?? "", diff --git a/scripts/session-analysis/usage-summary.mjs b/scripts/session-analysis/usage-summary.mjs index b1f2aae..1dcb279 100644 --- a/scripts/session-analysis/usage-summary.mjs +++ b/scripts/session-analysis/usage-summary.mjs @@ -12,7 +12,7 @@ Emit a bounded, read-only usage boundary as JSON. This command never accepts --output and never writes report or scratch files. Options: - --platform + --platform Session provider (default: qoder) --workspace Target workspace (default: current directory) --selection Selection strategy (default: all-eligible) diff --git a/test/fixtures/scripts-refactor-contract/root-help.txt b/test/fixtures/scripts-refactor-contract/root-help.txt index 25e8f1d..e864b42 100644 --- a/test/fixtures/scripts-refactor-contract/root-help.txt +++ b/test/fixtures/scripts-refactor-contract/root-help.txt @@ -20,8 +20,8 @@ Commands: repair-findings, record-fix-output, validate-canvas Project Evidence - session-analysis Collect and normalize Qoder, Codex, Claude, Cursor, and Qwen - session evidence + session-analysis Collect and normalize Qoder, Codex, Claude, Cursor, Qwen, and + Copilot session evidence dependency-governance Detect dependency governance files, automation, audit signals, and stale dependency evidence cloc Count code, comments, and blank lines diff --git a/test/fixtures/scripts-refactor-contract/session-help.txt b/test/fixtures/scripts-refactor-contract/session-help.txt index 3c7952e..66aab31 100644 --- a/test/fixtures/scripts-refactor-contract/session-help.txt +++ b/test/fixtures/scripts-refactor-contract/session-help.txt @@ -1,4 +1,4 @@ -Usage: session-analysis --platform --workspace [options] +Usage: session-analysis --platform --workspace [options] Commands: sources, sessions, facets, insights, facts, file-reads, show, events, claude-facets diff --git a/test/scripts-refactor-contract.test.mjs b/test/scripts-refactor-contract.test.mjs index 4eaed8f..1012fda 100644 --- a/test/scripts-refactor-contract.test.mjs +++ b/test/scripts-refactor-contract.test.mjs @@ -70,12 +70,12 @@ test("scripts refactor contract freezes machine-readable CLI output", () => { { label: "command inventory", args: ["commands", "--json"], - sha256: "6a015f7a50ccf60bcdc02f490e16790f011e84925e12440f4967812aad16e408", + sha256: "544235e08c2248f15a757bea00c584b7221e0011a84eec34b0b8515385a9b4dd", }, { label: "OpenCLI schema", args: ["schema"], - sha256: "cc5788593fe6f305db290974d4c808647b190a2850c621433f3269dc416ab179", + sha256: "34bd94d621f400ce4a7975bbb9d9f1681b7bd41cef52f99befbfe578944b2825", }, { label: "Harness command description", From ffd6ab6cd9610764286546ae720171b70c55c29c Mon Sep 17 00:00:00 2001 From: ShawnX Date: Thu, 30 Jul 2026 08:33:33 +0800 Subject: [PATCH 2/4] feat: add Copilot support in session analysis and plugin manifest tests --- test/agent-customize.test.mjs | 135 +++++++++++++++++++++++ test/plugin-manifests.test.mjs | 30 ++++- test/session-analysis-providers.test.mjs | 113 +++++++++++++++++++ 3 files changed, 277 insertions(+), 1 deletion(-) diff --git a/test/agent-customize.test.mjs b/test/agent-customize.test.mjs index f2d8b29..959dbff 100644 --- a/test/agent-customize.test.mjs +++ b/test/agent-customize.test.mjs @@ -1626,3 +1626,138 @@ test("Qwen provider collects user and project MCPs, skills, hooks, and rules", a await rm(fixture.root, { recursive: true, force: true }); } }); + +async function makeCopilotFixture() { + const root = await mkdtemp(path.join(os.tmpdir(), "better-harness-agent-customize-copilot-")); + const copilotHome = path.join(root, ".copilot"); + const workspace = path.join(root, "workspace", "better-harness"); + + const pluginRoot = path.join(copilotHome, "installed-plugins", "acme", "delivery"); + await writeJson(path.join(pluginRoot, ".github", "plugin", "plugin.json"), { + name: "delivery", + version: "1.0.0", + displayName: "Delivery", + description: "Delivery workflow plugin.", + skills: "./skills/", + }); + await writeText( + path.join(pluginRoot, "skills", "ship-release", "SKILL.md"), + "---\nname: ship-release\ndescription: Ship a release.\n---\n", + ); + await writeJson(path.join(pluginRoot, ".mcp.json"), { + mcpServers: { deliveryMcp: { command: "node", args: ["mcp/server.cjs"] } }, + }); + + await writeJson(path.join(copilotHome, "config.json"), { + installedPlugins: [ + { + name: "delivery", + marketplace: "acme", + version: "1.0.0", + enabled: true, + installed_at: "2026-07-20T00:00:00.000Z", + cache_path: pluginRoot, + }, + { + name: "missing-plugin", + marketplace: "acme", + version: "0.1.0", + enabled: true, + cache_path: path.join(copilotHome, "installed-plugins", "acme", "missing-plugin"), + }, + ], + }); + + await writeText(path.join(copilotHome, "copilot-instructions.md"), "# User Copilot Guide\n\nPersonal defaults.\n"); + await writeText( + path.join(copilotHome, "skills", "user-skill", "SKILL.md"), + "---\nname: user-skill\ndescription: A personal skill.\n---\n", + ); + await writeJson(path.join(copilotHome, "mcp-config.json"), { + mcpServers: { userMcp: { command: "node", args: ["user.cjs"] } }, + }); + + await writeText(path.join(workspace, "AGENTS.md"), "# Agents\n\nProject guidance.\n"); + await writeText(path.join(workspace, ".github", "copilot-instructions.md"), "# Copilot\n\nProject Copilot guidance.\n"); + await writeText( + path.join(workspace, ".github", "instructions", "tests.instructions.md"), + "---\napplyTo: \"**/*.test.mjs\"\ndescription: Test guidance.\n---\n", + ); + await writeText( + path.join(workspace, ".github", "skills", "review-change", "SKILL.md"), + "---\nname: review-change\ndescription: Review a change.\n---\n", + ); + await writeText( + path.join(workspace, ".github", "agents", "reviewer.agent.md"), + "---\nname: reviewer\ndescription: Review code.\n---\n", + ); + await writeJson(path.join(workspace, ".github", "hooks", "guard.json"), { + hooks: { Stop: [{ hooks: [{ type: "command", command: "node scripts/review-trigger/cli.mjs" }] }] }, + }); + await writeJson(path.join(workspace, ".github", "mcp.json"), { + mcpServers: { projectMcp: { command: "node", args: ["project.cjs"] } }, + }); + + return { root, copilotHome, workspace }; +} + +test("Copilot inventory separates plugin, user, and project assets", async () => { + const fixture = await makeCopilotFixture(); + try { + const inventory = await collectAgentCustomizeInventory({ + provider: "copilot", + workspace: fixture.workspace, + copilotHome: fixture.copilotHome, + includeUserHome: true, + }); + + assert.equal(inventory.provider, "copilot"); + assert.equal(inventory.copilotHome, fixture.copilotHome); + assert.equal(inventory.diagnostics.installedPluginState, "copilot-config"); + + // The record whose cache path is absent must not become an installed plugin. + assert.equal(inventory.plugins.length, 1); + assert.equal(inventory.plugins[0].id, "acme/delivery"); + assert.equal(inventory.plugins[0].installMatch, "copilot-marketplace"); + assert.equal(inventory.plugins[0].skills.length, 1); + + const skillNames = inventory.manage.skills.map((skill) => skill.name).sort(); + assert.deepEqual(skillNames, ["review-change", "ship-release", "user-skill"]); + + const scopesByName = new Map(inventory.manage.skills.map((skill) => [skill.name, skill.scope])); + assert.equal(scopesByName.get("review-change"), "project"); + assert.equal(scopesByName.get("user-skill"), "user"); + assert.equal(scopesByName.get("ship-release"), "plugin"); + + const ruleNames = inventory.manage.rules.map((rule) => rule.name); + assert.ok(ruleNames.includes("AGENTS.md")); + assert.ok(ruleNames.includes("copilot-instructions.md")); + + const mcpNames = inventory.manage.mcps.map((mcp) => mcp.name).sort(); + assert.deepEqual(mcpNames, ["deliveryMcp", "projectMcp", "userMcp"]); + + assert.equal(inventory.manage.subagents.some((agent) => agent.name === "reviewer"), true); + assert.equal(inventory.manage.hooks.length > 0, true); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +test("Copilot inventory without user-home authority keeps project scope only", async () => { + const fixture = await makeCopilotFixture(); + try { + const inventory = await collectAgentCustomizeInventory({ + provider: "copilot", + workspace: fixture.workspace, + copilotHome: fixture.copilotHome, + includeUserHome: false, + }); + + assert.equal(inventory.plugins.length, 0); + assert.equal(inventory.diagnostics.installedPluginState, "not-authorized"); + assert.deepEqual(inventory.manage.skills.map((skill) => skill.name), ["review-change"]); + assert.deepEqual(inventory.manage.mcps.map((mcp) => mcp.name), ["projectMcp"]); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); diff --git a/test/plugin-manifests.test.mjs b/test/plugin-manifests.test.mjs index 57c4328..2d4701e 100644 --- a/test/plugin-manifests.test.mjs +++ b/test/plugin-manifests.test.mjs @@ -105,6 +105,8 @@ test("host plugin manifests expose canonical Better Harness resources", () => { const cursor = readJson(".cursor-plugin/plugin.json"); const codex = readJson(".codex-plugin/plugin.json"); const qwen = readJson("qwen-extension.json"); + const copilot = readJson(".github/plugin/plugin.json"); + const copilotMarketplace = readJson(".github/plugin/marketplace.json"); const cursorMarketplace = readJson(".cursor-plugin/marketplace.json"); const packageJson = readJson("package.json"); const packageLock = readJson("package-lock.json"); @@ -159,6 +161,16 @@ test("host plugin manifests expose canonical Better Harness resources", () => { assert.equal(qwen.displayName, qoder.displayName); assert.equal(qwen.contextFileName, "QWEN.md"); assert.equal(qwen.skills, "./skills/"); + + assert.equal(copilot.name, qoder.name); + assert.equal(copilot.version, qoder.version); + assert.equal(copilot.description, qoder.description); + assert.deepEqual(copilot.author, qoder.author); + assert.equal(copilot.category, qoder.category); + assert.equal(copilot.skills, "./skills/"); + assert.equal(copilot.hooks, undefined); + assert.equal(copilot.license, "MIT"); + assert.equal(cursor.license, "MIT"); assert.equal(qoder.license, "MIT"); assert.equal(packageJson.license, "MIT"); @@ -169,7 +181,7 @@ test("host plugin manifests expose canonical Better Harness resources", () => { assert.equal(packageLock.packages[""].name, packageJson.name); assert.equal(packageLock.packages[""].license, packageJson.license); assert.match(packageJson.scripts["publish:dry-run"], /registry\.npmjs\.org/u); - for (const manifest of [qoder, claude, cursor, codex]) { + for (const manifest of [qoder, claude, cursor, codex, copilot]) { assert.equal(manifest.homepage, "https://github.com/QoderAI/better-harness"); assert.equal(manifest.repository, "https://github.com/QoderAI/better-harness"); } @@ -200,6 +212,17 @@ test("host plugin manifests expose canonical Better Harness resources", () => { assert.equal(claudeMarketplace.plugins[0].version, claude.version); assert.equal(claudeMarketplace.plugins[0].source, "./"); assert.deepEqual(claudeMarketplace.plugins[0].author, claude.author); + + assert.equal(copilotMarketplace.name, "better-harness"); + // Copilot marketplace owners accept name and email only. + assert.deepEqual(copilotMarketplace.owner, { name: copilot.author.name, email: copilot.author.email }); + assert.equal(copilotMarketplace.metadata.version, copilot.version); + assert.equal(copilotMarketplace.plugins.length, 1); + assert.equal(copilotMarketplace.plugins[0].name, copilot.name); + assert.equal(copilotMarketplace.plugins[0].description, copilot.description); + assert.equal(copilotMarketplace.plugins[0].version, copilot.version); + assert.equal(copilotMarketplace.plugins[0].source, "./"); + assert.equal(copilotMarketplace.plugins[0].skills, "./skills/"); }); test("npm packaging includes every host manifest while the runtime bundle stays Qoder-specific", () => { @@ -210,6 +233,7 @@ test("npm packaging includes every host manifest while the runtime bundle stays ".claude-plugin/", ".codex-plugin/", ".cursor-plugin/", + ".github/plugin/", ".qoder-plugin/", "qwen-extension.json", "AGENTS.md", @@ -240,6 +264,7 @@ test("npm packaging includes every host manifest while the runtime bundle stays assert.doesNotMatch(bundleScript, /"\.claude-plugin"/u); assert.doesNotMatch(bundleScript, /"\.codex-plugin"/u); assert.doesNotMatch(bundleScript, /"\.cursor-plugin"/u); + assert.doesNotMatch(bundleScript, /"\.github"/u); assert.doesNotMatch(bundleScript, /"schemas"/u); const verifyScript = readText("scripts/npm-package/verify-pack.mjs"); @@ -255,6 +280,9 @@ test("npm packaging includes every host manifest while the runtime bundle stays assert.match(verifyScript, /package\/\.claude-plugin\/plugin\.json/u); assert.match(verifyScript, /package\/\.claude-plugin\/marketplace\.json/u); assert.match(verifyScript, /package\/\.codex-plugin\/plugin\.json/u); + assert.match(verifyScript, /package\/\.github\/plugin\/plugin\.json/u); + assert.match(verifyScript, /package\/\.github\/plugin\/marketplace\.json/u); + assert.match(verifyScript, /"\.github\/plugin\/plugin\.json"/u); assert.match(verifyScript, /package\/\.cursor-plugin\/plugin\.json/u); assert.match(verifyScript, /package\/\.cursor-plugin\/marketplace\.json/u); assert.match(verifyScript, /package\/qwen-extension\.json/u); diff --git a/test/session-analysis-providers.test.mjs b/test/session-analysis-providers.test.mjs index ff72622..4306e9e 100644 --- a/test/session-analysis-providers.test.mjs +++ b/test/session-analysis-providers.test.mjs @@ -18,6 +18,10 @@ import { QwenSessionAnalyzer, workspaceToQwenSlugVariants, } from "../scripts/session-analysis/platforms/qwen.mjs"; +import { + CopilotSessionAnalyzer, + parseWorkspaceDescriptor, +} from "../scripts/session-analysis/platforms/copilot.mjs"; import { measureLongSessionRows } from "../scripts/session-analysis/long-sessions.mjs"; async function fixtureRoot(prefix) { @@ -33,9 +37,11 @@ test("root dispatcher creates Claude and Cursor provider analyzers", async () => assert.ok(await createAnalyzer("claude") instanceof ClaudeSessionAnalyzer); assert.ok(await createAnalyzer("cursor") instanceof CursorSessionAnalyzer); assert.ok(await createAnalyzer("qwen") instanceof QwenSessionAnalyzer); + assert.ok(await createAnalyzer("copilot") instanceof CopilotSessionAnalyzer); assert.ok(await createCapabilityAnalyzer("claude") instanceof ClaudeSessionAnalyzer); assert.ok(await createCapabilityAnalyzer("cursor") instanceof CursorSessionAnalyzer); assert.ok(await createCapabilityAnalyzer("qwen") instanceof QwenSessionAnalyzer); + assert.ok(await createCapabilityAnalyzer("copilot") instanceof CopilotSessionAnalyzer); }); test("Claude, Cursor, and Qwen workspace slugs cover Unix and Windows layouts", () => { @@ -486,3 +492,110 @@ test("Qwen provider rejects a transcript whose embedded cwd belongs to another w const result = await new QwenSessionAnalyzer().analyze({ command: "sources", workspace, home }); assert.equal(result.sessions.length, 0); }); + +test("Copilot workspace descriptors parse the session cwd binding", () => { + const descriptor = parseWorkspaceDescriptor([ + "id: 831cdd03-a101-4246-8809-5d7a80dd48be", + "cwd: C:\\workspace\\project", + "client_name: github/autopilot", + "summary_count: 5", + ].join("\n")); + assert.equal(descriptor.id, "831cdd03-a101-4246-8809-5d7a80dd48be"); + assert.equal(descriptor.cwd, "C:\\workspace\\project"); +}); + +test("Copilot provider pairs tool lifecycle, hooks, and subagent delegation", async () => { + const root = await fixtureRoot("copilot-provider-"); + const home = path.join(root, ".copilot"); + const workspace = path.join(root, "workspace", "project"); + const sessionDir = path.join(home, "session-state", "session-a"); + await mkdir(workspace, { recursive: true }); + await mkdir(sessionDir, { recursive: true }); + await writeFile(path.join(sessionDir, "workspace.yaml"), `id: session-a\ncwd: ${workspace}\n`); + await writeJsonl(path.join(sessionDir, "events.jsonl"), [ + { type: "session.start", id: "e1", timestamp: "2026-07-20T01:00:00.000Z", data: { sessionId: "session-a", selectedModel: "test-model", context: { cwd: workspace } } }, + { type: "user.message", id: "e2", timestamp: "2026-07-20T01:00:01.000Z", data: { content: "run the tests" } }, + { type: "hook.start", id: "e3", timestamp: "2026-07-20T01:00:02.000Z", data: { hookInvocationId: "h1", hookType: "userPromptSubmitted" } }, + { type: "hook.end", id: "e4", timestamp: "2026-07-20T01:00:03.000Z", data: { hookInvocationId: "h1", hookType: "userPromptSubmitted", success: true } }, + { type: "assistant.message", id: "e5", timestamp: "2026-07-20T01:00:04.000Z", data: { model: "test-model", content: "running" } }, + { type: "tool.execution_start", id: "e6", timestamp: "2026-07-20T01:00:05.000Z", data: { toolCallId: "t1", toolName: "bash", arguments: { command: "npm test" } } }, + { type: "tool.execution_complete", id: "e7", timestamp: "2026-07-20T01:00:06.000Z", data: { toolCallId: "t1", success: true, result: { content: "Tests: 1 failed, 2 passed" } } }, + { type: "subagent.started", id: "e8", timestamp: "2026-07-20T01:00:07.000Z", data: { toolCallId: "s1", agentName: "research" } }, + { type: "subagent.completed", id: "e9", timestamp: "2026-07-20T01:00:08.000Z", data: { toolCallId: "s1", agentName: "research", totalTokens: 42, totalToolCalls: 3 } }, + { type: "brand.new.event", id: "e10", timestamp: "2026-07-20T01:00:09.000Z", data: {} }, + ]); + + const analyzer = new CopilotSessionAnalyzer(); + const scope = await analyzer.resolveScope({ workspace, home }); + const roots = await analyzer.discoverSourceRoots(scope); + const sessions = await analyzer.discoverSessions(scope, roots); + assert.equal(sessions.length, 1); + assert.equal(sessions[0].sessionId, "session-a"); + + const events = await analyzer.readSession(sessions[0], scope, { includeCommandText: true }); + const byType = new Map(events.map((event) => [event.type, event])); + assert.equal(byType.get("tool.call").toolName, "bash"); + assert.equal(byType.get("tool.call").commandText, "npm test"); + assert.equal(byType.get("tool.result").success, true); + assert.deepEqual(byType.get("tool.result").resultFacts, { testsFailed: 1, testsPassed: 2 }); + assert.equal(byType.get("hook.call").hookEvent, "userPromptSubmitted"); + assert.equal(byType.get("hook.result").success, true); + assert.equal(byType.get("subagent.start").subagentName, "research"); + assert.equal(byType.get("subagent.stop").subagentTotalTokens, 42); + assert.ok(byType.has("metadata.brand.new.event")); + assert.equal(events.some((event) => event.modelUsage), false); + + const coverage = analyzer.factsSourceCoverage(scope); + assert.equal(coverage.status, "observed"); + assert.equal(coverage.usage.perResponseUsageObserved, false); +}); + +test("Copilot provider keeps transcript-less workspace sessions explicit", async () => { + const root = await fixtureRoot("copilot-partial-"); + const home = path.join(root, ".copilot"); + const workspace = path.join(root, "workspace", "project"); + const withTranscript = path.join(home, "session-state", "session-a"); + const withoutTranscript = path.join(home, "session-state", "session-b"); + await mkdir(workspace, { recursive: true }); + await mkdir(withTranscript, { recursive: true }); + await mkdir(withoutTranscript, { recursive: true }); + await writeFile(path.join(withTranscript, "workspace.yaml"), `id: session-a\ncwd: ${workspace}\n`); + await writeFile(path.join(withoutTranscript, "workspace.yaml"), `id: session-b\ncwd: ${workspace}\n`); + await writeJsonl(path.join(withTranscript, "events.jsonl"), [ + { type: "user.message", id: "e1", timestamp: "2026-07-20T01:00:00.000Z", data: { content: "hello" } }, + ]); + + const analyzer = new CopilotSessionAnalyzer(); + const scope = await analyzer.resolveScope({ workspace, home }); + const roots = await analyzer.discoverSourceRoots(scope); + const sessions = await analyzer.discoverSessions(scope, roots); + assert.equal(sessions.length, 1); + + const coverage = analyzer.factsSourceCoverage(scope); + assert.equal(coverage.status, "partial"); + assert.equal(coverage.transcript.workspaceSessions, 2); + assert.equal(coverage.transcript.withoutTranscript, 1); + + const warnings = await analyzer.analysisWarnings(scope, roots, sessions); + assert.ok(warnings.some((warning) => warning.code === "copilot-session-transcript-partial")); + assert.ok(warnings.some((warning) => warning.code === "copilot-per-response-usage-unobserved")); +}); + +test("Copilot provider ignores sessions from another workspace", async () => { + const root = await fixtureRoot("copilot-foreign-"); + const home = path.join(root, ".copilot"); + const workspace = path.join(root, "workspace", "project"); + const sessionDir = path.join(home, "session-state", "session-foreign"); + await mkdir(workspace, { recursive: true }); + await mkdir(sessionDir, { recursive: true }); + await writeFile(path.join(sessionDir, "workspace.yaml"), `id: session-foreign\ncwd: ${path.join(root, "workspace", "other")}\n`); + await writeJsonl(path.join(sessionDir, "events.jsonl"), [ + { type: "user.message", id: "e1", timestamp: "2026-07-20T01:00:00.000Z", data: { content: "foreign" } }, + ]); + + const analyzer = new CopilotSessionAnalyzer(); + const scope = await analyzer.resolveScope({ workspace, home }); + const roots = await analyzer.discoverSourceRoots(scope); + assert.equal((await analyzer.discoverSessions(scope, roots)).length, 0); + assert.equal(analyzer.factsSourceCoverage(scope).status, "absent"); +}); From 8f362ea3c4c60d99d5d4c34f5ed15d1f8feb086b Mon Sep 17 00:00:00 2001 From: ShawnX Date: Thu, 30 Jul 2026 08:37:03 +0800 Subject: [PATCH 3/4] docs(copilot): document copilot host support and fix provider routing Adds the Copilot host support spec, a dedicated Copilot asset-route reference, and README/architecture/glossary coverage for the two new adapters. Corrects agent-facing provider enumerations that had silently omitted Qwen and Copilot: the agent-customize inventory CLI flags, the global-assets scope and home-directory list, the routing index entry, the ADR host-adapter comment, and the session-evidence --provider enum. Without these an agent reading the routing docs could never discover --provider copilot. Test assertions that pin docs prose (adapter table, plugin metadata roots, supported-platform list, report routing) are updated in lockstep. Spec: docs/specs/2026-07-29-copilot-host-support.md Verified: npm test 872/872, npm run pack:verify PASS (npm 314 / runtime zip 338), doc-link graph 34 files / 50 links with zero drift. Co-authored-by: Copilot App (Claude Opus 5) <223556219+Copilot@users.noreply.github.com> --- README.md | 29 +++- README.zh-CN.md | 24 ++- docs/ARCHITECTURE.md | 2 +- docs/adapters/README.md | 33 +++- docs/adrs/directory-structure.md | 14 +- docs/community.md | 2 +- docs/concepts.md | 2 +- docs/glossary.md | 4 +- docs/specs/2026-07-29-copilot-host-support.md | 146 ++++++++++++++++++ references/agent-customize/README.md | 4 +- references/agent-customize/global-assets.md | 31 ++-- .../agent-customize/platforms/copilot.md | 104 +++++++++++++ references/agent-customize/routing.md | 47 +++++- .../session-evidence/sessions-diagnostics.md | 47 +++++- templates/reporting/routing.md | 2 +- test/agent-customize-architecture.test.mjs | 17 +- test/better-harness-skill.test.mjs | 2 +- test/coding-agent-platform-notes.test.mjs | 3 +- test/style-templates.test.mjs | 2 +- 19 files changed, 455 insertions(+), 60 deletions(-) create mode 100644 docs/specs/2026-07-29-copilot-host-support.md create mode 100644 references/agent-customize/platforms/copilot.md diff --git a/README.md b/README.md index db03eb5..19e00f5 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,7 @@ Pick your coding agent — you can be looking at your first report in minutes: | **Codex Desktop** | Add the repository under **Settings > Plugins > + Add > From Marketplace**, install Better Harness, start a new task, then invoke `@better-harness`. | | **Codex CLI** | Add the Git marketplace, run `codex plugin add better-harness@better-harness`, then invoke `$better-harness:better-harness`. | | **Qoder Desktop / CLI** | Nothing to install when Qoder Desktop is installed — Better Harness is built in and available to both. Open your repository and use the report prompt below. | +| **GitHub Copilot CLI** | Add the repository marketplace, install `better-harness@better-harness`, start a new session, then use the report prompt below. | | **Cursor** | Load the plugin from source — see [Installation](#installation). | Once installed, ask Better Harness to generate the host's durable report: @@ -159,8 +160,8 @@ Once installed, ask Better Harness to generate the host's durable report: Better Harness scopes behavior claims to relevant Task Episodes and the surrounding project mechanisms. Qoder produces a Canvas report; Claude Code, -Codex, and Cursor produce self-contained HTML with paired Markdown. Missing or -partial evidence remains explicit. See the +Codex, Cursor, Qwen Code, and GitHub Copilot produce self-contained HTML with +paired Markdown. Missing or partial evidence remains explicit. See the [Host Adapter Matrix](docs/adapters/README.md) for current coverage and output differences. @@ -309,6 +310,30 @@ cursor-agent --plugin-dir /path/to/better-harness Cursor session evidence is supported through workspace-matched transcripts, metadata, and audit logs. Partial or unavailable coverage remains explicit. +### GitHub Copilot + +Register this repository as a Copilot plugin marketplace, then install Better +Harness: + +```bash +copilot plugin marketplace add QoderAI/better-harness +copilot plugin install better-harness@better-harness +``` + +Verify that the Skill loaded: + +```bash +copilot plugin list +``` + +Prefer marketplace installs. Direct repository, URL, and local-path installs are +deprecated in Copilot CLI. + +Copilot session evidence is supported through workspace-matched Copilot CLI +transcripts under `~/.copilot/session-state/`. Copilot records no per-response +token usage, and VS Code Copilot Chat has no supported durable transcript; both +remain explicit evidence boundaries. + ## Develop and package from source Development requires Node.js `>=22.20.0 <25.0.0` and npm diff --git a/README.zh-CN.md b/README.zh-CN.md index cc19e34..d003748 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -145,6 +145,7 @@ Better Harness 开放了三个相互关联的层次，而不只是一个斜杠 | **Codex Desktop** | 在 **Settings > Plugins > + Add > From Marketplace** 中添加本仓库，安装 Better Harness，启动新任务，然后调用 `@better-harness`。 | | **Codex CLI** | 添加 Git Marketplace，运行 `codex plugin add better-harness@better-harness`，然后调用 `$better-harness:better-harness`。 | | **Qoder Desktop / CLI** | 安装 Qoder Desktop 后无需额外安装——Better Harness 已内置，并可在桌面端和 CLI 中使用。打开仓库并使用下方的报告提示词。 | +| **GitHub Copilot CLI** | 添加本仓库 Marketplace，安装 `better-harness@better-harness`，启动新会话，然后使用下方的报告提示词。 | | **Cursor** | 从源码加载插件——参见[安装](#installation)。 | 安装完成后，让 Better Harness 生成当前宿主支持的持久化报告： @@ -154,7 +155,7 @@ Better Harness 开放了三个相互关联的层次，而不只是一个斜杠 ``` Better Harness 会将行为断言限定在相关的任务过程片段（Task Episode）及其周边项目机制内。 -Qoder 生成 Canvas 报告；Claude Code、Codex 和 Cursor 生成自包含的 HTML 报告及配套 Markdown。 +Qoder 生成 Canvas 报告；Claude Code、Codex、Cursor、Qwen Code 和 GitHub Copilot 生成自包含的 HTML 报告及配套 Markdown。 缺失或不完整的证据会被明确标注。有关当前覆盖范围和输出差异，请参阅 [宿主适配器矩阵](docs/adapters/README.md)。 @@ -297,6 +298,27 @@ cursor-agent --plugin-dir /path/to/better-harness Cursor 会话证据来自与工作区匹配的会话记录、元数据和审计日志。 覆盖范围不完整或不可用时会被明确标注。 +### GitHub Copilot + +将本仓库注册为 Copilot 插件 Marketplace，然后安装 Better Harness： + +```bash +copilot plugin marketplace add QoderAI/better-harness +copilot plugin install better-harness@better-harness +``` + +验证 Skill 已加载： + +```bash +copilot plugin list +``` + +请优先使用 Marketplace 安装。Copilot CLI 已弃用直接从仓库、URL 或本地路径安装。 + +Copilot 会话证据来自 `~/.copilot/session-state/` 下与工作区匹配的 Copilot CLI 会话记录。 +Copilot 不记录逐次响应的 token 用量，VS Code Copilot Chat 也没有受支持的持久化会话记录； +两者均作为明确的证据边界保留。 + ## 从源码开发和打包 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a4397ec..f4ec0bb 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -66,7 +66,7 @@ reference is `docs/adrs/directory-structure.md`. in the capability-owned agent-customize and session-analysis providers. The Codex shell owns local install/discovery metadata only; Codex evidence collection remains in the capability-owned provider and session-analysis modules. The public npm - package ships all five plugin metadata roots, while the Qoder runtime bundle + package ships all six plugin metadata roots, while the Qoder runtime bundle includes only `.qoder-plugin/`. ## Template Boundaries diff --git a/docs/adapters/README.md b/docs/adapters/README.md index cbc48d5..0a749f2 100644 --- a/docs/adapters/README.md +++ b/docs/adapters/README.md @@ -1,9 +1,9 @@ # Host Adapter Matrix -This is the single entry point for Claude Code, Codex, Qoder, Cursor, and Qwen -host boundaries. Do not create `docs/adapters/claude-code.md`, +This is the single entry point for Claude Code, Codex, Qoder, Cursor, Qwen, and +GitHub Copilot host boundaries. Do not create `docs/adapters/claude-code.md`, `docs/adapters/codex.md`, `docs/adapters/qoder.md`, `docs/adapters/cursor.md`, -or `docs/adapters/qwen.md` by default. +`docs/adapters/qwen.md`, or `docs/adapters/copilot.md` by default. Host differences enter only this matrix, capability-local configured-asset providers, real session-evidence adapters, and output modes. Canonical product @@ -11,10 +11,10 @@ judgment stays in `skills/`, `models/`, `references/`, `templates/`, and `scripts//`. The `@qoderai/better-harness` npm package includes the Qoder, Claude Code, -Codex, Cursor, and Qwen plugin metadata roots. The generated Qoder runtime -bundle includes only the Qoder shell, `.qoder-plugin/`; non-Qoder generated host -artifacts remain source-local. Claude Code installs its shell through the -repository's native marketplace manifest. +Codex, Cursor, Qwen, and GitHub Copilot plugin metadata roots. The generated +Qoder runtime bundle includes only the Qoder shell, `.qoder-plugin/`; non-Qoder +generated host artifacts remain source-local. Claude Code installs its shell +through the repository's native marketplace manifest. | Host | Positioning | Shell | Configured Assets | Session Evidence | Default Output | Rules / Prompts | Smoke | | --- | --- | --- | --- | --- | --- | --- | --- | @@ -23,6 +23,7 @@ repository's native marketplace manifest. | Qoder | First-class product host | `.qoder-plugin/` | `scripts/agent-customize/providers/qoder.mjs` | `scripts/session-analysis/platforms/qoder.mjs` | `better-harness` | `.qoder/rules` + `AGENTS.md` + output templates | `better-harness harness render --mode qoder-canvas --validate` | | Cursor | Analysis-capable source-local host | `.cursor-plugin/` | `scripts/agent-customize/providers/cursor.mjs` | `scripts/session-analysis/platforms/cursor.mjs` | self-contained HTML + Markdown | `.cursor` + `.codex` compatibility + `AGENTS.md` | `agent --plugin-dir . --mode ask --print` -> Cursor evidence bundle -> validated `html` render | | Qwen Code | Analysis-capable source-local host | `qwen-extension.json` | `scripts/agent-customize/providers/qwen.mjs` | `scripts/session-analysis/platforms/qwen.mjs` | self-contained HTML + Markdown | `.qwen` + `QWEN.md` + `AGENTS.md` | `harness prepare --platform qwen` -> finalize with `html-report` validation | +| GitHub Copilot | Analysis-capable source-local host | `.github/plugin/` | `scripts/agent-customize/providers/copilot.mjs` | `scripts/session-analysis/platforms/copilot.mjs` | self-contained HTML + Markdown | `.github` + `AGENTS.md` + `~/.copilot` | `copilot plugin marketplace add .` -> `copilot plugin install better-harness@better-harness` -> configured-asset baseline -> validated `html` render | ## Discovery And Evidence @@ -55,6 +56,22 @@ repository's native marketplace manifest. JSONL transcripts under `~/.qwen/projects//chats/`. The `qwen-extension.json` manifest is native Qwen install/discovery metadata included in the public npm package; it does not own Qwen evidence collection. +- GitHub Copilot configured assets are inventoried through + `scripts/agent-customize/providers/copilot.mjs`, covering `AGENTS.md`, + `.github/copilot-instructions.md`, `.github/instructions/`, `.github/skills/`, + `.agents/skills/`, `.github/agents/`, `.github/prompts/`, `.github/hooks/`, + `.mcp.json`, `.github/mcp.json`, and the user-scope `~/.copilot` equivalents. + Installed-Plugin records come from the `installedPlugins` array in + `~/.copilot/config.json` and stay separate from marketplace catalogs and + runtime-use claims. Session evidence comes from + `scripts/session-analysis/platforms/copilot.mjs`, which reads + workspace-matching `~/.copilot/session-state//events.jsonl` bound through + each session's `workspace.yaml`. Copilot transcripts record no per-response + model token usage, and a matched session directory without `events.jsonl` + stays an explicit partial coverage boundary. `~/.copilot/session-store.db` is + documented as automatically managed and is not an evidence source. The + `.github/plugin/` shell is native Copilot install/discovery metadata included + in the public npm package; it does not own Copilot evidence collection. ## Output Modes @@ -62,7 +79,7 @@ Canonical templates live under `templates/reporting/`. - `qoder-canvas.md`: Qoder Canvas output contract, covering renderer-owned `findings.json`, Canvas-only `canvas.json`, and `report.canvas.tsx`. -- `html-visual.md`: portable Claude Code/Codex/Cursor/Qwen visual output contract, covering +- `html-visual.md`: portable Claude Code/Codex/Cursor/Qwen/Copilot visual output contract, covering `findings.json`, `report.md`, and `report.html`. - Markdown-only output has no visual companion. diff --git a/docs/adrs/directory-structure.md b/docs/adrs/directory-structure.md index 11e461c..0062d13 100644 --- a/docs/adrs/directory-structure.md +++ b/docs/adrs/directory-structure.md @@ -42,6 +42,11 @@ Legend: .codex-plugin/ # [active] thin Codex shell plugin.json # thin discovery/install metadata only +.github/plugin/ # [active] thin GitHub Copilot shell + plugin.json marketplace.json # native install/discovery metadata only + +qwen-extension.json # [active] thin Qwen Code shell + .agents/ skills// # [active] host-local only; shared logic -> root skills/ SKILL.md @@ -71,7 +76,7 @@ scripts/ core-change-watch/ # [active] static structure/core-path/history evidence session-analysis.mjs # [active] thin shim; new exports -> scripts/session-analysis/ session-analysis/ # [active] session evidence collection/normalization - platforms/.mjs # Qoder/Codex/Claude/Cursor/Qwen host adapters + platforms/.mjs # Qoder/Codex/Claude/Cursor/Qwen/Copilot host adapters ides// # target editor-local evidence not covered by host adapters / # [target] new capability owner cli.mjs # use cli.mjs for new capabilities @@ -206,9 +211,10 @@ Use the tree first. These rules resolve common collisions: - Shared workflows go to root `skills/`; host-local wrappers or generated mirrors go to `.agents/skills/`. - Host plugin directories such as `.claude-plugin/`, `.qoder-plugin/`, - `.cursor-plugin/`, and `.codex-plugin/` are install/discovery shells for one - host. Existing active shells may be hand-maintained narrowly, but the Qoder - public npm package ships all five plugin metadata roots, while the Qoder + `.cursor-plugin/`, `.codex-plugin/`, and `.github/plugin/` are + install/discovery shells for one host. Existing active shells may be + hand-maintained narrowly, but the Qoder + public npm package ships all six plugin metadata roots, while the Qoder runtime bundle ships only `.qoder-plugin/`. New host shells start from the `docs/adapters/README.md` matrix; split to `docs/adapters/.md` and add a source-local `scripts/packaging/` builder only for an accepted host-artifact diff --git a/docs/community.md b/docs/community.md index c71051a..fd14a10 100644 --- a/docs/community.md +++ b/docs/community.md @@ -32,7 +32,7 @@ This is the complete reference. For the common cases, see Start Here above. | Style grammar | Yes | `templates/style/` | Directive-only visual language; no runnable skeletons | Selected by report/style routing | Style-template tests and no copied runtime skeletons | | Structured knowledge | Candidate only | `knowledge-base/{official,community}/...` | `knowledge.md`, interim `schema.json`, fixtures, namespace uniqueness | Docs-only until registry spec, compiler, and binding tests exist | Namespace check, schema/fixture review, migration note | | Examples and operating models | Yes | `case-studies/` | Named example, scope, evidence boundary, non-runtime status | Reference material only unless separately bound | Link/path check; no runtime-policy claims | -| Host shell and packaging | Thin, or generated only after a split trigger | `.claude-plugin/`, `.qoder-plugin/`, `.cursor-plugin/`, `.codex-plugin/`, `qwen-extension.json`, future lifecycle shells | Install/discovery metadata and pointers to canonical owners | Public npm package includes all five current metadata roots; the Qoder runtime bundle includes only `.qoder-plugin/`, and generated host artifacts stay source-local | `scripts/npm-package/` verification, or split adapter note plus target builder | +| Host shell and packaging | Thin, or generated only after a split trigger | `.claude-plugin/`, `.qoder-plugin/`, `.cursor-plugin/`, `.codex-plugin/`, `.github/plugin/`, `qwen-extension.json`, future lifecycle shells | Install/discovery metadata and pointers to canonical owners | Public npm package includes all six current metadata roots; the Qoder runtime bundle includes only `.qoder-plugin/`, and generated host artifacts stay source-local | `scripts/npm-package/` verification, or split adapter note plus target builder | ## Non-Extension Boundaries diff --git a/docs/concepts.md b/docs/concepts.md index f5eb3bf..7f7467a 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -73,7 +73,7 @@ until you need diagnosis. See [../models/routing.md](../models/routing.md). | Project evidence | `better-harness core-change-watch` | Project, history, core-path, and diff signals | | Change confidence | `hooks/git-scripts/blast-radius` | Symbol-graph blast radius of a change | | Dependency governance | `better-harness dependency-governance` | Update automation, audit, stale-dep signals | -| Session evidence | `better-harness session-analysis` | Normalize Qoder, Codex, Claude, Cursor, or Qwen session behavior | +| Session evidence | `better-harness session-analysis` | Normalize Qoder, Codex, Claude, Cursor, Qwen, or Copilot session behavior | | Agent assets | `better-harness coding-agent-practices inventory` | Inventory configured agent surfaces | | Guardrails | `hooks/`, `scripts/agent-guardrails` | Secret scanning and lifecycle checks | diff --git a/docs/glossary.md b/docs/glossary.md index 78bb8b6..0e885de 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -60,7 +60,7 @@ for extension surfaces, read [community.md](community.md). | `core-change-watch` | Project, history, core-path, and current-diff evidence collection. | [scripts/core-change-watch](../scripts/core-change-watch) | | Blast radius | The symbol-graph reach of a change, computed with tree-sitter (JS/TS, Go, Python) as a git hook. | [hooks/git-scripts/blast-radius](../hooks/git-scripts/blast-radius) | | `dependency-governance` | Update-automation, audit, and stale-dependency signals. | [scripts/dependency-governance](../scripts/dependency-governance) | -| `session-analysis` | Normalizes Qoder, Codex, Claude, Cursor, or Qwen agent session behavior into evidence. | [scripts/session-analysis](../scripts/session-analysis) | +| `session-analysis` | Normalizes Qoder, Codex, Claude, Cursor, Qwen, or Copilot agent session behavior into evidence. | [scripts/session-analysis](../scripts/session-analysis) | | Guardrails | Change-time enforcement: secret scanning and lifecycle hook checks. | [hooks](../hooks), [scripts/agent-guardrails](../scripts/agent-guardrails) | ## The Action Loop (Report → Change) @@ -79,7 +79,7 @@ for extension surfaces, read [community.md](community.md). |---|---|---| | Skill | A repeatable agent workflow defined by `SKILL.md` frontmatter plus a concise workflow. | [community.md](community.md); report use: [report contract](../skills/better-harness/SKILL.md#report-output) | | Host adapter | Per-host discovery and evidence-shape glue (e.g. Qoder, Codex); keeps the engine host-neutral. | [adapters/README.md](adapters/README.md) | -| Host shell | Thin host metadata (`.claude-plugin/`, `.qoder-plugin/`, `.cursor-plugin/`, `.codex-plugin/`, `qwen-extension.json`, or a future lifecycle shell) that exposes canonical behavior without owning product logic; the public npm package ships all five current metadata roots, while the Qoder runtime bundle includes only `.qoder-plugin/`. | [ARCHITECTURE.md](ARCHITECTURE.md) | +| Host shell | Thin host metadata (`.claude-plugin/`, `.qoder-plugin/`, `.cursor-plugin/`, `.codex-plugin/`, `.github/plugin/`, `qwen-extension.json`, or a future lifecycle shell) that exposes canonical behavior without owning product logic; the public npm package ships all six current metadata roots, while the Qoder runtime bundle includes only `.qoder-plugin/`. | [ARCHITECTURE.md](ARCHITECTURE.md) | | Canonical owner | The single directory that owns a behavior's product judgment; host shells and mirrors point back to it. | [ARCHITECTURE.md](ARCHITECTURE.md) | ## "I Want To… → Use" diff --git a/docs/specs/2026-07-29-copilot-host-support.md b/docs/specs/2026-07-29-copilot-host-support.md new file mode 100644 index 0000000..6eb1867 --- /dev/null +++ b/docs/specs/2026-07-29-copilot-host-support.md @@ -0,0 +1,146 @@ +# Add GitHub Copilot as a supported host + +## Traceability + +- Spec ID: `copilot-host-support` +- Status: Implemented + +## Intent + +Make GitHub Copilot a first-class, evidence-safe Better Harness host alongside +Qoder, Codex, Claude Code, Cursor, and Qwen Code. + +Copilot already loads the canonical `better-harness` Skill today, but only by +falling back to the Claude Code shell: Copilot resolves plugin manifests in the +order `.plugin/plugin.json`, `plugin.json`, `.github/plugin/plugin.json`, +`.claude-plugin/plugin.json`, and marketplace manifests in the order +`marketplace.json`, `.plugin/marketplace.json`, `.github/plugin/marketplace.json`, +`.claude-plugin/marketplace.json`. That fallback is undocumented for users, +gives Copilot no shell of its own, and leaves the load order dependent on which +other host shells happen to exist. + +The larger gap is evidence. Every provider enum rejects `copilot`, so the +workflow cannot inventory Copilot's configured assets or read Copilot session +transcripts. A Copilot user can run the Skill but cannot get a Copilot-scoped +Harness report. + +Copilot's transcript is the richest of any supported host: `events.jsonl` +records hook, subagent, plan, compaction, and permission lifecycle events +directly. That evidence must be normalized without inventing coverage Copilot +does not record. Copilot does not persist per-response token usage in the +transcript, and VS Code Copilot Chat has no documented durable transcript, so +both stay explicit boundaries rather than zero values. + +## Acceptance Scenarios + +- **CHS-AC-1 (native shell):** The repository exposes a Copilot-native plugin + manifest at `.github/plugin/plugin.json` and a marketplace manifest at + `.github/plugin/marketplace.json`. Both carry the `package.json` version, and + the manifest declares `skills` explicitly rather than relying on a default. + Copilot resolves them ahead of `.claude-plugin/`, so host selection no longer + depends on the Claude shell. +- **CHS-AC-2 (version alignment):** `package.json` and the Qoder, Codex, Claude + Code, Cursor, Qwen, and Copilot host manifests expose the same package + version, and the packaging verification gate covers the new manifests. +- **CHS-AC-3 (configured assets):** `agent-customize` supports + `--provider copilot` through a capability-owned provider module that + inventories Copilot Rules, Skills, Agents, Hooks, MCP, and installed-Plugin + metadata from documented Copilot locations. Installed-Plugin records stay + separate from marketplace catalogs and never become runtime-use claims. +- **CHS-AC-4 (session evidence):** `session-analysis` supports + `--platform copilot` through a capability-owned platform module that reads + workspace-matching `~/.copilot/session-state//events.jsonl`, and both the + public and capability-owned analyzer factories resolve it. +- **CHS-AC-5 (evidence boundaries):** Copilot facts report + `usageFieldsObserved: false` because the transcript records no per-response + token usage. Session Diagnostics states that VS Code Copilot Chat has no + supported durable transcript and that `session-store.db` is documented as + auto-managed and is not an evidence source. +- **CHS-AC-6 (bundle propagation):** `harness evidence-bundle --platform copilot` + freezes a Copilot context and returns all three lanes, and `--copilot-home` + routes isolated configuration paths into the Agent Customize lane. +- **CHS-AC-7 (host routing):** The host adapter matrix carries a Copilot row with + discovery paths, evidence sources, default output, and a smoke command. + Portable HTML routing includes Copilot, and Qoder remains the only Canvas host. +- **CHS-AC-8 (provider behavior):** Deterministic fixtures cover Copilot asset + inventory and transcript normalization, including tool call/result pairing, + hook lifecycle, subagent delegation, and absent or unreadable transcripts. Raw + prompts, commands, output, paths, and secrets do not enter production facts. +- **CHS-AC-9 (documentation integrity):** Markdown links and the generated + Better Harness documentation routing graph remain current, and the README + documents the marketplace install path rather than the deprecated direct + install. + +## Non-goals + +- Modify `~/.copilot`, publish to `github/copilot-plugins` or + `github/awesome-copilot`, or install a user-level Copilot Skill. +- Read `~/.copilot/session-store.db`. It is documented as automatically managed + and its schema is internal. +- Treat VS Code Copilot Chat `debug-logs` as durable session evidence. +- Add Copilot OpenTelemetry ingestion for token usage. +- Add a `scripts/packaging/` Copilot host-artifact builder. That path is scoped + to an accepted host-artifact contract and currently supports Codex only. +- Automate Copilot cloud agent `enabledPlugins` configuration. +- Add Copilot sources to `agent-lint` host instructions, which currently support + Qoder, Claude Code, and Codex only. +- Add Copilot metadata to the Qoder runtime bundle. +- Treat configured Copilot assets, zero candidates, or a loaded Skill as proof of + runtime quality or Skill invocation. + +## Plan and Tasks + +1. Add `.github/plugin/plugin.json` and `.github/plugin/marketplace.json` as a + thin Copilot shell, and include `.github/plugin/` in the public package files + list. (CHS-AC-1, CHS-AC-2) +2. Add `scripts/agent-customize/providers/copilot.mjs` as the capability-local + owner and register it in the provider registry. Resolve Copilot home through + `COPILOT_HOME` and an explicit override, and build the same Manage + collections as existing providers. (CHS-AC-3) +3. Add `scripts/session-analysis/platforms/copilot.mjs` and register it in both + the public `session-analysis.mjs` dispatch and the capability-owned + `analyzer.mjs` dispatch. (CHS-AC-4, CHS-AC-5) +4. Register `copilot` in the remaining provider enums: evidence-bundle contract + and lanes, report run, task-loop source and report, report quality, + coding-agent-practices asset baseline, asset integrity, and inventory, + lifecycle demand signals, session core facts, selection profile, usage + summary, and agent-lint usage. (CHS-AC-6) +5. Add `references/agent-customize/platforms/copilot.md`, a Copilot asset route, + and a Copilot section in Session Diagnostics. (CHS-AC-3, CHS-AC-5) +6. Add the Copilot host matrix row, update the metadata-root counts across the + architecture, community, glossary, and concepts docs, and route Copilot + through portable HTML reporting. (CHS-AC-7) +7. Document the Copilot marketplace install path in both READMEs. (CHS-AC-9) +8. Extend tests and packaging verification for the new shell, provider, + platform, and enums, replacing the assertion that excluded Copilot from the + host matrix. (CHS-AC-2, CHS-AC-8, CHS-AC-9) + +## Test and Review Evidence + +- `node --test` +- `node --test test/doc-link-graph.test.mjs` +- `npm run pack:verify` +- `copilot plugin marketplace add ` then + `copilot plugin install better-harness@better-harness`, expecting the reported + skill install count. +- `node scripts/better-harness.mjs harness evidence-bundle --platform copilot + --workspace . --depth quick --format json` + +## Risk + +Copilot's plugin surface is evolving: direct repository installs are deprecated +in favor of marketplace installs, and the manifest resolution order is the only +guarantee that a Copilot-native shell takes precedence over the Claude shell. +Both are pinned by the host matrix row and manifest tests so a Copilot change +surfaces as a failing contract rather than silent host misrouting. + +Copilot transcript event names are observed from a real session rather than a +published schema. The platform module therefore treats unknown event types as +ignorable and keeps coverage explicit instead of inferring activity. + +## Unknowns + +- [NEEDS CLARIFICATION: whether Copilot publishes a versioned JSON schema for + `plugin.json` that the manifest test should validate against.] +- [NEEDS CLARIFICATION: whether `~/.copilot/session-state` layout is stable + enough to promote from observed to documented in the host matrix.] diff --git a/references/agent-customize/README.md b/references/agent-customize/README.md index ac74fc2..92f31f4 100644 --- a/references/agent-customize/README.md +++ b/references/agent-customize/README.md @@ -14,8 +14,8 @@ authority, routing, overlap, observed use, and maintenance boundaries. `mcp-review.md`, `memory-review.md`, `hooks-review.md`, `custom-agents-review.md`, and `knowledge-assets-review.md`. - Inventory and authority: `global-assets.md`. -- Provider-specific notes: `platforms/claude.md`, `platforms/codex.md`, and - `platforms/qoder.md`. +- Provider-specific notes: `platforms/claude.md`, `platforms/codex.md`, + `platforms/qoder.md`, `platforms/qwen.md`, and `platforms/copilot.md`. ## Does Not Own diff --git a/references/agent-customize/global-assets.md b/references/agent-customize/global-assets.md index 98fd443..77c2c70 100644 --- a/references/agent-customize/global-assets.md +++ b/references/agent-customize/global-assets.md @@ -1,19 +1,21 @@ # Global Coding-Agent Assets Use this reference when a readiness run, screenshot, or user request points to -Cursor, Qoder, Codex, or Claude settings, installed assets, global skills, user hooks, -commands, agents, plugins, MCPs, or memories. Treat this as a configured asset -inventory, not a session behavior report. +Cursor, Qoder, Codex, Claude, Qwen, or Copilot settings, installed assets, +global skills, user hooks, commands, agents, plugins, MCPs, or memories. Treat +this as a configured asset inventory, not a session behavior report. ## Scope -- Project assets: `.cursor`, `.qoder`, `.codex`, `.claude`, `.agents`, project rules, - skills, agents, commands, hooks, workflows, settings, and MCP config. -- User/global assets: `~/.{cursor,qoder,codex,claude}` skills, hooks, commands, - agents, rules, settings, and MCP config. +- Project assets: `.cursor`, `.qoder`, `.codex`, `.claude`, `.agents`, + `.github`, project rules, skills, agents, commands, hooks, workflows, + settings, and MCP config. +- User/global assets: `~/.{cursor,qoder,codex,claude,qwen,copilot}` skills, + hooks, commands, agents, rules, settings, and MCP config. - Plugin/marketplace assets: provider plugin caches and install evidence under - `~/.cursor`, `~/.qoder`, `~/.codex`, and `~/.claude`, including plugin-declared Skills, - MCPs, Commands, Hooks, Rules, and Subagents. + `~/.cursor`, `~/.qoder`, `~/.codex`, `~/.claude`, `~/.qwen`, and + `~/.copilot`, including plugin-declared Skills, MCPs, Commands, Hooks, Rules, + and Subagents. - Memories: `~/.qoder/memories/**` plus Qoder `SharedClientCache` `app-config.json` memory keys and `cache/db/*.db*` file presence; and Codex generated-memory metadata under `~/.codex/memories/` plus supported @@ -26,14 +28,15 @@ inventory, not a session behavior report. Run the read-only inventory when user-home or installed assets are in scope: ```bash - /scripts/agent-customize/cli.mjs inventory --provider --workspace - /scripts/coding-agent-practices/inventory.mjs --workspace --include-user-home --include-memories --format markdown - coding-agent-practices asset-integrity --workspace --language --json [--include-memories] [--include-user-home] + /scripts/agent-customize/cli.mjs inventory --provider --workspace + /scripts/coding-agent-practices/inventory.mjs --workspace --include-user-home --include-memories --format markdown + coding-agent-practices asset-integrity --workspace --language --json [--include-memories] [--include-user-home] ``` Use `--cursor-home `, `--qoder-home `, `--codex-home `, -`--claude-home `, `--claude-state `, -`--codex-app-path `, or `--shared-cache ` for fixtures, alternate +`--claude-home `, `--qwen-home `, `--copilot-home `, +`--claude-state `, `--codex-app-path `, or `--shared-cache ` +for fixtures, alternate installs, or non-standard homes. Use the `agent-customize` command as the provider-specific configured asset source of truth; use the `coding-agent-practices` wrapper when the report also needs the matrix shape or diff --git a/references/agent-customize/platforms/copilot.md b/references/agent-customize/platforms/copilot.md new file mode 100644 index 0000000..971bfe9 --- /dev/null +++ b/references/agent-customize/platforms/copilot.md @@ -0,0 +1,104 @@ +# GitHub Copilot Best Practices + +Use this file for GitHub Copilot-specific operating practice. Use `../routing.md` +for host-neutral owner selection and `codex.md`/`claude.md`/`qwen.md` for other +host references. Do not copy Copilot-only workflow advice into shared docs +unless there is matching surface evidence. + +## Operating Frame + +Treat Copilot as a configured teammate across two surfaces: Copilot CLI and the +IDE agent. Both read the same repository instruction and Skill locations, so +durable guidance placed under `.github/` and `AGENTS.md` serves every surface. +Move repeated guidance into instructions, configure Copilot for the real +workflow, connect external systems through MCP, turn repeated work into Skills, +and automate only stable workflows through hooks. + +## Prompt Shape + +A strong first prompt has four parts: + +- **Goal**: the change, bug, review, artifact, or decision needed. +- **Context**: files, folders, docs, examples, logs, errors, or other material + Copilot should inspect. +- **Constraints**: architecture rules, safety limits, review standards, platform + requirements, and do-not-touch boundaries. +- **Done when**: tests, checks, behavior, output files, or review evidence that + prove the task is complete. + +## Durable Guidance + +Copilot combines every matching instruction file rather than choosing one, so +keep each file scoped and non-contradictory: + +- `AGENTS.md` for repo layout, commands, conventions, and what "done" means. +- `.github/copilot-instructions.md` for Copilot-specific repository guidance. +- `.github/instructions/*.instructions.md` for path-scoped guidance. A file with + no `applyTo` glob is never auto-applied. +- `~/.copilot/copilot-instructions.md` and `~/.copilot/instructions/` for + personal defaults that should not live in the repository. + +Keep context files short and practical. Put large or conditional detail in +linked references. When Copilot repeats a mistake, update durable guidance only +when the lesson is reusable. + +## Configuration + +Copilot settings cascade, with later layers overriding earlier ones: built-in +defaults, managed policy, `~/.copilot/settings.json`, +`.github/copilot/settings.json`, then environment variables and command flags. + +- Keep permission and approval settings tight until a trusted workflow needs + more access. +- Use `COPILOT_HOME` to isolate a Copilot home for testing. +- `~/.copilot/config.json` is automatically managed application state, not user + configuration. Read it for installed-plugin metadata only. + +## Skills, Agents, and Plugins + +Turn a repeated workflow into a Skill when it has stable triggers, inputs, +steps, outputs, and validation. Skill and Agent resolution is first-found-wins, +and plugin-provided assets are the lowest local tier, so a project or personal +asset always wins over a plugin asset of the same name. + +- Project Skills: `.github/skills/`, `.agents/skills/`, `.claude/skills/`. +- Personal Skills: `~/.copilot/skills/`, `~/.agents/skills/`. +- Custom Agents: `.github/agents/*.agent.md` and `~/.copilot/agents/`. +- Plugins resolve a manifest from `.plugin/plugin.json`, `plugin.json`, + `.github/plugin/plugin.json`, or `.claude-plugin/plugin.json`, in that order. + Marketplaces resolve `marketplace.json` from the equivalent roots. + +Prefer marketplace installs. Direct repository, URL, and local-path installs are +deprecated in favor of `plugin@marketplace` installs. + +## External Context + +Use MCP when Copilot needs context or actions outside the repository. MCP servers +are configured in `~/.copilot/mcp-config.json` (user) or `.mcp.json` and +`.github/mcp.json` (project). MCP resolution is last-wins, so a plugin server +overrides a user server of the same name. Start with one or two MCP tools that +remove a real repeated cost. + +## Automation + +Copilot hooks load from `.github/hooks/*.json`, `~/.copilot/hooks/`, inline +`hooks` in settings, and plugin hook files. Event names accept a camelCase form +(`sessionStart`, `preToolUse`, `agentStop`) and a PascalCase form +(`SessionStart`, `PreToolUse`, `Stop`) that matches the IDE format. Command +hooks accept `bash` and `powershell` variants, so keep automation +cross-platform. + +Automate only stable workflows, and prefer a prompt reminder when a blocking +hook would be disproportionate. + +## Evidence Boundaries + +- Configured Skills, Agents, hooks, MCP servers, and installed Plugins prove a + mechanism exists. They never prove it ran. +- Copilot CLI transcripts live at + `~/.copilot/session-state//events.jsonl` with a `workspace.yaml` + descriptor. See `../../session-evidence/sessions-diagnostics.md` for the + supported collection route. +- `~/.copilot/session-store.db` is documented as automatically managed. Never + read or decode it. +- Copilot transcripts record no per-response model token usage. diff --git a/references/agent-customize/routing.md b/references/agent-customize/routing.md index d47c419..f6a13cf 100644 --- a/references/agent-customize/routing.md +++ b/references/agent-customize/routing.md @@ -31,12 +31,13 @@ Route by ownership before choosing a vendor-specific feature: - Agent guides (`AGENTS.md`, `CLAUDE.md`, Copilot, Cursor, Qoder rules) -> `agents-md-review.md`. -- Cursor/Qoder/Codex/Claude project or user assets -> `global-assets.md`; for - Claude-specific configured-asset scope, then load `platforms/claude.md`; for - Codex-specific operating practice, then load `platforms/codex.md`; for - Qoder-specific feature taxonomy, then load `platforms/qoder.md`. For - installed, user-home, settings screenshot, plugin cache, or memory scope, run - the Global/User Asset Pass. +- Cursor/Qoder/Codex/Claude/Qwen/Copilot project or user assets -> + `global-assets.md`; for Claude-specific configured-asset scope, then load + `platforms/claude.md`; for Codex-specific operating practice, then load + `platforms/codex.md`; for Qoder-specific feature taxonomy, then load + `platforms/qoder.md`; for Copilot-specific operating practice, then load + `platforms/copilot.md`. For installed, user-home, settings screenshot, plugin + cache, or memory scope, run the Global/User Asset Pass. - Prior decision, user correction, remembered preference, stale recall, cross-window adoption, or memory-safety question -> `memory-review.md` after `global-assets.md` establishes the configured/storage boundary. Memory files @@ -336,3 +337,37 @@ Use the Global/User Asset Pass from `global-assets.md` when the user asks about Qwen global assets such as `~/.qwen/skills` or `~/.qwen/hooks`, installed extensions, or memories. Keep configured inventory evidence separate from observed session behavior. + +## Copilot Asset Route + +For GitHub Copilot-specific actions, use `platforms/copilot.md` as the operating +practice reference for prompt shape, instruction files, `.github` and +`~/.copilot` configuration, testing and review loops, MCP, Skills, Agents, +hooks, and plugins. Presence is not execution proof. + +Inspect configured surfaces before projecting readiness evidence: + +- `AGENTS.md`, `.github/copilot-instructions.md`, and + `.github/instructions/*.instructions.md` for durable repo context. Copilot + combines every matching instruction file instead of choosing one, and an + instruction file without an `applyTo` glob is never auto-applied. +- `~/.copilot/settings.json` and `.github/copilot/settings.json` for model, + approval, permission, MCP, and hook defaults. `~/.copilot/config.json` is + automatically managed state, not user configuration. +- `.github/skills`, `.agents/skills`, `~/.copilot/skills`, and `~/.agents/skills` + for repeatable workflows. Resolution is first-found-wins and plugin-provided + Skills are the lowest local tier. +- `.github/agents/*.agent.md` and `~/.copilot/agents/` for custom Agents. +- MCP configuration (`~/.copilot/mcp-config.json`, project `.mcp.json`, and + `.github/mcp.json`) and connector availability for external context. +- `.github/hooks/*.json` and `~/.copilot/hooks/` for lifecycle automation. +- Installed Plugins recorded in the `installedPlugins` array of + `~/.copilot/config.json`, with plugin roots under + `~/.copilot/installed-plugins///`. Keep installed records + separate from marketplace catalogs and from runtime-use claims. +- Session, diff, test, build, and review evidence for observed execution. + +Use the Global/User Asset Pass from `global-assets.md` when the user asks about +Copilot global assets such as `~/.copilot/skills` or `~/.copilot/hooks`, or +installed plugins. Keep configured inventory evidence separate from observed +session behavior. diff --git a/references/session-evidence/sessions-diagnostics.md b/references/session-evidence/sessions-diagnostics.md index 7b3a64a..d5394c1 100644 --- a/references/session-evidence/sessions-diagnostics.md +++ b/references/session-evidence/sessions-diagnostics.md @@ -12,22 +12,22 @@ fails validation: ```bash # Discover evidence roots for a workspace - scripts/session-analysis.mjs sources --platform --workspace /path/to/repo + scripts/session-analysis.mjs sources --platform --workspace /path/to/repo # Session list with event counts and time range - scripts/session-analysis.mjs facets --platform --workspace /path/to/repo --limit 20 + scripts/session-analysis.mjs facets --platform --workspace /path/to/repo --limit 20 # Compact insight cards and action candidates - scripts/session-analysis.mjs insights --platform --workspace /path/to/repo --limit 20 + scripts/session-analysis.mjs insights --platform --workspace /path/to/repo --limit 20 # Read single session events - scripts/session-analysis.mjs show --platform --workspace /path/to/repo --session-id --include-events + scripts/session-analysis.mjs show --platform --workspace /path/to/repo --session-id --include-events # Diagnose the facts admission funnel and resolve candidate refs to local sessions - scripts/session-analysis.mjs facts --platform --workspace /path/to/repo --selection all-eligible --limit 5 --debug --output /tmp/session-facts-debug.json + scripts/session-analysis.mjs facts --platform --workspace /path/to/repo --selection all-eligible --limit 5 --debug --output /tmp/session-facts-debug.json # Expand one debug locator with normalized commands and user text - scripts/session-analysis.mjs show --platform --workspace /path/to/repo --session-id --include-events --include-command-text --include-user-text + scripts/session-analysis.mjs show --platform --workspace /path/to/repo --session-id --include-events --include-command-text --include-user-text ``` `facts --debug` is an operator-only diagnostic route. It exposes raw session @@ -38,7 +38,7 @@ opening only the candidate sessions needed to explain a surprising aggregate. Command and user text flags are also local-only and must not be used for broad transcript dumps. -Supported platforms: `qoder`, `codex`, `claude`, `cursor`, and `qwen`. Do not invent +Supported platforms: `qoder`, `codex`, `claude`, `cursor`, `qwen`, and `copilot`. Do not invent unsupported platform names. Always pass the absolute target workspace and load the matching Platform Notes before interpreting source roots or workspace bindings. @@ -138,7 +138,7 @@ in scope: - Generated reports: source evidence JSON, usage JSON, analysis packets, review packets, final reports, and quality-check output. - For installed or global assets, run - `scripts/agent-customize/cli.mjs inventory --provider ` + `scripts/agent-customize/cli.mjs inventory --provider ` or the `scripts/coding-agent-practices/inventory.mjs` wrapper, and treat its output as configured asset inventory, not session behavior. @@ -247,3 +247,34 @@ tool results as provider-labelled coverage. Generated files under Route configured Qwen rules (`QWEN.md`), Skills, hooks, extensions, and other project/user assets through `../agent-customize/global-assets.md`; configured presence does not prove use. + +### Copilot + +For GitHub Copilot CLI, the analyzer reads workspace-matching transcripts under +`~/.copilot/session-state//events.jsonl`. Each session directory +carries a `workspace.yaml` descriptor whose `cwd` binds the session to a +workspace; the analyzer also accepts the `context.cwd` recorded on the +`session.start` event. Copilot writes one typed lifecycle record per line, so +transcript records are the primary source: `user.message`, `assistant.message`, +`tool.execution_start`, `tool.execution_complete`, `hook.start`, `hook.end`, +`subagent.started`, `subagent.completed`, `session.plan_changed`, +`session.compaction_start`, `session.compaction_complete`, +`session.permissions_changed`, and `external_tool.*`. Unrecognized types stay +explicit `metadata.*` events rather than being dropped or reinterpreted. + +Keep three Copilot boundaries explicit: + +- Copilot transcripts record no per-response model token usage. Subagent + totals (`totalTokens`) and `preCompactionTokens` are aggregates and never + become per-response usage. Usage evidence requires the opt-in OpenTelemetry + export, which this workflow does not read. +- A session directory can match the workspace and carry no `events.jsonl`. That + is partial coverage, not zero activity. +- `~/.copilot/session-store.db` is documented as automatically managed. Never + read or decode it. + +VS Code Copilot Chat has no supported durable transcript; its `debug-logs` +output is undocumented and stays `unobserved`. Route configured Copilot rules +(`AGENTS.md`, `.github/copilot-instructions.md`, `.github/instructions/`), +Skills, Agents, hooks, MCP, and installed Plugins through +`../agent-customize/global-assets.md`; configured presence does not prove use. diff --git a/templates/reporting/routing.md b/templates/reporting/routing.md index 55edd72..f1c9799 100644 --- a/templates/reporting/routing.md +++ b/templates/reporting/routing.md @@ -28,6 +28,6 @@ files from other routes. | Route | Use when | Artifacts | Runtime owner | | --- | --- | --- | --- | | Qoder Canvas report | Active host is Qoder | renderer-owned `findings.json`, `canvas.json`, `report.canvas.tsx` | `qoder-canvas.md` | -| Portable HTML report | Active host is Claude Code, Codex, or Cursor, or a portable visual is explicitly requested | renderer-owned `findings.json`, `report.md`, `report.html` | `html-visual.md` | +| Portable HTML report | Active host is Claude Code, Codex, Cursor, Qwen Code, or GitHub Copilot, or a portable visual is explicitly requested | renderer-owned `findings.json`, `report.md`, `report.html` | `html-visual.md` | | Markdown only | Markdown without a visual companion is explicitly requested | `report.md`, `findings.json` | none | | Inline only | Inline or no-files output is explicitly requested | none; inline analysis writes nothing | none | diff --git a/test/agent-customize-architecture.test.mjs b/test/agent-customize-architecture.test.mjs index 071f1bc..5e2d874 100644 --- a/test/agent-customize-architecture.test.mjs +++ b/test/agent-customize-architecture.test.mjs @@ -22,6 +22,8 @@ test("agent-customize inventory keeps host collectors behind provider modules", "scripts/agent-customize/providers/qoder.mjs", "scripts/agent-customize/providers/codex.mjs", "scripts/agent-customize/providers/claude.mjs", + "scripts/agent-customize/providers/qwen.mjs", + "scripts/agent-customize/providers/copilot.mjs", "scripts/agent-customize/providers/index.mjs", ]) { await assert.doesNotReject(() => readRepoFile(relativePath), `${relativePath} should exist`); @@ -36,6 +38,8 @@ test("agent-customize inventory keeps host collectors behind provider modules", assert.match(providerIndex, /qoder/u); assert.match(providerIndex, /codex/u); assert.match(providerIndex, /claude/u); + assert.match(providerIndex, /qwen/u); + assert.match(providerIndex, /copilot/u); }); test("host architecture docs keep matrix, providers, and thin shells separate", async () => { @@ -50,8 +54,8 @@ test("host architecture docs keep matrix, providers, and thin shells separate", assert.match(adapterReadme, /# Host Adapter Matrix/u); assert.match(adapterReadme, /`docs\/adapters\/qoder\.md`/u); assert.match(adapterReadme, /Codex \| Analysis-capable source-local host \| `\.codex-plugin\/`/u); - assert.match(adapterReadme, /npm package includes the Qoder, Claude Code,\s+Codex, Cursor, and Qwen plugin metadata roots/u); - assert.match(adapterReadme, /generated Qoder runtime\s+bundle\s+includes only the Qoder shell/u); + assert.match(adapterReadme, /npm package includes the Qoder, Claude Code,\s+Codex, Cursor, Qwen, and GitHub Copilot plugin metadata roots/u); + assert.match(adapterReadme, /generated\s+Qoder runtime bundle includes only the Qoder shell/u); assert.match(adapterReadme, /Cursor \| Analysis-capable source-local host[^\n]+platforms\/cursor\.mjs/u); assert.doesNotMatch(adapterReadme, /Cursor has no session-evidence adapter/u); assert.match(adapterReadme, /Split a host into `docs\/adapters\/\.md` only when/u); @@ -67,16 +71,17 @@ test("host architecture docs keep matrix, providers, and thin shells separate", assert.match(directoryAdr, /scripts\/packaging\/` owns source-local[\s\S]*excluded from public package\/runtime/u); assert.match(architecture, /The Codex shell\s+owns local install\/discovery metadata only/u); - assert.match(architecture, /public npm\s+package ships all five plugin metadata roots[\s\S]*Qoder runtime bundle\s+includes only `\.qoder-plugin\/`/u); + assert.match(architecture, /public npm\s+package ships all six plugin metadata roots[\s\S]*Qoder runtime bundle\s+includes only `\.qoder-plugin\/`/u); assert.match(architecture, /do not create a generic detector or signal umbrella/u); assert.match(community, /`docs\/adapters\/README\.md` matrix row/u); - assert.match(community, /Public npm package includes all five current metadata roots[\s\S]*Qoder runtime bundle includes only `\.qoder-plugin\/`/u); + assert.match(community, /Public npm package includes all six current metadata roots[\s\S]*Qoder runtime bundle includes only `\.qoder-plugin\/`/u); assert.match(community, /owning `models\/\.md`, `scripts\/\/`, or `skills\/\/references\/`/u); - assert.match(glossary, /public npm package ships all five current metadata roots[\s\S]*Qoder runtime bundle includes only `\.qoder-plugin\/`/u); + assert.match(glossary, /public npm package ships all six current metadata roots[\s\S]*Qoder runtime bundle includes only `\.qoder-plugin\/`/u); assert.match(glossary, /Start with \[model routing\]\(\.\.\/models\/routing\.md\)/u); assert.match(adapterReadme, /Claude Code\s+\|/u); - assert.doesNotMatch(adapterReadme, /GitHub Copilot\s+\|/u); + assert.match(adapterReadme, /GitHub Copilot \| Analysis-capable source-local host \| `\.github\/plugin\/`[^\n]+platforms\/copilot\.mjs/u); + assert.match(directoryAdr, /\.github\/plugin\/\s+# \[active\] thin GitHub Copilot shell/u); assert.doesNotMatch(community, /`docs\/adapters\/\.md`\s+\| Discovery paths/u); assert.doesNotMatch(architecture, /Do not add `\.codex-plugin\/` until Codex has/u); assert.doesNotMatch(directoryAdr, /detector\/\s+# auxiliary detectors/u); diff --git a/test/better-harness-skill.test.mjs b/test/better-harness-skill.test.mjs index 0af8c02..5c8ad87 100644 --- a/test/better-harness-skill.test.mjs +++ b/test/better-harness-skill.test.mjs @@ -77,7 +77,7 @@ test("non-Qoder providers default to validated durable HTML with an explicit inl assert.match(skill, /HTML artifacts: findings\.json, report\.md, report\.html/); assert.match(skill, /Succeed only on\s+`status: pass`/); assert.match(skill, /Never hand-write\s+Canvas, Markdown, or HTML/); - assert.match(routing, /Portable HTML report \| Active host is Claude Code, Codex, or Cursor/); + assert.match(routing, /Portable HTML report \| Active host is Claude Code, Codex, Cursor, Qwen Code, or GitHub Copilot/); assert.match(routing, /Inline only \| Inline or no-files output is explicitly requested \| none; inline analysis writes nothing/); assert.match(adapters, /Claude Code[^\n]+scripts\/session-analysis\/platforms\/claude\.mjs[^\n]+self-contained HTML \+ Markdown/); assert.match(adapters, /Cursor[^\n]+scripts\/session-analysis\/platforms\/cursor\.mjs[^\n]+self-contained HTML \+ Markdown/); diff --git a/test/coding-agent-platform-notes.test.mjs b/test/coding-agent-platform-notes.test.mjs index 2228516..06d56e2 100644 --- a/test/coding-agent-platform-notes.test.mjs +++ b/test/coding-agent-platform-notes.test.mjs @@ -43,7 +43,8 @@ test("session diagnostics keeps the shared workflow before platform source roots assertAfter(content, "~/.claude/projects", "## Platform Notes", "Sessions Diagnostics"); assertAfter(content, "~/.cursor/projects", "## Platform Notes", "Sessions Diagnostics"); assertAfter(content, "~/.qwen/projects", "## Platform Notes", "Sessions Diagnostics"); - assert.match(content, /Supported platforms: `qoder`, `codex`, `claude`, `cursor`, and `qwen`/); + assertAfter(content, "~/.copilot/session-state", "## Platform Notes", "Sessions Diagnostics"); + assert.match(content, /Supported platforms: `qoder`, `codex`, `claude`, `cursor`, `qwen`, and `copilot`/); assert.match(content, /Never decode Cursor `store\.db`/); assert.ok(content.indexOf("session-analysis.mjs sources") < content.indexOf("## Platform Notes")); }); diff --git a/test/style-templates.test.mjs b/test/style-templates.test.mjs index 9878b33..9c2f631 100644 --- a/test/style-templates.test.mjs +++ b/test/style-templates.test.mjs @@ -75,7 +75,7 @@ test("harness report routing owns output-mode selection and exclusions", () => { assert.match(reportRouting, /Choose exactly one output route/); assert.match(reportRouting, /Qoder Canvas report/); assert.match(reportRouting, /Portable HTML report/); - assert.match(reportRouting, /Active host is Claude Code, Codex, or Cursor/); + assert.match(reportRouting, /Active host is Claude Code, Codex, Cursor, Qwen Code, or GitHub Copilot/); assert.match(reportRouting, /Markdown only/); assert.match(reportRouting, /Inline only/); assert.match(reportRouting, /inline analysis writes nothing/); From 58b93fd4492eecc517766eb7553757cd8cac3f71 Mon Sep 17 00:00:00 2001 From: ShawnX Date: Thu, 30 Jul 2026 10:16:54 +0800 Subject: [PATCH 4/4] fix(copilot): preserve copilot-home, usage, and permission evidence Applies the four review findings on PR #22. The agent-customize CLI parsed --copilot-home but never forwarded it to the collector, so every CLI run inventoried the real ~/.copilot instead of the requested path. That broke the CLI half of CHS-AC-6 while the collector API stayed correct. scripts/agent-customize/cli.mjs now passes copilotHome through, and a CLI regression test asserts the inventory resolves under the fixture home and that no evidence path points at the real user home. The Copilot platform module dropped the outputTokens that assistant messages do record. isModelRequestEvent excludes assistant-typed events, so usage cannot ride the message itself: each assistant message that reports usage now also emits a companion model.response.completed carrying outputTokens, messageId as responseId, and requestId. Input and cache tokens stay absent rather than zero. 10,462 of 11,313 assistant messages across 491 local sessions carry a non-zero outputTokens, so the coverage is genuinely partial and the warning is renamed copilot-per-response-usage-partial. permission.requested and permission.completed now normalize into the shared control.permission lifecycle, retaining only requestId, the request kind, and the result decision; intents, paths, and commands are never retained. The decision rides the result event alone, because permissionObservation counts any event carrying a decision and Copilot emits a request even when policy auto-approves, so decorating both would double-count and label all 1,177 requests as prompted friction. toolInvocationId is deliberately left unset: dedupeEvents keys on it, and 32 toolCallIds repeat across separate permission requests, which would have silently dropped legitimate events. Source coverage now emits the canonical session-core-facts fields that safeSourceCoverage whitelists, keeping the Copilot transcript counters beside them so analysisWarnings keeps working. Two latent bugs surfaced here: coverage read probe fields off session descriptors that never carried them, so withRequest was always zero, and the relevant-set ladder narrowed even without a requested window, hiding transcript-less sessions instead of reporting them as unreadable. CHS-AC-5 and CHS-AC-6 in docs/specs/2026-07-29-copilot-host-support.md now describe partial per-response usage and the payload-free permission lifecycle, and Session Diagnostics documents the two added boundaries. Validated with npm test (874/874), npm run pack:verify (PASS, npm 314 / runtime zip 338), and node --test test/doc-link-graph.test.mjs (6/6, no mermaid drift). Co-authored-by: Copilot App (Claude Opus 5) <223556219+Copilot@users.noreply.github.com> --- docs/specs/2026-07-29-copilot-host-support.md | 23 +-- .../session-evidence/sessions-diagnostics.md | 20 ++- scripts/agent-customize/cli.mjs | 1 + .../session-analysis/platforms/copilot.mjs | 136 ++++++++++++++++-- test/agent-customize.test.mjs | 40 ++++++ test/session-analysis-providers.test.mjs | 113 ++++++++++++++- 6 files changed, 303 insertions(+), 30 deletions(-) diff --git a/docs/specs/2026-07-29-copilot-host-support.md b/docs/specs/2026-07-29-copilot-host-support.md index 6eb1867..0c08359 100644 --- a/docs/specs/2026-07-29-copilot-host-support.md +++ b/docs/specs/2026-07-29-copilot-host-support.md @@ -27,9 +27,9 @@ Harness report. Copilot's transcript is the richest of any supported host: `events.jsonl` records hook, subagent, plan, compaction, and permission lifecycle events directly. That evidence must be normalized without inventing coverage Copilot -does not record. Copilot does not persist per-response token usage in the -transcript, and VS Code Copilot Chat has no documented durable transcript, so -both stay explicit boundaries rather than zero values. +does not record. Copilot records output tokens per assistant message but no +input tokens, cache tokens, or cost, and VS Code Copilot Chat has no documented +durable transcript, so both stay explicit boundaries rather than zero values. ## Acceptance Scenarios @@ -51,14 +51,19 @@ both stay explicit boundaries rather than zero values. `--platform copilot` through a capability-owned platform module that reads workspace-matching `~/.copilot/session-state//events.jsonl`, and both the public and capability-owned analyzer factories resolve it. -- **CHS-AC-5 (evidence boundaries):** Copilot facts report - `usageFieldsObserved: false` because the transcript records no per-response - token usage. Session Diagnostics states that VS Code Copilot Chat has no - supported durable transcript and that `session-store.db` is documented as - auto-managed and is not an evidence source. +- **CHS-AC-5 (evidence boundaries):** Copilot facts carry the per-response + `outputTokens` the transcript records and omit input tokens, cache tokens, and + cost rather than reporting them as zero, so per-response usage coverage is + partial and complete usage still requires the opt-in OpenTelemetry export. + Permission request and result events normalize into the shared permission + lifecycle without retaining prompt intents, paths, or commands. Session + Diagnostics states that VS Code Copilot Chat has no supported durable + transcript and that `session-store.db` is documented as auto-managed and is not + an evidence source. - **CHS-AC-6 (bundle propagation):** `harness evidence-bundle --platform copilot` freezes a Copilot context and returns all three lanes, and `--copilot-home` - routes isolated configuration paths into the Agent Customize lane. + routes isolated configuration paths into the Agent Customize lane through both + the collector API and the `agent-customize` CLI. - **CHS-AC-7 (host routing):** The host adapter matrix carries a Copilot row with discovery paths, evidence sources, default output, and a smoke command. Portable HTML routing includes Copilot, and Qoder remains the only Canvas host. diff --git a/references/session-evidence/sessions-diagnostics.md b/references/session-evidence/sessions-diagnostics.md index d5394c1..6cff846 100644 --- a/references/session-evidence/sessions-diagnostics.md +++ b/references/session-evidence/sessions-diagnostics.md @@ -259,15 +259,23 @@ transcript records are the primary source: `user.message`, `assistant.message`, `tool.execution_start`, `tool.execution_complete`, `hook.start`, `hook.end`, `subagent.started`, `subagent.completed`, `session.plan_changed`, `session.compaction_start`, `session.compaction_complete`, +`permission.requested`, `permission.completed`, `session.permissions_changed`, and `external_tool.*`. Unrecognized types stay explicit `metadata.*` events rather than being dropped or reinterpreted. -Keep three Copilot boundaries explicit: - -- Copilot transcripts record no per-response model token usage. Subagent - totals (`totalTokens`) and `preCompactionTokens` are aggregates and never - become per-response usage. Usage evidence requires the opt-in OpenTelemetry - export, which this workflow does not read. +Keep four Copilot boundaries explicit: + +- Copilot records `outputTokens` per assistant message and nothing else. Carry + that field as partial per-response usage; never fill input tokens, cache + tokens, or cost with zero. Subagent totals (`totalTokens`) and + `preCompactionTokens` are aggregates and never become per-response usage. + Complete usage evidence requires the opt-in OpenTelemetry export, which this + workflow does not read. +- Permission evidence comes from the `permission.requested` / + `permission.completed` pair, joined on `requestId`. Retain only the request + kind and the result decision. The prompt intent, paths, and commands are + payloads and are never retained. `session.permissions_changed` reports a mode + change, not a decision. - A session directory can match the workspace and carry no `events.jsonl`. That is partial coverage, not zero activity. - `~/.copilot/session-store.db` is documented as automatically managed. Never diff --git a/scripts/agent-customize/cli.mjs b/scripts/agent-customize/cli.mjs index 5605b6d..baf9edc 100644 --- a/scripts/agent-customize/cli.mjs +++ b/scripts/agent-customize/cli.mjs @@ -73,6 +73,7 @@ async function main() { codexHome: options["codex-home"], claudeHome: options["claude-home"], qwenHome: options["qwen-home"], + copilotHome: options["copilot-home"], claudeStatePath: options["claude-state"] ?? options["claude-state-path"], codexAppPath: options["codex-app-path"], qoderSharedClientCacheRoot: options["qoder-shared-client-cache-root"] ?? options["shared-client-cache-root"], diff --git a/scripts/session-analysis/platforms/copilot.mjs b/scripts/session-analysis/platforms/copilot.mjs index 17b6c51..b9a528e 100644 --- a/scripts/session-analysis/platforms/copilot.mjs +++ b/scripts/session-analysis/platforms/copilot.mjs @@ -77,6 +77,25 @@ function resultText(data) { return result?.content ?? result?.detailedContent ?? ""; } +// Copilot permission records carry protocol enums (`read`, `write`, `shell`, +// `approved`, `denied-interactively-by-user`, ...). The guard keeps unexpected +// values out of normalized events so prompt text can never leak through a field +// that is only meant to hold a bounded token. +const PERMISSION_TOKEN_PATTERN = /^[a-z][a-z0-9._-]{0,63}$/u; + +function permissionToken(value) { + const token = String(value ?? "").trim().toLowerCase(); + return PERMISSION_TOKEN_PATTERN.test(token) ? token : null; +} + +function permissionDecisionFor(resultKind) { + const token = permissionToken(resultKind); + if (!token) return null; + if (token.startsWith("approved") || token.startsWith("allowed")) return "allowed"; + if (token.startsWith("denied") || token.startsWith("rejected")) return "denied"; + return token; +} + /** * Normalize one Copilot `events.jsonl` record. * @@ -116,6 +135,8 @@ function transcriptEvents(raw, sourceRef, options) { if (rawType === "assistant.message") { const text = typeof data.content === "string" ? data.content : ""; + const outputTokens = Number(data.outputTokens); + const usageObserved = Number.isFinite(outputTokens) && outputTokens >= 0; const event = { ...base, type: "assistant", @@ -126,8 +147,29 @@ function transcriptEvents(raw, sourceRef, options) { }; if (text) event.userVisibleAssistantMessage = true; if (options.includeContent && text) event.content = text; - if (data.model) event.model = data.model; + // When usage is observed the model is attributed on the companion response + // event instead, so a single response is not counted against two events. + if (data.model && !usageObserved) event.model = data.model; events.push(event); + if (usageObserved) { + // Copilot reports output tokens per assistant message but never input + // tokens or cost, and `isModelRequestEvent` deliberately ignores plain + // `assistant` events. Emit a companion response event carrying only the + // fields that were actually observed -- omitted token fields read as 0 in + // `session-efficiency`, so no input usage or cost is invented here. + events.push({ + ...base, + type: "model.response.completed", + category: "model", + model: data.model ?? null, + modelUsage: { outputTokens }, + usageFieldsObserved: true, + responseId: data.messageId ?? null, + requestId: data.requestId ?? null, + evidenceRef: evidenceRef(sourceRef, "model.response.completed"), + summary: "assistant response usage (output tokens only)", + }); + } return events; } @@ -248,6 +290,39 @@ function transcriptEvents(raw, sourceRef, options) { return events; } + if (rawType === "permission.requested" || rawType === "permission.completed") { + // Copilot records permission handling as a request/result pair correlated by + // `requestId`. Only the correlation id and bounded protocol enums are kept -- + // the prompt payload (`intention`, `path`, `paths`, `commands`) is dropped. + // + // The decision rides the result event alone. Attaching one to the request as + // well would double-count every permission in the episode summaries, and + // Copilot emits a request even when policy auto-approves, so treating the + // request as a prompt would overstate friction. + const requested = rawType === "permission.requested"; + const event = { + ...base, + type: "control.permission", + category: "control", + lifecyclePhase: requested ? "request" : "result", + evidenceRef: evidenceRef(sourceRef, rawType), + summary: requested ? "permission requested" : "permission resolved", + }; + // Deliberately not `toolInvocationId`: a tool call can be re-prompted, and + // `dedupeEvents` keys on that field, which would drop real observations. + if (typeof data.requestId === "string" && data.requestId) { + event.permissionRequestId = data.requestId; + } + const kind = permissionToken(data.permissionRequest?.kind); + if (kind) event.permissionKind = kind; + if (!requested) { + const decision = permissionDecisionFor(data.result?.kind); + if (decision) event.permissionDecision = decision; + } + events.push(event); + return events; + } + if (rawType === "session.permissions_changed" || rawType === "session.mode_changed") { events.push({ ...base, @@ -305,6 +380,8 @@ async function probeSessionDirectory(sessionDir, workspace) { transcriptAvailable: await pathExists(transcriptPath), cwd: descriptor.cwd, records: 0, + conversationRecords: 0, + requestRecords: 0, firstSeen: null, lastSeen: null, workspaceMatch: isWorkspaceMatch(descriptor.cwd, workspace), @@ -315,6 +392,12 @@ async function probeSessionDirectory(sessionDir, workspace) { await forEachJsonLine(transcriptPath, (raw) => { summary.records += 1; + if (raw?.type === "user.message" || raw?.type === "assistant.message") { + summary.conversationRecords += 1; + } + if (raw?.type === "assistant.message" && Number.isFinite(Number(raw?.data?.outputTokens))) { + summary.requestRecords += 1; + } if (raw?.type === "session.start") { const data = raw?.data ?? {}; if (data.sessionId) summary.sessionId = data.sessionId; @@ -335,16 +418,35 @@ async function probeSessionDirectory(sessionDir, workspace) { * A Copilot session directory can exist and match the workspace while carrying * no `events.jsonl`. That state stays explicit instead of collapsing into zero * activity or a clean result. + * + * The payload populates the canonical `session-core-facts` transcript fields so + * public facts never report unmapped evidence as a confirmed zero. Copilot has + * no terminal source, so `terminalOnly` is a measured zero rather than an + * unknown, and sessions with a missing or empty transcript surface as + * `unreadable`. Copilot-specific counters are kept alongside for warnings and + * are dropped by the bounded public schema. */ -function buildCopilotSourceCoverage({ scope, roots, matched, inWindow }) { +function buildCopilotSourceCoverage({ scope, roots, matched, inWindow, inWindowProbes = [] }) { const root = roots.find((entry) => entry.kind === "copilot-session-jsonl"); const workspaceSessions = matched.length; const withTranscript = matched.filter((probe) => probe.transcriptAvailable); const withoutTranscript = workspaceSessions - withTranscript.length; + const timeUnobservedProbes = matched.filter((probe) => !probe.firstSeen && !probe.lastSeen); const timeUnobserved = withTranscript.filter((probe) => !probe.firstSeen && !probe.lastSeen).length; const emptyTranscripts = withTranscript.filter((probe) => probe.records === 0).length; const requestedWindow = scope.sinceTime !== null || scope.untilTime !== null; + // Mirrors the Cursor precedent. Without a requested window every matched + // session stays relevant, so a session whose transcript is missing or empty + // surfaces as `unreadable` instead of disappearing from the denominator. + let relevant = matched; + if (requestedWindow) { + relevant = inWindowProbes.length > 0 ? inWindowProbes : timeUnobservedProbes; + } + const withConversation = relevant.filter((probe) => probe.conversationRecords > 0).length; + const withRequest = relevant.filter((probe) => probe.requestRecords > 0).length; + const unreadable = relevant.filter((probe) => !probe.transcriptAvailable || probe.records === 0).length; + let status = "observed"; if (!root?.exists || workspaceSessions === 0) { status = "absent"; @@ -361,17 +463,27 @@ function buildCopilotSourceCoverage({ scope, roots, matched, inWindow }) { transcript: { sourceAvailable: Boolean(root?.exists), workspaceSessions, + inWindowSessions: inWindow.length, + outOfWindowSessions: Math.max(workspaceSessions - inWindow.length - timeUnobservedProbes.length, 0), + timeUnobservedSessions: timeUnobserved, + relevantSessions: relevant.length, + withConversation, + withRequest, + // Copilot exposes no terminal-only source, so this is measured, not unknown. + terminalOnly: 0, + unreadable, + // Copilot-specific detail retained for adapter warnings. withTranscript: withTranscript.length, withoutTranscript, emptyTranscripts, - timeUnobservedSessions: timeUnobserved, - inWindowSessions: inWindow.length, }, usage: { - // Copilot transcripts carry no per-response model usage. Subagent and + // Copilot records output tokens per assistant message but never input + // tokens or cost, so per-response usage is partial. Subagent and // compaction totals are aggregates and are never projected as per-response // usage. - perResponseUsageObserved: false, + perResponseUsageObserved: withRequest > 0, + perResponseUsageFields: ["outputTokens"], }, }; } @@ -459,13 +571,15 @@ export class CopilotSessionAnalyzer extends SessionAnalyzer { if (probe?.workspaceMatch) matched.push(probe); } - const inWindow = matched + const inWindowProbes = matched .filter((probe) => probe.transcriptAvailable) .filter((probe) => { const timestamp = probe.lastSeen ?? probe.firstSeen; if ((scope.sinceTime !== null || scope.untilTime !== null) && !timestamp) return false; return withinTimeRange(timestamp, scope); - }) + }); + + const inWindow = inWindowProbes .map((probe) => ({ sessionId: probe.sessionId, workspace: scope.workspace, @@ -485,7 +599,7 @@ export class CopilotSessionAnalyzer extends SessionAnalyzer { })) .sort((left, right) => (timestampMillis(right.lastSeen) ?? 0) - (timestampMillis(left.lastSeen) ?? 0)); - scope._copilotSourceCoverage = buildCopilotSourceCoverage({ scope, roots, matched, inWindow }); + scope._copilotSourceCoverage = buildCopilotSourceCoverage({ scope, roots, matched, inWindow, inWindowProbes }); return inWindow; } @@ -519,8 +633,8 @@ export class CopilotSessionAnalyzer extends SessionAnalyzer { async analysisWarnings(scope, _roots, _sessions) { const coverage = scope._copilotSourceCoverage; const warnings = [{ - code: "copilot-per-response-usage-unobserved", - message: "Copilot transcripts do not record per-response model token usage; usage evidence requires the opt-in OpenTelemetry export.", + code: "copilot-per-response-usage-partial", + message: "Copilot transcripts record output tokens per assistant response but no input tokens, cache tokens, or cost; complete usage evidence requires the opt-in OpenTelemetry export.", }]; if (!coverage || coverage.status === "absent") { warnings.push({ diff --git a/test/agent-customize.test.mjs b/test/agent-customize.test.mjs index 959dbff..1009ab7 100644 --- a/test/agent-customize.test.mjs +++ b/test/agent-customize.test.mjs @@ -1761,3 +1761,43 @@ test("Copilot inventory without user-home authority keeps project scope only", a await rm(fixture.root, { recursive: true, force: true }); } }); + +test("agent-customize CLI honours --copilot-home instead of the real user home", async () => { + // Regression: the CLI advertised --copilot-home but never forwarded it, so an + // isolated home still resolved ~/.copilot and scanned the caller's real assets. + const fixture = await makeCopilotFixture(); + try { + const result = runAgentCustomizeCli([ + "inventory", + "--provider", + "copilot", + "--workspace", + fixture.workspace, + "--copilot-home", + fixture.copilotHome, + ]); + + assert.equal(result.status, 0, result.stderr); + const inventory = JSON.parse(result.stdout); + + // The resolved home must be the override, never the caller's real ~/.copilot. + assert.equal(inventory.copilotHome, fixture.copilotHome); + assert.notEqual(inventory.copilotHome, path.join(os.homedir(), ".copilot")); + + // The inventory itself must also come from the override. + assert.equal(inventory.diagnostics.installedPluginState, "copilot-config"); + assert.deepEqual(inventory.plugins.map((plugin) => plugin.id), ["acme/delivery"]); + assert.deepEqual( + inventory.manage.skills.map((skill) => skill.name).sort(), + ["review-change", "ship-release", "user-skill"], + ); + for (const item of [...inventory.manage.skills, ...inventory.manage.mcps]) { + assert.ok( + !item.evidence?.path || !item.evidence.path.startsWith(path.join(os.homedir(), ".copilot")), + `inventory item escaped the override home: ${item.evidence?.path}`, + ); + } + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); diff --git a/test/session-analysis-providers.test.mjs b/test/session-analysis-providers.test.mjs index 4306e9e..53abae4 100644 --- a/test/session-analysis-providers.test.mjs +++ b/test/session-analysis-providers.test.mjs @@ -517,12 +517,13 @@ test("Copilot provider pairs tool lifecycle, hooks, and subagent delegation", as { type: "user.message", id: "e2", timestamp: "2026-07-20T01:00:01.000Z", data: { content: "run the tests" } }, { type: "hook.start", id: "e3", timestamp: "2026-07-20T01:00:02.000Z", data: { hookInvocationId: "h1", hookType: "userPromptSubmitted" } }, { type: "hook.end", id: "e4", timestamp: "2026-07-20T01:00:03.000Z", data: { hookInvocationId: "h1", hookType: "userPromptSubmitted", success: true } }, - { type: "assistant.message", id: "e5", timestamp: "2026-07-20T01:00:04.000Z", data: { model: "test-model", content: "running" } }, + { type: "assistant.message", id: "e5", timestamp: "2026-07-20T01:00:04.000Z", data: { model: "test-model", content: "running", messageId: "m1", requestId: "r1", outputTokens: 128 } }, { type: "tool.execution_start", id: "e6", timestamp: "2026-07-20T01:00:05.000Z", data: { toolCallId: "t1", toolName: "bash", arguments: { command: "npm test" } } }, { type: "tool.execution_complete", id: "e7", timestamp: "2026-07-20T01:00:06.000Z", data: { toolCallId: "t1", success: true, result: { content: "Tests: 1 failed, 2 passed" } } }, { type: "subagent.started", id: "e8", timestamp: "2026-07-20T01:00:07.000Z", data: { toolCallId: "s1", agentName: "research" } }, { type: "subagent.completed", id: "e9", timestamp: "2026-07-20T01:00:08.000Z", data: { toolCallId: "s1", agentName: "research", totalTokens: 42, totalToolCalls: 3 } }, { type: "brand.new.event", id: "e10", timestamp: "2026-07-20T01:00:09.000Z", data: {} }, + { type: "assistant.message", id: "e11", timestamp: "2026-07-20T01:00:10.000Z", data: { model: "test-model", content: "done", messageId: "m2" } }, ]); const analyzer = new CopilotSessionAnalyzer(); @@ -543,11 +544,107 @@ test("Copilot provider pairs tool lifecycle, hooks, and subagent delegation", as assert.equal(byType.get("subagent.start").subagentName, "research"); assert.equal(byType.get("subagent.stop").subagentTotalTokens, 42); assert.ok(byType.has("metadata.brand.new.event")); - assert.equal(events.some((event) => event.modelUsage), false); + + // Copilot reports output tokens per assistant message. They ride a companion + // response event because `isModelRequestEvent` ignores plain assistant events, + // and only the observed field is carried -- no input tokens or cost. + const responses = events.filter((event) => event.type === "model.response.completed"); + assert.equal(responses.length, 1); + assert.deepEqual(responses[0].modelUsage, { outputTokens: 128 }); + assert.equal(responses[0].usageFieldsObserved, true); + assert.equal(responses[0].responseId, "m1"); + assert.equal(responses[0].requestId, "r1"); + assert.equal(responses[0].model, "test-model"); + const assistants = events.filter((event) => event.type === "assistant"); + assert.equal(assistants.length, 2); + // Model attribution moves to the companion so one response is not counted twice. + assert.equal(assistants[0].model, undefined); + // An assistant message without observed usage keeps its own attribution. + assert.equal(assistants[1].model, "test-model"); const coverage = analyzer.factsSourceCoverage(scope); assert.equal(coverage.status, "observed"); - assert.equal(coverage.usage.perResponseUsageObserved, false); + assert.equal(coverage.usage.perResponseUsageObserved, true); + assert.deepEqual(coverage.usage.perResponseUsageFields, ["outputTokens"]); + assert.equal(coverage.transcript.withConversation, 1); + assert.equal(coverage.transcript.withRequest, 1); + assert.equal(coverage.transcript.terminalOnly, 0); + assert.equal(coverage.transcript.unreadable, 0); +}); + +test("Copilot provider maps the permission request and result pair without payloads", async () => { + const root = await fixtureRoot("copilot-permission-"); + const home = path.join(root, ".copilot"); + const workspace = path.join(root, "workspace", "project"); + const sessionDir = path.join(home, "session-state", "session-p"); + await mkdir(workspace, { recursive: true }); + await mkdir(sessionDir, { recursive: true }); + await writeFile(path.join(sessionDir, "workspace.yaml"), `id: session-p\ncwd: ${workspace}\n`); + await writeJsonl(path.join(sessionDir, "events.jsonl"), [ + { type: "user.message", id: "e1", timestamp: "2026-07-20T01:00:00.000Z", data: { content: "read it" } }, + { + type: "permission.requested", + id: "e2", + timestamp: "2026-07-20T01:00:01.000Z", + data: { + requestId: "req-1", + permissionRequest: { kind: "read", toolCallId: "t1", intention: "Read file: C:\\secret\\notes.md", path: "C:\\secret\\notes.md" }, + promptRequest: { kind: "path", accessKind: "read", paths: ["C:\\secret\\notes.md"], toolCallId: "t1" }, + }, + }, + { type: "permission.completed", id: "e3", timestamp: "2026-07-20T01:00:02.000Z", data: { requestId: "req-1", toolCallId: "t1", result: { kind: "approved" } } }, + { + type: "permission.requested", + id: "e4", + timestamp: "2026-07-20T01:00:03.000Z", + data: { + requestId: "req-2", + permissionRequest: { kind: "shell", toolCallId: "t2", intention: "Run: rm -rf /" }, + promptRequest: { kind: "commands", commands: ["rm -rf /"], toolCallId: "t2" }, + }, + }, + { type: "permission.completed", id: "e5", timestamp: "2026-07-20T01:00:04.000Z", data: { requestId: "req-2", toolCallId: "t2", result: { kind: "denied-interactively-by-user" } } }, + // A re-prompt reuses the tool call id; both requests must survive dedupe. + { + type: "permission.requested", + id: "e6", + timestamp: "2026-07-20T01:00:05.000Z", + data: { requestId: "req-3", permissionRequest: { kind: "shell", toolCallId: "t2" } }, + }, + { type: "permission.completed", id: "e7", timestamp: "2026-07-20T01:00:06.000Z", data: { requestId: "req-3", toolCallId: "t2", result: { kind: "approved-for-location" } } }, + { type: "session.permissions_changed", id: "e8", timestamp: "2026-07-20T01:00:07.000Z", data: {} }, + ]); + + const analyzer = new CopilotSessionAnalyzer(); + const scope = await analyzer.resolveScope({ workspace, home }); + const roots = await analyzer.discoverSourceRoots(scope); + const sessions = await analyzer.discoverSessions(scope, roots); + const events = await analyzer.readSession(sessions[0], scope, { includeCommandText: true }); + + const permissions = events.filter((event) => event.type === "control.permission"); + assert.equal(permissions.length, 6); + assert.deepEqual( + permissions.map((event) => [event.lifecyclePhase, event.permissionRequestId, event.permissionKind ?? null, event.permissionDecision ?? null]), + [ + ["request", "req-1", "read", null], + ["result", "req-1", null, "allowed"], + ["request", "req-2", "shell", null], + ["result", "req-2", null, "denied"], + ["request", "req-3", "shell", null], + ["result", "req-3", null, "allowed"], + ], + ); + // The lifecycle stays additive: the mode/permission change event is unaffected. + assert.ok(events.some((event) => event.type === "control.change")); + + // No prompt payload may survive normalization, even with content opted in. + const serialized = JSON.stringify(permissions); + for (const payload of ["secret", "notes.md", "rm -rf", "Read file", "intention", "paths"]) { + assert.ok(!serialized.includes(payload), `permission events leaked ${payload}`); + } + // Tool call ids are deliberately not carried on `toolInvocationId`: dedupe keys + // on that field and a re-prompt would be dropped as a duplicate. + assert.ok(permissions.every((event) => event.toolInvocationId === undefined)); }); test("Copilot provider keeps transcript-less workspace sessions explicit", async () => { @@ -575,10 +672,18 @@ test("Copilot provider keeps transcript-less workspace sessions explicit", async assert.equal(coverage.status, "partial"); assert.equal(coverage.transcript.workspaceSessions, 2); assert.equal(coverage.transcript.withoutTranscript, 1); + // The missing transcript survives inside the canonical contract instead of + // being dropped when public facts are bounded. + assert.equal(coverage.transcript.relevantSessions, 2); + assert.equal(coverage.transcript.withConversation, 1); + assert.equal(coverage.transcript.withRequest, 0); + assert.equal(coverage.transcript.unreadable, 1); + assert.equal(coverage.transcript.terminalOnly, 0); + assert.equal(coverage.usage.perResponseUsageObserved, false); const warnings = await analyzer.analysisWarnings(scope, roots, sessions); assert.ok(warnings.some((warning) => warning.code === "copilot-session-transcript-partial")); - assert.ok(warnings.some((warning) => warning.code === "copilot-per-response-usage-unobserved")); + assert.ok(warnings.some((warning) => warning.code === "copilot-per-response-usage-partial")); }); test("Copilot provider ignores sessions from another workspace", async () => {