From 18da3355f3a4986821e9c7bd890885617429a11a Mon Sep 17 00:00:00 2001 From: TheLarkInn Date: Fri, 28 Aug 2026 09:13:43 +0000 Subject: [PATCH 1/4] Add deterministic AI reporter qualification gates Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d6318e80-5da9-4858-a147-817e8692f10e --- ...eporter-r8a-ai-gates_2026-08-28-08-45.json | 11 + common/reviews/api/rush-reporter.api.md | 124 ++++ libraries/reporter/README.md | 22 + .../scripts/runAiReporterQualification.js | 20 + libraries/reporter/src/index.ts | 16 + .../qualification/AiReporterQualification.ts | 309 ++++++++ .../AiReporterQualificationCorpus.ts | 700 ++++++++++++++++++ .../reporter/src/reporters/AiReporter.ts | 127 +++- .../reporter/src/reporters/JsonReporter.ts | 7 +- .../src/reporters/ReporterRedaction.ts | 34 +- .../src/test/AiReporterQualification.test.ts | 130 ++++ .../reporter/src/test/JsonAiReporter.test.ts | 76 ++ 12 files changed, 1538 insertions(+), 38 deletions(-) create mode 100644 common/changes/@rushstack/rush-reporter/copilot-reporter-r8a-ai-gates_2026-08-28-08-45.json create mode 100644 libraries/reporter/scripts/runAiReporterQualification.js create mode 100644 libraries/reporter/src/qualification/AiReporterQualification.ts create mode 100644 libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts create mode 100644 libraries/reporter/src/test/AiReporterQualification.test.ts diff --git a/common/changes/@rushstack/rush-reporter/copilot-reporter-r8a-ai-gates_2026-08-28-08-45.json b/common/changes/@rushstack/rush-reporter/copilot-reporter-r8a-ai-gates_2026-08-28-08-45.json new file mode 100644 index 0000000000..9267c15ce6 --- /dev/null +++ b/common/changes/@rushstack/rush-reporter/copilot-reporter-r8a-ai-gates_2026-08-28-08-45.json @@ -0,0 +1,11 @@ +{ + "changes": [ + { + "packageName": "@rushstack/rush-reporter", + "comment": "Add deterministic AI reporter qualification gates and privacy-safe actionable context.", + "type": "minor" + } + ], + "packageName": "@rushstack/rush-reporter", + "email": "TheLarkInn@users.noreply.github.com" +} diff --git a/common/reviews/api/rush-reporter.api.md b/common/reviews/api/rush-reporter.api.md index a11c6d1b87..7834936c5a 100644 --- a/common/reviews/api/rush-reporter.api.md +++ b/common/reviews/api/rush-reporter.api.md @@ -7,6 +7,12 @@ import type { Readable } from 'node:stream'; import type { Writable } from 'node:stream'; +// @beta +export const AI_REPORTER_QUALIFICATION_SCHEMA_VERSION: '1.0'; + +// @beta +export const AI_REPORTER_QUALIFICATION_THRESHOLDS: IAiReporterQualificationThresholds; + // @beta export class AiReporter implements IReporter { constructor(options: IAiReporterOptions); @@ -139,6 +145,9 @@ export function detectAgent(env: Record, configuredV // @beta export function encodeNdjsonRecord(value: unknown, options?: INdjsonOptions): string; +// @beta +export function evaluateAiReporterQualification(cases: readonly IAiReporterQualificationCaseResult[], thresholds?: IAiReporterQualificationThresholds): IAiReporterQualificationResult; + // @beta export function evaluatePluginApplyGate(manifests: readonly IRushPluginManifest[], options: IPluginApplyGateOptions): IPluginApplyDecision[]; @@ -170,6 +179,9 @@ export class FileReporter implements IReporter { // @beta export function filterEventsForLogLevel(logLevel: ReporterLogLevel, events: readonly IReporterEventEnvelope[]): IReporterEventEnvelope[]; +// @beta +export function formatAiReporterQualificationFailures(result: IAiReporterQualificationResult): string; + // @beta export function getBlockedPlugins(decisions: readonly IPluginApplyDecision[]): IPluginApplyDecision[]; @@ -182,6 +194,9 @@ export function getLogLevelRank(level: ReporterLogLevel): number; // @beta export function getPrivacyClassificationRank(classification: ReporterPrivacyClassification): number; +// @beta +export function getQualifiedAiReporterDecision(env: Record, configuredAgentEnvironmentVariables: readonly string[], qualification: IAiReporterQualificationResult | undefined): IQualifiedAiReporterDecision; + // @beta export function getReporterMigrationPhase(id: ReporterMigrationPhaseId): IReporterMigrationPhase; @@ -219,11 +234,19 @@ export interface IAiDiagnostic { // (undocumented) readonly code: string; // (undocumented) + readonly context?: Readonly>; + // (undocumented) + readonly detailKey?: string; + // (undocumented) + readonly diagnosticId?: string; + // (undocumented) readonly remediation?: readonly IRushRemediationAction[]; // (undocumented) readonly severity: string; // (undocumented) readonly summary?: string; + // (undocumented) + readonly summaryKey?: string; } // @beta @@ -280,6 +303,92 @@ export interface IAiReporterOptions { readonly write: (text: string) => void; } +// @beta +export interface IAiReporterQualificationCaseResult { + // (undocumented) + readonly actionable: boolean; + // (undocumented) + readonly aiOutputBytes: number; + // (undocumented) + readonly deterministic: boolean; + // (undocumented) + readonly expectedResult: 'succeeded' | 'failed'; + // (undocumented) + readonly failures: readonly string[]; + // (undocumented) + readonly fullLogValid: boolean; + // (undocumented) + readonly legacyOutputBytes: number; + // (undocumented) + readonly name: string; + // (undocumented) + readonly normalizedAiOutputSha256: string; + // (undocumented) + readonly plaintextOutputBytes: number; + // (undocumented) + readonly privacySafe: boolean; + // (undocumented) + readonly scenario: string; + // (undocumented) + readonly stdoutContractValid: boolean; + // (undocumented) + readonly warningContractValid: boolean; +} + +// @beta +export interface IAiReporterQualificationGateResult { + // (undocumented) + readonly actual: number; + // (undocumented) + readonly failedCases: readonly string[]; + // (undocumented) + readonly id: string; + // (undocumented) + readonly passed: boolean; + // (undocumented) + readonly threshold: string; +} + +// @beta +export interface IAiReporterQualificationResult { + // (undocumented) + readonly cases: readonly IAiReporterQualificationCaseResult[]; + // (undocumented) + readonly gates: readonly IAiReporterQualificationGateResult[]; + // (undocumented) + readonly passed: boolean; + // (undocumented) + readonly schemaVersion: typeof AI_REPORTER_QUALIFICATION_SCHEMA_VERSION; + // (undocumented) + readonly thresholds: IAiReporterQualificationThresholds; +} + +// @beta +export interface IAiReporterQualificationThresholds { + // (undocumented) + readonly deterministicRunCount: number; + // (undocumented) + readonly maximumAggregateAiToLegacyPercent: number; + // (undocumented) + readonly maximumAggregateAiToPlaintextPercent: number; + // (undocumented) + readonly maximumOutputBytesPerCase: number; + // (undocumented) + readonly minimumActionableFailurePercent: number; + // (undocumented) + readonly minimumControlCases: number; + // (undocumented) + readonly minimumFailureCases: number; + // (undocumented) + readonly minimumFullLogPassPercent: number; + // (undocumented) + readonly minimumPrivacyPassPercent: number; + // (undocumented) + readonly minimumStdoutContractPassPercent: number; + // (undocumented) + readonly minimumWarningContractPassPercent: number; +} + // @beta export interface IAutomaticReporterPlan { readonly emergencyDestination: 'stderr'; @@ -702,6 +811,18 @@ export interface IProblemMatcherResult { readonly unmatchedLineCount: number; } +// @beta +export interface IQualifiedAiReporterDecision { + // (undocumented) + readonly agentDetected: boolean; + // (undocumented) + readonly eligible: boolean; + // (undocumented) + readonly reason: 'RUSH_REPORTER=legacy' | 'agent not detected' | 'qualification unavailable' | 'qualification failed' | 'qualified'; + // (undocumented) + readonly reporter?: 'ai'; +} + // @beta export interface IRenderLiveRegionOptions { readonly color: IColorizer; @@ -1468,6 +1589,9 @@ export function resolveReporterCompatibility(frontend: IReporterFrontendDescript // @beta export function resolveReporterSelection(input: IReporterSelectionInput): IReporterSelection; +// @beta +export function runAiReporterQualificationCorpusAsync(): Promise; + // @beta export function runProblemMatchers(events: readonly IReporterEventEnvelope[], matchers: readonly IProblemMatcher[], options?: IRunProblemMatchersOptions): IProblemMatcherResult; diff --git a/libraries/reporter/README.md b/libraries/reporter/README.md index b326abb9f1..90b622a8ee 100644 --- a/libraries/reporter/README.md +++ b/libraries/reporter/README.md @@ -4,6 +4,28 @@ Canonical event protocol, reporter manager, and built-in reporters for Rush. This package is released as a public beta. Exported contracts may change before the stable release. +## AI reporter qualification + +The network-free qualification corpus runs representative bootstrap/version, configuration, input, +dependency-tool, operation, cache, network/auth, plugin, cancellation, and internal failures plus +successful and warning-only controls through the AI, detailed plaintext, legacy, and full-log reporters. + +| Gate | Blocking threshold | +| --- | --- | +| Failure/control coverage | At least 10 failure cases and 2 successful controls | +| Actionability | 100% of failures retain stable code, category, context, and remediation | +| Output size | At most 64 KiB per case; aggregate AI bytes at most 50% of legacy and plaintext | +| Determinism | Byte-identical normalized AI output across 3 runs | +| Privacy | 100% secret redaction and no private producer identity leakage | +| Full log | 100% absolute, existing, owner-only where supported, complete, and failure-correlated | +| Stdout/warnings | 100% payload-only NDJSON and warning suppression/detail compliance | + +Run `rushx build && node scripts/runAiReporterQualification.js` from this project to print the +machine-readable result. Machine-specific paths are normalized before hashing and are not stored. Passing +these gates only produces a reusable qualification decision; it does not enable environment-based automatic +reporter selection. The pre-major Rush frontend remains explicit/repository-opt-in, and +`RUSH_REPORTER=legacy` remains authoritative. + ## Links - [CHANGELOG.md](https://github.com/microsoft/rushstack/blob/main/libraries/reporter/CHANGELOG.md) - Find out diff --git a/libraries/reporter/scripts/runAiReporterQualification.js b/libraries/reporter/scripts/runAiReporterQualification.js new file mode 100644 index 0000000000..53e714d6fa --- /dev/null +++ b/libraries/reporter/scripts/runAiReporterQualification.js @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +const { + formatAiReporterQualificationFailures, + runAiReporterQualificationCorpusAsync +} = require('../lib-commonjs'); + +runAiReporterQualificationCorpusAsync() + .then((result) => { + process.stdout.write(`${JSON.stringify(result, undefined, 2)}\n`); + if (!result.passed) { + process.stderr.write(`${formatAiReporterQualificationFailures(result)}\n`); + process.exitCode = 1; + } + }) + .catch((error) => { + process.stderr.write(`${error.stack || error.message}\n`); + process.exitCode = 1; + }); diff --git a/libraries/reporter/src/index.ts b/libraries/reporter/src/index.ts index 2167262849..4b2509960f 100644 --- a/libraries/reporter/src/index.ts +++ b/libraries/reporter/src/index.ts @@ -337,6 +337,22 @@ export { isWithinMemoryBudget } from './perf/PerformanceBudgets'; +export type { + IAiReporterQualificationThresholds, + IAiReporterQualificationCaseResult, + IAiReporterQualificationGateResult, + IAiReporterQualificationResult, + IQualifiedAiReporterDecision +} from './qualification/AiReporterQualification'; +export { + AI_REPORTER_QUALIFICATION_SCHEMA_VERSION, + AI_REPORTER_QUALIFICATION_THRESHOLDS, + evaluateAiReporterQualification, + formatAiReporterQualificationFailures, + getQualifiedAiReporterDecision +} from './qualification/AiReporterQualification'; +export { runAiReporterQualificationCorpusAsync } from './qualification/AiReporterQualificationCorpus'; + export type { ReporterMigrationPhaseId, IReporterMigrationPhase } from './migration/MigrationPhase'; export { REPORTER_MIGRATION_PHASES, getReporterMigrationPhase } from './migration/MigrationPhase'; export type { diff --git a/libraries/reporter/src/qualification/AiReporterQualification.ts b/libraries/reporter/src/qualification/AiReporterQualification.ts new file mode 100644 index 0000000000..e38f50e3be --- /dev/null +++ b/libraries/reporter/src/qualification/AiReporterQualification.ts @@ -0,0 +1,309 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { detectAgent } from '../config/AgentDetection'; +import { isLegacyEmergencyFallbackRequested } from '../reporters/LegacyReporter'; +import { REPORTER_PERFORMANCE_BUDGETS } from '../perf/PerformanceBudgets'; + +/** + * The version of the machine-readable AI reporter qualification result. + * + * @beta + */ +export const AI_REPORTER_QUALIFICATION_SCHEMA_VERSION: '1.0' = '1.0'; + +/** + * The blocking thresholds for the deterministic AI reporter corpus. + * + * @beta + */ +export interface IAiReporterQualificationThresholds { + readonly minimumFailureCases: number; + readonly minimumControlCases: number; + readonly minimumActionableFailurePercent: number; + readonly maximumOutputBytesPerCase: number; + readonly maximumAggregateAiToLegacyPercent: number; + readonly maximumAggregateAiToPlaintextPercent: number; + readonly deterministicRunCount: number; + readonly minimumPrivacyPassPercent: number; + readonly minimumFullLogPassPercent: number; + readonly minimumStdoutContractPassPercent: number; + readonly minimumWarningContractPassPercent: number; +} + +/** + * The frozen qualification thresholds used by CI. + * + * @beta + */ +export const AI_REPORTER_QUALIFICATION_THRESHOLDS: IAiReporterQualificationThresholds = { + minimumFailureCases: 10, + minimumControlCases: 2, + minimumActionableFailurePercent: 100, + maximumOutputBytesPerCase: REPORTER_PERFORMANCE_BUDGETS.maxAiOutputBytes, + maximumAggregateAiToLegacyPercent: 50, + maximumAggregateAiToPlaintextPercent: 50, + deterministicRunCount: 3, + minimumPrivacyPassPercent: 100, + minimumFullLogPassPercent: 100, + minimumStdoutContractPassPercent: 100, + minimumWarningContractPassPercent: 100 +}; + +/** + * A safe, path-free measurement for one deterministic corpus case. + * + * @beta + */ +export interface IAiReporterQualificationCaseResult { + readonly name: string; + readonly scenario: string; + readonly expectedResult: 'succeeded' | 'failed'; + readonly aiOutputBytes: number; + readonly plaintextOutputBytes: number; + readonly legacyOutputBytes: number; + readonly normalizedAiOutputSha256: string; + readonly actionable: boolean; + readonly deterministic: boolean; + readonly privacySafe: boolean; + readonly fullLogValid: boolean; + readonly stdoutContractValid: boolean; + readonly warningContractValid: boolean; + readonly failures: readonly string[]; +} + +/** + * The result of one blocking qualification gate. + * + * @beta + */ +export interface IAiReporterQualificationGateResult { + readonly id: string; + readonly passed: boolean; + readonly actual: number; + readonly threshold: string; + readonly failedCases: readonly string[]; +} + +/** + * The complete machine-readable AI reporter qualification result. + * + * @beta + */ +export interface IAiReporterQualificationResult { + readonly schemaVersion: typeof AI_REPORTER_QUALIFICATION_SCHEMA_VERSION; + readonly passed: boolean; + readonly thresholds: IAiReporterQualificationThresholds; + readonly cases: readonly IAiReporterQualificationCaseResult[]; + readonly gates: readonly IAiReporterQualificationGateResult[]; +} + +function percent(passing: number, total: number): number { + return total === 0 ? 0 : (passing / total) * 100; +} + +function ratioPercent(numerator: number, denominator: number): number { + return denominator === 0 ? Number.POSITIVE_INFINITY : (numerator / denominator) * 100; +} + +function getHighestRatioCaseNames( + cases: readonly IAiReporterQualificationCaseResult[], + getDenominator: (testCase: IAiReporterQualificationCaseResult) => number +): string[] { + return [...cases] + .sort( + (left, right) => + ratioPercent(right.aiOutputBytes, getDenominator(right)) - + ratioPercent(left.aiOutputBytes, getDenominator(left)) + ) + .slice(0, 3) + .map(({ name }) => name); +} + +function createPercentageGate( + id: string, + cases: readonly IAiReporterQualificationCaseResult[], + predicate: (testCase: IAiReporterQualificationCaseResult) => boolean, + minimumPercent: number +): IAiReporterQualificationGateResult { + const failedCases: string[] = cases.filter((testCase) => !predicate(testCase)).map(({ name }) => name); + return { + id, + passed: percent(cases.length - failedCases.length, cases.length) >= minimumPercent, + actual: percent(cases.length - failedCases.length, cases.length), + threshold: `>= ${minimumPercent}%`, + failedCases + }; +} + +/** + * Evaluates safe corpus measurements against the frozen blocking thresholds. + * + * @beta + */ +export function evaluateAiReporterQualification( + cases: readonly IAiReporterQualificationCaseResult[], + thresholds: IAiReporterQualificationThresholds = AI_REPORTER_QUALIFICATION_THRESHOLDS +): IAiReporterQualificationResult { + const failureCases: readonly IAiReporterQualificationCaseResult[] = cases.filter( + ({ expectedResult }) => expectedResult === 'failed' + ); + const controlCases: readonly IAiReporterQualificationCaseResult[] = cases.filter( + ({ expectedResult }) => expectedResult === 'succeeded' + ); + const totalAiBytes: number = cases.reduce((sum, testCase) => sum + testCase.aiOutputBytes, 0); + const totalLegacyBytes: number = cases.reduce((sum, testCase) => sum + testCase.legacyOutputBytes, 0); + const totalPlaintextBytes: number = cases.reduce((sum, testCase) => sum + testCase.plaintextOutputBytes, 0); + const aggregateAiToLegacyPercent: number = ratioPercent(totalAiBytes, totalLegacyBytes); + const aggregateAiToPlaintextPercent: number = ratioPercent(totalAiBytes, totalPlaintextBytes); + + const gates: IAiReporterQualificationGateResult[] = [ + { + id: 'corpus.failure-cases', + passed: failureCases.length >= thresholds.minimumFailureCases, + actual: failureCases.length, + threshold: `>= ${thresholds.minimumFailureCases}`, + failedCases: [] + }, + { + id: 'corpus.control-cases', + passed: controlCases.length >= thresholds.minimumControlCases, + actual: controlCases.length, + threshold: `>= ${thresholds.minimumControlCases}`, + failedCases: [] + }, + createPercentageGate( + 'actionability', + failureCases, + ({ actionable }) => actionable, + thresholds.minimumActionableFailurePercent + ), + { + id: 'size.absolute', + passed: cases.every(({ aiOutputBytes }) => aiOutputBytes <= thresholds.maximumOutputBytesPerCase), + actual: Math.max(0, ...cases.map(({ aiOutputBytes }) => aiOutputBytes)), + threshold: `<= ${thresholds.maximumOutputBytesPerCase} bytes`, + failedCases: cases + .filter(({ aiOutputBytes }) => aiOutputBytes > thresholds.maximumOutputBytesPerCase) + .map(({ name }) => name) + }, + { + id: 'size.vs-legacy', + passed: aggregateAiToLegacyPercent <= thresholds.maximumAggregateAiToLegacyPercent, + actual: aggregateAiToLegacyPercent, + threshold: `<= ${thresholds.maximumAggregateAiToLegacyPercent}%`, + failedCases: + aggregateAiToLegacyPercent <= thresholds.maximumAggregateAiToLegacyPercent + ? [] + : getHighestRatioCaseNames(cases, ({ legacyOutputBytes }) => legacyOutputBytes) + }, + { + id: 'size.vs-plaintext', + passed: aggregateAiToPlaintextPercent <= thresholds.maximumAggregateAiToPlaintextPercent, + actual: aggregateAiToPlaintextPercent, + threshold: `<= ${thresholds.maximumAggregateAiToPlaintextPercent}%`, + failedCases: + aggregateAiToPlaintextPercent <= thresholds.maximumAggregateAiToPlaintextPercent + ? [] + : getHighestRatioCaseNames(cases, ({ plaintextOutputBytes }) => plaintextOutputBytes) + }, + createPercentageGate('determinism', cases, ({ deterministic }) => deterministic, 100), + createPercentageGate( + 'privacy', + cases, + ({ privacySafe }) => privacySafe, + thresholds.minimumPrivacyPassPercent + ), + createPercentageGate( + 'full-log', + cases, + ({ fullLogValid }) => fullLogValid, + thresholds.minimumFullLogPassPercent + ), + createPercentageGate( + 'stdout-contract', + cases, + ({ stdoutContractValid }) => stdoutContractValid, + thresholds.minimumStdoutContractPassPercent + ), + createPercentageGate( + 'warning-contract', + cases, + ({ warningContractValid }) => warningContractValid, + thresholds.minimumWarningContractPassPercent + ) + ]; + + return { + schemaVersion: AI_REPORTER_QUALIFICATION_SCHEMA_VERSION, + passed: gates.every(({ passed }) => passed), + thresholds, + cases, + gates + }; +} + +/** + * Formats failed gates with actionable case names for CI output. + * + * @beta + */ +export function formatAiReporterQualificationFailures(result: IAiReporterQualificationResult): string { + return result.gates + .filter(({ passed }) => !passed) + .map( + ({ id, actual, threshold, failedCases }) => + `${id}: actual=${Number.isFinite(actual) ? actual.toFixed(2) : String(actual)}, ` + + `required=${threshold}` + + (failedCases.length > 0 ? `; cases=${failedCases.join(', ')}` : '') + ) + .join('\n'); +} + +/** + * The isolated decision produced for a future automatic-selection integration. + * + * @beta + */ +export interface IQualifiedAiReporterDecision { + readonly agentDetected: boolean; + readonly eligible: boolean; + readonly reporter?: 'ai'; + readonly reason: + | 'RUSH_REPORTER=legacy' + | 'agent not detected' + | 'qualification unavailable' + | 'qualification failed' + | 'qualified'; +} + +/** + * Resolves whether an agent environment is eligible for a future AI reporter selection. + * + * @remarks + * This helper does not alter reporter selection by itself. The pre-major Rush + * frontend remains opt-in-only until rollout integration explicitly consumes a + * passing result. + * + * @beta + */ +export function getQualifiedAiReporterDecision( + env: Record, + configuredAgentEnvironmentVariables: readonly string[], + qualification: IAiReporterQualificationResult | undefined +): IQualifiedAiReporterDecision { + const agentDetected: boolean = detectAgent(env, configuredAgentEnvironmentVariables); + if (isLegacyEmergencyFallbackRequested(env)) { + return { agentDetected, eligible: false, reason: 'RUSH_REPORTER=legacy' }; + } + if (!agentDetected) { + return { agentDetected: false, eligible: false, reason: 'agent not detected' }; + } + if (!qualification) { + return { agentDetected: true, eligible: false, reason: 'qualification unavailable' }; + } + if (!qualification.passed) { + return { agentDetected: true, eligible: false, reason: 'qualification failed' }; + } + return { agentDetected: true, eligible: true, reporter: 'ai', reason: 'qualified' }; +} diff --git a/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts b/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts new file mode 100644 index 0000000000..a23978d10f --- /dev/null +++ b/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts @@ -0,0 +1,700 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { createHash } from 'node:crypto'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import type { IReporterEventEnvelope } from '../events/IReporterEventEnvelope'; +import type { ReporterPrivacyClassification } from '../events/ReporterPrivacyClassification'; +import type { IAiDiagnostic, IAiFinalRecord } from '../reporters/AiReporter'; +import { AiReporter } from '../reporters/AiReporter'; +import { FileReporter, type IFileReporterArtifact } from '../reporters/FileReporter'; +import { LegacyReporter } from '../reporters/LegacyReporter'; +import { PlaintextReporter } from '../reporters/PlaintextReporter'; +import { + AI_REPORTER_QUALIFICATION_THRESHOLDS, + evaluateAiReporterQualification, + type IAiReporterQualificationCaseResult, + type IAiReporterQualificationResult +} from './AiReporterQualification'; + +const FIXED_TIMESTAMP: string = '2026-08-28T08:45:34.000Z'; +const FIXED_TIME_MS: number = Date.parse(FIXED_TIMESTAMP); +const FIXED_PID: number = 4242; +const RAW_EVIDENCE: string = `deterministic external evidence ${'x'.repeat(4096)}\n`; +const CLASSIFIED_SECRET: string = 'qualification-fake-secret-token'; +const PRIVATE_PRODUCER: string = '@private/example-rush-plugin'; +const PRIVATE_COMPONENT: string = 'PrivatePluginImplementation'; + +interface ICorpusDiagnostic { + readonly code: string; + readonly category: string; + readonly summaryKey: string; + readonly parameters: Readonly< + Record< + string, + { readonly value: string | number | boolean; readonly privacy: ReporterPrivacyClassification } + > + >; + readonly remediation: readonly { + readonly descriptionKey: string; + readonly command?: string; + readonly documentationUrl?: string; + readonly automatedExecutionSafety: 'safe' | 'requires-confirmation' | 'unsafe'; + }[]; + readonly privacy?: ReporterPrivacyClassification; + readonly sourcePackage?: string; + readonly sourceComponent?: string; +} + +interface ICorpusCase { + readonly name: string; + readonly scenario: string; + readonly expectedResult: 'succeeded' | 'failed'; + readonly diagnostic?: ICorpusDiagnostic; + readonly operationStatus?: 'success' | 'failure' | 'aborted' | 'fromCache'; + readonly warningOnly?: boolean; +} + +interface ICaseRun { + readonly normalizedAiOutput: string; + readonly normalizedPlaintextOutput: string; + readonly normalizedLegacyOutput: string; + readonly result: Omit; +} + +const CORPUS: readonly ICorpusCase[] = [ + { + name: 'bootstrap-unsupported-node', + scenario: 'Rush bootstrap rejects an unsupported Node.js version', + expectedResult: 'failed', + diagnostic: { + code: 'RUSH_ENVIRONMENT_UNSUPPORTED_NODE', + category: 'environment', + summaryKey: 'diagnostic.RUSH_ENVIRONMENT_UNSUPPORTED_NODE.summary', + parameters: { + actualVersion: { value: '16.20.0', privacy: 'public' }, + expectedRange: { value: '>=20.0.0', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.install-supported-node', + documentationUrl: 'https://rushjs.io/pages/maintainer/setup_new_repo/', + automatedExecutionSafety: 'requires-confirmation' + } + ] + } + }, + { + name: 'configuration-invalid-json', + scenario: 'rush.json contains invalid JSON', + expectedResult: 'failed', + diagnostic: { + code: 'RUSH_CONFIG_INVALID_JSON', + category: 'configuration', + summaryKey: 'diagnostic.RUSH_CONFIG_INVALID_JSON.summary', + parameters: { + file: { value: 'rush.json', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.fix-rush-json', + command: 'node common/scripts/install-run-rush.js check', + automatedExecutionSafety: 'safe' + } + ] + } + }, + { + name: 'input-unknown-project', + scenario: 'an invalid --only project is rejected', + expectedResult: 'failed', + diagnostic: { + code: 'RUSH_INPUT_UNKNOWN_PROJECT', + category: 'input', + summaryKey: 'diagnostic.RUSH_INPUT_UNKNOWN_PROJECT.summary', + parameters: { + projectName: { value: '@example/missing', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.list-projects', + command: 'rush list', + automatedExecutionSafety: 'safe' + } + ] + } + }, + { + name: 'dependency-package-manager', + scenario: 'pnpm install exits unsuccessfully', + expectedResult: 'failed', + operationStatus: 'failure', + diagnostic: { + code: 'RUSH_DEPENDENCY_TOOL_FAILED', + category: 'dependency-tool', + summaryKey: 'diagnostic.RUSH_DEPENDENCY_TOOL_FAILED.summary', + parameters: { + command: { value: 'pnpm install', privacy: 'public' }, + exitCode: { value: 1, privacy: 'public' }, + logPath: { value: '/private/install.log', privacy: 'local-sensitive' } + }, + remediation: [ + { + descriptionKey: 'remediation.rush-update-purge', + command: 'rush update --purge', + automatedExecutionSafety: 'requires-confirmation' + } + ] + } + }, + { + name: 'operation-build-failure', + scenario: 'a project build operation fails', + expectedResult: 'failed', + operationStatus: 'failure', + diagnostic: { + code: 'RUSH_OPERATION_FAILED', + category: 'operation', + summaryKey: 'diagnostic.RUSH_OPERATION_FAILED.summary', + parameters: { + projectName: { value: '@example/app', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.rebuild-project', + command: 'rush rebuild --to @example/app', + automatedExecutionSafety: 'safe' + } + ] + } + }, + { + name: 'cache-restore-failure', + scenario: 'a build-cache restore is invalid and requires a local rebuild', + expectedResult: 'failed', + operationStatus: 'failure', + diagnostic: { + code: 'RUSH_OPERATION_FAILED', + category: 'operation', + summaryKey: 'diagnostic.RUSH_OPERATION_FAILED.summary', + parameters: { + cacheKey: { value: 'cache-entry-42', privacy: 'public' }, + projectName: { value: '@example/cache-consumer', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.disable-build-cache', + command: 'rush rebuild --to @example/cache-consumer --disable-build-cache', + automatedExecutionSafety: 'safe' + } + ] + } + }, + { + name: 'network-auth-unauthorized', + scenario: 'the registry returns an authentication-shaped failure', + expectedResult: 'failed', + diagnostic: { + code: 'RUSH_NETWORK_AUTH_UNAUTHORIZED', + category: 'network-auth', + summaryKey: 'diagnostic.RUSH_NETWORK_AUTH_UNAUTHORIZED.summary', + parameters: { + registryUrl: { value: 'https://registry.example.test/', privacy: 'public' }, + token: { value: CLASSIFIED_SECRET, privacy: 'secret' } + }, + remediation: [ + { + descriptionKey: 'remediation.refresh-registry-auth', + documentationUrl: 'https://rushjs.io/pages/maintainer/npm_registry_auth/', + automatedExecutionSafety: 'requires-confirmation' + } + ] + } + }, + { + name: 'plugin-api-incompatible', + scenario: 'a private plugin is incompatible with the current Rush API', + expectedResult: 'failed', + diagnostic: { + code: 'RUSH_PLUGIN_API_INCOMPATIBLE', + category: 'configuration', + summaryKey: 'diagnostic.RUSH_PLUGIN_API_INCOMPATIBLE.summary', + parameters: { + pluginName: { value: PRIVATE_PRODUCER, privacy: 'secret' }, + rushVersion: { value: '5.200.0', privacy: 'public' }, + rushVersionRange: { value: '^5.100.0', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.update-plugin', + command: 'rush update', + automatedExecutionSafety: 'requires-confirmation' + } + ], + privacy: 'local-sensitive', + sourcePackage: PRIVATE_PRODUCER, + sourceComponent: PRIVATE_COMPONENT + } + }, + { + name: 'logical-cancellation', + scenario: 'a command is cancelled and reports an aborted operation', + expectedResult: 'failed', + operationStatus: 'aborted', + diagnostic: { + code: 'RUSH_COMMAND_FAILED', + category: 'operation', + summaryKey: 'diagnostic.RUSH_COMMAND_FAILED.summary', + parameters: { + commandName: { value: 'build', privacy: 'public' }, + reason: { value: 'cancelled', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.retry-command', + command: 'rush build', + automatedExecutionSafety: 'safe' + } + ] + } + }, + { + name: 'internal-unexpected-error', + scenario: 'Rush reports an unexpected internal failure', + expectedResult: 'failed', + diagnostic: { + code: 'RUSH_INTERNAL_UNEXPECTED', + category: 'internal', + summaryKey: 'diagnostic.RUSH_INTERNAL_UNEXPECTED.summary', + parameters: { + incident: { value: 'incident-42', privacy: 'public' }, + stack: { value: CLASSIFIED_SECRET, privacy: 'secret' } + }, + remediation: [ + { + descriptionKey: 'remediation.report-rush-bug', + documentationUrl: 'https://github.com/microsoft/rushstack/issues/new/choose', + automatedExecutionSafety: 'unsafe' + } + ] + } + }, + { + name: 'success-no-warning', + scenario: 'a successful operation emits no warnings', + expectedResult: 'succeeded', + operationStatus: 'success' + }, + { + name: 'success-warning-only', + scenario: 'a successful command emits one bounded warning', + expectedResult: 'succeeded', + operationStatus: 'fromCache', + warningOnly: true, + diagnostic: { + code: 'RUSH_EXTERNAL_TOOL_PROBLEM', + category: 'operation', + summaryKey: 'diagnostic.RUSH_EXTERNAL_TOOL_PROBLEM.summary', + parameters: { + code: { value: 'W42', privacy: 'public' }, + message: { value: 'deprecated option', privacy: 'public' }, + tool: { value: 'fixture-tool', privacy: 'public' } + }, + remediation: [ + { + descriptionKey: 'remediation.remove-deprecated-option', + command: 'rush check', + automatedExecutionSafety: 'safe' + } + ] + } + } +]; + +export function normalizeAiReporterQualificationOutput( + text: string, + logPath: string, + tempRoot: string +): string { + const replacePath = (value: string, machinePath: string, token: string): string => { + const jsonEscapedPath: string = JSON.stringify(machinePath).slice(1, -1); + return value.split(machinePath).join(token).split(jsonEscapedPath).join(token); + }; + return replacePath(replacePath(text, logPath, ''), tempRoot, '').replace( + /\\/g, + '/' + ); +} + +function sha256(text: string): string { + return createHash('sha256').update(text).digest('hex'); +} + +function parseAiOutput(output: string): { + readonly records: readonly Record[]; + readonly valid: boolean; +} { + try { + const records: Record[] = output + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); + return { records, valid: records.length > 0 && output.endsWith('\n') }; + } catch { + return { records: [], valid: false }; + } +} + +function createEvents(testCase: ICorpusCase, logPath: string): IReporterEventEnvelope[] { + const events: IReporterEventEnvelope[] = []; + let sequence: number = 0; + const add = ( + type: IReporterEventEnvelope['type'], + payload: unknown, + options: { + readonly privacy?: ReporterPrivacyClassification; + readonly sourcePackage?: string; + readonly sourceComponent?: string; + readonly scope?: IReporterEventEnvelope['scope']; + } = {} + ): void => { + sequence++; + events.push({ + protocolVersion: { major: 1, minor: 1 }, + eventId: `${testCase.name}-event-${sequence}`, + sessionId: `${testCase.name}-session`, + sequence, + timestamp: FIXED_TIMESTAMP, + source: { + packageName: options.sourcePackage ?? '@microsoft/rush-lib', + packageVersion: '5.200.0', + component: options.sourceComponent + }, + scope: options.scope, + privacy: options.privacy ?? 'public', + required: type !== 'activityChanged', + type, + payload + }); + }; + + add('commandStarted', { commandName: 'build' }, { scope: { commandName: 'build' } }); + if (testCase.operationStatus) { + add( + 'operationRegistered', + { operationId: 'fixture#build', projectName: '@example/app', phaseName: '_phase:build' }, + { + scope: { + commandName: 'build', + operationId: 'fixture#build', + projectName: '@example/app', + phaseName: '_phase:build' + } + } + ); + } + add( + 'externalOutput', + { stream: 'stderr', text: RAW_EVIDENCE }, + testCase.operationStatus + ? { + privacy: 'local-sensitive', + scope: { + commandName: 'build', + operationId: 'fixture#build', + projectName: '@example/app', + phaseName: '_phase:build' + } + } + : { privacy: 'local-sensitive', scope: { commandName: 'build' } } + ); + if (testCase.operationStatus) { + add( + 'operationStatusChanged', + { operationId: 'fixture#build', status: testCase.operationStatus, durationMs: 250 }, + { scope: { commandName: 'build', operationId: 'fixture#build', projectName: '@example/app' } } + ); + add( + 'operationCompleted', + { operationId: 'fixture#build', status: testCase.operationStatus, durationMs: 250 }, + { scope: { commandName: 'build', operationId: 'fixture#build', projectName: '@example/app' } } + ); + } + if (testCase.diagnostic) { + const diagnosticId: string = `${testCase.name}-diagnostic`; + add( + 'diagnosticEmitted', + { + diagnosticId, + code: testCase.diagnostic.code, + category: testCase.diagnostic.category, + severity: testCase.warningOnly ? 'warning' : 'error', + summaryKey: testCase.diagnostic.summaryKey, + parameters: testCase.diagnostic.parameters, + remediation: testCase.diagnostic.remediation, + source: { + kind: 'tool', + toolName: testCase.diagnostic.sourcePackage ?? 'rush' + } + }, + { + privacy: testCase.diagnostic.privacy, + sourcePackage: testCase.diagnostic.sourcePackage, + sourceComponent: testCase.diagnostic.sourceComponent, + scope: { commandName: 'build' } + } + ); + } + if (testCase.expectedResult === 'failed') { + add( + 'diagnosticEmitted', + { + diagnosticId: `${testCase.name}-warning`, + code: 'RUSH_EXTERNAL_TOOL_PROBLEM', + category: 'operation', + severity: 'warning', + summaryKey: 'diagnostic.RUSH_EXTERNAL_TOOL_PROBLEM.summary', + parameters: { + tool: { value: 'fixture-tool', privacy: 'public' }, + code: { value: 'W01', privacy: 'public' }, + message: { value: 'secondary warning', privacy: 'public' } + } + }, + { scope: { commandName: 'build' } } + ); + } + add( + 'extension', + { name: 'private.fixture.secret', payload: { token: CLASSIFIED_SECRET } }, + { + privacy: 'secret', + sourcePackage: PRIVATE_PRODUCER, + sourceComponent: PRIVATE_COMPONENT, + scope: { commandName: 'build' } + } + ); + add( + 'artifactAvailable', + { artifactId: `${testCase.name}-log`, role: 'log', path: logPath, format: 'plaintext', complete: true }, + { privacy: 'local-sensitive', scope: { commandName: 'build' } } + ); + add( + 'commandResult', + { + commandName: 'build', + succeeded: testCase.expectedResult === 'succeeded', + exitCode: testCase.expectedResult === 'succeeded' ? 0 : 1 + }, + { scope: { commandName: 'build' } } + ); + add( + 'sessionCompleted', + { exitCode: testCase.expectedResult === 'succeeded' ? 0 : 1 }, + { scope: { commandName: 'build' } } + ); + return events; +} + +async function runCaseAsync( + testCase: ICorpusCase, + caseDirectory: string, + tempRoot: string +): Promise { + let aiOutput: string = ''; + let plaintextOutput: string = ''; + let legacyOutput: string = ''; + const fileReporter: FileReporter = new FileReporter({ + commonTempFolder: caseDirectory, + actionName: testCase.name, + pid: FIXED_PID, + nowMs: () => FIXED_TIME_MS + }); + const aiReporter: AiReporter = new AiReporter({ write: (text: string) => (aiOutput += text) }); + const plaintextReporter: PlaintextReporter = new PlaintextReporter({ + write: (text: string) => (plaintextOutput += text), + variant: 'detailed', + color: false, + nowMs: () => FIXED_TIME_MS + }); + const legacyReporter: LegacyReporter = new LegacyReporter({ + write: (text: string) => (legacyOutput += text), + maxParallelism: 4 + }); + + await fileReporter.initializeAsync(); + const logPath: string = fileReporter.getArtifact().path!; + const events: readonly IReporterEventEnvelope[] = createEvents(testCase, logPath); + for (const event of events) { + fileReporter.report(event); + aiReporter.report(event); + plaintextReporter.report(event); + legacyReporter.report(event); + } + await fileReporter.closeAsync(); + await aiReporter.closeAsync(); + await plaintextReporter.closeAsync(); + await legacyReporter.closeAsync(); + + const normalizedAiOutput: string = normalizeAiReporterQualificationOutput(aiOutput, logPath, tempRoot); + const normalizedPlaintextOutput: string = normalizeAiReporterQualificationOutput( + plaintextOutput, + logPath, + tempRoot + ); + const normalizedLegacyOutput: string = normalizeAiReporterQualificationOutput( + legacyOutput, + logPath, + tempRoot + ); + const parsedAi: { + readonly records: readonly Record[]; + readonly valid: boolean; + } = parseAiOutput(aiOutput); + const final: IAiFinalRecord | undefined = parsedAi.records.at(-1) as IAiFinalRecord | undefined; + const diagnostic: ICorpusDiagnostic | undefined = testCase.diagnostic; + const matchingDiagnostic: IAiDiagnostic | undefined = diagnostic + ? final?.diagnostics.find(({ code }) => code === diagnostic.code) + : undefined; + const expectedContextKeys: readonly string[] = diagnostic ? Object.keys(diagnostic.parameters).sort() : []; + const actualContextKeys: readonly string[] = Object.keys(matchingDiagnostic?.context ?? {}).sort(); + const actionable: boolean = + testCase.expectedResult === 'succeeded' + ? true + : Boolean( + final?.result === 'failed' && + diagnostic && + final.errorCodes.includes(diagnostic.code) && + matchingDiagnostic?.category === diagnostic.category && + matchingDiagnostic.summaryKey === diagnostic.summaryKey && + expectedContextKeys.every((key) => actualContextKeys.includes(key)) && + matchingDiagnostic.remediation?.some(({ command, documentationUrl }) => + Boolean(command || documentationUrl) + ) + ); + + const artifact: IFileReporterArtifact = fileReporter.getArtifact(); + const logExists: boolean = artifact.path !== undefined && fs.existsSync(artifact.path); + const logContent: string = logExists ? await fs.promises.readFile(artifact.path!, 'utf8') : ''; + const ownerOnly: boolean = + process.platform === 'win32' || + (logExists && (await fs.promises.stat(artifact.path!)).mode % 0o1000 === 0o600); + const failureCorrelated: boolean = + testCase.expectedResult === 'succeeded' || + Boolean( + diagnostic && + logContent.includes(diagnostic.code) && + logContent.includes(`${testCase.name}-diagnostic`) && + logContent.includes(`${testCase.name}-session`) + ); + const fullLogValid: boolean = Boolean( + artifact.available && + artifact.complete && + artifact.path && + path.isAbsolute(artifact.path) && + logExists && + ownerOnly && + logContent.includes('"type":"commandResult"') && + logContent.includes(RAW_EVIDENCE.trim()) && + failureCorrelated + ); + const combinedPresentedOutput: string = `${aiOutput}\n${plaintextOutput}\n${legacyOutput}\n${logContent}`; + const privacySafe: boolean = + !combinedPresentedOutput.includes(CLASSIFIED_SECRET) && + !combinedPresentedOutput.includes(PRIVATE_PRODUCER) && + !combinedPresentedOutput.includes(PRIVATE_COMPONENT); + const warningContractValid: boolean = + testCase.expectedResult === 'failed' + ? final?.warningCount === 1 && final.diagnostics.every(({ severity }) => severity === 'error') + : testCase.warningOnly + ? final?.warningCount === 1 && final.diagnostics.some(({ severity }) => severity === 'warning') + : final?.warningCount === 0; + const failures: string[] = []; + if (!actionable) failures.push('missing stable code/category/context/remediation'); + if (!privacySafe) failures.push('classified secret or private producer identity leaked'); + if (!fullLogValid) failures.push('full log path, permissions, completeness, or correlation invalid'); + if (!parsedAi.valid) failures.push('AI stdout was not payload-only NDJSON'); + if (!warningContractValid) failures.push('warning suppression/detail contract regressed'); + + return { + normalizedAiOutput, + normalizedPlaintextOutput, + normalizedLegacyOutput, + result: { + name: testCase.name, + scenario: testCase.scenario, + expectedResult: testCase.expectedResult, + aiOutputBytes: Buffer.byteLength(normalizedAiOutput, 'utf8'), + plaintextOutputBytes: Buffer.byteLength(normalizedPlaintextOutput, 'utf8'), + legacyOutputBytes: Buffer.byteLength(normalizedLegacyOutput, 'utf8'), + actionable, + privacySafe, + fullLogValid, + stdoutContractValid: parsedAi.valid, + warningContractValid, + failures + } + }; +} + +/** + * Runs the deterministic, network-free AI reporter qualification corpus. + * + * @remarks + * External package-manager, cache, registry, plugin, cancellation, and internal + * failures are represented by stable canonical event fixtures. Each case runs + * through AI, detailed plaintext, legacy, and full-detail file reporters three + * times. Machine-specific paths are normalized before hashing and are never + * stored in the returned result. + * + * @beta + */ +export async function runAiReporterQualificationCorpusAsync(): Promise { + const tempRoot: string = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'rush-ai-reporter-qualification-') + ); + try { + const runs: ICaseRun[][] = []; + for ( + let runIndex: number = 0; + runIndex < AI_REPORTER_QUALIFICATION_THRESHOLDS.deterministicRunCount; + runIndex++ + ) { + const runDirectory: string = path.join(tempRoot, `run-${runIndex}`); + await fs.promises.mkdir(runDirectory); + const run: ICaseRun[] = []; + for (const testCase of CORPUS) { + const caseDirectory: string = path.join(runDirectory, testCase.name); + await fs.promises.mkdir(caseDirectory); + run.push(await runCaseAsync(testCase, caseDirectory, tempRoot)); + } + runs.push(run); + } + + const results: IAiReporterQualificationCaseResult[] = runs[0].map( + (firstRun: ICaseRun, caseIndex: number) => { + const normalizedOutputs: readonly string[] = runs.map( + (run: readonly ICaseRun[]) => run[caseIndex].normalizedAiOutput + ); + const deterministic: boolean = normalizedOutputs.every( + (output: string) => output === normalizedOutputs[0] + ); + const failures: string[] = [...firstRun.result.failures]; + if (!deterministic) { + failures.push('normalized AI output differed across repeated runs'); + } + return { + ...firstRun.result, + deterministic, + normalizedAiOutputSha256: sha256(firstRun.normalizedAiOutput), + failures + }; + } + ); + return evaluateAiReporterQualification(results); + } finally { + await fs.promises.rm(tempRoot, { recursive: true, force: true }); + } +} diff --git a/libraries/reporter/src/reporters/AiReporter.ts b/libraries/reporter/src/reporters/AiReporter.ts index 80d9a8f26e..8420626522 100644 --- a/libraries/reporter/src/reporters/AiReporter.ts +++ b/libraries/reporter/src/reporters/AiReporter.ts @@ -3,8 +3,10 @@ import type { IReporterProtocolVersion } from '../events/ReporterProtocolVersion'; import type { IReporterEventEnvelope } from '../events/IReporterEventEnvelope'; +import type { ReporterJsonValue } from '../events/ReporterJsonValue'; import type { IReporter } from '../manager/IReporter'; import type { IRushRemediationAction } from '../diagnostics/IRushRemediationAction'; +import type { IClassifiedDiagnosticValue } from '../diagnostics/IClassifiedDiagnosticValue'; import { REPORTER_PERFORMANCE_BUDGETS } from '../perf/PerformanceBudgets'; import { REPORTER_PROTOCOL_VERSION } from '../protocol/ReporterProtocol'; @@ -21,8 +23,8 @@ const TERMINAL_STATUSES: ReadonlySet = new Set([ ]); interface IAiDiagnosticState { - readonly errorDiagnostics: IAiDiagnostic[]; - readonly warningDiagnostics: IAiDiagnostic[]; + readonly errorDiagnostics: ICollectedAiDiagnostic[]; + readonly warningDiagnostics: ICollectedAiDiagnostic[]; readonly errorCodes: Set; readonly diagnosticCategoryCounts: { [category: string]: number }; errorDiagnosticsTruncated: boolean; @@ -60,13 +62,21 @@ function createDiagnosticState(): IAiDiagnosticState { * @beta */ export interface IAiDiagnostic { + readonly diagnosticId?: string; readonly code: string; readonly category: string; readonly severity: string; readonly summary?: string; + readonly summaryKey?: string; + readonly detailKey?: string; + readonly context?: Readonly>; readonly remediation?: readonly IRushRemediationAction[]; } +interface ICollectedAiDiagnostic extends IAiDiagnostic { + readonly causeDiagnosticIds?: readonly string[]; +} + /** * The AI reporter's log reference. * @@ -255,11 +265,11 @@ export class AiReporter implements IReporter { break; } case 'diagnosticEmitted': { - const diagnostic: IAiDiagnostic & { iterationId?: number } = event.payload as IAiDiagnostic & { - iterationId?: number; + const diagnostic: { readonly iterationId?: number } = event.payload as { + readonly iterationId?: number; }; this._collectDiagnostic( - diagnostic, + event, diagnostic.iterationId === undefined ? this._globalDiagnostics : this._getWatchCycle(diagnostic.iterationId).diagnostics @@ -294,7 +304,7 @@ export class AiReporter implements IReporter { severity?: string; text?: string; }; - if (payload.severity === 'error' && payload.text) { + if (event.privacy !== 'secret' && payload.severity === 'error' && payload.text) { this._fallbackErrorCount++; if (this._fallbackErrorMessages.length < this._maxDetailedDiagnostics) { this._fallbackErrorMessages.push(payload.text.trim()); @@ -348,35 +358,45 @@ export class AiReporter implements IReporter { } } - private _collectDiagnostic(diagnostic: IAiDiagnostic, state: IAiDiagnosticState): void { + private _collectDiagnostic(event: IReporterEventEnvelope, state: IAiDiagnosticState): void { + if (event.privacy === 'secret') { + return; + } + const diagnostic: IAiDiagnostic & { + readonly causeDiagnosticIds?: readonly string[]; + readonly parameters?: Readonly>; + } = event.payload as IAiDiagnostic & { + readonly causeDiagnosticIds?: readonly string[]; + readonly parameters?: Readonly>; + }; if (diagnostic.category !== undefined) { state.diagnosticCategoryCounts[diagnostic.category] = (state.diagnosticCategoryCounts[diagnostic.category] ?? 0) + 1; } + const collected: ICollectedAiDiagnostic = { + diagnosticId: diagnostic.diagnosticId, + code: diagnostic.code, + category: diagnostic.category, + severity: diagnostic.severity, + summary: diagnostic.summary, + summaryKey: diagnostic.summaryKey, + detailKey: diagnostic.detailKey, + context: this._projectDiagnosticContext(diagnostic.parameters), + remediation: diagnostic.remediation, + causeDiagnosticIds: diagnostic.causeDiagnosticIds + }; if (diagnostic.severity === 'error') { state.errorCount++; state.errorCodes.add(diagnostic.code); if (state.errorDiagnostics.length < this._maxDetailedDiagnostics) { - state.errorDiagnostics.push({ - code: diagnostic.code, - category: diagnostic.category, - severity: 'error', - summary: diagnostic.summary, - remediation: diagnostic.remediation - }); + state.errorDiagnostics.push(collected); } else { state.errorDiagnosticsTruncated = true; } } else if (diagnostic.severity === 'warning') { state.warningCount++; if (state.warningDiagnostics.length < this._maxDetailedDiagnostics) { - state.warningDiagnostics.push({ - code: diagnostic.code, - category: diagnostic.category, - severity: 'warning', - summary: diagnostic.summary, - remediation: diagnostic.remediation - }); + state.warningDiagnostics.push(collected); } else { state.warningDiagnosticsTruncated = true; } @@ -415,6 +435,63 @@ export class AiReporter implements IReporter { } } + private _projectDiagnosticContext( + parameters: Readonly> | undefined + ): Readonly> | undefined { + if (!parameters) { + return undefined; + } + const context: Record = {}; + for (const name of Object.keys(parameters).sort()) { + const parameter: IClassifiedDiagnosticValue = parameters[name]; + context[name] = parameter.privacy === 'public' ? parameter.value : (`[${parameter.privacy}]` as const); + } + return Object.keys(context).length > 0 ? context : undefined; + } + + private _orderDiagnostics(diagnostics: readonly ICollectedAiDiagnostic[]): IAiDiagnostic[] { + const byId: Map = new Map(); + for (const diagnostic of diagnostics) { + if (diagnostic.diagnosticId) { + byId.set(diagnostic.diagnosticId, diagnostic); + } + } + + const ordered: IAiDiagnostic[] = []; + const visited: Set = new Set(); + const visiting: Set = new Set(); + const visit = (diagnostic: ICollectedAiDiagnostic): void => { + if (visited.has(diagnostic) || visiting.has(diagnostic)) { + return; + } + visiting.add(diagnostic); + for (const causeId of diagnostic.causeDiagnosticIds ?? []) { + const cause: ICollectedAiDiagnostic | undefined = byId.get(causeId); + if (cause) { + visit(cause); + } + } + visiting.delete(diagnostic); + visited.add(diagnostic); + ordered.push({ + diagnosticId: diagnostic.diagnosticId, + code: diagnostic.code, + category: diagnostic.category, + severity: diagnostic.severity, + summary: diagnostic.summary, + summaryKey: diagnostic.summaryKey, + detailKey: diagnostic.detailKey, + context: diagnostic.context, + remediation: diagnostic.remediation + }); + }; + + for (const diagnostic of diagnostics) { + visit(diagnostic); + } + return ordered; + } + private _emitFinal(succeeded: boolean, exitCode: number): void { if (this._finalEmitted) { return; @@ -426,14 +503,14 @@ export class AiReporter implements IReporter { const errorCountWithoutFallback: number = this._globalDiagnostics.errorCount + cycleDiagnostics.errorCount; const warningCount: number = this._globalDiagnostics.warningCount + cycleDiagnostics.warningCount; - const collectedErrorDiagnostics: IAiDiagnostic[] = [ + const collectedErrorDiagnostics: IAiDiagnostic[] = this._orderDiagnostics([ ...this._globalDiagnostics.errorDiagnostics, ...cycleDiagnostics.errorDiagnostics - ]; - const collectedWarningDiagnostics: IAiDiagnostic[] = [ + ]); + const collectedWarningDiagnostics: IAiDiagnostic[] = this._orderDiagnostics([ ...this._globalDiagnostics.warningDiagnostics, ...cycleDiagnostics.warningDiagnostics - ]; + ]); const fallbackDiagnostics: IAiDiagnostic[] = errorCountWithoutFallback === 0 ? this._fallbackErrorMessages.map((summary) => ({ diff --git a/libraries/reporter/src/reporters/JsonReporter.ts b/libraries/reporter/src/reporters/JsonReporter.ts index 55d08ffdfb..0e79bbdaa6 100644 --- a/libraries/reporter/src/reporters/JsonReporter.ts +++ b/libraries/reporter/src/reporters/JsonReporter.ts @@ -50,16 +50,15 @@ export class JsonReporter implements IReporter { } public report(event: IReporterEventEnvelope): void { + const redactedEvent: IReporterEventEnvelope = redactReporterEvent(event); try { - this._write( - encodeNdjsonRecord(redactReporterEvent(event), { maxRecordBytes: this._maxRecordBytes }) - ); + this._write(encodeNdjsonRecord(redactedEvent, { maxRecordBytes: this._maxRecordBytes })); } catch (error) { if (error instanceof NdjsonRecordTooLargeError) { this._write( encodeNdjsonRecord( { - ...event, + ...redactedEvent, privacy: 'public', type: 'extension', payload: { diff --git a/libraries/reporter/src/reporters/ReporterRedaction.ts b/libraries/reporter/src/reporters/ReporterRedaction.ts index 5532c8006b..2494f14f3e 100644 --- a/libraries/reporter/src/reporters/ReporterRedaction.ts +++ b/libraries/reporter/src/reporters/ReporterRedaction.ts @@ -8,25 +8,41 @@ interface IClassifiedValue { readonly privacy: string; } -export function redactReporterEvent( - event: IReporterEventEnvelope -): IReporterEventEnvelope { +export function redactReporterEvent(event: IReporterEventEnvelope): IReporterEventEnvelope { let payload: unknown = event.payload; + const source: IReporterEventEnvelope['source'] = + event.privacy === 'public' + ? event.source + : { + packageName: '[private-producer]', + packageVersion: '[private-version]' + }; if (event.privacy === 'secret') { payload = '[secret]'; } else if (event.type === 'diagnosticEmitted') { - const diagnostic: { readonly parameters?: Readonly> } = - event.payload as { - readonly parameters?: Readonly>; - }; + const diagnostic: { + readonly parameters?: Readonly>; + readonly source?: unknown; + } = event.payload as { + readonly parameters?: Readonly>; + readonly source?: unknown; + }; + const redactedDiagnostic: { + parameters?: Record; + source?: unknown; + } = { ...diagnostic }; if (diagnostic.parameters) { const parameters: Record = {}; for (const [name, classified] of Object.entries(diagnostic.parameters)) { parameters[name] = classified.privacy === 'secret' ? { value: '[secret]', privacy: 'secret' } : classified; } - payload = { ...diagnostic, parameters }; + redactedDiagnostic.parameters = parameters; } + if (event.privacy !== 'public') { + delete redactedDiagnostic.source; + } + payload = redactedDiagnostic; } - return { ...event, payload }; + return { ...event, source, payload }; } diff --git a/libraries/reporter/src/test/AiReporterQualification.test.ts b/libraries/reporter/src/test/AiReporterQualification.test.ts new file mode 100644 index 0000000000..89ac5e01ad --- /dev/null +++ b/libraries/reporter/src/test/AiReporterQualification.test.ts @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { + AI_REPORTER_QUALIFICATION_THRESHOLDS, + evaluateAiReporterQualification, + formatAiReporterQualificationFailures, + getQualifiedAiReporterDecision, + runAiReporterQualificationCorpusAsync, + type IAiReporterQualificationCaseResult, + type IAiReporterQualificationResult +} from '../index'; +import { normalizeAiReporterQualificationOutput } from '../qualification/AiReporterQualificationCorpus'; + +describe('AI reporter deterministic qualification corpus', () => { + let qualification: IAiReporterQualificationResult; + + beforeAll(async () => { + qualification = await runAiReporterQualificationCorpusAsync(); + }); + + it('passes every blocking gate with machine-readable safe results', () => { + if (!qualification.passed) { + throw new Error(formatAiReporterQualificationFailures(qualification)); + } + expect(qualification.schemaVersion).toBe('1.0'); + expect(qualification.cases).toHaveLength(12); + expect(qualification.cases.filter(({ expectedResult }) => expectedResult === 'failed')).toHaveLength(10); + expect(qualification.cases.every(({ failures }) => failures.length === 0)).toBe(true); + const serialized: string = JSON.stringify(qualification); + expect(serialized).not.toContain('rush-ai-reporter-qualification-'); + expect(serialized).not.toContain('qualification-fake-secret-token'); + expect(serialized).not.toContain('@private/example-rush-plugin'); + }); + + it('enforces the documented size and repeat thresholds', () => { + expect(AI_REPORTER_QUALIFICATION_THRESHOLDS).toMatchObject({ + minimumActionableFailurePercent: 100, + maximumOutputBytesPerCase: 64 * 1024, + maximumAggregateAiToLegacyPercent: 50, + maximumAggregateAiToPlaintextPercent: 50, + deterministicRunCount: 3, + minimumPrivacyPassPercent: 100, + minimumFullLogPassPercent: 100, + minimumStdoutContractPassPercent: 100, + minimumWarningContractPassPercent: 100 + }); + }); + + it('normalizes Windows and POSIX paths without storing machine-specific separators', () => { + expect( + normalizeAiReporterQualificationOutput( + '{"path":"C:\\\\repo\\\\temp\\\\rush.log"}', + 'C:\\repo\\temp\\rush.log', + 'C:\\repo\\temp' + ) + ).toBe('{"path":""}'); + expect( + normalizeAiReporterQualificationOutput( + '{"path":"/repo/temp/rush.log","root":"/repo/temp"}', + '/repo/temp/rush.log', + '/repo/temp' + ) + ).toBe('{"path":"","root":""}'); + }); + + it('reports actionable per-case failures when a blocking gate regresses', () => { + const cases: IAiReporterQualificationCaseResult[] = qualification.cases.map( + (testCase: IAiReporterQualificationCaseResult, index: number) => + index === 0 ? { ...testCase, actionable: false } : testCase + ); + const failed: IAiReporterQualificationResult = evaluateAiReporterQualification(cases); + + expect(failed.passed).toBe(false); + expect(formatAiReporterQualificationFailures(failed)).toContain( + 'actionability: actual=90.00, required=>= 100%; cases=bootstrap-unsupported-node' + ); + }); +}); + +describe('qualified AI reporter decision', () => { + function passedQualification(): IAiReporterQualificationResult { + const emptyCases: readonly IAiReporterQualificationCaseResult[] = []; + return { + schemaVersion: '1.0', + passed: true, + thresholds: AI_REPORTER_QUALIFICATION_THRESHOLDS, + cases: emptyCases, + gates: [] + }; + } + + it('recognizes built-in COPILOT_CLI and configured agent variables', () => { + expect(getQualifiedAiReporterDecision({ COPILOT_CLI: '1' }, [], passedQualification())).toMatchObject({ + agentDetected: true, + eligible: true, + reporter: 'ai' + }); + expect( + getQualifiedAiReporterDecision({ MY_AGENT: 'yes' }, ['MY_AGENT'], passedQualification()) + ).toMatchObject({ + agentDetected: true, + eligible: true, + reporter: 'ai' + }); + }); + + it('blocks selection when qualification is absent or failed', () => { + expect(getQualifiedAiReporterDecision({ COPILOT_CLI: '1' }, [], undefined)).toMatchObject({ + eligible: false, + reason: 'qualification unavailable' + }); + expect( + getQualifiedAiReporterDecision({ COPILOT_CLI: '1' }, [], { ...passedQualification(), passed: false }) + ).toMatchObject({ + eligible: false, + reason: 'qualification failed' + }); + }); + + it('keeps RUSH_REPORTER=legacy authoritative even after qualification passes', () => { + expect( + getQualifiedAiReporterDecision({ COPILOT_CLI: '1', RUSH_REPORTER: 'legacy' }, [], passedQualification()) + ).toMatchObject({ + agentDetected: true, + eligible: false, + reason: 'RUSH_REPORTER=legacy' + }); + }); +}); diff --git a/libraries/reporter/src/test/JsonAiReporter.test.ts b/libraries/reporter/src/test/JsonAiReporter.test.ts index 516cbd9260..df730fa90b 100644 --- a/libraries/reporter/src/test/JsonAiReporter.test.ts +++ b/libraries/reporter/src/test/JsonAiReporter.test.ts @@ -73,6 +73,35 @@ describe('JsonReporter', () => { expect(Buffer.byteLength(output.trim(), 'utf8')).toBeLessThanOrEqual(512); }); + it('does not expose a private producer identity in an oversized redacted record', () => { + let output: string = ''; + const reporter: JsonReporter = new JsonReporter({ + write: (text: string) => (output += text), + maxRecordBytes: 512 + }); + reporter.report({ + ...ev('extension', { name: 'private.fixture', payload: 'x'.repeat(1000) }), + source: { + packageName: '@private/example-rush-plugin', + packageVersion: '1.0.0', + component: 'PrivatePluginImplementation' + }, + privacy: 'local-sensitive' + }); + + expect(output).not.toContain('@private/example-rush-plugin'); + expect(output).not.toContain('PrivatePluginImplementation'); + expect(parseLines(output)[0]).toMatchObject({ + source: { + packageName: '[private-producer]', + packageVersion: '[private-version]' + }, + payload: { + name: 'rush.reporter.record-too-large' + } + }); + }); + it('redacts secret diagnostic fields from stdout', () => { let output: string = ''; const reporter: JsonReporter = new JsonReporter({ write: (text: string) => (output += text) }); @@ -389,6 +418,53 @@ describe('AiReporter', () => { expect(final.truncated).toBe(false); }); + it('orders root-cause diagnostics before diagnostics that reference them', () => { + const { final } = run([ + ev('diagnosticEmitted', { + diagnosticId: 'outer', + code: 'RUSH_OPERATION_FAILED', + category: 'operation', + severity: 'error', + causeDiagnosticIds: ['root'] + }), + ev('diagnosticEmitted', { + diagnosticId: 'root', + code: 'RUSH_DEPENDENCY_TOOL_FAILED', + category: 'dependency-tool', + severity: 'error' + }), + ev('commandResult', { commandName: 'build', succeeded: false, exitCode: 1 }) + ]); + + expect(final.diagnostics.map(({ diagnosticId }) => diagnosticId)).toEqual(['root', 'outer']); + }); + + it('projects classified context without exposing secret values', () => { + const { final } = run([ + ev('diagnosticEmitted', { + diagnosticId: 'auth', + code: 'RUSH_NETWORK_AUTH_UNAUTHORIZED', + category: 'network-auth', + severity: 'error', + summaryKey: 'diagnostic.RUSH_NETWORK_AUTH_UNAUTHORIZED.summary', + parameters: { + registryUrl: { value: 'https://registry.example.test/', privacy: 'public' }, + token: { value: 'qualification-fake-secret-token', privacy: 'secret' } + } + }), + ev('commandResult', { commandName: 'install', succeeded: false, exitCode: 1 }) + ]); + + expect(final.diagnostics[0]).toMatchObject({ + summaryKey: 'diagnostic.RUSH_NETWORK_AUTH_UNAUTHORIZED.summary', + context: { + registryUrl: 'https://registry.example.test/', + token: '[secret]' + } + }); + expect(JSON.stringify(final)).not.toContain('qualification-fake-secret-token'); + }); + it('excludes raw external output and keeps stdout pure JSON', () => { let output: string = ''; const reporter: AiReporter = new AiReporter({ write: (text: string) => (output += text) }); From 45363498980445f0a53a811565a900e85106156b Mon Sep 17 00:00:00 2001 From: TheLarkInn Date: Fri, 28 Aug 2026 10:12:38 +0000 Subject: [PATCH 2/4] Harden AI reporter qualification gates Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d6318e80-5da9-4858-a147-817e8692f10e --- ...eporter-r8a-ai-gates_2026-08-28-08-45.json | 2 +- common/reviews/api/rush-reporter.api.md | 10 +- libraries/reporter/README.md | 8 +- .../qualification/AiReporterQualification.ts | 85 ++++++++- .../AiReporterQualificationCorpus.ts | 166 +++++++++++++++--- .../reporter/src/reporters/AiReporter.ts | 32 +++- .../src/reporters/ReporterRedaction.ts | 11 +- .../src/test/AiReporterQualification.test.ts | 65 ++++++- .../reporter/src/test/FileReporter.test.ts | 31 ++++ .../reporter/src/test/JsonAiReporter.test.ts | 72 ++++++-- 10 files changed, 414 insertions(+), 68 deletions(-) diff --git a/common/changes/@rushstack/rush-reporter/copilot-reporter-r8a-ai-gates_2026-08-28-08-45.json b/common/changes/@rushstack/rush-reporter/copilot-reporter-r8a-ai-gates_2026-08-28-08-45.json index 9267c15ce6..aba0aed31d 100644 --- a/common/changes/@rushstack/rush-reporter/copilot-reporter-r8a-ai-gates_2026-08-28-08-45.json +++ b/common/changes/@rushstack/rush-reporter/copilot-reporter-r8a-ai-gates_2026-08-28-08-45.json @@ -2,7 +2,7 @@ "changes": [ { "packageName": "@rushstack/rush-reporter", - "comment": "Add deterministic AI reporter qualification gates and privacy-safe actionable context.", + "comment": "Add adversarial deterministic AI reporter qualification gates and privacy-safe actionable context.", "type": "minor" } ], diff --git a/common/reviews/api/rush-reporter.api.md b/common/reviews/api/rush-reporter.api.md index 7834936c5a..da53dc102b 100644 --- a/common/reviews/api/rush-reporter.api.md +++ b/common/reviews/api/rush-reporter.api.md @@ -195,7 +195,7 @@ export function getLogLevelRank(level: ReporterLogLevel): number; export function getPrivacyClassificationRank(classification: ReporterPrivacyClassification): number; // @beta -export function getQualifiedAiReporterDecision(env: Record, configuredAgentEnvironmentVariables: readonly string[], qualification: IAiReporterQualificationResult | undefined): IQualifiedAiReporterDecision; +export function getQualifiedAiReporterDecision(env: Record, configuredAgentEnvironmentVariables: readonly string[], qualification: IAiReporterQualificationResult | undefined, privacyPrerequisiteAccepted?: boolean): IQualifiedAiReporterDecision; // @beta export function getReporterMigrationPhase(id: ReporterMigrationPhaseId): IReporterMigrationPhase; @@ -372,10 +372,16 @@ export interface IAiReporterQualificationThresholds { // (undocumented) readonly maximumAggregateAiToPlaintextPercent: number; // (undocumented) + readonly maximumCompactCaseAiOutputBytes: number; + // (undocumented) readonly maximumOutputBytesPerCase: number; // (undocumented) + readonly maximumPerCaseAiToBaselinePercent: number; + // (undocumented) readonly minimumActionableFailurePercent: number; // (undocumented) + readonly minimumComparableBaselineBytes: number; + // (undocumented) readonly minimumControlCases: number; // (undocumented) readonly minimumFailureCases: number; @@ -818,7 +824,7 @@ export interface IQualifiedAiReporterDecision { // (undocumented) readonly eligible: boolean; // (undocumented) - readonly reason: 'RUSH_REPORTER=legacy' | 'agent not detected' | 'qualification unavailable' | 'qualification failed' | 'qualified'; + readonly reason: 'RUSH_REPORTER=legacy' | 'agent not detected' | 'qualification unavailable' | 'qualification failed' | 'privacy prerequisite unavailable' | 'qualified'; // (undocumented) readonly reporter?: 'ai'; } diff --git a/libraries/reporter/README.md b/libraries/reporter/README.md index 90b622a8ee..11f194bc3e 100644 --- a/libraries/reporter/README.md +++ b/libraries/reporter/README.md @@ -9,12 +9,13 @@ This package is released as a public beta. Exported contracts may change before The network-free qualification corpus runs representative bootstrap/version, configuration, input, dependency-tool, operation, cache, network/auth, plugin, cancellation, and internal failures plus successful and warning-only controls through the AI, detailed plaintext, legacy, and full-log reporters. +Scenario-specific external output is included only where the real failure or control would produce it. | Gate | Blocking threshold | | --- | --- | | Failure/control coverage | At least 10 failure cases and 2 successful controls | | Actionability | 100% of failures retain stable code, category, context, and remediation | -| Output size | At most 64 KiB per case; aggregate AI bytes at most 50% of legacy and plaintext | +| Output size | At most 64 KiB per case; compact cases at most 2 KiB; AI no larger than comparable per-case baselines; aggregate AI bytes at most 50% of legacy and plaintext | | Determinism | Byte-identical normalized AI output across 3 runs | | Privacy | 100% secret redaction and no private producer identity leakage | | Full log | 100% absolute, existing, owner-only where supported, complete, and failure-correlated | @@ -23,8 +24,9 @@ successful and warning-only controls through the AI, detailed plaintext, legacy, Run `rushx build && node scripts/runAiReporterQualification.js` from this project to print the machine-readable result. Machine-specific paths are normalized before hashing and are not stored. Passing these gates only produces a reusable qualification decision; it does not enable environment-based automatic -reporter selection. The pre-major Rush frontend remains explicit/repository-opt-in, and -`RUSH_REPORTER=legacy` remains authoritative. +reporter selection. That decision also requires the separate telemetry privacy prerequisite to be accepted. +The pre-major Rush frontend remains explicit/repository-opt-in, and `RUSH_REPORTER=legacy` remains +authoritative. ## Links diff --git a/libraries/reporter/src/qualification/AiReporterQualification.ts b/libraries/reporter/src/qualification/AiReporterQualification.ts index e38f50e3be..6d659e6a78 100644 --- a/libraries/reporter/src/qualification/AiReporterQualification.ts +++ b/libraries/reporter/src/qualification/AiReporterQualification.ts @@ -22,6 +22,9 @@ export interface IAiReporterQualificationThresholds { readonly minimumControlCases: number; readonly minimumActionableFailurePercent: number; readonly maximumOutputBytesPerCase: number; + readonly maximumCompactCaseAiOutputBytes: number; + readonly minimumComparableBaselineBytes: number; + readonly maximumPerCaseAiToBaselinePercent: number; readonly maximumAggregateAiToLegacyPercent: number; readonly maximumAggregateAiToPlaintextPercent: number; readonly deterministicRunCount: number; @@ -41,6 +44,9 @@ export const AI_REPORTER_QUALIFICATION_THRESHOLDS: IAiReporterQualificationThres minimumControlCases: 2, minimumActionableFailurePercent: 100, maximumOutputBytesPerCase: REPORTER_PERFORMANCE_BUDGETS.maxAiOutputBytes, + maximumCompactCaseAiOutputBytes: 2 * 1024, + minimumComparableBaselineBytes: 1024, + maximumPerCaseAiToBaselinePercent: 100, maximumAggregateAiToLegacyPercent: 50, maximumAggregateAiToPlaintextPercent: 50, deterministicRunCount: 3, @@ -103,7 +109,11 @@ function percent(passing: number, total: number): number { } function ratioPercent(numerator: number, denominator: number): number { - return denominator === 0 ? Number.POSITIVE_INFINITY : (numerator / denominator) * 100; + return denominator === 0 + ? numerator === 0 + ? 0 + : Number.MAX_SAFE_INTEGER + : (numerator / denominator) * 100; } function getHighestRatioCaseNames( @@ -120,6 +130,31 @@ function getHighestRatioCaseNames( .map(({ name }) => name); } +function createPerCaseRatioGate( + id: string, + cases: readonly IAiReporterQualificationCaseResult[], + getDenominator: (testCase: IAiReporterQualificationCaseResult) => number, + minimumComparableBaselineBytes: number, + maximumPercent: number +): IAiReporterQualificationGateResult { + const comparableCases: readonly IAiReporterQualificationCaseResult[] = cases.filter( + (testCase) => getDenominator(testCase) >= minimumComparableBaselineBytes + ); + const failedCases: string[] = comparableCases + .filter((testCase) => ratioPercent(testCase.aiOutputBytes, getDenominator(testCase)) > maximumPercent) + .map(({ name }) => name); + return { + id, + passed: failedCases.length === 0, + actual: Math.max( + 0, + ...comparableCases.map((testCase) => ratioPercent(testCase.aiOutputBytes, getDenominator(testCase))) + ), + threshold: `<= ${maximumPercent}% when baseline >= ${minimumComparableBaselineBytes} bytes`, + failedCases + }; +} + function createPercentageGate( id: string, cases: readonly IAiReporterQualificationCaseResult[], @@ -187,6 +222,47 @@ export function evaluateAiReporterQualification( .filter(({ aiOutputBytes }) => aiOutputBytes > thresholds.maximumOutputBytesPerCase) .map(({ name }) => name) }, + { + id: 'size.compact-case', + passed: cases.every( + ({ aiOutputBytes, legacyOutputBytes, plaintextOutputBytes }) => + Math.max(legacyOutputBytes, plaintextOutputBytes) >= thresholds.minimumComparableBaselineBytes || + aiOutputBytes <= thresholds.maximumCompactCaseAiOutputBytes + ), + actual: Math.max( + 0, + ...cases + .filter( + ({ legacyOutputBytes, plaintextOutputBytes }) => + Math.max(legacyOutputBytes, plaintextOutputBytes) < thresholds.minimumComparableBaselineBytes + ) + .map(({ aiOutputBytes }) => aiOutputBytes) + ), + threshold: + `<= ${thresholds.maximumCompactCaseAiOutputBytes} bytes when both baselines are below ` + + `${thresholds.minimumComparableBaselineBytes} bytes`, + failedCases: cases + .filter( + ({ aiOutputBytes, legacyOutputBytes, plaintextOutputBytes }) => + Math.max(legacyOutputBytes, plaintextOutputBytes) < thresholds.minimumComparableBaselineBytes && + aiOutputBytes > thresholds.maximumCompactCaseAiOutputBytes + ) + .map(({ name }) => name) + }, + createPerCaseRatioGate( + 'size.per-case-vs-legacy', + cases, + ({ legacyOutputBytes }) => legacyOutputBytes, + thresholds.minimumComparableBaselineBytes, + thresholds.maximumPerCaseAiToBaselinePercent + ), + createPerCaseRatioGate( + 'size.per-case-vs-plaintext', + cases, + ({ plaintextOutputBytes }) => plaintextOutputBytes, + thresholds.minimumComparableBaselineBytes, + thresholds.maximumPerCaseAiToBaselinePercent + ), { id: 'size.vs-legacy', passed: aggregateAiToLegacyPercent <= thresholds.maximumAggregateAiToLegacyPercent, @@ -274,6 +350,7 @@ export interface IQualifiedAiReporterDecision { | 'agent not detected' | 'qualification unavailable' | 'qualification failed' + | 'privacy prerequisite unavailable' | 'qualified'; } @@ -290,7 +367,8 @@ export interface IQualifiedAiReporterDecision { export function getQualifiedAiReporterDecision( env: Record, configuredAgentEnvironmentVariables: readonly string[], - qualification: IAiReporterQualificationResult | undefined + qualification: IAiReporterQualificationResult | undefined, + privacyPrerequisiteAccepted: boolean = false ): IQualifiedAiReporterDecision { const agentDetected: boolean = detectAgent(env, configuredAgentEnvironmentVariables); if (isLegacyEmergencyFallbackRequested(env)) { @@ -305,5 +383,8 @@ export function getQualifiedAiReporterDecision( if (!qualification.passed) { return { agentDetected: true, eligible: false, reason: 'qualification failed' }; } + if (!privacyPrerequisiteAccepted) { + return { agentDetected: true, eligible: false, reason: 'privacy prerequisite unavailable' }; + } return { agentDetected: true, eligible: true, reporter: 'ai', reason: 'qualified' }; } diff --git a/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts b/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts index a23978d10f..3982add9d0 100644 --- a/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts +++ b/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts @@ -23,11 +23,20 @@ import { const FIXED_TIMESTAMP: string = '2026-08-28T08:45:34.000Z'; const FIXED_TIME_MS: number = Date.parse(FIXED_TIMESTAMP); const FIXED_PID: number = 4242; -const RAW_EVIDENCE: string = `deterministic external evidence ${'x'.repeat(4096)}\n`; const CLASSIFIED_SECRET: string = 'qualification-fake-secret-token'; +const CLASSIFIED_SECRET_PRODUCER: string = '@secret/qualification-fixture'; +const CLASSIFIED_SECRET_COMPONENT: string = 'SecretQualificationFixture'; const PRIVATE_PRODUCER: string = '@private/example-rush-plugin'; const PRIVATE_COMPONENT: string = 'PrivatePluginImplementation'; +function createExternalOutput(lines: readonly string[], repetitions: number): string { + const output: string[] = []; + for (let index: number = 0; index < repetitions; index++) { + output.push(lines[index % lines.length].split('{index}').join(String(index + 1))); + } + return `${output.join('\n')}\n`; +} + interface ICorpusDiagnostic { readonly code: string; readonly category: string; @@ -54,6 +63,7 @@ interface ICorpusCase { readonly scenario: string; readonly expectedResult: 'succeeded' | 'failed'; readonly diagnostic?: ICorpusDiagnostic; + readonly externalOutput?: string; readonly operationStatus?: 'success' | 'failure' | 'aborted' | 'fromCache'; readonly warningOnly?: boolean; } @@ -131,6 +141,14 @@ const CORPUS: readonly ICorpusCase[] = [ name: 'dependency-package-manager', scenario: 'pnpm install exits unsuccessfully', expectedResult: 'failed', + externalOutput: createExternalOutput( + [ + 'ERR_PNPM_FETCH_401 GET https://registry.example.test/@example/pkg: Unauthorized - 401', + 'Progress: resolved {index}, reused 0, downloaded 0, added 0', + 'The authorization header was rejected while resolving @example/pkg.' + ], + 60 + ), operationStatus: 'failure', diagnostic: { code: 'RUSH_DEPENDENCY_TOOL_FAILED', @@ -154,6 +172,14 @@ const CORPUS: readonly ICorpusCase[] = [ name: 'operation-build-failure', scenario: 'a project build operation fails', expectedResult: 'failed', + externalOutput: createExternalOutput( + [ + 'src/example-{index}.ts(12,7): error TS2322: Type string is not assignable to type number.', + 'Found 1 error in src/example-{index}.ts', + 'Project @example/app failed during the _phase:build operation.' + ], + 66 + ), operationStatus: 'failure', diagnostic: { code: 'RUSH_OPERATION_FAILED', @@ -175,6 +201,14 @@ const CORPUS: readonly ICorpusCase[] = [ name: 'cache-restore-failure', scenario: 'a build-cache restore is invalid and requires a local rebuild', expectedResult: 'failed', + externalOutput: createExternalOutput( + [ + 'Build cache entry cache-entry-42 failed integrity validation for @example/cache-consumer.', + 'Expected archive member lib/index.js but the restored file was missing.', + 'Discarding invalid cache entry and requiring a local rebuild ({index}).' + ], + 30 + ), operationStatus: 'failure', diagnostic: { code: 'RUSH_OPERATION_FAILED', @@ -197,6 +231,14 @@ const CORPUS: readonly ICorpusCase[] = [ name: 'network-auth-unauthorized', scenario: 'the registry returns an authentication-shaped failure', expectedResult: 'failed', + externalOutput: createExternalOutput( + [ + 'GET https://registry.example.test/@example/private returned 401 Unauthorized.', + 'The registry challenge did not include credentials; refresh the configured authentication.', + 'Request attempt {index} failed without exposing an authorization value.' + ], + 24 + ), diagnostic: { code: 'RUSH_NETWORK_AUTH_UNAUTHORIZED', category: 'network-auth', @@ -218,6 +260,14 @@ const CORPUS: readonly ICorpusCase[] = [ name: 'plugin-api-incompatible', scenario: 'a private plugin is incompatible with the current Rush API', expectedResult: 'failed', + externalOutput: createExternalOutput( + [ + 'Loading the configured Rush plugin from the repository plugin manifest.', + 'Validating plugin API compatibility before invoking plugin hooks ({index}).', + 'Plugin activation stopped because the declared Rush version range is incompatible.' + ], + 15 + ), diagnostic: { code: 'RUSH_PLUGIN_API_INCOMPATIBLE', category: 'configuration', @@ -243,6 +293,14 @@ const CORPUS: readonly ICorpusCase[] = [ name: 'logical-cancellation', scenario: 'a command is cancelled and reports an aborted operation', expectedResult: 'failed', + externalOutput: createExternalOutput( + [ + 'Building @example/app: completed work item {index}.', + 'Cancellation requested; waiting for the active child process to stop.', + 'The _phase:build operation exited before producing final outputs.' + ], + 24 + ), operationStatus: 'aborted', diagnostic: { code: 'RUSH_COMMAND_FAILED', @@ -265,6 +323,14 @@ const CORPUS: readonly ICorpusCase[] = [ name: 'internal-unexpected-error', scenario: 'Rush reports an unexpected internal failure', expectedResult: 'failed', + externalOutput: createExternalOutput( + [ + 'Unexpected internal failure while finalizing the command graph.', + 'Diagnostic incident incident-42 was recorded for correlation.', + 'See the owner-only full-detail log for stack frame {index}.' + ], + 30 + ), diagnostic: { code: 'RUSH_INTERNAL_UNEXPECTED', category: 'internal', @@ -286,12 +352,28 @@ const CORPUS: readonly ICorpusCase[] = [ name: 'success-no-warning', scenario: 'a successful operation emits no warnings', expectedResult: 'succeeded', + externalOutput: createExternalOutput( + [ + 'Building @example/app source file {index}.', + 'Emitted lib/example-{index}.js and lib/example-{index}.d.ts.', + 'Completed incremental compilation work item {index}.' + ], + 42 + ), operationStatus: 'success' }, { name: 'success-warning-only', scenario: 'a successful command emits one bounded warning', expectedResult: 'succeeded', + externalOutput: createExternalOutput( + [ + 'Restored @example/app output group {index} from the local build cache.', + 'Validated cached output metadata for work item {index}.', + 'The deprecated option warning is represented by a structured diagnostic.' + ], + 24 + ), operationStatus: 'fromCache', warningOnly: true, diagnostic: { @@ -396,21 +478,23 @@ function createEvents(testCase: ICorpusCase, logPath: string): IReporterEventEnv } ); } - add( - 'externalOutput', - { stream: 'stderr', text: RAW_EVIDENCE }, - testCase.operationStatus - ? { - privacy: 'local-sensitive', - scope: { - commandName: 'build', - operationId: 'fixture#build', - projectName: '@example/app', - phaseName: '_phase:build' + if (testCase.externalOutput !== undefined) { + add( + 'externalOutput', + { stream: 'stderr', text: testCase.externalOutput }, + testCase.operationStatus + ? { + privacy: 'local-sensitive', + scope: { + commandName: 'build', + operationId: 'fixture#build', + projectName: '@example/app', + phaseName: '_phase:build' + } } - } - : { privacy: 'local-sensitive', scope: { commandName: 'build' } } - ); + : { privacy: 'local-sensitive', scope: { commandName: 'build' } } + ); + } if (testCase.operationStatus) { add( 'operationStatusChanged', @@ -471,8 +555,8 @@ function createEvents(testCase: ICorpusCase, logPath: string): IReporterEventEnv { name: 'private.fixture.secret', payload: { token: CLASSIFIED_SECRET } }, { privacy: 'secret', - sourcePackage: PRIVATE_PRODUCER, - sourceComponent: PRIVATE_COMPONENT, + sourcePackage: CLASSIFIED_SECRET_PRODUCER, + sourceComponent: CLASSIFIED_SECRET_COMPONENT, scope: { commandName: 'build' } } ); @@ -560,6 +644,13 @@ async function runCaseAsync( : undefined; const expectedContextKeys: readonly string[] = diagnostic ? Object.keys(diagnostic.parameters).sort() : []; const actualContextKeys: readonly string[] = Object.keys(matchingDiagnostic?.context ?? {}).sort(); + const contextValuesMatch: boolean = diagnostic + ? Object.entries(diagnostic.parameters).every(([name, parameter]) => { + const expectedValue: string | number | boolean = + parameter.privacy === 'public' ? parameter.value : `[${parameter.privacy}]`; + return matchingDiagnostic?.context?.[name] === expectedValue; + }) + : true; const actionable: boolean = testCase.expectedResult === 'succeeded' ? true @@ -570,17 +661,20 @@ async function runCaseAsync( matchingDiagnostic?.category === diagnostic.category && matchingDiagnostic.summaryKey === diagnostic.summaryKey && expectedContextKeys.every((key) => actualContextKeys.includes(key)) && + contextValuesMatch && matchingDiagnostic.remediation?.some(({ command, documentationUrl }) => Boolean(command || documentationUrl) ) ); const artifact: IFileReporterArtifact = fileReporter.getArtifact(); - const logExists: boolean = artifact.path !== undefined && fs.existsSync(artifact.path); - const logContent: string = logExists ? await fs.promises.readFile(artifact.path!, 'utf8') : ''; + const aiLogPath: string | undefined = final?.log?.path; + const logExists: boolean = aiLogPath !== undefined && fs.existsSync(aiLogPath); + const logContent: string = + logExists && aiLogPath !== undefined ? await fs.promises.readFile(aiLogPath, 'utf8') : ''; const ownerOnly: boolean = process.platform === 'win32' || - (logExists && (await fs.promises.stat(artifact.path!)).mode % 0o1000 === 0o600); + (logExists && aiLogPath !== undefined && (await fs.promises.stat(aiLogPath)).mode % 0o1000 === 0o600); const failureCorrelated: boolean = testCase.expectedResult === 'succeeded' || Boolean( @@ -589,22 +683,36 @@ async function runCaseAsync( logContent.includes(`${testCase.name}-diagnostic`) && logContent.includes(`${testCase.name}-session`) ); + const localSensitiveProducerPreserved: boolean = + diagnostic?.sourcePackage === undefined || + (logContent.includes(diagnostic.sourcePackage) && + (diagnostic.sourceComponent === undefined || logContent.includes(diagnostic.sourceComponent))); const fullLogValid: boolean = Boolean( - artifact.available && + final?.log && + final.log.path === artifact.path && + final.log.format === 'plaintext' && + final.log.complete === artifact.complete && + artifact.available && artifact.complete && - artifact.path && - path.isAbsolute(artifact.path) && + aiLogPath && + path.isAbsolute(aiLogPath) && logExists && ownerOnly && + logContent.includes('"type":"commandStarted"') && logContent.includes('"type":"commandResult"') && - logContent.includes(RAW_EVIDENCE.trim()) && - failureCorrelated + logContent.includes('"type":"sessionCompleted"') && + (testCase.externalOutput === undefined || logContent.includes(testCase.externalOutput.trim())) && + failureCorrelated && + localSensitiveProducerPreserved ); - const combinedPresentedOutput: string = `${aiOutput}\n${plaintextOutput}\n${legacyOutput}\n${logContent}`; + const machinePresentedOutput: string = `${aiOutput}\n${plaintextOutput}\n${legacyOutput}`; + const allLocalOutput: string = `${machinePresentedOutput}\n${logContent}`; const privacySafe: boolean = - !combinedPresentedOutput.includes(CLASSIFIED_SECRET) && - !combinedPresentedOutput.includes(PRIVATE_PRODUCER) && - !combinedPresentedOutput.includes(PRIVATE_COMPONENT); + !allLocalOutput.includes(CLASSIFIED_SECRET) && + !allLocalOutput.includes(CLASSIFIED_SECRET_PRODUCER) && + !allLocalOutput.includes(CLASSIFIED_SECRET_COMPONENT) && + !machinePresentedOutput.includes(PRIVATE_PRODUCER) && + !machinePresentedOutput.includes(PRIVATE_COMPONENT); const warningContractValid: boolean = testCase.expectedResult === 'failed' ? final?.warningCount === 1 && final.diagnostics.every(({ severity }) => severity === 'error') diff --git a/libraries/reporter/src/reporters/AiReporter.ts b/libraries/reporter/src/reporters/AiReporter.ts index 8420626522..becf49623e 100644 --- a/libraries/reporter/src/reporters/AiReporter.ts +++ b/libraries/reporter/src/reporters/AiReporter.ts @@ -27,6 +27,8 @@ interface IAiDiagnosticState { readonly warningDiagnostics: ICollectedAiDiagnostic[]; readonly errorCodes: Set; readonly diagnosticCategoryCounts: { [category: string]: number }; + suppressedSecretErrorCount: number; + suppressedSecretWarningCount: number; errorDiagnosticsTruncated: boolean; warningDiagnosticsTruncated: boolean; errorCount: number; @@ -49,6 +51,8 @@ function createDiagnosticState(): IAiDiagnosticState { warningDiagnostics: [], errorCodes: new Set(), diagnosticCategoryCounts: {}, + suppressedSecretErrorCount: 0, + suppressedSecretWarningCount: 0, errorDiagnosticsTruncated: false, warningDiagnosticsTruncated: false, errorCount: 0, @@ -359,9 +363,6 @@ export class AiReporter implements IReporter { } private _collectDiagnostic(event: IReporterEventEnvelope, state: IAiDiagnosticState): void { - if (event.privacy === 'secret') { - return; - } const diagnostic: IAiDiagnostic & { readonly causeDiagnosticIds?: readonly string[]; readonly parameters?: Readonly>; @@ -369,6 +370,14 @@ export class AiReporter implements IReporter { readonly causeDiagnosticIds?: readonly string[]; readonly parameters?: Readonly>; }; + if (event.privacy === 'secret') { + if (diagnostic.severity === 'error') { + state.suppressedSecretErrorCount++; + } else if (diagnostic.severity === 'warning') { + state.suppressedSecretWarningCount++; + } + return; + } if (diagnostic.category !== undefined) { state.diagnosticCategoryCounts[diagnostic.category] = (state.diagnosticCategoryCounts[diagnostic.category] ?? 0) + 1; @@ -502,7 +511,12 @@ export class AiReporter implements IReporter { const cycleDiagnostics: IAiDiagnosticState = cycle.diagnostics; const errorCountWithoutFallback: number = this._globalDiagnostics.errorCount + cycleDiagnostics.errorCount; - const warningCount: number = this._globalDiagnostics.warningCount + cycleDiagnostics.warningCount; + const suppressedSecretErrorCount: number = + this._globalDiagnostics.suppressedSecretErrorCount + cycleDiagnostics.suppressedSecretErrorCount; + const suppressedSecretWarningCount: number = + this._globalDiagnostics.suppressedSecretWarningCount + cycleDiagnostics.suppressedSecretWarningCount; + const warningCount: number = + this._globalDiagnostics.warningCount + cycleDiagnostics.warningCount + suppressedSecretWarningCount; const collectedErrorDiagnostics: IAiDiagnostic[] = this._orderDiagnostics([ ...this._globalDiagnostics.errorDiagnostics, ...cycleDiagnostics.errorDiagnostics @@ -524,7 +538,10 @@ export class AiReporter implements IReporter { const errorDiagnostics: IAiDiagnostic[] = hasFallbackErrors ? fallbackDiagnostics : collectedErrorDiagnostics; - const errorCount: number = errorCountWithoutFallback + (hasFallbackErrors ? this._fallbackErrorCount : 0); + const errorCount: number = + errorCountWithoutFallback + + suppressedSecretErrorCount + + (hasFallbackErrors ? this._fallbackErrorCount : 0); const errorCodes: string[] = hasFallbackErrors ? ['RUSH_COMMAND_FAILED'] : [...new Set([...this._globalDiagnostics.errorCodes, ...cycleDiagnostics.errorCodes])].sort(); @@ -571,8 +588,11 @@ export class AiReporter implements IReporter { truncated: hasFailures ? this._globalDiagnostics.errorDiagnosticsTruncated || cycleDiagnostics.errorDiagnosticsTruncated || + suppressedSecretErrorCount > 0 || (hasFallbackErrors && this._fallbackErrorsTruncated) - : this._globalDiagnostics.warningDiagnosticsTruncated || cycleDiagnostics.warningDiagnosticsTruncated + : this._globalDiagnostics.warningDiagnosticsTruncated || + cycleDiagnostics.warningDiagnosticsTruncated || + suppressedSecretWarningCount > 0 }; if (this._logPath !== undefined) { diff --git a/libraries/reporter/src/reporters/ReporterRedaction.ts b/libraries/reporter/src/reporters/ReporterRedaction.ts index 2494f14f3e..f576e3f507 100644 --- a/libraries/reporter/src/reporters/ReporterRedaction.ts +++ b/libraries/reporter/src/reporters/ReporterRedaction.ts @@ -11,12 +11,12 @@ interface IClassifiedValue { export function redactReporterEvent(event: IReporterEventEnvelope): IReporterEventEnvelope { let payload: unknown = event.payload; const source: IReporterEventEnvelope['source'] = - event.privacy === 'public' - ? event.source - : { + event.privacy === 'secret' + ? { packageName: '[private-producer]', packageVersion: '[private-version]' - }; + } + : event.source; if (event.privacy === 'secret') { payload = '[secret]'; } else if (event.type === 'diagnosticEmitted') { @@ -39,9 +39,6 @@ export function redactReporterEvent(event: IReporterEventEnvelope): IRe } redactedDiagnostic.parameters = parameters; } - if (event.privacy !== 'public') { - delete redactedDiagnostic.source; - } payload = redactedDiagnostic; } return { ...event, source, payload }; diff --git a/libraries/reporter/src/test/AiReporterQualification.test.ts b/libraries/reporter/src/test/AiReporterQualification.test.ts index 89ac5e01ad..91d1233989 100644 --- a/libraries/reporter/src/test/AiReporterQualification.test.ts +++ b/libraries/reporter/src/test/AiReporterQualification.test.ts @@ -2,11 +2,13 @@ // See LICENSE in the project root for license information. import { + AiReporter, AI_REPORTER_QUALIFICATION_THRESHOLDS, evaluateAiReporterQualification, formatAiReporterQualificationFailures, getQualifiedAiReporterDecision, runAiReporterQualificationCorpusAsync, + type IReporterEventEnvelope, type IAiReporterQualificationCaseResult, type IAiReporterQualificationResult } from '../index'; @@ -37,6 +39,9 @@ describe('AI reporter deterministic qualification corpus', () => { expect(AI_REPORTER_QUALIFICATION_THRESHOLDS).toMatchObject({ minimumActionableFailurePercent: 100, maximumOutputBytesPerCase: 64 * 1024, + maximumCompactCaseAiOutputBytes: 2 * 1024, + minimumComparableBaselineBytes: 1024, + maximumPerCaseAiToBaselinePercent: 100, maximumAggregateAiToLegacyPercent: 50, maximumAggregateAiToPlaintextPercent: 50, deterministicRunCount: 3, @@ -76,6 +81,31 @@ describe('AI reporter deterministic qualification corpus', () => { 'actionability: actual=90.00, required=>= 100%; cases=bootstrap-unsupported-node' ); }); + + it('fails with an actionable case list when the AI reporter omits its log reference', async () => { + const originalReport: typeof AiReporter.prototype.report = AiReporter.prototype.report; + const reportSpy: jest.SpiedFunction = jest + .spyOn(AiReporter.prototype, 'report') + .mockImplementation(function (this: AiReporter, event: IReporterEventEnvelope): void { + if (event.type !== 'artifactAvailable') { + originalReport.call(this, event); + } + }); + try { + const failed: IAiReporterQualificationResult = await runAiReporterQualificationCorpusAsync(); + expect(failed.passed).toBe(false); + expect(formatAiReporterQualificationFailures(failed)).toContain( + 'full-log: actual=0.00, required=>= 100%; cases=' + ); + expect( + failed.cases.every(({ failures }) => + failures.includes('full log path, permissions, completeness, or correlation invalid') + ) + ).toBe(true); + } finally { + reportSpy.mockRestore(); + } + }); }); describe('qualified AI reporter decision', () => { @@ -90,28 +120,44 @@ describe('qualified AI reporter decision', () => { }; } - it('recognizes built-in COPILOT_CLI and configured agent variables', () => { + it('recognizes agent variables without activating selection before the privacy prerequisite', () => { expect(getQualifiedAiReporterDecision({ COPILOT_CLI: '1' }, [], passedQualification())).toMatchObject({ agentDetected: true, - eligible: true, - reporter: 'ai' + eligible: false, + reason: 'privacy prerequisite unavailable' }); expect( getQualifiedAiReporterDecision({ MY_AGENT: 'yes' }, ['MY_AGENT'], passedQualification()) + ).toMatchObject({ + agentDetected: true, + eligible: false, + reason: 'privacy prerequisite unavailable' + }); + }); + + it('returns a reusable AI decision only after qualification and privacy are accepted', () => { + expect( + getQualifiedAiReporterDecision({ COPILOT_CLI: '1' }, [], passedQualification(), true) ).toMatchObject({ agentDetected: true, eligible: true, - reporter: 'ai' + reporter: 'ai', + reason: 'qualified' }); }); it('blocks selection when qualification is absent or failed', () => { - expect(getQualifiedAiReporterDecision({ COPILOT_CLI: '1' }, [], undefined)).toMatchObject({ + expect(getQualifiedAiReporterDecision({ COPILOT_CLI: '1' }, [], undefined, true)).toMatchObject({ eligible: false, reason: 'qualification unavailable' }); expect( - getQualifiedAiReporterDecision({ COPILOT_CLI: '1' }, [], { ...passedQualification(), passed: false }) + getQualifiedAiReporterDecision( + { COPILOT_CLI: '1' }, + [], + { ...passedQualification(), passed: false }, + true + ) ).toMatchObject({ eligible: false, reason: 'qualification failed' @@ -120,7 +166,12 @@ describe('qualified AI reporter decision', () => { it('keeps RUSH_REPORTER=legacy authoritative even after qualification passes', () => { expect( - getQualifiedAiReporterDecision({ COPILOT_CLI: '1', RUSH_REPORTER: 'legacy' }, [], passedQualification()) + getQualifiedAiReporterDecision( + { COPILOT_CLI: '1', RUSH_REPORTER: 'legacy' }, + [], + passedQualification(), + true + ) ).toMatchObject({ agentDetected: true, eligible: false, diff --git a/libraries/reporter/src/test/FileReporter.test.ts b/libraries/reporter/src/test/FileReporter.test.ts index eded1e7b0a..5ac39d2348 100644 --- a/libraries/reporter/src/test/FileReporter.test.ts +++ b/libraries/reporter/src/test/FileReporter.test.ts @@ -340,6 +340,37 @@ describe('FileReporter', () => { }); }); + it('retains local-sensitive producer identity but redacts secret producer identity', async () => { + await withTempDir(async (base: string) => { + const reporter: FileReporter = new FileReporter({ commonTempFolder: base, nowMs: () => FIXED_NOW }); + reporter.report({ + ...ev('extension', { name: 'local.plugin.event' }, 'local-sensitive'), + source: { + packageName: '@private/example-rush-plugin', + packageVersion: '1.0.0', + component: 'PrivatePluginImplementation' + } + }); + reporter.report({ + ...ev('extension', { name: 'secret.plugin.event' }, 'secret'), + source: { + packageName: '@secret/example-rush-plugin', + packageVersion: '2.0.0', + component: 'SecretPluginImplementation' + } + }); + await reporter.closeAsync(); + + const content: string = await fs.promises.readFile(reporter.getArtifact().path!, 'utf8'); + expect(content).toContain('@private/example-rush-plugin'); + expect(content).toContain('PrivatePluginImplementation'); + expect(content).not.toContain('@secret/example-rush-plugin'); + expect(content).not.toContain('SecretPluginImplementation'); + expect(content).toContain('[private-producer]'); + expect(content).toContain('[private-version]'); + }); + }); + it('deletes logs older than the retention window and caps the session count', async () => { await withTempDir(async (base: string) => { const logsDir: string = path.join(base, RUSH_LOGS_DIR_NAME); diff --git a/libraries/reporter/src/test/JsonAiReporter.test.ts b/libraries/reporter/src/test/JsonAiReporter.test.ts index df730fa90b..8ec242f4a5 100644 --- a/libraries/reporter/src/test/JsonAiReporter.test.ts +++ b/libraries/reporter/src/test/JsonAiReporter.test.ts @@ -73,28 +73,28 @@ describe('JsonReporter', () => { expect(Buffer.byteLength(output.trim(), 'utf8')).toBeLessThanOrEqual(512); }); - it('does not expose a private producer identity in an oversized redacted record', () => { + it('does not expose a secret diagnostic value in an oversized redacted record', () => { let output: string = ''; const reporter: JsonReporter = new JsonReporter({ write: (text: string) => (output += text), maxRecordBytes: 512 }); reporter.report({ - ...ev('extension', { name: 'private.fixture', payload: 'x'.repeat(1000) }), - source: { - packageName: '@private/example-rush-plugin', - packageVersion: '1.0.0', - component: 'PrivatePluginImplementation' - }, + ...ev('diagnosticEmitted', { + code: 'RUSH_DEPENDENCY_TOOL_FAILED', + parameters: { + token: { value: 'qualification-fake-secret-token', privacy: 'secret' }, + detail: { value: 'x'.repeat(1000), privacy: 'public' } + } + }), privacy: 'local-sensitive' }); - expect(output).not.toContain('@private/example-rush-plugin'); - expect(output).not.toContain('PrivatePluginImplementation'); + expect(output).not.toContain('qualification-fake-secret-token'); expect(parseLines(output)[0]).toMatchObject({ source: { - packageName: '[private-producer]', - packageVersion: '[private-version]' + packageName: '@microsoft/rush-lib', + packageVersion: '5.177.2' }, payload: { name: 'rush.reporter.record-too-large' @@ -465,6 +465,56 @@ describe('AiReporter', () => { expect(JSON.stringify(final)).not.toContain('qualification-fake-secret-token'); }); + it('counts secret diagnostics while marking their omitted details as truncated', () => { + const { final } = run([ + { + ...ev('diagnosticEmitted', { + diagnosticId: 'secret-error', + code: 'RUSH_INTERNAL_UNEXPECTED', + category: 'internal', + severity: 'error', + summary: 'qualification-fake-secret-token' + }), + privacy: 'secret' + }, + ev('commandResult', { commandName: 'build', succeeded: false, exitCode: 1 }) + ]); + + expect(final.errorCount).toBe(1); + expect(final.errorCodes).toEqual([]); + expect(final.diagnosticCategoryCounts).toEqual({}); + expect(final.diagnostics).toEqual([]); + expect(final.truncated).toBe(true); + expect(JSON.stringify(final)).not.toContain('qualification-fake-secret-token'); + }); + + it('preserves fallback errors when secret diagnostics are also suppressed', () => { + const { final } = run([ + { + ...ev('diagnosticEmitted', { + diagnosticId: 'secret-error', + code: 'RUSH_INTERNAL_UNEXPECTED', + category: 'internal', + severity: 'error', + summary: 'qualification-fake-secret-token' + }), + privacy: 'secret' + }, + ev('messageEmitted', { severity: 'error', text: 'First visible fallback error.' }), + ev('messageEmitted', { severity: 'error', text: 'Second visible fallback error.' }), + ev('commandResult', { commandName: 'build', succeeded: false, exitCode: 1 }) + ]); + + expect(final.errorCount).toBe(3); + expect(final.errorCodes).toEqual(['RUSH_COMMAND_FAILED']); + expect(final.diagnostics.map(({ summary }) => summary)).toEqual([ + 'First visible fallback error.', + 'Second visible fallback error.' + ]); + expect(final.truncated).toBe(true); + expect(JSON.stringify(final)).not.toContain('qualification-fake-secret-token'); + }); + it('excludes raw external output and keeps stdout pure JSON', () => { let output: string = ''; const reporter: AiReporter = new AiReporter({ write: (text: string) => (output += text) }); From c46a4ea8036805e70de1143f25d206b4c653e42c Mon Sep 17 00:00:00 2001 From: TheLarkInn Date: Fri, 28 Aug 2026 22:29:33 +0000 Subject: [PATCH 3/4] Fix AI fallback privacy gates Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d6318e80-5da9-4858-a147-817e8692f10e --- libraries/reporter/README.md | 4 + .../AiReporterQualificationCorpus.ts | 164 +++++++++++++++--- .../reporter/src/reporters/AiReporter.ts | 7 +- .../reporter/src/reporters/JsonReporter.ts | 33 +++- .../src/test/AiReporterQualification.test.ts | 9 +- .../reporter/src/test/JsonAiReporter.test.ts | 114 +++++++++++- 6 files changed, 296 insertions(+), 35 deletions(-) diff --git a/libraries/reporter/README.md b/libraries/reporter/README.md index 11f194bc3e..1afb1d2ebf 100644 --- a/libraries/reporter/README.md +++ b/libraries/reporter/README.md @@ -28,6 +28,10 @@ reporter selection. That decision also requires the separate telemetry privacy p The pre-major Rush frontend remains explicit/repository-opt-in, and `RUSH_REPORTER=legacy` remains authoritative. +AI fallback message text is emitted only for public envelopes. Non-public fallback errors remain countable +and refer to the protected full-detail log. JSON oversized-record markers preserve the original privacy +classification and omit non-public source and scope metadata. + ## Links - [CHANGELOG.md](https://github.com/microsoft/rushstack/blob/main/libraries/reporter/CHANGELOG.md) - Find out diff --git a/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts b/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts index 3982add9d0..ca320a9b0a 100644 --- a/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts +++ b/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts @@ -11,6 +11,7 @@ import type { ReporterPrivacyClassification } from '../events/ReporterPrivacyCla import type { IAiDiagnostic, IAiFinalRecord } from '../reporters/AiReporter'; import { AiReporter } from '../reporters/AiReporter'; import { FileReporter, type IFileReporterArtifact } from '../reporters/FileReporter'; +import { JsonReporter } from '../reporters/JsonReporter'; import { LegacyReporter } from '../reporters/LegacyReporter'; import { PlaintextReporter } from '../reporters/PlaintextReporter'; import { @@ -28,6 +29,11 @@ const CLASSIFIED_SECRET_PRODUCER: string = '@secret/qualification-fixture'; const CLASSIFIED_SECRET_COMPONENT: string = 'SecretQualificationFixture'; const PRIVATE_PRODUCER: string = '@private/example-rush-plugin'; const PRIVATE_COMPONENT: string = 'PrivatePluginImplementation'; +const LOCAL_SENSITIVE_FALLBACK_MESSAGE: string = 'qualification-local-sensitive-fallback-message'; +const OVERSIZED_LOCAL_SENSITIVE_VALUE: string = 'qualification-oversized-local-sensitive-value'; +const OVERSIZED_LOCAL_SENSITIVE_PRODUCER: string = '@private/oversized-qualification-fixture'; +const OVERSIZED_LOCAL_SENSITIVE_COMPONENT: string = 'OversizedPrivateQualificationFixture'; +const OVERSIZED_LOCAL_SENSITIVE_SCOPE: string = '@private/oversized-qualification-project'; function createExternalOutput(lines: readonly string[], repetitions: number): string { const output: string[] = []; @@ -64,6 +70,10 @@ interface ICorpusCase { readonly expectedResult: 'succeeded' | 'failed'; readonly diagnostic?: ICorpusDiagnostic; readonly externalOutput?: string; + readonly fallbackMessages?: readonly { + readonly text: string; + readonly privacy: ReporterPrivacyClassification; + }[]; readonly operationStatus?: 'success' | 'failure' | 'aborted' | 'fromCache'; readonly warningOnly?: boolean; } @@ -348,6 +358,21 @@ const CORPUS: readonly ICorpusCase[] = [ ] } }, + { + name: 'fallback-mixed-privacy', + scenario: 'legacy parser errors include public and local-sensitive fallback messages', + expectedResult: 'failed', + fallbackMessages: [ + { + text: 'The requested command could not be parsed.', + privacy: 'public' + }, + { + text: LOCAL_SENSITIVE_FALLBACK_MESSAGE, + privacy: 'local-sensitive' + } + ] + }, { name: 'success-no-warning', scenario: 'a successful operation emits no warnings', @@ -532,6 +557,19 @@ function createEvents(testCase: ICorpusCase, logPath: string): IReporterEventEnv } ); } + for (const message of testCase.fallbackMessages ?? []) { + add( + 'messageEmitted', + { + severity: 'error', + text: message.text + }, + { + privacy: message.privacy, + scope: { commandName: 'build' } + } + ); + } if (testCase.expectedResult === 'failed') { add( 'diagnosticEmitted', @@ -550,6 +588,23 @@ function createEvents(testCase: ICorpusCase, logPath: string): IReporterEventEnv { scope: { commandName: 'build' } } ); } + add( + 'extension', + { + name: 'qualification.oversized-local-sensitive', + payload: OVERSIZED_LOCAL_SENSITIVE_VALUE.repeat(128) + }, + { + privacy: 'local-sensitive', + sourcePackage: OVERSIZED_LOCAL_SENSITIVE_PRODUCER, + sourceComponent: OVERSIZED_LOCAL_SENSITIVE_COMPONENT, + scope: { + commandName: 'build', + operationId: 'oversized-local-sensitive-operation', + projectName: OVERSIZED_LOCAL_SENSITIVE_SCOPE + } + } + ); add( 'extension', { name: 'private.fixture.secret', payload: { token: CLASSIFIED_SECRET } }, @@ -588,6 +643,7 @@ async function runCaseAsync( tempRoot: string ): Promise { let aiOutput: string = ''; + let jsonOutput: string = ''; let plaintextOutput: string = ''; let legacyOutput: string = ''; const fileReporter: FileReporter = new FileReporter({ @@ -597,6 +653,10 @@ async function runCaseAsync( nowMs: () => FIXED_TIME_MS }); const aiReporter: AiReporter = new AiReporter({ write: (text: string) => (aiOutput += text) }); + const jsonReporter: JsonReporter = new JsonReporter({ + write: (text: string) => (jsonOutput += text), + maxRecordBytes: 1024 + }); const plaintextReporter: PlaintextReporter = new PlaintextReporter({ write: (text: string) => (plaintextOutput += text), variant: 'detailed', @@ -614,11 +674,13 @@ async function runCaseAsync( for (const event of events) { fileReporter.report(event); aiReporter.report(event); + jsonReporter.report(event); plaintextReporter.report(event); legacyReporter.report(event); } await fileReporter.closeAsync(); await aiReporter.closeAsync(); + await jsonReporter.closeAsync(); await plaintextReporter.closeAsync(); await legacyReporter.closeAsync(); @@ -637,6 +699,10 @@ async function runCaseAsync( readonly records: readonly Record[]; readonly valid: boolean; } = parseAiOutput(aiOutput); + const parsedJson: { + readonly records: readonly Record[]; + readonly valid: boolean; + } = parseAiOutput(jsonOutput); const final: IAiFinalRecord | undefined = parsedAi.records.at(-1) as IAiFinalRecord | undefined; const diagnostic: ICorpusDiagnostic | undefined = testCase.diagnostic; const matchingDiagnostic: IAiDiagnostic | undefined = diagnostic @@ -651,21 +717,41 @@ async function runCaseAsync( return matchingDiagnostic?.context?.[name] === expectedValue; }) : true; + const fallbackMessages: readonly { + readonly text: string; + readonly privacy: ReporterPrivacyClassification; + }[] = testCase.fallbackMessages ?? []; + const publicFallbackMessages: readonly string[] = fallbackMessages + .filter(({ privacy }) => privacy === 'public') + .map(({ text }) => text); const actionable: boolean = testCase.expectedResult === 'succeeded' ? true - : Boolean( - final?.result === 'failed' && - diagnostic && - final.errorCodes.includes(diagnostic.code) && - matchingDiagnostic?.category === diagnostic.category && - matchingDiagnostic.summaryKey === diagnostic.summaryKey && - expectedContextKeys.every((key) => actualContextKeys.includes(key)) && - contextValuesMatch && - matchingDiagnostic.remediation?.some(({ command, documentationUrl }) => - Boolean(command || documentationUrl) - ) - ); + : diagnostic + ? Boolean( + final?.result === 'failed' && + final.errorCodes.includes(diagnostic.code) && + matchingDiagnostic?.category === diagnostic.category && + matchingDiagnostic.summaryKey === diagnostic.summaryKey && + expectedContextKeys.every((key) => actualContextKeys.includes(key)) && + contextValuesMatch && + matchingDiagnostic.remediation?.some(({ command, documentationUrl }) => + Boolean(command || documentationUrl) + ) + ) + : Boolean( + fallbackMessages.length > 0 && + final?.result === 'failed' && + final.errorCodes.includes('RUSH_COMMAND_FAILED') && + final.errorCount === fallbackMessages.length && + final.diagnosticCategoryCounts.command === fallbackMessages.length && + final.diagnostics.length === publicFallbackMessages.length && + final.diagnostics.every( + ({ category, severity, summary }, index) => + category === 'command' && severity === 'error' && summary === publicFallbackMessages[index] + ) && + final.truncated + ); const artifact: IFileReporterArtifact = fileReporter.getArtifact(); const aiLogPath: string | undefined = final?.log?.path; @@ -677,12 +763,13 @@ async function runCaseAsync( (logExists && aiLogPath !== undefined && (await fs.promises.stat(aiLogPath)).mode % 0o1000 === 0o600); const failureCorrelated: boolean = testCase.expectedResult === 'succeeded' || - Boolean( - diagnostic && - logContent.includes(diagnostic.code) && + (diagnostic + ? logContent.includes(diagnostic.code) && logContent.includes(`${testCase.name}-diagnostic`) && logContent.includes(`${testCase.name}-session`) - ); + : fallbackMessages.length > 0 && + fallbackMessages.every(({ text }) => logContent.includes(text)) && + logContent.includes(`${testCase.name}-session`)); const localSensitiveProducerPreserved: boolean = diagnostic?.sourcePackage === undefined || (logContent.includes(diagnostic.sourcePackage) && @@ -703,16 +790,45 @@ async function runCaseAsync( logContent.includes('"type":"sessionCompleted"') && (testCase.externalOutput === undefined || logContent.includes(testCase.externalOutput.trim())) && failureCorrelated && - localSensitiveProducerPreserved + localSensitiveProducerPreserved && + logContent.includes(OVERSIZED_LOCAL_SENSITIVE_VALUE) && + logContent.includes(OVERSIZED_LOCAL_SENSITIVE_PRODUCER) && + logContent.includes(OVERSIZED_LOCAL_SENSITIVE_COMPONENT) && + logContent.includes(OVERSIZED_LOCAL_SENSITIVE_SCOPE) ); - const machinePresentedOutput: string = `${aiOutput}\n${plaintextOutput}\n${legacyOutput}`; - const allLocalOutput: string = `${machinePresentedOutput}\n${logContent}`; + const oversizedMarker: Record | undefined = parsedJson.records.find( + ({ payload }) => + ( + payload as + | { readonly name?: string; readonly payload?: { readonly originalType?: string } } + | undefined + )?.name === 'rush.reporter.record-too-large' && + (payload as { readonly payload?: { readonly originalType?: string } }).payload?.originalType === + 'extension' + ); + const oversizedMarkerValid: boolean = + oversizedMarker?.privacy === 'local-sensitive' && + oversizedMarker.scope === undefined && + (oversizedMarker.source as { readonly packageName?: string; readonly packageVersion?: string }) + ?.packageName === '[private-producer]' && + (oversizedMarker.source as { readonly packageName?: string; readonly packageVersion?: string }) + ?.packageVersion === '[private-version]'; + const machinePresentedOutput: string = `${aiOutput}\n${jsonOutput}`; + const humanPresentedOutput: string = `${plaintextOutput}\n${legacyOutput}`; + const allLocalOutput: string = `${machinePresentedOutput}\n${humanPresentedOutput}\n${logContent}`; const privacySafe: boolean = !allLocalOutput.includes(CLASSIFIED_SECRET) && !allLocalOutput.includes(CLASSIFIED_SECRET_PRODUCER) && !allLocalOutput.includes(CLASSIFIED_SECRET_COMPONENT) && - !machinePresentedOutput.includes(PRIVATE_PRODUCER) && - !machinePresentedOutput.includes(PRIVATE_COMPONENT); + !machinePresentedOutput.includes(LOCAL_SENSITIVE_FALLBACK_MESSAGE) && + !machinePresentedOutput.includes(OVERSIZED_LOCAL_SENSITIVE_VALUE) && + !machinePresentedOutput.includes(OVERSIZED_LOCAL_SENSITIVE_PRODUCER) && + !machinePresentedOutput.includes(OVERSIZED_LOCAL_SENSITIVE_COMPONENT) && + !machinePresentedOutput.includes(OVERSIZED_LOCAL_SENSITIVE_SCOPE) && + !aiOutput.includes(PRIVATE_PRODUCER) && + !aiOutput.includes(PRIVATE_COMPONENT) && + !humanPresentedOutput.includes(PRIVATE_PRODUCER) && + !humanPresentedOutput.includes(PRIVATE_COMPONENT); const warningContractValid: boolean = testCase.expectedResult === 'failed' ? final?.warningCount === 1 && final.diagnostics.every(({ severity }) => severity === 'error') @@ -723,7 +839,9 @@ async function runCaseAsync( if (!actionable) failures.push('missing stable code/category/context/remediation'); if (!privacySafe) failures.push('classified secret or private producer identity leaked'); if (!fullLogValid) failures.push('full log path, permissions, completeness, or correlation invalid'); - if (!parsedAi.valid) failures.push('AI stdout was not payload-only NDJSON'); + if (!parsedAi.valid || !parsedJson.valid || !oversizedMarkerValid) { + failures.push('machine stdout or oversized-record marker contract regressed'); + } if (!warningContractValid) failures.push('warning suppression/detail contract regressed'); return { @@ -740,7 +858,7 @@ async function runCaseAsync( actionable, privacySafe, fullLogValid, - stdoutContractValid: parsedAi.valid, + stdoutContractValid: parsedAi.valid && parsedJson.valid && oversizedMarkerValid, warningContractValid, failures } diff --git a/libraries/reporter/src/reporters/AiReporter.ts b/libraries/reporter/src/reporters/AiReporter.ts index becf49623e..9b7eb034df 100644 --- a/libraries/reporter/src/reporters/AiReporter.ts +++ b/libraries/reporter/src/reporters/AiReporter.ts @@ -308,9 +308,12 @@ export class AiReporter implements IReporter { severity?: string; text?: string; }; - if (event.privacy !== 'secret' && payload.severity === 'error' && payload.text) { + if (payload.severity === 'error' && payload.text) { this._fallbackErrorCount++; - if (this._fallbackErrorMessages.length < this._maxDetailedDiagnostics) { + if ( + event.privacy === 'public' && + this._fallbackErrorMessages.length < this._maxDetailedDiagnostics + ) { this._fallbackErrorMessages.push(payload.text.trim()); } else { this._fallbackErrorsTruncated = true; diff --git a/libraries/reporter/src/reporters/JsonReporter.ts b/libraries/reporter/src/reporters/JsonReporter.ts index 0e79bbdaa6..a6c68ffd33 100644 --- a/libraries/reporter/src/reporters/JsonReporter.ts +++ b/libraries/reporter/src/reporters/JsonReporter.ts @@ -50,16 +50,43 @@ export class JsonReporter implements IReporter { } public report(event: IReporterEventEnvelope): void { - const redactedEvent: IReporterEventEnvelope = redactReporterEvent(event); + const machineEvent: IReporterEventEnvelope = + event.type === 'messageEmitted' && event.privacy === 'local-sensitive' + ? { + ...event, + payload: { + ...(event.payload as Record), + text: '[local-sensitive]' + } + } + : event; + const redactedEvent: IReporterEventEnvelope = redactReporterEvent(machineEvent); try { this._write(encodeNdjsonRecord(redactedEvent, { maxRecordBytes: this._maxRecordBytes })); } catch (error) { if (error instanceof NdjsonRecordTooLargeError) { + const source: IReporterEventEnvelope['source'] = + redactedEvent.privacy === 'public' + ? redactedEvent.source + : { + packageName: '[private-producer]', + packageVersion: '[private-version]' + }; this._write( encodeNdjsonRecord( { - ...redactedEvent, - privacy: 'public', + protocolVersion: redactedEvent.protocolVersion, + eventId: redactedEvent.eventId, + sessionId: redactedEvent.sessionId, + parentSessionId: redactedEvent.parentSessionId, + parentOperationId: redactedEvent.parentOperationId, + sequence: redactedEvent.sequence, + sourceSequence: redactedEvent.sourceSequence, + timestamp: redactedEvent.timestamp, + source, + scope: redactedEvent.privacy === 'public' ? redactedEvent.scope : undefined, + privacy: redactedEvent.privacy, + required: redactedEvent.required, type: 'extension', payload: { name: 'rush.reporter.record-too-large', diff --git a/libraries/reporter/src/test/AiReporterQualification.test.ts b/libraries/reporter/src/test/AiReporterQualification.test.ts index 91d1233989..b917f1958b 100644 --- a/libraries/reporter/src/test/AiReporterQualification.test.ts +++ b/libraries/reporter/src/test/AiReporterQualification.test.ts @@ -26,12 +26,15 @@ describe('AI reporter deterministic qualification corpus', () => { throw new Error(formatAiReporterQualificationFailures(qualification)); } expect(qualification.schemaVersion).toBe('1.0'); - expect(qualification.cases).toHaveLength(12); - expect(qualification.cases.filter(({ expectedResult }) => expectedResult === 'failed')).toHaveLength(10); + expect(qualification.cases).toHaveLength(13); + expect(qualification.cases.filter(({ expectedResult }) => expectedResult === 'failed')).toHaveLength(11); expect(qualification.cases.every(({ failures }) => failures.length === 0)).toBe(true); const serialized: string = JSON.stringify(qualification); expect(serialized).not.toContain('rush-ai-reporter-qualification-'); expect(serialized).not.toContain('qualification-fake-secret-token'); + expect(serialized).not.toContain('qualification-local-sensitive-fallback-message'); + expect(serialized).not.toContain('qualification-oversized-local-sensitive-value'); + expect(serialized).not.toContain('@private/oversized-qualification-fixture'); expect(serialized).not.toContain('@private/example-rush-plugin'); }); @@ -78,7 +81,7 @@ describe('AI reporter deterministic qualification corpus', () => { expect(failed.passed).toBe(false); expect(formatAiReporterQualificationFailures(failed)).toContain( - 'actionability: actual=90.00, required=>= 100%; cases=bootstrap-unsupported-node' + 'actionability: actual=90.91, required=>= 100%; cases=bootstrap-unsupported-node' ); }); diff --git a/libraries/reporter/src/test/JsonAiReporter.test.ts b/libraries/reporter/src/test/JsonAiReporter.test.ts index 8ec242f4a5..d6e70dd6b5 100644 --- a/libraries/reporter/src/test/JsonAiReporter.test.ts +++ b/libraries/reporter/src/test/JsonAiReporter.test.ts @@ -56,21 +56,36 @@ describe('JsonReporter', () => { let output: string = ''; const reporter: JsonReporter = new JsonReporter({ write: (text: string) => (output += text), - maxRecordBytes: 512 + maxRecordBytes: 768 + }); + reporter.report({ + ...ev( + 'externalOutput', + { stream: 'stdout', text: 'x'.repeat(1000) }, + { operationId: 'operation-a', projectName: '@example/project' } + ), + sessionId: 'child-session', + parentSessionId: 'parent-session', + parentOperationId: 'parent-operation', + sourceSequence: 3 }); - reporter.report(ev('externalOutput', { stream: 'stdout', text: 'x'.repeat(1000) })); const records: Record[] = parseLines(output); expect(records).toHaveLength(1); expect((records[0].payload as { name: string }).name).toBe('rush.reporter.record-too-large'); expect(records[0]).toMatchObject({ timestamp: '2026-01-01T00:00:00.000Z', + sessionId: 'child-session', + parentSessionId: 'parent-session', + parentOperationId: 'parent-operation', + sourceSequence: 3, source: { packageName: '@microsoft/rush-lib', packageVersion: '5.177.2' }, + scope: { operationId: 'operation-a', projectName: '@example/project' }, privacy: 'public', required: true, type: 'extension' }); - expect(Buffer.byteLength(output.trim(), 'utf8')).toBeLessThanOrEqual(512); + expect(Buffer.byteLength(output.trim(), 'utf8')).toBeLessThanOrEqual(768); }); it('does not expose a secret diagnostic value in an oversized redacted record', () => { @@ -87,17 +102,68 @@ describe('JsonReporter', () => { detail: { value: 'x'.repeat(1000), privacy: 'public' } } }), + source: { + packageName: '@private/oversized-reporter', + packageVersion: '1.0.0', + component: 'OversizedPrivateComponent' + }, + scope: { + operationId: 'oversized-private-operation', + projectName: '@private/oversized-project' + }, privacy: 'local-sensitive' }); expect(output).not.toContain('qualification-fake-secret-token'); + expect(output).not.toContain('@private/oversized-reporter'); + expect(output).not.toContain('OversizedPrivateComponent'); + expect(output).not.toContain('oversized-private-operation'); + expect(output).not.toContain('@private/oversized-project'); + expect(parseLines(output)[0]).toMatchObject({ + source: { + packageName: '[private-producer]', + packageVersion: '[private-version]' + }, + privacy: 'local-sensitive', + payload: { + name: 'rush.reporter.record-too-large' + } + }); + expect(parseLines(output)[0].scope).toBeUndefined(); + }); + + it('redacts local-sensitive message text from JSON stdout without dropping envelope metadata', () => { + let output: string = ''; + const reporter: JsonReporter = new JsonReporter({ write: (text: string) => (output += text) }); + reporter.report({ + ...ev( + 'messageEmitted', + { + severity: 'error', + text: 'qualification-local-sensitive-message' + }, + { + operationId: 'operation-a', + projectName: '@example/project' + } + ), + privacy: 'local-sensitive' + }); + + expect(output).not.toContain('qualification-local-sensitive-message'); expect(parseLines(output)[0]).toMatchObject({ source: { packageName: '@microsoft/rush-lib', packageVersion: '5.177.2' }, + scope: { + operationId: 'operation-a', + projectName: '@example/project' + }, + privacy: 'local-sensitive', payload: { - name: 'rush.reporter.record-too-large' + severity: 'error', + text: '[local-sensitive]' } }); }); @@ -184,6 +250,46 @@ describe('AiReporter', () => { ]); }); + it('counts non-public fallback errors without exposing their message text', () => { + const { final } = run([ + ev('commandStarted', { commandName: 'build' }), + { + ...ev('messageEmitted', { + severity: 'error', + text: 'qualification-local-sensitive-message' + }), + privacy: 'local-sensitive' + }, + { + ...ev('messageEmitted', { + severity: 'error', + text: 'qualification-secret-message' + }), + privacy: 'secret' + }, + ev('artifactAvailable', { + role: 'log', + path: '/protected/rush.log', + format: 'plaintext', + complete: true + }), + ev('commandResult', { commandName: 'build', succeeded: false, exitCode: 1 }) + ]); + + expect(final.errorCodes).toEqual(['RUSH_COMMAND_FAILED']); + expect(final.errorCount).toBe(2); + expect(final.diagnosticCategoryCounts.command).toBe(2); + expect(final.diagnostics).toEqual([]); + expect(final.truncated).toBe(true); + expect(final.log).toEqual({ + path: '/protected/rush.log', + format: 'plaintext', + complete: true + }); + expect(JSON.stringify(final)).not.toContain('qualification-local-sensitive-message'); + expect(JSON.stringify(final)).not.toContain('qualification-secret-message'); + }); + it('counts fallback errors even when detailed diagnostics are disabled', async () => { let output: string = ''; const reporter: AiReporter = new AiReporter({ From 6dac2ffdcdb761232829e9160ec13031d3fb65a6 Mon Sep 17 00:00:00 2001 From: TheLarkInn Date: Fri, 28 Aug 2026 23:12:06 +0000 Subject: [PATCH 4/4] Redact secret reporter envelope context Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: d6318e80-5da9-4858-a147-817e8692f10e --- libraries/reporter/README.md | 4 + .../AiReporterQualificationCorpus.ts | 93 ++++++++++ .../reporter/src/reporters/AiReporter.ts | 11 ++ .../src/reporters/ReporterRedaction.ts | 32 ++-- .../src/test/AiReporterQualification.test.ts | 8 + .../reporter/src/test/JsonAiReporter.test.ts | 169 ++++++++++++++++++ 6 files changed, 306 insertions(+), 11 deletions(-) diff --git a/libraries/reporter/README.md b/libraries/reporter/README.md index 1afb1d2ebf..d2d7de0c6b 100644 --- a/libraries/reporter/README.md +++ b/libraries/reporter/README.md @@ -32,6 +32,10 @@ AI fallback message text is emitted only for public envelopes. Non-public fallba and refer to the protected full-detail log. JSON oversized-record markers preserve the original privacy classification and omit non-public source and scope metadata. +Secret envelopes retain only protocol, event identity, ordering, timing, type, privacy, and fully redacted +source and payload fields. Contextual parent, command, operation, project, phase, and scope metadata is +removed. + ## Links - [CHANGELOG.md](https://github.com/microsoft/rushstack/blob/main/libraries/reporter/CHANGELOG.md) - Find out diff --git a/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts b/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts index ca320a9b0a..33a9119f90 100644 --- a/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts +++ b/libraries/reporter/src/qualification/AiReporterQualificationCorpus.ts @@ -27,6 +27,14 @@ const FIXED_PID: number = 4242; const CLASSIFIED_SECRET: string = 'qualification-fake-secret-token'; const CLASSIFIED_SECRET_PRODUCER: string = '@secret/qualification-fixture'; const CLASSIFIED_SECRET_COMPONENT: string = 'SecretQualificationFixture'; +const CLASSIFIED_SECRET_COMMAND: string = 'qualification-secret-command'; +const CLASSIFIED_SECRET_OPERATION: string = 'qualification-secret-operation'; +const CLASSIFIED_SECRET_PROJECT: string = '@private/qualification-secret-project'; +const CLASSIFIED_SECRET_PHASE: string = 'qualification-secret-phase'; +const CLASSIFIED_SECRET_PARENT_SESSION: string = 'qualification-secret-parent-session'; +const CLASSIFIED_SECRET_PARENT_OPERATION: string = 'qualification-secret-parent-operation'; +const CLASSIFIED_SECRET_MESSAGE: string = 'qualification-secret-message-text'; +const CLASSIFIED_SECRET_DIAGNOSTIC: string = 'qualification-secret-diagnostic-summary'; const PRIVATE_PRODUCER: string = '@private/example-rush-plugin'; const PRIVATE_COMPONENT: string = 'PrivatePluginImplementation'; const LOCAL_SENSITIVE_FALLBACK_MESSAGE: string = 'qualification-local-sensitive-fallback-message'; @@ -637,6 +645,74 @@ function createEvents(testCase: ICorpusCase, logPath: string): IReporterEventEnv return events; } +function createSecretProjectionProbeEvents(testCase: ICorpusCase): IReporterEventEnvelope[] { + const createProbe = ( + eventId: string, + sequence: number, + type: IReporterEventEnvelope['type'], + payload: unknown, + scope: IReporterEventEnvelope['scope'] = { + commandName: CLASSIFIED_SECRET_COMMAND, + operationId: CLASSIFIED_SECRET_OPERATION, + projectName: CLASSIFIED_SECRET_PROJECT, + phaseName: CLASSIFIED_SECRET_PHASE + } + ): IReporterEventEnvelope => ({ + protocolVersion: { major: 1, minor: 1 }, + eventId, + sessionId: `${testCase.name}-session`, + parentSessionId: CLASSIFIED_SECRET_PARENT_SESSION, + parentOperationId: CLASSIFIED_SECRET_PARENT_OPERATION, + sequence, + sourceSequence: sequence - 10000, + timestamp: FIXED_TIMESTAMP, + source: { + packageName: CLASSIFIED_SECRET_PRODUCER, + packageVersion: '1.0.0', + component: CLASSIFIED_SECRET_COMPONENT + }, + scope, + privacy: 'secret', + required: true, + type, + payload + }); + + return [ + createProbe( + `${testCase.name}-secret-command`, + 10001, + 'commandStarted', + { + commandName: CLASSIFIED_SECRET_COMMAND + }, + { + commandName: CLASSIFIED_SECRET_COMMAND + } + ), + createProbe(`${testCase.name}-secret-operation-registered`, 10002, 'operationRegistered', { + operationId: CLASSIFIED_SECRET_OPERATION, + projectName: CLASSIFIED_SECRET_PROJECT, + phaseName: CLASSIFIED_SECRET_PHASE + }), + createProbe(`${testCase.name}-secret-operation-completed`, 10003, 'operationCompleted', { + operationId: CLASSIFIED_SECRET_OPERATION, + status: 'failure' + }), + createProbe(`${testCase.name}-secret-message`, 10004, 'messageEmitted', { + severity: 'info', + text: CLASSIFIED_SECRET_MESSAGE + }), + createProbe(`${testCase.name}-secret-diagnostic`, 10005, 'diagnosticEmitted', { + diagnosticId: `${testCase.name}-secret-diagnostic`, + code: 'RUSH_INTERNAL_UNEXPECTED', + category: 'internal', + severity: 'info', + summary: CLASSIFIED_SECRET_DIAGNOSTIC + }) + ]; +} + async function runCaseAsync( testCase: ICorpusCase, caseDirectory: string, @@ -678,6 +754,15 @@ async function runCaseAsync( plaintextReporter.report(event); legacyReporter.report(event); } + for (const event of createSecretProjectionProbeEvents(testCase)) { + // Operation grouping is a separate file-sidecar policy. Exercise shared + // file redaction with records that do not enter the grouping path. + if (event.type === 'messageEmitted' || event.type === 'diagnosticEmitted') { + fileReporter.report(event); + } + aiReporter.report(event); + jsonReporter.report(event); + } await fileReporter.closeAsync(); await aiReporter.closeAsync(); await jsonReporter.closeAsync(); @@ -820,6 +905,14 @@ async function runCaseAsync( !allLocalOutput.includes(CLASSIFIED_SECRET) && !allLocalOutput.includes(CLASSIFIED_SECRET_PRODUCER) && !allLocalOutput.includes(CLASSIFIED_SECRET_COMPONENT) && + !allLocalOutput.includes(CLASSIFIED_SECRET_COMMAND) && + !allLocalOutput.includes(CLASSIFIED_SECRET_OPERATION) && + !allLocalOutput.includes(CLASSIFIED_SECRET_PROJECT) && + !allLocalOutput.includes(CLASSIFIED_SECRET_PHASE) && + !allLocalOutput.includes(CLASSIFIED_SECRET_PARENT_SESSION) && + !allLocalOutput.includes(CLASSIFIED_SECRET_PARENT_OPERATION) && + !allLocalOutput.includes(CLASSIFIED_SECRET_MESSAGE) && + !allLocalOutput.includes(CLASSIFIED_SECRET_DIAGNOSTIC) && !machinePresentedOutput.includes(LOCAL_SENSITIVE_FALLBACK_MESSAGE) && !machinePresentedOutput.includes(OVERSIZED_LOCAL_SENSITIVE_VALUE) && !machinePresentedOutput.includes(OVERSIZED_LOCAL_SENSITIVE_PRODUCER) && diff --git a/libraries/reporter/src/reporters/AiReporter.ts b/libraries/reporter/src/reporters/AiReporter.ts index 9b7eb034df..6e6f4667bb 100644 --- a/libraries/reporter/src/reporters/AiReporter.ts +++ b/libraries/reporter/src/reporters/AiReporter.ts @@ -205,6 +205,17 @@ export class AiReporter implements IReporter { public report(event: IReporterEventEnvelope): void { this._protocolVersion = event.protocolVersion; + if (event.privacy === 'secret') { + switch (event.type) { + case 'diagnosticEmitted': + case 'messageEmitted': + case 'commandResult': + case 'sessionCompleted': + break; + default: + return; + } + } switch (event.type) { case 'commandStarted': { this._commandName = (event.payload as { commandName: string }).commandName; diff --git a/libraries/reporter/src/reporters/ReporterRedaction.ts b/libraries/reporter/src/reporters/ReporterRedaction.ts index f576e3f507..373631a01b 100644 --- a/libraries/reporter/src/reporters/ReporterRedaction.ts +++ b/libraries/reporter/src/reporters/ReporterRedaction.ts @@ -9,17 +9,27 @@ interface IClassifiedValue { } export function redactReporterEvent(event: IReporterEventEnvelope): IReporterEventEnvelope { - let payload: unknown = event.payload; - const source: IReporterEventEnvelope['source'] = - event.privacy === 'secret' - ? { - packageName: '[private-producer]', - packageVersion: '[private-version]' - } - : event.source; if (event.privacy === 'secret') { - payload = '[secret]'; - } else if (event.type === 'diagnosticEmitted') { + return { + protocolVersion: event.protocolVersion, + eventId: event.eventId, + sessionId: event.sessionId, + sequence: event.sequence, + sourceSequence: event.sourceSequence, + timestamp: event.timestamp, + source: { + packageName: '[private-producer]', + packageVersion: '[private-version]' + }, + privacy: 'secret', + required: event.required, + type: event.type, + payload: '[secret]' + }; + } + + let payload: unknown = event.payload; + if (event.type === 'diagnosticEmitted') { const diagnostic: { readonly parameters?: Readonly>; readonly source?: unknown; @@ -41,5 +51,5 @@ export function redactReporterEvent(event: IReporterEventEnvelope): IRe } payload = redactedDiagnostic; } - return { ...event, source, payload }; + return { ...event, payload }; } diff --git a/libraries/reporter/src/test/AiReporterQualification.test.ts b/libraries/reporter/src/test/AiReporterQualification.test.ts index b917f1958b..56c9351fed 100644 --- a/libraries/reporter/src/test/AiReporterQualification.test.ts +++ b/libraries/reporter/src/test/AiReporterQualification.test.ts @@ -32,6 +32,14 @@ describe('AI reporter deterministic qualification corpus', () => { const serialized: string = JSON.stringify(qualification); expect(serialized).not.toContain('rush-ai-reporter-qualification-'); expect(serialized).not.toContain('qualification-fake-secret-token'); + expect(serialized).not.toContain('qualification-secret-command'); + expect(serialized).not.toContain('qualification-secret-operation'); + expect(serialized).not.toContain('@private/qualification-secret-project'); + expect(serialized).not.toContain('qualification-secret-phase'); + expect(serialized).not.toContain('qualification-secret-parent-session'); + expect(serialized).not.toContain('qualification-secret-parent-operation'); + expect(serialized).not.toContain('qualification-secret-message-text'); + expect(serialized).not.toContain('qualification-secret-diagnostic-summary'); expect(serialized).not.toContain('qualification-local-sensitive-fallback-message'); expect(serialized).not.toContain('qualification-oversized-local-sensitive-value'); expect(serialized).not.toContain('@private/oversized-qualification-fixture'); diff --git a/libraries/reporter/src/test/JsonAiReporter.test.ts b/libraries/reporter/src/test/JsonAiReporter.test.ts index d6e70dd6b5..5365f497cc 100644 --- a/libraries/reporter/src/test/JsonAiReporter.test.ts +++ b/libraries/reporter/src/test/JsonAiReporter.test.ts @@ -10,6 +10,13 @@ import { type ITelemetryAggregate } from '../index'; +const SECRET_COMMAND: string = 'qualification-secret-command'; +const SECRET_OPERATION: string = 'qualification-secret-operation'; +const SECRET_PROJECT: string = '@private/qualification-secret-project'; +const SECRET_PHASE: string = 'qualification-secret-phase'; +const SECRET_PARENT_SESSION: string = 'qualification-secret-parent-session'; +const SECRET_PARENT_OPERATION: string = 'qualification-secret-parent-operation'; + function ev( type: string, payload: unknown = {}, @@ -168,6 +175,92 @@ describe('JsonReporter', () => { }); }); + it('allowlists metadata for normal and oversized secret envelopes', () => { + let output: string = ''; + const reporter: JsonReporter = new JsonReporter({ + write: (text: string) => (output += text), + maxRecordBytes: 512 + }); + const secretMetadata: Partial> = { + parentSessionId: SECRET_PARENT_SESSION, + parentOperationId: SECRET_PARENT_OPERATION, + sourceSequence: 7, + source: { + packageName: '@private/qualification-secret-producer', + packageVersion: '1.0.0', + component: 'QualificationSecretComponent' + }, + scope: { + commandName: SECRET_COMMAND, + operationId: SECRET_OPERATION, + projectName: SECRET_PROJECT, + phaseName: SECRET_PHASE + }, + privacy: 'secret' + }; + reporter.report({ + ...ev('messageEmitted', { + severity: 'error', + text: 'qualification-secret-message-text' + }), + ...secretMetadata, + eventId: 'secret-message' + } as IReporterEventEnvelope); + reporter.report({ + ...ev('diagnosticEmitted', { + code: 'RUSH_INTERNAL_UNEXPECTED', + summary: 'qualification-secret-diagnostic-summary', + detail: 'x'.repeat(2000) + }), + ...secretMetadata, + eventId: 'secret-diagnostic', + sequence: 2 + } as IReporterEventEnvelope); + + const records: Record[] = parseLines(output); + expect(records).toHaveLength(2); + for (const record of records) { + expect(Object.keys(record).sort()).toEqual( + [ + 'eventId', + 'payload', + 'privacy', + 'protocolVersion', + 'required', + 'sequence', + 'sessionId', + 'source', + 'sourceSequence', + 'timestamp', + 'type' + ].sort() + ); + expect(record).toMatchObject({ + source: { + packageName: '[private-producer]', + packageVersion: '[private-version]' + }, + privacy: 'secret', + payload: '[secret]' + }); + } + for (const sentinel of [ + SECRET_COMMAND, + SECRET_OPERATION, + SECRET_PROJECT, + SECRET_PHASE, + SECRET_PARENT_SESSION, + SECRET_PARENT_OPERATION, + '@private/qualification-secret-producer', + 'QualificationSecretComponent', + 'qualification-secret-message-text', + 'qualification-secret-diagnostic-summary' + ]) { + expect(output).not.toContain(sentinel); + } + expect(output).not.toContain('rush.reporter.record-too-large'); + }); + it('redacts secret diagnostic fields from stdout', () => { let output: string = ''; const reporter: JsonReporter = new JsonReporter({ write: (text: string) => (output += text) }); @@ -265,6 +358,14 @@ describe('AiReporter', () => { severity: 'error', text: 'qualification-secret-message' }), + parentSessionId: SECRET_PARENT_SESSION, + parentOperationId: SECRET_PARENT_OPERATION, + scope: { + commandName: SECRET_COMMAND, + operationId: SECRET_OPERATION, + projectName: SECRET_PROJECT, + phaseName: SECRET_PHASE + }, privacy: 'secret' }, ev('artifactAvailable', { @@ -288,6 +389,60 @@ describe('AiReporter', () => { }); expect(JSON.stringify(final)).not.toContain('qualification-local-sensitive-message'); expect(JSON.stringify(final)).not.toContain('qualification-secret-message'); + expect(JSON.stringify(final)).not.toContain(SECRET_COMMAND); + expect(JSON.stringify(final)).not.toContain(SECRET_OPERATION); + expect(JSON.stringify(final)).not.toContain(SECRET_PROJECT); + expect(JSON.stringify(final)).not.toContain(SECRET_PHASE); + expect(JSON.stringify(final)).not.toContain(SECRET_PARENT_SESSION); + expect(JSON.stringify(final)).not.toContain(SECRET_PARENT_OPERATION); + }); + + it('ignores secret lifecycle context in AI status and final scope', () => { + const secretEnvelope: Partial> = { + parentSessionId: SECRET_PARENT_SESSION, + parentOperationId: SECRET_PARENT_OPERATION, + sourceSequence: 7, + scope: { + commandName: SECRET_COMMAND, + operationId: SECRET_OPERATION, + projectName: SECRET_PROJECT, + phaseName: SECRET_PHASE + }, + privacy: 'secret' + }; + const { records, final } = run([ + { + ...ev('commandStarted', { commandName: SECRET_COMMAND }), + ...secretEnvelope + } as IReporterEventEnvelope, + { + ...ev('operationRegistered', { + operationId: SECRET_OPERATION, + projectName: SECRET_PROJECT, + phaseName: SECRET_PHASE + }), + ...secretEnvelope + } as IReporterEventEnvelope, + { + ...ev('operationCompleted', { + operationId: SECRET_OPERATION, + status: 'failure' + }), + ...secretEnvelope + } as IReporterEventEnvelope, + ev('commandResult', { commandName: 'build', succeeded: false, exitCode: 1 }) + ]); + + expect(records.filter(({ kind }) => kind === 'ai.status')).toEqual([]); + expect(final.scope).toEqual({ failedProjects: [] }); + expect(final.operationCounts).toEqual({}); + const serialized: string = JSON.stringify(records); + expect(serialized).not.toContain(SECRET_COMMAND); + expect(serialized).not.toContain(SECRET_OPERATION); + expect(serialized).not.toContain(SECRET_PROJECT); + expect(serialized).not.toContain(SECRET_PHASE); + expect(serialized).not.toContain(SECRET_PARENT_SESSION); + expect(serialized).not.toContain(SECRET_PARENT_OPERATION); }); it('counts fallback errors even when detailed diagnostics are disabled', async () => { @@ -581,6 +736,14 @@ describe('AiReporter', () => { severity: 'error', summary: 'qualification-fake-secret-token' }), + parentSessionId: SECRET_PARENT_SESSION, + parentOperationId: SECRET_PARENT_OPERATION, + scope: { + commandName: SECRET_COMMAND, + operationId: SECRET_OPERATION, + projectName: SECRET_PROJECT, + phaseName: SECRET_PHASE + }, privacy: 'secret' }, ev('commandResult', { commandName: 'build', succeeded: false, exitCode: 1 }) @@ -592,6 +755,12 @@ describe('AiReporter', () => { expect(final.diagnostics).toEqual([]); expect(final.truncated).toBe(true); expect(JSON.stringify(final)).not.toContain('qualification-fake-secret-token'); + expect(JSON.stringify(final)).not.toContain(SECRET_COMMAND); + expect(JSON.stringify(final)).not.toContain(SECRET_OPERATION); + expect(JSON.stringify(final)).not.toContain(SECRET_PROJECT); + expect(JSON.stringify(final)).not.toContain(SECRET_PHASE); + expect(JSON.stringify(final)).not.toContain(SECRET_PARENT_SESSION); + expect(JSON.stringify(final)).not.toContain(SECRET_PARENT_OPERATION); }); it('preserves fallback errors when secret diagnostics are also suppressed', () => {