Add deterministic AI reporter qualification gates

common/changes/@rushstack/rush-reporter/copilot-reporter-r8a-ai-gates_2026-08-28-08-45.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/rush-reporter",
+      "comment": "Add adversarial deterministic AI reporter qualification gates and privacy-safe actionable context.",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@rushstack/rush-reporter",
+  "email": "TheLarkInn@users.noreply.github.com"
+}

common/reviews/api/rush-reporter.api.md

@@ -7,6 +7,12 @@
 import type { Readable } from 'node:stream';
 import type { Writable } from 'node:stream';
 
+// @beta
+export const AI_REPORTER_QUALIFICATION_SCHEMA_VERSION: '1.0';
+
+// @beta
+export const AI_REPORTER_QUALIFICATION_THRESHOLDS: IAiReporterQualificationThresholds;
+
 // @beta
 export class AiReporter implements IReporter {
     constructor(options: IAiReporterOptions);
@@ -139,6 +145,9 @@ export function detectAgent(env: Record<string, string | undefined>, configuredV
 // @beta
 export function encodeNdjsonRecord(value: unknown, options?: INdjsonOptions): string;
 
+// @beta
+export function evaluateAiReporterQualification(cases: readonly IAiReporterQualificationCaseResult[], thresholds?: IAiReporterQualificationThresholds): IAiReporterQualificationResult;
+
 // @beta
 export function evaluatePluginApplyGate(manifests: readonly IRushPluginManifest[], options: IPluginApplyGateOptions): IPluginApplyDecision[];
 
@@ -170,6 +179,9 @@ export class FileReporter implements IReporter {
 // @beta
 export function filterEventsForLogLevel(logLevel: ReporterLogLevel, events: readonly IReporterEventEnvelope<unknown>[]): IReporterEventEnvelope<unknown>[];
 
+// @beta
+export function formatAiReporterQualificationFailures(result: IAiReporterQualificationResult): string;
+
 // @beta
 export function getBlockedPlugins(decisions: readonly IPluginApplyDecision[]): IPluginApplyDecision[];
 
@@ -182,6 +194,9 @@ export function getLogLevelRank(level: ReporterLogLevel): number;
 // @beta
 export function getPrivacyClassificationRank(classification: ReporterPrivacyClassification): number;
 
+// @beta
+export function getQualifiedAiReporterDecision(env: Record<string, string | undefined>, configuredAgentEnvironmentVariables: readonly string[], qualification: IAiReporterQualificationResult | undefined, privacyPrerequisiteAccepted?: boolean): IQualifiedAiReporterDecision;
+
 // @beta
 export function getReporterMigrationPhase(id: ReporterMigrationPhaseId): IReporterMigrationPhase;
 
@@ -219,11 +234,19 @@ export interface IAiDiagnostic {
     // (undocumented)
     readonly code: string;
     // (undocumented)
+    readonly context?: Readonly<Record<string, ReporterJsonValue | '[local-sensitive]' | '[secret]'>>;
+    // (undocumented)
+    readonly detailKey?: string;
+    // (undocumented)
+    readonly diagnosticId?: string;
+    // (undocumented)
     readonly remediation?: readonly IRushRemediationAction[];
     // (undocumented)
     readonly severity: string;
     // (undocumented)
     readonly summary?: string;
+    // (undocumented)
+    readonly summaryKey?: string;
 }
 
 // @beta
@@ -280,6 +303,98 @@ export interface IAiReporterOptions {
     readonly write: (text: string) => void;
 }
 
+// @beta
+export interface IAiReporterQualificationCaseResult {
+    // (undocumented)
+    readonly actionable: boolean;
+    // (undocumented)
+    readonly aiOutputBytes: number;
+    // (undocumented)
+    readonly deterministic: boolean;
+    // (undocumented)
+    readonly expectedResult: 'succeeded' | 'failed';
+    // (undocumented)
+    readonly failures: readonly string[];
+    // (undocumented)
+    readonly fullLogValid: boolean;
+    // (undocumented)
+    readonly legacyOutputBytes: number;
+    // (undocumented)
+    readonly name: string;
+    // (undocumented)
+    readonly normalizedAiOutputSha256: string;
+    // (undocumented)
+    readonly plaintextOutputBytes: number;
+    // (undocumented)
+    readonly privacySafe: boolean;
+    // (undocumented)
+    readonly scenario: string;
+    // (undocumented)
+    readonly stdoutContractValid: boolean;
+    // (undocumented)
+    readonly warningContractValid: boolean;
+}
+
+// @beta
+export interface IAiReporterQualificationGateResult {
+    // (undocumented)
+    readonly actual: number;
+    // (undocumented)
+    readonly failedCases: readonly string[];
+    // (undocumented)
+    readonly id: string;
+    // (undocumented)
+    readonly passed: boolean;
+    // (undocumented)
+    readonly threshold: string;
+}
+
+// @beta
+export interface IAiReporterQualificationResult {
+    // (undocumented)
+    readonly cases: readonly IAiReporterQualificationCaseResult[];
+    // (undocumented)
+    readonly gates: readonly IAiReporterQualificationGateResult[];
+    // (undocumented)
+    readonly passed: boolean;
+    // (undocumented)
+    readonly schemaVersion: typeof AI_REPORTER_QUALIFICATION_SCHEMA_VERSION;
+    // (undocumented)
+    readonly thresholds: IAiReporterQualificationThresholds;
+}
+
+// @beta
+export interface IAiReporterQualificationThresholds {
+    // (undocumented)
+    readonly deterministicRunCount: number;
+    // (undocumented)
+    readonly maximumAggregateAiToLegacyPercent: number;
+    // (undocumented)
+    readonly maximumAggregateAiToPlaintextPercent: number;
+    // (undocumented)
+    readonly maximumCompactCaseAiOutputBytes: number;
+    // (undocumented)
+    readonly maximumOutputBytesPerCase: number;
+    // (undocumented)
+    readonly maximumPerCaseAiToBaselinePercent: number;
+    // (undocumented)
+    readonly minimumActionableFailurePercent: number;
+    // (undocumented)
+    readonly minimumComparableBaselineBytes: number;
+    // (undocumented)
+    readonly minimumControlCases: number;
+    // (undocumented)
+    readonly minimumFailureCases: number;
+    // (undocumented)
+    readonly minimumFullLogPassPercent: number;
+    // (undocumented)
+    readonly minimumPrivacyPassPercent: number;
+    // (undocumented)
+    readonly minimumStdoutContractPassPercent: number;
+    // (undocumented)
+    readonly minimumWarningContractPassPercent: number;
+}
+
 // @beta
 export interface IAutomaticReporterPlan {
     readonly emergencyDestination: 'stderr';
@@ -702,6 +817,18 @@ export interface IProblemMatcherResult {
     readonly unmatchedLineCount: number;
 }
 
+// @beta
+export interface IQualifiedAiReporterDecision {
+    // (undocumented)
+    readonly agentDetected: boolean;
+    // (undocumented)
+    readonly eligible: boolean;
+    // (undocumented)
+    readonly reason: 'RUSH_REPORTER=legacy' | 'agent not detected' | 'qualification unavailable' | 'qualification failed' | 'privacy prerequisite unavailable' | 'qualified';
+    // (undocumented)
+    readonly reporter?: 'ai';
+}
+
 // @beta
 export interface IRenderLiveRegionOptions {
     readonly color: IColorizer;
@@ -1468,6 +1595,9 @@ export function resolveReporterCompatibility(frontend: IReporterFrontendDescript
 // @beta
 export function resolveReporterSelection(input: IReporterSelectionInput): IReporterSelection;
 
+// @beta
+export function runAiReporterQualificationCorpusAsync(): Promise<IAiReporterQualificationResult>;
+
 // @beta
 export function runProblemMatchers(events: readonly IReporterEventEnvelope<unknown>[], matchers: readonly IProblemMatcher[], options?: IRunProblemMatchersOptions): IProblemMatcherResult;
 

libraries/reporter/README.md

@@ -4,6 +4,38 @@ Canonical event protocol, reporter manager, and built-in reporters for Rush.
 
 This package is released as a public beta. Exported contracts may change before the stable release.
 
+## AI reporter qualification
+
+The network-free qualification corpus runs representative bootstrap/version, configuration, input,
+dependency-tool, operation, cache, network/auth, plugin, cancellation, and internal failures plus
+successful and warning-only controls through the AI, detailed plaintext, legacy, and full-log reporters.
+Scenario-specific external output is included only where the real failure or control would produce it.
+
+| Gate | Blocking threshold |
+| --- | --- |
+| Failure/control coverage | At least 10 failure cases and 2 successful controls |
+| Actionability | 100% of failures retain stable code, category, context, and remediation |
+| Output size | At most 64 KiB per case; compact cases at most 2 KiB; AI no larger than comparable per-case baselines; aggregate AI bytes at most 50% of legacy and plaintext |
+| Determinism | Byte-identical normalized AI output across 3 runs |
+| Privacy | 100% secret redaction and no private producer identity leakage |
+| Full log | 100% absolute, existing, owner-only where supported, complete, and failure-correlated |
+| Stdout/warnings | 100% payload-only NDJSON and warning suppression/detail compliance |
+
+Run `rushx build && node scripts/runAiReporterQualification.js` from this project to print the
+machine-readable result. Machine-specific paths are normalized before hashing and are not stored. Passing
+these gates only produces a reusable qualification decision; it does not enable environment-based automatic
+reporter selection. That decision also requires the separate telemetry privacy prerequisite to be accepted.
+The pre-major Rush frontend remains explicit/repository-opt-in, and `RUSH_REPORTER=legacy` remains
+authoritative.
+
+AI fallback message text is emitted only for public envelopes. Non-public fallback errors remain countable
+and refer to the protected full-detail log. JSON oversized-record markers preserve the original privacy
+classification and omit non-public source and scope metadata.
+
+Secret envelopes retain only protocol, event identity, ordering, timing, type, privacy, and fully redacted
+source and payload fields. Contextual parent, command, operation, project, phase, and scope metadata is
+removed.
+
 ## Links
 
 - [CHANGELOG.md](https://github.com/microsoft/rushstack/blob/main/libraries/reporter/CHANGELOG.md) - Find out

libraries/reporter/scripts/runAiReporterQualification.js

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+const {
+  formatAiReporterQualificationFailures,
+  runAiReporterQualificationCorpusAsync
+} = require('../lib-commonjs');
+
+runAiReporterQualificationCorpusAsync()
+  .then((result) => {
+    process.stdout.write(`${JSON.stringify(result, undefined, 2)}\n`);
+    if (!result.passed) {
+      process.stderr.write(`${formatAiReporterQualificationFailures(result)}\n`);
+      process.exitCode = 1;
+    }
+  })
+  .catch((error) => {
+    process.stderr.write(`${error.stack || error.message}\n`);
+    process.exitCode = 1;
+  });

libraries/reporter/src/index.ts

@@ -337,6 +337,22 @@ export {
   isWithinMemoryBudget
 } from './perf/PerformanceBudgets';
 
+export type {
+  IAiReporterQualificationThresholds,
+  IAiReporterQualificationCaseResult,
+  IAiReporterQualificationGateResult,
+  IAiReporterQualificationResult,
+  IQualifiedAiReporterDecision
+} from './qualification/AiReporterQualification';
+export {
+  AI_REPORTER_QUALIFICATION_SCHEMA_VERSION,
+  AI_REPORTER_QUALIFICATION_THRESHOLDS,
+  evaluateAiReporterQualification,
+  formatAiReporterQualificationFailures,
+  getQualifiedAiReporterDecision
+} from './qualification/AiReporterQualification';
+export { runAiReporterQualificationCorpusAsync } from './qualification/AiReporterQualificationCorpus';
+
 export type { ReporterMigrationPhaseId, IReporterMigrationPhase } from './migration/MigrationPhase';
 export { REPORTER_MIGRATION_PHASES, getReporterMigrationPhase } from './migration/MigrationPhase';
 export type {