refactor: consolidate design policy and remove obsolete analysis

Author: Bill Leoutsakos

design-diff.config.json

@@ -111,8 +111,7 @@
       "list": "apps/docs/lib/openapi-specs.ts",
       "export": "OPENAPI_SPEC_FILES",
       "root": "apps/docs",
-      "renderer": "apps/docs/lib/openapi.ts",
-      "contentOnly": true
+      "renderer": "apps/docs/lib/openapi.ts"
     }
   ],
   "environmentAdapters": [

scripts/design-diff/README.md

@@ -133,8 +133,8 @@ Large opaque resolver trees can themselves exceed the supported analysis budget;
 uncertainty, not evidence of an appearance edit.
 
 Operational failures remain distinct from completed policy exemptions. Configured file-loaded
-OpenAPI inputs are validated as Git data; content is exempt and malformed inputs retain diagnostic
-evidence. Lockfile rendering-dependency changes retain diagnostics but do not qualify by themselves.
+OpenAPI inputs in the head revision are validated as Git data; content is exempt and malformed
+inputs retain diagnostic evidence. Lockfile rendering-dependency changes retain diagnostics but do not qualify by themselves.
 
 ## Files
 
@@ -151,7 +151,7 @@ scripts/design-diff/
   finite.ts, environment.ts            Finite key and environment projections
   appearance.ts                       Supported appearance evidence projection
   tailwind.ts                         Pinned compiler and trusted class helpers
-  compare.ts, policy.ts, movement.ts   Matching, decisions and categories
+  compare.ts, policy.ts                Matching, decisions and categories
   group.ts, report.ts, semantic.ts     Grouping, full hashes and bounded JSON
   inputs.ts                           Configured file inputs
   infrastructure.ts                   Rendering dependency diagnostics
@@ -200,8 +200,10 @@ Fixtures stay inside test strings/JSON so application builds and styling scans d
 
 The frozen manifest contains the original 120 comparisons plus 60 later sampled holdouts. Its
 original labels use a broader visual/content policy and must not be treated as policy-5 ground
-truth. Preserve original labels and document policy-specific review separately. The prior policy-4
-run completed 178/180 comparisons, flagging 112; two timed out. That is historical context only.
+truth. Both cohorts have now informed development and are no longer unseen validation. Keep the
+engine and configuration fixed while evaluating the next sample, label it independently, and
+report disagreements before changing rules. The production CLI never reads benchmark labels or PR
+numbers to make decisions; repository conventions belong in configuration, not per-PR exceptions.
 
 The runner verifies exact commits and GitHub file sets. Its cache identity includes immutable
 engine SHA, configuration, lockfile, runtime and comparison commits. It records elapsed time,

scripts/design-diff/analyze.ts

@@ -7,7 +7,7 @@ import { extractTsx } from '#design-diff/extract/tsx'
 import { GitReader } from '#design-diff/git'
 import { groupFindings } from '#design-diff/group'
 import { renderingLock } from '#design-diff/infrastructure'
-import { fileLoadedInputs } from '#design-diff/inputs'
+import { fileLoadedInputDiagnostics } from '#design-diff/inputs'
 import { reclaimMemory } from '#design-diff/memory'
 import { limitations } from '#design-diff/policy'
 import { Resolver } from '#design-diff/resolve'
@@ -165,15 +165,6 @@ export async function analyze(
                 'Native menus, palettes and embedded rendering need review'
               )
             )
-          for (const definition of defs) {
-            if (
-              definition.movement &&
-              [...tree.texts].some(
-                ([name, text]) => name.endsWith('.css') && /\b(?:svg|rect|circle)\b|\*/.test(text)
-              )
-            )
-              definition.movement = undefined
-          }
           const normalized: Definition[] = []
           for (const definition of defs) normalized.push(await tailwind.normalize(definition))
           return normalized
@@ -250,7 +241,7 @@ export async function analyze(
           ...b.flatMap((definition) => definition.unresolved),
         ]),
       ].sort()
-      findings.push(...compareDefinitions(a, b, affected))
+      findings.push(...compareDefinitions(a, b))
       if (!changed.has(file) && (a.length || b.length)) indirectExamples++
       if (
         changed.has(file) &&
@@ -271,8 +262,7 @@ export async function analyze(
                   after.entries.get(file)?.oid ?? '',
                   'Rendering infrastructure is not executed'
                 )
-              : undefined,
-            'Rendering infrastructure changed'
+              : undefined
           )
         )
       }
@@ -350,7 +340,7 @@ export async function analyze(
         )
     }
   }
-  findings.push(...fileLoadedInputs(before, after, config))
+  findings.push(...fileLoadedInputDiagnostics(after, config))
   if (findings.length && !before.graph) {
     before.buildGraph()
     after.buildGraph()

scripts/design-diff/compare.ts

@@ -1,6 +1,5 @@
 import { createHash } from 'node:crypto'
 import { appearanceValue } from '#design-diff/appearance'
-import { pureMovement } from '#design-diff/movement'
 import { changedCategory } from '#design-diff/policy'
 import { previewChange } from '#design-diff/report'
 import type { Change, Definition } from '#design-diff/types'
@@ -9,44 +8,27 @@ function signature(definition: Definition) {
   return JSON.stringify([definition.value, definition.conditions])
 }
 
-export function finding(
-  before: Definition | undefined,
-  after: Definition | undefined,
-  reason?: string
-): Change {
+export function finding(before: Definition | undefined, after: Definition | undefined): Change {
   const definition = after ?? before
   if (!definition) throw new Error('Finding needs evidence')
   const unresolved = [
     ...new Set([...(before?.unresolved ?? []), ...(after?.unresolved ?? [])]),
   ].sort()
-  const movement = before && after && pureMovement(before, after)
   const category = changedCategory(before, after)
-  const uncertain =
-    unresolved.length ||
-    definition.kind === 'review' ||
-    ['movement', 'unresolved'].includes(category)
   const a = before && appearanceValue(before)
   const b = after && appearanceValue(after)
   const supported =
     (!before || a !== undefined) &&
     (!after || b !== undefined) &&
     (a !== undefined || b !== undefined)
   const changed = JSON.stringify(a) !== JSON.stringify(b)
-  const flag = supported && changed && !movement && category !== 'movement'
+  const flag = supported && changed && category !== 'movement'
   const result: Omit<Change, 'id'> = {
     decision: flag ? 'flag' : 'exempt',
-    category: movement ? 'movement' : category,
-    reason:
-      (flag
-        ? 'Supported authored appearance values changed'
-        : !movement
-          ? 'No established change to authored appearance under the designer policy'
-          : reason) ??
-      (movement
-        ? 'Static geometry establishes movement within unchanged bounds'
-        : uncertain
-          ? 'Potential visual effect; static evidence is incomplete'
-          : 'Visual definition changed'),
+    category,
+    reason: flag
+      ? 'Supported authored appearance values changed'
+      : 'No established change to authored appearance under the designer policy',
     before: before
       ? {
           value: before.value,
@@ -84,11 +66,7 @@ export function finding(
   })
 }
 
-export function compareDefinitions(
-  before: Definition[],
-  after: Definition[],
-  _changed?: Set<string>
-): Change[] {
+export function compareDefinitions(before: Definition[], after: Definition[]): Change[] {
   const signatures = new Map<Definition, string | undefined>()
   const appearance = (definition: Definition) => {
     if (!signatures.has(definition))

scripts/design-diff/extract/tsx.ts

@@ -2,7 +2,6 @@ import type { NodePath } from '@babel/traverse'
 import * as t from '@babel/types'
 import { appearanceAttributes, mediaElement } from '#design-diff/appearance'
 import { location, propertyName, symbolName, traverse } from '#design-diff/ast'
-import { svgMovement } from '#design-diff/movement'
 import { child, children, object, type Resolver } from '#design-diff/resolve'
 import type { Data, Definition, Evidence } from '#design-diff/types'
 
@@ -79,7 +78,6 @@ export function extractTsx(resolver: Resolver, file: string): Definition[] {
       symbol,
       location: location(file, path.node),
       conditions,
-      movement: svgMovement(path),
     })
   }
   const literal = (value: Data): Evidence => ({ value, dependencies: [file], unresolved: [] })

scripts/design-diff/group.ts

@@ -96,11 +96,7 @@ export function groupFindings(
           dependencies: [file, reference.location.file],
           unresolved: ['Resolved component reference; its runtime appearance is not established'],
         })
-        example = finding(
-          definition(previous?.oid),
-          definition(next?.oid),
-          'Resolved reference to a changed component'
-        )
+        example = finding(definition(previous?.oid), definition(next?.oid))
       }
     }
     const limitations = new Set([

scripts/design-diff/inputs.ts

@@ -1,30 +1,31 @@
 import path from 'node:path'
-import { canonicalJson } from '#design-diff/ast'
-import { compareDefinitions, finding } from '#design-diff/compare'
+import { finding } from '#design-diff/compare'
 import { Resolver } from '#design-diff/resolve'
 import type { SourceTree } from '#design-diff/source'
-import type { Change, Config, Data, Definition } from '#design-diff/types'
+import type { Change, Config } from '#design-diff/types'
 
-/** Explicit file-loading conventions are data edges, never runtime filesystem reads. */
-export function fileLoadedInputs(before: SourceTree, after: SourceTree, config: Config): Change[] {
-  const read = (
-    tree: SourceTree,
-    input: NonNullable<Config['fileInputs']>[number]
-  ): Definition[] => {
-    if (!tree.entries.has(input.list) && !tree.entries.has(input.renderer)) return []
-    const definitions: Definition[] = []
-    const emit = (file: string, value: Data, unresolved: string[] = []) =>
-      definitions.push({
-        key: file,
-        kind: unresolved.length ? 'review' : 'content',
-        property: 'file-loaded-documentation',
-        value,
-        location: { file, line: 1, column: 1 },
-        symbol: input.export,
-        conditions: [{ renderer: input.renderer, list: input.list }],
-        dependencies: [file, input.list, input.renderer],
-        unresolved,
-      })
+/** Content never qualifies; validate current file-loaded documentation only for coverage notes. */
+export function fileLoadedInputDiagnostics(tree: SourceTree, config: Config): Change[] {
+  const diagnostics = new Map<string, Change>()
+  for (const input of config.fileInputs ?? []) {
+    if (!tree.entries.has(input.list) && !tree.entries.has(input.renderer)) continue
+    const emit = (file: string, reason: string) => {
+      if (diagnostics.has(file)) return
+      diagnostics.set(
+        file,
+        finding(undefined, {
+          key: file,
+          kind: 'review',
+          property: 'file-loaded-documentation',
+          value: tree.entries.get(file)?.oid ?? 'missing',
+          location: { file, line: 1, column: 1 },
+          symbol: input.export,
+          conditions: [{ renderer: input.renderer, list: input.list }],
+          dependencies: [file, input.list, input.renderer],
+          unresolved: [reason],
+        })
+      )
+    }
     try {
       const resolver = new Resolver(tree)
       const exported = resolver.module(input.list).exports.get(input.export)
@@ -37,36 +38,22 @@ export function fileLoadedInputs(before: SourceTree, after: SourceTree, config:
         !evidence.value.every((file) => typeof file === 'string')
       )
         throw new Error('Configured input list is not a static string array')
-      if (!input.contentOnly) emit(input.list, evidence.value)
       for (const name of evidence.value as string[]) {
         const file = path.posix.normalize(path.posix.join(input.root, name))
         if (!file.startsWith(`${input.root}/`) || !file.endsWith('.json'))
           throw new Error('Unsupported configured input path')
         try {
-          const value = JSON.parse(tree.texts.get(file) ?? '')
-          if (!input.contentOnly) emit(file, canonicalJson(value))
+          JSON.parse(tree.texts.get(file) ?? '')
         } catch {
-          emit(file, tree.entries.get(file)?.oid ?? 'missing', [
-            'Configured documentation JSON is missing, malformed or exceeds the source budget',
-          ])
+          emit(
+            file,
+            'Configured documentation JSON is missing, malformed or exceeds the source budget'
+          )
         }
       }
     } catch {
-      emit(input.list, tree.entries.get(input.list)?.oid ?? 'missing', [
-        'Configured documentation list or renderer could not be resolved',
-      ])
+      emit(input.list, 'Configured documentation list or renderer could not be resolved')
     }
-    return definitions
-  }
-  const findings: Change[] = []
-  for (const input of config.fileInputs ?? []) {
-    const a = read(before, input)
-    const b = read(after, input)
-    findings.push(...compareDefinitions(a, b))
-    /** Invalid documentation remains an explicit coverage note without notifying the designer. */
-    for (const definition of b.filter((definition) => definition.unresolved.length))
-      if (!findings.some((finding) => finding.after?.location.file === definition.location.file))
-        findings.push(finding(undefined, definition))
   }
-  return findings
+  return [...diagnostics.values()]
 }