test: retry recursive teardown removals — kill a class of phantom failures

tests/scripts/kb-smoke.cjs

@@ -137,7 +137,7 @@ async function main() {
   check('loaded store serves hybrid', afterLoadHyb.length > 0);
 
   console.log('\n' + (failures === 0 ? 'ALL PASSED' : failures + ' FAILURES'));
-  fs.rmSync(dir, { recursive: true, force: true });
+  fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   process.exit(failures === 0 ? 0 : 1);
 }
 

tests/scripts/test-conventions-lint.cjs

@@ -756,7 +756,7 @@ function withTemp(fn) {
   try {
     return fn(dir);
   } finally {
-    fs.rmSync(dir, { recursive: true, force: true });
+    fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
   }
 }
 

tests/scripts/test-engine-agent-state.cjs

@@ -228,7 +228,7 @@ describe('engine agent — lifecycle store', () => {
   it('deleting the topic cache dir is a complete cleanse — state is colocated', () => {
     runJson(dir, ['dispatch', 'pay', 'research', 'alpha', '--kind', 'review']);
     runJson(dir, ['dispatch', 'pay', 'research', 'beta', '--kind', 'review']);
-    fs.rmSync(path.join(dir, '.workflows', '.cache', 'pay', 'research', 'alpha'), { recursive: true, force: true });
+    fs.rmSync(path.join(dir, '.workflows', '.cache', 'pay', 'research', 'alpha'), { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
     assert.deepStrictEqual(runJson(dir, ['scan', 'pay', 'research', 'alpha']).in_flight, [],
       'the restart rm -rf removes rows with the content');
     assert.deepStrictEqual(runJson(dir, ['scan', 'pay', 'research', 'beta']).in_flight.map((r) => r.id), ['review-001'],

tests/scripts/test-engine-boot.cjs

@@ -153,7 +153,7 @@ function knowledgeCalls(project) {
 describe('engine boot', () => {
   let fix;
   beforeEach(() => { fix = setupSkillsFixture(); });
-  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   it('happy path: no pending migrations, knowledge ready — compact runs', () => {
     const res = runEngine(fix.engine, fix.project, ['boot'], { STUB_CHECK: 'ready' });
@@ -297,7 +297,7 @@ describe('engine boot system-config detection', () => {
     home = path.join(fix.root, 'home');
     fs.mkdirSync(home, { recursive: true });
   });
-  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   function writeSystemConfig(content) {
     writeFile(home, '.config/workflows/config.json', content);
@@ -368,7 +368,7 @@ describe('engine boot (real scripts)', () => {
     root = fs.mkdtempSync(path.join(os.tmpdir(), 'engine-boot-real-'));
     project = setupProject(root);
   });
-  afterEach(() => { fs.rmSync(root, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   it('runs the real migrate.cjs and knowledge CLI against an isolated project', async () => {
     const first = runEngine(REAL_ENGINE, project, ['boot']);
@@ -418,7 +418,7 @@ describe('engine commit --workflows', () => {
     root = fs.mkdtempSync(path.join(os.tmpdir(), 'engine-boot-commit-'));
     project = setupProject(root);
   });
-  afterEach(() => { fs.rmSync(root, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   it('stages the whole .workflows tree — many work units plus .state — and commits', () => {
     writeFile(project, '.workflows/payments/manifest.json', '{"name":"payments","v":2}\n');

tests/scripts/test-engine-discovery-session.cjs

@@ -153,7 +153,7 @@ const CLOSE = ['discovery-session', 'close', 'payments', '-m', 'discovery(paymen
 
 describe('engine discovery-session close — happy path', () => {
   let fix;
-  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   it('clears the marker, indexes the closed log, and commits the session dirt with the caller message', () => {
     fix = setupFixture();
@@ -225,7 +225,7 @@ describe('engine discovery-session close — happy path', () => {
 
 describe('engine discovery-session close — guards refuse loudly, everything pristine', () => {
   let fix;
-  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   /** Assert the refusal leaves every `.workflows/` byte identical, no commit, no KB call. */
   function refusedPristine(args, pattern) {
@@ -279,7 +279,7 @@ function closedEpicManifest() {
 
 describe('engine discovery-session open — happy path', () => {
   let fix;
-  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   it('allocates past the highest on-disk log, moves the draft into place, sets the marker, and does NOT commit', () => {
     fix = setupFixture({ epic: closedEpicManifest() });
@@ -356,7 +356,7 @@ describe('engine discovery-session open — happy path', () => {
 
 describe('engine discovery-session open — guards refuse loudly, everything pristine', () => {
   let fix;
-  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   /** Assert the refusal leaves every `.workflows/` byte identical (the draft included), no commit, no KB call. */
   function refusedPristine(args, pattern) {

tests/scripts/test-engine-manifest-fields.cjs

@@ -76,7 +76,7 @@ function readWorkUnit(dir, name) {
 describe('engine manifest — reads keep the CLI stdout contract', () => {
   let dir;
   beforeEach(() => { dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'engine-fields-'))); });
-  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   it('get: scalar raw, subtree pretty JSON, missing path empty exit 0', () => {
     writeWorkUnit(dir, 'auth', 'feature');
@@ -164,7 +164,7 @@ describe('engine manifest — reads keep the CLI stdout contract', () => {
 describe('engine manifest — mutations answer with the engine JSON contract', () => {
   let dir;
   beforeEach(() => { dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'engine-fields-'))); });
-  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   it('set: one-line JSON listing what was written', () => {
     writeWorkUnit(dir, 'auth', 'feature');
@@ -323,7 +323,7 @@ describe('engine manifest — mutations answer with the engine JSON contract', (
 describe('engine manifest — the batched set holds the work-unit lock', () => {
   let dir;
   beforeEach(() => { dir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'engine-fields-'))); });
-  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
 
@@ -367,7 +367,7 @@ describe('engine manifest apply — the batch form of set/delete (D7)', () => {
       },
     });
   });
-  afterEach(() => fs.rmSync(dir, { recursive: true, force: true }));
+  afterEach(() => fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }));
 
   function payload(ops) {
     const p = path.join(dir, 'ops.json');
@@ -443,7 +443,7 @@ describe('engine manifest set — two grammars, never mixed', () => {
     dir = fs.mkdtempSync(path.join(os.tmpdir(), 'manifest-grammar-'));
     writeWorkUnit(dir, 'auth', 'feature');
   });
-  afterEach(() => fs.rmSync(dir, { recursive: true, force: true }));
+  afterEach(() => fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }));
 
   it('uniform batch: every pair assigned, first pair included', () => {
     const res = runJson(dir, ['set', 'auth', 'status=completed', 'note=hello']);
@@ -481,7 +481,7 @@ describe('staging, candidate, and tracking state is vocabulary-guarded', () => {
       discovery: { items: {} },
     } });
   });
-  afterEach(() => fs.rmSync(dir, { recursive: true, force: true }));
+  afterEach(() => fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }));
 
   it('staging task decisions take only the gate vocabulary', () => {
     runJson(dir, ['set', 'pay.review.pay', 'staging.c1.gate_mode=gated', 'staging.c1.tasks.1=pending', 'staging.c1.tasks.2=pending']);
@@ -536,7 +536,7 @@ describe('work-unit-level fields never shadow the phases tree', () => {
     dir = fs.mkdtempSync(path.join(os.tmpdir(), 'shadow-guard-'));
     writeWorkUnit(dir, 'pay', 'feature');
   });
-  afterEach(() => fs.rmSync(dir, { recursive: true, force: true }));
+  afterEach(() => fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }));
 
   it('set refuses a phase-name-headed field in both grammars', () => {
     const err = runFails(dir, ['set', 'pay', 'specification.foo', 'bar']);
@@ -582,7 +582,7 @@ describe('storage_paths is guarded at write time — set and apply', () => {
     dir = fs.mkdtempSync(path.join(os.tmpdir(), 'storage-paths-'));
     writeWorkUnit(dir, 'pay', 'feature', { phases: { planning: { items: { pay: { status: 'in-progress' } } } } });
   });
-  afterEach(() => fs.rmSync(dir, { recursive: true, force: true }));
+  afterEach(() => fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }));
 
   it('set refuses traversal, absolute, dot, and non-array values; a legal array lands', () => {
     for (const bad of ['\'["../evil"]\'', '\'["/abs"]\'', '\'["."]\'', '\'[""]\'', '"nope"']) {

tests/scripts/test-engine-manifest-io.cjs

@@ -85,7 +85,7 @@ function sleep(ms) {
 describe('manifest-io — shared lock constants and IO contract', () => {
   let dir;
   beforeEach(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'manifest-io-')); });
-  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   it('carries the shared lock discipline: 30s stale, 50ms retry, 10s timeout, 60s tmp orphan', () => {
     assert.strictEqual(io.LOCK_STALE_MS, 30000);
@@ -205,7 +205,7 @@ describe('stale-lock break is atomic — one winner, mutual exclusion holds', ()
   const IO_PATH = path.join(__dirname, '../../skills/workflow-engine/scripts/kernel/manifest-io.cjs');
   let dir;
   beforeEach(() => { dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lock-break-')); });
-  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   /** Spawn a node child running `script` with argv, resolving with its exit code. */
   function child(script, args) {
@@ -289,7 +289,7 @@ describe('structurally corrupt manifests refuse writes', () => {
     dir = fs.mkdtempSync(path.join(os.tmpdir(), 'engine-root-'));
     fs.mkdirSync(path.join(dir, '.workflows/unit'), { recursive: true });
   });
-  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   /** Run the engine expecting `{ok:false}` exit 1; returns the parsed stderr JSON. */
   function engineFail(args) {
@@ -376,7 +376,7 @@ describe('structurally corrupt manifests refuse writes', () => {
 describe('engine writes take the work-unit lock', () => {
   let dir;
   beforeEach(() => { dir = setupGitFixture(); writeEpicFixture(dir); });
-  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   it('a stale .lock is broken per the constants: the write proceeds, the lock is cleaned up', () => {
     const lock = path.join(dir, '.workflows/payments/.lock');
@@ -436,7 +436,7 @@ describe('engine project-manifest writes take the project lock', () => {
     git(dir, ['add', '-A']);
     git(dir, ['commit', '-q', '-m', 'init']);
   });
-  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   it('a stale .project-lock is broken: create registers and cleans up', () => {
     const lock = path.join(dir, '.workflows/.project-lock');

tests/scripts/test-engine-render-surfaces.cjs

@@ -14,7 +14,7 @@ function setup() {
   return fs.mkdtempSync(path.join(os.tmpdir(), 'render-surfaces-'));
 }
 function teardown(dir) {
-  fs.rmSync(dir, { recursive: true, force: true });
+  fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
 }
 function writeManifest(dir, name, data) {
   const mdir = path.join(dir, '.workflows', name);

tests/scripts/test-engine-transactions.cjs

@@ -1167,7 +1167,7 @@ describe('engine commit', () => {
     fs.writeFileSync(path.join(dir, '.workflows/payments/manifest.json'), JSON.stringify(m, null, 2) + '\n');
     writeFile(dir, '.tick/tasks-auth.jsonl', '{"id":"auth-1-1"}\n');
     commitAll(dir, 'seed tick storage');
-    fs.rmSync(path.join(dir, '.tick'), { recursive: true, force: true });
+    fs.rmSync(path.join(dir, '.tick'), { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
 
     const res = engine(dir, ['commit', 'payments', '-m', 'planning(payments): restart plan', '--plan', 'auth']);
     assert.strictEqual(res.committed, shortHead(dir));
@@ -1231,7 +1231,7 @@ describe('knowledge store rides along on every engine commit', () => {
   });
 
   it('exists-guarded: no .knowledge directory, no pathspec, no git error', () => {
-    fs.rmSync(path.join(dir, '.workflows/.knowledge'), { recursive: true, force: true });
+    fs.rmSync(path.join(dir, '.workflows/.knowledge'), { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
     git(dir, ['add', '-A']);
     git(dir, ['commit', '-q', '-m', 'drop store']);
     writeFile(dir, '.workflows/payments/discussion/auth.md', '# Auth\n');

tests/scripts/test-engine-workunit-absorb.cjs

@@ -175,7 +175,7 @@ const ABSORB = ['workunit', 'absorb', 'auth-flow', '--into', 'payments', '--topi
 
 describe('engine workunit absorb — happy path', () => {
   let fix;
-  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   it('moves everything, mirrors statuses, deletes the feature, commits all three pathspecs once', () => {
     fix = setupFixture();
@@ -305,7 +305,7 @@ describe('engine workunit absorb — happy path', () => {
 
 describe('engine workunit absorb — guards refuse loudly, both work units pristine', () => {
   let fix;
-  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true }); });
+  afterEach(() => { fs.rmSync(fix.root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 
   /** Assert the refusal leaves every `.workflows/` byte identical, no commit. */
   function refusedPristine(args, pattern) {
@@ -335,7 +335,7 @@ describe('engine workunit absorb — guards refuse loudly, both work units prist
     const feature = featureManifest({ phases: { research: { items: { exploration: { status: 'completed' } } } } });
     fix = setupFixture({ feature });
     refusedPristine(ABSORB, /has no discussion — absorb moves the discussion in/);
-    fs.rmSync(fix.root, { recursive: true, force: true });
+    fs.rmSync(fix.root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
 
     fix = setupFixture();
     fs.rmSync(path.join(fix.project, '.workflows/auth-flow/discussion/auth-flow.md'));
@@ -349,7 +349,7 @@ describe('engine workunit absorb — guards refuse loudly, both work units prist
     withSpec.phases.specification = { items: { 'auth-flow': { status: 'in-progress' } } };
     fix = setupFixture({ feature: withSpec });
     refusedPristine(ABSORB, /has specification work — absorb is only for features before specification/);
-    fs.rmSync(fix.root, { recursive: true, force: true });
+    fs.rmSync(fix.root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
 
     const withPlan = featureManifest();
     withPlan.phases.planning = { items: {} };