Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 22 additions & 2 deletions src/automation/autonomousRunner.infraError.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import type { TaskItem } from '../orchestration/decisionEngine.js';
import type { AutonomousConfig } from './runnerTypes.js';
import type { ITaskSource } from './taskSource.js';
import { RETRY_RAMP_FROM_ATTEMPT } from './durableRunCoordinator.js';
import { HostDiskGate } from '../support/hostHealth.js';

// Regression for INT-2010: adapter CLI/infra failures (the worker/reviewer never
// ran) must NOT drive a completable issue to durable STUCK. They get a backoff
Expand Down Expand Up @@ -42,14 +43,14 @@ const task = (): TaskItem => ({
createdAt: Date.now(),
});

const result = (finalStatus: PipelineResult['finalStatus']): PipelineResult => ({
const result = (finalStatus: PipelineResult['finalStatus'], error = 'codex CLI failed with code 1'): PipelineResult => ({
success: false,
sessionId: 'pipeline-1',
iterations: 0,
totalDuration: 5,
finalStatus,
stages: [],
workerResult: { success: false, summary: '', filesChanged: [], commands: [], output: '', error: 'codex CLI failed with code 1' },
workerResult: { success: false, summary: '', filesChanged: [], commands: [], output: '', error },
});

function mockTaskSource() {
Expand Down Expand Up @@ -107,6 +108,25 @@ describe('AutonomousRunner infra_error handling (INT-2010)', () => {
expect(history.every((entry: { failureCause?: string }) => entry.failureCause === 'infra')).toBe(true);
});

it('parks dispatch on ENOSPC without scheduling a retry', async () => {
const source = mockTaskSource();
runnerExecution.setTaskSource(source);
const runner = new AutonomousRunner(cfg());
const scheduler = (runner as unknown as { scheduler: TaskScheduler }).scheduler;
const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});

for (let attempt = 0; attempt < 2; attempt++) {
scheduler.startTask(task(), '/repo', async () => result('infra_error', 'tester: git exited with code 128: No space left on device'));
await new Promise((resolve) => setTimeout(resolve, 30));
}

expect((runner as unknown as { failedTaskRetryTimes: Map<string, number> }).failedTaskRetryTimes.has('ISSUE-1')).toBe(false);
expect(errorLog).toHaveBeenCalledWith(expect.stringContaining('stopped dispatch after a host resource failure'));
const gate = new HostDiskGate(() => []);
expect(gate.resourceFailure('No space left on device').notice).toBeDefined();
expect(gate.resourceFailure('No space left on device').notice).toBeUndefined();
});

it('retries a pair-level stuck result with fresh outer context before requiring a human', async () => {
const source = mockTaskSource();
runnerExecution.setTaskSource(source);
Expand Down
9 changes: 8 additions & 1 deletion src/automation/autonomousRunner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ import { SANDBOX_OUTCOME_UNKNOWN_PARK_REASON } from '../sandboxExecutor/protocol
import { INFRA_CIRCUIT_PARK_REASON } from './infraFailureCircuit.js';
import { listHeldIssueIds } from './execHold.js';
import { inputFingerprint, mayRedispatch } from './dispatchFingerprint.js';
import { HostDiskGate, dispatchWritePaths } from '../support/hostHealth.js';
import { HostDiskGate, dispatchWritePaths, isHostResourceError } from '../support/hostHealth.js';
import { VERIFIED_WORKER_BLOCKER_PARK_REASON } from '../agents/pairPipelineTypes.js';
import { decideExplicitReadmission, OPERATOR_QUESTION_PARK_MARKER } from './explicitDispatchReadmission.js';
import { broadcastEvent, type SwarmStats } from '../core/eventHub.js';
Expand Down Expand Up @@ -946,6 +946,13 @@ export class AutonomousRunner {
// edited files) STUCK in production. (INT-2010)
if (result.finalStatus === 'infra_error') {
const detail = pickPipelineFailureDetail(result) || 'infra/CLI execution error';
if (isHostResourceError(detail)) {
const hold = this.diskGate.resourceFailure(detail);
console.error(`[Scheduler] ${taskCtx} stopped dispatch after a host resource failure: ${detail}`);
if (hold.notice) await reportToDiscord(`⚠️ ${hold.notice}`);
this.scheduleNextHeartbeat();
return;
}
if (task.issueId) {
// Fixed mid-range backoff — we intentionally don't bump failure counts,
// so there's no attempt number to scale by.
Expand Down
10 changes: 10 additions & 0 deletions src/support/hostHealth.ts
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,16 @@ export class HostDiskGate {
private readonly minFreeBytes: () => number = () => DEFAULT_MIN_FREE_BYTES,
) {}

/** Latch a host-resource failure into the same pause/one-notice path as a low-space reading. */
resourceFailure(detail: string): DispatchGateDecision {
const first = !this.low;
this.low = true;
const line = `Host resource failure: ${detail}`;
return first
? { pause: true, line, notice: `Dispatch paused: ${line}. New work waits until host resources recover.` }
: { pause: true, line };
}

/** Measure now and decide. A floor of zero or less switches the check off. */
evaluate(): DispatchGateDecision {
const floor = this.minFreeBytes();
Expand Down
Loading