From 9e8904bd3d3ae925b8b8ce6ddf4a473ad00b8cf1 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sun, 25 Oct 2026 22:38:48 -0400 Subject: [PATCH 1/5] fix: preserve CLI reports with durable checkpoints and atomic publication --- README.md | 4 + docs/REFERENCE.md | 28 ++++-- skills/php2shell-check/SKILL.md | 14 +-- test_output_durability.py | 132 +++++++++++++++++++++++++ test_php2shell.py | 2 +- test_wordpress_check.py | 2 +- wordpress_check.py | 164 ++++++++++++++++++++++++++------ 7 files changed, 296 insertions(+), 50 deletions(-) create mode 100644 test_output_durability.py diff --git a/README.md b/README.md index 10dab20..601367e 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,10 @@ Python 3.10+ on Windows, macOS, or Linux. Nothing to install. Use `--logo path/to/logo.png` and `--brand "Your Company"` to brand it for your own organization. CSV and JSON remain available for spreadsheets and pipelines. +File outputs refuse reuse by default; `--force` explicitly permits atomic final replacement. Every completed row is fsynced into `OUTPUT.partial.json` before progress, so an interrupted or force-killed run can be rendered offline with `python wordpress_check.py --recover OUTPUT.partial.json --format html --output recovered.html`. Recovery never requests targets and labels incomplete evidence (`130`). Existing checkpoints are always protected, including with `--force`; choose fresh paths and protect both artifacts. Successful final files are atomically published. Stdout cannot recover after forced termination. + +CLI JSON now has a run envelope with `complete`, completed/pending/total target counts and `results` (previously a top-level array); CSV appends corresponding run metadata and an empty summary, and HTML labels completeness. See [recovery and file handling](docs/REFERENCE.md#file-handling-and-exit-codes) for naming, limits and conservative retained checkpoints. + ## What the results mean | Status | Meaning | diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 8ecbf99..a55a73e 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -129,7 +129,9 @@ are rejected. Input is a text list, **not a CSV or an Excel workbook**. | --- | --- | --- | | `urls` | None | Zero or more HTTP(S) URLs or bare domains. | | `--input PATH` | None | UTF-8 text file containing one website per line. | -| `--output PATH` | stdout | Write results to this file. An existing output file is overwritten. | +| `--output PATH` | stdout | Atomically publish the final report; refuse an existing path by default. Keep a durable `PATH.partial.json` checkpoint. | +| `--force` | off | Explicitly replace an existing final output after successful rendering; never replace an existing checkpoint. | +| `--recover PATH` | none | Render a persisted checkpoint without targets, requests, or automatic retries. | | `--format csv\|json\|html` | `csv` | Output format. `html` writes a self-contained, branded report. | | `--logo PATH` | `assets/logo.png` | Logo inlined into the HTML report. A missing file is omitted. | | `--brand NAME` | `CompleteTech LLC` | Organization name shown on the HTML report. | @@ -203,6 +205,8 @@ the coarse `blocked` value as proof that the owner deliberately blocked you. | `php2shell_notes` | Evidence limits and the reason for this interpretation. | | `php2shell_recommendation` | Administrator review or patch-verification guidance. | +CLI JSON is now a run envelope: `complete`, `completed_targets`, `pending_targets`, `total_targets`, and `results`. Consumers of the previous top-level array must read `results`. The result dictionaries and their evidence arrays are unchanged. CSV appends `record_type` and these four run fields to the existing result columns; an empty report has a summary row. HTML labels run completeness and counts separately from assessment status. + JSON preserves arrays in the evidence, version-candidate, CVE, source, and note fields. CSV joins array values with ` | ` and prefixes potentially dangerous spreadsheet-formula strings with an apostrophe. Empty version strings mean "not disclosed", not version zero. @@ -286,14 +290,22 @@ Do not use `request()` directly to bypass the policy checks in `check()`. ## File handling and exit codes -Results are flushed after each website. CSV retains completed rows after an -interruption; JSON can be left without its closing bracket. HTML is written when the run ends, including after an interruption. There is no automatic -resume or retry. Preserve any partial result file before rerunning with the same -`--output` path, because the CLI opens it for replacement. +With `--output report.html`, the CLI first creates `report.html.partial.json` exclusively. Each completed result is committed to a private temporary file, flushed and fsynced, and atomically replaces that checkpoint before progress is printed. Checkpoints contain completed result dictionaries and explicit conservative incomplete/completed/pending counts, not pending target URLs. They contain assessment evidence and must be protected like the report. Never commit real checkpoints. + +The final JSON, CSV or HTML is rendered completely and fsynced before publication in the destination directory. Default publication uses an atomic hard link and fails closed if hard links are unavailable or another writer creates the destination first. `--force` explicitly allows atomic replacement of the final file; failures before publication preserve its prior contents. Existing checkpoint names are always refused, even with `--force`; choose a fresh output name. Directories and symbolic-link output paths are refused. + +A handled Ctrl-C returns `130`, retains the incomplete checkpoint, and does not publish a final file or replace a previous report. A forced process termination cannot execute cleanup, but previously checkpointed results remain reconstructable. Recovery performs no requests and never reruns completed targets: + +```sh +python wordpress_check.py --recover report.html.partial.json --format html --output recovered.html +python wordpress_check.py --recover report.html.partial.json --format json --output recovered.json +``` + +Recovery returns `130` for an incomplete checkpoint, even if all targets were checkpointed before final publication. The conservative checkpoint is intentionally retained after success and is not final-output attestation. Keep the final report as the completion receipt. Recovery requires a regular non-symlink checkpoint, validates its structure/counts/result types and reads at most 64 MiB (100,000 result rows). Keep both the original checkpoint and any reconstructed artifact; continuation requires a separately authorized new run of pending targets only. The tool never resumes automatically. + +Without `--output`, results accumulate in memory and stdout receives one well-formed report on completion or a handled interrupt. Stdout/pipes do not provide durable recovery after forced termination; use `--output` when preserving evidence matters. Final output/partial names and report/input paths must differ. -Exit code `0` means the run completed, not that every website was reachable or -identified. Inspect each result's status. Argument/input-file errors exit with -code `2`. Unexpected I/O errors may raise a Python exception. +Exit code `0` means the run completed, not that every website was reachable or patched. `130` means interruption or incomplete recovery; `2` means argument/input/recovery/publication error. Inspect operational and assessment status separately. Disk exhaustion or abrupt termination during a new checkpoint leaves the previous atomic checkpoint intact; the in-progress result may not have been committed. No file mechanism guarantees recovery from storage hardware failure. ## Tests and contributing diff --git a/skills/php2shell-check/SKILL.md b/skills/php2shell-check/SKILL.md index 81ce838..eba63cb 100644 --- a/skills/php2shell-check/SKILL.md +++ b/skills/php2shell-check/SKILL.md @@ -63,7 +63,7 @@ matching revision when known. ## Run and preserve the output Choose a new output path outside version control; create its parent directory -first. `--output` overwrites an existing file. The following paths and domains +first. `--output` refuses existing files; `--force` explicitly replaces only the final artifact after rendering. Existing `.partial.json` checkpoints are never overwritten by a new run. The following paths and domains are examples: replace them with the authorized inputs and chosen output paths, and do not scan example domains as a test. @@ -92,16 +92,12 @@ public contact value the user intends to disclose in requests. Inspect each result even if the process exits with `0`: that means the run completed, not that every fetch succeeded or every site is patched. Argument or -input-file errors exit with `2`; unexpected I/O errors can raise exceptions. +input/recovery/publication errors exit with `2`; interruption or incomplete recovery exits with `130`. For blocked or failed targets, report the reason without silently retrying. -On interruption, preserve the output before any rerun. Completed CSV rows are -flushed incrementally; JSON may lack its closing bracket. HTML is rendered from -completed rows during normal finalization, including a handled interruption -path, but a force-killed process may leave it empty. There is no automatic -resume. Determine which targets completed, then run only authorized outstanding -targets into a **new** output file if continuation is requested or already in -scope; do not imply the original run completed. +Use `--output` for durable evidence. The CLI atomically checkpoints every completed row to `OUTPUT.partial.json` before printing progress; forced termination preserves prior checkpoints. Ctrl-C returns `130` and leaves an explicitly incomplete checkpoint without replacing any previous final report. Recover JSON/CSV/HTML offline with `python wordpress_check.py --recover OUTPUT.partial.json --format html --output NEW.html`; do not supply targets. Recovery never makes requests and returns `130` for incomplete evidence. JSON now wraps rows under `results` with completeness and completed/pending/total counts; CSV appends run fields and an empty summary, HTML labels run state. Protect checkpoints like reports. A conservative incomplete checkpoint remains even after final success, so preserve the final artifact as the completion receipt. + +Stdout has no forced-termination durability. Recovery is bounded to regular non-symlink files up to 64 MiB / 100,000 rows; never claim uncommitted in-progress results survived. There is no automatic resume: use completed/pending counts and saved rows to prepare only authorized outstanding targets into a fresh output if continuation is authorized. Never imply the original run completed. ## Interpret and deliver diff --git a/test_output_durability.py b/test_output_durability.py new file mode 100644 index 0000000..bb1ed47 --- /dev/null +++ b/test_output_durability.py @@ -0,0 +1,132 @@ +import csv +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from unittest.mock import patch + +import wordpress_check as m + + +class DurableOutputTests(unittest.TestCase): + def call(self, args): + with redirect_stderr(io.StringIO()), redirect_stdout(io.StringIO()): + return m.main(args) + + def test_existing_output_and_failed_force_remain_unchanged(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / 'report.json' + for fmt in ('json', 'csv', 'html'): + target.write_text('PREVIOUS REPORT', encoding='utf-8') + with patch.object(m.Checker, 'check') as check, self.assertRaises(SystemExit): + self.call(['one.example', '--output', str(target), '--format', fmt]) + check.assert_not_called() + self.assertEqual(target.read_text(), 'PREVIOUS REPORT') + with patch.object(m.Checker, 'check', side_effect=KeyboardInterrupt): + self.assertEqual(self.call(['one.example', '--output', str(target), '--force']), 130) + self.assertEqual(target.read_text(), 'PREVIOUS REPORT') + journal = json.loads(Path(str(target) + '.partial.json').read_text()) + self.assertFalse(journal['complete']) + self.assertEqual(journal['completed_targets'], 0) + self.assertEqual(journal['pending_targets'], 1) + + def test_interrupted_completed_rows_recover_all_formats_without_requests(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / 'initial.json' + result = m.Result(input_url='https://one.example', title='=SUM(1)', http_status=200) + with patch.object(m.Checker, 'check', side_effect=[result, KeyboardInterrupt]): + self.assertEqual(self.call(['one.example', 'two.example', '--output', str(target)]), 130) + self.assertFalse(target.exists()) + journal = Path(str(target) + '.partial.json') + report = m.load_checkpoint(journal) + self.assertEqual((report['completed_targets'], report['pending_targets']), (1, 1)) + for fmt in ('json', 'csv', 'html'): + recovered = Path(directory) / ('recovered.' + fmt) + with patch.object(m.Checker, 'check', side_effect=AssertionError('network')): + self.assertEqual(self.call(['--recover', str(journal), '--format', fmt, '--output', str(recovered)]), 130) + text = recovered.read_text(encoding='utf-8') + if fmt == 'json': + self.assertFalse(json.loads(text)['complete']) + elif fmt == 'csv': + row = next(csv.DictReader(io.StringIO(text))) + self.assertEqual(row['title'], "'=SUM(1)") + self.assertEqual(row['pending_targets'], '1') + else: + self.assertIn('incomplete / recovered', text) + self.assertIn('https://one.example', text) + self.assertTrue(journal.exists()) + + def test_forced_process_exit_preserves_checkpoint_for_html(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / 'killed.html' + script = """import os,sys +import wordpress_check as m +calls=0 +def check(self,url): + global calls + calls+=1 + if calls==2: os._exit(73) + return m.Result(input_url=url,title='') +m.Checker.check=check +m.main(['https://one.example','https://two.example','--output',sys.argv[1],'--format','html']) +""" + run = subprocess.run([sys.executable, '-c', script, str(target)], capture_output=True, timeout=20) + self.assertEqual(run.returncode, 73) + self.assertFalse(target.exists()) + recovered = Path(directory) / 'recovered.html' + self.assertEqual(self.call(['--recover', str(target) + '.partial.json', '--format', 'html', '--output', str(recovered)]), 130) + self.assertIn('one.example', recovered.read_text(encoding='utf-8')) + self.assertNotIn('', recovered.read_text(encoding='utf-8')) + + def test_atomic_publish_race_and_render_failure_preserve_output(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / 'out.json' + original_link = os.link + def race(source, destination): + if Path(destination) == target: + target.write_text('RACING REPORT', encoding='utf-8') + return original_link(source, destination) + with patch.object(m.Checker, 'check', return_value=m.Result(input_url='one.example')), patch.object(m.os, 'link', side_effect=race): + self.assertEqual(self.call(['one.example', '--output', str(target)]), 2) + self.assertEqual(target.read_text(), 'RACING REPORT') + self.assertEqual(m.load_checkpoint(Path(str(target) + '.partial.json'))['completed_targets'], 1) + target2 = Path(directory) / 'render.html' + target2.write_text('OLDER', encoding='utf-8') + with patch.object(m.Checker, 'check', return_value=m.Result(input_url='one.example')), patch.object(m, 'render_html', side_effect=ValueError('render failed')): + self.assertEqual(self.call(['one.example', '--output', str(target2), '--force', '--format', 'html']), 2) + self.assertEqual(target2.read_text(), 'OLDER') + + def test_success_and_explicit_force_with_conservative_retained_journal(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / 'out.json' + target.write_text('OLD', encoding='utf-8') + with patch.object(m.Checker, 'check', return_value=m.Result(input_url='one.example')): + self.assertEqual(self.call(['one.example', '--output', str(target), '--format', 'json', '--force']), 0) + report = json.loads(target.read_text()) + self.assertTrue(report['complete']) + self.assertEqual(report['pending_targets'], 0) + # Existing journal is never silently overwritten, including --force. + with patch.object(m.Checker, 'check') as check: + self.assertEqual(self.call(['one.example', '--output', str(target), '--force']), 2) + check.assert_not_called() + self.assertEqual(json.loads(target.read_text()), report) + + def test_empty_incomplete_stdout_and_invalid_recovery(self): + with patch.object(m.Checker, 'check', side_effect=KeyboardInterrupt), redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()): + self.assertEqual(m.main(['one.example', '--format', 'json']), 130) + self.assertEqual(json.loads(stdout.getvalue())['pending_targets'], 1) + with tempfile.TemporaryDirectory() as directory: + journal = Path(directory) / 'bad.json' + journal.write_text('{"results":[]}', encoding='utf-8') + with patch.object(m.Checker, 'check') as check: + self.assertEqual(self.call(['--recover', str(journal)]), 2) + check.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/test_php2shell.py b/test_php2shell.py index a19c2cb..07ec44c 100644 --- a/test_php2shell.py +++ b/test_php2shell.py @@ -152,7 +152,7 @@ def test_cli_json_and_csv_include_php2shell_fields(self): output = io.StringIO() with patch.object(Checker, 'request', side_effect=replies), redirect_stdout(output), redirect_stderr(io.StringIO()): self.assertEqual(main(['https://example.test/', '--format', fmt]), 0) - rows = json.loads(output.getvalue()) if fmt == 'json' else list(csv.DictReader(io.StringIO(output.getvalue()))) + rows = json.loads(output.getvalue())['results'] if fmt == 'json' else list(csv.DictReader(io.StringIO(output.getvalue()))) self.assertEqual(rows[0]['php2shell_status'], 'potentially_affected') self.assertEqual(rows[0]['php2shell_exploitability'], 'not_tested') diff --git a/test_wordpress_check.py b/test_wordpress_check.py index af08fc6..d874851 100644 --- a/test_wordpress_check.py +++ b/test_wordpress_check.py @@ -284,7 +284,7 @@ def test_cli_csv_json_and_deduplication(self): with redirect_stderr(io.StringIO()): self.assertEqual(main(["--input", str(source), "--format", fmt, "--output", str(output), "--delay", "0.001"]), 0) content = output.read_text(encoding="utf-8") - rows = json.loads(content) if fmt == "json" else list(csv.DictReader(io.StringIO(content))) + rows = json.loads(content)["results"] if fmt == "json" else list(csv.DictReader(io.StringIO(content))) self.assertEqual(len(rows), 1) self.assertEqual(rows[0]["status"], "wordpress") diff --git a/wordpress_check.py b/wordpress_check.py index 25df40d..504f1b6 100644 --- a/wordpress_check.py +++ b/wordpress_check.py @@ -9,6 +9,10 @@ import base64 import csv import html +import io +import os +import stat +import tempfile import json import math import mimetypes @@ -629,11 +633,96 @@ def esc(value): """ + +def publish_report(path, text, *, replace=False): + """Publish one fully fsynced artifact; no-clobber is enforced at publication.""" + descriptor, temporary = tempfile.mkstemp(prefix=".php2shell-", dir=path.parent) + temporary = Path(temporary) + try: + with os.fdopen(descriptor, "w", encoding="utf-8", newline="") as stream: + stream.write(text) + stream.flush() + os.fsync(stream.fileno()) + if replace: + os.replace(temporary, path) + else: + os.link(temporary, path) # Fail closed when atomic no-clobber is unavailable. + finally: + temporary.unlink(missing_ok=True) + + +def run_report(rows, total, *, complete=False): + return {"journal_schema": "php2shell-run-v1", "complete": complete, + "completed_targets": len(rows), "pending_targets": total - len(rows), + "total_targets": total, "results": list(rows)} + + +def load_checkpoint(path): + if path.is_symlink(): + raise ValueError("Recovery refuses symbolic-link checkpoints") + with path.open("rb") as stream: + if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + raise ValueError("Recovery requires a regular checkpoint file") + raw = stream.read(64 * 1024 * 1024 + 1) + if len(raw) > 64 * 1024 * 1024: + raise ValueError("Checkpoint exceeds 64 MiB recovery limit") + report = json.loads(raw) + if not isinstance(report, dict) or set(report) != {"journal_schema", "complete", "completed_targets", "pending_targets", "total_targets", "results"}: + raise ValueError("Invalid checkpoint fields") + rows = report["results"] + if (report["journal_schema"] != "php2shell-run-v1" or type(report["complete"]) is not bool + or not isinstance(rows, list) or len(rows) > 100000 + or any(type(report[k]) is not int or report[k] < 0 for k in ("completed_targets", "pending_targets", "total_targets")) + or report["completed_targets"] != len(rows) + or report["completed_targets"] + report["pending_targets"] != report["total_targets"] + or (report["complete"] and report["pending_targets"])): + raise ValueError("Invalid checkpoint completeness") + example = asdict(Result(input_url="")) + for row in rows: + if not isinstance(row, dict) or set(row) != set(example): + raise ValueError("Invalid checkpoint result fields") + if row["php2shell_exploitability"] != "not_tested": + raise ValueError("Checkpoint cannot attest exploitability") + for key, value in row.items(): + default = example[key] + if isinstance(default, list): + valid = isinstance(value, list) and all(isinstance(item, str) for item in value) + elif key == "http_status": + valid = value is None or (type(value) is int and 100 <= value <= 599) + else: + valid = type(value) is type(default) + if not valid: + raise ValueError("Invalid checkpoint result value") + return report + + +def format_report(report, args): + metadata = {key: report[key] for key in ("complete", "completed_targets", "pending_targets", "total_targets")} + if args.format == "json": + return json.dumps({**metadata, "results": report["results"]}, ensure_ascii=False, indent=2) + "\n" + if args.format == "html": + rendered = render_html(report["results"], logo=args.logo, brand=args.brand) + state = "complete" if report["complete"] else "incomplete / recovered" + banner = f"

Run {state}: {report['completed_targets']} completed; {report['pending_targets']} pending; {report['total_targets']} total targets.

" + return rendered.replace('

Results

', banner + '

Results

', 1) + stream = io.StringIO(newline="") + fields = list(Result.__dataclass_fields__) + ["record_type", *metadata] + writer = csv.DictWriter(stream, fieldnames=fields) + writer.writeheader() + for row in report["results"]: + writer.writerow({key: csv_safe(value) for key, value in {**row, "record_type": "result", **metadata}.items()}) + if not report["results"]: + writer.writerow({"record_type": "summary", **metadata}) + return stream.getvalue() + + def main(argv=None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("urls", nargs="*", help="Website URLs or bare domains (defaults to HTTPS)") parser.add_argument("--input", type=Path, help="UTF-8 text file: one URL per line; # comments allowed") - parser.add_argument("--output", type=Path, help="Save results here; otherwise write to stdout") + parser.add_argument("--output", type=Path, help="Atomically save results; existing paths refused unless --force") + parser.add_argument("--force", action="store_true", help="Explicitly replace an existing final output only after successful completion") + parser.add_argument("--recover", type=Path, help="Render a persisted .partial.json checkpoint without making requests") parser.add_argument("--format", choices=("csv", "json", "html"), default="csv", help="csv, json, or a branded html report") parser.add_argument("--delay", type=positive_float, default=1.0, help="Minimum seconds between requests to an origin (default: 1)") parser.add_argument("--timeout", type=positive_float, default=15.0, help="Socket timeout per request (default: 15 seconds)") @@ -651,41 +740,54 @@ def main(argv=None) -> int: except OSError as exc: parser.error(str(exc)) urls = list(dict.fromkeys(urls)) - if not urls: + if not urls and not args.recover: parser.error("provide website URLs or --input websites.txt") + if args.recover and (urls or args.input): + parser.error("--recover cannot be combined with targets or --input") + if args.force and not args.output: + parser.error("--force requires --output") if args.input and args.output and args.input.resolve() == args.output.resolve(): parser.error("--output must differ from --input") - checker = Checker(timeout=args.timeout, delay=args.delay, contact=args.contact, authorized_origins=args.authorized_origin) - output = args.output.open("w", encoding="utf-8", newline="") if args.output else sys.stdout - writer = csv.DictWriter(output, fieldnames=list(Result.__dataclass_fields__)) if args.format == "csv" else None - report_rows = [] + checkpoint = Path(str(args.output) + ".partial.json") if args.output and not args.recover else None + if args.output: + if args.output.is_symlink() or (args.output.exists() and (not args.force or not args.output.is_file())): + parser.error("Output already exists or is not a regular file; choose a new path or explicitly --force") + if args.recover and args.output.resolve() == args.recover.resolve(): + parser.error("Recovery output must differ from its checkpoint") try: - if writer: - writer.writeheader() - elif args.format == "json": - output.write("[\n") - for index, url in enumerate(urls): - result = checker.check(url) - print(f"[{index + 1}/{len(urls)}] {url}: {result.status}", file=sys.stderr) - row = asdict(result) - if writer: - writer.writerow({key: csv_safe(value) for key, value in row.items()}) - elif args.format == "html": - report_rows.append(row) - else: - if index: - output.write(",\n") - output.write(json.dumps(row, ensure_ascii=False, indent=2)) - output.flush() - if args.format == "json": - output.write("\n]\n") - finally: - # An interrupted html run still reports the websites that completed. - if args.format == "html": - output.write(render_html(report_rows, logo=args.logo, brand=args.brand)) + if args.recover: + report = load_checkpoint(args.recover) + else: + report = run_report([], len(urls)) + if checkpoint: + # Exclusive creation protects a previous interrupted run from accidental reuse. + publish_report(checkpoint, json.dumps(report, ensure_ascii=False)) + checker = Checker(timeout=args.timeout, delay=args.delay, contact=args.contact, authorized_origins=args.authorized_origin) + try: + for index, url in enumerate(urls): + result = checker.check(url) + report = run_report([*report["results"], asdict(result)], len(urls)) + if checkpoint: + publish_report(checkpoint, json.dumps(report, ensure_ascii=False), replace=True) + print(f"[{index + 1}/{len(urls)}] {url}: {result.status}", file=sys.stderr) + report["complete"] = True + except KeyboardInterrupt: + if not checkpoint: + sys.stdout.write(format_report(report, args)) + sys.stdout.flush() + print("Interrupted: run incomplete; completed results remain in the checkpoint." if checkpoint else "Interrupted: output is incomplete.", file=sys.stderr) + return 130 + rendered = format_report(report, args) if args.output: - output.close() - return 0 + publish_report(args.output, rendered, replace=args.force) + else: + sys.stdout.write(rendered) + sys.stdout.flush() + # Retain the conservative checkpoint, even after success: recovery never invents completion. + return 0 if report["complete"] else 130 + except (OSError, ValueError, TypeError) as exc: + print(f"Report failed: {exc}. Existing output and persisted checkpoint are preserved.", file=sys.stderr) + return 2 if __name__ == "__main__": From c97bc86ace1b0106249a6f97038cb2cd44ce7771 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sun, 25 Oct 2026 22:40:34 -0400 Subject: [PATCH 2/5] fix: keep checkpoint publication within recovery bounds --- docs/REFERENCE.md | 2 +- test_output_durability.py | 10 ++++++++++ wordpress_check.py | 23 ++++++++++++++++++----- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index a55a73e..788249f 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -301,7 +301,7 @@ python wordpress_check.py --recover report.html.partial.json --format html --out python wordpress_check.py --recover report.html.partial.json --format json --output recovered.json ``` -Recovery returns `130` for an incomplete checkpoint, even if all targets were checkpointed before final publication. The conservative checkpoint is intentionally retained after success and is not final-output attestation. Keep the final report as the completion receipt. Recovery requires a regular non-symlink checkpoint, validates its structure/counts/result types and reads at most 64 MiB (100,000 result rows). Keep both the original checkpoint and any reconstructed artifact; continuation requires a separately authorized new run of pending targets only. The tool never resumes automatically. +Recovery returns `130` for an incomplete checkpoint, even if all targets were checkpointed before final publication. The conservative checkpoint is intentionally retained after success and is not final-output attestation. Keep the final report as the completion receipt. Runs accept at most 100,000 targets; split larger lists explicitly. Checkpoint publication refuses content beyond the 64 MiB recovery bound and preserves the previous checkpoint, with an error rather than an unrecoverable larger artifact. Recovery requires a regular non-symlink checkpoint, validates its structure/counts/result types and reads at most 64 MiB (100,000 result rows). Keep both the original checkpoint and any reconstructed artifact; continuation requires a separately authorized new run of pending targets only. The tool never resumes automatically. Without `--output`, results accumulate in memory and stdout receives one well-formed report on completion or a handled interrupt. Stdout/pipes do not provide durable recovery after forced termination; use `--output` when preserving evidence matters. Final output/partial names and report/input paths must differ. diff --git a/test_output_durability.py b/test_output_durability.py index bb1ed47..df62ac0 100644 --- a/test_output_durability.py +++ b/test_output_durability.py @@ -116,6 +116,16 @@ def test_success_and_explicit_force_with_conservative_retained_journal(self): check.assert_not_called() self.assertEqual(json.loads(target.read_text()), report) + def test_checkpoint_limit_preserves_previous_committed_state(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / 'limited.json' + initial_size = len(m.checkpoint_json(m.run_report([], 1)).encode('utf-8')) + with patch.object(m, 'MAX_CHECKPOINT_BYTES', initial_size + 1), patch.object(m.Checker, 'check', return_value=m.Result(input_url='one.example')): + self.assertEqual(self.call(['one.example', '--output', str(target)]), 2) + self.assertFalse(target.exists()) + report = m.load_checkpoint(Path(str(target) + '.partial.json')) + self.assertEqual((report['completed_targets'], report['pending_targets']), (0, 1)) + def test_empty_incomplete_stdout_and_invalid_recovery(self): with patch.object(m.Checker, 'check', side_effect=KeyboardInterrupt), redirect_stdout(io.StringIO()) as stdout, redirect_stderr(io.StringIO()): self.assertEqual(m.main(['one.example', '--format', 'json']), 130) diff --git a/wordpress_check.py b/wordpress_check.py index 504f1b6..d0e63a6 100644 --- a/wordpress_check.py +++ b/wordpress_check.py @@ -634,6 +634,17 @@ def esc(value): +MAX_CHECKPOINT_BYTES = 64 * 1024 * 1024 +MAX_CHECKPOINT_ROWS = 100000 + + +def checkpoint_json(report): + text = json.dumps(report, ensure_ascii=False) + if len(text.encode("utf-8")) > MAX_CHECKPOINT_BYTES: + raise ValueError("Checkpoint exceeds recovery byte limit; last committed checkpoint retained") + return text + + def publish_report(path, text, *, replace=False): """Publish one fully fsynced artifact; no-clobber is enforced at publication.""" descriptor, temporary = tempfile.mkstemp(prefix=".php2shell-", dir=path.parent) @@ -663,15 +674,15 @@ def load_checkpoint(path): with path.open("rb") as stream: if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): raise ValueError("Recovery requires a regular checkpoint file") - raw = stream.read(64 * 1024 * 1024 + 1) - if len(raw) > 64 * 1024 * 1024: + raw = stream.read(MAX_CHECKPOINT_BYTES + 1) + if len(raw) > MAX_CHECKPOINT_BYTES: raise ValueError("Checkpoint exceeds 64 MiB recovery limit") report = json.loads(raw) if not isinstance(report, dict) or set(report) != {"journal_schema", "complete", "completed_targets", "pending_targets", "total_targets", "results"}: raise ValueError("Invalid checkpoint fields") rows = report["results"] if (report["journal_schema"] != "php2shell-run-v1" or type(report["complete"]) is not bool - or not isinstance(rows, list) or len(rows) > 100000 + or not isinstance(rows, list) or len(rows) > MAX_CHECKPOINT_ROWS or any(type(report[k]) is not int or report[k] < 0 for k in ("completed_targets", "pending_targets", "total_targets")) or report["completed_targets"] != len(rows) or report["completed_targets"] + report["pending_targets"] != report["total_targets"] @@ -742,6 +753,8 @@ def main(argv=None) -> int: urls = list(dict.fromkeys(urls)) if not urls and not args.recover: parser.error("provide website URLs or --input websites.txt") + if len(urls) > MAX_CHECKPOINT_ROWS: + parser.error("At most 100,000 targets per run; split inputs explicitly") if args.recover and (urls or args.input): parser.error("--recover cannot be combined with targets or --input") if args.force and not args.output: @@ -761,14 +774,14 @@ def main(argv=None) -> int: report = run_report([], len(urls)) if checkpoint: # Exclusive creation protects a previous interrupted run from accidental reuse. - publish_report(checkpoint, json.dumps(report, ensure_ascii=False)) + publish_report(checkpoint, checkpoint_json(report)) checker = Checker(timeout=args.timeout, delay=args.delay, contact=args.contact, authorized_origins=args.authorized_origin) try: for index, url in enumerate(urls): result = checker.check(url) report = run_report([*report["results"], asdict(result)], len(urls)) if checkpoint: - publish_report(checkpoint, json.dumps(report, ensure_ascii=False), replace=True) + publish_report(checkpoint, checkpoint_json(report), replace=True) print(f"[{index + 1}/{len(urls)}] {url}: {result.status}", file=sys.stderr) report["complete"] = True except KeyboardInterrupt: From 8a514e85af87c3c62c1699071eef804fffd5a86b Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sun, 25 Oct 2026 22:43:50 -0400 Subject: [PATCH 3/5] fix: reject nonregular recovery inputs without blocking --- test_output_durability.py | 16 ++++++++++++++++ wordpress_check.py | 11 ++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/test_output_durability.py b/test_output_durability.py index df62ac0..3bb48b0 100644 --- a/test_output_durability.py +++ b/test_output_durability.py @@ -116,6 +116,22 @@ def test_success_and_explicit_force_with_conservative_retained_journal(self): check.assert_not_called() self.assertEqual(json.loads(target.read_text()), report) + @unittest.skipUnless(hasattr(os, 'mkfifo') and os.name == 'posix', 'POSIX FIFO fixture') + def test_fifo_recovery_rejected_without_blocking(self): + with tempfile.TemporaryDirectory() as directory: + fifo = Path(directory) / 'checkpoint.fifo' + os.mkfifo(fifo, 0o600) + run = subprocess.run([sys.executable, 'wordpress_check.py', '--recover', str(fifo)], + capture_output=True, timeout=10) + self.assertEqual(run.returncode, 2) + self.assertIn(b'regular checkpoint', run.stderr) + + def test_directory_recovery_rejected(self): + with tempfile.TemporaryDirectory() as directory: + with patch.object(m.Checker, 'check') as check: + self.assertEqual(self.call(['--recover', directory]), 2) + check.assert_not_called() + def test_checkpoint_limit_preserves_previous_committed_state(self): with tempfile.TemporaryDirectory() as directory: target = Path(directory) / 'limited.json' diff --git a/wordpress_check.py b/wordpress_check.py index d0e63a6..34fb409 100644 --- a/wordpress_check.py +++ b/wordpress_check.py @@ -671,10 +671,15 @@ def run_report(rows, total, *, complete=False): def load_checkpoint(path): if path.is_symlink(): raise ValueError("Recovery refuses symbolic-link checkpoints") - with path.open("rb") as stream: - if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode): + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(path, flags) + try: + if not stat.S_ISREG(os.fstat(descriptor).st_mode): raise ValueError("Recovery requires a regular checkpoint file") - raw = stream.read(MAX_CHECKPOINT_BYTES + 1) + with os.fdopen(descriptor, "rb", closefd=False) as stream: + raw = stream.read(MAX_CHECKPOINT_BYTES + 1) + finally: + os.close(descriptor) if len(raw) > MAX_CHECKPOINT_BYTES: raise ValueError("Checkpoint exceeds 64 MiB recovery limit") report = json.loads(raw) From dada7c5f282c1aaaa569c6d7de70becaf2c6f9af Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sun, 25 Oct 2026 22:45:30 -0400 Subject: [PATCH 4/5] docs: align synthetic HTML example with run completeness --- docs/example-report.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/example-report.html b/docs/example-report.html index 68e8af5..0ee999e 100644 --- a/docs/example-report.html +++ b/docs/example-report.html @@ -34,7 +34,7 @@
CompleteTech LLC logo
CompleteTech LLC

PHP2Shell Exposure Report

CVE-2026-109618 & CVE-2026-109619 · generated 2026-10-21 14:00 UTC · rules as of 2026-10-21

6Websites checked
2Potentially affected
2Fixed version reported
2Unknown / not disclosed
3WordPress detected
-

Results

+

Run complete: 6 completed; 0 pending; 6 total targets.

Results

WebsitePHP2ShellPHP reportedUpstream fixDetectionHTTPNotes
https://example.com/Potentially affected8.3.358.3.36wordpress200Advertised PHP 8.3.35 is older than the 8.3.36 upstream fix for both listed CVEs.
https://shop.example.org/Potentially affected8.2.348.2.35no_evidence200Advertised PHP 8.2.34 is older than the 8.2.35 upstream fix for both listed CVEs.
https://blog.example.com/Unknown8.3.6-1ubuntu0.24.04.138.3.36wordpress200PHP 8.3.6-1ubuntu0.24.04.13 contains a build qualifier. Confirm the exact vendor package; its upstream base alone cannot establish patch status.
https://www.example.edu/Unknown——no_evidence200No public PHP version was disclosed; lack of a header does not establish that PHP is absent.
https://example.net/Fixed version reported8.4.278.4.27wordpress200Advertised PHP 8.4.27 is at or above the 8.4.27 upstream fix. This is not a verified clean bill of health.
https://portal.example.org/Fixed version reported8.5.128.5.12no_evidence200Advertised PHP 8.5.12 is at or above the 8.5.12 upstream fix. This is not a verified clean bill of health.

Upstream fix thresholds

From 66955e11c25e6c64247b5be15da323a1cce3e975 Mon Sep 17 00:00:00 2001 From: Timothy Wayne Gregg Date: Sun, 25 Oct 2026 22:52:47 -0400 Subject: [PATCH 5/5] fix: recover all HTTP statuses preserved by the fetcher --- docs/REFERENCE.md | 2 +- test_output_durability.py | 69 +++++++++++++++++++++++++++++++++++++++ wordpress_check.py | 2 +- 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 788249f..dacee89 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -301,7 +301,7 @@ python wordpress_check.py --recover report.html.partial.json --format html --out python wordpress_check.py --recover report.html.partial.json --format json --output recovered.json ``` -Recovery returns `130` for an incomplete checkpoint, even if all targets were checkpointed before final publication. The conservative checkpoint is intentionally retained after success and is not final-output attestation. Keep the final report as the completion receipt. Runs accept at most 100,000 targets; split larger lists explicitly. Checkpoint publication refuses content beyond the 64 MiB recovery bound and preserves the previous checkpoint, with an error rather than an unrecoverable larger artifact. Recovery requires a regular non-symlink checkpoint, validates its structure/counts/result types and reads at most 64 MiB (100,000 result rows). Keep both the original checkpoint and any reconstructed artifact; continuation requires a separately authorized new run of pending targets only. The tool never resumes automatically. +Recovery returns `130` for an incomplete checkpoint, even if all targets were checkpointed before final publication. The conservative checkpoint is intentionally retained after success and is not final-output attestation. Keep the final report as the completion receipt. Runs accept at most 100,000 targets; split larger lists explicitly. Checkpoint publication refuses content beyond the 64 MiB recovery bound and preserves the previous checkpoint, with an error rather than an unrecoverable larger artifact. Recovery requires a regular non-symlink checkpoint, validates its structure/counts/result types and reads at most 64 MiB (100,000 result rows). Recovery retains integer HTTP statuses 100–999 (or null), matching the fetcher, including nonstandard responses; this does not turn a failed fetch into a successful assessment. Boolean, string, fractional and out-of-domain statuses remain invalid, and exploitability must remain `not_tested`. Keep both the original checkpoint and any reconstructed artifact; continuation requires a separately authorized new run of pending targets only. The tool never resumes automatically. Without `--output`, results accumulate in memory and stdout receives one well-formed report on completion or a handled interrupt. Stdout/pipes do not provide durable recovery after forced termination; use `--output` when preserving evidence matters. Final output/partial names and report/input paths must differ. diff --git a/test_output_durability.py b/test_output_durability.py index 3bb48b0..8752723 100644 --- a/test_output_durability.py +++ b/test_output_durability.py @@ -6,7 +6,9 @@ import subprocess import sys import tempfile +import threading import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from contextlib import redirect_stderr, redirect_stdout from unittest.mock import patch @@ -18,6 +20,73 @@ def call(self, args): with redirect_stderr(io.StringIO()), redirect_stdout(io.StringIO()): return m.main(args) + def test_generated_nonstandard_status_checkpoint_recovers_without_requests(self): + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(404 if self.path == '/robots.txt' else 999) + self.send_header('Content-Length', '0') + self.end_headers() + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + original_check = m.Checker.check + calls = [] + def check_then_interrupt(checker, url): + calls.append(url) + if len(calls) == 2: + raise KeyboardInterrupt + return original_check(checker, url) + try: + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / 'interrupted.json' + origin = 'http://127.0.0.1:' + str(server.server_port) + with patch.object(m.Checker, 'check', check_then_interrupt): + self.assertEqual(self.call([origin + '/999', origin + '/pending', '--delay', '0.001', + '--output', str(target), '--format', 'json']), 130) + checkpoint = Path(str(target) + '.partial.json') + report = m.load_checkpoint(checkpoint) + self.assertEqual(report['results'][0]['http_status'], 999) + self.assertEqual((report['completed_targets'], report['pending_targets']), (1, 1)) + for fmt in ('json', 'csv', 'html'): + output = Path(directory) / ('recovered.' + fmt) + with patch.object(m.Checker, 'check', side_effect=AssertionError('network')): + self.assertEqual(self.call(['--recover', str(checkpoint), '--output', str(output), + '--format', fmt]), 130) + text = output.read_text(encoding='utf-8') + self.assertIn('999', text) + if fmt == 'json': + recovered = json.loads(text) + self.assertFalse(recovered['complete']) + self.assertEqual(recovered['results'][0]['php2shell_exploitability'], 'not_tested') + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + def test_checkpoint_status_domain_and_unsafe_types(self): + with tempfile.TemporaryDirectory() as directory: + checkpoint = Path(directory) / 'boundary.json' + for status in (None, 100, 599, 600, 999): + with self.subTest(accepted=status): + report = m.run_report([m.asdict(m.Result(input_url='one.example', http_status=status))], 2) + checkpoint.write_text(m.checkpoint_json(report), encoding='utf-8') + self.assertEqual(m.load_checkpoint(checkpoint)['results'][0]['http_status'], status) + for status in (99, 1000, -1, True, 999.0, '999', [], {}): + with self.subTest(rejected=status): + report = m.run_report([m.asdict(m.Result(input_url='one.example', http_status=status))], 2) + checkpoint.write_text(m.checkpoint_json(report), encoding='utf-8') + with self.assertRaisesRegex(ValueError, 'Invalid checkpoint result value'): + m.load_checkpoint(checkpoint) + report = m.run_report([m.asdict(m.Result(input_url='one.example', http_status=999, + php2shell_exploitability='confirmed'))], 2) + checkpoint.write_text(m.checkpoint_json(report), encoding='utf-8') + with self.assertRaisesRegex(ValueError, 'cannot attest exploitability'): + m.load_checkpoint(checkpoint) + def test_existing_output_and_failed_force_remain_unchanged(self): with tempfile.TemporaryDirectory() as directory: target = Path(directory) / 'report.json' diff --git a/wordpress_check.py b/wordpress_check.py index 34fb409..501b1be 100644 --- a/wordpress_check.py +++ b/wordpress_check.py @@ -704,7 +704,7 @@ def load_checkpoint(path): if isinstance(default, list): valid = isinstance(value, list) and all(isinstance(item, str) for item in value) elif key == "http_status": - valid = value is None or (type(value) is int and 100 <= value <= 599) + valid = value is None or (type(value) is int and 100 <= value <= 999) else: valid = type(value) is type(default) if not valid: