feat(eval): add task analysis and trace publishing

CHANGELOG.md

@@ -29,3 +29,7 @@
 - Add `gpt-5.6-luna` and `gpt-5.6-terra` to the native evaluation catalog.
 - Report repair-overlay task provenance and original-versus-repaired score
   sensitivity when the original job is available.
+- Export native run metadata into aggregate task rows and add task-matrix
+  variance analysis with paired harness/reasoning diagnostics and SVG plots.
+- Add private S3 trace-bundle publishing through the standard AWS credential
+  chain with SHA-256 verification and resumable object checks.

README.md

@@ -435,6 +435,53 @@ clawbench dynamics-report \
 #   results/gptoss_dynamics/dynamics.json
 ```
 
+### Analyzing a native ShellBench matrix
+
+Native aggregation now preserves the task revision, harness, provider model,
+reasoning effort, and parent-run score/eligibility on every task row. Build
+task-level variance diagnostics from those normalized exports:
+
+```bash
+python scripts/native_eval/aggregate.py runs-full-YYYYMMDD results/native_summaries
+
+# Install the optional plotting dependency, or pass --no-plots.
+pip install -e '.[analysis]'
+clawbench task-analysis \
+  --summaries-dir results/native_summaries \
+  --output-dir results/task_analysis
+```
+
+The report writes task-cell summaries, paired harness and reasoning deltas,
+task variance rankings, a machine-readable JSON summary, `ANALYSIS.md`, and
+SVG box plots. Different task revisions and task counts remain separate
+datasets and are never silently pooled.
+
+### Publishing trace bundles to private S3
+
+Trace bundles can be uploaded without placing static credentials in command
+arguments or repository files. The command uses the standard AWS credential
+chain and does not expose access-key options:
+
+```bash
+pip install -e '.[s3]'
+
+export AWS_PROFILE="your-private-profile"
+export SHELLBENCH_TRACE_BUCKET="your-private-bucket"
+
+clawbench trace-upload \
+  --run-dir runs-full-YYYYMMDD \
+  --prefix runs-full-YYYYMMDD
+```
+
+The uploader selects final and checkpoint artifact archives, trajectory
+validation reports, trace-gap audits, and the run index. It sets no public ACL,
+uses S3-managed encryption, and records SHA-256 metadata; the destination
+bucket policy remains authoritative. Existing objects with matching size and
+digest are skipped, and a verified `S3_UPLOAD_MANIFEST.json` is written locally
+and uploaded last.
+
+Use `--dry-run` to inspect the selected files without authenticating.
+
 ### Running locally with small models (Ollama)
 
 A single consumer GPU running an open-weight model is enough to develop plugin profiles and validate algorithmic ideas — no API keys or cloud spend required.
@@ -518,6 +565,8 @@ clawbench/
 │   ├── dynamics.py                 # Trajectory metrics + sensitivity analysis
 │   ├── dynamics_archive.py         # Cached-run loading + offline report assembly
 │   ├── dynamics_plots.py           # Offline dynamics visualizations
+│   ├── task_analysis.py            # Native matrix task variance + paired deltas
+│   ├── trace_upload.py             # Private S3 trace publishing + verification
 │   └── cli.py                      # CLI entry points
 │
 ├── tasks-public/                   # Core v1 PUBLIC release (19 tasks)

clawbench/cli.py

@@ -254,6 +254,111 @@ def dynamics_report(
     click.echo(f"Saved {len(plots)} plots to {output_dir}/")
 
 
+@cli.command("task-analysis")
+@click.option(
+    "--summaries-dir",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    required=True,
+    help="Directory containing native aggregate_results.csv and per_task_results.csv.",
+)
+@click.option(
+    "--output-dir",
+    type=click.Path(path_type=Path),
+    default=Path("results/task_analysis"),
+    show_default=True,
+    help="Directory where task diagnostics, Markdown, and plots will be written.",
+)
+@click.option(
+    "--no-plots",
+    is_flag=True,
+    help="Write CSV, JSON, and Markdown outputs without rendering SVG plots.",
+)
+def task_analysis(summaries_dir: Path, output_dir: Path, no_plots: bool) -> None:
+    """Analyze task variance across harness, model, and reasoning cells."""
+    from clawbench.task_analysis import analyze_task_matrix
+
+    try:
+        report = analyze_task_matrix(
+            summaries_dir,
+            output_dir,
+            generate_plots=not no_plots,
+        )
+    except (FileNotFoundError, RuntimeError, ValueError) as exc:
+        raise click.ClickException(str(exc)) from exc
+
+    click.echo(
+        f"Analyzed {report['source']['eligible_run_count']} eligible runs "
+        f"across {len(report['datasets'])} task dataset(s)"
+    )
+    click.echo(f"Task analysis saved to {output_dir}")
+
+
+@cli.command("trace-upload")
+@click.option(
+    "--run-dir",
+    type=click.Path(exists=True, file_okay=False, path_type=Path),
+    required=True,
+    help="Native run directory containing raw/, summaries/, and manifests/.",
+)
+@click.option(
+    "--bucket",
+    envvar="SHELLBENCH_TRACE_BUCKET",
+    default="",
+    help="Private destination bucket. May be set with SHELLBENCH_TRACE_BUCKET.",
+)
+@click.option(
+    "--prefix",
+    default=None,
+    help="Object prefix. Defaults to the run directory name.",
+)
+@click.option(
+    "--workers",
+    type=click.IntRange(1, 32),
+    default=4,
+    show_default=True,
+    help="Number of files to upload concurrently.",
+)
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    help="Print the bundle plan without authenticating or uploading.",
+)
+def trace_upload(
+    run_dir: Path,
+    bucket: str,
+    prefix: str | None,
+    workers: int,
+    dry_run: bool,
+) -> None:
+    """Publish native trace archives to private S3 with hash verification."""
+    from clawbench.trace_upload import trace_bundle_plan, upload_trace_bundle
+
+    try:
+        if dry_run:
+            plan = trace_bundle_plan(run_dir)
+            click.echo(json.dumps(plan, indent=2, sort_keys=True))
+            return
+        if not bucket:
+            raise ValueError(
+                "set --bucket or SHELLBENCH_TRACE_BUCKET before uploading"
+            )
+        report = upload_trace_bundle(
+            run_dir,
+            bucket=bucket,
+            prefix=prefix,
+            workers=workers,
+            progress=click.echo,
+        )
+    except (FileNotFoundError, RuntimeError, ValueError) as exc:
+        raise click.ClickException(str(exc)) from exc
+
+    click.echo(
+        f"Verified {report['file_count']} trace objects "
+        f"({report['uploaded_count']} uploaded, {report['skipped_count']} unchanged)"
+    )
+    click.echo(f"Upload manifest saved to {report['manifest_path']}")
+
+
 def _write_dynamics_report(
     task_runs: dict[str, list],
     output_dir: Path,