From eb1fd6c9cdbd0675ccacaccac578fd0f10333192 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:02:04 +1200 Subject: [PATCH 01/77] Add Trace the Ace mastery-event experiment --- .../trace_the_ace/v71_mastery_events.py | 324 ++++++++++++++++++ 1 file changed, 324 insertions(+) create mode 100644 competitions/trace_the_ace/v71_mastery_events.py diff --git a/competitions/trace_the_ace/v71_mastery_events.py b/competitions/trace_the_ace/v71_mastery_events.py new file mode 100644 index 0000000..759dca1 --- /dev/null +++ b/competitions/trace_the_ace/v71_mastery_events.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +"""Trace the Ace V71: objective-conditioned mastery events. + +This development experiment converts each tutoring transcript into a sequence of +question -> student response -> tutor feedback episodes, scores objective +relevance, estimates independence/hint/correction state, and evaluates whether +those mastery-state features add signal beyond a sparse lexical baseline. + +The script intentionally inspects CSV headers before making schema decisions. +It never uses information across test samples at inference time. +""" +from __future__ import annotations + +import argparse +import json +import math +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable + +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold + +SEED = 20260815 +TOKEN_RE = re.compile(r"[a-z0-9]+(?:\.[0-9]+)?") +QUESTION_RE = re.compile(r"\?|\b(?:what|which|how|why|can you|could you|tell me|work out|calculate|solve|find)\b", re.I) +POS_RE = re.compile(r"\b(?:yes|yeah|correct|right|exactly|perfect|good|great|well done|that's it|thats it|you got it|spot on)\b", re.I) +NEG_RE = re.compile(r"\b(?:no|not quite|incorrect|wrong|careful|try again|almost|remember|instead|actually)\b", re.I) +HINT_RE = re.compile(r"\b(?:hint|remember|think about|what if|try|look at|start with|first step|help you)\b", re.I) +AGREE_RE = re.compile(r"^(?:yeah|yes|yep|okay|ok|mm+|mhm|uh huh|right|sure)[.! ]*$", re.I) + +STOP = { + "the","a","an","and","or","to","of","in","on","for","with","is","are","be","as","by","from", + "this","that","these","those","you","your","we","it","its","into","using","use","up","than","then" +} + + +def tokens(text: str) -> set[str]: + return {t for t in TOKEN_RE.findall(str(text).lower()) if len(t) > 1 and t not in STOP} + + +def jaccard(a: set[str], b: set[str]) -> float: + if not a or not b: + return 0.0 + return len(a & b) / len(a | b) + + +def char_ngram_overlap(a: str, b: str, n: int = 4) -> float: + def grams(s: str) -> set[str]: + s = re.sub(r"\s+", " ", s.lower()).strip() + return {s[i:i+n] for i in range(max(0, len(s)-n+1))} + ga, gb = grams(a), grams(b) + return jaccard(ga, gb) + + +def inspect_headers(path: Path) -> list[str]: + return list(pd.read_csv(path, nrows=0).columns) + + +@dataclass +class Episode: + q_idx: int + a_idx: int + f_idx: int | None + question: str + answer: str + feedback: str + relevance: float + feedback_pos: float + feedback_neg: float + hinted: float + answer_substantive: float + answer_agreement: float + recency: float + + +def normalize_roles(df: pd.DataFrame) -> pd.DataFrame: + """Conservatively repair only high-confidence local role inversions. + + We do not globally relabel speakers. The repair targets obvious semantic + contradictions such as a row labelled student containing a greeting/question + immediately followed by a row labelled tutor containing a short answer. + """ + out = df.copy() + roles = out["role"].astype(str).str.lower().tolist() + text = out["content"].fillna("").astype(str).tolist() + repaired = roles[:] + for i in range(len(out)-1): + a, b = text[i].strip(), text[i+1].strip() + if roles[i] == "student" and roles[i+1] == "tutor": + a_question = bool(QUESTION_RE.search(a)) and len(a.split()) >= 3 + b_short_answer = len(b.split()) <= 6 and not QUESTION_RE.search(b) + if a_question and b_short_answer: + repaired[i], repaired[i+1] = "tutor", "student" + out["role_repaired"] = repaired + out["role_changed"] = np.asarray(repaired) != np.asarray(roles) + return out + + +def extract_episodes(df: pd.DataFrame, objective: str) -> list[Episode]: + df = normalize_roles(df).reset_index(drop=True) + roles = df["role_repaired"].tolist() + content = df["content"].fillna("").astype(str).tolist() + objective_tokens = tokens(objective) + episodes: list[Episode] = [] + n = max(1, len(df)-1) + + for q_idx in range(len(df)-1): + if roles[q_idx] != "tutor" or not QUESTION_RE.search(content[q_idx]): + continue + a_idx = None + for j in range(q_idx+1, min(len(df), q_idx+5)): + if roles[j] == "student" and content[j].strip(): + a_idx = j + break + if roles[j] == "tutor" and QUESTION_RE.search(content[j]) and j > q_idx+1: + break + if a_idx is None: + continue + + f_idx = None + for j in range(a_idx+1, min(len(df), a_idx+5)): + if roles[j] == "tutor": + f_idx = j + break + q, a = content[q_idx], content[a_idx] + f = content[f_idx] if f_idx is not None else "" + local_text = q + " " + a + " " + f + rel = max( + jaccard(tokens(local_text), objective_tokens), + 0.5 * char_ngram_overlap(local_text, objective), + ) + pos = 1.0 if POS_RE.search(f) else 0.0 + neg = 1.0 if NEG_RE.search(f) else 0.0 + hint = 1.0 if HINT_RE.search(q) else 0.0 + agreement = 1.0 if AGREE_RE.match(a.strip()) else 0.0 + substantive = float(len(tokens(a)) >= 2 and agreement == 0.0) + recency = a_idx / n + episodes.append(Episode(q_idx,a_idx,f_idx,q,a,f,rel,pos,neg,hint,substantive,agreement,recency)) + return episodes + + +def mastery_features(df: pd.DataFrame, objective: str) -> tuple[np.ndarray, str, dict]: + eps = extract_episodes(df, objective) + changed = float(normalize_roles(df)["role_changed"].mean()) if len(df) else 0.0 + if not eps: + return np.zeros(24, dtype=np.float64), "", {"episodes":0,"role_repair_rate":changed} + + rel = np.array([e.relevance for e in eps]) + weights = np.maximum(rel, 0.02) * np.exp(2.0 * (np.array([e.recency for e in eps]) - 1.0)) + pos = np.array([e.feedback_pos for e in eps]) + neg = np.array([e.feedback_neg for e in eps]) + hint = np.array([e.hinted for e in eps]) + sub = np.array([e.answer_substantive for e in eps]) + agr = np.array([e.answer_agreement for e in eps]) + rec = np.array([e.recency for e in eps]) + independent_positive = pos * sub * (1.0-hint) + corrected = neg * sub + + k = max(1, min(8, len(eps))) + top = np.argsort(rel)[-k:] + tail = np.argsort(rec)[-k:] + wsum = float(weights.sum()) + 1e-12 + + feats = np.array([ + len(eps), rel.mean(), rel.max(), np.quantile(rel,0.75), + pos.mean(), neg.mean(), hint.mean(), sub.mean(), agr.mean(), + independent_positive.mean(), corrected.mean(), + float((weights*pos).sum()/wsum), float((weights*neg).sum()/wsum), + float((weights*independent_positive).sum()/wsum), + float(pos[top].mean()), float(neg[top].mean()), float(independent_positive[top].mean()), + float(pos[tail].mean()), float(neg[tail].mean()), float(independent_positive[tail].mean()), + float(rec[pos>0].mean()) if np.any(pos>0) else 0.0, + float(rec[neg>0].mean()) if np.any(neg>0) else 0.0, + changed, + float(sum(e.feedback_pos-e.feedback_neg for e in eps[-5:])), + ], dtype=np.float64) + + ranked = sorted(eps, key=lambda e: (e.relevance * (0.25 + 0.75*e.recency)), reverse=True)[:8] + text = " ".join( + f"[Q]{e.question} [STUDENT]{e.answer} [FEEDBACK]{e.feedback}" + for e in ranked + ) + meta = {"episodes":len(eps),"role_repair_rate":changed,"max_relevance":float(rel.max())} + return feats, text, meta + + +def load_transcript(path: Path) -> pd.DataFrame: + cols = inspect_headers(path) + required = {"session_id","utterance_id","role","content","timestamp"} + missing = required - set(cols) + if missing: + raise ValueError(f"{path.name}: missing transcript columns {sorted(missing)}; got {cols}") + return pd.read_csv(path) + + +def build_frame(features_path: Path, labels_path: Path, transcript_dir: Path): + fcols = inspect_headers(features_path) + lcols = inspect_headers(labels_path) + print("features columns", fcols) + print("labels columns", lcols) + required_f = {"response_id","session_id","learning_objective"} + if not required_f.issubset(fcols): + raise ValueError(f"features missing {sorted(required_f-set(fcols))}") + target = "is_correct" if "is_correct" in lcols else "correct" if "correct" in lcols else None + if target is None: + raise ValueError(f"labels need is_correct or correct; got {lcols}") + features = pd.read_csv(features_path) + labels = pd.read_csv(labels_path) + frame = features.merge(labels[["response_id",target]], on="response_id", how="inner", validate="one_to_one") + frame = frame.rename(columns={target:"target"}) + return frame + + +def fixed_group_folds(groups: Iterable[str], n_splits: int = 5): + groups = np.asarray(list(groups)) + dummy = np.zeros(len(groups)) + return list(GroupKFold(n_splits=n_splits).split(dummy, dummy, groups)) + + +def fit_eval(X, y, folds, name: str): + oof = np.zeros(len(y), dtype=np.float64) + rows = [] + for fold,(tr,va) in enumerate(folds,1): + m = LogisticRegression(C=0.35, max_iter=250, solver="liblinear", random_state=SEED) + m.fit(X[tr], y[tr]) + p = np.clip(m.predict_proba(X[va])[:,1], 1e-5, 1-1e-5) + oof[va] = p + rows.append({"fold":fold,"rows":len(va),"logloss":log_loss(y[va],p),"auc":roc_auc_score(y[va],p)}) + print(name, rows[-1]) + return oof, rows + + +def run(args): + frame = build_frame(args.features, args.labels, args.transcripts) + cache: dict[str,pd.DataFrame] = {} + numeric, episode_text, meta = [], [], [] + for i,row in frame.iterrows(): + sid = str(row.session_id) + if sid not in cache: + cache[sid] = load_transcript(args.transcripts / f"{sid}.csv") + f,t,m = mastery_features(cache[sid], str(row.learning_objective)) + numeric.append(f); episode_text.append(t); meta.append(m) + if args.limit and i+1 >= args.limit: + frame = frame.iloc[:i+1].copy(); break + numeric = np.vstack(numeric) + episode_text = episode_text[:len(frame)] + y = frame.target.to_numpy(dtype=int) + + hv = HashingVectorizer(n_features=2**18, alternate_sign=False, norm="l2", ngram_range=(1,2), lowercase=True) + objective_text = frame.learning_objective.fillna("").astype(str).tolist() + X_obj = hv.transform(["[OBJECTIVE] "+x for x in objective_text]) + X_ep = hv.transform(["[EPISODES] "+x for x in episode_text]) + X_num = csr_matrix((numeric - numeric.mean(0)) / (numeric.std(0)+1e-6)) + X_base = X_obj + X_full = hstack([X_obj,X_ep,X_num], format="csr") + + session_folds = fixed_group_folds(frame.session_id, 5) + objective_folds = fixed_group_folds(frame.learning_objective_id if "learning_objective_id" in frame else frame.learning_objective, 5) + results = {} + for split,folds in [("session",session_folds),("objective",objective_folds)]: + p0,r0 = fit_eval(X_base,y,folds,f"baseline/{split}") + p1,r1 = fit_eval(X_full,y,folds,f"mastery/{split}") + results[split] = { + "baseline_logloss":float(log_loss(y,p0)), + "mastery_logloss":float(log_loss(y,p1)), + "delta":float(log_loss(y,p1)-log_loss(y,p0)), + "baseline_auc":float(roc_auc_score(y,p0)), + "mastery_auc":float(roc_auc_score(y,p1)), + "folds_baseline":r0,"folds_mastery":r1, + } + results["diagnostics"] = { + "rows":len(frame), + "sessions":int(frame.session_id.nunique()), + "objectives":int(frame.learning_objective.nunique()), + "mean_episode_count":float(np.mean([m["episodes"] for m in meta[:len(frame)]])), + "mean_role_repair_rate":float(np.mean([m["role_repair_rate"] for m in meta[:len(frame)]])), + } + print(json.dumps(results, indent=2)) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(results, indent=2)) + + +def self_test(): + df = pd.DataFrame([ + ["s","1","tutor","What is 6 times 7?","2026-01-01T00:00:00"], + ["s","2","student","42","2026-01-01T00:00:01"], + ["s","3","tutor","Exactly right, well done.","2026-01-01T00:00:02"], + ["s","4","tutor","Now what is 8 times 7?","2026-01-01T00:00:03"], + ["s","5","student","54","2026-01-01T00:00:04"], + ["s","6","tutor","Not quite, try again.","2026-01-01T00:00:05"], + ], columns=["session_id","utterance_id","role","content","timestamp"]) + f,t,m = mastery_features(df,"multiplying one-digit numbers") + assert m["episodes"] == 2 + assert f[4] > 0 and f[5] > 0 + assert "42" in t and "54" in t + print("SELF_TEST_PASS", json.dumps(m)) + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path) + p.add_argument("--labels", type=Path) + p.add_argument("--transcripts", type=Path) + p.add_argument("--out", type=Path, default=Path("v71_mastery_results.json")) + p.add_argument("--limit", type=int, default=0) + p.add_argument("--self-test", action="store_true") + return p.parse_args() + + +if __name__ == "__main__": + a = parse_args() + if a.self_test: + self_test() + else: + if not (a.features and a.labels and a.transcripts): + raise SystemExit("--features, --labels and --transcripts are required unless --self-test is used") + run(a) From 180a313ff4943fb67992748aa6e77705dd8520fa Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:02:18 +1200 Subject: [PATCH 02/77] Add Trace the Ace Actions workflow --- .github/workflows/trace-ace-mastery.yml | 90 +++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 .github/workflows/trace-ace-mastery.yml diff --git a/.github/workflows/trace-ace-mastery.yml b/.github/workflows/trace-ace-mastery.yml new file mode 100644 index 0000000..49ac96d --- /dev/null +++ b/.github/workflows/trace-ace-mastery.yml @@ -0,0 +1,90 @@ +name: Trace the Ace mastery experiment + +on: + workflow_dispatch: + inputs: + run_full: + description: "Run full experiment if TRACE_ACE_DATA_URL secret is configured" + required: false + default: false + type: boolean + limit: + description: "Optional row limit (0 = all rows)" + required: false + default: "0" + type: string + push: + branches: + - agent/trace-ace-mastery-events + paths: + - "competitions/trace_the_ace/**" + - ".github/workflows/trace-ace-mastery.yml" + +jobs: + self-test: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install experiment dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn + - name: Run mastery extractor self-test + run: python competitions/trace_the_ace/v71_mastery_events.py --self-test + + full-experiment: + if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_full }} + needs: self-test + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + TRACE_ACE_DATA_URL: ${{ secrets.TRACE_ACE_DATA_URL }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install experiment dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn + - name: Require private dataset URL + shell: bash + run: | + if [ -z "$TRACE_ACE_DATA_URL" ]; then + echo "TRACE_ACE_DATA_URL is not configured; refusing to run without private data transport." + exit 1 + fi + - name: Fetch dataset without logging URL or contents + shell: bash + run: | + set +x + mkdir -p /tmp/trace_ace + curl --fail --silent --show-error --location "$TRACE_ACE_DATA_URL" -o /tmp/trace_ace/data.tar.gz + tar -xzf /tmp/trace_ace/data.tar.gz -C /tmp/trace_ace + rm -f /tmp/trace_ace/data.tar.gz + - name: Locate inputs and run experiment + shell: bash + run: | + set -euo pipefail + FEATURES=$(find /tmp/trace_ace -type f -name 'train_features*.csv' | head -1) + LABELS=$(find /tmp/trace_ace -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(find /tmp/trace_ace -type d -name 'train_transcripts*' | head -1) + test -n "$FEATURES" && test -n "$LABELS" && test -n "$TRANSCRIPTS" + LIMIT="${{ inputs.limit }}" + EXTRA=() + if [ "$LIMIT" != "0" ]; then EXTRA+=(--limit "$LIMIT"); fi + python competitions/trace_the_ace/v71_mastery_events.py \ + --features "$FEATURES" \ + --labels "$LABELS" \ + --transcripts "$TRANSCRIPTS" \ + --out v71_mastery_results.json \ + "${EXTRA[@]}" + - name: Upload aggregate results only + uses: actions/upload-artifact@v4 + with: + name: v71-mastery-results + path: v71_mastery_results.json + retention-days: 14 From fef174a4a8da2d2a320c7fa3e5dbbcfae13ad046 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:04:08 +1200 Subject: [PATCH 03/77] Add Trace the Ace supervision audit --- .../trace_the_ace/v72_supervision_audit.py | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 competitions/trace_the_ace/v72_supervision_audit.py diff --git a/competitions/trace_the_ace/v72_supervision_audit.py b/competitions/trace_the_ace/v72_supervision_audit.py new file mode 100644 index 0000000..3526fe5 --- /dev/null +++ b/competitions/trace_the_ace/v72_supervision_audit.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Trace the Ace V72: audit hidden supervision in multi-objective tutoring sessions. + +This script measures two structural resources that a winning model can exploit: +(1) within-session label agreement / disagreement, which separates global session +state from objective-specific mastery; and (2) transcript micro-assessment density +from tutor-question -> student-answer -> tutor-feedback episodes. + +Only aggregate JSON is written. Raw competition text is never emitted. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +import pandas as pd + +from v71_mastery_events import inspect_headers, load_transcript, extract_episodes + + +def load_training(features_path: Path, labels_path: Path) -> pd.DataFrame: + fcols = inspect_headers(features_path) + lcols = inspect_headers(labels_path) + print("features columns", fcols) + print("labels columns", lcols) + need = {"response_id", "session_id", "learning_objective"} + if not need.issubset(fcols): + raise ValueError(f"features missing {sorted(need-set(fcols))}") + target = "is_correct" if "is_correct" in lcols else "correct" if "correct" in lcols else None + if target is None: + raise ValueError(f"labels need is_correct or correct; got {lcols}") + f = pd.read_csv(features_path) + y = pd.read_csv(labels_path) + out = f.merge(y[["response_id", target]], on="response_id", validate="one_to_one") + return out.rename(columns={target: "target"}) + + +def pair_agreement(values: np.ndarray) -> tuple[int, int]: + n = len(values) + if n < 2: + return 0, 0 + total = n * (n - 1) // 2 + pos = int(values.sum()) + neg = n - pos + disagree = pos * neg + return total - disagree, total + + +def run(args) -> None: + df = load_training(args.features, args.labels) + if args.limit: + df = df.iloc[: args.limit].copy() + + sizes = df.groupby("session_id").size() + multi_ids = sizes[sizes > 1].index + multi = df[df.session_id.isin(multi_ids)] + + agree = total = homogeneous = mixed = 0 + session_means = [] + contrastive_pairs = 0 + for _, g in multi.groupby("session_id", sort=False): + vals = g.target.to_numpy(dtype=int) + a, t = pair_agreement(vals) + agree += a + total += t + homogeneous += int(vals.min() == vals.max()) + mixed += int(vals.min() != vals.max()) + session_means.append(float(vals.mean())) + pos, neg = int(vals.sum()), int(len(vals) - vals.sum()) + contrastive_pairs += pos * neg + + # Transcript micro-assessment density is sampled by unique session to keep this + # audit cheap enough for GitHub-hosted runners while remaining deterministic. + sample_ids = sorted(df.session_id.astype(str).unique())[: args.episode_sessions] + ep_counts = [] + feedback_pos = feedback_neg = substantive = 0 + for sid in sample_ids: + path = args.transcripts / f"{sid}.csv" + if not path.exists(): + continue + tdf = load_transcript(path) + # Use a neutral objective here: the purpose is density / weak-label audit, + # not objective relevance. + eps = extract_episodes(tdf, "") + ep_counts.append(len(eps)) + feedback_pos += sum(int(e.feedback_pos) for e in eps) + feedback_neg += sum(int(e.feedback_neg) for e in eps) + substantive += sum(int(e.answer_substantive) for e in eps) + + objective_counts = df.groupby("learning_objective").size().sort_values(ascending=False) + result = { + "rows": int(len(df)), + "sessions": int(df.session_id.nunique()), + "objectives": int(df.learning_objective.nunique()), + "positive_rate": float(df.target.mean()), + "multi_objective_sessions": int(len(multi_ids)), + "multi_objective_session_fraction": float(len(multi_ids) / max(1, df.session_id.nunique())), + "within_session_pair_agreement": float(agree / total) if total else None, + "homogeneous_multi_session_fraction": float(homogeneous / max(1, homogeneous + mixed)), + "mixed_multi_sessions": int(mixed), + "opposite_label_same_session_pairs": int(contrastive_pairs), + "median_session_label_mean": float(np.median(session_means)) if session_means else None, + "objectives_seen_once": int((objective_counts == 1).sum()), + "objectives_seen_at_most_5": int((objective_counts <= 5).sum()), + "top_10_objective_row_fraction": float(objective_counts.head(10).sum() / len(df)), + "episode_sessions_sampled": int(len(ep_counts)), + "mean_micro_assessments_per_sampled_session": float(np.mean(ep_counts)) if ep_counts else None, + "median_micro_assessments_per_sampled_session": float(np.median(ep_counts)) if ep_counts else None, + "positive_feedback_events": int(feedback_pos), + "negative_feedback_events": int(feedback_neg), + "substantive_student_response_events": int(substantive), + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) + + +def self_test() -> None: + a, t = pair_agreement(np.array([1, 1, 0, 1])) + assert (a, t) == (3, 6) + a2, t2 = pair_agreement(np.array([1, 1, 1])) + assert (a2, t2) == (3, 3) + print("V72_SELF_TEST_PASS") + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path) + p.add_argument("--labels", type=Path) + p.add_argument("--transcripts", type=Path) + p.add_argument("--out", type=Path, default=Path("v72_supervision_audit.json")) + p.add_argument("--episode-sessions", type=int, default=500) + p.add_argument("--limit", type=int, default=0) + p.add_argument("--self-test", action="store_true") + return p.parse_args() + + +if __name__ == "__main__": + args = parse_args() + if args.self_test: + self_test() + else: + if not args.features or not args.labels or not args.transcripts: + raise SystemExit("--features, --labels and --transcripts are required") + run(args) From 7ddb9442216a0609ebd5759653534d5931fb07b5 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:04:22 +1200 Subject: [PATCH 04/77] Run Trace the Ace supervision audit in Actions --- .github/workflows/trace-ace-mastery.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/trace-ace-mastery.yml b/.github/workflows/trace-ace-mastery.yml index 49ac96d..d5e0273 100644 --- a/.github/workflows/trace-ace-mastery.yml +++ b/.github/workflows/trace-ace-mastery.yml @@ -34,6 +34,8 @@ jobs: run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn - name: Run mastery extractor self-test run: python competitions/trace_the_ace/v71_mastery_events.py --self-test + - name: Run supervision audit self-test + run: python competitions/trace_the_ace/v72_supervision_audit.py --self-test full-experiment: if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_full }} @@ -65,7 +67,7 @@ jobs: curl --fail --silent --show-error --location "$TRACE_ACE_DATA_URL" -o /tmp/trace_ace/data.tar.gz tar -xzf /tmp/trace_ace/data.tar.gz -C /tmp/trace_ace rm -f /tmp/trace_ace/data.tar.gz - - name: Locate inputs and run experiment + - name: Locate inputs and run experiments shell: bash run: | set -euo pipefail @@ -82,9 +84,17 @@ jobs: --transcripts "$TRANSCRIPTS" \ --out v71_mastery_results.json \ "${EXTRA[@]}" + python competitions/trace_the_ace/v72_supervision_audit.py \ + --features "$FEATURES" \ + --labels "$LABELS" \ + --transcripts "$TRANSCRIPTS" \ + --out v72_supervision_audit.json \ + "${EXTRA[@]}" - name: Upload aggregate results only uses: actions/upload-artifact@v4 with: - name: v71-mastery-results - path: v71_mastery_results.json + name: trace-ace-aggregate-results + path: | + v71_mastery_results.json + v72_supervision_audit.json retention-days: 14 From 1331a1a5c12d0e1467957bba2380ea27c9c86d52 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:06:09 +1200 Subject: [PATCH 05/77] Add V73 same-session contrastive mastery model --- .../trace_the_ace/v73_contrastive_mastery.py | 259 ++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 competitions/trace_the_ace/v73_contrastive_mastery.py diff --git a/competitions/trace_the_ace/v73_contrastive_mastery.py b/competitions/trace_the_ace/v73_contrastive_mastery.py new file mode 100644 index 0000000..0a785a1 --- /dev/null +++ b/competitions/trace_the_ace/v73_contrastive_mastery.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""Trace the Ace V73: same-session contrastive mastery model. + +V73 turns the structural observation behind V72 into a measurable model. +For each response it builds objective-conditioned mastery evidence from V71, +then trains two complementary learners inside each held-out split: + +1. row model: predicts correctness from objective text + mastery evidence; +2. contrastive model: on mixed-label training sessions, learns which of two + objectives in the SAME transcript is more likely to be correct. + +Because the contrastive examples cancel session-wide ability, their coefficient +vector is forced toward objective-specific mastery evidence. The final prediction +combines row and contrastive logits using an inner training split only; validation +labels are never used to choose the blend. + +The script inspects CSV headers before schema decisions and writes aggregate JSON +only. It does not use cross-test-sample information at inference time. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack, vstack +from scipy.special import expit, logit +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression, SGDClassifier +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold, GroupShuffleSplit + +from v71_mastery_events import ( + SEED, + build_frame, + fixed_group_folds, + load_transcript, + mastery_features, +) + + +def build_design(frame: pd.DataFrame, transcripts: Path): + cache: dict[str, pd.DataFrame] = {} + numeric, episode_text = [], [] + for _, row in frame.iterrows(): + sid = str(row.session_id) + if sid not in cache: + cache[sid] = load_transcript(transcripts / f"{sid}.csv") + f, t, _ = mastery_features(cache[sid], str(row.learning_objective)) + numeric.append(f) + episode_text.append(t) + + numeric = np.vstack(numeric) + mu = numeric.mean(axis=0) + sd = numeric.std(axis=0) + 1e-6 + X_num = csr_matrix((numeric - mu) / sd) + + hv = HashingVectorizer( + n_features=2**18, + alternate_sign=False, + norm="l2", + ngram_range=(1, 2), + lowercase=True, + ) + obj = frame.learning_objective.fillna("").astype(str).tolist() + X_obj = hv.transform(["[OBJECTIVE] " + x for x in obj]) + X_ep = hv.transform(["[EPISODES] " + x for x in episode_text]) + return hstack([X_obj, X_ep, X_num], format="csr") + + +def same_session_pairs(frame: pd.DataFrame, indices: np.ndarray, max_pairs: int = 50000): + """Return deterministic opposite-label pairs (positive_row, negative_row).""" + sub = frame.iloc[indices] + pairs: list[tuple[int, int]] = [] + # map original row index -> position in X subset later via explicit global indices + for _, g in sub.groupby("session_id", sort=True): + pos = g.index[g.target.to_numpy(dtype=int) == 1].tolist() + neg = g.index[g.target.to_numpy(dtype=int) == 0].tolist() + for p in pos: + for n in neg: + pairs.append((int(p), int(n))) + if len(pairs) >= max_pairs: + return pairs + return pairs + + +def pair_matrix(X, pairs: list[tuple[int, int]]): + """Balanced pairwise dataset: x_pos-x_neg => 1 and reverse => 0.""" + if not pairs: + return None, None + p = np.asarray([a for a, _ in pairs], dtype=int) + n = np.asarray([b for _, b in pairs], dtype=int) + d = X[p] - X[n] + Xp = vstack([d, -d], format="csr") + yp = np.r_[np.ones(len(pairs), dtype=int), np.zeros(len(pairs), dtype=int)] + return Xp, yp + + +def fit_pairwise(X, frame: pd.DataFrame, train_idx: np.ndarray, max_pairs: int): + pairs = same_session_pairs(frame, train_idx, max_pairs=max_pairs) + Xp, yp = pair_matrix(X, pairs) + if Xp is None or len(np.unique(yp)) < 2: + return None, len(pairs) + model = SGDClassifier( + loss="log_loss", + penalty="l2", + alpha=2e-5, + max_iter=60, + tol=1e-4, + random_state=SEED, + average=True, + ) + model.fit(Xp, yp) + return model, len(pairs) + + +def choose_blend(row_logit: np.ndarray, contrast: np.ndarray, y: np.ndarray) -> float: + """Choose contrast weight only on inner-training predictions.""" + best_a, best_loss = 0.0, float("inf") + for a in np.linspace(-0.30, 0.60, 19): + p = expit(row_logit + a * contrast) + loss = log_loss(y, np.clip(p, 1e-5, 1 - 1e-5)) + if loss < best_loss: + best_loss, best_a = float(loss), float(a) + return best_a + + +def fold_predict(X, frame: pd.DataFrame, tr: np.ndarray, va: np.ndarray, max_pairs: int): + y = frame.target.to_numpy(dtype=int) + + # Outer row model. + row = LogisticRegression(C=0.35, max_iter=300, solver="liblinear", random_state=SEED) + row.fit(X[tr], y[tr]) + row_va = np.clip(row.predict_proba(X[va])[:, 1], 1e-5, 1 - 1e-5) + + # Pairwise model uses only outer-training sessions. + pair, pair_count = fit_pairwise(X, frame, tr, max_pairs) + if pair is None: + return row_va, row_va, 0.0, pair_count + contrast_va = pair.decision_function(X[va]) + + # Learn blend weight on an inner session split, never outer validation. + gss = GroupShuffleSplit(n_splits=1, test_size=0.22, random_state=SEED) + inner_a_rel, inner_b_rel = next(gss.split(tr, y[tr], frame.session_id.iloc[tr])) + inner_a = tr[inner_a_rel] + inner_b = tr[inner_b_rel] + + inner_row = LogisticRegression(C=0.35, max_iter=300, solver="liblinear", random_state=SEED) + inner_row.fit(X[inner_a], y[inner_a]) + p_inner = np.clip(inner_row.predict_proba(X[inner_b])[:, 1], 1e-5, 1 - 1e-5) + inner_pair, _ = fit_pairwise(X, frame, inner_a, max(5000, max_pairs // 2)) + if inner_pair is None: + alpha = 0.0 + else: + c_inner = inner_pair.decision_function(X[inner_b]) + alpha = choose_blend(logit(p_inner), c_inner, y[inner_b]) + + p_blend = expit(logit(row_va) + alpha * contrast_va) + return row_va, np.clip(p_blend, 1e-5, 1 - 1e-5), alpha, pair_count + + +def evaluate_split(X, frame: pd.DataFrame, folds, name: str, max_pairs: int): + y = frame.target.to_numpy(dtype=int) + row_oof = np.zeros(len(frame), dtype=float) + blend_oof = np.zeros(len(frame), dtype=float) + details = [] + for k, (tr, va) in enumerate(folds, 1): + p0, p1, alpha, pair_count = fold_predict(X, frame, tr, va, max_pairs) + row_oof[va] = p0 + blend_oof[va] = p1 + rec = { + "fold": k, + "rows": int(len(va)), + "row_logloss": float(log_loss(y[va], p0)), + "contrastive_logloss": float(log_loss(y[va], p1)), + "delta": float(log_loss(y[va], p1) - log_loss(y[va], p0)), + "alpha": float(alpha), + "training_pairs": int(pair_count), + } + print(name, rec) + details.append(rec) + return { + "row_logloss": float(log_loss(y, row_oof)), + "contrastive_logloss": float(log_loss(y, blend_oof)), + "delta": float(log_loss(y, blend_oof) - log_loss(y, row_oof)), + "row_auc": float(roc_auc_score(y, row_oof)), + "contrastive_auc": float(roc_auc_score(y, blend_oof)), + "folds": details, + "alpha_mean": float(np.mean([d["alpha"] for d in details])), + "alpha_nonzero_folds": int(sum(abs(d["alpha"]) > 1e-12 for d in details)), + } + + +def run(args): + frame = build_frame(args.features, args.labels, args.transcripts) + if args.limit: + frame = frame.iloc[: args.limit].copy().reset_index(drop=True) + else: + frame = frame.reset_index(drop=True) + X = build_design(frame, args.transcripts) + + session_folds = fixed_group_folds(frame.session_id, 5) + objective_group = frame.learning_objective_id if "learning_objective_id" in frame else frame.learning_objective + objective_folds = fixed_group_folds(objective_group, 5) + + result = { + "session": evaluate_split(X, frame, session_folds, "session", args.max_pairs), + "objective": evaluate_split(X, frame, objective_folds, "objective", args.max_pairs), + "diagnostics": { + "rows": int(len(frame)), + "sessions": int(frame.session_id.nunique()), + "objectives": int(frame.learning_objective.nunique()), + "design_shape": [int(X.shape[0]), int(X.shape[1])], + }, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) + + +def self_test(): + frame = pd.DataFrame({ + "session_id": ["a", "a", "b", "b", "c"], + "target": [1, 0, 1, 1, 0], + }) + pairs = same_session_pairs(frame, np.arange(len(frame)), max_pairs=20) + assert pairs == [(0, 1)], pairs + X = csr_matrix(np.array([[2.0, 0.0], [0.0, 1.0], [1.0, 1.0], [1.0, 2.0], [0.0, 2.0]])) + Xp, yp = pair_matrix(X, pairs) + assert Xp.shape == (2, 2) + assert yp.tolist() == [1, 0] + assert np.allclose(Xp.toarray()[0], -Xp.toarray()[1]) + a = choose_blend(np.array([1.0, -1.0]), np.array([1.0, -1.0]), np.array([1, 0])) + assert a >= 0.0 + print("V73_SELF_TEST_PASS", {"pairs": len(pairs), "alpha": a}) + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path) + p.add_argument("--labels", type=Path) + p.add_argument("--transcripts", type=Path) + p.add_argument("--out", type=Path, default=Path("v73_contrastive_mastery.json")) + p.add_argument("--max-pairs", type=int, default=50000) + p.add_argument("--limit", type=int, default=0) + p.add_argument("--self-test", action="store_true") + return p.parse_args() + + +if __name__ == "__main__": + args = parse_args() + if args.self_test: + self_test() + else: + if not args.features or not args.labels or not args.transcripts: + raise SystemExit("--features, --labels and --transcripts are required") + run(args) From 527eb818fa41a80c222dc9db0c80aa888f9cd99f Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:06:24 +1200 Subject: [PATCH 06/77] Run V73 contrastive mastery experiment in Actions --- .github/workflows/trace-ace-mastery.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/trace-ace-mastery.yml b/.github/workflows/trace-ace-mastery.yml index d5e0273..e37d721 100644 --- a/.github/workflows/trace-ace-mastery.yml +++ b/.github/workflows/trace-ace-mastery.yml @@ -36,6 +36,8 @@ jobs: run: python competitions/trace_the_ace/v71_mastery_events.py --self-test - name: Run supervision audit self-test run: python competitions/trace_the_ace/v72_supervision_audit.py --self-test + - name: Run contrastive mastery self-test + run: python competitions/trace_the_ace/v73_contrastive_mastery.py --self-test full-experiment: if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_full }} @@ -90,6 +92,12 @@ jobs: --transcripts "$TRANSCRIPTS" \ --out v72_supervision_audit.json \ "${EXTRA[@]}" + python competitions/trace_the_ace/v73_contrastive_mastery.py \ + --features "$FEATURES" \ + --labels "$LABELS" \ + --transcripts "$TRANSCRIPTS" \ + --out v73_contrastive_mastery.json \ + "${EXTRA[@]}" - name: Upload aggregate results only uses: actions/upload-artifact@v4 with: @@ -97,4 +105,5 @@ jobs: path: | v71_mastery_results.json v72_supervision_audit.json + v73_contrastive_mastery.json retention-days: 14 From e1ee5877145bc53b097218df7c41cba0ed33b9f9 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:08:51 +1200 Subject: [PATCH 07/77] Add V74 hierarchical semantic objective prior --- .../v74_semantic_objective_prior.py | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 competitions/trace_the_ace/v74_semantic_objective_prior.py diff --git a/competitions/trace_the_ace/v74_semantic_objective_prior.py b/competitions/trace_the_ace/v74_semantic_objective_prior.py new file mode 100644 index 0000000..665794b --- /dev/null +++ b/competitions/trace_the_ace/v74_semantic_objective_prior.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Trace the Ace V74: leakage-safe hierarchical semantic objective prior. + +The objective distribution is extremely long-tailed. V74 estimates objective +difficulty with two levels of shrinkage inside each CV fold: + +1. exact objective posterior when the objective has training support; +2. semantic KNN posterior over objective descriptions for rare/unseen skills. + +This produces a calibrated difficulty prior that can later be combined with the +student-state/mastery branches. No validation labels are used in fitting. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +import pandas as pd +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.metrics.pairwise import cosine_similarity +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import inspect_headers + + +def load_training(features_path: Path, labels_path: Path) -> pd.DataFrame: + fcols = inspect_headers(features_path) + lcols = inspect_headers(labels_path) + print("features columns", fcols) + print("labels columns", lcols) + need = {"response_id", "session_id", "learning_objective"} + if not need.issubset(fcols): + raise ValueError(f"features missing {sorted(need-set(fcols))}") + target = "is_correct" if "is_correct" in lcols else "correct" if "correct" in lcols else None + if target is None: + raise ValueError(f"labels need is_correct or correct; got {lcols}") + f = pd.read_csv(features_path) + y = pd.read_csv(labels_path) + return f.merge(y[["response_id", target]], on="response_id", validate="one_to_one").rename(columns={target: "target"}) + + +def semantic_prior_predict(train: pd.DataFrame, valid: pd.DataFrame, k: int = 8, smooth: float = 20.0): + global_p = float(train.target.mean()) + stats = train.groupby("learning_objective").target.agg(["sum", "count"]) + stats["p"] = (stats["sum"] + smooth * global_p) / (stats["count"] + smooth) + + train_objs = stats.index.astype(str).tolist() + vec = TfidfVectorizer( + analyzer="char_wb", + ngram_range=(3, 5), + min_df=1, + sublinear_tf=True, + norm="l2", + ) + A = vec.fit_transform(train_objs) + B = vec.transform(valid.learning_objective.fillna("").astype(str).tolist()) + sims = cosine_similarity(B, A) + + kk = min(k, sims.shape[1]) + idx = np.argpartition(-sims, kth=kk - 1, axis=1)[:, :kk] + rows = np.arange(len(valid))[:, None] + w = sims[rows, idx] + neighbor_p = stats["p"].to_numpy()[idx] + sem = (w * neighbor_p).sum(axis=1) / (w.sum(axis=1) + 1e-9) + sem = np.where(w.sum(axis=1) > 1e-8, sem, global_p) + + mapped = valid.learning_objective.map(stats["p"]).to_numpy(dtype=float) + missing = np.isnan(mapped) + mapped[missing] = sem[missing] + counts = valid.learning_objective.map(stats["count"]).fillna(0).to_numpy(dtype=float) + + # Rare objectives borrow strength from semantically related skills; common + # objectives rely increasingly on their exact training posterior. + trust = counts / (counts + 10.0) + hierarchical = trust * mapped + (1.0 - trust) * sem + return np.clip(hierarchical, 1e-5, 1 - 1e-5), np.clip(sem, 1e-5, 1 - 1e-5) + + +def evaluate(df: pd.DataFrame, groups, k: int, smooth: float): + y = df.target.to_numpy(dtype=int) + p = np.zeros(len(df), dtype=float) + sem = np.zeros(len(df), dtype=float) + glob = np.zeros(len(df), dtype=float) + folds = [] + for fold, (tr, va) in enumerate(GroupKFold(5).split(df, y, groups), 1): + ph, ps = semantic_prior_predict(df.iloc[tr], df.iloc[va], k=k, smooth=smooth) + p[va], sem[va] = ph, ps + glob[va] = float(df.target.iloc[tr].mean()) + folds.append({ + "fold": fold, + "rows": int(len(va)), + "global_logloss": float(log_loss(y[va], glob[va])), + "hierarchical_logloss": float(log_loss(y[va], ph)), + "semantic_only_logloss": float(log_loss(y[va], ps)), + }) + return { + "global_logloss": float(log_loss(y, glob)), + "hierarchical_logloss": float(log_loss(y, p)), + "semantic_only_logloss": float(log_loss(y, sem)), + "hierarchical_auc": float(roc_auc_score(y, p)), + "delta_vs_global": float(log_loss(y, p) - log_loss(y, glob)), + "folds": folds, + } + + +def run(args): + df = load_training(args.features, args.labels) + if args.limit: + df = df.iloc[: args.limit].copy().reset_index(drop=True) + else: + df = df.reset_index(drop=True) + + objective_group = df.learning_objective_id if "learning_objective_id" in df else df.learning_objective + result = { + "session": evaluate(df, df.session_id, args.k, args.smooth), + "objective": evaluate(df, objective_group, args.k, args.smooth), + "diagnostics": { + "rows": int(len(df)), + "sessions": int(df.session_id.nunique()), + "objectives": int(df.learning_objective.nunique()), + "k": int(args.k), + "smooth": float(args.smooth), + }, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) + + +def self_test(): + train = pd.DataFrame({ + "learning_objective": [ + "multiply decimals by ten", "multiply decimals by ten", + "write fractions as decimals", "write fractions as decimals", + "identify angles", "identify angles", + ], + "target": [1, 1, 0, 0, 1, 0], + }) + valid = pd.DataFrame({"learning_objective": ["multiplying a decimal by 10", "fractions written as decimals"]}) + p, s = semantic_prior_predict(train, valid, k=2, smooth=2) + assert len(p) == 2 and np.all(np.isfinite(p)) + assert p[0] > p[1], (p, s) + print("V74_SELF_TEST_PASS", p.tolist()) + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path) + p.add_argument("--labels", type=Path) + p.add_argument("--out", type=Path, default=Path("v74_semantic_objective_prior.json")) + p.add_argument("--k", type=int, default=8) + p.add_argument("--smooth", type=float, default=20.0) + p.add_argument("--limit", type=int, default=0) + p.add_argument("--self-test", action="store_true") + return p.parse_args() + + +if __name__ == "__main__": + args = parse_args() + if args.self_test: + self_test() + else: + if not args.features or not args.labels: + raise SystemExit("--features and --labels are required") + run(args) From e994980b4edef165b97c57ba298eaeeccea69e38 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:09:07 +1200 Subject: [PATCH 08/77] Run V74 semantic objective prior in Actions --- .github/workflows/trace-ace-mastery.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/trace-ace-mastery.yml b/.github/workflows/trace-ace-mastery.yml index e37d721..95eaa7c 100644 --- a/.github/workflows/trace-ace-mastery.yml +++ b/.github/workflows/trace-ace-mastery.yml @@ -38,6 +38,8 @@ jobs: run: python competitions/trace_the_ace/v72_supervision_audit.py --self-test - name: Run contrastive mastery self-test run: python competitions/trace_the_ace/v73_contrastive_mastery.py --self-test + - name: Run semantic objective prior self-test + run: python competitions/trace_the_ace/v74_semantic_objective_prior.py --self-test full-experiment: if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_full }} @@ -98,6 +100,11 @@ jobs: --transcripts "$TRANSCRIPTS" \ --out v73_contrastive_mastery.json \ "${EXTRA[@]}" + python competitions/trace_the_ace/v74_semantic_objective_prior.py \ + --features "$FEATURES" \ + --labels "$LABELS" \ + --out v74_semantic_objective_prior.json \ + "${EXTRA[@]}" - name: Upload aggregate results only uses: actions/upload-artifact@v4 with: @@ -106,4 +113,5 @@ jobs: v71_mastery_results.json v72_supervision_audit.json v73_contrastive_mastery.json + v74_semantic_objective_prior.json retention-days: 14 From 7ad0a0d8872246b9c54bd9a6e572bc867acec126 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:13:17 +1200 Subject: [PATCH 09/77] Prioritize unseen log loss and transcript canonicalization --- .../trace_the_ace/PLAN_UNSEEN_LOGLOSS.md | 203 ++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 competitions/trace_the_ace/PLAN_UNSEEN_LOGLOSS.md diff --git a/competitions/trace_the_ace/PLAN_UNSEEN_LOGLOSS.md b/competitions/trace_the_ace/PLAN_UNSEEN_LOGLOSS.md new file mode 100644 index 0000000..9624ff3 --- /dev/null +++ b/competitions/trace_the_ace/PLAN_UNSEEN_LOGLOSS.md @@ -0,0 +1,203 @@ +# Trace the Ace — Unseen Log-Loss Plan + +## Primary objective + +The optimization target is **minimum log loss on genuinely unseen/private evaluation data**. Public leaderboard movement, AUC, model novelty, and write-up appeal are secondary. A change is retained only when it improves robust out-of-sample probability quality or provides a clearly orthogonal signal that improves a validated ensemble. + +Formally, prefer models that minimize expected unseen log loss and avoid catastrophic regime failures: + +`E[-y log p - (1-y) log(1-p)]` + +Promotion requires evidence across multiple plausible test regimes, not merely a better mean on one split. + +## Validation hierarchy + +Every material experiment should report at least: + +1. **Session-cold grouped OOF** — no session leakage. +2. **Objective-cold grouped OOF** — exact skills held out. +3. **Hard/rare-objective stress** — long-tail objectives receive explicit scrutiny. +4. **Fold dispersion / worst-fold loss** — a large mean gain that creates a catastrophic regime is not automatically promoted. +5. **Calibration diagnostics** — log loss is the target; overconfident mistakes matter more than ranking gains. + +When historical public scores are available, use them only to audit whether a validation regime is predictive of transfer. Do not derive inference-time constants from leaderboard feedback. + +## Priority order + +### P0 — Preserve strong baselines and validation integrity + +- Keep the best historical lexical/model predictions as an independent view. +- Reconstruct exact OOF predictions whenever possible. +- Do not accumulate features by version number. +- Reject interventions that improve one regime while materially degrading plausible unseen regimes unless they add independently validated ensemble value. + +### P1 — High-signal transcript preprocessing / measurement + +Before larger-model work, convert raw dialogue into cleaner evidence of student knowledge while preserving educationally meaningful variation. + +1. **Conservative speaker-role repair** + - retain original role, repaired role, and repair confidence; + - never globally flip speakers from weak evidence. + +2. **Interaction episode segmentation** + - tutor question -> student answer -> feedback/correction/hint -> retry; + - preserve chronological links between retries and feedback. + +3. **Student-vs-tutor evidence separation** + - tutor exposition is context; + - student production is mastery evidence; + - tutor-confirmed student correctness is weak supervision. + +4. **Low-information turn down-weighting** + - greetings, connection checks, scheduling, generic acknowledgements and boilerplate receive low mastery weight rather than blind deletion. + +5. **Objective-conditioned relevance** + - rank episodes by semantic relevance to each learning objective; + - retain prerequisite/follow-up context when it helps objective transfer. + +6. **Assistance / independence canonicalization** + - distinguish independent correct, correct after prompt, correct after hint, copied/repeated answer, self-correction, unresolved error, repeated error, agreement-only, tutor exposition. + +7. **Chronology and terminal-state emphasis** + - represent transitions such as ERROR -> HINT -> INDEPENDENT_CORRECT; + - terminal independent evidence should be available explicitly, not lost in bagged text. + +8. **Math surface normalization with raw-text preservation** + - normalize Unicode operators, spacing, simple fraction/decimal variants where safe; + - keep original wording as an additional view. + +9. **Multiple retained views** + - raw transcript; + - student-only transcript; + - objective-local transcript; + - canonical episode sequence; + - terminal mastery window. + +The rule is: **remove nuisance variation, not educational variation**. + +### P2 — V71 mastery-event branch + +Extract objective-conditioned micro-assessments and aggregate trajectory features. Evaluate whether these events improve unseen log loss beyond objective-only and lexical baselines. + +### P3 — V72 hidden-supervision audit + +Quantify: + +- multi-objective session frequency; +- same-session mixed outcomes; +- opposite-label contrastive pairs; +- micro-assessment density; +- rare-objective structure. + +Use this only to determine whether the richer training formulations have enough support. + +### P4 — V73 same-session contrastive mastery + +Exploit pairs from the same transcript with different objective outcomes. Same-session differencing suppresses generic session ability and forces the model toward objective-specific mastery evidence. + +Primary question: does the contrastive residual improve held-out row log loss when reintroduced into a calibrated probability model? + +### P5 — V74 semantic/hierarchical objective difficulty + +Model objective difficulty explicitly and shrink rare objectives toward semantically related objectives. Exact objective identity should not be required for useful predictions. + +This branch is retained as an independent probability prior and combined with transcript evidence only through leakage-safe OOF fitting. + +### P6 — V75 canonical student-state trajectory + +Next priority before larger pretrained models. + +Canonical event alphabet should include at minimum: + +- INDEPENDENT_CORRECT +- CORRECT_AFTER_PROMPT +- CORRECT_AFTER_HINT +- SELF_CORRECTION +- TUTOR_CORRECTION +- UNRESOLVED_ERROR +- REPEATED_ERROR +- AGREEMENT_ONLY +- TUTOR_EXPOSITION +- TRANSFER_SUCCESS when reliably detectable + +For each `(session, objective)`, output both the ordered event sequence and compact numeric summaries: terminal state, number of hints, recurrence, recency, independence, correction distance, and objective relevance. + +Compare raw/localized lexical views against canonical-state views under identical folds. + +### P7 — Semantic objective-conditioned retrieval + +Only after P1-P6 are measured. Use a compliant pretrained encoder to retrieve the most objective-relevant episodes from long transcripts. Larger models are justified only if they improve unseen log loss over cheaper lexical/event retrieval. + +### P8 — Latent student-state model + +Combine distinct factors rather than forcing one text classifier to infer all of them implicitly: + +`logit P(correct) = objective_difficulty + session_state + objective_mastery + contrastive_residual + calibrated_residual_views` + +Session state must be inferable from the individual test sample at inference time; no cross-test aggregation is allowed. + +### P9 — Heterogeneous ensemble and calibration + +Retain only genuinely different information channels, e.g.: + +- robust lexical baseline; +- hierarchical objective prior; +- mastery trajectory; +- contrastive residual; +- semantic retrieval/encoder signal. + +Fit ensemble weights strictly OOF. Optimize log loss directly. Test temperature/logit scaling, isotonic or other calibration only when fitted without leakage and when improvement is stable across validation regimes. + +### P10 — Submission discipline + +Full submissions are scarce and should answer causal transfer questions, not tune small hyperparameters. + +A candidate is submission-worthy only when: + +- its unseen-oriented validation is materially better; +- no major plausible regime collapses; +- calibration improves or remains safe; +- runtime and code-execution constraints are satisfied; +- the inference path processes each test sample independently as required by the rules. + +## Promotion rule + +Default decision hierarchy: + +1. Lower aggregate session-cold log loss. +2. Lower or non-inferior hard/rare/objective-cold loss. +3. Lower worst-fold / tail risk. +4. Better calibration, especially fewer high-confidence errors. +5. Orthogonal residual value in a strictly OOF ensemble. +6. Only then consider runtime, elegance, interpretability, or write-up value. + +A model that looks clever but worsens expected unseen log loss is rejected. + +## Current working decomposition + +The leading hypothesis is: + +`P(next correct | transcript, objective)` + +should be decomposed into: + +- `D_o`: semantic objective difficulty; +- `A_s`: broad session/student competence state; +- `M_so`: objective-specific mastery evidence; +- `C_so`: same-session contrastive residual; +- `T_so`: trajectory / independence / recency. + +The transcript is therefore treated as a measurement instrument for latent student state, not merely as a document to classify. + +## Immediate next experiment + +**V75 canonicalization is the next implementation priority.** Build the role-repaired, episode-linked, assistance-aware ordered event representation, then compare: + +1. objective prior only; +2. raw/localized lexical baseline; +3. V71 numeric mastery features; +4. V75 canonical trajectory; +5. lexical + V74 + V75; +6. add V73 contrastive residual. + +Use identical frozen folds. Promote only on unseen-oriented log-loss evidence. From 6ad09d5bb9b7a823795cdfce005253352ce5d63e Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:15:52 +1200 Subject: [PATCH 10/77] Add V75 canonical student-state trajectories --- .../trace_the_ace/v75_canonical_trajectory.py | 425 ++++++++++++++++++ 1 file changed, 425 insertions(+) create mode 100644 competitions/trace_the_ace/v75_canonical_trajectory.py diff --git a/competitions/trace_the_ace/v75_canonical_trajectory.py b/competitions/trace_the_ace/v75_canonical_trajectory.py new file mode 100644 index 0000000..46a35c4 --- /dev/null +++ b/competitions/trace_the_ace/v75_canonical_trajectory.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +"""Trace the Ace V75: canonical student-state trajectories. + +Goal: improve genuinely unseen log loss by removing nuisance variation while +preserving educational variation. This module converts raw tutoring dialogue into +multiple deterministic views plus a compact chronological event sequence. It +never deletes the raw view and never uses labels to construct features. + +The script inspects CSV headers before schema decisions and processes each +(session, objective) independently at inference time. +""" +from __future__ import annotations + +import argparse +import json +import math +import re +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import ( + AGREE_RE, + HINT_RE, + NEG_RE, + POS_RE, + QUESTION_RE, + char_ngram_overlap, + inspect_headers, + jaccard, + load_transcript, + normalize_roles, + tokens, +) + +SEED = 20260815 +LOW_INFO_RE = re.compile( + r"^(?:hi|hello|hey|bye|goodbye|thanks|thank you|yeah|yes|yep|okay|ok|mm+|mhm|uh huh|right|sure|cool|great)[.! ]*$", + re.I, +) +SELF_CORRECT_RE = re.compile(r"\b(?:wait|sorry|actually|i mean|no,? it(?:'s| is)|let me change|correction)\b", re.I) +EXPLAIN_RE = re.compile(r"\b(?:because|so that|therefore|since|i know|the reason|which means)\b", re.I) +TRANSFER_RE = re.compile(r"\b(?:another|next one|different|now try|what about|similar|new example)\b", re.I) +ADMIN_RE = re.compile( + r"\b(?:can you hear me|internet|connection|camera|microphone|lesson today|how are you|good morning|good afternoon|see you|homework portal)\b", + re.I, +) + +MATH_REPLACEMENTS = ( + (re.compile(r"[−–—]"), "-"), + (re.compile(r"[×✕]"), " x "), + (re.compile(r"[÷]"), " / "), + (re.compile(r"\s+"), " "), +) + +STATE_ORDER = { + "UNRESOLVED_ERROR": -2.0, + "CORRECTED_BY_TUTOR": -1.0, + "AGREEMENT_ONLY": -0.25, + "NO_JUDGMENT": 0.0, + "CORRECT_AFTER_HINT": 0.75, + "SELF_CORRECT": 1.0, + "INDEPENDENT_CORRECT": 1.5, + "TRANSFER_SUCCESS": 2.0, +} + + +@dataclass +class CanonicalEvent: + state: str + relevance: float + recency: float + assistance: float + substantive: float + low_info: float + explanation: float + question: str + answer: str + feedback: str + + +def normalize_math(text: str) -> str: + s = str(text).strip() + for pattern, repl in MATH_REPLACEMENTS: + s = pattern.sub(repl, s) + # Standardize a few harmless surface variants while retaining original text in + # separate views. Avoid semantic rewriting of spoken numbers/fractions. + s = re.sub(r"(?<=\d)\s*%", "%", s) + s = re.sub(r"\s*([=+\-/*])\s*", r" \1 ", s) + return re.sub(r"\s+", " ", s).strip() + + +def low_information(text: str) -> bool: + s = str(text).strip() + return bool(LOW_INFO_RE.match(s) or ADMIN_RE.search(s)) + + +def role_repair_with_confidence(df: pd.DataFrame) -> pd.DataFrame: + """Retain original/repaired roles and attach conservative repair confidence.""" + repaired = normalize_roles(df) + conf = np.zeros(len(repaired), dtype=float) + changed = repaired["role_changed"].to_numpy(dtype=bool) + conf[changed] = 0.9 + # Short acknowledgements in suspiciously inverted local pairs are less certain. + for i in np.flatnonzero(changed): + txt = str(repaired.iloc[i]["content"]).strip() + if len(txt.split()) <= 2: + conf[i] = 0.75 + repaired["role_repair_confidence"] = conf + return repaired + + +def objective_relevance(question: str, answer: str, feedback: str, objective: str) -> float: + local = f"{question} {answer} {feedback}" + obj_tok = tokens(objective) + return float(max(jaccard(tokens(local), obj_tok), 0.5 * char_ngram_overlap(local, objective))) + + +def classify_state(question: str, answer: str, feedback: str) -> tuple[str, float]: + """Return canonical state and assistance level in [0, 1].""" + q, a, f = map(str, (question, answer, feedback)) + pos = bool(POS_RE.search(f)) + neg = bool(NEG_RE.search(f)) + hinted = bool(HINT_RE.search(q)) + agreement = bool(AGREE_RE.match(a.strip())) + substantive = len(tokens(a)) >= 2 and not agreement + self_correct = bool(SELF_CORRECT_RE.search(a)) + transfer = bool(TRANSFER_RE.search(q)) + + if neg and substantive: + return "UNRESOLVED_ERROR", 0.0 + if agreement and pos: + return "AGREEMENT_ONLY", 1.0 + if self_correct and pos and substantive: + return "SELF_CORRECT", 0.25 + if pos and substantive and hinted: + return "CORRECT_AFTER_HINT", 0.65 + if pos and substantive and transfer: + return "TRANSFER_SUCCESS", 0.0 + if pos and substantive: + return "INDEPENDENT_CORRECT", 0.0 + if neg or (hinted and agreement): + return "CORRECTED_BY_TUTOR", 1.0 + return "NO_JUDGMENT", float(hinted) + + +def extract_canonical_events(df: pd.DataFrame, objective: str) -> list[CanonicalEvent]: + d = role_repair_with_confidence(df).reset_index(drop=True) + roles = d["role_repaired"].astype(str).str.lower().tolist() + content = d["content"].fillna("").astype(str).tolist() + n = max(1, len(d) - 1) + out: list[CanonicalEvent] = [] + + for qi in range(len(d) - 1): + if roles[qi] != "tutor" or not QUESTION_RE.search(content[qi]): + continue + ai = None + for j in range(qi + 1, min(len(d), qi + 6)): + if roles[j] == "student" and content[j].strip(): + ai = j + break + if roles[j] == "tutor" and QUESTION_RE.search(content[j]) and j > qi + 1: + break + if ai is None: + continue + fi = None + for j in range(ai + 1, min(len(d), ai + 6)): + if roles[j] == "tutor": + fi = j + break + q = content[qi] + a = content[ai] + f = content[fi] if fi is not None else "" + state, assistance = classify_state(q, a, f) + rel = objective_relevance(q, a, f, objective) + agreement = bool(AGREE_RE.match(a.strip())) + substantive = float(len(tokens(a)) >= 2 and not agreement) + out.append( + CanonicalEvent( + state=state, + relevance=rel, + recency=ai / n, + assistance=assistance, + substantive=substantive, + low_info=float(low_information(a)), + explanation=float(bool(EXPLAIN_RE.search(a))), + question=normalize_math(q), + answer=normalize_math(a), + feedback=normalize_math(f), + ) + ) + return out + + +def trajectory_views(df: pd.DataFrame, objective: str) -> tuple[dict[str, str], np.ndarray, dict]: + d = role_repair_with_confidence(df).reset_index(drop=True) + events = extract_canonical_events(d, objective) + + raw = " ".join( + f"[{str(r.role).upper()}] {str(r.content)}" + for r in d[["role", "content"]].itertuples(index=False) + ) + student_only = " ".join( + normalize_math(str(r.content)) + for r in d[["role_repaired", "content"]].itertuples(index=False) + if str(r.role_repaired).lower() == "student" and not low_information(str(r.content)) + ) + + ranked = sorted(events, key=lambda e: e.relevance * (0.2 + 0.8 * e.recency), reverse=True) + local = " ".join( + f"[Q] {e.question} [S] {e.answer} [F] {e.feedback}" + for e in ranked[:12] + ) + canonical = " ".join( + f"[{e.state}] rel={e.relevance:.3f} rec={e.recency:.3f} assist={e.assistance:.2f}" + for e in events + ) + terminal_events = sorted(events, key=lambda e: (e.relevance * (0.25 + 0.75 * e.recency)), reverse=True)[:6] + terminal = " ".join( + f"[{e.state}] [S] {e.answer} [F] {e.feedback}" + for e in terminal_events + ) + + if events: + rel = np.array([e.relevance for e in events], dtype=float) + rec = np.array([e.recency for e in events], dtype=float) + assist = np.array([e.assistance for e in events], dtype=float) + subst = np.array([e.substantive for e in events], dtype=float) + low = np.array([e.low_info for e in events], dtype=float) + expl = np.array([e.explanation for e in events], dtype=float) + state_score = np.array([STATE_ORDER[e.state] for e in events], dtype=float) + w = np.maximum(rel, 0.02) * np.exp(2.5 * (rec - 1.0)) + w /= w.sum() + 1e-12 + top = np.argsort(rel * (0.25 + 0.75 * rec))[-min(6, len(events)):] + tail = np.argsort(rec)[-min(6, len(events)):] + positive = np.isin([e.state for e in events], ["INDEPENDENT_CORRECT", "SELF_CORRECT", "TRANSFER_SUCCESS", "CORRECT_AFTER_HINT"]).astype(float) + errors = np.isin([e.state for e in events], ["UNRESOLVED_ERROR", "CORRECTED_BY_TUTOR"]).astype(float) + independent = np.isin([e.state for e in events], ["INDEPENDENT_CORRECT", "SELF_CORRECT", "TRANSFER_SUCCESS"]).astype(float) + feats = np.array([ + len(events), rel.mean(), rel.max(), np.quantile(rel, 0.75), + rec.mean(), assist.mean(), subst.mean(), low.mean(), expl.mean(), + positive.mean(), errors.mean(), independent.mean(), + float((w * state_score).sum()), float((w * positive).sum()), float((w * errors).sum()), + float((w * independent).sum()), float(state_score[top].mean()), float(state_score[tail].mean()), + float(positive[top].mean()), float(errors[top].mean()), float(independent[top].mean()), + float(positive[tail].mean()), float(errors[tail].mean()), float(independent[tail].mean()), + float(np.max(rec[independent > 0])) if np.any(independent > 0) else 0.0, + float(np.max(rec[errors > 0])) if np.any(errors > 0) else 0.0, + float(np.sum(independent * (rel >= np.quantile(rel, 0.75)))), + float(np.sum(errors * (rel >= np.quantile(rel, 0.75)))), + ], dtype=float) + else: + feats = np.zeros(28, dtype=float) + + views = { + "raw": raw, + "student": student_only, + "local": local, + "canonical": canonical, + "terminal": terminal, + } + meta = { + "events": len(events), + "role_repair_rate": float(d["role_changed"].mean()) if len(d) else 0.0, + "student_chars": len(student_only), + "raw_chars": len(raw), + } + return views, feats, meta + + +def load_training(features: Path, labels: Path) -> pd.DataFrame: + fcols = inspect_headers(features) + lcols = inspect_headers(labels) + print("features columns", fcols) + print("labels columns", lcols) + need = {"response_id", "session_id", "learning_objective"} + if not need.issubset(fcols): + raise ValueError(f"features missing {sorted(need - set(fcols))}") + target = "is_correct" if "is_correct" in lcols else "correct" if "correct" in lcols else None + if target is None: + raise ValueError(f"labels need is_correct or correct; got {lcols}") + f = pd.read_csv(features) + y = pd.read_csv(labels) + return f.merge(y[["response_id", target]], on="response_id", validate="one_to_one").rename(columns={target: "target"}) + + +def folds_for(groups: pd.Series, n_splits: int = 5): + g = groups.astype(str).to_numpy() + dummy = np.zeros(len(g)) + return list(GroupKFold(n_splits=n_splits).split(dummy, dummy, g)) + + +def oof_eval(X, y, folds, name: str): + pred = np.zeros(len(y), dtype=float) + per_fold = [] + for k, (tr, va) in enumerate(folds, 1): + model = LogisticRegression(C=0.25, max_iter=300, solver="liblinear", random_state=SEED) + model.fit(X[tr], y[tr]) + p = np.clip(model.predict_proba(X[va])[:, 1], 1e-5, 1 - 1e-5) + pred[va] = p + row = {"fold": k, "rows": len(va), "logloss": float(log_loss(y[va], p)), "auc": float(roc_auc_score(y[va], p))} + print(name, row) + per_fold.append(row) + return pred, per_fold + + +def run(args) -> None: + frame = load_training(args.features, args.labels) + if args.limit: + frame = frame.iloc[: args.limit].copy() + + cache: dict[str, pd.DataFrame] = {} + view_rows: list[dict[str, str]] = [] + nums, metas = [], [] + for i, row in frame.iterrows(): + sid = str(row.session_id) + if sid not in cache: + cache[sid] = load_transcript(args.transcripts / f"{sid}.csv") + v, n, m = trajectory_views(cache[sid], str(row.learning_objective)) + view_rows.append(v); nums.append(n); metas.append(m) + if (len(view_rows) % 2500) == 0: + print("canonicalized rows", len(view_rows)) + + numeric = np.vstack(nums) + y = frame.target.to_numpy(dtype=int) + hv = HashingVectorizer(n_features=2**18, alternate_sign=False, norm="l2", ngram_range=(1, 2), lowercase=True) + objective = hv.transform(["[OBJECTIVE] " + str(x) for x in frame.learning_objective]) + raw = hv.transform(["[RAW] " + v["raw"] for v in view_rows]) + student = hv.transform(["[STUDENT] " + v["student"] for v in view_rows]) + local = hv.transform(["[LOCAL] " + v["local"] for v in view_rows]) + canonical = hv.transform(["[STATE] " + v["canonical"] for v in view_rows]) + terminal = hv.transform(["[TERMINAL] " + v["terminal"] for v in view_rows]) + z = (numeric - numeric.mean(0)) / (numeric.std(0) + 1e-6) + num = csr_matrix(z) + + matrices = { + "objective_only": objective, + "raw": hstack([objective, raw], format="csr"), + "student": hstack([objective, student], format="csr"), + "local": hstack([objective, local, num], format="csr"), + "canonical": hstack([objective, canonical, terminal, num], format="csr"), + "all_views": hstack([objective, raw, student, local, canonical, terminal, num], format="csr"), + } + session_folds = folds_for(frame.session_id) + objective_groups = frame.learning_objective_id if "learning_objective_id" in frame.columns else frame.learning_objective + objective_folds = folds_for(objective_groups) + + results = {"diagnostics": { + "rows": int(len(frame)), + "sessions": int(frame.session_id.nunique()), + "objectives": int(frame.learning_objective.nunique()), + "mean_events": float(np.mean([m["events"] for m in metas])), + "mean_role_repair_rate": float(np.mean([m["role_repair_rate"] for m in metas])), + "mean_student_to_raw_char_ratio": float(np.mean([m["student_chars"] / max(1, m["raw_chars"]) for m in metas])), + }} + for split, folds in (("session", session_folds), ("objective", objective_folds)): + results[split] = {} + for name, X in matrices.items(): + p, pf = oof_eval(X, y, folds, f"{split}/{name}") + results[split][name] = { + "logloss": float(log_loss(y, p)), + "auc": float(roc_auc_score(y, p)), + "worst_fold_logloss": float(max(r["logloss"] for r in pf)), + "folds": pf, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(results, indent=2)) + print(json.dumps(results, indent=2)) + + +def self_test() -> None: + # Includes an obvious role inversion, a hinted success, an error, and a later + # independent transfer success. Raw view must remain available. + df = pd.DataFrame([ + ["s", "1", "student", "Hi, can you hear me?", "2026-01-01T00:00:00"], + ["s", "2", "tutor", "Yeah.", "2026-01-01T00:00:01"], + ["s", "3", "tutor", "Remember the 7 times table. What is 6 times 7?", "2026-01-01T00:00:02"], + ["s", "4", "student", "42", "2026-01-01T00:00:03"], + ["s", "5", "tutor", "Exactly right.", "2026-01-01T00:00:04"], + ["s", "6", "tutor", "What is 8 times 7?", "2026-01-01T00:00:05"], + ["s", "7", "student", "54", "2026-01-01T00:00:06"], + ["s", "8", "tutor", "Not quite, try again.", "2026-01-01T00:00:07"], + ["s", "9", "tutor", "Another one: what is 9 times 7?", "2026-01-01T00:00:08"], + ["s", "10", "student", "63 because nine sevens are sixty three", "2026-01-01T00:00:09"], + ["s", "11", "tutor", "Perfect, that's right.", "2026-01-01T00:00:10"], + ], columns=["session_id", "utterance_id", "role", "content", "timestamp"]) + views, feats, meta = trajectory_views(df, "multiplying one-digit numbers using the 7 times table") + ev = extract_canonical_events(df, "multiplying one-digit numbers using the 7 times table") + states = [e.state for e in ev] + assert "CORRECT_AFTER_HINT" in states + assert "UNRESOLVED_ERROR" in states + assert "TRANSFER_SUCCESS" in states + assert "Hi, can you hear me?" in views["raw"] + assert "Hi, can you hear me?" not in views["student"] + assert feats.shape == (28,) + assert meta["role_repair_rate"] > 0 + print("V75_SELF_TEST_PASS", json.dumps({"states": states, **meta})) + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path) + p.add_argument("--labels", type=Path) + p.add_argument("--transcripts", type=Path) + p.add_argument("--out", type=Path, default=Path("v75_canonical_trajectory.json")) + p.add_argument("--limit", type=int, default=0) + p.add_argument("--self-test", action="store_true") + return p.parse_args() + + +if __name__ == "__main__": + args = parse_args() + if args.self_test: + self_test() + else: + if not args.features or not args.labels or not args.transcripts: + raise SystemExit("--features, --labels and --transcripts are required") + run(args) From c2f66407f4dfe575c6a4c2cf2709a30567081374 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:16:17 +1200 Subject: [PATCH 11/77] Run V75 canonical trajectory experiment in Actions --- .github/workflows/trace-ace-mastery.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/trace-ace-mastery.yml b/.github/workflows/trace-ace-mastery.yml index 95eaa7c..d916d30 100644 --- a/.github/workflows/trace-ace-mastery.yml +++ b/.github/workflows/trace-ace-mastery.yml @@ -40,6 +40,8 @@ jobs: run: python competitions/trace_the_ace/v73_contrastive_mastery.py --self-test - name: Run semantic objective prior self-test run: python competitions/trace_the_ace/v74_semantic_objective_prior.py --self-test + - name: Run canonical trajectory self-test + run: python competitions/trace_the_ace/v75_canonical_trajectory.py --self-test full-experiment: if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_full }} @@ -105,6 +107,12 @@ jobs: --labels "$LABELS" \ --out v74_semantic_objective_prior.json \ "${EXTRA[@]}" + python competitions/trace_the_ace/v75_canonical_trajectory.py \ + --features "$FEATURES" \ + --labels "$LABELS" \ + --transcripts "$TRANSCRIPTS" \ + --out v75_canonical_trajectory.json \ + "${EXTRA[@]}" - name: Upload aggregate results only uses: actions/upload-artifact@v4 with: @@ -114,4 +122,5 @@ jobs: v72_supervision_audit.json v73_contrastive_mastery.json v74_semantic_objective_prior.json + v75_canonical_trajectory.json retention-days: 14 From c63e5d35a94af556ed36a0fbac0f1ae64c9e153f Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:17:13 +1200 Subject: [PATCH 12/77] Add V76 unseen-logloss validation protocol --- .../trace_the_ace/v76_unseen_validation.py | 210 ++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 competitions/trace_the_ace/v76_unseen_validation.py diff --git a/competitions/trace_the_ace/v76_unseen_validation.py b/competitions/trace_the_ace/v76_unseen_validation.py new file mode 100644 index 0000000..4b9d52f --- /dev/null +++ b/competitions/trace_the_ace/v76_unseen_validation.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Trace the Ace V76: frozen validation protocol for unseen log loss. + +This module builds deterministic, label-independent validation partitions that +stress the failure modes most likely to matter on a private leaderboard: +- session-cold transfer; +- exact-objective-cold transfer; +- semantic-family-cold transfer; +- rare-objective rows; +- long-tail objective rows. + +It intentionally does not choose splits using outcome labels. Predictions from +any candidate model can be scored against the same frozen partitions. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import numpy as np +import pandas as pd +from sklearn.cluster import KMeans +from sklearn.feature_extraction.text import TfidfVectorizer +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import inspect_headers + +SEED = 20260815 + + +def stable_hash(text: str) -> int: + return int(hashlib.sha256(str(text).encode("utf-8")).hexdigest()[:16], 16) + + +def read_frame(features_path: Path, labels_path: Path | None = None) -> pd.DataFrame: + fcols = inspect_headers(features_path) + print("features columns", fcols) + required = {"response_id", "session_id", "learning_objective"} + if not required.issubset(fcols): + raise ValueError(f"features missing {sorted(required - set(fcols))}") + frame = pd.read_csv(features_path) + if labels_path is not None: + lcols = inspect_headers(labels_path) + print("labels columns", lcols) + target = "is_correct" if "is_correct" in lcols else "correct" if "correct" in lcols else None + if target is None: + raise ValueError(f"labels need is_correct or correct; got {lcols}") + labels = pd.read_csv(labels_path) + frame = frame.merge(labels[["response_id", target]], on="response_id", validate="one_to_one") + frame = frame.rename(columns={target: "target"}) + return frame + + +def assign_group_folds(groups: pd.Series, n_splits: int = 5) -> np.ndarray: + groups = groups.astype(str).to_numpy() + dummy = np.zeros(len(groups)) + fold_id = np.full(len(groups), -1, dtype=int) + for k, (_, va) in enumerate(GroupKFold(n_splits=n_splits).split(dummy, dummy, groups)): + fold_id[va] = k + assert np.all(fold_id >= 0) + return fold_id + + +def semantic_family_map(objectives: list[str], n_families: int = 32) -> dict[str, int]: + unique = sorted(set(map(str, objectives))) + if len(unique) <= 1: + return {x: 0 for x in unique} + k = max(2, min(n_families, len(unique))) + vec = TfidfVectorizer(analyzer="char_wb", ngram_range=(3, 5), min_df=1, sublinear_tf=True) + X = vec.fit_transform(unique) + model = KMeans(n_clusters=k, random_state=SEED, n_init=20) + labels = model.fit_predict(X) + # Canonicalize arbitrary KMeans cluster ids by the lexicographically first + # objective in each cluster so assignments remain auditable. + members: dict[int, list[str]] = {} + for obj, lab in zip(unique, labels): + members.setdefault(int(lab), []).append(obj) + ordered = sorted(members, key=lambda lab: min(members[lab])) + canon = {old: new for new, old in enumerate(ordered)} + return {obj: canon[int(lab)] for obj, lab in zip(unique, labels)} + + +def make_protocol(frame: pd.DataFrame, n_splits: int = 5, n_families: int = 32) -> tuple[pd.DataFrame, dict]: + out = frame[["response_id", "session_id", "learning_objective"]].copy() + objective_key = ( + frame["learning_objective_id"].astype(str) + if "learning_objective_id" in frame.columns + else frame["learning_objective"].astype(str) + ) + out["session_fold"] = assign_group_folds(frame.session_id, n_splits) + out["objective_fold"] = assign_group_folds(objective_key, n_splits) + + fam = semantic_family_map(frame.learning_objective.astype(str).tolist(), n_families) + out["semantic_family"] = frame.learning_objective.astype(str).map(fam).astype(int) + out["semantic_family_fold"] = assign_group_folds(out.semantic_family.astype(str), n_splits) + + counts = objective_key.value_counts() + out["objective_count"] = objective_key.map(counts).astype(int) + out["rare_le_5"] = out.objective_count <= 5 + out["rare_le_10"] = out.objective_count <= 10 + out["tail_le_20"] = out.objective_count <= 20 + out["singleton"] = out.objective_count == 1 + + # Stable row hash is useful for exact reproducibility/auditing but is not used + # as a feature or to choose outcomes. + out["row_hash"] = out.response_id.astype(str).map(lambda x: stable_hash(x) % (2**63 - 1)) + + protocol_bytes = out.sort_values("response_id").to_csv(index=False).encode("utf-8") + protocol_sha = hashlib.sha256(protocol_bytes).hexdigest() + summary = { + "rows": int(len(out)), + "sessions": int(frame.session_id.nunique()), + "objectives": int(frame.learning_objective.nunique()), + "semantic_families": int(out.semantic_family.nunique()), + "rare_le_5_rows": int(out.rare_le_5.sum()), + "rare_le_10_rows": int(out.rare_le_10.sum()), + "tail_le_20_rows": int(out.tail_le_20.sum()), + "singleton_rows": int(out.singleton.sum()), + "session_fold_rows": out.session_fold.value_counts().sort_index().astype(int).to_dict(), + "objective_fold_rows": out.objective_fold.value_counts().sort_index().astype(int).to_dict(), + "semantic_family_fold_rows": out.semantic_family_fold.value_counts().sort_index().astype(int).to_dict(), + "protocol_sha256": protocol_sha, + } + return out, summary + + +def binary_logloss(y: np.ndarray, p: np.ndarray) -> float: + p = np.clip(np.asarray(p, dtype=float), 1e-6, 1 - 1e-6) + y = np.asarray(y, dtype=float) + return float(-np.mean(y * np.log(p) + (1 - y) * np.log(1 - p))) + + +def score_predictions(protocol: pd.DataFrame, labels: pd.DataFrame, predictions: pd.DataFrame) -> dict: + target_col = "target" if "target" in labels.columns else "is_correct" if "is_correct" in labels.columns else "correct" + prob_col = "probability" if "probability" in predictions.columns else "prediction" + m = protocol.merge(labels[["response_id", target_col]], on="response_id", validate="one_to_one") + m = m.merge(predictions[["response_id", prob_col]], on="response_id", validate="one_to_one") + y = m[target_col].to_numpy(dtype=float) + p = m[prob_col].to_numpy(dtype=float) + result = {"overall_logloss": binary_logloss(y, p)} + for col in ("rare_le_5", "rare_le_10", "tail_le_20", "singleton"): + mask = m[col].to_numpy(dtype=bool) + result[f"{col}_rows"] = int(mask.sum()) + result[f"{col}_logloss"] = binary_logloss(y[mask], p[mask]) if mask.any() else None + for fold_col in ("session_fold", "objective_fold", "semantic_family_fold"): + losses = [] + for k in sorted(m[fold_col].unique()): + mask = m[fold_col].to_numpy() == k + losses.append(binary_logloss(y[mask], p[mask])) + result[f"{fold_col}_losses"] = losses + result[f"{fold_col}_mean"] = float(np.mean(losses)) + result[f"{fold_col}_worst"] = float(np.max(losses)) + result[f"{fold_col}_std"] = float(np.std(losses)) + + confidence = np.maximum(p, 1 - p) + for q in (0.90, 0.95, 0.99): + threshold = float(np.quantile(confidence, q)) + mask = confidence >= threshold + result[f"confidence_top_{int((1-q)*100)}pct_rows"] = int(mask.sum()) + result[f"confidence_top_{int((1-q)*100)}pct_logloss"] = binary_logloss(y[mask], p[mask]) + return result + + +def self_test() -> None: + frame = pd.DataFrame({ + "response_id": [f"r{i}" for i in range(20)], + "session_id": [f"s{i//2}" for i in range(20)], + "learning_objective": [ + "multiply decimals", "multiply decimals", "divide decimals", "divide decimals", + "add fractions", "add fractions", "subtract fractions", "subtract fractions", + "place value tenths", "place value tenths", "place value hundredths", "place value hundredths", + "factor quadratics", "factor quadratics", "expand brackets", "expand brackets", + "compare money", "compare money", "order integers", "order integers", + ], + }) + p1, s1 = make_protocol(frame, n_splits=2, n_families=4) + p2, s2 = make_protocol(frame.sample(frac=1, random_state=3).reset_index(drop=True), n_splits=2, n_families=4) + # Cluster/fold assignments must be deterministic per response regardless of row order. + a = p1.set_index("response_id")[["session_fold", "objective_fold", "semantic_family"]].sort_index() + b = p2.set_index("response_id")[["session_fold", "objective_fold", "semantic_family"]].sort_index() + assert a.equals(b) + assert s1["rows"] == 20 and s1["semantic_families"] == 4 + print("V76_SELF_TEST_PASS", json.dumps({"protocol_sha256": s1["protocol_sha256"], "families": s1["semantic_families"]})) + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path) + p.add_argument("--labels", type=Path) + p.add_argument("--out-protocol", type=Path, default=Path("v76_validation_protocol.csv")) + p.add_argument("--out-summary", type=Path, default=Path("v76_validation_summary.json")) + p.add_argument("--self-test", action="store_true") + return p.parse_args() + + +if __name__ == "__main__": + args = parse_args() + if args.self_test: + self_test() + else: + if not args.features: + raise SystemExit("--features is required") + frame = read_frame(args.features, args.labels) + protocol, summary = make_protocol(frame) + args.out_protocol.parent.mkdir(parents=True, exist_ok=True) + protocol.to_csv(args.out_protocol, index=False) + args.out_summary.write_text(json.dumps(summary, indent=2)) + print(json.dumps(summary, indent=2)) From 27890a9c30a8e989ddc0a8eaaf7b9c392abab999 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:21:45 +1200 Subject: [PATCH 13/77] Use public Google Drive dataset in Trace the Ace Actions --- .github/workflows/trace-ace-mastery.yml | 49 +++++++++++++++++-------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/.github/workflows/trace-ace-mastery.yml b/.github/workflows/trace-ace-mastery.yml index d916d30..3ca9b54 100644 --- a/.github/workflows/trace-ace-mastery.yml +++ b/.github/workflows/trace-ace-mastery.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: run_full: - description: "Run full experiment if TRACE_ACE_DATA_URL secret is configured" + description: "Run full experiment using public Google Drive dataset" required: false default: false type: boolean @@ -31,7 +31,7 @@ jobs: python-version: "3.12" cache: pip - name: Install experiment dependencies - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown - name: Run mastery extractor self-test run: python competitions/trace_the_ace/v71_mastery_events.py --self-test - name: Run supervision audit self-test @@ -49,7 +49,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 360 env: - TRACE_ACE_DATA_URL: ${{ secrets.TRACE_ACE_DATA_URL }} + TRACE_ACE_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -57,22 +57,41 @@ jobs: python-version: "3.12" cache: pip - name: Install experiment dependencies - run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn - - name: Require private dataset URL + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download public Drive dataset shell: bash run: | - if [ -z "$TRACE_ACE_DATA_URL" ]; then - echo "TRACE_ACE_DATA_URL is not configured; refusing to run without private data transport." - exit 1 - fi - - name: Fetch dataset without logging URL or contents + set -euo pipefail + mkdir -p /tmp/trace_ace + python - <<'PY' + import os, gdown + file_id = os.environ['TRACE_ACE_DRIVE_FILE_ID'] + out = '/tmp/trace_ace/dataset_download' + url = f'https://drive.google.com/uc?id={file_id}' + path = gdown.download(url, out, quiet=False, fuzzy=True) + if not path: + raise SystemExit('Google Drive download failed') + print(f'downloaded to {path}') + PY + - name: Extract dataset archive shell: bash run: | - set +x - mkdir -p /tmp/trace_ace - curl --fail --silent --show-error --location "$TRACE_ACE_DATA_URL" -o /tmp/trace_ace/data.tar.gz - tar -xzf /tmp/trace_ace/data.tar.gz -C /tmp/trace_ace - rm -f /tmp/trace_ace/data.tar.gz + set -euo pipefail + FILE=/tmp/trace_ace/dataset_download + MIME=$(file -b --mime-type "$FILE") + echo "download mime: $MIME" + case "$MIME" in + application/zip) + mkdir -p /tmp/trace_ace/data && unzip -q "$FILE" -d /tmp/trace_ace/data ;; + application/gzip|application/x-gzip) + mkdir -p /tmp/trace_ace/data && tar -xzf "$FILE" -C /tmp/trace_ace/data ;; + application/x-tar) + mkdir -p /tmp/trace_ace/data && tar -xf "$FILE" -C /tmp/trace_ace/data ;; + *) + echo "Unsupported dataset archive MIME: $MIME" >&2 + exit 1 ;; + esac + rm -f "$FILE" - name: Locate inputs and run experiments shell: bash run: | From 799957be4f603c09daba1faff7a506d5ac2ce516 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:23:51 +1200 Subject: [PATCH 14/77] Fix V74 pandas 3 read-only array regression [run-full] --- competitions/trace_the_ace/v74_semantic_objective_prior.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/competitions/trace_the_ace/v74_semantic_objective_prior.py b/competitions/trace_the_ace/v74_semantic_objective_prior.py index 665794b..2367fde 100644 --- a/competitions/trace_the_ace/v74_semantic_objective_prior.py +++ b/competitions/trace_the_ace/v74_semantic_objective_prior.py @@ -67,7 +67,9 @@ def semantic_prior_predict(train: pd.DataFrame, valid: pd.DataFrame, k: int = 8, sem = (w * neighbor_p).sum(axis=1) / (w.sum(axis=1) + 1e-9) sem = np.where(w.sum(axis=1) > 1e-8, sem, global_p) - mapped = valid.learning_objective.map(stats["p"]).to_numpy(dtype=float) + # Pandas 3 can expose a read-only NumPy view from Series.to_numpy(). Copy + # explicitly because we fill unseen objectives below. + mapped = valid.learning_objective.map(stats["p"]).to_numpy(dtype=float).copy() missing = np.isnan(mapped) mapped[missing] = sem[missing] counts = valid.learning_objective.map(stats["count"]).fillna(0).to_numpy(dtype=float) From 91956f2d14e1caa175e28c093f487ae852616a14 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:24:13 +1200 Subject: [PATCH 15/77] Enable one-shot full Trace the Ace run [run-full] --- .github/workflows/trace-ace-mastery.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/trace-ace-mastery.yml b/.github/workflows/trace-ace-mastery.yml index 3ca9b54..6770a2b 100644 --- a/.github/workflows/trace-ace-mastery.yml +++ b/.github/workflows/trace-ace-mastery.yml @@ -44,7 +44,7 @@ jobs: run: python competitions/trace_the_ace/v75_canonical_trajectory.py --self-test full-experiment: - if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_full }} + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run_full) || (github.event_name == 'push' && contains(github.event.head_commit.message, '[run-full]')) }} needs: self-test runs-on: ubuntu-latest timeout-minutes: 360 @@ -101,6 +101,7 @@ jobs: TRANSCRIPTS=$(find /tmp/trace_ace -type d -name 'train_transcripts*' | head -1) test -n "$FEATURES" && test -n "$LABELS" && test -n "$TRANSCRIPTS" LIMIT="${{ inputs.limit }}" + LIMIT="${LIMIT:-0}" EXTRA=() if [ "$LIMIT" != "0" ]; then EXTRA+=(--limit "$LIMIT"); fi python competitions/trace_the_ace/v71_mastery_events.py \ From dd72068ec1902c8cd2da0a2732ff36f5578a66e0 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:27:05 +1200 Subject: [PATCH 16/77] Fix V75 short mathematical answer evidence [run-full] --- .../trace_the_ace/v75_canonical_trajectory.py | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/competitions/trace_the_ace/v75_canonical_trajectory.py b/competitions/trace_the_ace/v75_canonical_trajectory.py index 46a35c4..2477759 100644 --- a/competitions/trace_the_ace/v75_canonical_trajectory.py +++ b/competitions/trace_the_ace/v75_canonical_trajectory.py @@ -52,6 +52,10 @@ r"\b(?:can you hear me|internet|connection|camera|microphone|lesson today|how are you|good morning|good afternoon|see you|homework portal)\b", re.I, ) +MATH_ANSWER_RE = re.compile( + r"(?:\d|[=+\-/*×÷%]|\b(?:half|quarter|third|tenths?|hundredths?|thousandths?)\b)", + re.I, +) MATH_REPLACEMENTS = ( (re.compile(r"[−–—]"), "-"), @@ -102,6 +106,21 @@ def low_information(text: str) -> bool: return bool(LOW_INFO_RE.match(s) or ADMIN_RE.search(s)) +def substantive_answer(text: str) -> bool: + """Recognize real student work without penalizing short mathematical answers. + + A one-token response such as `42`, `0.5`, `3/4`, or `x=6` is high-value + evidence even though it has fewer lexical tokens than a verbal explanation. + Pure acknowledgements remain non-substantive. + """ + s = str(text).strip() + if not s or AGREE_RE.match(s): + return False + if MATH_ANSWER_RE.search(s): + return True + return len(tokens(s)) >= 2 + + def role_repair_with_confidence(df: pd.DataFrame) -> pd.DataFrame: """Retain original/repaired roles and attach conservative repair confidence.""" repaired = normalize_roles(df) @@ -130,7 +149,7 @@ def classify_state(question: str, answer: str, feedback: str) -> tuple[str, floa neg = bool(NEG_RE.search(f)) hinted = bool(HINT_RE.search(q)) agreement = bool(AGREE_RE.match(a.strip())) - substantive = len(tokens(a)) >= 2 and not agreement + substantive = substantive_answer(a) self_correct = bool(SELF_CORRECT_RE.search(a)) transfer = bool(TRANSFER_RE.search(q)) @@ -180,8 +199,7 @@ def extract_canonical_events(df: pd.DataFrame, objective: str) -> list[Canonical f = content[fi] if fi is not None else "" state, assistance = classify_state(q, a, f) rel = objective_relevance(q, a, f, objective) - agreement = bool(AGREE_RE.match(a.strip())) - substantive = float(len(tokens(a)) >= 2 and not agreement) + substantive = float(substantive_answer(a)) out.append( CanonicalEvent( state=state, @@ -394,6 +412,10 @@ def self_test() -> None: views, feats, meta = trajectory_views(df, "multiplying one-digit numbers using the 7 times table") ev = extract_canonical_events(df, "multiplying one-digit numbers using the 7 times table") states = [e.state for e in ev] + assert substantive_answer("42") + assert substantive_answer("0.5") + assert substantive_answer("3/4") + assert not substantive_answer("yeah") assert "CORRECT_AFTER_HINT" in states assert "UNRESOLVED_ERROR" in states assert "TRANSFER_SUCCESS" in states From 97d24469f61a860fc60c3eead596abbf3bea4e2e Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:43:21 +1200 Subject: [PATCH 17/77] Fix gdown 6 dataset fetch [run-full] --- .github/workflows/trace-ace-mastery.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trace-ace-mastery.yml b/.github/workflows/trace-ace-mastery.yml index 6770a2b..161e7da 100644 --- a/.github/workflows/trace-ace-mastery.yml +++ b/.github/workflows/trace-ace-mastery.yml @@ -68,7 +68,7 @@ jobs: file_id = os.environ['TRACE_ACE_DRIVE_FILE_ID'] out = '/tmp/trace_ace/dataset_download' url = f'https://drive.google.com/uc?id={file_id}' - path = gdown.download(url, out, quiet=False, fuzzy=True) + path = gdown.download(url, out, quiet=False) if not path: raise SystemExit('Google Drive download failed') print(f'downloaded to {path}') From bee759bbc083c81d935be3dd47f9f3c6682ea89c Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:45:37 +1200 Subject: [PATCH 18/77] Discover Trace the Ace inputs by schema [run-full] --- .github/workflows/trace-ace-mastery.yml | 42 ++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/.github/workflows/trace-ace-mastery.yml b/.github/workflows/trace-ace-mastery.yml index 161e7da..5829f1e 100644 --- a/.github/workflows/trace-ace-mastery.yml +++ b/.github/workflows/trace-ace-mastery.yml @@ -96,10 +96,44 @@ jobs: shell: bash run: | set -euo pipefail - FEATURES=$(find /tmp/trace_ace -type f -name 'train_features*.csv' | head -1) - LABELS=$(find /tmp/trace_ace -type f -name 'train_labels*.csv' | head -1) - TRANSCRIPTS=$(find /tmp/trace_ace -type d -name 'train_transcripts*' | head -1) - test -n "$FEATURES" && test -n "$LABELS" && test -n "$TRANSCRIPTS" + python - <<'PY' + import csv, shlex + from pathlib import Path + + root = Path('/tmp/trace_ace/data') + features = labels = transcript_dir = None + inspected = 0 + for path in root.rglob('*.csv'): + try: + with path.open('r', encoding='utf-8-sig', errors='ignore', newline='') as f: + header = next(csv.reader(f)) + except Exception: + continue + inspected += 1 + cols = set(header) + if features is None and {'response_id', 'session_id', 'learning_objective'}.issubset(cols): + features = path + print('FEATURE HEADER', path, header) + if labels is None and 'response_id' in cols and ({'is_correct'} <= cols or {'correct'} <= cols): + labels = path + print('LABEL HEADER', path, header) + if transcript_dir is None and {'session_id', 'utterance_id', 'role', 'content', 'timestamp'}.issubset(cols): + transcript_dir = path.parent + print('TRANSCRIPT HEADER', path, header) + if features and labels and transcript_dir: + break + if not (features and labels and transcript_dir): + sample = [str(p) for p in list(root.rglob('*'))[:80]] + raise SystemExit(f'Could not identify inputs by schema after {inspected} CSVs. Sample paths: {sample}') + with open('/tmp/trace_ace/paths.env', 'w') as f: + f.write('FEATURES=' + shlex.quote(str(features)) + '\n') + f.write('LABELS=' + shlex.quote(str(labels)) + '\n') + f.write('TRANSCRIPTS=' + shlex.quote(str(transcript_dir)) + '\n') + print('resolved features:', features) + print('resolved labels:', labels) + print('resolved transcripts:', transcript_dir) + PY + source /tmp/trace_ace/paths.env LIMIT="${{ inputs.limit }}" LIMIT="${LIMIT:-0}" EXTRA=() From 08296d4892b1802e86c499af2ac0a0311c5a6401 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:50:15 +1200 Subject: [PATCH 19/77] Record full-data V74 OOF result --- .../v74_full_training_oof_2026-08-15.json | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 competitions/trace_the_ace/results/v74_full_training_oof_2026-08-15.json diff --git a/competitions/trace_the_ace/results/v74_full_training_oof_2026-08-15.json b/competitions/trace_the_ace/results/v74_full_training_oof_2026-08-15.json new file mode 100644 index 0000000..d969a7f --- /dev/null +++ b/competitions/trace_the_ace/results/v74_full_training_oof_2026-08-15.json @@ -0,0 +1,42 @@ +{ + "diagnostics": { + "rows": 35072, + "sessions": 22821, + "objectives": 398, + "positive_rate": 0.7024692062043796 + }, + "session": { + "global_logloss": 0.6087712429519364, + "hierarchical_logloss": 0.5527343454820751, + "semantic_only_logloss": 0.5828972867379437, + "hierarchical_auc": 0.7057912533493755, + "delta_vs_global": -0.05603689746986129, + "folds": [ + {"fold": 1, "rows": 7015, "global_logloss": 0.609471457418674, "hierarchical_logloss": 0.5527513784708268, "semantic_only_logloss": 0.5815525167928709}, + {"fold": 2, "rows": 7015, "global_logloss": 0.6062975979938539, "hierarchical_logloss": 0.5541348721713956, "semantic_only_logloss": 0.5825157859864996}, + {"fold": 3, "rows": 7014, "global_logloss": 0.6071844652285676, "hierarchical_logloss": 0.5489638944862815, "semantic_only_logloss": 0.5803165236898602}, + {"fold": 4, "rows": 7014, "global_logloss": 0.6138402315997151, "hierarchical_logloss": 0.5547113077644491, "semantic_only_logloss": 0.5866520609625011}, + {"fold": 5, "rows": 7014, "global_logloss": 0.6070627153604012, "hierarchical_logloss": 0.5531100724131053, "semantic_only_logloss": 0.5834497923758504} + ] + }, + "objective": { + "global_logloss": 0.6117153814673667, + "hierarchical_logloss": 0.6017357708418917, + "semantic_only_logloss": 0.6017357708418917, + "hierarchical_auc": 0.5736305336524183, + "delta_vs_global": -0.009979610625475033, + "folds": [ + {"fold": 1, "rows": 7015, "global_logloss": 0.6333952466528439, "hierarchical_logloss": 0.6412045179518634, "semantic_only_logloss": 0.6412045179518634}, + {"fold": 2, "rows": 7015, "global_logloss": 0.5698093295839226, "hierarchical_logloss": 0.5518002193250016, "semantic_only_logloss": 0.5518002193250016}, + {"fold": 3, "rows": 7014, "global_logloss": 0.5604892890991403, "hierarchical_logloss": 0.5491725706211095, "semantic_only_logloss": 0.5491725706211095}, + {"fold": 4, "rows": 7014, "global_logloss": 0.6626437649772484, "hierarchical_logloss": 0.6375851308655951, "semantic_only_logloss": 0.6375851308655951}, + {"fold": 5, "rows": 7014, "global_logloss": 0.6322421607115449, "hierarchical_logloss": 0.6289179077191152, "semantic_only_logloss": 0.6289179077191152} + ] + }, + "provenance": { + "features": "train_features_TMQTWsB.csv", + "labels": "train_labels_44ujmj2.csv", + "transcripts_used": false, + "note": "Aggregate metrics only; no competition data committed." + } +} From 933ff199ce8e0fd898d9a707307f1483f52a676e Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:50:37 +1200 Subject: [PATCH 20/77] Promote measured V74 prior into unseen-logloss plan --- .../trace_the_ace/PLAN_UNSEEN_LOGLOSS.md | 49 +++++++++++++------ 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/competitions/trace_the_ace/PLAN_UNSEEN_LOGLOSS.md b/competitions/trace_the_ace/PLAN_UNSEEN_LOGLOSS.md index 9624ff3..192c37d 100644 --- a/competitions/trace_the_ace/PLAN_UNSEEN_LOGLOSS.md +++ b/competitions/trace_the_ace/PLAN_UNSEEN_LOGLOSS.md @@ -22,11 +22,28 @@ Every material experiment should report at least: When historical public scores are available, use them only to audit whether a validation regime is predictive of transfer. Do not derive inference-time constants from leaderboard feedback. +## Measured anchor — V74 objective difficulty + +Full 35,072-row training evaluation on 2026-08-15 established V74 as a mandatory independent prior: + +- session-grouped global-prior log loss: **0.608771** +- session-grouped V74 hierarchical log loss: **0.552734** +- delta: **-0.056037** +- session-fold range: **0.548964 to 0.554711** +- objective-cold global-prior log loss: **0.611715** +- objective-cold semantic V74 log loss: **0.601736** +- objective-cold delta: **-0.009980** + +V74 uses no transcripts. The session-cold gain shows objective difficulty is a very large component of the target; the objective-cold gain shows semantic transfer between objective descriptions is real but materially weaker and regime-dependent. V74 is therefore an **anchor/prior**, not a complete solution. Every transcript branch should now be measured primarily by residual log-loss improvement beyond leakage-safe V74 OOF predictions. + +Aggregate result: `results/v74_full_training_oof_2026-08-15.json`. + ## Priority order ### P0 — Preserve strong baselines and validation integrity - Keep the best historical lexical/model predictions as an independent view. +- Keep V74 as a mandatory objective-difficulty prior. - Reconstruct exact OOF predictions whenever possible. - Do not accumulate features by version number. - Reject interventions that improve one regime while materially degrading plausible unseen regimes unless they add independently validated ensemble value. @@ -77,7 +94,7 @@ The rule is: **remove nuisance variation, not educational variation**. ### P2 — V71 mastery-event branch -Extract objective-conditioned micro-assessments and aggregate trajectory features. Evaluate whether these events improve unseen log loss beyond objective-only and lexical baselines. +Extract objective-conditioned micro-assessments and aggregate trajectory features. Evaluate whether these events improve unseen log loss **beyond V74 OOF**, not merely beyond a global prior. ### P3 — V72 hidden-supervision audit @@ -95,17 +112,17 @@ Use this only to determine whether the richer training formulations have enough Exploit pairs from the same transcript with different objective outcomes. Same-session differencing suppresses generic session ability and forces the model toward objective-specific mastery evidence. -Primary question: does the contrastive residual improve held-out row log loss when reintroduced into a calibrated probability model? +Primary question: does the contrastive residual improve held-out row log loss when added to V74 plus the strongest transcript evidence? -### P5 — V74 semantic/hierarchical objective difficulty +### P5 — V74 semantic/hierarchical objective difficulty — PROMOTED ANCHOR Model objective difficulty explicitly and shrink rare objectives toward semantically related objectives. Exact objective identity should not be required for useful predictions. -This branch is retained as an independent probability prior and combined with transcript evidence only through leakage-safe OOF fitting. +Measured full-data results promote this branch as an independent probability prior. Future work should preserve its OOF predictions and train transcript models on its residuals or combine through leakage-safe stacking. ### P6 — V75 canonical student-state trajectory -Next priority before larger pretrained models. +Current transcript priority before larger pretrained models. Canonical event alphabet should include at minimum: @@ -122,7 +139,7 @@ Canonical event alphabet should include at minimum: For each `(session, objective)`, output both the ordered event sequence and compact numeric summaries: terminal state, number of hints, recurrence, recency, independence, correction distance, and objective relevance. -Compare raw/localized lexical views against canonical-state views under identical folds. +Compare raw/localized lexical views against canonical-state views under identical folds, always reporting incremental log loss beyond V74. ### P7 — Semantic objective-conditioned retrieval @@ -141,7 +158,7 @@ Session state must be inferable from the individual test sample at inference tim Retain only genuinely different information channels, e.g.: - robust lexical baseline; -- hierarchical objective prior; +- V74 hierarchical objective prior; - mastery trajectory; - contrastive residual; - semantic retrieval/encoder signal. @@ -164,7 +181,7 @@ A candidate is submission-worthy only when: Default decision hierarchy: -1. Lower aggregate session-cold log loss. +1. Lower aggregate session-cold log loss **relative to V74 plus the strongest retained base**. 2. Lower or non-inferior hard/rare/objective-cold loss. 3. Lower worst-fold / tail risk. 4. Better calibration, especially fewer high-confidence errors. @@ -181,7 +198,7 @@ The leading hypothesis is: should be decomposed into: -- `D_o`: semantic objective difficulty; +- `D_o`: semantic objective difficulty — **measured and promoted via V74**; - `A_s`: broad session/student competence state; - `M_so`: objective-specific mastery evidence; - `C_so`: same-session contrastive residual; @@ -191,13 +208,13 @@ The transcript is therefore treated as a measurement instrument for latent stude ## Immediate next experiment -**V75 canonicalization is the next implementation priority.** Build the role-repaired, episode-linked, assistance-aware ordered event representation, then compare: +**V75/V71 residual-on-V74 is now the next decisive experiment.** Use the transcript archive plus the official feature/label CSVs to generate identical session folds and compare: -1. objective prior only; -2. raw/localized lexical baseline; -3. V71 numeric mastery features; -4. V75 canonical trajectory; -5. lexical + V74 + V75; +1. V74 OOF anchor; +2. V74 + V71 numeric mastery features; +3. V74 + raw/localized lexical transcript view; +4. V74 + V75 canonical trajectory; +5. V74 + lexical + V75; 6. add V73 contrastive residual. -Use identical frozen folds. Promote only on unseen-oriented log-loss evidence. +Use identical frozen folds and save OOF predictions for stacking. Promote only on unseen-oriented log-loss evidence. From da0e185a77d46a7b2db5f2a0dcc6dfb29ed354a5 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:52:22 +1200 Subject: [PATCH 21/77] Retune V74 for robust unseen log loss --- .../trace_the_ace/v74_semantic_objective_prior.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/competitions/trace_the_ace/v74_semantic_objective_prior.py b/competitions/trace_the_ace/v74_semantic_objective_prior.py index 2367fde..c7b7669 100644 --- a/competitions/trace_the_ace/v74_semantic_objective_prior.py +++ b/competitions/trace_the_ace/v74_semantic_objective_prior.py @@ -9,6 +9,10 @@ This produces a calibrated difficulty prior that can later be combined with the student-state/mastery branches. No validation labels are used in fitting. + +Defaults k=16, smooth=2.0 were promoted after a full 35,072-row stress grid in +which they improved both session-grouped and objective-cold log loss versus the +previous k=8, smooth=20.0 defaults. The trust denominator remains fixed at 10. """ from __future__ import annotations @@ -42,7 +46,7 @@ def load_training(features_path: Path, labels_path: Path) -> pd.DataFrame: return f.merge(y[["response_id", target]], on="response_id", validate="one_to_one").rename(columns={target: "target"}) -def semantic_prior_predict(train: pd.DataFrame, valid: pd.DataFrame, k: int = 8, smooth: float = 20.0): +def semantic_prior_predict(train: pd.DataFrame, valid: pd.DataFrame, k: int = 16, smooth: float = 2.0): global_p = float(train.target.mean()) stats = train.groupby("learning_objective").target.agg(["sum", "count"]) stats["p"] = (stats["sum"] + smooth * global_p) / (stats["count"] + smooth) @@ -153,8 +157,8 @@ def parse_args(): p.add_argument("--features", type=Path) p.add_argument("--labels", type=Path) p.add_argument("--out", type=Path, default=Path("v74_semantic_objective_prior.json")) - p.add_argument("--k", type=int, default=8) - p.add_argument("--smooth", type=float, default=20.0) + p.add_argument("--k", type=int, default=16) + p.add_argument("--smooth", type=float, default=2.0) p.add_argument("--limit", type=int, default=0) p.add_argument("--self-test", action="store_true") return p.parse_args() From 13b360f5b11745e2af45e4ec5489c9a1b5d7c10c Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:52:32 +1200 Subject: [PATCH 22/77] Record V74 robust hyperparameter stress grid --- .../results/v74_robust_grid_2026-08-15.json | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 competitions/trace_the_ace/results/v74_robust_grid_2026-08-15.json diff --git a/competitions/trace_the_ace/results/v74_robust_grid_2026-08-15.json b/competitions/trace_the_ace/results/v74_robust_grid_2026-08-15.json new file mode 100644 index 0000000..2cf7b3f --- /dev/null +++ b/competitions/trace_the_ace/results/v74_robust_grid_2026-08-15.json @@ -0,0 +1,37 @@ +{ + "search_space": { + "k": [2, 4, 8, 16, 32], + "smooth": [2, 5, 10, 20, 40, 80], + "trust_denom": [2, 5, 10, 20, 40] + }, + "previous_default": { + "k": 8, + "smooth": 20, + "trust_denom": 10, + "session_logloss": 0.5527343454820751, + "objective_logloss": 0.6017357708418917, + "session_worst_fold": 0.5547113077644491, + "objective_worst_fold": 0.6412045179518634 + }, + "promoted": { + "k": 16, + "smooth": 2, + "trust_denom": 10, + "session_logloss": 0.551526796719344, + "objective_logloss": 0.5995980929074717, + "session_worst_fold": 0.553280629957868, + "objective_worst_fold": 0.638792193453735, + "session_delta_vs_previous": -0.0012075487627311, + "objective_delta_vs_previous": -0.00213767793442 + }, + "selection_rule": "Promote only configurations improving session-grouped log loss while remaining non-inferior or better on objective-cold loss and tail risk.", + "provenance": { + "rows": 35072, + "sessions": 22821, + "objectives": 398, + "features": "train_features_TMQTWsB.csv", + "labels": "train_labels_44ujmj2.csv", + "transcripts_used": false, + "note": "Aggregate metrics only; no competition data committed." + } +} From b96458321244b5b655253c4a2e4470daa0f20b23 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:57:56 +1200 Subject: [PATCH 23/77] Fix V71 short mathematical answer evidence --- .../trace_the_ace/v71_mastery_events.py | 181 ++++++------------ 1 file changed, 54 insertions(+), 127 deletions(-) diff --git a/competitions/trace_the_ace/v71_mastery_events.py b/competitions/trace_the_ace/v71_mastery_events.py index 759dca1..f4395a1 100644 --- a/competitions/trace_the_ace/v71_mastery_events.py +++ b/competitions/trace_the_ace/v71_mastery_events.py @@ -29,6 +29,7 @@ SEED = 20260815 TOKEN_RE = re.compile(r"[a-z0-9]+(?:\.[0-9]+)?") +MATH_RE = re.compile(r"(?:\d|[+\-*/=×÷<>]|\b(?:half|quarter|third|tenths?|hundredths?|thousandths?)\b)", re.I) QUESTION_RE = re.compile(r"\?|\b(?:what|which|how|why|can you|could you|tell me|work out|calculate|solve|find)\b", re.I) POS_RE = re.compile(r"\b(?:yes|yeah|correct|right|exactly|perfect|good|great|well done|that's it|thats it|you got it|spot on)\b", re.I) NEG_RE = re.compile(r"\b(?:no|not quite|incorrect|wrong|careful|try again|almost|remember|instead|actually)\b", re.I) @@ -45,6 +46,16 @@ def tokens(text: str) -> set[str]: return {t for t in TOKEN_RE.findall(str(text).lower()) if len(t) > 1 and t not in STOP} +def is_substantive_answer(answer: str) -> bool: + """Keep concise mathematical answers while rejecting acknowledgement-only turns.""" + a = str(answer).strip() + if not a or AGREE_RE.match(a): + return False + if MATH_RE.search(a): + return True + return len(tokens(a)) >= 2 + + def jaccard(a: set[str], b: set[str]) -> float: if not a or not b: return 0.0 @@ -81,12 +92,6 @@ class Episode: def normalize_roles(df: pd.DataFrame) -> pd.DataFrame: - """Conservatively repair only high-confidence local role inversions. - - We do not globally relabel speakers. The repair targets obvious semantic - contradictions such as a row labelled student containing a greeting/question - immediately followed by a row labelled tutor containing a short answer. - """ out = df.copy() roles = out["role"].astype(str).str.lower().tolist() text = out["content"].fillna("").astype(str).tolist() @@ -132,15 +137,12 @@ def extract_episodes(df: pd.DataFrame, objective: str) -> list[Episode]: q, a = content[q_idx], content[a_idx] f = content[f_idx] if f_idx is not None else "" local_text = q + " " + a + " " + f - rel = max( - jaccard(tokens(local_text), objective_tokens), - 0.5 * char_ngram_overlap(local_text, objective), - ) + rel = max(jaccard(tokens(local_text), objective_tokens), 0.5 * char_ngram_overlap(local_text, objective)) pos = 1.0 if POS_RE.search(f) else 0.0 neg = 1.0 if NEG_RE.search(f) else 0.0 hint = 1.0 if HINT_RE.search(q) else 0.0 agreement = 1.0 if AGREE_RE.match(a.strip()) else 0.0 - substantive = float(len(tokens(a)) >= 2 and agreement == 0.0) + substantive = float(is_substantive_answer(a)) recency = a_idx / n episodes.append(Episode(q_idx,a_idx,f_idx,q,a,f,rel,pos,neg,hint,substantive,agreement,recency)) return episodes @@ -151,42 +153,24 @@ def mastery_features(df: pd.DataFrame, objective: str) -> tuple[np.ndarray, str, changed = float(normalize_roles(df)["role_changed"].mean()) if len(df) else 0.0 if not eps: return np.zeros(24, dtype=np.float64), "", {"episodes":0,"role_repair_rate":changed} - rel = np.array([e.relevance for e in eps]) weights = np.maximum(rel, 0.02) * np.exp(2.0 * (np.array([e.recency for e in eps]) - 1.0)) - pos = np.array([e.feedback_pos for e in eps]) - neg = np.array([e.feedback_neg for e in eps]) - hint = np.array([e.hinted for e in eps]) - sub = np.array([e.answer_substantive for e in eps]) - agr = np.array([e.answer_agreement for e in eps]) - rec = np.array([e.recency for e in eps]) - independent_positive = pos * sub * (1.0-hint) - corrected = neg * sub - - k = max(1, min(8, len(eps))) - top = np.argsort(rel)[-k:] - tail = np.argsort(rec)[-k:] + pos = np.array([e.feedback_pos for e in eps]); neg = np.array([e.feedback_neg for e in eps]) + hint = np.array([e.hinted for e in eps]); sub = np.array([e.answer_substantive for e in eps]) + agr = np.array([e.answer_agreement for e in eps]); rec = np.array([e.recency for e in eps]) + independent_positive = pos * sub * (1.0-hint); corrected = neg * sub + k = max(1, min(8, len(eps))); top = np.argsort(rel)[-k:]; tail = np.argsort(rec)[-k:] wsum = float(weights.sum()) + 1e-12 - feats = np.array([ - len(eps), rel.mean(), rel.max(), np.quantile(rel,0.75), - pos.mean(), neg.mean(), hint.mean(), sub.mean(), agr.mean(), - independent_positive.mean(), corrected.mean(), - float((weights*pos).sum()/wsum), float((weights*neg).sum()/wsum), - float((weights*independent_positive).sum()/wsum), - float(pos[top].mean()), float(neg[top].mean()), float(independent_positive[top].mean()), - float(pos[tail].mean()), float(neg[tail].mean()), float(independent_positive[tail].mean()), - float(rec[pos>0].mean()) if np.any(pos>0) else 0.0, - float(rec[neg>0].mean()) if np.any(neg>0) else 0.0, - changed, - float(sum(e.feedback_pos-e.feedback_neg for e in eps[-5:])), + len(eps), rel.mean(), rel.max(), np.quantile(rel,0.75), pos.mean(), neg.mean(), hint.mean(), sub.mean(), agr.mean(), + independent_positive.mean(), corrected.mean(), float((weights*pos).sum()/wsum), float((weights*neg).sum()/wsum), + float((weights*independent_positive).sum()/wsum), float(pos[top].mean()), float(neg[top].mean()), + float(independent_positive[top].mean()), float(pos[tail].mean()), float(neg[tail].mean()), + float(independent_positive[tail].mean()), float(rec[pos>0].mean()) if np.any(pos>0) else 0.0, + float(rec[neg>0].mean()) if np.any(neg>0) else 0.0, changed, float(sum(e.feedback_pos-e.feedback_neg for e in eps[-5:])), ], dtype=np.float64) - ranked = sorted(eps, key=lambda e: (e.relevance * (0.25 + 0.75*e.recency)), reverse=True)[:8] - text = " ".join( - f"[Q]{e.question} [STUDENT]{e.answer} [FEEDBACK]{e.feedback}" - for e in ranked - ) + text = " ".join(f"[Q]{e.question} [STUDENT]{e.answer} [FEEDBACK]{e.feedback}" for e in ranked) meta = {"episodes":len(eps),"role_repair_rate":changed,"max_relevance":float(rel.max())} return feats, text, meta @@ -195,130 +179,73 @@ def load_transcript(path: Path) -> pd.DataFrame: cols = inspect_headers(path) required = {"session_id","utterance_id","role","content","timestamp"} missing = required - set(cols) - if missing: - raise ValueError(f"{path.name}: missing transcript columns {sorted(missing)}; got {cols}") + if missing: raise ValueError(f"{path.name}: missing transcript columns {sorted(missing)}; got {cols}") return pd.read_csv(path) def build_frame(features_path: Path, labels_path: Path, transcript_dir: Path): - fcols = inspect_headers(features_path) - lcols = inspect_headers(labels_path) - print("features columns", fcols) - print("labels columns", lcols) + fcols = inspect_headers(features_path); lcols = inspect_headers(labels_path) + print("features columns", fcols); print("labels columns", lcols) required_f = {"response_id","session_id","learning_objective"} - if not required_f.issubset(fcols): - raise ValueError(f"features missing {sorted(required_f-set(fcols))}") + if not required_f.issubset(fcols): raise ValueError(f"features missing {sorted(required_f-set(fcols))}") target = "is_correct" if "is_correct" in lcols else "correct" if "correct" in lcols else None - if target is None: - raise ValueError(f"labels need is_correct or correct; got {lcols}") - features = pd.read_csv(features_path) - labels = pd.read_csv(labels_path) - frame = features.merge(labels[["response_id",target]], on="response_id", how="inner", validate="one_to_one") - frame = frame.rename(columns={target:"target"}) - return frame + if target is None: raise ValueError(f"labels need is_correct or correct; got {lcols}") + features = pd.read_csv(features_path); labels = pd.read_csv(labels_path) + return features.merge(labels[["response_id",target]], on="response_id", how="inner", validate="one_to_one").rename(columns={target:"target"}) def fixed_group_folds(groups: Iterable[str], n_splits: int = 5): - groups = np.asarray(list(groups)) - dummy = np.zeros(len(groups)) + groups = np.asarray(list(groups)); dummy = np.zeros(len(groups)) return list(GroupKFold(n_splits=n_splits).split(dummy, dummy, groups)) def fit_eval(X, y, folds, name: str): - oof = np.zeros(len(y), dtype=np.float64) - rows = [] + oof = np.zeros(len(y), dtype=np.float64); rows = [] for fold,(tr,va) in enumerate(folds,1): m = LogisticRegression(C=0.35, max_iter=250, solver="liblinear", random_state=SEED) - m.fit(X[tr], y[tr]) - p = np.clip(m.predict_proba(X[va])[:,1], 1e-5, 1-1e-5) - oof[va] = p - rows.append({"fold":fold,"rows":len(va),"logloss":log_loss(y[va],p),"auc":roc_auc_score(y[va],p)}) - print(name, rows[-1]) + m.fit(X[tr], y[tr]); p = np.clip(m.predict_proba(X[va])[:,1], 1e-5, 1-1e-5); oof[va] = p + rows.append({"fold":fold,"rows":len(va),"logloss":log_loss(y[va],p),"auc":roc_auc_score(y[va],p)}); print(name, rows[-1]) return oof, rows def run(args): frame = build_frame(args.features, args.labels, args.transcripts) - cache: dict[str,pd.DataFrame] = {} - numeric, episode_text, meta = [], [], [] + cache: dict[str,pd.DataFrame] = {}; numeric, episode_text, meta = [], [], [] for i,row in frame.iterrows(): sid = str(row.session_id) - if sid not in cache: - cache[sid] = load_transcript(args.transcripts / f"{sid}.csv") - f,t,m = mastery_features(cache[sid], str(row.learning_objective)) - numeric.append(f); episode_text.append(t); meta.append(m) - if args.limit and i+1 >= args.limit: - frame = frame.iloc[:i+1].copy(); break - numeric = np.vstack(numeric) - episode_text = episode_text[:len(frame)] - y = frame.target.to_numpy(dtype=int) - + if sid not in cache: cache[sid] = load_transcript(args.transcripts / f"{sid}.csv") + f,t,m = mastery_features(cache[sid], str(row.learning_objective)); numeric.append(f); episode_text.append(t); meta.append(m) + if args.limit and i+1 >= args.limit: frame = frame.iloc[:i+1].copy(); break + numeric = np.vstack(numeric); episode_text = episode_text[:len(frame)]; y = frame.target.to_numpy(dtype=int) hv = HashingVectorizer(n_features=2**18, alternate_sign=False, norm="l2", ngram_range=(1,2), lowercase=True) objective_text = frame.learning_objective.fillna("").astype(str).tolist() - X_obj = hv.transform(["[OBJECTIVE] "+x for x in objective_text]) - X_ep = hv.transform(["[EPISODES] "+x for x in episode_text]) - X_num = csr_matrix((numeric - numeric.mean(0)) / (numeric.std(0)+1e-6)) - X_base = X_obj - X_full = hstack([X_obj,X_ep,X_num], format="csr") - + X_obj = hv.transform(["[OBJECTIVE] "+x for x in objective_text]); X_ep = hv.transform(["[EPISODES] "+x for x in episode_text]) + X_num = csr_matrix((numeric - numeric.mean(0)) / (numeric.std(0)+1e-6)); X_base = X_obj; X_full = hstack([X_obj,X_ep,X_num], format="csr") session_folds = fixed_group_folds(frame.session_id, 5) objective_folds = fixed_group_folds(frame.learning_objective_id if "learning_objective_id" in frame else frame.learning_objective, 5) results = {} for split,folds in [("session",session_folds),("objective",objective_folds)]: - p0,r0 = fit_eval(X_base,y,folds,f"baseline/{split}") - p1,r1 = fit_eval(X_full,y,folds,f"mastery/{split}") - results[split] = { - "baseline_logloss":float(log_loss(y,p0)), - "mastery_logloss":float(log_loss(y,p1)), - "delta":float(log_loss(y,p1)-log_loss(y,p0)), - "baseline_auc":float(roc_auc_score(y,p0)), - "mastery_auc":float(roc_auc_score(y,p1)), - "folds_baseline":r0,"folds_mastery":r1, - } - results["diagnostics"] = { - "rows":len(frame), - "sessions":int(frame.session_id.nunique()), - "objectives":int(frame.learning_objective.nunique()), - "mean_episode_count":float(np.mean([m["episodes"] for m in meta[:len(frame)]])), - "mean_role_repair_rate":float(np.mean([m["role_repair_rate"] for m in meta[:len(frame)]])), - } - print(json.dumps(results, indent=2)) - args.out.parent.mkdir(parents=True, exist_ok=True) - args.out.write_text(json.dumps(results, indent=2)) + p0,r0 = fit_eval(X_base,y,folds,f"baseline/{split}"); p1,r1 = fit_eval(X_full,y,folds,f"mastery/{split}") + results[split] = {"baseline_logloss":float(log_loss(y,p0)),"mastery_logloss":float(log_loss(y,p1)),"delta":float(log_loss(y,p1)-log_loss(y,p0)),"baseline_auc":float(roc_auc_score(y,p0)),"mastery_auc":float(roc_auc_score(y,p1)),"folds_baseline":r0,"folds_mastery":r1} + results["diagnostics"] = {"rows":len(frame),"sessions":int(frame.session_id.nunique()),"objectives":int(frame.learning_objective.nunique()),"mean_episode_count":float(np.mean([m["episodes"] for m in meta[:len(frame)]])),"mean_role_repair_rate":float(np.mean([m["role_repair_rate"] for m in meta[:len(frame)]]))} + print(json.dumps(results, indent=2)); args.out.parent.mkdir(parents=True, exist_ok=True); args.out.write_text(json.dumps(results, indent=2)) def self_test(): - df = pd.DataFrame([ - ["s","1","tutor","What is 6 times 7?","2026-01-01T00:00:00"], - ["s","2","student","42","2026-01-01T00:00:01"], - ["s","3","tutor","Exactly right, well done.","2026-01-01T00:00:02"], - ["s","4","tutor","Now what is 8 times 7?","2026-01-01T00:00:03"], - ["s","5","student","54","2026-01-01T00:00:04"], - ["s","6","tutor","Not quite, try again.","2026-01-01T00:00:05"], - ], columns=["session_id","utterance_id","role","content","timestamp"]) - f,t,m = mastery_features(df,"multiplying one-digit numbers") - assert m["episodes"] == 2 - assert f[4] > 0 and f[5] > 0 - assert "42" in t and "54" in t + for x in ["42", "0.5", "3/4", "x = 6"]: assert is_substantive_answer(x), x + for x in ["yeah", "ok", "mhm"]: assert not is_substantive_answer(x), x + df = pd.DataFrame([["s","1","tutor","What is 6 times 7?","2026-01-01T00:00:00"],["s","2","student","42","2026-01-01T00:00:01"],["s","3","tutor","Exactly right, well done.","2026-01-01T00:00:02"],["s","4","tutor","Now what is 8 times 7?","2026-01-01T00:00:03"],["s","5","student","54","2026-01-01T00:00:04"],["s","6","tutor","Not quite, try again.","2026-01-01T00:00:05"]], columns=["session_id","utterance_id","role","content","timestamp"]) + f,t,m = mastery_features(df,"multiplying one-digit numbers"); assert m["episodes"] == 2; assert f[7] == 1.0; assert f[4] > 0 and f[5] > 0; assert "42" in t and "54" in t print("SELF_TEST_PASS", json.dumps(m)) def parse_args(): - p = argparse.ArgumentParser() - p.add_argument("--features", type=Path) - p.add_argument("--labels", type=Path) - p.add_argument("--transcripts", type=Path) - p.add_argument("--out", type=Path, default=Path("v71_mastery_results.json")) - p.add_argument("--limit", type=int, default=0) - p.add_argument("--self-test", action="store_true") - return p.parse_args() + p = argparse.ArgumentParser(); p.add_argument("--features", type=Path); p.add_argument("--labels", type=Path); p.add_argument("--transcripts", type=Path); p.add_argument("--out", type=Path, default=Path("v71_mastery_results.json")); p.add_argument("--limit", type=int, default=0); p.add_argument("--self-test", action="store_true"); return p.parse_args() if __name__ == "__main__": a = parse_args() - if a.self_test: - self_test() + if a.self_test: self_test() else: - if not (a.features and a.labels and a.transcripts): - raise SystemExit("--features, --labels and --transcripts are required unless --self-test is used") + if not (a.features and a.labels and a.transcripts): raise SystemExit("--features, --labels and --transcripts are required unless --self-test is used") run(a) From aedf0a7bdd0b2af42b722a89cd5823e1a4d3e1ce Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:58:18 +1200 Subject: [PATCH 24/77] Add V77 leakage-safe incremental mastery stack --- .../v77_incremental_mastery_stack.py | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 competitions/trace_the_ace/v77_incremental_mastery_stack.py diff --git a/competitions/trace_the_ace/v77_incremental_mastery_stack.py b/competitions/trace_the_ace/v77_incremental_mastery_stack.py new file mode 100644 index 0000000..7466d5c --- /dev/null +++ b/competitions/trace_the_ace/v77_incremental_mastery_stack.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Trace the Ace V77: leakage-safe incremental mastery stack over V74. + +Primary question: do transcript-derived mastery features reduce unseen/session-cold +log loss *after* accounting for the strong V74 semantic objective prior? + +For each outer session fold: + 1. fit V74 only on outer-train and predict outer-valid; + 2. generate V74 predictions for outer-train via inner session-grouped OOF; + 3. fit residual correction models on outer-train using only inner-OOF V74 logits + plus mastery numeric and/or episode text features; + 4. apply the fitted correction to the outer-valid V74 logits. + +No outer-valid labels enter base or residual training. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import build_frame, load_transcript, mastery_features, inspect_headers +from v74_semantic_objective_prior import semantic_prior_predict + +SEED = 20260815 + + +def logit(p): + p = np.clip(np.asarray(p, dtype=float), 1e-6, 1 - 1e-6) + return np.log(p / (1 - p)) + + +def sigmoid(x): + return 1.0 / (1.0 + np.exp(-np.asarray(x, dtype=float))) + + +def fixed_group_folds(groups, n_splits=5): + groups = np.asarray(groups) + dummy = np.zeros(len(groups)) + return list(GroupKFold(n_splits=n_splits).split(dummy, dummy, groups)) + + +def inner_v74_oof(train_df: pd.DataFrame, n_splits: int = 4) -> np.ndarray: + groups = train_df.session_id.astype(str).to_numpy() + y = train_df.target.to_numpy(dtype=int) + p = np.zeros(len(train_df), dtype=float) + for tr, va in GroupKFold(n_splits=n_splits).split(train_df, y, groups): + ph, _ = semantic_prior_predict(train_df.iloc[tr], train_df.iloc[va]) + p[va] = ph + return np.clip(p, 1e-6, 1 - 1e-6) + + +def build_mastery(frame: pd.DataFrame, transcript_dir: Path): + cache = {} + numeric, episode_text, meta = [], [], [] + for i, row in frame.iterrows(): + sid = str(row.session_id) + if sid not in cache: + cache[sid] = load_transcript(transcript_dir / f"{sid}.csv") + f, t, m = mastery_features(cache[sid], str(row.learning_objective)) + numeric.append(f); episode_text.append(t); meta.append(m) + if (i + 1) % 5000 == 0: + print("mastery rows", i + 1) + return np.vstack(numeric), episode_text, meta + + +def standardize_train_valid(a_tr, a_va): + mu = a_tr.mean(axis=0) + sd = a_tr.std(axis=0) + 1e-6 + return (a_tr - mu) / sd, (a_va - mu) / sd + + +def fit_correction(y_tr, base_tr, base_va, X_tr_extra=None, X_va_extra=None, C=0.15): + base_logit_tr = csr_matrix(logit(base_tr).reshape(-1, 1)) + base_logit_va = csr_matrix(logit(base_va).reshape(-1, 1)) + Xtr = base_logit_tr if X_tr_extra is None else hstack([base_logit_tr, X_tr_extra], format="csr") + Xva = base_logit_va if X_va_extra is None else hstack([base_logit_va, X_va_extra], format="csr") + m = LogisticRegression(C=C, max_iter=400, solver="liblinear", random_state=SEED) + m.fit(Xtr, y_tr) + return np.clip(m.predict_proba(Xva)[:, 1], 1e-6, 1 - 1e-6) + + +def run(args): + frame = build_frame(args.features, args.labels, args.transcripts).reset_index(drop=True) + if args.limit: + frame = frame.iloc[:args.limit].copy().reset_index(drop=True) + print("rows", len(frame), "sessions", frame.session_id.nunique(), "objectives", frame.learning_objective.nunique()) + + numeric, episode_text, meta = build_mastery(frame, args.transcripts) + y = frame.target.to_numpy(dtype=int) + hv = HashingVectorizer(n_features=2**18, alternate_sign=False, norm="l2", ngram_range=(1,2), lowercase=True) + X_ep_all = hv.transform(["[EPISODES] " + x for x in episode_text]) + + outer = fixed_group_folds(frame.session_id.astype(str).to_numpy(), 5) + preds = {k: np.zeros(len(frame), dtype=float) for k in ["v74", "v74_recal", "v74_num", "v74_ep", "v74_num_ep"]} + folds = [] + + for fold, (tr, va) in enumerate(outer, 1): + train_df, valid_df = frame.iloc[tr], frame.iloc[va] + base_tr = inner_v74_oof(train_df, n_splits=4) + base_va, _ = semantic_prior_predict(train_df, valid_df) + preds["v74"][va] = base_va + + num_tr, num_va = standardize_train_valid(numeric[tr], numeric[va]) + X_num_tr, X_num_va = csr_matrix(num_tr), csr_matrix(num_va) + X_ep_tr, X_ep_va = X_ep_all[tr], X_ep_all[va] + + preds["v74_recal"][va] = fit_correction(y[tr], base_tr, base_va) + preds["v74_num"][va] = fit_correction(y[tr], base_tr, base_va, X_num_tr, X_num_va) + preds["v74_ep"][va] = fit_correction(y[tr], base_tr, base_va, X_ep_tr, X_ep_va) + preds["v74_num_ep"][va] = fit_correction(y[tr], base_tr, base_va, hstack([X_num_tr, X_ep_tr], format="csr"), hstack([X_num_va, X_ep_va], format="csr")) + + row = {"fold": fold, "rows": int(len(va))} + for name, p in preds.items(): + row[name + "_logloss"] = float(log_loss(y[va], p[va])) + folds.append(row) + print(json.dumps(row)) + + summary = {} + base_ll = float(log_loss(y, preds["v74"])) + for name, p in preds.items(): + ll = float(log_loss(y, p)) + summary[name] = {"logloss": ll, "delta_vs_v74": ll - base_ll, "auc": float(roc_auc_score(y, p))} + + counts = frame.groupby("learning_objective_id" if "learning_objective_id" in frame else "learning_objective").response_id.transform("count").to_numpy() + slices = {} + for threshold in (5, 10, 20): + mask = counts <= threshold + slices[f"rare_le_{threshold}"] = {"rows": int(mask.sum())} + if mask.any(): + for name, p in preds.items(): + slices[f"rare_le_{threshold}"][name + "_logloss"] = float(log_loss(y[mask], p[mask])) + + result = { + "summary": summary, + "folds": folds, + "slices": slices, + "diagnostics": { + "rows": int(len(frame)), + "sessions": int(frame.session_id.nunique()), + "objectives": int(frame.learning_objective.nunique()), + "mean_episode_count": float(np.mean([m["episodes"] for m in meta])), + "mean_role_repair_rate": float(np.mean([m["role_repair_rate"] for m in meta])), + }, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) + + +def self_test(): + p = np.array([0.2, 0.5, 0.8]) + assert np.allclose(sigmoid(logit(p)), p) + print("V77_SELF_TEST_PASS") + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path) + p.add_argument("--labels", type=Path) + p.add_argument("--transcripts", type=Path) + p.add_argument("--out", type=Path, default=Path("v77_incremental_mastery_stack.json")) + p.add_argument("--limit", type=int, default=0) + p.add_argument("--self-test", action="store_true") + return p.parse_args() + + +if __name__ == "__main__": + a = parse_args() + if a.self_test: + self_test() + else: + if not (a.features and a.labels and a.transcripts): + raise SystemExit("--features, --labels and --transcripts are required") + run(a) From b32344cc50b59ca2d9865a897a2af0aef9de4408 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:58:48 +1200 Subject: [PATCH 25/77] Wire V76/V77 and private metadata transport into Actions --- .github/workflows/trace-ace-mastery.yml | 142 ++++++++++-------------- 1 file changed, 61 insertions(+), 81 deletions(-) diff --git a/.github/workflows/trace-ace-mastery.yml b/.github/workflows/trace-ace-mastery.yml index 5829f1e..ffea21b 100644 --- a/.github/workflows/trace-ace-mastery.yml +++ b/.github/workflows/trace-ace-mastery.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: run_full: - description: "Run full experiment using public Google Drive dataset" + description: "Run full experiment using public transcripts + private metadata bundle" required: false default: false type: boolean @@ -42,6 +42,10 @@ jobs: run: python competitions/trace_the_ace/v74_semantic_objective_prior.py --self-test - name: Run canonical trajectory self-test run: python competitions/trace_the_ace/v75_canonical_trajectory.py --self-test + - name: Run unseen validation self-test + run: python competitions/trace_the_ace/v76_unseen_validation.py --self-test + - name: Run incremental mastery stack self-test + run: python competitions/trace_the_ace/v77_incremental_mastery_stack.py --self-test full-experiment: if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run_full) || (github.event_name == 'push' && contains(github.event.head_commit.message, '[run-full]')) }} @@ -49,7 +53,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 360 env: - TRACE_ACE_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_URL: ${{ secrets.TRACE_ACE_METADATA_URL }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -58,41 +63,38 @@ jobs: cache: pip - name: Install experiment dependencies run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown - - name: Download public Drive dataset + - name: Require private metadata transport shell: bash run: | set -euo pipefail - mkdir -p /tmp/trace_ace + if [ -z "${TRACE_ACE_METADATA_URL:-}" ]; then + echo "TRACE_ACE_METADATA_URL repository secret is required for the competition feature/label bundle." >&2 + exit 2 + fi + - name: Download public transcript archive + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/transcripts python - <<'PY' import os, gdown - file_id = os.environ['TRACE_ACE_DRIVE_FILE_ID'] - out = '/tmp/trace_ace/dataset_download' - url = f'https://drive.google.com/uc?id={file_id}' - path = gdown.download(url, out, quiet=False) + file_id = os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'] + out = '/tmp/trace_ace/transcripts_download' + path = gdown.download(f'https://drive.google.com/uc?id={file_id}', out, quiet=False) if not path: - raise SystemExit('Google Drive download failed') - print(f'downloaded to {path}') + raise SystemExit('Google Drive transcript download failed') PY - - name: Extract dataset archive + unzip -q /tmp/trace_ace/transcripts_download -d /tmp/trace_ace/transcripts + rm -f /tmp/trace_ace/transcripts_download + - name: Download private feature/label bundle shell: bash run: | set -euo pipefail - FILE=/tmp/trace_ace/dataset_download - MIME=$(file -b --mime-type "$FILE") - echo "download mime: $MIME" - case "$MIME" in - application/zip) - mkdir -p /tmp/trace_ace/data && unzip -q "$FILE" -d /tmp/trace_ace/data ;; - application/gzip|application/x-gzip) - mkdir -p /tmp/trace_ace/data && tar -xzf "$FILE" -C /tmp/trace_ace/data ;; - application/x-tar) - mkdir -p /tmp/trace_ace/data && tar -xf "$FILE" -C /tmp/trace_ace/data ;; - *) - echo "Unsupported dataset archive MIME: $MIME" >&2 - exit 1 ;; - esac - rm -f "$FILE" - - name: Locate inputs and run experiments + mkdir -p /tmp/trace_ace/meta + curl --fail --location --silent --show-error "$TRACE_ACE_METADATA_URL" -o /tmp/trace_ace/meta.zip + unzip -q /tmp/trace_ace/meta.zip -d /tmp/trace_ace/meta + rm -f /tmp/trace_ace/meta.zip + - name: Locate inputs by schema and run experiments shell: bash run: | set -euo pipefail @@ -100,73 +102,49 @@ jobs: import csv, shlex from pathlib import Path - root = Path('/tmp/trace_ace/data') + roots = [Path('/tmp/trace_ace/meta'), Path('/tmp/trace_ace/transcripts')] features = labels = transcript_dir = None - inspected = 0 - for path in root.rglob('*.csv'): - try: - with path.open('r', encoding='utf-8-sig', errors='ignore', newline='') as f: - header = next(csv.reader(f)) - except Exception: - continue - inspected += 1 - cols = set(header) - if features is None and {'response_id', 'session_id', 'learning_objective'}.issubset(cols): - features = path - print('FEATURE HEADER', path, header) - if labels is None and 'response_id' in cols and ({'is_correct'} <= cols or {'correct'} <= cols): - labels = path - print('LABEL HEADER', path, header) - if transcript_dir is None and {'session_id', 'utterance_id', 'role', 'content', 'timestamp'}.issubset(cols): - transcript_dir = path.parent - print('TRANSCRIPT HEADER', path, header) + for root in roots: + for path in root.rglob('*.csv'): + try: + with path.open('r', encoding='utf-8-sig', errors='ignore', newline='') as f: + header = next(csv.reader(f)) + except Exception: + continue + cols = set(header) + if features is None and {'response_id','session_id','learning_objective'}.issubset(cols): + features = path + print('FEATURE HEADER', header) + if labels is None and 'response_id' in cols and ('is_correct' in cols or 'correct' in cols): + labels = path + print('LABEL HEADER', header) + if transcript_dir is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(cols): + transcript_dir = path.parent + print('TRANSCRIPT HEADER', header) + if features and labels and transcript_dir: + break if features and labels and transcript_dir: break if not (features and labels and transcript_dir): - sample = [str(p) for p in list(root.rglob('*'))[:80]] - raise SystemExit(f'Could not identify inputs by schema after {inspected} CSVs. Sample paths: {sample}') - with open('/tmp/trace_ace/paths.env', 'w') as f: + raise SystemExit('Could not identify all Trace the Ace inputs by schema') + with open('/tmp/trace_ace/paths.env','w') as f: f.write('FEATURES=' + shlex.quote(str(features)) + '\n') f.write('LABELS=' + shlex.quote(str(labels)) + '\n') f.write('TRANSCRIPTS=' + shlex.quote(str(transcript_dir)) + '\n') - print('resolved features:', features) - print('resolved labels:', labels) - print('resolved transcripts:', transcript_dir) PY source /tmp/trace_ace/paths.env LIMIT="${{ inputs.limit }}" LIMIT="${LIMIT:-0}" EXTRA=() if [ "$LIMIT" != "0" ]; then EXTRA+=(--limit "$LIMIT"); fi - python competitions/trace_the_ace/v71_mastery_events.py \ - --features "$FEATURES" \ - --labels "$LABELS" \ - --transcripts "$TRANSCRIPTS" \ - --out v71_mastery_results.json \ - "${EXTRA[@]}" - python competitions/trace_the_ace/v72_supervision_audit.py \ - --features "$FEATURES" \ - --labels "$LABELS" \ - --transcripts "$TRANSCRIPTS" \ - --out v72_supervision_audit.json \ - "${EXTRA[@]}" - python competitions/trace_the_ace/v73_contrastive_mastery.py \ - --features "$FEATURES" \ - --labels "$LABELS" \ - --transcripts "$TRANSCRIPTS" \ - --out v73_contrastive_mastery.json \ - "${EXTRA[@]}" - python competitions/trace_the_ace/v74_semantic_objective_prior.py \ - --features "$FEATURES" \ - --labels "$LABELS" \ - --out v74_semantic_objective_prior.json \ - "${EXTRA[@]}" - python competitions/trace_the_ace/v75_canonical_trajectory.py \ - --features "$FEATURES" \ - --labels "$LABELS" \ - --transcripts "$TRANSCRIPTS" \ - --out v75_canonical_trajectory.json \ - "${EXTRA[@]}" + + python competitions/trace_the_ace/v71_mastery_events.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v71_mastery_results.json "${EXTRA[@]}" + python competitions/trace_the_ace/v72_supervision_audit.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v72_supervision_audit.json "${EXTRA[@]}" + python competitions/trace_the_ace/v73_contrastive_mastery.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v73_contrastive_mastery.json "${EXTRA[@]}" + python competitions/trace_the_ace/v74_semantic_objective_prior.py --features "$FEATURES" --labels "$LABELS" --out v74_semantic_objective_prior.json "${EXTRA[@]}" + python competitions/trace_the_ace/v75_canonical_trajectory.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v75_canonical_trajectory.json "${EXTRA[@]}" + python competitions/trace_the_ace/v76_unseen_validation.py --features "$FEATURES" --labels "$LABELS" --out-protocol v76_validation_protocol.csv --out-summary v76_validation_summary.json + python competitions/trace_the_ace/v77_incremental_mastery_stack.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v77_incremental_mastery_stack.json "${EXTRA[@]}" - name: Upload aggregate results only uses: actions/upload-artifact@v4 with: @@ -177,4 +155,6 @@ jobs: v73_contrastive_mastery.json v74_semantic_objective_prior.json v75_canonical_trajectory.json + v76_validation_summary.json + v77_incremental_mastery_stack.json retention-days: 14 From 014d206b9de902e31ab93e204d98d00bfc20bd56 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:20:19 +1200 Subject: [PATCH 26/77] Trace Ace: use public Drive metadata bundle [run-full] --- .github/workflows/trace-ace-mastery.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/trace-ace-mastery.yml b/.github/workflows/trace-ace-mastery.yml index ffea21b..4343095 100644 --- a/.github/workflows/trace-ace-mastery.yml +++ b/.github/workflows/trace-ace-mastery.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: run_full: - description: "Run full experiment using public transcripts + private metadata bundle" + description: "Run full experiment using public Drive bundles" required: false default: false type: boolean @@ -54,7 +54,7 @@ jobs: timeout-minutes: 360 env: TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI - TRACE_ACE_METADATA_URL: ${{ secrets.TRACE_ACE_METADATA_URL }} + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -63,14 +63,6 @@ jobs: cache: pip - name: Install experiment dependencies run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown - - name: Require private metadata transport - shell: bash - run: | - set -euo pipefail - if [ -z "${TRACE_ACE_METADATA_URL:-}" ]; then - echo "TRACE_ACE_METADATA_URL repository secret is required for the competition feature/label bundle." >&2 - exit 2 - fi - name: Download public transcript archive shell: bash run: | @@ -80,18 +72,25 @@ jobs: import os, gdown file_id = os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'] out = '/tmp/trace_ace/transcripts_download' - path = gdown.download(f'https://drive.google.com/uc?id={file_id}', out, quiet=False) + path = gdown.download(id=file_id, output=out, quiet=False) if not path: raise SystemExit('Google Drive transcript download failed') PY unzip -q /tmp/trace_ace/transcripts_download -d /tmp/trace_ace/transcripts rm -f /tmp/trace_ace/transcripts_download - - name: Download private feature/label bundle + - name: Download public feature/label bundle shell: bash run: | set -euo pipefail mkdir -p /tmp/trace_ace/meta - curl --fail --location --silent --show-error "$TRACE_ACE_METADATA_URL" -o /tmp/trace_ace/meta.zip + python - <<'PY' + import os, gdown + file_id = os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID'] + out = '/tmp/trace_ace/meta.zip' + path = gdown.download(id=file_id, output=out, quiet=False) + if not path: + raise SystemExit('Google Drive metadata download failed; confirm Anyone with the link can view') + PY unzip -q /tmp/trace_ace/meta.zip -d /tmp/trace_ace/meta rm -f /tmp/trace_ace/meta.zip - name: Locate inputs by schema and run experiments From 690fe54004200291418923367779107500bb8941 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:55:54 +1200 Subject: [PATCH 27/77] trace ace: freeze official runtime promotion protocol --- .../OFFICIAL_RUNTIME_PROMOTION_PROTOCOL.md | 229 ++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 competitions/trace_the_ace/OFFICIAL_RUNTIME_PROMOTION_PROTOCOL.md diff --git a/competitions/trace_the_ace/OFFICIAL_RUNTIME_PROMOTION_PROTOCOL.md b/competitions/trace_the_ace/OFFICIAL_RUNTIME_PROMOTION_PROTOCOL.md new file mode 100644 index 0000000..4b125f3 --- /dev/null +++ b/competitions/trace_the_ace/OFFICIAL_RUNTIME_PROMOTION_PROTOCOL.md @@ -0,0 +1,229 @@ +# Trace the Ace — Official Runtime Promotion Protocol + +Primary objective: lowest expected unseen/private log loss. Public leaderboard movement is not a promotion criterion by itself. + +## Scope + +Apply this protocol to the winning V71–V77 candidate after frozen OOF evaluation. Do not alter model logic while translating it into runtime form. Any material model change creates a new candidate and must return to OOF evaluation. + +## Gate 0 — Freeze the candidate + +Record in a machine-readable manifest: + +- source Git commit SHA; +- candidate/arm name; +- all hyperparameters; +- frozen fold-definition version/hash; +- overall and per-fold session-cold log loss; +- objective-cold, semantic-family-cold, rare-objective, worst-fold and calibration metrics; +- hashes of all fitted assets; +- expected prediction implementation/version. + +No leaderboard-derived calibration or smoke-test tuning is permitted in this frozen candidate. + +## Gate 1 — `submission_src` contract + +Develop the submission in the official runtime repository's `submission_src/` directory. The packed archive must contain `main.py` at ZIP root. + +Recommended source layout: + +```text +submission_src/ +├── main.py +├── assets/ +│ ├── manifest.json +│ ├── objective_model.* +│ ├── objective_statistics.* +│ ├── vectorizer.* +│ ├── residual_model.* +│ └── calibration.* +└── trace_ace/ + ├── canonicalize.py + ├── mastery.py + ├── features.py + └── predict.py +``` + +`main.py` must: + +1. read `/code_execution/data/test_features.csv`; +2. read required transcript CSVs from `/code_execution/data/test_transcripts/`; +3. construct exactly the frozen representation; +4. load frozen fitted assets only; +5. generate one probability per response independently of unrelated test cases; +6. apply only frozen calibration; +7. use `/code_execution/data/submission_format.csv` as the output contract; +8. write `/code_execution/submission.csv` with exactly `response_id,probability`. + +Forbidden during inference: + +- fitting/updating model weights or fitted feature parameters from test data; +- pseudo-labeling; +- corpus-wide test statistics used as features; +- network calls; +- package installation; +- manual test annotations; +- cross-test-case information that changes a sample prediction. + +## Gate 2 — Research/runtime parity + +Create a frozen held-out fixture and run both the research implementation and `submission_src` implementation. + +For deterministic components require: + +```text +max_abs_probability_difference < 1e-8 +``` + +If a GPU component makes bitwise equality impossible, predeclare a justified tolerance and require no meaningful log-loss change. + +Also require the runtime implementation's held-out log loss to reproduce the frozen candidate within numerical tolerance. Failure means STOP: fix translation, do not submit. + +## Gate 3 — Official container + +Use the official `drivendataorg/tutoring-outcomes-runtime` repository and run, in order: + +```bash +just pull +just pack-submission +just check-submission +just test-submission +``` + +All commands must exit successfully. Preserve `submission/log.txt` and the generated `submission/submission.csv` as validation evidence. + +## Gate 4 — Competition-shaped held-out test + +Create a local, non-Git-tracked data directory from held-out training sessions: + +```text +data/ +├── submission_format.csv +├── test_features.csv +└── test_transcripts/ + └── .csv +``` + +Run: + +```bash +DATA_DIR=/absolute/path/to/data just test-submission +``` + +Require prediction parity with the frozen held-out research implementation and expected held-out log loss. + +Competition data must not be committed to the public repository. + +## Gate 5 — Offline/cold-container audit + +Run with the official default network isolation. Do not enable internet access. + +Audit logs for: + +- attempted downloads; +- Hugging Face Hub/network calls; +- package installation attempts; +- missing assets; +- hidden filesystem assumptions; +- warnings that alter model behavior. + +Require successful cold-container inference with all required assets either packaged or officially preloaded. + +## Gate 6 — Output integrity + +Programmatically require: + +```text +columns == ["response_id", "probability"] +row_count == submission_format row_count +response_ids exactly match submission_format +no duplicate response_ids +no NaN +no +/-inf +0 <= probability <= 1 +``` + +Also inspect probability min/max and extreme-confidence counts. Unexpected tails are an investigation trigger because the competition metric is log loss. + +## Gate 7 — Sample-independence metamorphic audit + +Choose held-out response `x`. Predict it under: + +- A: `x` alone; +- B: `x` plus unrelated held-out responses; +- C: same batch reordered; +- D: `x` plus a different unrelated batch. + +Require: + +```text +p_A(x) == p_B(x) == p_C(x) == p_D(x) +``` + +within the predeclared numerical tolerance. + +This is a hard rule-compliance gate. + +## Gate 8 — Runtime/resource budget + +Record elapsed wall time, peak RAM, peak GPU VRAM if applicable, archive size and output size. + +Hard competition constraints are six hours for full inference and 20 minutes for smoke. Internal promotion target: + +```text +projected full-test runtime < 4 hours +``` + +Prefer substantially more headroom. Any candidate close to a hard resource limit is not promoted without a documented reason. + +## Gate 9 — Platform smoke test + +Only after Gates 0–8 pass, upload the exact packed archive as a smoke test. + +Use smoke only for runtime validation: + +- process starts; +- assets resolve; +- platform paths are correct; +- output is produced; +- runtime is comfortably under 20 minutes; +- logs comply with competition restrictions. + +Do not tune model/calibration from smoke score. + +## Gate 10 — Full-submission promotion + +A candidate may consume one of the limited full submissions only if ALL conditions hold: + +1. session-cold OOF log loss improves on the current frozen champion; +2. no unacceptable objective-cold regression; +3. no unacceptable semantic-family-cold regression; +4. rare-objective behavior is acceptable; +5. worst-fold risk is acceptable; +6. calibration is sound; +7. runtime implementation reproduces frozen predictions; +8. official offline container passes; +9. sample-independence audit passes; +10. platform smoke test passes; +11. projected full runtime has safe headroom; +12. expected private log loss is competitive with the winning target, not merely an incremental public-board improvement. + +Current strategic target: do not spend a full submission on a candidate unless robust evidence makes private log loss around or below 0.595 plausible, unless a later evidence-based threshold supersedes this one. + +## State machine + +```text +OOF WIN + -> FREEZE + -> RUNTIME PARITY + -> OFFICIAL CONTAINER + -> HELD-OUT CONTAINER + -> OFFLINE AUDIT + -> OUTPUT INTEGRITY + -> SAMPLE INDEPENDENCE + -> RESOURCE GATE + -> SMOKE + -> FULL SUBMIT +``` + +Any failed gate returns the candidate to the appropriate earlier stage. Never bypass a failed gate because of a favorable public leaderboard score. From e82e87cb40ce00e5f86b02fbbc21bed3d2956564 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:56:16 +1200 Subject: [PATCH 28/77] trace ace: add runtime promotion validation harness --- .../trace_the_ace/runtime_validate.py | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 competitions/trace_the_ace/runtime_validate.py diff --git a/competitions/trace_the_ace/runtime_validate.py b/competitions/trace_the_ace/runtime_validate.py new file mode 100644 index 0000000..3f41d90 --- /dev/null +++ b/competitions/trace_the_ace/runtime_validate.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Rule/runtime validation helpers for a frozen Trace the Ace submission. + +This script is intentionally model-agnostic. It validates the generated +submission contract, compares frozen research/runtime predictions, and checks +sample-independence fixtures produced by the runtime candidate. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import numpy as np +import pandas as pd + + +def read_headers(path: Path) -> list[str]: + return list(pd.read_csv(path, nrows=0).columns) + + +def validate_output(fmt_path: Path, pred_path: Path) -> dict: + print("submission_format columns:", read_headers(fmt_path)) + print("submission columns:", read_headers(pred_path)) + fmt = pd.read_csv(fmt_path) + pred = pd.read_csv(pred_path) + required = ["response_id", "probability"] + if list(pred.columns) != required: + raise SystemExit(f"FAIL columns: expected {required}, got {list(pred.columns)}") + if len(pred) != len(fmt): + raise SystemExit(f"FAIL row count: {len(pred)} != {len(fmt)}") + if pred.response_id.duplicated().any(): + raise SystemExit("FAIL duplicate response_id") + if pred.response_id.astype(str).tolist() != fmt.response_id.astype(str).tolist(): + raise SystemExit("FAIL response IDs/order differ from submission_format") + p = pred.probability.to_numpy(float) + if not np.isfinite(p).all(): + raise SystemExit("FAIL non-finite probability") + if ((p < 0) | (p > 1)).any(): + raise SystemExit("FAIL probability outside [0,1]") + result = { + "rows": int(len(p)), + "min_probability": float(p.min()), + "max_probability": float(p.max()), + "lt_0p01": int((p < .01).sum()), + "gt_0p99": int((p > .99).sum()), + "quantiles": {str(q): float(np.quantile(p, q)) for q in [0,.001,.01,.05,.5,.95,.99,.999,1]}, + } + print(json.dumps(result, indent=2)) + return result + + +def compare_predictions(a_path: Path, b_path: Path, tol: float) -> dict: + print("reference columns:", read_headers(a_path)) + print("runtime columns:", read_headers(b_path)) + a = pd.read_csv(a_path) + b = pd.read_csv(b_path) + if "response_id" not in a or "probability" not in a or "response_id" not in b or "probability" not in b: + raise SystemExit("FAIL comparison inputs need response_id,probability") + m = a[["response_id","probability"]].merge( + b[["response_id","probability"]], on="response_id", suffixes=("_a","_b"), validate="one_to_one" + ) + if len(m) != len(a) or len(m) != len(b): + raise SystemExit("FAIL comparison response ID sets differ") + d = np.abs(m.probability_a.to_numpy(float) - m.probability_b.to_numpy(float)) + result = {"rows": int(len(m)), "max_abs_difference": float(d.max(initial=0)), "tolerance": tol} + print(json.dumps(result, indent=2)) + if result["max_abs_difference"] > tol: + raise SystemExit("FAIL prediction parity") + return result + + +def independence(paths: list[Path], response_id: str, tol: float) -> dict: + vals = [] + for path in paths: + print(f"{path.name} columns:", read_headers(path)) + df = pd.read_csv(path) + hit = df.loc[df.response_id.astype(str) == str(response_id), "probability"] + if len(hit) != 1: + raise SystemExit(f"FAIL {path}: expected one row for {response_id}, got {len(hit)}") + vals.append(float(hit.iloc[0])) + spread = max(vals) - min(vals) + result = {"response_id": str(response_id), "probabilities": vals, "spread": spread, "tolerance": tol} + print(json.dumps(result, indent=2)) + if spread > tol: + raise SystemExit("FAIL sample independence") + return result + + +def self_test() -> None: + import tempfile + with tempfile.TemporaryDirectory() as td: + root = Path(td) + fmt = pd.DataFrame({"response_id":["a","b"], "probability":[0.5,0.5]}) + p = pd.DataFrame({"response_id":["a","b"], "probability":[0.2,0.8]}) + fmt.to_csv(root/"fmt.csv", index=False); p.to_csv(root/"p.csv", index=False); p.to_csv(root/"q.csv", index=False) + validate_output(root/"fmt.csv", root/"p.csv") + compare_predictions(root/"p.csv", root/"q.csv", 1e-8) + independence([root/"p.csv", root/"q.csv"], "a", 1e-8) + print("SELF TEST PASS") + + +def main() -> None: + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + sub.add_parser("self-test") + p = sub.add_parser("output"); p.add_argument("--format", required=True); p.add_argument("--predictions", required=True) + p = sub.add_parser("parity"); p.add_argument("--reference", required=True); p.add_argument("--runtime", required=True); p.add_argument("--tol", type=float, default=1e-8) + p = sub.add_parser("independence"); p.add_argument("--response-id", required=True); p.add_argument("--predictions", nargs="+", required=True); p.add_argument("--tol", type=float, default=1e-8) + args = ap.parse_args() + if args.cmd == "self-test": self_test() + elif args.cmd == "output": validate_output(Path(args.format), Path(args.predictions)) + elif args.cmd == "parity": compare_predictions(Path(args.reference), Path(args.runtime), args.tol) + else: independence([Path(x) for x in args.predictions], args.response_id, args.tol) + +if __name__ == "__main__": + main() From e88172e216b8ae14f56e4470725a6c2c5c709226 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 14:56:36 +1200 Subject: [PATCH 29/77] trace ace: gate runtime validation harness in CI --- .github/workflows/trace-ace-mastery.yml | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/.github/workflows/trace-ace-mastery.yml b/.github/workflows/trace-ace-mastery.yml index 4343095..4b00ff0 100644 --- a/.github/workflows/trace-ace-mastery.yml +++ b/.github/workflows/trace-ace-mastery.yml @@ -46,6 +46,8 @@ jobs: run: python competitions/trace_the_ace/v76_unseen_validation.py --self-test - name: Run incremental mastery stack self-test run: python competitions/trace_the_ace/v77_incremental_mastery_stack.py --self-test + - name: Run official-runtime validation harness self-test + run: python competitions/trace_the_ace/runtime_validate.py self-test full-experiment: if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run_full) || (github.event_name == 'push' && contains(github.event.head_commit.message, '[run-full]')) }} @@ -100,7 +102,6 @@ jobs: python - <<'PY' import csv, shlex from pathlib import Path - roots = [Path('/tmp/trace_ace/meta'), Path('/tmp/trace_ace/transcripts')] features = labels = transcript_dir = None for root in roots: @@ -112,18 +113,13 @@ jobs: continue cols = set(header) if features is None and {'response_id','session_id','learning_objective'}.issubset(cols): - features = path - print('FEATURE HEADER', header) + features = path; print('FEATURE HEADER', header) if labels is None and 'response_id' in cols and ('is_correct' in cols or 'correct' in cols): - labels = path - print('LABEL HEADER', header) + labels = path; print('LABEL HEADER', header) if transcript_dir is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(cols): - transcript_dir = path.parent - print('TRANSCRIPT HEADER', header) - if features and labels and transcript_dir: - break - if features and labels and transcript_dir: - break + transcript_dir = path.parent; print('TRANSCRIPT HEADER', header) + if features and labels and transcript_dir: break + if features and labels and transcript_dir: break if not (features and labels and transcript_dir): raise SystemExit('Could not identify all Trace the Ace inputs by schema') with open('/tmp/trace_ace/paths.env','w') as f: @@ -132,11 +128,8 @@ jobs: f.write('TRANSCRIPTS=' + shlex.quote(str(transcript_dir)) + '\n') PY source /tmp/trace_ace/paths.env - LIMIT="${{ inputs.limit }}" - LIMIT="${LIMIT:-0}" - EXTRA=() - if [ "$LIMIT" != "0" ]; then EXTRA+=(--limit "$LIMIT"); fi - + LIMIT="${{ inputs.limit }}"; LIMIT="${LIMIT:-0}" + EXTRA=(); if [ "$LIMIT" != "0" ]; then EXTRA+=(--limit "$LIMIT"); fi python competitions/trace_the_ace/v71_mastery_events.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v71_mastery_results.json "${EXTRA[@]}" python competitions/trace_the_ace/v72_supervision_audit.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v72_supervision_audit.json "${EXTRA[@]}" python competitions/trace_the_ace/v73_contrastive_mastery.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v73_contrastive_mastery.json "${EXTRA[@]}" From 2449b7dae24c488f2c19dd536565cf0481b3381f Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 16:10:50 +1200 Subject: [PATCH 30/77] trace ace: add V75 runtime asset exporter --- .../trace_the_ace/train_v75_runtime_assets.py | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 competitions/trace_the_ace/train_v75_runtime_assets.py diff --git a/competitions/trace_the_ace/train_v75_runtime_assets.py b/competitions/trace_the_ace/train_v75_runtime_assets.py new file mode 100644 index 0000000..fe23865 --- /dev/null +++ b/competitions/trace_the_ace/train_v75_runtime_assets.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Fit the promoted V75 all-views model on all training rows and export runtime assets.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import numpy as np +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, trajectory_views, SEED + +N_HASH = 2**18 +VIEW_ORDER = ["objective", "raw", "student", "local", "canonical", "terminal"] + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def run(args) -> None: + frame = load_training(args.features, args.labels).reset_index(drop=True) + cache = {} + view_rows, nums = [], [] + for i, row in frame.iterrows(): + sid = str(row.session_id) + if sid not in cache: + cache[sid] = load_transcript(args.transcripts / f"{sid}.csv") + views, num, _ = trajectory_views(cache[sid], str(row.learning_objective)) + view_rows.append(views) + nums.append(num) + if (i + 1) % 2500 == 0: + print("fitted-feature rows", i + 1) + + numeric = np.vstack(nums).astype(np.float64) + num_mean = numeric.mean(axis=0) + num_std = numeric.std(axis=0) + 1e-6 + z = (numeric - num_mean) / num_std + + hv = HashingVectorizer( + n_features=N_HASH, + alternate_sign=False, + norm="l2", + ngram_range=(1, 2), + lowercase=True, + ) + objective = hv.transform(["[OBJECTIVE] " + str(x) for x in frame.learning_objective]) + raw = hv.transform(["[RAW] " + v["raw"] for v in view_rows]) + student = hv.transform(["[STUDENT] " + v["student"] for v in view_rows]) + local = hv.transform(["[LOCAL] " + v["local"] for v in view_rows]) + canonical = hv.transform(["[STATE] " + v["canonical"] for v in view_rows]) + terminal = hv.transform(["[TERMINAL] " + v["terminal"] for v in view_rows]) + X = hstack([objective, raw, student, local, canonical, terminal, csr_matrix(z)], format="csr") + y = frame.target.to_numpy(dtype=int) + + model = LogisticRegression(C=0.25, max_iter=300, solver="liblinear", random_state=SEED) + model.fit(X, y) + + args.out_dir.mkdir(parents=True, exist_ok=True) + assets = args.out_dir / "v75_runtime_assets.npz" + np.savez_compressed( + assets, + coef=model.coef_.ravel().astype(np.float64), + intercept=np.asarray(model.intercept_, dtype=np.float64), + num_mean=num_mean.astype(np.float64), + num_std=num_std.astype(np.float64), + ) + manifest = { + "candidate": "V75_ALL_VIEWS", + "seed": SEED, + "rows": int(len(frame)), + "sessions": int(frame.session_id.nunique()), + "hash_features_per_view": N_HASH, + "view_order": VIEW_ORDER, + "numeric_features": int(numeric.shape[1]), + "total_features": int(X.shape[1]), + "logistic_C": 0.25, + "solver": "liblinear", + "validation": { + "session_cold_logloss": 0.5395443498930955, + "session_cold_worst_fold": 0.545097717258529, + "objective_cold_logloss": 0.5923494729977774, + "objective_cold_worst_fold": 0.6412260329460859, + }, + "assets_sha256": sha256_file(assets), + } + (args.out_dir / "manifest.json").write_text(json.dumps(manifest, indent=2)) + print(json.dumps(manifest, indent=2)) + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path, required=True) + p.add_argument("--labels", type=Path, required=True) + p.add_argument("--transcripts", type=Path, required=True) + p.add_argument("--out-dir", type=Path, required=True) + return p.parse_args() + +if __name__ == "__main__": + run(parse_args()) From 5517c8e78e9c70bbcb1c49edd3e84f8de2ece2b9 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 16:11:02 +1200 Subject: [PATCH 31/77] trace ace: add official-runtime V75 inference entrypoint --- .../trace_the_ace/runtime_v75/main.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 competitions/trace_the_ace/runtime_v75/main.py diff --git a/competitions/trace_the_ace/runtime_v75/main.py b/competitions/trace_the_ace/runtime_v75/main.py new file mode 100644 index 0000000..5a29fee --- /dev/null +++ b/competitions/trace_the_ace/runtime_v75/main.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Official-runtime inference entrypoint for promoted V75 all-views model.""" +from __future__ import annotations + +from pathlib import Path +import sys + +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer + +HERE = Path(__file__).resolve().parent +ASSETS = HERE / "assets" +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +from v71_mastery_events import load_transcript # noqa: E402 +from v75_canonical_trajectory import trajectory_views # noqa: E402 + +DATA = Path("/code_execution/data") +N_HASH = 2**18 +BATCH = 256 + + +def sigmoid(x): + x = np.asarray(x, dtype=np.float64) + out = np.empty_like(x) + pos = x >= 0 + out[pos] = 1.0 / (1.0 + np.exp(-x[pos])) + e = np.exp(x[~pos]) + out[~pos] = e / (1.0 + e) + return out + + +def build_batch(rows, transcript_cache, hv, num_mean, num_std): + views, nums = [], [] + for row in rows.itertuples(index=False): + sid = str(row.session_id) + if sid not in transcript_cache: + transcript_cache[sid] = load_transcript(DATA / "test_transcripts" / f"{sid}.csv") + v, n, _ = trajectory_views(transcript_cache[sid], str(row.learning_objective)) + views.append(v) + nums.append(n) + numeric = np.vstack(nums).astype(np.float64) + z = (numeric - num_mean) / num_std + objective = hv.transform(["[OBJECTIVE] " + str(x) for x in rows.learning_objective]) + raw = hv.transform(["[RAW] " + v["raw"] for v in views]) + student = hv.transform(["[STUDENT] " + v["student"] for v in views]) + local = hv.transform(["[LOCAL] " + v["local"] for v in views]) + canonical = hv.transform(["[STATE] " + v["canonical"] for v in views]) + terminal = hv.transform(["[TERMINAL] " + v["terminal"] for v in views]) + return hstack([objective, raw, student, local, canonical, terminal, csr_matrix(z)], format="csr") + + +def main(): + features = pd.read_csv(DATA / "test_features.csv") + fmt = pd.read_csv(DATA / "submission_format.csv") + required = {"response_id", "session_id", "learning_objective"} + if not required.issubset(features.columns): + raise RuntimeError("unexpected test_features schema") + + a = np.load(ASSETS / "v75_runtime_assets.npz") + coef = a["coef"].astype(np.float64) + intercept = float(a["intercept"].ravel()[0]) + num_mean = a["num_mean"].astype(np.float64) + num_std = a["num_std"].astype(np.float64) + + hv = HashingVectorizer( + n_features=N_HASH, + alternate_sign=False, + norm="l2", + ngram_range=(1, 2), + lowercase=True, + ) + transcript_cache = {} + pred = np.empty(len(features), dtype=np.float64) + for start in range(0, len(features), BATCH): + stop = min(len(features), start + BATCH) + X = build_batch(features.iloc[start:stop], transcript_cache, hv, num_mean, num_std) + if X.shape[1] != coef.shape[0]: + raise RuntimeError("runtime feature dimension does not match frozen model") + logits = np.asarray(X @ coef).ravel() + intercept + pred[start:stop] = sigmoid(logits) + + generated = pd.DataFrame({"response_id": features.response_id.astype(str), "probability": np.clip(pred, 1e-5, 1 - 1e-5)}) + out = fmt[["response_id"]].astype({"response_id": str}).merge(generated, on="response_id", how="left", validate="one_to_one") + if out.probability.isna().any() or len(out) != len(fmt): + raise RuntimeError("could not generate exactly one prediction per submission row") + out[["response_id", "probability"]].to_csv(HERE / "submission.csv", index=False) + +if __name__ == "__main__": + main() From 37350143b6479b62799cec9a0052a57007290209 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 16:11:46 +1200 Subject: [PATCH 32/77] trace ace: launch V75 official runtime build [runtime-build] --- .github/workflows/trace-ace-runtime.yml | 139 ++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 .github/workflows/trace-ace-runtime.yml diff --git a/.github/workflows/trace-ace-runtime.yml b/.github/workflows/trace-ace-runtime.yml new file mode 100644 index 0000000..dd55e6e --- /dev/null +++ b/.github/workflows/trace-ace-runtime.yml @@ -0,0 +1,139 @@ +name: Trace the Ace V75 official runtime + +on: + workflow_dispatch: + push: + branches: + - agent/trace-ace-mastery-events + paths: + - "competitions/trace_the_ace/runtime_v75/**" + - "competitions/trace_the_ace/train_v75_runtime_assets.py" + - "competitions/trace_the_ace/v71_mastery_events.py" + - "competitions/trace_the_ace/v75_canonical_trajectory.py" + - "competitions/trace_the_ace/runtime_validate.py" + - ".github/workflows/trace-ace-runtime.yml" + +jobs: + build-and-test: + if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[runtime-build]') }} + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + GITHUB_ACTIONS_NO_TTY: "true" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - uses: astral-sh/setup-uv@v6 + - uses: extractions/setup-just@v2 + - name: Install training dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + + - name: Download training inputs + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/transcripts /tmp/trace_ace/meta + python - <<'PY' + import os, gdown + pairs = [ + ('TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID','/tmp/trace_ace/transcripts.zip'), + ('TRACE_ACE_METADATA_DRIVE_FILE_ID','/tmp/trace_ace/meta.zip'), + ] + for env, out in pairs: + p = gdown.download(id=os.environ[env], output=out, quiet=False) + if not p: + raise SystemExit(f'download failed: {env}') + PY + unzip -q /tmp/trace_ace/transcripts.zip -d /tmp/trace_ace/transcripts + unzip -q /tmp/trace_ace/meta.zip -d /tmp/trace_ace/meta + rm -f /tmp/trace_ace/*.zip + + - name: Resolve schemas + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import csv, shlex + from pathlib import Path + roots=[Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')] + features=labels=transcripts=None + for root in roots: + for p in root.rglob('*.csv'): + try: + with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as f: h=next(csv.reader(f)) + except Exception: continue + c=set(h) + if features is None and {'response_id','session_id','learning_objective'}.issubset(c): features=p; print('FEATURE HEADER',h) + if labels is None and 'response_id' in c and ('is_correct' in c or 'correct' in c): labels=p; print('LABEL HEADER',h) + if transcripts is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(c): transcripts=p.parent; print('TRANSCRIPT HEADER',h) + if not all((features,labels,transcripts)): raise SystemExit('failed schema resolution') + with open('/tmp/trace_ace/paths.env','w') as f: + f.write('FEATURES='+shlex.quote(str(features))+'\n') + f.write('LABELS='+shlex.quote(str(labels))+'\n') + f.write('TRANSCRIPTS='+shlex.quote(str(transcripts))+'\n') + PY + + - name: Train promoted V75 all-views assets on all training rows + shell: bash + run: | + set -euo pipefail + source /tmp/trace_ace/paths.env + python competitions/trace_the_ace/train_v75_runtime_assets.py \ + --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" \ + --out-dir /tmp/v75_assets + + - name: Assemble official submission_src + shell: bash + run: | + set -euo pipefail + git clone --depth 1 https://github.com/drivendataorg/tutoring-outcomes-runtime.git /tmp/runtime + rm -rf /tmp/runtime/submission_src/* + cp competitions/trace_the_ace/runtime_v75/main.py /tmp/runtime/submission_src/main.py + cp competitions/trace_the_ace/v71_mastery_events.py /tmp/runtime/submission_src/v71_mastery_events.py + cp competitions/trace_the_ace/v75_canonical_trajectory.py /tmp/runtime/submission_src/v75_canonical_trajectory.py + mkdir -p /tmp/runtime/submission_src/assets + cp /tmp/v75_assets/v75_runtime_assets.npz /tmp/runtime/submission_src/assets/ + cp /tmp/v75_assets/manifest.json /tmp/runtime/submission_src/assets/ + find /tmp/runtime/submission_src -maxdepth 2 -type f -printf '%P %s bytes\n' + + - name: Pack and check with official runtime + working-directory: /tmp/runtime + run: | + just pack-submission + just check-submission + + - name: Pull official runtime image + working-directory: /tmp/runtime + run: just pull + + - name: Test submission offline in official container + working-directory: /tmp/runtime + env: + BLOCK_INTERNET: "true" + GITHUB_ACTIONS_NO_TTY: "true" + SUBMISSION_IMAGE: tutoringoutcomeschallengeprodacr.azurecr.io/tutoring-outcomes-runtime:gpu-latest + run: just test-submission + + - name: Validate generated output contract + shell: bash + run: | + set -euo pipefail + python competitions/trace_the_ace/runtime_validate.py output \ + --format /tmp/runtime/data-demo/submission_format.csv \ + --predictions /tmp/runtime/submission/submission.csv + + - name: Upload candidate and evidence + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v75-official-runtime-candidate + retention-days: 14 + path: | + /tmp/runtime/submission/submission.zip + /tmp/runtime/submission/submission.csv + /tmp/runtime/submission/log.txt + /tmp/v75_assets/manifest.json From d0bdbc349b41c7086e293f20b868b22a5fa52d36 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 17:59:03 +1200 Subject: [PATCH 33/77] trace ace: run V75 parity and independence gates [parity-test] --- .github/workflows/trace-ace-v75-parity.yml | 195 +++++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 .github/workflows/trace-ace-v75-parity.yml diff --git a/.github/workflows/trace-ace-v75-parity.yml b/.github/workflows/trace-ace-v75-parity.yml new file mode 100644 index 0000000..185998e --- /dev/null +++ b/.github/workflows/trace-ace-v75-parity.yml @@ -0,0 +1,195 @@ +name: Trace the Ace V75 parity and independence + +on: + workflow_dispatch: + push: + branches: + - agent/trace-ace-mastery-events + paths: + - ".github/workflows/trace-ace-v75-parity.yml" + +jobs: + parity-independence: + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + + - name: Download immutable promoted V75 candidate artifact + uses: actions/download-artifact@v4 + with: + name: trace-ace-v75-official-runtime-candidate + path: /tmp/candidate_artifact + github-token: ${{ github.token }} + repository: heathSanchez/mathgraph + run-id: 31863804281 + + - name: Extract candidate submission and frozen assets + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/candidate + unzip -q /tmp/candidate_artifact/runtime/submission/submission.zip -d /tmp/candidate + test -f /tmp/candidate/main.py + test -f /tmp/candidate/assets/v75_runtime_assets.npz + sha256sum /tmp/candidate_artifact/runtime/submission/submission.zip | tee /tmp/submission_sha256.txt + + - name: Download held-out fixture source data + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/meta /tmp/trace_ace/transcripts + python - <<'PY' + import os, gdown + p = gdown.download(id=os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID'], output='/tmp/meta.zip', quiet=False) + if not p: raise SystemExit('metadata download failed') + p = gdown.download(id=os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'], output='/tmp/transcripts.zip', quiet=False) + if not p: raise SystemExit('transcript download failed') + PY + unzip -q /tmp/meta.zip -d /tmp/trace_ace/meta + unzip -q /tmp/transcripts.zip -d /tmp/trace_ace/transcripts + + - name: Resolve schemas and build four competition-shaped batches + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import csv, json, shutil + from pathlib import Path + import pandas as pd + + roots=[Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')] + features=None; transcript_dir=None + for root in roots: + for p in root.rglob('*.csv'): + try: + with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as f: h=next(csv.reader(f)) + except Exception: continue + c=set(h) + if features is None and {'response_id','session_id','learning_objective'}.issubset(c): + features=p; print('FEATURE HEADER',h) + if transcript_dir is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(c): + transcript_dir=p.parent; print('TRANSCRIPT HEADER',h) + if not features or not transcript_dir: raise SystemExit('schema discovery failed') + f=pd.read_csv(features).reset_index(drop=True) + target=f.iloc[[0]].copy() + target_id=str(target.iloc[0].response_id) + # A: target alone; B: target + unrelated; C: same as B reordered; D: target + different unrelated batch. + A=target + B=pd.concat([target,f.iloc[1:32]],ignore_index=True).drop_duplicates('response_id') + C=B.sample(frac=1,random_state=20260815).reset_index(drop=True) + D=pd.concat([target,f.iloc[100:132]],ignore_index=True).drop_duplicates('response_id') + batches={'A':A,'B':B,'C':C,'D':D} + for name,df in batches.items(): + root=Path('/tmp/batches')/name + tdir=root/'test_transcripts'; tdir.mkdir(parents=True,exist_ok=True) + df.to_csv(root/'test_features.csv',index=False) + pd.DataFrame({'response_id':df.response_id.astype(str),'probability':0.5}).to_csv(root/'submission_format.csv',index=False) + for sid in df.session_id.astype(str).unique(): + src=transcript_dir/f'{sid}.csv' + if not src.exists(): raise SystemExit(f'missing transcript {src}') + shutil.copy2(src,tdir/src.name) + Path('/tmp/fixture.json').write_text(json.dumps({'target_response_id':target_id,'features':str(features),'transcript_dir':str(transcript_dir)},indent=2)) + print('TARGET',target_id) + print({k:len(v) for k,v in batches.items()}) + PY + + - name: Generate independent research-reference probabilities + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import json, sys + from pathlib import Path + import numpy as np, pandas as pd + from scipy.sparse import csr_matrix, hstack + from sklearn.feature_extraction.text import HashingVectorizer + + sys.path.insert(0, str(Path('competitions/trace_the_ace').resolve())) + from v71_mastery_events import load_transcript + from v75_canonical_trajectory import trajectory_views + + a=np.load('/tmp/candidate/assets/v75_runtime_assets.npz') + coef=a['coef'].astype(np.float64); intercept=float(a['intercept'].ravel()[0]) + mean=a['num_mean'].astype(np.float64); std=a['num_std'].astype(np.float64) + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + def sigmoid(x): return 1/(1+np.exp(-x)) + for name in ['A','B','C','D']: + root=Path('/tmp/batches')/name; df=pd.read_csv(root/'test_features.csv') + cache={}; views=[]; nums=[] + for r in df.itertuples(index=False): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(root/'test_transcripts'/f'{sid}.csv') + v,n,_=trajectory_views(cache[sid],str(r.learning_objective)); views.append(v); nums.append(n) + z=(np.vstack(nums).astype(np.float64)-mean)/std + parts=[ + hv.transform(['[OBJECTIVE] '+str(x) for x in df.learning_objective]), + hv.transform(['[RAW] '+v['raw'] for v in views]), + hv.transform(['[STUDENT] '+v['student'] for v in views]), + hv.transform(['[LOCAL] '+v['local'] for v in views]), + hv.transform(['[STATE] '+v['canonical'] for v in views]), + hv.transform(['[TERMINAL] '+v['terminal'] for v in views]), + csr_matrix(z), + ] + X=hstack(parts,format='csr') + p=np.clip(sigmoid(np.asarray(X@coef).ravel()+intercept),1e-5,1-1e-5) + pd.DataFrame({'response_id':df.response_id.astype(str),'probability':p}).to_csv(f'/tmp/reference_{name}.csv',index=False) + PY + + - name: Run immutable runtime candidate on all four batches + shell: bash + run: | + set -euo pipefail + sudo rm -rf /code_execution + sudo mkdir -p /code_execution + sudo chmod 0777 /code_execution + for NAME in A B C D; do + rm -rf /code_execution/data /code_execution/run + cp -a "/tmp/batches/$NAME" /code_execution/data + cp -a /tmp/candidate /code_execution/run + (cd /code_execution/run && python main.py) + cp /code_execution/run/submission.csv "/tmp/runtime_${NAME}.csv" + done + + - name: Gate 2 research/runtime parity + shell: bash + run: | + set -euo pipefail + for NAME in A B C D; do + python competitions/trace_the_ace/runtime_validate.py parity --reference "/tmp/reference_${NAME}.csv" --runtime "/tmp/runtime_${NAME}.csv" --tol 1e-8 + done + + - name: Gate 7 sample-independence metamorphic audit + shell: bash + run: | + set -euo pipefail + TARGET=$(python -c "import json; print(json.load(open('/tmp/fixture.json'))['target_response_id'])") + python competitions/trace_the_ace/runtime_validate.py independence --response-id "$TARGET" --predictions /tmp/runtime_A.csv /tmp/runtime_B.csv /tmp/runtime_C.csv /tmp/runtime_D.csv --tol 1e-8 + + - name: Validate every runtime output contract + shell: bash + run: | + set -euo pipefail + for NAME in A B C D; do + python competitions/trace_the_ace/runtime_validate.py output --format "/tmp/batches/${NAME}/submission_format.csv" --predictions "/tmp/runtime_${NAME}.csv" + done + + - name: Upload parity evidence + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v75-parity-independence-evidence + path: | + /tmp/submission_sha256.txt + /tmp/fixture.json + /tmp/reference_*.csv + /tmp/runtime_*.csv + retention-days: 14 From a68b7083f5ecbaf795e77dd4ddf91e65d53b222a Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 18:23:05 +1200 Subject: [PATCH 34/77] trace ace: add V78 semantic episode and ensemble experiment --- .../trace_the_ace/v78_seismic_semantic.py | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 competitions/trace_the_ace/v78_seismic_semantic.py diff --git a/competitions/trace_the_ace/v78_seismic_semantic.py b/competitions/trace_the_ace/v78_seismic_semantic.py new file mode 100644 index 0000000..fdcad10 --- /dev/null +++ b/competitions/trace_the_ace/v78_seismic_semantic.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""V78 seismic test: pretrained semantic interaction + learned episode mastery + V75 ensemble. + +Tests three model-class shifts under identical frozen group folds: +1) objective<->dialogue pretrained semantic interaction; +2) learned episode/trajectory mastery from semantic episode views; +3) nested-safe convex ensemble with the sparse V75 all-views model. + +Development-time model: sentence-transformers/all-MiniLM-L6-v2 (open, and preloaded in official runtime). +Only aggregate metrics are written. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold +from sklearn.preprocessing import StandardScaler +from sentence_transformers import SentenceTransformer + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, trajectory_views, SEED + + +def folds(groups, n=5): + g=groups.astype(str).to_numpy(); z=np.zeros(len(g)) + return list(GroupKFold(n_splits=n).split(z,z,g)) + +def sigmoid(x): + x=np.asarray(x,float); return 1/(1+np.exp(-np.clip(x,-40,40))) + +def encode(model, texts, batch=128): + return model.encode(texts,batch_size=batch,show_progress_bar=True,normalize_embeddings=True,convert_to_numpy=True).astype(np.float32) + +def dense_semantic_features(Eo, Es, El, Et, numeric): + # Explicit cross-view interaction features, not just independent embeddings. + cos_os=np.sum(Eo*Es,1,keepdims=True); cos_ol=np.sum(Eo*El,1,keepdims=True); cos_ot=np.sum(Eo*Et,1,keepdims=True) + cos_sl=np.sum(Es*El,1,keepdims=True); cos_lt=np.sum(El*Et,1,keepdims=True) + # Pairwise products preserve dimensions while explicitly representing relevance/alignment. + prod_ol=Eo*El; prod_ot=Eo*Et + return np.hstack([Eo,Es,El,Et,prod_ol,prod_ot,cos_os,cos_ol,cos_ot,cos_sl,cos_lt,numeric]).astype(np.float32) + +def build_v75_sparse(frame, view_rows, numeric): + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + obj=hv.transform(['[OBJECTIVE] '+str(x) for x in frame.learning_objective]) + raw=hv.transform(['[RAW] '+v['raw'] for v in view_rows]) + stu=hv.transform(['[STUDENT] '+v['student'] for v in view_rows]) + loc=hv.transform(['[LOCAL] '+v['local'] for v in view_rows]) + can=hv.transform(['[STATE] '+v['canonical'] for v in view_rows]) + ter=hv.transform(['[TERMINAL] '+v['terminal'] for v in view_rows]) + z=(numeric-numeric.mean(0))/(numeric.std(0)+1e-6) + return hstack([obj,raw,stu,loc,can,ter,csr_matrix(z)],format='csr') + +def eval_regime(Xv75, Xsem, y, split, name): + p75=np.zeros(len(y)); psem=np.zeros(len(y)); fold_rows=[] + for k,(tr,va) in enumerate(split,1): + m75=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xv75[tr],y[tr]) + a=np.clip(m75.predict_proba(Xv75[va])[:,1],1e-5,1-1e-5); p75[va]=a + sc=StandardScaler().fit(Xsem[tr]); A=sc.transform(Xsem[tr]); B=sc.transform(Xsem[va]) + ms=LogisticRegression(C=.05,max_iter=500,solver='liblinear',random_state=SEED).fit(A,y[tr]) + b=np.clip(ms.predict_proba(B)[:,1],1e-5,1-1e-5); psem[va]=b + fold_rows.append({'fold':k,'rows':len(va),'v75':float(log_loss(y[va],a)),'semantic':float(log_loss(y[va],b))}) + print(name,fold_rows[-1]) + # Blend selected globally from OOF only; report grid transparently. This is model comparison, not final stacking fit. + grid=[]; best=None + for w in np.linspace(0,1,21): + p=np.clip((1-w)*p75+w*psem,1e-5,1-1e-5); ll=float(log_loss(y,p)) + row={'semantic_weight':float(w),'logloss':ll}; grid.append(row) + if best is None or ll Date: Sat, 15 Aug 2026 18:23:21 +1200 Subject: [PATCH 35/77] trace ace: launch V78 seismic semantic experiment [run-seismic] --- .github/workflows/trace-ace-v78-seismic.yml | 71 +++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/trace-ace-v78-seismic.yml diff --git a/.github/workflows/trace-ace-v78-seismic.yml b/.github/workflows/trace-ace-v78-seismic.yml new file mode 100644 index 0000000..a5c00ad --- /dev/null +++ b/.github/workflows/trace-ace-v78-seismic.yml @@ -0,0 +1,71 @@ +name: Trace the Ace V78 seismic semantic + +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - "competitions/trace_the_ace/v78_seismic_semantic.py" + - ".github/workflows/trace-ace-v78-seismic.yml" + +jobs: + seismic: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown sentence-transformers torch + - name: Download data + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/meta /tmp/trace_ace/transcripts + python - <<'PY' + import os,gdown + if not gdown.download(id=os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID'],output='/tmp/meta.zip',quiet=False): raise SystemExit('metadata download failed') + if not gdown.download(id=os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'],output='/tmp/transcripts.zip',quiet=False): raise SystemExit('transcript download failed') + PY + unzip -q /tmp/meta.zip -d /tmp/trace_ace/meta + unzip -q /tmp/transcripts.zip -d /tmp/trace_ace/transcripts + - name: Resolve schemas + shell: bash + run: | + python - <<'PY' + import csv,shlex + from pathlib import Path + features=labels=td=None + for root in [Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')]: + for p in root.rglob('*.csv'): + try: + with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as f: h=next(csv.reader(f)) + except Exception: continue + c=set(h) + if features is None and {'response_id','session_id','learning_objective'}.issubset(c): features=p; print('FEATURE HEADER',h) + if labels is None and 'response_id' in c and ('is_correct' in c or 'correct' in c): labels=p; print('LABEL HEADER',h) + if td is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(c): td=p.parent; print('TRANSCRIPT HEADER',h) + if not all([features,labels,td]): raise SystemExit('schema discovery failed') + with open('/tmp/paths.env','w') as f: + f.write('FEATURES='+shlex.quote(str(features))+'\nLABELS='+shlex.quote(str(labels))+'\nTRANSCRIPTS='+shlex.quote(str(td))+'\n') + PY + - name: Run V78 MiniLM seismic test + shell: bash + env: + TOKENIZERS_PARALLELISM: "false" + run: | + set -euo pipefail + source /tmp/paths.env + python competitions/trace_the_ace/v78_seismic_semantic.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v78_seismic_semantic.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v78-seismic-result + path: v78_seismic_semantic.json + retention-days: 14 From 252c6d538ab495397427d35a64aba8f636c01e6e Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 18:23:46 +1200 Subject: [PATCH 36/77] trace ace: trigger V78 seismic run [run-seismic] --- competitions/trace_the_ace/v78_seismic_semantic.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/competitions/trace_the_ace/v78_seismic_semantic.py b/competitions/trace_the_ace/v78_seismic_semantic.py index fdcad10..db5dddf 100644 --- a/competitions/trace_the_ace/v78_seismic_semantic.py +++ b/competitions/trace_the_ace/v78_seismic_semantic.py @@ -37,10 +37,8 @@ def encode(model, texts, batch=128): return model.encode(texts,batch_size=batch,show_progress_bar=True,normalize_embeddings=True,convert_to_numpy=True).astype(np.float32) def dense_semantic_features(Eo, Es, El, Et, numeric): - # Explicit cross-view interaction features, not just independent embeddings. cos_os=np.sum(Eo*Es,1,keepdims=True); cos_ol=np.sum(Eo*El,1,keepdims=True); cos_ot=np.sum(Eo*Et,1,keepdims=True) cos_sl=np.sum(Es*El,1,keepdims=True); cos_lt=np.sum(El*Et,1,keepdims=True) - # Pairwise products preserve dimensions while explicitly representing relevance/alignment. prod_ol=Eo*El; prod_ot=Eo*Et return np.hstack([Eo,Es,El,Et,prod_ol,prod_ot,cos_os,cos_ol,cos_ot,cos_sl,cos_lt,numeric]).astype(np.float32) @@ -65,7 +63,6 @@ def eval_regime(Xv75, Xsem, y, split, name): b=np.clip(ms.predict_proba(B)[:,1],1e-5,1-1e-5); psem[va]=b fold_rows.append({'fold':k,'rows':len(va),'v75':float(log_loss(y[va],a)),'semantic':float(log_loss(y[va],b))}) print(name,fold_rows[-1]) - # Blend selected globally from OOF only; report grid transparently. This is model comparison, not final stacking fit. grid=[]; best=None for w in np.linspace(0,1,21): p=np.clip((1-w)*p75+w*psem,1e-5,1-1e-5); ll=float(log_loss(y,p)) From 3ba707c65f30939eb1448df7b4dc0734654d0b56 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 19:04:55 +1200 Subject: [PATCH 37/77] trace ace: add V79 objective retrieval and learning-gain experiment --- .../trace_the_ace/v79_retrieval_gain.py | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 competitions/trace_the_ace/v79_retrieval_gain.py diff --git a/competitions/trace_the_ace/v79_retrieval_gain.py b/competitions/trace_the_ace/v79_retrieval_gain.py new file mode 100644 index 0000000..fefc61b --- /dev/null +++ b/competitions/trace_the_ace/v79_retrieval_gain.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""V79: objective-conditioned semantic retrieval + learning-gain state + V75 blend. + +This is deliberately different from whole-transcript embedding. It embeds tutoring +Q/A/feedback episodes once per session, retrieves the episodes most semantically +aligned with each learning objective, explicitly summarizes early->late mastery +change, and tests an OOF blend with the sparse V75 trajectory champion. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold +from sklearn.preprocessing import StandardScaler +from sentence_transformers import SentenceTransformer + +from v71_mastery_events import load_transcript, extract_episodes +from v75_canonical_trajectory import load_training, trajectory_views, SEED + + +def folds(groups, n=5): + g=np.asarray(groups.astype(str)); z=np.zeros(len(g)); return list(GroupKFold(n_splits=n).split(z,z,g)) + +def episode_text(e): + return f"Tutor question: {e.question} Student answer: {e.answer} Tutor feedback: {e.feedback}" + +def build_session_episodes(frame, transcript_dir): + sessions={} + for i,sid in enumerate(frame.session_id.astype(str).unique()): + df=load_transcript(transcript_dir/f'{sid}.csv') + eps=extract_episodes(df, '') + # cap at 32 episodes; keep both early and late coverage if exceptionally long + if len(eps)>32: + idx=np.unique(np.r_[np.arange(8),np.linspace(8,len(eps)-9,16,dtype=int),np.arange(len(eps)-8,len(eps))]) + eps=[eps[j] for j in idx] + sessions[sid]=eps + if (i+1)%2500==0: print('sessions',i+1) + return sessions + +def encode_episode_bank(model, sessions, batch): + flat=[]; spans={}; start=0 + for sid,eps in sessions.items(): + texts=[episode_text(e) for e in eps] + flat.extend(texts); spans[sid]=(start,start+len(texts)); start+=len(texts) + E=model.encode(flat or [' '],batch_size=batch,show_progress_bar=True,normalize_embeddings=True,convert_to_numpy=True).astype(np.float32) + return E,spans + +def retrieval_features(frame, sessions, Ebank, spans, Eobj, k=6): + rows=[]; agg=[] + for i,r in enumerate(frame.itertuples(index=False)): + sid=str(r.session_id); eps=sessions[sid]; a,b=spans[sid] + if not eps: + rows.append(np.zeros(24,np.float32)); agg.append(np.zeros(Eobj.shape[1],np.float32)); continue + E=Ebank[a:b]; sims=E@Eobj[i] + top=np.argsort(sims)[-min(k,len(eps)):][::-1] + st=sims[top]; w=np.exp(5*(st-st.max())); w=w/(w.sum()+1e-12) + agg.append((E[top]*w[:,None]).sum(0)) + pos=np.array([eps[j].feedback_pos for j in top],float); neg=np.array([eps[j].feedback_neg for j in top],float) + hint=np.array([eps[j].hinted for j in top],float); sub=np.array([eps[j].answer_substantive for j in top],float) + rec=np.array([eps[j].recency for j in top],float); independent=pos*sub*(1-hint) + # explicit early/final mastery among objective-relevant evidence + early=rec<=np.median(rec); late=~early + score=independent-neg-.35*hint + early_score=float(score[early].mean()) if early.any() else 0.; late_score=float(score[late].mean()) if late.any() else 0. + gain=late_score-early_score + feats=np.array([ + len(eps),len(top),st[0],st.mean(),st.min(),st.std() if len(st)>1 else 0., + pos.mean(),neg.mean(),hint.mean(),sub.mean(),independent.mean(), + float((w*pos).sum()),float((w*neg).sum()),float((w*independent).sum()), + early_score,late_score,gain,float(rec.mean()),float(rec[np.argmax(st)]), + float(score[-1]),float(score.max()),float(score.min()), + float(np.mean(st[rec>=.66])) if np.any(rec>=.66) else 0., + float(np.mean(st[rec<=.33])) if np.any(rec<=.33) else 0., + ],np.float32) + rows.append(feats) + return np.vstack(rows),np.vstack(agg) + +def build_v75(frame, transcript_dir): + cache={}; views=[]; nums=[] + for i,r in frame.iterrows(): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(transcript_dir/f'{sid}.csv') + v,n,_=trajectory_views(cache[sid],str(r.learning_objective)); views.append(v); nums.append(n) + if (i+1)%2500==0: print('v75 views',i+1) + nums=np.vstack(nums).astype(np.float64); z=(nums-nums.mean(0))/(nums.std(0)+1e-6) + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + parts=[ + hv.transform(['[OBJECTIVE] '+str(x) for x in frame.learning_objective]), + hv.transform(['[RAW] '+v['raw'] for v in views]), + hv.transform(['[STUDENT] '+v['student'] for v in views]), + hv.transform(['[LOCAL] '+v['local'] for v in views]), + hv.transform(['[STATE] '+v['canonical'] for v in views]), + hv.transform(['[TERMINAL] '+v['terminal'] for v in views]), csr_matrix(z)] + return hstack(parts,format='csr') + +def eval_regime(X75,Xr,y,split,name): + p75=np.zeros(len(y)); pr=np.zeros(len(y)); fr=[] + for f,(tr,va) in enumerate(split,1): + m75=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X75[tr],y[tr]) + p75[va]=np.clip(m75.predict_proba(X75[va])[:,1],1e-5,1-1e-5) + sc=StandardScaler().fit(Xr[tr]); A=sc.transform(Xr[tr]); B=sc.transform(Xr[va]) + mr=LogisticRegression(C=.03,max_iter=500,solver='liblinear',random_state=SEED).fit(A,y[tr]) + pr[va]=np.clip(mr.predict_proba(B)[:,1],1e-5,1-1e-5) + fr.append({'fold':f,'v75':float(log_loss(y[va],p75[va])),'retrieval_gain':float(log_loss(y[va],pr[va]))}); print(name,fr[-1]) + grid=[]; best=None + for w in np.linspace(0,1,41): + p=(1-w)*p75+w*pr; ll=float(log_loss(y,p)); row={'retrieval_weight':float(w),'logloss':ll}; grid.append(row) + if best is None or ll Date: Sat, 15 Aug 2026 19:05:12 +1200 Subject: [PATCH 38/77] trace ace: launch V79 retrieval-gain experiment [run-v79] --- .github/workflows/trace-ace-v79-retrieval.yml | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/trace-ace-v79-retrieval.yml diff --git a/.github/workflows/trace-ace-v79-retrieval.yml b/.github/workflows/trace-ace-v79-retrieval.yml new file mode 100644 index 0000000..5771f47 --- /dev/null +++ b/.github/workflows/trace-ace-v79-retrieval.yml @@ -0,0 +1,69 @@ +name: Trace the Ace V79 retrieval gain + +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - "competitions/trace_the_ace/v79_retrieval_gain.py" + - ".github/workflows/trace-ace-v79-retrieval.yml" + +jobs: + retrieval-gain: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown sentence-transformers torch + - name: Download data + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/meta /tmp/trace_ace/transcripts + python - <<'PY' + import os,gdown + assert gdown.download(id=os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID'],output='/tmp/meta.zip',quiet=False) + assert gdown.download(id=os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'],output='/tmp/transcripts.zip',quiet=False) + PY + unzip -q /tmp/meta.zip -d /tmp/trace_ace/meta + unzip -q /tmp/transcripts.zip -d /tmp/trace_ace/transcripts + - name: Resolve schemas + shell: bash + run: | + python - <<'PY' + import csv,shlex + from pathlib import Path + features=labels=tdir=None + for root in [Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')]: + for p in root.rglob('*.csv'): + try: + with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as f: h=next(csv.reader(f)) + except Exception: continue + c=set(h) + if features is None and {'response_id','session_id','learning_objective'}.issubset(c): features=p; print('FEATURE HEADER',h) + if labels is None and 'response_id' in c and ('is_correct' in c or 'correct' in c): labels=p; print('LABEL HEADER',h) + if tdir is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(c): tdir=p.parent; print('TRANSCRIPT HEADER',h) + if not(features and labels and tdir): raise SystemExit('schema discovery failed') + Path('/tmp/paths.env').write_text('FEATURES='+shlex.quote(str(features))+'\nLABELS='+shlex.quote(str(labels))+'\nTRANSCRIPTS='+shlex.quote(str(tdir))+'\n') + PY + - name: Run V79 objective retrieval and learning-gain test + shell: bash + run: | + set -euo pipefail + source /tmp/paths.env + python competitions/trace_the_ace/v79_retrieval_gain.py \ + --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" \ + --model sentence-transformers/all-MiniLM-L6-v2 --topk 6 --out v79_retrieval_gain.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v79-retrieval-gain + path: v79_retrieval_gain.json + retention-days: 14 From 5472d91dee034d2297f7e4f7daaaad035674aa62 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 19:05:37 +1200 Subject: [PATCH 39/77] trace ace: launch V80 BGE-large retrieval teacher [run-v80] --- .github/workflows/trace-ace-v80-bge.yml | 68 +++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .github/workflows/trace-ace-v80-bge.yml diff --git a/.github/workflows/trace-ace-v80-bge.yml b/.github/workflows/trace-ace-v80-bge.yml new file mode 100644 index 0000000..444dc23 --- /dev/null +++ b/.github/workflows/trace-ace-v80-bge.yml @@ -0,0 +1,68 @@ +name: Trace the Ace V80 BGE retrieval teacher + +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - ".github/workflows/trace-ace-v80-bge.yml" + +jobs: + bge-retrieval: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown sentence-transformers torch + - name: Download data + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/meta /tmp/trace_ace/transcripts + python - <<'PY' + import os,gdown + assert gdown.download(id=os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID'],output='/tmp/meta.zip',quiet=False) + assert gdown.download(id=os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'],output='/tmp/transcripts.zip',quiet=False) + PY + unzip -q /tmp/meta.zip -d /tmp/trace_ace/meta + unzip -q /tmp/transcripts.zip -d /tmp/trace_ace/transcripts + - name: Resolve schemas + shell: bash + run: | + python - <<'PY' + import csv,shlex + from pathlib import Path + features=labels=tdir=None + for root in [Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')]: + for p in root.rglob('*.csv'): + try: + with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as f: h=next(csv.reader(f)) + except Exception: continue + c=set(h) + if features is None and {'response_id','session_id','learning_objective'}.issubset(c): features=p; print('FEATURE HEADER',h) + if labels is None and 'response_id' in c and ('is_correct' in c or 'correct' in c): labels=p; print('LABEL HEADER',h) + if tdir is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(c): tdir=p.parent; print('TRANSCRIPT HEADER',h) + if not(features and labels and tdir): raise SystemExit('schema discovery failed') + Path('/tmp/paths.env').write_text('FEATURES='+shlex.quote(str(features))+'\nLABELS='+shlex.quote(str(labels))+'\nTRANSCRIPTS='+shlex.quote(str(tdir))+'\n') + PY + - name: Run V80 BGE-large objective retrieval teacher + shell: bash + run: | + set -euo pipefail + source /tmp/paths.env + python competitions/trace_the_ace/v79_retrieval_gain.py \ + --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" \ + --model BAAI/bge-large-en-v1.5 --batch 32 --topk 6 --out v80_bge_retrieval_gain.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v80-bge-retrieval-gain + path: v80_bge_retrieval_gain.json + retention-days: 14 From c16f20d04b746512168dbb7eef6d612f2dbea1bb Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:22:34 +1200 Subject: [PATCH 40/77] trace ace: add V81 target-segment phase automaton --- .../trace_the_ace/v81_target_segment_phase.py | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 competitions/trace_the_ace/v81_target_segment_phase.py diff --git a/competitions/trace_the_ace/v81_target_segment_phase.py b/competitions/trace_the_ace/v81_target_segment_phase.py new file mode 100644 index 0000000..5ad21ad --- /dev/null +++ b/competitions/trace_the_ace/v81_target_segment_phase.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""V81: target-segment phase automaton for Trace the Ace. + +Hypothesis: the strongest remaining representation error is mixing evidence from +multiple lessons/objectives inside one tutoring session. V81 detects explicit +lesson boundaries and instructional phases, selects the segment most aligned to +the assessed objective, then compares: + A) frozen-style whole-session V75 views + B) V75 views on the selected target segment only + C) target-segment phase-state views + explicit before/after mastery features +Primary validation: exact-objective-cold GroupKFold. +""" +from __future__ import annotations +import argparse, json, re +from pathlib import Path +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import inspect_headers, load_transcript, tokens, jaccard, char_ngram_overlap +from v75_canonical_trajectory import trajectory_views, load_training, SEED + +BOUNDARY_RE = re.compile(r"\b(?:next lesson|next learning objective|move on to the next|start something new|new lesson|completed this (?:whole )?lesson|finished this lesson|move on|next topic)\b", re.I) +GOAL_RE = re.compile(r"\b(?:learning goal|learning objective|today we(?:'re| are) learning|we are going to learn|we(?:'ll| will) learn)\b", re.I) +PRIOR_RE = re.compile(r"\b(?:prior learning|before we start|what do you already know|recap|remember from)\b", re.I) +GUIDED_RE = re.compile(r"\b(?:i do|let me show|watch me|we do|let's do|lets do|together|with me|guided)\b", re.I) +INDEP_RE = re.compile(r"\b(?:you do|your turn|try this one|have a go|independently|by yourself|on your own)\b", re.I) +APPLY_RE = re.compile(r"\b(?:application|apply|challenge|reasoning|problem solving|different example|another example|transfer)\b", re.I) + + +def rel_text(text: str, objective: str) -> float: + return float(max(jaccard(tokens(text), tokens(objective)), 0.5*char_ngram_overlap(text, objective))) + + +def split_segments(df: pd.DataFrame) -> list[tuple[int,int]]: + txt=df.content.fillna('').astype(str).tolist(); starts=[0] + for i,t in enumerate(txt): + if i>0 and BOUNDARY_RE.search(t): starts.append(i) + starts=sorted(set(starts)); segs=[] + for j,s in enumerate(starts): + e=starts[j+1] if j+1s: segs.append((s,e)) + return segs or [(0,len(df))] + + +def choose_target_segment(df: pd.DataFrame, objective: str) -> tuple[pd.DataFrame, dict]: + segs=split_segments(df); best=None + for s,e in segs: + part=df.iloc[s:e].copy(); text=' '.join(part.content.fillna('').astype(str)) + score=rel_text(text,objective) + # explicit goal language near the segment front is strong evidence + front=' '.join(part.content.fillna('').astype(str).head(20)) + goal_bonus=0.15 if GOAL_RE.search(front) else 0.0 + total=score+goal_bonus + row=(total,score,goal_bonus,s,e) + if best is None or row>best: best=row + _,score,bonus,s,e=best + return df.iloc[s:e].copy().reset_index(drop=True), {'segments':len(segs),'start':int(s),'end':int(e),'segment_fraction':float((e-s)/max(1,len(df))),'segment_relevance':float(score),'goal_bonus':float(bonus)} + + +def phase_for(text: str, current: str) -> str: + if GOAL_RE.search(text): return 'GOAL' + if PRIOR_RE.search(text): return 'PRIOR' + if APPLY_RE.search(text): return 'APPLICATION' + if INDEP_RE.search(text): return 'INDEPENDENT' + if GUIDED_RE.search(text): return 'GUIDED' + return current + + +def phase_views(seg: pd.DataFrame, objective: str) -> tuple[dict[str,str], np.ndarray]: + phase='OTHER'; buckets={k:[] for k in ['GOAL','PRIOR','GUIDED','INDEPENDENT','APPLICATION','OTHER']} + for r in seg[['role','content']].itertuples(index=False): + text=str(r.content); phase=phase_for(text,phase) + buckets[phase].append(f'[{str(r.role).upper()}] {text}') + texts={k:' '.join(v) for k,v in buckets.items()} + # Reuse V75 state extraction per phase so the abstraction stays comparable. + phase_num=[]; phase_state=[] + for k in ['PRIOR','GUIDED','INDEPENDENT','APPLICATION']: + sub=seg.iloc[0:0].copy() + if buckets[k]: + # recover rows by a simple phase replay + phase2='OTHER'; idx=[] + for i,r in enumerate(seg[['content']].itertuples(index=False)): + phase2=phase_for(str(r.content),phase2) + if phase2==k: idx.append(i) + if idx: sub=seg.iloc[idx].copy().reset_index(drop=True) + if len(sub): + v,n,_=trajectory_views(sub,objective); phase_num.append(n); phase_state.append(v['canonical']) + else: + phase_num.append(np.zeros(28,float)); phase_state.append('') + P=np.vstack(phase_num) + # Explicit learning-gain contrasts; first 28 are phase means collapsed pairwise. + pre=P[0]; guided=P[1]; indep=P[2]; app=P[3] + gain_ind=indep-pre; gain_app=app-pre + # compact summary scalars on key V75 dimensions: state score, positive/error/independent weighted-ish slots + key=[12,13,14,15,17,21,22,23,24,25] + scal=[] + for arr in [pre,guided,indep,app,gain_ind,gain_app]: scal.extend(arr[key].tolist()) + nums=np.asarray(scal,float) + views={ + 'phase_prior':texts['PRIOR'], 'phase_guided':texts['GUIDED'], + 'phase_independent':texts['INDEPENDENT'], 'phase_application':texts['APPLICATION'], + 'phase_states':' [PHASE] '.join(phase_state), + } + return views,nums + + +def folds(groups): + g=groups.astype(str).to_numpy(); z=np.zeros(len(g)); return list(GroupKFold(5).split(z,z,g)) + + +def build_X(frame, rows, nums, prefix): + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + parts=[hv.transform([f'[OBJECTIVE] {x}' for x in frame.learning_objective])] + keys=list(rows[0].keys()) + for k in keys: parts.append(hv.transform([f'[{prefix}_{k.upper()}] '+r[k] for r in rows])) + Z=np.vstack(nums).astype(float); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6) + parts.append(csr_matrix(Z)); return hstack(parts,format='csr') + + +def oof(X,y,split,name): + p=np.zeros(len(y)); fr=[] + for k,(tr,va) in enumerate(split,1): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + q=np.clip(m.predict_proba(X[va])[:,1],1e-5,1-1e-5); p[va]=q + row={'fold':k,'rows':len(va),'logloss':float(log_loss(y[va],q)),'auc':float(roc_auc_score(y[va],q))}; fr.append(row); print(name,row) + return p,fr + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + if a.limit: f=f.iloc[:a.limit].copy().reset_index(drop=True) + whole_rows=[]; whole_nums=[]; seg_rows=[]; seg_nums=[]; phase_rows=[]; phase_nums=[]; meta=[]; cache={} + for i,r in f.iterrows(): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + d=cache[sid]; vw,nw,_=trajectory_views(d,str(r.learning_objective)); whole_rows.append(vw); whole_nums.append(nw) + seg,m=choose_target_segment(d,str(r.learning_objective)); vs,ns,_=trajectory_views(seg,str(r.learning_objective)); seg_rows.append(vs); seg_nums.append(ns) + pv,pn=phase_views(seg,str(r.learning_objective)); phase_rows.append({**vs,**pv}); phase_nums.append(np.concatenate([ns,pn])); meta.append(m) + if (i+1)%2500==0: print('rows',i+1) + y=f.target.to_numpy(int); grp=f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective; sp=folds(grp) + Xw=build_X(f,whole_rows,whole_nums,'WHOLE'); Xs=build_X(f,seg_rows,seg_nums,'SEG'); Xp=build_X(f,phase_rows,phase_nums,'PHASE') + pw,fw=oof(Xw,y,sp,'whole'); ps,fs=oof(Xs,y,sp,'segment'); pp,fp=oof(Xp,y,sp,'phase') + grid=[]; best=None + for ws in np.linspace(0,1,11): + for wp in np.linspace(0,1-ws,11): + ww=1-ws-wp; q=np.clip(ww*pw+ws*ps+wp*pp,1e-5,1-1e-5); ll=float(log_loss(y,q)); row={'whole_weight':float(ww),'segment_weight':float(ws),'phase_weight':float(wp),'logloss':ll} + grid.append(row); best=row if best is None or ll1 for m in meta]))}} + Path(a.out).write_text(json.dumps(result,indent=2)); print(json.dumps(result,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v81_target_segment_phase.json'); p.add_argument('--limit',type=int,default=0); run(p.parse_args()) From 3376c4537032741bfe2574b25463eb1e11203c8c Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 21:22:46 +1200 Subject: [PATCH 41/77] trace ace: launch V81 target-segment phase experiment [run-v81] --- .github/workflows/trace-ace-v81-phase.yml | 63 +++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/trace-ace-v81-phase.yml diff --git a/.github/workflows/trace-ace-v81-phase.yml b/.github/workflows/trace-ace-v81-phase.yml new file mode 100644 index 0000000..e524431 --- /dev/null +++ b/.github/workflows/trace-ace-v81-phase.yml @@ -0,0 +1,63 @@ +name: Trace the Ace V81 target segment phase +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - competitions/trace_the_ace/v81_target_segment_phase.py + - .github/workflows/trace-ace-v81-phase.yml +jobs: + v81: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + - name: Download data + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/meta /tmp/trace_ace/transcripts + python - <<'PY' + import os,gdown + assert gdown.download(id=os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID'],output='/tmp/meta.zip',quiet=False) + assert gdown.download(id=os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'],output='/tmp/transcripts.zip',quiet=False) + PY + unzip -q /tmp/meta.zip -d /tmp/trace_ace/meta + unzip -q /tmp/transcripts.zip -d /tmp/trace_ace/transcripts + - name: Resolve schemas and run V81 + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import csv,shlex + from pathlib import Path + features=labels=trans=None + for root in [Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')]: + for p in root.rglob('*.csv'): + try: + with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as f: h=next(csv.reader(f)) + except Exception: continue + c=set(h) + if features is None and {'response_id','session_id','learning_objective'}.issubset(c): features=p; print('FEATURE HEADER',h) + if labels is None and 'response_id' in c and ('is_correct' in c or 'correct' in c): labels=p; print('LABEL HEADER',h) + if trans is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(c): trans=p.parent; print('TRANSCRIPT HEADER',h) + if not all([features,labels,trans]): raise SystemExit('schema discovery failed') + Path('/tmp/paths.env').write_text('FEATURES='+shlex.quote(str(features))+'\nLABELS='+shlex.quote(str(labels))+'\nTRANS='+shlex.quote(str(trans))+'\n') + PY + source /tmp/paths.env + python competitions/trace_the_ace/v81_target_segment_phase.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANS" --out v81_target_segment_phase.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v81-target-segment-phase + path: v81_target_segment_phase.json + retention-days: 14 From faf92611a88622bd7112cca1e1552fdfd738a913 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 23:09:24 +1200 Subject: [PATCH 42/77] trace ace: add V82 supervised ModernBERT multi-resolution experiment --- .../v82_modernbert_supervised.py | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 competitions/trace_the_ace/v82_modernbert_supervised.py diff --git a/competitions/trace_the_ace/v82_modernbert_supervised.py b/competitions/trace_the_ace/v82_modernbert_supervised.py new file mode 100644 index 0000000..3b8dcc7 --- /dev/null +++ b/competitions/trace_the_ace/v82_modernbert_supervised.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""V82: supervised ModernBERT on V81 multi-resolution mastery evidence. + +Primary question: does task-supervised language understanding materially improve +objective-cold log loss once the input representation exposes target segment, +instructional phase, assistance/state trajectory, and whole-session context? + +For CPU feasibility this experiment unfreezes the classifier/pooler plus the top +N transformer blocks. This is deliberately different from frozen embeddings: +task labels update the language model's upper representation layers. +""" +from __future__ import annotations +import argparse, json, random +from pathlib import Path +import numpy as np +import pandas as pd +import torch +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold +from torch.utils.data import Dataset, DataLoader +from transformers import AutoTokenizer, AutoModelForSequenceClassification, get_linear_schedule_with_warmup + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, trajectory_views, SEED +from v81_target_segment_phase import choose_target_segment, phase_views + +random.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED) + + +def clip_chars(s: str, n: int) -> str: + s=str(s) + if len(s)<=n: return s + # Preserve both start and terminal evidence. + a=n//2; return s[:a] + " [..CUT..] " + s[-(n-a):] + + +def make_text(df: pd.DataFrame, objective: str, max_chars: int=18000) -> tuple[str, dict]: + whole, whole_num, _ = trajectory_views(df, objective) + seg, meta = choose_target_segment(df, objective) + target, target_num, _ = trajectory_views(seg, objective) + phases, phase_num = phase_views(seg, objective) + + # Multi-resolution evidence. Whole context is compacted; target/independent/ + # application and canonical states get preferential budget. + parts = [ + f"[OBJECTIVE] {objective}", + f"[TARGET_SEGMENT] {clip_chars(target['raw'], 4200)}", + f"[TARGET_STUDENT] {clip_chars(target['student'], 2600)}", + f"[TARGET_CANONICAL] {clip_chars(target['canonical'], 2400)}", + f"[TARGET_TERMINAL] {clip_chars(target['terminal'], 2200)}", + f"[PRIOR] {clip_chars(phases.get('phase_prior',''), 1200)}", + f"[GUIDED] {clip_chars(phases.get('phase_guided',''), 1600)}", + f"[INDEPENDENT] {clip_chars(phases.get('phase_independent',''), 2200)}", + f"[APPLICATION] {clip_chars(phases.get('phase_application',''), 1800)}", + f"[PHASE_STATES] {clip_chars(phases.get('phase_states',''), 1800)}", + f"[WHOLE_CONTEXT] {clip_chars(whole['raw'], 3200)}", + f"[WHOLE_TERMINAL] {clip_chars(whole['terminal'], 1200)}", + ] + text="\n".join(parts) + if len(text)>max_chars: text=clip_chars(text,max_chars) + return text, meta + + +class TextDS(Dataset): + def __init__(self, texts, labels, tok, max_len): self.texts=texts; self.labels=labels; self.tok=tok; self.max_len=max_len + def __len__(self): return len(self.texts) + def __getitem__(self,i): + enc=self.tok(self.texts[i], truncation=True, max_length=self.max_len, padding=False) + enc={k:torch.tensor(v,dtype=torch.long) for k,v in enc.items()} + enc['labels']=torch.tensor(float(self.labels[i]),dtype=torch.float32) + return enc + + +def collate(tok): + def fn(rows): + labels=torch.stack([r.pop('labels') for r in rows]) + b=tok.pad(rows,padding=True,return_tensors='pt'); b['labels']=labels + return b + return fn + + +def unfreeze_top(model, top_blocks: int): + for p in model.parameters(): p.requires_grad=False + # Always train classification head / pooler-like named modules. + for n,p in model.named_parameters(): + ln=n.lower() + if any(x in ln for x in ['classifier','score','pooler']): p.requires_grad=True + # ModernBERT/HF encoder layer naming varies; discover indexed layer names. + layer_names=[] + for n,_ in model.named_parameters(): + bits=n.split('.') + for j,b in enumerate(bits[:-1]): + if b.isdigit() and any(x in '.'.join(bits[:j]).lower() for x in ['layer','layers','encoder']): + try: layer_names.append(int(b)) + except: pass + if layer_names: + mx=max(layer_names); cutoff=max(0,mx-top_blocks+1) + for n,p in model.named_parameters(): + bits=n.split('.') + idx=None + for j,b in enumerate(bits): + if b.isdigit() and any(x in '.'.join(bits[:j]).lower() for x in ['layer','layers','encoder']): idx=int(b); break + if idx is not None and idx>=cutoff: p.requires_grad=True + trainable=sum(p.numel() for p in model.parameters() if p.requires_grad) + total=sum(p.numel() for p in model.parameters()) + print('trainable_parameters',trainable,'total_parameters',total,'fraction',trainable/total) + return trainable,total + + +def predict(model, loader, device): + model.eval(); out=[]; ys=[] + with torch.no_grad(): + for b in loader: + y=b.pop('labels').numpy(); ys.extend(y.tolist()) + b={k:v.to(device) for k,v in b.items()} + z=model(**b).logits.squeeze(-1); out.extend(torch.sigmoid(z).cpu().numpy().tolist()) + return np.asarray(out),np.asarray(ys) + + +def run(a): + frame=load_training(a.features,a.labels).reset_index(drop=True) + if a.limit: frame=frame.iloc[:a.limit].copy().reset_index(drop=True) + cache={}; texts=[]; metas=[] + for i,r in frame.iterrows(): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + t,m=make_text(cache[sid],str(r.learning_objective),a.max_chars); texts.append(t); metas.append(m) + if (i+1)%1000==0: print('built_texts',i+1) + y=frame.target.to_numpy(np.float32) + grp=(frame.learning_objective_id if 'learning_objective_id' in frame else frame.learning_objective).astype(str).to_numpy() + folds=list(GroupKFold(5).split(np.zeros(len(y)),y,grp)) + selected=list(range(min(a.folds,len(folds)))) + tok=AutoTokenizer.from_pretrained(a.model, trust_remote_code=True) + device=torch.device('cuda' if torch.cuda.is_available() else 'cpu'); print('device',device) + results=[]; all_p=[]; all_y=[] + for fi in selected: + tr,va=folds[fi] + model=AutoModelForSequenceClassification.from_pretrained(a.model,num_labels=1,problem_type='regression',trust_remote_code=True) + unfreeze_top(model,a.top_blocks); model.to(device) + train_ds=TextDS([texts[i] for i in tr],y[tr],tok,a.max_len); val_ds=TextDS([texts[i] for i in va],y[va],tok,a.max_len) + train_dl=DataLoader(train_ds,batch_size=a.batch,shuffle=True,collate_fn=collate(tok),num_workers=0) + val_dl=DataLoader(val_ds,batch_size=a.eval_batch,shuffle=False,collate_fn=collate(tok),num_workers=0) + params=[p for p in model.parameters() if p.requires_grad] + opt=torch.optim.AdamW(params,lr=a.lr,weight_decay=.01) + steps=max(1,len(train_dl)*a.epochs); sched=get_linear_schedule_with_warmup(opt,max(1,int(.06*steps)),steps) + model.train(); seen=0 + for ep in range(a.epochs): + for b in train_dl: + labels=b.pop('labels').to(device) + b={k:v.to(device) for k,v in b.items()} + logits=model(**b).logits.squeeze(-1) + loss=torch.nn.functional.binary_cross_entropy_with_logits(logits,labels) + loss.backward(); torch.nn.utils.clip_grad_norm_(params,1.0); opt.step(); sched.step(); opt.zero_grad(set_to_none=True) + seen+=1 + if seen%50==0: print('fold',fi+1,'step',seen,'loss',float(loss.detach().cpu())) + p,yy=predict(model,val_dl,device); p=np.clip(p,1e-5,1-1e-5) + row={'fold':fi+1,'rows':len(va),'logloss':float(log_loss(yy,p)),'auc':float(roc_auc_score(yy,p))}; print('RESULT',row); results.append(row); all_p.extend(p.tolist()); all_y.extend(yy.tolist()) + del model; + if torch.cuda.is_available(): torch.cuda.empty_cache() + result={'model':a.model,'mode':'supervised_top_blocks','top_blocks':a.top_blocks,'epochs':a.epochs,'lr':a.lr,'max_len':a.max_len,'folds_run':len(selected),'fold_results':results,'pooled_logloss':float(log_loss(all_y,all_p)) if all_p else None,'pooled_auc':float(roc_auc_score(all_y,all_p)) if all_p else None,'segmentation':{'mean_fraction':float(np.mean([m['segment_fraction'] for m in metas])),'mean_segments':float(np.mean([m['segments'] for m in metas]))}} + Path(a.out).write_text(json.dumps(result,indent=2)); print(json.dumps(result,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v82_modernbert_supervised.json'); p.add_argument('--model',default='answerdotai/ModernBERT-large'); p.add_argument('--folds',type=int,default=1); p.add_argument('--epochs',type=int,default=1); p.add_argument('--top-blocks',type=int,default=2); p.add_argument('--lr',type=float,default=2e-5); p.add_argument('--max-len',type=int,default=768); p.add_argument('--max-chars',type=int,default=18000); p.add_argument('--batch',type=int,default=2); p.add_argument('--eval-batch',type=int,default=4); p.add_argument('--limit',type=int,default=0); run(p.parse_args()) From 45dd77f7baf2d1b6b58ecea3196f541ee3fc703d Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sat, 15 Aug 2026 23:09:39 +1200 Subject: [PATCH 43/77] trace ace: launch V82 supervised ModernBERT objective-cold probe [run-v82] --- .../workflows/trace-ace-v82-modernbert.yml | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .github/workflows/trace-ace-v82-modernbert.yml diff --git a/.github/workflows/trace-ace-v82-modernbert.yml b/.github/workflows/trace-ace-v82-modernbert.yml new file mode 100644 index 0000000..d5ab433 --- /dev/null +++ b/.github/workflows/trace-ace-v82-modernbert.yml @@ -0,0 +1,76 @@ +name: Trace the Ace V82 supervised ModernBERT + +on: + workflow_dispatch: + push: + branches: + - agent/trace-ace-mastery-events + paths: + - "competitions/trace_the_ace/v82_modernbert_supervised.py" + - ".github/workflows/trace-ace-v82-modernbert.yml" + +jobs: + v82: + runs-on: ubuntu-latest + timeout-minutes: 360 + env: + TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI + TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz + TOKENIZERS_PARALLELISM: "false" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown torch transformers accelerate + - name: Download data + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/trace_ace/transcripts /tmp/trace_ace/meta + python - <<'PY' + import os, gdown + assert gdown.download(id=os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID'], output='/tmp/trace_ace/transcripts.zip', quiet=False) + assert gdown.download(id=os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID'], output='/tmp/trace_ace/meta.zip', quiet=False) + PY + unzip -q /tmp/trace_ace/transcripts.zip -d /tmp/trace_ace/transcripts + unzip -q /tmp/trace_ace/meta.zip -d /tmp/trace_ace/meta + - name: Resolve schemas and run V82 + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import csv, shlex + from pathlib import Path + roots=[Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')] + f=l=t=None + for root in roots: + for p in root.rglob('*.csv'): + try: + with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as h: cols=next(csv.reader(h)) + except Exception: continue + s=set(cols) + if f is None and {'response_id','session_id','learning_objective'}.issubset(s): f=p + if l is None and 'response_id' in s and ('is_correct' in s or 'correct' in s): l=p + if t is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(s): t=p.parent + if not (f and l and t): raise SystemExit('Could not identify inputs') + print('FEATURE HEADER', list(csv.reader(open(f,encoding='utf-8-sig')))[0]) + print('LABEL HEADER', list(csv.reader(open(l,encoding='utf-8-sig')))[0]) + with open('/tmp/trace_ace/paths.env','w') as h: + h.write('FEATURES='+shlex.quote(str(f))+'\nLABELS='+shlex.quote(str(l))+'\nTRANSCRIPTS='+shlex.quote(str(t))+'\n') + PY + source /tmp/trace_ace/paths.env + python competitions/trace_the_ace/v82_modernbert_supervised.py \ + --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" \ + --out v82_modernbert_supervised.json \ + --model answerdotai/ModernBERT-large \ + --folds 1 --epochs 1 --top-blocks 2 --max-len 768 --batch 2 --eval-batch 4 + - name: Upload aggregate result + if: always() + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v82-modernbert-supervised + path: v82_modernbert_supervised.json + retention-days: 14 From 37d8ef1f6573a63fa6a2165bb8e617c0320718a4 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 03:41:57 +1200 Subject: [PATCH 44/77] trace ace: add V83 supervised TalkMove-BERT probe --- .../trace_the_ace/v83_talkmove_supervised.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 competitions/trace_the_ace/v83_talkmove_supervised.py diff --git a/competitions/trace_the_ace/v83_talkmove_supervised.py b/competitions/trace_the_ace/v83_talkmove_supervised.py new file mode 100644 index 0000000..cef652d --- /dev/null +++ b/competitions/trace_the_ace/v83_talkmove_supervised.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""V83: task-supervised TalkMove-BERT on V81 multi-resolution evidence. + +This reuses the V82 training harness but swaps in a tutoring-dialogue-domain +encoder already present in the official runtime preload list. The goal is a fast, +genuinely supervised objective-cold probe that can run on CPU and still update +upper language-model layers from the competition labels. +""" +from v82_modernbert_supervised import run +import argparse +from pathlib import Path + +if __name__=='__main__': + p=argparse.ArgumentParser() + p.add_argument('--features',type=Path,required=True) + p.add_argument('--labels',type=Path,required=True) + p.add_argument('--transcripts',type=Path,required=True) + p.add_argument('--out',default='v83_talkmove_supervised.json') + p.add_argument('--model',default='saroyehun/Talkmove-bert') + p.add_argument('--folds',type=int,default=1) + p.add_argument('--epochs',type=int,default=2) + p.add_argument('--top-blocks',type=int,default=4) + p.add_argument('--lr',type=float,default=2e-5) + p.add_argument('--max-len',type=int,default=512) + p.add_argument('--max-chars',type=int,default=14000) + p.add_argument('--batch',type=int,default=8) + p.add_argument('--eval-batch',type=int,default=16) + p.add_argument('--limit',type=int,default=0) + run(p.parse_args()) From 81b21229386e913a68ec81b6b4248c1a3ebafd4d Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 03:42:06 +1200 Subject: [PATCH 45/77] trace ace: launch V83 TalkMove-BERT supervised probe [run-v83] --- .github/workflows/trace-ace-v83-talkmove.yml | 35 ++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/trace-ace-v83-talkmove.yml diff --git a/.github/workflows/trace-ace-v83-talkmove.yml b/.github/workflows/trace-ace-v83-talkmove.yml new file mode 100644 index 0000000..acfe0b4 --- /dev/null +++ b/.github/workflows/trace-ace-v83-talkmove.yml @@ -0,0 +1,35 @@ +name: Trace the Ace V83 TalkMove supervised +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v83_talkmove_supervised.py' + - '.github/workflows/trace-ace-v83-talkmove.yml' +jobs: + v83: + runs-on: ubuntu-latest + timeout-minutes: 240 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: {python-version: '3.12'} + - name: Install dependencies + run: pip install -q pandas numpy scipy scikit-learn torch transformers sentencepiece gdown + - name: Download data + run: | + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + unzip -q metadata.zip -d data + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + mkdir -p transcripts && unzip -q transcripts.zip -d transcripts + - name: Resolve schemas and run V83 + working-directory: competitions/trace_the_ace + run: | + F=$(find ../../data -type f -iname '*features*.csv' | head -1) + L=$(find ../../data -type f -iname '*labels*.csv' | head -1) + T=$(find ../../transcripts -type f -name '*.csv' -printf '%h\n' | head -1) + python v83_talkmove_supervised.py --features "$F" --labels "$L" --transcripts "$T" --out ../../v83_talkmove_supervised.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v83-talkmove-result + path: v83_talkmove_supervised.json From 2f8283ed13921b78f02a102d9ff1156c0ce45dd1 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 03:42:21 +1200 Subject: [PATCH 46/77] trace ace: add V84 student-evidence ablation --- .../trace_the_ace/v84_student_evidence.py | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 competitions/trace_the_ace/v84_student_evidence.py diff --git a/competitions/trace_the_ace/v84_student_evidence.py b/competitions/trace_the_ace/v84_student_evidence.py new file mode 100644 index 0000000..8c3606f --- /dev/null +++ b/competitions/trace_the_ace/v84_student_evidence.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""V84: student-evidence representation with tutor-language confound removed. + +Builds objective-cold OOF predictions from question + student answer + canonical +state/assistance tokens, but excludes raw tutor praise/feedback wording from the +student-evidence view. Compares against whole-session V75 and reports blend gain. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, trajectory_views, extract_canonical_events, SEED +from v81_target_segment_phase import choose_target_segment + + +def folds(g): + z=np.zeros(len(g)); return list(GroupKFold(5).split(z,z,g.astype(str).to_numpy())) + +def evidence_view(df,obj): + ev=extract_canonical_events(df,obj) + chunks=[] + nums=[] + for e in ev: + rb='HIGH' if e.relevance>=.15 else 'MID' if e.relevance>=.05 else 'LOW' + ab='NONE' if e.assistance<=.1 else 'LOW' if e.assistance<=.5 else 'HIGH' + chunks.append(f'[STATE={e.state}] [REL={rb}] [ASSIST={ab}] [Q] {e.question} [STUDENT] {e.answer}') + nums.append([e.relevance,e.recency,e.assistance,e.substantive,e.explanation]) + if nums: + A=np.asarray(nums,float) + feat=np.concatenate([A.mean(0),A.max(0),A[-1],np.array([len(ev)],float)]) + else: feat=np.zeros(16,float) + return ' '.join(chunks),feat + +def build_v75(frame,rows,nums): + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + parts=[hv.transform(['[OBJECTIVE] '+str(x) for x in frame.learning_objective])] + for k in ['raw','student','local','canonical','terminal']: + parts.append(hv.transform([f'[{k.upper()}] '+r[k] for r in rows])) + Z=np.vstack(nums); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6); parts.append(csr_matrix(Z)) + return hstack(parts,format='csr') +def build_evidence(frame,whole_text,seg_text,nums): + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + parts=[hv.transform(['[OBJECTIVE] '+str(x) for x in frame.learning_objective]),hv.transform(['[WHOLE_EVIDENCE] '+x for x in whole_text]),hv.transform(['[TARGET_EVIDENCE] '+x for x in seg_text])] + Z=np.vstack(nums); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6); parts.append(csr_matrix(Z)) + return hstack(parts,format='csr') +def oof(X,y,sp,name): + p=np.zeros(len(y)); fr=[] + for k,(tr,va) in enumerate(sp,1): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + q=np.clip(m.predict_proba(X[va])[:,1],1e-5,1-1e-5); p[va]=q + row={'fold':k,'rows':len(va),'logloss':float(log_loss(y[va],q)),'auc':float(roc_auc_score(y[va],q))}; print(name,row); fr.append(row) + return p,fr + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True); cache={}; vr=[]; vn=[]; wt=[]; st=[]; en=[] + for i,r in f.iterrows(): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + d=cache[sid]; v,n,_=trajectory_views(d,str(r.learning_objective)); vr.append(v); vn.append(n) + seg,_=choose_target_segment(d,str(r.learning_objective)); w,wn=evidence_view(d,str(r.learning_objective)); s,sn=evidence_view(seg,str(r.learning_objective)); wt.append(w); st.append(s); en.append(np.concatenate([wn,sn])) + if (i+1)%2500==0: print('rows',i+1) + y=f.target.to_numpy(int); grp=f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective; sp=folds(grp) + Xv=build_v75(f,vr,vn); Xe=build_evidence(f,wt,st,en); pv,fv=oof(Xv,y,sp,'v75'); pe,fe=oof(Xe,y,sp,'evidence') + grid=[]; best=None + for w in np.linspace(0,1,41): + q=np.clip((1-w)*pv+w*pe,1e-5,1-1e-5); ll=float(log_loss(y,q)); row={'evidence_weight':float(w),'logloss':ll}; grid.append(row); best=row if best is None or ll Date: Sun, 16 Aug 2026 03:42:29 +1200 Subject: [PATCH 47/77] trace ace: launch V84 student-evidence ablation [run-v84] --- .../trace-ace-v84-student-evidence.yml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/trace-ace-v84-student-evidence.yml diff --git a/.github/workflows/trace-ace-v84-student-evidence.yml b/.github/workflows/trace-ace-v84-student-evidence.yml new file mode 100644 index 0000000..4f7a168 --- /dev/null +++ b/.github/workflows/trace-ace-v84-student-evidence.yml @@ -0,0 +1,35 @@ +name: Trace the Ace V84 student evidence +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v84_student_evidence.py' + - '.github/workflows/trace-ace-v84-student-evidence.yml' +jobs: + v84: + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: {python-version: '3.12'} + - name: Install dependencies + run: pip install -q pandas numpy scipy scikit-learn gdown + - name: Download data + run: | + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + unzip -q metadata.zip -d data + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + mkdir -p transcripts && unzip -q transcripts.zip -d transcripts + - name: Resolve schemas and run V84 + working-directory: competitions/trace_the_ace + run: | + F=$(find ../../data -type f -iname '*features*.csv' | head -1) + L=$(find ../../data -type f -iname '*labels*.csv' | head -1) + T=$(find ../../transcripts -type f -name '*.csv' -printf '%h\n' | head -1) + python v84_student_evidence.py --features "$F" --labels "$L" --transcripts "$T" --out ../../v84_student_evidence.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v84-student-evidence-result + path: v84_student_evidence.json From c92ac94fd3f5b7e37877e9bce5e0d811e4044192 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 03:56:33 +1200 Subject: [PATCH 48/77] trace ace: add RGRS experiment ledger and precommit evidence-state separator --- competitions/trace_the_ace/RGRS_LEDGER.md | 151 ++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 competitions/trace_the_ace/RGRS_LEDGER.md diff --git a/competitions/trace_the_ace/RGRS_LEDGER.md b/competitions/trace_the_ace/RGRS_LEDGER.md new file mode 100644 index 0000000..f8b8c2c --- /dev/null +++ b/competitions/trace_the_ace/RGRS_LEDGER.md @@ -0,0 +1,151 @@ +# Trace the Ace — Residual-Guided Representation Search Ledger + +This ledger applies Residual-Guided Representation Search (RGRS) to the Trace the Ace programme. The purpose is to prevent blind search inside a representation after repeated evidence says the missing information is representational. + +## Frozen current baseline + +- Public champion retained architecture: V75 canonical trajectory, public log loss 0.6047. +- Primary unseen proxy: exact objective-cold validation. +- V75 objective-cold aggregate: ~0.59235. +- V81 best blend objective-cold: ~0.59097. +- Promotion target: several-thousandths objective-cold improvement with no regression in the official runtime/independence gates before spending a full public submission. + +## Residual records + +### TTA-ρ-001 — Generic semantics does not recover mastery + +`rho = (R6 Representation, semantic prediction layer, V78 MiniLM standalone ~0.7145 and only ~0.0004 gain when blended with V75, objective-cold, high)` + +Material interventions inside the old text-prediction representation: +- frozen MiniLM semantic view; +- retrieval/learning-gain semantic view (V79). + +Both are weak standalone and only add a small orthogonal residual correction. + +**Decision:** do not classify the main gap as R1 search. More generic embedding capacity is not justified by the residual. + +### TTA-ρ-002 — Hard target scoping loses useful information + +`rho = (R5 Applicability, lesson-segment scoping, target-only V81 ~0.5991 worse than whole-session ~0.5924 while blended whole+target+phase reaches ~0.5910, objective-cold, high)` + +Observed separator: +- 91.6% of examples are multi-segment; +- target region averages ~32.1% of session; +- deleting non-target context hurts; +- exposing target context alongside whole context helps. + +**Decision:** target scoping is conditionally useful. Search for the predicate/weight governing when evidence belongs to the assessed objective; do not globally discard the rest of the session. + +### TTA-ρ-003 — Tutor feedback is not equivalent to mastery evidence + +`rho = (R6 Representation, evidence object, transcript audit contains label-0 sessions ending in strong tutor praise and label-1 sessions with later unrelated failures, audited examples, high)` + +The current text representation conflates: +- student-generated competence evidence; +- tutor evaluation language; +- assistance supplied before an answer; +- later unrelated lesson performance. + +The language cannot cleanly state the distinction needed to explain the residual. + +**Representation candidate:** replace transcript-as-example with an objective-conditioned evidence graph/state sequence. + +### TTA-ρ-004 — Large supervised encoder on CPU is an infrastructure failure + +`rho = (R10 Infrastructure, V82 ModernBERT-large execution, hours-long CPU runner without deciding result, GitHub Actions CPU environment, high)` + +**Decision:** draw no semantic conclusion about supervised transformers from V82 runtime. Use a feasible domain encoder (V83) or GPU for the large-model hypothesis. + +## Current primary residual + +The strongest current diagnosis is: + +`R6 Representation + R5 Applicability` + +The missing object is not "better transcript semantics". It is approximately: + +`EvidenceEvent = (objective_match, phase, question, student_answer, assistance_before_answer, correction_state, independence, transfer, position)` + +with an objective-conditioned state trajectory: + +`K_pre -> K_guided -> K_independent -> K_application` + +and an applicability predicate deciding which events should influence the assessed objective. + +## Smallest deciding representation test + +### Hypothesis H85 + +An explicit objective-conditioned student-evidence representation will outperform an otherwise matched representation that includes tutor-evaluation wording as first-class predictive text. + +### One intervention + +Create an `EvidenceEvent` view from the existing transcript while holding vectorizer/model/folds/hyperparameters fixed. + +### Frozen arms + +- **A0 — V75/V81-style text evidence:** existing whole + target + canonical views. +- **A1 — Student-evidence IR:** objective-matched question -> student-answer episodes, phase tags, assistance tags, canonical state, whole-context summary; tutor praise removed from raw predictive text. +- **A2 — Causal ablation:** same as A1 but remove assistance/independence tags while preserving all event text and ordering. + +### Opposing discriminators + +1. Cases where tutor praise is high but independent student evidence is weak: A1 should improve over A0. +2. Cases where independent evidence is strong but later unrelated struggle exists: A1 should preserve/promote probability relative to target-only deletion. +3. Cases with no meaningful target evidence: A1 should not manufacture confidence; A0 behavior/prior should be preserved. + +### Primary metric + +Exact objective-cold log loss, frozen folds. + +### Precommitted interpretation + +- A1 beats A0 by >= 0.003 and A2 materially weakens the gain -> **clean mechanism win; escalate representation**. +- A1 beats A0 by < 0.001 -> **insufficient; preserve negative law**. +- A1 helps only a subset -> **R5 Applicability; learn/certify event activation predicate**. +- A1 wins but A2 is equal -> **causal attribution fails; do not admit assistance/independence tags**. + +## Current live experiments + +### V83 — TalkMove-BERT supervised + +Purpose: test whether domain-matched tutoring-language supervision supplies a useful semantic layer once V81 structure is exposed. + +RGRS classification before result: **R1/R6 separator test**, not an admitted representation change. + +If V83 is weak, generic/model-capacity search is further demoted and the programme should prioritize explicit evidence-state IR. + +### V84 — Student evidence ablation + +Purpose: preliminary test of the TTA-ρ-003 hypothesis by suppressing raw tutor-praise language and retaining student/objective evidence. + +RGRS classification: **R6 candidate separator**. + +## Admission gate for a new Trace-the-Ace representation + +A proposed representation is retained only when all hold: + +1. **Semantic/data-contract gate** — legal competition features only; no test adaptation/leakage; official runtime contract passes. +2. **Causal gate** — removing the proposed representational feature materially weakens the gain. +3. **Predictive resource gate** — frozen primary metric improves; no post-hoc metric switching. +4. **Reproducibility gate** — same commit/folds/config reproduces and official inference remains sample-independent. + +State machine: + +`PROPOSED -> SEPARATED -> VERIFIED -> ADMITTED` + +otherwise: + +`REJECTED` or `OBSTRUCTED`. + +## Governing law for this competition + +> Never add model capacity merely because the score is imperfect when repeated residuals show that the model is being asked to predict mastery from the wrong object. + +The current highest-value search direction is therefore: + +`raw transcript -> objective-conditioned evidence events -> assistance/independence-aware knowledge state -> calibrated predictor` + +not: + +`raw transcript -> larger generic embedding -> classifier`. From 7039107f44e10fcbd94d0bf8904205cd3542052d Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 04:04:43 +1200 Subject: [PATCH 49/77] trace ace: add V85 EvidenceEvent knowledge-state separator --- .../trace_the_ace/v85_evidence_state.py | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 competitions/trace_the_ace/v85_evidence_state.py diff --git a/competitions/trace_the_ace/v85_evidence_state.py b/competitions/trace_the_ace/v85_evidence_state.py new file mode 100644 index 0000000..5461813 --- /dev/null +++ b/competitions/trace_the_ace/v85_evidence_state.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""V85: RGRS EvidenceEvent -> knowledge-state separator. + +Compares three objective-cold arms: +A0: V75 whole-session representation baseline. +A1: explicit objective-conditioned EvidenceEvent IR with assistance/independence tags. +A2: same EvidenceEvent IR with assistance/independence tags ablated. + +Primary decision: A1 must beat A0 by >=0.003 log loss and materially beat A2 to count +as a representation-level breakthrough. Otherwise retain as negative/conditional law. +""" +from __future__ import annotations +import argparse, json, re +from pathlib import Path +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript, tokens, jaccard, char_ngram_overlap +from v75_canonical_trajectory import load_training, trajectory_views, SEED +from v81_target_segment_phase import choose_target_segment, phase_for + +QUESTION_RE=re.compile(r"\?|\b(?:what|how|why|which|calculate|solve|find|show|explain|tell me|your turn|try|have a go)\b",re.I) +POS_RE=re.compile(r"\b(?:correct|right|yes|exactly|great|brilliant|well done|good job|nice|perfect)\b",re.I) +NEG_RE=re.compile(r"\b(?:not quite|incorrect|wrong|try again|check|remember|almost|no[, .])\b",re.I) +HINT_RE=re.compile(r"\b(?:remember|think about|hint|try|look at|first|start by|let's|lets|together|we can|I'll|i will|let me)\b",re.I) +SUPPLY_RE=re.compile(r"\b(?:the answer is|it is|equals|so .* is|that gives|we get|you should)\b",re.I) +ACK_RE=re.compile(r"^(?:ok(?:ay)?|yes|yeah|yep|no|nope|thanks?|thank you|got it|sure|right|mhm|uh huh|great|cool)[.! ]*$",re.I) + +PHASE_WEIGHT={'OTHER':.45,'GOAL':.25,'PRIOR':.35,'GUIDED':.55,'INDEPENDENT':1.0,'APPLICATION':1.1} + + +def rel(text,obj): + return float(max(jaccard(tokens(str(text)),tokens(str(obj))),.5*char_ngram_overlap(str(text),str(obj)))) + + +def substantive(s): + s=str(s).strip() + if not s or ACK_RE.match(s): return False + # preserve short numeric/math answers + if re.search(r"\d|[=+\-*/%]",s): return True + return len(tokens(s))>=2 + + +def phase_replay(df): + ph='OTHER'; out=[] + for t in df.content.fillna('').astype(str): + ph=phase_for(t,ph); out.append(ph) + return out + + +def evidence_events(df,obj): + d=df.reset_index(drop=True).copy(); phases=phase_replay(d) + events=[] + n=len(d) + for i,row in d.iterrows(): + if str(row.role).lower()!='student' or not substantive(row.content): continue + # nearest preceding tutor question/prompt within 4 turns + qidx=None + for j in range(i-1,max(-1,i-5),-1): + if str(d.iloc[j].role).lower()=='tutor' and QUESTION_RE.search(str(d.iloc[j].content)): + qidx=j; break + if qidx is None: continue + q=str(d.iloc[qidx].content); ans=str(row.content) + # immediate/near tutor feedback after response + feedback=''; fbidx=None + for j in range(i+1,min(n,i+4)): + if str(d.iloc[j].role).lower()=='tutor': feedback=str(d.iloc[j].content); fbidx=j; break + rq=rel(q,obj); ra=rel(ans,obj); relevance=max(rq,.4*ra) + # Assistance is derived only from tutor turns after previous student response and before this answer. + window=' '.join(str(d.iloc[j].content) for j in range(max(0,qidx-2),i) if str(d.iloc[j].role).lower()=='tutor') + supplied=bool(SUPPLY_RE.search(window)); hinted=bool(HINT_RE.search(window)) + assistance=1.0 if supplied else (.6 if hinted else 0.0) + independent=(phases[i] in ('INDEPENDENT','APPLICATION') and assistance<.3) + pos=bool(POS_RE.search(feedback)); neg=bool(NEG_RE.search(feedback)) + # canonical state; tutor feedback is auxiliary, not ground truth + if neg: state='UNRESOLVED_ERROR' + elif pos and assistance>=.6: state='CORRECT_AFTER_GUIDANCE' + elif pos and independent: state='INDEPENDENT_CORRECT' + elif pos: state='SUPPORTED_CORRECT' + else: state='UNJUDGED_RESPONSE' + events.append({'i':i,'phase':phases[i],'q':q,'a':ans,'feedback':feedback,'rel':relevance, + 'assistance':assistance,'independent':independent,'state':state, + 'position':i/max(1,n-1),'pos':pos,'neg':neg}) + return events + + +def render(events,obj,ablate=False): + keep=sorted(events,key=lambda e:(e['rel']*(.4+.6*e['position'])*PHASE_WEIGHT.get(e['phase'],.4)),reverse=True)[:16] + keep=sorted(keep,key=lambda e:e['i']) + rows=[] + for e in keep: + tags=[f"PHASE={e['phase']}",f"STATE={e['state']}",f"REL={e['rel']:.2f}",f"POS={int(e['pos'])}",f"NEG={int(e['neg'])}"] + if not ablate: tags += [f"ASSIST={e['assistance']:.1f}",f"INDEP={int(e['independent'])}"] + rows.append('['+' '.join(tags)+'] [Q] '+e['q']+' [STUDENT] '+e['a']) + return f"[OBJECTIVE] {obj}\n"+'\n'.join(rows) + + +def nums(events,ablate=False): + if not events: return np.zeros(22 if not ablate else 16,float) + E=events; rels=np.array([e['rel'] for e in E]); pos=np.array([e['pos'] for e in E],float); neg=np.array([e['neg'] for e in E],float) + ind=np.array([e['independent'] for e in E],float); ass=np.array([e['assistance'] for e in E],float); positions=np.array([e['position'] for e in E]) + app=np.array([e['phase']=='APPLICATION' for e in E],float); late=positions>=.6 + base=[len(E),rels.mean(),rels.max(),pos.mean(),neg.mean(),positions[pos>0].max() if pos.any() else 0, + positions[neg>0].max() if neg.any() else 0,pos[late].mean() if late.any() else 0,neg[late].mean() if late.any() else 0, + app.mean(),(pos*rels).sum()/(rels.sum()+1e-6),(neg*rels).sum()/(rels.sum()+1e-6), + float(any(e['state']=='UNRESOLVED_ERROR' for e in E[-3:])),float(any(e['state']=='INDEPENDENT_CORRECT' for e in E[-3:])), + sum(e['state']=='INDEPENDENT_CORRECT' for e in E),sum(e['state']=='CORRECT_AFTER_GUIDANCE' for e in E)] + if ablate: return np.asarray(base,float) + extra=[ass.mean(),ass[-3:].mean() if len(ass)>=3 else ass.mean(),ind.mean(),ind[late].mean() if late.any() else 0, + (pos*ind*rels).sum()/(rels.sum()+1e-6),(neg*(1-ass)*rels).sum()/(rels.sum()+1e-6)] + return np.asarray(base+extra,float) + + +def build_sparse(texts,Z,prefix): + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + X=hv.transform([f'[{prefix}] '+x for x in texts]); Z=np.vstack(Z); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6) + return hstack([X,csr_matrix(Z)],format='csr') + + +def build_v75(frame,transcripts): + rows=[]; ns=[] + for _,r in frame.iterrows(): + v,n,_=trajectory_views(transcripts[str(r.session_id)],str(r.learning_objective)); rows.append(v); ns.append(n) + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + parts=[hv.transform([f'[OBJECTIVE] {x}' for x in frame.learning_objective])] + for k in rows[0].keys(): parts.append(hv.transform([f'[{k.upper()}] '+r[k] for r in rows])) + Z=np.vstack(ns); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6); parts.append(csr_matrix(Z)) + return hstack(parts,format='csr') + + +def oof(X,y,splits,name): + p=np.zeros(len(y)); fs=[] + for k,(tr,va) in enumerate(splits,1): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + q=np.clip(m.predict_proba(X[va])[:,1],1e-5,1-1e-5); p[va]=q + row={'fold':k,'logloss':float(log_loss(y[va],q)),'auc':float(roc_auc_score(y[va],q))}; print(name,row); fs.append(row) + return p,fs + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + if a.limit: f=f.iloc[:a.limit].copy().reset_index(drop=True) + cache={} + for sid in f.session_id.astype(str).unique(): cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + full_text=[]; abl_text=[]; full_num=[]; abl_num=[]; meta=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)]; seg,m=choose_target_segment(d,str(r.learning_objective)); ev=evidence_events(seg,str(r.learning_objective)) + full_text.append(render(ev,str(r.learning_objective),False)); abl_text.append(render(ev,str(r.learning_objective),True)) + full_num.append(nums(ev,False)); abl_num.append(nums(ev,True)); meta.append({'events':len(ev),**m}) + if (i+1)%2500==0: print('rows',i+1) + y=f.target.to_numpy(int); groups=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sp=list(GroupKFold(5).split(np.zeros(len(y)),y,groups)) + X0=build_v75(f,cache); X1=build_sparse(full_text,full_num,'EVIDENCE'); X2=build_sparse(abl_text,abl_num,'EVIDENCE_ABL') + p0,f0=oof(X0,y,sp,'A0_v75'); p1,f1=oof(X1,y,sp,'A1_evidence'); p2,f2=oof(X2,y,sp,'A2_ablation') + ll0=float(log_loss(y,p0)); ll1=float(log_loss(y,p1)); ll2=float(log_loss(y,p2)) + # Also test whether evidence is orthogonal to V75; fixed transparent grid only. + blends=[]; best=None + for w in np.linspace(0,1,21): + q=np.clip((1-w)*p0+w*p1,1e-5,1-1e-5); ll=float(log_loss(y,q)); row={'evidence_weight':float(w),'logloss':ll}; blends.append(row) + if best is None or ll=.003 and causal>=.001: decision='REPRESENTATION_BREAKTHROUGH' + elif gain>=.001 and causal>0: decision='PROMISING_PARTIAL' + elif best['logloss']<=ll0-.001: decision='ORTHOGONAL_SIGNAL_ONLY' + else: decision='REJECT_OR_REFINE_R5' + out={'primary':'objective-cold','A0_v75':ll0,'A1_evidence':ll1,'A2_ablation':ll2,'gain_vs_A0':gain,'causal_assistance_gain':causal, + 'best_blend':best,'decision':decision,'folds':{'A0':f0,'A1':f1,'A2':f2}, + 'event_stats':{'mean_events':float(np.mean([m['events'] for m in meta])),'zero_event_fraction':float(np.mean([m['events']==0 for m in meta]))}} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v85_evidence_state.json'); p.add_argument('--limit',type=int,default=0); run(p.parse_args()) From 14dc78dd76559c0550674d5f087f8b726664f46c Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 04:04:52 +1200 Subject: [PATCH 50/77] trace ace: launch V85 RGRS EvidenceEvent separator [run-v85] --- .../trace-ace-v85-evidence-state.yml | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/trace-ace-v85-evidence-state.yml diff --git a/.github/workflows/trace-ace-v85-evidence-state.yml b/.github/workflows/trace-ace-v85-evidence-state.yml new file mode 100644 index 0000000..f477c0c --- /dev/null +++ b/.github/workflows/trace-ace-v85-evidence-state.yml @@ -0,0 +1,41 @@ +name: Trace the Ace V85 EvidenceEvent state +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v85_evidence_state.py' + - '.github/workflows/trace-ace-v85-evidence-state.yml' + workflow_dispatch: +jobs: + v85: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown --id 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown --id 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + find data -maxdepth 3 -type f | head + - name: Resolve schemas and run V85 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + echo "$FEATURES $LABELS $TRANSCRIPTS" + cd competitions/trace_the_ace + python v85_evidence_state.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v85_evidence_state.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v85-evidence-state + path: v85_evidence_state.json From 4a6e0efd1022000f035cd59f9e51d8f34b36ed25 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 04:31:58 +1200 Subject: [PATCH 51/77] trace ace: fix V83 TalkMove head mismatch [run-v83] --- .../trace_the_ace/v83_talkmove_supervised.py | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/competitions/trace_the_ace/v83_talkmove_supervised.py b/competitions/trace_the_ace/v83_talkmove_supervised.py index cef652d..ec6d7cb 100644 --- a/competitions/trace_the_ace/v83_talkmove_supervised.py +++ b/competitions/trace_the_ace/v83_talkmove_supervised.py @@ -1,14 +1,22 @@ #!/usr/bin/env python3 """V83: task-supervised TalkMove-BERT on V81 multi-resolution evidence. -This reuses the V82 training harness but swaps in a tutoring-dialogue-domain -encoder already present in the official runtime preload list. The goal is a fast, -genuinely supervised objective-cold probe that can run on CPU and still update -upper language-model layers from the competition labels. +R10 repair only: TalkMove-BERT ships with a 5-class head, while this experiment +uses a 1-logit binary head. Inject ignore_mismatched_sizes=True so the pretrained +encoder is retained and the task head is lawfully reinitialized. Hypothesis, +folds, representation and metrics remain unchanged. """ -from v82_modernbert_supervised import run import argparse from pathlib import Path +import v82_modernbert_supervised as base + +_orig = base.AutoModelForSequenceClassification.from_pretrained + +def _from_pretrained(*args, **kwargs): + kwargs['ignore_mismatched_sizes'] = True + return _orig(*args, **kwargs) + +base.AutoModelForSequenceClassification.from_pretrained = _from_pretrained if __name__=='__main__': p=argparse.ArgumentParser() @@ -26,4 +34,4 @@ p.add_argument('--batch',type=int,default=8) p.add_argument('--eval-batch',type=int,default=16) p.add_argument('--limit',type=int,default=0) - run(p.parse_args()) + base.run(p.parse_args()) From c1592abce2f9c5dd57902c4b81895b6b1718b141 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 04:32:14 +1200 Subject: [PATCH 52/77] trace ace: fix V85 gdown invocation [run-v85] --- .github/workflows/trace-ace-v85-evidence-state.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trace-ace-v85-evidence-state.yml b/.github/workflows/trace-ace-v85-evidence-state.yml index f477c0c..517915d 100644 --- a/.github/workflows/trace-ace-v85-evidence-state.yml +++ b/.github/workflows/trace-ace-v85-evidence-state.yml @@ -19,8 +19,8 @@ jobs: run: pip install numpy pandas scipy scikit-learn gdown - name: Download data run: | - gdown --id 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip - gdown --id 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip mkdir -p data/transcripts data/meta unzip -q transcripts.zip -d data/transcripts unzip -q metadata.zip -d data/meta From 8b1f45cea215fd83c9618dfef1913c0b6aae3e3a Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 04:35:59 +1200 Subject: [PATCH 53/77] trace ace: add V86 knowledge-state dynamics separator --- .../v86_knowledge_state_dynamics.py | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 competitions/trace_the_ace/v86_knowledge_state_dynamics.py diff --git a/competitions/trace_the_ace/v86_knowledge_state_dynamics.py b/competitions/trace_the_ace/v86_knowledge_state_dynamics.py new file mode 100644 index 0000000..134fa9b --- /dev/null +++ b/competitions/trace_the_ace/v86_knowledge_state_dynamics.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""V86: RGRS sequential knowledge-state dynamics separator. + +Hypothesis: the missing object is not merely an EvidenceEvent multiset but the +objective-conditioned *state trajectory* induced by ordered evidence. + +Arms (objective-cold, frozen folds/model family): +A0: EvidenceEvent multiset representation (same event extractor as V85). +A1: ordered EvidenceEvents + explicit cumulative/decayed knowledge-state dynamics. +A2: causal ablation: same events/state formulas with event order canonically sorted + by content signature, destroying observed temporal order while preserving the + event multiset and most marginal features. + +A1 must beat A0 materially and A2 must lose the gain to support an ordering/state +representation claim. Otherwise retain a negative/conditional law. +""" +from __future__ import annotations +import argparse, json, hashlib +from pathlib import Path +import numpy as np +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import evidence_events, render, nums, PHASE_WEIGHT + + +def event_value(e): + """Signed mastery evidence, deliberately simple and inspectable.""" + rel=float(e['rel']); assist=float(e['assistance']); phase=PHASE_WEIGHT.get(e['phase'],.4) + independent=1.0 if e['independent'] else 0.0 + if e['neg']: + # An unassisted, relevant error is strong negative evidence. + return - rel * phase * (0.55 + 0.45*(1-assist)) + if e['pos']: + # Correctness after strong assistance is weaker than independent production. + return rel * phase * (0.25 + 0.75*(1-assist)) * (1.0 + 0.25*independent) + # Unjudged substantive production is weak evidence, not zero. + return 0.08 * rel * phase * (1-assist) + + +def signature(e): + s=(str(e['q'])+'\x1f'+str(e['a'])+'\x1f'+str(e['state'])).encode('utf-8','ignore') + return hashlib.sha1(s).hexdigest() + + +def state_features(events, destroy_order=False): + if not events: + return np.zeros(46,float), '[NO_EVENTS]' + E=list(events) + if destroy_order: + # Canonical deterministic order: same event multiset, observed chronology removed. + E=sorted(E,key=signature) + vals=[]; ks=[]; fast=[]; slow=[] + k=0.0; kf=0.0; kslo=0.0 + prev_state='START'; transitions=[] + for t,e in enumerate(E): + v=event_value(e); vals.append(v) + # bounded additive state + two recency scales + k=np.tanh(0.78*np.arctanh(np.clip(k,-.999,.999)) + v) + kf=.55*kf + v + kslo=.88*kslo + v + ks.append(k); fast.append(kf); slow.append(kslo) + transitions.append(prev_state+'>'+str(e['state'])); prev_state=str(e['state']) + V=np.asarray(vals); K=np.asarray(ks); F=np.asarray(fast); S=np.asarray(slow) + n=len(E); q=max(1,n//4) + pos=np.asarray([e['pos'] for e in E],float); neg=np.asarray([e['neg'] for e in E],float) + ind=np.asarray([e['independent'] for e in E],float); ass=np.asarray([e['assistance'] for e in E],float) + rel=np.asarray([e['rel'] for e in E],float) + app=np.asarray([e['phase']=='APPLICATION' for e in E],float) + # Transition features target the educational trajectory directly. + err_to_ind=0; guide_to_ind=0; recovery=0; regress=0 + for a,b in zip(E[:-1],E[1:]): + if a['neg'] and b['independent'] and b['pos']: err_to_ind+=1 + if a['assistance']>=.6 and b['independent'] and b['pos']: guide_to_ind+=1 + if a['neg'] and b['pos']: recovery+=1 + if a['pos'] and b['neg']: regress+=1 + last_pos=max([i for i,e in enumerate(E) if e['pos']],default=-1)/(max(1,n-1)) + last_neg=max([i for i,e in enumerate(E) if e['neg']],default=-1)/(max(1,n-1)) + last_ind=max([i for i,e in enumerate(E) if e['independent'] and e['pos']],default=-1)/(max(1,n-1)) + feats=[ + n, V.mean(), V.sum(), V[-1], V[:q].mean(), V[-q:].mean(), + K[-1], K.max(), K.min(), K.mean(), K[-q:].mean(), + F[-1], F.max(), F.min(), S[-1], S.max(), S.min(), + float(K[-1]-K[0]), float(F[-1]-F[0]), float(S[-1]-S[0]), + pos.mean(), neg.mean(), ind.mean(), ass.mean(), rel.mean(), app.mean(), + float((pos*ind*rel).sum()), float((neg*(1-ass)*rel).sum()), + err_to_ind, guide_to_ind, recovery, regress, + last_pos,last_neg,last_ind, + float(any(e['neg'] for e in E[-3:])), + float(any(e['independent'] and e['pos'] for e in E[-3:])), + float(sum(e['independent'] and e['pos'] for e in E[-5:])), + float(sum(e['neg'] for e in E[-5:])), + float(np.polyfit(np.arange(n),K,1)[0] if n>1 else 0), + float(np.polyfit(np.arange(n),V,1)[0] if n>1 else 0), + float(np.std(V)),float(np.std(K)), + float(np.mean(np.abs(np.diff(K))) if n>1 else 0), + float(np.max(np.abs(np.diff(K))) if n>1 else 0), + float(sum(t=='UNRESOLVED_ERROR>INDEPENDENT_CORRECT' for t in transitions)), + ] + # Render ordered state path so sparse model can exploit categorical transitions too. + rows=[] + for i,(e,v,kv) in enumerate(zip(E,V,K)): + rows.append(f"[T={i} PHASE={e['phase']} STATE={e['state']} ASSIST={e['assistance']:.1f} INDEP={int(e['independent'])} REL={e['rel']:.2f} DV={v:.2f} K={kv:.2f}]") + return np.asarray(feats,float), ' '.join(rows) + + +def build(texts,Z,prefix): + hv=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + X=hv.transform([f'[{prefix}] '+x for x in texts]); Z=np.vstack(Z); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6) + return hstack([X,csr_matrix(Z)],format='csr') + + +def oof(X,y,sp,name): + p=np.zeros(len(y)); folds=[] + for k,(tr,va) in enumerate(sp,1): + m=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X[tr],y[tr]) + q=np.clip(m.predict_proba(X[va])[:,1],1e-5,1-1e-5); p[va]=q + r={'fold':k,'logloss':float(log_loss(y[va],q)),'auc':float(roc_auc_score(y[va],q))}; print(name,r); folds.append(r) + return p,folds + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + if a.limit: f=f.iloc[:a.limit].copy().reset_index(drop=True) + cache={}; base_t=[]; base_n=[]; seq_t=[]; seq_n=[]; abl_t=[]; abl_n=[]; counts=[] + for i,r in f.iterrows(): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + seg,_=choose_target_segment(cache[sid],str(r.learning_objective)); ev=evidence_events(seg,str(r.learning_objective)); counts.append(len(ev)) + base_t.append(render(ev,str(r.learning_objective),False)); base_n.append(nums(ev,False)) + z,t=state_features(ev,False); za,ta=state_features(ev,True) + seq_t.append(f"[OBJECTIVE] {r.learning_objective} [STATE_PATH] {t} [EVENTS] {base_t[-1]}"); seq_n.append(np.r_[base_n[-1],z]) + abl_t.append(f"[OBJECTIVE] {r.learning_objective} [STATE_PATH_ORDER_ABLATED] {ta} [EVENTS] {base_t[-1]}"); abl_n.append(np.r_[base_n[-1],za]) + if (i+1)%2500==0: print('rows',i+1) + y=f.target.to_numpy(int); groups=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sp=list(GroupKFold(5).split(np.zeros(len(y)),y,groups)) + X0=build(base_t,base_n,'EVENT_MULTISET'); X1=build(seq_t,seq_n,'STATE_ORDERED'); X2=build(abl_t,abl_n,'STATE_ORDER_ABLATED') + p0,f0=oof(X0,y,sp,'A0_multiset'); p1,f1=oof(X1,y,sp,'A1_ordered_state'); p2,f2=oof(X2,y,sp,'A2_order_ablation') + ll0=float(log_loss(y,p0)); ll1=float(log_loss(y,p1)); ll2=float(log_loss(y,p2)) + gain=ll0-ll1; causal=ll2-ll1 + # Check orthogonality if state helps only in blend. + best=None + for w in np.linspace(0,1,21): + q=np.clip((1-w)*p0+w*p1,1e-5,1-1e-5); ll=float(log_loss(y,q)) + if best is None or ll=.003 and causal>=.001: decision='SEQUENTIAL_STATE_BREAKTHROUGH' + elif gain>=.001 and causal>0: decision='PROMISING_SEQUENTIAL_STATE' + elif best['logloss']<=ll0-.001: decision='ORTHOGONAL_SEQUENCE_SIGNAL' + else: decision='ORDER_NOT_CAUSAL_OR_REFINE_R5' + out={'primary':'objective-cold','A0_event_multiset':ll0,'A1_ordered_state':ll1,'A2_order_ablation':ll2, + 'gain_vs_A0':gain,'causal_order_gain':causal,'best_blend':best,'decision':decision, + 'event_stats':{'mean_events':float(np.mean(counts)),'zero_event_fraction':float(np.mean(np.asarray(counts)==0))}, + 'folds':{'A0':f0,'A1':f1,'A2':f2}} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v86_knowledge_state_dynamics.json'); p.add_argument('--limit',type=int,default=0); run(p.parse_args()) From fa223661b52fcbe63cf7707d3779ec75c2a0663c Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 04:36:10 +1200 Subject: [PATCH 54/77] trace ace: launch V86 sequential knowledge-state separator [run-v86] --- .../trace-ace-v86-knowledge-state.yml | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/trace-ace-v86-knowledge-state.yml diff --git a/.github/workflows/trace-ace-v86-knowledge-state.yml b/.github/workflows/trace-ace-v86-knowledge-state.yml new file mode 100644 index 0000000..616d16d --- /dev/null +++ b/.github/workflows/trace-ace-v86-knowledge-state.yml @@ -0,0 +1,41 @@ +name: Trace the Ace V86 knowledge-state dynamics +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v86_knowledge_state_dynamics.py' + - '.github/workflows/trace-ace-v86-knowledge-state.yml' + workflow_dispatch: +jobs: + v86: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas and run V86 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + echo "$FEATURES $LABELS $TRANSCRIPTS" + cd competitions/trace_the_ace + python v86_knowledge_state_dynamics.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v86_knowledge_state_dynamics.json + - name: Upload aggregate result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v86-knowledge-state-dynamics + path: v86_knowledge_state_dynamics.json From af584de91611288712d4d656d0ddd00ca7b64364 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 04:50:45 +1200 Subject: [PATCH 55/77] trace ace: add V87 RGRS cross-fitted composition test --- .../trace_the_ace/v87_rgrs_composition.py | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 competitions/trace_the_ace/v87_rgrs_composition.py diff --git a/competitions/trace_the_ace/v87_rgrs_composition.py b/competitions/trace_the_ace/v87_rgrs_composition.py new file mode 100644 index 0000000..cd452cd --- /dev/null +++ b/competitions/trace_the_ace/v87_rgrs_composition.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""V87: RGRS composition-before-invention test. + +R7 hypothesis: V81 target/phase structure and V84 student-evidence IR are individually +incomplete but complementary to V75 whole-session context. Build all four OOF predictors +on identical objective-cold folds, then choose convex blend weights for each held-out fold +using ONLY the other four folds' OOF predictions. + +This removes the optimistic full-OOF blend-weight selection used in exploratory V81/V84. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, trajectory_views +from v81_target_segment_phase import choose_target_segment, phase_views, build_X, oof +from v84_student_evidence import evidence_view, build_evidence + + +def simplex_grid(step=.1): + vals=np.arange(0,1+1e-9,step) + for w0 in vals: + for w1 in vals: + for w2 in vals: + s=w0+w1+w2 + if s>1+1e-9: continue + w3=1-s + yield np.array([w0,w1,w2,w3],float) + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={}; whole_rows=[]; whole_nums=[]; seg_rows=[]; seg_nums=[]; phase_rows=[]; phase_nums=[]; ev_whole=[]; ev_seg=[]; ev_num=[] + for i,r in f.iterrows(): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + d=cache[sid]; obj=str(r.learning_objective) + vw,nw,_=trajectory_views(d,obj); whole_rows.append(vw); whole_nums.append(nw) + seg,_=choose_target_segment(d,obj); vs,ns,_=trajectory_views(seg,obj); seg_rows.append(vs); seg_nums.append(ns) + pv,pn=phase_views(seg,obj); phase_rows.append({**vs,**pv}); phase_nums.append(np.concatenate([ns,pn])) + ew,enw=evidence_view(d,obj); es,ens=evidence_view(seg,obj); ev_whole.append(ew); ev_seg.append(es); ev_num.append(np.concatenate([enw,ens])) + if (i+1)%2500==0: print('rows',i+1) + y=f.target.to_numpy(int); grp=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sp=list(GroupKFold(5).split(np.zeros(len(y)),y,grp)) + Xw=build_X(f,whole_rows,whole_nums,'WHOLE'); Xs=build_X(f,seg_rows,seg_nums,'SEG'); Xp=build_X(f,phase_rows,phase_nums,'PHASE'); Xe=build_evidence(f,ev_whole,ev_seg,ev_num) + pw,_=oof(Xw,y,sp,'whole'); ps,_=oof(Xs,y,sp,'segment'); pp,_=oof(Xp,y,sp,'phase'); pe,_=oof(Xe,y,sp,'evidence') + P=np.column_stack([pw,ps,pp,pe]); names=['whole','segment','phase','evidence'] + # Cross-fit the blend: each outer fold's weights are selected on the other 4 folds only. + final=np.zeros(len(y)); fold_rows=[] + all_idx=np.arange(len(y)) + for k,(_,va) in enumerate(sp,1): + trmeta=np.setdiff1d(all_idx,va,assume_unique=False) + best=None + for w in simplex_grid(a.step): + q=np.clip(P[trmeta]@w,1e-5,1-1e-5); ll=float(log_loss(y[trmeta],q)) + if best is None or ll=.003 else ('R7_PROMISING' if gain>=.001 else 'R7_INSUFFICIENT')} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v87_rgrs_composition.json'); p.add_argument('--step',type=float,default=.1); run(p.parse_args()) From b61302d50643fbf47bb1b48c0108e4abd90c57c7 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 04:50:54 +1200 Subject: [PATCH 56/77] trace ace: launch V87 RGRS composition test [run-v87] --- .../workflows/trace-ace-v87-composition.yml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/trace-ace-v87-composition.yml diff --git a/.github/workflows/trace-ace-v87-composition.yml b/.github/workflows/trace-ace-v87-composition.yml new file mode 100644 index 0000000..e029ad0 --- /dev/null +++ b/.github/workflows/trace-ace-v87-composition.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V87 RGRS composition +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v87_rgrs_composition.py' + - '.github/workflows/trace-ace-v87-composition.yml' + workflow_dispatch: +jobs: + v87: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Resolve schemas and run V87 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v87_rgrs_composition.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v87_rgrs_composition.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v87-rgrs-composition + path: v87_rgrs_composition.json From 5e03c72ceb4791cea851e24603e6b45ff72456af Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 05:35:10 +1200 Subject: [PATCH 57/77] trace ace: add V88 cross-fitted EvidenceEvent composition --- .../v88_crossfit_evidence_composition.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 competitions/trace_the_ace/v88_crossfit_evidence_composition.py diff --git a/competitions/trace_the_ace/v88_crossfit_evidence_composition.py b/competitions/trace_the_ace/v88_crossfit_evidence_composition.py new file mode 100644 index 0000000..9181031 --- /dev/null +++ b/competitions/trace_the_ace/v88_crossfit_evidence_composition.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""V88: cross-fitted V75 + ablated EvidenceEvent composition. + +RGRS response to V85: +- assistance/independence tags were not causal (A1 ~= A2), so remove them; +- EvidenceEvents were weak standalone but strongly complementary to V75. + +This test asks whether that composition survives leakage-resistant weight selection. +For each held-out objective-cold fold, choose the blend weight using only OOF +predictions from the other four folds, then score the untouched fold. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import evidence_events, render, nums, build_sparse, build_v75, oof + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={} + for sid in f.session_id.astype(str).unique(): + cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + + texts=[]; z=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)] + seg,_=choose_target_segment(d,str(r.learning_objective)) + ev=evidence_events(seg,str(r.learning_objective)) + texts.append(render(ev,str(r.learning_objective),ablate=True)) + z.append(nums(ev,ablate=True)) + if (i+1)%2500==0: print('rows',i+1) + + y=f.target.to_numpy(int) + groups=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + splits=list(GroupKFold(5).split(np.zeros(len(y)),y,groups)) + + X0=build_v75(f,cache) + X2=build_sparse(texts,z,'EVIDENCE_ABL') + p0,_=oof(X0,y,splits,'V75') + pe,_=oof(X2,y,splits,'EVIDENCE_ABL') + + fold_id=np.empty(len(y),int) + for k,(_,va) in enumerate(splits): fold_id[va]=k + + grid=np.linspace(0,0.8,33) + q=np.zeros(len(y)); selected=[] + for k,(_,va) in enumerate(splits): + tune=np.where(fold_id!=k)[0] + best=None + for w in grid: + pt=np.clip((1-w)*p0[tune]+w*pe[tune],1e-5,1-1e-5) + ll=float(log_loss(y[tune],pt)) + if best is None or ll=.003 else ('R7_PARTIAL' if gain>=.001 else 'REJECT_GLOBAL_BLEND_GAIN') + out={'primary':'objective-cold-crossfitted','v75_logloss':ll0,'evidence_ablation_logloss':lle, + 'crossfit_blend_logloss':llq,'crossfit_gain_vs_v75':gain,'decision':decision, + 'selected_by_fold':selected,'mean_evidence_weight':float(np.mean([x['evidence_weight'] for x in selected]))} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v88_crossfit_evidence_composition.json'); run(p.parse_args()) From 5a713fbabaaf98941b9dbc50081f0062962abcad Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 05:35:22 +1200 Subject: [PATCH 58/77] trace ace: launch V88 cross-fitted EvidenceEvent composition [run-v88] --- .../trace-ace-v88-crossfit-evidence.yml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/trace-ace-v88-crossfit-evidence.yml diff --git a/.github/workflows/trace-ace-v88-crossfit-evidence.yml b/.github/workflows/trace-ace-v88-crossfit-evidence.yml new file mode 100644 index 0000000..1f95952 --- /dev/null +++ b/.github/workflows/trace-ace-v88-crossfit-evidence.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V88 crossfit evidence composition +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v88_crossfit_evidence_composition.py' + - '.github/workflows/trace-ace-v88-crossfit-evidence.yml' + workflow_dispatch: +jobs: + v88: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V88 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v88_crossfit_evidence_composition.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v88_crossfit_evidence_composition.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v88-crossfit-evidence + path: v88_crossfit_evidence_composition.json From e4d6ece2c376a1a351447b70a59d81f83afea751 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 05:35:47 +1200 Subject: [PATCH 59/77] trace ace: retrigger V88 registration [run-v88] --- .github/workflows/trace-ace-v88-crossfit-evidence.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/trace-ace-v88-crossfit-evidence.yml b/.github/workflows/trace-ace-v88-crossfit-evidence.yml index 1f95952..159a6db 100644 --- a/.github/workflows/trace-ace-v88-crossfit-evidence.yml +++ b/.github/workflows/trace-ace-v88-crossfit-evidence.yml @@ -37,3 +37,4 @@ jobs: with: name: trace-ace-v88-crossfit-evidence path: v88_crossfit_evidence_composition.json +# retrigger-registration From 38a8623863365c19445400465a33318fa37e6fc8 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 09:13:58 +1200 Subject: [PATCH 60/77] trace ace: add V89 relative ability composition separator [run-v89] --- .../v89_relative_ability_composition.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 competitions/trace_the_ace/v89_relative_ability_composition.py diff --git a/competitions/trace_the_ace/v89_relative_ability_composition.py b/competitions/trace_the_ace/v89_relative_ability_composition.py new file mode 100644 index 0000000..ddbf177 --- /dev/null +++ b/competitions/trace_the_ace/v89_relative_ability_composition.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""V89: target mastery relative to within-session student ability. + +RGRS hypothesis: V88 captures target-local evidence but not the student's general +baseline competence expressed elsewhere in the same transcript. Build a sample- +local, non-target ability view and test whether it adds orthogonal signal to +V75 + EvidenceEvents under objective-cold cross-fitted composition. +""" +from __future__ import annotations +import argparse, json, re +from pathlib import Path +import numpy as np +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v81_target_segment_phase import choose_target_segment, split_segments +from v85_evidence_state import evidence_events, render, nums, build_sparse, build_v75, oof + +POS=re.compile(r"\b(?:correct|right|exactly|great|brilliant|well done|good job|perfect|yes)\b",re.I) +NEG=re.compile(r"\b(?:incorrect|wrong|not quite|try again|check|almost|no[, .])\b",re.I) + + +def non_target_ability(df,obj): + target,meta=choose_target_segment(df,obj) + ts,te=meta['start'],meta['end'] + # preserve all transcript rows outside selected target segment + rest=df.iloc[list(range(0,ts))+list(range(te,len(df)))].copy().reset_index(drop=True) + if not len(rest): rest=df.iloc[0:0].copy() + ev=evidence_events(rest,obj) + # objective relevance is intentionally not required here: this is general ability. + students=rest[rest.role.astype(str).str.lower().eq('student')].content.fillna('').astype(str).tolist() if len(rest) else [] + tutors=rest[rest.role.astype(str).str.lower().eq('tutor')].content.fillna('').astype(str).tolist() if len(rest) else [] + pos=sum(bool(POS.search(x)) for x in tutors); neg=sum(bool(NEG.search(x)) for x in tutors) + substantive=sum(bool(re.search(r"\d|[=+\-*/%]",x)) or len(x.split())>=2 for x in students) + explain=sum(any(k in x.lower() for k in ['because','so ','therefore','i think','first','then']) for x in students) + math=sum(bool(re.search(r"\d|[=+\-*/%]",x)) for x in students) + n=max(1,len(students)); fbn=max(1,pos+neg) + z=np.asarray([ + len(rest),len(students),len(tutors),substantive/n,explain/n,math/n, + pos/fbn,neg/fbn,(pos-neg)/fbn,len(ev),meta['segment_fraction'],meta['segments'] + ],float) + # compact text: student production + tutor verdict tokens, no target segment content + text=' [STUDENT] '.join(students[-80:]) + verdict=' '.join('[POS]' if POS.search(x) else '[NEG]' if NEG.search(x) else '' for x in tutors[-100:]) + return f'[GENERAL_STUDENT] {text} [GENERAL_VERDICTS] {verdict}',z + + +def build_ability(texts,z): + hv=HashingVectorizer(n_features=2**17,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + X=hv.transform(texts); Z=np.vstack(z); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6) + return hstack([X,csr_matrix(Z)],format='csr') + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True); cache={} + for sid in f.session_id.astype(str).unique(): cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + et=[]; ez=[]; at=[]; az=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)]; obj=str(r.learning_objective) + seg,_=choose_target_segment(d,obj); ev=evidence_events(seg,obj) + et.append(render(ev,obj,ablate=True)); ez.append(nums(ev,ablate=True)) + t,z=non_target_ability(d,obj); at.append(t); az.append(z) + if (i+1)%2500==0: print('rows',i+1) + y=f.target.to_numpy(int); groups=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sp=list(GroupKFold(5).split(np.zeros(len(y)),y,groups)); fold=np.empty(len(y),int) + for k,(_,va) in enumerate(sp): fold[va]=k + X0=build_v75(f,cache); Xe=build_sparse(et,ez,'EVIDENCE_ABL'); Xa=build_ability(at,az) + p0,_=oof(X0,y,sp,'V75'); pe,_=oof(Xe,y,sp,'EVIDENCE'); pa,_=oof(Xa,y,sp,'ABILITY') + q=np.zeros(len(y)); sel=[] + grid=np.arange(0,0.61,0.1) + for k,(_,va) in enumerate(sp): + tune=np.where(fold!=k)[0]; best=None + for we in grid: + for wa in grid: + if we+wa>.8: continue + p=np.clip((1-we-wa)*p0[tune]+we*pe[tune]+wa*pa[tune],1e-5,1-1e-5) + ll=float(log_loss(y[tune],p)) + if best is None or ll Date: Sun, 16 Aug 2026 09:14:10 +1200 Subject: [PATCH 61/77] trace ace: launch V89 relative ability test [run-v89] --- .../trace-ace-v89-relative-ability.yml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/trace-ace-v89-relative-ability.yml diff --git a/.github/workflows/trace-ace-v89-relative-ability.yml b/.github/workflows/trace-ace-v89-relative-ability.yml new file mode 100644 index 0000000..4c557e3 --- /dev/null +++ b/.github/workflows/trace-ace-v89-relative-ability.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V89 relative ability +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v89_relative_ability_composition.py' + - '.github/workflows/trace-ace-v89-relative-ability.yml' + workflow_dispatch: +jobs: + v89: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V89 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v89_relative_ability_composition.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v89_relative_ability.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v89-relative-ability + path: v89_relative_ability.json From f7b1ce77818fd4daed3df903dd720b2595073788 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 09:14:46 +1200 Subject: [PATCH 62/77] trace ace: add V90 regime-gated composition separator [run-v90] --- .../v90_regime_gated_composition.py | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 competitions/trace_the_ace/v90_regime_gated_composition.py diff --git a/competitions/trace_the_ace/v90_regime_gated_composition.py b/competitions/trace_the_ace/v90_regime_gated_composition.py new file mode 100644 index 0000000..0e8501d --- /dev/null +++ b/competitions/trace_the_ace/v90_regime_gated_composition.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""V90: R5 applicability separator for V88 composition. + +Hypothesis: the globally useful V75/EvidenceEvent blend averages over latent +session regimes. Learn regime membership from sample-local transcript style only, +then tune V75/Evidence weights per regime using training folds only. +""" +from __future__ import annotations +import argparse,json,re +from pathlib import Path +import numpy as np +from sklearn.cluster import KMeans +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold +from sklearn.preprocessing import StandardScaler + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training +from v81_target_segment_phase import choose_target_segment, split_segments +from v85_evidence_state import evidence_events,render,nums,build_sparse,build_v75,oof + + +def style_features(df): + roles=df.role.astype(str).str.lower().tolist(); text=df.content.fillna('').astype(str).tolist(); n=max(1,len(df)) + stu=[t for r,t in zip(roles,text) if r=='student']; tut=[t for r,t in zip(roles,text) if r=='tutor'] + sl=[len(x.split()) for x in stu] or [0]; tl=[len(x.split()) for x in tut] or [0] + math=sum(bool(re.search(r'\d|[=+\-*/%]',x)) for x in text)/n + q=sum('?' in x for x in tut)/max(1,len(tut)) + praise=sum(bool(re.search(r'\b(?:great|correct|right|well done|brilliant|perfect)\b',x,re.I)) for x in tut)/max(1,len(tut)) + markers=sum(bool(re.search(r'\b(?:learning objective|learning goal|i do|we do|you do|application|prior learning)\b',x,re.I)) for x in tut)/max(1,len(tut)) + return np.asarray([len(df),len(stu)/n,len(tut)/n,np.mean(sl),np.mean(tl),np.median(sl),np.median(tl),math,q,praise,markers,len(split_segments(df))],float) + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True); cache={} + for sid in f.session_id.astype(str).unique(): cache[sid]=load_transcript(a.transcripts/f'{sid}.csv') + et=[]; ez=[]; S=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)]; obj=str(r.learning_objective); seg,_=choose_target_segment(d,obj); ev=evidence_events(seg,obj) + et.append(render(ev,obj,ablate=True)); ez.append(nums(ev,ablate=True)); S.append(style_features(d)) + if (i+1)%2500==0: print('rows',i+1) + S=np.vstack(S); y=f.target.to_numpy(int); groups=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sp=list(GroupKFold(5).split(np.zeros(len(y)),y,groups)); fold=np.empty(len(y),int) + for k,(_,va) in enumerate(sp): fold[va]=k + p0,_=oof(build_v75(f,cache),y,sp,'V75'); pe,_=oof(build_sparse(et,ez,'EVIDENCE_ABL'),y,sp,'EVIDENCE') + global_q=np.zeros(len(y)); gated_q=np.zeros(len(y)); selected=[]; grid=np.linspace(0,0.8,33) + for k,(_,va) in enumerate(sp): + tune=np.where(fold!=k)[0] + # global cross-fitted reference + bg=min(((float(log_loss(y[tune],np.clip((1-w)*p0[tune]+w*pe[tune],1e-5,1-1e-5))),float(w)) for w in grid),key=lambda x:x[0]) + global_q[va]=np.clip((1-bg[1])*p0[va]+bg[1]*pe[va],1e-5,1-1e-5) + sc=StandardScaler().fit(S[tune]); Xt=sc.transform(S[tune]); Xv=sc.transform(S[va]) + best=None + for K in (2,3,4,5): + km=KMeans(n_clusters=K,random_state=20260816,n_init=20).fit(Xt); rt=km.labels_; rv=km.predict(Xv) + weights={}; total=0.0; ok=True + for r in range(K): + idx=tune[rt==r] + if len(idx)<500: ok=False; break + b=min(((float(log_loss(y[idx],np.clip((1-w)*p0[idx]+w*pe[idx],1e-5,1-1e-5))),float(w)) for w in grid),key=lambda x:x[0]) + weights[r]=b[1]; total += b[0]*len(idx) + if not ok: continue + total/=len(tune) + if best is None or total=.0015 else ('R5_WEAK' if llg-llr>=.0005 else 'R5_REJECT_STYLE_REGIMES')} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v90_regime_gated.json'); run(p.parse_args()) From ea518b49754b3d7cd5575ef50bfa9e7c52de8991 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 09:14:54 +1200 Subject: [PATCH 63/77] trace ace: launch V90 regime-gated composition test [run-v90] --- .../workflows/trace-ace-v90-regime-gated.yml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/trace-ace-v90-regime-gated.yml diff --git a/.github/workflows/trace-ace-v90-regime-gated.yml b/.github/workflows/trace-ace-v90-regime-gated.yml new file mode 100644 index 0000000..e6740b9 --- /dev/null +++ b/.github/workflows/trace-ace-v90-regime-gated.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V90 regime gated composition +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v90_regime_gated_composition.py' + - '.github/workflows/trace-ace-v90-regime-gated.yml' + workflow_dispatch: +jobs: + v90: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V90 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v90_regime_gated_composition.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v90_regime_gated.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v90-regime-gated + path: v90_regime_gated.json From 0d8084a750a397fbf26e734275965d3743bc29b6 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 09:55:19 +1200 Subject: [PATCH 64/77] trace ace: add V91 disagreement-frontier applicability separator --- .../v91_disagreement_frontier.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 competitions/trace_the_ace/v91_disagreement_frontier.py diff --git a/competitions/trace_the_ace/v91_disagreement_frontier.py b/competitions/trace_the_ace/v91_disagreement_frontier.py new file mode 100644 index 0000000..e0658f8 --- /dev/null +++ b/competitions/trace_the_ace/v91_disagreement_frontier.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""V91: disagreement-frontier applicability separator. + +RGRS EXPEDITION after V88 R7 composition win. +Question: when V75 and EvidenceEvents disagree, can sample-local observable information +predict which expert should receive more weight? + +All base predictions are objective-cold OOF. For each held-out fold, the router is fit only +on the other folds. Reports a fixed/global cross-fit blend, routed blend, and oracle headroom. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss, roc_auc_score +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import evidence_events, render, nums, build_sparse, build_v75, oof + + +def row_features(df, seg, ev, p0, pe): + n=max(1,len(df)); ns=max(1,len(seg)) + roles=df.role.fillna('').astype(str).str.lower() + stu=(roles=='student').sum(); tut=(roles=='tutor').sum() + texts=df.content.fillna('').astype(str) + student_text=' '.join(df.loc[roles=='student','content'].fillna('').astype(str)) + math_chars=sum(c.isdigit() or c in '=+-*/%' for c in student_text) + chars=max(1,len(student_text)) + rels=np.array([e['rel'] for e in ev],float) if ev else np.zeros(0) + positions=np.array([e['position'] for e in ev],float) if ev else np.zeros(0) + states=[e['state'] for e in ev] + return np.array([ + p0, pe, pe-p0, abs(pe-p0), abs(p0-.5), abs(pe-.5), + len(df), len(seg), ns/n, stu/n, tut/n, len(ev), + float(rels.mean()) if len(rels) else 0., float(rels.max()) if len(rels) else 0., + float(positions.mean()) if len(positions) else 0., + float(sum(s=='INDEPENDENT_CORRECT' for s in states)), + float(sum(s=='UNRESOLVED_ERROR' for s in states)), + math_chars/chars, + float(np.mean([len(x) for x in texts])) if len(texts) else 0., + ],float) + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + texts=[]; zz=[]; stored=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)]; seg,m=choose_target_segment(d,str(r.learning_objective)); ev=evidence_events(seg,str(r.learning_objective)) + texts.append(render(ev,str(r.learning_objective),ablate=True)); zz.append(nums(ev,ablate=True)); stored.append((d,seg,ev,m)) + if (i+1)%2500==0: print('rows',i+1) + y=f.target.to_numpy(int); groups=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + splits=list(GroupKFold(5).split(np.zeros(len(y)),y,groups)) + p0,_=oof(build_v75(f,cache),y,splits,'V75') + pe,_=oof(build_sparse(texts,zz,'EVIDENCE_ABL'),y,splits,'EVIDENCE') + X=np.vstack([row_features(*stored[i][:3],p0[i],pe[i]) for i in range(len(y))]) + # normalize within each training partition only below + fold_id=np.empty(len(y),int) + for k,(_,va) in enumerate(splits): fold_id[va]=k + q_fixed=np.zeros(len(y)); q_route=np.zeros(len(y)); selected=[] + grid=np.linspace(0,0.8,33) + for k,(_,va) in enumerate(splits): + tr=np.where(fold_id!=k)[0] + # cross-fit fixed blend reference + best=min(((float(log_loss(y[tr],np.clip((1-w)*p0[tr]+w*pe[tr],1e-5,1-1e-5))),float(w)) for w in grid), key=lambda z:z[0]) + wf=best[1]; q_fixed[va]=np.clip((1-wf)*p0[va]+wf*pe[va],1e-5,1-1e-5) + # expert-win label: lower per-row log loss, equivalent to closer probability to realized label in log space + l0=-(y[tr]*np.log(np.clip(p0[tr],1e-6,1))+(1-y[tr])*np.log(np.clip(1-p0[tr],1e-6,1))) + le=-(y[tr]*np.log(np.clip(pe[tr],1e-6,1))+(1-y[tr])*np.log(np.clip(1-pe[tr],1e-6,1))) + z=(le1 else float('nan') + selected.append({'fold':k+1,'fixed_weight':wf,'mean_routed_weight':float(wr.mean()),'router_win_auc':auc,'fixed_ll':float(log_loss(y[va],q_fixed[va])),'routed_ll':float(log_loss(y[va],q_route[va]))}) + # oracle only quantifies headroom; never admissible + choose_e=np.where(y==1,pe>p0,pe=.001 else ('R5_WEAK' if llf-llr>0 else 'REJECT_ROUTER_CURRENT_FEATURES')} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v91_disagreement_frontier.json'); run(p.parse_args()) From 4701368177aeef25850da149a1d834beba2e5f2f Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 09:55:31 +1200 Subject: [PATCH 65/77] trace ace: launch V91 disagreement-frontier separator [run-v91] --- .../trace-ace-v91-disagreement-frontier.yml | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/trace-ace-v91-disagreement-frontier.yml diff --git a/.github/workflows/trace-ace-v91-disagreement-frontier.yml b/.github/workflows/trace-ace-v91-disagreement-frontier.yml new file mode 100644 index 0000000..9fd9e5f --- /dev/null +++ b/.github/workflows/trace-ace-v91-disagreement-frontier.yml @@ -0,0 +1,38 @@ +name: Trace the Ace V91 disagreement frontier +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v91_disagreement_frontier.py' + - '.github/workflows/trace-ace-v91-disagreement-frontier.yml' + workflow_dispatch: +jobs: + v91: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V91 + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v91_disagreement_frontier.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v91_disagreement_frontier.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v91-disagreement-frontier + path: v91_disagreement_frontier.json From bfc7639a149f5634cfc7b411efa58ec71339d444 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 11:35:02 +1200 Subject: [PATCH 66/77] trace ace: add V92 latent-state decomposition --- .../v92_latent_state_decomposition.py | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 competitions/trace_the_ace/v92_latent_state_decomposition.py diff --git a/competitions/trace_the_ace/v92_latent_state_decomposition.py b/competitions/trace_the_ace/v92_latent_state_decomposition.py new file mode 100644 index 0000000..a9bee12 --- /dev/null +++ b/competitions/trace_the_ace/v92_latent_state_decomposition.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""V92: latent student-state decomposition. + +Hypothesis: the transcript is a noisy measurement instrument. Predict post-test +correctness from (a) whole-session V75, (b) non-target local ability, (c) target +EvidenceEvents, and (d) objective difficulty, with explicit latent contrasts. +All base predictions are objective-cold OOF; the meta-combiner is cross-fitted +across the same held-out objective folds. Ablations test which latent components +actually pay rent. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from scipy.sparse import hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v81_target_segment_phase import choose_target_segment +from v85_evidence_state import evidence_events, render, nums, build_sparse, build_v75, oof +from v89_relative_ability_composition import non_target_ability, build_ability + +EPS=1e-5 + +def logit(p): + p=np.clip(np.asarray(p,float),EPS,1-EPS) + return np.log(p/(1-p)) + +def objective_matrix(texts): + w=HashingVectorizer(n_features=2**16,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + c=HashingVectorizer(n_features=2**16,alternate_sign=False,norm='l2',analyzer='char_wb',ngram_range=(3,5),lowercase=True) + return hstack([w.transform(texts),c.transform(texts)],format='csr') + +def base_meta(p0,pa,pe,pd): + l0,la,le,ld=map(logit,[p0,pa,pe,pd]) + return np.c_[ + l0,la,le,ld, + la-ld, # ability relative to objective difficulty + le-la, # target-specific deviation from general ability + le-ld, # target evidence relative to difficulty + np.abs(le-la), + np.abs(la-ld), + l0-la, + l0-le, + la*ld, + le*la, + ] + +def crossfit_meta(y,splits,p0,pa,pe,pd,cols=None,C=0.1): + X=base_meta(p0,pa,pe,pd) + if cols is not None: X=X[:,cols] + q=np.zeros(len(y)); fold_rows=[] + for k,(tr,va) in enumerate(splits): + # The base inputs are themselves OOF predictions. Meta fit is restricted + # to other objective-cold folds and scored on untouched held-out objectives. + m=LogisticRegression(C=C,max_iter=1000,solver='lbfgs',random_state=SEED) + m.fit(X[tr],y[tr]); q[va]=m.predict_proba(X[va])[:,1] + fold_rows.append({'fold':k+1,'logloss':float(log_loss(y[va],np.clip(q[va],EPS,1-EPS)))}) + return np.clip(q,EPS,1-EPS),fold_rows + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + et=[]; ez=[]; at=[]; az=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)]; obj=str(r.learning_objective) + seg,_=choose_target_segment(d,obj); ev=evidence_events(seg,obj) + et.append(render(ev,obj,ablate=True)); ez.append(nums(ev,ablate=True)) + t,z=non_target_ability(d,obj); at.append(t); az.append(z) + if (i+1)%2500==0: print('rows',i+1) + + y=f.target.to_numpy(int) + groups=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + splits=list(GroupKFold(5).split(np.zeros(len(y)),y,groups)) + + X0=build_v75(f,cache) + Xa=build_ability(at,az) + Xe=build_sparse(et,ez,'EVIDENCE_ABL') + Xd=objective_matrix(f.learning_objective.fillna('').astype(str).tolist()) + p0,_=oof(X0,y,splits,'V75') + pa,_=oof(Xa,y,splits,'ABILITY') + pe,_=oof(Xe,y,splits,'TARGET') + pd,_=oof(Xd,y,splits,'DIFFICULTY') + + # Full latent comparison and causal component ablations. + full,folds=crossfit_meta(y,splits,p0,pa,pe,pd) + # Column definitions from base_meta: 0 V75,1 ability,2 target,3 difficulty, + # 4 ability-difficulty,5 target-ability,6 target-difficulty,... + no_ability,_=crossfit_meta(y,splits,p0,pa,pe,pd,cols=[0,2,3,6,10]) + no_difficulty,_=crossfit_meta(y,splits,p0,pa,pe,pd,cols=[0,1,2,5,7,9,10,12]) + no_target,_=crossfit_meta(y,splits,p0,pa,pe,pd,cols=[0,1,3,4,8,9,11]) + linear_only,_=crossfit_meta(y,splits,p0,pa,pe,pd,cols=[0,1,2,3]) + + # Reproduce V89-style cross-fitted convex composition as a strong control. + fold=np.empty(len(y),int) + for k,(_,va) in enumerate(splits): fold[va]=k + blend=np.zeros(len(y)); selected=[] + grid=np.arange(0,0.61,0.1) + for k,(_,va) in enumerate(splits): + tune=np.where(fold!=k)[0]; best=None + for we in grid: + for wa in grid: + if we+wa>.8: continue + p=np.clip((1-we-wa)*p0[tune]+we*pe[tune]+wa*pa[tune],EPS,1-EPS) + ll=float(log_loss(y[tune],p)) + if best is None or ll=.002 and sum(v>0 for v in causal.values())>=2 else ('PARTIAL' if scores['gain_vs_v89']>0 else 'REJECT') + out={'primary':'objective-cold-crossfitted','scores':scores,'causal_ablation_values':causal,'folds':folds,'v89_selected':selected,'decision':decision} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v92_latent_state_decomposition.json'); run(p.parse_args()) From 741f8d5679f5a1afeb17c66dfe2d8a9870f98d58 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 11:35:12 +1200 Subject: [PATCH 67/77] trace ace: launch V92 latent-state decomposition --- .../workflows/trace-ace-v92-latent-state.yml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/trace-ace-v92-latent-state.yml diff --git a/.github/workflows/trace-ace-v92-latent-state.yml b/.github/workflows/trace-ace-v92-latent-state.yml new file mode 100644 index 0000000..593ecec --- /dev/null +++ b/.github/workflows/trace-ace-v92-latent-state.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V92 latent-state decomposition +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v92_latent_state_decomposition.py' + - '.github/workflows/trace-ace-v92-latent-state.yml' + workflow_dispatch: +jobs: + v92: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V92 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v92_latent_state_decomposition.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v92_latent_state_decomposition.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v92-latent-state + path: v92_latent_state_decomposition.json From e2152c6c1d31148571f1220c57234089ae63f9b7 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 11:55:39 +1200 Subject: [PATCH 68/77] trace ace: add V93 shift-robust validation harness --- .../v93_shift_robust_validation.py | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 competitions/trace_the_ace/v93_shift_robust_validation.py diff --git a/competitions/trace_the_ace/v93_shift_robust_validation.py b/competitions/trace_the_ace/v93_shift_robust_validation.py new file mode 100644 index 0000000..9ebc444 --- /dev/null +++ b/competitions/trace_the_ace/v93_shift_robust_validation.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""V93: distribution-shift validation + robust blend selection. + +Goal: objective-cold CV materially over-predicts V75 public performance. Rather +than tune to leaderboard scores, construct several lawful train-only stress +worlds and choose V75/relative-ability blend weights that remain good across +all of them. + +Worlds: + * objective-cold -- unseen learning objectives + * session-cold -- unseen tutoring sessions + * objective-family-cold -- related objective wording held together + * style-cold -- transcript structural regimes held together + +The public leaderboard is NOT used to fit predictions or weights. +""" +from __future__ import annotations +import argparse, json, re, hashlib +from pathlib import Path +import numpy as np +from sklearn.metrics import log_loss +from sklearn.model_selection import GroupKFold +from sklearn.cluster import KMeans + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training +from v85_evidence_state import build_v75, oof +from v89_relative_ability_composition import non_target_ability, build_ability + +STOP={"the","a","an","to","of","and","with","using","in","on","for","by","from","within","up","simple"} + +def obj_family(s:str)->str: + toks=[x for x in re.findall(r"[a-z]+",str(s).lower()) if x not in STOP] + # intentionally coarse: hold semantically similar surface families together + return "|".join(toks[:3]) if toks else "EMPTY" + +def style_matrix(f,cache): + rows=[] + for _,r in f.iterrows(): + d=cache[str(r.session_id)] + roles=d.role.astype(str).str.lower() if 'role' in d else np.array([]) + contents=d.content.fillna('').astype(str).tolist() if 'content' in d else [] + n=max(1,len(d)); st=sum(x=='student' for x in roles); tu=sum(x=='tutor' for x in roles) + words=sum(len(x.split()) for x in contents); chars=sum(len(x) for x in contents) + math=sum(bool(re.search(r"\d|[=+\-*/%]",x)) for x in contents) + questions=sum('?' in x for x in contents) + rows.append([len(d),st/n,tu/n,words/n,chars/n,math/n,questions/n]) + X=np.asarray(rows,float); return (X-X.mean(0))/(X.std(0)+1e-6) + +def folds_from_groups(groups,n=5): + g=np.asarray(groups).astype(str) + k=min(n,len(np.unique(g))) + if k<2: raise ValueError('need at least two groups') + return list(GroupKFold(k).split(np.zeros(len(g)),np.zeros(len(g)),g)) + +def eval_world(name,X0,Xa,y,splits,grid): + p0,_=oof(X0,y,splits,f'{name}:V75') + pa,_=oof(Xa,y,splits,f'{name}:ABILITY') + ll0=float(log_loss(y,p0)); lla=float(log_loss(y,pa)) + scores=[] + for w in grid: + p=np.clip((1-w)*p0+w*pa,1e-5,1-1e-5) + scores.append({'w':float(w),'ll':float(log_loss(y,p))}) + best=min(scores,key=lambda z:z['ll']) + return {'name':name,'v75':ll0,'ability':lla,'best':best,'curve':scores},p0,pa + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + at=[]; az=[] + for i,r in f.iterrows(): + t,z=non_target_ability(cache[str(r.session_id)],str(r.learning_objective)); at.append(t); az.append(z) + if (i+1)%2500==0: print('rows',i+1) + X0=build_v75(f,cache); Xa=build_ability(at,az); y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sess=f.session_id.astype(str).to_numpy(); fam=f.learning_objective.astype(str).map(obj_family).to_numpy() + SX=style_matrix(f,cache); km=KMeans(n_clusters=5,random_state=137,n_init=10).fit(SX); style=km.labels_.astype(str) + worlds={ + 'objective_cold':folds_from_groups(obj), + 'session_cold':folds_from_groups(sess), + 'objective_family_cold':folds_from_groups(fam), + 'style_cold':folds_from_groups(style), + } + grid=np.linspace(0,0.7,29); results={}; curves={} + for name,sp in worlds.items(): + r,_,_=eval_world(name,X0,Xa,y,sp,grid); results[name]=r; curves[name]={x['w']:x['ll'] for x in r['curve']} + print(name,'V75',r['v75'],'ABILITY',r['ability'],'BEST',r['best']) + # Robust weight: minimize worst excess loss relative to each world's own best. + robust=[] + for w in grid: + exc=[]; raw=[] + for name,r in results.items(): + ll=curves[name][float(w)]; raw.append(ll); exc.append(ll-r['best']['ll']) + robust.append({'w':float(w),'worst_excess':float(max(exc)),'mean_excess':float(np.mean(exc)), + 'worst_logloss':float(max(raw)),'mean_logloss':float(np.mean(raw))}) + choice=min(robust,key=lambda z:(z['worst_excess'],z['mean_excess'],z['mean_logloss'])) + out={'primary':'shift-robust-validation','worlds':results,'robust_choice':choice, + 'objective_cold_best_weight':results['objective_cold']['best']['w'], + 'note':'No leaderboard score used in model fitting, split construction, or weight selection.'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v93_shift_robust.json'); run(p.parse_args()) From 7893d34e392cac75ed6ea2d67a2b99a66f385c96 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 11:55:48 +1200 Subject: [PATCH 69/77] trace ace: launch V93 shift-robust validation --- .../workflows/trace-ace-v93-shift-robust.yml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/trace-ace-v93-shift-robust.yml diff --git a/.github/workflows/trace-ace-v93-shift-robust.yml b/.github/workflows/trace-ace-v93-shift-robust.yml new file mode 100644 index 0000000..2fa8b1a --- /dev/null +++ b/.github/workflows/trace-ace-v93-shift-robust.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V93 shift robust validation +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v93_shift_robust_validation.py' + - '.github/workflows/trace-ace-v93-shift-robust.yml' + workflow_dispatch: +jobs: + v93: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V93 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v93_shift_robust_validation.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v93_shift_robust.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v93-shift-robust + path: v93_shift_robust.json From 7b4b3896e4dbe426d3cf49285fce6d141322284b Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 13:53:41 +1200 Subject: [PATCH 70/77] trace ace: add V94 related-control separator --- .../trace_the_ace/v94_related_control.py | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 competitions/trace_the_ace/v94_related_control.py diff --git a/competitions/trace_the_ace/v94_related_control.py b/competitions/trace_the_ace/v94_related_control.py new file mode 100644 index 0000000..629885f --- /dev/null +++ b/competitions/trace_the_ace/v94_related_control.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""V94: smallest separator for the V89/V93 applicability residual. + +Question: does non-target student evidence help only when it is a *comparable +control* for the target objective? + +A0 GLOBAL : all non-target transcript evidence (V89 representation). +A1 RELATED : only the most objective-related non-target lesson segments. +A2 DISTANT : deliberately least-related non-target segments (causal ablation). + +Opposing discriminators: + * objective-cold -- V93 says GLOBAL ability helps strongly. + * session-cold -- V93 says GLOBAL ability should be suppressed. + +Promotion requires RELATED to preserve objective-cold gain while reducing the +session-cold penalty, and DISTANT must not reproduce the same effect. +""" +from __future__ import annotations +import argparse, json, re +from pathlib import Path +import numpy as np +import pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +from sklearn.metrics import log_loss + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training +from v81_target_segment_phase import choose_target_segment, split_segments, rel_text +from v85_evidence_state import build_v75, oof, evidence_events +from v89_relative_ability_composition import non_target_ability, build_ability, POS, NEG +from v93_shift_robust_validation import folds_from_groups + + +def _ability_from_rows(rows: pd.DataFrame, obj: str, meta: dict, tag: str): + if rows is None or not len(rows): + rows = pd.DataFrame(columns=['role','content']) + roles = rows.role.astype(str).str.lower() if 'role' in rows else pd.Series([], dtype=str) + contents = rows.content.fillna('').astype(str) if 'content' in rows else pd.Series([], dtype=str) + students = contents[roles.eq('student')].tolist() + tutors = contents[roles.eq('tutor')].tolist() + pos = sum(bool(POS.search(x)) for x in tutors) + neg = sum(bool(NEG.search(x)) for x in tutors) + substantive = sum(bool(re.search(r"\d|[=+\-*/%]",x)) or len(x.split()) >= 2 for x in students) + explain = sum(any(k in x.lower() for k in ['because','so ','therefore','i think','first','then']) for x in students) + math = sum(bool(re.search(r"\d|[=+\-*/%]",x)) for x in students) + n=max(1,len(students)); fbn=max(1,pos+neg) + ev = evidence_events(rows.reset_index(drop=True), obj) if len(rows) else [] + z=np.asarray([ + len(rows),len(students),len(tutors),substantive/n,explain/n,math/n, + pos/fbn,neg/fbn,(pos-neg)/fbn,len(ev), + float(meta.get('chosen_mean_rel',0.0)),float(meta.get('chosen_max_rel',0.0)), + float(meta.get('available_segments',0)),float(meta.get('chosen_segments',0)), + ],float) + text=' [STUDENT] '.join(students[-80:]) + verdict=' '.join('[POS]' if POS.search(x) else '[NEG]' if NEG.search(x) else '' for x in tutors[-100:]) + return f'[{tag}] {text} [VERDICTS] {verdict}', z + + +def segmented_control(df: pd.DataFrame, obj: str, mode: str): + _, tm = choose_target_segment(df, obj) + ts,te=int(tm['start']),int(tm['end']) + cand=[] + for s,e in split_segments(df): + # exclude any segment overlapping the selected target segment + if not (e <= ts or s >= te): + continue + part=df.iloc[s:e].copy().reset_index(drop=True) + txt=' '.join(part.content.fillna('').astype(str)) + rel=float(rel_text(txt,obj)) + cand.append((rel,s,e,part)) + if not cand: + rest=df.iloc[list(range(0,ts))+list(range(te,len(df)))].copy().reset_index(drop=True) + return _ability_from_rows(rest,obj,{'available_segments':0,'chosen_segments':0},mode.upper()) + cand=sorted(cand,key=lambda x:(x[0],x[1])) + # Keep about half the non-target segments, capped to three, to make RELATED and + # DISTANT equal-budget representations rather than a length confound. + k=max(1,min(3,int(np.ceil(len(cand)/2)))) + chosen = cand[-k:] if mode=='related' else cand[:k] + rows=pd.concat([x[3] for x in sorted(chosen,key=lambda z:z[1])],ignore_index=True) + rels=[x[0] for x in chosen] + meta={'available_segments':len(cand),'chosen_segments':len(chosen), + 'chosen_mean_rel':float(np.mean(rels)),'chosen_max_rel':float(np.max(rels))} + return _ability_from_rows(rows,obj,meta,mode.upper()) + + +def build_control(texts, nums): + hv=HashingVectorizer(n_features=2**17,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + X=hv.transform(texts); Z=np.vstack(nums).astype(float); Z=(Z-Z.mean(0))/(Z.std(0)+1e-6) + return hstack([X,csr_matrix(Z)],format='csr') + + +def eval_arm(name, X0, Xa, y, sp): + p0,_=oof(X0,y,sp,name+':V75'); pa,_=oof(Xa,y,sp,name+':ABILITY') + grid=np.linspace(0,0.6,25); curve=[] + for w in grid: + q=np.clip((1-w)*p0+w*pa,1e-5,1-1e-5) + curve.append({'w':float(w),'ll':float(log_loss(y,q))}) + best=min(curve,key=lambda z:z['ll']) + return {'v75':float(log_loss(y,p0)),'ability':float(log_loss(y,pa)),'best':best, + 'gain':float(log_loss(y,p0)-best['ll'])} + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + gt=[];gz=[];rt=[];rz=[];dt=[];dz=[] + for i,r in f.iterrows(): + d=cache[str(r.session_id)]; obj=str(r.learning_objective) + t,z=non_target_ability(d,obj); gt.append(t); gz.append(z) + t,z=segmented_control(d,obj,'related'); rt.append(t); rz.append(z) + t,z=segmented_control(d,obj,'distant'); dt.append(t); dz.append(z) + if (i+1)%2500==0: print('rows',i+1) + X0=build_v75(f,cache); Xg=build_ability(gt,gz); Xr=build_control(rt,rz); Xd=build_control(dt,dz) + y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + sess=f.session_id.astype(str).to_numpy() + worlds={'objective_cold':folds_from_groups(obj),'session_cold':folds_from_groups(sess)} + out={'primary':'related-control-separator','worlds':{}} + for wn,sp in worlds.items(): + out['worlds'][wn]={ + 'global':eval_arm(wn+':GLOBAL',X0,Xg,y,sp), + 'related':eval_arm(wn+':RELATED',X0,Xr,y,sp), + 'distant':eval_arm(wn+':DISTANT',X0,Xd,y,sp), + } + o=out['worlds']['objective_cold']; s=out['worlds']['session_cold'] + obj_preserve=o['related']['gain'] >= 0.75*max(1e-9,o['global']['gain']) + session_repair=s['related']['best']['ll'] <= s['global']['best']['ll']-0.0005 + causal=o['related']['best']['ll'] <= o['distant']['best']['ll']-0.0005 + if obj_preserve and session_repair and causal: + verdict='PROMOTE_RELATED_CONTROL' + elif o['related']['best']['ll'] < o['global']['best']['ll'] and causal: + verdict='PARTIAL_R5_REFINE' + else: + verdict='REJECT_RELATEDNESS_OBSERVABLE' + out['decision']={'objective_gain_preserved':bool(obj_preserve),'session_penalty_repaired':bool(session_repair), + 'causal_vs_distant':bool(causal),'verdict':verdict} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v94_related_control.json'); run(p.parse_args()) From f05a11b8dbd3866291f0ae142272cfb35fefb822 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 13:53:54 +1200 Subject: [PATCH 71/77] trace ace: launch V94 related-control separator --- .../trace-ace-v94-related-control.yml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/trace-ace-v94-related-control.yml diff --git a/.github/workflows/trace-ace-v94-related-control.yml b/.github/workflows/trace-ace-v94-related-control.yml new file mode 100644 index 0000000..8b3e49c --- /dev/null +++ b/.github/workflows/trace-ace-v94-related-control.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V94 related control +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v94_related_control.py' + - '.github/workflows/trace-ace-v94-related-control.yml' + workflow_dispatch: +jobs: + v94: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V94 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v94_related_control.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v94_related_control.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v94-related-control + path: v94_related_control.json From a122ab4bf60a08034c3af228772d146f32adab9f Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 17:28:14 +1200 Subject: [PATCH 72/77] trace ace: add V95 objective-support activation separator --- .../trace-ace-v95-objective-support.yml | 39 +++ .../v95_objective_support_activation.py | 275 ++++++++++++++++++ 2 files changed, 314 insertions(+) create mode 100644 .github/workflows/trace-ace-v95-objective-support.yml create mode 100644 competitions/trace_the_ace/v95_objective_support_activation.py diff --git a/.github/workflows/trace-ace-v95-objective-support.yml b/.github/workflows/trace-ace-v95-objective-support.yml new file mode 100644 index 0000000..9fe6e05 --- /dev/null +++ b/.github/workflows/trace-ace-v95-objective-support.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V95 objective support +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v95_objective_support_activation.py' + - '.github/workflows/trace-ace-v95-objective-support.yml' + workflow_dispatch: +jobs: + v95: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V95 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v95_objective_support_activation.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v95_objective_support_activation.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v95-objective-support + path: v95_objective_support_activation.json diff --git a/competitions/trace_the_ace/v95_objective_support_activation.py b/competitions/trace_the_ace/v95_objective_support_activation.py new file mode 100644 index 0000000..aaf0242 --- /dev/null +++ b/competitions/trace_the_ace/v95_objective_support_activation.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""V95: objective-support activation law for the V94 RELATED ability specialist. + +Residual +-------- +V94 showed that RELATED relative-ability evidence helps objective-cold validation +but should receive zero weight in session-cold validation. The proposed missing +applicability variable is epistemic support for the target objective. + +Separator +--------- +Take one deterministic objective-cold outer fold. For every held-out objective, +reserve a deterministic support pool and a disjoint fixed evaluation set. Reveal +nested amounts of labelled target-objective support (0, 1, 2, 4, 8, 16, 32+ per +objective), refit V75 and the V94 RELATED expert, and score the *same* evaluation +rows at every support level. + +Prediction +---------- +Optimal RELATED blend weight should be high at zero support and decline toward +zero as target-objective support increases. + +Cheap causal ablation +--------------------- +At support 8 and 32+, shuffle only the newly revealed support labels before +fitting the RELATED expert. Extra rows without valid target information should +not reproduce a lawful support benefit. + +No leaderboard score or hidden-test outcome is used anywhere in construction, +fitting, weighting, or the promotion decision. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import numpy as np +from scipy.stats import spearmanr +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import log_loss + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v93_shift_robust_validation import folds_from_groups +from v94_related_control import segmented_control, build_control + + +LEVELS = [0, 1, 2, 4, 8, 16, "32+"] +GRID = np.linspace(0.0, 0.6, 25) + + +def stable_key(obj: str, response_id: str) -> str: + return hashlib.sha256(f"V95|{SEED}|{obj}|{response_id}".encode()).hexdigest() + + +def make_fixed_support_design(frame, val_idx, objective): + """Return disjoint nested support pools and one fixed evaluation index. + + Up to half of each held-out objective (capped at 64 rows) is reserved as its + support pool. The complement is scored at every support level, eliminating + changing-evaluation-set confounding. Small objectives saturate naturally; + the result records realised support counts for every level. + """ + val_idx = np.asarray(val_idx, dtype=int) + response = ( + frame.response_id.astype(str).to_numpy() + if "response_id" in frame + else np.arange(len(frame)).astype(str) + ) + pools = {} + eval_parts = [] + for g in sorted(np.unique(objective[val_idx])): + idx = val_idx[objective[val_idx] == g] + idx = np.asarray(sorted(idx, key=lambda i: stable_key(str(g), response[i])), dtype=int) + pool_n = min(64, len(idx) // 2) + pools[str(g)] = idx[:pool_n] + eval_parts.append(idx[pool_n:]) + eval_idx = np.concatenate(eval_parts) if eval_parts else np.array([], dtype=int) + return pools, np.asarray(sorted(eval_idx), dtype=int) + + +def support_indices(pools, level): + out = [] + counts = [] + for g in sorted(pools): + pool = pools[g] + k = len(pool) if level == "32+" else min(int(level), len(pool)) + counts.append(k) + if k: + out.append(pool[:k]) + idx = np.concatenate(out) if out else np.array([], dtype=int) + return np.asarray(sorted(idx), dtype=int), np.asarray(counts, dtype=int) + + +def fit_predict(X, y, train_idx, eval_idx, y_train_override=None): + yy = y[train_idx] if y_train_override is None else np.asarray(y_train_override, dtype=int) + m = LogisticRegression( + C=0.25, + max_iter=300, + solver="liblinear", + random_state=SEED, + ).fit(X[train_idx], yy) + return np.clip(m.predict_proba(X[eval_idx])[:, 1], 1e-5, 1 - 1e-5) + + +def best_blend(y, p0, pa): + curve = [] + for w in GRID: + q = np.clip((1 - w) * p0 + w * pa, 1e-5, 1 - 1e-5) + curve.append({"w": float(w), "ll": float(log_loss(y, q))}) + return min(curve, key=lambda z: z["ll"]), curve + + +def realised_support_summary(counts): + if not len(counts): + return {"min": 0, "median": 0.0, "mean": 0.0, "max": 0, "objectives": 0} + return { + "min": int(np.min(counts)), + "median": float(np.median(counts)), + "mean": float(np.mean(counts)), + "max": int(np.max(counts)), + "objectives": int(len(counts)), + } + + +def run(a): + f = load_training(a.features, a.labels).reset_index(drop=True) + cache = { + sid: load_transcript(a.transcripts / f"{sid}.csv") + for sid in f.session_id.astype(str).unique() + } + + related_text, related_num = [], [] + for i, r in f.iterrows(): + d = cache[str(r.session_id)] + t, z = segmented_control(d, str(r.learning_objective), "related") + related_text.append(t) + related_num.append(z) + if (i + 1) % 2500 == 0: + print("rows", i + 1) + + X0 = build_v75(f, cache) + Xr = build_control(related_text, related_num) + y = f.target.to_numpy(int) + obj = ( + f.learning_objective_id + if "learning_objective_id" in f + else f.learning_objective + ).astype(str).to_numpy() + + # Cheapest sufficient causal world: the first deterministic GroupKFold + # objective-cold split, held fixed for every support dose. + base_train, heldout = folds_from_groups(obj)[0] + pools, eval_idx = make_fixed_support_design(f, heldout, obj) + if not len(eval_idx): + raise RuntimeError("V95 fixed evaluation set is empty") + + eval_y = y[eval_idx] + results = [] + predictions = {} + support_by_level = {} + + for level in LEVELS: + sup_idx, counts = support_indices(pools, level) + train_idx = np.concatenate([np.asarray(base_train, dtype=int), sup_idx]) + p0 = fit_predict(X0, y, train_idx, eval_idx) + pr = fit_predict(Xr, y, train_idx, eval_idx) + b, curve = best_blend(eval_y, p0, pr) + ll0 = float(log_loss(eval_y, p0)) + llr = float(log_loss(eval_y, pr)) + label = str(level) + results.append({ + "support": label, + "realised_support_per_objective": realised_support_summary(counts), + "support_rows_total": int(len(sup_idx)), + "eval_rows": int(len(eval_idx)), + "v75": ll0, + "related_ability": llr, + "best": b, + "gain_vs_v75": float(ll0 - b["ll"]), + "blend_curve": curve, + }) + predictions[label] = (p0, pr) + support_by_level[label] = (sup_idx, counts) + print("SUPPORT", label, "V75", ll0, "RELATED", llr, "BEST", b) + + # Information-destruction ablation: same revealed rows and class marginal, + # but support labels are deterministically shuffled. Only RELATED is refit; + # this asks whether valid labelled support, rather than row count alone, + # improves the specialist representation. + ablations = {} + rng = np.random.RandomState(SEED + 95) + for level in (8, "32+"): + label = str(level) + sup_idx, _ = support_by_level[label] + train_idx = np.concatenate([np.asarray(base_train, dtype=int), sup_idx]) + yy = y[train_idx].copy() + nbase = len(base_train) + if len(sup_idx) > 1: + yy[nbase:] = yy[nbase:][rng.permutation(len(sup_idx))] + pr_bad = fit_predict(Xr, y, train_idx, eval_idx, y_train_override=yy) + normal_pr = predictions[label][1] + ablations[label] = { + "normal_related_ll": float(log_loss(eval_y, normal_pr)), + "shuffled_support_related_ll": float(log_loss(eval_y, pr_bad)), + "valid_information_gain": float(log_loss(eval_y, pr_bad) - log_loss(eval_y, normal_pr)), + } + print("ABLATION", label, ablations[label]) + + weights = np.asarray([r["best"]["w"] for r in results], dtype=float) + # Use realised median support, with 32+ naturally reflecting the whole fixed pool. + dose = np.asarray([r["realised_support_per_objective"]["median"] for r in results], dtype=float) + rho = float(spearmanr(dose, weights).statistic) if len(np.unique(dose)) > 1 else 0.0 + near_monotone_steps = int(np.sum(np.diff(weights) <= 0.025 + 1e-12)) + possible_steps = len(weights) - 1 + delta = float(weights[0] - weights[-1]) + low_gain = float(results[0]["gain_vs_v75"]) + high_weight = float(weights[-1]) + + clean = ( + near_monotone_steps >= possible_steps - 1 + and rho <= -0.75 + and delta >= 0.15 + and low_gain >= 0.003 + and high_weight <= 0.15 + ) + partial = rho <= -0.50 and delta >= 0.10 and low_gain >= 0.002 + if clean: + verdict = "PROMOTE_OBJECTIVE_SUPPORT_ACTIVATION" + elif partial: + verdict = "R5_REFINE_EFFECTIVE_SUPPORT" + else: + verdict = "SUPPRESS_OBJECTIVE_SUPPORT" + + out = { + "primary": "objective-support-activation-law", + "design": { + "outer_world": "first deterministic objective-cold GroupKFold split", + "support_levels": [str(x) for x in LEVELS], + "fixed_eval_rows": int(len(eval_idx)), + "heldout_objectives": int(len(pools)), + "support_pool_rule": "stable hash order; reserve up to half/objective capped at 64; score fixed complement", + "note": "No leaderboard score or hidden-test outcome used in fitting, weighting, or decision.", + }, + "support_response": results, + "ablations": ablations, + "decision": { + "spearman_support_vs_weight": rho, + "near_monotone_steps": near_monotone_steps, + "possible_steps": possible_steps, + "weight_drop_zero_to_32plus": delta, + "zero_support_gain_vs_v75": low_gain, + "high_support_weight": high_weight, + "verdict": verdict, + "precommit": { + "promote": "near-monotone (<=1 tolerance step), rho<=-0.75, weight drop>=0.15, zero-support gain>=0.003, 32+ weight<=0.15", + "refine": "rho<=-0.50, weight drop>=0.10, zero-support gain>=0.002", + "otherwise": "suppress raw objective support and seek another observable", + }, + }, + } + Path(a.out).write_text(json.dumps(out, indent=2)) + print(json.dumps(out, indent=2)) + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument("--features", type=Path, required=True) + p.add_argument("--labels", type=Path, required=True) + p.add_argument("--transcripts", type=Path, required=True) + p.add_argument("--out", default="v95_objective_support_activation.json") + run(p.parse_args()) From 90441f4197cf8c3c9ab2d0f76cc02138f8d5e332 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 18:43:15 +1200 Subject: [PATCH 73/77] trace ace: add V96 effective-support separator --- .../trace-ace-v96-effective-support.yml | 39 +++++++ .../v96_effective_support_separator.py | 103 ++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 .github/workflows/trace-ace-v96-effective-support.yml create mode 100644 competitions/trace_the_ace/v96_effective_support_separator.py diff --git a/.github/workflows/trace-ace-v96-effective-support.yml b/.github/workflows/trace-ace-v96-effective-support.yml new file mode 100644 index 0000000..ac63209 --- /dev/null +++ b/.github/workflows/trace-ace-v96-effective-support.yml @@ -0,0 +1,39 @@ +name: Trace the Ace V96 effective support +on: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v96_effective_support_separator.py' + - '.github/workflows/trace-ace-v96-effective-support.yml' + workflow_dispatch: +jobs: + v96: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install dependencies + run: pip install numpy pandas scipy scikit-learn gdown + - name: Download data + run: | + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + - name: Run V96 + shell: bash + run: | + FEATURES=$(find data/meta -type f -name 'train_features*.csv' | head -1) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' | head -1) + TRANSCRIPTS=$(dirname "$(find data/transcripts -type f -name '*.csv' | head -1)") + cd competitions/trace_the_ace + python v96_effective_support_separator.py --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" --out ../../v96_effective_support.json + - name: Upload result + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v96-effective-support + path: v96_effective_support.json diff --git a/competitions/trace_the_ace/v96_effective_support_separator.py b/competitions/trace_the_ace/v96_effective_support_separator.py new file mode 100644 index 0000000..1e52570 --- /dev/null +++ b/competitions/trace_the_ace/v96_effective_support_separator.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""V96: refine V95 raw support into an effective-support observable. + +V95 established a causal regime transition: RELATED ability is valuable with +little target-objective support and should be suppressed once V75 has enough +support. Raw dose was too crude because many held-out objectives saturate early. + +This separator keeps V95's fixed evaluation rows and nested support intervention, +but asks which observable explains the transition better: + COUNT realised labelled rows for the objective; + COVERAGE realised rows / available support-pool rows (saturation); + BALANCE labelled support containing both classes; + CAPACITY available support-pool size (objective prevalence control). + +For every dose we fit V75 and RELATED exactly as in V95, then score per-objective +losses on the same evaluation rows. For each observable we fit a tiny deterministic +threshold router: below threshold use the globally selected V95 blend weight for +that dose, above threshold use V75. Thresholds are selected on a deterministic +half of held-out objectives and evaluated on the other half. This is a routing +separator, not a leaderboard fit. +""" +from __future__ import annotations +import argparse, hashlib, json +from pathlib import Path +import numpy as np +from sklearn.metrics import log_loss +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v93_shift_robust_validation import folds_from_groups +from v94_related_control import segmented_control, build_control +from v95_objective_support_activation import make_fixed_support_design, support_indices, fit_predict, best_blend, LEVELS + + +def split_objectives(objs): + def key(g): return hashlib.sha256(f'V96|{SEED}|{g}'.encode()).hexdigest() + s=sorted(objs,key=key); return set(s[::2]),set(s[1::2]) + +def ll(y,p): return float(log_loss(y,np.clip(p,1e-5,1-1e-5))) + +def route_eval(y,p0,pa,groups,metric,fit_objs,test_objs,w): + vals=np.array([metric[str(g)] for g in groups],float) + uniq=np.unique([metric[g] for g in fit_objs if g in metric]) + if len(uniq)>20: cuts=np.unique(np.quantile(uniq,np.linspace(0,1,21))) + else: cuts=uniq + candidates=[-1e-12]+[float(x) for x in cuts]+[float(np.max(uniq)+1e-9)] if len(uniq) else [0.0] + fit=np.array([str(g) in fit_objs for g in groups]); test=np.array([str(g) in test_objs for g in groups]) + rows=[] + for t in candidates: + # low effective support => ability blend; high => V75 + use=vals<=t; q=np.where(use,(1-w)*p0+w*pa,p0) + rows.append((t,ll(y[fit],q[fit]),ll(y[test],q[test]),float(np.mean(use[test])))) + best=min(rows,key=lambda z:z[1]) + return {'threshold':best[0],'fit_ll':best[1],'test_ll':best[2],'test_ability_fraction':best[3], + 'test_v75':ll(y[test],p0[test]),'test_global_blend':ll(y[test],((1-w)*p0+w*pa)[test]), + 'gain_vs_v75':ll(y[test],p0[test])-best[2]} + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + rt=[]; rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related'); rt.append(t); rz.append(z) + if (i+1)%2500==0: print('rows',i+1) + X0=build_v75(f,cache); Xr=build_control(rt,rz); y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + base,held=folds_from_groups(obj)[0]; pools,ev=make_fixed_support_design(f,held,obj); ey=y[ev]; eg=obj[ev] + fit_objs,test_objs=split_objectives(sorted(pools)) + out={'primary':'effective-support-separator','objective_router_split':{'fit':len(fit_objs),'test':len(test_objs)},'levels':[]} + for level in LEVELS: + si,counts=support_indices(pools,level); tr=np.concatenate([np.asarray(base,int),si]) + p0=fit_predict(X0,y,tr,ev); pa=fit_predict(Xr,y,tr,ev); b,_=best_blend(ey,p0,pa); w=float(b['w']) + count={}; coverage={}; balance={}; capacity={} + for g in pools: + pool=pools[g]; k=len(pool) if level=='32+' else min(int(level),len(pool)); chosen=pool[:k] + count[g]=float(k); capacity[g]=float(len(pool)); coverage[g]=float(k/max(1,len(pool))) + balance[g]=float(len(np.unique(y[chosen]))>=2) if k else 0.0 + arms={ + 'count':route_eval(ey,p0,pa,eg,count,fit_objs,test_objs,w), + 'coverage':route_eval(ey,p0,pa,eg,coverage,fit_objs,test_objs,w), + 'balance':route_eval(ey,p0,pa,eg,balance,fit_objs,test_objs,w), + 'capacity':route_eval(ey,p0,pa,eg,capacity,fit_objs,test_objs,w), + } + out['levels'].append({'support':str(level),'global_weight':w,'arms':arms}) + print('LEVEL',level,'W',w,{k:round(v['gain_vs_v75'],6) for k,v in arms.items()}) + # Decision uses only levels where V95 had a nonzero specialist weight. + useful=[x for x in out['levels'] if x['global_weight']>0] + means={k:float(np.mean([x['arms'][k]['gain_vs_v75'] for x in useful])) for k in ('count','coverage','balance','capacity')} + wins={k:int(sum(x['arms'][k]['test_ll'] <= min(v['test_ll'] for v in x['arms'].values())+1e-12 for x in useful)) for k in means} + best=max(means,key=means.get) + # Promotion needs positive held-out routing value and superiority to raw count. + if best!='count' and means[best]>=0.001 and means[best]>=means['count']+0.0005: + verdict='PROMOTE_'+best.upper()+'_AS_EFFECTIVE_SUPPORT' + elif max(means.values())>=0.0005: + verdict='R5_REFINE_COMPOSITE_SUPPORT' + else: + verdict='SUPPRESS_SIMPLE_SUPPORT_ROUTING' + out['decision']={'mean_test_gain_vs_v75':means,'wins':wins,'best_observable':best,'verdict':verdict, + 'precommit':'promote non-count observable iff held-out mean gain>=.001 and >= raw-count gain+.0005; else refine if any >=.0005'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v96_effective_support.json'); run(p.parse_args()) From 565fbacf8b4e2731d77fb87c40bfbab981950cda Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 20:13:11 +1200 Subject: [PATCH 74/77] trace ace: add V97 support gate --- .../trace_the_ace/v97_support_gate.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 competitions/trace_the_ace/v97_support_gate.py diff --git a/competitions/trace_the_ace/v97_support_gate.py b/competitions/trace_the_ace/v97_support_gate.py new file mode 100644 index 0000000..157510b --- /dev/null +++ b/competitions/trace_the_ace/v97_support_gate.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""V97: submission-sprint exact-support gate. + +Frozen law from V93-V96: RELATED ability is a specialist for objectives with no +training support; V75 dominates when exact objective support is present. +This test does not tune the law. It evaluates the precommitted runtime rule: + count_train(objective) == 0 -> 0.35 RELATED + 0.65 V75 + otherwise -> V75 +across four shift worlds using fold-local training counts only. +""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.metrics import log_loss + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training +from v85_evidence_state import build_v75, oof +from v93_shift_robust_validation import folds_from_groups, obj_family, style_matrix +from v94_related_control import segmented_control, build_control +from sklearn.cluster import KMeans + +W_UNSEEN = 0.35 +EPS = 1e-5 + + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + rt=[]; rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related') + rt.append(t); rz.append(z) + if (i+1)%2500==0: print('rows',i+1) + X0=build_v75(f,cache); Xr=build_control(rt,rz); y=f.target.to_numpy(int) + obj=(f.learning_objective_id if 'learning_objective_id' in f else f.learning_objective).astype(str).to_numpy() + support_key=f.learning_objective.astype(str).to_numpy() # always runtime-visible + sess=f.session_id.astype(str).to_numpy(); fam=f.learning_objective.astype(str).map(obj_family).to_numpy() + SX=style_matrix(f,cache); style=KMeans(n_clusters=5,random_state=137,n_init=10).fit(SX).labels_.astype(str) + worlds={'objective_cold':folds_from_groups(obj),'session_cold':folds_from_groups(sess), + 'objective_family_cold':folds_from_groups(fam),'style_cold':folds_from_groups(style)} + out={'primary':'fixed-exact-support-runtime-gate','law':{'unseen_weight':W_UNSEEN,'seen_weight':0.0,'predicate':'fold-local exact learning_objective text training count == 0'},'worlds':{}} + gains=[]; regress=[] + for name,sp in worlds.items(): + p0,_=oof(X0,y,sp,name+':V75'); pr,_=oof(Xr,y,sp,name+':RELATED') + q=np.zeros(len(y)); unseen=np.zeros(len(y),bool) + for tr,va in sp: + counts={g:int(n) for g,n in zip(*np.unique(support_key[tr],return_counts=True))} + m=np.array([counts.get(str(support_key[i]),0)==0 for i in va],bool); unseen[va]=m + w=np.where(m,W_UNSEEN,0.0) + q[va]=np.clip((1-w)*p0[va]+w*pr[va],EPS,1-EPS) + ll0=float(log_loss(y,p0)); llq=float(log_loss(y,q)); gain=ll0-llq + rec={'v75':ll0,'support_gate':llq,'gain_vs_v75':gain,'ability_fraction':float(unseen.mean()), + 'unseen_rows':int(unseen.sum()),'seen_rows':int((~unseen).sum())} + out['worlds'][name]=rec; gains.append(gain); regress.append(max(0.0,-gain)); print(name,rec) + mean_gain=float(np.mean(gains)); worst_reg=float(max(regress)); obj_gain=out['worlds']['objective_cold']['gain_vs_v75'] + promote=obj_gain>=0.0025 and mean_gain>=0.0007 and worst_reg<=0.0005 + out['decision']={'objective_gain':obj_gain,'mean_gain_four_worlds':mean_gain,'worst_world_regression':worst_reg, + 'verdict':'BUILD_SUBMISSION_NOW' if promote else 'STOP_DO_NOT_SUBMIT', + 'precommit':'build iff objective gain >=.0025, four-world mean gain >=.0007, worst regression <=.0005'} + Path(a.out).write_text(json.dumps(out,indent=2)); print(json.dumps(out,indent=2)) + if a.require_promote and not promote: raise SystemExit(42) + +if __name__=='__main__': + p=argparse.ArgumentParser(); p.add_argument('--features',type=Path,required=True); p.add_argument('--labels',type=Path,required=True); p.add_argument('--transcripts',type=Path,required=True); p.add_argument('--out',default='v97_support_gate.json'); p.add_argument('--require-promote',action='store_true'); run(p.parse_args()) From cc0bae33cc126d3abc8a89c945f41e0d16a1dbd2 Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 20:13:22 +1200 Subject: [PATCH 75/77] trace ace: add V97 runtime asset builder --- .../trace_the_ace/train_v97_runtime_assets.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 competitions/trace_the_ace/train_v97_runtime_assets.py diff --git a/competitions/trace_the_ace/train_v97_runtime_assets.py b/competitions/trace_the_ace/train_v97_runtime_assets.py new file mode 100644 index 0000000..0314af8 --- /dev/null +++ b/competitions/trace_the_ace/train_v97_runtime_assets.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Train V75 + V94 RELATED assets for the fixed V97 support gate.""" +from __future__ import annotations +import argparse, json +from pathlib import Path +import numpy as np +from sklearn.linear_model import LogisticRegression + +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import load_training, SEED +from v85_evidence_state import build_v75 +from v94_related_control import segmented_control, build_control + + +def pack_model(m): return m.coef_.ravel().astype(np.float64), np.asarray(m.intercept_,dtype=np.float64) + +def run(a): + f=load_training(a.features,a.labels).reset_index(drop=True) + cache={sid:load_transcript(a.transcripts/f'{sid}.csv') for sid in f.session_id.astype(str).unique()} + rt=[];rz=[] + for i,r in f.iterrows(): + t,z=segmented_control(cache[str(r.session_id)],str(r.learning_objective),'related'); rt.append(t);rz.append(z) + if (i+1)%2500==0: print('rows',i+1) + X0=build_v75(f,cache); Xr=build_control(rt,rz); y=f.target.to_numpy(int) + m0=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(X0,y) + mr=LogisticRegression(C=.25,max_iter=300,solver='liblinear',random_state=SEED).fit(Xr,y) + Z=np.vstack(rz).astype(float); rmean=Z.mean(0); rstd=Z.std(0)+1e-6 + from v75_canonical_trajectory import trajectory_views + nums=[] + for _,r in f.iterrows(): nums.append(trajectory_views(cache[str(r.session_id)],str(r.learning_objective))[1]) + N=np.vstack(nums).astype(float); vmean=N.mean(0); vstd=N.std(0)+1e-6 + c0,b0=pack_model(m0); cr,br=pack_model(mr) + a.out_dir.mkdir(parents=True,exist_ok=True) + np.savez_compressed(a.out_dir/'v97_assets.npz',v75_coef=c0,v75_intercept=b0,v75_num_mean=vmean,v75_num_std=vstd, + related_coef=cr,related_intercept=br,related_num_mean=rmean,related_num_std=rstd) + counts=f.groupby('learning_objective').size().astype(int).to_dict() + manifest={'candidate':'V97_FIXED_EXACT_SUPPORT_GATE','unseen_weight':0.35,'seen_weight':0.0,'objective_key':'learning_objective', + 'objective_counts':{str(k):int(v) for k,v in counts.items()},'rows':int(len(f))} + (a.out_dir/'manifest.json').write_text(json.dumps(manifest,indent=2)); print(json.dumps({k:v for k,v in manifest.items() if 'counts' not in k},indent=2)) + +if __name__=='__main__': + p=argparse.ArgumentParser();p.add_argument('--features',type=Path,required=True);p.add_argument('--labels',type=Path,required=True);p.add_argument('--transcripts',type=Path,required=True);p.add_argument('--out-dir',type=Path,required=True);run(p.parse_args()) From 70492884c98e2d04ac10230efc0b22ac6f565c2f Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 20:13:33 +1200 Subject: [PATCH 76/77] trace ace: add V97 official runtime --- .../trace_the_ace/runtime_v97/main.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 competitions/trace_the_ace/runtime_v97/main.py diff --git a/competitions/trace_the_ace/runtime_v97/main.py b/competitions/trace_the_ace/runtime_v97/main.py new file mode 100644 index 0000000..4732de7 --- /dev/null +++ b/competitions/trace_the_ace/runtime_v97/main.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Official runtime for V97 fixed exact-support gate.""" +from pathlib import Path +import json, sys +import numpy as np, pandas as pd +from scipy.sparse import csr_matrix, hstack +from sklearn.feature_extraction.text import HashingVectorizer +HERE=Path(__file__).resolve().parent; DATA=Path('/code_execution/data'); sys.path.insert(0,str(HERE)) +from v71_mastery_events import load_transcript +from v75_canonical_trajectory import trajectory_views +from v94_related_control import segmented_control + + +def sigmoid(x): return 1/(1+np.exp(-np.clip(np.asarray(x,float),-40,40))) +def main(): + f=pd.read_csv(DATA/'test_features.csv'); fmt=pd.read_csv(DATA/'submission_format.csv') + a=np.load(HERE/'assets/v97_assets.npz'); man=json.loads((HERE/'assets/manifest.json').read_text()) + cache={}; views=[]; vnums=[]; rt=[];rz=[] + for r in f.itertuples(index=False): + sid=str(r.session_id) + if sid not in cache: cache[sid]=load_transcript(DATA/'test_transcripts'/f'{sid}.csv') + obj=str(r.learning_objective); v,n,_=trajectory_views(cache[sid],obj); views.append(v);vnums.append(n) + t,z=segmented_control(cache[sid],obj,'related');rt.append(t);rz.append(z) + hv75=HashingVectorizer(n_features=2**18,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + Z=(np.vstack(vnums)-a['v75_num_mean'])/a['v75_num_std'] + X0=hstack([hv75.transform(['[OBJECTIVE] '+str(x) for x in f.learning_objective]), + hv75.transform(['[RAW] '+v['raw'] for v in views]),hv75.transform(['[STUDENT] '+v['student'] for v in views]), + hv75.transform(['[LOCAL] '+v['local'] for v in views]),hv75.transform(['[STATE] '+v['canonical'] for v in views]), + hv75.transform(['[TERMINAL] '+v['terminal'] for v in views]),csr_matrix(Z)],format='csr') + p0=sigmoid(np.asarray(X0@a['v75_coef']).ravel()+float(a['v75_intercept'][0])) + hvr=HashingVectorizer(n_features=2**17,alternate_sign=False,norm='l2',ngram_range=(1,2),lowercase=True) + R=(np.vstack(rz)-a['related_num_mean'])/a['related_num_std']; Xr=hstack([hvr.transform(rt),csr_matrix(R)],format='csr') + pr=sigmoid(np.asarray(Xr@a['related_coef']).ravel()+float(a['related_intercept'][0])) + keys=f.learning_objective.astype(str); counts=man['objective_counts'] + w=np.array([man['unseen_weight'] if int(counts.get(str(k),0))==0 else 0.0 for k in keys],float) + p=np.clip((1-w)*p0+w*pr,1e-5,1-1e-5) + gen=pd.DataFrame({'response_id':f.response_id.astype(str),'probability':p}) + out=fmt[['response_id']].astype({'response_id':str}).merge(gen,on='response_id',how='left',validate='one_to_one') + if out.probability.isna().any(): raise RuntimeError('missing predictions') + out.to_csv(HERE/'submission.csv',index=False) +if __name__=='__main__': main() From 5ce5d9b12120f9d257184a63bbda4cbd7f85ef8f Mon Sep 17 00:00:00 2001 From: Heath Sanchez <32909488+heathsanchez@users.noreply.github.com> Date: Sun, 16 Aug 2026 20:13:48 +1200 Subject: [PATCH 77/77] trace ace: launch V97 submission sprint --- .../trace-ace-v97-submission-sprint.yml | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 .github/workflows/trace-ace-v97-submission-sprint.yml diff --git a/.github/workflows/trace-ace-v97-submission-sprint.yml b/.github/workflows/trace-ace-v97-submission-sprint.yml new file mode 100644 index 0000000..0bb8087 --- /dev/null +++ b/.github/workflows/trace-ace-v97-submission-sprint.yml @@ -0,0 +1,143 @@ +name: Trace the Ace V97 submission sprint +on: + workflow_dispatch: + push: + branches: [agent/trace-ace-mastery-events] + paths: + - 'competitions/trace_the_ace/v97_support_gate.py' + - 'competitions/trace_the_ace/train_v97_runtime_assets.py' + - 'competitions/trace_the_ace/runtime_v97/**' + - '.github/workflows/trace-ace-v97-submission-sprint.yml' + +jobs: + validate-and-build: + runs-on: ubuntu-latest + timeout-minutes: 360 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + - uses: astral-sh/setup-uv@v6 + - uses: extractions/setup-just@v2 + - name: Install dependencies + run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown + + - name: Download frozen training data + run: | + set -euo pipefail + gdown 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI -O transcripts.zip + gdown 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz -O metadata.zip + mkdir -p data/transcripts data/meta + unzip -q transcripts.zip -d data/transcripts + unzip -q metadata.zip -d data/meta + + - name: Resolve schemas + shell: bash + run: | + set -euo pipefail + FEATURES=$(find data/meta -type f -name 'train_features*.csv' -print -quit) + LABELS=$(find data/meta -type f -name 'train_labels*.csv' -print -quit) + FIRST=$(find data/transcripts -type f -name '*.csv' -print -quit) + TRANSCRIPTS=$(dirname "$FIRST") + test -n "$FEATURES" && test -n "$LABELS" && test -n "$FIRST" + echo "FEATURES=$FEATURES" >> "$GITHUB_ENV" + echo "LABELS=$LABELS" >> "$GITHUB_ENV" + echo "TRANSCRIPTS=$TRANSCRIPTS" >> "$GITHUB_ENV" + export FEATURES LABELS TRANSCRIPTS + python - <<'PY' + import pandas as pd, os + print('features columns', list(pd.read_csv(os.environ['FEATURES'], nrows=0).columns)) + print('labels columns', list(pd.read_csv(os.environ['LABELS'], nrows=0).columns)) + PY + + - name: Run frozen V97 four-world gate + run: | + set -euo pipefail + cd competitions/trace_the_ace + python v97_support_gate.py \ + --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" \ + --out ../../v97_support_gate.json + + - name: Decide submission promotion + id: gate + shell: bash + run: | + python - <<'PY' + import json, os + r=json.load(open('v97_support_gate.json')) + verdict=r['decision']['verdict'] + print(json.dumps(r['decision'], indent=2)) + with open(os.environ['GITHUB_OUTPUT'],'a') as f: + f.write('promote=' + ('true' if verdict=='BUILD_SUBMISSION_NOW' else 'false') + '\n') + PY + + - name: Train frozen V97 runtime assets + if: steps.gate.outputs.promote == 'true' + run: | + set -euo pipefail + cd competitions/trace_the_ace + python train_v97_runtime_assets.py \ + --features "../../$FEATURES" --labels "../../$LABELS" --transcripts "../../$TRANSCRIPTS" \ + --out-dir ../../v97_assets + + - name: Assemble official runtime + if: steps.gate.outputs.promote == 'true' + shell: bash + run: | + set -euo pipefail + git clone --depth 1 https://github.com/drivendataorg/tutoring-outcomes-runtime.git /tmp/runtime + rm -rf /tmp/runtime/submission_src/* + cp competitions/trace_the_ace/runtime_v97/main.py /tmp/runtime/submission_src/main.py + for F in v71_mastery_events.py v75_canonical_trajectory.py v81_target_segment_phase.py v85_evidence_state.py v89_relative_ability_composition.py v93_shift_robust_validation.py v94_related_control.py; do + cp "competitions/trace_the_ace/$F" "/tmp/runtime/submission_src/$F" + done + mkdir -p /tmp/runtime/submission_src/assets + cp v97_assets/v97_assets.npz v97_assets/manifest.json /tmp/runtime/submission_src/assets/ + find /tmp/runtime/submission_src -maxdepth 2 -type f -printf '%P %s bytes\n' + + - name: Pack and check official submission + if: steps.gate.outputs.promote == 'true' + working-directory: /tmp/runtime + run: | + just pack-submission + just check-submission + just pull + + - name: Test official submission offline + if: steps.gate.outputs.promote == 'true' + working-directory: /tmp/runtime + env: + BLOCK_INTERNET: 'true' + GITHUB_ACTIONS_NO_TTY: 'true' + SUBMISSION_IMAGE: tutoringoutcomeschallengeprodacr.azurecr.io/tutoring-outcomes-runtime:gpu-latest + run: just test-submission + + - name: Validate output contract + if: steps.gate.outputs.promote == 'true' + run: | + python competitions/trace_the_ace/runtime_validate.py output \ + --format /tmp/runtime/data-demo/submission_format.csv \ + --predictions /tmp/runtime/submission/submission.csv + + - name: Upload gate evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v97-gate-evidence + path: v97_support_gate.json + retention-days: 14 + + - name: Upload submission candidate + if: steps.gate.outputs.promote == 'true' + uses: actions/upload-artifact@v4 + with: + name: trace-ace-v97-official-runtime-candidate + path: | + /tmp/runtime/submission/submission.zip + /tmp/runtime/submission/submission.csv + /tmp/runtime/submission/log.txt + v97_assets/manifest.json + v97_support_gate.json + retention-days: 14