Add Trace the Ace mastery-event experiments

.github/workflows/trace-ace-mastery.yml

@@ -0,0 +1,152 @@
+name: Trace the Ace mastery experiment
+
+on:
+  workflow_dispatch:
+    inputs:
+      run_full:
+        description: "Run full experiment using public Drive bundles"
+        required: false
+        default: false
+        type: boolean
+      limit:
+        description: "Optional row limit (0 = all rows)"
+        required: false
+        default: "0"
+        type: string
+  push:
+    branches:
+      - agent/trace-ace-mastery-events
+    paths:
+      - "competitions/trace_the_ace/**"
+      - ".github/workflows/trace-ace-mastery.yml"
+
+jobs:
+  self-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install experiment dependencies
+        run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+      - name: Run mastery extractor self-test
+        run: python competitions/trace_the_ace/v71_mastery_events.py --self-test
+      - name: Run supervision audit self-test
+        run: python competitions/trace_the_ace/v72_supervision_audit.py --self-test
+      - name: Run contrastive mastery self-test
+        run: python competitions/trace_the_ace/v73_contrastive_mastery.py --self-test
+      - name: Run semantic objective prior self-test
+        run: python competitions/trace_the_ace/v74_semantic_objective_prior.py --self-test
+      - name: Run canonical trajectory self-test
+        run: python competitions/trace_the_ace/v75_canonical_trajectory.py --self-test
+      - name: Run unseen validation self-test
+        run: python competitions/trace_the_ace/v76_unseen_validation.py --self-test
+      - name: Run incremental mastery stack self-test
+        run: python competitions/trace_the_ace/v77_incremental_mastery_stack.py --self-test
+      - name: Run official-runtime validation harness self-test
+        run: python competitions/trace_the_ace/runtime_validate.py self-test
+
+  full-experiment:
+    if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run_full) || (github.event_name == 'push' && contains(github.event.head_commit.message, '[run-full]')) }}
+    needs: self-test
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    env:
+      TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI
+      TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Install experiment dependencies
+        run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+      - name: Download public transcript archive
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p /tmp/trace_ace/transcripts
+          python - <<'PY'
+          import os, gdown
+          file_id = os.environ['TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID']
+          out = '/tmp/trace_ace/transcripts_download'
+          path = gdown.download(id=file_id, output=out, quiet=False)
+          if not path:
+              raise SystemExit('Google Drive transcript download failed')
+          PY
+          unzip -q /tmp/trace_ace/transcripts_download -d /tmp/trace_ace/transcripts
+          rm -f /tmp/trace_ace/transcripts_download
+      - name: Download public feature/label bundle
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p /tmp/trace_ace/meta
+          python - <<'PY'
+          import os, gdown
+          file_id = os.environ['TRACE_ACE_METADATA_DRIVE_FILE_ID']
+          out = '/tmp/trace_ace/meta.zip'
+          path = gdown.download(id=file_id, output=out, quiet=False)
+          if not path:
+              raise SystemExit('Google Drive metadata download failed; confirm Anyone with the link can view')
+          PY
+          unzip -q /tmp/trace_ace/meta.zip -d /tmp/trace_ace/meta
+          rm -f /tmp/trace_ace/meta.zip
+      - name: Locate inputs by schema and run experiments
+        shell: bash
+        run: |
+          set -euo pipefail
+          python - <<'PY'
+          import csv, shlex
+          from pathlib import Path
+          roots = [Path('/tmp/trace_ace/meta'), Path('/tmp/trace_ace/transcripts')]
+          features = labels = transcript_dir = None
+          for root in roots:
+              for path in root.rglob('*.csv'):
+                  try:
+                      with path.open('r', encoding='utf-8-sig', errors='ignore', newline='') as f:
+                          header = next(csv.reader(f))
+                  except Exception:
+                      continue
+                  cols = set(header)
+                  if features is None and {'response_id','session_id','learning_objective'}.issubset(cols):
+                      features = path; print('FEATURE HEADER', header)
+                  if labels is None and 'response_id' in cols and ('is_correct' in cols or 'correct' in cols):
+                      labels = path; print('LABEL HEADER', header)
+                  if transcript_dir is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(cols):
+                      transcript_dir = path.parent; print('TRANSCRIPT HEADER', header)
+                  if features and labels and transcript_dir: break
+              if features and labels and transcript_dir: break
+          if not (features and labels and transcript_dir):
+              raise SystemExit('Could not identify all Trace the Ace inputs by schema')
+          with open('/tmp/trace_ace/paths.env','w') as f:
+              f.write('FEATURES=' + shlex.quote(str(features)) + '\n')
+              f.write('LABELS=' + shlex.quote(str(labels)) + '\n')
+              f.write('TRANSCRIPTS=' + shlex.quote(str(transcript_dir)) + '\n')
+          PY
+          source /tmp/trace_ace/paths.env
+          LIMIT="${{ inputs.limit }}"; LIMIT="${LIMIT:-0}"
+          EXTRA=(); if [ "$LIMIT" != "0" ]; then EXTRA+=(--limit "$LIMIT"); fi
+          python competitions/trace_the_ace/v71_mastery_events.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v71_mastery_results.json "${EXTRA[@]}"
+          python competitions/trace_the_ace/v72_supervision_audit.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v72_supervision_audit.json "${EXTRA[@]}"
+          python competitions/trace_the_ace/v73_contrastive_mastery.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v73_contrastive_mastery.json "${EXTRA[@]}"
+          python competitions/trace_the_ace/v74_semantic_objective_prior.py --features "$FEATURES" --labels "$LABELS" --out v74_semantic_objective_prior.json "${EXTRA[@]}"
+          python competitions/trace_the_ace/v75_canonical_trajectory.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v75_canonical_trajectory.json "${EXTRA[@]}"
+          python competitions/trace_the_ace/v76_unseen_validation.py --features "$FEATURES" --labels "$LABELS" --out-protocol v76_validation_protocol.csv --out-summary v76_validation_summary.json
+          python competitions/trace_the_ace/v77_incremental_mastery_stack.py --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" --out v77_incremental_mastery_stack.json "${EXTRA[@]}"
+      - name: Upload aggregate results only
+        uses: actions/upload-artifact@v4
+        with:
+          name: trace-ace-aggregate-results
+          path: |
+            v71_mastery_results.json
+            v72_supervision_audit.json
+            v73_contrastive_mastery.json
+            v74_semantic_objective_prior.json
+            v75_canonical_trajectory.json
+            v76_validation_summary.json
+            v77_incremental_mastery_stack.json
+          retention-days: 14

.github/workflows/trace-ace-runtime.yml

@@ -0,0 +1,139 @@
+name: Trace the Ace V75 official runtime
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - agent/trace-ace-mastery-events
+    paths:
+      - "competitions/trace_the_ace/runtime_v75/**"
+      - "competitions/trace_the_ace/train_v75_runtime_assets.py"
+      - "competitions/trace_the_ace/v71_mastery_events.py"
+      - "competitions/trace_the_ace/v75_canonical_trajectory.py"
+      - "competitions/trace_the_ace/runtime_validate.py"
+      - ".github/workflows/trace-ace-runtime.yml"
+
+jobs:
+  build-and-test:
+    if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[runtime-build]') }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    env:
+      TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID: 1nOjremWhpZ_QKSLvZfGcNkS_C3kMMBUI
+      TRACE_ACE_METADATA_DRIVE_FILE_ID: 1EpqoamY0vFI2qE57R6wdqU5HwuoVk3Zz
+      GITHUB_ACTIONS_NO_TTY: "true"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - uses: astral-sh/setup-uv@v6
+      - uses: extractions/setup-just@v2
+      - name: Install training dependencies
+        run: python -m pip install --disable-pip-version-check numpy pandas scipy scikit-learn gdown
+
+      - name: Download training inputs
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p /tmp/trace_ace/transcripts /tmp/trace_ace/meta
+          python - <<'PY'
+          import os, gdown
+          pairs = [
+              ('TRACE_ACE_TRANSCRIPTS_DRIVE_FILE_ID','/tmp/trace_ace/transcripts.zip'),
+              ('TRACE_ACE_METADATA_DRIVE_FILE_ID','/tmp/trace_ace/meta.zip'),
+          ]
+          for env, out in pairs:
+              p = gdown.download(id=os.environ[env], output=out, quiet=False)
+              if not p:
+                  raise SystemExit(f'download failed: {env}')
+          PY
+          unzip -q /tmp/trace_ace/transcripts.zip -d /tmp/trace_ace/transcripts
+          unzip -q /tmp/trace_ace/meta.zip -d /tmp/trace_ace/meta
+          rm -f /tmp/trace_ace/*.zip
+
+      - name: Resolve schemas
+        shell: bash
+        run: |
+          set -euo pipefail
+          python - <<'PY'
+          import csv, shlex
+          from pathlib import Path
+          roots=[Path('/tmp/trace_ace/meta'),Path('/tmp/trace_ace/transcripts')]
+          features=labels=transcripts=None
+          for root in roots:
+            for p in root.rglob('*.csv'):
+              try:
+                with p.open('r',encoding='utf-8-sig',errors='ignore',newline='') as f: h=next(csv.reader(f))
+              except Exception: continue
+              c=set(h)
+              if features is None and {'response_id','session_id','learning_objective'}.issubset(c): features=p; print('FEATURE HEADER',h)
+              if labels is None and 'response_id' in c and ('is_correct' in c or 'correct' in c): labels=p; print('LABEL HEADER',h)
+              if transcripts is None and {'session_id','utterance_id','role','content','timestamp'}.issubset(c): transcripts=p.parent; print('TRANSCRIPT HEADER',h)
+          if not all((features,labels,transcripts)): raise SystemExit('failed schema resolution')
+          with open('/tmp/trace_ace/paths.env','w') as f:
+            f.write('FEATURES='+shlex.quote(str(features))+'\n')
+            f.write('LABELS='+shlex.quote(str(labels))+'\n')
+            f.write('TRANSCRIPTS='+shlex.quote(str(transcripts))+'\n')
+          PY
+
+      - name: Train promoted V75 all-views assets on all training rows
+        shell: bash
+        run: |
+          set -euo pipefail
+          source /tmp/trace_ace/paths.env
+          python competitions/trace_the_ace/train_v75_runtime_assets.py \
+            --features "$FEATURES" --labels "$LABELS" --transcripts "$TRANSCRIPTS" \
+            --out-dir /tmp/v75_assets
+
+      - name: Assemble official submission_src
+        shell: bash
+        run: |
+          set -euo pipefail
+          git clone --depth 1 https://github.com/drivendataorg/tutoring-outcomes-runtime.git /tmp/runtime
+          rm -rf /tmp/runtime/submission_src/*
+          cp competitions/trace_the_ace/runtime_v75/main.py /tmp/runtime/submission_src/main.py
+          cp competitions/trace_the_ace/v71_mastery_events.py /tmp/runtime/submission_src/v71_mastery_events.py
+          cp competitions/trace_the_ace/v75_canonical_trajectory.py /tmp/runtime/submission_src/v75_canonical_trajectory.py
+          mkdir -p /tmp/runtime/submission_src/assets
+          cp /tmp/v75_assets/v75_runtime_assets.npz /tmp/runtime/submission_src/assets/
+          cp /tmp/v75_assets/manifest.json /tmp/runtime/submission_src/assets/
+          find /tmp/runtime/submission_src -maxdepth 2 -type f -printf '%P %s bytes\n'
+
+      - name: Pack and check with official runtime
+        working-directory: /tmp/runtime
+        run: |
+          just pack-submission
+          just check-submission
+
+      - name: Pull official runtime image
+        working-directory: /tmp/runtime
+        run: just pull
+
+      - name: Test submission offline in official container
+        working-directory: /tmp/runtime
+        env:
+          BLOCK_INTERNET: "true"
+          GITHUB_ACTIONS_NO_TTY: "true"
+          SUBMISSION_IMAGE: tutoringoutcomeschallengeprodacr.azurecr.io/tutoring-outcomes-runtime:gpu-latest
+        run: just test-submission
+
+      - name: Validate generated output contract
+        shell: bash
+        run: |
+          set -euo pipefail
+          python competitions/trace_the_ace/runtime_validate.py output \
+            --format /tmp/runtime/data-demo/submission_format.csv \
+            --predictions /tmp/runtime/submission/submission.csv
+
+      - name: Upload candidate and evidence
+        uses: actions/upload-artifact@v4
+        with:
+          name: trace-ace-v75-official-runtime-candidate
+          retention-days: 14
+          path: |
+            /tmp/runtime/submission/submission.zip
+            /tmp/runtime/submission/submission.csv
+            /tmp/runtime/submission/log.txt
+            /tmp/v75_assets/manifest.json