From 1e6a0fce3e249be3ba59dfb86d0f4a0f0a664826 Mon Sep 17 00:00:00 2001 From: wms2537 <51080539+wms2537@users.noreply.github.com> Date: Wed, 19 Aug 2026 10:58:45 +0800 Subject: [PATCH 01/11] research: initialize JumpBench neural workspace --- experiments/jumpbench/README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 experiments/jumpbench/README.md diff --git a/experiments/jumpbench/README.md b/experiments/jumpbench/README.md new file mode 100644 index 0000000..225877c --- /dev/null +++ b/experiments/jumpbench/README.md @@ -0,0 +1,3 @@ +# JumpBench neural evaluation workspace + +Temporary branch-only workspace for frozen neural experiments on constructive representational acquisition. Results are uploaded by GitHub Actions and are not part of the SciAgent skill release. From 0b6258301bb761b34989a833f7d6be6edcb1e9f7 Mon Sep 17 00:00:00 2001 From: wms2537 <51080539+wms2537@users.noreply.github.com> Date: Wed, 19 Aug 2026 11:13:00 +0800 Subject: [PATCH 02/11] research: add LittleLearner neural calibration probe --- experiments/jumpbench/neural_probe.py | 119 ++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 experiments/jumpbench/neural_probe.py diff --git a/experiments/jumpbench/neural_probe.py b/experiments/jumpbench/neural_probe.py new file mode 100644 index 0000000..e09ebd1 --- /dev/null +++ b/experiments/jumpbench/neural_probe.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import argparse +import json +import os +import platform +import time +from pathlib import Path + +import torch +from huggingface_hub import model_info +from transformers import AutoModelForCausalLM, AutoTokenizer + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--output", required=True) + return parser.parse_args() + + +def generate(model, tokenizer, prompt: str, max_new_tokens: int = 96) -> str: + encoded = tokenizer(prompt, return_tensors="pt") + with torch.inference_mode(): + output = model.generate( + **encoded, + max_new_tokens=max_new_tokens, + do_sample=False, + pad_token_id=tokenizer.eos_token_id, + eos_token_id=tokenizer.eos_token_id, + use_cache=True, + ) + continuation = output[0, encoded["input_ids"].shape[1] :] + return tokenizer.decode(continuation, skip_special_tokens=True) + + +def main() -> None: + args = parse_args() + torch.set_num_threads(min(4, os.cpu_count() or 1)) + started = time.time() + info = model_info(args.model, revision=args.revision) + tokenizer = AutoTokenizer.from_pretrained(args.model, revision=args.revision) + load_start = time.time() + model = AutoModelForCausalLM.from_pretrained( + args.model, + revision=args.revision, + torch_dtype=torch.float32, + low_cpu_mem_usage=True, + ) + model.eval() + load_seconds = time.time() - load_start + + prompts = { + "arithmetic": "Complete the answer with only the result. 17 + 28 =", + "repeat_expression": ( + "A newly discovered binary operator is defined by " + "M(x,y)=x*x+x*y-y. Repeat only the expression after the equals sign.\n" + "M(x,y)=" + ), + "apply_taught_expression": ( + "Use the exact rule f(x,y)=x*x+x*y-y. Compute f(2,-1). " + "Write only the integer answer.\nAnswer:" + ), + "infer_expression": ( + "An unknown polynomial f uses only x, y, integers, +, -, and *. " + "It has these exact values:\n" + "f(0,0)=0\n" + "f(1,0)=1\n" + "f(0,1)=-1\n" + "f(1,1)=1\n" + "f(2,-1)=3\n" + "Write one expression for f(x,y), and nothing else.\n" + "f(x,y)=" + ), + "choose_candidate": ( + "An unknown function has exact values f(0,0)=0, f(1,0)=1, " + "f(0,1)=-1, f(1,1)=1, and f(2,-1)=3. Choose the correct rule.\n" + "A. x*x+x*y-y\n" + "B. x*x-x*y+y\n" + "C. x*y+x-y\n" + "D. x*x+y*y-y\n" + "Write only A, B, C, or D.\nAnswer:" + ), + } + generations = {} + for name, prompt in prompts.items(): + t0 = time.time() + text = generate(model, tokenizer, prompt) + generations[name] = { + "prompt": prompt, + "generation": text, + "seconds": time.time() - t0, + } + print(f"[{name}] {text!r}", flush=True) + + parameter_count = sum(parameter.numel() for parameter in model.parameters()) + payload = { + "model": args.model, + "requested_revision": args.revision, + "resolved_revision": info.sha, + "parameter_count": parameter_count, + "model_type": getattr(model.config, "model_type", None), + "architectures": getattr(model.config, "architectures", None), + "transformers_version": __import__("transformers").__version__, + "torch_version": torch.__version__, + "python": platform.python_version(), + "load_seconds": load_seconds, + "total_seconds": time.time() - started, + "generations": generations, + } + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + print(json.dumps({k: payload[k] for k in ("model", "resolved_revision", "parameter_count", "load_seconds", "total_seconds")}, indent=2)) + + +if __name__ == "__main__": + main() From 5d30c519be692a8050b84eb8b3d03adb362361f8 Mon Sep 17 00:00:00 2001 From: wms2537 <51080539+wms2537@users.noreply.github.com> Date: Wed, 19 Aug 2026 11:13:25 +0800 Subject: [PATCH 03/11] research: run matched 0.6B LittleLearner calibration matrix --- .github/workflows/jumpbench-neural-eval.yml | 68 +++++++++++---------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/.github/workflows/jumpbench-neural-eval.yml b/.github/workflows/jumpbench-neural-eval.yml index ca329b3..0fde723 100644 --- a/.github/workflows/jumpbench-neural-eval.yml +++ b/.github/workflows/jumpbench-neural-eval.yml @@ -1,9 +1,6 @@ name: JumpBench neural evaluation on: - push: - branches: - - research/jumpbench-neural-eval pull_request: branches: - master @@ -13,9 +10,26 @@ permissions: contents: read jobs: - discover-models: + calibrate: + name: calibrate-${{ matrix.label }} runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - label: bounded-base-0p6b + model: littlelearner/littlelearner-0.6b-base + revision: 822ef3a9700a952f74a3923b45829d53bdc76bee + - label: unfiltered-base-0p6b + model: littlelearner/unfiltered-0.6b-base + revision: fb0bb25397dca503e1a9741b2e7e826170dc525d + - label: bounded-grpo-0p6b + model: littlelearner/littlelearner-0.6b-grpo-math-expert + revision: 301a228c83efdf45759e88e697589b9e18ae8222 + - label: unfiltered-grpo-0p6b + model: littlelearner/unfiltered-0.6b-grpo-math-expert + revision: 819453e53d7d8d304c0e986a5a9f2ad6a57243ca steps: - uses: actions/checkout@v4 - name: Environment @@ -23,35 +37,23 @@ jobs: python --version free -h nproc - - name: Discover LittleLearner model repositories + - name: Install CPU inference dependencies + run: | + python -m pip install --upgrade --quiet pip + python -m pip install --quiet --index-url https://download.pytorch.org/whl/cpu torch + python -m pip install --quiet "transformers>=4.52,<5" "accelerate>=1.2" safetensors huggingface_hub + - name: Run frozen calibration probe + env: + HF_HUB_DISABLE_TELEMETRY: "1" + TOKENIZERS_PARALLELISM: "false" run: | - set -euo pipefail - python - <<'PY' - import json, urllib.parse, urllib.request - queries = [ - {"author": "littlelearner", "limit": 100, "full": "true"}, - {"search": "LittleLearner", "limit": 100, "full": "true"}, - {"search": "littlelearner-ll", "limit": 100, "full": "true"}, - {"filter": "arxiv:2608.13545", "limit": 100, "full": "true"}, - ] - found = {} - for params in queries: - url = "https://huggingface.co/api/models?" + urllib.parse.urlencode(params) - print("QUERY", url) - with urllib.request.urlopen(url, timeout=60) as response: - payload = json.load(response) - print("COUNT", len(payload)) - for model in payload: - model_id = model.get("id") or model.get("modelId") - if model_id: - found[model_id] = model - print("MODEL", model_id, "sha=", model.get("sha"), "private=", model.get("private")) - with open("hf_models.json", "w", encoding="utf-8") as handle: - json.dump(found, handle, indent=2) - print("UNIQUE_MODELS", len(found)) - PY + mkdir -p neural-results + python experiments/jumpbench/neural_probe.py \ + --model "${{ matrix.model }}" \ + --revision "${{ matrix.revision }}" \ + --output "neural-results/${{ matrix.label }}.json" - uses: actions/upload-artifact@v4 with: - name: jumpbench-hf-model-discovery - path: hf_models.json + name: jumpbench-calibration-${{ matrix.label }} + path: neural-results/${{ matrix.label }}.json if-no-files-found: error From 82634fd910dc2ca3f2b351fdbe88b26f6f3aee45 Mon Sep 17 00:00:00 2001 From: wms2537 <51080539+wms2537@users.noreply.github.com> Date: Wed, 19 Aug 2026 11:23:49 +0800 Subject: [PATCH 04/11] research: add frozen neural acquisition evaluator --- experiments/jumpbench/neural_eval.py | 127 +++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 experiments/jumpbench/neural_eval.py diff --git a/experiments/jumpbench/neural_eval.py b/experiments/jumpbench/neural_eval.py new file mode 100644 index 0000000..be4c37a --- /dev/null +++ b/experiments/jumpbench/neural_eval.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import argparse, ast, hashlib, json, os, platform, re, time +from pathlib import Path +import numpy as np +import torch +from huggingface_hub import model_info +from transformers import AutoModelForCausalLM, AutoTokenizer + +ZERO=(); ONE=((0,0,1),); X=((1,0,1),); Y=((0,1,1),) +def fd(d): return tuple(sorted((i,j,int(c)) for (i,j),c in d.items() if int(c))) +def dd(p): return {(i,j):c for i,j,c in p} +def add(a,b): + d=dd(a) + for i,j,c in b:d[(i,j)]=d.get((i,j),0)+c + return fd(d) +def neg(a): return tuple((i,j,-c) for i,j,c in a) +def sub(a,b): return add(a,neg(b)) +def mul(a,b): + d={} + for ai,aj,ac in a: + for bi,bj,bc in b:d[(ai+bi,aj+bj)]=d.get((ai+bi,aj+bj),0)+ac*bc + return fd(d) +def power(a,n): + r=ONE + for _ in range(n):r=mul(r,a) + return r +def val(p,x,y): return int(sum(c*x**i*y**j for i,j,c in p)) +def ast_poly(n): + if isinstance(n,ast.Expression):return ast_poly(n.body) + if isinstance(n,ast.Name) and n.id in {'x','y'}:return X if n.id=='x' else Y + if isinstance(n,ast.Constant) and isinstance(n.value,int):return ZERO if n.value==0 else ((0,0,int(n.value)),) + if isinstance(n,ast.UnaryOp) and isinstance(n.op,ast.USub):return neg(ast_poly(n.operand)) + if isinstance(n,ast.UnaryOp) and isinstance(n.op,ast.UAdd):return ast_poly(n.operand) + if isinstance(n,ast.BinOp): + a,b=ast_poly(n.left),ast_poly(n.right) + if isinstance(n.op,ast.Add):return add(a,b) + if isinstance(n.op,ast.Sub):return sub(a,b) + if isinstance(n.op,ast.Mult):return mul(a,b) + if isinstance(n.op,ast.Pow) and isinstance(n.right,ast.Constant) and 0<=int(n.right.value)<=6:return power(a,int(n.right.value)) + raise ValueError(ast.dump(n)) +def parse_expr(text): + s=text.replace('²','**2').replace('^','**').replace('−','-').replace('×','*') + s=re.sub(r'\bxy\b','x*y',s); cs=[] + for m in re.finditer(r'(?:Expression|Answer|f\s*\(\s*x\s*,\s*y\s*\))\s*[:=]\s*([^\n`]+)',s,re.I):cs.append(m.group(1)) + cs+=re.findall(r'```(?:python)?\s*([^`]+)```',s,re.I|re.S);cs += [x for x in s.splitlines() if x.strip()]+[s] + for c in reversed(cs): + c=c.strip().strip('`$ .;,') + if '=' in c:c=c.split('=')[-1].strip() + c=re.sub(r'\b([0-9]+)\s*([xy])\b',r'\1*\2',c).split(' where ')[0].split(' because ')[0].strip() + if not c or len(c)>300:continue + try:return ast_poly(ast.parse(c,mode='eval')) + except Exception:pass + return None +def ev(rows):return '\n'.join(f"f({r['x']},{r['y']})={r['value']}" for r in rows) +DEMO='''Example 1 +f(-1,-1)=-2 +f(0,2)=2 +f(2,1)=3 +Expression: x + y + +Example 2 +f(-1,2)=-4 +f(0,-1)=1 +f(2,3)=3 +Expression: x*y - y +''' +def free_prompt(rows):return 'Infer the exact integer polynomial from observations. Use only x, y, integer constants, +, -, and *. Return one expression.\n\n'+DEMO+'\nTarget\n'+ev(rows)+'\nExpression:' +def rank_prompt(rows):return 'Infer the integer polynomial f from the observations. The next text must be the exact expression for f.\n'+ev(rows)+'\nExpression:' +def teach_prompt(e):return f'The exact rule has been taught: f(x,y) = {e}.\nRepeat an algebraically equivalent rule using x and y.\nExpression:' +def query_prompt(r): + hs='\n'.join(f"H{i+1}: {c['expression']}" for i,c in enumerate(r['query_choice']['candidates'])) + qs='\n'.join(f"{o['label']}: evaluate at ({o['x']},{o['y']})" for o in r['query_choice']['query_options']) + return 'Choose one experiment that separates the candidate polynomial hypotheses as strongly as possible.\n'+hs+'\nQueries:\n'+qs+'\nAnswer:' +def transfer_prompt(e,x,y):return f"A reusable binary operator is M(a,b) = {e.replace('x','a').replace('y','b')}. Compute M(M({x},{y}),{y}). Write only the integer.\nAnswer:" +def generate(m,t,p,n): + z=t(p,return_tensors='pt') + with torch.inference_mode():o=m.generate(**z,max_new_tokens=n,do_sample=False,pad_token_id=t.eos_token_id,eos_token_id=t.eos_token_id,use_cache=True) + return t.decode(o[0,z['input_ids'].shape[1]:],skip_special_tokens=True) +def scores(m,t,p,cs): + pi=t(p,add_special_tokens=True)['input_ids'];seq=[];pls=[] + for c in cs: + ci=t(' '+c,add_special_tokens=False)['input_ids'];seq.append(pi+ci);pls.append(len(pi)) + ml=max(map(len,seq));pad=t.pad_token_id if t.pad_token_id is not None else t.eos_token_id + ids=torch.full((len(seq),ml),pad,dtype=torch.long);att=torch.zeros_like(ids) + for i,s in enumerate(seq):ids[i,:len(s)]=torch.tensor(s);att[i,:len(s)]=1 + with torch.inference_mode():lp=torch.log_softmax(m(input_ids=ids,attention_mask=att).logits[:,:-1,:],dim=-1) + out=[] + for i,s in enumerate(seq): + lab=ids[i,pls[i]:len(s)];pos=torch.arange(pls[i]-1,len(s)-1);v=lp[i,pos,lab] + out.append({'continuation':cs[i],'sum_logprob':float(v.sum()),'mean_logprob':float(v.mean()),'tokens':int(v.numel())}) + return out +def rank(sc,ci):return sorted(range(len(sc)),key=lambda i:(-sc[i]['mean_logprob'],i)).index(ci)+1 +def numeric_options(correct,decoys): + vs=[correct] + for x in decoys: + if x not in vs:vs.append(x) + if len(vs)==8:break + d=1 + while len(vs)<8: + for x in (correct+d,correct-d): + if x not in vs:vs.append(x) + if len(vs)==8:break + d+=1 + return list(map(str,vs)) +def main(): + a=argparse.ArgumentParser();a.add_argument('--model',required=True);a.add_argument('--revision',required=True);a.add_argument('--manifest',required=True);a.add_argument('--output',required=True);a.add_argument('--max-tasks',type=int,default=0);a=a.parse_args() + torch.set_num_threads(min(4,os.cpu_count() or 1));mb=Path(a.manifest).read_bytes();man=json.loads(mb);recs=man['records'][:a.max_tasks or None] + info=model_info(a.model,revision=a.revision);tok=AutoTokenizer.from_pretrained(a.model,revision=a.revision);model=AutoModelForCausalLM.from_pretrained(a.model,revision=a.revision,torch_dtype=torch.float32,low_cpu_mem_usage=True);model.eval();start=time.time();rows=[] + for no,r in enumerate(recs,1): + target=tuple(tuple(map(int,z)) for z in r['target_polynomial']);opts=r['candidate_options'];cont=[o['expression'] for o in opts];ci=next(i for i,o in enumerate(opts) if o['pool_index']==r['target_pool_index']);row={'task_id':r['task_id'],'definition_cost':r['definition_cost'],'target_pool_index':r['target_pool_index']} + for cond in ('active','random','passive'): + sc=scores(model,tok,rank_prompt(r['evidence'][cond]),cont);rk=rank(sc,ci);row[f'recognition_{cond}_rank']=rk;row[f'recognition_{cond}_top1']=int(rk==1);row[f'recognition_{cond}_scores']=sc + sc=scores(model,tok,teach_prompt(r['target_expression']),cont);rk=rank(sc,ci);row['teach_rank']=rk;row['teach_top1']=int(rk==1);row['teach_scores']=sc + for cond in ('active','passive'): + g=generate(model,tok,free_prompt(r['evidence'][cond]),64);p=parse_expr(g);row[f'free_{cond}_generation']=g;row[f'free_{cond}_parseable']=int(p is not None);row[f'free_{cond}_exact']=int(p==target);row[f'free_{cond}_parsed']=None if p is None else [list(z) for z in p] + g=generate(model,tok,teach_prompt(r['target_expression']),64);p=parse_expr(g);row['teach_echo_generation']=g;row['teach_echo_parseable']=int(p is not None);row['teach_echo_exact']=int(p==target) + labels=[o['label'] for o in r['query_choice']['query_options']];sc=scores(model,tok,query_prompt(r),labels);sel=max(range(len(sc)),key=lambda i:sc[i]['mean_logprob']);row['query_selected_label']=labels[sel];row['query_correct']=int(labels[sel] in r['query_choice']['correct_labels']);row['query_scores']=sc + x,y=1,2;inner=val(target,x,y);correct=val(target,inner,y);ds=[] + for o in opts: + p=parse_expr(o['expression']) + if p is not None:q=val(p,x,y);ds.append(val(p,q,y)) + nos=numeric_options(correct,ds);sc=scores(model,tok,transfer_prompt(r['target_expression'],x,y),nos);rk=rank(sc,nos.index(str(correct)));row['transfer_rank']=rk;row['transfer_top1']=int(rk==1);row['transfer_correct_value']=correct;row['transfer_options']=nos;row['transfer_scores']=sc;rows.append(row);print(a.model,no,'/',len(recs),r['task_id'],flush=True) + metrics=['recognition_active_top1','recognition_random_top1','recognition_passive_top1','teach_top1','free_active_parseable','free_active_exact','free_passive_parseable','free_passive_exact','teach_echo_parseable','teach_echo_exact','query_correct','transfer_top1'];summary={k:float(np.mean([r[k] for r in rows])) for k in metrics} + out={'benchmark':'JumpBench neural acquisition v0.1','model':a.model,'requested_revision':a.revision,'resolved_revision':info.sha,'parameter_count':sum(p.numel() for p in model.parameters()),'manifest_file_sha256':hashlib.sha256(mb).hexdigest(),'manifest_semantic_sha256':man.get('sha256_without_sha_field'),'n_tasks':len(rows),'summary':summary,'rows':rows,'runtime_seconds':time.time()-start,'environment':{'python':platform.python_version(),'torch':torch.__version__,'transformers':__import__('transformers').__version__}} + Path(a.output).parent.mkdir(parents=True,exist_ok=True);Path(a.output).write_text(json.dumps(out,indent=2)+'\n');print(json.dumps({'model':a.model,'summary':summary,'runtime_seconds':out['runtime_seconds']},indent=2)) +if __name__=='__main__':main() From b0784d5a2d7e014d30fb2c9382370c023f3740c8 Mon Sep 17 00:00:00 2001 From: wms2537 <51080539+wms2537@users.noreply.github.com> Date: Wed, 19 Aug 2026 11:24:52 +0800 Subject: [PATCH 05/11] research: add neural evaluator development manifest --- experiments/jumpbench/neural_dev_manifest.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 experiments/jumpbench/neural_dev_manifest.json diff --git a/experiments/jumpbench/neural_dev_manifest.json b/experiments/jumpbench/neural_dev_manifest.json new file mode 100644 index 0000000..857a9e8 --- /dev/null +++ b/experiments/jumpbench/neural_dev_manifest.json @@ -0,0 +1 @@ +{"benchmark":"JumpBench neural acquisition development pilot","seed":26081970,"candidate_grammar_size":2222,"target_cost_tiers":[9,11],"n_targets":4,"evidence_budget":6,"excluded_target_count":295,"records":[{"task_id":"N0000","target_pool_index":296,"definition_cost":9,"target_expression":"x*y + y*y + x + 2*y","target_polynomial":[[0,1,2],[0,2,1],[1,0,1],[1,1,1]],"evidence":{"active":[{"x":3,"y":-3,"value":-3},{"x":-2,"y":3,"value":7},{"x":3,"y":2,"value":17},{"x":3,"y":3,"value":27},{"x":3,"y":1,"value":9},{"x":3,"y":0,"value":3}],"random":[{"x":-1,"y":-1,"value":-1},{"x":-3,"y":2,"value":-1},{"x":2,"y":0,"value":2},{"x":0,"y":0,"value":0},{"x":2,"y":1,"value":7},{"x":2,"y":-3,"value":-1}],"passive":[{"x":-1,"y":-1,"value":-1},{"x":1,"y":0,"value":1},{"x":0,"y":0,"value":0},{"x":1,"y":-1,"value":-1},{"x":0,"y":1,"value":3},{"x":-1,"y":0,"value":-1}]},"candidate_options":[{"label":"A","pool_index":296,"expression":"x*y + y*y + x + 2*y"},{"label":"B","pool_index":1369,"expression":"x*y + 2*y*y + x - y"},{"label":"C","pool_index":655,"expression":"2*x*y + y*y + x - y"},{"label":"D","pool_index":579,"expression":"2*x*y + 2*y*y - 2*x - y"},{"label":"E","pool_index":1800,"expression":"x*x + x*y + 3*y"},{"label":"F","pool_index":62,"expression":"x*x + x*y + y"},{"label":"G","pool_index":1699,"expression":"x*x + 2*x*y + y*y - 2*x - y"},{"label":"H","pool_index":109,"expression":"2*x*y + 2*y*y - x"}],"correct_candidate_label":"A","query_choice":{"query_options":[{"label":"A","x":-2,"y":3},{"label":"B","x":-3,"y":2},{"label":"C","x":0,"y":3},{"label":"D","x":3,"y":0},{"label":"E","x":2,"y":1},{"label":"F","x":-1,"y":-1},{"label":"G","x":2,"y":-3},{"label":"H","x":1,"y":-2}],"correct_labels":["G"],"candidates":[{"pool_index":109,"expression":"2*x*y + 2*y*y - x"},{"pool_index":296,"expression":"x*y + y*y + x + 2*y"},{"pool_index":655,"expression":"2*x*y + y*y + x - y"},{"pool_index":1800,"expression":"x*x + x*y + 3*y"},{"pool_index":62,"expression":"x*x + x*y + y"},{"pool_index":1699,"expression":"x*x + 2*x*y + y*y - 2*x - y"},{"pool_index":579,"expression":"2*x*y + 2*y*y - 2*x - y"},{"pool_index":1369,"expression":"x*y + 2*y*y + x - y"}]}},{"task_id":"N0001","target_pool_index":1296,"definition_cost":11,"target_expression":"4*x*x + x - y","target_polynomial":[[0,1,-1],[1,0,1],[2,0,4]],"evidence":{"active":[{"x":3,"y":-3,"value":42},{"x":3,"y":0,"value":39},{"x":3,"y":3,"value":36},{"x":3,"y":2,"value":37},{"x":3,"y":1,"value":38},{"x":3,"y":-1,"value":40}],"random":[{"x":1,"y":1,"value":4},{"x":3,"y":-1,"value":40},{"x":-2,"y":3,"value":11},{"x":1,"y":2,"value":3},{"x":0,"y":2,"value":-2},{"x":-2,"y":1,"value":13}],"passive":[{"x":1,"y":-1,"value":6},{"x":0,"y":1,"value":-1},{"x":-1,"y":1,"value":2},{"x":1,"y":0,"value":5},{"x":-1,"y":0,"value":3},{"x":1,"y":1,"value":4}]},"candidate_options":[{"label":"A","pool_index":1312,"expression":"4*x*x - x + y"},{"label":"B","pool_index":1206,"expression":"3*x*x - x*y + 3*x - y"},{"label":"C","pool_index":1323,"expression":"4*y*y + x - y"},{"label":"D","pool_index":1296,"expression":"4*x*x + x - y"},{"label":"E","pool_index":1298,"expression":"4*x*x + y + 1"},{"label":"F","pool_index":1299,"expression":"4*x*x + y - 1"},{"label":"G","pool_index":1188,"expression":"3*x*x + y*y"},{"label":"H","pool_index":1295,"expression":"4*x*x + x + y"}],"correct_candidate_label":"D","query_choice":{"query_options":[{"label":"A","x":2,"y":-3},{"label":"B","x":1,"y":3},{"label":"C","x":0,"y":-1},{"label":"D","x":-3,"y":2},{"label":"E","x":1,"y":-2},{"label":"F","x":3,"y":-1},{"label":"G","x":-2,"y":-3},{"label":"H","x":-1,"y":-2}],"correct_labels":["G"],"candidates":[{"pool_index":1296,"expression":"4*x*x + x - y"},{"pool_index":1188,"expression":"3*x*x + y*y"},{"pool_index":1295,"expression":"4*x*x + x + y"},{"pool_index":1312,"expression":"4*x*x - x + y"},{"pool_index":1299,"expression":"4*x*x + y - 1"},{"pool_index":1298,"expression":"4*x*x + y + 1"},{"pool_index":1206,"expression":"3*x*x - x*y + 3*x - y"},{"pool_index":1323,"expression":"4*y*y + x - y"}]}},{"task_id":"N0002","target_pool_index":465,"definition_cost":9,"target_expression":"x*x - 2*x*y + y*y - 1","target_polynomial":[[0,0,-1],[0,2,1],[1,1,-2],[2,0,1]],"evidence":{"active":[{"x":3,"y":-3,"value":35},{"x":-2,"y":3,"value":24},{"x":3,"y":3,"value":-1},{"x":3,"y":2,"value":0},{"x":3,"y":1,"value":3},{"x":3,"y":0,"value":8}],"random":[{"x":1,"y":-2,"value":8},{"x":1,"y":-1,"value":3},{"x":2,"y":-3,"value":24},{"x":0,"y":-3,"value":8},{"x":1,"y":2,"value":0},{"x":0,"y":0,"value":-1}],"passive":[{"x":-1,"y":0,"value":0},{"x":1,"y":-1,"value":3},{"x":1,"y":1,"value":-1},{"x":-1,"y":1,"value":3},{"x":-1,"y":-1,"value":-1},{"x":0,"y":0,"value":-1}]},"candidate_options":[{"label":"A","pool_index":944,"expression":"2*x*x - 2*x*y - x + y - 1"},{"label":"B","pool_index":1924,"expression":"x*x - 2*x*y + y - 1"},{"label":"C","pool_index":465,"expression":"x*x - 2*x*y + y*y - 1"},{"label":"D","pool_index":947,"expression":"2*x*x - 2*x*y - x - y - 1"},{"label":"E","pool_index":199,"expression":"2*x*x - 2*x*y - 1"},{"label":"F","pool_index":1965,"expression":"x*x - 2*x*y - x + 3*y"},{"label":"G","pool_index":1914,"expression":"x*x - 2*x*y + 2*y - 1"},{"label":"H","pool_index":2090,"expression":"x*x - x*y - 3*x + 2*y + 2"}],"correct_candidate_label":"C","query_choice":{"query_options":[{"label":"A","x":-1,"y":1},{"label":"B","x":1,"y":-2},{"label":"C","x":2,"y":-2},{"label":"D","x":2,"y":0},{"label":"E","x":-3,"y":-2},{"label":"F","x":2,"y":3},{"label":"G","x":1,"y":3},{"label":"H","x":3,"y":-1}],"correct_labels":["H"],"candidates":[{"pool_index":1924,"expression":"x*x - 2*x*y + y - 1"},{"pool_index":944,"expression":"2*x*x - 2*x*y - x + y - 1"},{"pool_index":199,"expression":"2*x*x - 2*x*y - 1"},{"pool_index":2090,"expression":"x*x - x*y - 3*x + 2*y + 2"},{"pool_index":465,"expression":"x*x - 2*x*y + y*y - 1"},{"pool_index":1965,"expression":"x*x - 2*x*y - x + 3*y"},{"pool_index":1914,"expression":"x*x - 2*x*y + 2*y - 1"},{"pool_index":947,"expression":"2*x*x - 2*x*y - x - y - 1"}]}},{"task_id":"N0003","target_pool_index":793,"definition_cost":11,"target_expression":"2*x*x + 2*x - y - 1","target_polynomial":[[0,0,-1],[0,1,-1],[1,0,2],[2,0,2]],"evidence":{"active":[{"x":3,"y":-3,"value":26},{"x":2,"y":3,"value":8},{"x":3,"y":3,"value":20},{"x":3,"y":2,"value":21},{"x":3,"y":1,"value":22},{"x":3,"y":0,"value":23}],"random":[{"x":1,"y":-3,"value":6},{"x":3,"y":-1,"value":24},{"x":1,"y":0,"value":3},{"x":-2,"y":0,"value":3},{"x":2,"y":-1,"value":12},{"x":-3,"y":3,"value":8}],"passive":[{"x":0,"y":1,"value":-2},{"x":1,"y":0,"value":3},{"x":0,"y":0,"value":-1},{"x":-1,"y":0,"value":-1},{"x":0,"y":-1,"value":0},{"x":1,"y":-1,"value":4}]},"candidate_options":[{"label":"A","pool_index":1742,"expression":"x*x + 3*x - y + 1"},{"label":"B","pool_index":992,"expression":"2*x*x - x*y + 2*y"},{"label":"C","pool_index":860,"expression":"2*x*x + x - y + 1"},{"label":"D","pool_index":793,"expression":"2*x*x + 2*x - y - 1"},{"label":"E","pool_index":892,"expression":"2*x*x + x*y - x - y - 1"},{"label":"F","pool_index":179,"expression":"2*x*x + x + y"},{"label":"G","pool_index":911,"expression":"2*x*x + y*y - 1"},{"label":"H","pool_index":1214,"expression":"3*x*x - y - 1"}],"correct_candidate_label":"D","query_choice":{"query_options":[{"label":"A","x":-1,"y":1},{"label":"B","x":3,"y":-1},{"label":"C","x":3,"y":0},{"label":"D","x":1,"y":-3},{"label":"E","x":-1,"y":2},{"label":"F","x":-3,"y":0},{"label":"G","x":-3,"y":1},{"label":"H","x":-1,"y":3}],"correct_labels":["C","F"],"candidates":[{"pool_index":860,"expression":"2*x*x + x - y + 1"},{"pool_index":179,"expression":"2*x*x + x + y"},{"pool_index":1742,"expression":"x*x + 3*x - y + 1"},{"pool_index":793,"expression":"2*x*x + 2*x - y - 1"},{"pool_index":1214,"expression":"3*x*x - y - 1"},{"pool_index":892,"expression":"2*x*x + x*y - x - y - 1"},{"pool_index":911,"expression":"2*x*x + y*y - 1"},{"pool_index":992,"expression":"2*x*x - x*y + 2*y"}]}}],"sha256_without_sha_field":"cba344bd066f9555b3bcf62f92a490de394c124aa2797a8b7c0c4d9b6569501e"} \ No newline at end of file From efa567aa5b14f538308a666ee6643477b569fa40 Mon Sep 17 00:00:00 2001 From: wms2537 <51080539+wms2537@users.noreply.github.com> Date: Wed, 19 Aug 2026 11:26:29 +0800 Subject: [PATCH 06/11] research: run four-model JumpBench neural development probe --- .github/workflows/jumpbench-neural-eval.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/jumpbench-neural-eval.yml b/.github/workflows/jumpbench-neural-eval.yml index 0fde723..8d8a9d4 100644 --- a/.github/workflows/jumpbench-neural-eval.yml +++ b/.github/workflows/jumpbench-neural-eval.yml @@ -10,10 +10,10 @@ permissions: contents: read jobs: - calibrate: - name: calibrate-${{ matrix.label }} + neural-dev: + name: neural-dev-${{ matrix.label }} runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 120 strategy: fail-fast: false matrix: @@ -37,23 +37,25 @@ jobs: python --version free -h nproc + sha256sum experiments/jumpbench/neural_eval.py experiments/jumpbench/neural_dev_manifest.json - name: Install CPU inference dependencies run: | python -m pip install --upgrade --quiet pip python -m pip install --quiet --index-url https://download.pytorch.org/whl/cpu torch - python -m pip install --quiet "transformers>=4.52,<5" "accelerate>=1.2" safetensors huggingface_hub - - name: Run frozen calibration probe + python -m pip install --quiet "transformers>=4.52,<5" "accelerate>=1.2" safetensors huggingface_hub numpy + - name: Run development-only neural acquisition probe env: HF_HUB_DISABLE_TELEMETRY: "1" TOKENIZERS_PARALLELISM: "false" run: | mkdir -p neural-results - python experiments/jumpbench/neural_probe.py \ + python experiments/jumpbench/neural_eval.py \ --model "${{ matrix.model }}" \ --revision "${{ matrix.revision }}" \ + --manifest experiments/jumpbench/neural_dev_manifest.json \ --output "neural-results/${{ matrix.label }}.json" - uses: actions/upload-artifact@v4 with: - name: jumpbench-calibration-${{ matrix.label }} + name: jumpbench-neural-dev-${{ matrix.label }} path: neural-results/${{ matrix.label }}.json if-no-files-found: error From 3380d5954a0fdb200b376b6664752fc1f6a5c308 Mon Sep 17 00:00:00 2001 From: wms2537 <51080539+wms2537@users.noreply.github.com> Date: Wed, 19 Aug 2026 11:37:54 +0800 Subject: [PATCH 07/11] research: calibrate forced-choice scoring and freeze parser semantics --- experiments/jumpbench/neural_eval.py | 554 +++++++++++++++++++++------ 1 file changed, 445 insertions(+), 109 deletions(-) diff --git a/experiments/jumpbench/neural_eval.py b/experiments/jumpbench/neural_eval.py index be4c37a..6466ff5 100644 --- a/experiments/jumpbench/neural_eval.py +++ b/experiments/jumpbench/neural_eval.py @@ -1,59 +1,137 @@ from __future__ import annotations -import argparse, ast, hashlib, json, os, platform, re, time +import argparse +import ast +import hashlib +import json +import os +import platform +import re +import time from pathlib import Path +from typing import Iterable + import numpy as np import torch from huggingface_hub import model_info from transformers import AutoModelForCausalLM, AutoTokenizer -ZERO=(); ONE=((0,0,1),); X=((1,0,1),); Y=((0,1,1),) -def fd(d): return tuple(sorted((i,j,int(c)) for (i,j),c in d.items() if int(c))) -def dd(p): return {(i,j):c for i,j,c in p} -def add(a,b): - d=dd(a) - for i,j,c in b:d[(i,j)]=d.get((i,j),0)+c - return fd(d) -def neg(a): return tuple((i,j,-c) for i,j,c in a) -def sub(a,b): return add(a,neg(b)) -def mul(a,b): - d={} - for ai,aj,ac in a: - for bi,bj,bc in b:d[(ai+bi,aj+bj)]=d.get((ai+bi,aj+bj),0)+ac*bc - return fd(d) -def power(a,n): - r=ONE - for _ in range(n):r=mul(r,a) - return r -def val(p,x,y): return int(sum(c*x**i*y**j for i,j,c in p)) -def ast_poly(n): - if isinstance(n,ast.Expression):return ast_poly(n.body) - if isinstance(n,ast.Name) and n.id in {'x','y'}:return X if n.id=='x' else Y - if isinstance(n,ast.Constant) and isinstance(n.value,int):return ZERO if n.value==0 else ((0,0,int(n.value)),) - if isinstance(n,ast.UnaryOp) and isinstance(n.op,ast.USub):return neg(ast_poly(n.operand)) - if isinstance(n,ast.UnaryOp) and isinstance(n.op,ast.UAdd):return ast_poly(n.operand) - if isinstance(n,ast.BinOp): - a,b=ast_poly(n.left),ast_poly(n.right) - if isinstance(n.op,ast.Add):return add(a,b) - if isinstance(n.op,ast.Sub):return sub(a,b) - if isinstance(n.op,ast.Mult):return mul(a,b) - if isinstance(n.op,ast.Pow) and isinstance(n.right,ast.Constant) and 0<=int(n.right.value)<=6:return power(a,int(n.right.value)) - raise ValueError(ast.dump(n)) -def parse_expr(text): - s=text.replace('²','**2').replace('^','**').replace('−','-').replace('×','*') - s=re.sub(r'\bxy\b','x*y',s); cs=[] - for m in re.finditer(r'(?:Expression|Answer|f\s*\(\s*x\s*,\s*y\s*\))\s*[:=]\s*([^\n`]+)',s,re.I):cs.append(m.group(1)) - cs+=re.findall(r'```(?:python)?\s*([^`]+)```',s,re.I|re.S);cs += [x for x in s.splitlines() if x.strip()]+[s] - for c in reversed(cs): - c=c.strip().strip('`$ .;,') - if '=' in c:c=c.split('=')[-1].strip() - c=re.sub(r'\b([0-9]+)\s*([xy])\b',r'\1*\2',c).split(' where ')[0].split(' because ')[0].strip() - if not c or len(c)>300:continue - try:return ast_poly(ast.parse(c,mode='eval')) - except Exception:pass - return None -def ev(rows):return '\n'.join(f"f({r['x']},{r['y']})={r['value']}" for r in rows) -DEMO='''Example 1 +Poly = tuple[tuple[int, int, int], ...] +ZERO: Poly = () +ONE: Poly = ((0, 0, 1),) +X: Poly = ((1, 0, 1),) +Y: Poly = ((0, 1, 1),) +EVALUATOR_VERSION = "jumpbench-neural-v0.2-calibrated" + + +def from_dict(values: dict[tuple[int, int], int]) -> Poly: + return tuple(sorted((i, j, int(c)) for (i, j), c in values.items() if int(c))) + + +def as_dict(poly: Poly) -> dict[tuple[int, int], int]: + return {(i, j): c for i, j, c in poly} + + +def add(a: Poly, b: Poly) -> Poly: + out = as_dict(a) + for i, j, c in b: + out[(i, j)] = out.get((i, j), 0) + c + return from_dict(out) + + +def neg(a: Poly) -> Poly: + return tuple((i, j, -c) for i, j, c in a) + + +def sub(a: Poly, b: Poly) -> Poly: + return add(a, neg(b)) + + +def mul(a: Poly, b: Poly) -> Poly: + out: dict[tuple[int, int], int] = {} + for ai, aj, ac in a: + for bi, bj, bc in b: + key = (ai + bi, aj + bj) + out[key] = out.get(key, 0) + ac * bc + return from_dict(out) + + +def power(a: Poly, exponent: int) -> Poly: + result = ONE + for _ in range(exponent): + result = mul(result, a) + return result + + +def value(poly: Poly, x: int, y: int) -> int: + return int(sum(c * x**i * y**j for i, j, c in poly)) + + +def ast_to_poly(node: ast.AST) -> Poly: + if isinstance(node, ast.Expression): + return ast_to_poly(node.body) + if isinstance(node, ast.Name) and node.id in {"x", "y"}: + return X if node.id == "x" else Y + if isinstance(node, ast.Constant) and isinstance(node.value, int): + return ZERO if node.value == 0 else ((0, 0, int(node.value)),) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): + return neg(ast_to_poly(node.operand)) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.UAdd): + return ast_to_poly(node.operand) + if isinstance(node, ast.BinOp): + left = ast_to_poly(node.left) + right = ast_to_poly(node.right) + if isinstance(node.op, ast.Add): + return add(left, right) + if isinstance(node.op, ast.Sub): + return sub(left, right) + if isinstance(node.op, ast.Mult): + return mul(left, right) + if isinstance(node.op, ast.Pow) and isinstance(node.right, ast.Constant): + exponent = int(node.right.value) + if 0 <= exponent <= 6: + return power(left, exponent) + raise ValueError(ast.dump(node)) + + +def parse_expression(text: str) -> Poly | None: + """Parse the first submitted expression; later self-contradictions do not overwrite it.""" + normalized = text.replace("²", "**2").replace("^", "**").replace("−", "-").replace("×", "*") + normalized = re.sub(r"\bxy\b", "x*y", normalized) + candidates: list[str] = [] + for match in re.finditer( + r"(?:Expression|Answer|f\s*\(\s*x\s*,\s*y\s*\))\s*[:=]\s*([^\n`]+)", + normalized, + re.I, + ): + candidates.append(match.group(1)) + candidates.extend(re.findall(r"```(?:python)?\s*([^`]+)```", normalized, re.I | re.S)) + candidates.extend(line for line in normalized.splitlines() if line.strip()) + candidates.append(normalized) + seen: set[str] = set() + for candidate in candidates: + candidate = candidate.strip().strip("`$ .;,") + if candidate in seen: + continue + seen.add(candidate) + if "=" in candidate: + candidate = candidate.split("=", 1)[-1].strip() + candidate = re.sub(r"\b([0-9]+)\s*([xy])\b", r"\1*\2", candidate) + candidate = candidate.split(" where ")[0].split(" because ")[0].strip() + if not candidate or len(candidate) > 300: + continue + try: + return ast_to_poly(ast.parse(candidate, mode="eval")) + except Exception: + continue + return None + + +def evidence_text(rows: list[dict]) -> str: + return "\n".join(f"f({row['x']},{row['y']})={row['value']}" for row in rows) + + +DEMONSTRATIONS = """Example 1 f(-1,-1)=-2 f(0,2)=2 f(2,1)=3 @@ -64,64 +142,322 @@ def ev(rows):return '\n'.join(f"f({r['x']},{r['y']})={r['value']}" for r in rows f(0,-1)=1 f(2,3)=3 Expression: x*y - y -''' -def free_prompt(rows):return 'Infer the exact integer polynomial from observations. Use only x, y, integer constants, +, -, and *. Return one expression.\n\n'+DEMO+'\nTarget\n'+ev(rows)+'\nExpression:' -def rank_prompt(rows):return 'Infer the integer polynomial f from the observations. The next text must be the exact expression for f.\n'+ev(rows)+'\nExpression:' -def teach_prompt(e):return f'The exact rule has been taught: f(x,y) = {e}.\nRepeat an algebraically equivalent rule using x and y.\nExpression:' -def query_prompt(r): - hs='\n'.join(f"H{i+1}: {c['expression']}" for i,c in enumerate(r['query_choice']['candidates'])) - qs='\n'.join(f"{o['label']}: evaluate at ({o['x']},{o['y']})" for o in r['query_choice']['query_options']) - return 'Choose one experiment that separates the candidate polynomial hypotheses as strongly as possible.\n'+hs+'\nQueries:\n'+qs+'\nAnswer:' -def transfer_prompt(e,x,y):return f"A reusable binary operator is M(a,b) = {e.replace('x','a').replace('y','b')}. Compute M(M({x},{y}),{y}). Write only the integer.\nAnswer:" -def generate(m,t,p,n): - z=t(p,return_tensors='pt') - with torch.inference_mode():o=m.generate(**z,max_new_tokens=n,do_sample=False,pad_token_id=t.eos_token_id,eos_token_id=t.eos_token_id,use_cache=True) - return t.decode(o[0,z['input_ids'].shape[1]:],skip_special_tokens=True) -def scores(m,t,p,cs): - pi=t(p,add_special_tokens=True)['input_ids'];seq=[];pls=[] - for c in cs: - ci=t(' '+c,add_special_tokens=False)['input_ids'];seq.append(pi+ci);pls.append(len(pi)) - ml=max(map(len,seq));pad=t.pad_token_id if t.pad_token_id is not None else t.eos_token_id - ids=torch.full((len(seq),ml),pad,dtype=torch.long);att=torch.zeros_like(ids) - for i,s in enumerate(seq):ids[i,:len(s)]=torch.tensor(s);att[i,:len(s)]=1 - with torch.inference_mode():lp=torch.log_softmax(m(input_ids=ids,attention_mask=att).logits[:,:-1,:],dim=-1) - out=[] - for i,s in enumerate(seq): - lab=ids[i,pls[i]:len(s)];pos=torch.arange(pls[i]-1,len(s)-1);v=lp[i,pos,lab] - out.append({'continuation':cs[i],'sum_logprob':float(v.sum()),'mean_logprob':float(v.mean()),'tokens':int(v.numel())}) - return out -def rank(sc,ci):return sorted(range(len(sc)),key=lambda i:(-sc[i]['mean_logprob'],i)).index(ci)+1 -def numeric_options(correct,decoys): - vs=[correct] - for x in decoys: - if x not in vs:vs.append(x) - if len(vs)==8:break - d=1 - while len(vs)<8: - for x in (correct+d,correct-d): - if x not in vs:vs.append(x) - if len(vs)==8:break - d+=1 - return list(map(str,vs)) -def main(): - a=argparse.ArgumentParser();a.add_argument('--model',required=True);a.add_argument('--revision',required=True);a.add_argument('--manifest',required=True);a.add_argument('--output',required=True);a.add_argument('--max-tasks',type=int,default=0);a=a.parse_args() - torch.set_num_threads(min(4,os.cpu_count() or 1));mb=Path(a.manifest).read_bytes();man=json.loads(mb);recs=man['records'][:a.max_tasks or None] - info=model_info(a.model,revision=a.revision);tok=AutoTokenizer.from_pretrained(a.model,revision=a.revision);model=AutoModelForCausalLM.from_pretrained(a.model,revision=a.revision,torch_dtype=torch.float32,low_cpu_mem_usage=True);model.eval();start=time.time();rows=[] - for no,r in enumerate(recs,1): - target=tuple(tuple(map(int,z)) for z in r['target_polynomial']);opts=r['candidate_options'];cont=[o['expression'] for o in opts];ci=next(i for i,o in enumerate(opts) if o['pool_index']==r['target_pool_index']);row={'task_id':r['task_id'],'definition_cost':r['definition_cost'],'target_pool_index':r['target_pool_index']} - for cond in ('active','random','passive'): - sc=scores(model,tok,rank_prompt(r['evidence'][cond]),cont);rk=rank(sc,ci);row[f'recognition_{cond}_rank']=rk;row[f'recognition_{cond}_top1']=int(rk==1);row[f'recognition_{cond}_scores']=sc - sc=scores(model,tok,teach_prompt(r['target_expression']),cont);rk=rank(sc,ci);row['teach_rank']=rk;row['teach_top1']=int(rk==1);row['teach_scores']=sc - for cond in ('active','passive'): - g=generate(model,tok,free_prompt(r['evidence'][cond]),64);p=parse_expr(g);row[f'free_{cond}_generation']=g;row[f'free_{cond}_parseable']=int(p is not None);row[f'free_{cond}_exact']=int(p==target);row[f'free_{cond}_parsed']=None if p is None else [list(z) for z in p] - g=generate(model,tok,teach_prompt(r['target_expression']),64);p=parse_expr(g);row['teach_echo_generation']=g;row['teach_echo_parseable']=int(p is not None);row['teach_echo_exact']=int(p==target) - labels=[o['label'] for o in r['query_choice']['query_options']];sc=scores(model,tok,query_prompt(r),labels);sel=max(range(len(sc)),key=lambda i:sc[i]['mean_logprob']);row['query_selected_label']=labels[sel];row['query_correct']=int(labels[sel] in r['query_choice']['correct_labels']);row['query_scores']=sc - x,y=1,2;inner=val(target,x,y);correct=val(target,inner,y);ds=[] - for o in opts: - p=parse_expr(o['expression']) - if p is not None:q=val(p,x,y);ds.append(val(p,q,y)) - nos=numeric_options(correct,ds);sc=scores(model,tok,transfer_prompt(r['target_expression'],x,y),nos);rk=rank(sc,nos.index(str(correct)));row['transfer_rank']=rk;row['transfer_top1']=int(rk==1);row['transfer_correct_value']=correct;row['transfer_options']=nos;row['transfer_scores']=sc;rows.append(row);print(a.model,no,'/',len(recs),r['task_id'],flush=True) - metrics=['recognition_active_top1','recognition_random_top1','recognition_passive_top1','teach_top1','free_active_parseable','free_active_exact','free_passive_parseable','free_passive_exact','teach_echo_parseable','teach_echo_exact','query_correct','transfer_top1'];summary={k:float(np.mean([r[k] for r in rows])) for k in metrics} - out={'benchmark':'JumpBench neural acquisition v0.1','model':a.model,'requested_revision':a.revision,'resolved_revision':info.sha,'parameter_count':sum(p.numel() for p in model.parameters()),'manifest_file_sha256':hashlib.sha256(mb).hexdigest(),'manifest_semantic_sha256':man.get('sha256_without_sha_field'),'n_tasks':len(rows),'summary':summary,'rows':rows,'runtime_seconds':time.time()-start,'environment':{'python':platform.python_version(),'torch':torch.__version__,'transformers':__import__('transformers').__version__}} - Path(a.output).parent.mkdir(parents=True,exist_ok=True);Path(a.output).write_text(json.dumps(out,indent=2)+'\n');print(json.dumps({'model':a.model,'summary':summary,'runtime_seconds':out['runtime_seconds']},indent=2)) -if __name__=='__main__':main() +""" + + +def discovery_prompt(rows: list[dict]) -> str: + return ( + "Infer the exact integer polynomial from observations. Use only x, y, integer constants, +, -, and *. " + "Write only one algebraically equivalent expression on the first line.\n\n" + + DEMONSTRATIONS + + "\nTarget\n" + + evidence_text(rows) + + "\nExpression:" + ) + + +def ranking_prompt(rows: list[dict]) -> str: + return ( + "Infer the exact integer polynomial f from the observations. " + "The continuation is one candidate expression.\n" + + evidence_text(rows) + + "\nExpression:" + ) + + +def ranking_null_prompt() -> str: + return "Infer the exact integer polynomial f. The continuation is one candidate expression.\nExpression:" + + +def teach_prompt(expression: str) -> str: + return ( + f"The exact rule has been taught: f(x,y) = {expression}.\n" + "Write only one algebraically equivalent expression on the first line.\nExpression:" + ) + + +def teach_null_prompt() -> str: + return "An exact polynomial rule has been taught. Write one algebraically equivalent rule.\nExpression:" + + +def query_prompt(record: dict) -> str: + hypotheses = "\n".join( + f"H{index + 1}: {candidate['expression']}" + for index, candidate in enumerate(record["query_choice"]["candidates"]) + ) + options = "\n".join( + f"evaluate at ({option['x']},{option['y']})" + for option in record["query_choice"]["query_options"] + ) + return ( + "Choose the single experiment whose possible outputs best distinguish these polynomial hypotheses.\n" + + hypotheses + + "\nAvailable experiments:\n" + + options + + "\nThe best experiment is evaluate at" + ) + + +def query_null_prompt() -> str: + return "Choose one experiment. The best experiment is evaluate at" + + +def transfer_prompt(expression: str, x: int, y: int) -> str: + renamed = expression.replace("x", "a").replace("y", "b") + return ( + f"A reusable binary operator is M(a,b) = {renamed}. " + f"Compute M(M({x},{y}),{y}). Write only the integer.\nAnswer:" + ) + + +def transfer_null_prompt() -> str: + return "Compute the requested integer. Write only the integer.\nAnswer:" + + +def generate(model, tokenizer, prompt: str, max_new_tokens: int) -> str: + encoded = tokenizer(prompt, return_tensors="pt") + with torch.inference_mode(): + output = model.generate( + **encoded, + max_new_tokens=max_new_tokens, + do_sample=False, + pad_token_id=tokenizer.eos_token_id, + eos_token_id=tokenizer.eos_token_id, + use_cache=True, + ) + return tokenizer.decode(output[0, encoded["input_ids"].shape[1] :], skip_special_tokens=True) + + +def continuation_scores(model, tokenizer, prompt: str, continuations: list[str]) -> list[dict]: + prompt_ids = tokenizer(prompt, add_special_tokens=True)["input_ids"] + sequences: list[list[int]] = [] + prompt_lengths: list[int] = [] + for continuation in continuations: + continuation_ids = tokenizer(" " + continuation, add_special_tokens=False)["input_ids"] + sequences.append(prompt_ids + continuation_ids) + prompt_lengths.append(len(prompt_ids)) + max_length = max(len(sequence) for sequence in sequences) + pad_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id + input_ids = torch.full((len(sequences), max_length), pad_id, dtype=torch.long) + attention = torch.zeros_like(input_ids) + for row, sequence in enumerate(sequences): + input_ids[row, : len(sequence)] = torch.tensor(sequence) + attention[row, : len(sequence)] = 1 + with torch.inference_mode(): + logits = model(input_ids=input_ids, attention_mask=attention).logits + log_probs = torch.log_softmax(logits[:, :-1, :], dim=-1) + outputs = [] + for row, sequence in enumerate(sequences): + start = prompt_lengths[row] + labels = input_ids[row, start : len(sequence)] + positions = torch.arange(start - 1, len(sequence) - 1) + token_log_probs = log_probs[row, positions, labels] + outputs.append( + { + "continuation": continuations[row], + "sum_logprob": float(token_log_probs.sum()), + "mean_logprob": float(token_log_probs.mean()), + "tokens": int(token_log_probs.numel()), + } + ) + return outputs + + +def calibrated_scores(model, tokenizer, prompt: str, continuations: list[str], null_prompt: str) -> list[dict]: + conditional = continuation_scores(model, tokenizer, prompt, continuations) + prior = continuation_scores(model, tokenizer, null_prompt, continuations) + outputs = [] + for conditional_item, prior_item in zip(conditional, prior, strict=True): + if conditional_item["tokens"] != prior_item["tokens"]: + raise AssertionError("continuation tokenization changed across prompts") + item = dict(conditional_item) + item["null_mean_logprob"] = prior_item["mean_logprob"] + item["calibrated_mean_logprob"] = conditional_item["mean_logprob"] - prior_item["mean_logprob"] + item["calibrated_sum_logprob"] = conditional_item["sum_logprob"] - prior_item["sum_logprob"] + outputs.append(item) + return outputs + + +def rank_of_correct(scores: list[dict], correct_index: int, score_key: str = "calibrated_mean_logprob") -> int: + ordering = sorted(range(len(scores)), key=lambda index: (-scores[index][score_key], index)) + return ordering.index(correct_index) + 1 + + +def unique_numeric_options(correct: int, decoys: Iterable[int]) -> list[str]: + values = [correct] + for value_ in decoys: + if value_ not in values: + values.append(value_) + if len(values) == 8: + break + delta = 1 + while len(values) < 8: + for candidate in (correct + delta, correct - delta): + if candidate not in values: + values.append(candidate) + if len(values) == 8: + break + delta += 1 + return [str(value_) for value_ in values] + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--manifest", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--start-index", type=int, default=0) + parser.add_argument("--max-tasks", type=int, default=0) + args = parser.parse_args() + + torch.set_num_threads(min(4, os.cpu_count() or 1)) + manifest_bytes = Path(args.manifest).read_bytes() + manifest = json.loads(manifest_bytes) + stop = args.start_index + args.max_tasks if args.max_tasks else None + records = manifest["records"][args.start_index:stop] + info = model_info(args.model, revision=args.revision) + tokenizer = AutoTokenizer.from_pretrained(args.model, revision=args.revision) + model = AutoModelForCausalLM.from_pretrained( + args.model, + revision=args.revision, + torch_dtype=torch.float32, + low_cpu_mem_usage=True, + ) + model.eval() + started = time.time() + rows = [] + + for task_number, record in enumerate(records, start=1): + target = tuple(tuple(int(v) for v in term) for term in record["target_polynomial"]) + options = record["candidate_options"] + continuations = [option["expression"] for option in options] + correct_index = next( + index for index, option in enumerate(options) + if option["pool_index"] == record["target_pool_index"] + ) + row: dict = { + "task_id": record["task_id"], + "definition_cost": record["definition_cost"], + "target_pool_index": record["target_pool_index"], + } + + for condition in ("active", "random", "passive"): + scores = calibrated_scores( + model, + tokenizer, + ranking_prompt(record["evidence"][condition]), + continuations, + ranking_null_prompt(), + ) + calibrated_rank = rank_of_correct(scores, correct_index) + raw_rank = rank_of_correct(scores, correct_index, "mean_logprob") + row[f"recognition_{condition}_rank"] = calibrated_rank + row[f"recognition_{condition}_top1"] = int(calibrated_rank == 1) + row[f"recognition_{condition}_raw_rank"] = raw_rank + row[f"recognition_{condition}_scores"] = scores + + teach_scores = calibrated_scores( + model, tokenizer, teach_prompt(record["target_expression"]), continuations, teach_null_prompt() + ) + teach_rank = rank_of_correct(teach_scores, correct_index) + row["teach_rank"] = teach_rank + row["teach_top1"] = int(teach_rank == 1) + row["teach_raw_rank"] = rank_of_correct(teach_scores, correct_index, "mean_logprob") + row["teach_scores"] = teach_scores + + for condition in ("active", "passive"): + generation = generate(model, tokenizer, discovery_prompt(record["evidence"][condition]), 48) + parsed = parse_expression(generation) + row[f"free_{condition}_generation"] = generation + row[f"free_{condition}_parseable"] = int(parsed is not None) + row[f"free_{condition}_exact"] = int(parsed == target) + row[f"free_{condition}_parsed"] = None if parsed is None else [list(term) for term in parsed] + + echo_generation = generate(model, tokenizer, teach_prompt(record["target_expression"]), 48) + echo_parsed = parse_expression(echo_generation) + row["teach_echo_generation"] = echo_generation + row["teach_echo_parseable"] = int(echo_parsed is not None) + row["teach_echo_exact"] = int(echo_parsed == target) + row["teach_echo_parsed"] = None if echo_parsed is None else [list(term) for term in echo_parsed] + + query_options = record["query_choice"]["query_options"] + query_continuations = [f"({option['x']},{option['y']})" for option in query_options] + query_scores = calibrated_scores( + model, tokenizer, query_prompt(record), query_continuations, query_null_prompt() + ) + selected_query_index = max( + range(len(query_scores)), key=lambda index: query_scores[index]["calibrated_mean_logprob"] + ) + selected_query = query_options[selected_query_index] + correct_coordinates = { + (option["x"], option["y"]) + for option in query_options + if option["label"] in record["query_choice"]["correct_labels"] + } + row["query_selected_coordinate"] = [selected_query["x"], selected_query["y"]] + row["query_correct"] = int((selected_query["x"], selected_query["y"]) in correct_coordinates) + row["query_scores"] = query_scores + + x_value, y_value = 1, 2 + inner = value(target, x_value, y_value) + correct_transfer = value(target, inner, y_value) + decoys = [] + for option in options: + parsed_option = parse_expression(option["expression"]) + if parsed_option is not None: + decoys.append(value(parsed_option, value(parsed_option, x_value, y_value), y_value)) + numeric_options = unique_numeric_options(correct_transfer, decoys) + transfer_scores = calibrated_scores( + model, + tokenizer, + transfer_prompt(record["target_expression"], x_value, y_value), + numeric_options, + transfer_null_prompt(), + ) + transfer_correct_index = numeric_options.index(str(correct_transfer)) + transfer_rank = rank_of_correct(transfer_scores, transfer_correct_index) + row["transfer_rank"] = transfer_rank + row["transfer_top1"] = int(transfer_rank == 1) + row["transfer_raw_rank"] = rank_of_correct(transfer_scores, transfer_correct_index, "mean_logprob") + row["transfer_correct_value"] = correct_transfer + row["transfer_options"] = numeric_options + row["transfer_scores"] = transfer_scores + rows.append(row) + print(args.model, task_number, "/", len(records), record["task_id"], flush=True) + + metrics = [ + "recognition_active_top1", "recognition_random_top1", "recognition_passive_top1", + "teach_top1", "free_active_parseable", "free_active_exact", "free_passive_parseable", + "free_passive_exact", "teach_echo_parseable", "teach_echo_exact", "query_correct", "transfer_top1", + ] + summary = {key: float(np.mean([row[key] for row in rows])) for key in metrics} + output = { + "benchmark": "JumpBench neural acquisition v0.2", + "evaluator_version": EVALUATOR_VERSION, + "scoring_rule": "conditional mean token log probability minus content-free mean token log probability", + "model": args.model, + "requested_revision": args.revision, + "resolved_revision": info.sha, + "parameter_count": sum(parameter.numel() for parameter in model.parameters()), + "manifest_file_sha256": hashlib.sha256(manifest_bytes).hexdigest(), + "manifest_semantic_sha256": manifest.get("sha256_without_sha_field"), + "start_index": args.start_index, + "n_tasks": len(rows), + "summary": summary, + "rows": rows, + "runtime_seconds": time.time() - started, + "environment": { + "python": platform.python_version(), + "torch": torch.__version__, + "transformers": __import__("transformers").__version__, + }, + } + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"model": args.model, "summary": summary, "runtime_seconds": output["runtime_seconds"]}, indent=2)) + + +if __name__ == "__main__": + main() From 7862e93496f2e8c4c2c38f28b97a11686b4710da Mon Sep 17 00:00:00 2001 From: wms2537 <51080539+wms2537@users.noreply.github.com> Date: Wed, 19 Aug 2026 11:59:39 +0800 Subject: [PATCH 08/11] research: freeze JumpBench neural evaluator artifact --- experiments/jumpbench/frozen/neural_eval_v0.3.py.gz.b64 | 1 + 1 file changed, 1 insertion(+) create mode 100644 experiments/jumpbench/frozen/neural_eval_v0.3.py.gz.b64 diff --git a/experiments/jumpbench/frozen/neural_eval_v0.3.py.gz.b64 b/experiments/jumpbench/frozen/neural_eval_v0.3.py.gz.b64 new file mode 100644 index 0000000..12a4e7f --- /dev/null +++ b/experiments/jumpbench/frozen/neural_eval_v0.3.py.gz.b64 @@ -0,0 +1 @@ +H4sICCIphWoCA25ldXJhbF9ldmFsLnB5ALU825bjxnHv/IoOkhwBsyAyM6v4KLRpn7GyPlGOd1dnd6XY4dJYkGyS2AEBCJcZUqN597Nf8h/5hORP8iWpqu5GXwByOI6yR+IA3XXv6uqq7ibXVbFjcbxum7bicczSXVlUDUvyvGiSJi3yejRSbdWmTKqad+91ox63Sb3N0oV6/VwXuXouavVUZkmzLqqdeq86Sk2646M1SlImDRJSYnwLr6KjOZRpvlHt3zS8ShYZ72TL2115AIlYXnZEi2opkbftZgPY62TJ423bUd8VK57Fab4uJI8qyWuUkFe1grlpm+I1wv2uqL5O2jrJfv86pNYPxS3P0x95NRp9W2QHNmVNW2Z8Jj7TvAmZ+piHLIqi+ejfX717O2ES2g9Gb9+80q/+Zcjgv6sgDEZ/MJqvdPMfbegr2fzq+5vff3fz4e27+PtX795/8/YNAHifwSILni+345y3VZKN7y6jl2PQ80eee6PRaMXXDLWOV+my8e+SrOX1hOGLo8FcfAZs/GviPhkx+FdxcJhcqOzXYCm+8v00ZJ8J2l8GAQNLMmoKQraEViaYRGnDd7UfsHStQAMpT1ILaUpkQ8yI63GhLFkeBLMJMEPWQhhijPQeFY/Vyk8E8ZAtDC5at6JtwIJKmCSgRofkQoBK8JlgPQc0eIs2vPGV5pcBe8GWppza6gCrVM/5Rol1ytLSxONl4AqUKEp1u3hKQUmSTBES60U3Brs2O8M+pzwFjPDw2NksARkTEDIRUmqzYecCOhfQuXBtiv9uOXl6koIBF0QFHz4HFhAaHwBNy8OrNDvwvADaTxq/LO55pbXm+7LIeQ5Kgjo909Vthu4Bc7fTMUbpIXhsuK9wA61Lh4KmFS8hk14lpRKtUhqaJsYcCNl+IkLJQUsEf62xxJlUtzsfNd5fXKTw53Bx8XloJhjTrYmbIsY2P4cYN8GW6Ob9B0dnnKh1mtdNki85QYYE+WpfguA1LBKWtsK3HNrRolgdgpPk3iQ7HsDKs2KEkK5Q4Adv74XMO3iPPRZ/QEoKdAohb+8xntWc/fEkm68LamsEKwckIuOTH/d1wvDd8SRAZHspmKr4jQOhAQKMz6ek+S5PqsPbcliYopRA79tFXxwKGa6ZixJWxnwV/Bxcb1arMwZWcTzJ8LdpDuw0sYyvRYx1iGG7nuBVutkOglGHhutz7ZSwdXCCH3IL2fnE7GEwiGHMfS6x1zDjh6lhnHgutW+L++HRJBKO59tcVcgCQ3fOS1jShS1gEOGS/WqqkeD5FzY9QxMRWIUuXWQUYStJYdJ8jwxeVVVRoSdHK8hZiH8XoijhjHkXaPyG7yEu103VxSj2E3sDhIUMnud9iyis2XK2Tqu6wbHZpQ3kJ0yT+SWDXJRXrObZerwscsj8cE3AdJetCpjgkLfe8eq+glyFpU0EZIl8DslhkkHSt8J0D0SJKg5pLdja+6//xDh1cXHtBbrxT6LNbPqfP/8FG8dm23//B8F5QZ9JxSN0rsr7uNgfPi4Qbn9xgD8aSmAtYfDTFWgFWVyW1s0MbISr4mzeLVO7pFluaani0ToFcDCBryeb5/9mokP6Tzd5DYP30/pjffHRh489/B/C/wdsCAL4nE2mc/jjz/70Mf80fxF4YUdMSxcaAST6RrwZDqjFjpKy5PnKJymjTVW0pX8VuMpFYHSEkiokGSyp3qdPn0D28tBsYSn6jRAJBYJ2y1IhyQAOA3/eHyWdpTkne9EDmEsTiOoySxtslwksPkZg6bT0+/SkQu5I1Zzn4MK8GyJ49HWS2RFA1sao9k0GqN2zEkL+9T79HYt+GXpW7LAokxTWvMV5kOYt7xoRJMI42SFa5LypZ4noUBuWEu3nAyaWLrPx1VzJPaid4fz+7HL8TzCkNLj7wzyguQA9Vxcfr+FpQMSTErD7La8484LZ5bxrW/AllHhdqyMZrfyNQRZdhOeGddiv2cvLyyes2lSH4QXJWOQwGFLo08RDKlenHoeg7AVaKr5f8rJhr+gPzNsT3CUjjJYyvPK7dAU1Io8xlvlVca+CB0ZDUfKBFaxE0/uYe9HnIs39tbf2HwBn9sX+i/ljKB4P8BhMxTMtH/DukV9DE8UeYILR/Z9fvX775v2HdzcfoGB9jxWr573aJzsoKNjVaO2Pr8LxVTAdX8PzZXgdTPHhOoSmlyMdpSZsD3n+YTRSqNcCFeDHXxImErki1Jc91IsDGwMyRndhj1VaLzHwH+IS6oTyTJPoGOp9k69hWcHFh++TZYMLKt9AC45qXuzSJKMKhBWLmld3YoMlYt+BzxU5LGV7yPHDDmkp1+s6ZC+g5gtpfb+ImKf5/RutUIQLo8qSbMMXsLQuIS4eGP+hTWEMcJXWSx/AGYsjxa+POYxpR/MFs4fG6ICx/5BUUGPZ8H0vCmwkw+gCTy3vkDjepvnm/9Pawt4IY9vcMOIH6JQThXohhyJr6pmuzRf93KrnbZYp/Ycn3DlaRuyZSlhyCYkaniy3ShRuzBOVbR0fiTXZUIhXtTANt0kN0ZSDqyUtZJITtvb34SHArQFN+dEy5s/ly0ctLvR72t43udTEMPCQUhFTEp8QFhGHzf1Da8YZviyqldhVsaXaHsoCdKx5jTGyC76G6f/lARIhjnHw6nHCHroBn32hLYVR2Np5IZTQTgl43u6glmu4FGfmCRmX2yJdcm8+83Q64s0DaVzahSlF7nxEQlyzWmSSNMx/EMBq1VBvtHDYMooukbAOyyPeJXdvbkjUDxZfb4tCFgZQIm0ymhC8Snc4TPdb7CsLMNYCeoq2KdsGRxv8ChYFmFWbNq23jEbCdAw9PE5k0B12QLi5S9IM964N9vXExpUKOZEXBCd5DLFhjhu2td1d2OZpd5d2QS/WlEU8eYqdnFe0bc6rI6FjcPvKkCJPdlTpGOGpK4xo7ycxSyUsfbyFFxyLQzfQ2NZk30WKuxyM9icadPmavfaTcEFRSPJ9NBeCtfc1KNCCcq/91/7DHrzz8BjQh57rML3RhWQQhkEXdZIba5RNzrC/5IlUK4gcYHO+6uifwVdw3PBczF061QhZo44nQibYy8HYJfs45/cx9dcDIwLLGVCgAleR8AWFUMoMi11eF1U99cpGjsR92mzFiUuU4nJFayJK4gfWRnlJW+skYtRJbGWsFxdSgNBqtsWe2q826KqIa0oGp79LsprbnWWyEkhxupp2Cka8qLtmG8HsOQsBCoh4CSsNn36oWoO75bKa0Iqjtr6wzewyVPafeWkOLUAeAltUb5OSz67mbDIPWX2blnFd8iUEIGUQ5KWcz0wDYkhoYVo95RUmirmBQK6hEzIxlgITJRt0EqgYB6WzNJKVMLg7eIriSB94hGBsXEhmsJhumq0ClMcMxt6GnfrkjkIjpx4SpnEV8JiHZzUGyKAy5FUD2lgaqeLfsNWLHnedK9paKmSsLjUBWfSh6ws4nEnJnqAUX3EspN5EnS8lEtjo/6k1uSNzSmCRe6ynpvL3DS0UuOM9PBnENrAyDTHCqLCGOOhbotZBaOgCL0K0kK2aQ8mnAi0r8o0QPGkg6tDoKpLAGfhm6S33O356GwVy8dAyhM5utATaMToKM0KcMMuq846pCH26pyPQyXeUwNX5kTIrNmlTq0ip1Zt2T6FmGO+S+nbavQaRwDaJ4Qq00IOBLXWxbsh7CHg2AZGhbqb4skp3UyibRzpm1/Zke6ZtoYitMOzbTo520hMHUiJwKtqCtgZC4LrW1POmqFOVegrdEnEOJ/DGeEZuzw9o0oMmnNY0UPcs2Hf0Qynh3FnNurlqrQEPvQ1xz5z73sSOT2SKsI9TtzsUDuUBlHVWJI3viBzhoV8QDCDveJI/iY1Aw+gi2HmUIPTQcLizHt7j8FonDaUWpyRLF+gqq//j0hQyI7fSRaq7WAH6ioYQkvXps5ZGh7WMn1UKE+BcQoaEg9QGZ5chcYx3JULBlJ5xsv2Ylr4BI7txqlSgtlhrJ9a+r0NwpgZ3zv5mahDX7c4GJZ3V3NQ1r5CKOK+xHFrpLF6WW5yCK5YsIUDX0pq1uReNmkyp2PVd4WyomUcmtHx5zmyh7U4H3XC3HpG+XRyQ8V/Fx5y1w2xsCJuL1Xcs2GhDDc4x3FuKi3UMbgmVc+ML/zR31tAbqS+mjQBZoBFcfMsPNJ2wmj9mPvsOAtTmUCLmGzzLEFeBRBCmyEu8IVTgfY5pluwWq4RJpv5Y9M7ofT7r+NNNEmgKbCUlm4j6fEsDvO5xJdVv8xSifUxrUrpU2wMKXuqKefcBbKKuks26S05dhBHKiVtLOEUlAT1TqYsuf0hq5qyTnZgzdXef7Hkl70PJMRXw1nED2k8ABXjT4CsbfVHx5JZaIOw0ic4wtilUvybur0zU3hmTMgxYkAh1vgGeSQ3OebF1knRCvb6KA4c0Z2lqa6s1fqFUlg6Co6bs6AyQID5Xl5ySNBfluD46pvMWdHp1zzG6qTYt7nvQiXLlBwYYHovFiez3vfGYYj+eSOHeX8VXsiA7gVHxuxR3O56FtEvydM3r5llIIjg8C4XSpzHNLMCjhFzOmnXSZs308gkp9+MGktL6FC4g4KySJMRBP7aBnUciN8N0ruYNTuS42cL4r2CNhYH7MmRFHS3LFgJcm+O+Cgz1VVceCQvFi0ND0xbvkPpIOFJduJ0EtQYB+DYWwOPlVciQiZlFS57eNkUpnARyL7RSrHZfJY99TKqjVzstVDXR6ZvwWdzTpCRfspl5ss2bz1z6E+Q7l8XVulClAd1eldqJ7EO51ZQa1Vsw6tJdzEwA3brCGtG1uBKmEaz1OcTvcykSiKTmXpPtEdW1kqZu3A4Y4hIaqTq4Q2zWhpTRvrwOjULnPkav2PFdDCJszB0YQ9wI9zB9NZoJrldYPaQ7HuGH7MGzHJGXdeEeRxGdcYGpnRiqoW3zWlYt0ysjcDZ0bqYuCfviE1PrOxmpkBIsRTvxik/mxrdAj/XGs9p/t3ffFXgXaJ29cXcLhDSUO/Ce3oCFpMTef5dUTCLG2gtEcjz+soM69RhHDZqaNpik21sOlEhlUWSCB2ZS0745dLdTe8ghFCcquOE7sqsbHMp0BeWNJilanDrMg6iV5pS+QbSBwKsx3B4Xsy/k5JQCGvtxZC/WKn+kxdpLlk16x3EbHK/dFTt8KhMYN2h0VmqRXdH1B7fo6q2vznS0KmQqZvp3vJzDW6maOhKFINZJPh+oMC0/PE7d2kC3wQLnwkmnI+LibBhMg53UN3BKnftnYYdOne1QK+5nawrqG+koD51JHomPKBBsyZ9BAtaEKyJBF+ldA0zNrY5z5LnXMqmXZ+ALAxG2eNRuLI5ez3XHfh1tHU07c8gMW4FTZYdDh74DkUJAHRl4U/ijzoOm8TQZsoJ+HQKzxs4UwBo2i641PmcLedxHDeLG4JkNZ0aiI/FHnvCITdsTB1T9azenYwn78qvAOdWB7HGlsknzvqgWYWgqrCvOLR/W4GQK/XoOMjHHWrIbVimXsXV+lhR0/cAlAn4hPD44W5YV0aAd+3TNtDB6D5+OXPx+3iFg53r4+XJbxOeO5zOmqzmSxOP4UDoiDLqxA0P6O21H0frDZwo0OIYuCXvgLH36ozfI3x4yR4Cnx81AMAbPuiZhpInnXKfQ+L3Ece2dvszhJJEWdZf4X782DNyjCcIhkcOhKxEDi0HNM4idwF+AqxQXD0ucXEHtcZkqDO90mRBqt+v43uSTQoE4ljVnQzL3s/VlAcZJ80QUxnZK7KucG1PRLgE/mHVG/16OJcSxJJ4OTsCtT9zjUQKKMxbDAI/2bJFmVMpqfWjWOFaQqriNoNJ8iKqUoZu9/rnUAnHe3DNxMCi7XmfNBj1X97H8CtIhll8xwjOsa+OgMqf6XXxLTMSUsIcV9MZe3UjpoRLBAUSxpamPJvqjL8d9MrAUS58YCuND5WavALSpGLG3vzEo5LR2UX0LPWSDjT2T6SctjrOHDPqc3lzuzKw2mDWpv2Xv8CvVnCV0cYcJ768BDoM3VJJ1I68P1oW8gEi+wZoUoJb0jWyDWH2oG75LFM46uSvEJVG1eWvGvshwxUYt4OjkJ+rRgP09bcs6mp6yjdMyU8wmc/ai1zlRvXPjary8NfW81WB0RsHqXlI7kZUMuYZFy9HkCKMjhWvQ17a3mWLTl8cduLftelkwQO1YFWPbNjzC3k1PTKqiOjBbjgDbpY0t2EB1owFOFTjnyf9UseMi0fjKYzrbtkcQu+yoP1BHMMzSym4zEboIBs/mvSA0obkba21Cev/gifsNcucxCAf2s9g6a+utuiFG04bjUTHFdn0/1izrRWUnxjG0u8Sm02CXrAJln6Zs1LuAQaWKZKBTbqdDJNKqURE2wDV1C6LDG8zr7fYO1l7/EchyZMFL3lhrd7uE8q8HOiYVtyryUlykwHsc4tvozhde5vLnEPAr7dAk7f84sm5F6nzMox9uAEa33oR5/9ruyt9iAxO/5MCS5Q9tKu6lMPxVB9Ma8n5uUcVQT9firknv9yEMeHRE3GbDe+rIzLwugTqJeIrXYvAcf5Es0ixtDgzWq7amJYbnzRiH4AS0KZ84J5sMHgF43fXXuDsck5D98wAArovszobF0xC8K2mAwegnO1xixWERAOGNma5VXWWh4elaaZDorKBrqq3rLl53NLROMx4Dy+t//AXQlj+EEokG9wAp2vL9Kt1Agz9Iq4YFHRbtpaanuuiHDTzRHOMVMvAahAL2PFt5JjXj0EhZz2gyAHNxLOWJy1X03RWTjPB0YS98Mk0PsLiZDX/MVjBvugNjcHQiBDDOVNhYnbWYzprfpVWR47EhQDtb9OLrnNCufjcmEi3Ksd3dYI9OhZArnQ7FCi6OXTjjZ14API7FT73EsW93BcM0HsWjOXlj/Nka66hRtAcuDLoT3q7f3a7Syhcv4nYsfj8aMsC4uDUOZ01M+i6y+JoRnVDid6VreW1Y3JbIm+l1IL4r4MlrxDCzp17brMdfed31pdwi8DA0I4dGf2h85aXlXs/80ZAIbyVDRh/HeOE+jumXGuIYT+Dj2JvI01c8jh/9LwhedBmMSAAA \ No newline at end of file From 583597823729f5877a9a97d9ca00ab085d07a95a Mon Sep 17 00:00:00 2001 From: wms2537 <51080539+wms2537@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:02:14 +0800 Subject: [PATCH 09/11] research: freeze disjoint JumpBench neural manifest --- .../jumpbench/frozen/neural_confirmatory_manifest.json.gz.b64 | 1 + 1 file changed, 1 insertion(+) create mode 100644 experiments/jumpbench/frozen/neural_confirmatory_manifest.json.gz.b64 diff --git a/experiments/jumpbench/frozen/neural_confirmatory_manifest.json.gz.b64 b/experiments/jumpbench/frozen/neural_confirmatory_manifest.json.gz.b64 new file mode 100644 index 0000000..426bc3e --- /dev/null +++ b/experiments/jumpbench/frozen/neural_confirmatory_manifest.json.gz.b64 @@ -0,0 +1 @@ +H4sICFkphWoCA25ldXJhbF9jb25maXJtYXRvcnlfbWFuaWZlc3QuanNvbgDtndlyHLmSpu/rKWS6JIU2wAEEgLkr7javMNYmY0nUObJWSdVaapJzrN99kksyk8EMwB1wR0Syu8zOUlrSEwEkgPz4/7//67c3b97+cfP1wz//vP7+H2//15u3//vXn3+d3P3Cm683v75ff3lz/eE/f33+8fnn529f3/yt/w3evrv7O/+4+Xrz/frnt+/v/775/mP9e3d/94/rL9dfP9x8VN/+uvvjP9Svr5//89eNWv/n++39L/7560/1t3l4iR83Nx/XfwsGHU0K5v7XPlx//fj54/XPm/f/+H795/o9vf/x+f/d3P2p9T/3f+Ln9fd/3Px8/+Hbj5/vf35e117/5v9Z/8abN+nd/f8Ys/6ff7//s1/fP/zpuz/j4v0v3fz9+eN6dDfv//j1cf07698YHn599eHLr483H98/vf6vr3e/a529//3vNx++ff+4Lfav+/++fz8//uP957uBvD3V63/evtv+zv0r/fXt25f3n79+vFmt/4wxg376Ax9vPn3+ev9g74dz/9vjv32z+uv7zY/NA7ZHq6PbN+rN7f1/r/bU+nL79dufn6+/PL3Ru3+2/+/Nm6fyd//A7r8o8/Qv//5u798176ZeiPh3n/2L3f7V30Yv8TRd6+H86+mPvb3+8PPz3zfPhvhm5/fv/8zd07bvnv/a7frX1PgX/77+8uvm/jfSzm/817v8S8Oel5565UB4XUt4XeMbXximXtg0vrCZeOHWN6wn5273hfctxbff1zvLtz/LawYIz2noMpzCCxvCilGUidWEiW1+4alHrFyXNb53yfx1vd54MfuM2JPaN7dq8pV94ytPvXDr3E4tc014XUV6FLb1lTXmUbw4t55K7dxjHq9CzxbR7rt4++X6j5u74/rt7293i76dvDc8HIzZe8Fvewa/v+hJtqg3iKKru8KEkqeZkkP0mYrwWBGO7gZ6vK58fPf/CbXPcsPNjhaeRkuteZ59xBAQj3j9v+uylKIX2aIOEEXdEWUhXeYLJuRCsqRHe5UpGgIgpvNhId39x+zUfXEPXV/+198A1l8Lnj7Xez62b++/57z/8M9vn8c31off2bcXvNiTJvaDzPUEuz9PfOhz+yD5pU+xbxp9x5n47ObeNVBf+nzfS+/99kB+6Qv0u6Y/kct9r633HpTUl77CPhFdvEZtPjv3rz1e/OsFs/8vbT5ohU9L7iNP+9DnH0x2P8PvaLQigCrybJ8mFfAGPYq6MYyOc/qBThrO86MUf5iSigwaVeTFTYxSpDAvk1eRvXfS33ZqT1ErU6BWETLQKuWZ1d27vXunq/sZhseH30qunuOkenAF9dyqWBUwVedEXgYo35MoyMsIIS87COEAC0LIC6IQJALHxry6jodlahWFX+hGLjI1A67108PxwpQHMfAhLxqP0Y0ESbHQNBLp0UtYjVqI0jEsmg7IK8YCNLi/SpCAQRZ3xRIweHmN4aFeZkiIync1aaQtR7t8DrQ9FXz4osSGu6IJuIE+3W/ZmFdIQ3HAD8N9+H5AA1E5+hUHS1lXTPjLhzJJPd754vV8ZVEA2ElXALb3IAUeAgYlvlFNwCwLlNlLwHhQ0nkV76kGYMDyPC6xbxpY8FeRvc6Mv0YbDWWrodCJCLQq1bDF5+FR5pAiEZ3n1wzcRYNSYHTyUM8eSqkhoabmxU2CUGN0hFMPccr0Bww8nDzDqmEYFGDYYKFBwrWLBzfzwArDylIsIQkYD0qbVQIGRgqIKeOEuIlKQkhMDUJITInJwAZRGdj03GoxTQ9JyUJ5VsMyxDd7L4IcejtTvdIFwNgkw13EuiHJ10CKiykpgNWsptSoZ9ETjXnrsjBj+zNVRRQrneTLGkLZFRsds1As+/xCxSUIgwE53mOyCi6rCvMaU3hFnt0sIfNDKBZl1IR57VBjvBNqqDeWCYut1zBQ1vALtR+FjJ32JWNWTBvGI4U6Rb9rYCFjuq80zLCQMZZJvESDQh5h2N5JtI1o7IQRjY23ctpmTtLuWEAX2rvDkHgPqtjE2UgblSONqraMxs3SqqmINbQZqh2LQ47lxQlHKjMERJmxRrCOjNk8GRuibQBjYy3gHQRlBWO2gYs1sK2FaL2U80L2RisEtqS4FonWULhWEMJaWpRqcXg1pZaLoi3xtICJNRSgpUxsfRrTLDvIGhz7+xDtItCTEgS4UgI1077b9LA4GrTFkQtppaCL2pz72xOj2svHshCJkWLt3Momf4j6/NbFBLFcQtS1fPwKsoY/d/T8Xs+l7go4kyEXvjLgER+S+0FWQquz2f2MloVZgZidce82zWNn5NFFneM9hzx+Ri1nZ2SySl7VYbwKbHXBia2g4GjM7Go065xHudpG+wqlhEsoT1ut9Gl0ytHOOUKdgDWY1toLDcFeWFdkdOfB33oo2q2IkbwxCbZcQbCVUqt7UT3J2Uwzk2LLzSpDqcUbEK0XMyAmKbmVEcJSXgheNId5TWbhBDYutV8GZKX0Y5Pf1kFMbmVCI04DFnZBMmRKcRzFGL21wFgoTjJll5DXJvUkGpyePbmUBo9gRLv3MsMluEq2ZFpTuwaIzZ2K+BZO86MPxNGvmMBVQvjm7tM8noZPG3YWXqWEqL29FPIgrJRFg/tGfEyqneNYbvCkpUYbdY5pgU6hWPpZ4klDWtd5V7qlxRRZgnRL0KuoOcBZb0UWj+P0Ev3SXJosI6DJuuKEWxo8CkRMHK0UrjI6SGuPUgplQZkxM+cXaXgJVWvPeUWpoiGQ56tOPDU6jKjHEWWaCqSqeOgSao0POvpRV43IfB6ReRfrEdnt9v3ywjGoF2wxRdnPycaiGBqTEmxFqWwuITKmpWyIRlSwpWcdTs13esuBJkmkB6SgGMnGS6KaSRaKiaErzfCpFZtb3dGMKfV+cTizJxFzyRY8VJtL2WMA7v3dgImJ2eiLxTc/DuemYesbXCSiClrxrB0xacAgmsdoY2BCYd7nqm4vf5wgbOc+mq0JXGZEHYYsedsuJ2BlX+s5RRHe+3X8fLwU6nUxf0SX4lF1TSjCWciXEYypZ4kWO0e/a8GYeh7yBSzvup8Z8ZwVfIWhAB0y+w0pQCuvHpvcvEnsSSPR2ss9jFYGkGWOG8qMDh7c0UOTwdmiXS97eaLxpkjmTdXrwEaPGFnmZlYNt4aC/suF2BjYtZkRe8Tcc7GFU9nDz9tKUowrHZon8eCithKf+EvKiqeFJpb0Jb3dWgZCb3nWdn/9Gi4uQpvV142ohdRkva2IajbCJRdOlA/YcuUIJssbPR+dQ+WiP1x+2DotamS+1b1Xg4lkuXLyE9V4mYNYYFxZUrUbA8NlSdy98Ba+Yli2vHnjA+bxPp9NCsO6nF25pXhi5nlymLprt+QaLfIIrC7E7KWXgnbKqzo7ZQXB+p01TstodFDTaAsnRc07h4xOH21lFEoy2qZpGzUpqslhoppa2hM6TBxYSwxYwBWonvDRyYU/u/olc9UxqpBnVHEbT1cZnbWrF2NVYdHi5A1JwLX8DonNhjDgsJpREBV4IUYFUjosEHMo8kEqYqxTK9cElvVohOK+tJCtVVFC20MzWLNE7CAtxZICSl4I+4ip9zqZCFmDvuayJyaNaalH94plORUYX2zNSI09ylOqCEVKdbv5ASNxpFlSpU3E2x+JhbOdEpNHNg1UFZObzdEyySAmF/j6IyaLHypxoDliFWFANaNssRtedYVWPPTnBI87BKmVWXyeluVgS/juiHbxjkOJ/ohnnNRqtLPh9jaKmUzjeuO1mMjGRyHtMCTZ/Uyk2P3qC41uE7j7BKVlYfLojpWqaXJispRK9XVgQDZh5DIQxgK/8pGBX22ViJ1DtjRGKUUVaJnDp19Bin6ZQYp+WSn6BYdGv4AvN17TOg0sYQpokWIgJCuDJSASmsFxtth4kJI9aTHoo6S0cmoRbK1aDdeRf0VEZtJxlUEtS8CC00UgZau8eDkOFmNEo5IVKwbLmgC3bbPoTzkPwRIRgnFptySZVN6ICIYUt3ZcoR27yuJkDHds8SL+Pr+Oi8eKyNJe8FROEIU3Ilo5IMaUwcVDrS7leluidVx6QUAsonKjJndYWnaUIWZHHTeJrbqRGFR+1HGryVIXnJyZE5hSZnSrodxrKM8sRsLcrKrnZnSWYU+zakaWCj5Eu53Eah/iVlK5HJmXaoBkcPAmRiPlYVQGhCCZkYJkRsrHmIQYmXSEPUglkU8umrDUqWV5FiQdV6vdDfeOl6kP00vIbBejn3KdIKUC92frqhggFUHVQ+4DX0KXB0RC16oCnWQ1YjYNxbL2aBM8AUxoDLQpu/xWVcK086y50FqUuXDznYBRIBaLUfHb71WKL6PLFs2xm2UFTDDMISDvjqemQSd2MnvTRSadmO4bz6XlVGJMUIzlXaOZmOYRiVk5kZhZukhsdIxQDhJSXBLCDLhnPyMpxRAp6hM7NSllymD8k6tGOdr4wKMeeRQXJYqCTW78pFXgAb0KGsqMLoHYa2AtCTM6T8Kc1fVx87tPH+ryuDKhWjw8iqr3Wkoe1yDFssRIFgjhjiQEsgYhkGXZQBaJ3ahmf+y0UCUuIQaJ5hwcGoEHx8eHBiKff4D4WZbprslyUmFUZgHpWST8pl8Zy7JASJ2/ZaJZ3ppCFPnT/YwJZO3cjgqXT2BL5LIa+Wi5+NVgXUFntGnBZY+4RF3WaPQC4kvjglBetparlaIp0t6tuOK2klWdzs+qBLPkeRRcLCqr3gIuEENVunMHRVgsquJM4RrtofhdlIAOrEbnMVVm1FtTTFzfc8JShgCkxPXbSjyFQCx7917KSIwmDKVWOgcB88DqAr6cxaM1GOV71aEoU0BRGlpR1CtK3VoIh3JSGKpZxGKl/IzA0W+OAqKaydmkmoTPdkiy8Alm+2i5qKnmpa6k4smBgXGp+qXO7zzs3gFRqqGgE+I6Yh0bdZ9PZkcU5e1QpEI1upuT7Jf7AYeijonhTHnH4YA2wtkqTVcOSzkNKPEan67KOUBrmzglVcW+kpt7PafbEExIqGVsjziDt0w0uId8/GafWo/Cqs5eDaviiRvvDKt05/itxZsNeeRxV3Wm0Zlx1fj8wp9gJJw0IGo06oM0INU09WKnkJDDsE1ZYgaQOGbvVkzSVAFBi1btxowDwVloW2VVEdFfceLcruZZUDAZJm+aTIaPXTR3SOKhmQ25ui0amjBs+VRNdyE+jFRNSam7tBBUa4V1SDEUO1Sz/YPsKaHkNEVTWsDkKimeQ0KAxsxjVIQlPKmuajQtlCtnekfvD3P5FJMxReXK5l4Cm5ReRrrmfEDX57IsJocJl3+oygvWQHtXdi1WarDOs/Ps8fN8yw7a1uM2hHHTl9hldolF9NC52jKmrIVxvMS2yS61RsbzrsCNSf1zgs2F0mK8TZmlOxl5crIu8HBz+X0aJYgbp5dxdL5QTxhK1FLBMofaaCg4yQeE2mrPeU0yNnqDMTZOnxYUljS6/9TdgEgFPaXgbeMCGV84aFcO0sqIhGFxKMxsnsh532B23Gr5Dk5lxoXhJqP8X2XoF8n1tojMr0FKoCZEKxWwoTSKbCou4PHLrUSxsH3bE4pY+bwvvQzdlBHiUm4BEWVSrQEOIQ1fMI0qD88chmxUCOKyiV866mIq/f5spqYwfJcC0kdIzb/KkjMdHHK0wETLdq6uExqE2sV0mbVpOorPl1WYlgxShvewmmsp2cVraQsJcnFfgiH4LK98jrUMckEyK5b3xWLPvMIuvVYL5QkjIRvtNdTdhhThHhwiwv3FBk5CHw6HPprQ2/jUpZy7FCGf10UFXHuq2HjHp+35lEKjSwPl2tAvjK0OgLmCJG3brJIOwHb54x2ua8Vf0Ku5o8GQs9cZ9qW8EMOyWsplKaUZMnrxeV8g5UZVJCpnpZRmtHVOQXrtjlOeXojASz9lWNYS0rPk5kDMDmmWYGRVZi5VmPMWxQKevBBcQMskpEONz2pphkKA/cOtbfeiyBcAZoKJyOL01pLZIDDwqCCwY8YgMKcNeqxcMWAGBsQ4b1nFXzYm/IKqRFmXr8VhyeKCPEVLkORIFpfB0si1c5RDWSwZZld1wWsz+ytHZxb+1KIkUEVckPiqpVckDKhYs9s9Ch50jaGY8l84bUnx6yaii7U0PByda/iTjVBjdF7jT2wSLPMWCcv2F6rjWD7PsXaaY1Y4K3dndxO0z6rlUiAo5sIpsg6VhEFqlLpYqfQuKwWsundwtFJyLi+bfD+JB0xrXJJlaD1ghORRJLSj3ALWjKLpN+08WWNKyhAoh8HkYvWUmIuR1AW1+jPUEYSFbIj4043u4VrEKOwaBoukFkqmpWNCtHSk5tJnOZiB3IjtUU3yfo5/heiQM0vnbjkGFpxH1K2R7F1m1zF+tFxdHEPRAjm+l9eRsKvZSZhZfNSY4gF4Z1jt1WFaH7Wc81EuGV83orBLziaOo10ct49TKMhg0YxKNffvCwET9L9n3yTVAHSNFvnY6NyhnDyUKpHyxFYNo9Ho0cBRS7vQlFBdSV82YqhjYUMhNj/Ettj8B4NnKwB7HtFlDz6ii9bzjsJ1rBCIklJkWSGkINV/MbFRqM5CoekcLb2EtQhCgEtVTy5jlD6CXaIplDId7XQ8y1GsraPu6ABUvVP61Vzewp0zf+Jni4pRgWVSsAhPIRUUnGZD/TFp6Ioo+crRJg8B1a6PJkM6z0a+u4gsCWx6K5kemvlke2eww+QyD6aASZC7bVJb/T6/2uowjYNMjMmKMSZlxPLsFQsZw7dfFNRbqSWZB+VbF46OXMyhS4qACrgUrdsGNdfoUMUeq6SwfBeRrRfri4xOauxZTZsNi5uNenfg6FTEnovVFCnkKZJNur354q4WbNZ4LOD3Bx4qxpJyFoKUsTAIYSwQS8cKQiBLuH3jVFm/hOekSBI8qax/LxVj5bsscv6YeY4HRYNNfgExVhQmBFLOP7Ek/9kchUEj+hnWBL3nYNbORWfi1vXyJsOVkeUQAq7NFZ+tbyMYdIM/qoUyR7ditu7z2a2Lj8raCgEIo+aUVflgCLNcUz1LvkIEtFt2f14YhYGd9I2Yt2JCKyuns2Ix2PVu6ag5MNUFVq9k5ERW9OdxVcftKgDYOWd6VtJEE111GlQwKL1VZosjRVw5UrXKKqNts2bjpPRGBIPqjZg5HSnkEiidJZtaAYAhVaolgKObI+XuWM3PYp6fgdmK6aqbPe5mmzErshr4mWoJiT98gqbSoSE0JZUvf2iGRL5kLppeC6RAV6s5VsmphYyQwkyxhImRMvOGmXK5BCVbcvY+uwAXpZiJslq91rVdo8YlvlNlRFmKNkDBDaje1Kq0TvNiIo/WTK1G6p62Lo0R16Vx80N6xczTjDe6+Mi37hbLBNJMdLj4tYke7m1dGsuLjKpUy2fOO4cfayUyO51dNsbTk1GxuB5P0ZxP8aR0KTlrIlMHzAs5jd6lmI6uX0zXKac3MWlsTHuDJkpHbLu9zNlBClJ3DolhRhsZTb/kCUqvVX0E2egAoh9BlGKjQ5ZyzJI4GaJIvVRxdDmjXc+qAVkqRM+b2B49v7q3VPJmdf33Tp43Ut+ijZTNEcSS56Wyk6IQ3uLrnqilvkCbZUAoUiyWgq5kBVjesyXNoJ4pcmsJvQP1QnKxQAhwiQnQpvoVyBMuk0Uf9uhZq2piAlWOcgEi82r3h9yKTyvmTDltf5tkwRe5FVEJ8Iotc379NQIIz5g+3ItspJomRqpxkS3wvjhqOsDMwq3BGDSyfRAB1KZvnXVFXEYuh14fZEtFLUa4tJgvkqcv5qWctO+qStpXwbfOGPkWoJKKJk8wUgiXJodw1Xbv80AaFBzVD2u0TWM3alL6vTEEWPNiY6aUKkTfTxzvpLkxmLD41lkZ3wdpN8JawAW6pADb9vxsVIDNgrmEBGDm8AVgTkoApqQy6ZtfuLcCzAkhMi+KyHi+3GsSFWr1w1kcFHo3U8SV3CdTM3TZk7ZRtq+bGbRlUj0U5Xp66s7LRpyQWY+QY42+0TclgkWvyaogrmww6xJKFvTg0AAuHyVG//XYMNywJYR5h1S7rRixmIkWO7ubn4TzkTFnDak2TWGXbdFYSIC7fYp8qbVLnr8S7Vfn/ox0EHQmp1g7F8w5u0BDSJCLDGNyTNrS6ps5l966hBThvDhHKEofnPbr5R5KqWENWiDVJmUbHwz0o4GiYvIoFVP11NhijNjeDZ/0uLyukK7VBop5hxbkrZiQmMkjsbjFgZVtGnddsJv0Pt6kelFzo0iy2FKkY1YMi4lJx8SyxeDgssX4xGNiD0qugx2QFrpfxEKnxO+HVnICRA0QW0w+S0MB0sLp0p+QE+WZRXyE6iWjfSPGIinti84ycpjMmUAKY+JySZoh4IdNddDlIFnMCuY2F7pn3auYUNmgMco1R062z3GyaCwh4ow3VV8y1i0HyQaH7UJ6POrMSQFlF31zxXiMeydinRBPxSDcmVg/S7SATLB5oyQmAzmL5KKS9UcfeMpHfoHJUbGgiCscUZQRmUBMFKvOworE5LcWKje+WtAuFxRFocap1VxDT4doLCmHbV//iDpwBgWzpLbQTM7U4pL4W8RgC6FeNJUMdGzfyPKOSVowsaB4KcOk4WsNSfIIxiXMgCIl39Pan5IwA0jJwYyW0sbFeeRgsIBsLb0EKZhdxGJUpj17UBx5xWTxCIYxV99pRE5URdp5Pg3MF8dKH2VWBWYQ8eoVNXNoK3mDGCUccebnm+DKA+UP/Nq99+Zm9Ig1M3/wAUNLm9PyL/tSLSOWli/oiwS56C+55C8QS8vnaXlwiU9ZkxN/LSktf7y9UTY4EizxAQdLGgPlR8cS/mCiCYwQsKSlwuigwx91pH4CFnDDaMn3T5ZCGeunXaOi0fZm+tdxK1voIjmEli6SO8x1dgfkTOn5ZokGSJJqhySAMVJKLyHiYoWQFwgRL8SPklpkXizfqoGiP5JTNjWrvHT/DCknFj6WZBPCTO+EMDkzoZIL3DJdfIqsPG22kLA691aTCXJwAZWopPYmKrWAr5Ad7ObHlY41GyzEhKhZJ6I7z/c3GJD9DaiPOJsNFjDD5fc/2oLlc3tHBi7f4xCQFVu8j1dd4ZeVU3QJwi+eKDO0pksLeh/lYu8l6ZdcLFizposz917e+jY6LGjHBQXnjA4HyvFA8iI6dEhXXYXRcUM7cCgTU5j6ybsDKX8+oJ9WixLNYcuoieS0OgbmCtotm6ApB+z2kTwfP3YYYKVgVhCCmfqqi2dgYrIvUhs7ktVOSlAGUjn5Zli+2VEuGokUAyYWqaWbFWUgxklsbznTXGZHvYisLrWIBH4xw7BUdtxcwq/de8fEbej5vYIrCoxk+WMCYDFFotGQsxmkTQPRVUq1/uVTwQDXA3NHWcAExZK2pMdOH3gOjQ0UyyOX4dEZj/fwVrKx32fPBePRhSnLYZTr3hOS56XPsURP8yjDBFtCglhLSCOQCsZJxpzxFNdencxJW6IHsaUZ4EAzVlaaEEd3D8rtgyRBA2zTyb1nH0nCFclG0frnN7pQ1FwpSLlkRBMsAzTzBeGYaxaObYLteOPBoCEe7BUovwYx7GWk8JSU3VFJZd8rMfGXnQl7uSXAANJqlBOsDa1AbfoTBGKszlhZw6McnxJL1hfr4yim/urveZwr5ssjgYgik4GTbA59wObQM1EvTMp/1TCzPSFdMqgOjWrTb4lL+OWjRz1ge0QFmhdZvFa0Pu70tWdCW95gZ5ZL9BUSei09J3gUsHXySsAWTyr9qRx8OhOMd3/drSB5DI9KAmxdsMbdB3zcfR1VGG3XlA2bUMajEZBqADLjQwB/DJCGQtMvVfZkTAbZL3PPAU5hWCERhlPLrrzBF2HAVkNB67VdjHRs9SI5736OWfFVg2+xQe3FE4v/SrmXVMtHK6YjE5N7STV9HGSxlxaDSN37HGopSZaYMbGV7uLIUz/DIywhiktqyVAA3SCE0cT0Y3ouo2MM+Gx3NpGX9gTtjWUjXnmp0/4bFBP2GkLCi7wY475spGT3c6q7TNBA1rUds8m7zJBo5em9C66yaf6BkOY/HjiFip3OnwNm5fReWi4HTM4KyUTFQIyKMbG87klg5ez8mftARhNIGeANIp8YKMnpldIe7Ul6r1raU1JiFY5m0mOjBc7Xa77G51/NCUiL0k+Ucg0xdIk6rn3J/XUELeQJGgTfJvy6X8IMsi94XX5FWn43hQcNhwawmuPkjdCT0NUoQiSl3gh9s3dS2qooJRaS0grZLu47frmWFAtSUhPQG16JCR57rxhxegUmb9zbKum5YroANLIiG7Sygy4qpjaFafgmR6x2rlKTVR/uSkz6LDA5G6Q92tcFsI1PJYP24tHGmeVS0SRk2Vty4lqOSEExmGu7jGpZ1FlfFmXlWBQLezmVUySdiTntzuWS4y/kEtAu5YR2HVstnjGiqNHZiD0dKbIm0OgKlcIpRPbT3l2LUiN4lNnwtpoDJUNwgVaXGZ0rtJOF5M4cNGFSqvHZ6D6AuxFUg6VYAku2HSytarDS/8RgVfkBQYgrmUV03yMJo6QMgXztD/fftlDt82o4gZUCEBz5WkB54ebmhxydCfTy0ZJcxz+1BLikl2GKdEKgcz66JBqNtLwcLDOkQJTswBHdlJhtiag18h3U2CGzQVjBYgLPXkX2ldMB0Y+RSwtlYtAo5PUU41uJn8674icecHEiBnLQSii5hoiwdPgk2BCRJSMfbw9UrTKoK0b21DeUanxs1R5cJGhkcQlVM0Qq4asEjSRGezZmUmNBjZ6eNtfo6FzDnGx9A8Pq4FTKw6nkhoaI+G0fzNXO55A398rMk3tVztsCjO1wTkRmxbyDJgkhMrEMeqnIrCgEyJ4jPX7r4NRz8s20ozshC2KApjXkSwvJ9abfMMyUluWk1EFWSCelxOyORirey/RZ5z2D4hMETEO9kX5fccKyZIH4FjgFWm7Q+OpcfsLkHGnInKTMJK+JxanwKovO3FBEZ+PrLJN2K9mBvNDoDR1zLC0Yyhuo5GgX88u4BEEaiIE0QUchj4zLyjVXFJRxMYXT93MUcsq4Rrsddb8jdT/0Gtf9kCU33A2kapVDgkAc0nEjGQyGVrByXBbI42rwFo5uG/T7Bm1sQ8XY9rWxrKJtVpekYKlVCvZIBu87SLL2YwRSQpepj9nS9eFei0ymN2JSNLG+ifrghGhaiLN5WSEax8xSjIimGWtAb7FSc1YZj1rJVovyXoMUTS9hcqWMsVLhX7rWl9kzo0trVGT6wzWIy+loICLDsmg1syjNOKwljy2P3kbkw32MHSEONwfSHASsFZBYNWt8dH5Ar6cXjc6bABoUc+KfX46Z9GfaIFbyw9e+8YOmYLPLvtgM5LAZD8xB2x/1wgVohuVxXMgZTdF9F5kEaFYAmnHqz8ZbHHWTIynBEqLS3k2NAkW0QQVk7d3EKFAOAt5EWF/FOLwjsjKn3kb0Ath7sJOsihGdXlZbw2pNWNA8IMzkQZjbMsgaELbVFj4u2gVpzoCFaFElZ0uBYVbOlinErIIQZEtCKMwLkTArmvXFoZZRJC0USaMIYihMkT4/EFpf2jKwI6j/yPPn1UuZHFl0VmYZAi4vFX/W/p6J24285sx6i8gy2lyFLZvOTKN0T2Qn6Gk2cAxBTBSfsMzHWG7lR2yQmGNggx0QcOZuiI4tmh6MBaSxlz6ZWQFZMBZtb31YwI7NhuksIF2Ymwt5HQa76orBjFibRi3XptEItmm0YiFgLJK3CznJ22WlWXJhIWCcjRoBUHCqXjKmkSKuhiQoZwFtU3yxc5GCrbxFBVvtuVfQOltGTNfB6p6Wgx1QOOrF6UKyQlpAWyHbUvSNJZhH95yZ1fgLCt0ag3UNtsvN+95AQVYdmGpo1QgHD7BgaPyePPnCRkrMJWaZHA5NzBXZEJYWAohy/kOxCejqmJxWXBkxmZiX5VfdjYJmEeuRgvLEkFsf+VlPerVzcchcaOjKqmxufSHs/NlPQ5nQVTIGkXFFH2eWX0VEw8PbKk3V+Vyp7lkxVwRNKrziQlnSwrkrOQkbhWb93pVm8YTDd84UE6RZgl5IxZMOfyHWuZGAs3i8kLq0Qma2QnZUdXVLUx+dkfhTksRQrEMxlKYiPRVXUIzUn7jPkJYAaOISqC2E6kl5OyG6q4NbtqDt0kOrtsse7bY0YG2o2CDPsgdPt8Scik4IbnknBLeslFMRpOAWX2R+ddV3jLqvVgOXlhKUSZlxp+VkrfQD6jPkZldmibkUpUx/NGI2LGCV64PgWhGBe+zedKIWtLVzV5m8PD2/jDCl5euU71i40zmqInIsS7mGaElB/XUdCrLEKwBitncbqXOpt2wC4tip1XPcKzpDW+UVGWRX2dFTZ75Wx3XySnRcSo58WUEdl2CePsjFgMkF6lvBZo5GgHxx+hlHH3rqx57EigDFoyY2dlJW+4DKam9ujBgiakS2MZNrdDDUHQ2krgeRvijalHjjGw/1zkPxOyYgD+5ltTo25krCr23GW7Xwa2sIZpZ+mWVGgC0eqwUprCYVs9+c3z+JHIKUZEnK9mj4fI8LCdpXvlkqxWLvFVuQirJwkpTvUTgDrLtsTKptpxFCYIJ9OaXkaH4muuZtrhfl7ShlhEs2FiOg2xRujBHHbHFguzet6R/AVg36LGsPHGzxWW9+hH3MqCFDQL3tRPMpyAxWQVYzv5fZJpgDRiB4t7bYhGMWSKq12iyw09ciG1MsYO5UzmB5JpfYdY591yCmGmMyQVqxLDCRXpScFsjRwUk5Oin0JwKhZePeo5JkvxssYlB7zyiapguImq5KFZwecCq450cBTdCHV3O1Tc74HKec5CSuiWg1ML2w69CYL8jGQmMk2LaPxbxxYGxtJBdCt2xzjjZ0byNpQAhwKTHd2OJlY7T4LSfHt0Dsm79ULL8S6/hIMunSssiev+k2wtU1251lTfZUY+nOerfmXg44ctaRcLkhkLsK0nQ9+XAvdGs99YZRRJa8RZflCr13AfBDZWsaKdyxMhv4pUO5+FT2VVvHSKfJo14xwS7QujzPtrlb5FlX1MWjXTpBowy5wC8e0sWD/np3i2SSoHVvF6klAr9YPZLJW3T3vGqYMtrV6PvaUpsPugCEp1cbmkboEamq9Wjj3Z+y/5PKBMz0tMezja6HNRfEauw1FBRhHnyTIuxxyT7BR7uAULDnCMzWB4qp/+atJaXSyBRoMfQmlUempPLIpBybIBpHpqSG09t9p5wUX1bQSnymTZtOTmQ3zGPbbJY98TTi7Nq3UuyFDccmZmpZYUfytr7DAQGP0LOccuDN2lQMnLeMtk3QCcuC7FFNA8S8cdNDOV5/6wwhls4CuJ1L6mTpfZdQJtemd6iB8zWbtNEgHzVnNJkdDHKcq4ZWk+ezMze5UDJB5GZZkJvmeBx4Z6YWJG488jIW4dpVVQ/LCtx2yZmvr1HAKHN2kpLjwSOS47N7OMlg6AETVD91WlHcftGgKzX0UPQOWaV6HINBV1jVj8MmRBV7VI/0Eh7p7b0jVYO2UABt1sdG0Hb89PnYemFZlWZQT8oaQBkcPChLUgZMIxVsZqUcmOCkHJhWCJMNbJiMlCjvloAOSLooMT5jxLR6xvaQRnXUpkm5L50QXA1LWOWdF+Pi1GkmmkAIWj9mFac5iPjaXMo0Zz3JKmePuOLNbPKkjCtaqn22G6W2iG6UVBx5ke9lGhG9TOtD3PL9KDVQnzRxWV9ls9QcKkutVpl20ZWSMQXKozkZiGEyJhOmEnRhyvWiZPF3XqLFYzydKK2ELu2cEZSNTjD8GUZyxGlqaJRqiKganRXY04IUguXp46m1RwZS2H1Lttf4ZkG9W5BC2RwylK26Z2hE9QzNxa/VobKYR2XW6hYr5hgks2MyBf+jKKsDZcpEKVSmLByem9MemJvTirIyFuXd/kDUSYVWbFQ4aqFOlHL6Ri0UwIXT1x1OGwAW9+IB6tRIPJL00eRYNfKwzHukqOvuesHl4Cwmsz+71DBhMjtotN6HswGAGQgyI8sW+29DoujX2PybOiGcjLdsYGzn9ozSWdCQXNa0adxQHOrtjpsGqvHY5fx4jMe4OZGJIBfxz8PHBJtb8jyQC/TDBjEdGVPEP0jElJ2yKsl0Qtn1qukYIp9+4oAkadQ8WpxUX2R0+tHOPwqAC4mmgKtUjWnCaKoT9a0migdrTbTjI4x6iFUjsVRQjxkPDeoxuzHkLiq0vyXYzC4EaIEVA1piHknwQjxLLH4/CTEPw5dOBkKBTRRQZowYKKO9dN91zrFqSM0TVBCWf2kpcZLhINFyafYkCRgcWJzaXPKvMCQEXOJXfhkNCEESPf8+q/zKIq3nY+XK3F/fzVyxmWZN0awl0gAgigITxQoxIp8rb8x+GCJp7XJZIXfv27k5bYjZv+qLsKwYwrJdBV5WjF+BmLqLDoIu5LqBXorZZPGxY3ZJsWOm4BWc+LCTmg8CSm7VkuI/PoXw5xCJXmkUiGupMTpqaIcNpc6QCGOph1ejI4x2iJEWAODWMYPZ0emCgis2K7huDxJYZYRfr0G95aRULV5KuyUl3TIHptziI11ij19L2bTEViIIOfd6tzow1QlyTcotOQ+e7myfVF2Y0UyMy9SG9HVkXDsXjsLPOjlbTFoTEVKbFSvfAk0QT3Em75ukPSYP/uFHooyoy7qyXmzFyLrsEJFriSvuy/hCJ4VNWgl9+Wb1WrHskt1M5/PBUjjX7/NzLrp86EQuv793P0klGLPP02HzQk5hdlmVzFUNu1SRK87cUXL0kad86EmZWJqQiVULPKzDKKlW9a0CRmct9bSlDMVElMCpgQ6NTm/a+U1ShEX03EOtTs+idHr7e2HWAS9TitE39cBrQwDnyfQyDan7BlN2Xn1WEAIFzT6sSXWWkUJWUm5DL8SsFJ/dUIk12VQkJ6Nc70gxTVmrj85yiNUsTXJk+MAVTezke+SNMXIgdXCoDaQkcLOlc9lUTlGyR5zcaue2kvmBJD2v6XQmFVpBnWUQSp4VZ7/IYDEhYOydIrNVnwbKxKyGYBDqvjo/ZVaZBVkbp9uZz3p74clr6QupBJkVUyw7ujWkZkFWWq4zJEu4/qUcZ7uq65Q5c/zW+HzEnpAUogAFs9zkrkKxlhWSo/Zs0CQogrNI7gtxwoeUWVyoV1sjyyEYlFqu1btoSs9s4m7QUWNWx6qgwKpigqYk+tWzXqWLidYq0irAaMLmpFXgpExWQQhWiUVjJSFWFYVYVZJ1EoKU9gfE5EosJlFlhDpEin16DKsoT0ZfZZcAfYyUxk1MuKXE3vIhSKwgK/95SCOlf9XPGwj9kC1ZmxKVVVjt3I4mr7g16OYs+2wHXBfLTbQsF6pyySLrrhgbKgJY5LxyhWKBAcziJROyHKlycUA+23X9SlB12hVUCWqreLxtaG0VD6ViomudU+KtGKZiyZ+/QovYlhQSP9rSsJsayUToB3SJ2tSo0Z6F37VIPfoGQBd5cdyRCnlEgnoL3xkdn5QDlJRwnzDNAdqo2+iExJ+R1ZjK5jFVCrqBUm3ernt4NLyUyjRoqlw9pVqIpkos8cokIUol1ewwCkGqIASp+FodkjQm0MySJoFpq7pPme7CmKHxLeve9Msk4bQrxSEsXISaSkycFMTiuawQppqt22FAEA3GlKsYA6IR3MOtyR3RA7dzrMrZhOqtqCqCvnO4audqlrkpbq5eTKKqGMpNLO91/0yQyqWER3JcRkDrMB0VmWPbdbaX4e5I6/sZnvV1AYJY2hVPvlP3foaCpEqx+Ol6mwBZssWuqmRgM3sAR6cG7dwg4ZdCB7vJnYXUxc7huti1jCSgKE81pxodp/gDldJNMAZUN8HJiwqJiiUK36tkYgHT9XF0IahmVa4gqQpON0uq7OM7Xq9W3sQrqE+8aiBd9uDlWDSlihWS7ZC8g1aIdAFI5V0ZIdYFWhR2KY6pJREp0yyu4QC6JOuglGqn9Q0rnNlRAHWZzm45KyZvUlIRUqHxUSjTOdcdZgJd0UcMeXq8BXF5B2NKaN5F1yrlaFdCCNA29ycm76AtyLJ2G2SvqsLs8z5Cp4vIy1aN+iJL2vSAwoqbxXXMFoTlwBAq8yEwEw12yC9bcFIw2Plr6VsoiMH2n0N28WFYcootkDMW2r7GQrskY2E0AxKE7fnYU3gIGHSdFn9eTInAeFqUVQmlrHpxNpC8hk4jHpptLDO6Q1FuUSRIWVS8FW8SJDamB9KCe/n86iiZL6XCQ0sq/OqR4y5JzgWHD7lASjdipTyHUSogS6p9YXNW2KTfyrNBLqAsGi/nDpSCXFqKyzXzCuiuJtKyvkMp2RVLJ0AtldnfN3bLdJa22bnSsbR1CK3Ry2/lLYwLtENZHfnYlokDmqo9/CybS8vlbQ5x3T7+cH7FpuJqAC1Noe4REJFc1CnNarkiAGGofAlZHlJxRl8OlUKxLuanWIKR7kwUi6Ut4pkYaDrHMhsehMWDItFKLiUn5WoOx/qdlWEBkBhWrdNttH9id1BChdFBhDmKSEykE+wbX1jwVxbKs4KEeFYts2HiQGCJL64kNJkg0qXJk441FKRcujnJfSM3fACGrFIuJdq9sIFzLT8J3ogZpBRoKT2XlkJdXgp1aSHUxUe6NCWwvdUeBb0jmFS75McyOBdpLQzFyNBcei41CImNlOnyqBj1XNAjPE3uE9SRdTlEbPjmvkXznOXD4AtGt+0Pi3evNVzcKxjkmB++vRguaRdEdHw5m55rMInUvJFP05UPTV+NfuzM1tdQYzLi61bVVX5JA2lJm0oQdjl7/BaPmkvLpcSDXPoWi3bpvC4UvR6EWTEQpsUsjUUuOLOlcbTR0LYaipILFYC+94ymedsMusyLI4qUex8JSeuVNXzRDVi4YJBaNppEbNlYLVDzCYjjqhtTqTdB5vCuZmShwMgs+Aa7ozu6nbHf4XSAvGuI9VpKgryYZRGk5Fw2HZqcywohrrn6HYIViqRSIKXmgtSquppcjlrKf9aqUoJ6p2sb5JJKe2dhXIsQoJklrJn6QLGOjCulgAziPmaN6TKQVcXYo7pOdVm2BaZckrnZoS505dtevNwRW4a8drhuh1z+RACPmspjYluALMpy2mO6dLImyO/es/cUHd+j6xjW1WvpdcjSM3ARnQ6ZDIm6qyFRCToSFY8lsZgKNzfGgoKuZ+KYomALpz2um2JLM0IwmHEct2RmpYCOK2+pMz5s8McNyR9YKDK505OEacW+ihO3Awog1Q7bIRKFqX57/FS9/fHPa/DD+//7+ec/v/36+X79r+8/fb75ck+sPgzX7o9wMwzRxhtIOn64+fTRfLoOH5P95AHApT/+8DcaDHz49MmA+3QTP944Z00c9B/u09vf/uu3/w+luxEXdY4DAA== \ No newline at end of file From 21f138fdf7f65f605f0905b2cf3929164909ce1d Mon Sep 17 00:00:00 2001 From: wms2537 <51080539+wms2537@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:02:56 +0800 Subject: [PATCH 10/11] research: freeze neural confirmatory preregistration artifact --- .../frozen/neural_confirmatory_preregistration.md.gz.b64 | 1 + 1 file changed, 1 insertion(+) create mode 100644 experiments/jumpbench/frozen/neural_confirmatory_preregistration.md.gz.b64 diff --git a/experiments/jumpbench/frozen/neural_confirmatory_preregistration.md.gz.b64 b/experiments/jumpbench/frozen/neural_confirmatory_preregistration.md.gz.b64 new file mode 100644 index 0000000..5482f2a --- /dev/null +++ b/experiments/jumpbench/frozen/neural_confirmatory_preregistration.md.gz.b64 @@ -0,0 +1 @@ +H4sICJYphWoCA05FVVJBTF9DT05GSVJNQVRPUllfUFJFUkVHSVNUUkFUSU9OLm1kAKVYbY/cthH+vr+CgBEkOZz29Lor2Siai+u8FInj2gb6oWizFEXtMpZERZTufPn1fYaktNo7Jw3QL/berjQznHlmnmf4jP19avuvZSdO7LWcBt6wl7qr1dDyUQ8P7M0gB3lUZhz4qHT3nL2N6AH8PYlR3Un2IxeDZrfi10kZRY9sNldX3wz6N9k9v7picRjvgjAPouKa8XqUA6vknWx038puDHTXPDDBG1U684x3FStlrQeJj/jpJMWHXqtuZPfcMHnHm4mPsmJ4dDxJJtahtrxTtTTjdrN59oz9Y8JHG85LDrus4d1x4kfJWl3Jhg1S6DtEg9/kRy5G/CvFNPKykazXzUOnW4VctPZ09aBbhiDkcIeoKU55pyrkTF7biCstDeMuH/MvTLX9ABezpwd8aLh9ZNRswGuPbeqB9dwYekKXBt+7lNyfpD8s3lEVjs+OA29bPlyfnZVTdZSji8YIPaju6IzXHL9ypLO2JfnrZvP+pJDJj70cFNWAjciTobK6s/JzJWHBwL7LESAwGz3KIVjnSI48mNO7Zd+PLh+ddrbZ1A34b1CCCvc2gYnViTuLueBequNppHoa3dAh8bOr49/OcLm5KLeRPXeo2WzeoED9eHH6FczYZCxkgLXDyM0Hc+O8/kx4+nmBzS9Gdwdrg7NaT0NAzzIzoRbjlr3/PbgxpPPS7PqxR+afbzYBO8pODvQrziCr5+wQ78I8KvbR4QV+TXM28gHVZDVvVaOkoW/jlLWqU+3UBrfv3gdCmzEobLCf+iWKvA377m8SdfU2CYoN79m9Gk8WVWkeWD8PFxmbo/7D13nTsH5QdI6HtkTdxIUNCu6TGbNBuYKf+FCtgA1YAisGNTUMAPVe6XHX7QOaaVw9z/te8sG4FsYRjPrIRsDaAGRMcsw03pl7GOq1h/RtEHxn3fs3dIdu6/EOgPzrRG3KWw34uPBWbSJOWgkYxmxC29GD5+DI7aplz96X/kQeKuVB/eMMHCORkRFZE7pt1UhugI/D4TDKj+NG7Hha7uVulye5jIswF7KuoprvqyKpsziO06IsMxnGUSzqOorTWuaVTNMkyndhmdZkaOWsVphs7767DeJst/JSl3UkpajSeJ9k+4zzfRjuo90uLuswEVVUhrHI6qyo6jgrRZWJZLffZWWdp8kulkXsvKBP3ch309XQN8/AHUirrfuIQY7Bzg2GK1cDNcGhUePYyAb1QzfcXPwVhNtdGdDjX+VxLOuEFwiLF1lc71OeFHFSplkeF1WWlJXY70opD09NToBegxEoq5W9ugzLMs6SYl8JnoWJjGA7jcpY7mUe76J9WIkszqqDO8I7SZVbH6Kn/gIbqo7++NOHOQ69DmDjFFhMjV8lYcTjOBd5grLWKXJfyDyXu2Kf5UVZyCjnCCiO/8TBntjGMEmzRGZJta/yKglTEcoi3/GMF3XMK3zYx2kiOA552xAX2vagFqWeRQfq5k7ShzvlerGdgKBSskYfj7Lasm8HDC6aGEJXlhWMm7HUHDV+C/ChZXoa+2k0W/ZaO1wEppdC1UB87wd2xfvR0RxMYJ7oe5hf48lTvkaaX80fGdrPWDFy+AXipSTxEngiuQu3SeC47gBD7zwbkHsD2EdbdnX10gsOe0Shj53rTKiVV9S0ul66n19Mm/V0QrjENDABXcKpv0GPk+MtyxX2V3psaX1LleDSUX/AwZBKykLJS8z48YFm+GTsmDO8lRcGPzd/9OLUVaRj7BukqSj/Pr+IRPdB9OQgNJRGelc5j7KrrMjabmLKzzdkwTx0+MkoQ3mhA2GOAwTgXCOtSgIxYmhREs958edFfeh7WNQUGO90p6DyWKnu+KAQwVo+CC1rQEI52um37N0kMGgNagMlAsM+/GODMzdL7NdWYPAjYrXz2SvCtXI6DgpgSuhIrx4Pclttrzn0ABBTXtZJN1Y1iTNUOAqoj3oyzcOWvXpEH+7oM5G4klA4tUcxeQpOD712OZ0RzEwPfbbdpD5GUKgi1QQYArUU48ulaJCMHwjK1KK2xhIn8ZqaXHmGviyGmXoYpegJjLMgvLEeZgDRqQfdmO0mozDew5Gp5TDXfTHh9KGtcIuBYSUV6VwnEtVoZFPb6NZc7fWijQUDzGrANcOuEnyRfQyNqUXJhK8XtGLlzFQSpyDFA0GKt1GEQY/WQDlz8gISTAA/R07cnOwEmIkvSpOsCEMhxB50tk+ikIM0yyIORbmTUuZZJqOwiMooDcG0oJmIZ0meVnu5F0WeMbYSkdv+YRNi0NZ7LkAmoYg49Fy+T4u8rsM6C4s4rco42idlmO8xlZMsB1VHUVLh+zApipSzWRfKn71lKyrJ9A68kMpwtxdpEYlURsWO82JXhPu8ynmWRmEeZsUuq/YIfR/VVVXg8awo4qosirpkTuTf0L+z8UWXwr7lcNvjE+ig0dSrVrqbSRHywA+2oaPYbSiontsa/Kp2ofLcIrXo91dPJJAhxry6urX7Eu2Irr9b/tGKWN8agW2NQBI2+4fLTclgMlibc8Yq5pY5RB9fgzTnDWmRWAA3+Xz7dO2iAEi/VXbBQRx2EgJmA7/v3Obn8Nz2DZDHDv8Kkuvk3/+JD3a+PO11DBq/OUjiMnL7xq11f8IVrEfXkbX+vwzbgi3CmyoXzKfWAxeNnKe7mNf1AEhVleWMntR1qeHjBTqXnqRpKvDFOFOaw4kVbQ3EjqvmLOlmxjAuDBuWXyO8V4Xwezxm9y8wLPvi0P0lzQ9fbtk3JKGJaXtvzQrD867veHo1GR7xV7BibZAiGO563r4JPKBRv0e/ID77f+x8Yk1HHqxKsFOTG7tcUrpICsLFeK8Dmoezsx/Fa0BwuCmVpztqHKcQaMdkPRVlcntFa+nkO920gR+fJKlsVrfsrW3C2Q3KJxAPFu7AXY9UqsbMpqYwbgbbQTivdv5xrABBqfVItzk9K7LP/Mk4Df/N7VoNwa9qJqdg7ENgFQ+NeakJlo6m71fXBvOrduNW9UrULCX+3F5BWLpraGpPjgs+VRguBIAoLLCMwrcQC+hpvMZLul8BkdEo+OIQ3eSHa0rhRQHmxF/bxDLiMOMQavN/Dsl86Xb/kUFno65Z+JnD/MV9GGSkGFTvrnrOTeAXSid+F+nEBrukWor2wCIX8y3P+RLJrtNUj3+qxtiLF18W2i59yZ2Bm/nlR57O9adXZiFy3l8DT6NLNj3CffZs8txe/Lgq1pHVGvaIHcTFpx90ssBLXVqRrIxwjfV77lpZKSjbVY+SzLH7s92xF4jRw9A1U4N56S4ja3VB+XTvwYrZTxT5AWlnnKxWO5NP5AXGlmamQiinJJdt1WmXeQ90BbQVWZy7fv727Zuf3AsgTMlHu9CcQeKmKokztwTBwNkq3kA2rfShgWw8v9qhwOerLjcb5olgsfn93JpO8Q5TIz29vlrweJby4HNIa1kZy0Qg1u5ILL/cUtD6JtaXu2+jdWN7Dl3lbbbHyumJL5oC1tFinpQkZtisBG1zWym4SEykAfmhwWaFwrVnpVVEK3nryX8J7L2D4fB78S1xPB5zNkqretRv0izjC8uL6nxTOXa+uOOB65MlaIvfeXJRxbjrhUGeZGc+mTd/N2z8PdrlzbEtzpwqfr5rDs5ps3hV5eTvoiEOyJB+UgTn+WvXAxa6qz5YjT60CeKgg9OA60n0LU3tlgMKigoB6UKMTu6ubVaXu157omu7iLatndtnRoD5sUG5xAcX0U/EDBf3wutgfNUu8jCLCdoEOuoiewUDJRrABvZRZU5P8EPbYldTXxN6UFKPE79e3Swjaj6kzQC57fgw6PtZPYlmMk4irO7U0Sq8X5p2Bs0L5MMtpqMrkLRG3OQiA2RO9uOE2C46a/MavzVctUgtpkYpoYcrepnP9xAXK7M14lZUXtKBHUDd0Fxlxw7cTjbL/TWds0GmfrA3Sj+4GyULV2zmxkHL2LfB0NUk5kSi77TBAcFVJ7pJpFPZq7CBrqWMvXEQeIEk47yEwRC3gmSx6Y9i5SIFg0nW+Q7+L8gplH+JGgAA \ No newline at end of file From b0708a64315bcbd5ab0bcdea5d9cc1c751d4ec8a Mon Sep 17 00:00:00 2001 From: wms2537 <51080539+wms2537@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:03:42 +0800 Subject: [PATCH 11/11] research: run preregistered 0.6B JumpBench neural confirmatory study --- .github/workflows/jumpbench-neural-eval.yml | 52 ++++++++++++++++----- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/.github/workflows/jumpbench-neural-eval.yml b/.github/workflows/jumpbench-neural-eval.yml index 8d8a9d4..2bd1603 100644 --- a/.github/workflows/jumpbench-neural-eval.yml +++ b/.github/workflows/jumpbench-neural-eval.yml @@ -10,10 +10,10 @@ permissions: contents: read jobs: - neural-dev: - name: neural-dev-${{ matrix.label }} + neural-confirmatory: + name: confirm-${{ matrix.label }}-s${{ matrix.start }} runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 150 strategy: fail-fast: false matrix: @@ -21,41 +21,71 @@ jobs: - label: bounded-base-0p6b model: littlelearner/littlelearner-0.6b-base revision: 822ef3a9700a952f74a3923b45829d53bdc76bee + start: 0 + - label: bounded-base-0p6b + model: littlelearner/littlelearner-0.6b-base + revision: 822ef3a9700a952f74a3923b45829d53bdc76bee + start: 24 + - label: unfiltered-base-0p6b + model: littlelearner/unfiltered-0.6b-base + revision: fb0bb25397dca503e1a9741b2e7e826170dc525d + start: 0 - label: unfiltered-base-0p6b model: littlelearner/unfiltered-0.6b-base revision: fb0bb25397dca503e1a9741b2e7e826170dc525d + start: 24 - label: bounded-grpo-0p6b model: littlelearner/littlelearner-0.6b-grpo-math-expert revision: 301a228c83efdf45759e88e697589b9e18ae8222 + start: 0 + - label: bounded-grpo-0p6b + model: littlelearner/littlelearner-0.6b-grpo-math-expert + revision: 301a228c83efdf45759e88e697589b9e18ae8222 + start: 24 + - label: unfiltered-grpo-0p6b + model: littlelearner/unfiltered-0.6b-grpo-math-expert + revision: 819453e53d7d8d304c0e986a5a9f2ad6a57243ca + start: 0 - label: unfiltered-grpo-0p6b model: littlelearner/unfiltered-0.6b-grpo-math-expert revision: 819453e53d7d8d304c0e986a5a9f2ad6a57243ca + start: 24 steps: - uses: actions/checkout@v4 - - name: Environment + - name: Materialize and verify frozen artifacts run: | + set -euo pipefail + mkdir -p frozen-runtime + base64 --decode experiments/jumpbench/frozen/neural_eval_v0.3.py.gz.b64 | gzip -dc > frozen-runtime/neural_eval.py + base64 --decode experiments/jumpbench/frozen/neural_confirmatory_manifest.json.gz.b64 | gzip -dc > frozen-runtime/neural_confirmatory_manifest.json + base64 --decode experiments/jumpbench/frozen/neural_confirmatory_preregistration.md.gz.b64 | gzip -dc > frozen-runtime/NEURAL_CONFIRMATORY_PREREGISTRATION.md + echo "1435900ccc75bf7310ab4fb920cb6eee855e1091b14049bdc71a5384d7e7c985 frozen-runtime/neural_eval.py" | sha256sum --check + echo "fbf1eecd4273575aa70071662bf03cd1b02c5f59df25bcd5c36765bf84362e92 frozen-runtime/neural_confirmatory_manifest.json" | sha256sum --check + echo "3d7e0c8be15d1e44624e156f6354dfe6ab8899abf5440d9daf000644461271fc frozen-runtime/NEURAL_CONFIRMATORY_PREREGISTRATION.md" | sha256sum --check python --version free -h nproc - sha256sum experiments/jumpbench/neural_eval.py experiments/jumpbench/neural_dev_manifest.json - name: Install CPU inference dependencies run: | python -m pip install --upgrade --quiet pip python -m pip install --quiet --index-url https://download.pytorch.org/whl/cpu torch python -m pip install --quiet "transformers>=4.52,<5" "accelerate>=1.2" safetensors huggingface_hub numpy - - name: Run development-only neural acquisition probe + - name: Run preregistered neural acquisition shard env: HF_HUB_DISABLE_TELEMETRY: "1" TOKENIZERS_PARALLELISM: "false" run: | + set -euo pipefail mkdir -p neural-results - python experiments/jumpbench/neural_eval.py \ + python frozen-runtime/neural_eval.py \ --model "${{ matrix.model }}" \ --revision "${{ matrix.revision }}" \ - --manifest experiments/jumpbench/neural_dev_manifest.json \ - --output "neural-results/${{ matrix.label }}.json" + --manifest frozen-runtime/neural_confirmatory_manifest.json \ + --start-index "${{ matrix.start }}" \ + --max-tasks 24 \ + --output "neural-results/${{ matrix.label }}-s${{ matrix.start }}.json" - uses: actions/upload-artifact@v4 with: - name: jumpbench-neural-dev-${{ matrix.label }} - path: neural-results/${{ matrix.label }}.json + name: jumpbench-neural-confirm-${{ matrix.label }}-s${{ matrix.start }} + path: neural-results/${{ matrix.label }}-s${{ matrix.start }}.json if-no-files-found: error