diff --git a/.github/workflows/jumpbench-neural-eval.yml b/.github/workflows/jumpbench-neural-eval.yml index ca329b3..2bd1603 100644 --- a/.github/workflows/jumpbench-neural-eval.yml +++ b/.github/workflows/jumpbench-neural-eval.yml @@ -1,9 +1,6 @@ name: JumpBench neural evaluation on: - push: - branches: - - research/jumpbench-neural-eval pull_request: branches: - master @@ -13,45 +10,82 @@ permissions: contents: read jobs: - discover-models: + neural-confirmatory: + name: confirm-${{ matrix.label }}-s${{ matrix.start }} runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: + include: + - label: bounded-base-0p6b + model: littlelearner/littlelearner-0.6b-base + revision: 822ef3a9700a952f74a3923b45829d53bdc76bee + start: 0 + - label: bounded-base-0p6b + model: littlelearner/littlelearner-0.6b-base + revision: 822ef3a9700a952f74a3923b45829d53bdc76bee + start: 24 + - label: unfiltered-base-0p6b + model: littlelearner/unfiltered-0.6b-base + revision: fb0bb25397dca503e1a9741b2e7e826170dc525d + start: 0 + - label: unfiltered-base-0p6b + model: littlelearner/unfiltered-0.6b-base + revision: fb0bb25397dca503e1a9741b2e7e826170dc525d + start: 24 + - label: bounded-grpo-0p6b + model: littlelearner/littlelearner-0.6b-grpo-math-expert + revision: 301a228c83efdf45759e88e697589b9e18ae8222 + start: 0 + - label: bounded-grpo-0p6b + model: littlelearner/littlelearner-0.6b-grpo-math-expert + revision: 301a228c83efdf45759e88e697589b9e18ae8222 + start: 24 + - label: unfiltered-grpo-0p6b + model: littlelearner/unfiltered-0.6b-grpo-math-expert + revision: 819453e53d7d8d304c0e986a5a9f2ad6a57243ca + start: 0 + - label: unfiltered-grpo-0p6b + model: littlelearner/unfiltered-0.6b-grpo-math-expert + revision: 819453e53d7d8d304c0e986a5a9f2ad6a57243ca + start: 24 steps: - uses: actions/checkout@v4 - - name: Environment + - name: Materialize and verify frozen artifacts run: | + set -euo pipefail + mkdir -p frozen-runtime + base64 --decode experiments/jumpbench/frozen/neural_eval_v0.3.py.gz.b64 | gzip -dc > frozen-runtime/neural_eval.py + base64 --decode experiments/jumpbench/frozen/neural_confirmatory_manifest.json.gz.b64 | gzip -dc > frozen-runtime/neural_confirmatory_manifest.json + base64 --decode experiments/jumpbench/frozen/neural_confirmatory_preregistration.md.gz.b64 | gzip -dc > frozen-runtime/NEURAL_CONFIRMATORY_PREREGISTRATION.md + echo "1435900ccc75bf7310ab4fb920cb6eee855e1091b14049bdc71a5384d7e7c985 frozen-runtime/neural_eval.py" | sha256sum --check + echo "fbf1eecd4273575aa70071662bf03cd1b02c5f59df25bcd5c36765bf84362e92 frozen-runtime/neural_confirmatory_manifest.json" | sha256sum --check + echo "3d7e0c8be15d1e44624e156f6354dfe6ab8899abf5440d9daf000644461271fc frozen-runtime/NEURAL_CONFIRMATORY_PREREGISTRATION.md" | sha256sum --check python --version free -h nproc - - name: Discover LittleLearner model repositories + - name: Install CPU inference dependencies + run: | + python -m pip install --upgrade --quiet pip + python -m pip install --quiet --index-url https://download.pytorch.org/whl/cpu torch + python -m pip install --quiet "transformers>=4.52,<5" "accelerate>=1.2" safetensors huggingface_hub numpy + - name: Run preregistered neural acquisition shard + env: + HF_HUB_DISABLE_TELEMETRY: "1" + TOKENIZERS_PARALLELISM: "false" run: | set -euo pipefail - python - <<'PY' - import json, urllib.parse, urllib.request - queries = [ - {"author": "littlelearner", "limit": 100, "full": "true"}, - {"search": "LittleLearner", "limit": 100, "full": "true"}, - {"search": "littlelearner-ll", "limit": 100, "full": "true"}, - {"filter": "arxiv:2608.13545", "limit": 100, "full": "true"}, - ] - found = {} - for params in queries: - url = "https://huggingface.co/api/models?" + urllib.parse.urlencode(params) - print("QUERY", url) - with urllib.request.urlopen(url, timeout=60) as response: - payload = json.load(response) - print("COUNT", len(payload)) - for model in payload: - model_id = model.get("id") or model.get("modelId") - if model_id: - found[model_id] = model - print("MODEL", model_id, "sha=", model.get("sha"), "private=", model.get("private")) - with open("hf_models.json", "w", encoding="utf-8") as handle: - json.dump(found, handle, indent=2) - print("UNIQUE_MODELS", len(found)) - PY + mkdir -p neural-results + python frozen-runtime/neural_eval.py \ + --model "${{ matrix.model }}" \ + --revision "${{ matrix.revision }}" \ + --manifest frozen-runtime/neural_confirmatory_manifest.json \ + --start-index "${{ matrix.start }}" \ + --max-tasks 24 \ + --output "neural-results/${{ matrix.label }}-s${{ matrix.start }}.json" - uses: actions/upload-artifact@v4 with: - name: jumpbench-hf-model-discovery - path: hf_models.json + name: jumpbench-neural-confirm-${{ matrix.label }}-s${{ matrix.start }} + path: neural-results/${{ matrix.label }}-s${{ matrix.start }}.json if-no-files-found: error diff --git a/experiments/jumpbench/README.md b/experiments/jumpbench/README.md new file mode 100644 index 0000000..225877c --- /dev/null +++ b/experiments/jumpbench/README.md @@ -0,0 +1,3 @@ +# JumpBench neural evaluation workspace + +Temporary branch-only workspace for frozen neural experiments on constructive representational acquisition. Results are uploaded by GitHub Actions and are not part of the SciAgent skill release. diff --git a/experiments/jumpbench/frozen/neural_confirmatory_manifest.json.gz.b64 b/experiments/jumpbench/frozen/neural_confirmatory_manifest.json.gz.b64 new file mode 100644 index 0000000..426bc3e --- /dev/null +++ b/experiments/jumpbench/frozen/neural_confirmatory_manifest.json.gz.b64 @@ -0,0 +1 @@ +H4sICFkphWoCA25ldXJhbF9jb25maXJtYXRvcnlfbWFuaWZlc3QuanNvbgDtndlyHLmSpu/rKWS6JIU2wAEEgLkr7javMNYmY0nUObJWSdVaapJzrN99kksyk8EMwB1wR0Syu8zOUlrSEwEkgPz4/7//67c3b97+cfP1wz//vP7+H2//15u3//vXn3+d3P3Cm683v75ff3lz/eE/f33+8fnn529f3/yt/w3evrv7O/+4+Xrz/frnt+/v/775/mP9e3d/94/rL9dfP9x8VN/+uvvjP9Svr5//89eNWv/n++39L/7560/1t3l4iR83Nx/XfwsGHU0K5v7XPlx//fj54/XPm/f/+H795/o9vf/x+f/d3P2p9T/3f+Ln9fd/3Px8/+Hbj5/vf35e117/5v9Z/8abN+nd/f8Ys/6ff7//s1/fP/zpuz/j4v0v3fz9+eN6dDfv//j1cf07698YHn599eHLr483H98/vf6vr3e/a529//3vNx++ff+4Lfav+/++fz8//uP957uBvD3V63/evtv+zv0r/fXt25f3n79+vFmt/4wxg376Ax9vPn3+ev9g74dz/9vjv32z+uv7zY/NA7ZHq6PbN+rN7f1/r/bU+nL79dufn6+/PL3Ru3+2/+/Nm6fyd//A7r8o8/Qv//5u798176ZeiPh3n/2L3f7V30Yv8TRd6+H86+mPvb3+8PPz3zfPhvhm5/fv/8zd07bvnv/a7frX1PgX/77+8uvm/jfSzm/817v8S8Oel5565UB4XUt4XeMbXximXtg0vrCZeOHWN6wn5273hfctxbff1zvLtz/LawYIz2noMpzCCxvCilGUidWEiW1+4alHrFyXNb53yfx1vd54MfuM2JPaN7dq8pV94ytPvXDr3E4tc014XUV6FLb1lTXmUbw4t55K7dxjHq9CzxbR7rt4++X6j5u74/rt7293i76dvDc8HIzZe8Fvewa/v+hJtqg3iKKru8KEkqeZkkP0mYrwWBGO7gZ6vK58fPf/CbXPcsPNjhaeRkuteZ59xBAQj3j9v+uylKIX2aIOEEXdEWUhXeYLJuRCsqRHe5UpGgIgpvNhId39x+zUfXEPXV/+198A1l8Lnj7Xez62b++/57z/8M9vn8c31off2bcXvNiTJvaDzPUEuz9PfOhz+yD5pU+xbxp9x5n47ObeNVBf+nzfS+/99kB+6Qv0u6Y/kct9r633HpTUl77CPhFdvEZtPjv3rz1e/OsFs/8vbT5ohU9L7iNP+9DnH0x2P8PvaLQigCrybJ8mFfAGPYq6MYyOc/qBThrO86MUf5iSigwaVeTFTYxSpDAvk1eRvXfS33ZqT1ErU6BWETLQKuWZ1d27vXunq/sZhseH30qunuOkenAF9dyqWBUwVedEXgYo35MoyMsIIS87COEAC0LIC6IQJALHxry6jodlahWFX+hGLjI1A67108PxwpQHMfAhLxqP0Y0ESbHQNBLp0UtYjVqI0jEsmg7IK8YCNLi/SpCAQRZ3xRIweHmN4aFeZkiIync1aaQtR7t8DrQ9FXz4osSGu6IJuIE+3W/ZmFdIQ3HAD8N9+H5AA1E5+hUHS1lXTPjLhzJJPd754vV8ZVEA2ElXALb3IAUeAgYlvlFNwCwLlNlLwHhQ0nkV76kGYMDyPC6xbxpY8FeRvc6Mv0YbDWWrodCJCLQq1bDF5+FR5pAiEZ3n1wzcRYNSYHTyUM8eSqkhoabmxU2CUGN0hFMPccr0Bww8nDzDqmEYFGDYYKFBwrWLBzfzwArDylIsIQkYD0qbVQIGRgqIKeOEuIlKQkhMDUJITInJwAZRGdj03GoxTQ9JyUJ5VsMyxDd7L4IcejtTvdIFwNgkw13EuiHJ10CKiykpgNWsptSoZ9ETjXnrsjBj+zNVRRQrneTLGkLZFRsds1As+/xCxSUIgwE53mOyCi6rCvMaU3hFnt0sIfNDKBZl1IR57VBjvBNqqDeWCYut1zBQ1vALtR+FjJ32JWNWTBvGI4U6Rb9rYCFjuq80zLCQMZZJvESDQh5h2N5JtI1o7IQRjY23ctpmTtLuWEAX2rvDkHgPqtjE2UgblSONqraMxs3SqqmINbQZqh2LQ47lxQlHKjMERJmxRrCOjNk8GRuibQBjYy3gHQRlBWO2gYs1sK2FaL2U80L2RisEtqS4FonWULhWEMJaWpRqcXg1pZaLoi3xtICJNRSgpUxsfRrTLDvIGhz7+xDtItCTEgS4UgI1077b9LA4GrTFkQtppaCL2pz72xOj2svHshCJkWLt3Momf4j6/NbFBLFcQtS1fPwKsoY/d/T8Xs+l7go4kyEXvjLgER+S+0FWQquz2f2MloVZgZidce82zWNn5NFFneM9hzx+Ri1nZ2SySl7VYbwKbHXBia2g4GjM7Go065xHudpG+wqlhEsoT1ut9Gl0ytHOOUKdgDWY1toLDcFeWFdkdOfB33oo2q2IkbwxCbZcQbCVUqt7UT3J2Uwzk2LLzSpDqcUbEK0XMyAmKbmVEcJSXgheNId5TWbhBDYutV8GZKX0Y5Pf1kFMbmVCI04DFnZBMmRKcRzFGL21wFgoTjJll5DXJvUkGpyePbmUBo9gRLv3MsMluEq2ZFpTuwaIzZ2K+BZO86MPxNGvmMBVQvjm7tM8noZPG3YWXqWEqL29FPIgrJRFg/tGfEyqneNYbvCkpUYbdY5pgU6hWPpZ4klDWtd5V7qlxRRZgnRL0KuoOcBZb0UWj+P0Ev3SXJosI6DJuuKEWxo8CkRMHK0UrjI6SGuPUgplQZkxM+cXaXgJVWvPeUWpoiGQ56tOPDU6jKjHEWWaCqSqeOgSao0POvpRV43IfB6ReRfrEdnt9v3ywjGoF2wxRdnPycaiGBqTEmxFqWwuITKmpWyIRlSwpWcdTs13esuBJkmkB6SgGMnGS6KaSRaKiaErzfCpFZtb3dGMKfV+cTizJxFzyRY8VJtL2WMA7v3dgImJ2eiLxTc/DuemYesbXCSiClrxrB0xacAgmsdoY2BCYd7nqm4vf5wgbOc+mq0JXGZEHYYsedsuJ2BlX+s5RRHe+3X8fLwU6nUxf0SX4lF1TSjCWciXEYypZ4kWO0e/a8GYeh7yBSzvup8Z8ZwVfIWhAB0y+w0pQCuvHpvcvEnsSSPR2ss9jFYGkGWOG8qMDh7c0UOTwdmiXS97eaLxpkjmTdXrwEaPGFnmZlYNt4aC/suF2BjYtZkRe8Tcc7GFU9nDz9tKUowrHZon8eCithKf+EvKiqeFJpb0Jb3dWgZCb3nWdn/9Gi4uQpvV142ohdRkva2IajbCJRdOlA/YcuUIJssbPR+dQ+WiP1x+2DotamS+1b1Xg4lkuXLyE9V4mYNYYFxZUrUbA8NlSdy98Ba+Yli2vHnjA+bxPp9NCsO6nF25pXhi5nlymLprt+QaLfIIrC7E7KWXgnbKqzo7ZQXB+p01TstodFDTaAsnRc07h4xOH21lFEoy2qZpGzUpqslhoppa2hM6TBxYSwxYwBWonvDRyYU/u/olc9UxqpBnVHEbT1cZnbWrF2NVYdHi5A1JwLX8DonNhjDgsJpREBV4IUYFUjosEHMo8kEqYqxTK9cElvVohOK+tJCtVVFC20MzWLNE7CAtxZICSl4I+4ip9zqZCFmDvuayJyaNaalH94plORUYX2zNSI09ylOqCEVKdbv5ASNxpFlSpU3E2x+JhbOdEpNHNg1UFZObzdEyySAmF/j6IyaLHypxoDliFWFANaNssRtedYVWPPTnBI87BKmVWXyeluVgS/juiHbxjkOJ/ohnnNRqtLPh9jaKmUzjeuO1mMjGRyHtMCTZ/Uyk2P3qC41uE7j7BKVlYfLojpWqaXJispRK9XVgQDZh5DIQxgK/8pGBX22ViJ1DtjRGKUUVaJnDp19Bin6ZQYp+WSn6BYdGv4AvN17TOg0sYQpokWIgJCuDJSASmsFxtth4kJI9aTHoo6S0cmoRbK1aDdeRf0VEZtJxlUEtS8CC00UgZau8eDkOFmNEo5IVKwbLmgC3bbPoTzkPwRIRgnFptySZVN6ICIYUt3ZcoR27yuJkDHds8SL+Pr+Oi8eKyNJe8FROEIU3Ilo5IMaUwcVDrS7leluidVx6QUAsonKjJndYWnaUIWZHHTeJrbqRGFR+1HGryVIXnJyZE5hSZnSrodxrKM8sRsLcrKrnZnSWYU+zakaWCj5Eu53Eah/iVlK5HJmXaoBkcPAmRiPlYVQGhCCZkYJkRsrHmIQYmXSEPUglkU8umrDUqWV5FiQdV6vdDfeOl6kP00vIbBejn3KdIKUC92frqhggFUHVQ+4DX0KXB0RC16oCnWQ1YjYNxbL2aBM8AUxoDLQpu/xWVcK086y50FqUuXDznYBRIBaLUfHb71WKL6PLFs2xm2UFTDDMISDvjqemQSd2MnvTRSadmO4bz6XlVGJMUIzlXaOZmOYRiVk5kZhZukhsdIxQDhJSXBLCDLhnPyMpxRAp6hM7NSllymD8k6tGOdr4wKMeeRQXJYqCTW78pFXgAb0KGsqMLoHYa2AtCTM6T8Kc1fVx87tPH+ryuDKhWjw8iqr3Wkoe1yDFssRIFgjhjiQEsgYhkGXZQBaJ3ahmf+y0UCUuIQaJ5hwcGoEHx8eHBiKff4D4WZbprslyUmFUZgHpWST8pl8Zy7JASJ2/ZaJZ3ppCFPnT/YwJZO3cjgqXT2BL5LIa+Wi5+NVgXUFntGnBZY+4RF3WaPQC4kvjglBetparlaIp0t6tuOK2klWdzs+qBLPkeRRcLCqr3gIuEENVunMHRVgsquJM4RrtofhdlIAOrEbnMVVm1FtTTFzfc8JShgCkxPXbSjyFQCx7917KSIwmDKVWOgcB88DqAr6cxaM1GOV71aEoU0BRGlpR1CtK3VoIh3JSGKpZxGKl/IzA0W+OAqKaydmkmoTPdkiy8Alm+2i5qKnmpa6k4smBgXGp+qXO7zzs3gFRqqGgE+I6Yh0bdZ9PZkcU5e1QpEI1upuT7Jf7AYeijonhTHnH4YA2wtkqTVcOSzkNKPEan67KOUBrmzglVcW+kpt7PafbEExIqGVsjziDt0w0uId8/GafWo/Cqs5eDaviiRvvDKt05/itxZsNeeRxV3Wm0Zlx1fj8wp9gJJw0IGo06oM0INU09WKnkJDDsE1ZYgaQOGbvVkzSVAFBi1btxowDwVloW2VVEdFfceLcruZZUDAZJm+aTIaPXTR3SOKhmQ25ui0amjBs+VRNdyE+jFRNSam7tBBUa4V1SDEUO1Sz/YPsKaHkNEVTWsDkKimeQ0KAxsxjVIQlPKmuajQtlCtnekfvD3P5FJMxReXK5l4Cm5ReRrrmfEDX57IsJocJl3+oygvWQHtXdi1WarDOs/Ps8fN8yw7a1uM2hHHTl9hldolF9NC52jKmrIVxvMS2yS61RsbzrsCNSf1zgs2F0mK8TZmlOxl5crIu8HBz+X0aJYgbp5dxdL5QTxhK1FLBMofaaCg4yQeE2mrPeU0yNnqDMTZOnxYUljS6/9TdgEgFPaXgbeMCGV84aFcO0sqIhGFxKMxsnsh532B23Gr5Dk5lxoXhJqP8X2XoF8n1tojMr0FKoCZEKxWwoTSKbCou4PHLrUSxsH3bE4pY+bwvvQzdlBHiUm4BEWVSrQEOIQ1fMI0qD88chmxUCOKyiV866mIq/f5spqYwfJcC0kdIzb/KkjMdHHK0wETLdq6uExqE2sV0mbVpOorPl1WYlgxShvewmmsp2cVraQsJcnFfgiH4LK98jrUMckEyK5b3xWLPvMIuvVYL5QkjIRvtNdTdhhThHhwiwv3FBk5CHw6HPprQ2/jUpZy7FCGf10UFXHuq2HjHp+35lEKjSwPl2tAvjK0OgLmCJG3brJIOwHb54x2ua8Vf0Ku5o8GQs9cZ9qW8EMOyWsplKaUZMnrxeV8g5UZVJCpnpZRmtHVOQXrtjlOeXojASz9lWNYS0rPk5kDMDmmWYGRVZi5VmPMWxQKevBBcQMskpEONz2pphkKA/cOtbfeiyBcAZoKJyOL01pLZIDDwqCCwY8YgMKcNeqxcMWAGBsQ4b1nFXzYm/IKqRFmXr8VhyeKCPEVLkORIFpfB0si1c5RDWSwZZld1wWsz+ytHZxb+1KIkUEVckPiqpVckDKhYs9s9Ch50jaGY8l84bUnx6yaii7U0PByda/iTjVBjdF7jT2wSLPMWCcv2F6rjWD7PsXaaY1Y4K3dndxO0z6rlUiAo5sIpsg6VhEFqlLpYqfQuKwWsundwtFJyLi+bfD+JB0xrXJJlaD1ghORRJLSj3ALWjKLpN+08WWNKyhAoh8HkYvWUmIuR1AW1+jPUEYSFbIj4043u4VrEKOwaBoukFkqmpWNCtHSk5tJnOZiB3IjtUU3yfo5/heiQM0vnbjkGFpxH1K2R7F1m1zF+tFxdHEPRAjm+l9eRsKvZSZhZfNSY4gF4Z1jt1WFaH7Wc81EuGV83orBLziaOo10ct49TKMhg0YxKNffvCwET9L9n3yTVAHSNFvnY6NyhnDyUKpHyxFYNo9Ho0cBRS7vQlFBdSV82YqhjYUMhNj/Ettj8B4NnKwB7HtFlDz6ii9bzjsJ1rBCIklJkWSGkINV/MbFRqM5CoekcLb2EtQhCgEtVTy5jlD6CXaIplDId7XQ8y1GsraPu6ABUvVP61Vzewp0zf+Jni4pRgWVSsAhPIRUUnGZD/TFp6Ioo+crRJg8B1a6PJkM6z0a+u4gsCWx6K5kemvlke2eww+QyD6aASZC7bVJb/T6/2uowjYNMjMmKMSZlxPLsFQsZw7dfFNRbqSWZB+VbF46OXMyhS4qACrgUrdsGNdfoUMUeq6SwfBeRrRfri4xOauxZTZsNi5uNenfg6FTEnovVFCnkKZJNur354q4WbNZ4LOD3Bx4qxpJyFoKUsTAIYSwQS8cKQiBLuH3jVFm/hOekSBI8qax/LxVj5bsscv6YeY4HRYNNfgExVhQmBFLOP7Ek/9kchUEj+hnWBL3nYNbORWfi1vXyJsOVkeUQAq7NFZ+tbyMYdIM/qoUyR7ditu7z2a2Lj8raCgEIo+aUVflgCLNcUz1LvkIEtFt2f14YhYGd9I2Yt2JCKyuns2Ix2PVu6ag5MNUFVq9k5ERW9OdxVcftKgDYOWd6VtJEE111GlQwKL1VZosjRVw5UrXKKqNts2bjpPRGBIPqjZg5HSnkEiidJZtaAYAhVaolgKObI+XuWM3PYp6fgdmK6aqbPe5mmzErshr4mWoJiT98gqbSoSE0JZUvf2iGRL5kLppeC6RAV6s5VsmphYyQwkyxhImRMvOGmXK5BCVbcvY+uwAXpZiJslq91rVdo8YlvlNlRFmKNkDBDaje1Kq0TvNiIo/WTK1G6p62Lo0R16Vx80N6xczTjDe6+Mi37hbLBNJMdLj4tYke7m1dGsuLjKpUy2fOO4cfayUyO51dNsbTk1GxuB5P0ZxP8aR0KTlrIlMHzAs5jd6lmI6uX0zXKac3MWlsTHuDJkpHbLu9zNlBClJ3DolhRhsZTb/kCUqvVX0E2egAoh9BlGKjQ5ZyzJI4GaJIvVRxdDmjXc+qAVkqRM+b2B49v7q3VPJmdf33Tp43Ut+ijZTNEcSS56Wyk6IQ3uLrnqilvkCbZUAoUiyWgq5kBVjesyXNoJ4pcmsJvQP1QnKxQAhwiQnQpvoVyBMuk0Uf9uhZq2piAlWOcgEi82r3h9yKTyvmTDltf5tkwRe5FVEJ8Iotc379NQIIz5g+3ItspJomRqpxkS3wvjhqOsDMwq3BGDSyfRAB1KZvnXVFXEYuh14fZEtFLUa4tJgvkqcv5qWctO+qStpXwbfOGPkWoJKKJk8wUgiXJodw1Xbv80AaFBzVD2u0TWM3alL6vTEEWPNiY6aUKkTfTxzvpLkxmLD41lkZ3wdpN8JawAW6pADb9vxsVIDNgrmEBGDm8AVgTkoApqQy6ZtfuLcCzAkhMi+KyHi+3GsSFWr1w1kcFHo3U8SV3CdTM3TZk7ZRtq+bGbRlUj0U5Xp66s7LRpyQWY+QY42+0TclgkWvyaogrmww6xJKFvTg0AAuHyVG//XYMNywJYR5h1S7rRixmIkWO7ubn4TzkTFnDak2TWGXbdFYSIC7fYp8qbVLnr8S7Vfn/ox0EHQmp1g7F8w5u0BDSJCLDGNyTNrS6ps5l966hBThvDhHKEofnPbr5R5KqWENWiDVJmUbHwz0o4GiYvIoFVP11NhijNjeDZ/0uLyukK7VBop5hxbkrZiQmMkjsbjFgZVtGnddsJv0Pt6kelFzo0iy2FKkY1YMi4lJx8SyxeDgssX4xGNiD0qugx2QFrpfxEKnxO+HVnICRA0QW0w+S0MB0sLp0p+QE+WZRXyE6iWjfSPGIinti84ycpjMmUAKY+JySZoh4IdNddDlIFnMCuY2F7pn3auYUNmgMco1R062z3GyaCwh4ow3VV8y1i0HyQaH7UJ6POrMSQFlF31zxXiMeydinRBPxSDcmVg/S7SATLB5oyQmAzmL5KKS9UcfeMpHfoHJUbGgiCscUZQRmUBMFKvOworE5LcWKje+WtAuFxRFocap1VxDT4doLCmHbV//iDpwBgWzpLbQTM7U4pL4W8RgC6FeNJUMdGzfyPKOSVowsaB4KcOk4WsNSfIIxiXMgCIl39Pan5IwA0jJwYyW0sbFeeRgsIBsLb0EKZhdxGJUpj17UBx5xWTxCIYxV99pRE5URdp5Pg3MF8dKH2VWBWYQ8eoVNXNoK3mDGCUccebnm+DKA+UP/Nq99+Zm9Ig1M3/wAUNLm9PyL/tSLSOWli/oiwS56C+55C8QS8vnaXlwiU9ZkxN/LSktf7y9UTY4EizxAQdLGgPlR8cS/mCiCYwQsKSlwuigwx91pH4CFnDDaMn3T5ZCGeunXaOi0fZm+tdxK1voIjmEli6SO8x1dgfkTOn5ZokGSJJqhySAMVJKLyHiYoWQFwgRL8SPklpkXizfqoGiP5JTNjWrvHT/DCknFj6WZBPCTO+EMDkzoZIL3DJdfIqsPG22kLA691aTCXJwAZWopPYmKrWAr5Ad7ObHlY41GyzEhKhZJ6I7z/c3GJD9DaiPOJsNFjDD5fc/2oLlc3tHBi7f4xCQFVu8j1dd4ZeVU3QJwi+eKDO0pksLeh/lYu8l6ZdcLFizposz917e+jY6LGjHBQXnjA4HyvFA8iI6dEhXXYXRcUM7cCgTU5j6ybsDKX8+oJ9WixLNYcuoieS0OgbmCtotm6ApB+z2kTwfP3YYYKVgVhCCmfqqi2dgYrIvUhs7ktVOSlAGUjn5Zli+2VEuGokUAyYWqaWbFWUgxklsbznTXGZHvYisLrWIBH4xw7BUdtxcwq/de8fEbej5vYIrCoxk+WMCYDFFotGQsxmkTQPRVUq1/uVTwQDXA3NHWcAExZK2pMdOH3gOjQ0UyyOX4dEZj/fwVrKx32fPBePRhSnLYZTr3hOS56XPsURP8yjDBFtCglhLSCOQCsZJxpzxFNdencxJW6IHsaUZ4EAzVlaaEEd3D8rtgyRBA2zTyb1nH0nCFclG0frnN7pQ1FwpSLlkRBMsAzTzBeGYaxaObYLteOPBoCEe7BUovwYx7GWk8JSU3VFJZd8rMfGXnQl7uSXAANJqlBOsDa1AbfoTBGKszlhZw6McnxJL1hfr4yim/urveZwr5ssjgYgik4GTbA59wObQM1EvTMp/1TCzPSFdMqgOjWrTb4lL+OWjRz1ge0QFmhdZvFa0Pu70tWdCW95gZ5ZL9BUSei09J3gUsHXySsAWTyr9qRx8OhOMd3/drSB5DI9KAmxdsMbdB3zcfR1VGG3XlA2bUMajEZBqADLjQwB/DJCGQtMvVfZkTAbZL3PPAU5hWCERhlPLrrzBF2HAVkNB67VdjHRs9SI5736OWfFVg2+xQe3FE4v/SrmXVMtHK6YjE5N7STV9HGSxlxaDSN37HGopSZaYMbGV7uLIUz/DIywhiktqyVAA3SCE0cT0Y3ouo2MM+Gx3NpGX9gTtjWUjXnmp0/4bFBP2GkLCi7wY475spGT3c6q7TNBA1rUds8m7zJBo5em9C66yaf6BkOY/HjiFip3OnwNm5fReWi4HTM4KyUTFQIyKMbG87klg5ez8mftARhNIGeANIp8YKMnpldIe7Ul6r1raU1JiFY5m0mOjBc7Xa77G51/NCUiL0k+Ucg0xdIk6rn3J/XUELeQJGgTfJvy6X8IMsi94XX5FWn43hQcNhwawmuPkjdCT0NUoQiSl3gh9s3dS2qooJRaS0grZLu47frmWFAtSUhPQG16JCR57rxhxegUmb9zbKum5YroANLIiG7Sygy4qpjaFafgmR6x2rlKTVR/uSkz6LDA5G6Q92tcFsI1PJYP24tHGmeVS0SRk2Vty4lqOSEExmGu7jGpZ1FlfFmXlWBQLezmVUySdiTntzuWS4y/kEtAu5YR2HVstnjGiqNHZiD0dKbIm0OgKlcIpRPbT3l2LUiN4lNnwtpoDJUNwgVaXGZ0rtJOF5M4cNGFSqvHZ6D6AuxFUg6VYAku2HSytarDS/8RgVfkBQYgrmUV03yMJo6QMgXztD/fftlDt82o4gZUCEBz5WkB54ebmhxydCfTy0ZJcxz+1BLikl2GKdEKgcz66JBqNtLwcLDOkQJTswBHdlJhtiag18h3U2CGzQVjBYgLPXkX2ldMB0Y+RSwtlYtAo5PUU41uJn8674icecHEiBnLQSii5hoiwdPgk2BCRJSMfbw9UrTKoK0b21DeUanxs1R5cJGhkcQlVM0Qq4asEjSRGezZmUmNBjZ6eNtfo6FzDnGx9A8Pq4FTKw6nkhoaI+G0fzNXO55A398rMk3tVztsCjO1wTkRmxbyDJgkhMrEMeqnIrCgEyJ4jPX7r4NRz8s20ozshC2KApjXkSwvJ9abfMMyUluWk1EFWSCelxOyORirey/RZ5z2D4hMETEO9kX5fccKyZIH4FjgFWm7Q+OpcfsLkHGnInKTMJK+JxanwKovO3FBEZ+PrLJN2K9mBvNDoDR1zLC0Yyhuo5GgX88u4BEEaiIE0QUchj4zLyjVXFJRxMYXT93MUcsq4Rrsddb8jdT/0Gtf9kCU33A2kapVDgkAc0nEjGQyGVrByXBbI42rwFo5uG/T7Bm1sQ8XY9rWxrKJtVpekYKlVCvZIBu87SLL2YwRSQpepj9nS9eFei0ymN2JSNLG+ifrghGhaiLN5WSEax8xSjIimGWtAb7FSc1YZj1rJVovyXoMUTS9hcqWMsVLhX7rWl9kzo0trVGT6wzWIy+loICLDsmg1syjNOKwljy2P3kbkw32MHSEONwfSHASsFZBYNWt8dH5Ar6cXjc6bABoUc+KfX46Z9GfaIFbyw9e+8YOmYLPLvtgM5LAZD8xB2x/1wgVohuVxXMgZTdF9F5kEaFYAmnHqz8ZbHHWTIynBEqLS3k2NAkW0QQVk7d3EKFAOAt5EWF/FOLwjsjKn3kb0Ath7sJOsihGdXlZbw2pNWNA8IMzkQZjbMsgaELbVFj4u2gVpzoCFaFElZ0uBYVbOlinErIIQZEtCKMwLkTArmvXFoZZRJC0USaMIYihMkT4/EFpf2jKwI6j/yPPn1UuZHFl0VmYZAi4vFX/W/p6J24285sx6i8gy2lyFLZvOTKN0T2Qn6Gk2cAxBTBSfsMzHWG7lR2yQmGNggx0QcOZuiI4tmh6MBaSxlz6ZWQFZMBZtb31YwI7NhuksIF2Ymwt5HQa76orBjFibRi3XptEItmm0YiFgLJK3CznJ22WlWXJhIWCcjRoBUHCqXjKmkSKuhiQoZwFtU3yxc5GCrbxFBVvtuVfQOltGTNfB6p6Wgx1QOOrF6UKyQlpAWyHbUvSNJZhH95yZ1fgLCt0ag3UNtsvN+95AQVYdmGpo1QgHD7BgaPyePPnCRkrMJWaZHA5NzBXZEJYWAohy/kOxCejqmJxWXBkxmZiX5VfdjYJmEeuRgvLEkFsf+VlPerVzcchcaOjKqmxufSHs/NlPQ5nQVTIGkXFFH2eWX0VEw8PbKk3V+Vyp7lkxVwRNKrziQlnSwrkrOQkbhWb93pVm8YTDd84UE6RZgl5IxZMOfyHWuZGAs3i8kLq0Qma2QnZUdXVLUx+dkfhTksRQrEMxlKYiPRVXUIzUn7jPkJYAaOISqC2E6kl5OyG6q4NbtqDt0kOrtsse7bY0YG2o2CDPsgdPt8Scik4IbnknBLeslFMRpOAWX2R+ddV3jLqvVgOXlhKUSZlxp+VkrfQD6jPkZldmibkUpUx/NGI2LGCV64PgWhGBe+zedKIWtLVzV5m8PD2/jDCl5euU71i40zmqInIsS7mGaElB/XUdCrLEKwBitncbqXOpt2wC4tip1XPcKzpDW+UVGWRX2dFTZ75Wx3XySnRcSo58WUEdl2CePsjFgMkF6lvBZo5GgHxx+hlHH3rqx57EigDFoyY2dlJW+4DKam9ujBgiakS2MZNrdDDUHQ2krgeRvijalHjjGw/1zkPxOyYgD+5ltTo25krCr23GW7Xwa2sIZpZ+mWVGgC0eqwUprCYVs9+c3z+JHIKUZEnK9mj4fI8LCdpXvlkqxWLvFVuQirJwkpTvUTgDrLtsTKptpxFCYIJ9OaXkaH4muuZtrhfl7ShlhEs2FiOg2xRujBHHbHFguzet6R/AVg36LGsPHGzxWW9+hH3MqCFDQL3tRPMpyAxWQVYzv5fZJpgDRiB4t7bYhGMWSKq12iyw09ciG1MsYO5UzmB5JpfYdY591yCmGmMyQVqxLDCRXpScFsjRwUk5Oin0JwKhZePeo5JkvxssYlB7zyiapguImq5KFZwecCq450cBTdCHV3O1Tc74HKec5CSuiWg1ML2w69CYL8jGQmMk2LaPxbxxYGxtJBdCt2xzjjZ0byNpQAhwKTHd2OJlY7T4LSfHt0Dsm79ULL8S6/hIMunSssiev+k2wtU1251lTfZUY+nOerfmXg44ctaRcLkhkLsK0nQ9+XAvdGs99YZRRJa8RZflCr13AfBDZWsaKdyxMhv4pUO5+FT2VVvHSKfJo14xwS7QujzPtrlb5FlX1MWjXTpBowy5wC8e0sWD/np3i2SSoHVvF6klAr9YPZLJW3T3vGqYMtrV6PvaUpsPugCEp1cbmkboEamq9Wjj3Z+y/5PKBMz0tMezja6HNRfEauw1FBRhHnyTIuxxyT7BR7uAULDnCMzWB4qp/+atJaXSyBRoMfQmlUempPLIpBybIBpHpqSG09t9p5wUX1bQSnymTZtOTmQ3zGPbbJY98TTi7Nq3UuyFDccmZmpZYUfytr7DAQGP0LOccuDN2lQMnLeMtk3QCcuC7FFNA8S8cdNDOV5/6wwhls4CuJ1L6mTpfZdQJtemd6iB8zWbtNEgHzVnNJkdDHKcq4ZWk+ezMze5UDJB5GZZkJvmeBx4Z6YWJG488jIW4dpVVQ/LCtx2yZmvr1HAKHN2kpLjwSOS47N7OMlg6AETVD91WlHcftGgKzX0UPQOWaV6HINBV1jVj8MmRBV7VI/0Eh7p7b0jVYO2UABt1sdG0Hb89PnYemFZlWZQT8oaQBkcPChLUgZMIxVsZqUcmOCkHJhWCJMNbJiMlCjvloAOSLooMT5jxLR6xvaQRnXUpkm5L50QXA1LWOWdF+Pi1GkmmkAIWj9mFac5iPjaXMo0Zz3JKmePuOLNbPKkjCtaqn22G6W2iG6UVBx5ke9lGhG9TOtD3PL9KDVQnzRxWV9ls9QcKkutVpl20ZWSMQXKozkZiGEyJhOmEnRhyvWiZPF3XqLFYzydKK2ELu2cEZSNTjD8GUZyxGlqaJRqiKganRXY04IUguXp46m1RwZS2H1Lttf4ZkG9W5BC2RwylK26Z2hE9QzNxa/VobKYR2XW6hYr5hgks2MyBf+jKKsDZcpEKVSmLByem9MemJvTirIyFuXd/kDUSYVWbFQ4aqFOlHL6Ri0UwIXT1x1OGwAW9+IB6tRIPJL00eRYNfKwzHukqOvuesHl4Cwmsz+71DBhMjtotN6HswGAGQgyI8sW+29DoujX2PybOiGcjLdsYGzn9ozSWdCQXNa0adxQHOrtjpsGqvHY5fx4jMe4OZGJIBfxz8PHBJtb8jyQC/TDBjEdGVPEP0jElJ2yKsl0Qtn1qukYIp9+4oAkadQ8WpxUX2R0+tHOPwqAC4mmgKtUjWnCaKoT9a0migdrTbTjI4x6iFUjsVRQjxkPDeoxuzHkLiq0vyXYzC4EaIEVA1piHknwQjxLLH4/CTEPw5dOBkKBTRRQZowYKKO9dN91zrFqSM0TVBCWf2kpcZLhINFyafYkCRgcWJzaXPKvMCQEXOJXfhkNCEESPf8+q/zKIq3nY+XK3F/fzVyxmWZN0awl0gAgigITxQoxIp8rb8x+GCJp7XJZIXfv27k5bYjZv+qLsKwYwrJdBV5WjF+BmLqLDoIu5LqBXorZZPGxY3ZJsWOm4BWc+LCTmg8CSm7VkuI/PoXw5xCJXmkUiGupMTpqaIcNpc6QCGOph1ejI4x2iJEWAODWMYPZ0emCgis2K7huDxJYZYRfr0G95aRULV5KuyUl3TIHptziI11ij19L2bTEViIIOfd6tzow1QlyTcotOQ+e7myfVF2Y0UyMy9SG9HVkXDsXjsLPOjlbTFoTEVKbFSvfAk0QT3Em75ukPSYP/uFHooyoy7qyXmzFyLrsEJFriSvuy/hCJ4VNWgl9+Wb1WrHskt1M5/PBUjjX7/NzLrp86EQuv793P0klGLPP02HzQk5hdlmVzFUNu1SRK87cUXL0kad86EmZWJqQiVULPKzDKKlW9a0CRmct9bSlDMVElMCpgQ6NTm/a+U1ShEX03EOtTs+idHr7e2HWAS9TitE39cBrQwDnyfQyDan7BlN2Xn1WEAIFzT6sSXWWkUJWUm5DL8SsFJ/dUIk12VQkJ6Nc70gxTVmrj85yiNUsTXJk+MAVTezke+SNMXIgdXCoDaQkcLOlc9lUTlGyR5zcaue2kvmBJD2v6XQmFVpBnWUQSp4VZ7/IYDEhYOydIrNVnwbKxKyGYBDqvjo/ZVaZBVkbp9uZz3p74clr6QupBJkVUyw7ujWkZkFWWq4zJEu4/qUcZ7uq65Q5c/zW+HzEnpAUogAFs9zkrkKxlhWSo/Zs0CQogrNI7gtxwoeUWVyoV1sjyyEYlFqu1btoSs9s4m7QUWNWx6qgwKpigqYk+tWzXqWLidYq0irAaMLmpFXgpExWQQhWiUVjJSFWFYVYVZJ1EoKU9gfE5EosJlFlhDpEin16DKsoT0ZfZZcAfYyUxk1MuKXE3vIhSKwgK/95SCOlf9XPGwj9kC1ZmxKVVVjt3I4mr7g16OYs+2wHXBfLTbQsF6pyySLrrhgbKgJY5LxyhWKBAcziJROyHKlycUA+23X9SlB12hVUCWqreLxtaG0VD6ViomudU+KtGKZiyZ+/QovYlhQSP9rSsJsayUToB3SJ2tSo0Z6F37VIPfoGQBd5cdyRCnlEgnoL3xkdn5QDlJRwnzDNAdqo2+iExJ+R1ZjK5jFVCrqBUm3ernt4NLyUyjRoqlw9pVqIpkos8cokIUol1ewwCkGqIASp+FodkjQm0MySJoFpq7pPme7CmKHxLeve9Msk4bQrxSEsXISaSkycFMTiuawQppqt22FAEA3GlKsYA6IR3MOtyR3RA7dzrMrZhOqtqCqCvnO4audqlrkpbq5eTKKqGMpNLO91/0yQyqWER3JcRkDrMB0VmWPbdbaX4e5I6/sZnvV1AYJY2hVPvlP3foaCpEqx+Ol6mwBZssWuqmRgM3sAR6cG7dwg4ZdCB7vJnYXUxc7huti1jCSgKE81pxodp/gDldJNMAZUN8HJiwqJiiUK36tkYgHT9XF0IahmVa4gqQpON0uq7OM7Xq9W3sQrqE+8aiBd9uDlWDSlihWS7ZC8g1aIdAFI5V0ZIdYFWhR2KY6pJREp0yyu4QC6JOuglGqn9Q0rnNlRAHWZzm45KyZvUlIRUqHxUSjTOdcdZgJd0UcMeXq8BXF5B2NKaN5F1yrlaFdCCNA29ycm76AtyLJ2G2SvqsLs8z5Cp4vIy1aN+iJL2vSAwoqbxXXMFoTlwBAq8yEwEw12yC9bcFIw2Plr6VsoiMH2n0N28WFYcootkDMW2r7GQrskY2E0AxKE7fnYU3gIGHSdFn9eTInAeFqUVQmlrHpxNpC8hk4jHpptLDO6Q1FuUSRIWVS8FW8SJDamB9KCe/n86iiZL6XCQ0sq/OqR4y5JzgWHD7lASjdipTyHUSogS6p9YXNW2KTfyrNBLqAsGi/nDpSCXFqKyzXzCuiuJtKyvkMp2RVLJ0AtldnfN3bLdJa22bnSsbR1CK3Ry2/lLYwLtENZHfnYlokDmqo9/CybS8vlbQ5x3T7+cH7FpuJqAC1Noe4REJFc1CnNarkiAGGofAlZHlJxRl8OlUKxLuanWIKR7kwUi6Ut4pkYaDrHMhsehMWDItFKLiUn5WoOx/qdlWEBkBhWrdNttH9id1BChdFBhDmKSEykE+wbX1jwVxbKs4KEeFYts2HiQGCJL64kNJkg0qXJk441FKRcujnJfSM3fACGrFIuJdq9sIFzLT8J3ogZpBRoKT2XlkJdXgp1aSHUxUe6NCWwvdUeBb0jmFS75McyOBdpLQzFyNBcei41CImNlOnyqBj1XNAjPE3uE9SRdTlEbPjmvkXznOXD4AtGt+0Pi3evNVzcKxjkmB++vRguaRdEdHw5m55rMInUvJFP05UPTV+NfuzM1tdQYzLi61bVVX5JA2lJm0oQdjl7/BaPmkvLpcSDXPoWi3bpvC4UvR6EWTEQpsUsjUUuOLOlcbTR0LYaipILFYC+94ymedsMusyLI4qUex8JSeuVNXzRDVi4YJBaNppEbNlYLVDzCYjjqhtTqTdB5vCuZmShwMgs+Aa7ozu6nbHf4XSAvGuI9VpKgryYZRGk5Fw2HZqcywohrrn6HYIViqRSIKXmgtSquppcjlrKf9aqUoJ6p2sb5JJKe2dhXIsQoJklrJn6QLGOjCulgAziPmaN6TKQVcXYo7pOdVm2BaZckrnZoS505dtevNwRW4a8drhuh1z+RACPmspjYluALMpy2mO6dLImyO/es/cUHd+j6xjW1WvpdcjSM3ARnQ6ZDIm6qyFRCToSFY8lsZgKNzfGgoKuZ+KYomALpz2um2JLM0IwmHEct2RmpYCOK2+pMz5s8McNyR9YKDK505OEacW+ihO3Awog1Q7bIRKFqX57/FS9/fHPa/DD+//7+ec/v/36+X79r+8/fb75ck+sPgzX7o9wMwzRxhtIOn64+fTRfLoOH5P95AHApT/+8DcaDHz49MmA+3QTP944Z00c9B/u09vf/uu3/w+luxEXdY4DAA== \ No newline at end of file diff --git a/experiments/jumpbench/frozen/neural_confirmatory_preregistration.md.gz.b64 b/experiments/jumpbench/frozen/neural_confirmatory_preregistration.md.gz.b64 new file mode 100644 index 0000000..5482f2a --- /dev/null +++ b/experiments/jumpbench/frozen/neural_confirmatory_preregistration.md.gz.b64 @@ -0,0 +1 @@ +H4sICJYphWoCA05FVVJBTF9DT05GSVJNQVRPUllfUFJFUkVHSVNUUkFUSU9OLm1kAKVYbY/cthH+vr+CgBEkOZz29Lor2Siai+u8FInj2gb6oWizFEXtMpZERZTufPn1fYaktNo7Jw3QL/berjQznHlmnmf4jP19avuvZSdO7LWcBt6wl7qr1dDyUQ8P7M0gB3lUZhz4qHT3nL2N6AH8PYlR3Un2IxeDZrfi10kZRY9sNldX3wz6N9k9v7picRjvgjAPouKa8XqUA6vknWx038puDHTXPDDBG1U684x3FStlrQeJj/jpJMWHXqtuZPfcMHnHm4mPsmJ4dDxJJtahtrxTtTTjdrN59oz9Y8JHG85LDrus4d1x4kfJWl3Jhg1S6DtEg9/kRy5G/CvFNPKykazXzUOnW4VctPZ09aBbhiDkcIeoKU55pyrkTF7biCstDeMuH/MvTLX9ABezpwd8aLh9ZNRswGuPbeqB9dwYekKXBt+7lNyfpD8s3lEVjs+OA29bPlyfnZVTdZSji8YIPaju6IzXHL9ypLO2JfnrZvP+pJDJj70cFNWAjciTobK6s/JzJWHBwL7LESAwGz3KIVjnSI48mNO7Zd+PLh+ddrbZ1A34b1CCCvc2gYnViTuLueBequNppHoa3dAh8bOr49/OcLm5KLeRPXeo2WzeoED9eHH6FczYZCxkgLXDyM0Hc+O8/kx4+nmBzS9Gdwdrg7NaT0NAzzIzoRbjlr3/PbgxpPPS7PqxR+afbzYBO8pODvQrziCr5+wQ78I8KvbR4QV+TXM28gHVZDVvVaOkoW/jlLWqU+3UBrfv3gdCmzEobLCf+iWKvA377m8SdfU2CYoN79m9Gk8WVWkeWD8PFxmbo/7D13nTsH5QdI6HtkTdxIUNCu6TGbNBuYKf+FCtgA1YAisGNTUMAPVe6XHX7QOaaVw9z/te8sG4FsYRjPrIRsDaAGRMcsw03pl7GOq1h/RtEHxn3fs3dIdu6/EOgPzrRG3KWw34uPBWbSJOWgkYxmxC29GD5+DI7aplz96X/kQeKuVB/eMMHCORkRFZE7pt1UhugI/D4TDKj+NG7Hha7uVulye5jIswF7KuoprvqyKpsziO06IsMxnGUSzqOorTWuaVTNMkyndhmdZkaOWsVphs7767DeJst/JSl3UkpajSeJ9k+4zzfRjuo90uLuswEVVUhrHI6qyo6jgrRZWJZLffZWWdp8kulkXsvKBP3ch309XQN8/AHUirrfuIQY7Bzg2GK1cDNcGhUePYyAb1QzfcXPwVhNtdGdDjX+VxLOuEFwiLF1lc71OeFHFSplkeF1WWlJXY70opD09NToBegxEoq5W9ugzLMs6SYl8JnoWJjGA7jcpY7mUe76J9WIkszqqDO8I7SZVbH6Kn/gIbqo7++NOHOQ69DmDjFFhMjV8lYcTjOBd5grLWKXJfyDyXu2Kf5UVZyCjnCCiO/8TBntjGMEmzRGZJta/yKglTEcoi3/GMF3XMK3zYx2kiOA552xAX2vagFqWeRQfq5k7ShzvlerGdgKBSskYfj7Lasm8HDC6aGEJXlhWMm7HUHDV+C/ChZXoa+2k0W/ZaO1wEppdC1UB87wd2xfvR0RxMYJ7oe5hf48lTvkaaX80fGdrPWDFy+AXipSTxEngiuQu3SeC47gBD7zwbkHsD2EdbdnX10gsOe0Shj53rTKiVV9S0ul66n19Mm/V0QrjENDABXcKpv0GPk+MtyxX2V3psaX1LleDSUX/AwZBKykLJS8z48YFm+GTsmDO8lRcGPzd/9OLUVaRj7BukqSj/Pr+IRPdB9OQgNJRGelc5j7KrrMjabmLKzzdkwTx0+MkoQ3mhA2GOAwTgXCOtSgIxYmhREs958edFfeh7WNQUGO90p6DyWKnu+KAQwVo+CC1rQEI52um37N0kMGgNagMlAsM+/GODMzdL7NdWYPAjYrXz2SvCtXI6DgpgSuhIrx4Pclttrzn0ABBTXtZJN1Y1iTNUOAqoj3oyzcOWvXpEH+7oM5G4klA4tUcxeQpOD712OZ0RzEwPfbbdpD5GUKgi1QQYArUU48ulaJCMHwjK1KK2xhIn8ZqaXHmGviyGmXoYpegJjLMgvLEeZgDRqQfdmO0mozDew5Gp5TDXfTHh9KGtcIuBYSUV6VwnEtVoZFPb6NZc7fWijQUDzGrANcOuEnyRfQyNqUXJhK8XtGLlzFQSpyDFA0GKt1GEQY/WQDlz8gISTAA/R07cnOwEmIkvSpOsCEMhxB50tk+ikIM0yyIORbmTUuZZJqOwiMooDcG0oJmIZ0meVnu5F0WeMbYSkdv+YRNi0NZ7LkAmoYg49Fy+T4u8rsM6C4s4rco42idlmO8xlZMsB1VHUVLh+zApipSzWRfKn71lKyrJ9A68kMpwtxdpEYlURsWO82JXhPu8ynmWRmEeZsUuq/YIfR/VVVXg8awo4qosirpkTuTf0L+z8UWXwr7lcNvjE+ig0dSrVrqbSRHywA+2oaPYbSiontsa/Kp2ofLcIrXo91dPJJAhxry6urX7Eu2Irr9b/tGKWN8agW2NQBI2+4fLTclgMlibc8Yq5pY5RB9fgzTnDWmRWAA3+Xz7dO2iAEi/VXbBQRx2EgJmA7/v3Obn8Nz2DZDHDv8Kkuvk3/+JD3a+PO11DBq/OUjiMnL7xq11f8IVrEfXkbX+vwzbgi3CmyoXzKfWAxeNnKe7mNf1AEhVleWMntR1qeHjBTqXnqRpKvDFOFOaw4kVbQ3EjqvmLOlmxjAuDBuWXyO8V4Xwezxm9y8wLPvi0P0lzQ9fbtk3JKGJaXtvzQrD867veHo1GR7xV7BibZAiGO563r4JPKBRv0e/ID77f+x8Yk1HHqxKsFOTG7tcUrpICsLFeK8Dmoezsx/Fa0BwuCmVpztqHKcQaMdkPRVlcntFa+nkO920gR+fJKlsVrfsrW3C2Q3KJxAPFu7AXY9UqsbMpqYwbgbbQTivdv5xrABBqfVItzk9K7LP/Mk4Df/N7VoNwa9qJqdg7ENgFQ+NeakJlo6m71fXBvOrduNW9UrULCX+3F5BWLpraGpPjgs+VRguBIAoLLCMwrcQC+hpvMZLul8BkdEo+OIQ3eSHa0rhRQHmxF/bxDLiMOMQavN/Dsl86Xb/kUFno65Z+JnD/MV9GGSkGFTvrnrOTeAXSid+F+nEBrukWor2wCIX8y3P+RLJrtNUj3+qxtiLF18W2i59yZ2Bm/nlR57O9adXZiFy3l8DT6NLNj3CffZs8txe/Lgq1pHVGvaIHcTFpx90ssBLXVqRrIxwjfV77lpZKSjbVY+SzLH7s92xF4jRw9A1U4N56S4ja3VB+XTvwYrZTxT5AWlnnKxWO5NP5AXGlmamQiinJJdt1WmXeQ90BbQVWZy7fv727Zuf3AsgTMlHu9CcQeKmKokztwTBwNkq3kA2rfShgWw8v9qhwOerLjcb5olgsfn93JpO8Q5TIz29vlrweJby4HNIa1kZy0Qg1u5ILL/cUtD6JtaXu2+jdWN7Dl3lbbbHyumJL5oC1tFinpQkZtisBG1zWym4SEykAfmhwWaFwrVnpVVEK3nryX8J7L2D4fB78S1xPB5zNkqretRv0izjC8uL6nxTOXa+uOOB65MlaIvfeXJRxbjrhUGeZGc+mTd/N2z8PdrlzbEtzpwqfr5rDs5ps3hV5eTvoiEOyJB+UgTn+WvXAxa6qz5YjT60CeKgg9OA60n0LU3tlgMKigoB6UKMTu6ubVaXu157omu7iLatndtnRoD5sUG5xAcX0U/EDBf3wutgfNUu8jCLCdoEOuoiewUDJRrABvZRZU5P8EPbYldTXxN6UFKPE79e3Swjaj6kzQC57fgw6PtZPYlmMk4irO7U0Sq8X5p2Bs0L5MMtpqMrkLRG3OQiA2RO9uOE2C46a/MavzVctUgtpkYpoYcrepnP9xAXK7M14lZUXtKBHUDd0Fxlxw7cTjbL/TWds0GmfrA3Sj+4GyULV2zmxkHL2LfB0NUk5kSi77TBAcFVJ7pJpFPZq7CBrqWMvXEQeIEk47yEwRC3gmSx6Y9i5SIFg0nW+Q7+L8gplH+JGgAA \ No newline at end of file diff --git a/experiments/jumpbench/frozen/neural_eval_v0.3.py.gz.b64 b/experiments/jumpbench/frozen/neural_eval_v0.3.py.gz.b64 new file mode 100644 index 0000000..12a4e7f --- /dev/null +++ b/experiments/jumpbench/frozen/neural_eval_v0.3.py.gz.b64 @@ -0,0 +1 @@ +H4sICCIphWoCA25ldXJhbF9ldmFsLnB5ALU825bjxnHv/IoOkhwBsyAyM6v4KLRpn7GyPlGOd1dnd6XY4dJYkGyS2AEBCJcZUqN597Nf8h/5hORP8iWpqu5GXwByOI6yR+IA3XXv6uqq7ibXVbFjcbxum7bicczSXVlUDUvyvGiSJi3yejRSbdWmTKqad+91ox63Sb3N0oV6/VwXuXouavVUZkmzLqqdeq86Sk2646M1SlImDRJSYnwLr6KjOZRpvlHt3zS8ShYZ72TL2115AIlYXnZEi2opkbftZgPY62TJ423bUd8VK57Fab4uJI8qyWuUkFe1grlpm+I1wv2uqL5O2jrJfv86pNYPxS3P0x95NRp9W2QHNmVNW2Z8Jj7TvAmZ+piHLIqi+ejfX717O2ES2g9Gb9+80q/+Zcjgv6sgDEZ/MJqvdPMfbegr2fzq+5vff3fz4e27+PtX795/8/YNAHifwSILni+345y3VZKN7y6jl2PQ80eee6PRaMXXDLWOV+my8e+SrOX1hOGLo8FcfAZs/GviPhkx+FdxcJhcqOzXYCm+8v00ZJ8J2l8GAQNLMmoKQraEViaYRGnDd7UfsHStQAMpT1ILaUpkQ8yI63GhLFkeBLMJMEPWQhhijPQeFY/Vyk8E8ZAtDC5at6JtwIJKmCSgRofkQoBK8JlgPQc0eIs2vPGV5pcBe8GWppza6gCrVM/5Rol1ytLSxONl4AqUKEp1u3hKQUmSTBES60U3Brs2O8M+pzwFjPDw2NksARkTEDIRUmqzYecCOhfQuXBtiv9uOXl6koIBF0QFHz4HFhAaHwBNy8OrNDvwvADaTxq/LO55pbXm+7LIeQ5Kgjo909Vthu4Bc7fTMUbpIXhsuK9wA61Lh4KmFS8hk14lpRKtUhqaJsYcCNl+IkLJQUsEf62xxJlUtzsfNd5fXKTw53Bx8XloJhjTrYmbIsY2P4cYN8GW6Ob9B0dnnKh1mtdNki85QYYE+WpfguA1LBKWtsK3HNrRolgdgpPk3iQ7HsDKs2KEkK5Q4Adv74XMO3iPPRZ/QEoKdAohb+8xntWc/fEkm68LamsEKwckIuOTH/d1wvDd8SRAZHspmKr4jQOhAQKMz6ek+S5PqsPbcliYopRA79tFXxwKGa6ZixJWxnwV/Bxcb1arMwZWcTzJ8LdpDuw0sYyvRYx1iGG7nuBVutkOglGHhutz7ZSwdXCCH3IL2fnE7GEwiGHMfS6x1zDjh6lhnHgutW+L++HRJBKO59tcVcgCQ3fOS1jShS1gEOGS/WqqkeD5FzY9QxMRWIUuXWQUYStJYdJ8jwxeVVVRoSdHK8hZiH8XoijhjHkXaPyG7yEu103VxSj2E3sDhIUMnud9iyis2XK2Tqu6wbHZpQ3kJ0yT+SWDXJRXrObZerwscsj8cE3AdJetCpjgkLfe8eq+glyFpU0EZIl8DslhkkHSt8J0D0SJKg5pLdja+6//xDh1cXHtBbrxT6LNbPqfP/8FG8dm23//B8F5QZ9JxSN0rsr7uNgfPi4Qbn9xgD8aSmAtYfDTFWgFWVyW1s0MbISr4mzeLVO7pFluaani0ToFcDCBryeb5/9mokP6Tzd5DYP30/pjffHRh489/B/C/wdsCAL4nE2mc/jjz/70Mf80fxF4YUdMSxcaAST6RrwZDqjFjpKy5PnKJymjTVW0pX8VuMpFYHSEkiokGSyp3qdPn0D28tBsYSn6jRAJBYJ2y1IhyQAOA3/eHyWdpTkne9EDmEsTiOoySxtslwksPkZg6bT0+/SkQu5I1Zzn4MK8GyJ49HWS2RFA1sao9k0GqN2zEkL+9T79HYt+GXpW7LAokxTWvMV5kOYt7xoRJMI42SFa5LypZ4noUBuWEu3nAyaWLrPx1VzJPaid4fz+7HL8TzCkNLj7wzyguQA9Vxcfr+FpQMSTErD7La8484LZ5bxrW/AllHhdqyMZrfyNQRZdhOeGddiv2cvLyyes2lSH4QXJWOQwGFLo08RDKlenHoeg7AVaKr5f8rJhr+gPzNsT3CUjjJYyvPK7dAU1Io8xlvlVca+CB0ZDUfKBFaxE0/uYe9HnIs39tbf2HwBn9sX+i/ljKB4P8BhMxTMtH/DukV9DE8UeYILR/Z9fvX775v2HdzcfoGB9jxWr573aJzsoKNjVaO2Pr8LxVTAdX8PzZXgdTPHhOoSmlyMdpSZsD3n+YTRSqNcCFeDHXxImErki1Jc91IsDGwMyRndhj1VaLzHwH+IS6oTyTJPoGOp9k69hWcHFh++TZYMLKt9AC45qXuzSJKMKhBWLmld3YoMlYt+BzxU5LGV7yPHDDmkp1+s6ZC+g5gtpfb+ImKf5/RutUIQLo8qSbMMXsLQuIS4eGP+hTWEMcJXWSx/AGYsjxa+POYxpR/MFs4fG6ICx/5BUUGPZ8H0vCmwkw+gCTy3vkDjepvnm/9Pawt4IY9vcMOIH6JQThXohhyJr6pmuzRf93KrnbZYp/Ycn3DlaRuyZSlhyCYkaniy3ShRuzBOVbR0fiTXZUIhXtTANt0kN0ZSDqyUtZJITtvb34SHArQFN+dEy5s/ly0ctLvR72t43udTEMPCQUhFTEp8QFhGHzf1Da8YZviyqldhVsaXaHsoCdKx5jTGyC76G6f/lARIhjnHw6nHCHroBn32hLYVR2Np5IZTQTgl43u6glmu4FGfmCRmX2yJdcm8+83Q64s0DaVzahSlF7nxEQlyzWmSSNMx/EMBq1VBvtHDYMooukbAOyyPeJXdvbkjUDxZfb4tCFgZQIm0ymhC8Snc4TPdb7CsLMNYCeoq2KdsGRxv8ChYFmFWbNq23jEbCdAw9PE5k0B12QLi5S9IM964N9vXExpUKOZEXBCd5DLFhjhu2td1d2OZpd5d2QS/WlEU8eYqdnFe0bc6rI6FjcPvKkCJPdlTpGOGpK4xo7ycxSyUsfbyFFxyLQzfQ2NZk30WKuxyM9icadPmavfaTcEFRSPJ9NBeCtfc1KNCCcq/91/7DHrzz8BjQh57rML3RhWQQhkEXdZIba5RNzrC/5IlUK4gcYHO+6uifwVdw3PBczF061QhZo44nQibYy8HYJfs45/cx9dcDIwLLGVCgAleR8AWFUMoMi11eF1U99cpGjsR92mzFiUuU4nJFayJK4gfWRnlJW+skYtRJbGWsFxdSgNBqtsWe2q826KqIa0oGp79LsprbnWWyEkhxupp2Cka8qLtmG8HsOQsBCoh4CSsNn36oWoO75bKa0Iqjtr6wzewyVPafeWkOLUAeAltUb5OSz67mbDIPWX2blnFd8iUEIGUQ5KWcz0wDYkhoYVo95RUmirmBQK6hEzIxlgITJRt0EqgYB6WzNJKVMLg7eIriSB94hGBsXEhmsJhumq0ClMcMxt6GnfrkjkIjpx4SpnEV8JiHZzUGyKAy5FUD2lgaqeLfsNWLHnedK9paKmSsLjUBWfSh6ws4nEnJnqAUX3EspN5EnS8lEtjo/6k1uSNzSmCRe6ynpvL3DS0UuOM9PBnENrAyDTHCqLCGOOhbotZBaOgCL0K0kK2aQ8mnAi0r8o0QPGkg6tDoKpLAGfhm6S33O356GwVy8dAyhM5utATaMToKM0KcMMuq846pCH26pyPQyXeUwNX5kTIrNmlTq0ip1Zt2T6FmGO+S+nbavQaRwDaJ4Qq00IOBLXWxbsh7CHg2AZGhbqb4skp3UyibRzpm1/Zke6ZtoYitMOzbTo520hMHUiJwKtqCtgZC4LrW1POmqFOVegrdEnEOJ/DGeEZuzw9o0oMmnNY0UPcs2Hf0Qynh3FnNurlqrQEPvQ1xz5z73sSOT2SKsI9TtzsUDuUBlHVWJI3viBzhoV8QDCDveJI/iY1Aw+gi2HmUIPTQcLizHt7j8FonDaUWpyRLF+gqq//j0hQyI7fSRaq7WAH6ioYQkvXps5ZGh7WMn1UKE+BcQoaEg9QGZ5chcYx3JULBlJ5xsv2Ylr4BI7txqlSgtlhrJ9a+r0NwpgZ3zv5mahDX7c4GJZ3V3NQ1r5CKOK+xHFrpLF6WW5yCK5YsIUDX0pq1uReNmkyp2PVd4WyomUcmtHx5zmyh7U4H3XC3HpG+XRyQ8V/Fx5y1w2xsCJuL1Xcs2GhDDc4x3FuKi3UMbgmVc+ML/zR31tAbqS+mjQBZoBFcfMsPNJ2wmj9mPvsOAtTmUCLmGzzLEFeBRBCmyEu8IVTgfY5pluwWq4RJpv5Y9M7ofT7r+NNNEmgKbCUlm4j6fEsDvO5xJdVv8xSifUxrUrpU2wMKXuqKefcBbKKuks26S05dhBHKiVtLOEUlAT1TqYsuf0hq5qyTnZgzdXef7Hkl70PJMRXw1nED2k8ABXjT4CsbfVHx5JZaIOw0ic4wtilUvybur0zU3hmTMgxYkAh1vgGeSQ3OebF1knRCvb6KA4c0Z2lqa6s1fqFUlg6Co6bs6AyQID5Xl5ySNBfluD46pvMWdHp1zzG6qTYt7nvQiXLlBwYYHovFiez3vfGYYj+eSOHeX8VXsiA7gVHxuxR3O56FtEvydM3r5llIIjg8C4XSpzHNLMCjhFzOmnXSZs308gkp9+MGktL6FC4g4KySJMRBP7aBnUciN8N0ruYNTuS42cL4r2CNhYH7MmRFHS3LFgJcm+O+Cgz1VVceCQvFi0ND0xbvkPpIOFJduJ0EtQYB+DYWwOPlVciQiZlFS57eNkUpnARyL7RSrHZfJY99TKqjVzstVDXR6ZvwWdzTpCRfspl5ss2bz1z6E+Q7l8XVulClAd1eldqJ7EO51ZQa1Vsw6tJdzEwA3brCGtG1uBKmEaz1OcTvcykSiKTmXpPtEdW1kqZu3A4Y4hIaqTq4Q2zWhpTRvrwOjULnPkav2PFdDCJszB0YQ9wI9zB9NZoJrldYPaQ7HuGH7MGzHJGXdeEeRxGdcYGpnRiqoW3zWlYt0ysjcDZ0bqYuCfviE1PrOxmpkBIsRTvxik/mxrdAj/XGs9p/t3ffFXgXaJ29cXcLhDSUO/Ce3oCFpMTef5dUTCLG2gtEcjz+soM69RhHDZqaNpik21sOlEhlUWSCB2ZS0745dLdTe8ghFCcquOE7sqsbHMp0BeWNJilanDrMg6iV5pS+QbSBwKsx3B4Xsy/k5JQCGvtxZC/WKn+kxdpLlk16x3EbHK/dFTt8KhMYN2h0VmqRXdH1B7fo6q2vznS0KmQqZvp3vJzDW6maOhKFINZJPh+oMC0/PE7d2kC3wQLnwkmnI+LibBhMg53UN3BKnftnYYdOne1QK+5nawrqG+koD51JHomPKBBsyZ9BAtaEKyJBF+ldA0zNrY5z5LnXMqmXZ+ALAxG2eNRuLI5ez3XHfh1tHU07c8gMW4FTZYdDh74DkUJAHRl4U/ijzoOm8TQZsoJ+HQKzxs4UwBo2i641PmcLedxHDeLG4JkNZ0aiI/FHnvCITdsTB1T9azenYwn78qvAOdWB7HGlsknzvqgWYWgqrCvOLR/W4GQK/XoOMjHHWrIbVimXsXV+lhR0/cAlAn4hPD44W5YV0aAd+3TNtDB6D5+OXPx+3iFg53r4+XJbxOeO5zOmqzmSxOP4UDoiDLqxA0P6O21H0frDZwo0OIYuCXvgLH36ozfI3x4yR4Cnx81AMAbPuiZhpInnXKfQ+L3Ece2dvszhJJEWdZf4X782DNyjCcIhkcOhKxEDi0HNM4idwF+AqxQXD0ucXEHtcZkqDO90mRBqt+v43uSTQoE4ljVnQzL3s/VlAcZJ80QUxnZK7KucG1PRLgE/mHVG/16OJcSxJJ4OTsCtT9zjUQKKMxbDAI/2bJFmVMpqfWjWOFaQqriNoNJ8iKqUoZu9/rnUAnHe3DNxMCi7XmfNBj1X97H8CtIhll8xwjOsa+OgMqf6XXxLTMSUsIcV9MZe3UjpoRLBAUSxpamPJvqjL8d9MrAUS58YCuND5WavALSpGLG3vzEo5LR2UX0LPWSDjT2T6SctjrOHDPqc3lzuzKw2mDWpv2Xv8CvVnCV0cYcJ768BDoM3VJJ1I68P1oW8gEi+wZoUoJb0jWyDWH2oG75LFM46uSvEJVG1eWvGvshwxUYt4OjkJ+rRgP09bcs6mp6yjdMyU8wmc/ai1zlRvXPjary8NfW81WB0RsHqXlI7kZUMuYZFy9HkCKMjhWvQ17a3mWLTl8cduLftelkwQO1YFWPbNjzC3k1PTKqiOjBbjgDbpY0t2EB1owFOFTjnyf9UseMi0fjKYzrbtkcQu+yoP1BHMMzSym4zEboIBs/mvSA0obkba21Cev/gifsNcucxCAf2s9g6a+utuiFG04bjUTHFdn0/1izrRWUnxjG0u8Sm02CXrAJln6Zs1LuAQaWKZKBTbqdDJNKqURE2wDV1C6LDG8zr7fYO1l7/EchyZMFL3lhrd7uE8q8HOiYVtyryUlykwHsc4tvozhde5vLnEPAr7dAk7f84sm5F6nzMox9uAEa33oR5/9ruyt9iAxO/5MCS5Q9tKu6lMPxVB9Ma8n5uUcVQT9firknv9yEMeHRE3GbDe+rIzLwugTqJeIrXYvAcf5Es0ixtDgzWq7amJYbnzRiH4AS0KZ84J5sMHgF43fXXuDsck5D98wAArovszobF0xC8K2mAwegnO1xixWERAOGNma5VXWWh4elaaZDorKBrqq3rLl53NLROMx4Dy+t//AXQlj+EEokG9wAp2vL9Kt1Agz9Iq4YFHRbtpaanuuiHDTzRHOMVMvAahAL2PFt5JjXj0EhZz2gyAHNxLOWJy1X03RWTjPB0YS98Mk0PsLiZDX/MVjBvugNjcHQiBDDOVNhYnbWYzprfpVWR47EhQDtb9OLrnNCufjcmEi3Ksd3dYI9OhZArnQ7FCi6OXTjjZ14API7FT73EsW93BcM0HsWjOXlj/Nka66hRtAcuDLoT3q7f3a7Syhcv4nYsfj8aMsC4uDUOZ01M+i6y+JoRnVDid6VreW1Y3JbIm+l1IL4r4MlrxDCzp17brMdfed31pdwi8DA0I4dGf2h85aXlXs/80ZAIbyVDRh/HeOE+jumXGuIYT+Dj2JvI01c8jh/9LwhedBmMSAAA \ No newline at end of file diff --git a/experiments/jumpbench/neural_dev_manifest.json b/experiments/jumpbench/neural_dev_manifest.json new file mode 100644 index 0000000..857a9e8 --- /dev/null +++ b/experiments/jumpbench/neural_dev_manifest.json @@ -0,0 +1 @@ +{"benchmark":"JumpBench neural acquisition development pilot","seed":26081970,"candidate_grammar_size":2222,"target_cost_tiers":[9,11],"n_targets":4,"evidence_budget":6,"excluded_target_count":295,"records":[{"task_id":"N0000","target_pool_index":296,"definition_cost":9,"target_expression":"x*y + y*y + x + 2*y","target_polynomial":[[0,1,2],[0,2,1],[1,0,1],[1,1,1]],"evidence":{"active":[{"x":3,"y":-3,"value":-3},{"x":-2,"y":3,"value":7},{"x":3,"y":2,"value":17},{"x":3,"y":3,"value":27},{"x":3,"y":1,"value":9},{"x":3,"y":0,"value":3}],"random":[{"x":-1,"y":-1,"value":-1},{"x":-3,"y":2,"value":-1},{"x":2,"y":0,"value":2},{"x":0,"y":0,"value":0},{"x":2,"y":1,"value":7},{"x":2,"y":-3,"value":-1}],"passive":[{"x":-1,"y":-1,"value":-1},{"x":1,"y":0,"value":1},{"x":0,"y":0,"value":0},{"x":1,"y":-1,"value":-1},{"x":0,"y":1,"value":3},{"x":-1,"y":0,"value":-1}]},"candidate_options":[{"label":"A","pool_index":296,"expression":"x*y + y*y + x + 2*y"},{"label":"B","pool_index":1369,"expression":"x*y + 2*y*y + x - y"},{"label":"C","pool_index":655,"expression":"2*x*y + y*y + x - y"},{"label":"D","pool_index":579,"expression":"2*x*y + 2*y*y - 2*x - y"},{"label":"E","pool_index":1800,"expression":"x*x + x*y + 3*y"},{"label":"F","pool_index":62,"expression":"x*x + x*y + y"},{"label":"G","pool_index":1699,"expression":"x*x + 2*x*y + y*y - 2*x - y"},{"label":"H","pool_index":109,"expression":"2*x*y + 2*y*y - x"}],"correct_candidate_label":"A","query_choice":{"query_options":[{"label":"A","x":-2,"y":3},{"label":"B","x":-3,"y":2},{"label":"C","x":0,"y":3},{"label":"D","x":3,"y":0},{"label":"E","x":2,"y":1},{"label":"F","x":-1,"y":-1},{"label":"G","x":2,"y":-3},{"label":"H","x":1,"y":-2}],"correct_labels":["G"],"candidates":[{"pool_index":109,"expression":"2*x*y + 2*y*y - x"},{"pool_index":296,"expression":"x*y + y*y + x + 2*y"},{"pool_index":655,"expression":"2*x*y + y*y + x - y"},{"pool_index":1800,"expression":"x*x + x*y + 3*y"},{"pool_index":62,"expression":"x*x + x*y + y"},{"pool_index":1699,"expression":"x*x + 2*x*y + y*y - 2*x - y"},{"pool_index":579,"expression":"2*x*y + 2*y*y - 2*x - y"},{"pool_index":1369,"expression":"x*y + 2*y*y + x - y"}]}},{"task_id":"N0001","target_pool_index":1296,"definition_cost":11,"target_expression":"4*x*x + x - y","target_polynomial":[[0,1,-1],[1,0,1],[2,0,4]],"evidence":{"active":[{"x":3,"y":-3,"value":42},{"x":3,"y":0,"value":39},{"x":3,"y":3,"value":36},{"x":3,"y":2,"value":37},{"x":3,"y":1,"value":38},{"x":3,"y":-1,"value":40}],"random":[{"x":1,"y":1,"value":4},{"x":3,"y":-1,"value":40},{"x":-2,"y":3,"value":11},{"x":1,"y":2,"value":3},{"x":0,"y":2,"value":-2},{"x":-2,"y":1,"value":13}],"passive":[{"x":1,"y":-1,"value":6},{"x":0,"y":1,"value":-1},{"x":-1,"y":1,"value":2},{"x":1,"y":0,"value":5},{"x":-1,"y":0,"value":3},{"x":1,"y":1,"value":4}]},"candidate_options":[{"label":"A","pool_index":1312,"expression":"4*x*x - x + y"},{"label":"B","pool_index":1206,"expression":"3*x*x - x*y + 3*x - y"},{"label":"C","pool_index":1323,"expression":"4*y*y + x - y"},{"label":"D","pool_index":1296,"expression":"4*x*x + x - y"},{"label":"E","pool_index":1298,"expression":"4*x*x + y + 1"},{"label":"F","pool_index":1299,"expression":"4*x*x + y - 1"},{"label":"G","pool_index":1188,"expression":"3*x*x + y*y"},{"label":"H","pool_index":1295,"expression":"4*x*x + x + y"}],"correct_candidate_label":"D","query_choice":{"query_options":[{"label":"A","x":2,"y":-3},{"label":"B","x":1,"y":3},{"label":"C","x":0,"y":-1},{"label":"D","x":-3,"y":2},{"label":"E","x":1,"y":-2},{"label":"F","x":3,"y":-1},{"label":"G","x":-2,"y":-3},{"label":"H","x":-1,"y":-2}],"correct_labels":["G"],"candidates":[{"pool_index":1296,"expression":"4*x*x + x - y"},{"pool_index":1188,"expression":"3*x*x + y*y"},{"pool_index":1295,"expression":"4*x*x + x + y"},{"pool_index":1312,"expression":"4*x*x - x + y"},{"pool_index":1299,"expression":"4*x*x + y - 1"},{"pool_index":1298,"expression":"4*x*x + y + 1"},{"pool_index":1206,"expression":"3*x*x - x*y + 3*x - y"},{"pool_index":1323,"expression":"4*y*y + x - y"}]}},{"task_id":"N0002","target_pool_index":465,"definition_cost":9,"target_expression":"x*x - 2*x*y + y*y - 1","target_polynomial":[[0,0,-1],[0,2,1],[1,1,-2],[2,0,1]],"evidence":{"active":[{"x":3,"y":-3,"value":35},{"x":-2,"y":3,"value":24},{"x":3,"y":3,"value":-1},{"x":3,"y":2,"value":0},{"x":3,"y":1,"value":3},{"x":3,"y":0,"value":8}],"random":[{"x":1,"y":-2,"value":8},{"x":1,"y":-1,"value":3},{"x":2,"y":-3,"value":24},{"x":0,"y":-3,"value":8},{"x":1,"y":2,"value":0},{"x":0,"y":0,"value":-1}],"passive":[{"x":-1,"y":0,"value":0},{"x":1,"y":-1,"value":3},{"x":1,"y":1,"value":-1},{"x":-1,"y":1,"value":3},{"x":-1,"y":-1,"value":-1},{"x":0,"y":0,"value":-1}]},"candidate_options":[{"label":"A","pool_index":944,"expression":"2*x*x - 2*x*y - x + y - 1"},{"label":"B","pool_index":1924,"expression":"x*x - 2*x*y + y - 1"},{"label":"C","pool_index":465,"expression":"x*x - 2*x*y + y*y - 1"},{"label":"D","pool_index":947,"expression":"2*x*x - 2*x*y - x - y - 1"},{"label":"E","pool_index":199,"expression":"2*x*x - 2*x*y - 1"},{"label":"F","pool_index":1965,"expression":"x*x - 2*x*y - x + 3*y"},{"label":"G","pool_index":1914,"expression":"x*x - 2*x*y + 2*y - 1"},{"label":"H","pool_index":2090,"expression":"x*x - x*y - 3*x + 2*y + 2"}],"correct_candidate_label":"C","query_choice":{"query_options":[{"label":"A","x":-1,"y":1},{"label":"B","x":1,"y":-2},{"label":"C","x":2,"y":-2},{"label":"D","x":2,"y":0},{"label":"E","x":-3,"y":-2},{"label":"F","x":2,"y":3},{"label":"G","x":1,"y":3},{"label":"H","x":3,"y":-1}],"correct_labels":["H"],"candidates":[{"pool_index":1924,"expression":"x*x - 2*x*y + y - 1"},{"pool_index":944,"expression":"2*x*x - 2*x*y - x + y - 1"},{"pool_index":199,"expression":"2*x*x - 2*x*y - 1"},{"pool_index":2090,"expression":"x*x - x*y - 3*x + 2*y + 2"},{"pool_index":465,"expression":"x*x - 2*x*y + y*y - 1"},{"pool_index":1965,"expression":"x*x - 2*x*y - x + 3*y"},{"pool_index":1914,"expression":"x*x - 2*x*y + 2*y - 1"},{"pool_index":947,"expression":"2*x*x - 2*x*y - x - y - 1"}]}},{"task_id":"N0003","target_pool_index":793,"definition_cost":11,"target_expression":"2*x*x + 2*x - y - 1","target_polynomial":[[0,0,-1],[0,1,-1],[1,0,2],[2,0,2]],"evidence":{"active":[{"x":3,"y":-3,"value":26},{"x":2,"y":3,"value":8},{"x":3,"y":3,"value":20},{"x":3,"y":2,"value":21},{"x":3,"y":1,"value":22},{"x":3,"y":0,"value":23}],"random":[{"x":1,"y":-3,"value":6},{"x":3,"y":-1,"value":24},{"x":1,"y":0,"value":3},{"x":-2,"y":0,"value":3},{"x":2,"y":-1,"value":12},{"x":-3,"y":3,"value":8}],"passive":[{"x":0,"y":1,"value":-2},{"x":1,"y":0,"value":3},{"x":0,"y":0,"value":-1},{"x":-1,"y":0,"value":-1},{"x":0,"y":-1,"value":0},{"x":1,"y":-1,"value":4}]},"candidate_options":[{"label":"A","pool_index":1742,"expression":"x*x + 3*x - y + 1"},{"label":"B","pool_index":992,"expression":"2*x*x - x*y + 2*y"},{"label":"C","pool_index":860,"expression":"2*x*x + x - y + 1"},{"label":"D","pool_index":793,"expression":"2*x*x + 2*x - y - 1"},{"label":"E","pool_index":892,"expression":"2*x*x + x*y - x - y - 1"},{"label":"F","pool_index":179,"expression":"2*x*x + x + y"},{"label":"G","pool_index":911,"expression":"2*x*x + y*y - 1"},{"label":"H","pool_index":1214,"expression":"3*x*x - y - 1"}],"correct_candidate_label":"D","query_choice":{"query_options":[{"label":"A","x":-1,"y":1},{"label":"B","x":3,"y":-1},{"label":"C","x":3,"y":0},{"label":"D","x":1,"y":-3},{"label":"E","x":-1,"y":2},{"label":"F","x":-3,"y":0},{"label":"G","x":-3,"y":1},{"label":"H","x":-1,"y":3}],"correct_labels":["C","F"],"candidates":[{"pool_index":860,"expression":"2*x*x + x - y + 1"},{"pool_index":179,"expression":"2*x*x + x + y"},{"pool_index":1742,"expression":"x*x + 3*x - y + 1"},{"pool_index":793,"expression":"2*x*x + 2*x - y - 1"},{"pool_index":1214,"expression":"3*x*x - y - 1"},{"pool_index":892,"expression":"2*x*x + x*y - x - y - 1"},{"pool_index":911,"expression":"2*x*x + y*y - 1"},{"pool_index":992,"expression":"2*x*x - x*y + 2*y"}]}}],"sha256_without_sha_field":"cba344bd066f9555b3bcf62f92a490de394c124aa2797a8b7c0c4d9b6569501e"} \ No newline at end of file diff --git a/experiments/jumpbench/neural_eval.py b/experiments/jumpbench/neural_eval.py new file mode 100644 index 0000000..6466ff5 --- /dev/null +++ b/experiments/jumpbench/neural_eval.py @@ -0,0 +1,463 @@ +from __future__ import annotations + +import argparse +import ast +import hashlib +import json +import os +import platform +import re +import time +from pathlib import Path +from typing import Iterable + +import numpy as np +import torch +from huggingface_hub import model_info +from transformers import AutoModelForCausalLM, AutoTokenizer + +Poly = tuple[tuple[int, int, int], ...] +ZERO: Poly = () +ONE: Poly = ((0, 0, 1),) +X: Poly = ((1, 0, 1),) +Y: Poly = ((0, 1, 1),) +EVALUATOR_VERSION = "jumpbench-neural-v0.2-calibrated" + + +def from_dict(values: dict[tuple[int, int], int]) -> Poly: + return tuple(sorted((i, j, int(c)) for (i, j), c in values.items() if int(c))) + + +def as_dict(poly: Poly) -> dict[tuple[int, int], int]: + return {(i, j): c for i, j, c in poly} + + +def add(a: Poly, b: Poly) -> Poly: + out = as_dict(a) + for i, j, c in b: + out[(i, j)] = out.get((i, j), 0) + c + return from_dict(out) + + +def neg(a: Poly) -> Poly: + return tuple((i, j, -c) for i, j, c in a) + + +def sub(a: Poly, b: Poly) -> Poly: + return add(a, neg(b)) + + +def mul(a: Poly, b: Poly) -> Poly: + out: dict[tuple[int, int], int] = {} + for ai, aj, ac in a: + for bi, bj, bc in b: + key = (ai + bi, aj + bj) + out[key] = out.get(key, 0) + ac * bc + return from_dict(out) + + +def power(a: Poly, exponent: int) -> Poly: + result = ONE + for _ in range(exponent): + result = mul(result, a) + return result + + +def value(poly: Poly, x: int, y: int) -> int: + return int(sum(c * x**i * y**j for i, j, c in poly)) + + +def ast_to_poly(node: ast.AST) -> Poly: + if isinstance(node, ast.Expression): + return ast_to_poly(node.body) + if isinstance(node, ast.Name) and node.id in {"x", "y"}: + return X if node.id == "x" else Y + if isinstance(node, ast.Constant) and isinstance(node.value, int): + return ZERO if node.value == 0 else ((0, 0, int(node.value)),) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub): + return neg(ast_to_poly(node.operand)) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.UAdd): + return ast_to_poly(node.operand) + if isinstance(node, ast.BinOp): + left = ast_to_poly(node.left) + right = ast_to_poly(node.right) + if isinstance(node.op, ast.Add): + return add(left, right) + if isinstance(node.op, ast.Sub): + return sub(left, right) + if isinstance(node.op, ast.Mult): + return mul(left, right) + if isinstance(node.op, ast.Pow) and isinstance(node.right, ast.Constant): + exponent = int(node.right.value) + if 0 <= exponent <= 6: + return power(left, exponent) + raise ValueError(ast.dump(node)) + + +def parse_expression(text: str) -> Poly | None: + """Parse the first submitted expression; later self-contradictions do not overwrite it.""" + normalized = text.replace("²", "**2").replace("^", "**").replace("−", "-").replace("×", "*") + normalized = re.sub(r"\bxy\b", "x*y", normalized) + candidates: list[str] = [] + for match in re.finditer( + r"(?:Expression|Answer|f\s*\(\s*x\s*,\s*y\s*\))\s*[:=]\s*([^\n`]+)", + normalized, + re.I, + ): + candidates.append(match.group(1)) + candidates.extend(re.findall(r"```(?:python)?\s*([^`]+)```", normalized, re.I | re.S)) + candidates.extend(line for line in normalized.splitlines() if line.strip()) + candidates.append(normalized) + seen: set[str] = set() + for candidate in candidates: + candidate = candidate.strip().strip("`$ .;,") + if candidate in seen: + continue + seen.add(candidate) + if "=" in candidate: + candidate = candidate.split("=", 1)[-1].strip() + candidate = re.sub(r"\b([0-9]+)\s*([xy])\b", r"\1*\2", candidate) + candidate = candidate.split(" where ")[0].split(" because ")[0].strip() + if not candidate or len(candidate) > 300: + continue + try: + return ast_to_poly(ast.parse(candidate, mode="eval")) + except Exception: + continue + return None + + +def evidence_text(rows: list[dict]) -> str: + return "\n".join(f"f({row['x']},{row['y']})={row['value']}" for row in rows) + + +DEMONSTRATIONS = """Example 1 +f(-1,-1)=-2 +f(0,2)=2 +f(2,1)=3 +Expression: x + y + +Example 2 +f(-1,2)=-4 +f(0,-1)=1 +f(2,3)=3 +Expression: x*y - y +""" + + +def discovery_prompt(rows: list[dict]) -> str: + return ( + "Infer the exact integer polynomial from observations. Use only x, y, integer constants, +, -, and *. " + "Write only one algebraically equivalent expression on the first line.\n\n" + + DEMONSTRATIONS + + "\nTarget\n" + + evidence_text(rows) + + "\nExpression:" + ) + + +def ranking_prompt(rows: list[dict]) -> str: + return ( + "Infer the exact integer polynomial f from the observations. " + "The continuation is one candidate expression.\n" + + evidence_text(rows) + + "\nExpression:" + ) + + +def ranking_null_prompt() -> str: + return "Infer the exact integer polynomial f. The continuation is one candidate expression.\nExpression:" + + +def teach_prompt(expression: str) -> str: + return ( + f"The exact rule has been taught: f(x,y) = {expression}.\n" + "Write only one algebraically equivalent expression on the first line.\nExpression:" + ) + + +def teach_null_prompt() -> str: + return "An exact polynomial rule has been taught. Write one algebraically equivalent rule.\nExpression:" + + +def query_prompt(record: dict) -> str: + hypotheses = "\n".join( + f"H{index + 1}: {candidate['expression']}" + for index, candidate in enumerate(record["query_choice"]["candidates"]) + ) + options = "\n".join( + f"evaluate at ({option['x']},{option['y']})" + for option in record["query_choice"]["query_options"] + ) + return ( + "Choose the single experiment whose possible outputs best distinguish these polynomial hypotheses.\n" + + hypotheses + + "\nAvailable experiments:\n" + + options + + "\nThe best experiment is evaluate at" + ) + + +def query_null_prompt() -> str: + return "Choose one experiment. The best experiment is evaluate at" + + +def transfer_prompt(expression: str, x: int, y: int) -> str: + renamed = expression.replace("x", "a").replace("y", "b") + return ( + f"A reusable binary operator is M(a,b) = {renamed}. " + f"Compute M(M({x},{y}),{y}). Write only the integer.\nAnswer:" + ) + + +def transfer_null_prompt() -> str: + return "Compute the requested integer. Write only the integer.\nAnswer:" + + +def generate(model, tokenizer, prompt: str, max_new_tokens: int) -> str: + encoded = tokenizer(prompt, return_tensors="pt") + with torch.inference_mode(): + output = model.generate( + **encoded, + max_new_tokens=max_new_tokens, + do_sample=False, + pad_token_id=tokenizer.eos_token_id, + eos_token_id=tokenizer.eos_token_id, + use_cache=True, + ) + return tokenizer.decode(output[0, encoded["input_ids"].shape[1] :], skip_special_tokens=True) + + +def continuation_scores(model, tokenizer, prompt: str, continuations: list[str]) -> list[dict]: + prompt_ids = tokenizer(prompt, add_special_tokens=True)["input_ids"] + sequences: list[list[int]] = [] + prompt_lengths: list[int] = [] + for continuation in continuations: + continuation_ids = tokenizer(" " + continuation, add_special_tokens=False)["input_ids"] + sequences.append(prompt_ids + continuation_ids) + prompt_lengths.append(len(prompt_ids)) + max_length = max(len(sequence) for sequence in sequences) + pad_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else tokenizer.eos_token_id + input_ids = torch.full((len(sequences), max_length), pad_id, dtype=torch.long) + attention = torch.zeros_like(input_ids) + for row, sequence in enumerate(sequences): + input_ids[row, : len(sequence)] = torch.tensor(sequence) + attention[row, : len(sequence)] = 1 + with torch.inference_mode(): + logits = model(input_ids=input_ids, attention_mask=attention).logits + log_probs = torch.log_softmax(logits[:, :-1, :], dim=-1) + outputs = [] + for row, sequence in enumerate(sequences): + start = prompt_lengths[row] + labels = input_ids[row, start : len(sequence)] + positions = torch.arange(start - 1, len(sequence) - 1) + token_log_probs = log_probs[row, positions, labels] + outputs.append( + { + "continuation": continuations[row], + "sum_logprob": float(token_log_probs.sum()), + "mean_logprob": float(token_log_probs.mean()), + "tokens": int(token_log_probs.numel()), + } + ) + return outputs + + +def calibrated_scores(model, tokenizer, prompt: str, continuations: list[str], null_prompt: str) -> list[dict]: + conditional = continuation_scores(model, tokenizer, prompt, continuations) + prior = continuation_scores(model, tokenizer, null_prompt, continuations) + outputs = [] + for conditional_item, prior_item in zip(conditional, prior, strict=True): + if conditional_item["tokens"] != prior_item["tokens"]: + raise AssertionError("continuation tokenization changed across prompts") + item = dict(conditional_item) + item["null_mean_logprob"] = prior_item["mean_logprob"] + item["calibrated_mean_logprob"] = conditional_item["mean_logprob"] - prior_item["mean_logprob"] + item["calibrated_sum_logprob"] = conditional_item["sum_logprob"] - prior_item["sum_logprob"] + outputs.append(item) + return outputs + + +def rank_of_correct(scores: list[dict], correct_index: int, score_key: str = "calibrated_mean_logprob") -> int: + ordering = sorted(range(len(scores)), key=lambda index: (-scores[index][score_key], index)) + return ordering.index(correct_index) + 1 + + +def unique_numeric_options(correct: int, decoys: Iterable[int]) -> list[str]: + values = [correct] + for value_ in decoys: + if value_ not in values: + values.append(value_) + if len(values) == 8: + break + delta = 1 + while len(values) < 8: + for candidate in (correct + delta, correct - delta): + if candidate not in values: + values.append(candidate) + if len(values) == 8: + break + delta += 1 + return [str(value_) for value_ in values] + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--manifest", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--start-index", type=int, default=0) + parser.add_argument("--max-tasks", type=int, default=0) + args = parser.parse_args() + + torch.set_num_threads(min(4, os.cpu_count() or 1)) + manifest_bytes = Path(args.manifest).read_bytes() + manifest = json.loads(manifest_bytes) + stop = args.start_index + args.max_tasks if args.max_tasks else None + records = manifest["records"][args.start_index:stop] + info = model_info(args.model, revision=args.revision) + tokenizer = AutoTokenizer.from_pretrained(args.model, revision=args.revision) + model = AutoModelForCausalLM.from_pretrained( + args.model, + revision=args.revision, + torch_dtype=torch.float32, + low_cpu_mem_usage=True, + ) + model.eval() + started = time.time() + rows = [] + + for task_number, record in enumerate(records, start=1): + target = tuple(tuple(int(v) for v in term) for term in record["target_polynomial"]) + options = record["candidate_options"] + continuations = [option["expression"] for option in options] + correct_index = next( + index for index, option in enumerate(options) + if option["pool_index"] == record["target_pool_index"] + ) + row: dict = { + "task_id": record["task_id"], + "definition_cost": record["definition_cost"], + "target_pool_index": record["target_pool_index"], + } + + for condition in ("active", "random", "passive"): + scores = calibrated_scores( + model, + tokenizer, + ranking_prompt(record["evidence"][condition]), + continuations, + ranking_null_prompt(), + ) + calibrated_rank = rank_of_correct(scores, correct_index) + raw_rank = rank_of_correct(scores, correct_index, "mean_logprob") + row[f"recognition_{condition}_rank"] = calibrated_rank + row[f"recognition_{condition}_top1"] = int(calibrated_rank == 1) + row[f"recognition_{condition}_raw_rank"] = raw_rank + row[f"recognition_{condition}_scores"] = scores + + teach_scores = calibrated_scores( + model, tokenizer, teach_prompt(record["target_expression"]), continuations, teach_null_prompt() + ) + teach_rank = rank_of_correct(teach_scores, correct_index) + row["teach_rank"] = teach_rank + row["teach_top1"] = int(teach_rank == 1) + row["teach_raw_rank"] = rank_of_correct(teach_scores, correct_index, "mean_logprob") + row["teach_scores"] = teach_scores + + for condition in ("active", "passive"): + generation = generate(model, tokenizer, discovery_prompt(record["evidence"][condition]), 48) + parsed = parse_expression(generation) + row[f"free_{condition}_generation"] = generation + row[f"free_{condition}_parseable"] = int(parsed is not None) + row[f"free_{condition}_exact"] = int(parsed == target) + row[f"free_{condition}_parsed"] = None if parsed is None else [list(term) for term in parsed] + + echo_generation = generate(model, tokenizer, teach_prompt(record["target_expression"]), 48) + echo_parsed = parse_expression(echo_generation) + row["teach_echo_generation"] = echo_generation + row["teach_echo_parseable"] = int(echo_parsed is not None) + row["teach_echo_exact"] = int(echo_parsed == target) + row["teach_echo_parsed"] = None if echo_parsed is None else [list(term) for term in echo_parsed] + + query_options = record["query_choice"]["query_options"] + query_continuations = [f"({option['x']},{option['y']})" for option in query_options] + query_scores = calibrated_scores( + model, tokenizer, query_prompt(record), query_continuations, query_null_prompt() + ) + selected_query_index = max( + range(len(query_scores)), key=lambda index: query_scores[index]["calibrated_mean_logprob"] + ) + selected_query = query_options[selected_query_index] + correct_coordinates = { + (option["x"], option["y"]) + for option in query_options + if option["label"] in record["query_choice"]["correct_labels"] + } + row["query_selected_coordinate"] = [selected_query["x"], selected_query["y"]] + row["query_correct"] = int((selected_query["x"], selected_query["y"]) in correct_coordinates) + row["query_scores"] = query_scores + + x_value, y_value = 1, 2 + inner = value(target, x_value, y_value) + correct_transfer = value(target, inner, y_value) + decoys = [] + for option in options: + parsed_option = parse_expression(option["expression"]) + if parsed_option is not None: + decoys.append(value(parsed_option, value(parsed_option, x_value, y_value), y_value)) + numeric_options = unique_numeric_options(correct_transfer, decoys) + transfer_scores = calibrated_scores( + model, + tokenizer, + transfer_prompt(record["target_expression"], x_value, y_value), + numeric_options, + transfer_null_prompt(), + ) + transfer_correct_index = numeric_options.index(str(correct_transfer)) + transfer_rank = rank_of_correct(transfer_scores, transfer_correct_index) + row["transfer_rank"] = transfer_rank + row["transfer_top1"] = int(transfer_rank == 1) + row["transfer_raw_rank"] = rank_of_correct(transfer_scores, transfer_correct_index, "mean_logprob") + row["transfer_correct_value"] = correct_transfer + row["transfer_options"] = numeric_options + row["transfer_scores"] = transfer_scores + rows.append(row) + print(args.model, task_number, "/", len(records), record["task_id"], flush=True) + + metrics = [ + "recognition_active_top1", "recognition_random_top1", "recognition_passive_top1", + "teach_top1", "free_active_parseable", "free_active_exact", "free_passive_parseable", + "free_passive_exact", "teach_echo_parseable", "teach_echo_exact", "query_correct", "transfer_top1", + ] + summary = {key: float(np.mean([row[key] for row in rows])) for key in metrics} + output = { + "benchmark": "JumpBench neural acquisition v0.2", + "evaluator_version": EVALUATOR_VERSION, + "scoring_rule": "conditional mean token log probability minus content-free mean token log probability", + "model": args.model, + "requested_revision": args.revision, + "resolved_revision": info.sha, + "parameter_count": sum(parameter.numel() for parameter in model.parameters()), + "manifest_file_sha256": hashlib.sha256(manifest_bytes).hexdigest(), + "manifest_semantic_sha256": manifest.get("sha256_without_sha_field"), + "start_index": args.start_index, + "n_tasks": len(rows), + "summary": summary, + "rows": rows, + "runtime_seconds": time.time() - started, + "environment": { + "python": platform.python_version(), + "torch": torch.__version__, + "transformers": __import__("transformers").__version__, + }, + } + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"model": args.model, "summary": summary, "runtime_seconds": output["runtime_seconds"]}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/jumpbench/neural_probe.py b/experiments/jumpbench/neural_probe.py new file mode 100644 index 0000000..e09ebd1 --- /dev/null +++ b/experiments/jumpbench/neural_probe.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import argparse +import json +import os +import platform +import time +from pathlib import Path + +import torch +from huggingface_hub import model_info +from transformers import AutoModelForCausalLM, AutoTokenizer + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--revision", required=True) + parser.add_argument("--output", required=True) + return parser.parse_args() + + +def generate(model, tokenizer, prompt: str, max_new_tokens: int = 96) -> str: + encoded = tokenizer(prompt, return_tensors="pt") + with torch.inference_mode(): + output = model.generate( + **encoded, + max_new_tokens=max_new_tokens, + do_sample=False, + pad_token_id=tokenizer.eos_token_id, + eos_token_id=tokenizer.eos_token_id, + use_cache=True, + ) + continuation = output[0, encoded["input_ids"].shape[1] :] + return tokenizer.decode(continuation, skip_special_tokens=True) + + +def main() -> None: + args = parse_args() + torch.set_num_threads(min(4, os.cpu_count() or 1)) + started = time.time() + info = model_info(args.model, revision=args.revision) + tokenizer = AutoTokenizer.from_pretrained(args.model, revision=args.revision) + load_start = time.time() + model = AutoModelForCausalLM.from_pretrained( + args.model, + revision=args.revision, + torch_dtype=torch.float32, + low_cpu_mem_usage=True, + ) + model.eval() + load_seconds = time.time() - load_start + + prompts = { + "arithmetic": "Complete the answer with only the result. 17 + 28 =", + "repeat_expression": ( + "A newly discovered binary operator is defined by " + "M(x,y)=x*x+x*y-y. Repeat only the expression after the equals sign.\n" + "M(x,y)=" + ), + "apply_taught_expression": ( + "Use the exact rule f(x,y)=x*x+x*y-y. Compute f(2,-1). " + "Write only the integer answer.\nAnswer:" + ), + "infer_expression": ( + "An unknown polynomial f uses only x, y, integers, +, -, and *. " + "It has these exact values:\n" + "f(0,0)=0\n" + "f(1,0)=1\n" + "f(0,1)=-1\n" + "f(1,1)=1\n" + "f(2,-1)=3\n" + "Write one expression for f(x,y), and nothing else.\n" + "f(x,y)=" + ), + "choose_candidate": ( + "An unknown function has exact values f(0,0)=0, f(1,0)=1, " + "f(0,1)=-1, f(1,1)=1, and f(2,-1)=3. Choose the correct rule.\n" + "A. x*x+x*y-y\n" + "B. x*x-x*y+y\n" + "C. x*y+x-y\n" + "D. x*x+y*y-y\n" + "Write only A, B, C, or D.\nAnswer:" + ), + } + generations = {} + for name, prompt in prompts.items(): + t0 = time.time() + text = generate(model, tokenizer, prompt) + generations[name] = { + "prompt": prompt, + "generation": text, + "seconds": time.time() - t0, + } + print(f"[{name}] {text!r}", flush=True) + + parameter_count = sum(parameter.numel() for parameter in model.parameters()) + payload = { + "model": args.model, + "requested_revision": args.revision, + "resolved_revision": info.sha, + "parameter_count": parameter_count, + "model_type": getattr(model.config, "model_type", None), + "architectures": getattr(model.config, "architectures", None), + "transformers_version": __import__("transformers").__version__, + "torch_version": torch.__version__, + "python": platform.python_version(), + "load_seconds": load_seconds, + "total_seconds": time.time() - started, + "generations": generations, + } + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + print(json.dumps({k: payload[k] for k in ("model", "resolved_revision", "parameter_count", "load_seconds", "total_seconds")}, indent=2)) + + +if __name__ == "__main__": + main()