diff --git a/.github/workflows/run-bot-aib-tournament.yaml b/.github/workflows/run-bot-aib-tournament.yaml index 167f37d0..2b70fd8b 100644 --- a/.github/workflows/run-bot-aib-tournament.yaml +++ b/.github/workflows/run-bot-aib-tournament.yaml @@ -308,19 +308,19 @@ jobs: INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_qwen_3_5: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_QWEN_3_5" - metac_name: "metac-qwen-3-5+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_qwen_3_5: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_QWEN_3_5" + # metac_name: "metac-qwen-3-5+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} bot_gemini_3_1_pro: needs: precache_asknews @@ -336,20 +336,20 @@ jobs: INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - bot_gpt_5_5: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GPT_5_5" - metac_name: "metac-gpt-5-5+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_gpt_5_5: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GPT_5_5" + # metac_name: "metac-gpt-5-5+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKENS: ${{ secrets.METACULUS_TOKENS }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # bot_gpt_5_4: # needs: precache_asknews @@ -439,20 +439,20 @@ jobs: #################################### February 2026 new bots #################################### - bot_claude_opus_4_6_high_32k: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_CLAUDE_OPUS_4_6_HIGH_32K" - metac_name: "metac-claude-opus-4-6-high-32k+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_OPUS_4_6_HIGH_32K }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_claude_opus_4_6_high_32k: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_CLAUDE_OPUS_4_6_HIGH_32K" + # metac_name: "metac-claude-opus-4-6-high-32k+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_CLAUDE_OPUS_4_6_HIGH_32K }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} # bot_claude_opus_4_6: # needs: precache_asknews @@ -483,19 +483,19 @@ jobs: INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_qwen3_max_thinking: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_QWEN3_MAX_THINKING" - metac_name: "metac-qwen3-max-thinking+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_QWEN3_MAX_THINKING }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_qwen3_max_thinking: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_QWEN3_MAX_THINKING" + # metac_name: "metac-qwen3-max-thinking+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_QWEN3_MAX_THINKING }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # bot_minimax_m2_5: # needs: precache_asknews @@ -510,33 +510,33 @@ jobs: # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - bot_kimi_k2_5_high: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_KIMI_K2_5_HIGH" - metac_name: "metac-kimi-k2-5+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_KIMI_K2_5_HIGH }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_kimi_k2_5_high: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_KIMI_K2_5_HIGH" + # metac_name: "metac-kimi-k2-5+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_KIMI_K2_5_HIGH }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - bot_glm_5: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GLM_5" - metac_name: "metac-glm-5+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GLM_5 }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_glm_5: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GLM_5" + # metac_name: "metac-glm-5+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GLM_5 }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} #################################### December 2025 new bots #################################### @@ -628,19 +628,19 @@ jobs: # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - bot_glm_4_6: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GLM_4_6" - metac_name: "metac-glm-4-6+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GLM_4_6 }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_glm_4_6: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GLM_4_6" + # metac_name: "metac-glm-4-6+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GLM_4_6 }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # bot_llama_3_1_405b_instruct: # Disabled - low performance and non-negligible cost # needs: precache_asknews @@ -656,19 +656,19 @@ jobs: #################################### November 2025 new bots #################################### - bot_kimi_k2_high: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_KIMI_K2_HIGH" - metac_name: "metac-kimi-k2-high+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_KIMI_K2_HIGH }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_kimi_k2_high: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_KIMI_K2_HIGH" + # metac_name: "metac-kimi-k2-high+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_KIMI_K2_HIGH }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # bot_gpt_5_1_high: # needs: precache_asknews @@ -727,35 +727,35 @@ jobs: # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - bot_grok_4_1_fast_high: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GROK_4_1_FAST_HIGH" - metac_name: "metac-grok-4-1-fast-high+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4_1_FAST_HIGH }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_grok_4_1_fast_high: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GROK_4_1_FAST_HIGH" + # metac_name: "metac-grok-4-1-fast-high+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4_1_FAST_HIGH }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - bot_grok_4_1_fast: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GROK_4_1_FAST" - metac_name: "metac-grok-4-1-fast+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4_1_FAST }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_grok_4_1_fast: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GROK_4_1_FAST" + # metac_name: "metac-grok-4-1-fast+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_4_1_FAST }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # bot_qwen_3_max_high: # needs: precache_asknews @@ -813,19 +813,19 @@ jobs: # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_qwen_3_max: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_QWEN_3_MAX" - metac_name: "metac-qwen-3-max+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_QWEN_3_MAX }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_qwen_3_max: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_QWEN_3_MAX" + # metac_name: "metac-qwen-3-max+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_QWEN_3_MAX }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} bot_deepseek_3_2_reasoning: needs: precache_asknews @@ -994,19 +994,19 @@ jobs: # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_kimi_k2: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_KIMI_K2" - metac_name: "metac-kimi-k2+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_KIMI_K2 }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_kimi_k2: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_KIMI_K2" + # metac_name: "metac-kimi-k2+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_KIMI_K2 }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # The name of the bot in Metaculus is weird, and this just isn't needed much # bot_kimi_k2_variance_test: @@ -1470,18 +1470,18 @@ jobs: # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} # INPUT_PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }} - bot_deepseek_r1_gpt_4o_search_preview: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_DEEPSEEK_R1_GPT_4O_SEARCH_PREVIEW" - metac_name: "metac-deepseek-r1+gpt-4o-search-preview" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_GPT_4O_SEARCH_PREVIEW }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # bot_deepseek_r1_gpt_4o_search_preview: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_DEEPSEEK_R1_GPT_4O_SEARCH_PREVIEW" + # metac_name: "metac-deepseek-r1+gpt-4o-search-preview" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_DEEPSEEK_R1_GPT_4O_SEARCH_PREVIEW }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} # bot_deepseek_r1_gemini_2_5_pro_grounding: # needs: precache_asknews @@ -1561,35 +1561,35 @@ jobs: # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_o4_mini_high: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_O4_MINI_HIGH_TOKEN" - metac_name: "metac-o4-mini-high+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_O4_MINI_HIGH_TOKEN }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_o4_mini_high: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_O4_MINI_HIGH_TOKEN" + # metac_name: "metac-o4-mini-high+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_O4_MINI_HIGH_TOKEN }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - bot_o4_mini: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_O4_MINI_TOKEN" - metac_name: "metac-o4-mini+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_O4_MINI_TOKEN }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # bot_o4_mini: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_O4_MINI_TOKEN" + # metac_name: "metac-o4-mini+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_O4_MINI_TOKEN }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} bot_4_1: needs: precache_asknews @@ -1895,20 +1895,20 @@ jobs: INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} - bot_grok_3_latest: - needs: precache_asknews - uses: ./.github/workflows/run-bot-launcher.yaml - with: - bot_name: "METAC_GROK_3_LATEST_TOKEN" - metac_name: "metac-grok-3+asknews" - cache_key: asknews-cache-${{ github.run_id }} - secrets: - INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_3_LATEST_TOKEN }} - INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} - INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} - INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} + # bot_grok_3_latest: + # needs: precache_asknews + # uses: ./.github/workflows/run-bot-launcher.yaml + # with: + # bot_name: "METAC_GROK_3_LATEST_TOKEN" + # metac_name: "metac-grok-3+asknews" + # cache_key: asknews-cache-${{ github.run_id }} + # secrets: + # INPUT_METACULUS_TOKEN: ${{ secrets.METAC_GROK_3_LATEST_TOKEN }} + # INPUT_METACULUS_API_BASE_URL: ${{ secrets.METACULUS_API_BASE_URL }} + # INPUT_XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + # INPUT_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + # INPUT_ASKNEWS_CLIENT_ID: ${{ secrets.ASKNEWS_CLIENT_ID }} + # INPUT_ASKNEWS_SECRET: ${{ secrets.ASKNEWS_SECRET }} # bot_grok_3_mini_latest_high: # needs: precache_asknews diff --git a/code_tests/unit_tests/test_data_models/test_markdown_tree.py b/code_tests/unit_tests/test_data_models/test_markdown_tree.py index 62b8420c..02197012 100644 --- a/code_tests/unit_tests/test_data_models/test_markdown_tree.py +++ b/code_tests/unit_tests/test_data_models/test_markdown_tree.py @@ -431,6 +431,12 @@ def test_heading_level_skipped_if_no_level_1_header(self) -> None: class TestReportSectionsToMarkdown: + def test_report_sections_to_markdown_empty_list_no_heading_adjust(self) -> None: + assert MarkdownTree.report_sections_to_markdown([], None) == "" + + def test_report_sections_to_markdown_empty_list_with_heading_level(self) -> None: + assert MarkdownTree.report_sections_to_markdown([], 3) == "" + @pytest.mark.parametrize( "top_heading_level, expected_output", [ diff --git a/forecasting_tools/data_models/markdown_tree.py b/forecasting_tools/data_models/markdown_tree.py index 10c2fa9b..845c5c87 100644 --- a/forecasting_tools/data_models/markdown_tree.py +++ b/forecasting_tools/data_models/markdown_tree.py @@ -30,6 +30,9 @@ def report_sections_to_markdown( report_sections: list[MarkdownTree], top_heading_level: int | None = None, ) -> str: + if not report_sections: + return "" + if top_heading_level is None: return "\n".join( [section.text_of_section_and_subsections for section in report_sections] diff --git a/forecasting_tools/forecast_bots/official_bots/gpt_4_1_optimized_bot.py b/forecasting_tools/forecast_bots/official_bots/gpt_4_1_optimized_bot.py index a6cdef59..24b619c8 100644 --- a/forecasting_tools/forecast_bots/official_bots/gpt_4_1_optimized_bot.py +++ b/forecasting_tools/forecast_bots/official_bots/gpt_4_1_optimized_bot.py @@ -20,7 +20,6 @@ class GPT41OptimizedBot(FallTemplateBot2025): async def run_research(self, question: MetaculusQuestion) -> str: async with self._concurrency_limiter: - research = "" researcher = self.get_llm("researcher") if researcher != "asknews/news-summaries": logger.warning( diff --git a/pyproject.toml b/pyproject.toml index 679357ea..e863bb01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "forecasting-tools" -version = "0.2.89" +version = "0.2.90" description = "AI forecasting and research tools to help humans reason about and forecast the future" authors = ["Benjamin Wilson "] license = "MIT" diff --git a/run_bots.py b/run_bots.py index 50b12bdd..6abd3af2 100644 --- a/run_bots.py +++ b/run_bots.py @@ -50,6 +50,8 @@ 41362, # https://www.metaculus.com/questions/41362/ is rejected because common cause of errors (confusion on units?) 42495, # https://www.metaculus.com/questions/42495/ is rejected since too many MC options 42557, # https://www.metaculus.com/questions/42557/ is rejected since too many group question options that lower quality models fail at + 43310, # https://www.metaculus.com/questions/43310/ is rejected since too many MC options + 40280, # https://www.metaculus.com/questions/40280/ is rejected since noisy workflow errors ] @@ -751,7 +753,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR temperature=default_temperature, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_GEMINI_3_1_PRO": { "estimated_cost_per_question": roughly_gemini_2_5_pro_preview_cost, @@ -773,7 +775,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR timeout=gpt_5_timeout, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_GPT_5_4": { "estimated_cost_per_question": roughly_gpt_5_cost, @@ -827,7 +829,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR **claude_thinking_settings_32k, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_CLAUDE_OPUS_4_6": { "estimated_cost_per_question": roughly_opus_4_5_cost, @@ -857,7 +859,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR temperature=default_temperature, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_MINIMAX_M2_5": { "estimated_cost_per_question": roughly_deepseek_r1_cost, @@ -882,7 +884,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR timeout=kimi_k2_timeout, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_GLM_5": { "estimated_cost_per_question": roughly_deepseek_r1_cost, @@ -892,7 +894,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR temperature=default_temperature, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, ############################ Bots started in December 2025 ############################ "METAC_CLAUDE_OPUS_4_5_HIGH_32K": { @@ -967,7 +969,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR temperature=default_temperature, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_LLAMA_3_1_405B_INSTRUCT": { "estimated_cost_per_question": roughly_deepseek_r1_cost * 3, @@ -989,7 +991,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR timeout=kimi_k2_timeout, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_GPT_5_1_HIGH": { "estimated_cost_per_question": roughly_gpt_5_high_cost, @@ -1037,7 +1039,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR temperature=default_temperature, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_GROK_4_1_FAST": { "estimated_cost_per_question": guess_at_deepseek_v3_1_cost, @@ -1047,7 +1049,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR temperature=default_temperature, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, # "METAC_QWEN_3_MAX_HIGH": {} -> Accidentally made bot w/o realizing that qwen-3-max-thinking is just an upgrade to the base model (not a new parameter) # "METAC_DEEPSEEK_R1_CP_ENABLED": {}, # TODO: Make a framework for this bot @@ -1080,7 +1082,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR temperature=default_temperature, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_DEEPSEEK_3_2_REASONING": { "estimated_cost_per_question": guess_at_deepseek_v3_1_cost * 1.2, @@ -1220,7 +1222,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR }, "METAC_KIMI_K2": { **kimi_k2_basic_bot, - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_KIMI_K2_VARIANCE_TEST": { **kimi_k2_basic_bot, @@ -1571,7 +1573,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR model="openai/gpt-4o-search-preview", temperature=None ), ), - "tournaments": TournConfig.aib_only, + "tournaments": TournConfig.NONE, }, "METAC_DEEPSEEK_R1_GEMINI_2_5_PRO_GROUNDING": { "estimated_cost_per_question": guess_at_deepseek_plus_search, @@ -1643,7 +1645,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR # **flex_price_settings, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_O4_MINI_TOKEN": { "estimated_cost_per_question": 0.043, @@ -1655,7 +1657,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR # **flex_price_settings, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_4_1_TOKEN": { "estimated_cost_per_question": 0.07, @@ -1888,7 +1890,7 @@ def get_default_bot_dict() -> dict[str, RunBotConfig]: # NOSONAR temperature=default_temperature, ), ), - "tournaments": TournConfig.aib_and_site, + "tournaments": TournConfig.NONE, }, "METAC_GROK_3_MINI_LATEST_HIGH_TOKEN": { "estimated_cost_per_question": 0.10,