From f4f2600c8e90cbdddacb4c40b89196a2b140b3a0 Mon Sep 17 00:00:00 2001 From: wlnc Date: Mon, 25 May 2026 15:20:15 +0200 Subject: [PATCH 1/8] switch to ioh --- agents/rl_das/env.py | 4 ++-- baselines.py | 45 +++++++++++++++++++++++++++++++----- das/env/das_env.py | 2 +- das/training/common.py | 46 ++++++++++++++++++++++++++++++++----- das/training/expdas.py | 13 ++++++----- das/training/rldas.py | 8 +++---- pyproject.toml | 3 +-- tests/test_baselines.py | 38 +++++++++++++++++++++++++++++- uv.lock | 51 +++++++++++++++++------------------------ 9 files changed, 152 insertions(+), 58 deletions(-) diff --git a/agents/rl_das/env.py b/agents/rl_das/env.py index 38653ae..d438147 100644 --- a/agents/rl_das/env.py +++ b/agents/rl_das/env.py @@ -166,7 +166,7 @@ def _pop_features( class RLDASEnv(gym.Env): - """RL-DAS environment wrapping BBOB problems via a cocoex Suite. + """RL-DAS environment wrapping optimization problems via an IOHSuite. Uses a Population object as shared warm-started state across all DE sub-optimizers (matching the original RL-DAS design). @@ -176,7 +176,7 @@ class RLDASEnv(gym.Env): problem_ids: BBOB problem IDs to cycle through (one per episode). suite: - cocoex Suite object. + IOHSuite object. optimizers: List of instantiated DE optimizer objects (NL_SHADE_RSP, JDE21, MadDE). dim: diff --git a/baselines.py b/baselines.py index 0cb7dc9..be029be 100644 --- a/baselines.py +++ b/baselines.py @@ -36,15 +36,20 @@ import os import warnings -import cocoex import numpy as np from tqdm import tqdm from das.env.das_env import DASEnv +from das.env.ioh_suite import IOHSuite from das.optimizers.portfolio import get_portfolio from das.utils import set_seed from das.env.bbob_splits import ALL_DIMS, get_train_test_split -from das.training.common import compute_run_stats, load_global_optima +from das.training.common import ( + compute_run_stats, + load_global_optima, + ERT_TARGETS, + _ert_key, +) warnings.filterwarnings("ignore") @@ -212,6 +217,8 @@ def compute_oracle(all_results: dict[str, list[dict]]) -> tuple[list[dict], list ], "aocc": best_m["aocc"], "final_fitness": best_m["final_fitness"], + "hitting_times": best_m.get("hitting_times", {}), + "max_fe": best_m.get("max_fe", 0), "agent": "oracle-best", "best_agent": best_m["agent"], } @@ -225,6 +232,8 @@ def compute_oracle(all_results: dict[str, list[dict]]) -> tuple[list[dict], list ], "aocc": worst_m["aocc"], "final_fitness": worst_m["final_fitness"], + "hitting_times": worst_m.get("hitting_times", {}), + "max_fe": worst_m.get("max_fe", 0), "agent": "oracle-worst", "worst_agent": worst_m["agent"], } @@ -241,12 +250,29 @@ def compute_oracle(all_results: dict[str, list[dict]]) -> tuple[list[dict], list # ------------------------------------------------------------------ # +def _ert_for_target(records: list[dict], target_key: str) -> float | None: + """ERT = total_FEs / n_successful_runs (unsuccessful runs contribute max_fe).""" + total_fe = 0 + n_succ = 0 + for r in records: + m = next(iter(r.values())) + ht = m.get("hitting_times", {}).get(target_key) + mfe = m.get("max_fe", 0) + if ht is not None: + total_fe += ht + n_succ += 1 + else: + total_fe += mfe + return float(total_fe / n_succ) if n_succ > 0 else None + + def summarise(tag: str, records: list[dict]) -> dict: fitnesses = [next(iter(r.values()))["final_fitness"] for r in records] aocc_vals = [next(iter(r.values()))["aocc"] for r in records] auoc_vals = [ next(iter(r.values()))["area_under_optimization_curve"] for r in records ] + ert = {_ert_key(t): _ert_for_target(records, _ert_key(t)) for t in ERT_TARGETS} return { "agent": tag, "n_problems": len(fitnesses), @@ -256,6 +282,7 @@ def summarise(tag: str, records: list[dict]) -> dict: "worst_final_fitness": float(np.max(fitnesses)), "mean_aocc": float(np.mean(aocc_vals)), "mean_auoc": float(np.mean(auoc_vals)), + "ert": ert, } @@ -266,16 +293,23 @@ def save_results(records: list[dict], path: str) -> None: def print_summary(summaries: list[dict]) -> None: + _ERT_PRINT_TARGET = "1e-04" width = max(len(s["agent"]) for s in summaries) + 2 - header = f" {'Agent':<{width}} {'Mean fitness':>14} {'Median fitness':>14} {'Mean AUOC':>14}" + header = ( + f" {'Agent':<{width}} {'Mean fitness':>14} {'Median fitness':>14}" + f" {'Mean AUOC':>14} {'ERT(1e-04)':>12}" + ) print(header) print(" " + "-" * (len(header) - 2)) for s in summaries: + ert_val = s.get("ert", {}).get(_ERT_PRINT_TARGET) + ert_str = f"{ert_val:>12.1f}" if ert_val is not None else f"{'inf':>12}" print( f" {s['agent']:<{width}} " f"{s['mean_final_fitness']:>14.4e} " f"{s['median_final_fitness']:>14.4e} " - f"{s['mean_auoc']:>14.4e}" + f"{s['mean_auoc']:>14.4e} " + f"{ert_str}" ) @@ -334,8 +368,7 @@ def main(): optimizers = get_portfolio(args.portfolio) opt_names = args.portfolio - cocoex.utilities.MiniPrint() - suite = cocoex.Suite("bbob", "", "") + suite = IOHSuite() _, test_ids = get_train_test_split(args.mode, args.dims) global_optima = load_global_optima() diff --git a/das/env/das_env.py b/das/env/das_env.py index 22a51d0..1ab820d 100644 --- a/das/env/das_env.py +++ b/das/env/das_env.py @@ -29,7 +29,7 @@ class DASEnv(gym.Env): problem_ids: BBOB problem IDs to cycle through (one per episode). suite: - cocoex Suite object to fetch problems from. + IOHSuite (or compatible) object to fetch problems from. optimizers: Ordered list of sub-optimizer classes (defines the action space). fe_multiplier: diff --git a/das/training/common.py b/das/training/common.py index 63e6a73..326c989 100644 --- a/das/training/common.py +++ b/das/training/common.py @@ -15,12 +15,21 @@ def load_global_optima(path: str = "bbob_optima.jsonl") -> dict[str, float]: return {k: v for line in f for k, v in json.loads(line).items()} +# Standard BBOB precision targets (excess above the global optimum). +# Keys are used as JSON dict keys: "1e+02", "1e+01", ..., "1e-08". +ERT_TARGETS: tuple[float, ...] = tuple(10.0**k for k in range(2, -9, -1)) + + +def _ert_key(t: float) -> str: + return f"{t:.0e}" + + def compute_run_stats( fitness_history: list[tuple[int, float]], max_fe: int, global_minimum: float, -) -> dict[str, float]: - """Compute AUOC and AOCC from a best-so-far improvement history. +) -> dict: + """Compute AUOC, AOCC, hitting times and ERT budget from a run history. Parameters ---------- @@ -32,9 +41,23 @@ def compute_run_stats( global_minimum: Known optimum value; used to shift fitness so the excess is >= 0. Pass 0.0 when the optimum is unknown. + + Returns + ------- + dict with keys: + area_under_optimization_curve, aocc, final_fitness, + hitting_times – {target_key: first_fe | null} for each ERT_TARGET, + max_fe – the budget (needed for ERT aggregation). """ + empty_hits = {_ert_key(t): None for t in ERT_TARGETS} if not fitness_history or max_fe <= 0: - return {"area_under_optimization_curve": 0.0, "aocc": 0.0, "final_fitness": 0.0} + return { + "area_under_optimization_curve": 0.0, + "aocc": 0.0, + "final_fitness": 0.0, + "hitting_times": empty_hits, + "max_fe": max_fe, + } lb, ub = 1e-8, 1e8 log_lb, log_ub = -8.0, 8.0 @@ -63,10 +86,22 @@ def compute_run_stats( final_normalized = (np.log10(final_clipped) - log_lb) / (log_ub - log_lb) aocc_area += (1.0 - final_normalized) * final_width + # Hitting times: first FE where shifted best_y ≤ target + hitting_times: dict[str, int | None] = {} + for target in ERT_TARGETS: + hit_fe = None + for fe, best_y in fitness_history: + if best_y - global_minimum <= target: + hit_fe = fe + break + hitting_times[_ert_key(target)] = hit_fe + return { "area_under_optimization_curve": auoc_area / max_fe, "aocc": aocc_area / max_fe, "final_fitness": final_shifted, + "hitting_times": hitting_times, + "max_fe": max_fe, } @@ -74,10 +109,9 @@ def make_das_env(problem_ids: list[str], optimizers: list, cfg: dict): """Return a zero-argument factory for use with SB3's make_vec_env.""" def _init(): - import cocoex as _cx + from das.env.ioh_suite import IOHSuite - _cx.utilities.MiniPrint() - _suite = _cx.Suite("bbob", "", "") + _suite = IOHSuite() return DASEnv( problem_ids=problem_ids, suite=_suite, diff --git a/das/training/expdas.py b/das/training/expdas.py index 62eb0db..215e2f8 100644 --- a/das/training/expdas.py +++ b/das/training/expdas.py @@ -4,7 +4,6 @@ import os from concurrent.futures import ProcessPoolExecutor, as_completed -import cocoex as cx import numpy as np from das.env.bbob_splits import get_cv_folds, get_train_test_split @@ -30,7 +29,9 @@ def run_exp_das(args) -> None: f" n_epochs={args.n_epochs} total_episodes={total_episodes}" ) - suite = cx.Suite("bbob", "", "") + from das.env.ioh_suite import IOHSuite + + suite = IOHSuite() env_cfg = dict( suite=suite, @@ -98,13 +99,13 @@ def _run_single_fold( ) -> tuple[int, str, list[dict]]: """Train and evaluate one CV fold. Safe to call in a subprocess. - Each call creates its own cocoex Suite so the (non-picklable) Suite object - never crosses process boundaries. + Each call creates its own IOHSuite so no shared mutable state crosses + process boundaries. Returns (fold_idx, fold_tag, fold_results). """ # Lazy imports so the function can be pickled by ProcessPoolExecutor. - import cocoex as _cx + from das.env.ioh_suite import IOHSuite as _IOHSuite from agents.exponential_das import ExpDASAgent from agents.exponential_das import train as _train, evaluate as _evaluate @@ -121,7 +122,7 @@ def _run_single_fold( obs_dim = observation_dim(n_opt) buffer_capacity = args.buffer_capacity or (16 * args.n_checkpoints) - suite = _cx.Suite("bbob", "", "") + suite = _IOHSuite() env_cfg = dict( suite=suite, optimizers=optimizers, diff --git a/das/training/rldas.py b/das/training/rldas.py index 00e859f..a197786 100644 --- a/das/training/rldas.py +++ b/das/training/rldas.py @@ -10,7 +10,7 @@ def run_rl_das(args) -> None: - import cocoex as cx + from das.env.ioh_suite import IOHSuite from agents.rl_das import RLDASEnv, PPOAgent from agents.rl_das import train, evaluate from agents.rl_das.optimizers import get_rldas_portfolio @@ -20,7 +20,7 @@ def run_rl_das(args) -> None: train_ids, test_ids = get_train_test_split(args.mode, [args.dim]) print(f"Train: {len(train_ids)} problems | Test: {len(test_ids)} problems") - suite = cx.Suite("bbob", "", "") + suite = IOHSuite() if args.k_epoch is None: args.k_epoch = max(1, int(0.3 * args.n_checkpoints)) @@ -71,14 +71,14 @@ def run_rl_das(args) -> None: def run_cv_rl_das(args) -> None: - import cocoex as cx + from das.env.ioh_suite import IOHSuite from agents.rl_das import RLDASEnv, PPOAgent from agents.rl_das import train, evaluate from agents.rl_das.optimizers import get_rldas_portfolio optimizers = get_rldas_portfolio(args.portfolio) - suite = cx.Suite("bbob", "", "") + suite = IOHSuite() if args.k_epoch is None: args.k_epoch = max(1, int(0.3 * args.n_checkpoints)) diff --git a/pyproject.toml b/pyproject.toml index e244ac6..0e45f7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,8 +6,7 @@ authors = [{ name = "wniec", email = "niecwladek@gmail.com" }] requires-python = ">=3.11,<3.12" dependencies = [ "pypop7>=0.0.82,<0.0.83", - "coco-experiment>=2.8.1,<3", - "cocopp>=2.8.0,<3", + "ioh>=0.3.0", "stable-baselines3>=2.3.0", "gymnasium>=0.29.0", "tqdm>=4.67.1,<5", diff --git a/tests/test_baselines.py b/tests/test_baselines.py index a17cf49..cbe5438 100644 --- a/tests/test_baselines.py +++ b/tests/test_baselines.py @@ -59,7 +59,13 @@ def get_problem(self, problem_id: str) -> MockProblem: N_INDIVIDUALS = 10 PORTFOLIO = ["SPSO", "IPSO"] -METRICS_KEYS = {"area_under_optimization_curve", "aocc", "final_fitness"} +METRICS_KEYS = { + "area_under_optimization_curve", + "aocc", + "final_fitness", + "hitting_times", + "max_fe", +} def make_env(problem_ids=PROBLEM_IDS, suite=None): @@ -174,6 +180,36 @@ def test_aocc_no_convergence(self): stats = compute_run_stats(history, 1000, global_minimum=0.0) assert stats["aocc"] < 0.1 + def test_fitness_history_fe_increasing_fitness_decreasing(self): + # Run a real optimizer on MockProblem with a tiny budget so that + # the fitness_history invariant is checked on actual optimizer output. + problem = MockProblem("bbob_f001_i01_d02", dim=2) + optimizer_class = get_portfolio(["SPSO"])[0] + max_fe = 50 * problem.dimension + problem_config = { + "fitness_function": problem, + "ndim_problem": problem.dimension, + "lower_boundary": problem.lower_bounds, + "upper_boundary": problem.upper_bounds, + } + options = { + "max_function_evaluations": max_fe, + "target_fe": max_fe, + "n_individuals": 10, + "verbose": False, + } + result = optimizer_class(problem_config, options).optimize() + if isinstance(result, tuple): + result = result[0] + history = result.get("fitness_history", []) + assert len(history) > 0, "optimizer produced no fitness_history" + fes = [fe for fe, _ in history] + fits = [y for _, y in history] + assert fes == sorted(fes), "FE counts must be strictly increasing" + assert fits == sorted(fits, reverse=True), ( + "fitness values must be non-increasing" + ) + # ------------------------------------------------------------------ # # 1. Policy functions # diff --git a/uv.lock b/uv.lock index d601eb3..284d3ab 100644 --- a/uv.lock +++ b/uv.lock @@ -119,32 +119,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d4/d4/ec46cedab6a6145e21768baa8110db3e2e836a320d8499e4ef18bc894e61/cma-4.4.4-py3-none-any.whl", hash = "sha256:edb6d02eb2aac2d54650f16a8f0c70711ff17445957de7c9de92ff7fd4b7ef38", size = 328311, upload-time = "2026-02-25T22:18:09.602Z" }, ] -[[package]] -name = "coco-experiment" -version = "2.8.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a4/67/53fe6d644b34f6b84d644132d2961106b2acd4c14aa41f847f608c82892c/coco_experiment-2.8.2.tar.gz", hash = "sha256:df044be950027d20e30e4d7edf187ed991f96eeb4242e8d5c89ea2719f0a50f2", size = 3919117, upload-time = "2025-11-27T15:05:28.479Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/50/1a11c8e15244b03d19f685c0d7713778e0e1e02643f11f14bcec4d68d71e/coco_experiment-2.8.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c897170eb7a645c39a9b89a283b6b2cbbbb1bdbfe227c2d1b31f92a6e98873b8", size = 565882, upload-time = "2025-11-27T15:05:04.898Z" }, - { url = "https://files.pythonhosted.org/packages/5c/18/6c52a0de7d5c65f5bd50ab116babb04fc47e8d9acfe62043347b9ee58bfb/coco_experiment-2.8.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c99d956df654fd8fee77391b196b9a35d34912131170dd31ab5d8c8479979f7", size = 762758, upload-time = "2025-11-27T15:05:06.653Z" }, - { url = "https://files.pythonhosted.org/packages/70/6d/9d6035675bf4e33d1f5bb458a349f933330821d0a4df87c33bcabb3a600d/coco_experiment-2.8.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a83a58aaaf664cfeb0a3d0c6f80f6667c0c80c12f3005537dd71d9686e88c87a", size = 770966, upload-time = "2025-11-27T15:05:08.046Z" }, - { url = "https://files.pythonhosted.org/packages/9e/6f/2e5d52d5e57e175168a4cca037819d96e48d47dc71aef9e82752a4f9554c/coco_experiment-2.8.2-cp311-cp311-win_amd64.whl", hash = "sha256:0b945a08271655f0121b5f4181d1eb8deced060d9dde551a0c97d58cf140758a", size = 425934, upload-time = "2025-11-27T15:05:09.602Z" }, -] - -[[package]] -name = "cocopp" -version = "2.8.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "matplotlib" }, - { name = "numpy" }, - { name = "platformdirs" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/93/3f/4a7afb5ebca617d683f8c21c711f724f012ce66322be7f14b3a239cd4b72/cocopp-2.8.4.tar.gz", hash = "sha256:bd9af937483960b91b8fae429074f9f173429a91579064bef4375a7b27390b60", size = 8629374, upload-time = "2026-04-08T21:58:47.657Z" } - [[package]] name = "colorama" version = "0.4.6" @@ -287,9 +261,8 @@ name = "dynamicalgorithmselection2" version = "0.1.0" source = { editable = "." } dependencies = [ - { name = "coco-experiment" }, - { name = "cocopp" }, { name = "gymnasium" }, + { name = "ioh" }, { name = "matplotlib" }, { name = "numpy" }, { name = "pandas" }, @@ -311,9 +284,8 @@ dev = [ [package.metadata] requires-dist = [ - { name = "coco-experiment", specifier = ">=2.8.1,<3" }, - { name = "cocopp", specifier = ">=2.8.0,<3" }, { name = "gymnasium", specifier = ">=0.29.0" }, + { name = "ioh", specifier = ">=0.3.0" }, { name = "matplotlib", specifier = ">=3.8,<4" }, { name = "numpy", specifier = ">=1.20.0,<2.0" }, { name = "pandas", specifier = ">=2.0.3" }, @@ -461,6 +433,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "ioh" +version = "0.3.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/ed/dc4a133cc6c5533ec039c60db99f75d9f856f80bccb4f681bad9aff00f4b/ioh-0.3.18.tar.gz", hash = "sha256:7f9eb288c237241faa5316f8d5285e763c376b1a5ac7bbfb3966b76946fd3039", size = 7981547, upload-time = "2024-11-20T15:36:37.715Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/52/9dde2589457cf88c81d0a817d29a67c5f147211fb2cab76cfe6245800bde/ioh-0.3.18-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:6f7e33401a33d8419cbd7bb30e2778445846eecd76d809c5ee0a7dfc6229bf6a", size = 8247555, upload-time = "2024-11-20T15:34:17.44Z" }, + { url = "https://files.pythonhosted.org/packages/28/8d/b0aae06f7251d8752ed253a48d059290ef36d1d47ebdee2908bda842feda/ioh-0.3.18-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cc2fba55b6dca791870a6606eeb66300a40bd13ad8e9d78a91b768d2adb3334e", size = 8154997, upload-time = "2024-11-20T15:34:19.59Z" }, + { url = "https://files.pythonhosted.org/packages/72/4d/5795a415a842d99d6cea6549c86bc38dc5321f02113662d3bb4709f69b1b/ioh-0.3.18-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ebf582a8eb1275f25aab07006000861536634cea40766ffb203d098b7895ded5", size = 8583170, upload-time = "2024-11-20T15:34:22.699Z" }, + { url = "https://files.pythonhosted.org/packages/18/dd/730e9ecdba89282d73fda29866f53389a733afcc9e47cc2661746cb5ecc7/ioh-0.3.18-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae5ca03e9f8ea50dcd24e0d0266c8867c4c7226bf8d5be427e5712bbbf2764c6", size = 8518719, upload-time = "2024-11-20T15:34:26.086Z" }, + { url = "https://files.pythonhosted.org/packages/f5/59/f2276f5d54602183e430672a24153f04e6353702d53b16e475e2bb3ed287/ioh-0.3.18-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:2450e079bc45447adb0e2e2d5ffc684afe8086fd7432adb7c0f57d9cc25d7bf4", size = 8934836, upload-time = "2024-11-20T15:34:29.264Z" }, + { url = "https://files.pythonhosted.org/packages/5e/d7/07dac1d548426fe45996b121d1b478ce0edcc62e92fc1d09878b775b48ec/ioh-0.3.18-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ba65633b12a3393376039bfec82da0c3f83b4b4271890108c84ecdf56e5a6860", size = 8865254, upload-time = "2024-11-20T15:34:32.957Z" }, + { url = "https://files.pythonhosted.org/packages/88/e9/6bc412ff1f39374151766faaaa0589587a2bd7239db526576607b367f618/ioh-0.3.18-cp311-cp311-win32.whl", hash = "sha256:e2b4beb410d7bea371c6a3a5248d331415c2291bd99026b3331b606f82d3f63b", size = 8003888, upload-time = "2024-11-20T15:34:35.481Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ac/fa6ef7c11993117711cf5fef3a1cb9825247886d1bd0a9a10ef33619eadd/ioh-0.3.18-cp311-cp311-win_amd64.whl", hash = "sha256:9e2bb09e6a5502bf8539596b5c8450a0d26f8bed4160a802c871ec7b3badc514", size = 8114617, upload-time = "2024-11-20T15:34:38.513Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" From cba3417a3c93dde09615119385894f5a7e775b8a Mon Sep 17 00:00:00 2001 From: wlnc Date: Thu, 28 May 2026 17:44:54 +0200 Subject: [PATCH 2/8] switch to ioh --- README.md | 65 +- agents/exponential_das/trainer.py | 7 +- baselines.py | 16 +- bbob_optima.jsonl | 2160 ----------------------------- das/env/das_env.py | 36 +- das/training/callbacks.py | 4 - das/training/common.py | 20 +- das/training/expdas.py | 9 +- das/training/ppo.py | 12 +- evaluate.py | 8 +- exp_das_study.slurm | 15 +- pyproject.toml | 5 + tests/test_baselines.py | 16 +- tests/test_exp_das_integration.py | 208 +++ train.py | 9 +- 15 files changed, 354 insertions(+), 2236 deletions(-) delete mode 100644 bbob_optima.jsonl create mode 100644 tests/test_exp_das_integration.py diff --git a/README.md b/README.md index 4e5cac7..eb10beb 100644 --- a/README.md +++ b/README.md @@ -302,14 +302,65 @@ Submit all agents for a given seed and portfolio: bash runner.sh ``` -Individual SLURM scripts: +Each script accepts positional arguments: `SEED [PORTFOLIO...]` (RL-DAS takes only `SEED`). -| Script | Agent | -|---|---| -| `ppo_study.slurm` | PPO | -| `rl_das_study.slurm` | RL-DAS | -| `exp_das_study.slurm` | Exp-DAS | -| `baselines.slurm` | Baselines | +```bash +sbatch baselines.slurm 42 CPSO NM TDE +sbatch ppo_study.slurm 42 CPSO NM TDE +sbatch rl_das_study.slurm 42 +sbatch exp_das_study.slurm 42 CPSO NM TDE +``` + +### `baselines.slurm` + +Single job (no array). Runs all baseline agent types (`random`, `fixed:*`, `single:*`, oracle) across all dimensions. + +### `ppo_study.slurm` — array 0–9 + +| Task | CV mode | Dimensions | +|------|---------|------------| +| 0 | LOIO | 2 | +| 1 | LOIO | 3 | +| 2 | LOIO | 5 | +| 3 | LOIO | 10 | +| 4 | LOPO | 2 | +| 5 | LOPO | 3 | +| 6 | LOPO | 5 | +| 7 | LOPO | 10 | +| 8 | LOIO | 2, 3, 5, 10 (multi-dim) | +| 9 | LOPO | 2, 3, 5, 10 (multi-dim) | + +### `rl_das_study.slurm` — array 0–7 + +Fixed DE portfolio (`NL_SHADE_RSP / MADDE / JDE21`). One model per dimension. + +| Task | CV mode | Dimension | +|------|---------|-----------| +| 0 | LOIO | 2 | +| 1 | LOIO | 3 | +| 2 | LOIO | 5 | +| 3 | LOIO | 10 | +| 4 | LOPO | 2 | +| 5 | LOPO | 3 | +| 6 | LOPO | 5 | +| 7 | LOPO | 10 | + +### `exp_das_study.slurm` — array 0–11 + +| Task | CV mode | Dimensions | +|------|---------|------------| +| 0 | LOIO | 2, 5, 10 (multi-dim) | +| 1 | LOPO | 2, 5, 10 (multi-dim) | +| 2 | LOIO | 2, 3, 5, 10 (multi-dim) | +| 3 | LOPO | 2, 3, 5, 10 (multi-dim) | +| 4 | LOIO | 2 | +| 5 | LOPO | 2 | +| 6 | LOIO | 3 | +| 7 | LOPO | 3 | +| 8 | LOIO | 5 | +| 9 | LOPO | 5 | +| 10 | LOIO | 10 | +| 11 | LOPO | 10 | --- diff --git a/agents/exponential_das/trainer.py b/agents/exponential_das/trainer.py index bcba617..5334e53 100644 --- a/agents/exponential_das/trainer.py +++ b/agents/exponential_das/trainer.py @@ -20,7 +20,7 @@ from agents.exponential_das.agent import ExpDASAgent from das.env.das_env import DASEnv -from das.training.common import compute_run_stats +from das.training.common import compute_run_stats, get_ioh_optimum def train( @@ -148,11 +148,8 @@ def evaluate( env: DASEnv, agent: ExpDASAgent, n_episodes: int = 20, - global_optima: dict[str, float] | None = None, ) -> list[dict]: """Run the agent deterministically and return per-episode results.""" - if global_optima is None: - global_optima = {} results = [] for _ in range(n_episodes): obs, info = env.reset() @@ -170,7 +167,7 @@ def evaluate( fitness_history.extend(step_info.get("fitness_history_step", [])) max_fe = step_info.get("n_fe", 0) - global_minimum = global_optima.get(problem_id, 0.0) + global_minimum = get_ioh_optimum(problem_id) stats = compute_run_stats(fitness_history, max_fe, global_minimum) results.append( { diff --git a/baselines.py b/baselines.py index be029be..c3dee18 100644 --- a/baselines.py +++ b/baselines.py @@ -46,7 +46,7 @@ from das.env.bbob_splits import ALL_DIMS, get_train_test_split from das.training.common import ( compute_run_stats, - load_global_optima, + get_ioh_optimum, ERT_TARGETS, _ert_key, ) @@ -100,7 +100,6 @@ def collect_env_results( suite, optimizers: list, cfg: dict, - global_optima: dict[str, float], ) -> list[dict]: """Run policy_fn on every problem in test_ids via DASEnv.""" env = DASEnv( @@ -117,7 +116,7 @@ def collect_env_results( for problem_id in tqdm(test_ids, desc=f" {agent_tag}", smoothing=0.0): step_info, fitness_history = run_episode(env, policy_fn) max_fe = step_info.get("n_fe", 0) - global_minimum = global_optima.get(problem_id, 0.0) + global_minimum = get_ioh_optimum(problem_id) stats = compute_run_stats(fitness_history, max_fe, global_minimum) results.append({problem_id: {**stats, "agent": agent_tag}}) env.close() @@ -169,13 +168,12 @@ def collect_single_results( suite, fe_multiplier: int, n_individuals: int, - global_optima: dict[str, float], ) -> list[dict]: """Run the optimizer independently on every problem in test_ids.""" results = [] for problem_id in tqdm(test_ids, desc=f" {agent_tag}", smoothing=0.0): problem = suite.get_problem(problem_id) - global_minimum = global_optima.get(problem_id, 0.0) + global_minimum = get_ioh_optimum(problem_id) stats = run_single_algorithm( optimizer_class, problem, fe_multiplier, n_individuals, global_minimum ) @@ -351,7 +349,7 @@ def parse_args(): p.add_argument("-s", "--n-checkpoints", type=int, default=10) p.add_argument("-x", "--cdb", type=float, default=1.0) p.add_argument("-O", "--reward-option", type=int, default=1, choices=[1, 2, 3, 4]) - p.add_argument("-n", "--n-individuals", type=int, default=100) + p.add_argument("-n", "--n-individuals", type=int, default=None) p.add_argument("--seed", type=int, default=42) return p.parse_args() @@ -370,8 +368,6 @@ def main(): opt_names = args.portfolio suite = IOHSuite() _, test_ids = get_train_test_split(args.mode, args.dims) - global_optima = load_global_optima() - cfg = { "fe_multiplier": args.fe_multiplier, "n_checkpoints": args.n_checkpoints, @@ -417,7 +413,7 @@ def main(): if tag == "random": records = collect_env_results( - tag, random_policy, test_ids, suite, optimizers, cfg, global_optima + tag, random_policy, test_ids, suite, optimizers, cfg ) elif tag.startswith("fixed:"): @@ -435,7 +431,6 @@ def main(): suite, optimizers, cfg, - global_optima, ) elif tag.startswith("single:"): @@ -448,7 +443,6 @@ def main(): suite, args.fe_multiplier, args.n_individuals, - global_optima, ) else: diff --git a/bbob_optima.jsonl b/bbob_optima.jsonl deleted file mode 100644 index ec089d8..0000000 --- a/bbob_optima.jsonl +++ /dev/null @@ -1,2160 +0,0 @@ -{"bbob_f001_i01_d02": 79.48} -{"bbob_f001_i02_d02": 394.48} -{"bbob_f001_i03_d02": -247.11} -{"bbob_f001_i04_d02": -152.04} -{"bbob_f001_i05_d02": -25.25} -{"bbob_f001_i71_d02": 183.01} -{"bbob_f001_i72_d02": 183.52} -{"bbob_f001_i73_d02": -593.84} -{"bbob_f001_i74_d02": -1000.0} -{"bbob_f001_i75_d02": -25.31} -{"bbob_f001_i76_d02": 22.28} -{"bbob_f001_i77_d02": 356.14} -{"bbob_f001_i78_d02": -475.7} -{"bbob_f001_i79_d02": -1000.0} -{"bbob_f001_i80_d02": -41.58} -{"bbob_f002_i01_d02": -209.88} -{"bbob_f002_i02_d02": -92.09} -{"bbob_f002_i03_d02": -87.89} -{"bbob_f002_i04_d02": 320.19} -{"bbob_f002_i05_d02": -104.24} -{"bbob_f002_i71_d02": -67.85} -{"bbob_f002_i72_d02": -363.98} -{"bbob_f002_i73_d02": -18.65} -{"bbob_f002_i74_d02": 174.92} -{"bbob_f002_i75_d02": -565.96} -{"bbob_f002_i76_d02": 70.95} -{"bbob_f002_i77_d02": 47.28} -{"bbob_f002_i78_d02": -568.88} -{"bbob_f002_i79_d02": 20.69} -{"bbob_f002_i80_d02": 561.67} -{"bbob_f003_i01_d02": -462.09} -{"bbob_f003_i02_d02": 77.66} -{"bbob_f003_i03_d02": 115.68} -{"bbob_f003_i04_d02": -3.71} -{"bbob_f003_i05_d02": 132.18} -{"bbob_f003_i71_d02": -39.3} -{"bbob_f003_i72_d02": 42.96} -{"bbob_f003_i73_d02": 284.22} -{"bbob_f003_i74_d02": -32.17} -{"bbob_f003_i75_d02": -58.58} -{"bbob_f003_i76_d02": -604.92} -{"bbob_f003_i77_d02": 133.7} -{"bbob_f003_i78_d02": -20.98} -{"bbob_f003_i79_d02": 135.33} -{"bbob_f003_i80_d02": 40.66} -{"bbob_f004_i01_d02": -462.09} -{"bbob_f004_i02_d02": 77.66} -{"bbob_f004_i03_d02": 115.68} -{"bbob_f004_i04_d02": -3.71} -{"bbob_f004_i05_d02": 132.18} -{"bbob_f004_i71_d02": -39.3} -{"bbob_f004_i72_d02": 42.96} -{"bbob_f004_i73_d02": 284.22} -{"bbob_f004_i74_d02": -32.17} -{"bbob_f004_i75_d02": -58.58} -{"bbob_f004_i76_d02": -604.92} -{"bbob_f004_i77_d02": 133.7} -{"bbob_f004_i78_d02": -20.98} -{"bbob_f004_i79_d02": 135.33} -{"bbob_f004_i80_d02": 40.66} -{"bbob_f005_i01_d02": -9.21} -{"bbob_f005_i02_d02": 655.99} -{"bbob_f005_i03_d02": 66.71} -{"bbob_f005_i04_d02": 941.67} -{"bbob_f005_i05_d02": -37.45} -{"bbob_f005_i71_d02": -66.39} -{"bbob_f005_i72_d02": -226.01} -{"bbob_f005_i73_d02": 1000.0} -{"bbob_f005_i74_d02": -183.02} -{"bbob_f005_i75_d02": -102.62} -{"bbob_f005_i76_d02": 99.53} -{"bbob_f005_i77_d02": 149.59} -{"bbob_f005_i78_d02": -1000.0} -{"bbob_f005_i79_d02": -29.22} -{"bbob_f005_i80_d02": -40.96} -{"bbob_f006_i01_d02": 35.9} -{"bbob_f006_i02_d02": 31.37} -{"bbob_f006_i03_d02": -1000.0} -{"bbob_f006_i04_d02": 131.92} -{"bbob_f006_i05_d02": 590.84} -{"bbob_f006_i71_d02": 67.49} -{"bbob_f006_i72_d02": 30.85} -{"bbob_f006_i73_d02": -12.55} -{"bbob_f006_i74_d02": 538.39} -{"bbob_f006_i75_d02": -105.56} -{"bbob_f006_i76_d02": 98.03} -{"bbob_f006_i77_d02": 1000.0} -{"bbob_f006_i78_d02": 1.49} -{"bbob_f006_i79_d02": 596.35} -{"bbob_f006_i80_d02": -118.97} -{"bbob_f007_i01_d02": 92.94} -{"bbob_f007_i02_d02": 35.35} -{"bbob_f007_i03_d02": 3.82} -{"bbob_f007_i04_d02": 21.75} -{"bbob_f007_i05_d02": 64.79} -{"bbob_f007_i71_d02": -1000.0} -{"bbob_f007_i72_d02": 164.07} -{"bbob_f007_i73_d02": -165.68} -{"bbob_f007_i74_d02": 92.84} -{"bbob_f007_i75_d02": -367.92} -{"bbob_f007_i76_d02": 49.5} -{"bbob_f007_i77_d02": -12.77} -{"bbob_f007_i78_d02": 63.65} -{"bbob_f007_i79_d02": 699.19} -{"bbob_f007_i80_d02": 1000.0} -{"bbob_f008_i01_d02": 149.15} -{"bbob_f008_i02_d02": -1000.0} -{"bbob_f008_i03_d02": 98.62} -{"bbob_f008_i04_d02": -47.15} -{"bbob_f008_i05_d02": 37.52} -{"bbob_f008_i71_d02": 6.74} -{"bbob_f008_i72_d02": -189.79} -{"bbob_f008_i73_d02": -27.74} -{"bbob_f008_i74_d02": 6.93} -{"bbob_f008_i75_d02": -44.56} -{"bbob_f008_i76_d02": 1000.0} -{"bbob_f008_i77_d02": -679.71} -{"bbob_f008_i78_d02": 1000.0} -{"bbob_f008_i79_d02": 10.07} -{"bbob_f008_i80_d02": -7.54} -{"bbob_f009_i01_d02": 123.83} -{"bbob_f009_i02_d02": 47.51} -{"bbob_f009_i03_d02": 55.34} -{"bbob_f009_i04_d02": -90.33} -{"bbob_f009_i05_d02": 65.61} -{"bbob_f009_i71_d02": 934.82} -{"bbob_f009_i72_d02": -63.71} -{"bbob_f009_i73_d02": 1000.0} -{"bbob_f009_i74_d02": 31.97} -{"bbob_f009_i75_d02": 58.19} -{"bbob_f009_i76_d02": -4.84} -{"bbob_f009_i77_d02": -63.37} -{"bbob_f009_i78_d02": 2.75} -{"bbob_f009_i79_d02": -46.81} -{"bbob_f009_i80_d02": 61.34} -{"bbob_f010_i01_d02": -54.94} -{"bbob_f010_i02_d02": 59.13} -{"bbob_f010_i03_d02": -491.53} -{"bbob_f010_i04_d02": -170.2} -{"bbob_f010_i05_d02": -469.75} -{"bbob_f010_i71_d02": -51.91} -{"bbob_f010_i72_d02": 48.08} -{"bbob_f010_i73_d02": 30.58} -{"bbob_f010_i74_d02": 1000.0} -{"bbob_f010_i75_d02": -42.36} -{"bbob_f010_i76_d02": -71.87} -{"bbob_f010_i77_d02": -16.5} -{"bbob_f010_i78_d02": 1000.0} -{"bbob_f010_i79_d02": -1000.0} -{"bbob_f010_i80_d02": -152.74} -{"bbob_f011_i01_d02": 76.27} -{"bbob_f011_i02_d02": -22.55} -{"bbob_f011_i03_d02": 13.66} -{"bbob_f011_i04_d02": 937.84} -{"bbob_f011_i05_d02": -78.72} -{"bbob_f011_i71_d02": -41.51} -{"bbob_f011_i72_d02": 132.07} -{"bbob_f011_i73_d02": 97.25} -{"bbob_f011_i74_d02": -20.33} -{"bbob_f011_i75_d02": 94.31} -{"bbob_f011_i76_d02": -143.69} -{"bbob_f011_i77_d02": -353.38} -{"bbob_f011_i78_d02": -2.91} -{"bbob_f011_i79_d02": -2.97} -{"bbob_f011_i80_d02": 299.63} -{"bbob_f012_i01_d02": -621.11} -{"bbob_f012_i02_d02": -254.82} -{"bbob_f012_i03_d02": 56.61} -{"bbob_f012_i04_d02": -18.1} -{"bbob_f012_i05_d02": 37.77} -{"bbob_f012_i71_d02": -632.55} -{"bbob_f012_i72_d02": -85.3} -{"bbob_f012_i73_d02": 171.95} -{"bbob_f012_i74_d02": 68.16} -{"bbob_f012_i75_d02": 1000.0} -{"bbob_f012_i76_d02": -68.37} -{"bbob_f012_i77_d02": -55.48} -{"bbob_f012_i78_d02": -53.49} -{"bbob_f012_i79_d02": 117.27} -{"bbob_f012_i80_d02": 77.89} -{"bbob_f013_i01_d02": 29.97} -{"bbob_f013_i02_d02": -51.71} -{"bbob_f013_i03_d02": -279.95} -{"bbob_f013_i04_d02": 931.99} -{"bbob_f013_i05_d02": 113.31} -{"bbob_f013_i71_d02": 28.49} -{"bbob_f013_i72_d02": -740.58} -{"bbob_f013_i73_d02": -51.77} -{"bbob_f013_i74_d02": -222.98} -{"bbob_f013_i75_d02": 5.86} -{"bbob_f013_i76_d02": 204.86} -{"bbob_f013_i77_d02": 115.49} -{"bbob_f013_i78_d02": 195.05} -{"bbob_f013_i79_d02": -214.48} -{"bbob_f013_i80_d02": -45.9} -{"bbob_f014_i01_d02": -52.35} -{"bbob_f014_i02_d02": -179.54} -{"bbob_f014_i03_d02": 77.31} -{"bbob_f014_i04_d02": 39.92} -{"bbob_f014_i05_d02": -56.61} -{"bbob_f014_i71_d02": 32.7} -{"bbob_f014_i72_d02": -39.43} -{"bbob_f014_i73_d02": 419.72} -{"bbob_f014_i74_d02": 7.64} -{"bbob_f014_i75_d02": -127.68} -{"bbob_f014_i76_d02": -36.5} -{"bbob_f014_i77_d02": 172.87} -{"bbob_f014_i78_d02": 157.72} -{"bbob_f014_i79_d02": 28.19} -{"bbob_f014_i80_d02": -99.25} -{"bbob_f015_i01_d02": 1000.0} -{"bbob_f015_i02_d02": 70.03} -{"bbob_f015_i03_d02": -48.22} -{"bbob_f015_i04_d02": 25.47} -{"bbob_f015_i05_d02": -100.81} -{"bbob_f015_i71_d02": -353.13} -{"bbob_f015_i72_d02": 87.16} -{"bbob_f015_i73_d02": -34.75} -{"bbob_f015_i74_d02": -1000.0} -{"bbob_f015_i75_d02": -92.47} -{"bbob_f015_i76_d02": -215.3} -{"bbob_f015_i77_d02": -54.49} -{"bbob_f015_i78_d02": 50.49} -{"bbob_f015_i79_d02": -364.13} -{"bbob_f015_i80_d02": 159.27} -{"bbob_f016_i01_d02": 71.35} -{"bbob_f016_i02_d02": -355.22} -{"bbob_f016_i03_d02": 99.17} -{"bbob_f016_i04_d02": -517.91} -{"bbob_f016_i05_d02": -57.21} -{"bbob_f016_i71_d02": 66.26} -{"bbob_f016_i72_d02": -138.62} -{"bbob_f016_i73_d02": 731.39} -{"bbob_f016_i74_d02": 75.26} -{"bbob_f016_i75_d02": -49.13} -{"bbob_f016_i76_d02": -141.89} -{"bbob_f016_i77_d02": 109.57} -{"bbob_f016_i78_d02": 78.36} -{"bbob_f016_i79_d02": 114.09} -{"bbob_f016_i80_d02": -544.59} -{"bbob_f017_i01_d02": -16.94} -{"bbob_f017_i02_d02": 18.81} -{"bbob_f017_i03_d02": 273.15} -{"bbob_f017_i04_d02": 37.18} -{"bbob_f017_i05_d02": -147.9} -{"bbob_f017_i71_d02": 481.71} -{"bbob_f017_i72_d02": -36.46} -{"bbob_f017_i73_d02": 39.2} -{"bbob_f017_i74_d02": -32.25} -{"bbob_f017_i75_d02": 368.12} -{"bbob_f017_i76_d02": 47.39} -{"bbob_f017_i77_d02": -342.89} -{"bbob_f017_i78_d02": 91.02} -{"bbob_f017_i79_d02": 732.42} -{"bbob_f017_i80_d02": 35.42} -{"bbob_f018_i01_d02": -16.94} -{"bbob_f018_i02_d02": 18.81} -{"bbob_f018_i03_d02": 273.15} -{"bbob_f018_i04_d02": 37.18} -{"bbob_f018_i05_d02": -147.9} -{"bbob_f018_i71_d02": 481.71} -{"bbob_f018_i72_d02": -36.46} -{"bbob_f018_i73_d02": 39.2} -{"bbob_f018_i74_d02": -32.25} -{"bbob_f018_i75_d02": 368.12} -{"bbob_f018_i76_d02": 47.39} -{"bbob_f018_i77_d02": -342.89} -{"bbob_f018_i78_d02": 91.02} -{"bbob_f018_i79_d02": 732.42} -{"bbob_f018_i80_d02": 35.42} -{"bbob_f019_i01_d02": -102.55} -{"bbob_f019_i02_d02": 71.69} -{"bbob_f019_i03_d02": 214.01} -{"bbob_f019_i04_d02": -58.48} -{"bbob_f019_i05_d02": -613.08} -{"bbob_f019_i71_d02": 61.89} -{"bbob_f019_i72_d02": -84.56} -{"bbob_f019_i73_d02": -26.02} -{"bbob_f019_i74_d02": -30.15} -{"bbob_f019_i75_d02": -1000.0} -{"bbob_f019_i76_d02": -924.3} -{"bbob_f019_i77_d02": -10.35} -{"bbob_f019_i78_d02": -1000.0} -{"bbob_f019_i79_d02": 28.8} -{"bbob_f019_i80_d02": 35.34} -{"bbob_f020_i01_d02": -546.5} -{"bbob_f020_i02_d02": 1000.0} -{"bbob_f020_i03_d02": 208.38} -{"bbob_f020_i04_d02": -144.96} -{"bbob_f020_i05_d02": -6.299999999999999} -{"bbob_f020_i71_d02": -136.76} -{"bbob_f020_i72_d02": 366.75} -{"bbob_f020_i73_d02": -248.69} -{"bbob_f020_i74_d02": -67.86} -{"bbob_f020_i75_d02": -23.29} -{"bbob_f020_i76_d02": -10.279999999999998} -{"bbob_f020_i77_d02": 179.97} -{"bbob_f020_i78_d02": 83.76} -{"bbob_f020_i79_d02": -389.01} -{"bbob_f020_i80_d02": -145.58} -{"bbob_f021_i01_d02": 40.78} -{"bbob_f021_i02_d02": -1.6} -{"bbob_f021_i03_d02": -370.84} -{"bbob_f021_i04_d02": -32.56} -{"bbob_f021_i05_d02": 515.4} -{"bbob_f021_i71_d02": -86.28} -{"bbob_f021_i72_d02": 22.24} -{"bbob_f021_i73_d02": -23.01} -{"bbob_f021_i74_d02": -332.37} -{"bbob_f021_i75_d02": -123.13} -{"bbob_f021_i76_d02": -153.79} -{"bbob_f021_i77_d02": -48.6} -{"bbob_f021_i78_d02": 27.7} -{"bbob_f021_i79_d02": 29.93} -{"bbob_f021_i80_d02": -19.83} -{"bbob_f022_i01_d02": -1000.0} -{"bbob_f022_i02_d02": 1000.0} -{"bbob_f022_i03_d02": -49.13} -{"bbob_f022_i04_d02": 297.73} -{"bbob_f022_i05_d02": 51.57} -{"bbob_f022_i71_d02": 152.71} -{"bbob_f022_i72_d02": 99.8} -{"bbob_f022_i73_d02": -119.41} -{"bbob_f022_i74_d02": 10.77} -{"bbob_f022_i75_d02": 21.37} -{"bbob_f022_i76_d02": 16.73} -{"bbob_f022_i77_d02": 1000.0} -{"bbob_f022_i78_d02": -107.96} -{"bbob_f022_i79_d02": -302.59} -{"bbob_f022_i80_d02": -306.55} -{"bbob_f023_i01_d02": 6.87} -{"bbob_f023_i02_d02": 0.01} -{"bbob_f023_i03_d02": -130.61} -{"bbob_f023_i04_d02": -223.12} -{"bbob_f023_i05_d02": -44.43} -{"bbob_f023_i71_d02": 255.78000000000017} -{"bbob_f023_i72_d02": 1000.0000000000005} -{"bbob_f023_i73_d02": -186.95} -{"bbob_f023_i74_d02": 585.9500000000004} -{"bbob_f023_i75_d02": 287.16} -{"bbob_f023_i76_d02": -394.38} -{"bbob_f023_i77_d02": 54.43} -{"bbob_f023_i78_d02": 139.69000000000008} -{"bbob_f023_i79_d02": -57.94} -{"bbob_f023_i80_d02": 125.39} -{"bbob_f024_i01_d02": 102.61} -{"bbob_f024_i02_d02": 93.3} -{"bbob_f024_i03_d02": 13.64} -{"bbob_f024_i04_d02": 194.8} -{"bbob_f024_i05_d02": -133.59} -{"bbob_f024_i71_d02": -30.85} -{"bbob_f024_i72_d02": -9.2} -{"bbob_f024_i73_d02": -203.76} -{"bbob_f024_i74_d02": 33.99} -{"bbob_f024_i75_d02": -1000.0} -{"bbob_f024_i76_d02": -1000.0} -{"bbob_f024_i77_d02": -12.72} -{"bbob_f024_i78_d02": -59.54} -{"bbob_f024_i79_d02": -1000.0} -{"bbob_f024_i80_d02": 43.03} -{"bbob_f001_i01_d03": 79.48} -{"bbob_f001_i02_d03": 394.48} -{"bbob_f001_i03_d03": -247.11} -{"bbob_f001_i04_d03": -152.04} -{"bbob_f001_i05_d03": -25.25} -{"bbob_f001_i71_d03": 183.01} -{"bbob_f001_i72_d03": 183.52} -{"bbob_f001_i73_d03": -593.84} -{"bbob_f001_i74_d03": -1000.0} -{"bbob_f001_i75_d03": -25.31} -{"bbob_f001_i76_d03": 22.28} -{"bbob_f001_i77_d03": 356.14} -{"bbob_f001_i78_d03": -475.7} -{"bbob_f001_i79_d03": -1000.0} -{"bbob_f001_i80_d03": -41.58} -{"bbob_f002_i01_d03": -209.88} -{"bbob_f002_i02_d03": -92.09} -{"bbob_f002_i03_d03": -87.89} -{"bbob_f002_i04_d03": 320.19} -{"bbob_f002_i05_d03": -104.24} -{"bbob_f002_i71_d03": -67.85} -{"bbob_f002_i72_d03": -363.98} -{"bbob_f002_i73_d03": -18.65} -{"bbob_f002_i74_d03": 174.92} -{"bbob_f002_i75_d03": -565.96} -{"bbob_f002_i76_d03": 70.95} -{"bbob_f002_i77_d03": 47.28} -{"bbob_f002_i78_d03": -568.88} -{"bbob_f002_i79_d03": 20.69} -{"bbob_f002_i80_d03": 561.67} -{"bbob_f003_i01_d03": -462.09} -{"bbob_f003_i02_d03": 77.66} -{"bbob_f003_i03_d03": 115.68} -{"bbob_f003_i04_d03": -3.71} -{"bbob_f003_i05_d03": 132.18} -{"bbob_f003_i71_d03": -39.3} -{"bbob_f003_i72_d03": 42.96} -{"bbob_f003_i73_d03": 284.22} -{"bbob_f003_i74_d03": -32.17} -{"bbob_f003_i75_d03": -58.58} -{"bbob_f003_i76_d03": -604.92} -{"bbob_f003_i77_d03": 133.7} -{"bbob_f003_i78_d03": -20.98} -{"bbob_f003_i79_d03": 135.33} -{"bbob_f003_i80_d03": 40.66} -{"bbob_f004_i01_d03": -462.09} -{"bbob_f004_i02_d03": 77.66} -{"bbob_f004_i03_d03": 115.68} -{"bbob_f004_i04_d03": -3.71} -{"bbob_f004_i05_d03": 132.18} -{"bbob_f004_i71_d03": -39.3} -{"bbob_f004_i72_d03": 42.96} -{"bbob_f004_i73_d03": 284.22} -{"bbob_f004_i74_d03": -32.17} -{"bbob_f004_i75_d03": -58.58} -{"bbob_f004_i76_d03": -604.92} -{"bbob_f004_i77_d03": 133.7} -{"bbob_f004_i78_d03": -20.98} -{"bbob_f004_i79_d03": 135.33} -{"bbob_f004_i80_d03": 40.66} -{"bbob_f005_i01_d03": -9.21} -{"bbob_f005_i02_d03": 655.99} -{"bbob_f005_i03_d03": 66.71} -{"bbob_f005_i04_d03": 941.67} -{"bbob_f005_i05_d03": -37.45} -{"bbob_f005_i71_d03": -66.39} -{"bbob_f005_i72_d03": -226.01} -{"bbob_f005_i73_d03": 1000.0} -{"bbob_f005_i74_d03": -183.02} -{"bbob_f005_i75_d03": -102.62} -{"bbob_f005_i76_d03": 99.53} -{"bbob_f005_i77_d03": 149.59} -{"bbob_f005_i78_d03": -1000.0} -{"bbob_f005_i79_d03": -29.22} -{"bbob_f005_i80_d03": -40.96} -{"bbob_f006_i01_d03": 35.9} -{"bbob_f006_i02_d03": 31.37} -{"bbob_f006_i03_d03": -1000.0} -{"bbob_f006_i04_d03": 131.92} -{"bbob_f006_i05_d03": 590.84} -{"bbob_f006_i71_d03": 67.49} -{"bbob_f006_i72_d03": 30.85} -{"bbob_f006_i73_d03": -12.55} -{"bbob_f006_i74_d03": 538.39} -{"bbob_f006_i75_d03": -105.56} -{"bbob_f006_i76_d03": 98.03} -{"bbob_f006_i77_d03": 1000.0} -{"bbob_f006_i78_d03": 1.49} -{"bbob_f006_i79_d03": 596.35} -{"bbob_f006_i80_d03": -118.97} -{"bbob_f007_i01_d03": 92.94} -{"bbob_f007_i02_d03": 35.35} -{"bbob_f007_i03_d03": 3.82} -{"bbob_f007_i04_d03": 21.75} -{"bbob_f007_i05_d03": 64.79} -{"bbob_f007_i71_d03": -1000.0} -{"bbob_f007_i72_d03": 164.07} -{"bbob_f007_i73_d03": -165.68} -{"bbob_f007_i74_d03": 92.84} -{"bbob_f007_i75_d03": -367.92} -{"bbob_f007_i76_d03": 49.5} -{"bbob_f007_i77_d03": -12.77} -{"bbob_f007_i78_d03": 63.65} -{"bbob_f007_i79_d03": 699.19} -{"bbob_f007_i80_d03": 1000.0} -{"bbob_f008_i01_d03": 149.15} -{"bbob_f008_i02_d03": -1000.0} -{"bbob_f008_i03_d03": 98.62} -{"bbob_f008_i04_d03": -47.15} -{"bbob_f008_i05_d03": 37.52} -{"bbob_f008_i71_d03": 6.74} -{"bbob_f008_i72_d03": -189.79} -{"bbob_f008_i73_d03": -27.74} -{"bbob_f008_i74_d03": 6.93} -{"bbob_f008_i75_d03": -44.56} -{"bbob_f008_i76_d03": 1000.0} -{"bbob_f008_i77_d03": -679.71} -{"bbob_f008_i78_d03": 1000.0} -{"bbob_f008_i79_d03": 10.07} -{"bbob_f008_i80_d03": -7.54} -{"bbob_f009_i01_d03": 123.83} -{"bbob_f009_i02_d03": 47.51} -{"bbob_f009_i03_d03": 55.34} -{"bbob_f009_i04_d03": -90.33} -{"bbob_f009_i05_d03": 65.61} -{"bbob_f009_i71_d03": 934.82} -{"bbob_f009_i72_d03": -63.71} -{"bbob_f009_i73_d03": 1000.0} -{"bbob_f009_i74_d03": 31.97} -{"bbob_f009_i75_d03": 58.19} -{"bbob_f009_i76_d03": -4.84} -{"bbob_f009_i77_d03": -63.37} -{"bbob_f009_i78_d03": 2.75} -{"bbob_f009_i79_d03": -46.81} -{"bbob_f009_i80_d03": 61.34} -{"bbob_f010_i01_d03": -54.94} -{"bbob_f010_i02_d03": 59.13} -{"bbob_f010_i03_d03": -491.53} -{"bbob_f010_i04_d03": -170.2} -{"bbob_f010_i05_d03": -469.75} -{"bbob_f010_i71_d03": -51.91} -{"bbob_f010_i72_d03": 48.08} -{"bbob_f010_i73_d03": 30.58} -{"bbob_f010_i74_d03": 1000.0} -{"bbob_f010_i75_d03": -42.36} -{"bbob_f010_i76_d03": -71.87} -{"bbob_f010_i77_d03": -16.5} -{"bbob_f010_i78_d03": 1000.0} -{"bbob_f010_i79_d03": -1000.0} -{"bbob_f010_i80_d03": -152.74} -{"bbob_f011_i01_d03": 76.27} -{"bbob_f011_i02_d03": -22.55} -{"bbob_f011_i03_d03": 13.66} -{"bbob_f011_i04_d03": 937.84} -{"bbob_f011_i05_d03": -78.72} -{"bbob_f011_i71_d03": -41.51} -{"bbob_f011_i72_d03": 132.07} -{"bbob_f011_i73_d03": 97.25} -{"bbob_f011_i74_d03": -20.33} -{"bbob_f011_i75_d03": 94.31} -{"bbob_f011_i76_d03": -143.69} -{"bbob_f011_i77_d03": -353.38} -{"bbob_f011_i78_d03": -2.91} -{"bbob_f011_i79_d03": -2.97} -{"bbob_f011_i80_d03": 299.63} -{"bbob_f012_i01_d03": -621.11} -{"bbob_f012_i02_d03": -254.82} -{"bbob_f012_i03_d03": 56.61} -{"bbob_f012_i04_d03": -18.1} -{"bbob_f012_i05_d03": 37.77} -{"bbob_f012_i71_d03": -632.55} -{"bbob_f012_i72_d03": -85.3} -{"bbob_f012_i73_d03": 171.95} -{"bbob_f012_i74_d03": 68.16} -{"bbob_f012_i75_d03": 1000.0} -{"bbob_f012_i76_d03": -68.37} -{"bbob_f012_i77_d03": -55.48} -{"bbob_f012_i78_d03": -53.49} -{"bbob_f012_i79_d03": 117.27} -{"bbob_f012_i80_d03": 77.89} -{"bbob_f013_i01_d03": 29.97} -{"bbob_f013_i02_d03": -51.71} -{"bbob_f013_i03_d03": -279.95} -{"bbob_f013_i04_d03": 931.99} -{"bbob_f013_i05_d03": 113.31} -{"bbob_f013_i71_d03": 28.490000000000006} -{"bbob_f013_i72_d03": -740.58} -{"bbob_f013_i73_d03": -51.77} -{"bbob_f013_i74_d03": -222.98} -{"bbob_f013_i75_d03": 5.86} -{"bbob_f013_i76_d03": 204.86} -{"bbob_f013_i77_d03": 115.49} -{"bbob_f013_i78_d03": 195.05} -{"bbob_f013_i79_d03": -214.48} -{"bbob_f013_i80_d03": -45.9} -{"bbob_f014_i01_d03": -52.35} -{"bbob_f014_i02_d03": -179.54} -{"bbob_f014_i03_d03": 77.31} -{"bbob_f014_i04_d03": 39.92} -{"bbob_f014_i05_d03": -56.61} -{"bbob_f014_i71_d03": 32.7} -{"bbob_f014_i72_d03": -39.43} -{"bbob_f014_i73_d03": 419.72} -{"bbob_f014_i74_d03": 7.64} -{"bbob_f014_i75_d03": -127.68} -{"bbob_f014_i76_d03": -36.5} -{"bbob_f014_i77_d03": 172.87} -{"bbob_f014_i78_d03": 157.72} -{"bbob_f014_i79_d03": 28.19} -{"bbob_f014_i80_d03": -99.25} -{"bbob_f015_i01_d03": 1000.0} -{"bbob_f015_i02_d03": 70.03} -{"bbob_f015_i03_d03": -48.22} -{"bbob_f015_i04_d03": 25.47} -{"bbob_f015_i05_d03": -100.81} -{"bbob_f015_i71_d03": -353.13} -{"bbob_f015_i72_d03": 87.16} -{"bbob_f015_i73_d03": -34.75} -{"bbob_f015_i74_d03": -1000.0} -{"bbob_f015_i75_d03": -92.47} -{"bbob_f015_i76_d03": -215.3} -{"bbob_f015_i77_d03": -54.49} -{"bbob_f015_i78_d03": 50.49} -{"bbob_f015_i79_d03": -364.13} -{"bbob_f015_i80_d03": 159.27} -{"bbob_f016_i01_d03": 71.35} -{"bbob_f016_i02_d03": -355.22} -{"bbob_f016_i03_d03": 99.17} -{"bbob_f016_i04_d03": -517.91} -{"bbob_f016_i05_d03": -57.21} -{"bbob_f016_i71_d03": 66.26} -{"bbob_f016_i72_d03": -138.62} -{"bbob_f016_i73_d03": 731.39} -{"bbob_f016_i74_d03": 75.26} -{"bbob_f016_i75_d03": -49.13} -{"bbob_f016_i76_d03": -141.89} -{"bbob_f016_i77_d03": 109.57} -{"bbob_f016_i78_d03": 78.36} -{"bbob_f016_i79_d03": 114.09} -{"bbob_f016_i80_d03": -544.59} -{"bbob_f017_i01_d03": -16.94} -{"bbob_f017_i02_d03": 18.81} -{"bbob_f017_i03_d03": 273.15} -{"bbob_f017_i04_d03": 37.18} -{"bbob_f017_i05_d03": -147.9} -{"bbob_f017_i71_d03": 481.71} -{"bbob_f017_i72_d03": -36.46} -{"bbob_f017_i73_d03": 39.2} -{"bbob_f017_i74_d03": -32.25} -{"bbob_f017_i75_d03": 368.12} -{"bbob_f017_i76_d03": 47.39} -{"bbob_f017_i77_d03": -342.89} -{"bbob_f017_i78_d03": 91.02} -{"bbob_f017_i79_d03": 732.42} -{"bbob_f017_i80_d03": 35.42} -{"bbob_f018_i01_d03": -16.94} -{"bbob_f018_i02_d03": 18.81} -{"bbob_f018_i03_d03": 273.15} -{"bbob_f018_i04_d03": 37.18} -{"bbob_f018_i05_d03": -147.9} -{"bbob_f018_i71_d03": 481.71} -{"bbob_f018_i72_d03": -36.46} -{"bbob_f018_i73_d03": 39.2} -{"bbob_f018_i74_d03": -32.25} -{"bbob_f018_i75_d03": 368.12} -{"bbob_f018_i76_d03": 47.39} -{"bbob_f018_i77_d03": -342.89} -{"bbob_f018_i78_d03": 91.02} -{"bbob_f018_i79_d03": 732.42} -{"bbob_f018_i80_d03": 35.42} -{"bbob_f019_i01_d03": -102.55} -{"bbob_f019_i02_d03": 71.69} -{"bbob_f019_i03_d03": 214.01} -{"bbob_f019_i04_d03": -58.48} -{"bbob_f019_i05_d03": -613.08} -{"bbob_f019_i71_d03": 61.89} -{"bbob_f019_i72_d03": -84.56} -{"bbob_f019_i73_d03": -26.02} -{"bbob_f019_i74_d03": -30.15} -{"bbob_f019_i75_d03": -1000.0} -{"bbob_f019_i76_d03": -924.3} -{"bbob_f019_i77_d03": -10.35} -{"bbob_f019_i78_d03": -1000.0} -{"bbob_f019_i79_d03": 28.8} -{"bbob_f019_i80_d03": 35.34} -{"bbob_f020_i01_d03": -546.5} -{"bbob_f020_i02_d03": 1000.0} -{"bbob_f020_i03_d03": 208.38} -{"bbob_f020_i04_d03": -144.96} -{"bbob_f020_i05_d03": -6.299999999999999} -{"bbob_f020_i71_d03": -136.76} -{"bbob_f020_i72_d03": 366.75} -{"bbob_f020_i73_d03": -248.69} -{"bbob_f020_i74_d03": -67.86} -{"bbob_f020_i75_d03": -23.29} -{"bbob_f020_i76_d03": -10.279999999999998} -{"bbob_f020_i77_d03": 179.97} -{"bbob_f020_i78_d03": 83.76} -{"bbob_f020_i79_d03": -389.01} -{"bbob_f020_i80_d03": -145.58} -{"bbob_f021_i01_d03": 40.78} -{"bbob_f021_i02_d03": -1.6} -{"bbob_f021_i03_d03": -370.84} -{"bbob_f021_i04_d03": -32.56} -{"bbob_f021_i05_d03": 515.4} -{"bbob_f021_i71_d03": -86.28} -{"bbob_f021_i72_d03": 22.24} -{"bbob_f021_i73_d03": -23.01} -{"bbob_f021_i74_d03": -332.37} -{"bbob_f021_i75_d03": -123.13} -{"bbob_f021_i76_d03": -153.79} -{"bbob_f021_i77_d03": -48.6} -{"bbob_f021_i78_d03": 27.7} -{"bbob_f021_i79_d03": 29.93} -{"bbob_f021_i80_d03": -19.83} -{"bbob_f022_i01_d03": -1000.0} -{"bbob_f022_i02_d03": 1000.0} -{"bbob_f022_i03_d03": -49.13} -{"bbob_f022_i04_d03": 297.73} -{"bbob_f022_i05_d03": 51.57} -{"bbob_f022_i71_d03": 152.71} -{"bbob_f022_i72_d03": 99.8} -{"bbob_f022_i73_d03": -119.41} -{"bbob_f022_i74_d03": 10.77} -{"bbob_f022_i75_d03": 21.37} -{"bbob_f022_i76_d03": 16.73} -{"bbob_f022_i77_d03": 1000.0} -{"bbob_f022_i78_d03": -107.96} -{"bbob_f022_i79_d03": -302.59} -{"bbob_f022_i80_d03": -306.55} -{"bbob_f023_i01_d03": 6.87} -{"bbob_f023_i02_d03": 0.010000000000036268} -{"bbob_f023_i03_d03": -130.60999999999996} -{"bbob_f023_i04_d03": -223.11999999999992} -{"bbob_f023_i05_d03": -44.43} -{"bbob_f023_i71_d03": 255.7800000000001} -{"bbob_f023_i72_d03": 1000.0000000000002} -{"bbob_f023_i73_d03": -186.95} -{"bbob_f023_i74_d03": 585.9500000000002} -{"bbob_f023_i75_d03": 287.16} -{"bbob_f023_i76_d03": -394.38} -{"bbob_f023_i77_d03": 54.43} -{"bbob_f023_i78_d03": 139.69000000000008} -{"bbob_f023_i79_d03": -57.94} -{"bbob_f023_i80_d03": 125.39} -{"bbob_f024_i01_d03": 102.61} -{"bbob_f024_i02_d03": 93.3} -{"bbob_f024_i03_d03": 13.64} -{"bbob_f024_i04_d03": 194.8} -{"bbob_f024_i05_d03": -133.59} -{"bbob_f024_i71_d03": -30.85} -{"bbob_f024_i72_d03": -9.2} -{"bbob_f024_i73_d03": -203.76} -{"bbob_f024_i74_d03": 33.99} -{"bbob_f024_i75_d03": -1000.0} -{"bbob_f024_i76_d03": -1000.0} -{"bbob_f024_i77_d03": -12.72} -{"bbob_f024_i78_d03": -59.54} -{"bbob_f024_i79_d03": -1000.0} -{"bbob_f024_i80_d03": 43.03} -{"bbob_f001_i01_d05": 79.48} -{"bbob_f001_i02_d05": 394.48} -{"bbob_f001_i03_d05": -247.11} -{"bbob_f001_i04_d05": -152.04} -{"bbob_f001_i05_d05": -25.25} -{"bbob_f001_i71_d05": 183.01} -{"bbob_f001_i72_d05": 183.52} -{"bbob_f001_i73_d05": -593.84} -{"bbob_f001_i74_d05": -1000.0} -{"bbob_f001_i75_d05": -25.31} -{"bbob_f001_i76_d05": 22.28} -{"bbob_f001_i77_d05": 356.14} -{"bbob_f001_i78_d05": -475.7} -{"bbob_f001_i79_d05": -1000.0} -{"bbob_f001_i80_d05": -41.58} -{"bbob_f002_i01_d05": -209.88} -{"bbob_f002_i02_d05": -92.09} -{"bbob_f002_i03_d05": -87.89} -{"bbob_f002_i04_d05": 320.19} -{"bbob_f002_i05_d05": -104.24} -{"bbob_f002_i71_d05": -67.85} -{"bbob_f002_i72_d05": -363.98} -{"bbob_f002_i73_d05": -18.65} -{"bbob_f002_i74_d05": 174.92} -{"bbob_f002_i75_d05": -565.96} -{"bbob_f002_i76_d05": 70.95} -{"bbob_f002_i77_d05": 47.28} -{"bbob_f002_i78_d05": -568.88} -{"bbob_f002_i79_d05": 20.69} -{"bbob_f002_i80_d05": 561.67} -{"bbob_f003_i01_d05": -462.09} -{"bbob_f003_i02_d05": 77.66} -{"bbob_f003_i03_d05": 115.68} -{"bbob_f003_i04_d05": -3.71} -{"bbob_f003_i05_d05": 132.18} -{"bbob_f003_i71_d05": -39.3} -{"bbob_f003_i72_d05": 42.96} -{"bbob_f003_i73_d05": 284.22} -{"bbob_f003_i74_d05": -32.17} -{"bbob_f003_i75_d05": -58.58} -{"bbob_f003_i76_d05": -604.92} -{"bbob_f003_i77_d05": 133.7} -{"bbob_f003_i78_d05": -20.98} -{"bbob_f003_i79_d05": 135.33} -{"bbob_f003_i80_d05": 40.66} -{"bbob_f004_i01_d05": -462.09} -{"bbob_f004_i02_d05": 77.66} -{"bbob_f004_i03_d05": 115.68} -{"bbob_f004_i04_d05": -3.71} -{"bbob_f004_i05_d05": 132.18} -{"bbob_f004_i71_d05": -39.3} -{"bbob_f004_i72_d05": 42.96} -{"bbob_f004_i73_d05": 284.22} -{"bbob_f004_i74_d05": -32.17} -{"bbob_f004_i75_d05": -58.58} -{"bbob_f004_i76_d05": -604.92} -{"bbob_f004_i77_d05": 133.7} -{"bbob_f004_i78_d05": -20.98} -{"bbob_f004_i79_d05": 135.33} -{"bbob_f004_i80_d05": 40.66} -{"bbob_f005_i01_d05": -9.21} -{"bbob_f005_i02_d05": 655.99} -{"bbob_f005_i03_d05": 66.71} -{"bbob_f005_i04_d05": 941.67} -{"bbob_f005_i05_d05": -37.45} -{"bbob_f005_i71_d05": -66.39} -{"bbob_f005_i72_d05": -226.01} -{"bbob_f005_i73_d05": 1000.0} -{"bbob_f005_i74_d05": -183.02} -{"bbob_f005_i75_d05": -102.62} -{"bbob_f005_i76_d05": 99.53} -{"bbob_f005_i77_d05": 149.59} -{"bbob_f005_i78_d05": -1000.0} -{"bbob_f005_i79_d05": -29.22} -{"bbob_f005_i80_d05": -40.96} -{"bbob_f006_i01_d05": 35.9} -{"bbob_f006_i02_d05": 31.37} -{"bbob_f006_i03_d05": -1000.0} -{"bbob_f006_i04_d05": 131.92} -{"bbob_f006_i05_d05": 590.84} -{"bbob_f006_i71_d05": 67.49} -{"bbob_f006_i72_d05": 30.85} -{"bbob_f006_i73_d05": -12.55} -{"bbob_f006_i74_d05": 538.39} -{"bbob_f006_i75_d05": -105.56} -{"bbob_f006_i76_d05": 98.03} -{"bbob_f006_i77_d05": 1000.0} -{"bbob_f006_i78_d05": 1.49} -{"bbob_f006_i79_d05": 596.35} -{"bbob_f006_i80_d05": -118.97} -{"bbob_f007_i01_d05": 92.94} -{"bbob_f007_i02_d05": 35.35} -{"bbob_f007_i03_d05": 3.82} -{"bbob_f007_i04_d05": 21.75} -{"bbob_f007_i05_d05": 64.79} -{"bbob_f007_i71_d05": -1000.0} -{"bbob_f007_i72_d05": 164.07} -{"bbob_f007_i73_d05": -165.68} -{"bbob_f007_i74_d05": 92.84} -{"bbob_f007_i75_d05": -367.92} -{"bbob_f007_i76_d05": 49.5} -{"bbob_f007_i77_d05": -12.77} -{"bbob_f007_i78_d05": 63.65} -{"bbob_f007_i79_d05": 699.19} -{"bbob_f007_i80_d05": 1000.0} -{"bbob_f008_i01_d05": 149.15} -{"bbob_f008_i02_d05": -1000.0} -{"bbob_f008_i03_d05": 98.62} -{"bbob_f008_i04_d05": -47.15} -{"bbob_f008_i05_d05": 37.52} -{"bbob_f008_i71_d05": 6.74} -{"bbob_f008_i72_d05": -189.79} -{"bbob_f008_i73_d05": -27.74} -{"bbob_f008_i74_d05": 6.93} -{"bbob_f008_i75_d05": -44.56} -{"bbob_f008_i76_d05": 1000.0} -{"bbob_f008_i77_d05": -679.71} -{"bbob_f008_i78_d05": 1000.0} -{"bbob_f008_i79_d05": 10.07} -{"bbob_f008_i80_d05": -7.54} -{"bbob_f009_i01_d05": 123.83} -{"bbob_f009_i02_d05": 47.51} -{"bbob_f009_i03_d05": 55.34} -{"bbob_f009_i04_d05": -90.33} -{"bbob_f009_i05_d05": 65.61} -{"bbob_f009_i71_d05": 934.82} -{"bbob_f009_i72_d05": -63.71} -{"bbob_f009_i73_d05": 1000.0} -{"bbob_f009_i74_d05": 31.97} -{"bbob_f009_i75_d05": 58.19} -{"bbob_f009_i76_d05": -4.84} -{"bbob_f009_i77_d05": -63.37} -{"bbob_f009_i78_d05": 2.75} -{"bbob_f009_i79_d05": -46.81} -{"bbob_f009_i80_d05": 61.34} -{"bbob_f010_i01_d05": -54.94} -{"bbob_f010_i02_d05": 59.13} -{"bbob_f010_i03_d05": -491.53} -{"bbob_f010_i04_d05": -170.2} -{"bbob_f010_i05_d05": -469.75} -{"bbob_f010_i71_d05": -51.91} -{"bbob_f010_i72_d05": 48.08} -{"bbob_f010_i73_d05": 30.58} -{"bbob_f010_i74_d05": 1000.0} -{"bbob_f010_i75_d05": -42.36} -{"bbob_f010_i76_d05": -71.87} -{"bbob_f010_i77_d05": -16.5} -{"bbob_f010_i78_d05": 1000.0} -{"bbob_f010_i79_d05": -1000.0} -{"bbob_f010_i80_d05": -152.74} -{"bbob_f011_i01_d05": 76.27} -{"bbob_f011_i02_d05": -22.55} -{"bbob_f011_i03_d05": 13.66} -{"bbob_f011_i04_d05": 937.84} -{"bbob_f011_i05_d05": -78.72} -{"bbob_f011_i71_d05": -41.51} -{"bbob_f011_i72_d05": 132.07} -{"bbob_f011_i73_d05": 97.25} -{"bbob_f011_i74_d05": -20.33} -{"bbob_f011_i75_d05": 94.31} -{"bbob_f011_i76_d05": -143.69} -{"bbob_f011_i77_d05": -353.38} -{"bbob_f011_i78_d05": -2.91} -{"bbob_f011_i79_d05": -2.97} -{"bbob_f011_i80_d05": 299.63} -{"bbob_f012_i01_d05": -621.11} -{"bbob_f012_i02_d05": -254.82} -{"bbob_f012_i03_d05": 56.61} -{"bbob_f012_i04_d05": -18.1} -{"bbob_f012_i05_d05": 37.77} -{"bbob_f012_i71_d05": -632.55} -{"bbob_f012_i72_d05": -85.3} -{"bbob_f012_i73_d05": 171.95} -{"bbob_f012_i74_d05": 68.16} -{"bbob_f012_i75_d05": 1000.0} -{"bbob_f012_i76_d05": -68.37} -{"bbob_f012_i77_d05": -55.48} -{"bbob_f012_i78_d05": -53.49} -{"bbob_f012_i79_d05": 117.27} -{"bbob_f012_i80_d05": 77.89} -{"bbob_f013_i01_d05": 29.97} -{"bbob_f013_i02_d05": -51.71} -{"bbob_f013_i03_d05": -279.95} -{"bbob_f013_i04_d05": 931.99} -{"bbob_f013_i05_d05": 113.31} -{"bbob_f013_i71_d05": 28.490000000000006} -{"bbob_f013_i72_d05": -740.58} -{"bbob_f013_i73_d05": -51.77} -{"bbob_f013_i74_d05": -222.98} -{"bbob_f013_i75_d05": 5.86} -{"bbob_f013_i76_d05": 204.86} -{"bbob_f013_i77_d05": 115.49} -{"bbob_f013_i78_d05": 195.05} -{"bbob_f013_i79_d05": -214.48} -{"bbob_f013_i80_d05": -45.89999999999999} -{"bbob_f014_i01_d05": -52.35} -{"bbob_f014_i02_d05": -179.54} -{"bbob_f014_i03_d05": 77.31} -{"bbob_f014_i04_d05": 39.92} -{"bbob_f014_i05_d05": -56.61} -{"bbob_f014_i71_d05": 32.7} -{"bbob_f014_i72_d05": -39.43} -{"bbob_f014_i73_d05": 419.72} -{"bbob_f014_i74_d05": 7.64} -{"bbob_f014_i75_d05": -127.68} -{"bbob_f014_i76_d05": -36.5} -{"bbob_f014_i77_d05": 172.87} -{"bbob_f014_i78_d05": 157.72} -{"bbob_f014_i79_d05": 28.19} -{"bbob_f014_i80_d05": -99.25} -{"bbob_f015_i01_d05": 1000.0} -{"bbob_f015_i02_d05": 70.03} -{"bbob_f015_i03_d05": -48.22} -{"bbob_f015_i04_d05": 25.47} -{"bbob_f015_i05_d05": -100.81} -{"bbob_f015_i71_d05": -353.13} -{"bbob_f015_i72_d05": 87.16} -{"bbob_f015_i73_d05": -34.75} -{"bbob_f015_i74_d05": -1000.0} -{"bbob_f015_i75_d05": -92.47} -{"bbob_f015_i76_d05": -215.3} -{"bbob_f015_i77_d05": -54.49} -{"bbob_f015_i78_d05": 50.49} -{"bbob_f015_i79_d05": -364.13} -{"bbob_f015_i80_d05": 159.27} -{"bbob_f016_i01_d05": 71.35} -{"bbob_f016_i02_d05": -355.22} -{"bbob_f016_i03_d05": 99.17} -{"bbob_f016_i04_d05": -517.91} -{"bbob_f016_i05_d05": -57.21} -{"bbob_f016_i71_d05": 66.26} -{"bbob_f016_i72_d05": -138.62} -{"bbob_f016_i73_d05": 731.39} -{"bbob_f016_i74_d05": 75.26} -{"bbob_f016_i75_d05": -49.13} -{"bbob_f016_i76_d05": -141.89} -{"bbob_f016_i77_d05": 109.57} -{"bbob_f016_i78_d05": 78.36} -{"bbob_f016_i79_d05": 114.09} -{"bbob_f016_i80_d05": -544.59} -{"bbob_f017_i01_d05": -16.94} -{"bbob_f017_i02_d05": 18.81} -{"bbob_f017_i03_d05": 273.15} -{"bbob_f017_i04_d05": 37.18} -{"bbob_f017_i05_d05": -147.9} -{"bbob_f017_i71_d05": 481.71} -{"bbob_f017_i72_d05": -36.46} -{"bbob_f017_i73_d05": 39.2} -{"bbob_f017_i74_d05": -32.25} -{"bbob_f017_i75_d05": 368.12} -{"bbob_f017_i76_d05": 47.39} -{"bbob_f017_i77_d05": -342.89} -{"bbob_f017_i78_d05": 91.02} -{"bbob_f017_i79_d05": 732.42} -{"bbob_f017_i80_d05": 35.42} -{"bbob_f018_i01_d05": -16.94} -{"bbob_f018_i02_d05": 18.81} -{"bbob_f018_i03_d05": 273.15} -{"bbob_f018_i04_d05": 37.18} -{"bbob_f018_i05_d05": -147.9} -{"bbob_f018_i71_d05": 481.71} -{"bbob_f018_i72_d05": -36.46} -{"bbob_f018_i73_d05": 39.2} -{"bbob_f018_i74_d05": -32.25} -{"bbob_f018_i75_d05": 368.12} -{"bbob_f018_i76_d05": 47.39} -{"bbob_f018_i77_d05": -342.89} -{"bbob_f018_i78_d05": 91.02} -{"bbob_f018_i79_d05": 732.42} -{"bbob_f018_i80_d05": 35.42} -{"bbob_f019_i01_d05": -102.55} -{"bbob_f019_i02_d05": 71.69} -{"bbob_f019_i03_d05": 214.01} -{"bbob_f019_i04_d05": -58.48} -{"bbob_f019_i05_d05": -613.08} -{"bbob_f019_i71_d05": 61.89} -{"bbob_f019_i72_d05": -84.56} -{"bbob_f019_i73_d05": -26.02} -{"bbob_f019_i74_d05": -30.15} -{"bbob_f019_i75_d05": -1000.0} -{"bbob_f019_i76_d05": -924.3} -{"bbob_f019_i77_d05": -10.35} -{"bbob_f019_i78_d05": -1000.0} -{"bbob_f019_i79_d05": 28.8} -{"bbob_f019_i80_d05": 35.34} -{"bbob_f020_i01_d05": -546.5} -{"bbob_f020_i02_d05": 1000.0} -{"bbob_f020_i03_d05": 208.38} -{"bbob_f020_i04_d05": -144.96} -{"bbob_f020_i05_d05": -6.299999999999997} -{"bbob_f020_i71_d05": -136.76} -{"bbob_f020_i72_d05": 366.75} -{"bbob_f020_i73_d05": -248.69} -{"bbob_f020_i74_d05": -67.86} -{"bbob_f020_i75_d05": -23.289999999999996} -{"bbob_f020_i76_d05": -10.279999999999998} -{"bbob_f020_i77_d05": 179.97} -{"bbob_f020_i78_d05": 83.76} -{"bbob_f020_i79_d05": -389.01} -{"bbob_f020_i80_d05": -145.58} -{"bbob_f021_i01_d05": 40.78} -{"bbob_f021_i02_d05": -1.6} -{"bbob_f021_i03_d05": -370.84} -{"bbob_f021_i04_d05": -32.56} -{"bbob_f021_i05_d05": 515.4} -{"bbob_f021_i71_d05": -86.28} -{"bbob_f021_i72_d05": 22.24} -{"bbob_f021_i73_d05": -23.01} -{"bbob_f021_i74_d05": -332.37} -{"bbob_f021_i75_d05": -123.13} -{"bbob_f021_i76_d05": -153.79} -{"bbob_f021_i77_d05": -48.6} -{"bbob_f021_i78_d05": 27.7} -{"bbob_f021_i79_d05": 29.93} -{"bbob_f021_i80_d05": -19.83} -{"bbob_f022_i01_d05": -1000.0} -{"bbob_f022_i02_d05": 1000.0} -{"bbob_f022_i03_d05": -49.13} -{"bbob_f022_i04_d05": 297.73} -{"bbob_f022_i05_d05": 51.57} -{"bbob_f022_i71_d05": 152.71} -{"bbob_f022_i72_d05": 99.8} -{"bbob_f022_i73_d05": -119.41} -{"bbob_f022_i74_d05": 10.77} -{"bbob_f022_i75_d05": 21.37} -{"bbob_f022_i76_d05": 16.73} -{"bbob_f022_i77_d05": 1000.0} -{"bbob_f022_i78_d05": -107.96} -{"bbob_f022_i79_d05": -302.59} -{"bbob_f022_i80_d05": -306.55} -{"bbob_f023_i01_d05": 6.87} -{"bbob_f023_i02_d05": 0.010000000000005152} -{"bbob_f023_i03_d05": -130.60999999999999} -{"bbob_f023_i04_d05": -223.11999999999995} -{"bbob_f023_i05_d05": -44.43} -{"bbob_f023_i71_d05": 255.78} -{"bbob_f023_i72_d05": 1000.0} -{"bbob_f023_i73_d05": -186.95} -{"bbob_f023_i74_d05": 585.95} -{"bbob_f023_i75_d05": 287.16} -{"bbob_f023_i76_d05": -394.38} -{"bbob_f023_i77_d05": 54.430000000000035} -{"bbob_f023_i78_d05": 139.69000000000003} -{"bbob_f023_i79_d05": -57.94} -{"bbob_f023_i80_d05": 125.39} -{"bbob_f024_i01_d05": 102.61} -{"bbob_f024_i02_d05": 93.3} -{"bbob_f024_i03_d05": 13.64} -{"bbob_f024_i04_d05": 194.8} -{"bbob_f024_i05_d05": -133.59} -{"bbob_f024_i71_d05": -30.85} -{"bbob_f024_i72_d05": -9.2} -{"bbob_f024_i73_d05": -203.76} -{"bbob_f024_i74_d05": 33.99} -{"bbob_f024_i75_d05": -1000.0} -{"bbob_f024_i76_d05": -1000.0} -{"bbob_f024_i77_d05": -12.72} -{"bbob_f024_i78_d05": -59.54} -{"bbob_f024_i79_d05": -1000.0} -{"bbob_f024_i80_d05": 43.03} -{"bbob_f001_i01_d10": 79.48} -{"bbob_f001_i02_d10": 394.48} -{"bbob_f001_i03_d10": -247.11} -{"bbob_f001_i04_d10": -152.04} -{"bbob_f001_i05_d10": -25.25} -{"bbob_f001_i71_d10": 183.01} -{"bbob_f001_i72_d10": 183.52} -{"bbob_f001_i73_d10": -593.84} -{"bbob_f001_i74_d10": -1000.0} -{"bbob_f001_i75_d10": -25.31} -{"bbob_f001_i76_d10": 22.28} -{"bbob_f001_i77_d10": 356.14} -{"bbob_f001_i78_d10": -475.7} -{"bbob_f001_i79_d10": -1000.0} -{"bbob_f001_i80_d10": -41.58} -{"bbob_f002_i01_d10": -209.88} -{"bbob_f002_i02_d10": -92.09} -{"bbob_f002_i03_d10": -87.89} -{"bbob_f002_i04_d10": 320.19} -{"bbob_f002_i05_d10": -104.24} -{"bbob_f002_i71_d10": -67.85} -{"bbob_f002_i72_d10": -363.98} -{"bbob_f002_i73_d10": -18.65} -{"bbob_f002_i74_d10": 174.92} -{"bbob_f002_i75_d10": -565.96} -{"bbob_f002_i76_d10": 70.95} -{"bbob_f002_i77_d10": 47.28} -{"bbob_f002_i78_d10": -568.88} -{"bbob_f002_i79_d10": 20.69} -{"bbob_f002_i80_d10": 561.67} -{"bbob_f003_i01_d10": -462.09} -{"bbob_f003_i02_d10": 77.66} -{"bbob_f003_i03_d10": 115.68} -{"bbob_f003_i04_d10": -3.71} -{"bbob_f003_i05_d10": 132.18} -{"bbob_f003_i71_d10": -39.3} -{"bbob_f003_i72_d10": 42.96} -{"bbob_f003_i73_d10": 284.22} -{"bbob_f003_i74_d10": -32.17} -{"bbob_f003_i75_d10": -58.58} -{"bbob_f003_i76_d10": -604.92} -{"bbob_f003_i77_d10": 133.7} -{"bbob_f003_i78_d10": -20.98} -{"bbob_f003_i79_d10": 135.33} -{"bbob_f003_i80_d10": 40.66} -{"bbob_f004_i01_d10": -462.09} -{"bbob_f004_i02_d10": 77.66} -{"bbob_f004_i03_d10": 115.68} -{"bbob_f004_i04_d10": -3.71} -{"bbob_f004_i05_d10": 132.18} -{"bbob_f004_i71_d10": -39.3} -{"bbob_f004_i72_d10": 42.96} -{"bbob_f004_i73_d10": 284.22} -{"bbob_f004_i74_d10": -32.17} -{"bbob_f004_i75_d10": -58.58} -{"bbob_f004_i76_d10": -604.92} -{"bbob_f004_i77_d10": 133.7} -{"bbob_f004_i78_d10": -20.98} -{"bbob_f004_i79_d10": 135.33} -{"bbob_f004_i80_d10": 40.66} -{"bbob_f005_i01_d10": -9.21} -{"bbob_f005_i02_d10": 655.99} -{"bbob_f005_i03_d10": 66.71} -{"bbob_f005_i04_d10": 941.67} -{"bbob_f005_i05_d10": -37.45} -{"bbob_f005_i71_d10": -66.39} -{"bbob_f005_i72_d10": -226.01} -{"bbob_f005_i73_d10": 1000.0} -{"bbob_f005_i74_d10": -183.02} -{"bbob_f005_i75_d10": -102.62} -{"bbob_f005_i76_d10": 99.53} -{"bbob_f005_i77_d10": 149.59} -{"bbob_f005_i78_d10": -1000.0} -{"bbob_f005_i79_d10": -29.22} -{"bbob_f005_i80_d10": -40.96} -{"bbob_f006_i01_d10": 35.9} -{"bbob_f006_i02_d10": 31.37} -{"bbob_f006_i03_d10": -1000.0} -{"bbob_f006_i04_d10": 131.92} -{"bbob_f006_i05_d10": 590.84} -{"bbob_f006_i71_d10": 67.49} -{"bbob_f006_i72_d10": 30.85} -{"bbob_f006_i73_d10": -12.55} -{"bbob_f006_i74_d10": 538.39} -{"bbob_f006_i75_d10": -105.56} -{"bbob_f006_i76_d10": 98.03} -{"bbob_f006_i77_d10": 1000.0} -{"bbob_f006_i78_d10": 1.49} -{"bbob_f006_i79_d10": 596.35} -{"bbob_f006_i80_d10": -118.97} -{"bbob_f007_i01_d10": 92.94} -{"bbob_f007_i02_d10": 35.35} -{"bbob_f007_i03_d10": 3.82} -{"bbob_f007_i04_d10": 21.75} -{"bbob_f007_i05_d10": 64.79} -{"bbob_f007_i71_d10": -1000.0} -{"bbob_f007_i72_d10": 164.07} -{"bbob_f007_i73_d10": -165.68} -{"bbob_f007_i74_d10": 92.84} -{"bbob_f007_i75_d10": -367.92} -{"bbob_f007_i76_d10": 49.5} -{"bbob_f007_i77_d10": -12.77} -{"bbob_f007_i78_d10": 63.65} -{"bbob_f007_i79_d10": 699.19} -{"bbob_f007_i80_d10": 1000.0} -{"bbob_f008_i01_d10": 149.15} -{"bbob_f008_i02_d10": -1000.0} -{"bbob_f008_i03_d10": 98.62} -{"bbob_f008_i04_d10": -47.15} -{"bbob_f008_i05_d10": 37.52} -{"bbob_f008_i71_d10": 6.74} -{"bbob_f008_i72_d10": -189.79} -{"bbob_f008_i73_d10": -27.74} -{"bbob_f008_i74_d10": 6.93} -{"bbob_f008_i75_d10": -44.56} -{"bbob_f008_i76_d10": 1000.0} -{"bbob_f008_i77_d10": -679.71} -{"bbob_f008_i78_d10": 1000.0} -{"bbob_f008_i79_d10": 10.07} -{"bbob_f008_i80_d10": -7.54} -{"bbob_f009_i01_d10": 123.83} -{"bbob_f009_i02_d10": 47.51} -{"bbob_f009_i03_d10": 55.34} -{"bbob_f009_i04_d10": -90.33} -{"bbob_f009_i05_d10": 65.61} -{"bbob_f009_i71_d10": 934.82} -{"bbob_f009_i72_d10": -63.71} -{"bbob_f009_i73_d10": 1000.0} -{"bbob_f009_i74_d10": 31.97} -{"bbob_f009_i75_d10": 58.19} -{"bbob_f009_i76_d10": -4.84} -{"bbob_f009_i77_d10": -63.37} -{"bbob_f009_i78_d10": 2.75} -{"bbob_f009_i79_d10": -46.81} -{"bbob_f009_i80_d10": 61.34} -{"bbob_f010_i01_d10": -54.94} -{"bbob_f010_i02_d10": 59.13} -{"bbob_f010_i03_d10": -491.53} -{"bbob_f010_i04_d10": -170.2} -{"bbob_f010_i05_d10": -469.75} -{"bbob_f010_i71_d10": -51.91} -{"bbob_f010_i72_d10": 48.08} -{"bbob_f010_i73_d10": 30.58} -{"bbob_f010_i74_d10": 1000.0} -{"bbob_f010_i75_d10": -42.36} -{"bbob_f010_i76_d10": -71.87} -{"bbob_f010_i77_d10": -16.5} -{"bbob_f010_i78_d10": 1000.0} -{"bbob_f010_i79_d10": -1000.0} -{"bbob_f010_i80_d10": -152.74} -{"bbob_f011_i01_d10": 76.27} -{"bbob_f011_i02_d10": -22.55} -{"bbob_f011_i03_d10": 13.66} -{"bbob_f011_i04_d10": 937.84} -{"bbob_f011_i05_d10": -78.72} -{"bbob_f011_i71_d10": -41.51} -{"bbob_f011_i72_d10": 132.07} -{"bbob_f011_i73_d10": 97.25} -{"bbob_f011_i74_d10": -20.33} -{"bbob_f011_i75_d10": 94.31} -{"bbob_f011_i76_d10": -143.69} -{"bbob_f011_i77_d10": -353.38} -{"bbob_f011_i78_d10": -2.91} -{"bbob_f011_i79_d10": -2.97} -{"bbob_f011_i80_d10": 299.63} -{"bbob_f012_i01_d10": -621.11} -{"bbob_f012_i02_d10": -254.82} -{"bbob_f012_i03_d10": 56.61} -{"bbob_f012_i04_d10": -18.1} -{"bbob_f012_i05_d10": 37.77} -{"bbob_f012_i71_d10": -632.55} -{"bbob_f012_i72_d10": -85.3} -{"bbob_f012_i73_d10": 171.95} -{"bbob_f012_i74_d10": 68.16} -{"bbob_f012_i75_d10": 1000.0} -{"bbob_f012_i76_d10": -68.37} -{"bbob_f012_i77_d10": -55.48} -{"bbob_f012_i78_d10": -53.49} -{"bbob_f012_i79_d10": 117.27} -{"bbob_f012_i80_d10": 77.89} -{"bbob_f013_i01_d10": 29.97} -{"bbob_f013_i02_d10": -51.709999999999994} -{"bbob_f013_i03_d10": -279.95} -{"bbob_f013_i04_d10": 931.99} -{"bbob_f013_i05_d10": 113.31} -{"bbob_f013_i71_d10": 28.490000000000002} -{"bbob_f013_i72_d10": -740.58} -{"bbob_f013_i73_d10": -51.769999999999996} -{"bbob_f013_i74_d10": -222.98} -{"bbob_f013_i75_d10": 5.86} -{"bbob_f013_i76_d10": 204.86} -{"bbob_f013_i77_d10": 115.49} -{"bbob_f013_i78_d10": 195.05} -{"bbob_f013_i79_d10": -214.48} -{"bbob_f013_i80_d10": -45.89999999999999} -{"bbob_f014_i01_d10": -52.35} -{"bbob_f014_i02_d10": -179.54} -{"bbob_f014_i03_d10": 77.31} -{"bbob_f014_i04_d10": 39.92} -{"bbob_f014_i05_d10": -56.61} -{"bbob_f014_i71_d10": 32.7} -{"bbob_f014_i72_d10": -39.43} -{"bbob_f014_i73_d10": 419.72} -{"bbob_f014_i74_d10": 7.64} -{"bbob_f014_i75_d10": -127.68} -{"bbob_f014_i76_d10": -36.5} -{"bbob_f014_i77_d10": 172.87} -{"bbob_f014_i78_d10": 157.72} -{"bbob_f014_i79_d10": 28.19} -{"bbob_f014_i80_d10": -99.25} -{"bbob_f015_i01_d10": 1000.0} -{"bbob_f015_i02_d10": 70.03} -{"bbob_f015_i03_d10": -48.22} -{"bbob_f015_i04_d10": 25.47} -{"bbob_f015_i05_d10": -100.81} -{"bbob_f015_i71_d10": -353.13} -{"bbob_f015_i72_d10": 87.16} -{"bbob_f015_i73_d10": -34.75} -{"bbob_f015_i74_d10": -1000.0} -{"bbob_f015_i75_d10": -92.47} -{"bbob_f015_i76_d10": -215.3} -{"bbob_f015_i77_d10": -54.49} -{"bbob_f015_i78_d10": 50.49} -{"bbob_f015_i79_d10": -364.13} -{"bbob_f015_i80_d10": 159.27} -{"bbob_f016_i01_d10": 71.35} -{"bbob_f016_i02_d10": -355.22} -{"bbob_f016_i03_d10": 99.17} -{"bbob_f016_i04_d10": -517.91} -{"bbob_f016_i05_d10": -57.21} -{"bbob_f016_i71_d10": 66.26} -{"bbob_f016_i72_d10": -138.62} -{"bbob_f016_i73_d10": 731.39} -{"bbob_f016_i74_d10": 75.26} -{"bbob_f016_i75_d10": -49.13} -{"bbob_f016_i76_d10": -141.89} -{"bbob_f016_i77_d10": 109.57} -{"bbob_f016_i78_d10": 78.36} -{"bbob_f016_i79_d10": 114.09} -{"bbob_f016_i80_d10": -544.59} -{"bbob_f017_i01_d10": -16.94} -{"bbob_f017_i02_d10": 18.81} -{"bbob_f017_i03_d10": 273.15} -{"bbob_f017_i04_d10": 37.18} -{"bbob_f017_i05_d10": -147.9} -{"bbob_f017_i71_d10": 481.71} -{"bbob_f017_i72_d10": -36.46} -{"bbob_f017_i73_d10": 39.2} -{"bbob_f017_i74_d10": -32.25} -{"bbob_f017_i75_d10": 368.12} -{"bbob_f017_i76_d10": 47.39} -{"bbob_f017_i77_d10": -342.89} -{"bbob_f017_i78_d10": 91.02} -{"bbob_f017_i79_d10": 732.42} -{"bbob_f017_i80_d10": 35.42} -{"bbob_f018_i01_d10": -16.94} -{"bbob_f018_i02_d10": 18.81} -{"bbob_f018_i03_d10": 273.15} -{"bbob_f018_i04_d10": 37.18} -{"bbob_f018_i05_d10": -147.9} -{"bbob_f018_i71_d10": 481.71} -{"bbob_f018_i72_d10": -36.46} -{"bbob_f018_i73_d10": 39.2} -{"bbob_f018_i74_d10": -32.25} -{"bbob_f018_i75_d10": 368.12} -{"bbob_f018_i76_d10": 47.39} -{"bbob_f018_i77_d10": -342.89} -{"bbob_f018_i78_d10": 91.02} -{"bbob_f018_i79_d10": 732.42} -{"bbob_f018_i80_d10": 35.42} -{"bbob_f019_i01_d10": -102.55} -{"bbob_f019_i02_d10": 71.69} -{"bbob_f019_i03_d10": 214.01} -{"bbob_f019_i04_d10": -58.48} -{"bbob_f019_i05_d10": -613.08} -{"bbob_f019_i71_d10": 61.89} -{"bbob_f019_i72_d10": -84.56} -{"bbob_f019_i73_d10": -26.02} -{"bbob_f019_i74_d10": -30.15} -{"bbob_f019_i75_d10": -1000.0} -{"bbob_f019_i76_d10": -924.3} -{"bbob_f019_i77_d10": -10.35} -{"bbob_f019_i78_d10": -1000.0} -{"bbob_f019_i79_d10": 28.8} -{"bbob_f019_i80_d10": 35.34} -{"bbob_f020_i01_d10": -546.5} -{"bbob_f020_i02_d10": 1000.0} -{"bbob_f020_i03_d10": 208.38} -{"bbob_f020_i04_d10": -144.96} -{"bbob_f020_i05_d10": -6.299999999999999} -{"bbob_f020_i71_d10": -136.76} -{"bbob_f020_i72_d10": 366.75} -{"bbob_f020_i73_d10": -248.69} -{"bbob_f020_i74_d10": -67.86} -{"bbob_f020_i75_d10": -23.29} -{"bbob_f020_i76_d10": -10.279999999999998} -{"bbob_f020_i77_d10": 179.97} -{"bbob_f020_i78_d10": 83.76} -{"bbob_f020_i79_d10": -389.01} -{"bbob_f020_i80_d10": -145.58} -{"bbob_f021_i01_d10": 40.78} -{"bbob_f021_i02_d10": -1.6} -{"bbob_f021_i03_d10": -370.84} -{"bbob_f021_i04_d10": -32.56} -{"bbob_f021_i05_d10": 515.4} -{"bbob_f021_i71_d10": -86.28} -{"bbob_f021_i72_d10": 22.24} -{"bbob_f021_i73_d10": -23.01} -{"bbob_f021_i74_d10": -332.37} -{"bbob_f021_i75_d10": -123.13} -{"bbob_f021_i76_d10": -153.79} -{"bbob_f021_i77_d10": -48.6} -{"bbob_f021_i78_d10": 27.7} -{"bbob_f021_i79_d10": 29.93} -{"bbob_f021_i80_d10": -19.83} -{"bbob_f022_i01_d10": -1000.0} -{"bbob_f022_i02_d10": 1000.0} -{"bbob_f022_i03_d10": -49.13} -{"bbob_f022_i04_d10": 297.73} -{"bbob_f022_i05_d10": 51.57} -{"bbob_f022_i71_d10": 152.71} -{"bbob_f022_i72_d10": 99.8} -{"bbob_f022_i73_d10": -119.41} -{"bbob_f022_i74_d10": 10.77} -{"bbob_f022_i75_d10": 21.37} -{"bbob_f022_i76_d10": 16.73} -{"bbob_f022_i77_d10": 1000.0} -{"bbob_f022_i78_d10": -107.96} -{"bbob_f022_i79_d10": -302.59} -{"bbob_f022_i80_d10": -306.55} -{"bbob_f023_i01_d10": 6.870000000000004} -{"bbob_f023_i02_d10": 0.010000000000001999} -{"bbob_f023_i03_d10": -130.61} -{"bbob_f023_i04_d10": -223.12} -{"bbob_f023_i05_d10": -44.43} -{"bbob_f023_i71_d10": 255.78} -{"bbob_f023_i72_d10": 1000.0} -{"bbob_f023_i73_d10": -186.95} -{"bbob_f023_i74_d10": 585.95} -{"bbob_f023_i75_d10": 287.16} -{"bbob_f023_i76_d10": -394.38} -{"bbob_f023_i77_d10": 54.43000000000001} -{"bbob_f023_i78_d10": 139.69} -{"bbob_f023_i79_d10": -57.93999999999999} -{"bbob_f023_i80_d10": 125.39} -{"bbob_f024_i01_d10": 102.61} -{"bbob_f024_i02_d10": 93.3} -{"bbob_f024_i03_d10": 13.64} -{"bbob_f024_i04_d10": 194.8} -{"bbob_f024_i05_d10": -133.59} -{"bbob_f024_i71_d10": -30.85} -{"bbob_f024_i72_d10": -9.2} -{"bbob_f024_i73_d10": -203.76} -{"bbob_f024_i74_d10": 33.99} -{"bbob_f024_i75_d10": -1000.0} -{"bbob_f024_i76_d10": -1000.0} -{"bbob_f024_i77_d10": -12.72} -{"bbob_f024_i78_d10": -59.54} -{"bbob_f024_i79_d10": -1000.0} -{"bbob_f024_i80_d10": 43.03} -{"bbob_f001_i01_d20": 79.48} -{"bbob_f001_i02_d20": 394.48} -{"bbob_f001_i03_d20": -247.11} -{"bbob_f001_i04_d20": -152.04} -{"bbob_f001_i05_d20": -25.25} -{"bbob_f001_i71_d20": 183.01} -{"bbob_f001_i72_d20": 183.52} -{"bbob_f001_i73_d20": -593.84} -{"bbob_f001_i74_d20": -1000.0} -{"bbob_f001_i75_d20": -25.31} -{"bbob_f001_i76_d20": 22.28} -{"bbob_f001_i77_d20": 356.14} -{"bbob_f001_i78_d20": -475.7} -{"bbob_f001_i79_d20": -1000.0} -{"bbob_f001_i80_d20": -41.58} -{"bbob_f002_i01_d20": -209.88} -{"bbob_f002_i02_d20": -92.09} -{"bbob_f002_i03_d20": -87.89} -{"bbob_f002_i04_d20": 320.19} -{"bbob_f002_i05_d20": -104.24} -{"bbob_f002_i71_d20": -67.85} -{"bbob_f002_i72_d20": -363.98} -{"bbob_f002_i73_d20": -18.65} -{"bbob_f002_i74_d20": 174.92} -{"bbob_f002_i75_d20": -565.96} -{"bbob_f002_i76_d20": 70.95} -{"bbob_f002_i77_d20": 47.28} -{"bbob_f002_i78_d20": -568.88} -{"bbob_f002_i79_d20": 20.69} -{"bbob_f002_i80_d20": 561.67} -{"bbob_f003_i01_d20": -462.09} -{"bbob_f003_i02_d20": 77.66} -{"bbob_f003_i03_d20": 115.68} -{"bbob_f003_i04_d20": -3.71} -{"bbob_f003_i05_d20": 132.18} -{"bbob_f003_i71_d20": -39.3} -{"bbob_f003_i72_d20": 42.96} -{"bbob_f003_i73_d20": 284.22} -{"bbob_f003_i74_d20": -32.17} -{"bbob_f003_i75_d20": -58.58} -{"bbob_f003_i76_d20": -604.92} -{"bbob_f003_i77_d20": 133.7} -{"bbob_f003_i78_d20": -20.98} -{"bbob_f003_i79_d20": 135.33} -{"bbob_f003_i80_d20": 40.66} -{"bbob_f004_i01_d20": -462.09} -{"bbob_f004_i02_d20": 77.66} -{"bbob_f004_i03_d20": 115.68} -{"bbob_f004_i04_d20": -3.71} -{"bbob_f004_i05_d20": 132.18} -{"bbob_f004_i71_d20": -39.3} -{"bbob_f004_i72_d20": 42.96} -{"bbob_f004_i73_d20": 284.22} -{"bbob_f004_i74_d20": -32.17} -{"bbob_f004_i75_d20": -58.58} -{"bbob_f004_i76_d20": -604.92} -{"bbob_f004_i77_d20": 133.7} -{"bbob_f004_i78_d20": -20.98} -{"bbob_f004_i79_d20": 135.33} -{"bbob_f004_i80_d20": 40.66} -{"bbob_f005_i01_d20": -9.21} -{"bbob_f005_i02_d20": 655.99} -{"bbob_f005_i03_d20": 66.71} -{"bbob_f005_i04_d20": 941.67} -{"bbob_f005_i05_d20": -37.45} -{"bbob_f005_i71_d20": -66.39} -{"bbob_f005_i72_d20": -226.01} -{"bbob_f005_i73_d20": 1000.0} -{"bbob_f005_i74_d20": -183.02} -{"bbob_f005_i75_d20": -102.62} -{"bbob_f005_i76_d20": 99.53} -{"bbob_f005_i77_d20": 149.59} -{"bbob_f005_i78_d20": -1000.0} -{"bbob_f005_i79_d20": -29.22} -{"bbob_f005_i80_d20": -40.96} -{"bbob_f006_i01_d20": 35.9} -{"bbob_f006_i02_d20": 31.37} -{"bbob_f006_i03_d20": -1000.0} -{"bbob_f006_i04_d20": 131.92} -{"bbob_f006_i05_d20": 590.84} -{"bbob_f006_i71_d20": 67.49} -{"bbob_f006_i72_d20": 30.85} -{"bbob_f006_i73_d20": -12.55} -{"bbob_f006_i74_d20": 538.39} -{"bbob_f006_i75_d20": -105.56} -{"bbob_f006_i76_d20": 98.03} -{"bbob_f006_i77_d20": 1000.0} -{"bbob_f006_i78_d20": 1.49} -{"bbob_f006_i79_d20": 596.35} -{"bbob_f006_i80_d20": -118.97} -{"bbob_f007_i01_d20": 92.94} -{"bbob_f007_i02_d20": 35.35} -{"bbob_f007_i03_d20": 3.82} -{"bbob_f007_i04_d20": 21.75} -{"bbob_f007_i05_d20": 64.79} -{"bbob_f007_i71_d20": -1000.0} -{"bbob_f007_i72_d20": 164.07} -{"bbob_f007_i73_d20": -165.68} -{"bbob_f007_i74_d20": 92.84} -{"bbob_f007_i75_d20": -367.92} -{"bbob_f007_i76_d20": 49.5} -{"bbob_f007_i77_d20": -12.77} -{"bbob_f007_i78_d20": 63.65} -{"bbob_f007_i79_d20": 699.19} -{"bbob_f007_i80_d20": 1000.0} -{"bbob_f008_i01_d20": 149.15} -{"bbob_f008_i02_d20": -1000.0} -{"bbob_f008_i03_d20": 98.62} -{"bbob_f008_i04_d20": -47.15} -{"bbob_f008_i05_d20": 37.52} -{"bbob_f008_i71_d20": 6.74} -{"bbob_f008_i72_d20": -189.79} -{"bbob_f008_i73_d20": -27.74} -{"bbob_f008_i74_d20": 6.93} -{"bbob_f008_i75_d20": -44.56} -{"bbob_f008_i76_d20": 1000.0} -{"bbob_f008_i77_d20": -679.71} -{"bbob_f008_i78_d20": 1000.0} -{"bbob_f008_i79_d20": 10.07} -{"bbob_f008_i80_d20": -7.54} -{"bbob_f009_i01_d20": 123.83} -{"bbob_f009_i02_d20": 47.51} -{"bbob_f009_i03_d20": 55.34} -{"bbob_f009_i04_d20": -90.33} -{"bbob_f009_i05_d20": 65.61} -{"bbob_f009_i71_d20": 934.82} -{"bbob_f009_i72_d20": -63.71} -{"bbob_f009_i73_d20": 1000.0} -{"bbob_f009_i74_d20": 31.97} -{"bbob_f009_i75_d20": 58.19} -{"bbob_f009_i76_d20": -4.84} -{"bbob_f009_i77_d20": -63.37} -{"bbob_f009_i78_d20": 2.75} -{"bbob_f009_i79_d20": -46.81} -{"bbob_f009_i80_d20": 61.34} -{"bbob_f010_i01_d20": -54.94} -{"bbob_f010_i02_d20": 59.13} -{"bbob_f010_i03_d20": -491.53} -{"bbob_f010_i04_d20": -170.2} -{"bbob_f010_i05_d20": -469.75} -{"bbob_f010_i71_d20": -51.91} -{"bbob_f010_i72_d20": 48.08} -{"bbob_f010_i73_d20": 30.58} -{"bbob_f010_i74_d20": 1000.0} -{"bbob_f010_i75_d20": -42.36} -{"bbob_f010_i76_d20": -71.87} -{"bbob_f010_i77_d20": -16.5} -{"bbob_f010_i78_d20": 1000.0} -{"bbob_f010_i79_d20": -1000.0} -{"bbob_f010_i80_d20": -152.74} -{"bbob_f011_i01_d20": 76.27} -{"bbob_f011_i02_d20": -22.55} -{"bbob_f011_i03_d20": 13.66} -{"bbob_f011_i04_d20": 937.84} -{"bbob_f011_i05_d20": -78.72} -{"bbob_f011_i71_d20": -41.51} -{"bbob_f011_i72_d20": 132.07} -{"bbob_f011_i73_d20": 97.25} -{"bbob_f011_i74_d20": -20.33} -{"bbob_f011_i75_d20": 94.31} -{"bbob_f011_i76_d20": -143.69} -{"bbob_f011_i77_d20": -353.38} -{"bbob_f011_i78_d20": -2.91} -{"bbob_f011_i79_d20": -2.97} -{"bbob_f011_i80_d20": 299.63} -{"bbob_f012_i01_d20": -621.11} -{"bbob_f012_i02_d20": -254.82} -{"bbob_f012_i03_d20": 56.61} -{"bbob_f012_i04_d20": -18.1} -{"bbob_f012_i05_d20": 37.77} -{"bbob_f012_i71_d20": -632.55} -{"bbob_f012_i72_d20": -85.3} -{"bbob_f012_i73_d20": 171.95} -{"bbob_f012_i74_d20": 68.16} -{"bbob_f012_i75_d20": 1000.0} -{"bbob_f012_i76_d20": -68.37} -{"bbob_f012_i77_d20": -55.48} -{"bbob_f012_i78_d20": -53.49} -{"bbob_f012_i79_d20": 117.27} -{"bbob_f012_i80_d20": 77.89} -{"bbob_f013_i01_d20": 29.97} -{"bbob_f013_i02_d20": -51.709999999999994} -{"bbob_f013_i03_d20": -279.95} -{"bbob_f013_i04_d20": 931.99} -{"bbob_f013_i05_d20": 113.31} -{"bbob_f013_i71_d20": 28.490000000000016} -{"bbob_f013_i72_d20": -740.58} -{"bbob_f013_i73_d20": -51.769999999999996} -{"bbob_f013_i74_d20": -222.98} -{"bbob_f013_i75_d20": 5.86} -{"bbob_f013_i76_d20": 204.86000000000004} -{"bbob_f013_i77_d20": 115.49000000000001} -{"bbob_f013_i78_d20": 195.05} -{"bbob_f013_i79_d20": -214.48} -{"bbob_f013_i80_d20": -45.899999999999984} -{"bbob_f014_i01_d20": -52.35} -{"bbob_f014_i02_d20": -179.54} -{"bbob_f014_i03_d20": 77.31} -{"bbob_f014_i04_d20": 39.92} -{"bbob_f014_i05_d20": -56.61} -{"bbob_f014_i71_d20": 32.7} -{"bbob_f014_i72_d20": -39.43} -{"bbob_f014_i73_d20": 419.72} -{"bbob_f014_i74_d20": 7.64} -{"bbob_f014_i75_d20": -127.68} -{"bbob_f014_i76_d20": -36.5} -{"bbob_f014_i77_d20": 172.87} -{"bbob_f014_i78_d20": 157.72} -{"bbob_f014_i79_d20": 28.19} -{"bbob_f014_i80_d20": -99.25} -{"bbob_f015_i01_d20": 1000.0} -{"bbob_f015_i02_d20": 70.03} -{"bbob_f015_i03_d20": -48.22} -{"bbob_f015_i04_d20": 25.47} -{"bbob_f015_i05_d20": -100.81} -{"bbob_f015_i71_d20": -353.13} -{"bbob_f015_i72_d20": 87.16} -{"bbob_f015_i73_d20": -34.75} -{"bbob_f015_i74_d20": -1000.0} -{"bbob_f015_i75_d20": -92.47} -{"bbob_f015_i76_d20": -215.3} -{"bbob_f015_i77_d20": -54.49} -{"bbob_f015_i78_d20": 50.49} -{"bbob_f015_i79_d20": -364.13} -{"bbob_f015_i80_d20": 159.27} -{"bbob_f016_i01_d20": 71.35} -{"bbob_f016_i02_d20": -355.22} -{"bbob_f016_i03_d20": 99.17} -{"bbob_f016_i04_d20": -517.91} -{"bbob_f016_i05_d20": -57.21} -{"bbob_f016_i71_d20": 66.26} -{"bbob_f016_i72_d20": -138.62} -{"bbob_f016_i73_d20": 731.39} -{"bbob_f016_i74_d20": 75.26} -{"bbob_f016_i75_d20": -49.13} -{"bbob_f016_i76_d20": -141.89} -{"bbob_f016_i77_d20": 109.57} -{"bbob_f016_i78_d20": 78.36} -{"bbob_f016_i79_d20": 114.09} -{"bbob_f016_i80_d20": -544.59} -{"bbob_f017_i01_d20": -16.94} -{"bbob_f017_i02_d20": 18.81} -{"bbob_f017_i03_d20": 273.15} -{"bbob_f017_i04_d20": 37.18} -{"bbob_f017_i05_d20": -147.9} -{"bbob_f017_i71_d20": 481.71} -{"bbob_f017_i72_d20": -36.46} -{"bbob_f017_i73_d20": 39.2} -{"bbob_f017_i74_d20": -32.25} -{"bbob_f017_i75_d20": 368.12} -{"bbob_f017_i76_d20": 47.39} -{"bbob_f017_i77_d20": -342.89} -{"bbob_f017_i78_d20": 91.02} -{"bbob_f017_i79_d20": 732.42} -{"bbob_f017_i80_d20": 35.42} -{"bbob_f018_i01_d20": -16.94} -{"bbob_f018_i02_d20": 18.81} -{"bbob_f018_i03_d20": 273.15} -{"bbob_f018_i04_d20": 37.18} -{"bbob_f018_i05_d20": -147.9} -{"bbob_f018_i71_d20": 481.71} -{"bbob_f018_i72_d20": -36.46} -{"bbob_f018_i73_d20": 39.2} -{"bbob_f018_i74_d20": -32.25} -{"bbob_f018_i75_d20": 368.12} -{"bbob_f018_i76_d20": 47.39} -{"bbob_f018_i77_d20": -342.89} -{"bbob_f018_i78_d20": 91.02} -{"bbob_f018_i79_d20": 732.42} -{"bbob_f018_i80_d20": 35.42} -{"bbob_f019_i01_d20": -102.55} -{"bbob_f019_i02_d20": 71.69} -{"bbob_f019_i03_d20": 214.01} -{"bbob_f019_i04_d20": -58.48} -{"bbob_f019_i05_d20": -613.08} -{"bbob_f019_i71_d20": 61.89} -{"bbob_f019_i72_d20": -84.56} -{"bbob_f019_i73_d20": -26.02} -{"bbob_f019_i74_d20": -30.15} -{"bbob_f019_i75_d20": -1000.0} -{"bbob_f019_i76_d20": -924.3} -{"bbob_f019_i77_d20": -10.35} -{"bbob_f019_i78_d20": -1000.0} -{"bbob_f019_i79_d20": 28.8} -{"bbob_f019_i80_d20": 35.34} -{"bbob_f020_i01_d20": -546.5} -{"bbob_f020_i02_d20": 1000.0} -{"bbob_f020_i03_d20": 208.38} -{"bbob_f020_i04_d20": -144.96} -{"bbob_f020_i05_d20": -6.299999999999999} -{"bbob_f020_i71_d20": -136.76} -{"bbob_f020_i72_d20": 366.75} -{"bbob_f020_i73_d20": -248.69} -{"bbob_f020_i74_d20": -67.86} -{"bbob_f020_i75_d20": -23.29} -{"bbob_f020_i76_d20": -10.279999999999998} -{"bbob_f020_i77_d20": 179.97} -{"bbob_f020_i78_d20": 83.76} -{"bbob_f020_i79_d20": -389.01} -{"bbob_f020_i80_d20": -145.58} -{"bbob_f021_i01_d20": 40.78} -{"bbob_f021_i02_d20": -1.6} -{"bbob_f021_i03_d20": -370.84} -{"bbob_f021_i04_d20": -32.56} -{"bbob_f021_i05_d20": 515.4} -{"bbob_f021_i71_d20": -86.28} -{"bbob_f021_i72_d20": 22.24} -{"bbob_f021_i73_d20": -23.01} -{"bbob_f021_i74_d20": -332.37} -{"bbob_f021_i75_d20": -123.13} -{"bbob_f021_i76_d20": -153.79} -{"bbob_f021_i77_d20": -48.6} -{"bbob_f021_i78_d20": 27.7} -{"bbob_f021_i79_d20": 29.93} -{"bbob_f021_i80_d20": -19.83} -{"bbob_f022_i01_d20": -1000.0} -{"bbob_f022_i02_d20": 1000.0} -{"bbob_f022_i03_d20": -49.13} -{"bbob_f022_i04_d20": 297.73} -{"bbob_f022_i05_d20": 51.57} -{"bbob_f022_i71_d20": 152.71} -{"bbob_f022_i72_d20": 99.8} -{"bbob_f022_i73_d20": -119.41} -{"bbob_f022_i74_d20": 10.77} -{"bbob_f022_i75_d20": 21.37} -{"bbob_f022_i76_d20": 16.73} -{"bbob_f022_i77_d20": 1000.0} -{"bbob_f022_i78_d20": -107.96} -{"bbob_f022_i79_d20": -302.59} -{"bbob_f022_i80_d20": -306.55} -{"bbob_f023_i01_d20": 6.870000000000001} -{"bbob_f023_i02_d20": 0.010000000000001315} -{"bbob_f023_i03_d20": -130.61} -{"bbob_f023_i04_d20": -223.12} -{"bbob_f023_i05_d20": -44.43} -{"bbob_f023_i71_d20": 255.78} -{"bbob_f023_i72_d20": 1000.0} -{"bbob_f023_i73_d20": -186.95} -{"bbob_f023_i74_d20": 585.95} -{"bbob_f023_i75_d20": 287.16} -{"bbob_f023_i76_d20": -394.38} -{"bbob_f023_i77_d20": 54.43} -{"bbob_f023_i78_d20": 139.69} -{"bbob_f023_i79_d20": -57.94} -{"bbob_f023_i80_d20": 125.39} -{"bbob_f024_i01_d20": 102.61} -{"bbob_f024_i02_d20": 93.3} -{"bbob_f024_i03_d20": 13.64} -{"bbob_f024_i04_d20": 194.8} -{"bbob_f024_i05_d20": -133.59} -{"bbob_f024_i71_d20": -30.85} -{"bbob_f024_i72_d20": -9.2} -{"bbob_f024_i73_d20": -203.76} -{"bbob_f024_i74_d20": 33.99} -{"bbob_f024_i75_d20": -1000.0} -{"bbob_f024_i76_d20": -1000.0} -{"bbob_f024_i77_d20": -12.72} -{"bbob_f024_i78_d20": -59.54} -{"bbob_f024_i79_d20": -1000.0} -{"bbob_f024_i80_d20": 43.03} -{"bbob_f001_i01_d40": 79.48} -{"bbob_f001_i02_d40": 394.48} -{"bbob_f001_i03_d40": -247.11} -{"bbob_f001_i04_d40": -152.04} -{"bbob_f001_i05_d40": -25.25} -{"bbob_f001_i71_d40": 183.01} -{"bbob_f001_i72_d40": 183.52} -{"bbob_f001_i73_d40": -593.84} -{"bbob_f001_i74_d40": -1000.0} -{"bbob_f001_i75_d40": -25.31} -{"bbob_f001_i76_d40": 22.28} -{"bbob_f001_i77_d40": 356.14} -{"bbob_f001_i78_d40": -475.7} -{"bbob_f001_i79_d40": -1000.0} -{"bbob_f001_i80_d40": -41.58} -{"bbob_f002_i01_d40": -209.88} -{"bbob_f002_i02_d40": -92.09} -{"bbob_f002_i03_d40": -87.89} -{"bbob_f002_i04_d40": 320.19} -{"bbob_f002_i05_d40": -104.24} -{"bbob_f002_i71_d40": -67.85} -{"bbob_f002_i72_d40": -363.98} -{"bbob_f002_i73_d40": -18.65} -{"bbob_f002_i74_d40": 174.92} -{"bbob_f002_i75_d40": -565.96} -{"bbob_f002_i76_d40": 70.95} -{"bbob_f002_i77_d40": 47.28} -{"bbob_f002_i78_d40": -568.88} -{"bbob_f002_i79_d40": 20.69} -{"bbob_f002_i80_d40": 561.67} -{"bbob_f003_i01_d40": -462.09} -{"bbob_f003_i02_d40": 77.66} -{"bbob_f003_i03_d40": 115.68} -{"bbob_f003_i04_d40": -3.71} -{"bbob_f003_i05_d40": 132.18} -{"bbob_f003_i71_d40": -39.3} -{"bbob_f003_i72_d40": 42.96} -{"bbob_f003_i73_d40": 284.22} -{"bbob_f003_i74_d40": -32.17} -{"bbob_f003_i75_d40": -58.58} -{"bbob_f003_i76_d40": -604.92} -{"bbob_f003_i77_d40": 133.7} -{"bbob_f003_i78_d40": -20.98} -{"bbob_f003_i79_d40": 135.33} -{"bbob_f003_i80_d40": 40.66} -{"bbob_f004_i01_d40": -462.09} -{"bbob_f004_i02_d40": 77.66} -{"bbob_f004_i03_d40": 115.68} -{"bbob_f004_i04_d40": -3.71} -{"bbob_f004_i05_d40": 132.18} -{"bbob_f004_i71_d40": -39.3} -{"bbob_f004_i72_d40": 42.96} -{"bbob_f004_i73_d40": 284.22} -{"bbob_f004_i74_d40": -32.17} -{"bbob_f004_i75_d40": -58.58} -{"bbob_f004_i76_d40": -604.92} -{"bbob_f004_i77_d40": 133.7} -{"bbob_f004_i78_d40": -20.98} -{"bbob_f004_i79_d40": 135.33} -{"bbob_f004_i80_d40": 40.66} -{"bbob_f005_i01_d40": -9.21} -{"bbob_f005_i02_d40": 655.99} -{"bbob_f005_i03_d40": 66.71} -{"bbob_f005_i04_d40": 941.67} -{"bbob_f005_i05_d40": -37.45} -{"bbob_f005_i71_d40": -66.39} -{"bbob_f005_i72_d40": -226.01} -{"bbob_f005_i73_d40": 1000.0} -{"bbob_f005_i74_d40": -183.02} -{"bbob_f005_i75_d40": -102.62} -{"bbob_f005_i76_d40": 99.53} -{"bbob_f005_i77_d40": 149.59} -{"bbob_f005_i78_d40": -1000.0} -{"bbob_f005_i79_d40": -29.22} -{"bbob_f005_i80_d40": -40.96} -{"bbob_f006_i01_d40": 35.9} -{"bbob_f006_i02_d40": 31.37} -{"bbob_f006_i03_d40": -1000.0} -{"bbob_f006_i04_d40": 131.92} -{"bbob_f006_i05_d40": 590.84} -{"bbob_f006_i71_d40": 67.49} -{"bbob_f006_i72_d40": 30.85} -{"bbob_f006_i73_d40": -12.55} -{"bbob_f006_i74_d40": 538.39} -{"bbob_f006_i75_d40": -105.56} -{"bbob_f006_i76_d40": 98.03} -{"bbob_f006_i77_d40": 1000.0} -{"bbob_f006_i78_d40": 1.49} -{"bbob_f006_i79_d40": 596.35} -{"bbob_f006_i80_d40": -118.97} -{"bbob_f007_i01_d40": 92.94} -{"bbob_f007_i02_d40": 35.35} -{"bbob_f007_i03_d40": 3.82} -{"bbob_f007_i04_d40": 21.75} -{"bbob_f007_i05_d40": 64.79} -{"bbob_f007_i71_d40": -1000.0} -{"bbob_f007_i72_d40": 164.07} -{"bbob_f007_i73_d40": -165.68} -{"bbob_f007_i74_d40": 92.84} -{"bbob_f007_i75_d40": -367.92} -{"bbob_f007_i76_d40": 49.5} -{"bbob_f007_i77_d40": -12.77} -{"bbob_f007_i78_d40": 63.65} -{"bbob_f007_i79_d40": 699.19} -{"bbob_f007_i80_d40": 1000.0} -{"bbob_f008_i01_d40": 149.15} -{"bbob_f008_i02_d40": -1000.0} -{"bbob_f008_i03_d40": 98.62} -{"bbob_f008_i04_d40": -47.15} -{"bbob_f008_i05_d40": 37.52} -{"bbob_f008_i71_d40": 6.74} -{"bbob_f008_i72_d40": -189.79} -{"bbob_f008_i73_d40": -27.74} -{"bbob_f008_i74_d40": 6.93} -{"bbob_f008_i75_d40": -44.56} -{"bbob_f008_i76_d40": 1000.0} -{"bbob_f008_i77_d40": -679.71} -{"bbob_f008_i78_d40": 1000.0} -{"bbob_f008_i79_d40": 10.07} -{"bbob_f008_i80_d40": -7.54} -{"bbob_f009_i01_d40": 123.83} -{"bbob_f009_i02_d40": 47.51} -{"bbob_f009_i03_d40": 55.34} -{"bbob_f009_i04_d40": -90.33} -{"bbob_f009_i05_d40": 65.61} -{"bbob_f009_i71_d40": 934.82} -{"bbob_f009_i72_d40": -63.71} -{"bbob_f009_i73_d40": 1000.0} -{"bbob_f009_i74_d40": 31.97} -{"bbob_f009_i75_d40": 58.19} -{"bbob_f009_i76_d40": -4.84} -{"bbob_f009_i77_d40": -63.37} -{"bbob_f009_i78_d40": 2.75} -{"bbob_f009_i79_d40": -46.81} -{"bbob_f009_i80_d40": 61.34} -{"bbob_f010_i01_d40": -54.94} -{"bbob_f010_i02_d40": 59.13} -{"bbob_f010_i03_d40": -491.53} -{"bbob_f010_i04_d40": -170.2} -{"bbob_f010_i05_d40": -469.75} -{"bbob_f010_i71_d40": -51.91} -{"bbob_f010_i72_d40": 48.08} -{"bbob_f010_i73_d40": 30.58} -{"bbob_f010_i74_d40": 1000.0} -{"bbob_f010_i75_d40": -42.36} -{"bbob_f010_i76_d40": -71.87} -{"bbob_f010_i77_d40": -16.5} -{"bbob_f010_i78_d40": 1000.0} -{"bbob_f010_i79_d40": -1000.0} -{"bbob_f010_i80_d40": -152.74} -{"bbob_f011_i01_d40": 76.27} -{"bbob_f011_i02_d40": -22.55} -{"bbob_f011_i03_d40": 13.66} -{"bbob_f011_i04_d40": 937.84} -{"bbob_f011_i05_d40": -78.72} -{"bbob_f011_i71_d40": -41.51} -{"bbob_f011_i72_d40": 132.07} -{"bbob_f011_i73_d40": 97.25} -{"bbob_f011_i74_d40": -20.33} -{"bbob_f011_i75_d40": 94.31} -{"bbob_f011_i76_d40": -143.69} -{"bbob_f011_i77_d40": -353.38} -{"bbob_f011_i78_d40": -2.91} -{"bbob_f011_i79_d40": -2.97} -{"bbob_f011_i80_d40": 299.63} -{"bbob_f012_i01_d40": -621.11} -{"bbob_f012_i02_d40": -254.82} -{"bbob_f012_i03_d40": 56.61} -{"bbob_f012_i04_d40": -18.1} -{"bbob_f012_i05_d40": 37.77} -{"bbob_f012_i71_d40": -632.55} -{"bbob_f012_i72_d40": -85.3} -{"bbob_f012_i73_d40": 171.95} -{"bbob_f012_i74_d40": 68.16} -{"bbob_f012_i75_d40": 1000.0} -{"bbob_f012_i76_d40": -68.37} -{"bbob_f012_i77_d40": -55.48} -{"bbob_f012_i78_d40": -53.49} -{"bbob_f012_i79_d40": 117.27} -{"bbob_f012_i80_d40": 77.89} -{"bbob_f013_i01_d40": 29.97000000000001} -{"bbob_f013_i02_d40": -51.70999999999999} -{"bbob_f013_i03_d40": -279.95} -{"bbob_f013_i04_d40": 931.99} -{"bbob_f013_i05_d40": 113.31} -{"bbob_f013_i71_d40": 28.49000000000002} -{"bbob_f013_i72_d40": -740.58} -{"bbob_f013_i73_d40": -51.76999999999999} -{"bbob_f013_i74_d40": -222.98} -{"bbob_f013_i75_d40": 5.860000000000015} -{"bbob_f013_i76_d40": 204.86000000000004} -{"bbob_f013_i77_d40": 115.49000000000001} -{"bbob_f013_i78_d40": 195.05000000000004} -{"bbob_f013_i79_d40": -214.48} -{"bbob_f013_i80_d40": -45.899999999999984} -{"bbob_f014_i01_d40": -52.35} -{"bbob_f014_i02_d40": -179.54} -{"bbob_f014_i03_d40": 77.31} -{"bbob_f014_i04_d40": 39.92} -{"bbob_f014_i05_d40": -56.61} -{"bbob_f014_i71_d40": 32.7} -{"bbob_f014_i72_d40": -39.43} -{"bbob_f014_i73_d40": 419.72} -{"bbob_f014_i74_d40": 7.64} -{"bbob_f014_i75_d40": -127.68} -{"bbob_f014_i76_d40": -36.5} -{"bbob_f014_i77_d40": 172.87} -{"bbob_f014_i78_d40": 157.72} -{"bbob_f014_i79_d40": 28.19} -{"bbob_f014_i80_d40": -99.25} -{"bbob_f015_i01_d40": 1000.0} -{"bbob_f015_i02_d40": 70.03} -{"bbob_f015_i03_d40": -48.22} -{"bbob_f015_i04_d40": 25.47} -{"bbob_f015_i05_d40": -100.81} -{"bbob_f015_i71_d40": -353.13} -{"bbob_f015_i72_d40": 87.16} -{"bbob_f015_i73_d40": -34.75} -{"bbob_f015_i74_d40": -1000.0} -{"bbob_f015_i75_d40": -92.47} -{"bbob_f015_i76_d40": -215.3} -{"bbob_f015_i77_d40": -54.49} -{"bbob_f015_i78_d40": 50.49} -{"bbob_f015_i79_d40": -364.13} -{"bbob_f015_i80_d40": 159.27} -{"bbob_f016_i01_d40": 71.35} -{"bbob_f016_i02_d40": -355.22} -{"bbob_f016_i03_d40": 99.17} -{"bbob_f016_i04_d40": -517.91} -{"bbob_f016_i05_d40": -57.21} -{"bbob_f016_i71_d40": 66.26} -{"bbob_f016_i72_d40": -138.62} -{"bbob_f016_i73_d40": 731.39} -{"bbob_f016_i74_d40": 75.26} -{"bbob_f016_i75_d40": -49.13} -{"bbob_f016_i76_d40": -141.89} -{"bbob_f016_i77_d40": 109.57} -{"bbob_f016_i78_d40": 78.36} -{"bbob_f016_i79_d40": 114.09} -{"bbob_f016_i80_d40": -544.59} -{"bbob_f017_i01_d40": -16.94} -{"bbob_f017_i02_d40": 18.81} -{"bbob_f017_i03_d40": 273.15} -{"bbob_f017_i04_d40": 37.18} -{"bbob_f017_i05_d40": -147.9} -{"bbob_f017_i71_d40": 481.71} -{"bbob_f017_i72_d40": -36.46} -{"bbob_f017_i73_d40": 39.2} -{"bbob_f017_i74_d40": -32.25} -{"bbob_f017_i75_d40": 368.12} -{"bbob_f017_i76_d40": 47.39} -{"bbob_f017_i77_d40": -342.89} -{"bbob_f017_i78_d40": 91.02} -{"bbob_f017_i79_d40": 732.42} -{"bbob_f017_i80_d40": 35.42} -{"bbob_f018_i01_d40": -16.94} -{"bbob_f018_i02_d40": 18.81} -{"bbob_f018_i03_d40": 273.15} -{"bbob_f018_i04_d40": 37.18} -{"bbob_f018_i05_d40": -147.9} -{"bbob_f018_i71_d40": 481.71} -{"bbob_f018_i72_d40": -36.46} -{"bbob_f018_i73_d40": 39.2} -{"bbob_f018_i74_d40": -32.25} -{"bbob_f018_i75_d40": 368.12} -{"bbob_f018_i76_d40": 47.39} -{"bbob_f018_i77_d40": -342.89} -{"bbob_f018_i78_d40": 91.02} -{"bbob_f018_i79_d40": 732.42} -{"bbob_f018_i80_d40": 35.42} -{"bbob_f019_i01_d40": -102.55} -{"bbob_f019_i02_d40": 71.69} -{"bbob_f019_i03_d40": 214.01} -{"bbob_f019_i04_d40": -58.48} -{"bbob_f019_i05_d40": -613.08} -{"bbob_f019_i71_d40": 61.89} -{"bbob_f019_i72_d40": -84.56} -{"bbob_f019_i73_d40": -26.02} -{"bbob_f019_i74_d40": -30.15} -{"bbob_f019_i75_d40": -1000.0} -{"bbob_f019_i76_d40": -924.3} -{"bbob_f019_i77_d40": -10.35} -{"bbob_f019_i78_d40": -1000.0} -{"bbob_f019_i79_d40": 28.8} -{"bbob_f019_i80_d40": 35.34} -{"bbob_f020_i01_d40": -546.5} -{"bbob_f020_i02_d40": 1000.0} -{"bbob_f020_i03_d40": 208.38} -{"bbob_f020_i04_d40": -144.96} -{"bbob_f020_i05_d40": -6.299999999999999} -{"bbob_f020_i71_d40": -136.76} -{"bbob_f020_i72_d40": 366.75} -{"bbob_f020_i73_d40": -248.69} -{"bbob_f020_i74_d40": -67.86} -{"bbob_f020_i75_d40": -23.29} -{"bbob_f020_i76_d40": -10.279999999999998} -{"bbob_f020_i77_d40": 179.97} -{"bbob_f020_i78_d40": 83.76} -{"bbob_f020_i79_d40": -389.01} -{"bbob_f020_i80_d40": -145.58} -{"bbob_f021_i01_d40": 40.78} -{"bbob_f021_i02_d40": -1.6} -{"bbob_f021_i03_d40": -370.84} -{"bbob_f021_i04_d40": -32.56} -{"bbob_f021_i05_d40": 515.4} -{"bbob_f021_i71_d40": -86.28} -{"bbob_f021_i72_d40": 22.24} -{"bbob_f021_i73_d40": -23.01} -{"bbob_f021_i74_d40": -332.37} -{"bbob_f021_i75_d40": -123.13} -{"bbob_f021_i76_d40": -153.79} -{"bbob_f021_i77_d40": -48.6} -{"bbob_f021_i78_d40": 27.7} -{"bbob_f021_i79_d40": 29.93} -{"bbob_f021_i80_d40": -19.83} -{"bbob_f022_i01_d40": -1000.0} -{"bbob_f022_i02_d40": 1000.0} -{"bbob_f022_i03_d40": -49.13} -{"bbob_f022_i04_d40": 297.73} -{"bbob_f022_i05_d40": 51.57} -{"bbob_f022_i71_d40": 152.71} -{"bbob_f022_i72_d40": 99.8} -{"bbob_f022_i73_d40": -119.41} -{"bbob_f022_i74_d40": 10.77} -{"bbob_f022_i75_d40": 21.37} -{"bbob_f022_i76_d40": 16.73} -{"bbob_f022_i77_d40": 1000.0} -{"bbob_f022_i78_d40": -107.96} -{"bbob_f022_i79_d40": -302.59} -{"bbob_f022_i80_d40": -306.55} -{"bbob_f023_i01_d40": 6.870000000000001} -{"bbob_f023_i02_d40": 0.010000000000001395} -{"bbob_f023_i03_d40": -130.61} -{"bbob_f023_i04_d40": -223.12} -{"bbob_f023_i05_d40": -44.43} -{"bbob_f023_i71_d40": 255.78} -{"bbob_f023_i72_d40": 1000.0} -{"bbob_f023_i73_d40": -186.95} -{"bbob_f023_i74_d40": 585.95} -{"bbob_f023_i75_d40": 287.16} -{"bbob_f023_i76_d40": -394.38} -{"bbob_f023_i77_d40": 54.43} -{"bbob_f023_i78_d40": 139.69} -{"bbob_f023_i79_d40": -57.94} -{"bbob_f023_i80_d40": 125.39} -{"bbob_f024_i01_d40": 102.61} -{"bbob_f024_i02_d40": 93.3} -{"bbob_f024_i03_d40": 13.64} -{"bbob_f024_i04_d40": 194.8} -{"bbob_f024_i05_d40": -133.59} -{"bbob_f024_i71_d40": -30.85} -{"bbob_f024_i72_d40": -9.2} -{"bbob_f024_i73_d40": -203.76} -{"bbob_f024_i74_d40": 33.99} -{"bbob_f024_i75_d40": -1000.0} -{"bbob_f024_i76_d40": -1000.0} -{"bbob_f024_i77_d40": -12.72} -{"bbob_f024_i78_d40": -59.54} -{"bbob_f024_i79_d40": -1000.0} -{"bbob_f024_i80_d40": 43.03} diff --git a/das/env/das_env.py b/das/env/das_env.py index 1ab820d..d508b5c 100644 --- a/das/env/das_env.py +++ b/das/env/das_env.py @@ -41,7 +41,10 @@ class DASEnv(gym.Env): reward_option: 1=log-scaled, 2=linear, 3=sparse, 4=binary (see das/env/reward.py). n_individuals: - Population size shared across all sub-optimizers. + Population size per sub-optimizer. ``None`` (default) lets each + algorithm use its own built-in default. Pass a single ``int`` to + override all algorithms with the same value, or a list of ``int | + None`` with one entry per optimizer for a mixed setup. """ metadata = {"render_modes": []} @@ -55,7 +58,7 @@ def __init__( n_checkpoints: int = 10, checkpoint_division_base: float = 1.0, reward_option: int = 1, - n_individuals: int = 100, + n_individuals: int | list[int | None] | None = None, seed: int | None = None, ): super().__init__() @@ -66,7 +69,18 @@ def __init__( self.n_checkpoints = n_checkpoints self.cdb = checkpoint_division_base self.reward_option = reward_option - self.n_individuals = n_individuals + n_opt = len(optimizers) + if n_individuals is None or isinstance(n_individuals, int): + self.n_individuals: list[int | None] = [n_individuals] * n_opt + else: + pop = list(n_individuals) + if len(pop) == 1: + pop = pop * n_opt + if len(pop) != n_opt: + raise ValueError( + f"n_individuals has {len(pop)} entries but portfolio has {n_opt}" + ) + self.n_individuals = pop self._seed = seed n_actions = len(optimizers) @@ -109,8 +123,9 @@ def reset(self, seed=None, options=None): self._problem = self.suite.get_problem(problem_id) dim = self._problem.dimension self._max_fe = self.fe_multiplier * dim + known = [n for n in self.n_individuals if n is not None] self._checkpoints = get_checkpoints( - self.n_checkpoints, self._max_fe, self.n_individuals, self.cdb + self.n_checkpoints, self._max_fe, max(known) if known else 1, self.cdb ) # Reset episode bookkeeping @@ -175,13 +190,15 @@ def _run_optimizer(self, action: int, target_fe: int) -> dict: "lower_boundary": self._problem.lower_bounds, "upper_boundary": self._problem.upper_bounds, } + n_ind = self.n_individuals[action] options = { "max_function_evaluations": self._max_fe, "target_fe": target_fe, - "n_individuals": self.n_individuals, "best_so_far_y": self._best_y, "verbose": False, } + if n_ind is not None: + options["n_individuals"] = n_ind # Derive a deterministic seed for pypop7's default_rng, which is # independent of np.random and must be seeded explicitly. if self._seed is not None: @@ -210,9 +227,14 @@ def _run_optimizer(self, action: int, target_fe: int) -> dict: else: # Fallback: carry x/y from the population history if len(optimizer.x_history) > 0: + x_hist = optimizer.x_history + y_hist = optimizer.y_history + if n_ind is not None: + x_hist = x_hist[-n_ind:] + y_hist = y_hist[-n_ind:] self._optimizer_state = { - "x": np.array(optimizer.x_history[-self.n_individuals :]), - "y": np.array(optimizer.y_history[-self.n_individuals :]), + "x": np.array(x_hist), + "y": np.array(y_hist), } return result diff --git a/das/training/callbacks.py b/das/training/callbacks.py index 83a3c68..2031831 100644 --- a/das/training/callbacks.py +++ b/das/training/callbacks.py @@ -96,8 +96,6 @@ class DASEvalCallback(BaseCallback): Number of problems to evaluate (taken from the start of the problem list). name: Prefix for result files written to results/. - global_optima: - Mapping from problem_id to its global minimum (for AOCC computation). """ def __init__( @@ -106,7 +104,6 @@ def __init__( eval_freq: int = 10_000, n_eval_episodes: int = 20, name: str = "eval", - global_optima: dict[str, float] | None = None, verbose: int = 1, ): super().__init__(verbose) @@ -114,7 +111,6 @@ def __init__( self.eval_freq = eval_freq self.n_eval_episodes = n_eval_episodes self.name = name - self.global_optima = global_optima or {} self._best_mean_aocc = -np.inf def _on_step(self) -> bool: diff --git a/das/training/common.py b/das/training/common.py index 326c989..b8ac184 100644 --- a/das/training/common.py +++ b/das/training/common.py @@ -1,18 +1,26 @@ """Shared training utilities.""" import json -import os +import re import numpy as np from das.env.das_env import DASEnv +_BBOB_ID_RE = re.compile(r"^bbob_f(\d+)_i(\d+)_d(\d+)$") -def load_global_optima(path: str = "bbob_optima.jsonl") -> dict[str, float]: - if not os.path.exists(path): - return {} - with open(path) as f: - return {k: v for line in f for k, v in json.loads(line).items()} + +def get_ioh_optimum(problem_id: str) -> float: + """Return the global minimum for a BBOB problem via the IOH API.""" + m = _BBOB_ID_RE.match(problem_id) + if m is None: + return 0.0 + import ioh + + fid, iid, dim = int(m.group(1)), int(m.group(2)), int(m.group(3)) + return float( + ioh.get_problem(fid, iid, dim, problem_class=ioh.ProblemClass.BBOB).optimum.y + ) # Standard BBOB precision targets (excess above the global optimum). diff --git a/das/training/expdas.py b/das/training/expdas.py index 215e2f8..191da22 100644 --- a/das/training/expdas.py +++ b/das/training/expdas.py @@ -10,7 +10,7 @@ from das.env.das_env import DASEnv from das.env.observation import observation_dim from das.optimizers.portfolio import get_portfolio -from das.training.common import load_global_optima, write_jsonl +from das.training.common import write_jsonl def run_exp_das(args) -> None: @@ -78,12 +78,10 @@ def run_exp_das(args) -> None: if args.eval: print("\nFinal evaluation on test set …") - global_optima = load_global_optima() test_results = evaluate( test_env, agent, n_episodes=min(len(test_ids), 50), - global_optima=global_optima, ) mean_final_fitness = float( np.mean([next(iter(r.values()))["final_fitness"] for r in test_results]) @@ -168,11 +166,8 @@ def _run_single_fold( with open(result_path) as fh: fold_results = [json.loads(line) for line in fh] else: - global_optima = load_global_optima() eval_env = DASEnv(problem_ids=test_ids, **env_cfg) - raw = _evaluate( - eval_env, agent, n_episodes=len(test_ids), global_optima=global_optima - ) + raw = _evaluate(eval_env, agent, n_episodes=len(test_ids)) fold_results = [ {pid: {**m, "fold": fold_tag}} for r in raw for pid, m in r.items() ] diff --git a/das/training/ppo.py b/das/training/ppo.py index 5eaf452..79fdf2d 100644 --- a/das/training/ppo.py +++ b/das/training/ppo.py @@ -8,11 +8,11 @@ from das.env.bbob_splits import get_cv_folds, get_train_test_split from das.optimizers.portfolio import get_portfolio -from das.training.common import load_global_optima, make_das_env, write_jsonl +from das.training.common import get_ioh_optimum, make_das_env, write_jsonl def _eval_loop( - model, eval_env, problem_ids: list[str], global_optima: dict, desc: str = "eval" + model, eval_env, problem_ids: list[str], desc: str = "eval" ) -> list[dict]: results = [] for problem_id in tqdm(problem_ids, desc=f" {desc}", smoothing=0.0): @@ -25,7 +25,7 @@ def _eval_loop( if done[0]: info = infos[0] best_y = info.get("best_y", float("inf")) - optimum = global_optima.get(problem_id, 0.0) + optimum = get_ioh_optimum(problem_id) results.append( {"problem_id": problem_id, "best_y": best_y, "gap": best_y - optimum} ) @@ -155,8 +155,7 @@ def run_ppo(args) -> None: model_path = os.path.join("models", args.name) model = PPO.load(model_path) eval_env = _load_eval_env(model_path, test_ids, optimizers, cfg, args.seed) - global_optima = load_global_optima() - results = _eval_loop(model, eval_env, test_ids, global_optima) + results = _eval_loop(model, eval_env, test_ids) eval_env.close() out_path = os.path.join("results", f"{args.name}_eval.jsonl") write_jsonl(out_path, results) @@ -181,7 +180,6 @@ def run_cv_ppo(args) -> None: print(f"Portfolio : {args.portfolio}") print(f"Budget : {args.fe_multiplier}×dim | checkpoints={args.n_checkpoints}") - global_optima = load_global_optima() all_folds = get_cv_folds( args.cv_mode, args.dims, seed=args.seed, n_folds=args.n_folds ) @@ -220,7 +218,7 @@ def run_cv_ppo(args) -> None: else: eval_env = _load_eval_env(model_path, test_ids, optimizers, cfg, args.seed) fold_results = _eval_loop( - model, eval_env, test_ids, global_optima, desc=f"eval {fold_tag}" + model, eval_env, test_ids, desc=f"eval {fold_tag}" ) for r in fold_results: r["fold"] = fold_tag diff --git a/evaluate.py b/evaluate.py index da8fd06..80cbfd2 100644 --- a/evaluate.py +++ b/evaluate.py @@ -22,7 +22,7 @@ from das.env.bbob_splits import ALL_DIMS, get_train_test_split from das.optimizers.portfolio import get_portfolio from das.utils import set_seed -from das.training.common import compute_run_stats, load_global_optima, make_das_env +from das.training.common import compute_run_stats, get_ioh_optimum, make_das_env warnings.filterwarnings("ignore") @@ -44,7 +44,7 @@ def parse_args(): p.add_argument("-s", "--n-checkpoints", type=int, default=10) p.add_argument("-x", "--cdb", type=float, default=1.0) p.add_argument("-O", "--reward-option", type=int, default=1, choices=[1, 2, 3, 4]) - p.add_argument("-n", "--n-individuals", type=int, default=100) + p.add_argument("-n", "--n-individuals", type=int, default=None) p.add_argument("--coco", action="store_true", help="Write COCO observer data") p.add_argument("--seed", type=int, default=42) return p.parse_args() @@ -84,8 +84,6 @@ def main(): eval_env.training = False eval_env.norm_reward = False - global_optima = load_global_optima() - out_path = os.path.join("results", f"{args.name}_eval.jsonl") results_all = [] @@ -101,7 +99,7 @@ def main(): fitness_history.extend(step_info.get("fitness_history_step", [])) max_fe = step_info.get("n_fe", 0) - global_minimum = global_optima.get(problem_id, 0.0) + global_minimum = get_ioh_optimum(problem_id) stats = compute_run_stats(fitness_history, max_fe, global_minimum) results_all.append({problem_id: stats}) diff --git a/exp_das_study.slurm b/exp_das_study.slurm index b8b3ae9..ea9f3ab 100644 --- a/exp_das_study.slurm +++ b/exp_das_study.slurm @@ -8,7 +8,7 @@ #SBATCH --time=48:00:00 #SBATCH --partition=plgrid-gpu-a100 #SBATCH -A plgrldas2026-gpu-a100 -#SBATCH --array=0-3 +#SBATCH --array=0-11 # Args: SEED [PORTFOLIO...] SEED=${1:-42} @@ -50,4 +50,17 @@ elif [[ $SLURM_ARRAY_TASK_ID -eq 3 ]]; then echo "Exp-DAS | CV-LOPO | multi-dim (2 3 5 10)" python cv.py exp-das ${PORTFOLIO_STR}_EXPDAS_LOPO_ALLDIM_SEED${SEED} \ -p "${PORTFOLIO[@]}" --dims 2 3 5 10 --cv-mode LOPO --n-epochs 10 --seed $SEED + +# 4-11: single-dimension runs; task ID encodes (dim_idx * 2 + cv_mode) +# dim_idx: 0→2, 1→3, 2→5, 3→10 | cv_mode: 0→LOIO, 1→LOPO +else + DIMS=(2 3 5 10) + DIM_IDX=$(( (SLURM_ARRAY_TASK_ID - 4) / 2 )) + CV_IDX=$(( (SLURM_ARRAY_TASK_ID - 4) % 2 )) + DIM=${DIMS[$DIM_IDX]} + CV_MODES=('LOIO' 'LOPO') + CV_MODE=${CV_MODES[$CV_IDX]} + echo "Exp-DAS | CV-${CV_MODE} | dim=${DIM}" + python cv.py exp-das "${PORTFOLIO_STR}_EXPDAS_${CV_MODE}_${DIM}D_SEED${SEED}" \ + -p "${PORTFOLIO[@]}" --dims "${DIM}" --cv-mode "${CV_MODE}" --n-epochs 10 --seed "$SEED" fi \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 0e45f7c..457bb42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,11 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["das", "agents"] +[tool.pytest.ini_options] +markers = [ + "integration: end-to-end tests that use real IOH/pypop7 (slower)", +] + [dependency-groups] dev = [ "pre-commit>=4.6.0", diff --git a/tests/test_baselines.py b/tests/test_baselines.py index cbe5438..8a5ec6d 100644 --- a/tests/test_baselines.py +++ b/tests/test_baselines.py @@ -328,7 +328,6 @@ def test_returns_one_record_per_problem(self): suite, get_portfolio(PORTFOLIO), make_cfg(), - {}, ) assert len(records) == 2 @@ -341,7 +340,6 @@ def test_record_is_nested_dict(self): suite, get_portfolio(PORTFOLIO), make_cfg(), - {}, ) r = records[0] assert len(r) == 1 @@ -358,7 +356,6 @@ def test_all_metric_keys_present(self): suite, get_portfolio(PORTFOLIO), make_cfg(), - {}, ) metrics = _metrics(records[0]) assert METRICS_KEYS <= set(metrics.keys()) @@ -372,7 +369,6 @@ def test_aocc_in_unit_interval(self): suite, get_portfolio(PORTFOLIO), make_cfg(), - {}, ) for r in records: assert 0.0 <= _metrics(r)["aocc"] <= 1.0 @@ -386,7 +382,6 @@ def test_final_fitness_is_finite(self): suite, get_portfolio(PORTFOLIO), make_cfg(), - {}, ) for r in records: assert np.isfinite(_metrics(r)["final_fitness"]) @@ -409,7 +404,6 @@ def test_agent_tag_in_metrics(self): suite, get_portfolio(PORTFOLIO), make_cfg(), - {}, ) assert _metrics(records[0])["agent"] == "fixed:SPSO" @@ -423,7 +417,6 @@ def test_problem_ids_match_order(self): suite, get_portfolio(PORTFOLIO), make_cfg(), - {}, ) assert [_pid(r) for r in records] == ids @@ -482,7 +475,6 @@ def test_collect_single_returns_one_record_per_problem(self): suite, FE_MULTIPLIER, N_INDIVIDUALS, - {}, ) assert len(records) == 2 @@ -496,7 +488,6 @@ def test_collect_single_record_structure(self): suite, FE_MULTIPLIER, N_INDIVIDUALS, - {}, ) r = records[0] metrics = _metrics(r) @@ -513,7 +504,6 @@ def test_collect_single_aocc_in_unit_interval(self): suite, FE_MULTIPLIER, N_INDIVIDUALS, - {}, ) for r in records: assert 0.0 <= _metrics(r)["aocc"] <= 1.0 @@ -529,7 +519,6 @@ def test_single_vs_fixed_both_have_finite_final_fitness(self): suite, FE_MULTIPLIER, N_INDIVIDUALS, - {}, ) fixed_records = collect_env_results( "fixed:SPSO", @@ -538,7 +527,6 @@ def test_single_vs_fixed_both_have_finite_final_fitness(self): suite, get_portfolio(PORTFOLIO), make_cfg(), - {}, ) for r in single_records + fixed_records: assert np.isfinite(_metrics(r)["final_fitness"]) @@ -636,10 +624,10 @@ def test_oracle_with_real_runs(self): optimizers = get_portfolio(PORTFOLIO) records_spso = collect_env_results( - "fixed:SPSO", fixed_policy(0), ids, suite, optimizers, cfg, {} + "fixed:SPSO", fixed_policy(0), ids, suite, optimizers, cfg ) records_ipso = collect_env_results( - "fixed:IPSO", fixed_policy(1), ids, suite, optimizers, cfg, {} + "fixed:IPSO", fixed_policy(1), ids, suite, optimizers, cfg ) best, worst = compute_oracle( diff --git a/tests/test_exp_das_integration.py b/tests/test_exp_das_integration.py new file mode 100644 index 0000000..522d8fa --- /dev/null +++ b/tests/test_exp_das_integration.py @@ -0,0 +1,208 @@ +"""Integration test: Exponential-DAS on all 10D BBOB problems, random split. + +All 360 10D problem IDs are split randomly (2/3 train, 1/3 test). +A small sample from each partition is used for the actual train/eval run +so the test finishes in reasonable time while the split logic covers the +full benchmark. +""" + +import json + +import numpy as np +import pytest + +from agents.exponential_das.agent import ExpDASAgent +from agents.exponential_das.trainer import evaluate, train +from das.env.bbob_splits import ALL_FUNCTIONS, build_problem_ids +from das.env.das_env import DASEnv +from das.env.ioh_suite import IOHSuite +from das.env.observation import observation_dim +from das.optimizers.portfolio import get_portfolio +from das.training.common import write_jsonl + +# ------------------------------------------------------------------ +# Constants +# ------------------------------------------------------------------ + +DIM = 10 +PORTFOLIO = ["SPSOL", "TDE", "NM"] +FE_MULTIPLIER = 20 # 20 × 10 = 200 FEs per episode +N_CHECKPOINTS = 3 +CDB = 2.0 +N_INDIVIDUALS = 20 +SEED = 42 + +N_TRAIN_SAMPLE = 6 # problems sampled from the train partition +N_TEST_SAMPLE = 4 # problems sampled from the test partition + +# ------------------------------------------------------------------ +# Full random split of all 10D BBOB problems (360 IDs) +# 2/3 train / 1/3 test, fixed seed for reproducibility +# ------------------------------------------------------------------ + +_all_10d_ids = build_problem_ids(ALL_FUNCTIONS, [DIM]) +_perm = list(_all_10d_ids) +np.random.default_rng(SEED).shuffle(_perm) +_split_at = 2 * len(_perm) // 3 + +ALL_TRAIN_IDS: list[str] = _perm[:_split_at] +ALL_TEST_IDS: list[str] = _perm[_split_at:] + +# Subsample for the actual training / eval run +TRAIN_IDS: list[str] = ALL_TRAIN_IDS[:N_TRAIN_SAMPLE] +TEST_IDS: list[str] = ALL_TEST_IDS[:N_TEST_SAMPLE] + +EXPECTED_METRIC_KEYS = { + "area_under_optimization_curve", + "aocc", + "final_fitness", + "hitting_times", + "max_fe", + "reward", +} + + +# ------------------------------------------------------------------ +# Module-scoped fixture — pipeline runs once, shared across all tests +# ------------------------------------------------------------------ + + +@pytest.fixture(scope="module") +def pipeline(tmp_path_factory): + tmp = tmp_path_factory.mktemp("exp_das_integration") + suite = IOHSuite() + optimizers = get_portfolio(PORTFOLIO) + n_opt = len(optimizers) + obs_dim = observation_dim(n_opt) + + env_cfg = dict( + suite=suite, + optimizers=optimizers, + fe_multiplier=FE_MULTIPLIER, + n_checkpoints=N_CHECKPOINTS, + checkpoint_division_base=CDB, + reward_option=1, + n_individuals=N_INDIVIDUALS, + seed=SEED, + ) + + train_env = DASEnv(problem_ids=TRAIN_IDS, **env_cfg) + test_env = DASEnv(problem_ids=TEST_IDS, **env_cfg) + + agent = ExpDASAgent( + obs_dim=obs_dim, + n_actions=n_opt, + buffer_capacity=N_CHECKPOINTS, + actor_lr=3e-5, + critic_lr=1e-5, + ppo_epochs=1, + n_checkpoints=N_CHECKPOINTS, + device="cpu", + ) + + save_dir = str(tmp / "models") + result_path = tmp / "results" / "test_exp_das_eval.jsonl" + result_path.parent.mkdir() + + total_episodes = 2 * len(TRAIN_IDS) + train( + train_env=train_env, + test_env=None, + agent=agent, + total_episodes=total_episodes, + eval_interval=total_episodes + 1, + save_interval=total_episodes + 1, + save_dir=save_dir, + name="test_exp_das", + ) + + results = evaluate(test_env, agent, n_episodes=len(TEST_IDS)) + write_jsonl(str(result_path), results) + + train_env.close() + test_env.close() + + yield result_path, results + + +# ------------------------------------------------------------------ +# 1. Split-level checks (no IOH / optimizer needed) +# ------------------------------------------------------------------ + + +class TestRandomSplit: + def test_split_covers_all_10d_problems(self): + assert set(ALL_TRAIN_IDS) | set(ALL_TEST_IDS) == set(_all_10d_ids) + + def test_split_has_no_overlap(self): + assert not (set(ALL_TRAIN_IDS) & set(ALL_TEST_IDS)) + + def test_train_is_twice_the_size_of_test(self): + assert len(ALL_TRAIN_IDS) == 2 * len(ALL_TEST_IDS) + + def test_all_ids_are_10d(self): + for pid in ALL_TRAIN_IDS + ALL_TEST_IDS: + assert pid.endswith("_d10"), f"{pid} is not a 10D problem" + + def test_sample_ids_come_from_correct_partition(self): + assert all(pid in ALL_TRAIN_IDS for pid in TRAIN_IDS) + assert all(pid in ALL_TEST_IDS for pid in TEST_IDS) + + +# ------------------------------------------------------------------ +# 2. JSONL file checks +# ------------------------------------------------------------------ + + +@pytest.mark.integration +class TestExpDasIntegration: + def test_result_file_is_created(self, pipeline): + result_path, _ = pipeline + assert result_path.exists(), "eval JSONL file was not created" + + def test_result_file_has_one_record_per_test_problem(self, pipeline): + result_path, _ = pipeline + records = [json.loads(line) for line in result_path.read_text().splitlines()] + assert len(records) == len(TEST_IDS) + + def test_result_file_records_are_valid_json(self, pipeline): + result_path, _ = pipeline + for line in result_path.read_text().splitlines(): + assert isinstance(json.loads(line), dict) + + def test_result_file_problem_ids_match_test_set(self, pipeline): + result_path, _ = pipeline + records = [json.loads(line) for line in result_path.read_text().splitlines()] + assert {next(iter(r)) for r in records} == set(TEST_IDS) + + def test_all_metric_keys_present(self, pipeline): + _, results = pipeline + for record in results: + metrics = next(iter(record.values())) + missing = EXPECTED_METRIC_KEYS - set(metrics.keys()) + assert not missing, f"missing keys: {missing}" + + def test_aocc_in_unit_interval(self, pipeline): + _, results = pipeline + for record in results: + aocc = next(iter(record.values()))["aocc"] + assert 0.0 <= aocc <= 1.0, f"aocc={aocc} outside [0, 1]" + + def test_final_fitness_is_finite(self, pipeline): + _, results = pipeline + for record in results: + ff = next(iter(record.values()))["final_fitness"] + assert np.isfinite(ff), f"final_fitness={ff} is not finite" + + def test_max_fe_equals_budget(self, pipeline): + _, results = pipeline + expected = FE_MULTIPLIER * DIM + for record in results: + max_fe = next(iter(record.values()))["max_fe"] + assert max_fe == expected, f"max_fe={max_fe}, expected {expected}" + + def test_hitting_times_keys_are_strings(self, pipeline): + _, results = pipeline + for record in results: + ht = next(iter(record.values()))["hitting_times"] + assert all(isinstance(k, str) for k in ht) diff --git a/train.py b/train.py index 73a1530..7c64d2d 100644 --- a/train.py +++ b/train.py @@ -71,7 +71,12 @@ def _add_shared_args( default=10, help="Optimizer-selection steps per episode", ) - p.add_argument("--n-individuals", type=int, default=100, help="Population size") + p.add_argument( + "--n-individuals", + type=int, + default=None, + help="Population size override (default: each algorithm uses its own built-in default)", + ) p.add_argument("--seed", type=int, default=42) @@ -159,7 +164,7 @@ def _parse_args() -> argparse.Namespace: rl.set_defaults( eval=True, portfolio=["NL_SHADE_RSP", "MADDE", "JDE21"], - n_individuals=170, + n_individuals=None, ) # ---- Exp-DAS ---------------------------------------------------- From 20cbded75d820707913b920e4a3089c7e2136b3e Mon Sep 17 00:00:00 2001 From: wlnc Date: Thu, 28 May 2026 18:21:57 +0200 Subject: [PATCH 3/8] update scenarios + parallel tqdm --- agents/exponential_das/trainer.py | 25 ++++++++++++++++--------- das/training/expdas.py | 9 +++++++-- exp_das_study.slurm | 2 +- ppo_study.slurm | 2 +- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/agents/exponential_das/trainer.py b/agents/exponential_das/trainer.py index 5334e53..9ab61b5 100644 --- a/agents/exponential_das/trainer.py +++ b/agents/exponential_das/trainer.py @@ -17,6 +17,7 @@ import numpy as np import torch +from tqdm import tqdm from agents.exponential_das.agent import ExpDASAgent from das.env.das_env import DASEnv @@ -64,7 +65,8 @@ def train( episode_rewards: list[float] = [] best_test_reward = -np.inf - for ep in range(1, total_episodes + 1): + pbar = tqdm(range(1, total_episodes + 1), desc=f"train {name}", unit="ep") + for ep in pbar: obs, info = train_env.reset() done = False step_idx = 0 @@ -117,17 +119,22 @@ def train( ) entry["mean_test_reward"] = mean_test_r mean_train_r = float(np.mean(episode_rewards[-eval_interval:])) - print( - f"Ep {ep:5d}/{total_episodes}" - f" train={mean_train_r:.4f}" - f" test={mean_test_r:.4f}" - f" entropy={agent.entropy_coef:.4f}" - f" lr={agent.current_lr:.2e}" - f" kl={agent.last_kl:.4f}" + pbar.set_postfix( + train=f"{mean_train_r:.4f}", + test=f"{mean_test_r:.4f}", + ent=f"{agent.entropy_coef:.3f}", + kl=f"{agent.last_kl:.4f}", ) if mean_test_r > best_test_reward: best_test_reward = mean_test_r agent.save(os.path.join(save_dir, f"{name}_best.pt")) + else: + mean_train_r = float( + np.mean(episode_rewards[-min(eval_interval, len(episode_rewards)) :]) + ) + pbar.set_postfix( + train=f"{mean_train_r:.4f}", ent=f"{agent.entropy_coef:.3f}" + ) if ep % save_interval == 0: ckpt = os.path.join(save_dir, f"{name}_ep{ep}.pt") @@ -151,7 +158,7 @@ def evaluate( ) -> list[dict]: """Run the agent deterministically and return per-episode results.""" results = [] - for _ in range(n_episodes): + for _ in tqdm(range(n_episodes), desc="evaluate", unit="ep", leave=False): obs, info = env.reset() problem_id = info.get("problem_id", "") done = False diff --git a/das/training/expdas.py b/das/training/expdas.py index 191da22..cb3338d 100644 --- a/das/training/expdas.py +++ b/das/training/expdas.py @@ -5,6 +5,7 @@ from concurrent.futures import ProcessPoolExecutor, as_completed import numpy as np +from tqdm import tqdm from das.env.bbob_splits import get_cv_folds, get_train_test_split from das.env.das_env import DASEnv @@ -204,11 +205,15 @@ def run_cv_exp_das(args) -> None: pool.submit(_run_single_fold, fi, all_folds[fi], args): fi for fi in fold_indices } - for future in as_completed(futures): + pbar = tqdm( + as_completed(futures), total=len(futures), desc="folds", unit="fold" + ) + for future in pbar: fi, fold_tag, fold_results = future.result() results_by_fold[fi] = (fold_tag, fold_results) + pbar.set_postfix(fold=fold_tag) else: - for fi in fold_indices: + for fi in tqdm(fold_indices, desc="folds", unit="fold"): print(f"\n{'=' * 60}") fi_out, fold_tag, fold_results = _run_single_fold(fi, all_folds[fi], args) results_by_fold[fi_out] = (fold_tag, fold_results) diff --git a/exp_das_study.slurm b/exp_das_study.slurm index ea9f3ab..246f020 100644 --- a/exp_das_study.slurm +++ b/exp_das_study.slurm @@ -14,7 +14,7 @@ SEED=${1:-42} if [ "$#" -lt 2 ]; then - PORTFOLIO=('SPSO' 'IPSO' 'SPSOL') + PORTFOLIO=('NM' 'TDE' 'SPSOL') else PORTFOLIO=("${@:2}") fi diff --git a/ppo_study.slurm b/ppo_study.slurm index 6ca70cb..9d3fd61 100644 --- a/ppo_study.slurm +++ b/ppo_study.slurm @@ -14,7 +14,7 @@ SEED=${1:-42} if [ "$#" -lt 2 ]; then - PORTFOLIO=('SPSO' 'IPSO' 'SPSOL') + PORTFOLIO=('NM' 'TDE' 'SPSOL') else PORTFOLIO=("${@:2}") fi From e66d0d27abcd69401fdc6c2321f8c45bb99387a6 Mon Sep 17 00:00:00 2001 From: wlnc Date: Thu, 28 May 2026 18:43:37 +0200 Subject: [PATCH 4/8] parallelize cv in slurm --- exp_das_study.slurm | 12 ++++++------ ppo_study.slurm | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/exp_das_study.slurm b/exp_das_study.slurm index 246f020..72a95d1 100644 --- a/exp_das_study.slurm +++ b/exp_das_study.slurm @@ -3,7 +3,7 @@ #SBATCH --output=logs/expdas_%A_%a.out #SBATCH --error=logs/expdas_%A_%a.err #SBATCH --ntasks=1 -#SBATCH --cpus-per-task=1 +#SBATCH --cpus-per-task=3 #SBATCH --mem=32G #SBATCH --time=48:00:00 #SBATCH --partition=plgrid-gpu-a100 @@ -31,25 +31,25 @@ echo "Array job $SLURM_ARRAY_TASK_ID | SEED=$SEED | PORTFOLIO=${PORTFOLIO[*]}" if [[ $SLURM_ARRAY_TASK_ID -eq 0 ]]; then echo "Exp-DAS | CV-LOIO | multi-dim (2 5 10)" python cv.py exp-das ${PORTFOLIO_STR}_EXPDAS_LOIO_MULTI_SEED${SEED} \ - -p "${PORTFOLIO[@]}" --dims 2 5 10 --cv-mode LOIO --n-epochs 10 --seed $SEED + -p "${PORTFOLIO[@]}" --dims 2 5 10 --cv-mode LOIO --n-epochs 10 --seed $SEED --n-jobs 3 # 1: CV-LOPO multi-dim (2,5,10) elif [[ $SLURM_ARRAY_TASK_ID -eq 1 ]]; then echo "Exp-DAS | CV-LOPO | multi-dim (2 5 10)" python cv.py exp-das ${PORTFOLIO_STR}_EXPDAS_LOPO_MULTI_SEED${SEED} \ - -p "${PORTFOLIO[@]}" --dims 2 5 10 --cv-mode LOPO --n-epochs 10 --seed $SEED + -p "${PORTFOLIO[@]}" --dims 2 5 10 --cv-mode LOPO --n-epochs 10 --seed $SEED --n-jobs 3 # 2: CV-LOIO multi-dim (2,3,5,10) elif [[ $SLURM_ARRAY_TASK_ID -eq 2 ]]; then echo "Exp-DAS | CV-LOIO | multi-dim (2 3 5 10)" python cv.py exp-das ${PORTFOLIO_STR}_EXPDAS_LOIO_ALLDIM_SEED${SEED} \ - -p "${PORTFOLIO[@]}" --dims 2 3 5 10 --cv-mode LOIO --n-epochs 10 --seed $SEED + -p "${PORTFOLIO[@]}" --dims 2 3 5 10 --cv-mode LOIO --n-epochs 10 --seed $SEED --n-jobs 3 # 3: CV-LOPO multi-dim (2,3,5,10) elif [[ $SLURM_ARRAY_TASK_ID -eq 3 ]]; then echo "Exp-DAS | CV-LOPO | multi-dim (2 3 5 10)" python cv.py exp-das ${PORTFOLIO_STR}_EXPDAS_LOPO_ALLDIM_SEED${SEED} \ - -p "${PORTFOLIO[@]}" --dims 2 3 5 10 --cv-mode LOPO --n-epochs 10 --seed $SEED + -p "${PORTFOLIO[@]}" --dims 2 3 5 10 --cv-mode LOPO --n-epochs 10 --seed $SEED --n-jobs 3 # 4-11: single-dimension runs; task ID encodes (dim_idx * 2 + cv_mode) # dim_idx: 0→2, 1→3, 2→5, 3→10 | cv_mode: 0→LOIO, 1→LOPO @@ -62,5 +62,5 @@ else CV_MODE=${CV_MODES[$CV_IDX]} echo "Exp-DAS | CV-${CV_MODE} | dim=${DIM}" python cv.py exp-das "${PORTFOLIO_STR}_EXPDAS_${CV_MODE}_${DIM}D_SEED${SEED}" \ - -p "${PORTFOLIO[@]}" --dims "${DIM}" --cv-mode "${CV_MODE}" --n-epochs 10 --seed "$SEED" + -p "${PORTFOLIO[@]}" --dims "${DIM}" --cv-mode "${CV_MODE}" --n-epochs 10 --seed "$SEED" --n-jobs 3 fi \ No newline at end of file diff --git a/ppo_study.slurm b/ppo_study.slurm index 9d3fd61..e014be0 100644 --- a/ppo_study.slurm +++ b/ppo_study.slurm @@ -3,7 +3,7 @@ #SBATCH --output=logs/ppo_%A_%a.out #SBATCH --error=logs/ppo_%A_%a.err #SBATCH --ntasks=1 -#SBATCH --cpus-per-task=1 +#SBATCH --cpus-per-task=3 #SBATCH --mem=32G #SBATCH --time=48:00:00 #SBATCH --partition=plgrid-gpu-a100 @@ -34,24 +34,24 @@ if [[ $SLURM_ARRAY_TASK_ID -ge 0 && $SLURM_ARRAY_TASK_ID -le 3 ]]; then DIM=${DIMS[$SLURM_ARRAY_TASK_ID]} echo "PPO | CV-LOIO | dim=$DIM" python cv.py ppo ${PORTFOLIO_STR}_PPO_LOIO_DIM${DIM}_SEED${SEED} \ - -p "${PORTFOLIO[@]}" -d $DIM --cv-mode LOIO --n-epochs 3 --seed $SEED + -p "${PORTFOLIO[@]}" -d $DIM --cv-mode LOIO --n-epochs 3 --seed $SEED -j 3 # 4-7: single-dim CV-LOPO elif [[ $SLURM_ARRAY_TASK_ID -ge 4 && $SLURM_ARRAY_TASK_ID -le 7 ]]; then DIM=${DIMS[$((SLURM_ARRAY_TASK_ID - 4))]} echo "PPO | CV-LOPO | dim=$DIM" python cv.py ppo ${PORTFOLIO_STR}_PPO_LOPO_DIM${DIM}_SEED${SEED} \ - -p "${PORTFOLIO[@]}" -d $DIM --cv-mode LOPO --n-epochs 3 --seed $SEED + -p "${PORTFOLIO[@]}" -d $DIM --cv-mode LOPO --n-epochs 3 --seed $SEED -j 3 # 8: multi-dim CV-LOIO elif [[ $SLURM_ARRAY_TASK_ID -eq 8 ]]; then echo "PPO | CV-LOIO | multi-dim" python cv.py ppo ${PORTFOLIO_STR}_PPO_LOIO_MULTI_SEED${SEED} \ - -p "${PORTFOLIO[@]}" -d 2 3 5 10 --cv-mode LOIO --n-epochs 3 --seed $SEED + -p "${PORTFOLIO[@]}" -d 2 3 5 10 --cv-mode LOIO --n-epochs 3 --seed $SEED -j 3 # 9: multi-dim CV-LOPO elif [[ $SLURM_ARRAY_TASK_ID -eq 9 ]]; then echo "PPO | CV-LOPO | multi-dim" python cv.py ppo ${PORTFOLIO_STR}_PPO_LOPO_MULTI_SEED${SEED} \ - -p "${PORTFOLIO[@]}" -d 2 3 5 10 --cv-mode LOPO --n-epochs 3 --seed $SEED + -p "${PORTFOLIO[@]}" -d 2 3 5 10 --cv-mode LOPO --n-epochs 3 --seed $SEED -j 3 fi \ No newline at end of file From 4f5d897a6e4dcb8a109f1d8f00c379b70992006a Mon Sep 17 00:00:00 2001 From: wlnc Date: Thu, 28 May 2026 18:51:13 +0200 Subject: [PATCH 5/8] reduce number of epochs --- exp_das_study.slurm | 10 +++++----- test.slurm | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 test.slurm diff --git a/exp_das_study.slurm b/exp_das_study.slurm index 72a95d1..001f365 100644 --- a/exp_das_study.slurm +++ b/exp_das_study.slurm @@ -31,25 +31,25 @@ echo "Array job $SLURM_ARRAY_TASK_ID | SEED=$SEED | PORTFOLIO=${PORTFOLIO[*]}" if [[ $SLURM_ARRAY_TASK_ID -eq 0 ]]; then echo "Exp-DAS | CV-LOIO | multi-dim (2 5 10)" python cv.py exp-das ${PORTFOLIO_STR}_EXPDAS_LOIO_MULTI_SEED${SEED} \ - -p "${PORTFOLIO[@]}" --dims 2 5 10 --cv-mode LOIO --n-epochs 10 --seed $SEED --n-jobs 3 + -p "${PORTFOLIO[@]}" --dims 2 5 10 --cv-mode LOIO --n-epochs 3 --seed $SEED --n-jobs 3 # 1: CV-LOPO multi-dim (2,5,10) elif [[ $SLURM_ARRAY_TASK_ID -eq 1 ]]; then echo "Exp-DAS | CV-LOPO | multi-dim (2 5 10)" python cv.py exp-das ${PORTFOLIO_STR}_EXPDAS_LOPO_MULTI_SEED${SEED} \ - -p "${PORTFOLIO[@]}" --dims 2 5 10 --cv-mode LOPO --n-epochs 10 --seed $SEED --n-jobs 3 + -p "${PORTFOLIO[@]}" --dims 2 5 10 --cv-mode LOPO --n-epochs 3 --seed $SEED --n-jobs 3 # 2: CV-LOIO multi-dim (2,3,5,10) elif [[ $SLURM_ARRAY_TASK_ID -eq 2 ]]; then echo "Exp-DAS | CV-LOIO | multi-dim (2 3 5 10)" python cv.py exp-das ${PORTFOLIO_STR}_EXPDAS_LOIO_ALLDIM_SEED${SEED} \ - -p "${PORTFOLIO[@]}" --dims 2 3 5 10 --cv-mode LOIO --n-epochs 10 --seed $SEED --n-jobs 3 + -p "${PORTFOLIO[@]}" --dims 2 3 5 10 --cv-mode LOIO --n-epochs 3 --seed $SEED --n-jobs 3 # 3: CV-LOPO multi-dim (2,3,5,10) elif [[ $SLURM_ARRAY_TASK_ID -eq 3 ]]; then echo "Exp-DAS | CV-LOPO | multi-dim (2 3 5 10)" python cv.py exp-das ${PORTFOLIO_STR}_EXPDAS_LOPO_ALLDIM_SEED${SEED} \ - -p "${PORTFOLIO[@]}" --dims 2 3 5 10 --cv-mode LOPO --n-epochs 10 --seed $SEED --n-jobs 3 + -p "${PORTFOLIO[@]}" --dims 2 3 5 10 --cv-mode LOPO --n-epochs 3 --seed $SEED --n-jobs 3 # 4-11: single-dimension runs; task ID encodes (dim_idx * 2 + cv_mode) # dim_idx: 0→2, 1→3, 2→5, 3→10 | cv_mode: 0→LOIO, 1→LOPO @@ -62,5 +62,5 @@ else CV_MODE=${CV_MODES[$CV_IDX]} echo "Exp-DAS | CV-${CV_MODE} | dim=${DIM}" python cv.py exp-das "${PORTFOLIO_STR}_EXPDAS_${CV_MODE}_${DIM}D_SEED${SEED}" \ - -p "${PORTFOLIO[@]}" --dims "${DIM}" --cv-mode "${CV_MODE}" --n-epochs 10 --seed "$SEED" --n-jobs 3 + -p "${PORTFOLIO[@]}" --dims "${DIM}" --cv-mode "${CV_MODE}" --n-epochs 3 --seed "$SEED" --n-jobs 3 fi \ No newline at end of file diff --git a/test.slurm b/test.slurm new file mode 100644 index 0000000..e584aed --- /dev/null +++ b/test.slurm @@ -0,0 +1,29 @@ +#!/bin/bash +#SBATCH --job-name=das2_expdas +#SBATCH --output=logs/expdas_%A_%a.out +#SBATCH --error=logs/expdas_%A_%a.err +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=3 +#SBATCH --mem=32G +#SBATCH --time=48:00:00 +#SBATCH --partition=plgrid-gpu-a100 +#SBATCH -A plgrldas2026-gpu-a100 + +# Args: SEED [PORTFOLIO...] +SEED=${1:-42} + +if [ "$#" -lt 2 ]; then + PORTFOLIO=('NM' 'TDE' 'SPSOL') +else + PORTFOLIO=("${@:2}") +fi + +PORTFOLIO_STR=$(IFS="_"; echo "${PORTFOLIO[*]}") + +ENV_PATH="$SCRATCH/DynamicAlgorithmSelection2/.venv/bin/activate" +source "$ENV_PATH" +mkdir -p logs + +echo "Exp-DAS | CV-LOIO | dim=2" +python cv.py exp-das "NM_TDE_SPSOL_EXPDAS_LOIO_2D_SEED42" \ + -p "${PORTFOLIO[@]}" --dims 2 --cv-mode LOIO --n-epochs 10 --seed "$SEED" --n-jobs 3 \ No newline at end of file From 1408e30eeeab53ad859c0fc5fd336080707731ea Mon Sep 17 00:00:00 2001 From: wlnc Date: Thu, 28 May 2026 18:56:58 +0200 Subject: [PATCH 6/8] add ioh suite --- das/env/ioh_suite.py | 74 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 das/env/ioh_suite.py diff --git a/das/env/ioh_suite.py b/das/env/ioh_suite.py new file mode 100644 index 0000000..a0beed5 --- /dev/null +++ b/das/env/ioh_suite.py @@ -0,0 +1,74 @@ +"""IOH-based problem suite, replacing cocoex. + +Drop-in replacement for ``cocoex.Suite("bbob", "", "")``: + + suite = IOHSuite() + problem = suite.get_problem("bbob_f001_i01_d05") + y = problem(x) + dim = problem.dimension + lb = problem.lower_bounds # np.ndarray + ub = problem.upper_bounds # np.ndarray + +Problem IDs follow the format ``bbob_f{fid:03d}_i{iid:02d}_d{dim:02d}``. +""" + +from __future__ import annotations + +import re + +import numpy as np + +_ID_PATTERN = re.compile(r"^bbob_f(\d+)_i(\d+)_d(\d+)$") + + +class IOHProblemWrapper: + """Wraps an IOH BBOB problem with the interface expected by DASEnv.""" + + __slots__ = ("_p",) + + def __init__(self, p) -> None: + self._p = p + + @property + def dimension(self) -> int: + return int(self._p.meta_data.n_variables) + + @property + def lower_bounds(self) -> np.ndarray: + return np.asarray(self._p.bounds.lb, dtype=np.float64) + + @property + def upper_bounds(self) -> np.ndarray: + return np.asarray(self._p.bounds.ub, dtype=np.float64) + + def __call__(self, x) -> float: + return float(self._p(x)) + + +class IOHSuite: + """Drop-in replacement for ``cocoex.Suite("bbob", "", "")``. + + Stateless — a new IOH problem object is created for every ``get_problem`` + call, so the suite is safely shareable across threads and picklable. + """ + + def get_problem(self, problem_id: str) -> IOHProblemWrapper: + """Return a wrapped BBOB problem for the given *problem_id*. + + Parameters + ---------- + problem_id: + String of the form ``bbob_f{fid:03d}_i{iid:02d}_d{dim:02d}``. + """ + m = _ID_PATTERN.match(problem_id) + if m is None: + raise ValueError( + f"Cannot parse problem_id {problem_id!r}. " + "Expected format: bbob_f_i_d" + ) + fid, iid, dim = int(m.group(1)), int(m.group(2)), int(m.group(3)) + + import ioh + + p = ioh.get_problem(fid, iid, dim, problem_class=ioh.ProblemClass.BBOB) + return IOHProblemWrapper(p) From 9355708a6d84ac21d1aa1297c188d0070e95056d Mon Sep 17 00:00:00 2001 From: wlnc Date: Thu, 28 May 2026 19:08:42 +0200 Subject: [PATCH 7/8] fix smoke tests --- das/training/rldas.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/das/training/rldas.py b/das/training/rldas.py index a197786..07f32c6 100644 --- a/das/training/rldas.py +++ b/das/training/rldas.py @@ -31,9 +31,10 @@ def run_rl_das(args) -> None: dim=args.dim, fe_multiplier=args.fe_multiplier, n_checkpoints=args.n_checkpoints, - n_individuals=args.n_individuals, seed=args.seed, ) + if args.n_individuals is not None: + env_kwargs["n_individuals"] = args.n_individuals train_env = RLDASEnv(problem_ids=train_ids, **env_kwargs) test_env = RLDASEnv(problem_ids=test_ids, **env_kwargs) @@ -99,9 +100,10 @@ def run_cv_rl_das(args) -> None: dim=args.dim, fe_multiplier=args.fe_multiplier, n_checkpoints=args.n_checkpoints, - n_individuals=args.n_individuals, seed=args.seed, ) + if args.n_individuals is not None: + env_kwargs["n_individuals"] = args.n_individuals fold_summaries = [] From 5660d6c1bb6bc689423a5b71ceb89d16c0f65d38 Mon Sep 17 00:00:00 2001 From: wlnc Date: Fri, 29 May 2026 20:00:30 +0200 Subject: [PATCH 8/8] test.slurm fix --- test.slurm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.slurm b/test.slurm index e584aed..c96c7a1 100644 --- a/test.slurm +++ b/test.slurm @@ -26,4 +26,4 @@ mkdir -p logs echo "Exp-DAS | CV-LOIO | dim=2" python cv.py exp-das "NM_TDE_SPSOL_EXPDAS_LOIO_2D_SEED42" \ - -p "${PORTFOLIO[@]}" --dims 2 --cv-mode LOIO --n-epochs 10 --seed "$SEED" --n-jobs 3 \ No newline at end of file + -p "${PORTFOLIO[@]}" --dims 2 --cv-mode LOIO --n-epochs 3 --seed "$SEED" --n-jobs 3 \ No newline at end of file