diff --git a/CMakeLists.txt b/CMakeLists.txt index 519d2889..e7c9d63a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,7 @@ add_subdirectory(benchmarks/max) add_subdirectory(benchmarks/dot_product) add_subdirectory(benchmarks/gx_kernel) add_subdirectory(benchmarks/gy_kernel) -add_subdirectory(benchmarks/sobel) +# add_subdirectory(benchmarks/sobel) add_subdirectory(benchmarks/roberts_cross) add_subdirectory(benchmarks/hamming_dist) add_subdirectory(benchmarks/l2_distance) @@ -43,5 +43,5 @@ add_subdirectory(benchmarks/lin_reg) add_subdirectory(benchmarks/matrix_mul) add_subdirectory(benchmarks/poly_reg) add_subdirectory(benchmarks/polynomials_coyote) -add_subdirectory(benchmarks/poly_derivative) -add_subdirectory(benchmarks/discrete_cosin_transform) +# add_subdirectory(benchmarks/poly_derivative) +# add_subdirectory(benchmarks/discrete_cosin_transform) diff --git a/RL/checkpoints/model_14388982/rl_model_1040000_steps.zip b/RL/checkpoints/model_14388982/rl_model_1040000_steps.zip new file mode 100644 index 00000000..6decebd1 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_1040000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_1120000_steps.zip b/RL/checkpoints/model_14388982/rl_model_1120000_steps.zip new file mode 100644 index 00000000..b55f9af8 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_1120000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_1200000_steps.zip b/RL/checkpoints/model_14388982/rl_model_1200000_steps.zip new file mode 100644 index 00000000..7ee8d412 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_1200000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_1280000_steps.zip b/RL/checkpoints/model_14388982/rl_model_1280000_steps.zip new file mode 100644 index 00000000..55a68c83 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_1280000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_1360000_steps.zip b/RL/checkpoints/model_14388982/rl_model_1360000_steps.zip new file mode 100644 index 00000000..b2d0f598 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_1360000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_1440000_steps.zip b/RL/checkpoints/model_14388982/rl_model_1440000_steps.zip new file mode 100644 index 00000000..fd661bc3 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_1440000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_1520000_steps.zip b/RL/checkpoints/model_14388982/rl_model_1520000_steps.zip new file mode 100644 index 00000000..9c26db5b Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_1520000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_1600000_steps.zip b/RL/checkpoints/model_14388982/rl_model_1600000_steps.zip new file mode 100644 index 00000000..2b222644 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_1600000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_160000_steps.zip b/RL/checkpoints/model_14388982/rl_model_160000_steps.zip new file mode 100644 index 00000000..ba968dca Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_160000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_1680000_steps.zip b/RL/checkpoints/model_14388982/rl_model_1680000_steps.zip new file mode 100644 index 00000000..75308ab8 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_1680000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_1760000_steps.zip b/RL/checkpoints/model_14388982/rl_model_1760000_steps.zip new file mode 100644 index 00000000..0a12baf3 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_1760000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_1840000_steps.zip b/RL/checkpoints/model_14388982/rl_model_1840000_steps.zip new file mode 100644 index 00000000..47e7707d Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_1840000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_1920000_steps.zip b/RL/checkpoints/model_14388982/rl_model_1920000_steps.zip new file mode 100644 index 00000000..4afd405e Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_1920000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_2000000_steps.zip b/RL/checkpoints/model_14388982/rl_model_2000000_steps.zip new file mode 100644 index 00000000..ffeb72ec Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_2000000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_240000_steps.zip b/RL/checkpoints/model_14388982/rl_model_240000_steps.zip new file mode 100644 index 00000000..8cf9eefb Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_240000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_320000_steps.zip b/RL/checkpoints/model_14388982/rl_model_320000_steps.zip new file mode 100644 index 00000000..a35058e6 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_320000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_400000_steps.zip b/RL/checkpoints/model_14388982/rl_model_400000_steps.zip new file mode 100644 index 00000000..4c788e6d Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_400000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_480000_steps.zip b/RL/checkpoints/model_14388982/rl_model_480000_steps.zip new file mode 100644 index 00000000..037b709f Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_480000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_560000_steps.zip b/RL/checkpoints/model_14388982/rl_model_560000_steps.zip new file mode 100644 index 00000000..cdfe8c04 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_560000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_640000_steps.zip b/RL/checkpoints/model_14388982/rl_model_640000_steps.zip new file mode 100644 index 00000000..a0e6c700 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_640000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_720000_steps.zip b/RL/checkpoints/model_14388982/rl_model_720000_steps.zip new file mode 100644 index 00000000..e2f0a27b Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_720000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_800000_steps.zip b/RL/checkpoints/model_14388982/rl_model_800000_steps.zip new file mode 100644 index 00000000..e1365b97 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_800000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_80000_steps.zip b/RL/checkpoints/model_14388982/rl_model_80000_steps.zip new file mode 100644 index 00000000..e2daeafe Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_80000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_880000_steps.zip b/RL/checkpoints/model_14388982/rl_model_880000_steps.zip new file mode 100644 index 00000000..b90d2355 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_880000_steps.zip differ diff --git a/RL/checkpoints/model_14388982/rl_model_960000_steps.zip b/RL/checkpoints/model_14388982/rl_model_960000_steps.zip new file mode 100644 index 00000000..66bef7b6 Binary files /dev/null and b/RL/checkpoints/model_14388982/rl_model_960000_steps.zip differ diff --git a/RL/eval/best_model_model_14388982/best_model.zip b/RL/eval/best_model_model_14388982/best_model.zip new file mode 100644 index 00000000..6decebd1 Binary files /dev/null and b/RL/eval/best_model_model_14388982/best_model.zip differ diff --git a/RL/fhe_rl/__main__.py b/RL/fhe_rl/__main__.py index 69759d53..a0c68486 100644 --- a/RL/fhe_rl/__main__.py +++ b/RL/fhe_rl/__main__.py @@ -11,7 +11,7 @@ get_model_path, get_tokenizer_type, print_config ) - +from .morl import run_interactive, add_subparser def parse_arguments(args=None): """Parse command line arguments""" @@ -37,6 +37,63 @@ def parse_arguments(args=None): # Train command train_parser = subparsers.add_parser('train', help='Train the agent') + train_parser.add_argument( + '--dataset', + type=str, + default='./fhe_rl/datasets/final_llm_dataset.txt', + help='Path to the training expressions file' + ) + train_parser.add_argument( + '--n_envs', + type=int, + default=8, + help='Number of parallel environments (default: 8)' + ) + train_parser.add_argument( + '--total_timesteps', + type=int, + default=2_000_000, + help='Total environment steps to train for (default: 2_000_000)' + ) + # ── MORL / reward hyperparameters ───────────────────────────────────────── + train_parser.add_argument( + '--n_cycle', + type=int, + default=1, + help=( + 'Biased-preference cycle length N_cycle. ' + 'Every N_cycle+1 episodes: N_cycle episodes use the fixed ' + 'speed-focus preference [1,0], then 1 episode uses a random ' + 'preference from Ω. Set to 0 to always use random preferences. (default: 1)' + ) + ) + train_parser.add_argument( + '--n_budget', + type=int, + default=5, + help=( + 'Fixed normalisation budget for the rotation-key cost component ' + 'r_keys = (C_keys_old - C_keys_new) / N_budget (default: 5)' + ) + ) + train_parser.add_argument( + '--lambda_env', + type=float, + default=0.0, + help=( + 'Weight for the Pareto-envelope bonus ' + 'added to the linear reward (default: 0.0)' + ) + ) + train_parser.add_argument( + '--lambda_kl', + type=float, + default=0.0, + help=( + 'Weight for the KL-divergence exploration bonus ' + 'added to the linear reward (default: 0.0)' + ) + ) # Test command test_parser = subparsers.add_parser('test', help='Test the agent') @@ -45,24 +102,33 @@ def parse_arguments(args=None): run_parser = subparsers.add_parser('run', help='Run the agent') run_parser.add_argument('input_expr_file', help='Input expression file') run_parser.add_argument('output_vector_file', help='Output vector file') - + + run_parser.add_argument('--w_ops', type=float, default=0.5, help='Weight for operations') + run_parser.add_argument('--w_keys', type=float, default=0.5, help='Weight for keys') + + add_subparser(subparsers) + return parser.parse_args(args) def usage() -> None: print( "Usage:\n" - " python -m fhe_rl train [--tokenizer_type {dynamic,bpe}]\n" - " python -m fhe_rl test [--tokenizer_type {dynamic,bpe}]\n" - " python -m fhe_rl run [--tokenizer_type {dynamic,bpe}] " - " \n" - " python -m fhe_rl --show_config # Show current configuration\n" - "\n" - "Options:\n" - " --tokenizer_type {dynamic,bpe} Choose tokenizer type (overrides config)\n" - " --show_config Show current configuration\n" + " python -m fhe_rl train [options] Train the MORL agent\n" + " python -m fhe_rl interactive [--mode {direct,menu}]\n" + " Optimise FHE circuits interactively\n" + " direct: single preference -> one circuit\n" + " menu: preference range -> Pareto frontier + selection\n" "\n" - "All model paths are loaded from config.py." + "Train options:\n" + " --dataset PATH Training expressions file\n" + " --n_envs INT Number of parallel environments (default: 8)\n" + " --total_timesteps INT Total training steps (default: 2_000_000)\n" + " --n_cycle INT Biased-preference cycle length (default: 1)\n" + " --n_budget INT Key-cost normalisation budget (default: 5)\n" + " --lambda_env FLOAT Pareto-envelope bonus weight (default: 0.0)\n" + " --lambda_kl FLOAT KL-divergence bonus weight (default: 0.0)\n" + " --tokenizer_type {dynamic,bpe}\n" ) sys.exit(1) @@ -98,7 +164,16 @@ def main(args=None): # ────────────────────────────── TRAIN ───────────────────────────── if mode == "train": embeddings, tokenizer = load_embeddings_from_config(parsed_args.tokenizer_type) - train_agent("./fhe_rl/datasets/new_dataset_random.txt", embeddings, total_timesteps=2_000_000) + train_agent( + expressions_file=parsed_args.dataset, + embeddings_model=embeddings, + total_timesteps=parsed_args.total_timesteps, + num_envs=parsed_args.n_envs, + n_cycle=parsed_args.n_cycle, + n_budget=parsed_args.n_budget, + lambda_env=parsed_args.lambda_env, + lambda_kl=parsed_args.lambda_kl, + ) # ─────────────────────────────── TEST ───────────────────────────── elif mode == "test": @@ -112,8 +187,10 @@ def main(args=None): input_file = parsed_args.input_expr_file output_file = parsed_args.output_vector_file embeddings, tokenizer = load_embeddings_from_config(parsed_args.tokenizer_type) - run_agent(input_file, embeddings, agent_zip, output_file) + run_agent(input_file, embeddings, agent_zip, output_file, w_ops=parsed_args.w_ops, w_keys=parsed_args.w_keys) + elif mode == "interactive": + run_interactive(mode=getattr(parsed_args, "interactive_mode", None)) else: print("Invalid command. Use 'train', 'test' or 'run'.") usage() diff --git a/RL/fhe_rl/callbacks.py b/RL/fhe_rl/callbacks.py index 75273bc4..2890fe4c 100644 --- a/RL/fhe_rl/callbacks.py +++ b/RL/fhe_rl/callbacks.py @@ -1,25 +1,121 @@ from stable_baselines3.common.callbacks import BaseCallback - +from stable_baselines3.common.evaluation import evaluate_policy +import numpy as np +import torch +import os class EntCoefScheduler(BaseCallback): def __init__(self, schedule, verbose: int = 0): super().__init__(verbose) self.schedule = schedule + self.rollout_count = 0 def _on_training_start(self) -> None: p = self.model._current_progress_remaining self.model.ent_coef = float(self.schedule(p)) def _on_rollout_end(self) -> None: - p = self.model._current_progress_remaining - self.model.ent_coef = float(self.schedule(p)) + # p = self.model._current_progress_remaining + # self.model.ent_coef = float(self.schedule(p)) + self.rollout_count += 1 + + # --- CHANGE: Update entropy ONLY every 2nd rollout --- + if self.rollout_count % 2 == 0: + p = self.model._current_progress_remaining + self.model.ent_coef = float(self.schedule(p)) + if self.verbose > 0: + print(f"[Entropy] Updated to {self.model.ent_coef:.4f} at rollout {self.rollout_count}") def _on_step(self) -> bool: return True +class ParetoEvalCallback(BaseCallback): + def __init__(self, eval_env, pref_list, + best_model_save_path=None, + log_path=None, + eval_freq=512, + n_eval_episodes=5, + deterministic=True, + verbose=1): + super().__init__(verbose) + self.eval_env = eval_env + self.pref_list = pref_list + self.eval_freq = eval_freq + self.n_eval_episodes = n_eval_episodes + self.deterministic = deterministic + self.best_model_save_path = best_model_save_path + self.log_path = log_path + + # Track best performance (Average across the Pareto Front) + self.best_mean_reward = -np.inf + + # Ensure directories exist + if self.best_model_save_path is not None: + os.makedirs(self.best_model_save_path, exist_ok=True) + + def _on_step(self) -> bool: + if self.n_calls % self.eval_freq == 0: + if self.verbose > 0: + print(f"\nStep {self.num_timesteps}: Starting Pareto Evaluation ({len(self.pref_list)} points)") + + all_means = [] + all_lengths = [] + + for w in self.pref_list: + # Force eval env to this specific goal + self.eval_env.env_method("set_preference_vector", w) + + # Run evaluation + episode_rewards, episode_lengths = evaluate_policy( + self.model, + self.eval_env, + n_eval_episodes=self.n_eval_episodes, + deterministic=self.deterministic, + return_episode_rewards=True + ) + + mean_r = np.mean(episode_rewards) + all_means.append(mean_r) + all_lengths.append(np.mean(episode_lengths)) + + # 1. Calculate the Global Score + current_mean_reward = np.mean(all_means) + current_mean_length = np.mean(all_lengths) + + # 2. Log to Tensorboard/Logger (standard EvalCallback style) + self.logger.record("eval/pareto_avg_reward", current_mean_reward) + self.logger.record("eval/pareto_avg_ep_length", current_mean_length) + + # Optional: Log specific weight performance for deeper insight + for i, w in enumerate(self.pref_list): + self.logger.record(f"eval/reward_w_{w[1]}", all_means[i]) + + self.eval_env.env_method("unlock_preferences") + + if self.verbose > 0: + print(f"Eval num_timesteps={self.num_timesteps}, " + f"episode_reward={current_mean_reward:.2f} +/- {np.std(all_means):.2f}") + print(f"Episode length: {current_mean_length:.2f} +/- {np.std(all_lengths):.2f}") + + # 3. Check if this is the "Best" model found so far + if current_mean_reward > self.best_mean_reward: + if self.verbose > 0: + print("New best mean reward!") + + if self.best_model_save_path is not None: + self.model.save(os.path.join(self.best_model_save_path, "best_model")) + + self.best_mean_reward = current_mean_reward + + # Trigger potential logging for SB3 monitoring + self.logger.dump(step=self.num_timesteps) + + return True + def linear_schedule(start: float, end: float = 0.0): def sched(progress_remaining): return (start - end) * progress_remaining + end - return sched \ No newline at end of file + return sched + \ No newline at end of file diff --git a/RL/fhe_rl/config.py b/RL/fhe_rl/config.py index cccaed98..cbf90109 100644 --- a/RL/fhe_rl/config.py +++ b/RL/fhe_rl/config.py @@ -12,7 +12,7 @@ # Model paths configuration MODEL_PATHS = { - "agent_model": FHE_RL_DIR / "trained_models" / "agent_dynamic_llm_data.zip", + "agent_model": FHE_RL_DIR / "trained_models" / "agent_pareto_2_full.zip", "dynamic_embeddings_model": FHE_RL_DIR / "trained_models" / "embeddings_ROT_15_32_5m_10742576.pth", "bpe_embeddings_model": FHE_RL_DIR / "trained_models" / "model_Transformer_BPE_ddp_jobid_epoch_5000000.pth", "bpe_tokenizer": FHE_RL_DIR / "trained_models" / "bpe_tokenizer.pkl", diff --git a/RL/fhe_rl/datasets/benchmarks.txt b/RL/fhe_rl/datasets/benchmarks.txt index dfdd4501..60045007 100644 --- a/RL/fhe_rl/datasets/benchmarks.txt +++ b/RL/fhe_rl/datasets/benchmarks.txt @@ -1,48 +1 @@ -(Vec ( + ( * ( + ( * ( + ( * o3214 ( - 1 c34 ) ) ( * c34 ( + ( * ( + ( * o4321 ( - 1 c34 ) ) ( * c34 o3421 ) ) ( - 1 c24 ) ) ( * c24 ( + ( * o3241 ( - 1 c14 ) ) ( * c14 o3214 ) ) ) ) ) ) ( - 1 c23 ) ) ( * c23 ( + ( * ( + ( * o2314 ( - 1 c24 ) ) ( * c24 ( + ( * ( - 1 c34 ) ( + ( * o4231 ( - 1 c24 ) ) ( * c24 o2431 ) ) ) ( * c34 ( + ( * o2341 ( - 1 c14 ) ) ( * c14 o2314 ) ) ) ) ) ) ( - 1 c13 ) ) ( * c13 ( + ( * o2134 ( - 1 c24 ) ) ( * c24 ( + ( * ( - 1 c14 ) ( + ( * o4213 ( - 1 c24 ) ) ( * c24 o2413 ) ) ) ( * c14 ( + ( * o2143 ( - 1 c34 ) ) ( * c34 o2134 ) ) ) ) ) ) ) ) ) ) ( - 1 c12 ) ) ( * c12 ( + ( * ( - 1 c13 ) ( + ( * o3124 ( - 1 c34 ) ) ( * c34 ( + ( * ( - 1 c14 ) ( + ( * o4312 ( - 1 c34 ) ) ( * c34 o3412 ) ) ) ( * c14 ( + ( * o3142 ( - 1 c24 ) ) ( * c24 o3124 ) ) ) ) ) ) ) ( * c13 ( + ( * ( - 1 c23 ) ( + ( * o1324 ( - 1 c14 ) ) ( * c14 ( + ( * ( - 1 c34 ) ( + ( * o4132 ( - 1 c14 ) ) ( * c14 o1432 ) ) ) ( * c34 ( + ( * o1342 ( - 1 c24 ) ) ( * c24 o1324 ) ) ) ) ) ) ) ( * c23 ( + ( * o1234 ( - 1 c14 ) ) ( * c14 ( + ( * ( - 1 c24 ) ( + ( * o4123 ( - 1 c14 ) ) ( * c14 o1423 ) ) ) ( * c24 ( + ( * o1243 ( - 1 c34 ) ) ( * c34 o1234 ) ) ) ) ) ) ) ) ) ) ) ) ):sort_4 -(Vec ( + ( * ( + ( * ( + ( * ( + ( * o5 ( - 1 c45 ) ) ( * c45 o4 ) ) ( - 1 c34 ) ) ( * c34 ( + ( * o5 ( - 1 c35 ) ) ( * c35 o3 ) ) ) ) ( - 1 c23 ) ) ( * c23 ( + ( * ( + ( * o5 ( - 1 c45 ) ) ( * c45 o4 ) ) ( - 1 c24 ) ) ( * c24 ( + ( * o5 ( - 1 c25 ) ) ( * c25 o2 ) ) ) ) ) ) ( - 1 c12 ) ) ( * c12 ( + ( * ( + ( * ( + ( * o5 ( - 1 c45 ) ) ( * c45 o4 ) ) ( - 1 c34 ) ) ( * c34 ( + ( * o5 ( - 1 c35 ) ) ( * c35 o3 ) ) ) ) ( - 1 c13 ) ) ( * c13 ( + ( * ( + ( * o5 ( - 1 c45 ) ) ( * c45 o4 ) ) ( - 1 c14 ) ) ( * c14 ( + ( * o5 ( - 1 c15 ) ) ( * c15 o1 ) ) ) ) ) ) ) ) ):max_5 - -(Vec ( + ( + in_1_1 in_1_0 ) ( + in_0_1 in_0_0 ) ) ( + ( + ( + in_1_2 in_1_1 ) ( + in_1_0 in_0_2 ) ) ( + in_0_1 in_0_0 ) ) ( + ( + in_1_2 in_1_1 ) ( + in_0_2 in_0_1 ) ) ( + ( + ( + in_2_1 in_2_0 ) ( + in_1_1 in_1_0 ) ) ( + in_0_1 in_0_0 ) ) ( + ( + ( + in_2_2 ( + in_2_1 in_2_0 ) ) ( + in_1_2 in_1_1 ) ) ( + ( + in_1_0 in_0_2 ) ( + in_0_1 in_0_0 ) ) ) ( + ( + ( + in_2_2 in_2_1 ) ( + in_1_2 in_1_1 ) ) ( + in_0_2 in_0_1 ) ) ( + ( + in_2_1 in_2_0 ) ( + in_1_1 in_1_0 ) ) ( + ( + ( + in_2_2 in_2_1 ) ( + in_2_0 in_1_2 ) ) ( + in_1_1 in_1_0 ) ) ( + ( + in_2_2 in_2_1 ) ( + in_1_2 in_1_1 ) ) ):box_blur_3 -(Vec ( + ( + in_1_1 in_1_0 ) ( + in_0_1 in_0_0 ) ) ( + ( + ( + in_1_2 in_1_1 ) ( + in_1_0 in_0_2 ) ) ( + in_0_1 in_0_0 ) ) ( + ( + ( + in_1_3 in_1_2 ) ( + in_1_1 in_0_3 ) ) ( + in_0_2 in_0_1 ) ) ( + ( + in_1_3 in_1_2 ) ( + in_0_3 in_0_2 ) ) ( + ( + ( + in_2_1 in_2_0 ) ( + in_1_1 in_1_0 ) ) ( + in_0_1 in_0_0 ) ) ( + ( + ( + in_2_2 ( + in_2_1 in_2_0 ) ) ( + in_1_2 in_1_1 ) ) ( + ( + in_1_0 in_0_2 ) ( + in_0_1 in_0_0 ) ) ) ( + ( + ( + in_2_3 ( + in_2_2 in_2_1 ) ) ( + in_1_3 in_1_2 ) ) ( + ( + in_1_1 in_0_3 ) ( + in_0_2 in_0_1 ) ) ) ( + ( + ( + in_2_3 in_2_2 ) ( + in_1_3 in_1_2 ) ) ( + in_0_3 in_0_2 ) ) ( + ( + in_3_1 in_3_0 ) ( + ( + in_2_1 in_2_0 ) ( + in_1_1 in_1_0 ) ) ) ( + ( + ( + in_3_2 ( + in_3_1 in_3_0 ) ) ( + in_2_2 in_2_1 ) ) ( + ( + in_2_0 in_1_2 ) ( + in_1_1 in_1_0 ) ) ) ( + ( + ( + in_3_3 ( + in_3_2 in_3_1 ) ) ( + in_2_3 in_2_2 ) ) ( + ( + in_2_1 in_1_3 ) ( + in_1_2 in_1_1 ) ) ) ( + ( + ( + in_3_3 in_3_2 ) ( + in_2_3 in_2_2 ) ) ( + in_1_3 in_1_2 ) ) ( + ( + in_3_1 in_3_0 ) ( + in_2_1 in_2_0 ) ) ( + ( + ( + in_3_2 in_3_1 ) ( + in_3_0 in_2_2 ) ) ( + in_2_1 in_2_0 ) ) ( + ( + ( + in_3_3 in_3_2 ) ( + in_3_1 in_2_3 ) ) ( + in_2_2 in_2_1 ) ) ( + ( + in_3_3 in_3_2 ) ( + in_2_3 in_2_2 ) ) ):box_blur_4 -(Vec ( + ( + in_1_1 in_1_0 ) ( + in_0_1 in_0_0 ) ) ( + ( + ( + in_1_2 in_1_1 ) ( + in_1_0 in_0_2 ) ) ( + in_0_1 in_0_0 ) ) ( + ( + ( + in_1_3 in_1_2 ) ( + in_1_1 in_0_3 ) ) ( + in_0_2 in_0_1 ) ) ( + ( + ( + in_1_4 in_1_3 ) ( + in_1_2 in_0_4 ) ) ( + in_0_3 in_0_2 ) ) ( + ( + in_1_4 in_1_3 ) ( + in_0_4 in_0_3 ) ) ( + ( + ( + in_2_1 in_2_0 ) ( + in_1_1 in_1_0 ) ) ( + in_0_1 in_0_0 ) ) ( + ( + ( + in_2_2 ( + in_2_1 in_2_0 ) ) ( + in_1_2 in_1_1 ) ) ( + ( + in_1_0 in_0_2 ) ( + in_0_1 in_0_0 ) ) ) ( + ( + ( + in_2_3 ( + in_2_2 in_2_1 ) ) ( + in_1_3 in_1_2 ) ) ( + ( + in_1_1 in_0_3 ) ( + in_0_2 in_0_1 ) ) ) ( + ( + ( + in_2_4 ( + in_2_3 in_2_2 ) ) ( + in_1_4 in_1_3 ) ) ( + ( + in_1_2 in_0_4 ) ( + in_0_3 in_0_2 ) ) ) ( + ( + ( + in_2_4 in_2_3 ) ( + in_1_4 in_1_3 ) ) ( + in_0_4 in_0_3 ) ) ( + ( + ( + in_3_1 in_3_0 ) ( + in_2_1 in_2_0 ) ) ( + in_1_1 in_1_0 ) ) ( + ( + ( + in_3_2 ( + in_3_1 in_3_0 ) ) ( + in_2_2 in_2_1 ) ) ( + ( + in_2_0 in_1_2 ) ( + in_1_1 in_1_0 ) ) ) ( + ( + ( + in_3_3 ( + in_3_2 in_3_1 ) ) ( + in_2_3 in_2_2 ) ) ( + ( + in_2_1 in_1_3 ) ( + in_1_2 in_1_1 ) ) ) ( + ( + ( + in_3_4 ( + in_3_3 in_3_2 ) ) ( + in_2_4 in_2_3 ) ) ( + ( + in_2_2 in_1_4 ) ( + in_1_3 in_1_2 ) ) ) ( + ( + ( + in_3_4 in_3_3 ) ( + in_2_4 in_2_3 ) ) ( + in_1_4 in_1_3 ) ) ( + ( + ( + in_4_1 in_4_0 ) ( + in_3_1 in_3_0 ) ) ( + in_2_1 in_2_0 ) ) ( + ( + ( + in_4_2 ( + in_4_1 in_4_0 ) ) ( + in_3_2 in_3_1 ) ) ( + ( + in_3_0 in_2_2 ) ( + in_2_1 in_2_0 ) ) ) ( + ( + ( + in_4_3 ( + in_4_2 in_4_1 ) ) ( + in_3_3 in_3_2 ) ) ( + ( + in_3_1 in_2_3 ) ( + in_2_2 in_2_1 ) ) ) ( + ( + ( + in_4_4 ( + in_4_3 in_4_2 ) ) ( + in_3_4 in_3_3 ) ) ( + ( + in_3_2 in_2_4 ) ( + in_2_3 in_2_2 ) ) ) ( + ( + ( + in_4_4 in_4_3 ) ( + in_3_4 in_3_3 ) ) ( + in_2_4 in_2_3 ) ) ( + ( + in_4_1 in_4_0 ) ( + in_3_1 in_3_0 ) ) ( + ( + ( + in_4_2 in_4_1 ) ( + in_4_0 in_3_2 ) ) ( + in_3_1 in_3_0 ) ) ( + ( + ( + in_4_3 in_4_2 ) ( + in_4_1 in_3_3 ) ) ( + in_3_2 in_3_1 ) ) ( + ( + ( + in_4_4 in_4_3 ) ( + in_4_2 in_3_4 ) ) ( + in_3_3 in_3_2 ) ) ( + ( + in_4_4 in_4_3 ) ( + in_3_4 in_3_3 ) ) ):box_blur_5 -(Vec ( + v2_0 ( + 2 ( * v1_0 5 ) ) ) ( + v2_1 ( + 2 ( * v1_1 5 ) ) ) ( + v2_2 ( + 2 ( * v1_2 5 ) ) ) ( + v2_3 ( + 2 ( * v1_3 5 ) ) ) ):lin_reg_4 -(Vec ( + v2_0 ( + 2 ( * v1_0 5 ) ) ) ( + v2_1 ( + 2 ( * v1_1 5 ) ) ) ( + v2_2 ( + 2 ( * v1_2 5 ) ) ) ( + v2_3 ( + 2 ( * v1_3 5 ) ) ) ( + v2_4 ( + 2 ( * v1_4 5 ) ) ) ( + v2_5 ( + 2 ( * v1_5 5 ) ) ) ( + v2_6 ( + 2 ( * v1_6 5 ) ) ) ( + v2_7 ( + 2 ( * v1_7 5 ) ) ) ):lin_reg_8 -(Vec ( + v2_0 ( + 2 ( * v1_0 5 ) ) ) ( + v2_1 ( + 2 ( * v1_1 5 ) ) ) ( + v2_2 ( + 2 ( * v1_2 5 ) ) ) ( + v2_3 ( + 2 ( * v1_3 5 ) ) ) ( + v2_4 ( + 2 ( * v1_4 5 ) ) ) ( + v2_5 ( + 2 ( * v1_5 5 ) ) ) ( + v2_6 ( + 2 ( * v1_6 5 ) ) ) ( + v2_7 ( + 2 ( * v1_7 5 ) ) ) ( + v2_8 ( + 2 ( * v1_8 5 ) ) ) ( + v2_9 ( + 2 ( * v1_9 5 ) ) ) ( + v2_10 ( + 2 ( * v1_10 5 ) ) ) ( + v2_11 ( + 2 ( * v1_11 5 ) ) ) ( + v2_12 ( + 2 ( * v1_12 5 ) ) ) ( + v2_13 ( + 2 ( * v1_13 5 ) ) ) ( + v2_14 ( + 2 ( * v1_14 5 ) ) ) ( + v2_15 ( + 2 ( * v1_15 5 ) ) ) ):lin_reg_16 -(Vec ( + v2_0 ( + 2 ( * v1_0 5 ) ) ) ( + v2_1 ( + 2 ( * v1_1 5 ) ) ) ( + v2_2 ( + 2 ( * v1_2 5 ) ) ) ( + v2_3 ( + 2 ( * v1_3 5 ) ) ) ( + v2_4 ( + 2 ( * v1_4 5 ) ) ) ( + v2_5 ( + 2 ( * v1_5 5 ) ) ) ( + v2_6 ( + 2 ( * v1_6 5 ) ) ) ( + v2_7 ( + 2 ( * v1_7 5 ) ) ) ( + v2_8 ( + 2 ( * v1_8 5 ) ) ) ( + v2_9 ( + 2 ( * v1_9 5 ) ) ) ( + v2_10 ( + 2 ( * v1_10 5 ) ) ) ( + v2_11 ( + 2 ( * v1_11 5 ) ) ) ( + v2_12 ( + 2 ( * v1_12 5 ) ) ) ( + v2_13 ( + 2 ( * v1_13 5 ) ) ) ( + v2_14 ( + 2 ( * v1_14 5 ) ) ) ( + v2_15 ( + 2 ( * v1_15 5 ) ) ) ( + v2_16 ( + 2 ( * v1_16 5 ) ) ) ( + v2_17 ( + 2 ( * v1_17 5 ) ) ) ( + v2_18 ( + 2 ( * v1_18 5 ) ) ) ( + v2_19 ( + 2 ( * v1_19 5 ) ) ) ( + v2_20 ( + 2 ( * v1_20 5 ) ) ) ( + v2_21 ( + 2 ( * v1_21 5 ) ) ) ( + v2_22 ( + 2 ( * v1_22 5 ) ) ) ( + v2_23 ( + 2 ( * v1_23 5 ) ) ) ( + v2_24 ( + 2 ( * v1_24 5 ) ) ) ( + v2_25 ( + 2 ( * v1_25 5 ) ) ) ( + v2_26 ( + 2 ( * v1_26 5 ) ) ) ( + v2_27 ( + 2 ( * v1_27 5 ) ) ) ( + v2_28 ( + 2 ( * v1_28 5 ) ) ) ( + v2_29 ( + 2 ( * v1_29 5 ) ) ) ( + v2_30 ( + 2 ( * v1_30 5 ) ) ) ( + v2_31 ( + 2 ( * v1_31 5 ) ) ) ):lin_reg_32 -(Vec ( + ( + ( - ( + v1_0 v2_0 ) ( * ( * v1_0 v2_0 ) 2 ) ) ( - ( + v1_1 v2_1 ) ( * 2 ( * v1_1 v2_1 ) ) ) ) ( - ( + v1_2 v2_2 ) ( * 2 ( * v1_2 v2_2 ) ) ) ) ):hamming_dist_3 -(Vec ( + ( + ( - ( + v1_0 v2_0 ) ( * ( * v1_0 v2_0 ) 2 ) ) ( - ( + v1_1 v2_1 ) ( * 2 ( * v1_1 v2_1 ) ) ) ) ( + ( - ( + v1_2 v2_2 ) ( * 2 ( * v1_2 v2_2 ) ) ) ( - ( + v1_3 v2_3 ) ( * 2 ( * v1_3 v2_3 ) ) ) ) ) ):hamming_dist_4 -(Vec ( + ( + ( - ( + v1_0 v2_0 ) ( * ( * v1_0 v2_0 ) 2 ) ) ( - ( + v1_1 v2_1 ) ( * 2 ( * v1_1 v2_1 ) ) ) ) ( + ( + ( - ( + v1_2 v2_2 ) ( * 2 ( * v1_2 v2_2 ) ) ) ( - ( + v1_3 v2_3 ) ( * 2 ( * v1_3 v2_3 ) ) ) ) ( - ( + v1_4 v2_4 ) ( * 2 ( * v1_4 v2_4 ) ) ) ) ) ):hamming_dist_5 -(Vec ( + ( + ( + ( - ( + v1_0 v2_0 ) ( * ( * v1_0 v2_0 ) 2 ) ) ( - ( + v1_1 v2_1 ) ( * 2 ( * v1_1 v2_1 ) ) ) ) ( + ( - ( + v1_2 v2_2 ) ( * 2 ( * v1_2 v2_2 ) ) ) ( - ( + v1_3 v2_3 ) ( * 2 ( * v1_3 v2_3 ) ) ) ) ) ( + ( + ( - ( + v1_4 v2_4 ) ( * 2 ( * v1_4 v2_4 ) ) ) ( - ( + v1_5 v2_5 ) ( * 2 ( * v1_5 v2_5 ) ) ) ) ( + ( - ( + v1_6 v2_6 ) ( * 2 ( * v1_6 v2_6 ) ) ) ( - ( + v1_7 v2_7 ) ( * 2 ( * v1_7 v2_7 ) ) ) ) ) ) ):hamming_dist_8 -(Vec ( + ( + ( + ( + ( - ( + v1_0 v2_0 ) ( * ( * v1_0 v2_0 ) 2 ) ) ( - ( + v1_1 v2_1 ) ( * 2 ( * v1_1 v2_1 ) ) ) ) ( + ( - ( + v1_2 v2_2 ) ( * 2 ( * v1_2 v2_2 ) ) ) ( - ( + v1_3 v2_3 ) ( * 2 ( * v1_3 v2_3 ) ) ) ) ) ( + ( + ( - ( + v1_4 v2_4 ) ( * 2 ( * v1_4 v2_4 ) ) ) ( - ( + v1_5 v2_5 ) ( * 2 ( * v1_5 v2_5 ) ) ) ) ( + ( - ( + v1_6 v2_6 ) ( * 2 ( * v1_6 v2_6 ) ) ) ( - ( + v1_7 v2_7 ) ( * 2 ( * v1_7 v2_7 ) ) ) ) ) ) ( + ( + ( + ( - ( + v1_8 v2_8 ) ( * 2 ( * v1_8 v2_8 ) ) ) ( - ( + v1_9 v2_9 ) ( * 2 ( * v1_9 v2_9 ) ) ) ) ( + ( - ( + v1_10 v2_10 ) ( * 2 ( * v1_10 v2_10 ) ) ) ( - ( + v1_11 v2_11 ) ( * 2 ( * v1_11 v2_11 ) ) ) ) ) ( + ( + ( - ( + v1_12 v2_12 ) ( * 2 ( * v1_12 v2_12 ) ) ) ( - ( + v1_13 v2_13 ) ( * 2 ( * v1_13 v2_13 ) ) ) ) ( + ( - ( + v1_14 v2_14 ) ( * 2 ( * v1_14 v2_14 ) ) ) ( - ( + v1_15 v2_15 ) ( * 2 ( * v1_15 v2_15 ) ) ) ) ) ) ) ):hamming_dist_16 -(Vec ( + ( + ( + ( + ( + ( - ( + v1_0 v2_0 ) ( * ( * v1_0 v2_0 ) 2 ) ) ( - ( + v1_1 v2_1 ) ( * 2 ( * v1_1 v2_1 ) ) ) ) ( + ( - ( + v1_2 v2_2 ) ( * 2 ( * v1_2 v2_2 ) ) ) ( - ( + v1_3 v2_3 ) ( * 2 ( * v1_3 v2_3 ) ) ) ) ) ( + ( + ( - ( + v1_4 v2_4 ) ( * 2 ( * v1_4 v2_4 ) ) ) ( - ( + v1_5 v2_5 ) ( * 2 ( * v1_5 v2_5 ) ) ) ) ( + ( - ( + v1_6 v2_6 ) ( * 2 ( * v1_6 v2_6 ) ) ) ( - ( + v1_7 v2_7 ) ( * 2 ( * v1_7 v2_7 ) ) ) ) ) ) ( + ( + ( + ( - ( + v1_8 v2_8 ) ( * 2 ( * v1_8 v2_8 ) ) ) ( - ( + v1_9 v2_9 ) ( * 2 ( * v1_9 v2_9 ) ) ) ) ( + ( - ( + v1_10 v2_10 ) ( * 2 ( * v1_10 v2_10 ) ) ) ( - ( + v1_11 v2_11 ) ( * 2 ( * v1_11 v2_11 ) ) ) ) ) ( + ( + ( - ( + v1_12 v2_12 ) ( * 2 ( * v1_12 v2_12 ) ) ) ( - ( + v1_13 v2_13 ) ( * 2 ( * v1_13 v2_13 ) ) ) ) ( + ( - ( + v1_14 v2_14 ) ( * 2 ( * v1_14 v2_14 ) ) ) ( - ( + v1_15 v2_15 ) ( * 2 ( * v1_15 v2_15 ) ) ) ) ) ) ) ( + ( + ( + ( + ( - ( + v1_16 v2_16 ) ( * 2 ( * v1_16 v2_16 ) ) ) ( - ( + v1_17 v2_17 ) ( * 2 ( * v1_17 v2_17 ) ) ) ) ( + ( - ( + v1_18 v2_18 ) ( * 2 ( * v1_18 v2_18 ) ) ) ( - ( + v1_19 v2_19 ) ( * 2 ( * v1_19 v2_19 ) ) ) ) ) ( + ( + ( - ( + v1_20 v2_20 ) ( * 2 ( * v1_20 v2_20 ) ) ) ( - ( + v1_21 v2_21 ) ( * 2 ( * v1_21 v2_21 ) ) ) ) ( + ( - ( + v1_22 v2_22 ) ( * 2 ( * v1_22 v2_22 ) ) ) ( - ( + v1_23 v2_23 ) ( * 2 ( * v1_23 v2_23 ) ) ) ) ) ) ( + ( + ( + ( - ( + v1_24 v2_24 ) ( * 2 ( * v1_24 v2_24 ) ) ) ( - ( + v1_25 v2_25 ) ( * 2 ( * v1_25 v2_25 ) ) ) ) ( + ( - ( + v1_26 v2_26 ) ( * 2 ( * v1_26 v2_26 ) ) ) ( - ( + v1_27 v2_27 ) ( * 2 ( * v1_27 v2_27 ) ) ) ) ) ( + ( + ( - ( + v1_28 v2_28 ) ( * 2 ( * v1_28 v2_28 ) ) ) ( - ( + v1_29 v2_29 ) ( * 2 ( * v1_29 v2_29 ) ) ) ) ( + ( - ( + v1_30 v2_30 ) ( * 2 ( * v1_30 v2_30 ) ) ) ( - ( + v1_31 v2_31 ) ( * 2 ( * v1_31 v2_31 ) ) ) ) ) ) ) ) ):hamming_dist_32 -(Vec ( + c1_0 ( + c2_0 ( * c0_0 ( + c3_0 ( * c0_0 c4_0 ) ) ) ) ) ( + c1_1 ( + c2_1 ( * c0_1 ( + c3_1 ( * c0_1 c4_1 ) ) ) ) ) ( + c1_2 ( + c2_2 ( * c0_2 ( + c3_2 ( * c0_2 c4_2 ) ) ) ) ) ( + c1_3 ( + c2_3 ( * c0_3 ( + c3_3 ( * c0_3 c4_3 ) ) ) ) ) ):poly_reg_4 -(Vec ( + c1_0 ( + c2_0 ( * c0_0 ( + c3_0 ( * c0_0 c4_0 ) ) ) ) ) ( + c1_1 ( + c2_1 ( * c0_1 ( + c3_1 ( * c0_1 c4_1 ) ) ) ) ) ( + c1_2 ( + c2_2 ( * c0_2 ( + c3_2 ( * c0_2 c4_2 ) ) ) ) ) ( + c1_3 ( + c2_3 ( * c0_3 ( + c3_3 ( * c0_3 c4_3 ) ) ) ) ) ( + c1_4 ( + c2_4 ( * c0_4 ( + c3_4 ( * c0_4 c4_4 ) ) ) ) ) ( + c1_5 ( + c2_5 ( * c0_5 ( + c3_5 ( * c0_5 c4_5 ) ) ) ) ) ( + c1_6 ( + c2_6 ( * c0_6 ( + c3_6 ( * c0_6 c4_6 ) ) ) ) ) ( + c1_7 ( + c2_7 ( * c0_7 ( + c3_7 ( * c0_7 c4_7 ) ) ) ) ) ):poly_reg_8 -(Vec ( + c1_0 ( + c2_0 ( * c0_0 ( + c3_0 ( * c0_0 c4_0 ) ) ) ) ) ( + c1_1 ( + c2_1 ( * c0_1 ( + c3_1 ( * c0_1 c4_1 ) ) ) ) ) ( + c1_2 ( + c2_2 ( * c0_2 ( + c3_2 ( * c0_2 c4_2 ) ) ) ) ) ( + c1_3 ( + c2_3 ( * c0_3 ( + c3_3 ( * c0_3 c4_3 ) ) ) ) ) ( + c1_4 ( + c2_4 ( * c0_4 ( + c3_4 ( * c0_4 c4_4 ) ) ) ) ) ( + c1_5 ( + c2_5 ( * c0_5 ( + c3_5 ( * c0_5 c4_5 ) ) ) ) ) ( + c1_6 ( + c2_6 ( * c0_6 ( + c3_6 ( * c0_6 c4_6 ) ) ) ) ) ( + c1_7 ( + c2_7 ( * c0_7 ( + c3_7 ( * c0_7 c4_7 ) ) ) ) ) ( + c1_8 ( + c2_8 ( * c0_8 ( + c3_8 ( * c0_8 c4_8 ) ) ) ) ) ( + c1_9 ( + c2_9 ( * c0_9 ( + c3_9 ( * c0_9 c4_9 ) ) ) ) ) ( + c1_10 ( + c2_10 ( * c0_10 ( + c3_10 ( * c0_10 c4_10 ) ) ) ) ) ( + c1_11 ( + c2_11 ( * c0_11 ( + c3_11 ( * c0_11 c4_11 ) ) ) ) ) ( + c1_12 ( + c2_12 ( * c0_12 ( + c3_12 ( * c0_12 c4_12 ) ) ) ) ) ( + c1_13 ( + c2_13 ( * c0_13 ( + c3_13 ( * c0_13 c4_13 ) ) ) ) ) ( + c1_14 ( + c2_14 ( * c0_14 ( + c3_14 ( * c0_14 c4_14 ) ) ) ) ) ( + c1_15 ( + c2_15 ( * c0_15 ( + c3_15 ( * c0_15 c4_15 ) ) ) ) ) ):poly_reg_16 -(Vec ( + c1_0 ( + c2_0 ( * c0_0 ( + c3_0 ( * c0_0 c4_0 ) ) ) ) ) ( + c1_1 ( + c2_1 ( * c0_1 ( + c3_1 ( * c0_1 c4_1 ) ) ) ) ) ( + c1_2 ( + c2_2 ( * c0_2 ( + c3_2 ( * c0_2 c4_2 ) ) ) ) ) ( + c1_3 ( + c2_3 ( * c0_3 ( + c3_3 ( * c0_3 c4_3 ) ) ) ) ) ( + c1_4 ( + c2_4 ( * c0_4 ( + c3_4 ( * c0_4 c4_4 ) ) ) ) ) ( + c1_5 ( + c2_5 ( * c0_5 ( + c3_5 ( * c0_5 c4_5 ) ) ) ) ) ( + c1_6 ( + c2_6 ( * c0_6 ( + c3_6 ( * c0_6 c4_6 ) ) ) ) ) ( + c1_7 ( + c2_7 ( * c0_7 ( + c3_7 ( * c0_7 c4_7 ) ) ) ) ) ( + c1_8 ( + c2_8 ( * c0_8 ( + c3_8 ( * c0_8 c4_8 ) ) ) ) ) ( + c1_9 ( + c2_9 ( * c0_9 ( + c3_9 ( * c0_9 c4_9 ) ) ) ) ) ( + c1_10 ( + c2_10 ( * c0_10 ( + c3_10 ( * c0_10 c4_10 ) ) ) ) ) ( + c1_11 ( + c2_11 ( * c0_11 ( + c3_11 ( * c0_11 c4_11 ) ) ) ) ) ( + c1_12 ( + c2_12 ( * c0_12 ( + c3_12 ( * c0_12 c4_12 ) ) ) ) ) ( + c1_13 ( + c2_13 ( * c0_13 ( + c3_13 ( * c0_13 c4_13 ) ) ) ) ) ( + c1_14 ( + c2_14 ( * c0_14 ( + c3_14 ( * c0_14 c4_14 ) ) ) ) ) ( + c1_15 ( + c2_15 ( * c0_15 ( + c3_15 ( * c0_15 c4_15 ) ) ) ) ) ( + c1_16 ( + c2_16 ( * c0_16 ( + c3_16 ( * c0_16 c4_16 ) ) ) ) ) ( + c1_17 ( + c2_17 ( * c0_17 ( + c3_17 ( * c0_17 c4_17 ) ) ) ) ) ( + c1_18 ( + c2_18 ( * c0_18 ( + c3_18 ( * c0_18 c4_18 ) ) ) ) ) ( + c1_19 ( + c2_19 ( * c0_19 ( + c3_19 ( * c0_19 c4_19 ) ) ) ) ) ( + c1_20 ( + c2_20 ( * c0_20 ( + c3_20 ( * c0_20 c4_20 ) ) ) ) ) ( + c1_21 ( + c2_21 ( * c0_21 ( + c3_21 ( * c0_21 c4_21 ) ) ) ) ) ( + c1_22 ( + c2_22 ( * c0_22 ( + c3_22 ( * c0_22 c4_22 ) ) ) ) ) ( + c1_23 ( + c2_23 ( * c0_23 ( + c3_23 ( * c0_23 c4_23 ) ) ) ) ) ( + c1_24 ( + c2_24 ( * c0_24 ( + c3_24 ( * c0_24 c4_24 ) ) ) ) ) ( + c1_25 ( + c2_25 ( * c0_25 ( + c3_25 ( * c0_25 c4_25 ) ) ) ) ) ( + c1_26 ( + c2_26 ( * c0_26 ( + c3_26 ( * c0_26 c4_26 ) ) ) ) ) ( + c1_27 ( + c2_27 ( * c0_27 ( + c3_27 ( * c0_27 c4_27 ) ) ) ) ) ( + c1_28 ( + c2_28 ( * c0_28 ( + c3_28 ( * c0_28 c4_28 ) ) ) ) ) ( + c1_29 ( + c2_29 ( * c0_29 ( + c3_29 ( * c0_29 c4_29 ) ) ) ) ) ( + c1_30 ( + c2_30 ( * c0_30 ( + c3_30 ( * c0_30 c4_30 ) ) ) ) ) ( + c1_31 ( + c2_31 ( * c0_31 ( + c3_31 ( * c0_31 c4_31 ) ) ) ) ) ):poly_reg_32 -(Vec ( + ( + ( * ( - c2_0 c1_0 ) ( - c2_0 c1_0 ) ) ( * ( - c2_1 c1_1 ) ( - c2_1 c1_1 ) ) ) ( * ( - c2_2 c1_2 ) ( - c2_2 c1_2 ) ) ) ):l2_distance_3 -(Vec ( + ( + ( * ( - c2_0 c1_0 ) ( - c2_0 c1_0 ) ) ( * ( - c2_1 c1_1 ) ( - c2_1 c1_1 ) ) ) ( + ( * ( - c2_2 c1_2 ) ( - c2_2 c1_2 ) ) ( * ( - c2_3 c1_3 ) ( - c2_3 c1_3 ) ) ) ) ):l2_distance_4 -(Vec ( + ( + ( * ( - c2_0 c1_0 ) ( - c2_0 c1_0 ) ) ( * ( - c2_1 c1_1 ) ( - c2_1 c1_1 ) ) ) ( + ( + ( * ( - c2_2 c1_2 ) ( - c2_2 c1_2 ) ) ( * ( - c2_3 c1_3 ) ( - c2_3 c1_3 ) ) ) ( * ( - c2_4 c1_4 ) ( - c2_4 c1_4 ) ) ) ) ):l2_distance_5 -(Vec ( + ( + ( + ( * ( - c2_0 c1_0 ) ( - c2_0 c1_0 ) ) ( * ( - c2_1 c1_1 ) ( - c2_1 c1_1 ) ) ) ( + ( * ( - c2_2 c1_2 ) ( - c2_2 c1_2 ) ) ( * ( - c2_3 c1_3 ) ( - c2_3 c1_3 ) ) ) ) ( + ( + ( * ( - c2_4 c1_4 ) ( - c2_4 c1_4 ) ) ( * ( - c2_5 c1_5 ) ( - c2_5 c1_5 ) ) ) ( + ( * ( - c2_6 c1_6 ) ( - c2_6 c1_6 ) ) ( * ( - c2_7 c1_7 ) ( - c2_7 c1_7 ) ) ) ) ) ):l2_distance_8 -(Vec ( + ( + ( + ( + ( * ( - c2_0 c1_0 ) ( - c2_0 c1_0 ) ) ( * ( - c2_1 c1_1 ) ( - c2_1 c1_1 ) ) ) ( + ( * ( - c2_2 c1_2 ) ( - c2_2 c1_2 ) ) ( * ( - c2_3 c1_3 ) ( - c2_3 c1_3 ) ) ) ) ( + ( + ( * ( - c2_4 c1_4 ) ( - c2_4 c1_4 ) ) ( * ( - c2_5 c1_5 ) ( - c2_5 c1_5 ) ) ) ( + ( * ( - c2_6 c1_6 ) ( - c2_6 c1_6 ) ) ( * ( - c2_7 c1_7 ) ( - c2_7 c1_7 ) ) ) ) ) ( + ( + ( + ( * ( - c2_8 c1_8 ) ( - c2_8 c1_8 ) ) ( * ( - c2_9 c1_9 ) ( - c2_9 c1_9 ) ) ) ( + ( * ( - c2_10 c1_10 ) ( - c2_10 c1_10 ) ) ( * ( - c2_11 c1_11 ) ( - c2_11 c1_11 ) ) ) ) ( + ( + ( * ( - c2_12 c1_12 ) ( - c2_12 c1_12 ) ) ( * ( - c2_13 c1_13 ) ( - c2_13 c1_13 ) ) ) ( + ( * ( - c2_14 c1_14 ) ( - c2_14 c1_14 ) ) ( * ( - c2_15 c1_15 ) ( - c2_15 c1_15 ) ) ) ) ) ) ):l2_distance_16 -(Vec ( + ( + ( + ( + ( + ( * ( - c2_0 c1_0 ) ( - c2_0 c1_0 ) ) ( * ( - c2_1 c1_1 ) ( - c2_1 c1_1 ) ) ) ( + ( * ( - c2_2 c1_2 ) ( - c2_2 c1_2 ) ) ( * ( - c2_3 c1_3 ) ( - c2_3 c1_3 ) ) ) ) ( + ( + ( * ( - c2_4 c1_4 ) ( - c2_4 c1_4 ) ) ( * ( - c2_5 c1_5 ) ( - c2_5 c1_5 ) ) ) ( + ( * ( - c2_6 c1_6 ) ( - c2_6 c1_6 ) ) ( * ( - c2_7 c1_7 ) ( - c2_7 c1_7 ) ) ) ) ) ( + ( + ( + ( * ( - c2_8 c1_8 ) ( - c2_8 c1_8 ) ) ( * ( - c2_9 c1_9 ) ( - c2_9 c1_9 ) ) ) ( + ( * ( - c2_10 c1_10 ) ( - c2_10 c1_10 ) ) ( * ( - c2_11 c1_11 ) ( - c2_11 c1_11 ) ) ) ) ( + ( + ( * ( - c2_12 c1_12 ) ( - c2_12 c1_12 ) ) ( * ( - c2_13 c1_13 ) ( - c2_13 c1_13 ) ) ) ( + ( * ( - c2_14 c1_14 ) ( - c2_14 c1_14 ) ) ( * ( - c2_15 c1_15 ) ( - c2_15 c1_15 ) ) ) ) ) ) ( + ( + ( + ( + ( * ( - c2_16 c1_16 ) ( - c2_16 c1_16 ) ) ( * ( - c2_17 c1_17 ) ( - c2_17 c1_17 ) ) ) ( + ( * ( - c2_18 c1_18 ) ( - c2_18 c1_18 ) ) ( * ( - c2_19 c1_19 ) ( - c2_19 c1_19 ) ) ) ) ( + ( + ( * ( - c2_20 c1_20 ) ( - c2_20 c1_20 ) ) ( * ( - c2_21 c1_21 ) ( - c2_21 c1_21 ) ) ) ( + ( * ( - c2_22 c1_22 ) ( - c2_22 c1_22 ) ) ( * ( - c2_23 c1_23 ) ( - c2_23 c1_23 ) ) ) ) ) ( + ( + ( + ( * ( - c2_24 c1_24 ) ( - c2_24 c1_24 ) ) ( * ( - c2_25 c1_25 ) ( - c2_25 c1_25 ) ) ) ( + ( * ( - c2_26 c1_26 ) ( - c2_26 c1_26 ) ) ( * ( - c2_27 c1_27 ) ( - c2_27 c1_27 ) ) ) ) ( + ( + ( * ( - c2_28 c1_28 ) ( - c2_28 c1_28 ) ) ( * ( - c2_29 c1_29 ) ( - c2_29 c1_29 ) ) ) ( + ( * ( - c2_30 c1_30 ) ( - c2_30 c1_30 ) ) ( * ( - c2_31 c1_31 ) ( - c2_31 c1_31 ) ) ) ) ) ) ) ):l2_distance_32 -(Vec ( + ( + ( * v1_0 v2_0 ) ( * v1_1 v2_1 ) ) ( * v1_2 v2_2 ) ) ):dot_product_3 -(Vec ( + ( + ( * v1_0 v2_0 ) ( * v1_1 v2_1 ) ) ( + ( * v1_2 v2_2 ) ( * v1_3 v2_3 ) ) ) ):dot_product_4 -(Vec ( + ( + ( * v1_0 v2_0 ) ( * v1_1 v2_1 ) ) ( + ( + ( * v1_2 v2_2 ) ( * v1_3 v2_3 ) ) ( * v1_4 v2_4 ) ) ) ):dot_product_5 -(Vec ( + ( + ( + ( * v1_0 v2_0 ) ( * v1_1 v2_1 ) ) ( + ( * v1_2 v2_2 ) ( * v1_3 v2_3 ) ) ) ( + ( + ( * v1_4 v2_4 ) ( * v1_5 v2_5 ) ) ( + ( * v1_6 v2_6 ) ( * v1_7 v2_7 ) ) ) ) ):dot_product_8 -(Vec ( + ( + ( + ( + ( * v1_0 v2_0 ) ( * v1_1 v2_1 ) ) ( + ( * v1_2 v2_2 ) ( * v1_3 v2_3 ) ) ) ( + ( + ( * v1_4 v2_4 ) ( * v1_5 v2_5 ) ) ( + ( * v1_6 v2_6 ) ( * v1_7 v2_7 ) ) ) ) ( + ( + ( + ( * v1_8 v2_8 ) ( * v1_9 v2_9 ) ) ( + ( * v1_10 v2_10 ) ( * v1_11 v2_11 ) ) ) ( + ( + ( * v1_12 v2_12 ) ( * v1_13 v2_13 ) ) ( + ( * v1_14 v2_14 ) ( * v1_15 v2_15 ) ) ) ) ) ):dot_product_16 -(Vec ( + ( + ( + ( + ( + ( * v1_0 v2_0 ) ( * v1_1 v2_1 ) ) ( + ( * v1_2 v2_2 ) ( * v1_3 v2_3 ) ) ) ( + ( + ( * v1_4 v2_4 ) ( * v1_5 v2_5 ) ) ( + ( * v1_6 v2_6 ) ( * v1_7 v2_7 ) ) ) ) ( + ( + ( + ( * v1_8 v2_8 ) ( * v1_9 v2_9 ) ) ( + ( * v1_10 v2_10 ) ( * v1_11 v2_11 ) ) ) ( + ( + ( * v1_12 v2_12 ) ( * v1_13 v2_13 ) ) ( + ( * v1_14 v2_14 ) ( * v1_15 v2_15 ) ) ) ) ) ( + ( + ( + ( + ( * v1_16 v2_16 ) ( * v1_17 v2_17 ) ) ( + ( * v1_18 v2_18 ) ( * v1_19 v2_19 ) ) ) ( + ( + ( * v1_20 v2_20 ) ( * v1_21 v2_21 ) ) ( + ( * v1_22 v2_22 ) ( * v1_23 v2_23 ) ) ) ) ( + ( + ( + ( * v1_24 v2_24 ) ( * v1_25 v2_25 ) ) ( + ( * v1_26 v2_26 ) ( * v1_27 v2_27 ) ) ) ( + ( + ( * v1_28 v2_28 ) ( * v1_29 v2_29 ) ) ( + ( * v1_30 v2_30 ) ( * v1_31 v2_31 ) ) ) ) ) ) ):dot_product_32 -(Vec ( + in_1_1 ( * in_0_1 2 ) ) ( + ( + in_1_2 ( - in_1_0 ) ) ( + ( * in_0_2 2 ) ( * in_0_0 -2 ) ) ) ( + ( - in_1_1 ) ( * in_0_1 -2 ) ) ( + in_2_1 ( + in_0_1 ( * in_1_1 2 ) ) ) ( + ( + in_2_2 ( - in_2_0 ) ) ( + ( + in_0_2 ( - in_0_0 ) ) ( + ( * in_1_2 2 ) ( * in_1_0 -2 ) ) ) ) ( + ( - in_2_1 ) ( + ( - in_0_1 ) ( * in_1_1 -2 ) ) ) ( + in_1_1 ( * in_2_1 2 ) ) ( + ( + in_1_2 ( - in_1_0 ) ) ( + ( * in_2_2 2 ) ( * in_2_0 -2 ) ) ) ( + ( - in_1_1 ) ( * in_2_1 -2 ) ) ):gx_kernel_3 -(Vec ( + in_1_1 ( * in_0_1 2 ) ) ( + ( + in_1_2 ( - in_1_0 ) ) ( + ( * in_0_2 2 ) ( * in_0_0 -2 ) ) ) ( + ( + in_1_3 ( - in_1_1 ) ) ( + ( * in_0_3 2 ) ( * in_0_1 -2 ) ) ) ( + ( - in_1_2 ) ( * in_0_2 -2 ) ) ( + in_2_1 ( + in_0_1 ( * in_1_1 2 ) ) ) ( + ( + in_2_2 ( - in_2_0 ) ) ( + ( + in_0_2 ( - in_0_0 ) ) ( + ( * in_1_2 2 ) ( * in_1_0 -2 ) ) ) ) ( + ( + in_2_3 ( - in_2_1 ) ) ( + ( + in_0_3 ( - in_0_1 ) ) ( + ( * in_1_3 2 ) ( * in_1_1 -2 ) ) ) ) ( + ( - in_2_2 ) ( + ( - in_0_2 ) ( * in_1_2 -2 ) ) ) ( + in_3_1 ( + in_1_1 ( * in_2_1 2 ) ) ) ( + ( + in_3_2 ( - in_3_0 ) ) ( + ( + in_1_2 ( - in_1_0 ) ) ( + ( * in_2_2 2 ) ( * in_2_0 -2 ) ) ) ) ( + ( + in_3_3 ( - in_3_1 ) ) ( + ( + in_1_3 ( - in_1_1 ) ) ( + ( * in_2_3 2 ) ( * in_2_1 -2 ) ) ) ) ( + ( - in_3_2 ) ( + ( - in_1_2 ) ( * in_2_2 -2 ) ) ) ( + in_2_1 ( * in_3_1 2 ) ) ( + ( + in_2_2 ( - in_2_0 ) ) ( + ( * in_3_2 2 ) ( * in_3_0 -2 ) ) ) ( + ( + in_2_3 ( - in_2_1 ) ) ( + ( * in_3_3 2 ) ( * in_3_1 -2 ) ) ) ( + ( - in_2_2 ) ( * in_3_2 -2 ) ) ):gx_kernel_4 -(Vec ( + in_1_1 ( * in_0_1 2 ) ) ( + ( + in_1_2 ( - in_1_0 ) ) ( + ( * in_0_2 2 ) ( * in_0_0 -2 ) ) ) ( + ( + in_1_3 ( - in_1_1 ) ) ( + ( * in_0_3 2 ) ( * in_0_1 -2 ) ) ) ( + ( + in_1_4 ( - in_1_2 ) ) ( + ( * in_0_4 2 ) ( * in_0_2 -2 ) ) ) ( + ( - in_1_3 ) ( * in_0_3 -2 ) ) ( + in_2_1 ( + in_0_1 ( * in_1_1 2 ) ) ) ( + ( + in_2_2 ( - in_2_0 ) ) ( + ( + in_0_2 ( - in_0_0 ) ) ( + ( * in_1_2 2 ) ( * in_1_0 -2 ) ) ) ) ( + ( + in_2_3 ( - in_2_1 ) ) ( + ( + in_0_3 ( - in_0_1 ) ) ( + ( * in_1_3 2 ) ( * in_1_1 -2 ) ) ) ) ( + ( + in_2_4 ( - in_2_2 ) ) ( + ( + in_0_4 ( - in_0_2 ) ) ( + ( * in_1_4 2 ) ( * in_1_2 -2 ) ) ) ) ( + ( - in_2_3 ) ( + ( - in_0_3 ) ( * in_1_3 -2 ) ) ) ( + in_3_1 ( + in_1_1 ( * in_2_1 2 ) ) ) ( + ( + in_3_2 ( - in_3_0 ) ) ( + ( + in_1_2 ( - in_1_0 ) ) ( + ( * in_2_2 2 ) ( * in_2_0 -2 ) ) ) ) ( + ( + in_3_3 ( - in_3_1 ) ) ( + ( + in_1_3 ( - in_1_1 ) ) ( + ( * in_2_3 2 ) ( * in_2_1 -2 ) ) ) ) ( + ( + in_3_4 ( - in_3_2 ) ) ( + ( + in_1_4 ( - in_1_2 ) ) ( + ( * in_2_4 2 ) ( * in_2_2 -2 ) ) ) ) ( + ( - in_3_3 ) ( + ( - in_1_3 ) ( * in_2_3 -2 ) ) ) ( + in_4_1 ( + in_2_1 ( * in_3_1 2 ) ) ) ( + ( + in_4_2 ( - in_4_0 ) ) ( + ( + in_2_2 ( - in_2_0 ) ) ( + ( * in_3_2 2 ) ( * in_3_0 -2 ) ) ) ) ( + ( + in_4_3 ( - in_4_1 ) ) ( + ( + in_2_3 ( - in_2_1 ) ) ( + ( * in_3_3 2 ) ( * in_3_1 -2 ) ) ) ) ( + ( + in_4_4 ( - in_4_2 ) ) ( + ( + in_2_4 ( - in_2_2 ) ) ( + ( * in_3_4 2 ) ( * in_3_2 -2 ) ) ) ) ( + ( - in_4_3 ) ( + ( - in_2_3 ) ( * in_3_3 -2 ) ) ) ( + in_3_1 ( * in_4_1 2 ) ) ( + ( + in_3_2 ( - in_3_0 ) ) ( + ( * in_4_2 2 ) ( * in_4_0 -2 ) ) ) ( + ( + in_3_3 ( - in_3_1 ) ) ( + ( * in_4_3 2 ) ( * in_4_1 -2 ) ) ) ( + ( + in_3_4 ( - in_3_2 ) ) ( + ( * in_4_4 2 ) ( * in_4_2 -2 ) ) ) ( + ( - in_3_3 ) ( * in_4_3 -2 ) ) ):gx_kernel_5 -(Vec ( + ( + ( * a_0_0 b_0_0 ) ( * a_0_1 b_1_0 ) ) ( * a_0_2 b_2_0 ) ) ( + ( + ( * a_0_0 b_0_1 ) ( * a_0_1 b_1_1 ) ) ( * a_0_2 b_2_1 ) ) ( + ( + ( * a_0_0 b_0_2 ) ( * a_0_1 b_1_2 ) ) ( * a_0_2 b_2_2 ) ) ( + ( + ( * a_1_0 b_0_0 ) ( * a_1_1 b_1_0 ) ) ( * a_1_2 b_2_0 ) ) ( + ( + ( * a_1_0 b_0_1 ) ( * a_1_1 b_1_1 ) ) ( * a_1_2 b_2_1 ) ) ( + ( + ( * a_1_0 b_0_2 ) ( * a_1_1 b_1_2 ) ) ( * a_1_2 b_2_2 ) ) ( + ( + ( * a_2_0 b_0_0 ) ( * a_2_1 b_1_0 ) ) ( * a_2_2 b_2_0 ) ) ( + ( + ( * a_2_0 b_0_1 ) ( * a_2_1 b_1_1 ) ) ( * a_2_2 b_2_1 ) ) ( + ( + ( * a_2_0 b_0_2 ) ( * a_2_1 b_1_2 ) ) ( * a_2_2 b_2_2 ) ) ):matrix_mul_3 -(Vec ( + ( + ( * a_0_0 b_0_0 ) ( * a_0_1 b_1_0 ) ) ( + ( * a_0_2 b_2_0 ) ( * a_0_3 b_3_0 ) ) ) ( + ( + ( * a_0_0 b_0_1 ) ( * a_0_1 b_1_1 ) ) ( + ( * a_0_2 b_2_1 ) ( * a_0_3 b_3_1 ) ) ) ( + ( + ( * a_0_0 b_0_2 ) ( * a_0_1 b_1_2 ) ) ( + ( * a_0_2 b_2_2 ) ( * a_0_3 b_3_2 ) ) ) ( + ( + ( * a_0_0 b_0_3 ) ( * a_0_1 b_1_3 ) ) ( + ( * a_0_2 b_2_3 ) ( * a_0_3 b_3_3 ) ) ) ( + ( + ( * a_1_0 b_0_0 ) ( * a_1_1 b_1_0 ) ) ( + ( * a_1_2 b_2_0 ) ( * a_1_3 b_3_0 ) ) ) ( + ( + ( * a_1_0 b_0_1 ) ( * a_1_1 b_1_1 ) ) ( + ( * a_1_2 b_2_1 ) ( * a_1_3 b_3_1 ) ) ) ( + ( + ( * a_1_0 b_0_2 ) ( * a_1_1 b_1_2 ) ) ( + ( * a_1_2 b_2_2 ) ( * a_1_3 b_3_2 ) ) ) ( + ( + ( * a_1_0 b_0_3 ) ( * a_1_1 b_1_3 ) ) ( + ( * a_1_2 b_2_3 ) ( * a_1_3 b_3_3 ) ) ) ( + ( + ( * a_2_0 b_0_0 ) ( * a_2_1 b_1_0 ) ) ( + ( * a_2_2 b_2_0 ) ( * a_2_3 b_3_0 ) ) ) ( + ( + ( * a_2_0 b_0_1 ) ( * a_2_1 b_1_1 ) ) ( + ( * a_2_2 b_2_1 ) ( * a_2_3 b_3_1 ) ) ) ( + ( + ( * a_2_0 b_0_2 ) ( * a_2_1 b_1_2 ) ) ( + ( * a_2_2 b_2_2 ) ( * a_2_3 b_3_2 ) ) ) ( + ( + ( * a_2_0 b_0_3 ) ( * a_2_1 b_1_3 ) ) ( + ( * a_2_2 b_2_3 ) ( * a_2_3 b_3_3 ) ) ) ( + ( + ( * a_3_0 b_0_0 ) ( * a_3_1 b_1_0 ) ) ( + ( * a_3_2 b_2_0 ) ( * a_3_3 b_3_0 ) ) ) ( + ( + ( * a_3_0 b_0_1 ) ( * a_3_1 b_1_1 ) ) ( + ( * a_3_2 b_2_1 ) ( * a_3_3 b_3_1 ) ) ) ( + ( + ( * a_3_0 b_0_2 ) ( * a_3_1 b_1_2 ) ) ( + ( * a_3_2 b_2_2 ) ( * a_3_3 b_3_2 ) ) ) ( + ( + ( * a_3_0 b_0_3 ) ( * a_3_1 b_1_3 ) ) ( + ( * a_3_2 b_2_3 ) ( * a_3_3 b_3_3 ) ) ) ):matrix_mul_4 -(Vec ( + ( * ( + ( * o3 ( - 1 c23 ) ) ( * c23 o2 ) ) ( - 1 c12 ) ) ( * c12 ( + ( * o3 ( - 1 c13 ) ) ( * c13 o1 ) ) ) ) ):max_3 -(Vec ( + ( * ( + ( * ( + ( * o4 ( - 1 c34 ) ) ( * c34 o3 ) ) ( - 1 c23 ) ) ( * c23 ( + ( * o4 ( - 1 c24 ) ) ( * c24 o2 ) ) ) ) ( - 1 c12 ) ) ( * c12 ( + ( * ( + ( * o4 ( - 1 c34 ) ) ( * c34 o3 ) ) ( - 1 c13 ) ) ( * c13 ( + ( * o4 ( - 1 c14 ) ) ( * c14 o1 ) ) ) ) ) ) ):max_4 -(Vec ( + ( * ( + ( * ( + ( * ( + ( * o5 ( - 1 c45 ) ) ( * c45 o4 ) ) ( - 1 c34 ) ) ( * c34 ( + ( * o5 ( - 1 c35 ) ) ( * c35 o3 ) ) ) ) ( - 1 c23 ) ) ( * c23 ( + ( * ( + ( * o5 ( - 1 c45 ) ) ( * c45 o4 ) ) ( - 1 c24 ) ) ( * c24 ( + ( * o5 ( - 1 c25 ) ) ( * c25 o2 ) ) ) ) ) ) ( - 1 c12 ) ) ( * c12 ( + ( * ( + ( * ( + ( * o5 ( - 1 c45 ) ) ( * c45 o4 ) ) ( - 1 c34 ) ) ( * c34 ( + ( * o5 ( - 1 c35 ) ) ( * c35 o3 ) ) ) ) ( - 1 c13 ) ) ( * c13 ( + ( * ( + ( * o5 ( - 1 c45 ) ) ( * c45 o4 ) ) ( - 1 c14 ) ) ( * c14 ( + ( * o5 ( - 1 c15 ) ) ( * c15 o1 ) ) ) ) ) ) ) ) ):max_5 -(Vec ( + ( * ( + ( * ( + ( * o321 ( - 1 c23 ) ) ( * c23 o231 ) ) ( - 1 c13 ) ) ( * c13 o213 ) ) ( - 1 c12 ) ) ( * c12 ( + ( * ( - 1 c23 ) ( + ( * o312 ( - 1 c13 ) ) ( * c13 o132 ) ) ) ( * c23 o123 ) ) ) ) ):sort_3 -(Vec ( + ( * ( + ( * ( + ( * o3214 ( - 1 c34 ) ) ( * c34 ( + ( * ( + ( * o4321 ( - 1 c34 ) ) ( * c34 o3421 ) ) ( - 1 c24 ) ) ( * c24 ( + ( * o3241 ( - 1 c14 ) ) ( * c14 o3214 ) ) ) ) ) ) ( - 1 c23 ) ) ( * c23 ( + ( * ( + ( * o2314 ( - 1 c24 ) ) ( * c24 ( + ( * ( - 1 c34 ) ( + ( * o4231 ( - 1 c24 ) ) ( * c24 o2431 ) ) ) ( * c34 ( + ( * o2341 ( - 1 c14 ) ) ( * c14 o2314 ) ) ) ) ) ) ( - 1 c13 ) ) ( * c13 ( + ( * o2134 ( - 1 c24 ) ) ( * c24 ( + ( * ( - 1 c14 ) ( + ( * o4213 ( - 1 c24 ) ) ( * c24 o2413 ) ) ) ( * c14 ( + ( * o2143 ( - 1 c34 ) ) ( * c34 o2134 ) ) ) ) ) ) ) ) ) ) ( - 1 c12 ) ) ( * c12 ( + ( * ( - 1 c13 ) ( + ( * o3124 ( - 1 c34 ) ) ( * c34 ( + ( * ( - 1 c14 ) ( + ( * o4312 ( - 1 c34 ) ) ( * c34 o3412 ) ) ) ( * c14 ( + ( * o3142 ( - 1 c24 ) ) ( * c24 o3124 ) ) ) ) ) ) ) ( * c13 ( + ( * ( - 1 c23 ) ( + ( * o1324 ( - 1 c14 ) ) ( * c14 ( + ( * ( - 1 c34 ) ( + ( * o4132 ( - 1 c14 ) ) ( * c14 o1432 ) ) ) ( * c34 ( + ( * o1342 ( - 1 c24 ) ) ( * c24 o1324 ) ) ) ) ) ) ) ( * c23 ( + ( * o1234 ( - 1 c14 ) ) ( * c14 ( + ( * ( - 1 c24 ) ( + ( * o4123 ( - 1 c14 ) ) ( * c14 o1423 ) ) ) ( * c24 ( + ( * o1243 ( - 1 c34 ) ) ( * c34 o1234 ) ) ) ) ) ) ) ) ) ) ) ) ):sort_4 -(Vec ( * ( + x9602 x9604 ) ( + ( + x9608 x9610 ) ( + ( * x9613 x9615 ) x9617 ) ) ) ):tree_50-50-5_poly -(Vec ( * ( * ( * ( + ( * ( * x10106 x10108 ) x10110 ) x10112 ) x10115 ) ( * x10117 x10119 ) ) ( + ( + x10121 ( * x10125 ( + ( + ( * ( * ( * x10132 x10133 ) ( + x10135 x10136 ) ) ( + ( * x10139 x10140 ) x10142 ) ) ( * ( * x10146 x10147 ) ( * ( + x10149 x10150 ) x10152 ) ) ) ( + ( + x10154 ( * ( * x10159 ( + ( + x10162 x10163 ) x10165 ) ) x10167 ) ) ( + ( + x10170 x10173 ) ( + x10176 x10178 ) ) ) ) ) ) ( + x10181 x10183 ) ) ) ):tree_50-50-10_poly -(Vec ( + ( + ( * ( + ( * x9604 x9605 ) ( + x9607 x9608 ) ) ( + ( + x9611 x9612 ) ( + x9614 x9615 ) ) ) ( * ( * ( * x9619 x9620 ) ( * x9622 x9623 ) ) ( * ( + x9626 x9627 ) ( * x9629 x9630 ) ) ) ) ( * ( + ( + ( * x9634 x9635 ) ( + x9637 x9638 ) ) ( + ( * x9641 x9642 ) ( * x9644 x9645 ) ) ) ( + ( * ( + x9649 x9650 ) ( + x9652 x9653 ) ) ( + ( * x9656 x9657 ) ( * x9659 x9660 ) ) ) ) ) ):tree_100-50-5_poly -(Vec ( + ( + ( + ( * ( + ( + ( * ( + ( * ( + x10109 x10110 ) ( * x10112 x10113 ) ) ( * ( + x10116 x10117 ) ( + x10119 x10120 ) ) ) ( + ( * ( + x10124 x10125 ) ( * x10127 x10128 ) ) ( + ( + x10131 x10132 ) ( * x10134 x10135 ) ) ) ) ( + ( + ( + ( + x10140 x10141 ) ( * x10143 x10144 ) ) ( + ( + x10147 x10148 ) ( * x10150 x10151 ) ) ) ( * ( + ( + x10155 x10156 ) ( * x10158 x10159 ) ) ( + ( * x10162 x10163 ) ( * x10165 x10166 ) ) ) ) ) ( * ( * ( + ( * ( * x10172 x10173 ) ( + x10175 x10176 ) ) ( + ( + x10179 x10180 ) ( + x10182 x10183 ) ) ) ( + ( * ( * x10187 x10188 ) ( * x10190 x10191 ) ) ( + ( * x10194 x10195 ) ( * x10197 x10198 ) ) ) ) ( * ( * ( + ( * x10203 x10204 ) ( + x10206 x10207 ) ) ( * ( * x10210 x10211 ) ( + x10213 x10214 ) ) ) ( * ( * ( + x10218 x10219 ) ( + x10221 x10222 ) ) ( + ( * x10225 x10226 ) ( + x10228 x10229 ) ) ) ) ) ) ( + ( + ( + ( + ( + ( + x10236 x10237 ) ( + x10239 x10240 ) ) ( + ( * x10243 x10244 ) ( + x10246 x10247 ) ) ) ( + ( + ( + x10251 x10252 ) ( * x10254 x10255 ) ) ( + ( + x10258 x10259 ) ( * x10261 x10262 ) ) ) ) ( * ( * ( + ( * x10267 x10268 ) ( + x10270 x10271 ) ) ( * ( * x10274 x10275 ) ( + x10277 x10278 ) ) ) ( * ( + ( + x10282 x10283 ) ( * x10285 x10286 ) ) ( * ( + x10289 x10290 ) ( * x10292 x10293 ) ) ) ) ) ( + ( * ( + ( + ( * x10299 x10300 ) ( + x10302 x10303 ) ) ( * ( * x10306 x10307 ) ( * x10309 x10310 ) ) ) ( * ( * ( * x10314 x10315 ) ( * x10317 x10318 ) ) ( + ( + x10321 x10322 ) ( + x10324 x10325 ) ) ) ) ( * ( + ( * ( + x10330 x10331 ) ( * x10333 x10334 ) ) ( * ( + x10337 x10338 ) ( * x10340 x10341 ) ) ) ( * ( * ( + x10345 x10346 ) ( + x10348 x10349 ) ) ( + ( * x10352 x10353 ) ( + x10355 x10356 ) ) ) ) ) ) ) ( + ( * ( * ( + ( * ( + ( + x10364 x10365 ) ( + x10367 x10368 ) ) ( * ( * x10371 x10372 ) ( + x10374 x10375 ) ) ) ( + ( * ( + x10379 x10380 ) ( + x10382 x10383 ) ) ( * ( + x10386 x10387 ) ( + x10389 x10390 ) ) ) ) ( + ( + ( + ( * x10395 x10396 ) ( * x10398 x10399 ) ) ( * ( + x10402 x10403 ) ( + x10405 x10406 ) ) ) ( + ( * ( + x10410 x10411 ) ( + x10413 x10414 ) ) ( * ( * x10417 x10418 ) ( + x10420 x10421 ) ) ) ) ) ( + ( * ( * ( + ( * x10427 x10428 ) ( * x10430 x10431 ) ) ( * ( * x10434 x10435 ) ( * x10437 x10438 ) ) ) ( + ( * ( * x10442 x10443 ) ( + x10445 x10446 ) ) ( * ( * x10449 x10450 ) ( + x10452 x10453 ) ) ) ) ( * ( + ( + ( * x10458 x10459 ) ( * x10461 x10462 ) ) ( + ( + x10465 x10466 ) ( + x10468 x10469 ) ) ) ( * ( * ( * x10473 x10474 ) ( * x10476 x10477 ) ) ( + ( + x10480 x10481 ) ( + x10483 x10484 ) ) ) ) ) ) ( * ( * ( + ( * ( + ( + x10491 x10492 ) ( + x10494 x10495 ) ) ( + ( * x10498 x10499 ) ( * x10501 x10502 ) ) ) ( + ( + ( * x10506 x10507 ) ( * x10509 x10510 ) ) ( * ( * x10513 x10514 ) ( + x10516 x10517 ) ) ) ) ( * ( * ( + ( + x10522 x10523 ) ( + x10525 x10526 ) ) ( + ( + x10529 x10530 ) ( * x10532 x10533 ) ) ) ( + ( * ( * x10537 x10538 ) ( * x10540 x10541 ) ) ( * ( * x10544 x10545 ) ( * x10547 x10548 ) ) ) ) ) ( * ( * ( * ( * ( * x10554 x10555 ) ( * x10557 x10558 ) ) ( * ( * x10561 x10562 ) ( + x10564 x10565 ) ) ) ( * ( * ( + x10569 x10570 ) ( + x10572 x10573 ) ) ( + ( * x10576 x10577 ) ( * x10579 x10580 ) ) ) ) ( + ( * ( * ( + x10585 x10586 ) ( + x10588 x10589 ) ) ( + ( + x10592 x10593 ) ( * x10595 x10596 ) ) ) ( + ( * ( + x10600 x10601 ) ( * x10603 x10604 ) ) ( * ( * x10607 x10608 ) ( * x10610 x10611 ) ) ) ) ) ) ) ) ( + ( * ( + ( + ( * ( + ( + ( * x10620 x10621 ) ( + x10623 x10624 ) ) ( + ( + x10627 x10628 ) ( * x10630 x10631 ) ) ) ( * ( + ( * x10635 x10636 ) ( * x10638 x10639 ) ) ( + ( * x10642 x10643 ) ( + x10645 x10646 ) ) ) ) ( + ( * ( * ( * x10651 x10652 ) ( + x10654 x10655 ) ) ( + ( * x10658 x10659 ) ( + x10661 x10662 ) ) ) ( * ( + ( + x10666 x10667 ) ( * x10669 x10670 ) ) ( * ( * x10673 x10674 ) ( + x10676 x10677 ) ) ) ) ) ( * ( + ( * ( + ( + x10683 x10684 ) ( + x10686 x10687 ) ) ( + ( + x10690 x10691 ) ( + x10693 x10694 ) ) ) ( + ( * ( * x10698 x10699 ) ( + x10701 x10702 ) ) ( + ( + x10705 x10706 ) ( * x10708 x10709 ) ) ) ) ( + ( + ( * ( * x10714 x10715 ) ( * x10717 x10718 ) ) ( * ( + x10721 x10722 ) ( + x10724 x10725 ) ) ) ( + ( + ( + x10729 x10730 ) ( * x10732 x10733 ) ) ( + ( * x10736 x10737 ) ( * x10739 x10740 ) ) ) ) ) ) ( + ( + ( + ( + ( + ( * x10747 x10748 ) ( + x10750 x10751 ) ) ( * ( + x10754 x10755 ) ( + x10757 x10758 ) ) ) ( + ( + ( * x10762 x10763 ) ( + x10765 x10766 ) ) ( * ( + x10769 x10770 ) ( * x10772 x10773 ) ) ) ) ( * ( * ( * ( + x10778 x10779 ) ( * x10781 x10782 ) ) ( * ( + x10785 x10786 ) ( * x10788 x10789 ) ) ) ( + ( + ( * x10793 x10794 ) ( + x10796 x10797 ) ) ( * ( + x10800 x10801 ) ( + x10803 x10804 ) ) ) ) ) ( * ( + ( + ( + ( * x10810 x10811 ) ( * x10813 x10814 ) ) ( + ( + x10817 x10818 ) ( + x10820 x10821 ) ) ) ( + ( + ( + x10825 x10826 ) ( * x10828 x10829 ) ) ( * ( * x10832 x10833 ) ( + x10835 x10836 ) ) ) ) ( * ( * ( * ( * x10841 x10842 ) ( * x10844 x10845 ) ) ( * ( * x10848 x10849 ) ( * x10851 x10852 ) ) ) ( + ( + ( * x10856 x10857 ) ( * x10859 x10860 ) ) ( * ( + x10863 x10864 ) ( + x10866 x10867 ) ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * ( + x10875 x10876 ) ( + x10878 x10879 ) ) ( * ( * x10882 x10883 ) ( * x10885 x10886 ) ) ) ( * ( + ( + x10890 x10891 ) ( + x10893 x10894 ) ) ( * ( + x10897 x10898 ) ( + x10900 x10901 ) ) ) ) ( + ( + ( * ( + x10906 x10907 ) ( + x10909 x10910 ) ) ( + ( * x10913 x10914 ) ( * x10916 x10917 ) ) ) ( + ( + ( + x10921 x10922 ) ( + x10924 x10925 ) ) ( * ( * x10928 x10929 ) ( * x10931 x10932 ) ) ) ) ) ( * ( * ( * ( * ( * x10938 x10939 ) ( * x10941 x10942 ) ) ( * ( * x10945 x10946 ) ( + x10948 x10949 ) ) ) ( + ( * ( + x10953 x10954 ) ( * x10956 x10957 ) ) ( * ( * x10960 x10961 ) ( * x10963 x10964 ) ) ) ) ( + ( * ( + ( * x10969 x10970 ) ( * x10972 x10973 ) ) ( * ( + x10976 x10977 ) ( + x10979 x10980 ) ) ) ( + ( * ( * x10984 x10985 ) ( + x10987 x10988 ) ) ( * ( + x10991 x10992 ) ( * x10994 x10995 ) ) ) ) ) ) ( + ( * ( * ( * ( * ( + x11002 x11003 ) ( * x11005 x11006 ) ) ( * ( * x11009 x11010 ) ( + x11012 x11013 ) ) ) ( * ( + ( * x11017 x11018 ) ( * x11020 x11021 ) ) ( + ( * x11024 x11025 ) ( * x11027 x11028 ) ) ) ) ( * ( * ( * ( + x11033 x11034 ) ( + x11036 x11037 ) ) ( * ( + x11040 x11041 ) ( * x11043 x11044 ) ) ) ( * ( + ( + x11048 x11049 ) ( + x11051 x11052 ) ) ( * ( + x11055 x11056 ) ( + x11058 x11059 ) ) ) ) ) ( + ( * ( * ( * ( + x11065 x11066 ) ( + x11068 x11069 ) ) ( + ( + x11072 x11073 ) ( * x11075 x11076 ) ) ) ( * ( * ( + x11080 x11081 ) ( * x11083 x11084 ) ) ( + ( + x11087 x11088 ) ( + x11090 x11091 ) ) ) ) ( + ( * ( * ( * x11096 x11097 ) ( + x11099 x11100 ) ) ( + ( + x11103 x11104 ) ( * x11106 x11107 ) ) ) ( * ( + ( * x11111 x11112 ) ( * x11114 x11115 ) ) ( + ( + x11118 x11119 ) ( + x11121 x11122 ) ) ) ) ) ) ) ) ) ( + ( * ( + ( * ( + ( + ( * ( + ( * x11131 x11132 ) ( + x11134 x11135 ) ) ( + ( + x11138 x11139 ) ( * x11141 x11142 ) ) ) ( + ( + ( * x11146 x11147 ) ( + x11149 x11150 ) ) ( + ( * x11153 x11154 ) ( + x11156 x11157 ) ) ) ) ( + ( * ( * ( * x11162 x11163 ) ( * x11165 x11166 ) ) ( * ( * x11169 x11170 ) ( + x11172 x11173 ) ) ) ( + ( * ( * x11177 x11178 ) ( * x11180 x11181 ) ) ( * ( + x11184 x11185 ) ( * x11187 x11188 ) ) ) ) ) ( * ( * ( + ( * ( * x11194 x11195 ) ( * x11197 x11198 ) ) ( * ( + x11201 x11202 ) ( + x11204 x11205 ) ) ) ( + ( * ( + x11209 x11210 ) ( * x11212 x11213 ) ) ( * ( + x11216 x11217 ) ( * x11219 x11220 ) ) ) ) ( + ( * ( * ( * x11225 x11226 ) ( * x11228 x11229 ) ) ( + ( * x11232 x11233 ) ( + x11235 x11236 ) ) ) ( * ( * ( * x11240 x11241 ) ( * x11243 x11244 ) ) ( * ( + x11247 x11248 ) ( * x11250 x11251 ) ) ) ) ) ) ( + ( + ( * ( * ( * ( * x11258 x11259 ) ( * x11261 x11262 ) ) ( + ( * x11265 x11266 ) ( * x11268 x11269 ) ) ) ( + ( * ( + x11273 x11274 ) ( * x11276 x11277 ) ) ( * ( + x11280 x11281 ) ( + x11283 x11284 ) ) ) ) ( + ( * ( + ( * x11289 x11290 ) ( + x11292 x11293 ) ) ( * ( * x11296 x11297 ) ( * x11299 x11300 ) ) ) ( + ( * ( + x11304 x11305 ) ( * x11307 x11308 ) ) ( + ( + x11311 x11312 ) ( + x11314 x11315 ) ) ) ) ) ( * ( + ( + ( * ( + x11321 x11322 ) ( + x11324 x11325 ) ) ( + ( + x11328 x11329 ) ( * x11331 x11332 ) ) ) ( + ( * ( * x11336 x11337 ) ( + x11339 x11340 ) ) ( + ( + x11343 x11344 ) ( + x11346 x11347 ) ) ) ) ( * ( * ( + ( + x11352 x11353 ) ( + x11355 x11356 ) ) ( + ( + x11359 x11360 ) ( + x11362 x11363 ) ) ) ( + ( + ( + x11367 x11368 ) ( + x11370 x11371 ) ) ( * ( * x11374 x11375 ) ( * x11377 x11378 ) ) ) ) ) ) ) ( + ( * ( * ( + ( + ( + ( * x11386 x11387 ) ( * x11389 x11390 ) ) ( + ( * x11393 x11394 ) ( + x11396 x11397 ) ) ) ( + ( + ( * x11401 x11402 ) ( * x11404 x11405 ) ) ( + ( + x11408 x11409 ) ( * x11411 x11412 ) ) ) ) ( * ( + ( + ( * x11417 x11418 ) ( + x11420 x11421 ) ) ( * ( + x11424 x11425 ) ( + x11427 x11428 ) ) ) ( * ( + ( + x11432 x11433 ) ( + x11435 x11436 ) ) ( + ( + x11439 x11440 ) ( * x11442 x11443 ) ) ) ) ) ( * ( * ( + ( + ( + x11449 x11450 ) ( + x11452 x11453 ) ) ( + ( + x11456 x11457 ) ( * x11459 x11460 ) ) ) ( * ( * ( * x11464 x11465 ) ( * x11467 x11468 ) ) ( * ( * x11471 x11472 ) ( + x11474 x11475 ) ) ) ) ( + ( * ( * ( + x11480 x11481 ) ( + x11483 x11484 ) ) ( + ( + x11487 x11488 ) ( * x11490 x11491 ) ) ) ( * ( * ( * x11495 x11496 ) ( * x11498 x11499 ) ) ( * ( * x11502 x11503 ) ( * x11505 x11506 ) ) ) ) ) ) ( + ( + ( + ( * ( * ( * x11513 x11514 ) ( * x11516 x11517 ) ) ( + ( * x11520 x11521 ) ( + x11523 x11524 ) ) ) ( * ( + ( * x11528 x11529 ) ( + x11531 x11532 ) ) ( * ( * x11535 x11536 ) ( * x11538 x11539 ) ) ) ) ( * ( + ( + ( + x11544 x11545 ) ( * x11547 x11548 ) ) ( * ( * x11551 x11552 ) ( + x11554 x11555 ) ) ) ( + ( + ( * x11559 x11560 ) ( + x11562 x11563 ) ) ( + ( + x11566 x11567 ) ( + x11569 x11570 ) ) ) ) ) ( + ( * ( * ( + ( + x11576 x11577 ) ( * x11579 x11580 ) ) ( + ( * x11583 x11584 ) ( * x11586 x11587 ) ) ) ( * ( * ( + x11591 x11592 ) ( * x11594 x11595 ) ) ( * ( * x11598 x11599 ) ( + x11601 x11602 ) ) ) ) ( + ( * ( * ( * x11607 x11608 ) ( + x11610 x11611 ) ) ( + ( * x11614 x11615 ) ( + x11617 x11618 ) ) ) ( * ( * ( * x11622 x11623 ) ( * x11625 x11626 ) ) ( + ( + x11629 x11630 ) ( * x11632 x11633 ) ) ) ) ) ) ) ) ( * ( + ( + ( + ( * ( * ( + ( + x11642 x11643 ) ( * x11645 x11646 ) ) ( + ( * x11649 x11650 ) ( + x11652 x11653 ) ) ) ( + ( * ( * x11657 x11658 ) ( + x11660 x11661 ) ) ( + ( * x11664 x11665 ) ( + x11667 x11668 ) ) ) ) ( * ( * ( + ( + x11673 x11674 ) ( * x11676 x11677 ) ) ( + ( + x11680 x11681 ) ( + x11683 x11684 ) ) ) ( * ( + ( + x11688 x11689 ) ( * x11691 x11692 ) ) ( + ( * x11695 x11696 ) ( * x11698 x11699 ) ) ) ) ) ( + ( + ( + ( * ( * x11705 x11706 ) ( * x11708 x11709 ) ) ( * ( + x11712 x11713 ) ( + x11715 x11716 ) ) ) ( * ( * ( + x11720 x11721 ) ( * x11723 x11724 ) ) ( * ( + x11727 x11728 ) ( * x11730 x11731 ) ) ) ) ( * ( + ( + ( + x11736 x11737 ) ( + x11739 x11740 ) ) ( * ( * x11743 x11744 ) ( * x11746 x11747 ) ) ) ( + ( * ( * x11751 x11752 ) ( * x11754 x11755 ) ) ( + ( + x11758 x11759 ) ( + x11761 x11762 ) ) ) ) ) ) ( * ( * ( * ( + ( + ( + x11769 x11770 ) ( + x11772 x11773 ) ) ( + ( + x11776 x11777 ) ( * x11779 x11780 ) ) ) ( + ( + ( * x11784 x11785 ) ( * x11787 x11788 ) ) ( + ( * x11791 x11792 ) ( * x11794 x11795 ) ) ) ) ( + ( * ( + ( + x11800 x11801 ) ( + x11803 x11804 ) ) ( * ( * x11807 x11808 ) ( * x11810 x11811 ) ) ) ( * ( + ( * x11815 x11816 ) ( * x11818 x11819 ) ) ( + ( * x11822 x11823 ) ( * x11825 x11826 ) ) ) ) ) ( + ( * ( + ( + ( + x11832 x11833 ) ( + x11835 x11836 ) ) ( + ( * x11839 x11840 ) ( * x11842 x11843 ) ) ) ( + ( + ( * x11847 x11848 ) ( * x11850 x11851 ) ) ( + ( * x11854 x11855 ) ( * x11857 x11858 ) ) ) ) ( + ( * ( + ( + x11863 x11864 ) ( + x11866 x11867 ) ) ( + ( * x11870 x11871 ) ( * x11873 x11874 ) ) ) ( * ( + ( * x11878 x11879 ) ( * x11881 x11882 ) ) ( + ( * x11885 x11886 ) ( + x11888 x11889 ) ) ) ) ) ) ) ( * ( * ( * ( * ( + ( + ( + x11897 x11898 ) ( * x11900 x11901 ) ) ( + ( + x11904 x11905 ) ( + x11907 x11908 ) ) ) ( + ( + ( * x11912 x11913 ) ( + x11915 x11916 ) ) ( + ( * x11919 x11920 ) ( + x11922 x11923 ) ) ) ) ( + ( * ( + ( + x11928 x11929 ) ( * x11931 x11932 ) ) ( + ( * x11935 x11936 ) ( + x11938 x11939 ) ) ) ( * ( * ( * x11943 x11944 ) ( * x11946 x11947 ) ) ( + ( * x11950 x11951 ) ( + x11953 x11954 ) ) ) ) ) ( * ( + ( * ( + ( * x11960 x11961 ) ( + x11963 x11964 ) ) ( * ( + x11967 x11968 ) ( + x11970 x11971 ) ) ) ( + ( * ( + x11975 x11976 ) ( * x11978 x11979 ) ) ( * ( + x11982 x11983 ) ( * x11985 x11986 ) ) ) ) ( * ( * ( * ( + x11991 x11992 ) ( * x11994 x11995 ) ) ( + ( * x11998 x11999 ) ( + x12001 x12002 ) ) ) ( * ( + ( * x12006 x12007 ) ( + x12009 x12010 ) ) ( * ( * x12013 x12014 ) ( + x12016 x12017 ) ) ) ) ) ) ( + ( + ( + ( * ( * ( * x12024 x12025 ) ( + x12027 x12028 ) ) ( * ( * x12031 x12032 ) ( + x12034 x12035 ) ) ) ( * ( * ( * x12039 x12040 ) ( * x12042 x12043 ) ) ( * ( + x12046 x12047 ) ( * x12049 x12050 ) ) ) ) ( + ( + ( * ( * x12055 x12056 ) ( + x12058 x12059 ) ) ( + ( * x12062 x12063 ) ( * x12065 x12066 ) ) ) ( + ( + ( * x12070 x12071 ) ( * x12073 x12074 ) ) ( * ( * x12077 x12078 ) ( + x12080 x12081 ) ) ) ) ) ( * ( + ( * ( + ( * x12087 x12088 ) ( + x12090 x12091 ) ) ( * ( + x12094 x12095 ) ( * x12097 x12098 ) ) ) ( * ( * ( * x12102 x12103 ) ( + x12105 x12106 ) ) ( + ( * x12109 x12110 ) ( + x12112 x12113 ) ) ) ) ( * ( + ( * ( + x12118 x12119 ) ( * x12121 x12122 ) ) ( + ( * x12125 x12126 ) ( + x12128 x12129 ) ) ) ( * ( + ( * x12133 x12134 ) ( + x12136 x12137 ) ) ( + ( * x12140 x12141 ) ( + x12143 x12144 ) ) ) ) ) ) ) ) ) ) ):tree_100-50-10_poly -(Vec ( * ( + ( * ( * ( * x9604 x9605 ) ( * x9607 x9608 ) ) ( * ( * x9611 x9612 ) ( * x9614 x9615 ) ) ) ( * ( * ( * x9619 x9620 ) ( * x9622 x9623 ) ) ( * ( * x9626 x9627 ) ( * x9629 x9630 ) ) ) ) ( + ( * ( * ( * x9634 x9635 ) ( * x9637 x9638 ) ) ( * ( * x9641 x9642 ) ( * x9644 x9645 ) ) ) ( * ( * ( * x9649 x9650 ) ( * x9652 x9653 ) ) ( * ( * x9656 x9657 ) ( * x9659 x9660 ) ) ) ) ) ):tree_100-100-5_poly -(Vec ( + ( * ( * ( * ( * ( * ( * ( * ( * ( * x10109 x10110 ) ( * x10112 x10113 ) ) ( * ( * x10116 x10117 ) ( * x10119 x10120 ) ) ) ( * ( * ( * x10124 x10125 ) ( * x10127 x10128 ) ) ( * ( * x10131 x10132 ) ( * x10134 x10135 ) ) ) ) ( * ( * ( * ( * x10140 x10141 ) ( * x10143 x10144 ) ) ( * ( * x10147 x10148 ) ( * x10150 x10151 ) ) ) ( * ( * ( * x10155 x10156 ) ( * x10158 x10159 ) ) ( * ( * x10162 x10163 ) ( * x10165 x10166 ) ) ) ) ) ( * ( * ( * ( * ( * x10172 x10173 ) ( * x10175 x10176 ) ) ( * ( * x10179 x10180 ) ( * x10182 x10183 ) ) ) ( * ( * ( * x10187 x10188 ) ( * x10190 x10191 ) ) ( * ( * x10194 x10195 ) ( * x10197 x10198 ) ) ) ) ( * ( * ( * ( * x10203 x10204 ) ( * x10206 x10207 ) ) ( * ( * x10210 x10211 ) ( * x10213 x10214 ) ) ) ( * ( * ( * x10218 x10219 ) ( * x10221 x10222 ) ) ( * ( * x10225 x10226 ) ( * x10228 x10229 ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * x10236 x10237 ) ( * x10239 x10240 ) ) ( * ( * x10243 x10244 ) ( * x10246 x10247 ) ) ) ( * ( * ( * x10251 x10252 ) ( * x10254 x10255 ) ) ( * ( * x10258 x10259 ) ( * x10261 x10262 ) ) ) ) ( * ( * ( * ( * x10267 x10268 ) ( * x10270 x10271 ) ) ( * ( * x10274 x10275 ) ( * x10277 x10278 ) ) ) ( * ( * ( * x10282 x10283 ) ( * x10285 x10286 ) ) ( * ( * x10289 x10290 ) ( * x10292 x10293 ) ) ) ) ) ( * ( * ( * ( * ( * x10299 x10300 ) ( * x10302 x10303 ) ) ( * ( * x10306 x10307 ) ( * x10309 x10310 ) ) ) ( * ( * ( * x10314 x10315 ) ( * x10317 x10318 ) ) ( * ( * x10321 x10322 ) ( * x10324 x10325 ) ) ) ) ( * ( * ( * ( * x10330 x10331 ) ( * x10333 x10334 ) ) ( * ( * x10337 x10338 ) ( * x10340 x10341 ) ) ) ( * ( * ( * x10345 x10346 ) ( * x10348 x10349 ) ) ( * ( * x10352 x10353 ) ( * x10355 x10356 ) ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * ( * x10364 x10365 ) ( * x10367 x10368 ) ) ( * ( * x10371 x10372 ) ( * x10374 x10375 ) ) ) ( * ( * ( * x10379 x10380 ) ( * x10382 x10383 ) ) ( * ( * x10386 x10387 ) ( * x10389 x10390 ) ) ) ) ( * ( * ( * ( * x10395 x10396 ) ( * x10398 x10399 ) ) ( * ( * x10402 x10403 ) ( * x10405 x10406 ) ) ) ( * ( * ( * x10410 x10411 ) ( * x10413 x10414 ) ) ( * ( * x10417 x10418 ) ( * x10420 x10421 ) ) ) ) ) ( * ( * ( * ( * ( * x10427 x10428 ) ( * x10430 x10431 ) ) ( * ( * x10434 x10435 ) ( * x10437 x10438 ) ) ) ( * ( * ( * x10442 x10443 ) ( * x10445 x10446 ) ) ( * ( * x10449 x10450 ) ( * x10452 x10453 ) ) ) ) ( * ( * ( * ( * x10458 x10459 ) ( * x10461 x10462 ) ) ( * ( * x10465 x10466 ) ( * x10468 x10469 ) ) ) ( * ( * ( * x10473 x10474 ) ( * x10476 x10477 ) ) ( * ( * x10480 x10481 ) ( * x10483 x10484 ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * x10491 x10492 ) ( * x10494 x10495 ) ) ( * ( * x10498 x10499 ) ( * x10501 x10502 ) ) ) ( * ( * ( * x10506 x10507 ) ( * x10509 x10510 ) ) ( * ( * x10513 x10514 ) ( * x10516 x10517 ) ) ) ) ( * ( * ( * ( * x10522 x10523 ) ( * x10525 x10526 ) ) ( * ( * x10529 x10530 ) ( * x10532 x10533 ) ) ) ( * ( * ( * x10537 x10538 ) ( * x10540 x10541 ) ) ( * ( * x10544 x10545 ) ( * x10547 x10548 ) ) ) ) ) ( * ( * ( * ( * ( * x10554 x10555 ) ( * x10557 x10558 ) ) ( * ( * x10561 x10562 ) ( * x10564 x10565 ) ) ) ( * ( * ( * x10569 x10570 ) ( * x10572 x10573 ) ) ( * ( * x10576 x10577 ) ( * x10579 x10580 ) ) ) ) ( * ( * ( * ( * x10585 x10586 ) ( * x10588 x10589 ) ) ( * ( * x10592 x10593 ) ( * x10595 x10596 ) ) ) ( * ( * ( * x10600 x10601 ) ( * x10603 x10604 ) ) ( * ( * x10607 x10608 ) ( * x10610 x10611 ) ) ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * ( * ( * x10620 x10621 ) ( * x10623 x10624 ) ) ( * ( * x10627 x10628 ) ( * x10630 x10631 ) ) ) ( * ( * ( * x10635 x10636 ) ( * x10638 x10639 ) ) ( * ( * x10642 x10643 ) ( * x10645 x10646 ) ) ) ) ( * ( * ( * ( * x10651 x10652 ) ( * x10654 x10655 ) ) ( * ( * x10658 x10659 ) ( * x10661 x10662 ) ) ) ( * ( * ( * x10666 x10667 ) ( * x10669 x10670 ) ) ( * ( * x10673 x10674 ) ( * x10676 x10677 ) ) ) ) ) ( * ( * ( * ( * ( * x10683 x10684 ) ( * x10686 x10687 ) ) ( * ( * x10690 x10691 ) ( * x10693 x10694 ) ) ) ( * ( * ( * x10698 x10699 ) ( * x10701 x10702 ) ) ( * ( * x10705 x10706 ) ( * x10708 x10709 ) ) ) ) ( * ( * ( * ( * x10714 x10715 ) ( * x10717 x10718 ) ) ( * ( * x10721 x10722 ) ( * x10724 x10725 ) ) ) ( * ( * ( * x10729 x10730 ) ( * x10732 x10733 ) ) ( * ( * x10736 x10737 ) ( * x10739 x10740 ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * x10747 x10748 ) ( * x10750 x10751 ) ) ( * ( * x10754 x10755 ) ( * x10757 x10758 ) ) ) ( * ( * ( * x10762 x10763 ) ( * x10765 x10766 ) ) ( * ( * x10769 x10770 ) ( * x10772 x10773 ) ) ) ) ( * ( * ( * ( * x10778 x10779 ) ( * x10781 x10782 ) ) ( * ( * x10785 x10786 ) ( * x10788 x10789 ) ) ) ( * ( * ( * x10793 x10794 ) ( * x10796 x10797 ) ) ( * ( * x10800 x10801 ) ( * x10803 x10804 ) ) ) ) ) ( * ( * ( * ( * ( * x10810 x10811 ) ( * x10813 x10814 ) ) ( * ( * x10817 x10818 ) ( * x10820 x10821 ) ) ) ( * ( * ( * x10825 x10826 ) ( * x10828 x10829 ) ) ( * ( * x10832 x10833 ) ( * x10835 x10836 ) ) ) ) ( * ( * ( * ( * x10841 x10842 ) ( * x10844 x10845 ) ) ( * ( * x10848 x10849 ) ( * x10851 x10852 ) ) ) ( * ( * ( * x10856 x10857 ) ( * x10859 x10860 ) ) ( * ( * x10863 x10864 ) ( * x10866 x10867 ) ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * ( * x10875 x10876 ) ( * x10878 x10879 ) ) ( * ( * x10882 x10883 ) ( * x10885 x10886 ) ) ) ( * ( * ( * x10890 x10891 ) ( * x10893 x10894 ) ) ( * ( * x10897 x10898 ) ( * x10900 x10901 ) ) ) ) ( * ( * ( * ( * x10906 x10907 ) ( * x10909 x10910 ) ) ( * ( * x10913 x10914 ) ( * x10916 x10917 ) ) ) ( * ( * ( * x10921 x10922 ) ( * x10924 x10925 ) ) ( * ( * x10928 x10929 ) ( * x10931 x10932 ) ) ) ) ) ( * ( * ( * ( * ( * x10938 x10939 ) ( * x10941 x10942 ) ) ( * ( * x10945 x10946 ) ( * x10948 x10949 ) ) ) ( * ( * ( * x10953 x10954 ) ( * x10956 x10957 ) ) ( * ( * x10960 x10961 ) ( * x10963 x10964 ) ) ) ) ( * ( * ( * ( * x10969 x10970 ) ( * x10972 x10973 ) ) ( * ( * x10976 x10977 ) ( * x10979 x10980 ) ) ) ( * ( * ( * x10984 x10985 ) ( * x10987 x10988 ) ) ( * ( * x10991 x10992 ) ( * x10994 x10995 ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * x11002 x11003 ) ( * x11005 x11006 ) ) ( * ( * x11009 x11010 ) ( * x11012 x11013 ) ) ) ( * ( * ( * x11017 x11018 ) ( * x11020 x11021 ) ) ( * ( * x11024 x11025 ) ( * x11027 x11028 ) ) ) ) ( * ( * ( * ( * x11033 x11034 ) ( * x11036 x11037 ) ) ( * ( * x11040 x11041 ) ( * x11043 x11044 ) ) ) ( * ( * ( * x11048 x11049 ) ( * x11051 x11052 ) ) ( * ( * x11055 x11056 ) ( * x11058 x11059 ) ) ) ) ) ( * ( * ( * ( * ( * x11065 x11066 ) ( * x11068 x11069 ) ) ( * ( * x11072 x11073 ) ( * x11075 x11076 ) ) ) ( * ( * ( * x11080 x11081 ) ( * x11083 x11084 ) ) ( * ( * x11087 x11088 ) ( * x11090 x11091 ) ) ) ) ( * ( * ( * ( * x11096 x11097 ) ( * x11099 x11100 ) ) ( * ( * x11103 x11104 ) ( * x11106 x11107 ) ) ) ( * ( * ( * x11111 x11112 ) ( * x11114 x11115 ) ) ( * ( * x11118 x11119 ) ( * x11121 x11122 ) ) ) ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * ( * ( * ( * x11131 x11132 ) ( * x11134 x11135 ) ) ( * ( * x11138 x11139 ) ( * x11141 x11142 ) ) ) ( * ( * ( * x11146 x11147 ) ( * x11149 x11150 ) ) ( * ( * x11153 x11154 ) ( * x11156 x11157 ) ) ) ) ( * ( * ( * ( * x11162 x11163 ) ( * x11165 x11166 ) ) ( * ( * x11169 x11170 ) ( * x11172 x11173 ) ) ) ( * ( * ( * x11177 x11178 ) ( * x11180 x11181 ) ) ( * ( * x11184 x11185 ) ( * x11187 x11188 ) ) ) ) ) ( * ( * ( * ( * ( * x11194 x11195 ) ( * x11197 x11198 ) ) ( * ( * x11201 x11202 ) ( * x11204 x11205 ) ) ) ( * ( * ( * x11209 x11210 ) ( * x11212 x11213 ) ) ( * ( * x11216 x11217 ) ( * x11219 x11220 ) ) ) ) ( * ( * ( * ( * x11225 x11226 ) ( * x11228 x11229 ) ) ( * ( * x11232 x11233 ) ( * x11235 x11236 ) ) ) ( * ( * ( * x11240 x11241 ) ( * x11243 x11244 ) ) ( * ( * x11247 x11248 ) ( * x11250 x11251 ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * x11258 x11259 ) ( * x11261 x11262 ) ) ( * ( * x11265 x11266 ) ( * x11268 x11269 ) ) ) ( * ( * ( * x11273 x11274 ) ( * x11276 x11277 ) ) ( * ( * x11280 x11281 ) ( * x11283 x11284 ) ) ) ) ( * ( * ( * ( * x11289 x11290 ) ( * x11292 x11293 ) ) ( * ( * x11296 x11297 ) ( * x11299 x11300 ) ) ) ( * ( * ( * x11304 x11305 ) ( * x11307 x11308 ) ) ( * ( * x11311 x11312 ) ( * x11314 x11315 ) ) ) ) ) ( * ( * ( * ( * ( * x11321 x11322 ) ( * x11324 x11325 ) ) ( * ( * x11328 x11329 ) ( * x11331 x11332 ) ) ) ( * ( * ( * x11336 x11337 ) ( * x11339 x11340 ) ) ( * ( * x11343 x11344 ) ( * x11346 x11347 ) ) ) ) ( * ( * ( * ( * x11352 x11353 ) ( * x11355 x11356 ) ) ( * ( * x11359 x11360 ) ( * x11362 x11363 ) ) ) ( * ( * ( * x11367 x11368 ) ( * x11370 x11371 ) ) ( * ( * x11374 x11375 ) ( * x11377 x11378 ) ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * ( * x11386 x11387 ) ( * x11389 x11390 ) ) ( * ( * x11393 x11394 ) ( * x11396 x11397 ) ) ) ( * ( * ( * x11401 x11402 ) ( * x11404 x11405 ) ) ( * ( * x11408 x11409 ) ( * x11411 x11412 ) ) ) ) ( * ( * ( * ( * x11417 x11418 ) ( * x11420 x11421 ) ) ( * ( * x11424 x11425 ) ( * x11427 x11428 ) ) ) ( * ( * ( * x11432 x11433 ) ( * x11435 x11436 ) ) ( * ( * x11439 x11440 ) ( * x11442 x11443 ) ) ) ) ) ( * ( * ( * ( * ( * x11449 x11450 ) ( * x11452 x11453 ) ) ( * ( * x11456 x11457 ) ( * x11459 x11460 ) ) ) ( * ( * ( * x11464 x11465 ) ( * x11467 x11468 ) ) ( * ( * x11471 x11472 ) ( * x11474 x11475 ) ) ) ) ( * ( * ( * ( * x11480 x11481 ) ( * x11483 x11484 ) ) ( * ( * x11487 x11488 ) ( * x11490 x11491 ) ) ) ( * ( * ( * x11495 x11496 ) ( * x11498 x11499 ) ) ( * ( * x11502 x11503 ) ( * x11505 x11506 ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * x11513 x11514 ) ( * x11516 x11517 ) ) ( * ( * x11520 x11521 ) ( * x11523 x11524 ) ) ) ( * ( * ( * x11528 x11529 ) ( * x11531 x11532 ) ) ( * ( * x11535 x11536 ) ( * x11538 x11539 ) ) ) ) ( * ( * ( * ( * x11544 x11545 ) ( * x11547 x11548 ) ) ( * ( * x11551 x11552 ) ( * x11554 x11555 ) ) ) ( * ( * ( * x11559 x11560 ) ( * x11562 x11563 ) ) ( * ( * x11566 x11567 ) ( * x11569 x11570 ) ) ) ) ) ( * ( * ( * ( * ( * x11576 x11577 ) ( * x11579 x11580 ) ) ( * ( * x11583 x11584 ) ( * x11586 x11587 ) ) ) ( * ( * ( * x11591 x11592 ) ( * x11594 x11595 ) ) ( * ( * x11598 x11599 ) ( * x11601 x11602 ) ) ) ) ( * ( * ( * ( * x11607 x11608 ) ( * x11610 x11611 ) ) ( * ( * x11614 x11615 ) ( * x11617 x11618 ) ) ) ( * ( * ( * x11622 x11623 ) ( * x11625 x11626 ) ) ( * ( * x11629 x11630 ) ( * x11632 x11633 ) ) ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * ( * ( * x11642 x11643 ) ( * x11645 x11646 ) ) ( * ( * x11649 x11650 ) ( * x11652 x11653 ) ) ) ( * ( * ( * x11657 x11658 ) ( * x11660 x11661 ) ) ( * ( * x11664 x11665 ) ( * x11667 x11668 ) ) ) ) ( * ( * ( * ( * x11673 x11674 ) ( * x11676 x11677 ) ) ( * ( * x11680 x11681 ) ( * x11683 x11684 ) ) ) ( * ( * ( * x11688 x11689 ) ( * x11691 x11692 ) ) ( * ( * x11695 x11696 ) ( * x11698 x11699 ) ) ) ) ) ( * ( * ( * ( * ( * x11705 x11706 ) ( * x11708 x11709 ) ) ( * ( * x11712 x11713 ) ( * x11715 x11716 ) ) ) ( * ( * ( * x11720 x11721 ) ( * x11723 x11724 ) ) ( * ( * x11727 x11728 ) ( * x11730 x11731 ) ) ) ) ( * ( * ( * ( * x11736 x11737 ) ( * x11739 x11740 ) ) ( * ( * x11743 x11744 ) ( * x11746 x11747 ) ) ) ( * ( * ( * x11751 x11752 ) ( * x11754 x11755 ) ) ( * ( * x11758 x11759 ) ( * x11761 x11762 ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * x11769 x11770 ) ( * x11772 x11773 ) ) ( * ( * x11776 x11777 ) ( * x11779 x11780 ) ) ) ( * ( * ( * x11784 x11785 ) ( * x11787 x11788 ) ) ( * ( * x11791 x11792 ) ( * x11794 x11795 ) ) ) ) ( * ( * ( * ( * x11800 x11801 ) ( * x11803 x11804 ) ) ( * ( * x11807 x11808 ) ( * x11810 x11811 ) ) ) ( * ( * ( * x11815 x11816 ) ( * x11818 x11819 ) ) ( * ( * x11822 x11823 ) ( * x11825 x11826 ) ) ) ) ) ( * ( * ( * ( * ( * x11832 x11833 ) ( * x11835 x11836 ) ) ( * ( * x11839 x11840 ) ( * x11842 x11843 ) ) ) ( * ( * ( * x11847 x11848 ) ( * x11850 x11851 ) ) ( * ( * x11854 x11855 ) ( * x11857 x11858 ) ) ) ) ( * ( * ( * ( * x11863 x11864 ) ( * x11866 x11867 ) ) ( * ( * x11870 x11871 ) ( * x11873 x11874 ) ) ) ( * ( * ( * x11878 x11879 ) ( * x11881 x11882 ) ) ( * ( * x11885 x11886 ) ( * x11888 x11889 ) ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * ( * x11897 x11898 ) ( * x11900 x11901 ) ) ( * ( * x11904 x11905 ) ( * x11907 x11908 ) ) ) ( * ( * ( * x11912 x11913 ) ( * x11915 x11916 ) ) ( * ( * x11919 x11920 ) ( * x11922 x11923 ) ) ) ) ( * ( * ( * ( * x11928 x11929 ) ( * x11931 x11932 ) ) ( * ( * x11935 x11936 ) ( * x11938 x11939 ) ) ) ( * ( * ( * x11943 x11944 ) ( * x11946 x11947 ) ) ( * ( * x11950 x11951 ) ( * x11953 x11954 ) ) ) ) ) ( * ( * ( * ( * ( * x11960 x11961 ) ( * x11963 x11964 ) ) ( * ( * x11967 x11968 ) ( * x11970 x11971 ) ) ) ( * ( * ( * x11975 x11976 ) ( * x11978 x11979 ) ) ( * ( * x11982 x11983 ) ( * x11985 x11986 ) ) ) ) ( * ( * ( * ( * x11991 x11992 ) ( * x11994 x11995 ) ) ( * ( * x11998 x11999 ) ( * x12001 x12002 ) ) ) ( * ( * ( * x12006 x12007 ) ( * x12009 x12010 ) ) ( * ( * x12013 x12014 ) ( * x12016 x12017 ) ) ) ) ) ) ( * ( * ( * ( * ( * ( * x12024 x12025 ) ( * x12027 x12028 ) ) ( * ( * x12031 x12032 ) ( * x12034 x12035 ) ) ) ( * ( * ( * x12039 x12040 ) ( * x12042 x12043 ) ) ( * ( * x12046 x12047 ) ( * x12049 x12050 ) ) ) ) ( * ( * ( * ( * x12055 x12056 ) ( * x12058 x12059 ) ) ( * ( * x12062 x12063 ) ( * x12065 x12066 ) ) ) ( * ( * ( * x12070 x12071 ) ( * x12073 x12074 ) ) ( * ( * x12077 x12078 ) ( * x12080 x12081 ) ) ) ) ) ( * ( * ( * ( * ( * x12087 x12088 ) ( * x12090 x12091 ) ) ( * ( * x12094 x12095 ) ( * x12097 x12098 ) ) ) ( * ( * ( * x12102 x12103 ) ( * x12105 x12106 ) ) ( * ( * x12109 x12110 ) ( * x12112 x12113 ) ) ) ) ( * ( * ( * ( * x12118 x12119 ) ( * x12121 x12122 ) ) ( * ( * x12125 x12126 ) ( * x12128 x12129 ) ) ) ( * ( * ( * x12133 x12134 ) ( * x12136 x12137 ) ) ( * ( * x12140 x12141 ) ( * x12143 x12144 ) ) ) ) ) ) ) ) ) ) ):tree_100-100-10_poly \ No newline at end of file +(Vec ( + ( + ( * v1_0 v2_0 ) ( * v1_1 v2_1 ) ) ( + ( * v1_2 v2_2 ) ( * v1_3 v2_3 ) ) ) ):dot_product_4 \ No newline at end of file diff --git a/RL/fhe_rl/datasets/train_rotations.txt b/RL/fhe_rl/datasets/train_rotations.txt new file mode 100644 index 00000000..5d1cb755 --- /dev/null +++ b/RL/fhe_rl/datasets/train_rotations.txt @@ -0,0 +1 @@ +(Vec ( + ( + ( + ( - ( + v1_0 v2_0 ) ( * ( * v1_0 v2_0 ) 2 ) ) ( - ( + v1_1 v2_1 ) ( * 2 ( * v1_1 v2_1 ) ) ) ) ( + ( - ( + v1_2 v2_2 ) ( * 2 ( * v1_2 v2_2 ) ) ) ( - ( + v1_3 v2_3 ) ( * 2 ( * v1_3 v2_3 ) ) ) ) ) ( + ( + ( - ( + v1_4 v2_4 ) ( * 2 ( * v1_4 v2_4 ) ) ) ( - ( + v1_5 v2_5 ) ( * 2 ( * v1_5 v2_5 ) ) ) ) ( + ( - ( + v1_6 v2_6 ) ( * 2 ( * v1_6 v2_6 ) ) ) ( - ( + v1_7 v2_7 ) ( * 2 ( * v1_7 v2_7 ) ) ) ) ) ) ):hamming_dist_8 \ No newline at end of file diff --git a/RL/fhe_rl/env.py b/RL/fhe_rl/env.py index a95c527c..b48f3f37 100644 --- a/RL/fhe_rl/env.py +++ b/RL/fhe_rl/env.py @@ -29,7 +29,9 @@ class fheEnv(gym.Env): - def __init__(self, rules_list, expressions, max_positions=2,embeddings_model=None): + + def __init__(self, rules_list, expressions, max_positions=2,embeddings_model=None, pref_list=[[1.0, 0.0], [0.0, 1.0]], lambda_env=0.0, lambda_kl=0.0, + n_cycle=1, n_budget=5, env_idx=0): super().__init__() self.rules = rules_list @@ -39,20 +41,74 @@ def __init__(self, rules_list, expressions, max_positions=2,embeddings_model=Non self.max_steps = 75 self.max_expression_size = 10000 self.initial_cost = 0 + + self.embedding_dim = 256 self.initial_vectorization_potential = 0 self.vectorizations_applied = 0 self.vectorization_helper = 0 + + self.n_cycle = n_cycle + self.episode_count = 0 + + self.pref_list = [np.array(p, dtype=np.float32) for p in pref_list] + self.current_pref_idx = env_idx % len(self.pref_list) + self.current_w = self.pref_list[self.current_pref_idx] + + self.n_budget = n_budget + self.lambda_kl = lambda_kl + self.lambda_env = lambda_env + + self.pref_locked = False + self.embeded_expression = None + self.random_pref = True + self.action_space = spaces.Discrete(len(self.rules.keys()) * self.max_positions) self.observation_space = spaces.Dict({ "observation": spaces.Box( - low=-np.inf, high=np.inf, shape=(self.embedding_dim,), dtype=np.float32 + low=-np.inf, high=np.inf, shape=(258,), dtype=np.float32 ), "action_mask": spaces.Box(0, 1, (len(self.rules.keys()) * self.max_positions,), np.float32) }) - self.reset() + + # self.reset() - + def set_preference_vector(self, w): + """Set the preference vector for multi-objective optimization""" + self.current_w = np.array(w, dtype=np.float32) + self.pref_locked = True + + def unlock_preferences(self): + """CHANGE: Method to restore automatic cycling behavior""" + self.pref_locked = False + + def get_split_costs(self, expr_str): + parsed = parse_sexpr(expr_str) + ops = calculate_cost(parsed, w_keys=0.0) + keys = calculate_cost(parsed, w_keys=1.0) - ops + return ops, keys + + + def _sample_preference(self) -> tuple[np.ndarray, str]: + """ + Implements Algorithm 2 biased preference sampling: + episode_count mod (n_cycle + 1) < n_cycle → speed-focus [1, 0] + otherwise → random from Ω + When n_cycle == 0 every episode is a random exploration episode. + """ + if self.n_cycle > 0 and (self.episode_count % (self.n_cycle + 1)) < self.n_cycle: + # Speed-focus episode + w = self.pref_list[0] # [1.0, 0.0] by convention + mode_name = "FIXED SPEED FOCUS" + else: + # Random exploration episode + idx = self.np_random.integers(0, len(self.pref_list)) + w = self.pref_list[idx] + mode_name = "RANDOM SEARCH" + + self.episode_count += 1 + return w, mode_name + def reset(self, seed=None, options=None): super().reset(seed=seed) @@ -62,14 +118,40 @@ def reset(self, seed=None, options=None): self.current_index = (self.current_index + 1) % len(self.expressions) self.initial_expression = self.expression self.steps = 0 + self.initial_cost = self.current_cost = self.get_cost(self.expression) + + self.initial_ops, self.initial_keys = self.get_split_costs(self.expression) + self.curr_ops, self.curr_keys = self.get_split_costs(self.expression) + + + + if self.pref_locked: + mode_name = "LOCKED (EVALUATION)" + else: + self.current_w, mode_name = self._sample_preference() + + print(f"\n{BLUE}{'='*40} EPISODE START {'='*40}{RESET}") + print(f"{BOLD}{MAGENTA}Optimization Mode{RESET} : {CYAN}{mode_name}{RESET}") + + self.embeded_expression = self._embed_expression(self.expression) + return self._get_obs(), {} + + def _get_obs(self): + if self.embeded_expression is None: + emb = np.zeros(self.embedding_dim, dtype=np.float32) + else: + emb = self.embeded_expression + w = np.atleast_1d(self.current_w).astype(np.float32) + obs = np.concatenate([emb, w]).astype(np.float32) return { - "observation": self._embed_expression(self.expression), + "observation": obs, "action_mask": self.get_action_mask() - }, {} + } - def step(self, action: int): + if self.current_w[0] == 0.0 and self.current_w[1] == 1.0: + action = list(self.rules.keys()).index("END") * self.max_positions self.steps += 1 rule_idx = action // self.max_positions pos_idx = action % self.max_positions @@ -79,7 +161,8 @@ def step(self, action: int): reward = 0 print(f"\n{CYAN}{'-'*100}{RESET}") print(f"{BOLD}{MAGENTA}Old expression{RESET}: {YELLOW}{self.expression}{RESET}") - print(f"{BOLD}{MAGENTA}Old cost {RESET}: {RED}{self.current_cost}{RESET}") + print(f"{BOLD}{MAGENTA}Old exec cost {RESET}: {RED}{self.curr_ops}{RESET}") + print(f"{BOLD}{MAGENTA}Old keys cost {RESET}: {RED}{self.curr_keys}{RESET}") if rule_name == "END": terminated = True @@ -94,21 +177,29 @@ def step(self, action: int): temp = expr_to_str(new_expr_tree) self.expression = temp new_cost = self.get_cost(self.expression) - reward = self.calculate_intermediate_reward(new_cost) - self.current_cost = new_cost + + + new_ops, new_keys = self.get_split_costs(self.expression) + reward = self.calculate_intermediate_reward(new_ops, new_keys) + + self.curr_ops = new_ops + self.curr_keys = new_keys + + self.current_cost = new_cost if (self.steps >= self.max_steps): terminated = True reward = self.calculate_final_reward() info = {"expression": self.expression} reward_color = GREEN if reward >= 0 else RED print(f"{BOLD}{MAGENTA}New expression{RESET}: {YELLOW}{self.expression}{RESET}") - print(f"{BOLD}{MAGENTA}New cost {RESET}: {RED}{self.current_cost}{RESET}") + print(f"{BOLD}{MAGENTA}New exec cost {RESET}: {RED}{self.curr_ops}{RESET}") + print(f"{BOLD}{MAGENTA}New keys cost {RESET}: {RED}{self.curr_keys}{RESET}") print(f"{BOLD}{MAGENTA}Reward {RESET}: {reward_color}{reward}{RESET}") print(f"{BOLD}{MAGENTA}Rule name {RESET}: {CYAN}{rule_name}{RESET}") print(f"{BOLD}{MAGENTA}At position {RESET}: {BLUE}{pos_idx}{RESET}") print(f"{CYAN}{'-'*100}{RESET}") - embedding = self._embed_expression(self.expression) - if embedding is None: + self.embeded_expression = self._embed_expression(self.expression) + if self.embeded_expression is None: terminated = True truncated = True reward = self.calculate_final_reward() @@ -120,10 +211,18 @@ def step(self, action: int): "l": self.steps, "t": None } - return { - "observation": embedding, - "action_mask": self.get_action_mask() - }, reward, terminated, truncated, {"expression": self.expression} + + if self.curr_ops > self.initial_ops: + self.expression = self.initial_expression + info["expression"] = self.initial_expression + self.curr_ops = self.initial_ops + + print(f"\n{BOLD}{MAGENTA}Initial exec cost {RESET}: {YELLOW}{self.initial_ops}{RESET}") + print(f"{BOLD}{MAGENTA}Initial keys cost {RESET}: {YELLOW}{self.initial_keys}{RESET}") + print(f"{BOLD}{MAGENTA}Final exec cost {RESET}: {GREEN}{self.curr_ops}{RESET}") + print(f"{BOLD}{MAGENTA}Final keys cost {RESET}: {GREEN}{self.curr_keys}{RESET}") + obs = self._get_obs() + return obs, reward, terminated, truncated, {"expression": self.expression} def _valid_end_action(self,expr: str) -> bool: expr_tree = parse_sexpr(expr) @@ -152,14 +251,51 @@ def _valid_end_action(self,expr: str) -> bool: break return isValid + def _reward_vector(self, delta_ops_old, delta_ops_new, + delta_keys_old, delta_keys_new) -> np.ndarray: + """Compute the 2-D reward vector [r_ops, r_keys].""" + r_ops = (delta_ops_old - delta_ops_new) / delta_ops_old if delta_ops_old != 0 else 0.0 + r_keys = (delta_keys_old - delta_keys_new) / self.n_budget + return np.array([r_ops, r_keys], dtype=np.float32) + + def _kl_bonus(self, r_vec: np.ndarray) -> float: + """ + KL divergence of the preference-weighted improvement distribution + from a Uniform(Ω) baseline. + + """ + r_pos = np.maximum(r_vec, 0.0) + 1e-6 + r_tilde = r_pos / np.sum(r_pos) + uniform = np.full_like(r_tilde, 1.0 / len(r_tilde)) + return float(np.sum(r_tilde * np.log((r_tilde + 1e-6) / uniform))) + + def _pareto_envelope_bonus(self, r_vec: np.ndarray) -> float: + """max_{w ∈ Ω} w · r⃗ — reward under the best-fitting preference.""" + return float(max(np.dot(p, r_vec) for p in self.pref_list)) + + def _compose_reward(self, r_vec: np.ndarray) -> float: + """ + Total reward = linear term + optional bonuses. + R = w⃗ · r⃗ + λ_env · max_w w · r⃗ + λ_kl · KL(w̃ ‖ Uniform) + """ + reward = float(np.dot(self.current_w, r_vec)) + if self.lambda_env != 0.0: + reward += self.lambda_env * self._pareto_envelope_bonus(r_vec) + if self.lambda_kl != 0.0: + reward += self.lambda_kl * self._kl_bonus(r_vec) + return reward + + + + def calculate_intermediate_reward(self, new_ops: float, new_keys: float) -> float: + r_vec = self._reward_vector(self.curr_ops, new_ops, self.curr_keys, new_keys) + return self._compose_reward(r_vec) + def calculate_final_reward(self) -> float: - if self.initial_cost == 0: - return 0.0 - return (self.initial_cost - self.current_cost) / self.initial_cost * 100 - def calculate_intermediate_reward(self,new_cost) -> float: - if self.current_cost == 0: - return 0.0 - return ( ( self.current_cost - new_cost) / self.current_cost ) + r_vec = self._reward_vector(self.initial_ops, self.curr_ops, + self.initial_keys, self.curr_keys) + return self._compose_reward(r_vec) * 100 + def get_cost(self, expr: str) -> float: return calculate_cost(parse_sexpr(expr)) diff --git a/RL/fhe_rl/morl/__init__.py b/RL/fhe_rl/morl/__init__.py new file mode 100644 index 00000000..352262c2 --- /dev/null +++ b/RL/fhe_rl/morl/__init__.py @@ -0,0 +1,17 @@ +""" +fhe_rl/interactive/ +------------------- +Sub-package for CHEHAB-MORL interactive inference modes. + +Modules +------- +display.py — terminal colours, Pareto table, scatter plot +pareto_frontier.py — dominance filtering, get_pareto_frontier +benchmark_runner.py — run_one_point, execute_solution, benchmark discovery +bisection.py — coarse-grid builder, recursive bisection search +interactive.py — prompt helpers, Direct Mode, Menu Mode, CLI entry point +""" + +from .morl import run_interactive, add_subparser + +__all__ = ["run_interactive", "add_subparser"] \ No newline at end of file diff --git a/RL/fhe_rl/morl/benchmark_runner.py b/RL/fhe_rl/morl/benchmark_runner.py new file mode 100644 index 00000000..e7eb24a9 --- /dev/null +++ b/RL/fhe_rl/morl/benchmark_runner.py @@ -0,0 +1,274 @@ +""" +benchmark_runner.py — run compiled FHE benchmark binaries and manage solutions. + +Key design +---------- +- run_one_point() : runs the binary once, streams output live, parses the + two costs, then snapshots he/_gen_he_fhe.{cpp,hpp} + into a solutions/ cache keyed by (benchmark, slot, w_ops). +- save_solution_to_he() : copies the chosen cached .cpp/.hpp back into he/ — + no rebuild, no execution. +- cleanup_solutions_cache(): removes the entire solutions/ cache directory. + +Polynomials-coyote special case +--------------------------------- +The binary has a different argument order: + ./ + 1 1 +Pass extra_args={"tree_depth":…, "instance":…, "regime":…} to both functions. +""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +from typing import Optional + +from .display import _ok, _warn, _err + +# ── config ───────────────────────────────────────────────────────────────────── +_HERE = os.path.dirname(os.path.abspath(__file__)) +BUILD_FOLDER = os.path.join(_HERE, "..", "..", "..", "build", "benchmarks") +SOLUTIONS_FOLDER = "solutions" # root cache dir for saved circuits +COMPILE_TIMEOUT_SECONDS = 7200 +VECTORIZE_CODE = 1 +OPTIMIZATION_METHOD = 1 # 1 = RL +WINDOW_SIZE = 0 +CSE_ENABLED = 1 + +POLY_BENCHMARK = "polynomials_coyote" + + +# ── discovery ────────────────────────────────────────────────────────────────── + +def discover_benchmarks() -> list[str]: + """Return all sub-folder names found under BUILD_FOLDER.""" + if not os.path.isdir(BUILD_FOLDER): + return [] + return sorted( + d for d in os.listdir(BUILD_FOLDER) + if os.path.isdir(os.path.join(BUILD_FOLDER, d)) + ) + + +def is_poly_benchmark(benchmark: str) -> bool: + return benchmark == POLY_BENCHMARK + + +# ── command builders ─────────────────────────────────────────────────────────── + +def _build_cmd(benchmark: str, slot_count: int, w_ops: float, + extra_args: Optional[dict] = None) -> str: + """ + Build the shell command string for the benchmark binary. + + Regular benchmarks: + ./ 1 1 + + polynomials_coyote: + ./ + 1 1 + extra_args must contain: tree_depth, instance, regime + """ + w_keys = round(1.0 - w_ops, 10) + if is_poly_benchmark(benchmark): + ea = extra_args or {} + tree_depth = ea.get("tree_depth", 5) + instance = ea.get("instance", 1) + regime = ea.get("regime", "100-50") + return ( + f"./{benchmark} {tree_depth} {instance} {regime} " + f"{VECTORIZE_CODE} {OPTIMIZATION_METHOD} {WINDOW_SIZE} " + f"1 {CSE_ENABLED} 1 {w_ops} {w_keys}" + ) + else: + return ( + f"./{benchmark} {VECTORIZE_CODE} {slot_count} " + f"{OPTIMIZATION_METHOD} {WINDOW_SIZE} 1 {CSE_ENABLED} 1 " + f"{w_ops} {w_keys}" + ) + + +# ── solution cache helpers ───────────────────────────────────────────────────── + +def _solution_dir(benchmark: str, slot_count: int, w_ops: float, + extra_args: Optional[dict] = None) -> str: + """ + Return the directory where the generated HE files for one point are stored. + + Regular : solutions///w_/ + Poly : solutions/////w_/ + """ + w_tag = f"w_{w_ops:.6f}" + if is_poly_benchmark(benchmark) and extra_args: + regime = extra_args.get("regime", "100-50") + tree_depth = extra_args.get("tree_depth", 5) + instance = extra_args.get("instance", 1) + return os.path.join( + SOLUTIONS_FOLDER, benchmark, + str(regime), str(tree_depth), str(instance), w_tag, + ) + return os.path.join(SOLUTIONS_FOLDER, benchmark, str(slot_count), w_tag) + + +def _save_solution(build_path_he: str, sol_dir: str) -> bool: + """ + Copy _gen_he_fhe.cpp and _gen_he_fhe.hpp from he/ into sol_dir. + Returns True on success. + """ + os.makedirs(sol_dir, exist_ok=True) + saved = False + for ext in ("cpp", "hpp"): + src = os.path.join(build_path_he, f"_gen_he_fhe.{ext}") + dst = os.path.join(sol_dir, f"_gen_he_fhe.{ext}") + if os.path.isfile(src): + shutil.copy2(src, dst) + saved = True + else: + print(_warn(f" [save] {src} not found — skipping")) + return saved + + +def _restore_solution(sol_dir: str, build_path_he: str) -> bool: + """ + Copy the cached .cpp/.hpp back into he/ so the HE binary can be rebuilt. + Returns True if both files were restored. + """ + ok = True + for ext in ("cpp", "hpp"): + src = os.path.join(sol_dir, f"_gen_he_fhe.{ext}") + dst = os.path.join(build_path_he, f"_gen_he_fhe.{ext}") + if os.path.isfile(src): + shutil.copy2(src, dst) + else: + print(_err(f" [restore] {src} not found")) + ok = False + return ok + + +# ── core runner ──────────────────────────────────────────────────────────────── + +def run_one_point( + benchmark: str, + slot_count: int, + w_ops: float, + extra_args: Optional[dict] = None, +) -> Optional[dict]: + """ + Run the compiled benchmark binary once. + Streams every output line to the terminal and parses final ops/keys costs. + After a successful run the generated he/_gen_he_fhe.{cpp,hpp} are copied + into the solutions cache so execute_solution() can reuse them without + re-optimising. + """ + w_keys = round(1.0 - w_ops, 10) + build_path = os.path.join(BUILD_FOLDER, benchmark) + build_path_he = os.path.join(build_path, "he") + + # optional source generation (regular benchmarks only) + if not is_poly_benchmark(benchmark): + gen_script = os.path.join(build_path, f"generate_{benchmark}.py") + if os.path.isfile(gen_script): + proc = subprocess.Popen( + ["python3", gen_script, "--slot_count", str(slot_count)], + cwd=build_path, + ) + proc.wait() + + cmd = _build_cmd(benchmark, slot_count, w_ops, extra_args) + + final_ops_cost: Optional[float] = None + final_keys_cost: Optional[float] = None + + try: + proc = subprocess.Popen( + cmd, shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + universal_newlines=True, cwd=build_path, + ) + for line in proc.stdout: + print(line, end="", flush=True) + clean = re.sub(r'\x1b\[[0-9;]*m', '', line) + clean_lower = clean.lower() + if 'final exec cost' in clean_lower: + try: + final_ops_cost = float(clean.split(':')[1].strip()) + except (IndexError, ValueError): + pass + if 'final keys cost' in clean_lower: + try: + final_keys_cost = float(clean.split(':')[1].strip()) + except (IndexError, ValueError): + pass + + proc.wait(timeout=COMPILE_TIMEOUT_SECONDS) + + except subprocess.TimeoutExpired: + proc.kill() + print(_err(f"\n Timed out after {COMPILE_TIMEOUT_SECONDS}s")) + return None + except Exception as e: + print(_err(f"\n Error: {e}")) + return None + + if final_ops_cost is None or final_keys_cost is None: + print(_err(" Could not parse costs from output.")) + return None + + # ── snapshot the generated HE files ─────────────────────────────────────── + sol_dir = _solution_dir(benchmark, slot_count, w_ops, extra_args) + if _save_solution(build_path_he, sol_dir): + print(_ok(f" [saved] circuit → {sol_dir}")) + else: + print(_warn(" [save] no generated files found to cache")) + + return { + "benchmark": benchmark, + "slot_count": slot_count, + "w_ops": w_ops, + "w_keys": w_keys, + "final_exec_cost": final_ops_cost, + "final_keys_cost": final_keys_cost, + "sol_dir": sol_dir, # carry the path for execute_solution + "extra_args": extra_args, + } + + +# ── save chosen solution back into he/ ──────────────────────────────────────── + +def save_solution_to_he(point: dict) -> None: + """ + Copy the cached _gen_he_fhe.{cpp,hpp} for the chosen solution back into + build/benchmarks//he/ — no rebuild, no execution. + """ + benchmark = point["benchmark"] + slot_count = point["slot_count"] + w_ops = point["w_ops"] + w_keys = point["w_keys"] + extra_args = point.get("extra_args") + sol_dir = point.get("sol_dir") or _solution_dir( + benchmark, slot_count, w_ops, extra_args + ) + + build_path_he = os.path.join(BUILD_FOLDER, benchmark, "he") + + print() + print(f" Saving circuit from {sol_dir}") + print(f" into {build_path_he}") + + if not _restore_solution(sol_dir, build_path_he): + print(_err(" Cached solution files missing — cannot save.")) + return + + print(_ok(f" ✓ _gen_he_fhe.cpp/.hpp written to {build_path_he}")) + + +# ── cache cleanup ────────────────────────────────────────────────────────────── + +def cleanup_solutions_cache() -> None: + """Remove the entire solutions/ cache directory.""" + if os.path.isdir(SOLUTIONS_FOLDER): + shutil.rmtree(SOLUTIONS_FOLDER) + print(_ok(f" [cache] {SOLUTIONS_FOLDER}/ removed.")) \ No newline at end of file diff --git a/RL/fhe_rl/morl/bisection.py b/RL/fhe_rl/morl/bisection.py new file mode 100644 index 00000000..82cb20fe --- /dev/null +++ b/RL/fhe_rl/morl/bisection.py @@ -0,0 +1,109 @@ +""" +bisection.py — coarse-grid generation and recursive bisection search for +hidden Pareto trade-offs (Algorithm 3 in the thesis). +""" + +from __future__ import annotations +from typing import Optional + +from .benchmark_runner import run_one_point +from .display import _warn, _err + +DEFAULT_MAX_BISECT_DEPTH = 3 +_COARSE_STEP = 0.1 + + +# ── coarse grid ──────────────────────────────────────────────────────────────── + +def build_coarse_grid(w_lo: float, w_hi: float) -> list[float]: + """ + Build the evaluation grid within [w_lo, w_hi]: + - Always includes w_lo and w_hi. + - If the range is wider than 0.1, interior multiples of 0.1 are added + (e.g. 0.72-0.81 -> [0.72, 0.80, 0.81]). + - Narrow ranges (<= 0.1) evaluate only the two endpoints + (e.g. 0.72-0.79 -> [0.72, 0.79]). + """ + points = {round(w_lo, 10), round(w_hi, 10)} + if w_hi - w_lo > _COARSE_STEP + 1e-9: + cursor = round((w_lo // _COARSE_STEP + 1) * _COARSE_STEP, 2) + while cursor < w_hi - 1e-9: + points.add(round(cursor, 10)) + cursor = round(cursor + _COARSE_STEP, 2) + return sorted(points) + + +# ── recursive bisection ──────────────────────────────────────────────────────── + +def _bisect( + benchmark: str, + slot_count: int, + w_hi: float, + ks_hi: int, + w_lo: float, + ks_lo: int, + target: int, + found: set, + all_results: list[dict], + max_depth: int, + extra_args: Optional[dict] = None, + depth: int = 0, +) -> None: + if depth >= max_depth or target in found: + return + + w_mid = round((w_hi + w_lo) / 2, 10) + print(f" {' ' * depth}[bisect d={depth}] w_mid={w_mid:.6f} target_keys={target}") + + result = run_one_point(benchmark, slot_count, w_mid, extra_args) + if result is None: + return + + all_results.append(result) + ks_mid = (int(result["final_keys_cost"]) + if result["final_keys_cost"] is not None else None) + if ks_mid is not None: + found.add(ks_mid) + + print(f" {' ' * depth} -> ops={result['final_exec_cost']:.1f} " + f"keys={result['final_keys_cost']}") + + if ks_mid is None or ks_mid == target: + return + + if ks_mid < target: + _bisect(benchmark, slot_count, w_hi, ks_hi, w_mid, ks_mid, + target, found, all_results, max_depth, extra_args, depth + 1) + else: + _bisect(benchmark, slot_count, w_mid, ks_mid, w_lo, ks_lo, + target, found, all_results, max_depth, extra_args, depth + 1) + + +def bisect_interval( + benchmark: str, + slot_count: int, + w_hi: float, + ks_hi: int, + w_lo: float, + ks_lo: int, + all_results: list[dict], + max_depth: int, + extra_args: Optional[dict] = None, +) -> None: + """ + Search independently for every missing integer key-size level between + ks_lo and ks_hi. Each target gets its own max_depth budget. + """ + lo, hi = min(ks_lo, ks_hi), max(ks_lo, ks_hi) + targets = list(range(lo + 1, hi)) + if not targets: + return + + found: set = set() + for t in targets: + if t in found: + print(f" [skip] target={t} already found by a previous probe") + continue + print(_warn(f"\n searching for keys={t} in w_ops in [{w_lo:.4f}, {w_hi:.4f}]")) + _bisect(benchmark, slot_count, w_hi, ks_hi, w_lo, ks_lo, + t, found, all_results, max_depth, extra_args) \ No newline at end of file diff --git a/RL/fhe_rl/morl/display.py b/RL/fhe_rl/morl/display.py new file mode 100644 index 00000000..bb304ffc --- /dev/null +++ b/RL/fhe_rl/morl/display.py @@ -0,0 +1,88 @@ +""" +display.py — terminal colour helpers, Pareto table, and scatter plot. +""" + +from __future__ import annotations + +try: + import matplotlib.pyplot as plt + import matplotlib.ticker as mticker + _HAS_MPL = True +except ImportError: + _HAS_MPL = False + +# ── ANSI colours ─────────────────────────────────────────────────────────────── +RESET = "\033[0m" +BOLD = "\033[1m" +RED = "\033[31m" +GREEN = "\033[32m" +YELLOW = "\033[33m" +BLUE = "\033[34m" +MAGENTA = "\033[35m" +CYAN = "\033[36m" + + +def _h(text: str) -> str: return f"{BOLD}{CYAN}{text}{RESET}" +def _ok(text: str) -> str: return f"{GREEN}{text}{RESET}" +def _warn(text: str) -> str: return f"{YELLOW}{text}{RESET}" +def _err(text: str) -> str: return f"{RED}{text}{RESET}" + + +# ── table ────────────────────────────────────────────────────────────────────── + +def display_pareto_table(front: list[dict]) -> None: + """Pretty-print the Pareto frontier as a numbered table.""" + print() + print(_h( + f"{'#':>3} {'w_exec':>7} {'w_keys':>7} " + f"{'Exec cost':>10} {'Keys cost':>10}" + )) + print(CYAN + "─" * 45 + RESET) + for i, p in enumerate(front, 1): + print( + f"{BOLD}{i:>3}{RESET} " + f"{p['w_ops']:>7.4f} " + f"{p['w_keys']:>7.4f} " + f"{p['final_exec_cost']:>10.1f} " + f"{p['final_keys_cost']:>10.1f}" + ) + print() + + +# ── plot ─────────────────────────────────────────────────────────────────────── + +def display_pareto_plot(front: list[dict], benchmark: str, slot_count: int) -> None: + """Normalised scatter plot of the Pareto frontier.""" + if not _HAS_MPL: + print(_warn(" matplotlib not installed — skipping plot")) + return + if not front: + return + + ops_vals = [p["final_exec_cost"] for p in front] + keys_vals = [p["final_keys_cost"] for p in front] + + ops_n = ops_vals + keys_n = keys_vals + + fig, ax = plt.subplots(figsize=(7, 5)) + ax.scatter(ops_n, keys_n, color="steelblue", zorder=5, s=80) + + for i, (xv, yv) in enumerate(zip(ops_n, keys_n), 1): + ax.annotate(str(i), (xv, yv), + textcoords="offset points", xytext=(6, 4), + fontsize=9, color="navy") + + ax.set_xlabel("Normalised Execution Cost", fontsize=11) + ax.set_ylabel("Normalised Key Size", fontsize=11) + ax.set_title( + f"Pareto Frontier — {benchmark} (slot={slot_count})", + fontsize=12, fontweight="bold", + ) + # ax.xaxis.set_major_formatter(mticker.PercentFormatter(xmax=1)) + # ax.yaxis.set_major_formatter(mticker.PercentFormatter(xmax=1)) + ax.set_xlabel("Execution Cost", fontsize=11) + ax.set_ylabel("Key Size", fontsize=11) + ax.grid(True, linestyle="--", alpha=0.4) + plt.tight_layout() + plt.show() \ No newline at end of file diff --git a/RL/fhe_rl/morl/morl.py b/RL/fhe_rl/morl/morl.py new file mode 100644 index 00000000..c6d61d64 --- /dev/null +++ b/RL/fhe_rl/morl/morl.py @@ -0,0 +1,312 @@ +""" +interactive.py — CHEHAB-MORL interactive interface (prompts + orchestration only). + +Two modes (Section 5.2.6 of the thesis): + direct — single preference -> one optimised circuit + menu — preference range -> Pareto frontier + user selection + execution + +Usage: + python -m fhe_rl interactive + python -m fhe_rl interactive --mode direct + python -m fhe_rl interactive --mode menu +""" + +from __future__ import annotations +from typing import Optional + +from .benchmark_runner import ( + run_one_point, save_solution_to_he, cleanup_solutions_cache, + discover_benchmarks, is_poly_benchmark, + POLY_BENCHMARK, +) +from .bisection import build_coarse_grid, bisect_interval, DEFAULT_MAX_BISECT_DEPTH +from .pareto_frontier import get_pareto_frontier +from .display import ( + _h, _ok, _warn, _err, + display_pareto_table, display_pareto_plot, + BOLD, CYAN, MAGENTA, RESET, +) + + +# ═══════════════════════════════════════════════════════════════════════════════ +# PROMPT HELPERS +# ═══════════════════════════════════════════════════════════════════════════════ + +def _ask(prompt: str, default: str = "") -> str: + suffix = f" [{default}]" if default else "" + try: + val = input(f"{BOLD}{MAGENTA}{prompt}{suffix}: {RESET}").strip() + except (EOFError, KeyboardInterrupt): + print() + raise SystemExit(0) + return val if val else default + + +def _pick_benchmark() -> str: + benchmarks = discover_benchmarks() + if not benchmarks: + print(_warn(" No benchmarks found under build/benchmarks/.")) + return _ask("Enter benchmark name manually") + + print() + print(_h("Available benchmarks:")) + for i, b in enumerate(benchmarks, 1): + print(f" {i:>2}. {b}") + + while True: + raw = _ask("Pick a number or type a name", benchmarks[0]) + if raw.isdigit() and 0 <= int(raw) - 1 < len(benchmarks): + return benchmarks[int(raw) - 1] + if raw in benchmarks: + return raw + print(_err(" Invalid choice, try again.")) + + +def _pick_slot_count() -> int: + """Ask the user to enter a slot count directly (no discovery).""" + print() + while True: + raw = _ask("Slot count (e.g. 4, 8, 16, 32)", "16") + try: + val = int(raw) + if val > 0: + return val + except ValueError: + pass + print(_err(" Please enter a positive integer.")) + + +def _pick_poly_args() -> dict: + """Prompt for the extra arguments required by polynomials_coyote.""" + print() + print(_h("polynomials_coyote parameters:")) + tree_depth = int(_ask(" tree_depth", "5")) + instance = int(_ask(" instance", "1")) + regime = _ask(" regime (e.g. 100-50)", "100-50") + return {"tree_depth": tree_depth, "instance": instance, "regime": regime} + + +def _pick_w_ops(label: str = "w_exec (0-1)") -> float: + while True: + raw = _ask(label, "0.5") + try: + val = float(raw) + if 0.0 <= val <= 1.0: + return round(val, 10) + except ValueError: + pass + print(_err(" Must be a float in [0, 1].")) + + +def _pick_w_range() -> tuple[float, float]: + print() + print(_h("Define the w_exec search range (w_keys = 1 - w_exec)")) + while True: + lo = _pick_w_ops(" lower bound w_exec_lo") + hi = _pick_w_ops(" upper bound w_exec_hi") + if lo < hi: + return lo, hi + print(_err(" Lower bound must be strictly less than upper bound.")) + + +def _pick_bisect_depth() -> int: + while True: + raw = _ask("Maximum bisection depth", str(DEFAULT_MAX_BISECT_DEPTH)) + try: + val = int(raw) + if val >= 0: + return val + except ValueError: + pass + print(_err(" Must be a non-negative integer.")) + + +# ═══════════════════════════════════════════════════════════════════════════════ +# DIRECT MODE +# ═══════════════════════════════════════════════════════════════════════════════ + +def run_direct_mode() -> None: + print() + print(_h("--- DIRECT MODE ---")) + print("Generate a single optimised FHE circuit for a specific preference.\n") + + benchmark = _pick_benchmark() + extra_args = _pick_poly_args() if is_poly_benchmark(benchmark) else None + slot_count = 0 if is_poly_benchmark(benchmark) else _pick_slot_count() + w_ops = _pick_w_ops("w_exec (execution weight)") + + print() + print(_warn(f" Running {benchmark} slot={slot_count} " + f"w_exec={w_ops:.4f} w_keys={round(1-w_ops,4):.4f} ...")) + + result = run_one_point(benchmark, slot_count, w_ops, extra_args) + if result is None: + print(_err(" Benchmark timed out or failed.")) + return + + print() + print(_ok(" --- Result ---")) + print(f" Execution cost : {result['final_exec_cost']}") + print(f" Key size cost : {result['final_keys_cost']}") + print(_ok(" ---------------")) + + save_solution_to_he(result) + + +# ═══════════════════════════════════════════════════════════════════════════════ +# MENU MODE +# ═══════════════════════════════════════════════════════════════════════════════ + +def run_menu_mode() -> None: + print() + print(_h("--- MENU MODE (Pareto Frontier) ---")) + print("Scan a preference range, find hidden trade-offs, pick your circuit.\n") + + benchmark = _pick_benchmark() + extra_args = _pick_poly_args() if is_poly_benchmark(benchmark) else None + slot_count = 0 if is_poly_benchmark(benchmark) else _pick_slot_count() + w_lo, w_hi = _pick_w_range() + max_depth = _pick_bisect_depth() + + # ── Phase 1: coarse grid ────────────────────────────────────────────────── + grid = build_coarse_grid(w_lo, w_hi) + print() + print(_h(f"Phase 1 - {len(grid)} coarse-grid points: " + + " ".join(f"{w:.4f}" for w in grid))) + + all_results: list[dict] = [] + for w in grid: + print(_warn(f"\n w_exec={w:.4f} w_keys={round(1-w,4):.4f} ...")) + r = run_one_point(benchmark, slot_count, w, extra_args) + if r is not None: + all_results.append(r) + print(f" ops={r['final_exec_cost']} keys={r['final_keys_cost']}") + else: + print(_err(" timed out - skipped")) + + # ── Phase 2: bisection ──────────────────────────────────────────────────── + if max_depth > 0 and len(all_results) >= 2: + print() + print(_h("Phase 2 - adaptive bisection on key-count gaps ...")) + ordered = sorted(all_results, key=lambda r: r["w_ops"]) + for i in range(len(ordered) - 1): + lo_pt, hi_pt = ordered[i], ordered[i + 1] + ks_lo = lo_pt.get("final_keys_cost") + ks_hi = hi_pt.get("final_keys_cost") + if ks_lo is None or ks_hi is None or int(ks_lo) == int(ks_hi): + continue + bisect_interval( + benchmark, slot_count, + hi_pt["w_ops"], int(ks_hi), + lo_pt["w_ops"], int(ks_lo), + all_results, max_depth, + extra_args=extra_args, + ) + + # ── Pareto frontier ─────────────────────────────────────────────────────── + front = get_pareto_frontier(all_results) + if not front: + print(_err(" No valid solutions found.")) + return + + print() + print(_h(f"Pareto Frontier ({len(front)} non-dominated solutions):")) + display_pareto_table(front) + display_pareto_plot(front, benchmark, slot_count) + + # ── User selection ──────────────────────────────────────────────────────── + print(_h("Select a solution:")) + print(" 1. Pick by solution number") + print(" 2. Pick by custom w_exec weight") + print(" 3. Skip") + + chosen: Optional[dict] = None + mode_raw = _ask("\nSelection method", "1") + + if mode_raw == "3" or not mode_raw: + print(" No selection - exiting.") + return + elif mode_raw == "1": + raw = _ask(f"Solution number (1-{len(front)})", "1") + try: + idx = int(raw) - 1 + if 0 <= idx < len(front): + chosen = front[idx] + else: + print(_err(f" Index out of range (1-{len(front)}).")) + return + except ValueError: + print(_err(" Invalid number.")) + return + elif mode_raw == "2": + raw = _ask("Custom w_exec (0-1)", "0.5") + try: + cw = float(raw) + if not 0.0 <= cw <= 1.0: + raise ValueError + cwk = round(1.0 - cw, 10) + chosen = min(front, key=lambda p: abs(p["w_ops"] - cw)) + print(_ok(f" w_exec={cw:.4f} -> nearest solution is " + f"#{front.index(chosen)+1} " + f"(w_exec={chosen['w_ops']:.4f})")) + except ValueError: + print(_err(" Must be a float in [0, 1].")) + return + else: + print(_err(" Invalid choice.")) + return + + if chosen is None: + return + + print() + print(_ok(f" Selected solution #{front.index(chosen)+1}:")) + print(f" w_exec={chosen['w_ops']:.4f} " + f"ops={chosen['final_exec_cost']} " + f"keys={chosen['final_keys_cost']}") + + + save_solution_to_he(chosen) + cleanup_solutions_cache() + + +# ═══════════════════════════════════════════════════════════════════════════════ +# ENTRY POINT +# ═══════════════════════════════════════════════════════════════════════════════ + +def run_interactive(mode: Optional[str] = None) -> None: + print() + print(BOLD + CYAN + "CHEHAB-MORL Interactive Search" + RESET) + print(BOLD + CYAN + "================================" + RESET) + + if mode is None: + print() + print(" 1. Direct Mode - single preference -> one optimised circuit") + print(" 2. Menu Mode - preference range -> Pareto frontier + selection") + while True: + raw = _ask("\nChoose mode", "1") + if raw == "1": mode = "direct"; break + if raw == "2": mode = "menu"; break + print(_err(" Enter 1 or 2.")) + + if mode == "direct": + run_direct_mode() + elif mode == "menu": + run_menu_mode() + else: + print(_err(f"Unknown mode '{mode}'.")) + + +def add_subparser(subparsers) -> None: + """Register the 'interactive' sub-command on the existing argparse object.""" + p = subparsers.add_parser( + "interactive", + help="Interactive Direct / Menu mode for Pareto exploration", + ) + p.add_argument( + "--mode", + dest="interactive_mode", + choices=["direct", "menu"], + default=None, + help="Skip the mode prompt", + ) \ No newline at end of file diff --git a/RL/fhe_rl/morl/output.txt b/RL/fhe_rl/morl/output.txt new file mode 100644 index 00000000..277f4b28 --- /dev/null +++ b/RL/fhe_rl/morl/output.txt @@ -0,0 +1,132 @@ + window is 0 +Using DDP with world size: 1 +Assigned device: cpu +Using DDP with world size: 1 +Assigned device: cpu +Loading embeddings with tokenizer type: dynamic +Number of unique valid expressions (excluding validation): 1 +['(Vec ( + ( + ( * v1_0 v2_0 ) ( * v1_1 v2_1 ) ) ( + ( * v1_2 v2_2 ) ( * v1_3 v2_3 ) ) ) )'] + +======================================== EPISODE START ======================================== +Optimization Mode : LOCKED (EVALUATION) + +---------------------------------------------------------------------------------------------------- +Old expression: (Vec ( + ( + ( * v1_0 v2_0 ) ( * v1_1 v2_1 ) ) ( + ( * v1_2 v2_2 ) ( * v1_3 v2_3 ) ) ) ) +Old exec cost : 1756.0 +Old keys cost : 0.0 +New expression: (VecAdd (Vec (+ (* v1_0 v2_0) (* v1_1 v2_1)) (+ (* v1_2 v2_2) (* v1_3 v2_3))) (<< (Vec (+ (* v1_0 v2_0) (* v1_1 v2_1)) (+ (* v1_2 v2_2) (* v1_3 v2_3))) 1)) +New exec cost : 1559.0 +New keys cost : 1.0 +Reward : 0.11218678951263428 +Rule name : rot-add-vectorize +At position : 0 +---------------------------------------------------------------------------------------------------- + +---------------------------------------------------------------------------------------------------- +Old expression: (VecAdd (Vec (+ (* v1_0 v2_0) (* v1_1 v2_1)) (+ (* v1_2 v2_2) (* v1_3 v2_3))) (<< (Vec (+ (* v1_0 v2_0) (* v1_1 v2_1)) (+ (* v1_2 v2_2) (* v1_3 v2_3))) 1)) +Old exec cost : 1559.0 +Old keys cost : 1.0 +New expression: (VecAdd (VecAdd (Vec (* v1_0 v2_0) (* v1_2 v2_2) (* v1_1 v2_1) (* v1_3 v2_3)) (<< (Vec (* v1_0 v2_0) (* v1_2 v2_2) (* v1_1 v2_1) (* v1_3 v2_3)) 2)) (<< (VecAdd (Vec (* v1_0 v2_0) (* v1_2 v2_2) (* v1_1 v2_1) (* v1_3 v2_3)) (<< (Vec (* v1_0 v2_0) (* v1_2 v2_2) (* v1_1 v2_1) (* v1_3 v2_3)) 2)) 1)) +New exec cost : 1115.0 +New keys cost : 2.0 +Reward : 0.28479793667793274 +Rule name : rot-add-vectorize +At position : 0 +---------------------------------------------------------------------------------------------------- + +---------------------------------------------------------------------------------------------------- +Old expression: (VecAdd (VecAdd (Vec (* v1_0 v2_0) (* v1_2 v2_2) (* v1_1 v2_1) (* v1_3 v2_3)) (<< (Vec (* v1_0 v2_0) (* v1_2 v2_2) (* v1_1 v2_1) (* v1_3 v2_3)) 2)) (<< (VecAdd (Vec (* v1_0 v2_0) (* v1_2 v2_2) (* v1_1 v2_1) (* v1_3 v2_3)) (<< (Vec (* v1_0 v2_0) (* v1_2 v2_2) (* v1_1 v2_1) (* v1_3 v2_3)) 2)) 1)) +Old exec cost : 1115.0 +Old keys cost : 2.0 +New expression: (VecAdd (VecAdd (VecMul (Vec v1_0 v1_2 v1_1 v1_3) (Vec v2_0 v2_2 v2_1 v2_3)) (<< (VecMul (Vec v1_0 v1_2 v1_1 v1_3) (Vec v2_0 v2_2 v2_1 v2_3)) 2)) (<< (VecAdd (VecMul (Vec v1_0 v1_2 v1_1 v1_3) (Vec v2_0 v2_2 v2_1 v2_3)) (<< (VecMul (Vec v1_0 v1_2 v1_1 v1_3) (Vec v2_0 v2_2 v2_1 v2_3)) 2)) 1)) +New exec cost : 215.0 +New keys cost : 2.0 +Reward : 0.8071748614311218 +Rule name : mul-vectorize +At position : 0 +---------------------------------------------------------------------------------------------------- + +---------------------------------------------------------------------------------------------------- +Old expression: (VecAdd (VecAdd (VecMul (Vec v1_0 v1_2 v1_1 v1_3) (Vec v2_0 v2_2 v2_1 v2_3)) (<< (VecMul (Vec v1_0 v1_2 v1_1 v1_3) (Vec v2_0 v2_2 v2_1 v2_3)) 2)) (<< (VecAdd (VecMul (Vec v1_0 v1_2 v1_1 v1_3) (Vec v2_0 v2_2 v2_1 v2_3)) (<< (VecMul (Vec v1_0 v1_2 v1_1 v1_3) (Vec v2_0 v2_2 v2_1 v2_3)) 2)) 1)) +Old exec cost : 215.0 +Old keys cost : 2.0 +New expression: (VecAdd (VecAdd (VecMul (Vec v1_0 v1_2 v1_1 v1_3) (Vec v2_0 v2_2 v2_1 v2_3)) (<< (VecMul (Vec v1_0 v1_2 v1_1 v1_3) (Vec v2_0 v2_2 v2_1 v2_3)) 2)) (<< (VecAdd (VecMul (Vec v1_0 v1_2 v1_1 v1_3) (Vec v2_0 v2_2 v2_1 v2_3)) (<< (VecMul (Vec v1_0 v1_2 v1_1 v1_3) (Vec v2_0 v2_2 v2_1 v2_3)) 2)) 1)) +New exec cost : 215.0 +New keys cost : 2.0 +Reward : 87.75626420974731 +Rule name : END +At position : 0 +---------------------------------------------------------------------------------------------------- + +Initial exec cost : 1756.0 +Initial keys cost : 0.0 +Final exec cost : 215.0 +Final keys cost : 2.0 + +======================================== EPISODE START ======================================== +Optimization Mode : LOCKED (EVALUATION) +Call the code vectorizer +Executing RL Inference with weights: [1, 0] +==> stop_reached : 1 +===> First_Step_FVC : +===> Second_Step_FVC : +===> Third_Step_FVC : +==> stop_reached : 1 +5506.83 ms +---------------------------------------------------------------------------------------------------- +global_metrics +global_metrics not computed +---------------------------------------------------------------------------------------------------- +depth_metrics +ctxt_leaves_*: (depth, xdepth) +max: (5, 1) +avg: (5, 1) +min: (5, 1) +---------------------------------------------------------------------------------------------------- +terms_classes_metrics +|terms|: 8 +captured_terms: 8 (100%) +|ctxt_inputs|: 2 +|ptxt_leaves|: 0 +|constants|: 0 +|ptxt_ptxt_ops|: 0 +|constants_ops|: 0 +operations_static_cost: 1.77 +---------------------------------------------------------------------------------------------------- +|mul| (level, arg1_size, arg2_size): count +(L-0, 2, 2): 1 +total: 1 +---------------------------------------------------------------------------------------------------- +|square| (level, arg_size): count +total: 0 +---------------------------------------------------------------------------------------------------- +|encrypt|: 0 +---------------------------------------------------------------------------------------------------- +|relin| (level, arg_size): count +(L-0, 3): 1 +total: 1 +---------------------------------------------------------------------------------------------------- +|rotate| (level, arg_size): count +(L-0, 2): 2 +total: 2 +---------------------------------------------------------------------------------------------------- +|mul_plain| (level, ctxt_arg_size): count +total: 0 +---------------------------------------------------------------------------------------------------- +|mul_scalar| (level, ctxt_arg_size): count +total: 0 +---------------------------------------------------------------------------------------------------- +|mul_non_scalar| (level, ctxt_arg_size): count +total: 0 +---------------------------------------------------------------------------------------------------- +|mod_switch| (level, arg_size): count +total: 0 +---------------------------------------------------------------------------------------------------- +|he_add| (level, max_args_size): count +(L-0, 2): 2 +total: 2 +---------------------------------------------------------------------------------------------------- +|rotation_keys|: 2 +|relin_keys|: 1 + Could not parse costs — ops=None, keys=None +None diff --git a/RL/fhe_rl/morl/pareto_frontier.py b/RL/fhe_rl/morl/pareto_frontier.py new file mode 100644 index 00000000..973e9a9e --- /dev/null +++ b/RL/fhe_rl/morl/pareto_frontier.py @@ -0,0 +1,45 @@ +""" +pareto_frontier.py — Pareto dominance filtering for interactive search results. + +Named pareto_frontier.py to avoid collision with the existing pareto.py +(which contains generate_pref_list used during training). +""" + +from __future__ import annotations + + +def is_dominated(point: dict, others: list[dict]) -> bool: + """Return True if `point` is strictly dominated by at least one entry in `others`.""" + ops = point.get("final_exec_cost") + keys = point.get("final_keys_cost") + if ops is None or keys is None: + return True + for o in others: + if o is point: + continue + o_ops = o.get("final_exec_cost") + o_keys = o.get("final_keys_cost") + if o_ops is None or o_keys is None: + continue + if o_ops <= ops and o_keys <= keys and (o_ops < ops or o_keys < keys): + return True + return False + + +def get_pareto_frontier(points: list[dict]) -> list[dict]: + """Return non-dominated points sorted by execution cost (ascending).""" + valid = [ + p for p in points + if p.get("final_exec_cost") is not None + and p.get("final_keys_cost") is not None + ] + front = [p for p in valid if not is_dominated(p, valid)] + seen = set() + deduped = [] + for p in front: + key = (p["final_exec_cost"], p["final_keys_cost"]) + if key not in seen: + seen.add(key) + deduped.append(p) + front = deduped + return sorted(front, key=lambda p: p["final_exec_cost"]) \ No newline at end of file diff --git a/RL/fhe_rl/pareto.py b/RL/fhe_rl/pareto.py new file mode 100644 index 00000000..e9af2c79 --- /dev/null +++ b/RL/fhe_rl/pareto.py @@ -0,0 +1,18 @@ +import numpy as np + +def generate_pref_list(n_points: int): + """ + Generates a list of preference vectors. + Example n_points=2: [[1.0, 0.0], [0.0, 1.0]] + Example n_points=3: [[1.0, 0.0], [0.5, 0.5], [0.0, 1.0]] + """ + if n_points < 2: + return [[0.5, 0.5]] + + prefs = [] + for i in range(n_points): + # Linearly interpolate between 0 and 1 + w_keys = round(i / (n_points - 1), 1) + w_ops = round(1.0 - w_keys, 1) + prefs.append([w_ops, w_keys]) + return prefs[:-1] \ No newline at end of file diff --git a/RL/fhe_rl/policy.py b/RL/fhe_rl/policy.py index 9adf633a..ff3f7644 100644 --- a/RL/fhe_rl/policy.py +++ b/RL/fhe_rl/policy.py @@ -6,7 +6,7 @@ from typing import Any, Dict, Tuple, Union import numpy as np class CustomFeaturesExtractor(nn.Module): - def __init__(self, observation_space, features_dim: int = 256): + def __init__(self, observation_space, features_dim: int = 258): super().__init__() self._embed_dim = observation_space["observation"].shape[0] self._features_dim = features_dim @@ -21,18 +21,25 @@ class HierarchicalMaskablePolicy(nn.Module): def __init__(self, observation_space: gym.Space, action_space: gym.Space, lr_schedule: Union[float, Any], **kwargs): super().__init__() + + seed = kwargs.pop("seed", None) + if seed is not None: + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + np.random.seed(seed) + self.rule_dim: int = kwargs.pop("rule_dim", 5) self.max_positions: int = kwargs.pop("max_positions", 32) - features_dim: int = kwargs.pop("features_dim", 256) + features_dim: int = kwargs.pop("features_dim", 258) lr: float = kwargs.pop("lr", 3e-4) rule_hidden_dims = kwargs.pop("rule_hidden_dims", [128, 128]) pos_hidden_dims = kwargs.pop("pos_hidden_dims", [128, 128]) value_hidden_dims = kwargs.pop("value_hidden_dims", [256, 128, 64]) self.encoder = CustomFeaturesExtractor(observation_space, features_dim) - self.rule_head = mlp(features_dim, rule_hidden_dims, self.rule_dim, layernorm=True) - self.pos_head = mlp(features_dim + self.rule_dim, pos_hidden_dims, self.max_positions, layernorm=True) - self.value_net = mlp(features_dim, value_hidden_dims, 1, layernorm=True) + self.rule_head = mlp(features_dim, rule_hidden_dims, self.rule_dim, layernorm=True, seed=seed) + self.pos_head = mlp(features_dim + self.rule_dim, pos_hidden_dims, self.max_positions, layernorm=True, seed=seed) + self.value_net = mlp(features_dim, value_hidden_dims, 1, layernorm=True, seed=seed) actor_params = list(self.encoder.parameters()) + list(self.rule_head.parameters()) + list(self.pos_head.parameters()) critic_params = self.value_net.parameters() diff --git a/RL/fhe_rl/run.py b/RL/fhe_rl/run.py index b931ba8f..2f0fff9f 100644 --- a/RL/fhe_rl/run.py +++ b/RL/fhe_rl/run.py @@ -8,7 +8,10 @@ import sys, importlib from stable_baselines3.common.monitor import Monitor -def run_agent(expressions_file: str,embeddings_model, model_filepath: str,output_file: str): +def run_agent(expressions_file: str,embeddings_model, model_filepath: str,output_file: str, w_ops=0.5, w_keys=0.5): + + pref = [w_ops, w_keys] + start_time = time.perf_counter() expressions = load_expressions(expressions_file) if not len(expressions): @@ -16,15 +19,16 @@ def run_agent(expressions_file: str,embeddings_model, model_filepath: str,output sys.exit(1) return print(expressions) - rules_list = create_rules("rules.txt") + rules_list = create_rules("rules.txt", 'rotations_rules.txt') rules_list["END"] = None results = [] max_positions = 16 end_time = time.perf_counter() elapsed_seconds = end_time - start_time env = DummyVecEnv([ - lambda: Monitor(fheEnv(rules_list, expressions, max_positions=max_positions,embeddings_model=embeddings_model)) + lambda: Monitor(fheEnv(rules_list, expressions, max_positions=max_positions,embeddings_model=embeddings_model, pref_list=[pref])) ]) + env.env_method("set_preference_vector", pref) model = PPO( policy=HierarchicalMaskablePolicy, env=env diff --git a/RL/fhe_rl/schedules.py b/RL/fhe_rl/schedules.py new file mode 100644 index 00000000..f1dd26c4 --- /dev/null +++ b/RL/fhe_rl/schedules.py @@ -0,0 +1,48 @@ +import numpy as np + +def step_schedule(total_timesteps, transition_point=0.75, max_value=0.01): + """Step function: 0 until transition_point, then 0.01""" + def schedule(timestep): + if timestep >= total_timesteps * transition_point: + return max_value + return 0.0 + return schedule + + +def linear_schedule(total_timesteps, start_point=0.75, saturation_point=0.9, max_value=0.01): + """Linear increase from start_point to end""" + def schedule(timestep): + if timestep < total_timesteps * start_point: + return 0.0 + elif timestep < total_timesteps * saturation_point: + progress = (timestep - total_timesteps * start_point) / (total_timesteps * (saturation_point - start_point)) + return max_value * progress + else: + return max_value + return schedule + + +def sigmoid_schedule(total_timesteps, midpoint=0.7, saturation_point=0.8, steepness=10, max_value=0.01): + """Smooth sigmoid transition around midpoint""" + def schedule(timestep): + if timestep < total_timesteps * midpoint: + return 0.0 + elif timestep < total_timesteps * saturation_point: + x = (timestep / total_timesteps - midpoint) * steepness + return max_value / (1.0 + np.exp(-x)) + else: + return max_value + return schedule + + +def cosine_schedule(total_timesteps, start_point=0.75, saturation_point=0.9, max_value=0.01): + """Smooth cosine transition from start_point""" + def schedule(timestep): + if timestep < total_timesteps * start_point: + return 0.0 + elif timestep < total_timesteps * saturation_point: + progress = (timestep - total_timesteps * start_point) / (total_timesteps * (1 - start_point)) + return max_value * 0.5 * (1 - np.cos(np.pi * min(1.0, progress))) + else: + return max_value + return schedule \ No newline at end of file diff --git a/RL/fhe_rl/test.py b/RL/fhe_rl/test.py index faa93f1f..50dc5535 100644 --- a/RL/fhe_rl/test.py +++ b/RL/fhe_rl/test.py @@ -10,7 +10,7 @@ import torch def test_agent(expressions_file: str,embeddings_model, model_filepath: str): expressions = load_expressions(expressions_file) - rules_list = create_rules("rules.txt") + rules_list = create_rules("rules.txt", 'rotations_rules.txt') rules_list["END"] = None results = [] max_positions = 16 diff --git a/RL/fhe_rl/train.py b/RL/fhe_rl/train.py index 6367e469..cb5e088f 100644 --- a/RL/fhe_rl/train.py +++ b/RL/fhe_rl/train.py @@ -7,24 +7,69 @@ from .utils import load_expressions, create_rules, load_embeddings from .logger import log_training_details from stable_baselines3.common.vec_env import SubprocVecEnv, VecNormalize,DummyVecEnv -from .callbacks import linear_schedule, EntCoefScheduler -from stable_baselines3.common.callbacks import EvalCallback +from .callbacks import linear_schedule, EntCoefScheduler, ParetoEvalCallback +from stable_baselines3.common.callbacks import EvalCallback, CheckpointCallback -def train_agent(expressions_file: str, embeddings_model, total_timesteps: int = 1_000_000, num_envs: int = 8): + +from .pareto import generate_pref_list + +import random +import numpy as np +import torch + +def set_random_seed(seed: int = 42): + """Set random seed for reproducibility""" + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + os.environ['PYTHONHASHSEED'] = str(seed) + +def train_agent(expressions_file: str, embeddings_model, total_timesteps: int = 1_000_000, num_envs: int = 8 , seed: int = 42, + lambda_env: float = 0.0, lambda_kl: float = 0.0, n_cycle: int = 1, n_budget: int = 5): + set_random_seed(seed) + N=11 + pref_list = generate_pref_list(N) benchmarks = load_expressions("./fhe_rl/datasets/benchmarks.txt") expressions = load_expressions(expressions_file, benchmarks) max_positions = 16 - rules_list = create_rules("rules.txt") + rules_list = create_rules("rules.txt", "rotations_rules.txt") rules_list["END"] = None job_id = os.environ.get("SLURM_JOB_ID", "jobid") run_name = f"model_{job_id}" tensorboard_log_dir = f"./tensorboard/{run_name}" - def make_env(): return Monitor(fheEnv(rules_list, expressions, max_positions=max_positions, embeddings_model=embeddings_model)) - env = SubprocVecEnv([make_env for _ in range(num_envs)], start_method='spawn') - val_env = DummyVecEnv([ - lambda: Monitor(fheEnv(rules_list, benchmarks, max_positions=max_positions,embeddings_model=embeddings_model)) - ]) + + checkpoint_dir = f"./checkpoints/{run_name}" + os.makedirs(checkpoint_dir, exist_ok=True) + + + checkpoint_path = None + steps_done = 0 + if os.path.exists(checkpoint_dir): + checkpoints = [f for f in os.listdir(checkpoint_dir) if f.startswith("rl_model_") and f.endswith("_steps.zip")] + if checkpoints: + # Get the latest checkpoint + latest = max(checkpoints, key=lambda x: int(x.split("_")[2])) + checkpoint_path = os.path.join(checkpoint_dir, latest) + steps_done = int(latest.split("_")[2]) + print(f"Found checkpoint: {checkpoint_path}") + print(f"Resuming from step {steps_done}") + + lambda_env = 0.0 + def make_env(rank, expressions): + def _init(): + # Pass pref_list and rank to each env + return Monitor(fheEnv(rules_list, expressions, max_positions=max_positions, embeddings_model=embeddings_model, + pref_list=pref_list, lambda_env=lambda_env, lambda_kl=lambda_kl, n_cycle=n_cycle, n_budget=n_budget, env_idx=rank)) + return _init + env = SubprocVecEnv([make_env(i, expressions) for i in range(num_envs)]) + env.seed(seed) + val_env = DummyVecEnv([make_env(0, benchmarks)]) ent_schedule = linear_schedule(0.1) + ent_callback = EntCoefScheduler(ent_schedule, verbose=1) model_params = { "policy": HierarchicalMaskablePolicy, "env": env, @@ -39,16 +84,25 @@ def make_env(): return Monitor(fheEnv(rules_list, expressions, max_positions=max "ent_coef": 0.1, "verbose": 1, "tensorboard_log": tensorboard_log_dir, + "seed": seed, "policy_kwargs": { "ent_coef": 0.1, + "features_dim": 258, "rule_dim": len(rules_list), "max_positions": max_positions, - "rule_hidden_dims": [128, 64], - "pos_hidden_dims": [64, 64], - "value_hidden_dims": [256, 128, 64], + "rule_hidden_dims": [64, 32], + "pos_hidden_dims": [32, 32], + "value_hidden_dims": [128, 64, 32], + "seed": seed, } } - model = PPO(**model_params) + + if checkpoint_path: + print(f"Loading model from checkpoint: {checkpoint_path}") + model = PPO.load(checkpoint_path, env=env, tensorboard_log=tensorboard_log_dir) + else: + model = PPO(**model_params) + log_training_details( model_params, job_id, @@ -59,20 +113,44 @@ def make_env(): return Monitor(fheEnv(rules_list, expressions, max_positions=max notes="2 level hierarchical PPO max steps 75 and 8 envs" ) num_benchmarks = len(benchmarks) - eval_callback = EvalCallback( + + checkpoint_callback = CheckpointCallback( + save_freq=10000, + save_path=checkpoint_dir, + name_prefix="rl_model", + save_replay_buffer=False, + save_vecnormalize=True, + verbose=1 + ) + + + + pareto_eval_cb = ParetoEvalCallback( val_env, + pref_list=pref_list, best_model_save_path=f"./eval/best_model_{run_name}", log_path=tensorboard_log_dir, - eval_freq=10000, - n_eval_episodes=num_benchmarks, + eval_freq=10000, # Replicating your parameter + n_eval_episodes=num_benchmarks, # Replicating your parameter deterministic=True, - render=False, verbose=1 ) + + if checkpoint_path: + remaining_steps = total_timesteps - steps_done + print(f"Resuming training: {remaining_steps} steps remaining out of {total_timesteps} total") + else: + remaining_steps = total_timesteps + print(f"Starting fresh training: {total_timesteps} total steps") + + + + model.learn( - total_timesteps=total_timesteps, + total_timesteps=remaining_steps, log_interval=1, progress_bar=True, - callback=[eval_callback,EntCoefScheduler(ent_schedule)] + callback=[ent_callback, pareto_eval_cb, checkpoint_callback], + reset_num_timesteps = (checkpoint_path is None) ) - model.save(run_name) \ No newline at end of file + model.save(run_name) \ No newline at end of file diff --git a/RL/fhe_rl/trained_models/agent_env_0_0001.zip b/RL/fhe_rl/trained_models/agent_env_0_0001.zip new file mode 100644 index 00000000..ea454860 Binary files /dev/null and b/RL/fhe_rl/trained_models/agent_env_0_0001.zip differ diff --git a/RL/fhe_rl/trained_models/agent_kl_0_0001.zip b/RL/fhe_rl/trained_models/agent_kl_0_0001.zip new file mode 100644 index 00000000..31776096 Binary files /dev/null and b/RL/fhe_rl/trained_models/agent_kl_0_0001.zip differ diff --git a/RL/fhe_rl/trained_models/agent_pareto_2_full.zip b/RL/fhe_rl/trained_models/agent_pareto_2_full.zip new file mode 100644 index 00000000..8077b3df Binary files /dev/null and b/RL/fhe_rl/trained_models/agent_pareto_2_full.zip differ diff --git a/RL/fhe_rl/trained_models/agent_pareto_4M.zip b/RL/fhe_rl/trained_models/agent_pareto_4M.zip new file mode 100644 index 00000000..52ebcd0b Binary files /dev/null and b/RL/fhe_rl/trained_models/agent_pareto_4M.zip differ diff --git a/RL/fhe_rl/trained_models/try.zip b/RL/fhe_rl/trained_models/try.zip new file mode 100644 index 00000000..8077b3df Binary files /dev/null and b/RL/fhe_rl/trained_models/try.zip differ diff --git a/RL/fhe_rl/trained_models/try_best.zip b/RL/fhe_rl/trained_models/try_best.zip new file mode 100644 index 00000000..6decebd1 Binary files /dev/null and b/RL/fhe_rl/trained_models/try_best.zip differ diff --git a/RL/fhe_rl/utils.py b/RL/fhe_rl/utils.py index 1219aada..3d4dba05 100644 --- a/RL/fhe_rl/utils.py +++ b/RL/fhe_rl/utils.py @@ -38,8 +38,8 @@ def load_embeddings(tokenizer_type=None, checkpoint_path=None, device=None): else: model = load_embedding_model_dynamic(checkpoint_path, device) return model, None -def create_rules(path): - return _create_rules(path=path) +def create_rules(rules_path, rotations_rules_path): + return _create_rules(rules_path, rotations_rules_path) def load_embedding_model_dynamic(checkpoint_path=None, device=DEVICE): embeddings_model = TRAE() @@ -183,7 +183,7 @@ def load_expressions(file_path: str,validation_exprs = []): def mlp(in_dim, hidden_dims, out_dim, *, act=nn.GELU, layernorm=True, dropout=0.0, - residual=False): + residual=False, seed=None): """ Build an MLP: [in_dim] → hidden_dims* → [out_dim] @@ -192,6 +192,11 @@ def mlp(in_dim, hidden_dims, out_dim, *, hidden_dims : list[int] (e.g. [1024, 1024, 512]) residual : if True, adds skip-connections every two layers """ + + if seed is not None: + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + layers, prev = [], in_dim for i, h in enumerate(hidden_dims): layers.append(nn.Linear(prev, h)) diff --git a/RL/job_logs.xlsx b/RL/job_logs.xlsx new file mode 100644 index 00000000..d3b4ff2e Binary files /dev/null and b/RL/job_logs.xlsx differ diff --git a/RL/pytrs/cost.py b/RL/pytrs/cost.py index 27ad285a..f92af6a7 100644 --- a/RL/pytrs/cost.py +++ b/RL/pytrs/cost.py @@ -3,6 +3,7 @@ from serializer import expr_to_str import subprocess from rules import create_rules +from parser import parse_sexpr @@ -24,7 +25,8 @@ def operations_cost(expr: Expr) -> int: node_cost = OP * 250 elif op == "<<": node_cost = VEC_OP * 50 - visit_all_children = False + if not isinstance(expr.args[0], Op) or expr.args[0].op != "<<": + visit_all_children = False elif op == "Vec": node_cost = 0 elif op == "VecAdd": @@ -55,8 +57,12 @@ def operations_cost(expr: Expr) -> int: # 3) Recurse into children if visit_all_children: - for child in expr.args: - node_cost += operations_cost(child) + if op == "<<": + # Special case: only recurse into first child for shifts + node_cost += operations_cost(expr.args[0]) + else: + for child in expr.args: + node_cost += operations_cost(child) # else: # node_cost += operations_cost(expr.args[0] ) # Only recurse into the first child @@ -166,17 +172,102 @@ def rotations_cost(expr: Expr, parent: Expr = None) -> float: # Leaves (Const/Var) contribute zero return 0.0 + +def evaluate_const_expr(expr: Expr) -> int: + """ + Recursively evaluate a constant expression to an integer. + Returns None if the expression cannot be evaluated (contains variables). + """ + if isinstance(expr, Const): + return expr.value + + if isinstance(expr, Var): + # Can't evaluate variables + return None + + if isinstance(expr, Op): + # Recursively evaluate operands + operands = [] + for arg in expr.args: + result = evaluate_const_expr(arg) + if result is None: + return None # Can't evaluate if any operand has variables + operands.append(result) + + # Evaluate based on operator + if expr.op == '+': + return sum(operands) + elif expr.op == '-': + if len(operands) == 1: + return -operands[0] # Unary minus + return operands[0] - operands[1] + elif expr.op == '*': + result = 1 + for op in operands: + result *= op + return result + else: + # Unknown operator, can't evaluate + return None + + return None + + +def get_unique_rotations(expr: Expr, rotations=None) -> int: + """ + Count unique rotation offsets in the expression. + Safely handles both constant offsets and computed offsets. + """ + if rotations is None: + rotations = set() + + if isinstance(expr, Op): + if expr.op == "<<": + offset = expr.args[1] + + # Try to get the rotation value + rotation_value = None + if isinstance(offset, Const): + rotation_value = offset.value + elif isinstance(offset, Op): + # Try to evaluate the expression + try: + rotation_value = evaluate_const_expr(offset) + except Exception as e: + # If evaluation fails, just skip this rotation + pass + + # Add to set if we successfully got a value + if rotation_value is not None: + rotations.add(rotation_value) + + # Recursively process all children + for child in expr.args: + get_unique_rotations(child, rotations) + + return len(rotations) + +def get_total_rotations(expr: Expr) -> int: + count = 0 + if isinstance(expr, Op): + if expr.op == "<<": + count += 1 + for child in expr.args: + count += get_total_rotations(child) + return count + def calculate_cost(expr: Expr, w_ops=1.0, w_rot=1.0, w_depth=1.0, w_muldepth=1.0, - w_vec=-1.0 + w_keys=1.0 ) -> float: return ( w_ops * operations_cost(expr) + w_rot * rotations_cost(expr) + w_depth * get_normal_depth(expr) + - w_muldepth * get_multiplicative_depth(expr) + w_muldepth * get_multiplicative_depth(expr) + + w_keys * get_unique_rotations(expr) ) diff --git a/RL/pytrs/generate_rotations_rules.py b/RL/pytrs/generate_rotations_rules.py new file mode 100644 index 00000000..fe2af5b5 --- /dev/null +++ b/RL/pytrs/generate_rotations_rules.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +def generate_rotation_rules(): + """ + Generate rewrite rules for decomposing rotations. + Valid rotation amounts: 1, 2, 4, 8, 16, 32 + Special case: 9 = 3 + 3 + 3, and 3 = 1 + 2 + """ + rules = [] + + # Standard power-of-2 decompositions + # Each rotation can be split into two rotations of half the amount + rotations = [32, 16, 8, 4, 2] + + for rot in rotations: + half = rot // 2 + rule = f'Rewrite {{ name: "rotate_{rot}", searcher: (<< ?a {rot}), applier: (<< (<< ?a {half}) {half}) }}' + rules.append(rule) + + # Special case: rotation of 9 + # 9 = 3 + 3 + 3 + rule_9 = 'Rewrite { name: "rotate_9", searcher: (<< ?a 9), applier: (<< (<< (<< ?a 3) 3) 3) }' + rules.append(rule_9) + + # Special case: rotation of 3 + # 3 = 1 + 2 + rule_3 = 'Rewrite { name: "rotate_3", searcher: (<< ?a 3), applier: (<< (<< ?a 1) 2) }' + rules.append(rule_3) + + return rules + +def main(): + rules = generate_rotation_rules() + + # Write rules to file + with open("../rotations_rules.txt", "w") as f: + for rule in rules: + f.write(rule + "\n") + + print("Rules saved to rotations_rules.txt") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/RL/pytrs/rules.py b/RL/pytrs/rules.py index 0310a9a7..4478b170 100644 --- a/RL/pytrs/rules.py +++ b/RL/pytrs/rules.py @@ -13,11 +13,14 @@ -def create_rules(path: str) -> dict: +def create_rules(rules_path: str, rotations_rules_path: str) -> dict: - rules_text = open(path,'r').read().replace("?","") + rules_text = open(rules_path,'r').read().replace("?","") + rotations_rules_text = open(rotations_rules_path,'r').read().replace("?","") rules = parse_rules_from_text(rules_text) - + rotations_rules = parse_rules_from_text(rotations_rules_text) + rules.extend(rotations_rules) + rules_dict = {rule.name: rule for rule in rules} return rules_dict diff --git a/RL/pytrs/trs_example.py b/RL/pytrs/trs_example.py index 8d8e45c3..c82c7564 100644 --- a/RL/pytrs/trs_example.py +++ b/RL/pytrs/trs_example.py @@ -12,12 +12,12 @@ def example_usage(): - rules = create_rules(vector_size=9) # returns a dict of rules {rulename:rule} - - rule = rules["rotations-8"] + rules = create_rules('../rules.txt', '../rotations_rules.txt') # returns a dict of rules {rulename:rule} + print(rules) + rule = rules["rotate_4"] # Example expression: ( << (Vec a b c d e f g h i ) 1 ) - expr_str = "(Vec a b c d e f g h i)" + expr_str = "(<< (Vec a b c d e f g h i) 4)" diff --git a/RL/pytrs/try_job.sh b/RL/pytrs/try_job.sh new file mode 100644 index 00000000..09dceba6 --- /dev/null +++ b/RL/pytrs/try_job.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +#SBATCH -q c2 +#SBATCH -p nvidia +#SBATCH --gres=gpu:a100:1 +#SBATCH --mem 64GB +#SBATCH -c 16 +#SBATCH -t 5-00:00:00 + +#SBATCH --mail-user=im2821@nyu.edu +#SBATCH --mail-type=BEGIN,END,FAIL + +#SBATCH -o output.txt +#SBATCH -e error.txt + + + +source /share/apps/NYUAD5/miniconda/3-4.11.0/bin/activate +conda activate chehabEnv + +cd RL +python -m fhe_rl --tokenizer_type dynamic train diff --git a/RL/rotations_rules.txt b/RL/rotations_rules.txt new file mode 100644 index 00000000..2195800a --- /dev/null +++ b/RL/rotations_rules.txt @@ -0,0 +1,7 @@ +Rewrite { name: "rotate_32", searcher: (<< ?a 32), applier: (<< (<< ?a 16) 16) } +Rewrite { name: "rotate_16", searcher: (<< ?a 16), applier: (<< (<< ?a 8) 8) } +Rewrite { name: "rotate_8", searcher: (<< ?a 8), applier: (<< (<< ?a 4) 4) } +Rewrite { name: "rotate_4", searcher: (<< ?a 4), applier: (<< (<< ?a 2) 2) } +Rewrite { name: "rotate_2", searcher: (<< ?a 2), applier: (<< (<< ?a 1) 1) } +Rewrite { name: "rotate_9", searcher: (<< ?a 9), applier: (<< (<< (<< ?a 3) 3) 3) } +Rewrite { name: "rotate_3", searcher: (<< ?a 3), applier: (<< (<< ?a 1) 2) } diff --git a/RL/tensorboard/Pareto_2M/PPO_1/events.out.tfevents.1773616273.cn222.248044.0 b/RL/tensorboard/Pareto_2M/PPO_1/events.out.tfevents.1773616273.cn222.248044.0 new file mode 100644 index 00000000..ca824b40 Binary files /dev/null and b/RL/tensorboard/Pareto_2M/PPO_1/events.out.tfevents.1773616273.cn222.248044.0 differ diff --git a/RL/tensorboard/Pareto_2M_keys_10/PPO_1/events.out.tfevents.1773912960.cn205.1351812.0 b/RL/tensorboard/Pareto_2M_keys_10/PPO_1/events.out.tfevents.1773912960.cn205.1351812.0 new file mode 100644 index 00000000..f1526823 Binary files /dev/null and b/RL/tensorboard/Pareto_2M_keys_10/PPO_1/events.out.tfevents.1773912960.cn205.1351812.0 differ diff --git a/RL/tensorboard/Pareto_2M_keys_6/PPO_1/events.out.tfevents.1773763310.cn223.1512043.0 b/RL/tensorboard/Pareto_2M_keys_6/PPO_1/events.out.tfevents.1773763310.cn223.1512043.0 new file mode 100644 index 00000000..ef3e622f Binary files /dev/null and b/RL/tensorboard/Pareto_2M_keys_6/PPO_1/events.out.tfevents.1773763310.cn223.1512043.0 differ diff --git a/avg_reward_pareto.png b/avg_reward_pareto.png new file mode 100644 index 00000000..34d80bc9 Binary files /dev/null and b/avg_reward_pareto.png differ diff --git a/benchmarks/box_blur/box_blur.cpp b/benchmarks/box_blur/box_blur.cpp index 5c65bf99..17b4b374 100644 --- a/benchmarks/box_blur/box_blur.cpp +++ b/benchmarks/box_blur/box_blur.cpp @@ -74,7 +74,7 @@ void print_bool_arg(bool arg, const string &name, ostream &os) os << (arg ? name : "no_" + name); } int main(int argc, char **argv) -{ +{ bool vectorize_code = true; if (argc > 1) vectorize_code = stoi(argv[1]); @@ -103,11 +103,17 @@ int main(int argc, char **argv) if (argc > 7) const_folding = stoi(argv[7]); + float w_ops = 0.5; + float w_keys = 0.5; + + if (argc > 8) w_ops = stof(argv[8]); + if (argc > 9) w_keys = stof(argv[9]); + if (cse) { Compiler::enable_cse(); Compiler::enable_order_operands(); - } + } else { Compiler::disable_cse(); @@ -116,7 +122,7 @@ int main(int argc, char **argv) if (const_folding) Compiler::enable_const_folding(); - else + else Compiler::disable_const_folding(); //Compiler::enable_auto_enc_params_selection(); chrono::high_resolution_clock::time_point t; @@ -133,12 +139,12 @@ int main(int argc, char **argv) if (!header_os) throw logic_error("failed to create header file"); ofstream source_os(gen_path + ".cpp"); - if (!source_os) + if (!source_os) throw logic_error("failed to create source file"); cout << " window is " << window << endl; /********** vectorization Part *******************************/ if(VECTORIZATION_ENABLED){ - Compiler::gen_vectorized_code(func, window, optimization_method); // add a flag to specify if the benchmark is structured or no + Compiler::gen_vectorized_code(func, window,optimization_method, w_ops, w_keys); // add a flag to specify if the benchmark is structured or no } /********** Simplification & depth reduction Part ************/ if(SIMPLIFICATION_ENABLED){ @@ -160,34 +166,34 @@ int main(int argc, char **argv) } else { - const auto &func = Compiler::create_func(func_name,slot_count*slot_count, 20, false, true); - // update io file - std::string updated_inputs_file_name = "fhe_io_example_adapted.txt" ; - std::string inputs_file_name = "fhe_io_example.txt"; - util::copyFile(inputs_file_name,updated_inputs_file_name); - fhe(slot_count); - string gen_name = "_gen_he_" + func_name; - string gen_path = "he/" + gen_name; - ofstream header_os(gen_path + ".hpp"); - if (!header_os) - throw logic_error("failed to create header file"); - ofstream source_os(gen_path + ".cpp"); - if (!source_os) - throw logic_error("failed to create source file"); - cout << " window is " << window << endl; - auto ruleset = Compiler::Ruleset::simplification_ruleset; - auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; - Compiler::compile(func, ruleset, rewrite_heuristic); - Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os, 29); - /************/elapsed = chrono::high_resolution_clock::now() - t; - cout<<"Compile time \n :"; - cout << elapsed.count() << " ms\n"; - if (call_quantifier) - { - util::Quantifier quantifier{func}; - quantifier.run_all_analysis(); - quantifier.print_info(cout); - } + const auto &func = Compiler::create_func(func_name, slot_count, 20, false, true); + // update_io_file + std::string updated_inputs_file_name = "fhe_io_example_adapted.txt" ; + std::string inputs_file_name = "fhe_io_example.txt"; + util::copyFile(inputs_file_name,updated_inputs_file_name); + fhe(slot_count); + string gen_name = "_gen_he_" + func_name; + string gen_path = "he/" + gen_name; + ofstream header_os(gen_path + ".hpp"); + if (!header_os) + throw logic_error("failed to create header file"); + ofstream source_os(gen_path + ".cpp"); + if (!source_os) + throw logic_error("failed to create source file"); + cout << " window is " << window << endl; + auto ruleset = Compiler::Ruleset::simplification_ruleset; + auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; + Compiler::compile(func, ruleset, rewrite_heuristic); + Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); + /************/elapsed = chrono::high_resolution_clock::now() - t; + cout<<"Compile time : \n"; + cout << elapsed.count() << " ms\n"; + if (call_quantifier) + { + util::Quantifier quantifier{func}; + quantifier.run_all_analysis(); + quantifier.print_info(cout); + } } return 0; } \ No newline at end of file diff --git a/benchmarks/dot_product/dot_product.cpp b/benchmarks/dot_product/dot_product.cpp index b074c557..1d99a0cf 100644 --- a/benchmarks/dot_product/dot_product.cpp +++ b/benchmarks/dot_product/dot_product.cpp @@ -76,6 +76,12 @@ int main(int argc, char **argv) if (argc > 7) const_folding = stoi(argv[7]); + float w_ops = 0.5; + float w_keys = 0.5; + + if (argc > 8) w_ops = stof(argv[8]); + if (argc > 9) w_keys = stof(argv[9]); + if (cse) { Compiler::enable_cse(); @@ -111,7 +117,7 @@ int main(int argc, char **argv) cout << " window is " << window << endl; /********** vectorization Part *******************************/ if(VECTORIZATION_ENABLED){ - Compiler::gen_vectorized_code(func, window,optimization_method); // add a flag to specify if the benchmark is structured or no + Compiler::gen_vectorized_code(func, window,optimization_method, w_ops, w_keys); // add a flag to specify if the benchmark is structured or no } /********** Simplification & depth reduction Part ************/ if(SIMPLIFICATION_ENABLED){ diff --git a/benchmarks/gx_kernel/gx_kernel.cpp b/benchmarks/gx_kernel/gx_kernel.cpp index cae129f4..a4d00fe9 100644 --- a/benchmarks/gx_kernel/gx_kernel.cpp +++ b/benchmarks/gx_kernel/gx_kernel.cpp @@ -94,17 +94,23 @@ int main(int argc, char **argv) if (argc > 7) const_folding = stoi(argv[7]); + float w_ops = 0.5; + float w_keys = 0.5; + + if (argc > 8) w_ops = stof(argv[8]); + if (argc > 9) w_keys = stof(argv[9]); if (cse) { Compiler::enable_cse(); Compiler::enable_order_operands(); - } + } else { Compiler::disable_cse(); Compiler::disable_order_operands(); } + if (const_folding) Compiler::enable_const_folding(); else @@ -124,12 +130,12 @@ int main(int argc, char **argv) if (!header_os) throw logic_error("failed to create header file"); ofstream source_os(gen_path + ".cpp"); - if (!source_os) + if (!source_os) throw logic_error("failed to create source file"); cout << " window is " << window << endl; /********** vectorization Part *******************************/ if(VECTORIZATION_ENABLED){ - Compiler::gen_vectorized_code(func, window, optimization_method); // add a flag to specify if the benchmark is structured or no + Compiler::gen_vectorized_code(func, window,optimization_method, w_ops, w_keys); // add a flag to specify if the benchmark is structured or no } /********** Simplification & depth reduction Part ************/ if(SIMPLIFICATION_ENABLED){ @@ -151,34 +157,34 @@ int main(int argc, char **argv) } else { - const auto &func = Compiler::create_func(func_name,slot_count*slot_count, 20, false, true); - // update io file - std::string updated_inputs_file_name = "fhe_io_example_adapted.txt" ; - std::string inputs_file_name = "fhe_io_example.txt"; - util::copyFile(inputs_file_name,updated_inputs_file_name); - fhe(slot_count); - string gen_name = "_gen_he_" + func_name; - string gen_path = "he/" + gen_name; - ofstream header_os(gen_path + ".hpp"); - if (!header_os) - throw logic_error("failed to create header file"); - ofstream source_os(gen_path + ".cpp"); - if (!source_os) - throw logic_error("failed to create source file"); - cout << " window is " << window << endl; - auto ruleset = Compiler::Ruleset::simplification_ruleset; - auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; - Compiler::compile(func, ruleset, rewrite_heuristic); - Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os, 29); - /************/elapsed = chrono::high_resolution_clock::now() - t; - cout<<"Compile time : \n"; - cout << elapsed.count() << " ms\n"; - if (call_quantifier) - { - util::Quantifier quantifier{func}; - quantifier.run_all_analysis(); - quantifier.print_info(cout); - } + const auto &func = Compiler::create_func(func_name, slot_count, 20, false, true); + // update_io_file + std::string updated_inputs_file_name = "fhe_io_example_adapted.txt" ; + std::string inputs_file_name = "fhe_io_example.txt"; + util::copyFile(inputs_file_name,updated_inputs_file_name); + fhe(slot_count); + string gen_name = "_gen_he_" + func_name; + string gen_path = "he/" + gen_name; + ofstream header_os(gen_path + ".hpp"); + if (!header_os) + throw logic_error("failed to create header file"); + ofstream source_os(gen_path + ".cpp"); + if (!source_os) + throw logic_error("failed to create source file"); + cout << " window is " << window << endl; + auto ruleset = Compiler::Ruleset::simplification_ruleset; + auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; + Compiler::compile(func, ruleset, rewrite_heuristic); + Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); + /************/elapsed = chrono::high_resolution_clock::now() - t; + cout<<"Compile time : \n"; + cout << elapsed.count() << " ms\n"; + if (call_quantifier) + { + util::Quantifier quantifier{func}; + quantifier.run_all_analysis(); + quantifier.print_info(cout); + } } return 0; } diff --git a/benchmarks/gy_kernel/gy_kernel.cpp b/benchmarks/gy_kernel/gy_kernel.cpp index e9c84102..95f2b378 100644 --- a/benchmarks/gy_kernel/gy_kernel.cpp +++ b/benchmarks/gy_kernel/gy_kernel.cpp @@ -94,14 +94,19 @@ int main(int argc, char **argv) if (argc > 7) const_folding = stoi(argv[7]); + float w_ops = 0.5; + float w_keys = 0.5; + + if (argc > 8) w_ops = stof(argv[8]); + if (argc > 9) w_keys = stof(argv[9]); if (cse) { Compiler::enable_cse(); Compiler::enable_order_operands(); - } + } else - { + { Compiler::disable_cse(); Compiler::disable_order_operands(); } @@ -125,12 +130,12 @@ int main(int argc, char **argv) if (!header_os) throw logic_error("failed to create header file"); ofstream source_os(gen_path + ".cpp"); - if (!source_os) + if (!source_os) throw logic_error("failed to create source file"); cout << " window is " << window << endl; /********** vectorization Part *******************************/ if(VECTORIZATION_ENABLED){ - Compiler::gen_vectorized_code(func, window,optimization_method); // add a flag to specify if the benchmark is structured or no + Compiler::gen_vectorized_code(func, window,optimization_method, w_ops, w_keys); // add a flag to specify if the benchmark is structured or no } /********** Simplification & depth reduction Part ************/ if(SIMPLIFICATION_ENABLED){ @@ -152,34 +157,34 @@ int main(int argc, char **argv) } else { - const auto &func = Compiler::create_func(func_name,slot_count*slot_count, 20, false, true); - // update io file - std::string updated_inputs_file_name = "fhe_io_example_adapted.txt" ; - std::string inputs_file_name = "fhe_io_example.txt"; - util::copyFile(inputs_file_name,updated_inputs_file_name); - fhe(slot_count); - string gen_name = "_gen_he_" + func_name; - string gen_path = "he/" + gen_name; - ofstream header_os(gen_path + ".hpp"); - if (!header_os) - throw logic_error("failed to create header file"); - ofstream source_os(gen_path + ".cpp"); - if (!source_os) - throw logic_error("failed to create source file"); - cout << " window is " << window << endl; - auto ruleset = Compiler::Ruleset::simplification_ruleset; - auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; - Compiler::compile(func, ruleset, rewrite_heuristic); - Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os, 29); - /************/elapsed = chrono::high_resolution_clock::now() - t; - cout<<"Compile time : \n"; - cout << elapsed.count() << " ms\n"; - if (call_quantifier) - { - util::Quantifier quantifier{func}; - quantifier.run_all_analysis(); - quantifier.print_info(cout); - } + const auto &func = Compiler::create_func(func_name, slot_count, 20, false, true); + // update_io_file + std::string updated_inputs_file_name = "fhe_io_example_adapted.txt" ; + std::string inputs_file_name = "fhe_io_example.txt"; + util::copyFile(inputs_file_name,updated_inputs_file_name); + fhe(slot_count); + string gen_name = "_gen_he_" + func_name; + string gen_path = "he/" + gen_name; + ofstream header_os(gen_path + ".hpp"); + if (!header_os) + throw logic_error("failed to create header file"); + ofstream source_os(gen_path + ".cpp"); + if (!source_os) + throw logic_error("failed to create source file"); + cout << " window is " << window << endl; + auto ruleset = Compiler::Ruleset::simplification_ruleset; + auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; + Compiler::compile(func, ruleset, rewrite_heuristic); + Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); + /************/elapsed = chrono::high_resolution_clock::now() - t; + cout<<"Compile time : \n"; + cout << elapsed.count() << " ms\n"; + if (call_quantifier) + { + util::Quantifier quantifier{func}; + quantifier.run_all_analysis(); + quantifier.print_info(cout); + } } return 0; } diff --git a/benchmarks/hamming_dist/hamming_dist.cpp b/benchmarks/hamming_dist/hamming_dist.cpp index 3a1262d0..f9e3b73e 100644 --- a/benchmarks/hamming_dist/hamming_dist.cpp +++ b/benchmarks/hamming_dist/hamming_dist.cpp @@ -74,11 +74,17 @@ int main(int argc, char **argv) if (argc > 7) const_folding = stoi(argv[7]); + float w_ops = 0.5; + float w_keys = 0.5; + + if (argc > 8) w_ops = stof(argv[8]); + if (argc > 9) w_keys = stof(argv[9]); + if (cse) { Compiler::enable_cse(); Compiler::enable_order_operands(); - } + } else { Compiler::disable_cse(); @@ -104,12 +110,12 @@ int main(int argc, char **argv) if (!header_os) throw logic_error("failed to create header file"); ofstream source_os(gen_path + ".cpp"); - if (!source_os) + if (!source_os) throw logic_error("failed to create source file"); cout << " window is " << window << endl; /********** vectorization Part *******************************/ if(VECTORIZATION_ENABLED){ - Compiler::gen_vectorized_code(func, window,optimization_method); + Compiler::gen_vectorized_code(func, window,optimization_method, w_ops, w_keys); // add a flag to specify if the benchmark is structured or no } /********** Simplification & depth reduction Part ************/ if(SIMPLIFICATION_ENABLED){ @@ -117,12 +123,12 @@ int main(int argc, char **argv) auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; Compiler::compile(func, ruleset, rewrite_heuristic); } - /********** FHE code generation *****************************/ + /********** FHE code generation *****************************/ Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); /************/elapsed = chrono::high_resolution_clock::now() - t; cout << elapsed.count() << " ms\n"; - if (call_quantifier) + if (call_quantifier) { util::Quantifier quantifier{func}; quantifier.run_all_analysis(); @@ -151,6 +157,7 @@ int main(int argc, char **argv) Compiler::compile(func, ruleset, rewrite_heuristic); Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); /************/elapsed = chrono::high_resolution_clock::now() - t; + cout<<"Compile time : \n"; cout << elapsed.count() << " ms\n"; if (call_quantifier) { diff --git a/benchmarks/l2_distance/l2_distance.cpp b/benchmarks/l2_distance/l2_distance.cpp index 73e953c0..d94b22db 100644 --- a/benchmarks/l2_distance/l2_distance.cpp +++ b/benchmarks/l2_distance/l2_distance.cpp @@ -45,7 +45,6 @@ void print_bool_arg(bool arg, const string &name, ostream &os) { os << (arg ? name : "no_" + name); } - int main(int argc, char **argv) { bool vectorize_code = true; @@ -76,13 +75,17 @@ int main(int argc, char **argv) if (argc > 7) const_folding = stoi(argv[7]); + float w_ops = 0.5; + float w_keys = 0.5; + if (argc > 8) w_ops = stof(argv[8]); + if (argc > 9) w_keys = stof(argv[9]); if (cse) { Compiler::enable_cse(); Compiler::enable_order_operands(); - } + } else { Compiler::disable_cse(); @@ -110,11 +113,10 @@ int main(int argc, char **argv) ofstream source_os(gen_path + ".cpp"); if (!source_os) throw logic_error("failed to create source file"); - cout << " window is " << window << endl; /********** vectorization Part *******************************/ if(VECTORIZATION_ENABLED){ - Compiler::gen_vectorized_code(func, window,optimization_method); // add a flag to specify if the benchmark is structured or no + Compiler::gen_vectorized_code(func, window,optimization_method, w_ops, w_keys); // add a flag to specify if the benchmark is structured or no } /********** Simplification & depth reduction Part ************/ if(SIMPLIFICATION_ENABLED){ @@ -122,7 +124,6 @@ int main(int argc, char **argv) auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; Compiler::compile(func, ruleset, rewrite_heuristic); } - /********** FHE code generation *****************************/ Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); @@ -134,7 +135,7 @@ int main(int argc, char **argv) quantifier.run_all_analysis(); quantifier.print_info(cout); } - } + } else { const auto &func = Compiler::create_func(func_name, slot_count, 20, false, true); @@ -154,7 +155,7 @@ int main(int argc, char **argv) cout << " window is " << window << endl; auto ruleset = Compiler::Ruleset::simplification_ruleset; auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; - Compiler::compile(func, ruleset, rewrite_heuristic); + Compiler::compile(func, ruleset, rewrite_heuristic); Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); /************/elapsed = chrono::high_resolution_clock::now() - t; cout<<"Compile time : \n"; diff --git a/benchmarks/lin_reg/lin_reg.cpp b/benchmarks/lin_reg/lin_reg.cpp index 72eb5fbe..4099b5f5 100644 --- a/benchmarks/lin_reg/lin_reg.cpp +++ b/benchmarks/lin_reg/lin_reg.cpp @@ -45,7 +45,6 @@ void print_bool_arg(bool arg, const string &name, ostream &os) { os << (arg ? name : "no_" + name); } - int main(int argc, char **argv) { bool vectorize_code = true; @@ -76,12 +75,17 @@ int main(int argc, char **argv) if (argc > 7) const_folding = stoi(argv[7]); + float w_ops = 0.5; + float w_keys = 0.5; + + if (argc > 8) w_ops = stof(argv[8]); + if (argc > 9) w_keys = stof(argv[9]); if (cse) { Compiler::enable_cse(); Compiler::enable_order_operands(); - } + } else { Compiler::disable_cse(); @@ -112,7 +116,7 @@ int main(int argc, char **argv) cout << " window is " << window << endl; /********** vectorization Part *******************************/ if(VECTORIZATION_ENABLED){ - Compiler::gen_vectorized_code(func, window,optimization_method); // add a flag to specify if the benchmark is structured or no + Compiler::gen_vectorized_code(func, window,optimization_method, w_ops, w_keys); // add a flag to specify if the benchmark is structured or no } /********** Simplification & depth reduction Part ************/ if(SIMPLIFICATION_ENABLED){ diff --git a/benchmarks/matrix_mul/matrix_mul.cpp b/benchmarks/matrix_mul/matrix_mul.cpp index c21b6b2c..5047f031 100644 --- a/benchmarks/matrix_mul/matrix_mul.cpp +++ b/benchmarks/matrix_mul/matrix_mul.cpp @@ -82,7 +82,6 @@ void print_bool_arg(bool arg, const string &name, ostream &os) { os << (arg ? name : "no_" + name); } -/************************************************************/ int main(int argc, char **argv) { bool vectorize_code = true; @@ -91,7 +90,7 @@ int main(int argc, char **argv) int slot_count = 1 ; if (argc > 2) - slot_count = stoi(argv[2]); + slot_count = stoi(argv[2]); int optimization_method = 0; // 0 = egraph (default), 1 = RL if (argc > 3) @@ -105,7 +104,7 @@ int main(int argc, char **argv) if (argc > 5) call_quantifier = stoi(argv[5]); - bool cse = true; + bool cse = true; if (argc > 6) cse = stoi(argv[6]); @@ -113,12 +112,17 @@ int main(int argc, char **argv) if (argc > 7) const_folding = stoi(argv[7]); - + float w_ops = 0.5; + float w_keys = 0.5; + + if (argc > 8) w_ops = stof(argv[8]); + if (argc > 9) w_keys = stof(argv[9]); + if (cse) { Compiler::enable_cse(); Compiler::enable_order_operands(); - } + } else { Compiler::disable_cse(); @@ -132,73 +136,73 @@ int main(int argc, char **argv) //Compiler::enable_auto_enc_params_selection(); chrono::high_resolution_clock::time_point t; chrono::duration elapsed; - string func_name = "fhe"; + string func_name = "fhe"; /**************/t = chrono::high_resolution_clock::now(); if (vectorize_code) { - const auto &func = Compiler::create_func(func_name, 1, 20, false, true); - fhe(slot_count); - string gen_name = "_gen_he_" + func_name; - string gen_path = "he/" + gen_name; - ofstream header_os(gen_path + ".hpp"); - if (!header_os) - throw logic_error("failed to create header file"); - ofstream source_os(gen_path + ".cpp"); - if (!source_os) - throw logic_error("failed to create source file"); - cout << " window is " << window << endl; - /********** vectorization Part *******************************/ - if(VECTORIZATION_ENABLED){ - Compiler::gen_vectorized_code(func, window,optimization_method); // add a flag to specify if the benchmark is structured or no - } - /********** Simplification & depth reduction Part ************/ - if(SIMPLIFICATION_ENABLED){ - auto ruleset = Compiler::Ruleset::depth; - auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; - Compiler::compile(func, ruleset, rewrite_heuristic); - } - /********** FHE code generation *****************************/ - Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); - - /************/elapsed = chrono::high_resolution_clock::now() - t; - cout << elapsed.count() << " ms\n"; - if (call_quantifier) - { - util::Quantifier quantifier{func}; - quantifier.run_all_analysis(); - quantifier.print_info(cout); - } - } - else - { - const auto &func = Compiler::create_func(func_name,slot_count*slot_count, 20, false, true); - // update io file - std::string updated_inputs_file_name = "fhe_io_example_adapted.txt" ; - std::string inputs_file_name = "fhe_io_example.txt"; - util::copyFile(inputs_file_name,updated_inputs_file_name); - fhe(slot_count); - string gen_name = "_gen_he_" + func_name; - string gen_path = "he/" + gen_name; - ofstream header_os(gen_path + ".hpp"); - if (!header_os) - throw logic_error("failed to create header file"); - ofstream source_os(gen_path + ".cpp"); - if (!source_os) - throw logic_error("failed to create source file"); - cout << " window is " << window << endl; - auto ruleset = Compiler::Ruleset::simplification_ruleset; + const auto &func = Compiler::create_func(func_name, 1, 20, false, true); + fhe(slot_count); + string gen_name = "_gen_he_" + func_name; + string gen_path = "he/" + gen_name; + ofstream header_os(gen_path + ".hpp"); + if (!header_os) + throw logic_error("failed to create header file"); + ofstream source_os(gen_path + ".cpp"); + if (!source_os) + throw logic_error("failed to create source file"); + cout << " window is " << window << endl; + /********** vectorization Part *******************************/ + if(VECTORIZATION_ENABLED){ + Compiler::gen_vectorized_code(func, window,optimization_method, w_ops, w_keys); // add a flag to specify if the benchmark is structured or no + } + /********** Simplification & depth reduction Part ************/ + if(SIMPLIFICATION_ENABLED){ + auto ruleset = Compiler::Ruleset::depth; auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; Compiler::compile(func, ruleset, rewrite_heuristic); - Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os, 29); - /************/elapsed = chrono::high_resolution_clock::now() - t; - cout<<"Compile time : \n"; - cout << elapsed.count() << " ms\n"; - if (call_quantifier) - { + } + /********** FHE code generation *****************************/ + Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); + + /************/elapsed = chrono::high_resolution_clock::now() - t; + cout << elapsed.count() << " ms\n"; + if (call_quantifier) + { util::Quantifier quantifier{func}; quantifier.run_all_analysis(); quantifier.print_info(cout); - } + } + } + else + { + const auto &func = Compiler::create_func(func_name, slot_count, 20, false, true); + // update_io_file + std::string updated_inputs_file_name = "fhe_io_example_adapted.txt" ; + std::string inputs_file_name = "fhe_io_example.txt"; + util::copyFile(inputs_file_name,updated_inputs_file_name); + fhe(slot_count); + string gen_name = "_gen_he_" + func_name; + string gen_path = "he/" + gen_name; + ofstream header_os(gen_path + ".hpp"); + if (!header_os) + throw logic_error("failed to create header file"); + ofstream source_os(gen_path + ".cpp"); + if (!source_os) + throw logic_error("failed to create source file"); + cout << " window is " << window << endl; + auto ruleset = Compiler::Ruleset::simplification_ruleset; + auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; + Compiler::compile(func, ruleset, rewrite_heuristic); + Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); + /************/elapsed = chrono::high_resolution_clock::now() - t; + cout<<"Compile time : \n"; + cout << elapsed.count() << " ms\n"; + if (call_quantifier) + { + util::Quantifier quantifier{func}; + quantifier.run_all_analysis(); + quantifier.print_info(cout); + } } return 0; } \ No newline at end of file diff --git a/benchmarks/max/max.cpp b/benchmarks/max/max.cpp index a4accc58..e4ab6b4e 100644 --- a/benchmarks/max/max.cpp +++ b/benchmarks/max/max.cpp @@ -66,7 +66,7 @@ void print_bool_arg(bool arg, const string &name, ostream &os) } int main(int argc, char **argv) { - bool vectorize_code = true; + bool vectorize_code = true; if (argc > 1) vectorize_code = stoi(argv[1]); @@ -94,12 +94,17 @@ int main(int argc, char **argv) if (argc > 7) const_folding = stoi(argv[7]); + float w_ops = 0.5; + float w_keys = 0.5; + + if (argc > 8) w_ops = stof(argv[8]); + if (argc > 9) w_keys = stof(argv[9]); if (cse) { Compiler::enable_cse(); Compiler::enable_order_operands(); - } + } else { Compiler::disable_cse(); @@ -118,7 +123,7 @@ int main(int argc, char **argv) if (vectorize_code) { const auto &func = Compiler::create_func(func_name, 1, 20, false, true); - fhe(slot_count); + fhe(slot_count); string gen_name = "_gen_he_" + func_name; string gen_path = "he/" + gen_name; ofstream header_os(gen_path + ".hpp"); @@ -130,13 +135,13 @@ int main(int argc, char **argv) cout << " window is " << window << endl; /********** vectorization Part *******************************/ if(VECTORIZATION_ENABLED){ - Compiler::gen_vectorized_code(func, window,optimization_method); // add a flag to specify if the benchmark is structured or no + Compiler::gen_vectorized_code(func, window,optimization_method, w_ops, w_keys); // add a flag to specify if the benchmark is structured or no } /********** Simplification & depth reduction Part ************/ if(SIMPLIFICATION_ENABLED){ - auto ruleset = Compiler::Ruleset::depth; - auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; - Compiler::compile(func, ruleset, rewrite_heuristic); + auto ruleset = Compiler::Ruleset::depth; + auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; + Compiler::compile(func, ruleset, rewrite_heuristic); } /********** FHE code generation *****************************/ Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); @@ -152,7 +157,7 @@ int main(int argc, char **argv) } else { - const auto &func = Compiler::create_func(func_name, 1, 20, false, true); + const auto &func = Compiler::create_func(func_name, slot_count, 20, false, true); // update_io_file std::string updated_inputs_file_name = "fhe_io_example_adapted.txt" ; std::string inputs_file_name = "fhe_io_example.txt"; @@ -182,4 +187,4 @@ int main(int argc, char **argv) } } return 0; -} +} \ No newline at end of file diff --git a/benchmarks/poly_reg/poly_reg.cpp b/benchmarks/poly_reg/poly_reg.cpp index 53d545d4..2631e7e8 100644 --- a/benchmarks/poly_reg/poly_reg.cpp +++ b/benchmarks/poly_reg/poly_reg.cpp @@ -91,12 +91,17 @@ int main(int argc, char **argv) if (argc > 7) const_folding = stoi(argv[7]); + float w_ops = 0.5; + float w_keys = 0.5; + + if (argc > 8) w_ops = stof(argv[8]); + if (argc > 9) w_keys = stof(argv[9]); if (cse) { Compiler::enable_cse(); Compiler::enable_order_operands(); - } + } else { Compiler::disable_cse(); @@ -127,7 +132,7 @@ int main(int argc, char **argv) cout << " window is " << window << endl; /********** vectorization Part *******************************/ if(VECTORIZATION_ENABLED){ - Compiler::gen_vectorized_code(func, window,optimization_method); // add a flag to specify if the benchmark is structured or no + Compiler::gen_vectorized_code(func, window,optimization_method, w_ops, w_keys); // add a flag to specify if the benchmark is structured or no } /********** Simplification & depth reduction Part ************/ if(SIMPLIFICATION_ENABLED){ diff --git a/benchmarks/polynomials_coyote/polynomials_coyote.cpp b/benchmarks/polynomials_coyote/polynomials_coyote.cpp index 5bab200d..7cb50ec9 100644 --- a/benchmarks/polynomials_coyote/polynomials_coyote.cpp +++ b/benchmarks/polynomials_coyote/polynomials_coyote.cpp @@ -201,6 +201,12 @@ int main(int argc, char **argv) { int optimization_method = 0; // 0 = egraph (default), 1 = RL if (argc > 5) optimization_method = stoi(argv[5]); + + float w_ops = 1.0; + float w_keys = 0.0; + + if (argc > 10) w_ops = std::stof(argv[10]); + if (argc > 11) w_keys = std::stof(argv[11]); if (cse) { Compiler::enable_cse(); @@ -239,7 +245,7 @@ int main(int argc, char **argv) { cout << " window is " << window << endl; /********** vectorization Part *******************************/ if(VECTORIZATION_ENABLED){ - Compiler::gen_vectorized_code(func, window,optimization_method); // add a flag to specify if the benchmark is structured or no + Compiler::gen_vectorized_code(func, window,optimization_method, w_ops, w_keys); // add a flag to specify if the benchmark is structured or no } /********** Simplification & depth reduction Part ************/ if(SIMPLIFICATION_ENABLED){ diff --git a/benchmarks/roberts_cross/roberts_cross.cpp b/benchmarks/roberts_cross/roberts_cross.cpp index ee05a5a1..d12a088f 100644 --- a/benchmarks/roberts_cross/roberts_cross.cpp +++ b/benchmarks/roberts_cross/roberts_cross.cpp @@ -71,13 +71,13 @@ void fhe(int width){ /*******************************************************************************************/ void print_bool_arg(bool arg, const string &name, ostream &os) { - os << (arg ? name : "no_" + name); + os << (arg ? name : "no_" + name); } int main(int argc, char **argv) { bool vectorize_code = true; if (argc > 1) - vectorize_code = stoi(argv[1]); + vectorize_code = stoi(argv[1]); int slot_count = 1 ; if (argc > 2) @@ -103,11 +103,17 @@ int main(int argc, char **argv) if (argc > 7) const_folding = stoi(argv[7]); + float w_ops = 0.5; + float w_keys = 0.5; + + if (argc > 8) w_ops = stof(argv[8]); + if (argc > 9) w_keys = stof(argv[9]); + if (cse) { Compiler::enable_cse(); Compiler::enable_order_operands(); - } + } else { Compiler::disable_cse(); @@ -125,68 +131,69 @@ int main(int argc, char **argv) /**************/t = chrono::high_resolution_clock::now(); if (vectorize_code) { - const auto &func = Compiler::create_func(func_name, 1, 20, false, true); - fhe(slot_count); - string gen_name = "_gen_he_" + func_name; - string gen_path = "he/" + gen_name; - ofstream header_os(gen_path + ".hpp"); - if (!header_os) - throw logic_error("failed to create header file"); - ofstream source_os(gen_path + ".cpp"); - if (!source_os) - throw logic_error("failed to create source file"); - cout << " window is " << window << endl; - /********** vectorization Part *******************************/ - if(VECTORIZATION_ENABLED){ - Compiler::gen_vectorized_code(func, window,optimization_method); // add a flag to specify if the benchmark is structured or no - } - /********** Simplification & depth reduction Part ************/ - if(SIMPLIFICATION_ENABLED){ + const auto &func = Compiler::create_func(func_name, 1, 20, false, true); + fhe(slot_count); + string gen_name = "_gen_he_" + func_name; + string gen_path = "he/" + gen_name; + ofstream header_os(gen_path + ".hpp"); + if (!header_os) + throw logic_error("failed to create header file"); + ofstream source_os(gen_path + ".cpp"); + if (!source_os) + throw logic_error("failed to create source file"); + cout << " window is " << window << endl; + /********** vectorization Part *******************************/ + if(VECTORIZATION_ENABLED){ + Compiler::gen_vectorized_code(func, window,optimization_method, w_ops, w_keys); // add a flag to specify if the benchmark is structured or no + } + /********** Simplification & depth reduction Part ************/ + if(SIMPLIFICATION_ENABLED){ auto ruleset = Compiler::Ruleset::depth; auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; Compiler::compile(func, ruleset, rewrite_heuristic); - } - /********** FHE code generation *****************************/ - Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); - - /************/elapsed = chrono::high_resolution_clock::now() - t; - cout << elapsed.count() << " ms\n"; - if (call_quantifier) - { - util::Quantifier quantifier{func}; - quantifier.run_all_analysis(); - quantifier.print_info(cout); - } - } - else - { - const auto &func = Compiler::create_func(func_name,slot_count*slot_count, 20, false, true); - // update io file - std::string updated_inputs_file_name = "fhe_io_example_adapted.txt" ; - std::string inputs_file_name = "fhe_io_example.txt"; - util::copyFile(inputs_file_name,updated_inputs_file_name); - fhe(slot_count); - string gen_name = "_gen_he_" + func_name; - string gen_path = "he/" + gen_name; - ofstream header_os(gen_path + ".hpp"); - if (!header_os) - throw logic_error("failed to create header file"); - ofstream source_os(gen_path + ".cpp"); - if (!source_os) - throw logic_error("failed to create source file"); - cout << " window is " << window << endl; - auto ruleset = Compiler::Ruleset::simplification_ruleset; - auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; - Compiler::compile(func, ruleset, rewrite_heuristic); - Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os, 29); - /************/elapsed = chrono::high_resolution_clock::now() - t; - cout << elapsed.count() << " ms\n"; - if (call_quantifier) - { + } + /********** FHE code generation *****************************/ + Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); + + /************/elapsed = chrono::high_resolution_clock::now() - t; + cout << elapsed.count() << " ms\n"; + if (call_quantifier) + { util::Quantifier quantifier{func}; quantifier.run_all_analysis(); quantifier.print_info(cout); - } + } + } + else + { + const auto &func = Compiler::create_func(func_name, slot_count, 20, false, true); + // update_io_file + std::string updated_inputs_file_name = "fhe_io_example_adapted.txt" ; + std::string inputs_file_name = "fhe_io_example.txt"; + util::copyFile(inputs_file_name,updated_inputs_file_name); + fhe(slot_count); + string gen_name = "_gen_he_" + func_name; + string gen_path = "he/" + gen_name; + ofstream header_os(gen_path + ".hpp"); + if (!header_os) + throw logic_error("failed to create header file"); + ofstream source_os(gen_path + ".cpp"); + if (!source_os) + throw logic_error("failed to create source file"); + cout << " window is " << window << endl; + auto ruleset = Compiler::Ruleset::simplification_ruleset; + auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; + Compiler::compile(func, ruleset, rewrite_heuristic); + Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); + /************/elapsed = chrono::high_resolution_clock::now() - t; + cout<<"Compile time : \n"; + cout << elapsed.count() << " ms\n"; + if (call_quantifier) + { + util::Quantifier quantifier{func}; + quantifier.run_all_analysis(); + quantifier.print_info(cout); + } } return 0; } diff --git a/benchmarks/sort/sort.cpp b/benchmarks/sort/sort.cpp index d19a6963..8b5c933f 100644 --- a/benchmarks/sort/sort.cpp +++ b/benchmarks/sort/sort.cpp @@ -149,7 +149,7 @@ int main(int argc, char **argv) int window = 0; if (argc > 4) - window = stoi(argv[4]); + window = stoi(argv[4]); bool call_quantifier = true; if (argc > 5) @@ -163,11 +163,17 @@ int main(int argc, char **argv) if (argc > 7) const_folding = stoi(argv[7]); + float w_ops = 0.5; + float w_keys = 0.5; + + if (argc > 8) w_ops = stof(argv[8]); + if (argc > 9) w_keys = stof(argv[9]); + if (cse) { Compiler::enable_cse(); Compiler::enable_order_operands(); - } + } else { Compiler::disable_cse(); @@ -184,7 +190,7 @@ int main(int argc, char **argv) string func_name = "fhe"; /**************/t = chrono::high_resolution_clock::now(); if (vectorize_code) - { + { const auto &func = Compiler::create_func(func_name, 1, 20, false, true); fhe(slot_count); string gen_name = "_gen_he_" + func_name; @@ -198,13 +204,13 @@ int main(int argc, char **argv) cout << " window is " << window << endl; /********** vectorization Part *******************************/ if(VECTORIZATION_ENABLED){ - Compiler::gen_vectorized_code(func, window,optimization_method); // add a flag to specify if the benchmark is structured or no + Compiler::gen_vectorized_code(func, window,optimization_method, w_ops, w_keys); // add a flag to specify if the benchmark is structured or no } /********** Simplification & depth reduction Part ************/ - if(SIMPLIFICATION_ENABLED){ - auto ruleset = Compiler::Ruleset::depth; - auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; - Compiler::compile(func, ruleset, rewrite_heuristic); + if(SIMPLIFICATION_ENABLED){ + auto ruleset = Compiler::Ruleset::depth; + auto rewrite_heuristic = trs::RewriteHeuristic::bottom_up; + Compiler::compile(func, ruleset, rewrite_heuristic); } /********** FHE code generation *****************************/ Compiler::gen_he_code(func, header_os, gen_name + ".hpp", source_os); @@ -220,7 +226,7 @@ int main(int argc, char **argv) } else { - const auto &func = Compiler::create_func(func_name, 1, 20, false, true); + const auto &func = Compiler::create_func(func_name, slot_count, 20, false, true); // update_io_file std::string updated_inputs_file_name = "fhe_io_example_adapted.txt" ; std::string inputs_file_name = "fhe_io_example.txt"; @@ -250,4 +256,4 @@ int main(int argc, char **argv) } } return 0; -} +} \ No newline at end of file diff --git a/circuit_execution_time_s.png b/circuit_execution_time_s.png new file mode 100644 index 00000000..3a3b9220 Binary files /dev/null and b/circuit_execution_time_s.png differ diff --git a/desktop.ini b/desktop.ini new file mode 100644 index 00000000..5b0f36da --- /dev/null +++ b/desktop.ini @@ -0,0 +1,2 @@ +[LocalizedFileNames] +try_job (2).sh=@try_job (2).sh,0 diff --git a/eval_kl.csv b/eval_kl.csv new file mode 100644 index 00000000..8a3f7723 --- /dev/null +++ b/eval_kl.csv @@ -0,0 +1,51 @@ +benchmark,w_ops,w_keys,add,sub,multiply_plain,rotate_rows,negate,multiply,Depth,Multiplicative Depth,compile_time (s),circuit_execution_time (s),galois_keys_generation_time (s),total_execution_time (s),Remaining_noise_budget,rotation_keys_size (MB),Final_Cost +max_3,1.0,0.0,2,1,0,3,0,2,8,2,11.890,0.495,0.343,0.837,295.0,54.3391,0.0 +max_4,1.0,0.0,3,1,0,5,0,3,12,3,21.543,0.716,0.570,1.285,262.0,90.4814,0.0 +max_5,1.0,0.0,7,1,0,5,2,7,15,4,267.937,1.435,0.573,2.008,229.0,90.4814,0.0 +sort_3,1.0,0.0 +box_blur_4,1.0,0.0,8,0,0,1,0,0,4,0,11.332,0.238,0.114,0.352,357.0,18.1967,0.0 +box_blur_8,1.0,0.0 +box_blur_16,1.0,0.0 +box_blur_32,1.0,0.0 +lin_reg_4,1.0,0.0,2,0,0,1,0,1,4,1,11.550,0.242,0.114,0.356,325.5,18.1967,0.0 +lin_reg_8,1.0,0.0,2,0,0,1,0,1,4,1,10.879,0.243,0.113,0.356,325.5,18.1967,0.0 +lin_reg_16,1.0,0.0,2,0,0,1,0,1,4,1,11.036,0.243,0.114,0.357,325.5,18.1967,0.0 +lin_reg_32,1.0,0.0,2,0,1,0,0,1,3,1,11.425,0.055,0.000,0.055,359.0,0.125542,0.0 +hamming_dist_4,1.0,0.0,4,2,0,3,0,4,7,2,11.264,0.882,0.225,1.107,295.0,36.2679,0.0 +hamming_dist_8,1.0,0.0,8,4,0,5,0,8,8,2,12.259,1.733,0.226,1.959,294.5,36.2679,0.0 +hamming_dist_16,1.0,0.0,22,8,0,9,0,16,13,2,19.453,3.414,0.226,3.640,294.0,36.2679,0.0 +hamming_dist_32,1.0,0.0,38,16,0,17,0,32,11,2,50.748,6.771,0.225,6.996,293.0,36.2679 +dot_product_4,1.0,0.0,2,0,0,3,0,2,5,1,11.484,0.470,0.227,0.698,325.0,36.2679,0.0 +dot_product_8,1.0,0.0,4,0,0,5,0,4,6,1,11.212,0.888,0.227,1.115,324.5,36.2679,0.0 +dot_product_16,1.0,0.0,8,0,0,9,0,8,7,1,11.737,1.721,0.228,1.949,324.0,36.2679,0.0 +dot_product_32,1.0,0.0,27,0,0,17,0,16,19,1,17.637,3.394,0.227,3.621,323.0,36.2679,0.0 +poly_reg_4,1.0,0.0,3,0,0,2,0,2,7,2,11.038,0.475,0.228,0.703,296.0,36.2679,0.0 +poly_reg_8,1.0,0.0,3,0,0,2,0,2,7,2,11.054,0.476,0.228,0.704,296.0,36.2679,0.0 +poly_reg_16,1.0,0.0,3,0,0,1,0,2,6,2,11.578,0.459,0.116,0.575,296.0,18.1967,0.0 +poly_reg_32,1.0,0.0,3,0,0,0,0,2,5,2,11.805,0.417,0.000,0.417,296.0,0.125542,0.0 +l2_distance_4,1.0,0.0,2,2,0,3,0,2,6,1,11.406,0.519,0.228,0.747,325.0,36.2679,0.0 +l2_distance_8,1.0,0.0,4,4,0,5,0,4,7,1,11.630,0.981,0.226,1.207,324.5,36.2679,0.0 +l2_distance_16,1.0,0.0,12,8,0,9,0,8,11,1,14.770,1.928,0.228,2.156,323.5,36.2679,0.0 +l2_distance_32,1.0,0.0,27,16,0,17,0,16,20,1,36.233,3.785,0.227,4.011,323.0,36.2679,0.0 +gx_kernel_4,1.0,0.0,4,0,0,2,2,2,6,1,11.873,0.498,0.114,0.612,327.0,18.1967,0.0 +gx_kernel_8,1.0,0.0 +gx_kernel_16,1.0,0.0 +gx_kernel_32,1.0,0.0 +gy_kernel_4,1.0,0.0,5,0,0,2,3,4,6,1,12.414,0.853,0.113,0.967,327.0,18.1967,0.0 +gy_kernel_8,1.0,0.0 +gy_kernel_16,1.0,0.0 +gy_kernel_32,1.0,0.0 +roberts_cross_4,1.0,0.0,3,0,0,1,3,1,5,1,12.099,0.267,0.114,0.380,327.5,18.1967,0.0 +roberts_cross_8,1.0,0.0 +roberts_cross_16,1.0,0.0 +roberts_cross_32,1.0,0.0 +matrix_mul_4,1.0,0.0,3,0,0,4,0,4,4,1,11.486,0.834,0.112,0.947,325.0,18.1967,0.0 +matrix_mul_8,1.0,0.0 +matrix_mul_16,1.0,0.0 +matrix_mul_32,1.0,0.0 +tree_50-50_5_1,1.0,0.0,6,0,0,3,0,2,8,2,11.076,0.573,0.229,0.802,295.5,36.2679,0.0 +tree_50-50_10_1,1.0,0.0,13,0,0,5,0,7,16,5,71.692,1.603,0.571,2.174,197.0,90.4814,0.0 +tree_100-50_5_1,1.0,0.0,13,0,0,5,0,4,16,3,83.706,1.106,0.572,1.678,261.0,90.4814,0.0 +tree_100-50_10_1,1.0,0.0,519,0,0,2,0,506,15,9,89.099,96.410,0.228,96.638,66.0,36.2679 +tree_100-100_5_1,1.0,0.0,1,0,0,5,0,4,10,4,12.647,0.812,0.573,1.385,228.0,90.4814,0.0 +tree_100-100_10_1,1.0,0.0,1,0,0,2,0,256,12,9,84.072,42.592,0.229,42.821,63.0,36.2679 \ No newline at end of file diff --git a/eval_morl_final_2.csv b/eval_morl_final_2.csv new file mode 100644 index 00000000..a500e47c --- /dev/null +++ b/eval_morl_final_2.csv @@ -0,0 +1,585 @@ +benchmark,w_ops,w_keys,add,sub,multiply_plain,rotate_rows,negate,multiply,Depth,Multiplicative Depth,compile_time (s),final_exec_cost,final_keys_cost +lin_reg_4,1.0,0.0,2.0,0.0,0.0,2.0,0.0,1.0,5.0,1.0,13.881,230.0,2.0 +lin_reg_4,0.9,0.1,2.0,0.0,0.0,2.0,0.0,1.0,5.0,1.0,11.539,230.0,2.0 +lin_reg_4,0.8,0.2,2.0,0.0,0.0,1.0,0.0,1.0,4.0,1.0,11.108,163.0,1.0 +lin_reg_4,0.7,0.3,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.961,108.0,0.0 +lin_reg_4,0.6,0.4,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.875,108.0,0.0 +lin_reg_4,0.5,0.5,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.724,108.0,0.0 +lin_reg_4,0.4,0.6,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.917,108.0,0.0 +lin_reg_4,0.3,0.7,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.849,108.0,0.0 +lin_reg_4,0.2,0.8,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.87,108.0,0.0 +lin_reg_4,0.1,0.9,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.769,108.0,0.0 +lin_reg_8,1.0,0.0,2.0,0.0,0.0,2.0,0.0,1.0,5.0,1.0,11.18,250.0,2.0 +lin_reg_8,0.9,0.1,2.0,0.0,0.0,2.0,0.0,1.0,5.0,1.0,11.082,250.0,2.0 +lin_reg_8,0.8,0.2,2.0,0.0,0.0,2.0,0.0,1.0,5.0,1.0,11.066,250.0,2.0 +lin_reg_8,0.7,0.3,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.884,108.0,0.0 +lin_reg_8,0.6,0.4,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.854,108.0,0.0 +lin_reg_8,0.5,0.5,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.76,108.0,0.0 +lin_reg_8,0.4,0.6,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.714,108.0,0.0 +lin_reg_8,0.3,0.7,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.733,108.0,0.0 +lin_reg_8,0.2,0.8,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.698,108.0,0.0 +lin_reg_8,0.1,0.9,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.782,108.0,0.0 +lin_reg_8,0.75,0.25,2.0,0.0,0.0,1.0,0.0,1.0,4.0,1.0,10.874,167.0,1.0 +lin_reg_16,1.0,0.0,2.0,0.0,0.0,1.0,0.0,1.0,4.0,1.0,11.074,175.0,1.0 +lin_reg_16,0.9,0.1,2.0,0.0,0.0,1.0,0.0,1.0,4.0,1.0,11.127,175.0,1.0 +lin_reg_16,0.8,0.2,2.0,0.0,0.0,1.0,0.0,1.0,4.0,1.0,11.181,175.0,1.0 +lin_reg_16,0.7,0.3,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.927,108.0,0.0 +lin_reg_16,0.6,0.4,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.066,108.0,0.0 +lin_reg_16,0.5,0.5,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.975,108.0,0.0 +lin_reg_16,0.4,0.6,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.034,108.0,0.0 +lin_reg_16,0.3,0.7,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.978,108.0,0.0 +lin_reg_16,0.2,0.8,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.072,108.0,0.0 +lin_reg_16,0.1,0.9,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,10.885,108.0,0.0 +lin_reg_32,1.0,0.0,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.088,108.0,0.0 +lin_reg_32,0.9,0.1,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.374,108.0,0.0 +lin_reg_32,0.8,0.2,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.349,108.0,0.0 +lin_reg_32,0.7,0.3,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.31,108.0,0.0 +lin_reg_32,0.6,0.4,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.229,108.0,0.0 +lin_reg_32,0.5,0.5,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.358,108.0,0.0 +lin_reg_32,0.4,0.6,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.381,108.0,0.0 +lin_reg_32,0.3,0.7,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.268,108.0,0.0 +lin_reg_32,0.2,0.8,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.331,108.0,0.0 +lin_reg_32,0.1,0.9,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.253,108.0,0.0 +hamming_dist_4,1.0,0.0,3.0,1.0,0.0,4.0,0.0,2.0,8.0,2.0,11.412,453.0,3.0 +hamming_dist_4,0.9,0.1,4.0,1.0,0.0,3.0,0.0,3.0,8.0,2.0,11.406,636.0,3.0 +hamming_dist_4,0.8,0.2,4.0,1.0,0.0,2.0,0.0,3.0,8.0,2.0,11.382,570.0,2.0 +hamming_dist_4,0.7,0.3,6.0,2.0,1.0,1.0,0.0,5.0,7.0,2.0,11.268,867.0,1.0 +hamming_dist_4,0.6,0.4,6.0,2.0,1.0,1.0,0.0,5.0,7.0,2.0,11.338,867.0,1.0 +hamming_dist_4,0.5,0.5,6.0,2.0,1.0,1.0,0.0,5.0,7.0,2.0,11.359,867.0,1.0 +hamming_dist_4,0.4,0.6,7.0,4.0,4.0,0.0,0.0,8.0,5.0,2.0,12.972,5011.0,0.0 +hamming_dist_4,0.3,0.7,7.0,4.0,4.0,0.0,0.0,8.0,5.0,2.0,13.077,5011.0,0.0 +hamming_dist_4,0.2,0.8,7.0,4.0,4.0,0.0,0.0,8.0,5.0,2.0,12.827,5011.0,0.0 +hamming_dist_4,0.1,0.9,7.0,4.0,4.0,0.0,0.0,8.0,5.0,2.0,12.895,5011.0,0.0 +hamming_dist_8,1.0,0.0,4.0,1.0,0.0,5.0,0.0,2.0,10.0,2.0,14.795,618.0,4.0 +hamming_dist_8,0.9,0.1,5.0,1.0,0.0,4.0,0.0,3.0,10.0,2.0,13.652,753.0,4.0 +hamming_dist_8,0.8,0.2,6.0,2.0,1.0,2.0,0.0,5.0,9.0,2.0,12.621,774.0,2.0 +hamming_dist_8,0.7,0.3,12.0,4.0,3.0,1.0,0.0,9.0,9.0,2.0,18.718,1276.0,1.0 +hamming_dist_8,0.6,0.4,12.0,4.0,3.0,1.0,0.0,9.0,9.0,2.0,18.786,1276.0,1.0 +hamming_dist_8,0.5,0.5,12.0,4.0,3.0,1.0,0.0,9.0,9.0,2.0,18.86,1276.0,1.0 +hamming_dist_8,0.4,0.6,15.0,8.0,8.0,0.0,0.0,16.0,6.0,2.0,14.812,10012.0,0.0 +hamming_dist_8,0.3,0.7,15.0,8.0,8.0,0.0,0.0,16.0,6.0,2.0,14.757,10012.0,0.0 +hamming_dist_8,0.2,0.8,15.0,8.0,8.0,0.0,0.0,16.0,6.0,2.0,14.504,10012.0,0.0 +hamming_dist_8,0.1,0.9,15.0,8.0,8.0,0.0,0.0,16.0,6.0,2.0,14.817,10012.0,0.0 +hamming_dist_8,0.85,0.15,5.0,1.0,0.0,3.0,0.0,3.0,10.0,2.0,13.134,639.0,3.0 +hamming_dist_16,1.0,0.0,6.0,1.0,0.0,5.0,0.0,3.0,12.0,2.0,39.972,1062.0,5.0 +hamming_dist_16,0.9,0.1,6.0,1.0,0.0,4.0,0.0,3.0,12.0,2.0,35.465,756.0,4.0 +hamming_dist_16,0.8,0.2,25.0,9.0,7.0,2.0,0.0,19.0,12.0,2.0,29.865,2093.0,2.0 +hamming_dist_16,0.7,0.3,22.0,8.0,7.0,1.0,0.0,17.0,12.0,2.0,29.39,2090.0,1.0 +hamming_dist_16,0.6,0.4,20.0,8.0,7.0,1.0,0.0,17.0,13.0,2.0,29.552,2091.0,1.0 +hamming_dist_16,0.5,0.5,20.0,8.0,7.0,1.0,0.0,17.0,13.0,2.0,29.791,2091.0,1.0 +hamming_dist_16,0.4,0.6,31.0,16.0,16.0,0.0,0.0,32.0,7.0,2.0,19.503,20013.0,0.0 +hamming_dist_16,0.3,0.7,31.0,16.0,16.0,0.0,0.0,32.0,7.0,2.0,19.623,20013.0,0.0 +hamming_dist_16,0.2,0.8,31.0,16.0,16.0,0.0,0.0,32.0,7.0,2.0,19.463,20013.0,0.0 +hamming_dist_16,0.1,0.9,31.0,16.0,16.0,0.0,0.0,32.0,7.0,2.0,19.43,20013.0,0.0 +hamming_dist_16,0.85,0.15,7.0,2.0,1.0,3.0,0.0,5.0,11.0,2.0,26.21,843.0,3.0 +max_4,1.0,0.0,3.0,1.0,0.0,5.0,0.0,3.0,12.0,3.0,31.36,912.0,5.0 +max_4,0.9,0.1,5.0,2.0,0.0,4.0,1.0,5.0,11.0,3.0,20.864,808.0,4.0 +max_4,0.8,0.2,9.0,0.0,0.0,2.0,4.0,7.0,11.0,3.0,13.488,1076.0,2.0 +max_4,0.7,0.3,17.0,1.0,0.0,3.0,6.0,11.0,12.0,3.0,23.041,4728.0,2.0 +max_4,0.6,0.4,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,5823.0,2.0 +max_4,0.5,0.5,16.0,0.0,0.0,1.0,7.0,13.0,10.0,3.0,15.852,6667.0,1.0 +max_4,0.4,0.6,12.0,0.0,0.0,0.0,6.0,12.0,8.0,3.0,14.089,7262.0,0.0 +max_4,0.3,0.7,12.0,0.0,0.0,0.0,6.0,12.0,8.0,3.0,14.07,7262.0,0.0 +max_4,0.2,0.8,12.0,0.0,0.0,0.0,6.0,12.0,8.0,3.0,13.872,7262.0,0.0 +max_4,0.1,0.9,12.0,0.0,0.0,0.0,6.0,12.0,8.0,3.0,13.933,7262.0,0.0 +max_4,0.85,0.15,6.0,1.0,0.0,3.0,2.0,6.0,11.0,3.0,17.65,793.0,3.0 +max_3,1.0,0.0,2.0,1.0,0.0,4.0,0.0,2.0,9.0,2.0,14.691,501.0,4.0 +max_3,0.9,0.1,2.0,1.0,0.0,3.0,0.0,2.0,8.0,2.0,11.911,386.0,3.0 +max_3,0.8,0.2,2.0,1.0,0.0,3.0,0.0,2.0,8.0,2.0,11.738,386.0,3.0 +max_3,0.7,0.3,7.0,1.0,0.0,3.0,2.0,4.0,10.0,2.0,15.492,1624.0,2.0 +max_3,0.6,0.4,8.0,0.0,0.0,1.0,3.0,5.0,8.0,2.0,13.363,2412.0,1.0 +max_3,0.5,0.5,8.0,0.0,0.0,1.0,3.0,5.0,8.0,2.0,13.349,2662.0,1.0 +max_3,0.4,0.6,6.0,0.0,0.0,0.0,3.0,6.0,6.0,2.0,12.464,3009.0,0.0 +max_3,0.3,0.7,6.0,0.0,0.0,0.0,3.0,6.0,6.0,2.0,12.683,3259.0,0.0 +max_3,0.2,0.8,6.0,0.0,0.0,0.0,3.0,6.0,6.0,2.0,12.755,3259.0,0.0 +max_3,0.1,0.9,6.0,0.0,0.0,0.0,3.0,6.0,6.0,2.0,12.524,3260.0,0.0 +max_5,1.0,0.0,12.0,2.0,0.0,5.0,4.0,12.0,14.0,4.0,298.543,1632.0,4.0 +max_5,0.9,0.1,13.0,2.0,0.0,4.0,5.0,13.0,14.0,4.0,163.896,1619.0,4.0 +max_5,0.8,0.2,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,14108.0,3.0 +max_5,0.7,0.3,33.0,1.0,0.0,3.0,14.0,27.0,14.0,4.0,47.292,12731.0,2.0 +max_5,0.6,0.4,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,30891.0,2.0 +max_5,0.5,0.5,28.0,0.0,0.0,1.0,13.0,25.0,12.0,4.0,24.805,14670.0,1.0 +max_5,0.4,0.6,20.0,0.0,0.0,0.0,10.0,20.0,10.0,4.0,17.417,15265.0,0.0 +max_5,0.3,0.7,20.0,0.0,0.0,0.0,10.0,20.0,10.0,4.0,17.902,15265.0,0.0 +max_5,0.2,0.8,20.0,0.0,0.0,0.0,10.0,20.0,10.0,4.0,17.727,15265.0,0.0 +max_5,0.1,0.9,20.0,0.0,0.0,0.0,10.0,20.0,10.0,4.0,18.081,15265.0,0.0 +sort_3,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,912.0,5.0 +sort_3,0.9,0.1,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,2755.0,4.0 +sort_3,0.8,0.2,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,3171.0,2.0 +sort_3,0.7,0.3,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,3476.0,2.0 +sort_3,0.6,0.4,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,4071.0,2.0 +sort_3,0.5,0.5,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,4667.0,1.0 +sort_3,0.4,0.6,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,5262.0,0.0 +sort_3,0.3,0.7,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,5262.0,0.0 +sort_3,0.2,0.8,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,5262.0,0.0 +sort_3,0.1,0.9,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,5262.0,0.0 +sort_3,0.85,0.15,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,2755.0,4.0 +sort_3,0.825,0.175,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,2755.0,4.0 +sort_3,0.8125,0.1875,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,2755.0,4.0 +sort_3,0.80625,0.19375,0.0,0.0,0.0,0.0,0.0,0.0,0,0,0,3140.0,3.0 +box_blur_4,1.0,0.0,5.0,0.0,0.0,1.0,0.0,0.0,5.0,0.0,11.811,78.0,1.0 +box_blur_4,0.9,0.1,5.0,0.0,0.0,1.0,0.0,0.0,5.0,0.0,11.867,78.0,1.0 +box_blur_4,0.8,0.2,8.0,0.0,0.0,0.0,0.0,0.0,4.0,0.0,11.979,14.0,0.0 +box_blur_4,0.7,0.3,8.0,0.0,0.0,0.0,0.0,0.0,4.0,0.0,11.698,14.0,0.0 +box_blur_4,0.6,0.4,8.0,0.0,0.0,0.0,0.0,0.0,4.0,0.0,11.759,14.0,0.0 +box_blur_4,0.5,0.5,8.0,0.0,0.0,0.0,0.0,0.0,4.0,0.0,15.521,14.0,0.0 +box_blur_4,0.4,0.6,8.0,0.0,0.0,0.0,0.0,0.0,4.0,0.0,11.737,14.0,0.0 +box_blur_4,0.3,0.7,8.0,0.0,0.0,0.0,0.0,0.0,4.0,0.0,11.756,14.0,0.0 +box_blur_4,0.2,0.8,8.0,0.0,0.0,0.0,0.0,0.0,5.0,0.0,12.067,15.0,0.0 +box_blur_4,0.1,0.9,8.0,0.0,0.0,0.0,0.0,0.0,6.0,0.0,11.798,16.0,0.0 +box_blur_8,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_8,0.9,0.1,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_8,0.8,0.2,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_8,0.7,0.3,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_8,0.6,0.4,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_8,0.5,0.5,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_8,0.4,0.6,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_8,0.3,0.7,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_8,0.2,0.8,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_8,0.1,0.9,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_16,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_16,0.9,0.1,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_16,0.8,0.2,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_16,0.7,0.3,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_16,0.6,0.4,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_16,0.5,0.5,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_16,0.4,0.6,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_16,0.3,0.7,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_16,0.2,0.8,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_16,0.1,0.9,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_32,1.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_32,0.9,0.1,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_32,0.8,0.2,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_32,0.7,0.3,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_32,0.6,0.4,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_32,0.5,0.5,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_32,0.4,0.6,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_32,0.3,0.7,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_32,0.2,0.8,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +box_blur_32,0.1,0.9,0.0,0.0,0.0,0.0,0.0,0.0,,,,, +hamming_dist_32,1.0,0.0,7.0,1.0,0.0,5.0,0.0,3.0,14.0,2.0,363.44,1065.0,5.0 +hamming_dist_32,0.9,0.1,7.0,1.0,0.0,5.0,0.0,3.0,14.0,2.0,361.433,1065.0,5.0 +hamming_dist_32,0.8,0.2,34.0,16.0,15.0,1.0,0.0,33.0,10.0,2.0,73.009,5312.0,1.0 +hamming_dist_32,0.7,0.3,34.0,16.0,15.0,1.0,0.0,33.0,10.0,2.0,72.793,5312.0,1.0 +hamming_dist_32,0.6,0.4,34.0,16.0,15.0,1.0,0.0,33.0,10.0,2.0,73.543,5312.0,1.0 +hamming_dist_32,0.5,0.5,34.0,16.0,15.0,1.0,0.0,33.0,10.0,2.0,72.809,5312.0,1.0 +hamming_dist_32,0.4,0.6,34.0,16.0,15.0,1.0,0.0,33.0,10.0,2.0,72.794,5312.0,1.0 +hamming_dist_32,0.3,0.7,63.0,32.0,32.0,0.0,0.0,64.0,8.0,2.0,33.845,40014.0,0.0 +hamming_dist_32,0.2,0.8,63.0,32.0,32.0,0.0,0.0,64.0,8.0,2.0,33.194,40014.0,0.0 +hamming_dist_32,0.1,0.9,63.0,32.0,32.0,0.0,0.0,64.0,8.0,2.0,33.595,40014.0,0.0 +hamming_dist_32,0.85,0.15,28.0,8.0,7.0,5.0,0.0,17.0,17.0,2.0,116.382,2049.0,1.0 +hamming_dist_32,0.875,0.125,30.0,8.0,7.0,5.0,0.0,17.0,17.0,2.0,120.286,2048.0,1.0 +hamming_dist_32,0.8875,0.1125,10.0,3.0,2.0,4.0,0.0,7.0,12.0,2.0,162.958,1163.0,4.0 +hamming_dist_32,0.88125,0.11875,11.0,4.0,3.0,3.0,0.0,9.0,12.0,2.0,121.654,1250.0,3.0 +hamming_dist_32,0.878125,0.121875,30.0,8.0,7.0,5.0,0.0,17.0,17.0,2.0,121.057,2048.0,1.0 +hamming_dist_32,0.8796875,0.1203125,30.0,8.0,7.0,5.0,0.0,17.0,17.0,2.0,121.01,2048.0,1.0 +dot_product_4,1.0,0.0,2.0,0.0,0.0,3.0,0.0,1.0,6.0,1.0,14.497,282.0,3.0 +dot_product_4,0.9,0.1,2.0,0.0,0.0,3.0,0.0,1.0,6.0,1.0,10.107,282.0,3.0 +dot_product_4,0.8,0.2,2.0,0.0,0.0,2.0,0.0,1.0,5.0,1.0,9.989,215.0,2.0 +dot_product_4,0.7,0.3,2.0,0.0,0.0,1.0,0.0,2.0,4.0,1.0,9.74,260.0,1.0 +dot_product_4,0.6,0.4,2.0,0.0,0.0,1.0,0.0,2.0,4.0,1.0,9.663,260.0,1.0 +dot_product_4,0.5,0.5,2.0,0.0,0.0,1.0,0.0,2.0,4.0,1.0,9.735,260.0,1.0 +dot_product_4,0.4,0.6,3.0,0.0,0.0,0.0,0.0,4.0,3.0,1.0,9.394,1756.0,0.0 +dot_product_4,0.3,0.7,3.0,0.0,0.0,0.0,0.0,4.0,3.0,1.0,9.606,1756.0,0.0 +dot_product_4,0.2,0.8,3.0,0.0,0.0,0.0,0.0,4.0,3.0,1.0,9.506,1756.0,0.0 +dot_product_4,0.1,0.9,3.0,0.0,0.0,0.0,0.0,4.0,3.0,1.0,9.412,1756.0,0.0 +dot_product_8,1.0,0.0,3.0,0.0,0.0,4.0,0.0,1.0,8.0,1.0,10.054,399.0,4.0 +dot_product_8,0.9,0.1,3.0,0.0,0.0,4.0,0.0,1.0,8.0,1.0,9.991,399.0,4.0 +dot_product_8,0.8,0.2,3.0,0.0,0.0,3.0,0.0,1.0,7.0,1.0,9.907,284.0,3.0 +dot_product_8,0.7,0.3,4.0,0.0,0.0,1.0,0.0,4.0,5.0,1.0,9.724,463.0,1.0 +dot_product_8,0.6,0.4,4.0,0.0,0.0,1.0,0.0,4.0,5.0,1.0,9.765,463.0,1.0 +dot_product_8,0.5,0.5,6.0,0.0,0.0,1.0,0.0,4.0,6.0,1.0,9.963,464.0,1.0 +dot_product_8,0.4,0.6,7.0,0.0,0.0,0.0,0.0,8.0,4.0,1.0,11.121,3757.0,0.0 +dot_product_8,0.3,0.7,7.0,0.0,0.0,0.0,0.0,8.0,4.0,1.0,11.079,4008.0,0.0 +dot_product_8,0.2,0.8,7.0,0.0,0.0,0.0,0.0,8.0,4.0,1.0,11.256,4008.0,0.0 +dot_product_8,0.1,0.9,7.0,0.0,0.0,0.0,0.0,8.0,4.0,1.0,11.06,4008.0,0.0 +dot_product_8,0.75,0.25,3.0,0.0,0.0,2.0,0.0,2.0,6.0,1.0,9.588,317.0,2.0 +dot_product_16,1.0,0.0,4.0,0.0,0.0,5.0,0.0,1.0,10.0,1.0,12.214,708.0,5.0 +dot_product_16,0.9,0.1,4.0,0.0,0.0,4.0,0.0,1.0,9.0,1.0,11.553,401.0,4.0 +dot_product_16,0.8,0.2,10.0,0.0,0.0,3.0,0.0,4.0,11.0,1.0,11.062,572.0,1.0 +dot_product_16,0.7,0.3,14.0,0.0,0.0,1.0,0.0,8.0,14.0,1.0,14.89,876.0,1.0 +dot_product_16,0.6,0.4,13.0,0.0,0.0,1.0,0.0,8.0,12.0,1.0,15.213,874.0,1.0 +dot_product_16,0.5,0.5,8.0,0.0,0.0,1.0,0.0,8.0,10.0,1.0,14.638,871.0,1.0 +dot_product_16,0.4,0.6,15.0,0.0,0.0,0.0,0.0,16.0,5.0,1.0,12.561,8010.0,0.0 +dot_product_16,0.3,0.7,15.0,0.0,0.0,0.0,0.0,16.0,5.0,1.0,12.855,8010.0,0.0 +dot_product_16,0.2,0.8,15.0,0.0,0.0,0.0,0.0,16.0,5.0,1.0,12.427,8010.0,0.0 +dot_product_16,0.1,0.9,15.0,0.0,0.0,0.0,0.0,16.0,5.0,1.0,12.399,8010.0,0.0 +dot_product_16,0.85,0.15,4.0,0.0,0.0,4.0,0.0,1.0,9.0,1.0,11.363,401.0,4.0 +dot_product_16,0.825,0.175,4.0,0.0,0.0,3.0,0.0,2.0,8.0,1.0,10.889,386.0,3.0 +dot_product_16,0.8125,0.1875,10.0,0.0,0.0,3.0,0.0,4.0,11.0,1.0,10.9,572.0,1.0 +dot_product_16,0.81875,0.18125,4.0,0.0,0.0,3.0,0.0,2.0,8.0,1.0,10.819,386.0,3.0 +dot_product_16,0.815625,0.184375,4.0,0.0,0.0,3.0,0.0,2.0,8.0,1.0,10.976,386.0,3.0 +dot_product_16,0.8140625,0.1859375,10.0,0.0,0.0,3.0,0.0,4.0,11.0,1.0,11.053,572.0,1.0 +dot_product_32,1.0,0.0,5.0,0.0,0.0,5.0,0.0,1.0,11.0,1.0,29.367,710.0,5.0 +dot_product_32,0.9,0.1,5.0,0.0,0.0,4.0,0.0,2.0,10.0,1.0,21.738,503.0,4.0 +dot_product_32,0.8,0.2,16.0,0.0,0.0,1.0,0.0,16.0,12.0,1.0,22.162,1681.0,1.0 +dot_product_32,0.7,0.3,16.0,0.0,0.0,1.0,0.0,16.0,15.0,1.0,21.826,1684.0,1.0 +dot_product_32,0.6,0.4,16.0,0.0,0.0,1.0,0.0,16.0,15.0,1.0,21.844,1686.0,1.0 +dot_product_32,0.5,0.5,16.0,0.0,0.0,1.0,0.0,16.0,15.0,1.0,22.087,1686.0,1.0 +dot_product_32,0.4,0.6,31.0,0.0,0.0,0.0,0.0,32.0,6.0,1.0,15.69,16011.0,0.0 +dot_product_32,0.3,0.7,31.0,0.0,0.0,0.0,0.0,32.0,6.0,1.0,15.618,16011.0,0.0 +dot_product_32,0.2,0.8,31.0,0.0,0.0,0.0,0.0,32.0,6.0,1.0,15.392,16011.0,0.0 +dot_product_32,0.1,0.9,31.0,0.0,0.0,0.0,0.0,32.0,6.0,1.0,15.872,16011.0,0.0 +dot_product_32,0.85,0.15,6.0,0.0,0.0,3.0,0.0,4.0,9.0,1.0,17.29,589.0,3.0 +dot_product_32,0.825,0.175,15.0,0.0,0.0,3.0,0.0,8.0,18.0,1.0,19.431,1082.0,1.0 +dot_product_32,0.8375,0.1625,6.0,0.0,0.0,3.0,0.0,4.0,9.0,1.0,17.377,589.0,3.0 +dot_product_32,0.83125,0.16875,6.0,0.0,0.0,3.0,0.0,4.0,9.0,1.0,17.255,589.0,3.0 +dot_product_32,0.828125,0.171875,15.0,0.0,0.0,3.0,0.0,8.0,18.0,1.0,19.71,1082.0,1.0 +dot_product_32,0.8296875,0.1703125,15.0,0.0,0.0,3.0,0.0,8.0,18.0,1.0,19.83,1082.0,1.0 +l2_distance_4,1.0,0.0,2.0,1.0,0.0,3.0,0.0,1.0,7.0,1.0,9.99,284.0,3.0 +l2_distance_4,0.9,0.1,2.0,1.0,0.0,3.0,0.0,1.0,7.0,1.0,9.759,284.0,3.0 +l2_distance_4,0.8,0.2,2.0,1.0,0.0,2.0,0.0,1.0,6.0,1.0,10.128,218.0,2.0 +l2_distance_4,0.7,0.3,2.0,2.0,0.0,1.0,0.0,2.0,5.0,1.0,9.812,265.0,1.0 +l2_distance_4,0.6,0.4,2.0,2.0,0.0,1.0,0.0,2.0,5.0,1.0,9.866,265.0,1.0 +l2_distance_4,0.5,0.5,2.0,2.0,0.0,1.0,0.0,2.0,5.0,1.0,9.399,265.0,1.0 +l2_distance_4,0.4,0.6,3.0,4.0,0.0,0.0,0.0,4.0,4.0,1.0,9.729,3757.0,0.0 +l2_distance_4,0.3,0.7,3.0,4.0,0.0,0.0,0.0,4.0,4.0,1.0,9.643,3757.0,0.0 +l2_distance_4,0.2,0.8,3.0,4.0,0.0,0.0,0.0,4.0,4.0,1.0,11.152,4008.0,0.0 +l2_distance_4,0.1,0.9,3.0,4.0,0.0,0.0,0.0,4.0,4.0,1.0,11.225,4008.0,0.0 +l2_distance_8,1.0,0.0,3.0,1.0,0.0,4.0,0.0,1.0,9.0,1.0,11.122,401.0,4.0 +l2_distance_8,0.9,0.1,3.0,1.0,0.0,4.0,0.0,1.0,9.0,1.0,11.362,401.0,4.0 +l2_distance_8,0.8,0.2,3.0,2.0,0.0,2.0,0.0,2.0,7.0,1.0,10.304,322.0,2.0 +l2_distance_8,0.7,0.3,6.0,4.0,0.0,1.0,0.0,4.0,8.0,1.0,14.967,473.0,1.0 +l2_distance_8,0.6,0.4,5.0,4.0,0.0,1.0,0.0,4.0,8.0,1.0,15.104,473.0,1.0 +l2_distance_8,0.5,0.5,4.0,4.0,0.0,1.0,0.0,4.0,6.0,1.0,14.517,473.0,1.0 +l2_distance_8,0.4,0.6,7.0,8.0,0.0,0.0,0.0,8.0,5.0,1.0,12.257,8010.0,0.0 +l2_distance_8,0.3,0.7,7.0,8.0,0.0,0.0,0.0,8.0,5.0,1.0,12.659,8010.0,0.0 +l2_distance_8,0.2,0.8,7.0,8.0,0.0,0.0,0.0,8.0,5.0,1.0,12.606,8010.0,0.0 +l2_distance_8,0.1,0.9,7.0,8.0,0.0,0.0,0.0,8.0,5.0,1.0,12.608,8010.0,0.0 +l2_distance_8,0.85,0.15,3.0,1.0,0.0,3.0,0.0,1.0,8.0,1.0,10.991,287.0,3.0 +l2_distance_16,1.0,0.0,4.0,1.0,0.0,5.0,0.0,1.0,11.0,1.0,29.749,710.0,5.0 +l2_distance_16,0.9,0.1,4.0,1.0,0.0,4.0,0.0,1.0,10.0,1.0,21.716,404.0,4.0 +l2_distance_16,0.8,0.2,15.0,8.0,0.0,1.0,0.0,8.0,13.0,1.0,22.951,891.0,1.0 +l2_distance_16,0.7,0.3,8.0,8.0,0.0,1.0,0.0,8.0,8.0,1.0,21.944,887.0,1.0 +l2_distance_16,0.6,0.4,8.0,8.0,0.0,1.0,0.0,8.0,11.0,1.0,22.154,888.0,1.0 +l2_distance_16,0.5,0.5,8.0,8.0,0.0,1.0,0.0,8.0,11.0,1.0,21.989,888.0,1.0 +l2_distance_16,0.4,0.6,15.0,16.0,0.0,0.0,0.0,16.0,6.0,1.0,15.68,16011.0,0.0 +l2_distance_16,0.3,0.7,15.0,16.0,0.0,0.0,0.0,16.0,6.0,1.0,15.63,16011.0,0.0 +l2_distance_16,0.2,0.8,15.0,16.0,0.0,0.0,0.0,16.0,6.0,1.0,15.525,16011.0,0.0 +l2_distance_16,0.1,0.9,15.0,16.0,0.0,0.0,0.0,16.0,6.0,1.0,15.761,16011.0,0.0 +l2_distance_16,0.85,0.15,4.0,2.0,0.0,3.0,0.0,2.0,9.0,1.0,17.086,391.0,3.0 +l2_distance_16,0.825,0.175,10.0,4.0,0.0,3.0,0.0,4.0,14.0,1.0,29.273,582.0,1.0 +l2_distance_16,0.8375,0.1625,4.0,2.0,0.0,3.0,0.0,2.0,9.0,1.0,17.56,391.0,3.0 +l2_distance_16,0.83125,0.16875,10.0,4.0,0.0,3.0,0.0,4.0,14.0,1.0,29.634,582.0,1.0 +l2_distance_16,0.834375,0.165625,10.0,4.0,0.0,3.0,0.0,4.0,14.0,1.0,29.676,582.0,1.0 +l2_distance_16,0.8359375,0.1640625,10.0,4.0,0.0,3.0,0.0,4.0,14.0,1.0,29.788,582.0,1.0 +l2_distance_32,1.0,0.0,5.0,1.0,0.0,5.0,0.0,1.0,12.0,1.0,171.827,713.0,5.0 +l2_distance_32,0.9,0.1,6.0,4.0,0.0,3.0,0.0,4.0,10.0,1.0,62.655,598.0,3.0 +l2_distance_32,0.8,0.2,16.0,16.0,0.0,1.0,0.0,16.0,11.0,1.0,48.821,1713.0,1.0 +l2_distance_32,0.7,0.3,16.0,16.0,0.0,1.0,0.0,16.0,15.0,1.0,48.592,1717.0,1.0 +l2_distance_32,0.6,0.4,16.0,16.0,0.0,1.0,0.0,16.0,16.0,1.0,48.845,1719.0,1.0 +l2_distance_32,0.5,0.5,16.0,16.0,0.0,1.0,0.0,16.0,16.0,1.0,48.185,1719.0,1.0 +l2_distance_32,0.4,0.6,31.0,32.0,0.0,0.0,0.0,32.0,7.0,1.0,24.395,32012.0,0.0 +l2_distance_32,0.3,0.7,31.0,32.0,0.0,0.0,0.0,32.0,7.0,1.0,23.779,32012.0,0.0 +l2_distance_32,0.2,0.8,31.0,32.0,0.0,0.0,0.0,32.0,7.0,1.0,24.63,32012.0,0.0 +l2_distance_32,0.1,0.9,31.0,32.0,0.0,0.0,0.0,32.0,7.0,1.0,24.054,32012.0,0.0 +l2_distance_32,0.85,0.15,15.0,8.0,0.0,3.0,0.0,8.0,19.0,1.0,66.68,1098.0,1.0 +l2_distance_32,0.875,0.125,6.0,4.0,0.0,3.0,0.0,4.0,10.0,1.0,61.519,598.0,3.0 +l2_distance_32,0.8625,0.1375,15.0,8.0,0.0,3.0,0.0,8.0,19.0,1.0,67.531,1098.0,1.0 +l2_distance_32,0.86875,0.13125,15.0,8.0,0.0,3.0,0.0,8.0,19.0,1.0,67.468,1098.0,1.0 +l2_distance_32,0.871875,0.128125,15.0,8.0,0.0,3.0,0.0,8.0,19.0,1.0,68.646,1098.0,1.0 +l2_distance_32,0.8734375,0.1265625,6.0,4.0,0.0,3.0,0.0,4.0,10.0,1.0,61.552,598.0,3.0 +l2_distance_32,0.95,0.05,5.0,1.0,0.0,5.0,0.0,1.0,12.0,1.0,169.801,713.0,5.0 +l2_distance_32,0.925,0.075,5.0,1.0,0.0,5.0,0.0,1.0,12.0,1.0,173.63,713.0,5.0 +l2_distance_32,0.9125,0.0875,5.0,1.0,0.0,5.0,0.0,1.0,12.0,1.0,173.541,713.0,5.0 +l2_distance_32,0.90625,0.09375,6.0,4.0,0.0,3.0,0.0,4.0,10.0,1.0,63.185,598.0,3.0 +l2_distance_32,0.909375,0.090625,5.0,1.0,0.0,5.0,0.0,1.0,12.0,1.0,171.267,713.0,5.0 +l2_distance_32,0.9078125,0.0921875,6.0,4.0,0.0,3.0,0.0,4.0,10.0,1.0,63.816,598.0,3.0 +gx_kernel_4,1.0,0.0,4.0,0.0,1.0,1.0,2.0,3.0,6.0,1.0,13.013,381.0,1.0 +gx_kernel_4,0.9,0.1,4.0,0.0,1.0,1.0,2.0,4.0,7.0,2.0,13.537,483.0,1.0 +gx_kernel_4,0.8,0.2,4.0,0.0,1.0,1.0,2.0,4.0,7.0,2.0,13.469,483.0,1.0 +gx_kernel_4,0.7,0.3,5.0,0.0,2.0,0.0,2.0,4.0,6.0,2.0,11.922,417.0,0.0 +gx_kernel_4,0.6,0.4,5.0,0.0,2.0,0.0,2.0,4.0,6.0,2.0,11.936,417.0,0.0 +gx_kernel_4,0.5,0.5,5.0,0.0,2.0,0.0,2.0,5.0,6.0,3.0,12.111,518.0,0.0 +gx_kernel_4,0.4,0.6,5.0,0.0,2.0,0.0,2.0,5.0,6.0,3.0,12.137,518.0,0.0 +gx_kernel_4,0.3,0.7,5.0,0.0,2.0,0.0,2.0,5.0,6.0,3.0,12.12,518.0,0.0 +gx_kernel_4,0.2,0.8,5.0,0.0,2.0,0.0,2.0,5.0,6.0,3.0,12.095,518.0,0.0 +gx_kernel_4,0.1,0.9,3.0,2.0,2.0,0.0,0.0,5.0,6.0,3.0,12.269,516.0,0.0 +gx_kernel_8,1.0,0.0,,,,,,,,,,, +gx_kernel_8,0.9,0.1,,,,,,,,,,, +gx_kernel_8,0.8,0.2,,,,,,,,,,, +gx_kernel_8,0.7,0.3,,,,,,,,,,, +gx_kernel_8,0.6,0.4,,,,,,,,,,, +gx_kernel_8,0.5,0.5,,,,,,,,,,, +gx_kernel_8,0.4,0.6,,,,,,,,,,, +gx_kernel_8,0.3,0.7,,,,,,,,,,, +gx_kernel_8,0.2,0.8,,,,,,,,,,, +gx_kernel_8,0.1,0.9,,,,,,,,,,, +gx_kernel_16,1.0,0.0,,,,,,,,,,, +gx_kernel_16,0.9,0.1,,,,,,,,,,, +gx_kernel_16,0.8,0.2,,,,,,,,,,, +gx_kernel_16,0.7,0.3,,,,,,,,,,, +gx_kernel_16,0.6,0.4,,,,,,,,,,, +gx_kernel_16,0.5,0.5,,,,,,,,,,, +gx_kernel_16,0.4,0.6,,,,,,,,,,, +gx_kernel_16,0.3,0.7,,,,,,,,,,, +gx_kernel_16,0.2,0.8,,,,,,,,,,, +gx_kernel_16,0.1,0.9,,,,,,,,,,, +gx_kernel_32,1.0,0.0,,,,,,,,,,, +gx_kernel_32,0.9,0.1,,,,,,,,,,, +gx_kernel_32,0.8,0.2,,,,,,,,,,, +gx_kernel_32,0.7,0.3,,,,,,,,,,, +gx_kernel_32,0.6,0.4,,,,,,,,,,, +gx_kernel_32,0.5,0.5,,,,,,,,,,, +gx_kernel_32,0.4,0.6,,,,,,,,,,, +gx_kernel_32,0.3,0.7,,,,,,,,,,, +gx_kernel_32,0.2,0.8,,,,,,,,,,, +gx_kernel_32,0.1,0.9,,,,,,,,,,, +gy_kernel_4,1.0,0.0,3.0,0.0,0.0,1.0,3.0,3.0,6.0,1.0,12.608,381.0,1.0 +gy_kernel_4,0.9,0.1,3.0,0.0,0.0,1.0,3.0,4.0,7.0,2.0,13.435,483.0,1.0 +gy_kernel_4,0.8,0.2,7.0,0.0,0.0,1.0,4.0,7.0,7.0,2.0,13.765,1284.0,1.0 +gy_kernel_4,0.7,0.3,6.0,1.0,0.0,0.0,4.0,5.0,6.0,1.0,12.265,1015.0,0.0 +gy_kernel_4,0.6,0.4,6.0,1.0,0.0,0.0,4.0,6.0,7.0,2.0,12.709,1117.0,0.0 +gy_kernel_4,0.5,0.5,5.0,2.0,0.0,0.0,3.0,7.0,8.0,3.0,12.887,1218.0,0.0 +gy_kernel_4,0.4,0.6,5.0,2.0,0.0,0.0,3.0,8.0,8.0,3.0,13.301,1318.0,0.0 +gy_kernel_4,0.3,0.7,5.0,2.0,0.0,0.0,3.0,8.0,8.0,3.0,13.136,1318.0,0.0 +gy_kernel_4,0.2,0.8,5.0,2.0,0.0,0.0,3.0,8.0,8.0,3.0,13.378,1318.0,0.0 +gy_kernel_4,0.1,0.9,5.0,2.0,0.0,0.0,3.0,8.0,8.0,3.0,12.923,1318.0,0.0 +gy_kernel_8,1.0,0.0,,,,,,,,,,, +gy_kernel_8,0.9,0.1,,,,,,,,,,, +gy_kernel_8,0.8,0.2,,,,,,,,,,, +gy_kernel_8,0.7,0.3,,,,,,,,,,, +gy_kernel_8,0.6,0.4,,,,,,,,,,, +gy_kernel_8,0.5,0.5,,,,,,,,,,, +gy_kernel_8,0.4,0.6,,,,,,,,,,, +gy_kernel_8,0.3,0.7,,,,,,,,,,, +gy_kernel_8,0.2,0.8,,,,,,,,,,, +gy_kernel_8,0.1,0.9,,,,,,,,,,, +gy_kernel_16,1.0,0.0,,,,,,,,,,, +gy_kernel_16,0.9,0.1,,,,,,,,,,, +gy_kernel_16,0.8,0.2,,,,,,,,,,, +gy_kernel_16,0.7,0.3,,,,,,,,,,, +gy_kernel_16,0.6,0.4,,,,,,,,,,, +gy_kernel_16,0.5,0.5,,,,,,,,,,, +gy_kernel_16,0.4,0.6,,,,,,,,,,, +gy_kernel_16,0.3,0.7,,,,,,,,,,, +gy_kernel_16,0.2,0.8,,,,,,,,,,, +gy_kernel_16,0.1,0.9,,,,,,,,,,, +gy_kernel_32,1.0,0.0,,,,,,,,,,, +gy_kernel_32,0.9,0.1,,,,,,,,,,, +gy_kernel_32,0.8,0.2,,,,,,,,,,, +gy_kernel_32,0.7,0.3,,,,,,,,,,, +gy_kernel_32,0.6,0.4,,,,,,,,,,, +gy_kernel_32,0.5,0.5,,,,,,,,,,, +gy_kernel_32,0.4,0.6,,,,,,,,,,, +gy_kernel_32,0.3,0.7,,,,,,,,,,, +gy_kernel_32,0.2,0.8,,,,,,,,,,, +gy_kernel_32,0.1,0.9,,,,,,,,,,, +roberts_cross_4,1.0,0.0,3.0,0.0,0.0,1.0,3.0,1.0,5.0,1.0,12.884,181.0,1.0 +roberts_cross_4,0.9,0.1,3.0,0.0,0.0,1.0,3.0,1.0,5.0,1.0,15.048,181.0,1.0 +roberts_cross_4,0.8,0.2,3.0,0.0,0.0,1.0,3.0,1.0,5.0,1.0,13.225,181.0,1.0 +roberts_cross_4,0.7,0.3,1.0,3.0,0.0,0.0,2.0,2.0,4.0,1.0,13.363,214.0,0.0 +roberts_cross_4,0.6,0.4,1.0,3.0,0.0,0.0,2.0,2.0,4.0,1.0,12.956,214.0,0.0 +roberts_cross_4,0.5,0.5,1.0,3.0,0.0,0.0,2.0,2.0,4.0,1.0,12.992,214.0,0.0 +roberts_cross_4,0.4,0.6,1.0,3.0,0.0,0.0,2.0,2.0,4.0,1.0,13.056,214.0,0.0 +roberts_cross_4,0.3,0.7,1.0,3.0,0.0,0.0,2.0,2.0,4.0,1.0,12.746,214.0,0.0 +roberts_cross_4,0.2,0.8,1.0,3.0,0.0,0.0,2.0,2.0,4.0,1.0,12.696,214.0,0.0 +roberts_cross_4,0.1,0.9,1.0,3.0,0.0,0.0,2.0,2.0,4.0,1.0,13.037,214.0,0.0 +roberts_cross_8,1.0,0.0,,,,,,,,,,, +roberts_cross_8,0.9,0.1,,,,,,,,,,, +roberts_cross_8,0.8,0.2,,,,,,,,,,, +roberts_cross_8,0.7,0.3,,,,,,,,,,, +roberts_cross_8,0.6,0.4,,,,,,,,,,, +roberts_cross_8,0.5,0.5,,,,,,,,,,, +roberts_cross_8,0.4,0.6,,,,,,,,,,, +roberts_cross_8,0.3,0.7,,,,,,,,,,, +roberts_cross_8,0.2,0.8,,,,,,,,,,, +roberts_cross_8,0.1,0.9,,,,,,,,,,, +roberts_cross_16,1.0,0.0,,,,,,,,,,, +roberts_cross_16,0.9,0.1,,,,,,,,,,, +roberts_cross_16,0.8,0.2,,,,,,,,,,, +roberts_cross_16,0.7,0.3,,,,,,,,,,, +roberts_cross_16,0.6,0.4,,,,,,,,,,, +roberts_cross_16,0.5,0.5,,,,,,,,,,, +roberts_cross_16,0.4,0.6,,,,,,,,,,, +roberts_cross_16,0.3,0.7,,,,,,,,,,, +roberts_cross_16,0.2,0.8,,,,,,,,,,, +roberts_cross_16,0.1,0.9,,,,,,,,,,, +roberts_cross_32,1.0,0.0,,,,,,,,,,, +roberts_cross_32,0.9,0.1,,,,,,,,,,, +roberts_cross_32,0.8,0.2,,,,,,,,,,, +roberts_cross_32,0.7,0.3,,,,,,,,,,, +roberts_cross_32,0.6,0.4,,,,,,,,,,, +roberts_cross_32,0.5,0.5,,,,,,,,,,, +roberts_cross_32,0.4,0.6,,,,,,,,,,, +roberts_cross_32,0.3,0.7,,,,,,,,,,, +roberts_cross_32,0.2,0.8,,,,,,,,,,, +roberts_cross_32,0.1,0.9,,,,,,,,,,, +matrix_mul_4,1.0,0.0,2.0,0.0,0.0,1.0,0.0,2.0,4.0,1.0,11.935,275.0,1.0 +matrix_mul_4,0.9,0.1,2.0,0.0,0.0,1.0,0.0,2.0,4.0,1.0,12.105,275.0,1.0 +matrix_mul_4,0.8,0.2,3.0,0.0,0.0,0.0,0.0,4.0,3.0,1.0,11.774,409.0,0.0 +matrix_mul_4,0.7,0.3,3.0,0.0,0.0,0.0,0.0,4.0,3.0,1.0,11.886,409.0,0.0 +matrix_mul_4,0.6,0.4,3.0,0.0,0.0,0.0,0.0,4.0,3.0,1.0,11.789,409.0,0.0 +matrix_mul_4,0.5,0.5,3.0,0.0,0.0,0.0,0.0,4.0,3.0,1.0,11.87,409.0,0.0 +matrix_mul_4,0.4,0.6,3.0,0.0,0.0,0.0,0.0,4.0,3.0,1.0,11.832,409.0,0.0 +matrix_mul_4,0.3,0.7,3.0,0.0,0.0,0.0,0.0,4.0,3.0,1.0,11.878,409.0,0.0 +matrix_mul_4,0.2,0.8,3.0,0.0,0.0,0.0,0.0,4.0,3.0,1.0,11.555,409.0,0.0 +matrix_mul_4,0.1,0.9,3.0,0.0,0.0,0.0,0.0,4.0,3.0,1.0,11.702,409.0,0.0 +matrix_mul_8,1.0,0.0,,,,,,,,,,, +matrix_mul_8,0.9,0.1,,,,,,,,,,, +matrix_mul_8,0.8,0.2,,,,,,,,,,, +matrix_mul_8,0.7,0.3,,,,,,,,,,, +matrix_mul_8,0.6,0.4,,,,,,,,,,, +matrix_mul_8,0.5,0.5,,,,,,,,,,, +matrix_mul_8,0.4,0.6,,,,,,,,,,, +matrix_mul_8,0.3,0.7,,,,,,,,,,, +matrix_mul_8,0.2,0.8,,,,,,,,,,, +matrix_mul_8,0.1,0.9,,,,,,,,,,, +matrix_mul_16,1.0,0.0,,,,,,,,,,, +matrix_mul_16,0.9,0.1,,,,,,,,,,, +matrix_mul_16,0.8,0.2,,,,,,,,,,, +matrix_mul_16,0.7,0.3,,,,,,,,,,, +matrix_mul_16,0.6,0.4,,,,,,,,,,, +matrix_mul_16,0.5,0.5,,,,,,,,,,, +matrix_mul_16,0.4,0.6,,,,,,,,,,, +matrix_mul_16,0.3,0.7,,,,,,,,,,, +matrix_mul_16,0.2,0.8,,,,,,,,,,, +matrix_mul_16,0.1,0.9,,,,,,,,,,, +matrix_mul_32,1.0,0.0,,,,,,,,,,, +matrix_mul_32,0.9,0.1,,,,,,,,,,, +matrix_mul_32,0.8,0.2,,,,,,,,,,, +matrix_mul_32,0.7,0.3,,,,,,,,,,, +matrix_mul_32,0.6,0.4,,,,,,,,,,, +matrix_mul_32,0.5,0.5,,,,,,,,,,, +matrix_mul_32,0.4,0.6,,,,,,,,,,, +matrix_mul_32,0.3,0.7,,,,,,,,,,, +matrix_mul_32,0.2,0.8,,,,,,,,,,, +matrix_mul_32,0.1,0.9,,,,,,,,,,, +tree_50-50_5_1,1.0,0.0,5,0,0,3,0,2,9,2,10.771,784.0,3.0 +tree_50-50_5_1,0.9,0.1,5,0,0,3,0,2,9,2,10.94,784.0,3.0 +tree_50-50_5_1,0.8,0.2,5,0,0,2,0,2,7,2,10.919,717.0,2.0 +tree_50-50_5_1,0.7,0.3,6,0,0,1,0,2,6,2,10.978,1161.0,1.0 +tree_50-50_5_1,0.6,0.4,6,0,0,1,0,2,6,2,10.782,1161.0,1.0 +tree_50-50_5_1,0.5,0.5,5,0,0,0,0,2,4,2,10.75,1758.0,0.0 +tree_50-50_5_1,0.4,0.6,5,0,0,0,0,2,4,2,10.574,1758.0,0.0 +tree_50-50_5_1,0.3,0.7,5,0,0,0,0,2,4,2,11.045,1758.0,0.0 +tree_50-50_5_1,0.2,0.8,5,0,0,0,0,2,4,2,10.928,1758.0,0.0 +tree_50-50_5_1,0.1,0.9,5,0,0,0,0,2,4,2,10.605,1758.0,0.0 +tree_50-50_10_1,1.0,0.0,13,0,0,5,0,7,16,5,131.869,2769.0,5.0 +tree_50-50_10_1,0.9,0.1,19,0,0,2,0,16,12,5,51.092,25541.0,2.0 +tree_50-50_10_1,0.8,0.2,19,0,0,2,0,16,12,5,49.988,25541.0,2.0 +tree_50-50_10_1,0.7,0.3,19,0,0,2,0,16,12,5,49.565,25541.0,2.0 +tree_50-50_10_1,0.6,0.4,19,0,0,2,0,16,12,5,49.647,25541.0,2.0 +tree_50-50_10_1,0.5,0.5,17,0,0,1,0,17,10,5,17.832,8068.0,1.0 +tree_50-50_10_1,0.4,0.6,16,0,0,0,0,17,8,5,14.524,8517.0,0.0 +tree_50-50_10_1,0.3,0.7,16,0,0,0,0,17,8,5,14.903,8517.0,0.0 +tree_50-50_10_1,0.2,0.8,16,0,0,0,0,17,8,5,14.982,8517.0,0.0 +tree_50-50_10_1,0.1,0.9,16,0,0,0,0,17,8,5,14.72,8517.0,0.0 +tree_50-50_10_1,0.95,0.05,17,0,0,9,0,14,19,6,102.441,4683.0,2.0 +tree_50-50_10_1,0.925,0.075,17,0,0,9,0,14,19,6,104.597,4683.0,2.0 +tree_50-50_10_1,0.9125,0.0875,19,0,0,2,0,16,12,5,51.897,25541.0,2.0 +tree_50-50_10_1,0.90625,0.09375,19,0,0,2,0,16,12,5,50.297,25541.0,2.0 +tree_50-50_10_1,0.903125,0.096875,19,0,0,2,0,16,12,5,49.991,25541.0,2.0 +tree_50-50_10_1,0.9015625,0.0984375,19,0,0,2,0,16,12,5,50.179,25541.0,2.0 +lin_reg_4,0.0,1.0,2,0,1,0,0,1,3,1,21.125,3006.0,0.0 +lin_reg_8,0.0,1.0,2,0,1,0,0,1,3,1,7.347,6006.0,0.0 +lin_reg_16,0.0,1.0,2,0,1,0,0,1,3,1,5.606,12006.0,0.0 +lin_reg_32,0.0,1.0,2,0,1,0,0,1,3,1,5.151,24006.0,0.0 +box_blur_4,0.0,1.0,13,0,0,0,0,0,6,0,5.236,21006.0,0.0 +box_blur_8,0.0,1.0,0,0,0,0,0,0,, +box_blur_16,0.0,1.0,0,0,0,0,0,0,, +box_blur_32,0.0,1.0,0,0,0,0,0,0,, +matrix_mul_4,0.0,1.0,3,0,0,0,0,4,3,1,5.256,28006.0,0.0 +matrix_mul_8,0.0,1.0,0,0,0,0,0,0,, +matrix_mul_16,0.0,1.0,0,0,0,0,0,0,, +matrix_mul_32,0.0,1.0,0,0,0,0,0,0,, +max_3,0.0,1.0,6,0,0,0,3,6,6,2,5.383,3009.0,0.0 +max_4,0.0,1.0,12,0,0,0,6,12,8,3,5.774,7012.0,0.0 +max_5,0.0,1.0,20,0,0,0,10,20,10,4,5.434,15015.0,0.0 +sort_3,0.0,1.0,0,0,0,0,0,0,0,0,0,5012.0,0.0 +dot_product_4,0.0,1.0,3,0,0,0,0,4,3,1,6.158,1756.0,0.0 +dot_product_8,0.0,1.0,7,0,0,0,0,8,4,1,5.313,3757.0,0.0 +dot_product_16,0.0,1.0,15,0,0,0,0,16,5,1,5.127,7758.0,0.0 +dot_product_32,0.0,1.0,31,0,0,0,0,32,6,1,5.275,15759.0,0.0 +gx_kernel_4,0.0,1.0,10,0,5,0,2,5,7,1,9.682,21507.0,0.0 +gx_kernel_8,0.0,1.0,0,0,0,0,0,0,, +gx_kernel_16,0.0,1.0,0,0,0,0,0,0,, +gx_kernel_32,0.0,1.0,0,0,0,0,0,0,, +gy_kernel_4,0.0,1.0,13,0,8,0,1,8,8,1,5.211,21507.0,0.0 +gy_kernel_8,0.0,1.0,0,0,0,0,0,0,, +gy_kernel_16,0.0,1.0,0,0,0,0,0,0,, +gy_kernel_32,0.0,1.0,0,0,0,0,0,0,, +hamming_dist_4,0.0,1.0,7,4,4,0,0,8,5,2,5.033,4759.0,0.0 +hamming_dist_8,0.0,1.0,15,8,8,0,0,16,6,2,5.040,9760.0,0.0 +hamming_dist_16,0.0,1.0,31,16,16,0,0,32,7,2,5.245,19761.0,0.0 +hamming_dist_32,0.0,1.0,63,32,32,0,0,64,8,2,5.379,39762.0,0.0 +l2_distance_4,0.0,1.0,3,4,0,0,0,4,4,1,5.554,3757.0,0.0 +l2_distance_8,0.0,1.0,7,8,0,0,0,8,5,1,5.378,7758.0,0.0 +l2_distance_16,0.0,1.0,15,16,0,0,0,16,6,1,4.945,15759.0,0.0 +l2_distance_32,0.0,1.0,31,32,0,0,0,32,7,1,5.032,31760.0,0.0 +roberts_cross_4,0.0,1.0,6,0,0,0,2,3,6,1,5.943,31007.0,0.0 +roberts_cross_8,0.0,1.0,0,0,0,0,0,0,, +roberts_cross_16,0.0,1.0,0,0,0,0,0,0,, +roberts_cross_32,0.0,1.0,0,0,0,0,0,0,, +tree_50-50_5_1,0.0,1.0,5,0,0,0,0,2,4,2,5.220,1758.0,0.0 +tree_50-50_10_1,0.0,1.0,16,0,0,0,0,17,8,5,4.794,8265.0,0.0 +poly_reg_4,1.0,0.0,3,0,0,3,0,2,8,2,15.269,449.0,3.0 +poly_reg_4,0.9,0.1,3,0,0,2,0,2,7,2,11.766,334.0,2.0 +poly_reg_4,0.8,0.2,3,0,0,2,0,2,7,2,11.473,334.0,2.0 +poly_reg_4,0.7,0.3,3,0,0,0,0,2,5,2,10.845,212.0,0.0 +poly_reg_4,0.6,0.4,3,0,0,0,0,2,5,2,10.805,212.0,0.0 +poly_reg_4,0.5,0.5,3,0,0,0,0,2,5,2,11.363,212.0,0.0 +poly_reg_4,0.4,0.6,3,0,0,0,0,2,5,2,10.709,212.0,0.0 +poly_reg_4,0.3,0.7,3,0,0,0,0,2,5,2,10.774,212.0,0.0 +poly_reg_4,0.2,0.8,3,0,0,0,0,2,5,2,10.841,212.0,0.0 +poly_reg_4,0.1,0.9,3,0,0,0,0,2,5,2,10.774,212.0,0.0 +poly_reg_4,0.75,0.25,3,0,0,2,0,2,7,2,11.129,334.0,2.0 +poly_reg_4,0.725,0.275,3,0,0,1,0,2,6,2,10.950,267.0,1.0 +poly_reg_8,1.0,0.0,3,0,0,2,0,2,7,2,11.787,354.0,2.0 +poly_reg_8,0.9,0.1,3,0,0,2,0,2,7,2,11.753,354.0,2.0 +poly_reg_8,0.8,0.2,3,0,0,2,0,2,7,2,11.800,354.0,2.0 +poly_reg_8,0.7,0.3,3,0,0,0,0,2,5,2,10.921,212.0,0.0 +poly_reg_8,0.6,0.4,3,0,0,0,0,2,5,2,10.871,212.0,0.0 +poly_reg_8,0.5,0.5,3,0,0,0,0,2,5,2,10.931,212.0,0.0 +poly_reg_8,0.4,0.6,3,0,0,0,0,2,5,2,10.767,212.0,0.0 +poly_reg_8,0.3,0.7,3,0,0,0,0,2,5,2,10.949,212.0,0.0 +poly_reg_8,0.2,0.8,3,0,0,0,0,2,5,2,10.963,212.0,0.0 +poly_reg_8,0.1,0.9,3,0,0,0,0,2,5,2,10.870,212.0,0.0 +poly_reg_8,0.75,0.25,3,0,0,0,0,2,5,2,10.870,212.0,0.0 +poly_reg_8,0.775,0.225,3,0,0,1,0,2,6,2,11.235,271.0,1.0 +poly_reg_16,1.0,0.0,3,0,0,1,0,2,6,2,11.769,279.0,1.0 +poly_reg_16,0.9,0.1,3,0,0,1,0,2,6,2,11.653,279.0,1.0 +poly_reg_16,0.8,0.2,3,0,0,0,0,2,5,2,11.212,212.0,0.0 +poly_reg_16,0.7,0.3,3,0,0,0,0,2,5,2,11.212,212.0,0.0 +poly_reg_16,0.6,0.4,3,0,0,0,0,2,5,2,11.105,212.0,0.0 +poly_reg_16,0.5,0.5,3,0,0,0,0,2,5,2,11.229,212.0,0.0 +poly_reg_16,0.4,0.6,3,0,0,0,0,2,5,2,11.197,212.0,0.0 +poly_reg_16,0.3,0.7,3,0,0,0,0,2,5,2,11.092,212.0,0.0 +poly_reg_16,0.2,0.8,3,0,0,0,0,2,5,2,11.204,212.0,0.0 +poly_reg_16,0.1,0.9,3,0,0,0,0,2,5,2,11.951,212.0,0.0 +poly_reg_32,1.0,0.0,3,0,0,0,0,2,5,2,11.719,212.0,0.0 +poly_reg_32,0.9,0.1,3,0,0,0,0,2,5,2,11.835,212.0,0.0 +poly_reg_32,0.8,0.2,3,0,0,0,0,2,5,2,11.867,212.0,0.0 +poly_reg_32,0.7,0.3,3,0,0,0,0,2,5,2,11.716,212.0,0.0 +poly_reg_32,0.6,0.4,3,0,0,0,0,2,5,2,11.861,212.0,0.0 +poly_reg_32,0.5,0.5,3,0,0,0,0,2,5,2,11.809,212.0,0.0 +poly_reg_32,0.4,0.6,3,0,0,0,0,2,5,2,11.684,212.0,0.0 +poly_reg_32,0.3,0.7,3,0,0,0,0,2,5,2,11.876,212.0,0.0 +poly_reg_32,0.2,0.8,3,0,0,0,0,2,5,2,11.831,212.0,0.0 +poly_reg_32,0.1,0.9,3,0,0,0,0,2,4,2,11.767,211.0,0.0 +poly_reg_4,0.0,1.0,3,0,0,0,0,2,5,2,5.271,5009.0,0.0 +poly_reg_8,0.0,1.0,3,0,0,0,0,2,5,2,5.063,10009.0,0.0 +poly_reg_16,0.0,1.0,3,0,0,0,0,2,5,2,5.000,20009.0,0.0 +poly_reg_32,0.0,1.0,3,0,0,0,0,2,5,2,5.492,40009.0,0.0 +tree_100-50_5_1,1.0,0.0,12,0,0,5,0,4,15,3,116.586,1766.0,5.0 +tree_100-50_5_1,0.9,0.1,15,0,0,4,0,8,13,3,24.625,3358.0,4.0 +tree_100-50_5_1,0.8,0.2,24,0,0,2,0,9,11,3,48.384,22990.0,2.0 +tree_100-50_5_1,0.7,0.3,24,0,0,2,0,9,11,3,48.648,22990.0,2.0 +tree_100-50_5_1,0.6,0.4,24,0,0,2,0,9,11,3,48.939,22990.0,2.0 +tree_100-50_5_1,0.5,0.5,20,0,0,1,0,12,7,3,18.463,7563.0,1.0 +tree_100-50_5_1,0.4,0.6,19,0,0,0,0,12,5,3,15.216,8011.0,0.0 +tree_100-50_5_1,0.3,0.7,19,0,0,0,0,12,5,3,15.649,8011.0,0.0 +tree_100-50_5_1,0.2,0.8,19,0,0,0,0,12,5,3,15.692,8011.0,0.0 +tree_100-50_5_1,0.1,0.9,19,0,0,0,0,12,5,3,15.094,8011.0,0.0 +tree_100-50_5_1,0.85,0.15,19,0,0,4,1,9,12,3,38.809,3388.0,2.0 +tree_100-50_5_1,0.825,0.175,19,0,0,4,1,9,12,3,38.615,3388.0,2.0 +tree_100-50_5_1,0.8125,0.1875,19,0,0,4,1,9,12,3,38.866,3388.0,2.0 +tree_100-50_5_1,0.80625,0.19375,24,0,0,2,0,9,11,3,48.712,22990.0,2.0 +tree_100-50_5_1,0.803125,0.196875,24,0,0,2,0,9,11,3,48.749,22990.0,2.0 +tree_100-50_5_1,0.8015625,0.1984375,24,0,0,2,0,9,11,3,48.839,22990.0,2.0 +tree_100-50_5_1,0.875,0.125,15,0,0,4,0,8,13,3,23.46,3358.0,4.0 +tree_100-50_5_1,0.8875,0.1125,15,0,0,4,0,8,13,3,23.997,3358.0,4.0 +tree_100-50_5_1,0.89375,0.10625,15,0,0,4,0,8,13,3,23.271,3358.0,4.0 +tree_100-50_5_1,0.896875,0.103125,15,0,0,4,0,8,13,3,24.093,3358.0,4.0 +tree_100-50_5_1,0.8984375,0.1015625,15,0,0,4,0,8,13,3,23.123,3358.0,4.0 +tree_100-50_5_1,0.0,1.0,19,0,0,0,0,12,5,3,4.947,7760.0,0.0 diff --git a/evaluate.py b/evaluate.py new file mode 100644 index 00000000..6c8a69a0 --- /dev/null +++ b/evaluate.py @@ -0,0 +1,58 @@ +import re + +def parse_and_save(input_file, output_file): + rewards = [] + final_cost = None + + # 1. Read the input file + with open(input_file, 'r') as f: + lines = f.readlines() + + # 2. Parse line by line + for line in lines: + line = line.strip() + + # Capture Rewards + if line.startswith("Reward"): + # Split by ':' and take the second part + parts = line.split(":") + if len(parts) > 1: + try: + reward_value = float(parts[1].strip()) + rewards.append(reward_value) + except ValueError: + continue + + # Capture New Cost + # We update this every time we see it. + # By the end of the loop, it will hold the cost of the final expression. + if line.startswith("New cost"): + parts = line.split(":") + if len(parts) > 1: + try: + final_cost = float(parts[1].strip()) + except ValueError: + continue + + # 3. Calculate the Return (Sum of rewards) + total_return = sum(rewards) + + # 4. Save to file + with open(output_file, 'w') as f: + f.write("--- Trajectory Statistics ---\n") + f.write(f"Final Expression Cost: {final_cost}\n") + f.write(f"Total Return (Sum): {total_return}\n") + f.write(f"Reward History: {rewards}\n") + + # Optional: Save in a format easy to parse later (like CSV style) + f.write("\n--- Raw Data ---\n") + f.write(f"cost={final_cost}\n") + f.write(f"return={total_return}\n") + f.write(f"rewards={','.join(map(str, rewards))}\n") + + print(f"Successfully saved stats to {output_file}") + print(f"Return: {total_return}") + print(f"Final Cost: {final_cost}") + +# Run the parser +parse_and_save("output.txt", "trajectory_stats.txt") \ No newline at end of file diff --git a/get_rotations.py b/get_rotations.py new file mode 100644 index 00000000..8d78fb9f --- /dev/null +++ b/get_rotations.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +import os +import subprocess +import csv +import re +from pathlib import Path + +def get_benchmarks(benchmarks_dir): + """Get all benchmark directories.""" + try: + benchmarks_path = Path(benchmarks_dir) + if not benchmarks_path.exists(): + print(f"Error: Directory {benchmarks_dir} does not exist") + return [] + + # Get all subdirectories in the benchmarks folder + benchmarks = [d.name for d in benchmarks_path.iterdir() if d.is_dir()] + return benchmarks + except Exception as e: + print(f"Error getting benchmarks: {e}") + return [] + +def run_benchmark(benchmark_name, slot_count): + """Run a single benchmark with given slot_count and extract rotation count.""" + try: + benchmark_dir = f"build/benchmarks/{benchmark_name}" + + # Change to benchmark directory + os.chdir(benchmark_dir) + + # Run the generate script + generate_cmd = f"python3 generate_{benchmark_name}.py --slot_count {slot_count}" + print(f" Running: {generate_cmd}") + result = subprocess.run(generate_cmd, shell=True, capture_output=True, text=True, timeout=60) + + if result.returncode != 0: + print(f" Generate script failed for {benchmark_name} with slot_count {slot_count}") + os.chdir("../../..") + return None + + # Run the benchmark + benchmark_cmd = f"./{benchmark_name} 1 {slot_count} 1 0 1 1 1" + print(f" Running: {benchmark_cmd}") + result = subprocess.run(benchmark_cmd, shell=True, capture_output=True, text=True, timeout=60) + + # Go back to original directory + os.chdir("../../..") + + if result.returncode != 0: + print(f" Benchmark execution failed for {benchmark_name} with slot_count {slot_count}") + return None + + # Extract rotation count from output + output = result.stdout + rotation_count = extract_rotation_count(output) + + if rotation_count is not None: + print(f" ✓ Success: {benchmark_name}, slot_count={slot_count}, rotations={rotation_count}") + return rotation_count + else: + print(f" Could not extract rotation count for {benchmark_name} with slot_count {slot_count}") + return None + + except subprocess.TimeoutExpired: + print(f" Timeout for {benchmark_name} with slot_count {slot_count}") + try: + os.chdir("../../..") + except: + pass + return None + except Exception as e: + print(f" Error running {benchmark_name} with slot_count {slot_count}: {e}") + try: + os.chdir("../../..") + except: + pass + return None + +def extract_rotation_count(output): + """Extract the total rotation count from benchmark output.""" + try: + # Look for the |rotate| section + lines = output.split('\n') + in_rotate_section = False + + for line in lines: + # Check if we're entering the rotate section + if '|rotate|' in line: + in_rotate_section = True + continue + + # If we're in the rotate section, look for the total line + if in_rotate_section: + # Look for "total: " + match = re.search(r'total:\s*(\d+)', line) + if match: + return int(match.group(1)) + + # If we hit another operation section, we've left rotate + if line.strip().startswith('|') and '|rotate|' not in line: + in_rotate_section = False + + return None + except Exception as e: + print(f" Error extracting rotation count: {e}") + return None + +def main(): + benchmarks_dir = "build/benchmarks" + output_csv = "benchmark_results.csv" + slot_counts = range(2, 33) # 2 to 32 inclusive + + # Get all benchmarks + benchmarks = get_benchmarks(benchmarks_dir) + + if not benchmarks: + print("No benchmarks found!") + return + + print(f"Found {len(benchmarks)} benchmarks: {', '.join(benchmarks)}\n") + + # Create CSV file and write header + csv_exists = False + total_successful = 0 + + # Loop through each benchmark + for benchmark in benchmarks: + print(f"Processing benchmark: {benchmark}") + + # Loop through slot counts + for slot_count in slot_counts: + rotation_count = run_benchmark(benchmark, slot_count) + + if rotation_count is not None: + # Write to CSV immediately + mode = 'a' if csv_exists else 'w' + with open(output_csv, mode, newline='') as csvfile: + fieldnames = ['benchmark', 'slot_count', 'number_of_rotations'] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames) + + # Write header only if this is the first write + if not csv_exists: + writer.writeheader() + csv_exists = True + + writer.writerow({ + 'benchmark': benchmark, + 'slot_count': slot_count, + 'number_of_rotations': rotation_count + }) + + total_successful += 1 + print(f" → Saved to CSV (total entries: {total_successful})") + + if total_successful > 0: + print(f"\n✓ Results saved to {output_csv}") + print(f"Total successful runs: {total_successful}") + else: + print("\nNo successful runs to save!") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/initial_metrics.txt b/initial_metrics.txt new file mode 100644 index 00000000..4ca5f9dc --- /dev/null +++ b/initial_metrics.txt @@ -0,0 +1,6 @@ +cost = 2947 +time = 1700 +source /share/apps/NYUAD5/miniconda/3-4.11.0/bin/activate +conda activate chehabEnv + +python3 run_benchmarks.py diff --git a/output.txt b/output.txt new file mode 100644 index 00000000..e69de29b diff --git a/pareto_2_full_train.csv b/pareto_2_full_train.csv new file mode 100644 index 00000000..3274f86c --- /dev/null +++ b/pareto_2_full_train.csv @@ -0,0 +1,460 @@ +benchmark,w_ops,w_keys,add,sub,multiply_plain,rotate_rows,negate,multiply,Depth,Multplicative Depth,compile_time (s),circuit_execution_time (s),galois_keys_generation_time (s),total_execution_time (s),Remaining_noise_budget,rotation_key_size (MB),Final_Cost +max_3,1.0,0.0,2,1,0,4,0,2,9,2,11.772,0.501,0.446,0.947,292.5,72.4103,505.0 +max_3,0.9,0.1,2,1,0,3,0,2,8,2,11.298,0.482,0.334,0.816,295.0,54.3391,389.0 +max_3,0.8,0.2,2,1,0,3,0,2,8,2,11.378,0.482,0.334,0.816,295.0,54.3391,389.0 +max_3,0.7,0.3,7,1,0,3,2,4,10,2,17.955,0.869,0.227,1.096,295.0,36.2679 +max_3,0.6,0.4,8,0,0,1,3,5,8,2,14.831,0.961,0.115,1.076,295.0,18.1967 +max_3,0.5,0.5,8,0,0,1,3,5,8,2,14.731,0.961,0.116,1.077,295.0,18.1967 +max_3,0.4,0.6,6,0,0,0,3,6,6,2,13.925,1.019,0.000,1.019,295.0,0.125542 +max_3,0.3,0.7,6,0,0,0,3,6,6,2,13.780,1.020,0.000,1.020,295.0,0.125542 +max_3,0.2,0.8,6,0,0,0,3,6,6,2,13.879,1.021,0.000,1.021,295.0,0.125542 +max_3,0.1,0.9,6,0,0,0,3,6,6,2,13.783,1.016,0.000,1.016,295.0,0.125542 +max_4,1.0,0.0,3,1,0,5,0,3,12,3,33.132,0.720,0.570,1.290,262.0,90.4814,917.0 +max_4,0.9,0.1,5,2,0,4,1,5,11,3,22.475,1.084,0.456,1.540,262.0,72.4103,812.0 +max_4,0.8,0.2,9,0,0,2,4,7,11,3,14.674,1.337,0.228,1.565,262.0,36.2679,1078.0 +max_4,0.7,0.3,17,1,0,3,6,11,12,3,24.744,2.126,0.228,2.355,262.0,36.2679 +max_4,0.6,0.4 +max_4,0.5,0.5,16,0,0,1,7,13,10,3,17.827,2.248,0.115,2.363,262.0,18.1967 +max_4,0.4,0.6,12,0,0,0,6,12,8,3,14.545,1.965,0.000,1.965,262.0,0.125542 +max_4,0.3,0.7,12,0,0,0,6,12,8,3,14.588,1.969,0.000,1.969,262.0,0.125542 +max_4,0.2,0.8,12,0,0,0,6,12,8,3,14.552,1.966,0.000,1.966,262.0,0.125542 +max_4,0.1,0.9,12,0,0,0,6,12,8,3,14.622,1.963,0.000,1.963,262.0,0.125542 +max_5,1.0,0.0,12,2,0,5,4,12,14,4,305.018,2.347,0.453,2.800,228.5,72.4103,1636.0 +max_5,0.9,0.1,13,2,0,4,5,13,14,4,167.662,2.479,0.452,2.931,228.5,72.4103,1623.0 +max_5,0.8,0.2 +max_5,0.7,0.3,33,1,0,3,14,27,14,4,49.714,4.758,0.231,4.989,228.5,36.2679 +max_5,0.6,0.4 +max_5,0.5,0.5,28,0,0,1,13,25,12,4,27.084,4.186,0.117,4.303,228.5,18.1967 +max_5,0.4,0.6,20,0,0,0,10,20,10,4,18.475,3.260,0.000,3.260,229.0,0.125542 +max_5,0.3,0.7,20,0,0,0,10,20,10,4,18.665,3.247,0.000,3.247,229.0,0.125542 +max_5,0.2,0.8,20,0,0,0,10,20,10,4,18.714,3.243,0.000,3.243,229.0,0.125542 +max_5,0.1,0.9,20,0,0,0,10,20,10,4,18.702,3.246,0.000,3.246,229.0,0.125542 +sort_3,1.0,0.0 +sort_3,0.9,0.1 +sort_3,0.8,0.2 +sort_3,0.7,0.3 +sort_3,0.6,0.4 +sort_3,0.5,0.5 +sort_3,0.4,0.6 +sort_3,0.3,0.7 +sort_3,0.2,0.8 +sort_3,0.1,0.9 +box_blur_4,1.0,0.0,5,0,0,1,0,0,5,0,12.773,0.168,0.114,0.282,357.0,18.1967,79.0 +box_blur_4,0.9,0.1,5,0,0,1,0,0,5,0,13.001,0.168,0.115,0.282,357.0,18.1967,79.0 +box_blur_4,0.8,0.2,8,0,0,0,0,0,4,0,12.706,0.225,0.000,0.225,359.0,0.125542,14.0 +box_blur_4,0.7,0.3,8,0,0,0,0,0,4,0,12.767,0.225,0.000,0.225,359.5,0.125542,14.0 +box_blur_4,0.6,0.4,8,0,0,0,0,0,4,0,12.647,0.224,0.000,0.224,359.0,0.125542,14.0 +box_blur_4,0.5,0.5,8,0,0,0,0,0,4,0,12.734,0.221,0.000,0.221,359.0,0.125542,14.0 +box_blur_4,0.4,0.6,8,0,0,0,0,0,4,0,12.908,0.227,0.000,0.227,359.0,0.125542,14.0 +box_blur_4,0.3,0.7,8,0,0,0,0,0,4,0,12.997,0.227,0.000,0.227,359.0,0.125542,14.0 +box_blur_4,0.2,0.8,8,0,0,0,0,0,5,0,12.940,0.227,0.000,0.227,359.0,0.125542,15.0 +box_blur_4,0.1,0.9,8,0,0,0,0,0,6,0,13.250,0.227,0.000,0.227,359.0,0.125542,16.0 +box_blur_8,1.0,0.0 +box_blur_8,0.9,0.1 +box_blur_8,0.8,0.2 +box_blur_8,0.7,0.3 +box_blur_8,0.6,0.4 +box_blur_8,0.5,0.5 +box_blur_8,0.4,0.6 +box_blur_8,0.3,0.7 +box_blur_8,0.2,0.8 +box_blur_8,0.1,0.9 +box_blur_16,1.0,0.0 +box_blur_16,0.9,0.1 +box_blur_16,0.8,0.2 +box_blur_16,0.7,0.3 +box_blur_16,0.6,0.4 +box_blur_16,0.5,0.5 +box_blur_16,0.4,0.6 +box_blur_16,0.3,0.7 +box_blur_16,0.2,0.8 +box_blur_16,0.1,0.9 +box_blur_32,1.0,0.0 +box_blur_32,0.9,0.1 +box_blur_32,0.8,0.2 +box_blur_32,0.7,0.3 +box_blur_32,0.6,0.4 +box_blur_32,0.5,0.5 +box_blur_32,0.4,0.6 +box_blur_32,0.3,0.7 +box_blur_32,0.2,0.8 +box_blur_32,0.1,0.9 +lin_reg_4,1.0,0.0,2,0,0,2,0,1,5,1,15.219,0.286,0.229,0.515,327.0,36.2679,232.0 +lin_reg_4,0.9,0.1,2,0,0,2,0,1,5,1,11.847,0.286,0.229,0.515,327.0,36.2679,232.0 +lin_reg_4,0.8,0.2,2,0,0,1,0,1,4,1,12.016,0.244,0.115,0.360,328.0,18.1967,164.0 +lin_reg_4,0.7,0.3,2,0,1,0,0,1,3,1,11.788,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_4,0.6,0.4,2,0,1,0,0,1,3,1,11.825,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_4,0.5,0.5,2,0,1,0,0,1,3,1,11.856,0.055,0.000,0.055,358.5,0.125542,108.0 +lin_reg_4,0.4,0.6,2,0,1,0,0,1,3,1,12.344,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_4,0.3,0.7,2,0,1,0,0,1,3,1,11.868,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_4,0.2,0.8,2,0,1,0,0,1,3,1,12.179,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_4,0.1,0.9,2,0,1,0,0,1,3,1,11.749,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_8,1.0,0.0,2,0,0,2,0,1,5,1,12.076,0.286,0.228,0.514,327.5,36.2679,252.0 +lin_reg_8,0.9,0.1,2,0,0,2,0,1,5,1,12.144,0.285,0.228,0.513,327.0,36.2679,252.0 +lin_reg_8,0.8,0.2,2,0,0,2,0,1,5,1,11.965,0.286,0.229,0.516,327.0,36.2679,252.0 +lin_reg_8,0.7,0.3,2,0,1,0,0,1,3,1,11.810,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_8,0.6,0.4,2,0,1,0,0,1,3,1,11.979,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_8,0.5,0.5,2,0,1,0,0,1,3,1,11.974,0.055,0.000,0.055,358.5,0.125542,108.0 +lin_reg_8,0.4,0.6,2,0,1,0,0,1,3,1,11.969,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_8,0.3,0.7,2,0,1,0,0,1,3,1,11.921,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_8,0.2,0.8,2,0,1,0,0,1,3,1,11.785,0.055,0.000,0.055,358.5,0.125542,108.0 +lin_reg_8,0.1,0.9,2,0,1,0,0,1,3,1,11.902,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_16,1.0,0.0,2,0,0,1,0,1,4,1,12.113,0.244,0.115,0.359,328.0,18.1967,176.0 +lin_reg_16,0.9,0.1,2,0,0,1,0,1,4,1,12.137,0.244,0.115,0.359,328.0,18.1967,176.0 +lin_reg_16,0.8,0.2,2,0,0,1,0,1,4,1,12.548,0.244,0.115,0.359,328.0,18.1967,176.0 +lin_reg_16,0.7,0.3,2,0,1,0,0,1,3,1,12.068,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_16,0.6,0.4,2,0,1,0,0,1,3,1,11.979,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_16,0.5,0.5,2,0,1,0,0,1,3,1,11.815,0.055,0.000,0.055,358.0,0.125542,108.0 +lin_reg_16,0.4,0.6,2,0,1,0,0,1,3,1,11.822,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_16,0.3,0.7,2,0,1,0,0,1,3,1,11.800,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_16,0.2,0.8,2,0,1,0,0,1,3,1,12.172,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_16,0.1,0.9,2,0,1,0,0,1,3,1,12.315,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_32,1.0,0.0,2,0,1,0,0,1,3,1,12.372,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_32,0.9,0.1,2,0,1,0,0,1,3,1,12.203,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_32,0.8,0.2,2,0,1,0,0,1,3,1,12.221,0.055,0.000,0.055,358.5,0.125542,108.0 +lin_reg_32,0.7,0.3,2,0,1,0,0,1,3,1,12.274,0.055,0.000,0.055,358.5,0.125542,108.0 +lin_reg_32,0.6,0.4,2,0,1,0,0,1,3,1,12.139,0.055,0.000,0.055,358.5,0.125542,108.0 +lin_reg_32,0.5,0.5,2,0,1,0,0,1,3,1,12.233,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_32,0.4,0.6,2,0,1,0,0,1,3,1,12.249,0.054,0.000,0.054,359.0,0.125542,108.0 +lin_reg_32,0.3,0.7,2,0,1,0,0,1,3,1,12.072,0.054,0.000,0.054,359.0,0.125542,108.0 +lin_reg_32,0.2,0.8,2,0,1,0,0,1,3,1,12.063,0.054,0.000,0.054,359.0,0.125542,108.0 +lin_reg_32,0.1,0.9,2,0,1,0,0,1,3,1,12.258,0.054,0.000,0.054,359.0,0.125542,108.0 +hamming_dist_4,1.0,0.0,3,1,0,4,0,2,8,2,12.351,0.529,0.336,0.866,295.0,54.3391,456.0 +hamming_dist_4,0.9,0.1,4,1,0,3,0,3,8,2,12.163,0.677,0.336,1.013,295.0,54.3391,639.0 +hamming_dist_4,0.8,0.2,4,1,0,2,0,3,8,2,12.138,0.660,0.224,0.884,295.0,36.2679,572.0 +hamming_dist_4,0.7,0.3,6,2,1,1,0,5,7,2,12.032,0.806,0.113,0.919,295.0,18.1967,868.0 +hamming_dist_4,0.6,0.4,6,2,1,1,0,5,7,2,12.057,0.809,0.113,0.922,295.0,18.1967,868.0 +hamming_dist_4,0.5,0.5,6,2,1,1,0,5,7,2,12.299,0.821,0.116,0.936,295.0,18.1967,868.0 +hamming_dist_4,0.4,0.6,7,4,4,0,0,8,5,2,14.068,0.782,0.000,0.782,326.0,0.125542 +hamming_dist_4,0.3,0.7,7,4,4,0,0,8,5,2,14.365,0.784,0.000,0.784,327.0,0.125542 +hamming_dist_4,0.2,0.8,7,4,4,0,0,8,5,2,14.276,0.783,0.000,0.783,326.5,0.125542 +hamming_dist_4,0.1,0.9,7,4,4,0,0,8,5,2,14.267,0.783,0.000,0.783,326.0,0.125542 +hamming_dist_8,1.0,0.0,4,1,0,5,0,2,10,2,15.743,0.582,0.459,1.041,294.5,72.4103,622.0 +hamming_dist_8,0.9,0.1,5,1,0,4,0,3,10,2,14.908,0.735,0.459,1.194,294.0,72.4103,757.0 +hamming_dist_8,0.8,0.2,6,2,1,2,0,5,9,2,13.721,0.861,0.229,1.090,295.0,36.2679,776.0 +hamming_dist_8,0.7,0.3,12,4,3,1,0,9,9,2,21.028,1.204,0.115,1.319,295.0,18.1967 +hamming_dist_8,0.6,0.4,12,4,3,1,0,9,9,2,20.397,1.203,0.115,1.318,295.0,18.1967 +hamming_dist_8,0.5,0.5,12,4,3,1,0,9,9,2,21.206,1.204,0.115,1.318,295.0,18.1967 +hamming_dist_8,0.4,0.6,15,8,8,0,0,16,6,2,16.536,1.552,0.000,1.552,326.0,0.125542 +hamming_dist_8,0.3,0.7,15,8,8,0,0,16,6,2,16.087,1.553,0.000,1.553,326.0,0.125542 +hamming_dist_8,0.2,0.8,15,8,8,0,0,16,6,2,16.667,1.551,0.000,1.551,326.0,0.125542 +hamming_dist_8,0.1,0.9,15,8,8,0,0,16,6,2,16.618,1.565,0.000,1.566,326.0,0.125542 +hamming_dist_16,1.0,0.0,6,1,0,5,0,3,12,2,43.448,0.782,0.579,1.360,294.0,90.4814,1067.0 +hamming_dist_16,0.9,0.1,6,1,0,4,0,3,12,2,38.603,0.762,0.462,1.224,294.0,72.4103,760.0 +hamming_dist_16,0.8,0.2,25,9,7,2,0,19,12,2,32.818,2.342,0.230,2.572,294.0,36.2679 +hamming_dist_16,0.7,0.3,22,8,7,1,0,17,12,2,33.348,1.982,0.116,2.098,295.0,18.1967 +hamming_dist_16,0.6,0.4,20,8,7,1,0,17,13,2,32.338,1.986,0.117,2.103,295.0,18.1967 +hamming_dist_16,0.5,0.5,20,8,7,1,0,17,13,2,33.495,1.985,0.117,2.101,295.0,18.1967 +hamming_dist_16,0.4,0.6,31,16,16,0,0,32,7,2,21.655,3.129,0.000,3.129,325.0,0.125542 +hamming_dist_16,0.3,0.7,31,16,16,0,0,32,7,2,22.555,3.130,0.000,3.130,325.0,0.125542 +hamming_dist_16,0.2,0.8,31,16,16,0,0,32,7,2,22.215,3.129,0.000,3.129,325.5,0.125542 +hamming_dist_16,0.1,0.9,31,16,16,0,0,32,7,2,22.706,3.119,0.000,3.119,325.0,0.125542 +hamming_dist_32,1.0,0.0,7,1,0,5,0,3,14,2,361.765,0.789,0.566,1.355,292.0,90.4814,1070.0 +hamming_dist_32,0.9,0.1,7,1,0,5,0,3,14,2,363.081,0.788,0.562,1.349,292.5,90.4814,1070.0 +hamming_dist_32,0.8,0.2,34,16,15,1,0,33,10,2,74.757,3.456,0.112,3.569,295.0,18.1967 +hamming_dist_32,0.7,0.3,34,16,15,1,0,33,10,2,74.541,3.450,0.112,3.563,295.0,18.1967 +hamming_dist_32,0.6,0.4,34,16,15,1,0,33,10,2,74.031,3.492,0.114,3.606,295.0,18.1967 +hamming_dist_32,0.5,0.5,34,16,15,1,0,33,10,2,77.597,3.524,0.116,3.640,295.0,18.1967 +hamming_dist_32,0.4,0.6,34,16,15,1,0,33,10,2,74.919,3.511,0.115,3.626,295.0,18.1967 +hamming_dist_32,0.3,0.7,63,32,32,0,0,64,8,2,36.438,6.152,0.000,6.152,325.0,0.125542 +hamming_dist_32,0.2,0.8,63,32,32,0,0,64,8,2,36.347,6.138,0.000,6.138,325.0,0.125542 +hamming_dist_32,0.1,0.9,63,32,32,0,0,64,8,2,35.978,6.138,0.000,6.138,325.0,0.125542 +dot_product_4,1.0,0.0,2,0,0,3,0,1,6,1,15.355,0.298,0.334,0.632,324.5,54.3391,285.0 +dot_product_4,0.9,0.1,2,0,0,3,0,1,6,1,11.961,0.304,0.343,0.646,324.5,54.3391,285.0 +dot_product_4,0.8,0.2,2,0,0,2,0,1,5,1,11.772,0.287,0.230,0.517,327.0,36.2679,217.0 +dot_product_4,0.7,0.3,2,0,0,1,0,2,4,1,12.066,0.435,0.116,0.551,328.0,18.1967,261.0 +dot_product_4,0.6,0.4,2,0,0,1,0,2,4,1,12.021,0.435,0.116,0.551,327.5,18.1967,261.0 +dot_product_4,0.5,0.5,2,0,0,1,0,2,4,1,12.219,0.435,0.116,0.550,327.0,18.1967,261.0 +dot_product_4,0.4,0.6,3,0,0,0,0,4,3,1,12.097,0.773,0.000,0.773,328.0,0.125542,1756.0 +dot_product_4,0.3,0.7,3,0,0,0,0,4,3,1,12.132,0.776,0.000,0.776,327.5,0.125542,1756.0 +dot_product_4,0.2,0.8,3,0,0,0,0,4,3,1,12.023,0.776,0.000,0.776,327.0,0.125542,1756.0 +dot_product_4,0.1,0.9,3,0,0,0,0,4,3,1,12.096,0.775,0.000,0.775,328.0,0.125542,1756.0 +dot_product_8,1.0,0.0,3,0,0,4,0,1,8,1,12.517,0.346,0.455,0.801,324.0,72.4103,403.0 +dot_product_8,0.9,0.1,3,0,0,4,0,1,8,1,12.395,0.345,0.456,0.801,324.0,72.4103,403.0 +dot_product_8,0.8,0.2,3,0,0,3,0,1,7,1,12.499,0.327,0.343,0.670,326.5,54.3391,287.0 +dot_product_8,0.7,0.3,4,0,0,1,0,4,5,1,12.490,0.817,0.115,0.932,327.0,18.1967,464.0 +dot_product_8,0.6,0.4,4,0,0,1,0,4,5,1,12.386,0.816,0.115,0.930,327.0,18.1967,464.0 +dot_product_8,0.5,0.5,6,0,0,1,0,4,6,1,12.412,0.819,0.116,0.935,327.0,18.1967,465.0 +dot_product_8,0.4,0.6,7,0,0,0,0,8,4,1,13.797,1.545,0.000,1.545,327.0,0.125542 +dot_product_8,0.3,0.7,7,0,0,0,0,8,4,1,13.919,1.545,0.000,1.545,327.0,0.125542 +dot_product_8,0.2,0.8,7,0,0,0,0,8,4,1,14.076,1.536,0.000,1.536,327.0,0.125542 +dot_product_8,0.1,0.9,7,0,0,0,0,8,4,1,14.156,1.540,0.000,1.540,327.0,0.125542 +dot_product_16,1.0,0.0,4,0,0,5,0,1,10,1,14.922,0.385,0.568,0.953,324.0,90.4814,713.0 +dot_product_16,0.9,0.1,4,0,0,4,0,1,9,1,14.273,0.369,0.456,0.825,326.0,72.4103,405.0 +dot_product_16,0.8,0.2,10,0,0,3,0,4,11,1,13.717,0.897,0.114,1.011,326.5,18.1967,573.0 +dot_product_16,0.7,0.3,14,0,0,1,0,8,14,1,18.813,1.577,0.114,1.691,326.0,18.1967 +dot_product_16,0.6,0.4,13,0,0,1,0,8,12,1,18.823,1.572,0.114,1.686,326.5,18.1967 +dot_product_16,0.5,0.5,8,0,0,1,0,8,10,1,17.473,1.572,0.115,1.687,326.0,18.1967 +dot_product_16,0.4,0.6,15,0,0,0,0,16,5,1,14.928,3.023,0.000,3.023,326.0,0.125542 +dot_product_16,0.3,0.7,15,0,0,0,0,16,5,1,14.830,3.026,0.000,3.027,326.0,0.125542 +dot_product_16,0.2,0.8,15,0,0,0,0,16,5,1,14.758,3.022,0.000,3.022,326.5,0.125542 +dot_product_16,0.1,0.9,15,0,0,0,0,16,5,1,14.800,3.020,0.000,3.020,326.0,0.125542 +dot_product_32,1.0,0.0,5,0,0,5,0,1,11,1,35.174,0.407,0.558,0.965,325.5,90.4814,715.0 +dot_product_32,0.9,0.1,5,0,0,4,0,2,10,1,24.635,0.553,0.447,1.000,325.5,72.4103,507.0 +dot_product_32,0.8,0.2,16,0,0,1,0,16,12,1,26.619,3.051,0.112,3.164,326.0,18.1967 +dot_product_32,0.7,0.3,16,0,0,1,0,16,15,1,24.729,3.051,0.113,3.164,326.0,18.1967 +dot_product_32,0.6,0.4,16,0,0,1,0,16,15,1,26.329,3.052,0.113,3.164,326.0,18.1967 +dot_product_32,0.5,0.5,16,0,0,1,0,16,15,1,24.353,3.033,0.112,3.145,326.0,18.1967 +dot_product_32,0.4,0.6,31,0,0,0,0,32,6,1,17.870,5.958,0.000,5.958,326.0,0.125542 +dot_product_32,0.3,0.7,31,0,0,0,0,32,6,1,17.873,5.957,0.000,5.957,326.0,0.125542 +dot_product_32,0.2,0.8,31,0,0,0,0,32,6,1,17.899,5.957,0.000,5.957,326.0,0.125542 +dot_product_32,0.1,0.9,31,0,0,0,0,32,6,1,18.326,5.959,0.000,5.959,326.0,0.125542 +poly_reg_4,1.0,0.0,3,0,0,3,0,2,8,2,12.454,0.483,0.335,0.818,295.0,54.3391,452.0 +poly_reg_4,0.9,0.1,3,0,0,2,0,2,7,2,11.612,0.443,0.223,0.666,295.0,36.2679,336.0 +poly_reg_4,0.8,0.2,3,0,0,2,0,2,7,2,11.679,0.443,0.224,0.666,295.0,36.2679,336.0 +poly_reg_4,0.7,0.3,3,0,0,0,0,2,5,2,11.306,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_4,0.6,0.4,3,0,0,0,0,2,5,2,11.243,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_4,0.5,0.5,3,0,0,0,0,2,5,2,11.166,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_4,0.4,0.6,3,0,0,0,0,2,5,2,11.232,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_4,0.3,0.7,3,0,0,0,0,2,5,2,11.229,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_4,0.2,0.8,3,0,0,0,0,2,5,2,11.230,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_4,0.1,0.9,3,0,0,0,0,2,5,2,11.277,0.408,0.000,0.409,296.0,0.125542,212.0 +poly_reg_8,1.0,0.0,3,0,0,2,0,2,7,2,12.257,0.443,0.223,0.666,295.0,36.2679,356.0 +poly_reg_8,0.9,0.1,3,0,0,2,0,2,7,2,12.267,0.443,0.223,0.666,295.0,36.2679,356.0 +poly_reg_8,0.8,0.2,3,0,0,2,0,2,7,2,12.238,0.443,0.224,0.666,295.0,36.2679,356.0 +poly_reg_8,0.7,0.3,3,0,0,0,0,2,5,2,11.394,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_8,0.6,0.4,3,0,0,0,0,2,5,2,11.411,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_8,0.5,0.5,3,0,0,0,0,2,5,2,11.314,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_8,0.4,0.6,3,0,0,0,0,2,5,2,11.357,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_8,0.3,0.7,3,0,0,0,0,2,5,2,11.345,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_8,0.2,0.8,3,0,0,0,0,2,5,2,11.284,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_8,0.1,0.9,3,0,0,0,0,2,5,2,11.348,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_16,1.0,0.0,3,0,0,1,0,2,6,2,12.209,0.426,0.112,0.538,295.0,18.1967,280.0 +poly_reg_16,0.9,0.1,3,0,0,1,0,2,6,2,14.398,0.426,0.112,0.537,295.0,18.1967,280.0 +poly_reg_16,0.8,0.2,3,0,0,0,0,2,5,2,11.587,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_16,0.7,0.3,3,0,0,0,0,2,5,2,11.600,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_16,0.6,0.4,3,0,0,0,0,2,5,2,11.546,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_16,0.5,0.5,3,0,0,0,0,2,5,2,11.590,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_16,0.4,0.6,3,0,0,0,0,2,5,2,11.616,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_16,0.3,0.7,3,0,0,0,0,2,5,2,11.564,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_16,0.2,0.8,3,0,0,0,0,2,5,2,11.574,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_16,0.1,0.9,3,0,0,0,0,2,5,2,11.624,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_32,1.0,0.0,3,0,0,0,0,2,5,2,12.099,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_32,0.9,0.1,3,0,0,0,0,2,5,2,12.205,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_32,0.8,0.2,3,0,0,0,0,2,5,2,12.200,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_32,0.7,0.3,3,0,0,0,0,2,5,2,12.100,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_32,0.6,0.4,3,0,0,0,0,2,5,2,12.178,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_32,0.5,0.5,3,0,0,0,0,2,5,2,12.190,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_32,0.4,0.6,3,0,0,0,0,2,5,2,12.165,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_32,0.3,0.7,3,0,0,0,0,2,5,2,12.205,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_32,0.2,0.8,3,0,0,0,0,2,5,2,12.111,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_32,0.1,0.9,3,0,0,0,0,2,4,2,12.240,0.407,0.000,0.407,296.0,0.125542,211.0 +l2_distance_4,1.0,0.0,2,1,0,3,0,1,7,1,11.415,0.321,0.335,0.655,324.5,54.3391,287.0 +l2_distance_4,0.9,0.1,2,1,0,3,0,1,7,1,11.369,0.321,0.335,0.655,324.5,54.3391,287.0 +l2_distance_4,0.8,0.2,2,1,0,2,0,1,6,1,11.331,0.280,0.222,0.502,326.5,36.2679,220.0 +l2_distance_4,0.7,0.3,2,2,0,1,0,2,5,1,11.320,0.425,0.112,0.537,327.0,18.1967,266.0 +l2_distance_4,0.6,0.4,2,2,0,1,0,2,5,1,11.600,0.425,0.112,0.538,326.5,18.1967,266.0 +l2_distance_4,0.5,0.5,2,2,0,1,0,2,5,1,11.391,0.425,0.112,0.537,327.0,18.1967,266.0 +l2_distance_4,0.4,0.6,3,4,0,0,0,4,4,1,11.097,0.759,0.000,0.759,326.5,0.125542,3757.0 +l2_distance_4,0.3,0.7,3,4,0,0,0,4,4,1,11.097,0.759,0.000,0.759,327.0,0.125542,3757.0 +l2_distance_4,0.2,0.8,3,4,0,0,0,4,4,1,12.902,0.759,0.000,0.759,326.0,0.125542 +l2_distance_4,0.1,0.9,3,4,0,0,0,4,4,1,12.895,0.759,0.000,0.759,326.5,0.125542 +l2_distance_8,1.0,0.0,3,1,0,4,0,1,9,1,13.060,0.362,0.444,0.806,324.0,72.4103,405.0 +l2_distance_8,0.9,0.1,3,1,0,4,0,1,9,1,13.003,0.362,0.444,0.806,324.0,72.4103,405.0 +l2_distance_8,0.8,0.2,3,2,0,2,0,2,7,1,12.269,0.466,0.223,0.690,326.0,36.2679,324.0 +l2_distance_8,0.7,0.3,6,4,0,1,0,4,8,1,17.362,0.800,0.113,0.912,326.0,18.1967 +l2_distance_8,0.6,0.4,5,4,0,1,0,4,8,1,17.240,0.799,0.112,0.911,326.0,18.1967 +l2_distance_8,0.5,0.5,4,4,0,1,0,4,6,1,16.622,0.800,0.113,0.912,326.0,18.1967 +l2_distance_8,0.4,0.6,7,8,0,0,0,8,5,1,14.952,1.544,0.000,1.544,326.0,0.125542 +l2_distance_8,0.3,0.7,7,8,0,0,0,8,5,1,15.210,1.543,0.000,1.543,326.0,0.125542 +l2_distance_8,0.2,0.8,7,8,0,0,0,8,5,1,15.196,1.548,0.000,1.548,326.0,0.125542 +l2_distance_8,0.1,0.9,7,8,0,0,0,8,5,1,15.172,1.541,0.000,1.541,326.0,0.125542 +l2_distance_16,1.0,0.0,4,1,0,5,0,1,11,1,35.724,0.412,0.571,0.983,324.0,90.4814,715.0 +l2_distance_16,0.9,0.1,4,1,0,4,0,1,10,1,25.010,0.370,0.454,0.824,325.0,72.4103,408.0 +l2_distance_16,0.8,0.2,15,8,0,1,0,8,13,1,28.418,1.593,0.116,1.709,325.0,18.1967 +l2_distance_16,0.7,0.3,8,8,0,1,0,8,8,1,25.692,1.580,0.115,1.695,325.5,18.1967 +l2_distance_16,0.6,0.4,8,8,0,1,0,8,11,1,27.348,1.579,0.115,1.694,325.5,18.1967 +l2_distance_16,0.5,0.5,8,8,0,1,0,8,11,1,25.597,1.574,0.114,1.688,325.5,18.1967 +l2_distance_16,0.4,0.6,15,16,0,0,0,16,6,1,19.018,3.046,0.000,3.046,325.0,0.125542 +l2_distance_16,0.3,0.7,15,16,0,0,0,16,6,1,19.244,3.053,0.000,3.053,325.5,0.125542 +l2_distance_16,0.2,0.8,15,16,0,0,0,16,6,1,18.890,3.052,0.000,3.052,325.0,0.125542 +l2_distance_16,0.1,0.9,15,16,0,0,0,16,6,1,19.181,3.035,0.000,3.035,325.5,0.125542 +l2_distance_32,1.0,0.0,5,1,0,5,0,1,12,1,190.726,0.407,0.560,0.967,324.5,90.4814,718.0 +l2_distance_32,0.9,0.1,6,4,0,3,0,4,10,1,71.000,0.891,0.335,1.226,325.0,54.3391,601.0 +l2_distance_32,0.8,0.2,16,16,0,1,0,16,11,1,56.522,3.060,0.113,3.172,325.0,18.1967 +l2_distance_32,0.7,0.3,16,16,0,1,0,16,15,1,56.205,3.056,0.112,3.168,325.0,18.1967 +l2_distance_32,0.6,0.4,16,16,0,1,0,16,16,1,56.786,3.114,0.115,3.229,325.0,18.1967 +l2_distance_32,0.5,0.5,16,16,0,1,0,16,16,1,59.350,3.099,0.114,3.214,325.0,18.1967 +l2_distance_32,0.4,0.6,31,32,0,0,0,32,7,1,30.275,6.119,0.000,6.119,325.0,0.125542 +l2_distance_32,0.3,0.7,31,32,0,0,0,32,7,1,30.498,6.123,0.000,6.123,325.0,0.125542 +l2_distance_32,0.2,0.8,31,32,0,0,0,32,7,1,30.467,6.129,0.000,6.129,325.0,0.125542 +l2_distance_32,0.1,0.9,31,32,0,0,0,32,7,1,30.376,6.124,0.000,6.124,325.0,0.125542 +gx_kernel_4,1.0,0.0,4,0,1,1,2,3,6,1,15.805,0.487,0.112,0.599,327.5,18.1967,382.0 +gx_kernel_4,0.9,0.1,4,0,1,1,2,4,7,2,14.253,0.648,0.112,0.761,295.0,18.1967,484.0 +gx_kernel_4,0.8,0.2,4,0,1,1,2,4,7,2,14.006,0.648,0.112,0.761,295.0,18.1967,484.0 +gx_kernel_4,0.7,0.3,5,0,2,0,2,4,6,2,12.432,0.508,0.000,0.508,296.0,0.125542,417.0 +gx_kernel_4,0.6,0.4,5,0,2,0,2,4,6,2,12.387,0.508,0.000,0.508,296.0,0.125542,417.0 +gx_kernel_4,0.5,0.5,5,0,2,0,2,5,6,3,12.507,0.669,0.000,0.669,276.0,0.125542,518.0 +gx_kernel_4,0.4,0.6,5,0,2,0,2,5,6,3,12.457,0.669,0.000,0.669,276.0,0.125542,518.0 +gx_kernel_4,0.3,0.7,5,0,2,0,2,5,6,3,12.575,0.669,0.000,0.670,276.0,0.125542,518.0 +gx_kernel_4,0.2,0.8,5,0,2,0,2,5,6,3,12.478,0.670,0.000,0.670,276.5,0.125542,518.0 +gx_kernel_4,0.1,0.9,3,2,2,0,0,5,6,3,12.888,0.668,0.000,0.668,276.0,0.125542,516.0 +gx_kernel_8,1.0,0.0 +gx_kernel_8,0.9,0.1 +gx_kernel_8,0.8,0.2 +gx_kernel_8,0.7,0.3 +gx_kernel_8,0.6,0.4 +gx_kernel_8,0.5,0.5 +gx_kernel_8,0.4,0.6 +gx_kernel_8,0.3,0.7 +gx_kernel_8,0.2,0.8 +gx_kernel_8,0.1,0.9 +gx_kernel_16,1.0,0.0 +gx_kernel_16,0.9,0.1 +gx_kernel_16,0.8,0.2 +gx_kernel_16,0.7,0.3 +gx_kernel_16,0.6,0.4 +gx_kernel_16,0.5,0.5 +gx_kernel_16,0.4,0.6 +gx_kernel_16,0.3,0.7 +gx_kernel_16,0.2,0.8 +gx_kernel_16,0.1,0.9 +gx_kernel_32,1.0,0.0 +gx_kernel_32,0.9,0.1 +gx_kernel_32,0.8,0.2 +gx_kernel_32,0.7,0.3 +gx_kernel_32,0.6,0.4 +gx_kernel_32,0.5,0.5 +gx_kernel_32,0.4,0.6 +gx_kernel_32,0.3,0.7 +gx_kernel_32,0.2,0.8 +gx_kernel_32,0.1,0.9 +gy_kernel_4,1.0,0.0,3,0,0,1,3,3,6,1,13.138,0.613,0.112,0.725,327.0,18.1967,382.0 +gy_kernel_4,0.9,0.1,3,0,0,1,3,4,7,2,13.728,0.776,0.112,0.888,295.0,18.1967,484.0 +gy_kernel_4,0.8,0.2,7,0,0,1,4,7,7,2,13.923,1.357,0.112,1.469,294.5,18.1967,1285.0 +gy_kernel_4,0.7,0.3,6,1,0,0,4,5,6,1,12.744,1.015,0.000,1.015,327.0,0.125542,1015.0 +gy_kernel_4,0.6,0.4,6,1,0,0,4,6,7,2,12.979,1.177,0.000,1.177,295.0,0.125542,1117.0 +gy_kernel_4,0.5,0.5,5,2,0,0,3,7,8,3,13.048,1.340,0.000,1.340,263.0,0.125542,1218.0 +gy_kernel_4,0.4,0.6,5,2,0,0,3,8,8,3,13.417,1.504,0.000,1.504,263.0,0.125542,1318.0 +gy_kernel_4,0.3,0.7,5,2,0,0,3,8,8,3,13.631,1.504,0.000,1.504,263.0,0.125542,1318.0 +gy_kernel_4,0.2,0.8,5,2,0,0,3,8,8,3,13.347,1.504,0.000,1.504,263.0,0.125542,1318.0 +gy_kernel_4,0.1,0.9,5,2,0,0,3,8,8,3,13.359,1.504,0.000,1.504,263.0,0.125542,1318.0 +gy_kernel_8,1.0,0.0 +gy_kernel_8,0.9,0.1 +gy_kernel_8,0.8,0.2 +gy_kernel_8,0.7,0.3 +gy_kernel_8,0.6,0.4 +gy_kernel_8,0.5,0.5 +gy_kernel_8,0.4,0.6 +gy_kernel_8,0.3,0.7 +gy_kernel_8,0.2,0.8 +gy_kernel_8,0.1,0.9 +gy_kernel_16,1.0,0.0 +gy_kernel_16,0.9,0.1 +gy_kernel_16,0.8,0.2 +gy_kernel_16,0.7,0.3 +gy_kernel_16,0.6,0.4 +gy_kernel_16,0.5,0.5 +gy_kernel_16,0.4,0.6 +gy_kernel_16,0.3,0.7 +gy_kernel_16,0.2,0.8 +gy_kernel_16,0.1,0.9 +gy_kernel_32,1.0,0.0 +gy_kernel_32,0.9,0.1 +gy_kernel_32,0.8,0.2 +gy_kernel_32,0.7,0.3 +gy_kernel_32,0.6,0.4 +gy_kernel_32,0.5,0.5 +gy_kernel_32,0.4,0.6 +gy_kernel_32,0.3,0.7 +gy_kernel_32,0.2,0.8 +gy_kernel_32,0.1,0.9 +roberts_cross_4,1.0,0.0,3,0,0,1,3,1,5,1,13.618,0.262,0.112,0.374,327.0,18.1967,182.0 +roberts_cross_4,0.9,0.1,3,0,0,1,3,1,5,1,13.568,0.262,0.112,0.374,327.0,18.1967,182.0 +roberts_cross_4,0.8,0.2,3,0,0,1,3,1,5,1,13.544,0.262,0.112,0.374,327.5,18.1967,182.0 +roberts_cross_4,0.7,0.3,1,3,0,0,2,2,4,1,13.551,0.408,0.000,0.408,327.0,0.125542,214.0 +roberts_cross_4,0.6,0.4,1,3,0,0,2,2,4,1,13.379,0.408,0.000,0.408,327.0,0.125542,214.0 +roberts_cross_4,0.5,0.5,1,3,0,0,2,2,4,1,13.459,0.408,0.000,0.408,327.0,0.125542,214.0 +roberts_cross_4,0.4,0.6,1,3,0,0,2,2,4,1,13.921,0.408,0.000,0.408,327.0,0.125542,214.0 +roberts_cross_4,0.3,0.7,1,3,0,0,2,2,4,1,12.932,0.408,0.000,0.408,327.0,0.125542,214.0 +roberts_cross_4,0.2,0.8,1,3,0,0,2,2,4,1,12.962,0.408,0.000,0.408,327.0,0.125542,214.0 +roberts_cross_4,0.1,0.9,1,3,0,0,2,2,4,1,12.879,0.408,0.000,0.408,327.0,0.125542,214.0 +roberts_cross_8,1.0,0.0 +roberts_cross_8,0.9,0.1 +roberts_cross_8,0.8,0.2 +roberts_cross_8,0.7,0.3 +roberts_cross_8,0.6,0.4 +roberts_cross_8,0.5,0.5 +roberts_cross_8,0.4,0.6 +roberts_cross_8,0.3,0.7 +roberts_cross_8,0.2,0.8 +roberts_cross_8,0.1,0.9 +roberts_cross_16,1.0,0.0 +roberts_cross_16,0.9,0.1 +roberts_cross_16,0.8,0.2 +roberts_cross_16,0.7,0.3 +roberts_cross_16,0.6,0.4 +roberts_cross_16,0.5,0.5 +roberts_cross_16,0.4,0.6 +roberts_cross_16,0.3,0.7 +roberts_cross_16,0.2,0.8 +roberts_cross_16,0.1,0.9 +roberts_cross_32,1.0,0.0 +roberts_cross_32,0.9,0.1 +roberts_cross_32,0.8,0.2 +roberts_cross_32,0.7,0.3 +roberts_cross_32,0.6,0.4 +roberts_cross_32,0.5,0.5 +roberts_cross_32,0.4,0.6 +roberts_cross_32,0.3,0.7 +roberts_cross_32,0.2,0.8 +roberts_cross_32,0.1,0.9 +matrix_mul_4,1.0,0.0,2,0,0,1,0,2,4,1,12.261,0.424,0.113,0.537,327.0,18.1967,276.0 +matrix_mul_4,0.9,0.1,2,0,0,1,0,2,4,1,12.418,0.424,0.112,0.537,327.0,18.1967,276.0 +matrix_mul_4,0.8,0.2,3,0,0,0,0,4,3,1,11.955,0.757,0.000,0.757,327.5,0.125542,409.0 +matrix_mul_4,0.7,0.3,3,0,0,0,0,4,3,1,12.071,0.756,0.000,0.756,327.0,0.125542,409.0 +matrix_mul_4,0.6,0.4,3,0,0,0,0,4,3,1,11.945,0.757,0.000,0.757,327.5,0.125542,409.0 +matrix_mul_4,0.5,0.5,3,0,0,0,0,4,3,1,12.195,0.757,0.000,0.757,327.5,0.125542,409.0 +matrix_mul_4,0.4,0.6,3,0,0,0,0,4,3,1,12.135,0.757,0.000,0.757,327.5,0.125542,409.0 +matrix_mul_4,0.3,0.7,3,0,0,0,0,4,3,1,12.108,0.756,0.000,0.756,327.0,0.125542,409.0 +matrix_mul_4,0.2,0.8,3,0,0,0,0,4,3,1,12.083,0.757,0.000,0.757,327.0,0.125542,409.0 +matrix_mul_4,0.1,0.9,3,0,0,0,0,4,3,1,12.228,0.757,0.000,0.757,327.5,0.125542,409.0 +matrix_mul_8,1.0,0.0 +matrix_mul_8,0.9,0.1 +matrix_mul_8,0.8,0.2 +matrix_mul_8,0.7,0.3 +matrix_mul_8,0.6,0.4 +matrix_mul_8,0.5,0.5 +matrix_mul_8,0.4,0.6 +matrix_mul_8,0.3,0.7 +matrix_mul_8,0.2,0.8 +matrix_mul_8,0.1,0.9 +matrix_mul_16,1.0,0.0 +matrix_mul_16,0.9,0.1 +matrix_mul_16,0.8,0.2 +matrix_mul_16,0.7,0.3 +matrix_mul_16,0.6,0.4 +matrix_mul_16,0.5,0.5 +matrix_mul_16,0.4,0.6 +matrix_mul_16,0.3,0.7 +matrix_mul_16,0.2,0.8 +matrix_mul_16,0.1,0.9 +matrix_mul_32,1.0,0.0 +matrix_mul_32,0.9,0.1 +matrix_mul_32,0.8,0.2 +matrix_mul_32,0.7,0.3 +matrix_mul_32,0.6,0.4 +matrix_mul_32,0.5,0.5 +matrix_mul_32,0.4,0.6 +matrix_mul_32,0.3,0.7 +matrix_mul_32,0.2,0.8 +matrix_mul_32,0.1,0.9 +tree_50-50_5_1,1.0,0.0,5,0,0,3,0,2,9,2,14.572,0.530,0.334,0.864,295.0,54.3391,787.0 +tree_50-50_5_1,0.9,0.1,5,0,0,3,0,2,9,2,11.365,0.530,0.334,0.864,295.0,54.3391,787.0 +tree_50-50_5_1,0.8,0.2,5,0,0,2,0,2,7,2,11.337,0.514,0.223,0.737,296.0,36.2679,719.0 +tree_50-50_5_1,0.7,0.3,6,0,0,1,0,2,6,2,11.236,0.518,0.112,0.630,296.0,18.1967,1162.0 +tree_50-50_5_1,0.6,0.4,6,0,0,1,0,2,6,2,11.278,0.519,0.112,0.631,296.0,18.1967,1162.0 +tree_50-50_5_1,0.5,0.5,5,0,0,0,0,2,4,2,11.277,0.478,0.000,0.478,296.0,0.125542,1758.0 +tree_50-50_5_1,0.4,0.6,5,0,0,0,0,2,4,2,11.276,0.478,0.000,0.478,296.0,0.125542,1758.0 +tree_50-50_5_1,0.3,0.7,5,0,0,0,0,2,4,2,11.409,0.477,0.000,0.477,296.0,0.125542,1758.0 +tree_50-50_5_1,0.2,0.8,5,0,0,0,0,2,4,2,11.318,0.478,0.000,0.478,296.0,0.125542,1758.0 +tree_50-50_5_1,0.1,0.9,5,0,0,0,0,2,4,2,11.244,0.478,0.000,0.478,296.0,0.125542,1758.0 +tree_50-50_10_1,1.0,0.0,13,0,0,5,0,7,16,5,129.903,1.573,0.557,2.130,197.0,90.4814,2774.0 +tree_100-50_5_1,1.0,0.0,12,0,0,5,0,4,15,3,111.459,1.058,0.556,1.614,261.0,90.4814,1771.0 +tree_100-50_5_1,0.9,0.1,15,0,0,4,0,8,13,3,23.684,1.737,0.446,2.182,261.0,72.4103,3362.0 +tree_100-50_10_1,0.7,0.3 +tree_50-50_10_1,0.1,0.9,16.0,0.0,0.0,0.0,0.0,17.0,8.0,5.0,14.687,3.266,0.000,3.266,199,0.125542 +tree_100-50_5_1,0.1,0.9,19.0,0.0,0.0,0.0,0.0,12.0,5.0,3.0,13.600,2.490,0.000,2.490,263,0.125542 +tree_50-50_10_1,0.9,0.1,19.0,0.0,0.0,2.0,0.0,16.0,12.0,5.0,32.492,3.186,0.232,3.418,197,36.2679 +tree_50-50_10_1,0.8,0.2,19.0,0.0,0.0,2.0,0.0,16.0,12.0,5.0,32.642,3.185,0.231,3.416,198,36.2679 +tree_100-50_5_1,0.8,0.2,24.0,0.0,0.0,2.0,0.0,9.0,11.0,3.0,31.221,2.090,0.230,2.321,261,36.2679 \ No newline at end of file diff --git a/pareto_env_0_05.csv b/pareto_env_0_05.csv new file mode 100644 index 00000000..6bb8a4e2 --- /dev/null +++ b/pareto_env_0_05.csv @@ -0,0 +1,501 @@ +benchmark,w_ops,w_keys,add,sub,multiply_plain,rotate_rows,negate,multiply,Depth,Multiplicative Depth,compile_time (s),circuit_execution_time (s),galois_keys_generation_time (s),total_execution_time (s),Remaining_noise_budget,rotation_keys_size (MB),Final_Cost +max_3,1.0,0.0,2,1,0,3,0,2,8,2,12.093,0.481,0.334,0.815,295.0,54.3391,0.0 +max_3,0.9,0.1,5,0,0,2,2,3,9,2,11.181,0.632,0.222,0.854,295.0,36.2679,0.0 +max_3,0.8,0.2,7,1,0,3,2,4,10,2,11.075,0.858,0.222,1.080,295.0,36.2679,0.0 +max_3,0.7,0.3,7,1,0,2,2,5,10,2,12.817,0.978,0.223,1.202,294.5,36.2679 +max_3,0.6,0.4,7,1,0,2,2,5,10,2,12.864,0.979,0.223,1.202,295.0,36.2679 +max_3,0.5,0.5,7,1,0,2,2,5,10,2,12.808,0.978,0.223,1.202,295.0,36.2679 +max_3,0.4,0.6,8,0,0,1,3,5,8,2,12.349,0.937,0.112,1.049,295.0,18.1967 +max_3,0.3,0.7,6,0,0,0,3,6,6,2,10.935,0.992,0.000,0.992,295.0,0.125542,0.0 +max_3,0.2,0.8,6,0,0,0,3,6,6,2,10.878,0.991,0.000,0.991,295.0,0.125542,0.0 +max_3,0.1,0.9,6,0,0,0,3,6,6,2,11.929,0.991,0.000,0.992,295.0,0.125542 +max_4,1.0,0.0,4,1,0,4,1,4,12,3,16.561,0.856,0.444,1.300,262.0,72.4103,0.0 +max_4,0.9,0.1 +max_4,0.8,0.2,17,1,0,3,6,12,12,3,17.047,2.185,0.223,2.408,261.5,36.2679 +max_4,0.7,0.3,15,1,0,2,6,13,12,3,15.313,2.239,0.223,2.462,262.0,36.2679 +max_4,0.6,0.4,16,0,0,1,7,13,10,3,13.950,2.195,0.112,2.307,262.0,18.1967 +max_4,0.5,0.5,16,0,0,1,7,13,10,3,13.945,2.195,0.112,2.307,262.0,18.1967 +max_4,0.4,0.6,12,0,0,0,6,12,8,3,12.823,1.919,0.000,1.919,262.0,0.125542 +max_4,0.3,0.7,12,0,0,0,7,12,8,3,12.881,1.921,0.000,1.921,262.0,0.125542 +max_4,0.2,0.8,12,0,0,0,6,12,8,3,12.924,1.920,0.000,1.920,262.0,0.125542 +max_4,0.1,0.9,12,0,0,0,6,12,8,3,12.920,1.919,0.000,1.919,262.0,0.125542 +max_5,1.0,0.0,21,1,0,4,9,20,14,4,29.765,3.448,0.444,3.891,229.0,72.4103,0.0 +max_5,0.9,0.1 +max_5,0.8,0.2 +max_5,0.7,0.3,31,1,0,3,13,26,14,4,45.604,4.384,0.222,4.606,228.5,36.2679 +max_5,0.6,0.4,28,0,0,1,13,25,12,4,18.114,4.054,0.112,4.166,229.0,18.1967 +max_5,0.5,0.5,20,0,0,0,10,20,10,4,14.945,3.159,0.000,3.159,229.0,0.125542 +max_5,0.4,0.6,20,0,0,0,10,20,10,4,15.040,3.159,0.000,3.159,229.0,0.125542 +max_5,0.3,0.7,20,0,0,0,10,20,10,4,15.051,3.159,0.000,3.159,229.0,0.125542 +max_5,0.2,0.8,20,0,0,0,10,20,10,4,15.068,3.159,0.000,3.159,229.0,0.125542 +max_5,0.1,0.9,20,0,0,0,10,20,10,4,15.035,3.159,0.000,3.159,229.0,0.125542 +sort_3,1.0,0.0 +sort_3,0.9,0.1 +sort_3,0.8,0.2 +sort_3,0.7,0.3 +sort_3,0.6,0.4 +sort_3,0.5,0.5 +sort_3,0.4,0.6 +sort_3,0.3,0.7 +sort_3,0.2,0.8 +sort_3,0.1,0.9 +box_blur_4,1.0,0.0,8,0,0,0,0,0,4,0,11.504,0.217,0.000,0.217,359.0,0.125542,0.0 +box_blur_4,0.9,0.1,8,0,0,0,0,0,4,0,11.524,0.217,0.000,0.217,359.5,0.125542,0.0 +box_blur_4,0.8,0.2,8,0,0,0,0,0,4,0,11.543,0.217,0.000,0.217,359.0,0.125542,0.0 +box_blur_4,0.7,0.3,8,0,0,0,0,0,4,0,11.518,0.217,0.000,0.217,359.0,0.125542,0.0 +box_blur_4,0.6,0.4,8,0,0,0,0,0,4,0,11.522,0.217,0.000,0.217,359.0,0.125542,0.0 +box_blur_4,0.5,0.5,8,0,0,0,0,0,4,0,11.488,0.217,0.000,0.217,359.5,0.125542,0.0 +box_blur_4,0.4,0.6,8,0,0,0,0,0,4,0,11.519,0.217,0.000,0.217,359.5,0.125542,0.0 +box_blur_4,0.3,0.7,8,0,0,0,0,0,4,0,11.499,0.217,0.000,0.217,359.0,0.125542,0.0 +box_blur_4,0.2,0.8,8,0,0,0,0,0,4,0,11.471,0.217,0.000,0.217,359.0,0.125542,0.0 +box_blur_4,0.1,0.9,8,0,0,0,0,0,5,0,11.546,0.217,0.000,0.217,359.0,0.125542,0.0 +box_blur_8,1.0,0.0 +box_blur_8,0.9,0.1 +box_blur_8,0.8,0.2 +box_blur_8,0.7,0.3 +box_blur_8,0.6,0.4 +box_blur_8,0.5,0.5 +box_blur_8,0.4,0.6 +box_blur_8,0.3,0.7 +box_blur_8,0.2,0.8 +box_blur_8,0.1,0.9 +box_blur_16,1.0,0.0 +box_blur_16,0.9,0.1 +box_blur_16,0.8,0.2 +box_blur_16,0.7,0.3 +box_blur_16,0.6,0.4 +box_blur_16,0.5,0.5 +box_blur_16,0.4,0.6 +box_blur_16,0.3,0.7 +box_blur_16,0.2,0.8 +box_blur_16,0.1,0.9 +box_blur_32,1.0,0.0 +box_blur_32,0.9,0.1 +box_blur_32,0.8,0.2 +box_blur_32,0.7,0.3 +box_blur_32,0.6,0.4 +box_blur_32,0.5,0.5 +box_blur_32,0.4,0.6 +box_blur_32,0.3,0.7 +box_blur_32,0.2,0.8 +box_blur_32,0.1,0.9 +lin_reg_4,1.0,0.0,2,0,0,2,0,1,5,1,11.949,0.279,0.223,0.502,327.5,36.2679,0.0 +lin_reg_4,0.9,0.1,2,0,0,2,0,1,5,1,11.009,0.279,0.223,0.502,327.0,36.2679,0.0 +lin_reg_4,0.8,0.2,2,0,0,1,0,1,4,1,10.914,0.239,0.112,0.350,328.0,18.1967,0.0 +lin_reg_4,0.7,0.3,2,0,1,0,0,1,3,1,10.649,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_4,0.6,0.4,2,0,1,0,0,1,3,1,10.791,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_4,0.5,0.5,2,0,1,0,0,1,3,1,10.683,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_4,0.4,0.6,2,0,1,0,0,1,3,1,10.670,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_4,0.3,0.7,2,0,1,0,0,1,3,1,10.682,0.053,0.000,0.053,358.5,0.125542,0.0 +lin_reg_4,0.2,0.8,2,0,1,0,0,1,3,1,10.691,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_4,0.1,0.9,2,0,1,0,0,1,3,1,10.780,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_8,1.0,0.0,2,0,0,2,0,1,5,1,11.002,0.279,0.223,0.501,327.0,36.2679,0.0 +lin_reg_8,0.9,0.1,2,0,0,2,0,1,5,1,10.884,0.279,0.222,0.501,328.0,36.2679,0.0 +lin_reg_8,0.8,0.2,2,0,0,1,0,1,4,1,10.774,0.238,0.112,0.350,328.0,18.1967,0.0 +lin_reg_8,0.7,0.3,2,0,1,0,0,1,3,1,10.812,0.053,0.000,0.053,358.0,0.125542,0.0 +lin_reg_8,0.6,0.4,2,0,1,0,0,1,3,1,10.860,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_8,0.5,0.5,2,0,1,0,0,1,3,1,10.737,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_8,0.4,0.6,2,0,1,0,0,1,3,1,10.796,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_8,0.3,0.7,2,0,1,0,0,1,3,1,10.814,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_8,0.2,0.8,2,0,1,0,0,1,3,1,10.740,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_8,0.1,0.9,2,0,1,0,0,1,3,1,10.840,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_16,1.0,0.0,2,0,0,1,0,1,4,1,11.008,0.239,0.112,0.351,328.0,18.1967,0.0 +lin_reg_16,0.9,0.1,2,0,1,1,0,1,4,1,11.040,0.093,0.112,0.205,357.0,18.1967,0.0 +lin_reg_16,0.8,0.2,2,0,1,0,0,1,3,1,10.860,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_16,0.7,0.3,2,0,1,0,0,1,3,1,10.979,0.053,0.000,0.053,358.5,0.125542,0.0 +lin_reg_16,0.6,0.4,2,0,1,0,0,1,3,1,10.905,0.053,0.000,0.053,358.5,0.125542,0.0 +lin_reg_16,0.5,0.5,2,0,1,0,0,1,3,1,10.976,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_16,0.4,0.6,2,0,1,0,0,1,3,1,10.862,0.053,0.000,0.053,358.5,0.125542,0.0 +lin_reg_16,0.3,0.7,2,0,1,0,0,1,3,1,10.844,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_16,0.2,0.8,2,0,1,0,0,1,3,1,10.873,0.053,0.000,0.053,358.5,0.125542,0.0 +lin_reg_16,0.1,0.9,2,0,1,0,0,1,3,1,10.863,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_32,1.0,0.0,2,0,1,0,0,1,3,1,11.209,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_32,0.9,0.1,2,0,1,0,0,1,3,1,11.128,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_32,0.8,0.2,2,0,1,0,0,1,3,1,11.076,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_32,0.7,0.3,2,0,1,0,0,1,3,1,11.128,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_32,0.6,0.4,2,0,1,0,0,1,3,1,11.130,0.053,0.000,0.053,358.5,0.125542,0.0 +lin_reg_32,0.5,0.5,2,0,1,0,0,1,3,1,11.014,0.053,0.000,0.053,358.5,0.125542,0.0 +lin_reg_32,0.4,0.6,2,0,1,0,0,1,3,1,11.000,0.053,0.000,0.053,358.5,0.125542,0.0 +lin_reg_32,0.3,0.7,2,0,1,0,0,1,3,1,11.054,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_32,0.2,0.8,2,0,1,0,0,1,3,1,11.138,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_32,0.1,0.9,2,0,1,0,0,1,3,1,11.226,0.053,0.000,0.053,359.0,0.125542,0.0 +hamming_dist_4,1.0,0.0,4,1,0,3,0,3,10,2,11.370,0.673,0.334,1.008,294.0,54.3391,0.0 +hamming_dist_4,0.9,0.1,4,1,0,2,0,3,8,2,11.125,0.654,0.223,0.876,295.0,36.2679,0.0 +hamming_dist_4,0.8,0.2,6,2,1,1,0,5,7,2,11.173,0.800,0.112,0.912,295.0,18.1967,0.0 +hamming_dist_4,0.7,0.3,6,2,1,1,0,5,7,2,11.241,0.799,0.112,0.912,295.0,18.1967,0.0 +hamming_dist_4,0.6,0.4,6,2,1,1,0,5,7,2,11.147,0.800,0.112,0.912,295.0,18.1967,0.0 +hamming_dist_4,0.5,0.5,6,2,1,1,0,5,7,2,11.111,0.799,0.112,0.912,295.0,18.1967,0.0 +hamming_dist_4,0.4,0.6,6,2,1,1,0,5,7,2,11.158,0.800,0.112,0.912,295.0,18.1967,0.0 +hamming_dist_4,0.3,0.7,7,4,4,0,0,8,5,2,12.322,0.763,0.000,0.763,326.5,0.125542 +hamming_dist_4,0.2,0.8,7,4,4,0,0,8,5,2,12.328,0.763,0.000,0.763,326.5,0.125542 +hamming_dist_4,0.1,0.9,7,4,4,0,0,8,5,2,12.315,0.763,0.000,0.763,326.5,0.125542 +hamming_dist_8,1.0,0.0,8,4,3,5,0,9,10,2,12.918,1.290,0.334,1.624,293.0,54.3391,0.0 +hamming_dist_8,0.9,0.1,8,4,3,3,0,9,8,2,12.157,1.210,0.223,1.433,292.5,36.2679,0.0 +hamming_dist_8,0.8,0.2,10,4,3,1,0,9,8,2,12.046,1.174,0.112,1.287,295.0,18.1967,0.0 +hamming_dist_8,0.7,0.3,10,4,3,1,0,9,8,2,12.060,1.174,0.112,1.286,295.0,18.1967,0.0 +hamming_dist_8,0.6,0.4,10,4,3,1,0,9,8,2,12.334,1.174,0.112,1.287,295.0,18.1967,0.0 +hamming_dist_8,0.5,0.5,10,4,3,1,0,9,8,2,12.028,1.174,0.112,1.286,295.0,18.1967,0.0 +hamming_dist_8,0.4,0.6,15,8,8,0,0,16,6,2,13.629,1.513,0.000,1.513,326.0,0.125542 +hamming_dist_8,0.3,0.7,15,8,8,0,0,16,6,2,13.632,1.513,0.000,1.513,326.0,0.125542 +hamming_dist_8,0.2,0.8,15,8,8,0,0,16,6,2,13.643,1.513,0.000,1.513,326.0,0.125542 +hamming_dist_8,0.1,0.9,15,8,8,0,0,16,6,2,13.757,1.514,0.000,1.514,326.0,0.125542 +hamming_dist_16,1.0,0.0,16,8,7,3,0,17,9,2,20.618,1.966,0.223,2.189,293.0,36.2679,0.0 +hamming_dist_16,0.9,0.1,16,8,7,3,0,17,9,2,16.749,1.965,0.223,2.188,292.0,36.2679,0.0 +hamming_dist_16,0.8,0.2,18,8,7,1,0,17,9,2,16.484,1.922,0.112,2.034,295.0,18.1967,0.0 +hamming_dist_16,0.7,0.3,18,8,7,1,0,17,9,2,16.460,1.921,0.112,2.032,295.0,18.1967,0.0 +hamming_dist_16,0.6,0.4,18,8,7,1,0,17,11,2,16.731,1.922,0.112,2.033,295.0,18.1967,0.0 +hamming_dist_16,0.5,0.5,31,16,16,0,0,32,7,2,16.755,3.033,0.000,3.033,326.0,0.125542 +hamming_dist_16,0.4,0.6,31,16,16,0,0,32,7,2,16.440,3.032,0.000,3.032,325.5,0.125542 +hamming_dist_16,0.3,0.7,31,16,16,0,0,32,7,2,16.334,3.034,0.000,3.034,325.0,0.125542 +hamming_dist_16,0.2,0.8,31,16,16,0,0,32,7,2,16.281,3.033,0.000,3.034,325.0,0.125542 +hamming_dist_16,0.1,0.9,31,16,16,0,0,32,7,2,20.478,3.033,0.000,3.033,325.5,0.125542 +hamming_dist_32,1.0,0.0,34,16,15,1,0,33,10,2,43.697,3.421,0.112,3.533,295.0,18.1967 +hamming_dist_32,0.9,0.1,34,16,15,1,0,33,10,2,44.286,3.417,0.112,3.530,295.0,18.1967 +hamming_dist_32,0.8,0.2,34,16,15,1,0,33,10,2,44.169,3.417,0.112,3.530,295.0,18.1967 +hamming_dist_32,0.7,0.3,34,16,15,1,0,33,10,2,44.104,3.421,0.113,3.534,295.0,18.1967 +hamming_dist_32,0.6,0.4,63,32,32,0,0,64,8,2,25.032,5.991,0.000,5.991,325.0,0.125542 +hamming_dist_32,0.5,0.5,63,32,32,0,0,64,8,2,24.105,5.993,0.000,5.993,325.0,0.125542 +hamming_dist_32,0.4,0.6,63,32,32,0,0,64,8,2,24.988,5.993,0.000,5.993,325.0,0.125542 +hamming_dist_32,0.3,0.7,63,32,32,0,0,64,8,2,24.120,5.994,0.000,5.994,325.0,0.125542 +hamming_dist_32,0.2,0.8,63,32,32,0,0,64,8,2,24.971,5.994,0.000,5.994,325.0,0.125542 +hamming_dist_32,0.1,0.9,63,32,32,0,0,64,8,2,28.985,5.992,0.000,5.992,325.0,0.125542 +dot_product_4,1.0,0.0,2,0,0,3,0,1,6,1,11.932,0.296,0.334,0.631,325.0,54.3391,0.0 +dot_product_4,0.9,0.1,2,0,0,3,0,1,6,1,11.027,0.297,0.334,0.631,324.5,54.3391,0.0 +dot_product_4,0.8,0.2,2,0,0,2,0,1,5,1,10.943,0.279,0.223,0.502,328.0,36.2679,0.0 +dot_product_4,0.7,0.3,2,0,0,1,0,2,4,1,10.775,0.424,0.113,0.537,327.5,18.1967,0.0 +dot_product_4,0.6,0.4,2,0,0,1,0,2,4,1,10.733,0.424,0.113,0.537,327.0,18.1967,0.0 +dot_product_4,0.5,0.5,2,0,0,1,0,2,4,1,10.853,0.424,0.113,0.537,327.0,18.1967,0.0 +dot_product_4,0.4,0.6,2,0,0,1,0,2,4,1,10.751,0.424,0.113,0.537,327.5,18.1967,0.0 +dot_product_4,0.3,0.7,3,0,0,0,0,4,3,1,10.774,0.757,0.000,0.757,327.5,0.125542,0.0 +dot_product_4,0.2,0.8,3,0,0,0,0,4,3,1,10.740,0.757,0.000,0.757,327.5,0.125542,0.0 +dot_product_4,0.1,0.9,3,0,0,0,0,4,3,1,10.768,0.758,0.000,0.758,327.0,0.125542,0.0 +dot_product_8,1.0,0.0,3,0,0,3,0,1,7,1,10.992,0.320,0.335,0.655,327.0,54.3391,0.0 +dot_product_8,0.9,0.1,3,0,0,2,0,2,6,1,10.964,0.466,0.223,0.689,327.0,36.2679,0.0 +dot_product_8,0.8,0.2,4,0,0,1,0,4,5,1,10.968,0.798,0.112,0.910,327.0,18.1967,0.0 +dot_product_8,0.7,0.3,4,0,0,1,0,4,5,1,10.993,0.798,0.112,0.910,327.0,18.1967,0.0 +dot_product_8,0.6,0.4,4,0,0,1,0,4,5,1,10.976,0.798,0.112,0.910,327.0,18.1967,0.0 +dot_product_8,0.5,0.5,4,0,0,1,0,4,5,1,10.971,0.798,0.112,0.911,327.0,18.1967,0.0 +dot_product_8,0.4,0.6,4,0,0,1,0,4,5,1,11.091,0.798,0.112,0.911,327.0,18.1967,0.0 +dot_product_8,0.3,0.7,7,0,0,0,0,8,4,1,10.776,1.508,0.000,1.508,327.0,0.125542,0.0 +dot_product_8,0.2,0.8,7,0,0,0,0,8,4,1,10.770,1.507,0.000,1.507,327.0,0.125542,0.0 +dot_product_8,0.1,0.9,7,0,0,0,0,8,4,1,11.980,1.507,0.000,1.507,327.0,0.125542 +dot_product_16,1.0,0.0,5,0,0,2,0,4,7,1,11.472,0.840,0.223,1.063,326.5,36.2679,0.0 +dot_product_16,0.9,0.1,8,0,0,1,0,8,6,1,11.600,1.545,0.112,1.658,326.0,18.1967,0.0 +dot_product_16,0.8,0.2,8,0,0,1,0,8,6,1,11.542,1.545,0.112,1.657,326.0,18.1967,0.0 +dot_product_16,0.7,0.3,8,0,0,1,0,8,6,1,11.563,1.546,0.112,1.658,326.0,18.1967,0.0 +dot_product_16,0.6,0.4,8,0,0,1,0,8,6,1,11.705,1.545,0.112,1.658,326.0,18.1967,0.0 +dot_product_16,0.5,0.5,8,0,0,1,0,8,6,1,11.601,1.545,0.112,1.658,326.5,18.1967,0.0 +dot_product_16,0.4,0.6,15,0,0,0,0,16,5,1,12.974,2.991,0.000,2.991,326.5,0.125542 +dot_product_16,0.3,0.7,15,0,0,0,0,16,5,1,13.108,2.991,0.000,2.991,326.0,0.125542 +dot_product_16,0.2,0.8,15,0,0,0,0,16,5,1,13.029,2.992,0.000,2.992,327.0,0.125542 +dot_product_16,0.1,0.9,15,0,0,0,0,16,5,1,13.048,2.991,0.000,2.991,326.5,0.125542 +dot_product_32,1.0,0.0,16,0,0,1,0,16,7,1,14.305,3.039,0.113,3.152,326.0,18.1967,0.0 +dot_product_32,0.9,0.1,16,0,0,1,0,16,7,1,14.332,3.039,0.113,3.151,326.0,18.1967,0.0 +dot_product_32,0.8,0.2,16,0,0,1,0,16,7,1,14.407,3.039,0.113,3.152,326.0,18.1967,0.0 +dot_product_32,0.7,0.3,16,0,0,1,0,16,7,1,14.335,3.038,0.113,3.151,326.0,18.1967,0.0 +dot_product_32,0.6,0.4,16,0,0,1,0,16,18,1,15.740,3.036,0.112,3.149,326.0,18.1967,0.0 +dot_product_32,0.5,0.5,31,0,0,0,0,32,6,1,15.483,5.967,0.000,5.967,326.0,0.125542 +dot_product_32,0.4,0.6,31,0,0,0,0,32,6,1,15.174,5.965,0.000,5.965,326.0,0.125542 +dot_product_32,0.3,0.7,31,0,0,0,0,32,6,1,15.139,5.965,0.000,5.965,326.0,0.125542 +dot_product_32,0.2,0.8,31,0,0,0,0,32,6,1,15.076,5.965,0.000,5.965,326.0,0.125542 +dot_product_32,0.1,0.9,31,0,0,0,0,32,6,1,15.100,5.965,0.000,5.965,326.0,0.125542 +poly_reg_4,1.0,0.0,3,0,0,2,0,2,7,2,10.832,0.444,0.223,0.667,295.0,36.2679,0.0 +poly_reg_4,0.9,0.1,3,0,0,2,0,2,7,2,10.871,0.444,0.223,0.667,295.0,36.2679,0.0 +poly_reg_4,0.8,0.2,3,0,0,1,0,2,6,2,10.697,0.426,0.112,0.538,295.0,18.1967,0.0 +poly_reg_4,0.7,0.3,3,0,0,0,0,2,5,2,10.642,0.409,0.000,0.409,296.0,0.125542,0.0 +poly_reg_4,0.6,0.4,3,0,0,0,0,2,5,2,10.915,0.410,0.000,0.410,296.0,0.125542,0.0 +poly_reg_4,0.5,0.5,3,0,0,0,0,2,5,2,10.904,0.409,0.000,0.409,296.0,0.125542,0.0 +poly_reg_4,0.4,0.6,3,0,0,0,0,2,5,2,10.662,0.409,0.000,0.409,296.0,0.125542,0.0 +poly_reg_4,0.3,0.7,3,0,0,0,0,2,5,2,10.915,0.409,0.000,0.409,296.0,0.125542,0.0 +poly_reg_4,0.2,0.8,3,0,0,0,0,2,5,2,10.914,0.409,0.000,0.409,296.0,0.125542,0.0 +poly_reg_4,0.1,0.9,3,0,0,0,0,2,5,2,10.684,0.409,0.000,0.409,296.0,0.125542,0.0 +poly_reg_8,1.0,0.0,3,0,0,2,0,2,7,2,11.586,0.444,0.223,0.667,295.0,36.2679,0.0 +poly_reg_8,0.9,0.1,3,0,0,1,0,2,6,2,11.195,0.426,0.112,0.538,295.0,18.1967,0.0 +poly_reg_8,0.8,0.2,3,0,0,0,0,2,5,2,11.039,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_8,0.7,0.3,3,0,0,0,0,2,5,2,10.820,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_8,0.6,0.4,3,0,0,0,0,2,5,2,10.777,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_8,0.5,0.5,3,0,0,0,0,2,5,2,10.876,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_8,0.4,0.6,3,0,0,0,0,2,5,2,10.933,0.408,0.000,0.408,295.5,0.125542,0.0 +poly_reg_8,0.3,0.7,3,0,0,0,0,2,5,2,10.861,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_8,0.2,0.8,3,0,0,0,0,2,5,2,10.950,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_8,0.1,0.9,3,0,0,0,0,2,5,2,10.858,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_16,1.0,0.0,3,0,0,1,0,2,6,2,11.222,0.449,0.113,0.561,296.0,18.1967,0.0 +poly_reg_16,0.9,0.1,3,0,0,0,0,2,5,2,11.099,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_16,0.8,0.2,3,0,0,0,0,2,5,2,11.127,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_16,0.7,0.3,3,0,0,0,0,2,5,2,11.102,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_16,0.6,0.4,3,0,0,0,0,2,5,2,11.128,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_16,0.5,0.5,3,0,0,0,0,2,5,2,11.156,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_16,0.4,0.6,3,0,0,0,0,2,5,2,11.130,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_16,0.3,0.7,3,0,0,0,0,2,5,2,11.143,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_16,0.2,0.8,3,0,0,0,0,2,5,2,11.183,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_16,0.1,0.9,3,0,0,0,0,2,5,2,11.166,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_32,1.0,0.0,3,0,0,0,0,2,5,2,11.560,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_32,0.9,0.1,3,0,0,0,0,2,5,2,11.661,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_32,0.8,0.2,3,0,0,0,0,2,5,2,11.566,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_32,0.7,0.3,3,0,0,0,0,2,5,2,11.608,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_32,0.6,0.4,3,0,0,0,0,2,5,2,11.631,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_32,0.5,0.5,3,0,0,0,0,2,5,2,11.612,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_32,0.4,0.6,3,0,0,0,0,2,5,2,11.621,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_32,0.3,0.7,3,0,0,0,0,2,5,2,11.620,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_32,0.2,0.8,3,0,0,0,0,2,5,2,11.601,0.408,0.000,0.408,296.0,0.125542,0.0 +poly_reg_32,0.1,0.9,3,0,0,0,0,2,5,2,11.633,0.408,0.000,0.408,295.5,0.125542,0.0 +l2_distance_4,1.0,0.0,2,1,0,3,0,1,7,1,10.962,0.321,0.335,0.656,324.5,54.3391,0.0 +l2_distance_4,0.9,0.1,2,1,0,2,0,1,6,1,10.886,0.280,0.222,0.503,326.0,36.2679,0.0 +l2_distance_4,0.8,0.2,2,2,0,1,0,2,5,1,10.948,0.426,0.112,0.538,326.5,18.1967,0.0 +l2_distance_4,0.7,0.3,2,2,0,1,0,2,5,1,10.946,0.426,0.112,0.538,326.5,18.1967,0.0 +l2_distance_4,0.6,0.4,2,2,0,1,0,2,5,1,10.984,0.426,0.112,0.538,327.0,18.1967,0.0 +l2_distance_4,0.5,0.5,2,2,0,1,0,2,5,1,11.002,0.426,0.112,0.539,327.0,18.1967,0.0 +l2_distance_4,0.4,0.6,2,2,0,1,0,2,5,1,10.961,0.428,0.112,0.540,326.0,18.1967,0.0 +l2_distance_4,0.3,0.7,3,4,0,0,0,4,4,1,10.720,0.761,0.000,0.761,326.5,0.125542,0.0 +l2_distance_4,0.2,0.8,3,4,0,0,0,4,4,1,10.764,0.762,0.000,0.762,326.5,0.125542,0.0 +l2_distance_4,0.1,0.9,3,4,0,0,0,4,4,1,10.779,0.761,0.000,0.761,326.5,0.125542,0.0 +l2_distance_8,1.0,0.0,3,2,0,2,0,2,7,1,11.435,0.468,0.223,0.691,326.0,36.2679,0.0 +l2_distance_8,0.9,0.1,4,4,0,1,0,4,6,1,11.559,0.801,0.113,0.913,326.0,18.1967,0.0 +l2_distance_8,0.8,0.2,4,4,0,1,0,4,6,1,11.518,0.801,0.113,0.913,326.0,18.1967,0.0 +l2_distance_8,0.7,0.3,4,4,0,1,0,4,6,1,11.550,0.801,0.113,0.913,326.0,18.1967,0.0 +l2_distance_8,0.6,0.4,4,4,0,1,0,4,6,1,11.564,0.801,0.113,0.913,326.0,18.1967,0.0 +l2_distance_8,0.5,0.5,4,4,0,1,0,4,6,1,11.546,0.801,0.113,0.914,326.0,18.1967,0.0 +l2_distance_8,0.4,0.6,7,8,0,0,0,8,5,1,13.157,1.509,0.000,1.509,326.0,0.125542 +l2_distance_8,0.3,0.7,7,8,0,0,0,8,5,1,13.009,1.509,0.000,1.509,326.0,0.125542 +l2_distance_8,0.2,0.8,7,8,0,0,0,8,5,1,13.006,1.509,0.000,1.509,326.0,0.125542 +l2_distance_8,0.1,0.9,7,8,0,0,0,8,5,1,12.974,1.510,0.000,1.510,326.0,0.125542 +l2_distance_16,1.0,0.0,8,8,0,1,0,8,7,1,14.257,1.550,0.112,1.662,325.5,18.1967,0.0 +l2_distance_16,0.9,0.1,8,8,0,1,0,8,7,1,14.265,1.550,0.112,1.663,326.0,18.1967,0.0 +l2_distance_16,0.8,0.2,8,8,0,1,0,8,7,1,14.246,1.550,0.112,1.662,325.0,18.1967,0.0 +l2_distance_16,0.7,0.3,8,8,0,1,0,8,7,1,14.256,1.550,0.112,1.662,325.5,18.1967,0.0 +l2_distance_16,0.6,0.4,8,8,0,1,0,8,8,1,14.336,1.552,0.112,1.664,325.5,18.1967,0.0 +l2_distance_16,0.5,0.5,15,16,0,0,0,16,6,1,15.209,2.997,0.000,2.997,326.0,0.125542 +l2_distance_16,0.4,0.6,15,16,0,0,0,16,6,1,15.390,2.997,0.000,2.997,325.5,0.125542 +l2_distance_16,0.3,0.7,15,16,0,0,0,16,6,1,15.353,2.997,0.000,2.997,325.5,0.125542 +l2_distance_16,0.2,0.8,15,16,0,0,0,16,6,1,15.265,2.997,0.000,2.997,326.0,0.125542 +l2_distance_16,0.1,0.9,15,16,0,0,0,16,6,1,15.269,2.997,0.000,2.997,325.5,0.125542 +l2_distance_32,1.0,0.0,16,16,0,1,0,16,8,1,31.288,3.048,0.112,3.160,325.0,18.1967,0.0 +l2_distance_32,0.9,0.1,16,16,0,1,0,16,8,1,31.085,3.048,0.112,3.160,325.0,18.1967,0.0 +l2_distance_32,0.8,0.2,16,16,0,1,0,16,8,1,31.241,3.048,0.112,3.160,325.0,18.1967,0.0 +l2_distance_32,0.7,0.3,16,16,0,1,0,16,14,1,33.433,3.045,0.112,3.157,325.0,18.1967,0.0 +l2_distance_32,0.6,0.4,31,32,0,0,0,32,7,1,21.505,5.988,0.000,5.988,325.0,0.125542 +l2_distance_32,0.5,0.5,31,32,0,0,0,32,7,1,21.296,5.986,0.000,5.986,325.0,0.125542 +l2_distance_32,0.4,0.6,31,32,0,0,0,32,7,1,21.510,5.988,0.000,5.988,325.0,0.125542 +l2_distance_32,0.3,0.7,31,32,0,0,0,32,7,1,21.257,5.988,0.000,5.988,325.0,0.125542 +l2_distance_32,0.2,0.8,31,32,0,0,0,32,7,1,21.482,5.988,0.000,5.988,325.0,0.125542 +l2_distance_32,0.1,0.9,31,32,0,0,0,32,7,1,25.320,5.988,0.000,5.988,325.0,0.125542 +gx_kernel_4,1.0,0.0,4,0,0,2,2,2,6,1,12.750,0.490,0.112,0.602,327.0,18.1967,0.0 +gx_kernel_4,0.9,0.1,5,0,2,0,2,3,5,1,11.645,0.345,0.000,0.345,329.0,0.125542,0.0 +gx_kernel_4,0.8,0.2,5,0,2,0,2,3,5,1,11.637,0.344,0.000,0.344,328.5,0.125542,0.0 +gx_kernel_4,0.7,0.3,5,0,2,0,2,3,5,1,11.616,0.344,0.000,0.344,329.0,0.125542,0.0 +gx_kernel_4,0.6,0.4,5,0,2,0,2,3,5,1,11.608,0.344,0.000,0.344,328.5,0.125542,0.0 +gx_kernel_4,0.5,0.5,5,0,2,0,2,3,5,1,11.609,0.344,0.000,0.344,329.0,0.125542,0.0 +gx_kernel_4,0.4,0.6,5,0,2,0,2,3,5,1,11.621,0.344,0.000,0.344,328.0,0.125542,0.0 +gx_kernel_4,0.3,0.7,5,0,2,0,2,3,5,1,11.580,0.345,0.000,0.345,328.5,0.125542,0.0 +gx_kernel_4,0.2,0.8,5,0,2,0,2,3,5,1,11.595,0.344,0.000,0.344,328.0,0.125542,0.0 +gx_kernel_4,0.1,0.9,5,0,2,0,2,3,5,1,11.685,0.346,0.000,0.346,328.5,0.125542,0.0 +gx_kernel_8,1.0,0.0 +gx_kernel_8,0.9,0.1 +gx_kernel_8,0.8,0.2 +gx_kernel_8,0.7,0.3 +gx_kernel_8,0.6,0.4 +gx_kernel_8,0.5,0.5 +gx_kernel_8,0.4,0.6 +gx_kernel_8,0.3,0.7 +gx_kernel_8,0.2,0.8 +gx_kernel_8,0.1,0.9 +gx_kernel_16,1.0,0.0 +gx_kernel_16,0.9,0.1 +gx_kernel_16,0.8,0.2 +gx_kernel_16,0.7,0.3 +gx_kernel_16,0.6,0.4 +gx_kernel_16,0.5,0.5 +gx_kernel_16,0.4,0.6 +gx_kernel_16,0.3,0.7 +gx_kernel_16,0.2,0.8 +gx_kernel_16,0.1,0.9 +gx_kernel_32,1.0,0.0 +gx_kernel_32,0.9,0.1 +gx_kernel_32,0.8,0.2 +gx_kernel_32,0.7,0.3 +gx_kernel_32,0.6,0.4 +gx_kernel_32,0.5,0.5 +gx_kernel_32,0.4,0.6 +gx_kernel_32,0.3,0.7 +gx_kernel_32,0.2,0.8 +gx_kernel_32,0.1,0.9 +gy_kernel_4,1.0,0.0,5,1,0,2,4,4,7,1,12.965,0.864,0.112,0.976,327.0,18.1967,0.0 +gy_kernel_4,0.9,0.1,7,0,0,0,5,5,6,1,11.564,1.015,0.000,1.015,327.0,0.125542,0.0 +gy_kernel_4,0.8,0.2,7,0,0,0,5,5,6,1,11.550,1.015,0.000,1.015,327.0,0.125542,0.0 +gy_kernel_4,0.7,0.3,7,0,0,0,5,5,6,1,11.616,1.015,0.000,1.015,327.0,0.125542,0.0 +gy_kernel_4,0.6,0.4,7,0,0,0,5,5,6,1,11.585,1.016,0.000,1.016,327.0,0.125542,0.0 +gy_kernel_4,0.5,0.5,6,1,0,0,4,5,6,1,11.622,1.015,0.000,1.015,327.0,0.125542,0.0 +gy_kernel_4,0.4,0.6,6,1,0,0,4,5,6,1,11.614,1.015,0.000,1.015,327.0,0.125542,0.0 +gy_kernel_4,0.3,0.7,6,1,0,0,4,5,6,1,11.623,1.015,0.000,1.015,327.0,0.125542,0.0 +gy_kernel_4,0.2,0.8,5,2,0,0,3,5,6,1,11.782,1.015,0.000,1.015,327.0,0.125542,0.0 +gy_kernel_4,0.1,0.9,5,2,0,0,3,5,7,1,11.746,1.015,0.000,1.016,327.0,0.125542,0.0 +gy_kernel_8,1.0,0.0 +gy_kernel_8,0.9,0.1 +gy_kernel_8,0.8,0.2 +gy_kernel_8,0.7,0.3 +gy_kernel_8,0.6,0.4 +gy_kernel_8,0.5,0.5 +gy_kernel_8,0.4,0.6 +gy_kernel_8,0.3,0.7 +gy_kernel_8,0.2,0.8 +gy_kernel_8,0.1,0.9 +gy_kernel_16,1.0,0.0 +gy_kernel_16,0.9,0.1 +gy_kernel_16,0.8,0.2 +gy_kernel_16,0.7,0.3 +gy_kernel_16,0.6,0.4 +gy_kernel_16,0.5,0.5 +gy_kernel_16,0.4,0.6 +gy_kernel_16,0.3,0.7 +gy_kernel_16,0.2,0.8 +gy_kernel_16,0.1,0.9 +gy_kernel_32,1.0,0.0 +gy_kernel_32,0.9,0.1 +gy_kernel_32,0.8,0.2 +gy_kernel_32,0.7,0.3 +gy_kernel_32,0.6,0.4 +gy_kernel_32,0.5,0.5 +gy_kernel_32,0.4,0.6 +gy_kernel_32,0.3,0.7 +gy_kernel_32,0.2,0.8 +gy_kernel_32,0.1,0.9 +roberts_cross_4,1.0,0.0,3,0,0,1,3,1,5,1,11.988,0.262,0.112,0.374,327.0,18.1967,0.0 +roberts_cross_4,0.9,0.1,4,0,0,0,4,2,4,1,11.938,0.409,0.000,0.409,327.0,0.125542,0.0 +roberts_cross_4,0.8,0.2,4,0,0,0,4,2,4,1,11.927,0.409,0.000,0.409,327.0,0.125542,0.0 +roberts_cross_4,0.7,0.3,4,0,0,0,4,2,4,1,11.882,0.409,0.000,0.409,327.0,0.125542,0.0 +roberts_cross_4,0.6,0.4,4,0,0,0,4,2,4,1,11.878,0.409,0.000,0.409,327.0,0.125542,0.0 +roberts_cross_4,0.5,0.5,4,0,0,0,4,2,4,1,11.911,0.409,0.000,0.409,327.0,0.125542,0.0 +roberts_cross_4,0.4,0.6,4,0,0,0,4,2,4,1,11.938,0.409,0.000,0.409,327.0,0.125542,0.0 +roberts_cross_4,0.3,0.7,4,0,0,0,4,2,4,1,12.604,0.409,0.000,0.409,327.0,0.125542,0.0 +roberts_cross_4,0.2,0.8,4,0,0,0,4,2,4,1,11.922,0.408,0.000,0.408,327.0,0.125542,0.0 +roberts_cross_4,0.1,0.9,1,3,0,0,2,2,4,1,12.111,0.408,0.000,0.408,327.0,0.125542,0.0 +roberts_cross_8,1.0,0.0 +roberts_cross_8,0.9,0.1 +roberts_cross_8,0.8,0.2 +roberts_cross_8,0.7,0.3 +roberts_cross_8,0.6,0.4 +roberts_cross_8,0.5,0.5 +roberts_cross_8,0.4,0.6 +roberts_cross_8,0.3,0.7 +roberts_cross_8,0.2,0.8 +roberts_cross_8,0.1,0.9 +roberts_cross_16,1.0,0.0 +roberts_cross_16,0.9,0.1 +roberts_cross_16,0.8,0.2 +roberts_cross_16,0.7,0.3 +roberts_cross_16,0.6,0.4 +roberts_cross_16,0.5,0.5 +roberts_cross_16,0.4,0.6 +roberts_cross_16,0.3,0.7 +roberts_cross_16,0.2,0.8 +roberts_cross_16,0.1,0.9 +roberts_cross_32,1.0,0.0 +roberts_cross_32,0.9,0.1 +roberts_cross_32,0.8,0.2 +roberts_cross_32,0.7,0.3 +roberts_cross_32,0.6,0.4 +roberts_cross_32,0.5,0.5 +roberts_cross_32,0.4,0.6 +roberts_cross_32,0.3,0.7 +roberts_cross_32,0.2,0.8 +roberts_cross_32,0.1,0.9 +matrix_mul_4,1.0,0.0,3,0,0,0,0,4,3,1,11.680,0.757,0.000,0.757,327.5,0.125542,0.0 +matrix_mul_4,0.9,0.1,3,0,0,0,0,4,3,1,11.665,0.756,0.000,0.756,327.5,0.125542,0.0 +matrix_mul_4,0.8,0.2,3,0,0,0,0,4,3,1,11.639,0.757,0.000,0.757,327.0,0.125542,0.0 +matrix_mul_4,0.7,0.3,3,0,0,0,0,4,3,1,11.607,0.758,0.000,0.758,327.5,0.125542,0.0 +matrix_mul_4,0.6,0.4,3,0,0,0,0,4,3,1,11.590,0.758,0.000,0.759,327.5,0.125542,0.0 +matrix_mul_4,0.5,0.5,3,0,0,0,0,4,3,1,11.573,0.757,0.000,0.757,327.0,0.125542,0.0 +matrix_mul_4,0.4,0.6,3,0,0,0,0,4,3,1,11.549,0.757,0.000,0.757,327.5,0.125542,0.0 +matrix_mul_4,0.3,0.7,3,0,0,0,0,4,3,1,11.556,0.757,0.000,0.757,327.5,0.125542,0.0 +matrix_mul_4,0.2,0.8,3,0,0,0,0,4,3,1,11.573,0.757,0.000,0.757,327.0,0.125542,0.0 +matrix_mul_4,0.1,0.9,3,0,0,0,0,4,3,1,11.584,0.757,0.000,0.757,327.0,0.125542,0.0 +matrix_mul_8,1.0,0.0 +matrix_mul_8,0.9,0.1 +matrix_mul_8,0.8,0.2 +matrix_mul_8,0.7,0.3 +matrix_mul_8,0.6,0.4 +matrix_mul_8,0.5,0.5 +matrix_mul_8,0.4,0.6 +matrix_mul_8,0.3,0.7 +matrix_mul_8,0.2,0.8 +matrix_mul_8,0.1,0.9 +matrix_mul_16,1.0,0.0 +matrix_mul_16,0.9,0.1 +matrix_mul_16,0.8,0.2 +matrix_mul_16,0.7,0.3 +matrix_mul_16,0.6,0.4 +matrix_mul_16,0.5,0.5 +matrix_mul_16,0.4,0.6 +matrix_mul_16,0.3,0.7 +matrix_mul_16,0.2,0.8 +matrix_mul_16,0.1,0.9 +matrix_mul_32,1.0,0.0 +matrix_mul_32,0.9,0.1 +matrix_mul_32,0.8,0.2 +matrix_mul_32,0.7,0.3 +matrix_mul_32,0.6,0.4 +matrix_mul_32,0.5,0.5 +matrix_mul_32,0.4,0.6 +matrix_mul_32,0.3,0.7 +matrix_mul_32,0.2,0.8 +matrix_mul_32,0.1,0.9 +tree_50-50_5_1,1.0,0.0,6,0,0,3,0,2,8,2,10.939,0.557,0.222,0.779,295.5,36.2679,0.0 +tree_50-50_5_1,0.9,0.1,6,0,0,3,0,2,8,2,10.813,0.557,0.222,0.779,296.0,36.2679,0.0 +tree_50-50_5_1,0.8,0.2,6,0,0,3,0,2,8,2,10.868,0.557,0.222,0.779,296.0,36.2679,0.0 +tree_50-50_5_1,0.7,0.3,6,0,0,1,0,2,6,2,10.783,0.519,0.113,0.631,296.0,18.1967,0.0 +tree_50-50_5_1,0.6,0.4,5,0,0,0,0,2,4,2,10.681,0.478,0.000,0.478,296.0,0.125542,0.0 +tree_50-50_5_1,0.5,0.5,5,0,0,0,0,2,4,2,10.652,0.478,0.000,0.478,296.0,0.125542,0.0 +tree_50-50_5_1,0.4,0.6,5,0,0,0,0,2,4,2,10.796,0.478,0.000,0.478,296.0,0.125542,0.0 +tree_50-50_5_1,0.3,0.7,5,0,0,0,0,2,4,2,10.694,0.478,0.000,0.478,296.0,0.125542,0.0 +tree_50-50_5_1,0.2,0.8,5,0,0,0,0,2,4,2,10.822,0.478,0.000,0.478,296.0,0.125542,0.0 +tree_50-50_5_1,0.1,0.9,5,0,0,0,0,2,4,2,10.733,0.478,0.000,0.478,296.0,0.125542,0.0 +tree_50-50_10_1,1.0,0.0,16,0,0,4,0,9,14,5,26.236,1.956,0.447,2.403,197.0,72.4103,0.0 +tree_50-50_10_1,0.9,0.1,19,0,0,2,71,16,83,5,38.176,3.107,0.224,3.331,197.5,36.2679 +tree_50-50_10_1,0.8,0.2,21,0,0,3,61,20,73,5,37.682,3.695,0.112,3.808,198.0,18.1967 +tree_50-50_10_1,0.7,0.3,20,0,0,3,0,15,14,7,19.550,2.944,0.113,3.057,135.0,18.1967 +tree_50-50_10_1,0.6,0.4,17,0,0,1,0,17,10,5,14.422,3.235,0.113,3.348,198.0,18.1967 +tree_50-50_10_1,0.5,0.5,17,0,0,1,0,17,10,5,14.229,3.236,0.113,3.349,198.0,18.1967 +tree_50-50_10_1,0.4,0.6,16,0,0,0,0,17,8,5,12.897,3.192,0.000,3.192,198.0,0.125542 +tree_50-50_10_1,0.3,0.7,16,0,0,0,0,17,8,5,12.905,3.192,0.000,3.192,198.5,0.125542 +tree_50-50_10_1,0.2,0.8,16,0,0,0,0,17,8,5,12.904,3.192,0.000,3.192,198.5,0.125542 +tree_50-50_10_1,0.1,0.9,16,0,0,0,0,17,8,5,12.912,3.193,0.000,3.193,198.5,0.125542 +tree_100-50_5_1,1.0,0.0,12,0,0,4,0,5,14,4,23.640,1.209,0.447,1.656,229.5,72.4103,0.0 +tree_100-50_5_1,0.9,0.1,20,0,0,6,0,8,17,3,33.425,1.898,0.224,2.122,261.0,36.2679 +tree_100-50_5_1,0.8,0.2,27,0,0,3,0,12,12,3,17.977,2.539,0.112,2.651,262.0,18.1967 +tree_100-50_5_1,0.7,0.3,22,0,0,2,0,10,11,4,18.294,2.186,0.223,2.409,229.5,36.2679 +tree_100-50_5_1,0.6,0.4,22,0,0,1,0,11,9,3,15.136,2.338,0.113,2.451,262.0,18.1967 +tree_100-50_5_1,0.5,0.5,19,0,0,0,0,12,5,3,12.733,2.428,0.000,2.428,262.5,0.125542 +tree_100-50_5_1,0.4,0.6,19,0,0,0,0,12,5,3,12.719,2.428,0.000,2.428,263.0,0.125542 +tree_100-50_5_1,0.3,0.7,19,0,0,0,0,12,5,3,12.832,2.428,0.000,2.428,262.5,0.125542 +tree_100-50_5_1,0.2,0.8,19,0,0,0,0,12,5,3,12.727,2.428,0.000,2.428,262.5,0.125542 +tree_100-50_5_1,0.1,0.9,19,0,0,0,0,12,5,3,12.736,2.428,0.000,2.428,263.0,0.125542 +tree_100-50_10_1,1.0,0.0,526,0,0,2,0,502,17,10,232.018,93.260,0.223,93.483,35.0,36.2679 +tree_100-50_10_1,0.9,0.1,538,0,0,2,0,491,18,10,3644.950,91.713,0.224,91.937,35.0,36.2679 +tree_100-50_10_1,0.8,0.2,512,0,0,1,73,512,15,9,997.327,95.105,0.113,95.218,66.0,18.1967 +tree_100-50_10_1,0.7,0.3,512,0,0,1,10,512,13,9,996.085,94.888,0.112,95.000,65.5,18.1967 +tree_100-50_10_1,0.6,0.4,511,0,0,0,1,512,11,9,305.001,94.940,0.000,94.940,66.5,0.125542 +tree_100-50_10_1,0.5,0.5,511,0,0,0,1,512,11,9,303.726,94.929,0.000,94.929,66.0,0.125542 +tree_100-50_10_1,0.4,0.6,511,0,0,0,1,512,11,9,305.730,94.925,0.000,94.925,66.0,0.125542 +tree_100-50_10_1,0.3,0.7,511,0,0,0,1,512,11,9,305.207,94.954,0.000,94.954,66.0,0.125542 +tree_100-50_10_1,0.2,0.8,511,0,0,0,0,512,10,9,308.887,94.845,0.000,94.845,66.5,0.125542 +tree_100-50_10_1,0.1,0.9,511,0,0,0,0,512,10,9,309.651,94.857,0.000,94.857,66.5,0.125542 +tree_100-100_5_1,1.0,0.0,1,0,0,5,0,4,10,4,12.179,0.796,0.559,1.355,228.0,90.4814,0.0 +tree_100-100_5_1,0.9,0.1,1,0,0,4,0,4,9,4,11.650,0.778,0.448,1.226,230.5,72.4103,0.0 +tree_100-100_5_1,0.8,0.2,1,0,0,1,0,15,6,4,11.353,2.514,0.112,2.626,228.5,18.1967,0.0 +tree_100-100_5_1,0.7,0.3,1,0,0,1,0,15,6,4,11.348,2.514,0.112,2.626,229.0,18.1967,0.0 +tree_100-100_5_1,0.6,0.4,1,0,0,1,0,15,6,4,11.334,2.513,0.112,2.626,229.0,18.1967,0.0 +tree_100-100_5_1,0.5,0.5,1,0,0,1,0,15,9,7,11.526,2.520,0.112,2.632,132.0,18.1967,0.0 +tree_100-100_5_1,0.4,0.6,1,0,0,0,0,30,5,4,12.721,4.925,0.000,4.925,229.0,0.125542 +tree_100-100_5_1,0.3,0.7,1,0,0,0,0,30,5,4,12.716,4.927,0.000,4.927,229.0,0.125542 +tree_100-100_5_1,0.2,0.8,1,0,0,0,0,30,5,4,12.776,4.926,0.000,4.926,229.0,0.125542 +tree_100-100_5_1,0.1,0.9,1,0,0,0,0,30,5,4,12.732,4.924,0.000,4.924,229.0,0.125542 +tree_100-100_10_1,1.0,0.0,1,0,0,2,0,256,12,9,84.570,41.657,0.223,41.880,62.0,36.2679 +tree_100-100_10_1,0.9,0.1,1,0,0,1,0,511,11,9,987.280,82.949,0.112,83.061,62.0,18.1967 +tree_100-100_10_1,0.8,0.2,1,0,0,0,72,1022,82,9,368.400,165.899,0.000,165.899,62.5,0.125542 +tree_100-100_10_1,0.7,0.3,1,0,0,0,1,1022,11,9,305.348,165.512,0.000,165.512,62.0,0.125542 +tree_100-100_10_1,0.6,0.4,1,0,0,0,0,1022,10,9,304.071,165.645,0.000,165.645,62.5,0.125542 +tree_100-100_10_1,0.5,0.5,1,0,0,0,1,1022,11,9,305.381,165.501,0.000,165.501,63.0,0.125542 +tree_100-100_10_1,0.4,0.6,1,0,0,0,1,1022,11,9,302.784,165.476,0.000,165.476,62.5,0.125542 +tree_100-100_10_1,0.3,0.7,1,0,0,0,0,1022,10,9,304.495,165.636,0.000,165.636,61.5,0.125542 +tree_100-100_10_1,0.2,0.8,1,0,0,0,0,1022,10,9,305.321,165.636,0.000,165.636,62.0,0.125542 +tree_100-100_10_1,0.1,0.9,1,0,0,0,0,1022,10,9,304.948,165.641,0.000,165.641,62.5,0.125542 \ No newline at end of file diff --git a/pareto_keys_10.csv b/pareto_keys_10.csv new file mode 100644 index 00000000..3eed05e8 --- /dev/null +++ b/pareto_keys_10.csv @@ -0,0 +1,457 @@ +benchmark,w_ops,w_keys,add,sub,multiply_plain,rotate_rows,negate,multiply,Depth,Multplicative Depth,compile_time (s),circuit_execution_time (s),galois_keys_generation_time (s),total_execution_time (s),Remaining_noise_budget,rotation_key_size (MB),Final_Cost +max_3,1.0,0.0,2,1,0,3,0,2,8,2,14.372,0.482,0.334,0.815,295.0,54.3391,389.0 +max_3,0.9,0.1,2,1,0,3,0,2,8,2,11.789,0.482,0.333,0.815,295.0,54.3391,389.0 +max_3,0.8,0.2,5,1,0,3,1,3,9,2,11.495,0.694,0.223,0.918,295.0,36.2679,726.0 +max_3,0.7,0.3,5,1,0,3,1,3,9,2,11.636,0.694,0.223,0.917,295.0,36.2679,726.0 +max_3,0.6,0.4,5,1,0,3,1,3,9,2,11.616,0.694,0.223,0.917,295.0,36.2679,726.0 +max_3,0.5,0.5,5,1,0,2,1,4,8,2,11.437,0.815,0.223,1.039,295.0,36.2679,1321.0 +max_3,0.4,0.6,7,1,0,2,2,5,10,2,14.654,0.980,0.223,1.202,295.0,36.2679 +max_3,0.3,0.7,6,0,0,0,3,6,6,2,11.228,0.992,0.000,0.992,295.0,0.125542,3009.0 +max_3,0.2,0.8,6,0,0,0,3,6,6,2,11.177,0.993,0.000,0.993,295.0,0.125542,3009.0 +max_3,0.1,0.9,6,0,0,0,3,6,6,2,11.217,0.993,0.000,0.993,295.0,0.125542,3009.0 +max_4,1.0,0.0,4,1,0,4,1,4,12,3,22.779,0.853,0.445,1.299,262.0,72.4103,710.0 +max_4,0.9,0.1,6,1,0,3,2,6,11,3,17.395,1.179,0.333,1.512,262.0,54.3391,796.0 +max_4,0.8,0.2,17,1,0,3,5,11,12,3,22.726,2.072,0.222,2.295,261.5,36.2679 +max_4,0.7,0.3,17,1,0,3,5,11,12,3,22.745,2.072,0.222,2.294,261.5,36.2679 +max_4,0.6,0.4,17,1,0,3,6,12,12,3,22.489,2.185,0.223,2.408,262.0,36.2679 +max_4,0.5,0.5,17,1,0,3,6,12,12,3,22.265,2.186,0.223,2.409,262.0,36.2679 +max_4,0.4,0.6,15,1,0,2,6,13,12,3,19.089,2.244,0.222,2.466,262.0,36.2679 +max_4,0.3,0.7,16,0,0,1,7,13,10,3,15.721,2.200,0.112,2.312,262.0,18.1967 +max_4,0.2,0.8,12,0,0,0,6,12,8,3,13.617,1.923,0.000,1.923,262.0,0.125542 +max_4,0.1,0.9,12,0,0,0,6,12,8,3,10.822,1.928,0.000,1.929,262.0,0.125542,7012.0 +max_5,1.0,0.0,13,1,0,4,5,13,14,4,150.751,2.422,0.446,2.868,229.0,72.4103,1622.0 +max_5,0.9,0.1,14,1,0,3,6,14,14,4,82.370,2.537,0.335,2.871,229.0,54.3391,1608.0 +max_5,0.8,0.2,33,1,0,4,13,27,15,4,64.202,4.639,0.335,4.973,228.5,54.3391 +max_5,0.7,0.3,33,1,0,3,13,27,14,4,45.234,4.613,0.223,4.836,228.0,36.2679 +max_5,0.6,0.4,31,1,0,3,13,26,14,4,44.849,4.376,0.223,4.599,228.5,36.2679 +max_5,0.5,0.5,31,1,0,3,13,26,14,4,44.160,4.376,0.223,4.599,228.5,36.2679 +max_5,0.4,0.6,31,1,0,3,13,26,14,4,44.420,4.377,0.223,4.599,229.0,36.2679 +max_5,0.3,0.7,27,1,0,2,12,25,14,4,33.306,4.105,0.222,4.327,229.0,36.2679 +max_5,0.2,0.8,21,0,0,0,11,21,11,4,17.495,3.296,0.000,3.296,229.0,0.125542 +max_5,0.1,0.9,21,0,0,0,11,21,11,4,17.226,3.295,0.000,3.295,229.0,0.125542 +sort_3,1.0,0.0 +sort_3,0.9,0.1 +sort_3,0.8,0.2 +sort_3,0.7,0.3 +sort_3,0.6,0.4 +sort_3,0.5,0.5 +sort_3,0.4,0.6 +sort_3,0.3,0.7 +sort_3,0.2,0.8 +sort_3,0.1,0.9 +box_blur_4,1.0,0.0,5,0,0,1,0,0,5,0,11.660,0.163,0.113,0.276,357.0,18.1967,79.0 +box_blur_4,0.9,0.1,5,0,0,1,0,0,5,0,11.809,0.163,0.113,0.275,357.0,18.1967,79.0 +box_blur_4,0.8,0.2,8,0,0,0,0,0,4,0,11.470,0.217,0.000,0.217,359.0,0.125542,14.0 +box_blur_4,0.7,0.3,8,0,0,0,0,0,4,0,11.534,0.217,0.000,0.217,359.0,0.125542,14.0 +box_blur_4,0.6,0.4,8,0,0,0,0,0,4,0,11.626,0.217,0.000,0.217,359.0,0.125542,14.0 +box_blur_4,0.5,0.5,8,0,0,0,0,0,4,0,11.532,0.217,0.000,0.217,359.0,0.125542,14.0 +box_blur_4,0.4,0.6,8,0,0,0,0,0,4,0,11.715,0.217,0.000,0.217,359.0,0.125542,14.0 +box_blur_4,0.3,0.7,8,0,0,0,0,0,5,0,11.662,0.217,0.000,0.217,359.0,0.125542,15.0 +box_blur_4,0.2,0.8,8,0,0,0,0,0,5,0,11.646,0.217,0.000,0.217,359.0,0.125542,15.0 +box_blur_4,0.1,0.9,8,0,0,0,0,0,6,0,11.781,0.217,0.000,0.217,359.0,0.125542,16.0 +box_blur_8,1.0,0.0 +box_blur_8,0.9,0.1 +box_blur_8,0.8,0.2 +box_blur_8,0.7,0.3 +box_blur_8,0.6,0.4 +box_blur_8,0.5,0.5 +box_blur_8,0.4,0.6 +box_blur_8,0.3,0.7 +box_blur_8,0.2,0.8 +box_blur_8,0.1,0.9 +box_blur_16,1.0,0.0 +box_blur_16,0.9,0.1 +box_blur_16,0.8,0.2 +box_blur_16,0.7,0.3 +box_blur_16,0.6,0.4 +box_blur_16,0.5,0.5 +box_blur_16,0.4,0.6 +box_blur_16,0.3,0.7 +box_blur_16,0.2,0.8 +box_blur_16,0.1,0.9 +box_blur_32,1.0,0.0 +box_blur_32,0.9,0.1 +box_blur_32,0.8,0.2 +box_blur_32,0.7,0.3 +box_blur_32,0.6,0.4 +box_blur_32,0.5,0.5 +box_blur_32,0.4,0.6 +box_blur_32,0.3,0.7 +box_blur_32,0.2,0.8 +box_blur_32,0.1,0.9 +lin_reg_4,1.0,0.0,2,0,0,2,0,1,5,1,12.965,0.287,0.229,0.516,327.0,36.2679,232.0 +lin_reg_4,0.9,0.1,2,0,0,2,0,1,5,1,12.142,0.286,0.229,0.515,327.0,36.2679,232.0 +lin_reg_4,0.8,0.2,2,0,0,2,0,1,5,1,11.828,0.286,0.228,0.515,328.0,36.2679,232.0 +lin_reg_4,0.7,0.3,2,0,1,0,0,1,3,1,11.910,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_4,0.6,0.4,2,0,1,0,0,1,3,1,12.181,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_4,0.5,0.5,2,0,1,0,0,1,3,1,12.088,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_4,0.4,0.6,2,0,1,0,0,1,3,1,12.095,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_4,0.3,0.7,2,0,1,0,0,1,3,1,11.984,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_4,0.2,0.8,2,0,1,0,0,1,3,1,12.039,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_4,0.1,0.9,2,0,1,0,0,1,3,1,12.057,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_8,1.0,0.0,2,0,0,2,0,1,5,1,12.322,0.287,0.229,0.516,327.5,36.2679,252.0 +lin_reg_8,0.9,0.1,2,0,0,2,0,1,5,1,12.243,0.286,0.229,0.515,327.0,36.2679,252.0 +lin_reg_8,0.8,0.2,2,0,0,2,0,1,5,1,12.200,0.287,0.230,0.517,327.0,36.2679,252.0 +lin_reg_8,0.7,0.3,2,0,1,0,0,1,3,1,12.021,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_8,0.6,0.4,2,0,1,0,0,1,3,1,12.125,0.055,0.000,0.055,358.5,0.125542,108.0 +lin_reg_8,0.5,0.5,2,0,1,0,0,1,3,1,12.315,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_8,0.4,0.6,2,0,1,0,0,1,3,1,12.142,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_8,0.3,0.7,2,0,1,0,0,1,3,1,12.339,0.056,0.000,0.056,359.0,0.125542,108.0 +lin_reg_8,0.2,0.8,2,0,1,0,0,1,3,1,12.444,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_8,0.1,0.9,2,0,1,0,0,1,3,1,11.850,0.056,0.000,0.056,359.0,0.125542,108.0 +lin_reg_16,1.0,0.0,2,0,0,1,0,1,4,1,12.574,0.245,0.115,0.360,328.0,18.1967,176.0 +lin_reg_16,0.9,0.1,2,0,0,1,0,1,4,1,12.483,0.245,0.115,0.361,328.0,18.1967,176.0 +lin_reg_16,0.8,0.2,2,0,0,1,0,1,4,1,12.388,0.246,0.115,0.361,328.0,18.1967,176.0 +lin_reg_16,0.7,0.3,2,0,1,0,0,1,3,1,12.140,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_16,0.6,0.4,2,0,1,0,0,1,3,1,12.459,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_16,0.5,0.5,2,0,1,0,0,1,3,1,12.273,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_16,0.4,0.6,2,0,1,0,0,1,3,1,12.142,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_16,0.3,0.7,2,0,1,0,0,1,3,1,12.335,0.055,0.000,0.055,358.5,0.125542,108.0 +lin_reg_16,0.2,0.8,2,0,1,0,0,1,3,1,12.072,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_16,0.1,0.9,2,0,1,0,0,1,3,1,11.987,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_32,1.0,0.0,2,0,1,0,0,1,3,1,12.143,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_32,0.9,0.1,2,0,1,0,0,1,3,1,12.359,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_32,0.8,0.2,2,0,1,0,0,1,3,1,12.574,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_32,0.7,0.3,2,0,1,0,0,1,3,1,12.215,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_32,0.6,0.4,2,0,1,0,0,1,3,1,12.203,0.055,0.000,0.055,358.5,0.125542,108.0 +lin_reg_32,0.5,0.5,2,0,1,0,0,1,3,1,12.415,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_32,0.4,0.6,2,0,1,0,0,1,3,1,12.190,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_32,0.3,0.7,2,0,1,0,0,1,3,1,12.602,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_32,0.2,0.8,2,0,1,0,0,1,3,1,12.477,0.055,0.000,0.055,359.0,0.125542,108.0 +lin_reg_32,0.1,0.9,2,0,1,0,0,1,3,1,12.138,0.055,0.000,0.055,359.0,0.125542,108.0 +hamming_dist_4,1.0,0.0,4,1,0,3,0,3,8,2,12.386,0.682,0.339,1.021,295.0,54.3391,639.0 +hamming_dist_4,0.9,0.1,4,1,0,3,0,3,8,2,12.283,0.683,0.339,1.022,295.0,54.3391,639.0 +hamming_dist_4,0.8,0.2,4,1,0,2,0,3,8,2,12.326,0.663,0.226,0.889,295.0,36.2679,572.0 +hamming_dist_4,0.7,0.3,6,2,1,1,0,5,7,2,12.368,0.811,0.114,0.925,295.0,18.1967,868.0 +hamming_dist_4,0.6,0.4,6,2,1,1,0,5,7,2,12.252,0.812,0.114,0.926,295.0,18.1967,868.0 +hamming_dist_4,0.5,0.5,6,2,1,1,0,5,7,2,12.343,0.811,0.114,0.925,295.0,18.1967,868.0 +hamming_dist_4,0.4,0.6,6,2,1,1,0,5,7,2,12.336,0.812,0.114,0.926,295.0,18.1967,868.0 +hamming_dist_4,0.3,0.7,6,2,1,1,0,5,7,2,12.196,0.812,0.114,0.925,295.0,18.1967,868.0 +hamming_dist_4,0.2,0.8,7,4,4,0,0,8,5,2,11.697,0.777,0.000,0.777,326.0,0.125542,4759.0 +hamming_dist_4,0.1,0.9,7,4,4,0,0,8,5,2,11.984,0.776,0.000,0.776,326.0,0.125542,4759.0 +hamming_dist_8,1.0,0.0,5,1,0,4,0,3,10,2,14.922,0.726,0.452,1.177,294.0,72.4103,757.0 +hamming_dist_8,0.9,0.1,5,1,0,4,0,3,10,2,15.067,0.724,0.451,1.175,294.0,72.4103,757.0 +hamming_dist_8,0.8,0.2,6,2,1,2,0,5,9,2,13.981,0.853,0.225,1.078,295.0,36.2679,776.0 +hamming_dist_8,0.7,0.3,12,4,3,1,0,9,9,2,20.218,1.195,0.113,1.308,295.0,18.1967 +hamming_dist_8,0.6,0.4,12,4,3,1,0,9,9,2,20.220,1.194,0.113,1.307,295.0,18.1967 +hamming_dist_8,0.5,0.5,12,4,3,1,0,9,9,2,20.079,1.194,0.114,1.308,295.0,18.1967 +hamming_dist_8,0.4,0.6,12,4,3,1,0,9,9,2,20.117,1.192,0.113,1.305,295.0,18.1967 +hamming_dist_8,0.3,0.7,12,4,3,1,0,9,9,2,19.829,1.195,0.113,1.309,295.0,18.1967 +hamming_dist_8,0.2,0.8,15,8,8,0,0,16,6,2,16.201,1.561,0.000,1.562,326.0,0.125542 +hamming_dist_8,0.1,0.9,15,8,8,0,1,16,6,2,16.311,1.561,0.000,1.561,326.0,0.125542 +hamming_dist_16,1.0,0.0,6,1,0,4,0,3,12,2,38.854,0.758,0.459,1.217,294.0,72.4103,760.0 +hamming_dist_16,0.9,0.1,7,2,1,3,0,5,11,2,27.844,0.908,0.344,1.252,294.0,54.3391,846.0 +hamming_dist_16,0.8,0.2,24,8,7,1,0,17,11,2,33.267,1.982,0.116,2.098,295.0,18.1967 +hamming_dist_16,0.7,0.3,24,8,7,1,0,17,12,2,32.270,1.979,0.116,2.095,295.0,18.1967 +hamming_dist_16,0.6,0.4,24,8,7,1,0,17,12,2,33.700,2.007,0.117,2.124,295.0,18.1967 +hamming_dist_16,0.5,0.5,21,8,7,1,0,17,12,2,33.432,2.010,0.118,2.128,295.0,18.1967 +hamming_dist_16,0.4,0.6,22,8,7,1,0,17,12,2,34.029,2.000,0.117,2.117,295.0,18.1967 +hamming_dist_16,0.3,0.7,22,8,7,1,0,17,12,2,33.043,1.998,0.117,2.115,295.0,18.1967 +hamming_dist_16,0.2,0.8,31,16,16,0,1,32,8,2,23.095,3.163,0.000,3.163,325.0,0.125542 +hamming_dist_16,0.1,0.9,31,16,16,0,1,32,8,2,21.616,3.155,0.000,3.155,325.5,0.125542 +hamming_dist_32,1.0,0.0,7,1,0,5,0,3,14,2,370.169,0.800,0.574,1.374,292.5,90.4814,1070.0 +hamming_dist_32,0.9,0.1,18,8,7,2,0,17,12,2,92.953,2.008,0.228,2.236,295.0,36.2679,1997.0 +hamming_dist_32,0.8,0.2,34,16,15,1,0,33,10,2,77.518,3.496,0.114,3.610,295.0,18.1967 +hamming_dist_32,0.7,0.3,34,16,15,1,0,33,10,2,75.097,3.492,0.114,3.606,295.0,18.1967 +hamming_dist_32,0.6,0.4,34,16,15,1,0,33,10,2,76.088,3.476,0.113,3.590,295.0,18.1967 +hamming_dist_32,0.5,0.5,34,16,15,1,0,33,10,2,75.574,3.478,0.113,3.592,295.0,18.1967 +hamming_dist_32,0.4,0.6,34,16,15,1,0,33,10,2,74.201,3.474,0.113,3.586,295.0,18.1967 +hamming_dist_32,0.3,0.7,34,16,15,1,0,33,10,2,76.302,3.478,0.113,3.591,295.0,18.1967 +hamming_dist_32,0.2,0.8,63,32,32,0,1,64,9,2,35.138,6.090,0.000,6.090,325.0,0.125542 +hamming_dist_32,0.1,0.9,63,32,32,0,1,64,9,2,34.756,6.095,0.000,6.095,325.0,0.125542 +dot_product_4,1.0,0.0,2,0,0,3,0,1,6,1,14.223,0.300,0.336,0.636,325.0,54.3391,285.0 +dot_product_4,0.9,0.1,2,0,0,2,0,1,5,1,11.684,0.282,0.224,0.506,328.0,36.2679,217.0 +dot_product_4,0.8,0.2,2,0,0,1,0,2,4,1,11.823,0.429,0.113,0.542,327.0,18.1967,261.0 +dot_product_4,0.7,0.3,2,0,0,1,0,2,4,1,11.861,0.430,0.113,0.543,327.5,18.1967,261.0 +dot_product_4,0.6,0.4,2,0,0,1,0,2,4,1,11.732,0.429,0.113,0.542,327.5,18.1967,261.0 +dot_product_4,0.5,0.5,2,0,0,1,0,2,4,1,11.675,0.429,0.113,0.543,328.0,18.1967,261.0 +dot_product_4,0.4,0.6,2,0,0,1,0,2,4,1,11.820,0.428,0.113,0.541,327.0,18.1967,261.0 +dot_product_4,0.3,0.7,3,0,0,0,0,4,3,1,11.552,0.776,0.000,0.776,327.0,0.125542,1756.0 +dot_product_4,0.2,0.8,3,0,0,0,0,4,3,1,11.655,0.780,0.000,0.780,327.5,0.125542,1756.0 +dot_product_4,0.1,0.9,3,0,0,0,0,4,3,1,11.786,0.778,0.000,0.778,327.0,0.125542,1756.0 +dot_product_8,1.0,0.0,3,0,0,4,0,1,8,1,12.374,0.346,0.456,0.803,324.0,72.4103,403.0 +dot_product_8,0.9,0.1,3,0,0,3,0,1,7,1,12.142,0.327,0.342,0.668,327.0,54.3391,287.0 +dot_product_8,0.8,0.2,3,0,0,2,0,2,6,1,12.008,0.477,0.228,0.705,327.0,36.2679,319.0 +dot_product_8,0.7,0.3,4,0,0,1,0,4,5,1,12.115,0.818,0.116,0.934,327.0,18.1967,464.0 +dot_product_8,0.6,0.4,4,0,0,1,0,4,5,1,12.202,0.817,0.116,0.933,327.0,18.1967,464.0 +dot_product_8,0.5,0.5,6,0,0,1,0,4,6,1,11.858,0.817,0.116,0.933,327.0,18.1967,465.0 +dot_product_8,0.4,0.6,6,0,0,1,0,4,7,1,15.023,0.821,0.116,0.937,327.0,18.1967 +dot_product_8,0.3,0.7,6,0,0,1,0,4,7,1,15.076,0.818,0.116,0.934,327.0,18.1967 +dot_product_8,0.2,0.8,7,0,0,0,0,8,4,1,11.710,1.536,0.000,1.536,327.0,0.125542,3757.0 +dot_product_8,0.1,0.9,7,0,0,0,0,8,4,1,11.553,1.537,0.000,1.537,327.0,0.125542,3757.0 +dot_product_16,1.0,0.0,4,0,0,4,0,1,9,1,14.003,0.369,0.455,0.825,325.5,72.4103,405.0 +dot_product_16,0.9,0.1,4,0,0,4,0,1,9,1,13.957,0.367,0.453,0.821,327.0,72.4103,405.0 +dot_product_16,0.8,0.2,5,0,0,2,0,4,7,1,13.043,0.854,0.227,1.080,326.0,36.2679,522.0 +dot_product_16,0.7,0.3,15,0,0,1,0,8,10,1,14.277,1.576,0.114,1.690,326.0,18.1967,873.0 +dot_product_16,0.6,0.4,15,0,0,1,0,8,13,1,18.185,1.576,0.114,1.690,326.5,18.1967 +dot_product_16,0.5,0.5,15,0,0,1,0,8,13,1,18.121,1.578,0.114,1.692,326.0,18.1967 +dot_product_16,0.4,0.6,15,0,0,1,0,8,12,1,17.755,1.574,0.114,1.688,327.0,18.1967 +dot_product_16,0.3,0.7,15,0,0,1,0,8,12,1,17.802,1.579,0.115,1.694,326.5,18.1967 +dot_product_16,0.2,0.8,15,0,0,0,0,16,5,1,14.735,3.036,0.000,3.036,326.5,0.125542 +dot_product_16,0.1,0.9,15,0,0,0,1,16,6,1,14.752,3.043,0.000,3.043,326.5,0.125542 +dot_product_32,1.0,0.0,5,0,0,4,0,2,10,1,26.220,0.557,0.451,1.007,326.0,72.4103,507.0 +dot_product_32,0.9,0.1,6,0,0,3,0,4,9,1,19.761,0.892,0.337,1.229,326.0,54.3391,592.0 +dot_product_32,0.8,0.2,31,0,0,1,0,16,17,1,27.286,3.091,0.113,3.205,326.0,18.1967 +dot_product_32,0.7,0.3,30,0,0,1,0,16,14,1,25.542,3.094,0.114,3.208,326.0,18.1967 +dot_product_32,0.6,0.4,30,0,0,1,0,16,14,1,27.327,3.096,0.114,3.211,326.0,18.1967 +dot_product_32,0.5,0.5,16,0,0,1,0,16,13,1,25.001,3.072,0.114,3.185,326.0,18.1967 +dot_product_32,0.4,0.6,16,0,0,1,0,16,14,1,26.340,3.067,0.113,3.181,326.0,18.1967 +dot_product_32,0.3,0.7,16,0,0,1,0,16,14,1,24.806,3.073,0.113,3.186,326.0,18.1967 +dot_product_32,0.2,0.8,31,0,0,0,1,32,7,1,18.250,6.026,0.000,6.026,326.0,0.125542 +dot_product_32,0.1,0.9,31,0,0,0,1,32,7,1,18.248,6.017,0.000,6.017,326.0,0.125542 +poly_reg_4,1.0,0.0,3,0,0,2,0,2,7,2,11.725,0.448,0.225,0.673,295.0,36.2679,336.0 +poly_reg_4,0.9,0.1,3,0,0,2,0,2,7,2,11.655,0.447,0.224,0.671,295.0,36.2679,336.0 +poly_reg_4,0.8,0.2,3,0,0,2,0,2,7,2,11.721,0.446,0.224,0.670,295.0,36.2679,336.0 +poly_reg_4,0.7,0.3,3,0,0,0,0,2,5,2,11.712,0.411,0.000,0.411,296.0,0.125542,212.0 +poly_reg_4,0.6,0.4,3,0,0,0,0,2,5,2,11.216,0.411,0.000,0.411,296.0,0.125542,212.0 +poly_reg_4,0.5,0.5,3,0,0,0,0,2,5,2,15.381,0.412,0.000,0.412,296.0,0.125542,212.0 +poly_reg_4,0.4,0.6,3,0,0,0,0,2,5,2,10.974,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_4,0.3,0.7,3,0,0,0,0,2,5,2,11.033,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_4,0.2,0.8,3,0,0,0,0,2,5,2,11.004,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_4,0.1,0.9,3,0,0,0,0,2,5,2,11.027,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_8,1.0,0.0,3,0,0,2,0,2,7,2,12.052,0.444,0.224,0.667,295.0,36.2679,356.0 +poly_reg_8,0.9,0.1,3,0,0,2,0,2,7,2,12.018,0.444,0.224,0.667,295.0,36.2679,356.0 +poly_reg_8,0.8,0.2,3,0,0,2,0,2,7,2,12.022,0.444,0.224,0.667,295.0,36.2679,356.0 +poly_reg_8,0.7,0.3,3,0,0,0,0,2,5,2,11.178,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_8,0.6,0.4,3,0,0,0,0,2,5,2,11.185,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_8,0.5,0.5,3,0,0,0,0,2,5,2,11.095,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_8,0.4,0.6,3,0,0,0,0,2,5,2,11.155,0.408,0.000,0.409,296.0,0.125542,212.0 +poly_reg_8,0.3,0.7,3,0,0,0,0,2,5,2,11.143,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_8,0.2,0.8,3,0,0,0,0,2,5,2,11.105,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_8,0.1,0.9,3,0,0,0,0,2,5,2,11.147,0.408,0.000,0.408,296.0,0.125542,212.0 +poly_reg_16,1.0,0.0,3,0,0,1,0,2,6,2,11.960,0.426,0.112,0.538,295.0,18.1967,280.0 +poly_reg_16,0.9,0.1,3,0,0,1,0,2,6,2,12.027,0.427,0.112,0.539,295.0,18.1967,280.0 +poly_reg_16,0.8,0.2,3,0,0,0,0,2,5,2,11.452,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_16,0.7,0.3,3,0,0,0,0,2,5,2,11.321,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_16,0.6,0.4,3,0,0,0,0,2,5,2,11.380,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_16,0.5,0.5,3,0,0,0,0,2,5,2,11.412,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_16,0.4,0.6,3,0,0,0,0,2,5,2,11.430,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_16,0.3,0.7,3,0,0,0,0,2,5,2,11.513,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_16,0.2,0.8,3,0,0,0,0,2,5,2,11.428,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_16,0.1,0.9,3,0,0,0,0,2,5,2,11.398,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_32,1.0,0.0,3,0,0,0,0,2,5,2,12.143,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_32,0.9,0.1,3,0,0,0,0,2,5,2,12.008,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_32,0.8,0.2,3,0,0,0,0,2,5,2,12.002,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_32,0.7,0.3,3,0,0,0,0,2,5,2,12.018,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_32,0.6,0.4,3,0,0,0,0,2,5,2,12.014,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_32,0.5,0.5,3,0,0,0,0,2,5,2,11.971,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_32,0.4,0.6,3,0,0,0,0,2,5,2,12.065,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_32,0.3,0.7,3,0,0,0,0,2,5,2,12.089,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_32,0.2,0.8,3,0,0,0,0,2,5,2,12.111,0.409,0.000,0.409,296.0,0.125542,212.0 +poly_reg_32,0.1,0.9,3,0,0,0,0,2,5,2,12.062,0.409,0.000,0.409,296.0,0.125542,212.0 +l2_distance_4,1.0,0.0,2,1,0,3,0,1,7,1,11.257,0.321,0.335,0.656,324.0,54.3391,287.0 +l2_distance_4,0.9,0.1,2,1,0,2,0,1,6,1,11.175,0.281,0.222,0.503,326.0,36.2679,220.0 +l2_distance_4,0.8,0.2,2,1,0,2,0,1,6,1,11.190,0.281,0.223,0.503,326.0,36.2679,220.0 +l2_distance_4,0.7,0.3,2,2,0,1,0,2,5,1,11.199,0.426,0.112,0.539,326.0,18.1967,266.0 +l2_distance_4,0.6,0.4,2,2,0,1,0,2,5,1,11.237,0.426,0.113,0.540,326.5,18.1967,266.0 +l2_distance_4,0.5,0.5,2,2,0,1,0,2,5,1,11.191,0.426,0.112,0.538,326.5,18.1967,266.0 +l2_distance_4,0.4,0.6,2,2,0,1,0,2,5,1,11.218,0.426,0.112,0.538,326.0,18.1967,266.0 +l2_distance_4,0.3,0.7,3,4,0,0,0,4,4,1,10.891,0.760,0.000,0.760,327.0,0.125542,3757.0 +l2_distance_4,0.2,0.8,3,4,0,0,0,4,4,1,10.832,0.760,0.000,0.760,326.5,0.125542,3757.0 +l2_distance_4,0.1,0.9,3,4,0,0,0,4,4,1,10.978,0.760,0.000,0.760,326.5,0.125542,3757.0 +l2_distance_8,1.0,0.0,3,1,0,4,0,1,9,1,12.904,0.363,0.444,0.808,324.0,72.4103,405.0 +l2_distance_8,0.9,0.1,3,1,0,3,0,1,8,1,12.293,0.322,0.334,0.655,326.0,54.3391,290.0 +l2_distance_8,0.8,0.2,3,2,0,2,0,2,7,1,12.078,0.467,0.224,0.691,326.0,36.2679,324.0 +l2_distance_8,0.7,0.3,4,4,0,1,0,4,6,1,12.092,0.800,0.113,0.913,326.0,18.1967,473.0 +l2_distance_8,0.6,0.4,6,4,0,1,0,4,8,1,17.156,0.801,0.113,0.914,326.0,18.1967 +l2_distance_8,0.5,0.5,6,4,0,1,0,4,8,1,17.087,0.801,0.113,0.914,326.0,18.1967 +l2_distance_8,0.4,0.6,6,4,0,1,0,4,8,1,17.012,0.801,0.113,0.914,326.0,18.1967 +l2_distance_8,0.3,0.7,6,4,0,1,0,4,8,1,17.001,0.801,0.113,0.913,326.0,18.1967 +l2_distance_8,0.2,0.8,7,8,0,0,0,8,5,1,13.983,1.508,0.000,1.508,326.0,0.125542 +l2_distance_8,0.1,0.9,7,8,0,0,1,8,6,1,14.181,1.510,0.000,1.510,326.0,0.125542 +l2_distance_16,1.0,0.0,4,1,0,4,0,1,10,1,25.255,0.363,0.445,0.808,325.0,72.4103,408.0 +l2_distance_16,0.9,0.1,4,2,0,3,0,2,9,1,19.193,0.509,0.335,0.844,325.5,54.3391,394.0 +l2_distance_16,0.8,0.2,15,8,0,1,0,8,10,1,26.197,1.554,0.112,1.666,326.0,18.1967 +l2_distance_16,0.7,0.3,14,8,0,1,0,8,13,1,24.976,1.552,0.113,1.664,325.0,18.1967 +l2_distance_16,0.6,0.4,14,8,0,1,0,8,13,1,26.816,1.551,0.113,1.664,325.0,18.1967 +l2_distance_16,0.5,0.5,14,8,0,1,0,8,14,1,24.891,1.553,0.113,1.665,325.5,18.1967 +l2_distance_16,0.4,0.6,14,8,0,1,0,8,14,1,26.653,1.551,0.113,1.664,325.0,18.1967 +l2_distance_16,0.3,0.7,14,8,0,1,0,8,14,1,24.893,1.551,0.113,1.664,325.0,18.1967 +l2_distance_16,0.2,0.8,15,16,0,0,1,16,7,1,17.547,2.996,0.000,2.996,325.5,0.125542 +l2_distance_16,0.1,0.9,15,16,0,0,1,16,7,1,17.550,2.996,0.000,2.996,325.5,0.125542 +l2_distance_32,1.0,0.0,5,1,0,5,0,1,12,1,188.208,0.404,0.558,0.962,324.0,90.4814,718.0 +l2_distance_32,0.9,0.1,12,8,0,3,0,8,12,1,60.076,1.634,0.112,1.747,325.0,18.1967,994.0 +l2_distance_32,0.8,0.2,16,16,0,1,0,16,13,1,53.920,3.045,0.113,3.158,325.0,18.1967 +l2_distance_32,0.7,0.3,16,16,0,1,0,16,13,1,53.804,3.046,0.112,3.159,325.0,18.1967 +l2_distance_32,0.6,0.4,16,16,0,1,0,16,13,1,53.859,3.047,0.113,3.160,325.0,18.1967 +l2_distance_32,0.5,0.5,16,16,0,1,0,16,13,1,53.815,3.046,0.113,3.158,325.0,18.1967 +l2_distance_32,0.4,0.6,16,16,0,1,0,16,13,1,54.403,3.046,0.113,3.159,325.0,18.1967 +l2_distance_32,0.3,0.7,16,16,0,1,0,16,13,1,54.602,3.048,0.112,3.160,325.0,18.1967 +l2_distance_32,0.2,0.8,31,32,0,0,1,32,8,1,28.287,5.984,0.000,5.984,325.0,0.125542 +l2_distance_32,0.1,0.9,31,32,0,0,1,32,8,1,26.666,5.981,0.000,5.981,325.0,0.125542 +gx_kernel_4,1.0,0.0,5,0,1,1,2,3,7,1,15.797,0.514,0.113,0.626,327.0,18.1967,3381.0 +gx_kernel_4,0.9,0.1,4,0,1,1,2,3,6,1,14.301,0.489,0.112,0.601,327.0,18.1967,382.0 +gx_kernel_4,0.8,0.2,5,0,0,1,2,2,7,1,12.629,0.499,0.112,0.611,327.0,18.1967,1781.0 +gx_kernel_4,0.7,0.3,6,0,2,0,2,3,6,1,12.291,0.371,0.000,0.371,328.0,0.125542,1814.0 +gx_kernel_4,0.6,0.4,5,0,2,0,2,3,5,1,12.428,0.346,0.000,0.346,328.5,0.125542,315.0 +gx_kernel_4,0.5,0.5,5,0,2,0,2,3,5,1,12.368,0.346,0.000,0.346,328.5,0.125542,315.0 +gx_kernel_4,0.4,0.6,5,0,2,0,2,3,4,1,12.697,0.346,0.000,0.347,328.5,0.125542,314.0 +gx_kernel_4,0.3,0.7,4,1,2,0,1,3,4,1,12.604,0.347,0.000,0.347,328.0,0.125542,313.0 +gx_kernel_4,0.2,0.8,4,1,2,0,1,3,4,1,12.525,0.346,0.000,0.346,328.5,0.125542,313.0 +gx_kernel_4,0.1,0.9,5,0,2,0,4,3,6,1,12.492,0.346,0.000,0.346,328.0,0.125542,318.0 +gx_kernel_8,1.0,0.0 +gx_kernel_8,0.9,0.1 +gx_kernel_8,0.8,0.2 +gx_kernel_8,0.7,0.3 +gx_kernel_8,0.6,0.4 +gx_kernel_8,0.5,0.5 +gx_kernel_8,0.4,0.6 +gx_kernel_8,0.3,0.7 +gx_kernel_8,0.2,0.8 +gx_kernel_8,0.1,0.9 +gx_kernel_16,1.0,0.0 +gx_kernel_16,0.9,0.1 +gx_kernel_16,0.8,0.2 +gx_kernel_16,0.7,0.3 +gx_kernel_16,0.6,0.4 +gx_kernel_16,0.5,0.5 +gx_kernel_16,0.4,0.6 +gx_kernel_16,0.3,0.7 +gx_kernel_16,0.2,0.8 +gx_kernel_16,0.1,0.9 +gx_kernel_32,1.0,0.0 +gx_kernel_32,0.9,0.1 +gx_kernel_32,0.8,0.2 +gx_kernel_32,0.7,0.3 +gx_kernel_32,0.6,0.4 +gx_kernel_32,0.5,0.5 +gx_kernel_32,0.4,0.6 +gx_kernel_32,0.3,0.7 +gx_kernel_32,0.2,0.8 +gx_kernel_32,0.1,0.9 +gy_kernel_4,1.0,0.0,5,0,0,1,3,3,7,1,13.336,0.675,0.115,0.791,327.0,18.1967,2630.0 +gy_kernel_4,0.9,0.1,5,0,0,1,3,3,7,1,13.584,0.677,0.116,0.792,327.0,18.1967,2630.0 +gy_kernel_4,0.8,0.2,6,0,0,1,4,4,7,1,13.717,0.860,0.114,0.974,327.0,18.1967,984.0 +gy_kernel_4,0.7,0.3,7,0,0,0,5,5,6,1,12.966,1.037,0.000,1.037,327.0,0.125542,1016.0 +gy_kernel_4,0.6,0.4,7,0,0,0,5,5,6,1,13.120,1.038,0.000,1.038,327.0,0.125542,1016.0 +gy_kernel_4,0.5,0.5,6,1,0,0,4,5,6,1,13.192,1.037,0.000,1.037,327.0,0.125542,1015.0 +gy_kernel_4,0.4,0.6,6,1,0,0,4,5,6,1,13.154,1.037,0.000,1.037,327.0,0.125542,1015.0 +gy_kernel_4,0.3,0.7,6,1,0,0,4,5,6,1,13.300,1.036,0.000,1.036,327.0,0.125542,1016.0 +gy_kernel_4,0.2,0.8,6,1,0,0,4,5,6,1,18.155,1.037,0.000,1.037,327.0,0.125542 +gy_kernel_4,0.1,0.9,7,0,0,0,6,5,7,1,13.013,1.033,0.000,1.033,327.0,0.125542,1019.0 +gy_kernel_8,1.0,0.0 +gy_kernel_8,0.9,0.1 +gy_kernel_8,0.8,0.2 +gy_kernel_8,0.7,0.3 +gy_kernel_8,0.6,0.4 +gy_kernel_8,0.5,0.5 +gy_kernel_8,0.4,0.6 +gy_kernel_8,0.3,0.7 +gy_kernel_8,0.2,0.8 +gy_kernel_8,0.1,0.9 +gy_kernel_16,1.0,0.0 +gy_kernel_16,0.9,0.1 +gy_kernel_16,0.8,0.2 +gy_kernel_16,0.7,0.3 +gy_kernel_16,0.6,0.4 +gy_kernel_16,0.5,0.5 +gy_kernel_16,0.4,0.6 +gy_kernel_16,0.3,0.7 +gy_kernel_16,0.2,0.8 +gy_kernel_16,0.1,0.9 +gy_kernel_32,1.0,0.0 +gy_kernel_32,0.9,0.1 +gy_kernel_32,0.8,0.2 +gy_kernel_32,0.7,0.3 +gy_kernel_32,0.6,0.4 +gy_kernel_32,0.5,0.5 +gy_kernel_32,0.4,0.6 +gy_kernel_32,0.3,0.7 +gy_kernel_32,0.2,0.8 +gy_kernel_32,0.1,0.9 +roberts_cross_4,1.0,0.0,3,0,0,1,3,1,5,1,13.768,0.263,0.113,0.376,327.5,18.1967,182.0 +roberts_cross_4,0.9,0.1,4,0,0,1,3,1,6,1,13.231,0.288,0.112,0.400,327.0,18.1967,931.0 +roberts_cross_4,0.8,0.2,4,0,0,1,3,1,6,1,13.510,0.288,0.112,0.400,327.0,18.1967,931.0 +roberts_cross_4,0.7,0.3,3,0,0,1,3,1,5,1,13.336,0.262,0.112,0.374,327.5,18.1967,182.0 +roberts_cross_4,0.6,0.4,4,0,0,0,4,2,4,1,12.756,0.408,0.000,0.408,327.0,0.125542,218.0 +roberts_cross_4,0.5,0.5,4,0,0,0,4,2,4,1,14.088,0.408,0.000,0.408,327.0,0.125542,218.0 +roberts_cross_4,0.4,0.6,4,0,0,0,4,2,4,1,12.961,0.408,0.000,0.409,327.0,0.125542,218.0 +roberts_cross_4,0.3,0.7,4,0,0,0,4,2,4,1,13.032,0.408,0.000,0.408,327.0,0.125542,218.0 +roberts_cross_4,0.2,0.8,4,0,0,0,4,2,4,1,13.062,0.413,0.000,0.413,327.0,0.125542,218.0 +roberts_cross_4,0.1,0.9,4,0,0,0,2,2,4,1,13.589,0.417,0.000,0.417,327.0,0.125542,216.0 +roberts_cross_8,1.0,0.0 +roberts_cross_8,0.9,0.1 +roberts_cross_8,0.8,0.2 +roberts_cross_8,0.7,0.3 +roberts_cross_8,0.6,0.4 +roberts_cross_8,0.5,0.5 +roberts_cross_8,0.4,0.6 +roberts_cross_8,0.3,0.7 +roberts_cross_8,0.2,0.8 +roberts_cross_8,0.1,0.9 +roberts_cross_16,1.0,0.0 +roberts_cross_16,0.9,0.1 +roberts_cross_16,0.8,0.2 +roberts_cross_16,0.7,0.3 +roberts_cross_16,0.6,0.4 +roberts_cross_16,0.5,0.5 +roberts_cross_16,0.4,0.6 +roberts_cross_16,0.3,0.7 +roberts_cross_16,0.2,0.8 +roberts_cross_16,0.1,0.9 +roberts_cross_32,1.0,0.0 +roberts_cross_32,0.9,0.1 +roberts_cross_32,0.8,0.2 +roberts_cross_32,0.7,0.3 +roberts_cross_32,0.6,0.4 +roberts_cross_32,0.5,0.5 +roberts_cross_32,0.4,0.6 +roberts_cross_32,0.3,0.7 +roberts_cross_32,0.2,0.8 +roberts_cross_32,0.1,0.9 +matrix_mul_4,1.0,0.0,2,0,0,1,0,2,4,1,12.958,0.432,0.115,0.547,327.0,18.1967,276.0 +matrix_mul_4,0.9,0.1,2,0,0,1,0,2,4,1,12.771,0.432,0.114,0.547,327.0,18.1967,276.0 +matrix_mul_4,0.8,0.2,3,0,0,0,0,4,3,1,12.640,0.770,0.000,0.770,327.5,0.125542,409.0 +matrix_mul_4,0.7,0.3,3,0,0,0,0,4,3,1,12.592,0.770,0.000,0.770,327.0,0.125542,409.0 +matrix_mul_4,0.6,0.4,3,0,0,0,0,4,3,1,12.553,0.769,0.000,0.769,328.0,0.125542,409.0 +matrix_mul_4,0.5,0.5,3,0,0,0,0,4,3,1,12.702,0.772,0.000,0.772,327.5,0.125542,409.0 +matrix_mul_4,0.4,0.6,3,0,0,0,0,4,3,1,12.631,0.771,0.000,0.771,327.0,0.125542,409.0 +matrix_mul_4,0.3,0.7,3,0,0,0,0,4,3,1,12.775,0.768,0.000,0.768,327.0,0.125542,409.0 +matrix_mul_4,0.2,0.8,3,0,0,0,0,4,3,1,12.712,0.767,0.000,0.767,327.5,0.125542,409.0 +matrix_mul_4,0.1,0.9,3,0,0,0,0,4,3,1,12.607,0.767,0.000,0.767,327.5,0.125542,409.0 +matrix_mul_8,1.0,0.0 +matrix_mul_8,0.9,0.1 +matrix_mul_8,0.8,0.2 +matrix_mul_8,0.7,0.3 +matrix_mul_8,0.6,0.4 +matrix_mul_8,0.5,0.5 +matrix_mul_8,0.4,0.6 +matrix_mul_8,0.3,0.7 +matrix_mul_8,0.2,0.8 +matrix_mul_8,0.1,0.9 +matrix_mul_16,1.0,0.0 +matrix_mul_16,0.9,0.1 +matrix_mul_16,0.8,0.2 +matrix_mul_16,0.7,0.3 +matrix_mul_16,0.6,0.4 +matrix_mul_16,0.5,0.5 +matrix_mul_16,0.4,0.6 +matrix_mul_16,0.3,0.7 +matrix_mul_16,0.2,0.8 +matrix_mul_16,0.1,0.9 +matrix_mul_32,1.0,0.0 +matrix_mul_32,0.9,0.1 +matrix_mul_32,0.8,0.2 +matrix_mul_32,0.7,0.3 +matrix_mul_32,0.6,0.4 +matrix_mul_32,0.5,0.5 +matrix_mul_32,0.4,0.6 +matrix_mul_32,0.3,0.7 +matrix_mul_32,0.2,0.8 +matrix_mul_32,0.1,0.9 +tree_50-50_5_1,1.0,0.0,5,0,0,3,0,2,9,2,11.475,0.543,0.342,0.885,295.0,54.3391,787.0 +tree_50-50_5_1,0.9,0.1,5,0,0,3,0,2,9,2,11.309,0.540,0.339,0.880,295.0,54.3391,787.0 +tree_50-50_5_1,0.8,0.2,5,0,0,2,0,2,7,2,11.295,0.524,0.227,0.750,296.0,36.2679,719.0 +tree_50-50_5_1,0.7,0.3,6,0,0,3,0,2,8,2,11.285,0.568,0.226,0.794,296.0,36.2679,774.0 +tree_50-50_5_1,0.6,0.4,6,0,0,3,0,2,8,2,11.365,0.567,0.226,0.794,296.0,36.2679,774.0 +tree_50-50_5_1,0.5,0.5,6,0,0,3,0,2,8,2,11.484,0.567,0.226,0.794,296.0,36.2679,774.0 +tree_50-50_5_1,0.4,0.6,6,0,0,1,0,2,6,2,11.252,0.529,0.115,0.644,296.0,18.1967,1162.0 +tree_50-50_5_1,0.3,0.7,6,0,0,1,0,2,6,2,11.219,0.530,0.115,0.645,296.0,18.1967,1162.0 +tree_50-50_5_1,0.2,0.8,5,0,0,0,0,2,4,2,11.225,0.489,0.000,0.489,296.0,0.125542,1758.0 +tree_50-50_5_1,0.1,0.9,5,0,0,0,0,2,4,2,11.245,0.490,0.000,0.490,296.0,0.125542,1758.0 +tree_50-50_10_1,1.0,0.0,13,0,0,5,0,7,16,5,135.686,1.601,0.567,2.168,197.0,90.4814,2774.0 +tree_100-50_5_1,1.0,0.0,13,0,0,5,0,4,16,3,151.681,1.082,0.558,1.639,261.0,90.4814,1773.0 +tree_100-50_5_1,0.9,0.1,14,0,0,4,0,4,15,3,42.143,1.095,0.445,1.540,261.0,72.4103,1466.0 +tree_100-50_10_1,0.8,0.2 +tree_100-50_10_1,0.7,0.3 +tree_100-50_10_1,0.6,0.4 \ No newline at end of file diff --git a/results_RL.csv b/results_RL.csv new file mode 100644 index 00000000..30a497ac --- /dev/null +++ b/results_RL.csv @@ -0,0 +1,57 @@ +benchmark,w_ops,w_keys,add,sub,multiply_plain,rotate_rows,negate,multiply,Depth,Multiplicative Depth,compile_time (s),circuit_execution_time (s),galois_keys_generation_time (s),total_execution_time (s),Remaining_noise_budget,rotation_keys_size (MB),final_ops_cost,final_keys_cost +lin_reg_4,0.0,1.0,2,0,1,0,0,1,3,1,24.410,3006.0,0.0 +lin_reg_8,0.0,1.0,2,0,1,0,0,1,3,1,7.347,6006.0,0.0 +lin_reg_16,0.0,1.0,2,0,1,0,0,1,3,1,5.606,12006.0,0.0 +lin_reg_32,0.0,1.0,2,0,1,0,0,1,3,1,5.151,24006.0,0.0 +box_blur_4,0.0,1.0,13,0,0,0,0,0,6,0,5.236,21006.0,0.0 +box_blur_8,0.0,1.0,0,0,0,0,0,0,, +box_blur_16,0.0,1.0,0,0,0,0,0,0,, +box_blur_32,0.0,1.0,0,0,0,0,0,0,, +matrix_mul_4,0.0,1.0,3,0,0,0,0,4,3,1,5.256,28006.0,0.0 +matrix_mul_8,0.0,1.0,0,0,0,0,0,0,, +matrix_mul_16,0.0,1.0,0,0,0,0,0,0,, +matrix_mul_32,0.0,1.0,0,0,0,0,0,0,, +max_3,0.0,1.0,6,0,0,0,3,6,6,2,5.383,3009.0,0.0 +max_4,0.0,1.0,12,0,0,0,6,12,8,3,5.774,7012.0,0.0 +max_5,0.0,1.0,20,0,0,0,10,20,10,4,5.434,15015.0,0.0 +sort_3,0.0,1.0,0,0,0,0,0,0,,,5012.0,0.0 +dot_product_4,0.0,1.0,3,0,0,0,0,4,3,1,6.158,1756.0,0.0 +dot_product_8,0.0,1.0,7,0,0,0,0,8,4,1,5.313,3757.0,0.0 +dot_product_16,0.0,1.0,15,0,0,0,0,16,5,1,5.127,7758.0,0.0 +dot_product_32,0.0,1.0,31,0,0,0,0,32,6,1,5.275,15759.0,0.0 +gx_kernel_4,0.0,1.0,10,0,5,0,2,5,7,1,9.682,21507.0,0.0 +gx_kernel_8,0.0,1.0,0,0,0,0,0,0,, +gx_kernel_16,0.0,1.0,0,0,0,0,0,0,, +gx_kernel_32,0.0,1.0,0,0,0,0,0,0,, +gy_kernel_4,0.0,1.0,13,0,8,0,1,8,8,1,5.211,21507.0,0.0 +gy_kernel_8,0.0,1.0,0,0,0,0,0,0,, +gy_kernel_16,0.0,1.0,0,0,0,0,0,0,, +gy_kernel_32,0.0,1.0,0,0,0,0,0,0,, +hamming_dist_4,0.0,1.0,7,4,4,0,0,8,5,2,5.033,4759.0,0.0 +hamming_dist_8,0.0,1.0,15,8,8,0,0,16,6,2,5.040,9760.0,0.0 +hamming_dist_16,0.0,1.0,31,16,16,0,0,32,7,2,5.245,19761.0,0.0 +hamming_dist_32,0.0,1.0,63,32,32,0,0,64,8,2,5.379,39762.0,0.0 +l2_distance_4,0.0,1.0,3,4,0,0,0,4,4,1,5.554,3757.0,0.0 +l2_distance_8,0.0,1.0,7,8,0,0,0,8,5,1,5.378,7758.0,0.0 +l2_distance_16,0.0,1.0,15,16,0,0,0,16,6,1,4.945,15759.0,0.0 +l2_distance_32,0.0,1.0,31,32,0,0,0,32,7,1,5.032,31760.0,0.0 +poly_reg_4,0.0,1.0,3,0,0,0,0,2,5,2,5.271,5009.0,0.0 +poly_reg_8,0.0,1.0,3,0,0,0,0,2,5,2,5.063,10009.0,0.0 +poly_reg_16,0.0,1.0,3,0,0,0,0,2,5,2,5.000,20009.0,0.0 +poly_reg_32,0.0,1.0,3,0,0,0,0,2,5,2,5.492,40009.0,0.0 +roberts_cross_4,0.0,1.0,6,0,0,0,2,3,6,1,5.943,31007.0,0.0 +roberts_cross_8,0.0,1.0,0,0,0,0,0,0,, +roberts_cross_16,0.0,1.0,0,0,0,0,0,0,, +roberts_cross_32,0.0,1.0,0,0,0,0,0,0,, +tree_50-50_5_1,0.0,1.0,5,0,0,0,0,2,4,2,5.220,N/A,N/A,N/A,N/A,N/A,1758.0,0.0 +tree_50-50_5_2,0.0,1.0,4,0,0,0,0,2,4,2,5.506,N/A,N/A,N/A,N/A,N/A,1508.0,0.0 +tree_50-50_10_1,0.0,1.0,16,0,0,0,0,17,8,5,4.794,N/A,N/A,N/A,N/A,N/A,8265.0,0.0 +tree_50-50_10_2,0.0,1.0,16,0,0,0,0,16,11,5,4.931,N/A,N/A,N/A,N/A,N/A,8018.0,0.0 +tree_100-50_5_1,0.0,1.0,19,0,0,0,0,12,5,3,4.947,N/A,N/A,N/A,N/A,N/A,7760.0,0.0 +tree_100-50_5_2,0.0,1.0,16,0,0,0,0,15,5,3,4.881,N/A,N/A,N/A,N/A,N/A,7760.0,0.0 +tree_100-50_10_1,0.0,1.0,511,0,0,0,0,512,10,9,11.461,N/A,N/A,N/A,N/A,N/A,255771.0,0.0 +tree_100-50_10_2,0.0,1.0,504,0,0,0,0,519,10,8,39.266,N/A,N/A,N/A,N/A,N/A,255770.0,0.0 +tree_100-100_5_1,0.0,1.0,1,0,0,0,0,30,5,4,29.988,N/A,N/A,N/A,N/A,N/A,7761.0,0.0 +tree_100-100_5_2,0.0,1.0,2,0,0,0,0,29,5,4,5.176,N/A,N/A,N/A,N/A,N/A,7761.0,0.0 +tree_100-100_10_1,0.0,1.0,1,0,0,0,0,1022,10,9,10.503,N/A,N/A,N/A,N/A,N/A,255771.0,0.0 +tree_100-100_10_2,0.0,1.0,0,0,0,0,0,1023,10,10,35.803,N/A,N/A,N/A,N/A,N/A,255772.0,0.0 diff --git a/results_RL_adaptive.csv b/results_RL_adaptive.csv new file mode 100644 index 00000000..d1f3cebb --- /dev/null +++ b/results_RL_adaptive.csv @@ -0,0 +1,12 @@ +benchmark,w_ops,w_keys,add,sub,multiply_plain,rotate_rows,negate,multiply,Depth,Multiplicative Depth,compile_time (s),circuit_execution_time (s),galois_keys_generation_time (s),total_execution_time (s),Remaining_noise_budget,rotation_keys_size (MB),final_ops_cost,final_keys_cost +dot_product_8,1.0,0.0,3,0,0,4,0,1,8,1,26.313,399.0,4.0 +dot_product_8,0.9,0.1,3,0,0,4,0,1,8,1,5.088,399.0,4.0 +dot_product_8,0.8,0.2,3,0,0,3,0,1,7,1,5.038,284.0,3.0 +dot_product_8,0.7,0.3,4,0,0,1,0,4,5,1,4.968,463.0,1.0 +dot_product_8,0.6,0.4,4,0,0,1,0,4,5,1,4.971,463.0,1.0 +dot_product_8,0.5,0.5,6,0,0,1,0,4,6,1,4.907,464.0,1.0 +dot_product_8,0.4,0.6,7,0,0,0,0,8,4,1,5.654,3757.0,0.0 +dot_product_8,0.3,0.7,7,0,0,0,1,8,5,1,5.621,4008.0,0.0 +dot_product_8,0.2,0.8,7,0,0,0,1,8,5,1,5.552,4008.0,0.0 +dot_product_8,0.1,0.9,7,0,0,0,1,8,5,1,5.556,4008.0,0.0 +dot_product_8,0.75,0.25,3,0,0,2,0,2,6,1,4.818,317.0,2.0 diff --git a/results_RL_rotations.csv b/results_RL_rotations.csv new file mode 100644 index 00000000..b028ec6f --- /dev/null +++ b/results_RL_rotations.csv @@ -0,0 +1,102 @@ +benchmark,add,sub,multiply_plain,rotate_rows,negate,multiply,Depth,Multplicative Depth,compile_time (s),circuit_execution_time (s),galois_keys_generation_time (s),total_execution_time (s),Remaining_noise_budget,rotation_keys_size (MB) +max_3,6.0,1.0,0.0,3.0,1.0,3.0,10.0,2.0,12.524,0.718,0.222,0.940,295,36.2679 +max_4,10.0,0.0,0.0,2.0,4.0,7.0,11.0,3.0,11.818,1.304,0.223,1.528,262,36.2679 +max_5,18.0,1.0,0.0,5.0,7.0,15.0,15.0,4.0,12.566,2.692,0.446,3.138,229,72.4103 +sort_3,13.0,0.0,0.0,2.0,5.0,7.0,12.0,3.0,11.367,1.330,0.223,1.553,262,36.2679 +sort_4,43.0,2.0,0.0,6.0,16.0,39.0,20.0,6.0,18.898,6.628,0.556,7.184,163,90.4814 +box_blur_3,11.0,0.0,0.0,0.0,0.0,0.0,5.0,0.0,11.800,0.286,0.000,0.286,359,0.125542 +box_blur_4,9.0,0.0,0.0,0.0,0.0,0.0,5.0,0.0,11.724,0.240,0.000,0.240,359,0.125542 +box_blur_5,8.0,0.0,0.0,0.0,0.0,0.0,4.0,0.0,12.037,0.217,0.000,0.217,359,0.125542 +box_blur_8 +box_blur_15 +box_blur_16 +box_blur_24 +box_blur_31 +box_blur_32 +lin_reg_3 +lin_reg_4,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.247,0.053,0.000,0.053,358,0.125542 +lin_reg_5 +lin_reg_8,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.030,0.053,0.000,0.053,359,0.125542 +lin_reg_15 +lin_reg_16,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.679,0.053,0.000,0.053,358,0.125542 +lin_reg_24 +lin_reg_31 +lin_reg_32,2.0,0.0,1.0,0.0,0.0,1.0,3.0,1.0,11.208,0.053,0.000,0.053,359,0.125542 +hamming_dist_3,5.0,1.0,0.0,2.0,0.0,3.0,8.0,2.0,12.445,0.652,0.224,0.875,294,36.2679 +hamming_dist_4,6.0,2.0,1.0,1.0,0.0,5.0,7.0,2.0,11.328,0.800,0.112,0.912,295,18.1967 +hamming_dist_5,7.0,2.0,0.0,3.0,0.0,5.0,9.0,2.0,11.338,1.067,0.223,1.290,295,36.2679 +hamming_dist_8,10.0,4.0,1.0,3.0,0.0,9.0,8.0,2.0,14.883,1.620,0.223,1.843,295,36.2679 +hamming_dist_15,9.0,2.0,0.0,5.0,0.0,5.0,14.0,2.0,13.939,1.155,0.557,1.713,294,90.4814 +hamming_dist_16,8.0,1.0,0.0,5.0,0.0,3.0,15.0,2.0,15.860,0.776,0.558,1.334,293,90.4814 +hamming_dist_24,10.0,2.0,1.0,5.0,0.0,5.0,16.0,2.0,23.532,0.977,0.555,1.532,293,90.4814 +hamming_dist_31,10,2,1,5,0,5,15.0,2.0,28.238 +hamming_dist_32,8.0,1.0,0.0,5.0,0.0,3.0,14.0,2.0,33.373,0.778,0.558,1.336,293,90.4814 +poly_reg_3 +poly_reg_4,3.0,0.0,0.0,0.0,0.0,2.0,5.0,2.0,11.057,0.407,0.000,0.407,296,0.125542 +poly_reg_5 +poly_reg_8,4.0,0.0,0.0,1.0,0.0,2.0,7.0,2.0,11.191,0.471,0.113,0.583,296,18.1967 +poly_reg_15 +poly_reg_16,4.0,0.0,0.0,1.0,0.0,2.0,7.0,2.0,11.025,0.471,0.113,0.584,296,18.1967 +poly_reg_24 +poly_reg_31 +poly_reg_32,3.0,0.0,0.0,0.0,0.0,2.0,5.0,2.0,11.203,0.407,0.000,0.407,296,0.125542 +l2_distance_3,2.0,2.0,0.0,2.0,0.0,1.0,6.0,1.0,10.816,0.303,0.222,0.525,327,36.2679 +l2_distance_4,2.0,2.0,0.0,1.0,0.0,2.0,5.0,1.0,12.610,0.424,0.113,0.537,326,18.1967 +l2_distance_5,3.0,3.0,0.0,3.0,0.0,2.0,7.0,1.0,10.869,0.531,0.222,0.753,325,36.2679 +l2_distance_8,4.0,4.0,0.0,2.0,0.0,4.0,7.0,1.0,13.677,0.840,0.223,1.063,325,36.2679 +l2_distance_15,4.0,2.0,0.0,5.0,0.0,2.0,10.0,1.0,14.553,0.593,0.443,1.036,323,72.4103 +l2_distance_16,4.0,1.0,0.0,5.0,0.0,1.0,11.0,1.0,13.258,0.402,0.555,0.957,324,90.4814 +l2_distance_24,5.0,3.0,0.0,4.0,0.0,2.0,11.0,1.0,15.928,0.573,0.445,1.018,325,72.4103 +l2_distance_31,5.0,3.0,0.0,4.0,0.0,2.0,11.0,1.0,17.718,0.573,0.446,1.020,325,72.4103 +l2_distance_32,5.0,1.0,0.0,5.0,0.0,1.0,12.0,1.0,22.434,0.403,0.557,0.959,324,90.4814 +dot_product_3,2.0,0.0,0.0,2.0,0.0,1.0,5.0,1.0,10.870,0.279,0.223,0.502,327,36.2679 +dot_product_4,2.0,0.0,0.0,1.0,0.0,2.0,4.0,1.0,11.202,0.425,0.113,0.538,328,18.1967 +dot_product_5,3.0,0.0,0.0,2.0,0.0,2.0,6.0,1.0,11.364,0.467,0.222,0.689,327,36.2679 +dot_product_8,4.0,0.0,0.0,1.0,0.0,4.0,5.0,1.0,11.010,0.798,0.112,0.911,327,18.1967 +dot_product_15,8.0,0.0,0.0,2.0,0.0,7.0,7.0,1.0,11.694,1.398,0.223,1.622,327,36.2679 +dot_product_16,8.0,0.0,0.0,2.0,0.0,8.0,7.0,1.0,15.582,1.557,0.223,1.780,324,36.2679 +dot_product_24,16.0,0.0,0.0,9.0,0.0,8.0,8.0,1.0,16.066,1.880,0.222,2.102,326,36.2679 +dot_product_31,5.0,0.0,0.0,5.0,0.0,2.0,11.0,1.0,13.750,0.567,0.557,1.124,322,90.4814 +dot_product_32,5.0,0.0,0.0,5.0,0.0,1.0,11.0,1.0,13.426,0.403,0.554,0.957,326,90.4814 +gx_kernel_3,9.0,0.0,3.0,0.0,2.0,4.0,6.0,2.0,12.092,0.440,0.000,0.440,304,0.125542 +gx_kernel_4,7.0,0.0,3.0,0.0,1.0,3.0,5.0,1.0,11.074,0.238,0.000,0.238,340,0.125542 +gx_kernel_5,7.0,0.0,3.0,0.0,1.0,3.0,5.0,1.0,11.256,0.242,0.000,0.242,335,0.125542 +gx_kernel_8 +gx_kernel_15 +gx_kernel_16 +gx_kernel_24 +gx_kernel_31 +gx_kernel_32 +gy_kernel_3,10.0,0.0,5.0,0.0,2.0,5.0,6.0,1.0,11.332,0.344,0.000,0.344,335,0.125542 +gy_kernel_4,8.0,0.0,3.0,0.0,2.0,5.0,6.0,1.0,11.055,0.592,0.000,0.592,328,0.125542 +gy_kernel_5,8.0,0.0,3.0,0.0,2.0,5.0,6.0,1.0,12.593,0.597,0.000,0.597,328,0.125542 +gy_kernel_8 +gy_kernel_15 +gy_kernel_16 +gy_kernel_24 +gy_kernel_31 +gy_kernel_32 +roberts_cross_3,6.0,0.0,0.0,1.0,3.0,2.0,5.0,1.0,11.166,0.474,0.112,0.586,326,18.1967 +roberts_cross_4,6.0,0.0,0.0,1.0,3.0,2.0,5.0,1.0,11.316,0.474,0.112,0.586,326,18.1967 +roberts_cross_5,6.0,0.0,0.0,0.0,3.0,2.0,5.0,1.0,12.409,0.433,0.000,0.433,327,0.125542 +roberts_cross_8 +roberts_cross_15 +roberts_cross_16 +roberts_cross_24 +roberts_cross_31 +roberts_cross_32 +matrix_mul_3,2.0,0.0,0.0,0.0,0.0,3.0,3.0,1.0,11.034,0.570,0.000,0.570,328,0.125542 +matrix_mul_4,2.0,0.0,0.0,1.0,0.0,2.0,4.0,1.0,11.180,0.425,0.113,0.538,327,18.1967 +matrix_mul_5,4.0,0.0,0.0,0.0,0.0,5.0,4.0,1.0,11.815,0.943,0.000,0.943,327,0.125542 +matrix_mul_8 +matrix_mul_15 +matrix_mul_16 +matrix_mul_24 +matrix_mul_31 +matrix_mul_32 +tree_50-50_5_1,6.0,0.0,0.0,3.0,0.0,2.0,8.0,2.0,11.073,0.553,0.222,0.775,296,36.2679 +tree_50-50_10_1,16.0,0.0,0.0,5.0,0.0,9.0,15.0,5.0,13.389,1.976,0.558,2.533,197,90.4814 +tree_100-50_5_1,18.0,0.0,0.0,5.0,0.0,4.0,14.0,3.0,12.277,1.206,0.555,1.761,260,90.4814 +tree_100-50_10_1,524.0,0.0,0.0,2.0,0.0,502.0,16.0,10.0,31.384,93.194,0.223,93.417,35,36.2679 +tree_100-100_5_1,1.0,0.0,0.0,5.0,0.0,4.0,10.0,4.0,12.502,0.793,0.557,1.350,228,90.4814 +tree_100-100_10_1,1.0,0.0,0.0,2.0,0.0,256.0,12.0,9.0,24.696,41.549,0.223,41.772,63,36.2679 diff --git a/rotations_results.csv b/rotations_results.csv new file mode 100644 index 00000000..024f517f --- /dev/null +++ b/rotations_results.csv @@ -0,0 +1,59 @@ +benchmark,slot_count,number_of_rotations +box_blur,2,2 +box_blur,3,1 +box_blur,4,1 +box_blur,5,0 +dot_product,2,1 +dot_product,3,2 +dot_product,4,3 +dot_product,5,3 +dot_product,6,3 +dot_product,7,4 +dot_product,8,4 +dot_product,9,3 +dot_product,10,4 +dot_product,11,4 +dot_product,12,4 +dot_product,13,4 +dot_product,14,4 +dot_product,15,4 +dot_product,16,5 +dot_product,17,4 +dot_product,18,4 +dot_product,19,4 +dot_product,20,4 +dot_product,21,4 +dot_product,22,4 +dot_product,23,4 +dot_product,24,5 +dot_product,25,4 +dot_product,26,4 +dot_product,27,4 +dot_product,28,4 +dot_product,29,7 +dot_product,30,7 +dot_product,31,8 +dot_product,32,5 +gx_kernel,2,2 +gx_kernel,3,1 +gx_kernel,4,1 +gx_kernel,5,0 +gy_kernel,2,3 +gy_kernel,3,1 +gy_kernel,4,1 +gy_kernel,5,0 +hamming_dist,2,4 +hamming_dist,3,3 +hamming_dist,4,4 +hamming_dist,5,3 +hamming_dist,6,4 +hamming_dist,7,4 +hamming_dist,8,5 +hamming_dist,9,4 +hamming_dist,10,4 +hamming_dist,11,4 +hamming_dist,12,4 +hamming_dist,13,4 +hamming_dist,14,4 +hamming_dist,15,4 +hamming_dist,16,6 diff --git a/run_benchmarks.py b/run_benchmarks.py index 8b6c3467..2d577d26 100644 --- a/run_benchmarks.py +++ b/run_benchmarks.py @@ -4,51 +4,45 @@ import csv import re import statistics +from RL.fhe_rl.pareto import generate_pref_list # Specify the parent folder containing the benchmarks and build subfolders benchmarks_folder = "benchmarks" ################################ build_folder = os.path.join("build", "benchmarks") operations = ["add", "sub", "multiply_plain", "rotate_rows", "negate", "multiply"] -infos = ["benchmark"] -additional_infos =[ "Depth", "Multplicative Depth","compile_time (s)", "execution_time (s)","Remaining_noise_budget"] +infos = ["benchmark", "w_ops", "w_keys"] +additional_infos =[ "Depth", "Multiplicative Depth","compile_time (s)", "circuit_execution_time (s)",'galois_keys_generation_time (s)','total_execution_time (s)',"Remaining_noise_budget", 'rotation_keys_size (MB)', 'final_ops_cost', 'final_keys_cost'] infos.extend(operations) infos.extend(additional_infos) ############################################# -# Force system toolchain to avoid glibc mismatches with Conda compilers. -cmake_env = os.environ.copy() -cmake_env["CC"] = "/usr/bin/gcc" -cmake_env["CXX"] = "/usr/bin/g++" +# try: +# print("run=> cmake', '-S', '.', '-B', 'build' ") +# result = subprocess.run( +# ['cmake', '-S', '.', '-B', 'build'], +# check=True, +# stdout=subprocess.PIPE, +# stderr=subprocess.PIPE, +# universal_newlines=True +# ) +# print("run=> 'cmake', '--build', 'build'") +# result = subprocess.run( +# ['cmake', '--build', 'build'], +# check=True, +# stdout=subprocess.PIPE, +# stderr=subprocess.PIPE, +# universal_newlines=True +# ) +# except subprocess.CalledProcessError as e: +# print(f"Command failed with error:\n{e.stderr}") -try: - print("run=> cmake', '-S', '.', '-B', 'build' ") - result = subprocess.run( - ['cmake', '-S', '.', '-B', 'build'], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - env=cmake_env - ) - print("run=> 'cmake', '--build', 'build'") - result = subprocess.run( - ['cmake', '--build', 'build'], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - env=cmake_env - ) -except subprocess.CalledProcessError as e: - print(f"Command failed with error:\n{e.stderr}") - -benchmark_folders = ["max","sort","box_blur","lin_reg","hamming_dist","poly_reg","l2_distance","dot_product","gx_kernel","gy_kernel","roberts_cross","matrix_mul"] +benchmark_folders = ["dot_product"] #benchmark_folders = ["lin_reg","hamming_dist","poly_reg","l2_distance","dot_product","gx_kernel","gy_kernel","roberts_cross","matrix_mul","max","sort"] exceptions = ["max","sort","discrete_cosin_transform","poly_derivative"] benchmarks_slot_counts = { "max" : [3,4,5], - "sort" : [3,4], + "sort" : [3], "discrete_cosin_transform":[1], "poly_derivative":[1] } @@ -58,14 +52,15 @@ optimization_method = 1 # 0 = egraph (default), 1 = RL cse_enabled = 1 vectorize_code = 1 -slot_counts= [3,4,5,8,16,32] +slot_counts= [4,8,16,32] +pref_list = generate_pref_list(11) iterations = 2 #minimum 2 window_size = 0 depths = [5,10] -regimes = ["50-50","100-50","100-100"] +regimes = ["50-50","100-50"] number_instances_each_polynomial_configuration = 1 compile_time_timeout_seconds = 7200 -output_csv = f"results/results_{'RL' if optimization_method == 1 else 'EGraph'}.csv" +output_csv = f"results_{'RL' if optimization_method == 1 else 'EGraph'}.csv" ###################################### with open(output_csv, mode='w', newline='') as file: @@ -87,302 +82,165 @@ if subfolder_name in exceptions : updated_slot_counts = benchmarks_slot_counts[subfolder_name] for slot_count in updated_slot_counts: - try : - benchmark_compilation_timed_out = False - print("****************************************************************") - print(f"*****run {subfolder_name} , for slot_count : {slot_count}******") - operation_stats = { - "add": [], "sub": [], "multiply_plain": [], "rotate_rows": [], - "negate": [], "multiply": [], "Depth": [], "Multiplicative Depth": [], - "compile_time (s)": [], "execution_time (s)": [],"Remaining_noise_budget": [], - } - ###generate io_file for benchmark with slot_count - if not subfolder_name in exceptions : - pro = subprocess.Popen(['python3', 'generate_{}.py'.format(subfolder_name),'--slot_count',str(slot_count)],cwd=build_path) - pro.wait() - ###################################### - for iteration in range(iterations): - print(f"===> Running iteration : {iteration + 1}") - # Step 1: Run the first benchmark command - benchmark_run_command = f"./{subfolder_name} {vectorize_code} {slot_count} {optimization_method} {window_size} 1 {cse_enabled} 1 " - try: - result = subprocess.run( - benchmark_run_command, shell=True, check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - cwd=build_path, - timeout=compile_time_timeout_seconds - ) - lines = result.stdout.splitlines() - - # Collect compile time (ms) - compile_time_found = False - poly_mod_found = True - for line in lines: - if ' ms' in line: - #print(f"=======> compile_time line : {line}") - optimization_time = float(line.split()[0]) - operation_stats["compile_time (s)"].append(optimization_time) - compile_time_found = True - if 'poly_mod:' in line: - print(f"======> poly_mod : {line}") - poly_mod = float(line.split()[1]) - operation_stats["poly_modulus"].append(poly_mod) - poly_mod_found = True - if compile_time_found and poly_mod_found : - break - # Collect depth and multiplicative depth - #print(result.stdout) - depth_match = re.search(r'max:\s*\((\d+),\s*(\d+)\)', result.stdout) - print(depth_match) - depth = int(depth_match.group(1)) if depth_match else None - multiplicative_depth = int(depth_match.group(2)) if depth_match else None - #print(depth) - #print(multiplicative_depth) - print(f"Depth=>{depth}, multiplcative_depth=>{multiplicative_depth}") - operation_stats["Depth"].append(depth) - operation_stats["Multiplicative Depth"].append(multiplicative_depth) - except subprocess.TimeoutExpired: - print(f"Command `{benchmark_run_command}` timed out after {compile_time_timeout_seconds} seconds.") - benchmark_compilation_timed_out = True - except subprocess.CalledProcessError as e: - error_message = e.stderr if e.stderr else "No error message available." - print("Command for {} failed with error:\n{}".format(subfolder_name, error_message)) - continue - if benchmark_compilation_timed_out : - break - ## building and running fhe code - build_path_he = os.path.join(build_path, "he") - build_path_he_build = os.path.join(build_path_he, "build") - if os.path.isdir(build_path_he_build): - shutil.rmtree(build_path_he_build, ignore_errors=True) - result = subprocess.run( - ['cmake', '-S', '.', '-B', 'build'], - cwd=build_path_he, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - env=cmake_env - ) - result = subprocess.run( - ['cmake', '--build', 'build'], - cwd=build_path_he, - universal_newlines=True, - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=cmake_env - ) - build_path_he_build = os.path.join(build_path_he, "build") - # Step 2: Build and run fhe code - - if iteration == iterations-1 : - build_path_he = os.path.join(build_path, "he") - try: - # Run the compiled program - for counter in range(iterations): - command = f"./main" - result = subprocess.run( - command, shell=True, check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - cwd=build_path_he_build - ) - print("**fhe run done**") - #print(result.stderr) - if counter > 0 : - lines = result.stdout.splitlines() - # Collect execution time (ms) - comp = 0 - print(f"returned lines : \n {lines} \n\n") - for line in lines: - if 'execution_time_(ms):' in line: - #print(f"==> execution time {line.split()[0]}") - execution_time = float(line.split()[1]) - operation_stats["execution_time (s)"].append(execution_time) - #################################### - if 'Remaining_noise_budget:' in line: - Remaining_noise_budget=int(line.split()[1]) - operation_stats["Remaining_noise_budget"].append(Remaining_noise_budget) - ############## - if comp == 2 : - break - - except subprocess.CalledProcessError as e: - print(f"Failed in building fhe_code for benchmark: {subfolder_name}") - continue - - # Step 3: Parse operation counts from the generated C++ code - file_name = os.path.join(build_path_he, "_gen_he_fhe.cpp") - with open(file_name, "r") as file: - file_content = file.read() - for op in operations: - nb_occurrences = len(re.findall(rf'\b{op}', file_content)) - operation_stats[op].append(int(nb_occurrences)) - #################################################################### - bench_name = subfolder_name+"_"+str(slot_count) - row=[bench_name] - if not benchmark_compilation_timed_out : - for key, values in operation_stats.items(): - if values == []: - print(f"Warning: No values found for {key} in {subfolder_name} with slot_count {slot_count}.") - result = "N/A" - else : - result = statistics.median(values) - if key == "compile_time (s)" or key == "execution_time (s)" : - result = result / 1000 - result = format(result, ".3f") - row.append(result) if values else None - - print(f"{key} {values} {result}") - ########################################################################## - ########################################################################## - with open(output_csv, mode='a', newline='') as file: - writer = csv.writer(file) - writer.writerow(row) - except Exception as e: - print(f"Command for {subfolder_name} failed with error:\n{e}") - continue - -###################################################################################### -###################################################################################### -print("Run polynomial benchmarks !!!!!! ") -polynomial_folders = ["polynomials_coyote"] -for subfolder_name in polynomial_folders: - benchmark_path = os.path.join(benchmarks_folder, subfolder_name) - build_path = os.path.join(build_folder, subfolder_name) - # build_path = build/benchmarks/dot_product - ## informations to collect - for regime in regimes : - for tree_depth in depths : - for instance in range(1,number_instances_each_polynomial_configuration+1): - try: + for w in pref_list: + w_ops, w_keys = w + try : benchmark_compilation_timed_out = False + print("****************************************************************") + print(f"*****run {subfolder_name} , for slot_count : {slot_count}******") operation_stats = { "add": [], "sub": [], "multiply_plain": [], "rotate_rows": [], "negate": [], "multiply": [], "Depth": [], "Multiplicative Depth": [], - "compile_time (s)": [], "execution_time (s)": [],"Remaining_noise_budget": [] + "compile_time (s)": [], "circuit_execution_time (s)": [], "galois_keys_generation_time (s)": [], + "total_execution_time (s)": [], "Remaining_noise_budget": [], "rotation_keys_size (MB)": [], "final_ops_cost": [], "final_keys_cost": [] } - benchmark_name = f'tree_{regime}_{tree_depth}_{instance}' - print(f"Benchmark '{benchmark_name}' will be run...") + ###generate io_file for benchmark with slot_count + if not subfolder_name in exceptions : + pro = subprocess.Popen(['python3', 'generate_{}.py'.format(subfolder_name),'--slot_count',str(slot_count)],cwd=build_path) + pro.wait() + ###################################### for iteration in range(iterations): - optimization_time="" - execution_time="" - depth = "" - multiplicative_depth = "" - if os.path.isdir(build_path): - print(f"=========> Iteration : {iteration+1}") - command = f"./{subfolder_name} {tree_depth} {instance} {regime} {vectorize_code} {optimization_method}" - try: - result = subprocess.run( - command, shell=True, check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - cwd=build_path, - timeout=compile_time_timeout_seconds - ) - lines = result.stdout.splitlines() - compile_time_found = False - poly_mod_found = True - for line in lines: - if ' ms' in line: - print(f"=======> compile_time line : {line}") - optimization_time = float(line.split()[0]) - operation_stats["compile_time (s)"].append(optimization_time) - compile_time_found = True - if compile_time_found and poly_mod_found : - break - depth_match = re.search(r'max:\s*\((\d+),\s*(\d+)\)', result.stdout) - #print(f"\n\n {depth_match} \n\n") - depth = depth_match.group(1) if depth_match else None - multiplicative_depth = depth_match.group(2) if depth_match else None - operation_stats["Depth"].append(int(depth)) - operation_stats["Multiplicative Depth"].append(int(multiplicative_depth)) - print(f"Depth: {depth} --MultipliDepth {multiplicative_depth}") - except subprocess.TimeoutExpired: - print(f"Command `{command}` timed out after {compile_time_timeout_seconds} seconds.") - benchmark_compilation_timed_out = True - except subprocess.CalledProcessError as e: - print(f"Command for {subfolder_name} failed with error:\n{e.stderr}") - if benchmark_compilation_timed_out : - break - ######################################################################### - ## building and running fhe code - build_path_he = os.path.join(build_path, "he") - build_path_he_build = os.path.join(build_path_he, "build") - if os.path.isdir(build_path_he_build): - shutil.rmtree(build_path_he_build, ignore_errors=True) - try: - result = subprocess.run( - ['cmake', '-S', '.', '-B', 'build'], - check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, + print(f"===> Running iteration : {iteration + 1}") + # Step 1: Run the first benchmark command + benchmark_run_command = f"./{subfolder_name} {vectorize_code} {slot_count} {optimization_method} {window_size} 1 {cse_enabled} 1 {w_ops} {w_keys}" + try: + result = subprocess.run( + benchmark_run_command, shell=True, check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=build_path, + timeout=compile_time_timeout_seconds + ) + lines = result.stdout.splitlines() + + # Collect compile time (ms) + compile_time_found = False + poly_mod_found = True + temp_cost = 0.0 + for line in lines: + clean_line = re.sub(r'\x1b\[[0-9;]*m', '', line) + #print(line) + if 'final ops cost:' in clean_line: + try: + operation_stats["final_ops_cost"].append(float(clean_line.split(':')[1].strip())) + except (IndexError, ValueError): + pass + if 'final keys cost:' in clean_line: + try: + operation_stats["final_keys_cost"].append(float(clean_line.split(':')[1].strip())) + except (IndexError, ValueError): + pass + if ' ms' in line: + #print(f"=======> compile_time line : {line}") + optimization_time = float(line.split()[0]) + operation_stats["compile_time (s)"].append(optimization_time) + compile_time_found = True + if 'poly_mod:' in line: + print(f"======> poly_mod : {line}") + poly_mod = float(line.split()[1]) + operation_stats["poly_modulus"].append(poly_mod) + poly_mod_found = True + if compile_time_found and poly_mod_found : + break + # Collect depth and multiplicative depth + #print(result.stdout) + depth_match = re.search(r'max:\s*\((\d+),\s*(\d+)\)', result.stdout) + print(depth_match) + depth = int(depth_match.group(1)) if depth_match else None + multiplicative_depth = int(depth_match.group(2)) if depth_match else None + #print(depth) + #print(multiplicative_depth) + print(f"Depth=>{depth}, multiplcative_depth=>{multiplicative_depth}") + operation_stats["Depth"].append(depth) + operation_stats["Multiplicative Depth"].append(multiplicative_depth) + except subprocess.TimeoutExpired: + print(f"Command `{benchmark_run_command}` timed out after {compile_time_timeout_seconds} seconds.") + benchmark_compilation_timed_out = True + except subprocess.CalledProcessError as e: + error_message = e.stderr if e.stderr else "No error message available." + print("Command for {} failed with error:\n{}".format(subfolder_name, error_message)) + continue + if benchmark_compilation_timed_out : + break + ## building and running fhe code + build_path_he = os.path.join(build_path, "he") + result = subprocess.run(['cmake', '-S', '.', '-B', 'build'], cwd=build_path_he, - env=cmake_env - ) - result = subprocess.run( - ['cmake', '--build', 'build'], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True, - cwd=build_path_he, - env=cmake_env - ) - # result = subprocess.run(['sudo','cmake','--install','build'], check=True, capture_output=False, text=True) - except : - print(f"Failed in building fhe_code for benchmark:{subfolder_name} ,with error \n") - build_path_he_build = os.path.join(build_path_he, "build") - ########################################################################## - if iteration == iterations-1 : - try: - # Run the compiled program - for counter in range(iterations): - command = f"./main" - result = subprocess.run( - command, shell=True, check=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - cwd=build_path_he_build - ) - print("**fhe run done**") - #print(result.stderr) - if counter > 0 : - lines = result.stdout.splitlines() - comp = 0 - for line in lines: - if 'execution_time_(ms):' in line: - #print(f"==> execution time {line.split()[0]}") - execution_time = float(line.split()[1]) - operation_stats["execution_time (s)"].append(execution_time) - #################################### - if 'Remaining_noise_budget:' in line: - Remaining_noise_budget=int(line.split()[1]) - operation_stats["Remaining_noise_budget"].append(Remaining_noise_budget) - ############## - if comp == 2 : - break + universal_newlines=True + ) + result =subprocess.run(['cmake', '--build', 'build'], cwd=build_path_he,universal_newlines=True, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE,) + build_path_he_build = os.path.join(build_path_he, "build") + # Step 2: Build and run fhe code + + if iteration == iterations-1 : + build_path_he = os.path.join(build_path, "he") + try: + # Run the compiled program + for counter in range(iterations): + command = f"./main" + result = subprocess.run( + command, shell=True, check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=build_path_he_build + ) + print("**fhe run done**") + #print(result.stderr) + if counter > 0 : + lines = result.stdout.splitlines() + # Collect execution time (ms) + comp = 0 + print(f"returned lines : \n {lines} \n\n") + for line in lines: + if 'circuit_execution_time_(ms):' in line: + #print(f"==> execution time {line.split()[0]}") + execution_time = float(line.split()[1]) + operation_stats["circuit_execution_time (s)"].append(execution_time) + #################################### + if 'galois_keys_generation_time_(ms):' in line: + #print(f"==> execution time {line.split()[0]}") + execution_time = float(line.split()[1]) + operation_stats["galois_keys_generation_time (s)"].append(execution_time) + #################################### + if 'total_execution_time_(ms):' in line: + #print(f"==> execution time {line.split()[0]}") + execution_time = float(line.split()[1]) + operation_stats["total_execution_time (s)"].append(execution_time) + #################################### + if 'rotation_keys_size_(MB):' in line: + #print(f"==> execution time {line.split()[0]}") + keys_size = float(line.split()[1]) + operation_stats["rotation_keys_size (MB)"].append(keys_size) + #################################### + if 'Remaining_noise_budget:' in line: + Remaining_noise_budget=int(line.split()[1]) + operation_stats["Remaining_noise_budget"].append(Remaining_noise_budget) + ############## + if comp == 2 : + break + + except subprocess.CalledProcessError as e: + print(f"Failed in building fhe_code for benchmark: {subfolder_name}") + continue - except subprocess.CalledProcessError as e: - print(f"Failed in running fhe_code for benchmark: {subfolder_name}") - continue - ########################################################################### # Step 3: Parse operation counts from the generated C++ code file_name = os.path.join(build_path_he, "_gen_he_fhe.cpp") with open(file_name, "r") as file: file_content = file.read() for op in operations: nb_occurrences = len(re.findall(rf'\b{op}', file_content)) - operation_stats[op].append(int(nb_occurrences)) - ################################################################## - row=[benchmark_name] + operation_stats[op].append(int(nb_occurrences)) + #################################################################### + bench_name = subfolder_name+"_"+str(slot_count) + row=[bench_name, w_ops, w_keys] if not benchmark_compilation_timed_out : for key, values in operation_stats.items(): if values == []: @@ -390,16 +248,197 @@ result = "N/A" else : result = statistics.median(values) - if key == "compile_time (s)" or key == "execution_time (s)" : + if key in ["compile_time (s)","circuit_execution_time (s)","galois_keys_generation_time (s)","total_execution_time (s)"] : result = result / 1000 result = format(result, ".3f") row.append(result) if values else None - + print(f"{key} {values} {result}") + ########################################################################## + ########################################################################## with open(output_csv, mode='a', newline='') as file: writer = csv.writer(file) - writer.writerow(row) + writer.writerow(row) except Exception as e: print(f"Command for {subfolder_name} failed with error:\n{e}") - continue - + continue + +###################################################################################### +###################################################################################### +print("Run polynomial benchmarks !!!!!! ") +polynomial_folders = ["polynomials_coyote"] +for subfolder_name in polynomial_folders: + benchmark_path = os.path.join(benchmarks_folder, subfolder_name) + build_path = os.path.join(build_folder, subfolder_name) + # build_path = build/benchmarks/dot_product + ## informations to collect + for regime in regimes : + for tree_depth in depths : + for instance in range(1,number_instances_each_polynomial_configuration+1): + for w in pref_list: + w_ops, w_keys = w + try: + benchmark_compilation_timed_out = False + operation_stats = { + "add": [], "sub": [], "multiply_plain": [], "rotate_rows": [], + "negate": [], "multiply": [], "Depth": [], "Multiplicative Depth": [], + "compile_time (s)": [], "circuit_execution_time (s)": [], "galois_keys_generation_time (s)": [], + "total_execution_time (s)": [], "Remaining_noise_budget": [], 'rotation_keys_size (MB)': [], "final_ops_cost": [], "final_keys_cost": [] + } + benchmark_name = f'tree_{regime}_{tree_depth}_{instance}' + print(f"Benchmark '{benchmark_name}' will be run...") + for iteration in range(iterations): + optimization_time="" + execution_time="" + depth = "" + multiplicative_depth = "" + if os.path.isdir(build_path): + print(f"=========> Iteration : {iteration+1}") + command = f"./{subfolder_name} {tree_depth} {instance} {regime} {vectorize_code} {optimization_method} {window_size} 1 {cse_enabled} 1 {w_ops} {w_keys}" + try: + result = subprocess.run( + command, shell=True, check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=build_path, + timeout=compile_time_timeout_seconds + ) + lines = result.stdout.splitlines() + compile_time_found = False + poly_mod_found = True + for line in lines: + clean_line = re.sub(r'\x1b\[[0-9;]*m', '', line) + if 'final ops cost:' in clean_line: + try: + operation_stats["final_ops_cost"].append(float(clean_line.split(':')[1].strip())) + except (IndexError, ValueError): + pass + if 'final keys cost:' in clean_line: + try: + operation_stats["final_keys_cost"].append(float(clean_line.split(':')[1].strip())) + except (IndexError, ValueError): + pass + if ' ms' in line: + print(f"=======> compile_time line : {line}") + optimization_time = float(line.split()[0]) + operation_stats["compile_time (s)"].append(optimization_time) + compile_time_found = True + if compile_time_found and poly_mod_found : + break + depth_match = re.search(r'max:\s*\((\d+),\s*(\d+)\)', result.stdout) + #print(f"\n\n {depth_match} \n\n") + depth = depth_match.group(1) if depth_match else None + multiplicative_depth = depth_match.group(2) if depth_match else None + operation_stats["Depth"].append(int(depth)) + operation_stats["Multiplicative Depth"].append(int(multiplicative_depth)) + print(f"Depth: {depth} --MultipliDepth {multiplicative_depth}") + except subprocess.TimeoutExpired: + print(f"Command `{command}` timed out after {compile_time_timeout_seconds} seconds.") + benchmark_compilation_timed_out = True + except subprocess.CalledProcessError as e: + print(f"Command for {subfolder_name} failed with error:\n{e.stderr}") + if benchmark_compilation_timed_out : + break + ######################################################################### + ## building and running fhe code + build_path_he = os.path.join(build_path, "he") + try: + result = subprocess.run( + ['cmake', '-S', '.', '-B', 'build'], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=build_path_he + ) + result = subprocess.run( + ['cmake', '--build', 'build'], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=build_path_he + ) + # result = subprocess.run(['sudo','cmake','--install','build'], check=True, capture_output=False, text=True) + except : + print(f"Failed in building fhe_code for benchmark:{subfolder_name} ,with error \n") + build_path_he_build = os.path.join(build_path_he, "build") + ########################################################################## + if iteration == iterations-1 : + try: + # Run the compiled program + for counter in range(iterations): + command = f"./main" + result = subprocess.run( + command, shell=True, check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + cwd=build_path_he_build + ) + print("**fhe run done**") + #print(result.stderr) + if counter > 0 : + lines = result.stdout.splitlines() + comp = 0 + for line in lines: + if 'circuit_execution_time_(ms):' in line: + #print(f"==> execution time {line.split()[0]}") + execution_time = float(line.split()[1]) + operation_stats["circuit_execution_time (s)"].append(execution_time) + #################################### + if 'galois_keys_generation_time_(ms):' in line: + #print(f"==> execution time {line.split()[0]}") + execution_time = float(line.split()[1]) + operation_stats["galois_keys_generation_time (s)"].append(execution_time) + #################################### + if 'total_execution_time_(ms):' in line: + #print(f"==> execution time {line.split()[0]}") + execution_time = float(line.split()[1]) + operation_stats["total_execution_time (s)"].append(execution_time) + #################################### + if 'rotation_keys_size_(MB):' in line: + #print(f"==> execution time {line.split()[0]}") + keys_size = float(line.split()[1]) + operation_stats["rotation_keys_size (MB)"].append(keys_size) + #################################### + if 'Remaining_noise_budget:' in line: + Remaining_noise_budget=int(line.split()[1]) + operation_stats["Remaining_noise_budget"].append(Remaining_noise_budget) + ############## + if comp == 2 : + break + + except subprocess.CalledProcessError as e: + print(f"Failed in running fhe_code for benchmark: {subfolder_name}") + continue + ########################################################################### + # Step 3: Parse operation counts from the generated C++ code + file_name = os.path.join(build_path_he, "_gen_he_fhe.cpp") + with open(file_name, "r") as file: + file_content = file.read() + for op in operations: + nb_occurrences = len(re.findall(rf'\b{op}', file_content)) + operation_stats[op].append(int(nb_occurrences)) + ################################################################## + row=[benchmark_name, w_ops, w_keys] + if not benchmark_compilation_timed_out : + for key, values in operation_stats.items(): + if values == []: + print(f"Warning: No values found for {key} in {benchmark_name}.") + result = "N/A" + else : + result = statistics.median(values) + if key in ["compile_time (s)","circuit_execution_time (s)","galois_keys_generation_time (s)","total_execution_time (s)"] : + result = result / 1000 + result = format(result, ".3f") + row.append(result) if values else None + + print(f"{key} {values} {result}") + with open(output_csv, mode='a', newline='') as file: + writer = csv.writer(file) + writer.writerow(row) + except Exception as e: + print(f"Command for {subfolder_name} failed with error:\n{e}") + continue \ No newline at end of file diff --git a/run_morl_benchmarks.py b/run_morl_benchmarks.py new file mode 100644 index 00000000..59e1f7fc --- /dev/null +++ b/run_morl_benchmarks.py @@ -0,0 +1,649 @@ +import os +import subprocess +import csv +import re +import statistics +from RL.fhe_rl.pareto import generate_pref_list + +benchmarks_folder = "benchmarks" +build_folder = os.path.join("build", "benchmarks") +operations = ["add", "sub", "multiply_plain", "rotate_rows", "negate", "multiply"] +infos = ["benchmark", "w_ops", "w_keys"] +additional_infos = ["Depth", "Multiplicative Depth", "compile_time (s)", "circuit_execution_time (s)", + 'galois_keys_generation_time (s)', 'total_execution_time (s)', "Remaining_noise_budget", + 'rotation_keys_size (MB)', 'final_ops_cost', 'final_keys_cost'] +infos.extend(operations) +infos.extend(additional_infos) + +try: + print("run=> cmake', '-S', '.', '-B', 'build' ") + result = subprocess.run( + ['cmake', '-S', '.', '-B', 'build'], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True + ) + print("run=> 'cmake', '--build', 'build'") + result = subprocess.run( + ['cmake', '--build', 'build'], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True + ) +except subprocess.CalledProcessError as e: + print(f"Command failed with error:\n{e.stderr}") + +benchmark_folders = ["lin_reg"] +exceptions = ["max", "sort", "discrete_cosin_transform", "poly_derivative"] +benchmarks_slot_counts = { + "max": [3, 4, 5], + "sort": [3], + "discrete_cosin_transform": [1], + "poly_derivative": [1] +} + +optimization_method = 1 +cse_enabled = 1 +vectorize_code = 1 +slot_counts = [4, 8, 16, 32] +pref_list = [[0.0, 1.0]] +iterations = 1 +window_size = 0 +depths = [5] +regimes = ["100-50"] +number_instances_each_polynomial_configuration = 1 +compile_time_timeout_seconds = 7200 +output_csv = f"results_{'RL' if optimization_method == 1 else 'EGraph'}.csv" + +# ── adaptive bisection config ────────────────────────────────────────────────── +BASE_UNIT = 18 # known spacing between key-size levels +MAX_BISECT_DEPTH = 3 # safety cap on recursion + +written_w_ops = set() + +with open(output_csv, mode='w', newline='') as file: + writer = csv.writer(file) + writer.writerow(infos) + + +# ── helpers ──────────────────────────────────────────────────────────────────── +def get_key_size_level(key_size): + return int(key_size) + + +def run_benchmark(subfolder_name, slot_count, w_ops, w_keys, build_path, build_path_he, build_path_he_build): + """ + Run one (subfolder_name, slot_count, w_ops) point exactly like the original loop body. + Returns (row, key_size_level) or (None, None) on timeout. + """ + benchmark_compilation_timed_out = False + operation_stats = { + "add": [], "sub": [], "multiply_plain": [], "rotate_rows": [], + "negate": [], "multiply": [], "Depth": [], "Multiplicative Depth": [], + "compile_time (s)": [], "circuit_execution_time (s)": [], "galois_keys_generation_time (s)": [], + "total_execution_time (s)": [], "Remaining_noise_budget": [], "rotation_keys_size (MB)": [], + "final_ops_cost": [], "final_keys_cost": [] + } + + if not subfolder_name in exceptions: + pro = subprocess.Popen(['python3', 'generate_{}.py'.format(subfolder_name), '--slot_count', str(slot_count)], cwd=build_path) + pro.wait() + + for iteration in range(iterations): + print(f"===> Running iteration : {iteration + 1}") + benchmark_run_command = f"./{subfolder_name} {vectorize_code} {slot_count} {optimization_method} {window_size} 1 {cse_enabled} 1 {w_ops} {w_keys}" + try: + result = subprocess.run( + benchmark_run_command, shell=True, check=False, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, cwd=build_path, + timeout=compile_time_timeout_seconds + ) + lines = result.stdout.splitlines() + compile_time_found = False + poly_mod_found = True + for line in lines: + clean_line = re.sub(r'\x1b\[[0-9;]*m', '', line) + if 'final ops cost:' in clean_line: + try: + operation_stats["final_ops_cost"].append(float(clean_line.split(':')[1].strip())) + except (IndexError, ValueError): + pass + if 'final keys cost:' in clean_line: + try: + operation_stats["final_keys_cost"].append(float(clean_line.split(':')[1].strip())) + except (IndexError, ValueError): + pass + if ' ms' in line: + optimization_time = float(line.split()[0]) + operation_stats["compile_time (s)"].append(optimization_time) + compile_time_found = True + if 'poly_mod:' in line: + print(f"======> poly_mod : {line}") + poly_mod = float(line.split()[1]) + poly_mod_found = True + if compile_time_found and poly_mod_found: + break + depth_match = re.search(r'max:\s*\((\d+),\s*(\d+)\)', result.stdout) + depth = int(depth_match.group(1)) if depth_match else None + multiplicative_depth = int(depth_match.group(2)) if depth_match else None + print(f"Depth=>{depth}, multiplicative_depth=>{multiplicative_depth}") + operation_stats["Depth"].append(depth) + operation_stats["Multiplicative Depth"].append(multiplicative_depth) + except subprocess.TimeoutExpired: + print(f"Command `{benchmark_run_command}` timed out after {compile_time_timeout_seconds} seconds.") + benchmark_compilation_timed_out = True + except subprocess.CalledProcessError as e: + error_message = e.stderr if e.stderr else "No error message available." + print("Command for {} failed with error:\n{}".format(subfolder_name, error_message)) + continue + + if benchmark_compilation_timed_out: + break + + he_build_ok = (result.returncode == 0) + if he_build_ok: + result = subprocess.run(['cmake', '-S', '.', '-B', 'build'], + cwd=build_path_he, check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + result = subprocess.run(['cmake', '--build', 'build'], + cwd=build_path_he, check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True) + + if iteration == iterations - 1: + try: + for counter in range(iterations): + command = f"./main" + result = subprocess.run( + command, shell=True, check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, cwd=build_path_he_build + ) + print("**fhe run done**") + if counter > 0: + lines = result.stdout.splitlines() + comp = 0 + print(f"returned lines : \n {lines} \n\n") + for line in lines: + if 'circuit_execution_time_(ms):' in line: + operation_stats["circuit_execution_time (s)"].append(float(line.split()[1])) + if 'galois_keys_generation_time_(ms):' in line: + operation_stats["galois_keys_generation_time (s)"].append(float(line.split()[1])) + if 'total_execution_time_(ms):' in line: + operation_stats["total_execution_time (s)"].append(float(line.split()[1])) + if 'rotation_keys_size_(MB):' in line: + operation_stats["rotation_keys_size (MB)"].append(float(line.split()[1])) + if 'Remaining_noise_budget:' in line: + operation_stats["Remaining_noise_budget"].append(int(line.split()[1])) + if comp == 2: + break + except subprocess.CalledProcessError as e: + print(f"Failed in building fhe_code for benchmark: {subfolder_name}") + + file_name = os.path.join(build_path_he, "_gen_he_fhe.cpp") + with open(file_name, "r") as file: + file_content = file.read() + for op in operations: + nb_occurrences = len(re.findall(rf'\b{op}', file_content)) + operation_stats[op].append(int(nb_occurrences)) + + # ── build row ────────────────────────────────────────────────────────────── + bench_name = subfolder_name + "_" + str(slot_count) + row = [bench_name, w_ops, w_keys] + raw_key_size = None + + if not benchmark_compilation_timed_out: + for key, values in operation_stats.items(): + if values == []: + print(f"Warning: No values found for {key} in {subfolder_name} with slot_count {slot_count}.") + result = "N/A" + else: + result = statistics.median(values) + if key in ["compile_time (s)", "circuit_execution_time (s)", + "galois_keys_generation_time (s)", "total_execution_time (s)"]: + result = result / 1000 + result = format(result, ".3f") + if key == "final_keys_cost": + raw_key_size = statistics.median(values) # keep raw float for level comparison + row.append(result) + print(f"{key} {values} {result}") + + # ── write immediately (open → write → close, same as original) ──────────── + if w_ops not in written_w_ops: + with open(output_csv, mode='a', newline='') as file: + writer = csv.writer(file) + writer.writerow(row) + written_w_ops.add(w_ops) + else: + print(f" [skip] w_ops={w_ops} already written, skipping duplicate row") + + if benchmark_compilation_timed_out: + return None, None + + key_size_level = get_key_size_level(raw_key_size) if raw_key_size is not None else None + return row, key_size_level + + +def _bisect_one_target(subfolder_name, slot_count, w_hi, ks_hi, w_lo, ks_lo, + build_path, build_path_he, build_path_he_build, + target, found, depth=0): + """ + Search for a single integer key-size level `target` in (w_lo, w_hi). + `found` is a shared set — if the target (or any level) gets discovered + by a mid-point probe it is added there so sibling searches can skip it. + """ + if depth >= MAX_BISECT_DEPTH: + print(f" [max depth] target={target} stopping at [{w_lo:.6f}, {w_hi:.6f}]") + return + + print(f" {' ' * depth}[bisect d={depth}] [{w_lo:.6f}, {w_hi:.6f}] " + f"endpoints=[{ks_lo}, {ks_hi}] target={target}") + + w_mid = (w_hi + w_lo) / 2 + w_mid = round(w_mid, 10) # avoid floating point precision issues + w_mid_keys = round(1.0 - w_mid, 10) + + row, ks_mid = run_benchmark(subfolder_name, slot_count, w_mid, w_mid_keys, + build_path, build_path_he, build_path_he_build) + if row is None: + return + + print(f" {' ' * depth}[result d={depth}] w_ops={w_mid:.6f} → keys={ks_mid}") + found.add(ks_mid) + + if ks_mid == target: + return # found — done for this target + + elif ks_mid < target: + # target is in the upper half [w_mid, w_hi] + _bisect_one_target(subfolder_name, slot_count, w_hi, ks_hi, w_mid, ks_mid, + build_path, build_path_he, build_path_he_build, + target, found, depth=depth + 1) + + else: # ks_mid > target + # target is in the lower half [w_lo, w_mid] + _bisect_one_target(subfolder_name, slot_count, w_mid, ks_mid, w_lo, ks_lo, + build_path, build_path_he, build_path_he_build, + target, found, depth=depth + 1) + + +def bisect_search(subfolder_name, slot_count, w_hi, ks_hi, w_lo, ks_lo, + build_path, build_path_he, build_path_he_build): + """ + Entry point: search independently for every integer level between ks_lo + and ks_hi. Each target gets its own MAX_BISECT_DEPTH budget. + Targets already discovered by a previous probe are skipped. + """ + lo, hi = min(ks_lo, ks_hi), max(ks_lo, ks_hi) + missing_targets = list(range(lo + 1, hi)) # e.g. ks=1,ks=4 → [2, 3] + + if not missing_targets: + return + + found = set() # levels discovered by any probe, shared across targets + + for target in missing_targets: + if target in found: + print(f" [skip] target={target} already found by a previous probe") + continue + print(f"\n === independent search for target={target} " + f"in [{w_lo:.6f}, {w_hi:.6f}] ===") + _bisect_one_target(subfolder_name, slot_count, w_hi, ks_hi, w_lo, ks_lo, + build_path, build_path_he, build_path_he_build, + target, found) + +# ── main loop (your original structure) ─────────────────────────────────────── +for subfolder_name in benchmark_folders: + benchmark_path = os.path.join(benchmarks_folder, subfolder_name) + build_path = os.path.join(build_folder, subfolder_name) + if os.path.isdir(build_path): + updated_slot_counts = slot_counts + if subfolder_name in exceptions: + updated_slot_counts = benchmarks_slot_counts[subfolder_name] + + for slot_count in updated_slot_counts: + build_path_he = os.path.join(build_path, "he") + build_path_he_build = os.path.join(build_path_he, "build") + + # ── Phase 1: coarse grid (your original pref_list loop) ─────────── + written_w_ops = set() + known_points = [] # list of (w_ops, ks_level) + for w in pref_list: + w_ops, w_keys = w + print("****************************************************************") + print(f"*****run {subfolder_name} , for slot_count : {slot_count}******") + try: + row, ks_level = run_benchmark(subfolder_name, slot_count, w_ops, w_keys, + build_path, build_path_he, build_path_he_build) + if ks_level is not None: + known_points.append((w_ops, ks_level)) + except Exception as e: + print(f"Command for {subfolder_name} failed with error:\n{e}") + continue + + # ── Phase 2: bisect every interval where levels differ ───────────── + print(f"\n--- Adaptive bisection for {subfolder_name} slot_count={slot_count} ---") + known_points.sort(key=lambda x: x[0]) # ascending w_ops + for i in range(len(known_points) - 1): + w_lo, ks_lo = known_points[i] + w_hi, ks_hi = known_points[i + 1] + if ks_lo != ks_hi: + try: + bisect_search(subfolder_name, slot_count, w_hi, ks_hi, w_lo, ks_lo, + build_path, build_path_he, build_path_he_build) + except Exception as e: + print(f"Bisect failed for {subfolder_name} [{w_lo}, {w_hi}]: {e}") + continue +################################################################################################# +# ── poly-tree helpers ────────────────────────────────────────────────────────── + +def run_poly_benchmark(subfolder_name, build_path, build_path_he, build_path_he_build, + benchmark_name, tree_depth, instance, regime, + w_ops, w_keys): + """ + Run one (benchmark_name, w_ops, w_keys) point for polynomial trees. + Returns (row, ks_ops_level, ks_keys_level) or (None, None, None) on timeout. + """ + benchmark_compilation_timed_out = False + operation_stats = { + "add": [], "sub": [], "multiply_plain": [], "rotate_rows": [], + "negate": [], "multiply": [], "Depth": [], "Multiplicative Depth": [], + "compile_time (s)": [], "circuit_execution_time (s)": [], + "galois_keys_generation_time (s)": [], "total_execution_time (s)": [], + "Remaining_noise_budget": [], "rotation_keys_size (MB)": [], + "final_ops_cost": [], "final_keys_cost": [] + } + + for iteration in range(iterations): + optimization_time = "" + execution_time = "" + depth = "" + multiplicative_depth = "" + + if not os.path.isdir(build_path): + continue + + print(f"=========> Iteration : {iteration + 1}") + command = (f"./{subfolder_name} {tree_depth} {instance} {regime} " + f"{vectorize_code} {optimization_method} {window_size} " + f"1 {cse_enabled} 1 {w_ops} {w_keys}") + try: + result = subprocess.run( + command, shell=True, check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, cwd=build_path, + timeout=compile_time_timeout_seconds + ) + lines = result.stdout.splitlines() + compile_time_found = False + poly_mod_found = True + for line in lines: + clean_line = re.sub(r'\x1b\[[0-9;]*m', '', line) + if 'final ops cost:' in clean_line: + try: + operation_stats["final_ops_cost"].append(float(clean_line.split(':')[1].strip())) + except (IndexError, ValueError): + pass + if 'final keys cost:' in clean_line: + try: + operation_stats["final_keys_cost"].append(float(clean_line.split(':')[1].strip())) + except (IndexError, ValueError): + pass + if ' ms' in line: + optimization_time = float(line.split()[0]) + operation_stats["compile_time (s)"].append(optimization_time) + compile_time_found = True + if 'poly_mod:' in line: + print(f"======> poly_mod : {line}") + poly_mod_found = True + if compile_time_found and poly_mod_found: + break + + depth_match = re.search(r'max:\s*\((\d+),\s*(\d+)\)', result.stdout) + depth = int(depth_match.group(1)) if depth_match else None + multiplicative_depth = int(depth_match.group(2)) if depth_match else None + print(f"Depth: {depth} -- MultiplicativeDepth: {multiplicative_depth}") + operation_stats["Depth"].append(depth) + operation_stats["Multiplicative Depth"].append(multiplicative_depth) + + except subprocess.TimeoutExpired: + print(f"Command `{command}` timed out after {compile_time_timeout_seconds} seconds.") + benchmark_compilation_timed_out = True + except subprocess.CalledProcessError as e: + print(f"Command for {subfolder_name} failed with error:\n{e.stderr}") + + if benchmark_compilation_timed_out: + break + + # build HE code + try: + subprocess.run(['cmake', '-S', '.', '-B', 'build'], + check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, cwd=build_path_he) + subprocess.run(['cmake', '--build', 'build'], + check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, cwd=build_path_he) + except Exception: + print(f"Failed in building fhe_code for benchmark: {subfolder_name}") + + if iteration == iterations - 1: + try: + for counter in range(iterations): + result = subprocess.run( + "./main", shell=True, check=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, cwd=build_path_he_build + ) + print("**fhe run done**") + if counter > 0: + lines = result.stdout.splitlines() + comp = 0 + print(f"returned lines : \n {lines} \n\n") + for line in lines: + if 'circuit_execution_time_(ms):' in line: + operation_stats["circuit_execution_time (s)"].append(float(line.split()[1])) + if 'galois_keys_generation_time_(ms):' in line: + operation_stats["galois_keys_generation_time (s)"].append(float(line.split()[1])) + if 'total_execution_time_(ms):' in line: + operation_stats["total_execution_time (s)"].append(float(line.split()[1])) + if 'rotation_keys_size_(MB):' in line: + operation_stats["rotation_keys_size (MB)"].append(float(line.split()[1])) + if 'Remaining_noise_budget:' in line: + operation_stats["Remaining_noise_budget"].append(int(line.split()[1])) + if comp == 2: + break + except subprocess.CalledProcessError: + print(f"Failed in running fhe_code for benchmark: {subfolder_name}") + + # parse operation counts from generated C++ file + file_name = os.path.join(build_path_he, "_gen_he_fhe.cpp") + with open(file_name, "r") as f: + file_content = f.read() + for op in operations: + nb_occurrences = len(re.findall(rf'\b{op}', file_content)) + operation_stats[op].append(int(nb_occurrences)) + + # ── build row ────────────────────────────────────────────────────────────── + row = [benchmark_name, w_ops, w_keys] + raw_ops_cost = None + raw_keys_cost = None + + if not benchmark_compilation_timed_out: + for key, values in operation_stats.items(): + if not values: + print(f"Warning: No values found for {key} in {benchmark_name}.") + result_val = "N/A" + else: + result_val = statistics.median(values) + if key in ["compile_time (s)", "circuit_execution_time (s)", + "galois_keys_generation_time (s)", "total_execution_time (s)"]: + result_val = result_val / 1000 + result_val = format(result_val, ".3f") + if key == "final_ops_cost": + raw_ops_cost = statistics.median(values) + if key == "final_keys_cost": + raw_keys_cost = statistics.median(values) + row.append(result_val) + print(f"{key} {values} {result_val}") + + with open(output_csv, mode='a', newline='') as f: + writer = csv.writer(f) + writer.writerow(row) + + if benchmark_compilation_timed_out: + return None, None, None + + ops_level = int(raw_ops_cost) if raw_ops_cost is not None else None + keys_level = int(raw_keys_cost) if raw_keys_cost is not None else None + return row, ops_level, keys_level + + +def _poly_bisect_one_target(subfolder_name, build_path, build_path_he, build_path_he_build, + benchmark_name, tree_depth, instance, regime, + w_hi, ks_ops_hi, ks_keys_hi, + w_lo, ks_ops_lo, ks_keys_lo, + target_ops, target_keys, found, depth=0): + """ + Search for a single (target_ops, target_keys) level pair. + Bisects on w_ops; w_keys = 1 - w_ops. + """ + if depth >= MAX_BISECT_DEPTH: + print(f" [max depth] targets=({target_ops},{target_keys}) " + f"stopping at [{w_lo:.6f}, {w_hi:.6f}]") + return + + print(f" {' ' * depth}[poly bisect d={depth}] [{w_lo:.6f}, {w_hi:.6f}] " + f"endpoints=ops[{ks_ops_lo},{ks_ops_hi}] keys[{ks_keys_lo},{ks_keys_hi}] " + f"target=({target_ops},{target_keys})") + + w_mid = (w_hi + w_lo) / 2 + w_mid = round(w_mid, 10) # avoid floating point precision issues + w_mid_keys = round(1.0 - w_mid, 10) + + row, ops_mid, keys_mid = run_poly_benchmark( + subfolder_name, build_path, build_path_he, build_path_he_build, + benchmark_name, tree_depth, instance, regime, w_mid, w_mid_keys + ) + if row is None: + return + + print(f" {' ' * depth}[result d={depth}] w_ops={w_mid:.6f} → " + f"ops={ops_mid} keys={keys_mid}") + found.add((ops_mid, keys_mid)) + + if ops_mid == target_ops and keys_mid == target_keys: + return # found — done for this target + + # use ops cost as the primary bisection axis (same convention as first script) + if ops_mid < target_ops: + _poly_bisect_one_target( + subfolder_name, build_path, build_path_he, build_path_he_build, + benchmark_name, tree_depth, instance, regime, + w_hi, ks_ops_hi, ks_keys_hi, w_mid, ops_mid, keys_mid, + target_ops, target_keys, found, depth=depth + 1 + ) + else: + _poly_bisect_one_target( + subfolder_name, build_path, build_path_he, build_path_he_build, + benchmark_name, tree_depth, instance, regime, + w_mid, ops_mid, keys_mid, w_lo, ks_ops_lo, ks_keys_lo, + target_ops, target_keys, found, depth=depth + 1 + ) + + +def poly_bisect_search(subfolder_name, build_path, build_path_he, build_path_he_build, + benchmark_name, tree_depth, instance, regime, + w_hi, ks_ops_hi, ks_keys_hi, + w_lo, ks_ops_lo, ks_keys_lo): + """ + Entry point for poly-tree bisection between two adjacent coarse-grid points. + Searches independently for every missing (ops, keys) level pair. + Each target gets its own MAX_BISECT_DEPTH budget. + """ + ops_lo, ops_hi = min(ks_ops_lo, ks_ops_hi), max(ks_ops_lo, ks_ops_hi) + keys_lo, keys_hi = min(ks_keys_lo, ks_keys_hi), max(ks_keys_lo, ks_keys_hi) + + # build all missing (ops, keys) integer pairs between the two endpoints + missing_targets = [ + (o, k) + for o in range(ops_lo + 1, ops_hi) + for k in range(keys_lo + 1, keys_hi) + ] + + if not missing_targets: + return + + found = set() + + for target_ops, target_keys in missing_targets: + if (target_ops, target_keys) in found: + print(f" [skip] target=({target_ops},{target_keys}) already found") + continue + print(f"\n === poly independent search for target=({target_ops},{target_keys}) " + f"in [{w_lo:.6f}, {w_hi:.6f}] ===") + _poly_bisect_one_target( + subfolder_name, build_path, build_path_he, build_path_he_build, + benchmark_name, tree_depth, instance, regime, + w_hi, ks_ops_hi, ks_keys_hi, w_lo, ks_ops_lo, ks_keys_lo, + target_ops, target_keys, found + ) + + +# ── poly-tree main loop ──────────────────────────────────────────────────────── +print("Run polynomial benchmarks !!!!!!") +polynomial_folders = ["polynomials_coyote"] + +for subfolder_name in polynomial_folders: + build_path = os.path.join(build_folder, subfolder_name) + build_path_he = os.path.join(build_path, "he") + build_path_he_build = os.path.join(build_path_he, "build") + + for regime in regimes: + for tree_depth in depths: + for instance in range(1, number_instances_each_polynomial_configuration + 1): + benchmark_name = f'tree_{regime}_{tree_depth}_{instance}' + print(f"\nBenchmark '{benchmark_name}' will be run...") + + # ── Phase 1: coarse grid ─────────────────────────────────────── + known_points = [] # list of (w_ops, ops_level, keys_level) + written_w_ops_poly = set() + + for w in pref_list: + w_ops, w_keys = w + if w_ops in written_w_ops_poly: + continue + print("*" * 64) + try: + row, ops_level, keys_level = run_poly_benchmark( + subfolder_name, build_path, build_path_he, build_path_he_build, + benchmark_name, tree_depth, instance, regime, w_ops, w_keys + ) + written_w_ops_poly.add(w_ops) + if ops_level is not None and keys_level is not None: + known_points.append((w_ops, ops_level, keys_level)) + except Exception as e: + print(f"Command for {benchmark_name} failed with error:\n{e}") + continue + + # ── Phase 2: adaptive bisection ──────────────────────────────── + print(f"\n--- Adaptive bisection for {benchmark_name} ---") + known_points.sort(key=lambda x: x[0]) # ascending w_ops + + for i in range(len(known_points) - 1): + w_lo, ops_lo, keys_lo = known_points[i] + w_hi, ops_hi, keys_hi = known_points[i + 1] + if ops_lo != ops_hi or keys_lo != keys_hi: + try: + poly_bisect_search( + subfolder_name, build_path, build_path_he, build_path_he_build, + benchmark_name, tree_depth, instance, regime, + w_hi, ops_hi, keys_hi, w_lo, ops_lo, keys_lo + ) + except Exception as e: + print(f"Poly bisect failed for {benchmark_name} " + f"[{w_lo}, {w_hi}]: {e}") + continue \ No newline at end of file diff --git a/separated_rotations.csv b/separated_rotations.csv new file mode 100644 index 00000000..1c9bcbc4 --- /dev/null +++ b/separated_rotations.csv @@ -0,0 +1,531 @@ +benchmark,w_ops,w_keys,add,sub,multiply_plain,rotate_rows,negate,multiply,Depth,Multiplicative Depth,compile_time (s),circuit_execution_time (s),galois_keys_generation_time (s),total_execution_time (s),Remaining_noise_budget,rotation_key_size (MB),Final_Cost +gx_kernel_4,1.0,0.0,4,0,1,1,2,3,6,1,34.316,0.488,0.112,0.600,328.0,18.1967,0.0 +gx_kernel_4,0.9,0.1,4,0,1,1,2,3,6,1,32.102,0.487,0.112,0.599,327.5,18.1967,0.0 +gx_kernel_4,0.8,0.2,4,0,1,1,2,3,6,1,31.684,0.487,0.112,0.599,327.5,18.1967,0.0 +gx_kernel_4,0.7,0.3,4,0,1,2,2,3,7,1,32.086,0.529,0.112,0.641,327.0,18.1967,0.0 +gx_kernel_4,0.6,0.4,4,0,1,2,2,3,7,1,32.060,0.530,0.112,0.643,328.0,18.1967,0.0 +gx_kernel_4,0.5,0.5,4,0,1,2,2,3,7,1,32.431,0.530,0.112,0.643,327.5,18.1967,0.0 +gx_kernel_4,0.4,0.6,4,0,1,2,2,3,7,1,32.356,0.530,0.112,0.643,327.5,18.1967,0.0 +gx_kernel_4,0.3,0.7,4,0,1,2,2,3,7,1,32.309,0.531,0.112,0.643,327.5,18.1967,0.0 +gx_kernel_4,0.2,0.8,4,0,1,2,2,3,7,1,32.366,0.530,0.112,0.643,327.0,18.1967,0.0 +gx_kernel_4,0.1,0.9,4,0,1,4,2,3,9,1,32.497,0.612,0.112,0.724,328.0,18.1967,0.0 +gx_kernel_8,1.0,0.0 +gx_kernel_8,0.9,0.1 +gx_kernel_8,0.8,0.2 +gx_kernel_8,0.7,0.3 +gx_kernel_8,0.6,0.4 +gx_kernel_8,0.5,0.5 +gx_kernel_8,0.4,0.6 +gx_kernel_8,0.3,0.7 +gx_kernel_8,0.2,0.8 +gx_kernel_8,0.1,0.9 +gx_kernel_16,1.0,0.0 +gx_kernel_16,0.9,0.1 +gx_kernel_16,0.8,0.2 +gx_kernel_16,0.7,0.3 +gx_kernel_16,0.6,0.4 +gx_kernel_16,0.5,0.5 +gx_kernel_16,0.4,0.6 +gx_kernel_16,0.3,0.7 +gx_kernel_16,0.2,0.8 +gx_kernel_16,0.1,0.9 +gx_kernel_32,1.0,0.0 +gx_kernel_32,0.9,0.1 +gx_kernel_32,0.8,0.2 +gx_kernel_32,0.7,0.3 +gx_kernel_32,0.6,0.4 +gx_kernel_32,0.5,0.5 +gx_kernel_32,0.4,0.6 +gx_kernel_32,0.3,0.7 +gx_kernel_32,0.2,0.8 +gx_kernel_32,0.1,0.9 +gy_kernel_4,1.0,0.0,3,0,0,1,3,3,6,1,32.484,0.617,0.113,0.730,327.0,18.1967,190.5 +gy_kernel_4,0.9,0.1,3,0,0,1,3,3,6,1,32.201,0.619,0.113,0.732,327.0,18.1967,0.0 +gy_kernel_4,0.8,0.2,3,0,0,1,3,3,6,1,32.145,0.619,0.112,0.731,327.0,18.1967,0.0 +gy_kernel_4,0.7,0.3,3,0,0,2,3,3,7,1,32.341,0.660,0.112,0.772,327.0,18.1967,0.0 +gy_kernel_4,0.6,0.4,3,0,0,2,3,3,7,1,32.164,0.663,0.113,0.776,327.0,18.1967,0.0 +gy_kernel_4,0.5,0.5,3,0,0,2,3,3,7,1,32.239,0.662,0.113,0.776,327.0,18.1967,0.0 +gy_kernel_4,0.4,0.6,3,0,0,2,3,3,7,1,32.290,0.658,0.112,0.770,327.0,18.1967,0.0 +gy_kernel_4,0.3,0.7,3,0,0,4,3,3,9,1,32.343,0.740,0.113,0.853,327.0,18.1967,0.0 +gy_kernel_4,0.2,0.8,3,0,0,4,3,3,9,1,32.132,0.739,0.113,0.851,327.0,18.1967,0.0 +gy_kernel_4,0.1,0.9,3,0,0,4,3,3,9,1,32.171,0.744,0.114,0.857,327.0,18.1967,0.0 +gy_kernel_8,1.0,0.0 +gy_kernel_8,0.9,0.1 +gy_kernel_8,0.8,0.2 +gy_kernel_8,0.7,0.3 +gy_kernel_8,0.6,0.4 +gy_kernel_8,0.5,0.5 +gy_kernel_8,0.4,0.6 +gy_kernel_8,0.3,0.7 +gy_kernel_8,0.2,0.8 +gy_kernel_8,0.1,0.9 +gy_kernel_16,1.0,0.0 +gy_kernel_16,0.9,0.1 +gy_kernel_16,0.8,0.2 +gy_kernel_16,0.7,0.3 +gy_kernel_16,0.6,0.4 +gy_kernel_16,0.5,0.5 +gy_kernel_16,0.4,0.6 +gy_kernel_16,0.3,0.7 +gy_kernel_16,0.2,0.8 +gy_kernel_16,0.1,0.9 +gy_kernel_32,1.0,0.0 +gy_kernel_32,0.9,0.1 +gy_kernel_32,0.8,0.2 +gy_kernel_32,0.7,0.3 +gy_kernel_32,0.6,0.4 +gy_kernel_32,0.5,0.5 +gy_kernel_32,0.4,0.6 +gy_kernel_32,0.3,0.7 +gy_kernel_32,0.2,0.8 +gy_kernel_32,0.1,0.9 +roberts_cross_4,1.0,0.0,3,0,0,2,3,2,5,1,32.463,0.469,0.113,0.582,325.0,18.1967,173.0 +roberts_cross_4,0.9,0.1,3,0,0,2,3,2,5,1,32.358,0.470,0.113,0.584,325.0,18.1967,0.0 +roberts_cross_4,0.8,0.2,3,0,0,2,3,2,5,1,32.245,0.471,0.113,0.584,325.5,18.1967,0.0 +roberts_cross_4,0.7,0.3,3,0,0,2,3,2,5,1,32.257,0.471,0.114,0.585,325.0,18.1967,0.0 +roberts_cross_4,0.6,0.4,3,0,0,4,3,2,6,1,32.144,0.551,0.113,0.664,324.5,18.1967,0.0 +roberts_cross_4,0.5,0.5,3,0,0,8,3,2,8,1,32.324,0.715,0.112,0.827,323.5,18.1967,0.0 +roberts_cross_4,0.4,0.6,3,0,0,8,3,2,8,1,32.185,0.715,0.113,0.828,324.0,18.1967,0.0 +roberts_cross_4,0.3,0.7,3,0,0,8,3,2,8,1,32.270,0.717,0.113,0.830,324.0,18.1967,0.0 +roberts_cross_4,0.2,0.8,3,0,0,8,3,2,8,1,32.262,0.715,0.113,0.828,323.5,18.1967,0.0 +roberts_cross_4,0.1,0.9,3,0,0,8,3,2,8,1,32.315,0.714,0.113,0.827,324.0,18.1967,0.0 +roberts_cross_8,1.0,0.0 +roberts_cross_8,0.9,0.1 +roberts_cross_8,0.8,0.2 +roberts_cross_8,0.7,0.3 +roberts_cross_8,0.6,0.4 +roberts_cross_8,0.5,0.5 +roberts_cross_8,0.4,0.6 +roberts_cross_8,0.3,0.7 +roberts_cross_8,0.2,0.8 +roberts_cross_8,0.1,0.9 +roberts_cross_16,1.0,0.0 +roberts_cross_16,0.9,0.1 +roberts_cross_16,0.8,0.2 +roberts_cross_16,0.7,0.3 +roberts_cross_16,0.6,0.4 +roberts_cross_16,0.5,0.5 +roberts_cross_16,0.4,0.6 +roberts_cross_16,0.3,0.7 +roberts_cross_16,0.2,0.8 +roberts_cross_16,0.1,0.9 +roberts_cross_32,1.0,0.0 +roberts_cross_32,0.9,0.1 +roberts_cross_32,0.8,0.2 +roberts_cross_32,0.7,0.3 +roberts_cross_32,0.6,0.4 +roberts_cross_32,0.5,0.5 +roberts_cross_32,0.4,0.6 +roberts_cross_32,0.3,0.7 +roberts_cross_32,0.2,0.8 +roberts_cross_32,0.1,0.9 +matrix_mul_4,1.0,0.0,2,0,0,1,0,2,4,1,32.413,0.429,0.113,0.542,327.0,18.1967,0.0 +matrix_mul_4,0.9,0.1,2,0,0,1,0,2,4,1,31.909,0.427,0.113,0.541,327.0,18.1967,0.0 +matrix_mul_4,0.8,0.2,2,0,0,1,0,2,4,1,32.079,0.429,0.114,0.543,328.0,18.1967,0.0 +matrix_mul_4,0.7,0.3,2,0,0,1,0,2,4,1,31.942,0.426,0.113,0.539,327.5,18.1967,0.0 +matrix_mul_4,0.6,0.4,2,0,0,2,0,2,5,1,32.333,0.469,0.112,0.581,327.5,18.1967,0.0 +matrix_mul_4,0.5,0.5,2,0,0,2,0,2,5,1,32.231,0.468,0.112,0.580,328.0,18.1967,0.0 +matrix_mul_4,0.4,0.6,2,0,0,2,0,2,5,1,32.210,0.467,0.112,0.580,327.5,18.1967,0.0 +matrix_mul_4,0.3,0.7,2,0,0,2,0,2,5,1,32.139,0.468,0.113,0.581,327.0,18.1967,0.0 +matrix_mul_4,0.2,0.8,2,0,0,2,0,2,5,1,32.594,0.470,0.113,0.583,327.5,18.1967,0.0 +matrix_mul_4,0.1,0.9,2,0,0,4,0,2,7,1,32.030,0.550,0.113,0.663,327.0,18.1967,0.0 +matrix_mul_8,1.0,0.0 +matrix_mul_8,0.9,0.1 +matrix_mul_8,0.8,0.2 +matrix_mul_8,0.7,0.3 +matrix_mul_8,0.6,0.4 +matrix_mul_8,0.5,0.5 +matrix_mul_8,0.4,0.6 +matrix_mul_8,0.3,0.7 +matrix_mul_8,0.2,0.8 +matrix_mul_8,0.1,0.9 +matrix_mul_16,1.0,0.0 +matrix_mul_16,0.9,0.1 +matrix_mul_16,0.8,0.2 +matrix_mul_16,0.7,0.3 +matrix_mul_16,0.6,0.4 +matrix_mul_16,0.5,0.5 +matrix_mul_16,0.4,0.6 +matrix_mul_16,0.3,0.7 +matrix_mul_16,0.2,0.8 +matrix_mul_16,0.1,0.9 +matrix_mul_32,1.0,0.0 +matrix_mul_32,0.9,0.1 +matrix_mul_32,0.8,0.2 +matrix_mul_32,0.7,0.3 +matrix_mul_32,0.6,0.4 +matrix_mul_32,0.5,0.5 +matrix_mul_32,0.4,0.6 +matrix_mul_32,0.3,0.7 +matrix_mul_32,0.2,0.8 +matrix_mul_32,0.1,0.9 +lin_reg_4,1.0,0.0,2,0,0,2,0,1,5,1,33.548,0.279,0.223,0.501,327.0,36.2679,0.0 +lin_reg_4,0.9,0.1,2,0,0,2,0,1,5,1,31.845,0.279,0.223,0.502,327.0,36.2679,0.0 +lin_reg_4,0.8,0.2,2,0,0,2,0,1,5,1,45.630,0.279,0.223,0.502,327.5,36.2679,0.0 +lin_reg_4,0.7,0.3,2,0,0,2,0,1,5,1,31.764,0.279,0.223,0.502,327.5,36.2679,0.0 +lin_reg_4,0.6,0.4,2,0,0,2,0,1,5,1,31.571,0.279,0.223,0.502,327.0,36.2679,0.0 +lin_reg_4,0.5,0.5,2,0,0,2,0,1,5,1,31.836,0.279,0.223,0.502,327.0,36.2679,0.0 +lin_reg_4,0.4,0.6,2,0,0,2,0,1,5,1,31.383,0.279,0.223,0.502,327.5,36.2679,0.0 +lin_reg_4,0.3,0.7,2,0,0,2,0,1,5,1,31.823,0.279,0.223,0.502,327.0,36.2679,0.0 +lin_reg_4,0.2,0.8,2,0,0,2,0,1,5,1,31.514,0.279,0.223,0.502,327.5,36.2679,0.0 +lin_reg_4,0.1,0.9,2,0,0,3,0,1,6,1,32.014,0.320,0.112,0.432,327.5,18.1967,0.0 +lin_reg_8,1.0,0.0,2,0,0,2,0,1,5,1,31.834,0.280,0.223,0.503,327.0,36.2679,0.0 +lin_reg_8,0.9,0.1,2,0,0,2,0,1,5,1,31.792,0.282,0.224,0.506,327.0,36.2679,0.0 +lin_reg_8,0.8,0.2,2,0,0,2,0,1,5,1,32.078,0.280,0.223,0.503,328.0,36.2679,0.0 +lin_reg_8,0.7,0.3,2,0,0,2,0,1,5,1,31.954,0.281,0.223,0.505,327.5,36.2679,0.0 +lin_reg_8,0.6,0.4,2,0,0,3,0,1,6,1,32.044,0.324,0.114,0.438,327.0,18.1967,0.0 +lin_reg_8,0.5,0.5,2,0,0,3,0,1,6,1,32.149,0.321,0.113,0.435,327.5,18.1967,0.0 +lin_reg_8,0.4,0.6,2,0,0,3,0,1,6,1,31.586,0.320,0.113,0.433,327.0,18.1967,0.0 +lin_reg_8,0.3,0.7,2,0,0,3,0,1,6,1,32.043,0.320,0.112,0.433,328.0,18.1967,0.0 +lin_reg_8,0.2,0.8,2,0,0,3,0,1,6,1,31.984,0.319,0.112,0.432,327.0,18.1967,0.0 +lin_reg_8,0.1,0.9,2,0,0,5,0,1,8,1,31.901,0.400,0.112,0.512,327.5,18.1967,0.0 +lin_reg_16,1.0,0.0,2,0,0,1,0,1,4,1,31.490,0.239,0.112,0.351,328.0,18.1967,0.0 +lin_reg_16,0.9,0.1,2,0,0,1,0,1,4,1,31.867,0.239,0.112,0.351,328.0,18.1967,0.0 +lin_reg_16,0.8,0.2,2,0,0,1,0,1,4,1,31.592,0.239,0.112,0.351,328.0,18.1967,0.0 +lin_reg_16,0.7,0.3,2,0,0,1,0,1,4,1,31.951,0.239,0.112,0.351,328.0,18.1967,0.0 +lin_reg_16,0.6,0.4,2,0,0,1,0,1,4,1,32.002,0.239,0.112,0.351,328.0,18.1967,0.0 +lin_reg_16,0.5,0.5,2,0,0,1,0,1,4,1,31.897,0.242,0.113,0.354,328.0,18.1967,0.0 +lin_reg_16,0.4,0.6,2,0,0,2,0,1,5,1,32.089,0.282,0.113,0.395,328.0,18.1967,0.0 +lin_reg_16,0.3,0.7,2,0,0,2,0,1,5,1,32.164,0.282,0.113,0.395,328.0,18.1967,0.0 +lin_reg_16,0.2,0.8,2,0,0,2,0,1,5,1,32.292,0.282,0.113,0.395,328.0,18.1967,0.0 +lin_reg_16,0.1,0.9,2,0,0,2,0,1,5,1,32.018,0.282,0.113,0.395,328.0,18.1967,0.0 +lin_reg_32,1.0,0.0,2,0,1,0,0,1,3,1,32.349,0.054,0.000,0.054,358.5,0.125542,0.0 +lin_reg_32,0.9,0.1,2,0,1,0,0,1,3,1,32.233,0.054,0.000,0.054,359.0,0.125542,0.0 +lin_reg_32,0.8,0.2,2,0,1,0,0,1,3,1,31.910,0.054,0.000,0.054,358.5,0.125542,0.0 +lin_reg_32,0.7,0.3,2,0,1,0,0,1,3,1,32.214,0.054,0.000,0.054,358.5,0.125542,0.0 +lin_reg_32,0.6,0.4,2,0,1,0,0,1,3,1,31.996,0.054,0.000,0.054,359.0,0.125542,0.0 +lin_reg_32,0.5,0.5,2,0,1,0,0,1,3,1,32.329,0.054,0.000,0.054,359.0,0.125542,0.0 +lin_reg_32,0.4,0.6,2,0,1,0,0,1,3,1,32.029,0.053,0.000,0.053,359.0,0.125542,0.0 +lin_reg_32,0.3,0.7,2,0,1,0,0,1,3,1,32.210,0.054,0.000,0.054,359.0,0.125542,0.0 +lin_reg_32,0.2,0.8,2,0,1,0,0,1,3,1,32.008,0.054,0.000,0.054,359.0,0.125542,0.0 +lin_reg_32,0.1,0.9,2,0,1,0,0,1,3,1,32.022,0.053,0.000,0.053,359.0,0.125542,0.0 +hamming_dist_4,1.0,0.0,3,1,0,4,0,2,8,2,32.351,0.530,0.336,0.866,295.0,54.3391,0.0 +hamming_dist_4,0.9,0.1,3,1,0,4,0,2,8,2,32.410,0.530,0.336,0.866,295.0,54.3391,0.0 +hamming_dist_4,0.8,0.2,3,1,0,4,0,2,8,2,32.392,0.530,0.336,0.866,295.0,54.3391,0.0 +hamming_dist_4,0.7,0.3,3,1,0,5,0,2,9,2,32.300,0.569,0.224,0.793,295.0,36.2679,0.0 +hamming_dist_4,0.6,0.4,3,1,0,10,0,2,11,2,32.580,0.771,0.113,0.884,295.0,18.1967,0.0 +hamming_dist_4,0.5,0.5,3,1,0,10,0,2,11,2,32.433,0.772,0.113,0.885,294.5,18.1967,0.0 +hamming_dist_4,0.4,0.6,3,1,0,10,0,2,11,2,32.592,0.763,0.112,0.875,295.0,18.1967,0.0 +hamming_dist_4,0.3,0.7,3,1,0,10,0,2,11,2,32.545,0.771,0.113,0.884,295.0,18.1967,0.0 +hamming_dist_4,0.2,0.8,3,1,0,10,0,2,11,2,32.779,0.771,0.113,0.884,295.0,18.1967,0.0 +hamming_dist_4,0.1,0.9,3,1,0,10,0,2,11,2,32.764,0.768,0.112,0.881,295.0,18.1967,0.0 +hamming_dist_8,1.0,0.0,4,1,0,5,0,2,10,2,33.007,0.572,0.446,1.018,294.5,72.4103,0.0 +hamming_dist_8,0.9,0.1,4,1,0,5,0,2,10,2,32.807,0.572,0.446,1.018,294.0,72.4103,0.0 +hamming_dist_8,0.8,0.2,4,1,0,8,0,2,12,2,33.493,0.695,0.224,0.918,294.0,36.2679,0.0 +hamming_dist_8,0.7,0.3,4,1,0,8,0,2,12,2,33.535,0.687,0.223,0.910,294.0,36.2679,0.0 +hamming_dist_8,0.6,0.4,4,1,0,19,0,2,17,2,34.587,1.142,0.113,1.254,294.0,18.1967,0.0 +hamming_dist_8,0.5,0.5,4,1,0,19,0,2,17,2,34.652,1.129,0.112,1.241,294.0,18.1967,0.0 +hamming_dist_8,0.4,0.6,4,1,0,19,0,2,17,2,34.557,1.140,0.113,1.253,294.0,18.1967,0.0 +hamming_dist_8,0.3,0.7,4,1,0,19,0,2,17,2,34.483,1.141,0.113,1.254,294.0,18.1967,0.0 +hamming_dist_8,0.2,0.8,4,1,0,19,0,2,17,2,34.506,1.141,0.113,1.253,294.0,18.1967,0.0 +hamming_dist_8,0.1,0.9,4,1,0,19,0,2,17,2,36.158,1.141,0.113,1.254,294.0,18.1967,0.0 +hamming_dist_16,1.0,0.0,5,1,0,8,0,2,13,2,38.108,0.698,0.447,1.144,294.0,72.4103,0.0 +hamming_dist_16,0.9,0.1,5,1,0,8,0,2,13,2,38.008,0.698,0.447,1.145,293.5,72.4103,0.0 +hamming_dist_16,0.8,0.2,5,1,0,12,0,2,15,2,44.766,0.859,0.224,1.082,293.0,36.2679,0.0 +hamming_dist_16,0.7,0.3,5,1,0,12,0,2,15,2,44.736,0.859,0.224,1.082,293.0,36.2679,0.0 +hamming_dist_16,0.6,0.4,5,1,0,36,0,2,27,2,58.799,1.833,0.112,1.945,293.0,18.1967,0.0 +hamming_dist_16,0.5,0.5,5,1,0,36,0,2,27,2,58.578,1.834,0.112,1.946,293.0,18.1967,0.0 +hamming_dist_16,0.4,0.6,5,1,0,36,0,2,27,2,58.415,1.833,0.112,1.946,293.0,18.1967,0.0 +hamming_dist_16,0.3,0.7,5,1,0,36,0,2,27,2,58.545,1.834,0.112,1.946,293.0,18.1967,0.0 +hamming_dist_16,0.2,0.8,5,1,0,36,0,2,27,2,58.380,1.833,0.112,1.946,293.0,18.1967,0.0 +hamming_dist_16,0.1,0.9,5,1,0,36,0,2,27,2,58.316,1.833,0.112,1.945,293.0,18.1967,0.0 +hamming_dist_32,1.0,0.0,7,1,0,6,0,3,15,2,47.005,0.829,0.447,1.276,293.0,72.4103,0.0 +hamming_dist_32,0.9,0.1,7,1,0,6,0,3,15,2,47.067,0.825,0.446,1.271,293.0,72.4103,0.0 +hamming_dist_32,0.8,0.2,7,1,0,7,0,3,16,2,55.422,0.869,0.336,1.204,293.0,54.3391,0.0 +hamming_dist_32,0.7,0.3,7,1,0,8,0,3,17,2,64.819,0.911,0.223,1.135,292.5,36.2679,0.0 +hamming_dist_32,0.6,0.4,7,1,0,8,0,3,17,2,63.274,0.912,0.223,1.135,292.5,36.2679,0.0 +hamming_dist_32,0.5,0.5,7,1,0,8,0,3,17,2,64.692,0.910,0.223,1.134,292.5,36.2679,0.0 +hamming_dist_32,0.4,0.6,7,1,0,8,0,3,17,2,63.386,0.905,0.222,1.128,292.0,36.2679,0.0 +hamming_dist_32,0.3,0.7,7,1,0,8,0,3,17,2,64.173,0.911,0.223,1.134,292.5,36.2679,0.0 +hamming_dist_32,0.2,0.8,7,1,0,20,0,3,29,2,84.719,1.390,0.113,1.503,292.0,18.1967,0.0 +hamming_dist_32,0.1,0.9,7,1,0,20,0,3,29,2,84.792,1.395,0.113,1.507,292.0,18.1967,0.0 +max_3,1.0,0.0,4,1,0,3,1,3,9,2,28.692,0.559,0.276,0.835,293.0,54.3391,0.0 +max_3,0.9,0.1,4,1,0,3,1,3,9,2,26.595,0.558,0.276,0.835,293.0,54.3391,0.0 +max_3,0.8,0.2,4,1,0,3,1,3,9,2,26.849,0.558,0.276,0.834,292.5,54.3391,0.0 +max_3,0.7,0.3,4,1,0,3,1,3,9,2,34.024,0.559,0.276,0.835,293.0,54.3391,0.0 +max_3,0.6,0.4,4,1,0,4,1,3,10,2,26.832,0.591,0.184,0.776,293.0,36.2679,0.0 +max_3,0.5,0.5,4,1,0,4,1,3,10,2,26.800,0.591,0.184,0.776,293.0,36.2679,0.0 +max_3,0.4,0.6,4,1,0,6,1,3,11,2,26.566,0.657,0.093,0.749,292.0,18.1967,0.0 +max_3,0.3,0.7,4,1,0,6,1,3,11,2,26.902,0.657,0.093,0.749,292.0,18.1967,0.0 +max_3,0.2,0.8,4,1,0,6,1,3,11,2,26.487,0.656,0.093,0.749,291.5,18.1967,0.0 +max_3,0.1,0.9,4,1,0,6,1,3,11,2,26.836,0.657,0.093,0.749,292.0,18.1967,0.0 +max_4,1.0,0.0,8,1,0,3,3,7,11,3,26.539,1.133,0.276,1.409,262.0,54.3391,0.0 +max_4,0.9,0.1,8,1,0,3,3,7,11,3,27.066,1.138,0.278,1.415,262.0,54.3391,0.0 +max_4,0.8,0.2,8,1,0,3,3,7,11,3,26.817,1.144,0.279,1.423,262.0,54.3391,0.0 +max_4,0.7,0.3,8,1,0,6,3,7,14,3,27.571,1.240,0.093,1.334,262.0,18.1967,0.0 +max_4,0.6,0.4,8,1,0,6,3,7,14,3,27.511,1.240,0.093,1.333,261.5,18.1967,0.0 +max_4,0.5,0.5,8,1,0,6,3,7,14,3,27.656,1.241,0.093,1.334,262.0,18.1967,0.0 +max_4,0.4,0.6,8,1,0,6,3,7,14,3,27.526,1.245,0.094,1.338,262.0,18.1967,0.0 +max_4,0.3,0.7,8,1,0,6,3,7,14,3,27.687,1.242,0.093,1.336,262.0,18.1967,0.0 +max_4,0.2,0.8,8,1,0,6,3,7,14,3,27.432,1.244,0.093,1.337,262.0,18.1967,0.0 +max_4,0.1,0.9,8,1,0,6,3,7,14,3,27.608,1.239,0.093,1.332,262.0,18.1967,0.0 +max_5,1.0,0.0,12,2,0,5,4,12,15,4,36.062,1.912,0.369,2.280,228.5,72.4103,0.0 +max_5,0.9,0.1,12,2,0,5,4,12,15,4,36.927,1.933,0.371,2.303,229.0,72.4103,0.0 +max_5,0.8,0.2,12,2,0,8,4,12,17,4,56.950,2.046,0.186,2.232,228.5,36.2679,0.0 +max_5,0.7,0.3,12,2,0,8,4,12,17,4,56.933,2.045,0.188,2.233,229.0,36.2679,0.0 +max_5,0.6,0.4,12,2,0,8,4,12,17,4,56.871,2.039,0.185,2.224,229.0,36.2679,0.0 +max_5,0.5,0.5,12,2,0,7,4,12,16,4,56.992,2.004,0.185,2.190,228.0,36.2679,0.0 +max_5,0.4,0.6,12,2,0,7,4,12,16,4,56.945,2.011,0.186,2.197,229.0,36.2679,0.0 +max_5,0.3,0.7,12,2,0,19,4,12,22,4,79.688,2.414,0.094,2.508,228.0,18.1967,0.0 +max_5,0.2,0.8,12,2,0,19,4,12,22,4,79.492,2.410,0.093,2.503,229.0,18.1967,0.0 +max_5,0.1,0.9,12,2,0,19,4,12,22,4,78.784,2.410,0.094,2.504,228.5,18.1967,0.0 +sort_3,1.0,0.0 +sort_3,0.9,0.1 +sort_3,0.8,0.2 +sort_3,0.7,0.3 +sort_3,0.6,0.4 +sort_3,0.5,0.5 +sort_3,0.4,0.6 +sort_3,0.3,0.7 +sort_3,0.2,0.8 +sort_3,0.1,0.9 +box_blur_4,1.0,0.0,5,0,0,1,0,0,5,0,27.797,0.139,0.095,0.233,357.0,18.1967,0.0 +box_blur_4,0.9,0.1,5,0,0,1,0,0,5,0,26.682,0.137,0.093,0.230,357.0,18.1967,0.0 +box_blur_4,0.8,0.2,5,0,0,1,0,0,5,0,27.564,0.138,0.094,0.232,357.0,18.1967,0.0 +box_blur_4,0.7,0.3,5,0,0,1,0,0,5,0,26.570,0.137,0.094,0.231,357.0,18.1967,0.0 +box_blur_4,0.6,0.4,5,0,0,2,0,0,6,0,27.636,0.171,0.093,0.264,357.0,18.1967,0.0 +box_blur_4,0.5,0.5,5,0,0,2,0,0,6,0,26.550,0.171,0.093,0.264,356.5,18.1967,0.0 +box_blur_4,0.4,0.6,5,0,0,4,0,0,8,0,27.486,0.241,0.094,0.335,356.0,18.1967,0.0 +box_blur_4,0.3,0.7,5,0,0,4,0,0,8,0,26.651,0.241,0.095,0.336,355.5,18.1967,0.0 +box_blur_4,0.2,0.8,5,0,0,4,0,0,8,0,27.753,0.240,0.093,0.333,356.0,18.1967,0.0 +box_blur_4,0.1,0.9,5,0,0,4,0,0,8,0,26.599,0.239,0.093,0.333,356.0,18.1967,0.0 +box_blur_8,1.0,0.0 +box_blur_8,0.9,0.1 +box_blur_8,0.8,0.2 +box_blur_8,0.7,0.3 +box_blur_8,0.6,0.4 +box_blur_8,0.5,0.5 +box_blur_8,0.4,0.6 +box_blur_8,0.3,0.7 +box_blur_8,0.2,0.8 +box_blur_8,0.1,0.9 +box_blur_16,1.0,0.0 +box_blur_16,0.9,0.1 +box_blur_16,0.8,0.2 +box_blur_16,0.7,0.3 +box_blur_16,0.6,0.4 +box_blur_16,0.5,0.5 +box_blur_16,0.4,0.6 +box_blur_16,0.3,0.7 +box_blur_16,0.2,0.8 +box_blur_16,0.1,0.9 +box_blur_32,1.0,0.0 +box_blur_32,0.9,0.1 +box_blur_32,0.8,0.2 +box_blur_32,0.7,0.3 +box_blur_32,0.6,0.4 +box_blur_32,0.5,0.5 +box_blur_32,0.4,0.6 +box_blur_32,0.3,0.7 +box_blur_32,0.2,0.8 +box_blur_32,0.1,0.9 +dot_product_4,1.0,0.0,2,0,0,3,0,1,6,1,34.093,0.296,0.334,0.629,325.0,54.3391,0.0 +dot_product_4,0.9,0.1,2,0,0,3,0,1,6,1,31.935,0.296,0.333,0.629,325.0,54.3391,0.0 +dot_product_4,0.8,0.2,2,0,0,3,0,1,6,1,31.617,0.296,0.333,0.629,325.0,54.3391,0.0 +dot_product_4,0.7,0.3,2,0,0,3,0,1,6,1,31.911,0.296,0.333,0.629,324.5,54.3391,0.0 +dot_product_4,0.6,0.4,2,0,0,3,0,1,6,1,31.452,0.296,0.333,0.629,325.0,54.3391,0.0 +dot_product_4,0.5,0.5,2,0,0,4,0,1,7,1,31.870,0.335,0.223,0.558,325.0,36.2679,0.0 +dot_product_4,0.4,0.6,2,0,0,6,0,1,9,1,31.958,0.417,0.112,0.529,323.5,18.1967,0.0 +dot_product_4,0.3,0.7,2,0,0,6,0,1,9,1,31.974,0.417,0.112,0.530,323.5,18.1967,0.0 +dot_product_4,0.2,0.8,2,0,0,6,0,1,9,1,32.102,0.418,0.112,0.531,323.0,18.1967,0.0 +dot_product_4,0.1,0.9,2,0,0,6,0,1,9,1,32.299,0.418,0.113,0.530,324.0,18.1967,0.0 +dot_product_8,1.0,0.0,3,0,0,4,0,1,8,1,31.989,0.339,0.444,0.784,324.0,72.4103,0.0 +dot_product_8,0.9,0.1,3,0,0,4,0,1,8,1,32.189,0.339,0.445,0.784,324.0,72.4103,0.0 +dot_product_8,0.8,0.2,3,0,0,4,0,1,8,1,31.826,0.339,0.445,0.783,324.0,72.4103,0.0 +dot_product_8,0.7,0.3,3,0,0,5,0,1,9,1,32.053,0.379,0.334,0.713,323.5,54.3391,0.0 +dot_product_8,0.6,0.4,3,0,0,6,0,1,10,1,32.451,0.419,0.223,0.642,323.5,36.2679,0.0 +dot_product_8,0.5,0.5,3,0,0,6,0,1,10,1,32.271,0.418,0.223,0.640,323.5,36.2679,0.0 +dot_product_8,0.4,0.6,3,0,0,11,0,1,15,1,32.449,0.622,0.112,0.734,322.0,18.1967,0.0 +dot_product_8,0.3,0.7,3,0,0,11,0,1,15,1,32.419,0.619,0.112,0.731,322.0,18.1967,0.0 +dot_product_8,0.2,0.8,3,0,0,11,0,1,15,1,32.302,0.619,0.112,0.731,322.0,18.1967,0.0 +dot_product_8,0.1,0.9,3,0,0,11,0,1,15,1,32.401,0.619,0.112,0.731,322.0,18.1967,0.0 +dot_product_16,1.0,0.0,4,0,0,6,0,1,11,1,33.192,0.420,0.445,0.865,323.0,72.4103,0.0 +dot_product_16,0.9,0.1,4,0,0,6,0,1,11,1,32.919,0.420,0.444,0.864,323.0,72.4103,0.0 +dot_product_16,0.8,0.2,4,0,0,6,0,1,11,1,32.874,0.420,0.444,0.864,322.5,72.4103,0.0 +dot_product_16,0.7,0.3,4,0,0,9,0,1,14,1,33.993,0.540,0.223,0.763,322.5,36.2679,0.0 +dot_product_16,0.6,0.4,4,0,0,8,0,1,13,1,33.952,0.500,0.223,0.723,321.5,36.2679,0.0 +dot_product_16,0.5,0.5,4,0,0,8,0,1,13,1,33.994,0.500,0.223,0.723,322.0,36.2679,0.0 +dot_product_16,0.4,0.6,4,0,0,20,0,1,25,1,36.160,0.982,0.112,1.094,320.5,18.1967,0.0 +dot_product_16,0.3,0.7,4,0,0,20,0,1,25,1,36.126,0.982,0.112,1.094,320.5,18.1967,0.0 +dot_product_16,0.2,0.8,4,0,0,20,0,1,25,1,36.095,0.983,0.112,1.095,320.0,18.1967,0.0 +dot_product_16,0.1,0.9,4,0,0,20,0,1,25,1,36.060,0.981,0.112,1.093,320.0,18.1967,0.0 +dot_product_32,1.0,0.0,5,0,0,6,0,1,12,1,34.645,0.444,0.445,0.889,326.0,72.4103,0.0 +dot_product_32,0.9,0.1,5,0,0,6,0,1,12,1,34.643,0.443,0.445,0.888,325.5,72.4103,0.0 +dot_product_32,0.8,0.2,5,0,0,7,0,1,13,1,36.076,0.483,0.333,0.817,325.5,54.3391,0.0 +dot_product_32,0.7,0.3,5,0,0,8,0,1,14,1,37.794,0.524,0.223,0.747,325.0,36.2679,0.0 +dot_product_32,0.6,0.4,5,0,0,8,0,1,14,1,37.396,0.524,0.223,0.747,325.5,36.2679,0.0 +dot_product_32,0.5,0.5,5,0,0,8,0,1,14,1,37.424,0.524,0.223,0.747,325.5,36.2679,0.0 +dot_product_32,0.4,0.6,5,0,0,8,0,1,14,1,37.606,0.524,0.223,0.747,325.0,36.2679,0.0 +dot_product_32,0.3,0.7,5,0,0,20,0,1,26,1,42.467,1.003,0.113,1.116,325.0,18.1967,0.0 +dot_product_32,0.2,0.8,5,0,0,20,0,1,26,1,42.368,1.003,0.113,1.115,325.0,18.1967,0.0 +dot_product_32,0.1,0.9,5,0,0,20,0,1,26,1,42.201,1.003,0.113,1.115,325.0,18.1967,0.0 +poly_reg_4,1.0,0.0,3,0,0,3,0,2,8,2,32.229,0.483,0.335,0.818,294.5,54.3391,0.0 +poly_reg_4,0.9,0.1,3,0,0,3,0,2,8,2,32.121,0.484,0.335,0.818,295.0,54.3391,0.0 +poly_reg_4,0.8,0.2,3,0,0,3,0,2,8,2,32.148,0.483,0.335,0.818,295.0,54.3391,0.0 +poly_reg_4,0.7,0.3,3,0,0,4,0,2,9,2,32.479,0.525,0.222,0.747,295.0,36.2679,0.0 +poly_reg_4,0.6,0.4,3,0,0,4,0,2,9,2,32.476,0.525,0.222,0.747,295.0,36.2679,0.0 +poly_reg_4,0.5,0.5,3,0,0,4,0,2,9,2,32.370,0.525,0.222,0.747,295.0,36.2679,0.0 +poly_reg_4,0.4,0.6,3,0,0,4,0,2,9,2,32.592,0.525,0.222,0.747,295.0,36.2679,0.0 +poly_reg_4,0.3,0.7,3,0,0,4,0,2,9,2,32.691,0.525,0.222,0.747,295.0,36.2679,0.0 +poly_reg_4,0.2,0.8,3,0,0,4,0,2,9,2,32.096,0.525,0.222,0.747,295.0,36.2679,0.0 +poly_reg_4,0.1,0.9,3,0,0,6,0,2,11,2,32.788,0.606,0.112,0.718,295.0,18.1967,0.0 +poly_reg_8,1.0,0.0,3,0,0,2,0,2,7,2,32.192,0.465,0.223,0.688,295.0,36.2679,0.0 +poly_reg_8,0.9,0.1,3,0,0,2,0,2,7,2,31.786,0.466,0.223,0.688,295.0,36.2679,0.0 +poly_reg_8,0.8,0.2,3,0,0,2,0,2,7,2,32.073,0.466,0.223,0.689,295.5,36.2679,0.0 +poly_reg_8,0.7,0.3,3,0,0,2,0,2,7,2,31.883,0.465,0.223,0.688,295.5,36.2679,0.0 +poly_reg_8,0.6,0.4,3,0,0,2,0,2,7,2,32.401,0.466,0.223,0.688,295.5,36.2679,0.0 +poly_reg_8,0.5,0.5,3,0,0,3,0,2,8,2,32.227,0.506,0.112,0.619,295.5,18.1967,0.0 +poly_reg_8,0.4,0.6,3,0,0,3,0,2,8,2,31.940,0.506,0.112,0.619,295.0,18.1967,0.0 +poly_reg_8,0.3,0.7,3,0,0,3,0,2,8,2,32.083,0.506,0.112,0.619,295.0,18.1967,0.0 +poly_reg_8,0.2,0.8,3,0,0,3,0,2,8,2,32.367,0.506,0.112,0.619,295.5,18.1967,0.0 +poly_reg_8,0.1,0.9,3,0,0,3,0,2,8,2,32.305,0.506,0.112,0.619,295.5,18.1967,0.0 +poly_reg_16,1.0,0.0,3,0,0,1,0,2,6,2,32.094,0.425,0.112,0.537,295.0,18.1967,0.0 +poly_reg_16,0.9,0.1,3,0,0,1,0,2,6,2,32.129,0.425,0.112,0.537,295.0,18.1967,0.0 +poly_reg_16,0.8,0.2,3,0,0,1,0,2,6,2,31.860,0.425,0.112,0.537,295.0,18.1967,0.0 +poly_reg_16,0.7,0.3,3,0,0,1,0,2,6,2,32.084,0.425,0.112,0.537,295.0,18.1967,0.0 +poly_reg_16,0.6,0.4,3,0,0,1,0,2,6,2,31.897,0.425,0.112,0.537,295.0,18.1967,0.0 +poly_reg_16,0.5,0.5,3,0,0,1,0,2,6,2,32.093,0.425,0.112,0.537,295.0,18.1967,0.0 +poly_reg_16,0.4,0.6,3,0,0,1,0,2,6,2,31.812,0.425,0.112,0.537,295.5,18.1967,0.0 +poly_reg_16,0.3,0.7,3,0,0,1,0,2,6,2,32.084,0.425,0.112,0.537,295.5,18.1967,0.0 +poly_reg_16,0.2,0.8,3,0,0,2,0,2,7,2,31.801,0.465,0.112,0.577,295.5,18.1967,0.0 +poly_reg_16,0.1,0.9,3,0,0,2,0,2,7,2,32.043,0.465,0.112,0.577,296.0,18.1967,0.0 +poly_reg_32,1.0,0.0,3,0,0,0,0,2,5,2,32.048,0.407,0.000,0.407,296.0,0.125542,0.0 +poly_reg_32,0.9,0.1,3,0,0,0,0,2,5,2,32.110,0.407,0.000,0.407,296.0,0.125542,0.0 +poly_reg_32,0.8,0.2,3,0,0,0,0,2,5,2,31.924,0.407,0.000,0.407,296.0,0.125542,0.0 +poly_reg_32,0.7,0.3,3,0,0,0,0,2,5,2,32.162,0.407,0.000,0.407,296.0,0.125542,0.0 +poly_reg_32,0.6,0.4,3,0,0,0,0,2,5,2,32.102,0.407,0.000,0.407,296.0,0.125542,0.0 +poly_reg_32,0.5,0.5,3,0,0,0,0,2,5,2,32.093,0.407,0.000,0.407,296.0,0.125542,0.0 +poly_reg_32,0.4,0.6,3,0,0,0,0,2,5,2,32.078,0.407,0.000,0.407,296.0,0.125542,0.0 +poly_reg_32,0.3,0.7,3,0,0,0,0,2,5,2,32.110,0.407,0.000,0.407,296.0,0.125542,0.0 +poly_reg_32,0.2,0.8,3,0,0,0,0,2,5,2,32.272,0.407,0.000,0.407,296.0,0.125542,0.0 +poly_reg_32,0.1,0.9,3,0,0,0,0,2,5,2,31.975,0.407,0.000,0.407,296.0,0.125542,0.0 +l2_distance_4,1.0,0.0,2,1,0,4,0,1,8,1,31.814,0.338,0.446,0.783,324.0,72.4103,0.0 +l2_distance_4,0.9,0.1,2,1,0,4,0,1,8,1,31.991,0.338,0.446,0.783,323.5,72.4103,0.0 +l2_distance_4,0.8,0.2,2,1,0,4,0,1,8,1,31.998,0.338,0.446,0.783,324.0,72.4103,0.0 +l2_distance_4,0.7,0.3,2,1,0,6,0,1,10,1,32.092,0.418,0.223,0.641,323.5,36.2679,0.0 +l2_distance_4,0.6,0.4,2,1,0,11,0,1,15,1,32.336,0.617,0.113,0.729,322.0,18.1967,0.0 +l2_distance_4,0.5,0.5,2,1,0,11,0,1,15,1,32.562,0.617,0.112,0.729,322.0,18.1967,0.0 +l2_distance_4,0.4,0.6,2,1,0,11,0,1,15,1,32.621,0.617,0.112,0.729,322.0,18.1967,0.0 +l2_distance_4,0.3,0.7,2,1,0,11,0,1,15,1,32.909,0.617,0.112,0.729,322.5,18.1967,0.0 +l2_distance_4,0.2,0.8,2,1,0,11,0,1,15,1,32.773,0.617,0.112,0.729,321.5,18.1967,0.0 +l2_distance_4,0.1,0.9,2,1,0,11,0,1,15,1,32.593,0.617,0.112,0.729,322.0,18.1967,0.0 +l2_distance_8,1.0,0.0,3,1,0,4,0,1,9,1,32.470,0.362,0.443,0.806,323.5,72.4103,0.0 +l2_distance_8,0.9,0.1,3,1,0,4,0,1,9,1,32.401,0.365,0.443,0.808,324.0,72.4103,0.0 +l2_distance_8,0.8,0.2,3,1,0,5,0,1,10,1,32.494,0.403,0.334,0.737,324.0,54.3391,0.0 +l2_distance_8,0.7,0.3,3,1,0,6,0,1,11,1,32.720,0.442,0.223,0.665,323.0,36.2679,0.0 +l2_distance_8,0.6,0.4,3,1,0,11,0,1,16,1,33.000,0.643,0.112,0.755,322.0,18.1967,0.0 +l2_distance_8,0.5,0.5,3,1,0,11,0,1,16,1,33.098,0.643,0.112,0.755,322.0,18.1967,0.0 +l2_distance_8,0.4,0.6,3,1,0,11,0,1,16,1,33.170,0.643,0.112,0.755,322.0,18.1967,0.0 +l2_distance_8,0.3,0.7,3,1,0,11,0,1,16,1,33.141,0.643,0.112,0.755,322.0,18.1967,0.0 +l2_distance_8,0.2,0.8,3,1,0,11,0,1,16,1,33.123,0.643,0.112,0.755,322.0,18.1967,0.0 +l2_distance_8,0.1,0.9,3,1,0,11,0,1,16,1,33.091,0.643,0.112,0.756,321.5,18.1967,0.0 +box_blur_3,1.0,0.0,6,0,0,1,0,0,6,0,33.699,0.188,0.113,0.300,357.0,18.1967,0.0 +box_blur_3,0.9,0.1,6,0,0,1,0,0,6,0,31.900,0.188,0.113,0.300,357.0,18.1967,0.0 +box_blur_3,0.8,0.2,6,0,0,1,0,0,6,0,31.583,0.190,0.113,0.303,357.0,18.1967,0.0 +box_blur_3,0.7,0.3,6,0,0,1,0,0,6,0,32.186,0.189,0.113,0.302,357.0,18.1967,0.0 +box_blur_3,0.6,0.4,6,0,0,1,0,0,6,0,31.873,0.188,0.113,0.301,357.0,18.1967,0.0 +box_blur_3,0.5,0.5,6,0,0,1,0,0,6,0,32.250,0.190,0.114,0.304,357.0,18.1967,0.0 +box_blur_3,0.4,0.6,6,0,0,1,0,0,6,0,32.173,0.188,0.113,0.301,357.0,18.1967,0.0 +box_blur_3,0.3,0.7,6,0,0,1,0,0,6,0,32.117,0.189,0.113,0.302,357.0,18.1967,0.0 +box_blur_3,0.2,0.8,6,0,0,1,0,0,6,0,31.866,0.189,0.113,0.302,357.0,18.1967,0.0 +box_blur_3,0.1,0.9,6,0,0,1,0,0,6,0,32.149,0.188,0.113,0.301,357.0,18.1967,0.0 +box_blur_4,1.0,0.0,5,0,0,1,0,0,5,0,31.528,0.164,0.113,0.277,357.0,18.1967,0.0 +box_blur_4,0.9,0.1,5,0,0,1,0,0,5,0,32.124,0.164,0.113,0.277,357.0,18.1967,0.0 +box_blur_4,0.8,0.2,5,0,0,1,0,0,5,0,31.913,0.166,0.115,0.281,357.0,18.1967,0.0 +box_blur_4,0.7,0.3,5,0,0,1,0,0,5,0,32.235,0.165,0.114,0.278,357.0,18.1967,0.0 +box_blur_4,0.6,0.4,5,0,0,2,0,0,6,0,31.920,0.203,0.112,0.315,356.0,18.1967,0.0 +box_blur_4,0.5,0.5,5,0,0,2,0,0,6,0,32.048,0.203,0.112,0.315,357.0,18.1967,0.0 +box_blur_4,0.4,0.6,5,0,0,4,0,0,8,0,31.715,0.284,0.112,0.396,356.0,18.1967,0.0 +box_blur_4,0.3,0.7,5,0,0,4,0,0,8,0,32.200,0.284,0.112,0.396,356.0,18.1967,0.0 +box_blur_4,0.2,0.8,5,0,0,4,0,0,8,0,32.200,0.288,0.113,0.401,356.0,18.1967,0.0 +box_blur_4,0.1,0.9,5,0,0,4,0,0,8,0,32.424,0.288,0.113,0.401,356.0,18.1967,0.0 +box_blur_5,1.0,0.0,8,0,0,0,0,0,4,0,32.339,0.221,0.000,0.221,359.0,0.125542,0.0 +box_blur_5,0.9,0.1,8,0,0,0,0,0,4,0,32.564,0.221,0.000,0.221,359.5,0.125542,0.0 +box_blur_5,0.8,0.2,8,0,0,0,0,0,4,0,32.103,0.222,0.000,0.222,359.0,0.125542,0.0 +box_blur_5,0.7,0.3,8,0,0,0,0,0,4,0,32.522,0.222,0.000,0.222,359.0,0.125542,0.0 +box_blur_5,0.6,0.4,8,0,0,0,0,0,4,0,32.312,0.222,0.000,0.222,359.0,0.125542,0.0 +box_blur_5,0.5,0.5,8,0,0,0,0,0,4,0,32.279,0.222,0.000,0.222,359.0,0.125542,0.0 +box_blur_5,0.4,0.6,8,0,0,0,0,0,4,0,32.159,0.221,0.000,0.221,359.0,0.125542,0.0 +box_blur_5,0.3,0.7,8,0,0,0,0,0,4,0,32.483,0.219,0.000,0.219,359.0,0.125542,0.0 +box_blur_5,0.2,0.8,8,0,0,0,0,0,4,0,31.989,0.222,0.000,0.222,359.0,0.125542,0.0 +box_blur_5,0.1,0.9,8,0,0,0,0,0,4,0,32.603,0.222,0.000,0.222,359.0,0.125542,0.0 +box_blur_8,1.0,0.0 +box_blur_8,0.9,0.1 +box_blur_8,0.8,0.2 +box_blur_8,0.7,0.3 +box_blur_8,0.6,0.4 +box_blur_8,0.5,0.5 +box_blur_8,0.4,0.6 +box_blur_8,0.3,0.7 +box_blur_8,0.2,0.8 +box_blur_8,0.1,0.9 +box_blur_16,1.0,0.0 +box_blur_16,0.9,0.1 +box_blur_16,0.8,0.2 +box_blur_16,0.7,0.3 +box_blur_16,0.6,0.4 +box_blur_16,0.5,0.5 +box_blur_16,0.4,0.6 +box_blur_16,0.3,0.7 +box_blur_16,0.2,0.8 +box_blur_16,0.1,0.9 +box_blur_32,1.0,0.0 +box_blur_32,0.9,0.1 +box_blur_32,0.8,0.2 +box_blur_32,0.7,0.3 +box_blur_32,0.6,0.4 +box_blur_32,0.5,0.5 +box_blur_32,0.4,0.6 +box_blur_32,0.3,0.7 +box_blur_32,0.2,0.8 +box_blur_32,0.1,0.9 +tree_50-50_5_1,1.0,0.0,5,0,0,2,0,2,7,2,32.282,0.521,0.223,0.745,296.0,36.2679,717.0 +tree_50-50_5_1,0.9,0.1,5,0,0,2,0,2,7,2,32.457,0.521,0.223,0.745,296.0,36.2679,717.0 +tree_50-50_5_1,0.8,0.2,5,0,0,2,0,2,7,2,32.103,0.522,0.224,0.745,295.5,36.2679,717.0 +tree_50-50_5_1,0.7,0.3,5,0,0,3,0,2,8,2,32.321,0.561,0.113,0.674,296.0,18.1967,717.0 +tree_50-50_5_1,0.6,0.4,5,0,0,3,0,2,8,2,32.071,0.561,0.113,0.674,296.0,18.1967,717.0 +tree_50-50_5_1,0.5,0.5,5,0,0,3,0,2,8,2,32.208,0.561,0.113,0.674,295.5,18.1967,717.0 +tree_50-50_5_1,0.4,0.6,5,0,0,3,0,2,8,2,32.364,0.561,0.113,0.674,296.0,18.1967,717.0 +tree_50-50_5_1,0.3,0.7,5,0,0,3,0,2,8,2,32.321,0.561,0.113,0.674,296.0,18.1967,717.0 +tree_50-50_5_1,0.2,0.8,5,0,0,3,0,2,8,2,32.236,0.561,0.113,0.674,296.0,18.1967,717.0 +tree_50-50_5_1,0.1,0.9,5,0,0,3,0,2,8,2,32.479,0.561,0.113,0.674,296.0,18.1967,717.0 +tree_50-50_10_1,1.0,0.0,19,0,0,2,0,16,12,5,32.722,3.131,0.225,3.357,197.5,36.2679,7475.0 +tree_50-50_10_1,0.9,0.1,19,0,0,3,0,16,13,5,33.001,3.168,0.114,3.282,197.5,18.1967,7475.0 +tree_50-50_10_1,0.8,0.2,19,0,0,3,0,16,13,5,32.347,3.170,0.114,3.284,198.0,18.1967,7475.0 +tree_50-50_10_1,0.7,0.3,19,0,0,3,0,16,13,5,32.486,3.170,0.114,3.283,197.5,18.1967,7475.0 +tree_50-50_10_1,0.6,0.4,19,0,0,3,0,16,13,5,32.441,3.130,0.113,3.243,197.5,18.1967,7475.0 +tree_50-50_10_1,0.5,0.5,19,0,0,3,0,16,13,5,32.687,3.171,0.114,3.284,198.0,18.1967,7475.0 +tree_50-50_10_1,0.4,0.6,19,0,0,3,0,16,13,5,33.079,3.169,0.114,3.283,197.5,18.1967,7475.0 +tree_50-50_10_1,0.3,0.7,19,0,0,3,0,16,13,5,31.970,3.169,0.114,3.283,198.0,18.1967,7475.0 +tree_50-50_10_1,0.2,0.8,19,0,0,3,0,16,13,5,32.298,3.169,0.114,3.283,198.0,18.1967,7475.0 +tree_50-50_10_1,0.1,0.9,19,0,0,3,0,16,13,5,32.428,3.169,0.114,3.283,197.5,18.1967,7475.0 +tree_100-50_5_1,1.0,0.0,24,0,0,3,0,9,15,5,33.757,2.072,0.337,2.409,199.5,54.3391,2947.0 +tree_100-50_5_1,0.9,0.1,24,0,0,3,0,9,15,5,32.594,2.072,0.337,2.409,199.5,54.3391,2947.0 +tree_100-50_5_1,0.8,0.2,23,0,0,4,0,9,15,5,32.143,2.085,0.222,2.308,199.5,36.2679,2947.0 +tree_100-50_5_1,0.7,0.3,23,0,0,7,0,9,18,5,33.201,2.232,0.114,2.346,200.0,18.1967,2947.0 +tree_100-50_5_1,0.6,0.4,23,0,0,7,0,9,18,5,33.133,2.231,0.114,2.345,199.5,18.1967,2947.0 +tree_100-50_5_1,0.5,0.5,23,0,0,7,0,9,18,5,33.249,2.230,0.114,2.344,200.0,18.1967,2947.0 +tree_100-50_5_1,0.4,0.6,23,0,0,7,0,9,18,5,32.957,2.230,0.114,2.344,199.5,18.1967,2947.0 +tree_100-50_5_1,0.3,0.7,23,0,0,7,0,9,18,5,33.074,2.221,0.113,2.334,199.5,18.1967,2947.0 +tree_100-50_5_1,0.2,0.8,23,0,0,7,0,9,18,5,32.840,2.231,0.114,2.345,199.5,18.1967,2947.0 +tree_100-50_5_1,0.1,0.9,23,0,0,7,0,9,18,5,33.137,2.230,0.114,2.343,200.0,18.1967,2947.0 +l2_distance_16,1.0,0.0,4,1,0,6,0,1,12,1,35.095,0.444,0.445,0.888,323.0,72.4103,0.0 +l2_distance_16,0.9,0.1,4,1,0,6,0,1,12,1,34.991,0.444,0.445,0.889,323.0,72.4103,0.0 +l2_distance_16,0.8,0.2,4,1,0,8,0,1,14,1,38.116,0.523,0.223,0.746,321.0,36.2679,0.0 +l2_distance_16,0.7,0.3,4,1,0,8,0,1,14,1,37.633,0.524,0.223,0.747,322.0,36.2679,0.0 +l2_distance_16,0.6,0.4,4,1,0,8,0,1,14,1,37.686,0.524,0.223,0.747,321.5,36.2679,0.0 +l2_distance_16,0.5,0.5,4,1,0,20,0,1,26,1,42.505,1.003,0.112,1.116,319.5,18.1967,0.0 +l2_distance_16,0.4,0.6,4,1,0,20,0,1,26,1,42.673,1.004,0.112,1.116,320.0,18.1967,0.0 +l2_distance_16,0.3,0.7,4,1,0,20,0,1,26,1,42.468,1.004,0.112,1.116,320.5,18.1967,0.0 +l2_distance_16,0.2,0.8,4,1,0,20,0,1,26,1,42.572,1.004,0.112,1.116,320.5,18.1967,0.0 +l2_distance_16,0.1,0.9,4,1,0,20,0,1,26,1,42.586,1.004,0.112,1.116,320.0,18.1967,0.0 +l2_distance_32,1.0,0.0,5,1,0,6,0,1,13,1,41.364,0.445,0.444,0.889,324.0,72.4103,0.0 +l2_distance_32,0.9,0.1,5,1,0,6,0,1,13,1,41.393,0.445,0.444,0.889,324.5,72.4103,0.0 +l2_distance_32,0.8,0.2,5,1,0,7,0,1,14,1,45.934,0.482,0.334,0.817,324.5,54.3391,0.0 +l2_distance_32,0.7,0.3,5,1,0,8,0,1,15,1,50.852,0.524,0.223,0.747,324.0,36.2679,0.0 +l2_distance_32,0.6,0.4,5,1,0,8,0,1,15,1,50.993,0.524,0.223,0.747,324.5,36.2679,0.0 +l2_distance_32,0.5,0.5,5,1,0,8,0,1,15,1,52.553,0.524,0.223,0.747,323.5,36.2679,0.0 +l2_distance_32,0.4,0.6,5,1,0,20,0,1,27,1,65.925,1.004,0.112,1.116,324.0,18.1967,0.0 +l2_distance_32,0.3,0.7,5,1,0,20,0,1,27,1,68.579,1.004,0.112,1.116,324.0,18.1967,0.0 +l2_distance_32,0.2,0.8,5,1,0,20,0,1,27,1,64.965,1.003,0.112,1.115,324.0,18.1967,0.0 +l2_distance_32,0.1,0.9,5,1,0,20,0,1,27,1,68.155,1.003,0.112,1.115,324.0,18.1967,0.0 \ No newline at end of file diff --git a/src/fheco/code_gen/gen_func.cpp b/src/fheco/code_gen/gen_func.cpp index 9bfa5806..1fb7aab6 100644 --- a/src/fheco/code_gen/gen_func.cpp +++ b/src/fheco/code_gen/gen_func.cpp @@ -410,8 +410,16 @@ void gen_main_code(fheco::param_select::EncParams params,param_select::EncParams RelinKeys relin_keys; keygen.create_relin_keys(relin_keys); GaloisKeys galois_keys; - keygen.create_galois_keys(galois_keys); - //keygen.create_galois_keys(get_rotation_steps_fhe(), galois_keys); + //keygen.create_galois_keys(galois_keys); + chrono::high_resolution_clock::time_point keys_time; + chrono::duration keys_elapsed; + keys_time = chrono::high_resolution_clock::now(); + keygen.create_galois_keys(get_rotation_steps_fhe(), galois_keys); + keys_elapsed = chrono::high_resolution_clock::now() - keys_time; + + size_t galois_keys_total_size = galois_keys.save_size(); + cout << "rotation_keys_size_(MB): " << galois_keys_total_size / (1024.0 * 1024.0) << endl; + Encryptor encryptor(context, public_key); Evaluator evaluator(context); Decryptor decryptor(context, secret_key); @@ -437,7 +445,10 @@ void gen_main_code(fheco::param_select::EncParams params,param_select::EncParams get_clear_outputs( batch_encoder, decryptor, encrypted_outputs, encoded_outputs, func_slot_count, obtained_clear_outputs); print_encrypted_outputs_info(context, decryptor, encrypted_outputs, clog); - cout <<"execution_time_(ms): "< func, Ruleset ruleset, trs::RewriteH * * @param func Shared pointer to the function to be vectorized. */ -void Compiler::gen_vectorized_code(const std::shared_ptr &func, int optimization_method) +void Compiler::gen_vectorized_code(const std::shared_ptr &func, int optimization_method, float w_ops, float w_keys) { // Utility function to print expressions in prefix notation util::ExprPrinter expr_printer(func); @@ -311,7 +311,7 @@ void Compiler::gen_vectorized_code(const std::shared_ptr &func, int op /*********************************************************/ // Call the vectorizer function with the computed vector width std::cout << "Call the code vectorizer \n"; - call_vectorizer(vector_width, optimization_method); + call_vectorizer(vector_width, optimization_method, w_ops, w_keys); /***********************************************************/ // Call the script to build the source code that operates on vectors format_vectorized_code(func,false); @@ -337,7 +337,7 @@ void Compiler::gen_vectorized_code(const std::shared_ptr &func, int op * @param func Shared pointer to the function to be vectorized. * @param window The number of subvectors to divide the outputs into for vectorization. */ -void Compiler::gen_vectorized_code(const std::shared_ptr &func, int window, int optimization_method) +void Compiler::gen_vectorized_code(const std::shared_ptr &func, int window, int optimization_method, float w_ops, float w_keys) { if (window < 0) { @@ -368,7 +368,7 @@ void Compiler::gen_vectorized_code(const std::shared_ptr &func, int wi /***************************************************************/ if (window == 0) { - gen_vectorized_code(func, optimization_method); + gen_vectorized_code(func, optimization_method, w_ops, w_keys); return; } else @@ -427,7 +427,7 @@ void Compiler::gen_vectorized_code(const std::shared_ptr &func, int wi if (vector_full_width < window) { std::cout << "\nresult vector width smaller than window size ==> windows will be considered=0(deactivated)\n"; - gen_vectorized_code(func, optimization_method); + gen_vectorized_code(func, optimization_method, w_ops, w_keys); return; } int index = 0; @@ -462,7 +462,7 @@ void Compiler::gen_vectorized_code(const std::shared_ptr &func, int wi } expression_file << expression; expression_file.close(); - call_vectorizer(vector_width, optimization_method); + call_vectorizer(vector_width, optimization_method, w_ops, w_keys); /********************************************/ std::string vectorized_file = "../vectorized_code.txt"; /******************************************************/ @@ -509,7 +509,7 @@ void Compiler::gen_vectorized_code(const std::shared_ptr &func, int wi } } /***********************************************************************/ -void Compiler::call_vectorizer(int vector_width, int optimization_method) +void Compiler::call_vectorizer(int vector_width, int optimization_method, float w_ops, float w_keys) { if (optimization_method == 0) { @@ -517,7 +517,7 @@ void Compiler::call_vectorizer(int vector_width, int optimization_method) } else if (optimization_method == 1) { - call_rl_vectorizer(vector_width); + call_rl_vectorizer(vector_width, w_ops, w_keys); } else { @@ -541,7 +541,7 @@ void Compiler::call_egraph_vectorizer(int vector_width,int rewrite_rule_family_i } } -void Compiler::call_rl_vectorizer(int vector_width) +void Compiler::call_rl_vectorizer(int vector_width, float w_ops, float w_keys) { namespace fs = std::filesystem; @@ -580,8 +580,10 @@ void Compiler::call_rl_vectorizer(int vector_width) std::ostringstream cmd; cmd << "python -m fhe_rl run " << "'" << expr_file.string() << "' " - << "'" << vect_file.string() << "'"; - std::cout << "Executing: " << cmd.str() << '\n'; + << "'" << vect_file.string() << "' " + << "--w_ops " << w_ops << " " + << "--w_keys " << w_keys; + std::cout << "Executing RL Inference with weights: [" << w_ops << ", " << w_keys << "]\n"; const int rc = std::system(cmd.str().c_str()); /*----------------------------------------------------------------- 5. Restore caller’s working directory @@ -1732,7 +1734,7 @@ void Compiler::format_vectorized_code(const std::shared_ptr &func, boo } std::cout<<"==> stop_reached : "< &func, std::map map, queue &tokens); - static void gen_vectorized_code(const std::shared_ptr &func, int optimization_method); + static void gen_vectorized_code(const std::shared_ptr &func, int optimization_method, float w_ops, float w_keys); static void format_vectorized_code(const std::shared_ptr &func,bool final_expression_reached); - static void gen_vectorized_code(const std::shared_ptr &func, int window, int optimization_method); + static void gen_vectorized_code(const std::shared_ptr &func, int window, int optimization_method, float w_ops, float w_keys); static void gen_he_code( const std::shared_ptr &func, std::ostream &header_os, std::string_view header_name, - std::ostream &source_os, std::size_t rotation_keys_threshold = std::numeric_limits::max(), + std::ostream &source_os, std::size_t rotation_keys_threshold = 10, bool lazy_relin = false,param_select::EncParams::SecurityLevel security_level=param_select::EncParams::SecurityLevel::tc128); static inline const std::shared_ptr &active_func() { @@ -80,9 +80,9 @@ class Compiler static void call_egraph_vectorizer(int vector_width,int rewrite_rule_family_index); - static void call_rl_vectorizer(int vector_width); + static void call_rl_vectorizer(int vector_width, float w_ops, float w_keys); - static void call_vectorizer(int vector_width, int optimization_method); + static void call_vectorizer(int vector_width, int optimization_method, float w_ops, float w_keys); static void call_script(); diff --git a/src/fheco/passes/reduce_rotation_keys.cpp b/src/fheco/passes/reduce_rotation_keys.cpp index 69c022c1..575da906 100644 --- a/src/fheco/passes/reduce_rotation_keys.cpp +++ b/src/fheco/passes/reduce_rotation_keys.cpp @@ -21,8 +21,7 @@ unordered_set reduce_rotation_keys(const shared_ptr &func, size_t { auto steps = term->op_code().steps(); ++steps_freq[steps]; - if (!util::is_power_of_two(abs(steps))) - decomp_candidate_terms.insert(func->data_flow().get_term(term->id())); + decomp_candidate_terms.insert(func->data_flow().get_term(term->id())); } } @@ -38,10 +37,17 @@ unordered_set reduce_rotation_keys(const shared_ptr &func, size_t unordered_map> steps_nafs; for (auto steps : ordered_used_steps) { - if (util::is_power_of_two(abs(steps))) - steps_nafs.emplace(steps, vector{steps}); + if (util::is_power_of_two(abs(steps)) && abs(steps) > 1) + { + // Binary decomposition: split power of 2 into two halves + int half = steps / 2; + steps_nafs.emplace(steps, vector{half, half}); + } else + { + // NAF decomposition for non-powers of 2 steps_nafs.emplace(steps, naf(steps)); + } } unordered_map steps_costs; @@ -66,12 +72,12 @@ unordered_set reduce_rotation_keys(const shared_ptr &func, size_t for (size_t i = 0; i < init_steps_count; ++i) { auto min_cost_steps = ordered_used_steps.back(); - if (util::is_power_of_two(abs(min_cost_steps))) - { - used_steps.insert(min_cost_steps); - ordered_used_steps.pop_back(); - continue; - } + // if (util::is_power_of_two(abs(min_cost_steps))) + // { + // used_steps.insert(min_cost_steps); + // ordered_used_steps.pop_back(); + // continue; + // } steps_to_decomp.insert(min_cost_steps); ordered_used_steps.pop_back(); keys_count -= 1; diff --git a/train_command.sh b/train_command.sh new file mode 100644 index 00000000..7a4318f2 --- /dev/null +++ b/train_command.sh @@ -0,0 +1,8 @@ +python -m fhe_rl train \ + --dataset ./fhe_rl/datasets/my_data.txt \ + --n_envs 8 \ + --total_timesteps 2000000 \ + --n_cycle 1 \ + --n_budget 5 \ + --lambda_env 0.1 \ + --lambda_kl 0.05 \ No newline at end of file diff --git a/train_job.sh b/train_job.sh new file mode 100644 index 00000000..68fe54a6 --- /dev/null +++ b/train_job.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +#SBATCH -q c2 +#SBATCH -p compute +#SBATCH --mem 128GB +#SBATCH -c 8 +#SBATCH -t 10-00:00:00 + +#SBATCH --mail-user=im2821@nyu.edu +#SBATCH --mail-type=BEGIN,END,FAIL + +#SBATCH -o output.txt +#SBATCH -e error.txt + + + +source /share/apps/NYUAD5/miniconda/3-4.11.0/bin/activate +conda activate chehabEnv + +cd RL +python -m fhe_rl --tokenizer_type dynamic train \ No newline at end of file