diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 00000000..d4821959 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,7 @@ +--- +# Checks désactivés pour tout le projet (clang-tidy n'est utilisé que par l'IDE : +# ni le build CMake ni la CI ne le lancent). +# +# modernize-use-nodiscard : réclame [[nodiscard]] sur toute méthode const +# renvoyant une valeur — trop bruyant pour les nombreux getters du projet. +Checks: '-modernize-use-nodiscard' diff --git a/README.md b/README.md index b1b1c0f1..7d14730d 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ A battle-royal game with trained agent which controls tanks in realistic physic ## Description -Each agent receives the rendered frame of its camera as input, and it is trained to fire and hit enemies. +Each tank receives the rendered frame of its camera as input, and the agent is trained to fire and hit enemies. -When agent is trained (with SAC algorithm) you can fight against other tanks. +When the agent is trained (with the PPO or SAC algorithm), you can fight it through the tanks it handles. ## Installation diff --git a/arenai_agent/src/agents/agent_cli.cpp b/arenai_agent/src/agents/agent_cli.cpp index 7da3a8e4..7e0d671b 100644 --- a/arenai_agent/src/agents/agent_cli.cpp +++ b/arenai_agent/src/agents/agent_cli.cpp @@ -47,7 +47,4 @@ namespace arenai::agent { return algorithms; } - AgentCli get_default_agent_cli() { - return make_agent_cli("sac", sac_cli_fields()); - } }// namespace arenai::agent diff --git a/arenai_agent/src/agents/agent_cli.h b/arenai_agent/src/agents/agent_cli.h index ea013838..5426a1ba 100644 --- a/arenai_agent/src/agents/agent_cli.h +++ b/arenai_agent/src/agents/agent_cli.h @@ -29,10 +29,9 @@ namespace arenai::agent { create_factory; }; + // the first entry is the algorithm used when no subcommand is given std::vector make_agent_clis(); - AgentCli get_default_agent_cli(); - }// namespace arenai::agent #endif//ARENAI_ALGORITHMS_H diff --git a/arenai_agent/src/agents/factory_set.cpp b/arenai_agent/src/agents/factory_set.cpp index 62b171c8..5922821e 100644 --- a/arenai_agent/src/agents/factory_set.cpp +++ b/arenai_agent/src/agents/factory_set.cpp @@ -31,7 +31,8 @@ namespace arenai::agent { .channels, get_value( "group_norm_nums", parse_cli_group_norms, {{{1, 2, 3, 4, 6, 8}}}) - .groups), + .groups, + 0.f, 0.f), get_value("cuda", false) ? torch::kCUDA : torch::kCPU); } diff --git a/arenai_agent/src/agents/ppo/ppo_agent.cpp b/arenai_agent/src/agents/ppo/ppo_agent.cpp index cf268a40..2eb538db 100644 --- a/arenai_agent/src/agents/ppo/ppo_agent.cpp +++ b/arenai_agent/src/agents/ppo/ppo_agent.cpp @@ -29,11 +29,11 @@ namespace arenai::agent { std::vector TorchPpoAgent::act( const std::vector &states, const int vision_height, const int vision_width) { const auto [continuous_action, discrete_action] = - act(states_to_tensor(states, vision_height, vision_width)); + act(states_to_tensor(states, vision_height, vision_width), false); return tensor_to_actions(continuous_action, discrete_action); } - TorchAction TorchPpoAgent::act(const TorchState &state) { + TorchAction TorchPpoAgent::act(const TorchState &state, const bool sample) { TorchAction action; torch::Tensor continuous_log_prob; torch::Tensor discrete_log_prob; @@ -44,8 +44,9 @@ namespace arenai::agent { const auto &[vision, sensors] = state; const auto &[mu, sigma, discrete_proba] = actor->act(vision, sensors); - action.continuous_action = truncated_normal_sample(mu, sigma); - action.discrete_action = multinomial_sample(discrete_proba); + action.continuous_action = sample ? truncated_normal_sample(mu, sigma) : mu; + action.discrete_action = sample ? multinomial_sample(discrete_proba) + : multinomial_max_action(discrete_proba); // old log-probabilities, kept for the PPO importance ratio continuous_log_prob = diff --git a/arenai_agent/src/agents/ppo/ppo_agent.h b/arenai_agent/src/agents/ppo/ppo_agent.h index cbb503a9..bcf5440f 100644 --- a/arenai_agent/src/agents/ppo/ppo_agent.h +++ b/arenai_agent/src/agents/ppo/ppo_agent.h @@ -19,7 +19,7 @@ namespace arenai::agent { const std::shared_ptr &actor, torch::Device device, std::optional> collector = std::nullopt); - TorchAction act(const TorchState &state) override; + TorchAction act(const TorchState &state, bool sample) override; std::vector act(const std::vector &states, int vision_height, int vision_width) override; diff --git a/arenai_agent/src/agents/ppo/ppo_collector.cpp b/arenai_agent/src/agents/ppo/ppo_collector.cpp index 8f87e0cc..b0e30543 100644 --- a/arenai_agent/src/agents/ppo/ppo_collector.cpp +++ b/arenai_agent/src/agents/ppo/ppo_collector.cpp @@ -21,16 +21,14 @@ namespace arenai::agent { last_discrete_log_prob = discrete_log_prob; } - void PpoStepCollector::on_transition( - const torch::Tensor &rewards, const torch::Tensor &done, const torch::Tensor &truncated) { + void PpoStepCollector::on_transition(const torch::Tensor &rewards, const torch::Tensor &done) { rollout_buffer->add( {.state = last_state, .action = last_action, .continuous_log_prob = last_continuous_log_prob, .discrete_log_prob = last_discrete_log_prob, .reward = rewards, - .done = done, - .truncated = truncated}); + .done = done}); } void PpoStepCollector::on_episode_end(const TorchState &final_state) { diff --git a/arenai_agent/src/agents/ppo/ppo_collector.h b/arenai_agent/src/agents/ppo/ppo_collector.h index 89c7a1c6..f98cb8a8 100644 --- a/arenai_agent/src/agents/ppo/ppo_collector.h +++ b/arenai_agent/src/agents/ppo/ppo_collector.h @@ -22,9 +22,7 @@ namespace arenai::agent { const TorchState &state, const TorchAction &action, const torch::Tensor &continuous_log_prob, const torch::Tensor &discrete_log_prob); - void on_transition( - const torch::Tensor &rewards, const torch::Tensor &done, - const torch::Tensor &truncated) override; + void on_transition(const torch::Tensor &rewards, const torch::Tensor &done) override; void on_episode_end(const TorchState &final_state) override; diff --git a/arenai_agent/src/agents/ppo/ppo_factory.cpp b/arenai_agent/src/agents/ppo/ppo_factory.cpp index 173ede37..23b5f3af 100644 --- a/arenai_agent/src/agents/ppo/ppo_factory.cpp +++ b/arenai_agent/src/agents/ppo/ppo_factory.cpp @@ -16,17 +16,17 @@ namespace arenai::agent { : actor(std::make_shared( vision_height, vision_width, nb_sensors, nb_continuous_actions, nb_discrete_actions, params.hidden_size_sensors, params.actor_hidden_sizes, params.vision_channels, - params.group_norm_nums)), + params.group_norm_nums, params.initial_sigma, params.initial_fire_proba)), rollout_buffer(std::make_shared()), collector(std::make_shared(rollout_buffer)), agent(std::make_shared(actor, device, collector)), trainer(std::make_shared( - actor, rollout_buffer, vision_height, vision_width, nb_sensors, - params.actor_learning_rate, params.critic_learning_rate, params.hidden_size_sensors, - params.critic_hidden_sizes, params.vision_channels, params.group_norm_nums, device, - params.metric_window_size, params.gamma, params.gae_lambda, params.clip_epsilon, - params.target_kl, params.grad_norm_max, params.continuous_entropy_coef, - params.discrete_entropy_coef, params.epochs, params.rollout_size, + actor, rollout_buffer, vision_height, vision_width, nb_sensors, nb_continuous_actions, + params.actor_learning_rate, params.critic_learning_rate, params.alpha_learning_rate, + params.hidden_size_sensors, params.critic_hidden_sizes, params.vision_channels, + params.group_norm_nums, device, params.metric_window_size, params.gamma, + params.gae_lambda, params.clip_epsilon, params.target_kl, params.grad_norm_max, + params.target_sigma, params.target_fire_proba, params.epochs, params.rollout_size, params.minibatch_size)) {} std::shared_ptr PpoTorchAgentFactory::get_agent() { return agent; } diff --git a/arenai_agent/src/agents/ppo/ppo_hyperparams.cpp b/arenai_agent/src/agents/ppo/ppo_hyperparams.cpp index 12d2acc9..ef97e2f3 100644 --- a/arenai_agent/src/agents/ppo/ppo_hyperparams.cpp +++ b/arenai_agent/src/agents/ppo/ppo_hyperparams.cpp @@ -8,24 +8,27 @@ namespace arenai::agent { std::vector> ppo_cli_fields() { return { - {"--actor_learning_rate", &PpoHyperParams::actor_learning_rate}, - {"--critic_learning_rate", &PpoHyperParams::critic_learning_rate}, - {"--hidden_size_sensors", &PpoHyperParams::hidden_size_sensors}, - {"--actor_hidden_sizes", &PpoHyperParams::actor_hidden_sizes}, - {"--critic_hidden_sizes", &PpoHyperParams::critic_hidden_sizes}, - {"--vision_channels", &PpoHyperParams::vision_channels}, - {"--group_norm_nums", &PpoHyperParams::group_norm_nums}, - {"--metric_window_size", &PpoHyperParams::metric_window_size}, - {"--gamma", &PpoHyperParams::gamma}, - {"--gae_lambda", &PpoHyperParams::gae_lambda}, - {"--clip_epsilon", &PpoHyperParams::clip_epsilon}, - {"--target_kl", &PpoHyperParams::target_kl}, - {"--grad_norm_max", &PpoHyperParams::grad_norm_max}, - {"--continuous_entropy_coef", &PpoHyperParams::continuous_entropy_coef}, - {"--discrete_entropy_coef", &PpoHyperParams::discrete_entropy_coef}, - {"--epochs", &PpoHyperParams::epochs}, - {"--rollout_size", &PpoHyperParams::rollout_size}, - {"--minibatch_size", &PpoHyperParams::minibatch_size}, + {.name = "--actor_learning_rate", .member = &PpoHyperParams::actor_learning_rate}, + {.name = "--critic_learning_rate", .member = &PpoHyperParams::critic_learning_rate}, + {.name = "--alpha_learning_rate", .member = &PpoHyperParams::alpha_learning_rate}, + {.name = "--hidden_size_sensors", .member = &PpoHyperParams::hidden_size_sensors}, + {.name = "--actor_hidden_sizes", .member = &PpoHyperParams::actor_hidden_sizes}, + {.name = "--critic_hidden_sizes", .member = &PpoHyperParams::critic_hidden_sizes}, + {.name = "--vision_channels", .member = &PpoHyperParams::vision_channels}, + {.name = "--group_norm_nums", .member = &PpoHyperParams::group_norm_nums}, + {.name = "--initial_sigma", .member = &PpoHyperParams::initial_sigma}, + {.name = "--initial_fire_proba", .member = &PpoHyperParams::initial_fire_proba}, + {.name = "--metric_window_size", .member = &PpoHyperParams::metric_window_size}, + {.name = "--gamma", .member = &PpoHyperParams::gamma}, + {.name = "--gae_lambda", .member = &PpoHyperParams::gae_lambda}, + {.name = "--clip_epsilon", .member = &PpoHyperParams::clip_epsilon}, + {.name = "--target_kl", .member = &PpoHyperParams::target_kl}, + {.name = "--grad_norm_max", .member = &PpoHyperParams::grad_norm_max}, + {.name = "--target_sigma", .member = &PpoHyperParams::target_sigma}, + {.name = "--target_fire_proba", .member = &PpoHyperParams::target_fire_proba}, + {.name = "--epochs", .member = &PpoHyperParams::epochs}, + {.name = "--rollout_size", .member = &PpoHyperParams::rollout_size}, + {.name = "--minibatch_size", .member = &PpoHyperParams::minibatch_size}, }; } diff --git a/arenai_agent/src/agents/ppo/ppo_hyperparams.h b/arenai_agent/src/agents/ppo/ppo_hyperparams.h index f7544732..b44ee783 100644 --- a/arenai_agent/src/agents/ppo/ppo_hyperparams.h +++ b/arenai_agent/src/agents/ppo/ppo_hyperparams.h @@ -16,25 +16,24 @@ namespace arenai::agent { struct PpoHyperParams { float actor_learning_rate = 1e-4f; float critic_learning_rate = 3e-4f; + float alpha_learning_rate = 1e-3f; int hidden_size_sensors = 128; std::vector actor_hidden_sizes = {1024, 512}; std::vector critic_hidden_sizes = {1024, 512}; std::vector> vision_channels = {{3, 8}, {8, 16}, {16, 24}, {24, 32}, {32, 48}, {48, 64}}; std::vector group_norm_nums = {1, 2, 3, 4, 6, 8}; + float initial_sigma = 0.4f; + float initial_fire_proba = 0.025f; int metric_window_size = 256; - // 0.997 at 30 Hz -> ~11 s credit horizon (shell flight time + fights stay visible) - float gamma = 0.997f; - // 0.98: a shell resolving 30-60 steps after the fire still reaches the fire - // decision at x0.25-0.5 through GAE, instead of x0.04-0.2 with 0.95 + float gamma = 0.99f; float gae_lambda = 0.98f; float clip_epsilon = 0.2f; - // early-stop of the epoch loop when approx KL > 1.5 * target_kl; <= 0 disables it - float target_kl = 0.02f; + float target_kl = 0.05f; float grad_norm_max = 0.5f; - float continuous_entropy_coef = 0.0005f; - float discrete_entropy_coef = 0.005f; - int epochs = 4; + float target_sigma = 0.4f; + float target_fire_proba = 0.025f; + int epochs = 2; int rollout_size = 30 * 30; int minibatch_size = 1024; }; diff --git a/arenai_agent/src/agents/ppo/ppo_rollout_buffer.cpp b/arenai_agent/src/agents/ppo/ppo_rollout_buffer.cpp index 45aed232..cf382337 100644 --- a/arenai_agent/src/agents/ppo/ppo_rollout_buffer.cpp +++ b/arenai_agent/src/agents/ppo/ppo_rollout_buffer.cpp @@ -27,11 +27,7 @@ namespace arenai::agent { // tanks already terminated before this step have no valid transition to store const auto valid = already_terminated_.logical_not(); already_terminated_.logical_or_( - step.done.detach() - .cpu() - .to(torch::kBool) - .reshape({nb_tanks}) - .logical_or(step.truncated.detach().cpu().to(torch::kBool).reshape({nb_tanks}))); + step.done.detach().cpu().to(torch::kBool).reshape({nb_tanks})); steps_.push_back( {.step = @@ -42,8 +38,7 @@ namespace arenai::agent { .continuous_log_prob = step.continuous_log_prob.detach().cpu(), .discrete_log_prob = step.discrete_log_prob.detach().cpu(), .reward = step.reward.detach().cpu(), - .done = step.done.detach().cpu(), - .truncated = step.truncated.detach().cpu()}, + .done = step.done.detach().cpu()}, .valid = valid}); // the freshly added step is pending: its closing observation is not known yet @@ -93,7 +88,6 @@ namespace arenai::agent { stack([](const StoredStep &s) { return s.step.discrete_log_prob; }), .rewards = stack([](const StoredStep &s) { return s.step.reward; }), .dones = stack([](const StoredStep &s) { return s.step.done; }), - .truncateds = stack([](const StoredStep &s) { return s.step.truncated; }), .bootstrap_state = bootstrap_state, .valids = stack([](const StoredStep &s) { return s.valid; }).unsqueeze(-1)}; diff --git a/arenai_agent/src/agents/ppo/ppo_rollout_buffer.h b/arenai_agent/src/agents/ppo/ppo_rollout_buffer.h index 45aad4b3..924a5941 100644 --- a/arenai_agent/src/agents/ppo/ppo_rollout_buffer.h +++ b/arenai_agent/src/agents/ppo/ppo_rollout_buffer.h @@ -21,7 +21,6 @@ namespace arenai::agent { torch::Tensor discrete_log_prob; torch::Tensor reward; torch::Tensor done; - torch::Tensor truncated; }; // On-policy rollout stacked on the time dimension: every tensor is [T, nb_tanks, ...] @@ -32,7 +31,6 @@ namespace arenai::agent { torch::Tensor discrete_log_probs; torch::Tensor rewards; torch::Tensor dones; - torch::Tensor truncateds; // [nb_tanks, ...] observation closing the last step, for the value bootstrap TorchState bootstrap_state; // [T, nb_tanks, 1] whether the (step, tank) pair is a live transition @@ -62,7 +60,7 @@ namespace arenai::agent { // observation closing the last stored step, set by finish_episode() std::optional final_state_; - // [nb_tanks] tanks already done/truncated in the current episode + // [nb_tanks] tanks already done in the current episode torch::Tensor already_terminated_; }; diff --git a/arenai_agent/src/agents/ppo/ppo_trainer.cpp b/arenai_agent/src/agents/ppo/ppo_trainer.cpp index caa3145c..674399c1 100644 --- a/arenai_agent/src/agents/ppo/ppo_trainer.cpp +++ b/arenai_agent/src/agents/ppo/ppo_trainer.cpp @@ -4,6 +4,7 @@ #include "./ppo_trainer.h" +#include #include #include "../../distributions/multinomial.h" @@ -18,7 +19,6 @@ using namespace arenai; using namespace arenai::agent; namespace arenai::agent { - namespace { // merges the [T, nb_tanks] leading dimensions into a single row dimension torch::Tensor flatten_steps(const torch::Tensor &tensor) { @@ -27,42 +27,63 @@ namespace arenai::agent { sizes[0] = tensor.size(0) * tensor.size(1); return tensor.reshape(sizes); } + + constexpr float LOG_RATIO_MAX_ABS = 3.f; + + constexpr float KL_TRIM_FRACTION = 0.01f; + + constexpr float ALPHA_K_P = 2e-1f; + constexpr float ALPHA_K_I = 5e-3f; + constexpr float ALPHA_K_D = 1.f; + + constexpr float ALPHA_INITIAL = 1e-3f; }// namespace PpoTrainer::PpoTrainer( - std::shared_ptr actor, std::shared_ptr rollout_buffer, - const int vision_height, const int vision_width, const int nb_sensors, + const std::shared_ptr &actor, + const std::shared_ptr &rollout_buffer, const int vision_height, + const int vision_width, const int nb_sensors, const int nb_continuous_actions, const float actor_learning_rate, const float critic_learning_rate, - const int hidden_size_sensors, const std::vector &critic_hidden_sizes, + float alpha_learning_rate, const int hidden_size_sensors, + const std::vector &critic_hidden_sizes, const std::vector> &vision_channels, const std::vector &group_norm_nums, const torch::Device device, const int metric_window_size, const float gamma, const float gae_lambda, const float clip_epsilon, const float target_kl, const float grad_norm_max, - const float continuous_entropy_coef, const float discrete_entropy_coef, const int epochs, + const float target_sigma, const float target_fire_proba, const int epochs, const int rollout_size, const int minibatch_size) - : actor(std::move(actor)), rollout_buffer(std::move(rollout_buffer)), + : actor(actor), rollout_buffer(rollout_buffer), + continuous_alpha(std::make_shared( + ALPHA_K_P, ALPHA_K_I, ALPHA_K_D, ALPHA_INITIAL, nb_continuous_actions)), + discrete_alpha(std::make_shared( + ALPHA_K_P, ALPHA_K_I, ALPHA_K_D, ALPHA_INITIAL, 1)), critic(std::make_shared( vision_height, vision_width, nb_sensors, hidden_size_sensors, critic_hidden_sizes, vision_channels, group_norm_nums)), - actor_optim(std::make_unique( - this->actor->parameters(), torch::optim::AdamOptions(actor_learning_rate))), - critic_optim(std::make_unique( - critic->parameters(), torch::optim::AdamOptions(critic_learning_rate))), + actor_optim( + std::make_unique(this->actor->parameters(), actor_learning_rate)), + critic_optim( + std::make_unique(critic->parameters(), critic_learning_rate)), actor_mean_loss_metric(std::make_shared("π_μ", metric_window_size)), actor_std_loss_metric(std::make_shared("π_σ", metric_window_size)), critic_mean_loss_metric(std::make_shared("v_μ", metric_window_size)), critic_std_loss_metric(std::make_shared("v_σ", metric_window_size)), + explained_variance_metric(std::make_shared("ev", metric_window_size)), continuous_entropy_metric(std::make_shared("Hc", metric_window_size)), discrete_entropy_metric(std::make_shared("Hd", metric_window_size)), + continuous_alpha_metric(std::make_shared("α_c", metric_window_size, 2, true)), + discrete_alpha_metric(std::make_shared("α_d", metric_window_size, 2, true)), sigma_metric(std::make_shared("σ", metric_window_size)), clip_fraction_metric(std::make_shared("clip", metric_window_size)), - kl_metric(std::make_shared("kl", metric_window_size, 2, true)), gamma(gamma), - gae_lambda(gae_lambda), clip_epsilon(clip_epsilon), target_kl(target_kl), - grad_norm_max(grad_norm_max), continuous_entropy_coef(continuous_entropy_coef), - discrete_entropy_coef(discrete_entropy_coef), epochs(epochs), rollout_size(rollout_size), + kl_metric(std::make_shared("kl", metric_window_size, 2, true)), + skip_fraction_metric(std::make_shared("skip", metric_window_size)), + gamma(gamma), gae_lambda(gae_lambda), clip_epsilon(clip_epsilon), target_kl(target_kl), + grad_norm_max(grad_norm_max), target_sigma(target_sigma), + target_fire_proba(target_fire_proba), epochs(epochs), rollout_size(rollout_size), minibatch_size(minibatch_size) { - to(device); + + set_train(false); } void PpoTrainer::step() { @@ -96,90 +117,135 @@ namespace arenai::agent { const auto nb_valid_rows = valid_idx.size(0); if (nb_valid_rows == 0) return; - bool kl_stop = false; - for (int e = 0; e < epochs && !kl_stop; e++) { + const auto select = [&](const torch::Tensor &tensor, const torch::Tensor &idx) { + return tensor.index_select(0, idx).to(device); + }; + + for (int e = 0; e < epochs; e++) { const auto perm = valid_idx.index_select(0, torch::randperm(nb_valid_rows)); for (int64_t start = 0; start < nb_valid_rows; start += minibatch_size) { const auto idx = perm.slice(0, start, std::min(start + minibatch_size, nb_valid_rows)); - const auto select = [&](const torch::Tensor &tensor) { - return tensor.index_select(0, idx).to(device); - }; + const auto mb_vision = select(flat_vision, idx); + const auto mb_proprioception = select(flat_proprioception, idx); - const auto mb_vision = select(flat_vision); - const auto mb_proprioception = select(flat_proprioception); - const auto mb_continuous_actions = select(flat_continuous_actions); - const auto mb_discrete_actions = select(flat_discrete_actions); - const auto mb_old_log_probs = select(flat_old_log_probs); - const auto mb_advantages = select(flat_advantages); - const auto mb_returns = select(flat_returns); + train_actor( + mb_vision, mb_proprioception, select(flat_continuous_actions, idx), + select(flat_discrete_actions, idx), select(flat_old_log_probs, idx), + select(flat_advantages, idx)); + + train_critic(mb_vision, mb_proprioception, select(flat_returns, idx)); + } + } - // policy: clipped surrogate on the joint (continuous x discrete) ratio - const auto [mu, sigma, discrete_proba] = actor->act(mb_vision, mb_proprioception); + set_train(false); + } - const auto curr_continuous_log_probs = - truncated_normal_log_pdf(mb_continuous_actions, mu, sigma).sum(-1, true); + bool PpoTrainer::train_actor( + const torch::Tensor &vision, const torch::Tensor &proprioception, + const torch::Tensor &continuous_actions, const torch::Tensor &discrete_actions, + const torch::Tensor &old_log_probs, const torch::Tensor &advantages) const { + const auto [mu, sigma, discrete_proba] = actor->act(vision, proprioception); - const auto clamped_proba = torch::clamp(discrete_proba, EPSILON, 1.0 - EPSILON); - const auto curr_discrete_log_probs = - (mb_discrete_actions * torch::log(clamped_proba)).sum(-1, true); + const auto curr_continuous_log_probs = + truncated_normal_log_pdf(continuous_actions, mu, sigma).sum(-1, true); - const auto ratio = torch::exp( - curr_continuous_log_probs + curr_discrete_log_probs - mb_old_log_probs); + const auto clamped_proba = torch::clamp(discrete_proba, EPSILON, 1.0 - EPSILON); + const auto curr_discrete_log_probs = + torch::sum(discrete_actions * torch::log(clamped_proba), -1, true); - // approx KL (Schulman): E[(ratio - 1) - log ratio]; stop the update before - // this minibatch if the policy drifted too far from the rollout policy - const auto approx_kl = ((ratio - 1.f) - torch::log(ratio)).mean().item(); - kl_metric->add(approx_kl); - if (target_kl > 0.f && approx_kl > 1.5f * target_kl) { - kl_stop = true; - break; - } + const auto log_ratio = torch::clamp( + curr_continuous_log_probs + curr_discrete_log_probs - old_log_probs, -LOG_RATIO_MAX_ABS, + LOG_RATIO_MAX_ABS); - const auto clipped_ratio = - torch::clamp(ratio, 1.f - clip_epsilon, 1.f + clip_epsilon); + const auto ratio = torch::exp(log_ratio); - const auto surrogate = - torch::min(ratio * mb_advantages, clipped_ratio * mb_advantages); + const auto continuous_entropy = truncated_normal_entropy(mu, sigma); + const auto discrete_entropy = multinomial_entropy(discrete_proba); - const auto continuous_entropy = truncated_normal_entropy(mu, sigma).sum(-1, true); - const auto discrete_entropy = multinomial_entropy(discrete_proba); + const auto kl_per_row = (ratio - 1.f - log_ratio).flatten(); - const auto actor_loss = -torch::mean( - surrogate + continuous_entropy_coef * continuous_entropy - + discrete_entropy_coef * discrete_entropy); + const auto nb_kept = std::max( + 1, static_cast( + static_cast(kl_per_row.size(0)) * (1.f - KL_TRIM_FRACTION))); - actor_optim->zero_grad(); - actor_loss.backward(); - torch::nn::utils::clip_grad_norm_(actor->parameters(), grad_norm_max); - actor_optim->step(); + // per-row KL is non-negative: sorting ascending puts the outliers past nb_kept + const auto approx_kl = + std::get<0>(torch::sort(kl_per_row)).slice(0, 0, nb_kept).mean().item(); - // critic - const auto values = critic->value(mb_vision, mb_proprioception); - const auto critic_loss = torch::mse_loss(values, mb_returns, at::Reduction::Mean); + const bool kl_exceeded = target_kl > 0.f && approx_kl > 1.5f * target_kl; - critic_optim->zero_grad(); - critic_loss.backward(); - torch::nn::utils::clip_grad_norm_(critic->parameters(), grad_norm_max); - critic_optim->step(); + const auto entropy_bonus = + torch::sum(continuous_alpha->alpha().detach() * continuous_entropy, -1) + + discrete_alpha->alpha().squeeze(1).detach() * discrete_entropy; - // metrics - actor_mean_loss_metric->add(actor_loss.cpu().item()); - actor_std_loss_metric->add(actor_loss.cpu().item()); + if (!kl_exceeded) { + const auto clipped_ratio = torch::clamp(ratio, 1.f - clip_epsilon, 1.f + clip_epsilon); + const auto surrogate = torch::min(ratio * advantages, clipped_ratio * advantages); - critic_mean_loss_metric->add(critic_loss.cpu().item()); - critic_std_loss_metric->add(critic_loss.cpu().item()); + const auto actor_loss = -torch::mean(surrogate + entropy_bonus); - continuous_entropy_metric->add(continuous_entropy.mean().item()); - discrete_entropy_metric->add(discrete_entropy.mean().item()); - sigma_metric->add(sigma.mean().item()); + actor_optim->zero_grad(); + actor_loss.backward(); + torch::nn::utils::clip_grad_norm_(actor->parameters(), grad_norm_max); + actor_optim->step(); - clip_fraction_metric->add( - ((ratio - 1.f).abs() > clip_epsilon).to(torch::kFloat).mean().item()); - } + const auto loss_value = actor_loss.cpu().item(); + actor_mean_loss_metric->add(loss_value); + actor_std_loss_metric->add(loss_value); } + + const auto target_sigma_tensor = torch::ones_like(continuous_entropy) * target_sigma; + const auto target_continuous_entropy = + truncated_normal_entropy(mu.detach(), target_sigma_tensor); + + continuous_alpha->update(continuous_entropy, target_continuous_entropy); + + // train discrete alpha + const auto target_discrete_entropy = + torch::tensor(multinomial_target_entropy(target_fire_proba), discrete_entropy.device()); + + discrete_alpha->update(discrete_entropy, target_discrete_entropy); + + // metrics + continuous_entropy_metric->add(continuous_entropy.mean().item()); + sigma_metric->add(sigma.mean().item()); + continuous_alpha_metric->add(continuous_alpha->alpha().mean().item()); + + discrete_entropy_metric->add(discrete_entropy.mean().item()); + discrete_alpha_metric->add(discrete_alpha->alpha().mean().item()); + + kl_metric->add(approx_kl); + + clip_fraction_metric->add( + ((ratio - 1.f).abs() > clip_epsilon).to(torch::kFloat).mean().item()); + + skip_fraction_metric->add(kl_exceeded ? 1.f : 0.f); + + return kl_exceeded; + } + + void PpoTrainer::train_critic( + const torch::Tensor &vision, const torch::Tensor &proprioception, + const torch::Tensor &returns) const { + const auto values = critic->value(vision, proprioception); + const auto critic_loss = torch::mse_loss(values, returns, at::Reduction::Mean); + + critic_optim->zero_grad(); + critic_loss.backward(); + torch::nn::utils::clip_grad_norm_(critic->parameters(), grad_norm_max); + critic_optim->step(); + + const auto loss_value = critic_loss.cpu().item(); + critic_mean_loss_metric->add(loss_value); + critic_std_loss_metric->add(loss_value); + + const auto residual_var = (returns - values.detach()).var(false); + const auto returns_var = returns.var(false); + explained_variance_metric->add( + (1.f - residual_var / returns_var.clamp_min(EPSILON)).item()); } GaeResult PpoTrainer::compute_gae(const PpoRollout &rollout, const torch::Device device) const { @@ -216,37 +282,36 @@ namespace arenai::agent { const auto rewards = rollout.rewards.to(torch::kFloat); const auto dones = rollout.dones.to(torch::kFloat); - const auto truncateds = rollout.truncateds.to(torch::kFloat); const auto valids = rollout.valids.to(torch::kFloat); - // terminal: no bootstrap; truncated: bootstrap but stop the GAE recursion - const auto terminals = dones * (1.f - truncateds); - const auto boundaries = torch::max(dones, truncateds); - - const auto deltas = rewards + gamma * next_values * (1.f - terminals) - values; + const auto deltas = rewards + gamma * next_values * (1.f - dones) - values; auto advantages = torch::zeros_like(deltas); auto gae = torch::zeros({nb_tanks, 1}, deltas.options()); for (int64_t t = nb_steps - 1; t >= 0; t--) { - gae = deltas[t] + gamma * gae_lambda * (1.f - boundaries[t]) * gae; + gae = deltas[t] + gamma * gae_lambda * (1.f - dones[t]) * gae; advantages[t] = gae; } const auto returns = advantages + values; const auto nb_valid = valids.sum().clamp_min(1.f); - const auto advantage_mean = (advantages * valids).sum() / nb_valid; + const auto advantage_mean = torch::sum(advantages * valids) / nb_valid; const auto advantage_std = - (((advantages - advantage_mean).square() * valids).sum() / nb_valid).sqrt(); + torch::sqrt(torch::sum(torch::square(advantages - advantage_mean) * valids) / nb_valid); advantages = (advantages - advantage_mean) / (advantage_std + EPSILON); return {.advantages = advantages, .returns = returns}; } std::vector> PpoTrainer::get_metrics() { - return {actor_mean_loss_metric, actor_std_loss_metric, critic_mean_loss_metric, - critic_std_loss_metric, continuous_entropy_metric, discrete_entropy_metric, - sigma_metric, clip_fraction_metric, kl_metric}; + return {actor_mean_loss_metric, actor_std_loss_metric, + critic_mean_loss_metric, critic_std_loss_metric, + explained_variance_metric, continuous_entropy_metric, + continuous_alpha_metric, sigma_metric, + discrete_entropy_metric, discrete_alpha_metric, + clip_fraction_metric, kl_metric, + skip_fraction_metric}; } void PpoTrainer::save(const std::filesystem::path &output_folder) { @@ -275,16 +340,21 @@ namespace arenai::agent { void PpoTrainer::set_train(const bool train) const { actor->train(train); critic->train(train); + + continuous_alpha->train(train); + discrete_alpha->train(train); } void PpoTrainer::to(const torch::Device device) const { actor->to(device); critic->to(device); + + continuous_alpha->to(device); + discrete_alpha->to(device); } int PpoTrainer::count_parameters() { return count_parameters_impl(actor->parameters()) + count_parameters_impl(critic->parameters()); } - }// namespace arenai::agent diff --git a/arenai_agent/src/agents/ppo/ppo_trainer.h b/arenai_agent/src/agents/ppo/ppo_trainer.h index d37c81be..85e2ca05 100644 --- a/arenai_agent/src/agents/ppo/ppo_trainer.h +++ b/arenai_agent/src/agents/ppo/ppo_trainer.h @@ -6,6 +6,7 @@ #define ARENAI_PPO_TRAINER_H #include "../../networks/actor.h" +#include "../../networks/entropy.h" #include "../../networks/value_function.h" #include "../trainer.h" #include "./ppo_rollout_buffer.h" @@ -20,15 +21,16 @@ namespace arenai::agent { class PpoTrainer final : public AbstractTrainer { public: PpoTrainer( - std::shared_ptr actor, std::shared_ptr rollout_buffer, - int vision_height, int vision_width, int nb_sensors, float actor_learning_rate, - float critic_learning_rate, int hidden_size_sensors, + const std::shared_ptr &actor, + const std::shared_ptr &rollout_buffer, int vision_height, + int vision_width, int nb_sensors, int nb_continuous_actions, float actor_learning_rate, + float critic_learning_rate, float alpha_learning_rate, int hidden_size_sensors, const std::vector &critic_hidden_sizes, const std::vector> &vision_channels, const std::vector &group_norm_nums, torch::Device device, int metric_window_size, float gamma, float gae_lambda, float clip_epsilon, float target_kl, float grad_norm_max, - float continuous_entropy_coef, float discrete_entropy_coef, int epochs, - int rollout_size, int minibatch_size); + float target_sigma, float target_fire_proba, int epochs, int rollout_size, + int minibatch_size); void step() override; @@ -42,10 +44,13 @@ namespace arenai::agent { std::shared_ptr actor; std::shared_ptr rollout_buffer; + std::shared_ptr continuous_alpha; + std::shared_ptr discrete_alpha; + std::shared_ptr critic; - std::shared_ptr actor_optim; - std::shared_ptr critic_optim; + std::unique_ptr actor_optim; + std::unique_ptr critic_optim; std::shared_ptr actor_mean_loss_metric; std::shared_ptr actor_std_loss_metric; @@ -53,15 +58,26 @@ namespace arenai::agent { std::shared_ptr critic_mean_loss_metric; std::shared_ptr critic_std_loss_metric; + // share of the return variance the critic explains, per critic minibatch + std::shared_ptr explained_variance_metric; + + // both regulated by their constant entropy bonus std::shared_ptr continuous_entropy_metric; std::shared_ptr discrete_entropy_metric; + std::shared_ptr continuous_alpha_metric; + std::shared_ptr discrete_alpha_metric; + // mean sigma of the truncated normal: direct view of the aim spread, Hc only bounds it std::shared_ptr sigma_metric; + // both recorded on every attempted minibatch, skipped ones included std::shared_ptr clip_fraction_metric; std::shared_ptr kl_metric; + // fraction of minibatches the KL threshold skipped + std::shared_ptr skip_fraction_metric; + float gamma; float gae_lambda; float clip_epsilon; @@ -70,8 +86,8 @@ namespace arenai::agent { float grad_norm_max; - float continuous_entropy_coef; - float discrete_entropy_coef; + float target_sigma; + float target_fire_proba; int epochs; // rollout horizon: one train() consumes rollout_size complete steps in a single update @@ -81,6 +97,19 @@ namespace arenai::agent { void train() const; + // one backward pass on the actor for a single minibatch; returns true when the + // minibatch drifted past the KL threshold, in which case no update was applied and + // only the kl metric was recorded — the next minibatch is tried either way + bool train_actor( + const torch::Tensor &vision, const torch::Tensor &proprioception, + const torch::Tensor &continuous_actions, const torch::Tensor &discrete_actions, + const torch::Tensor &old_log_probs, const torch::Tensor &advantages) const; + + // one backward pass on the critic for a single minibatch + void train_critic( + const torch::Tensor &vision, const torch::Tensor &proprioception, + const torch::Tensor &returns) const; + // GAE advantages (normalized over the valid pairs) and value targets, // computed with the pre-update critic GaeResult compute_gae(const PpoRollout &rollout, torch::Device device) const; diff --git a/arenai_agent/src/agents/sac/sac_agent.cpp b/arenai_agent/src/agents/sac/sac_agent.cpp index 0275774e..fe07ee52 100644 --- a/arenai_agent/src/agents/sac/sac_agent.cpp +++ b/arenai_agent/src/agents/sac/sac_agent.cpp @@ -28,11 +28,11 @@ namespace arenai::agent { std::vector TorchSacAgent::act( const std::vector &states, const int vision_height, const int vision_width) { const auto [continuous_action, discrete_action] = - act(states_to_tensor(states, vision_height, vision_width)); + act(states_to_tensor(states, vision_height, vision_width), false); return tensor_to_actions(continuous_action, discrete_action); } - TorchAction TorchSacAgent::act(const TorchState &state) { + TorchAction TorchSacAgent::act(const TorchState &state, const bool sample) { TorchAction action; { @@ -43,8 +43,9 @@ namespace arenai::agent { const auto &[vision, sensors] = state; const auto &[mu, sigma, discrete_proba] = actor->act(vision, sensors); - action.continuous_action = truncated_normal_sample(mu, sigma); - action.discrete_action = multinomial_sample(discrete_proba); + action.continuous_action = sample ? truncated_normal_sample(mu, sigma) : mu; + action.discrete_action = sample ? multinomial_sample(discrete_proba) + : multinomial_max_action(discrete_proba); } if (collector.has_value()) collector.value()->on_act(state, action); diff --git a/arenai_agent/src/agents/sac/sac_agent.h b/arenai_agent/src/agents/sac/sac_agent.h index 9771c677..25d4481b 100644 --- a/arenai_agent/src/agents/sac/sac_agent.h +++ b/arenai_agent/src/agents/sac/sac_agent.h @@ -19,7 +19,7 @@ namespace arenai::agent { const std::shared_ptr &actor, torch::Device device, std::optional> collector = std::nullopt); - TorchAction act(const TorchState &state) override; + TorchAction act(const TorchState &state, bool sample) override; std::vector act(const std::vector &states, int vision_height, int vision_width) override; diff --git a/arenai_agent/src/agents/sac/sac_collector.cpp b/arenai_agent/src/agents/sac/sac_collector.cpp index c97c0e82..e8ef7c12 100644 --- a/arenai_agent/src/agents/sac/sac_collector.cpp +++ b/arenai_agent/src/agents/sac/sac_collector.cpp @@ -17,14 +17,9 @@ namespace arenai::agent { last_action = action; } - void SacStepCollector::on_transition( - const torch::Tensor &rewards, const torch::Tensor &done, const torch::Tensor &truncated) { + void SacStepCollector::on_transition(const torch::Tensor &rewards, const torch::Tensor &done) { replay_buffer->add( - {.state = last_state, - .action = last_action, - .reward = rewards, - .done = done, - .truncated = truncated}); + {.state = last_state, .action = last_action, .reward = rewards, .done = done}); } void SacStepCollector::on_episode_end(const TorchState &final_state) { diff --git a/arenai_agent/src/agents/sac/sac_collector.h b/arenai_agent/src/agents/sac/sac_collector.h index 209b9932..c93bd6e1 100644 --- a/arenai_agent/src/agents/sac/sac_collector.h +++ b/arenai_agent/src/agents/sac/sac_collector.h @@ -20,9 +20,7 @@ namespace arenai::agent { // concrete act-time channel, called by TrainableSacAgent::act void on_act(const TorchState &state, const TorchAction &action); - void on_transition( - const torch::Tensor &rewards, const torch::Tensor &done, - const torch::Tensor &truncated) override; + void on_transition(const torch::Tensor &rewards, const torch::Tensor &done) override; void on_episode_end(const TorchState &final_state) override; diff --git a/arenai_agent/src/agents/sac/sac_factory.cpp b/arenai_agent/src/agents/sac/sac_factory.cpp index d5166313..0e5e5ade 100644 --- a/arenai_agent/src/agents/sac/sac_factory.cpp +++ b/arenai_agent/src/agents/sac/sac_factory.cpp @@ -16,7 +16,7 @@ namespace arenai::agent { : actor(std::make_shared( vision_height, vision_width, nb_sensors, nb_continuous_actions, nb_discrete_actions, params.hidden_size_sensors, params.actor_hidden_sizes, params.vision_channels, - params.group_norm_nums)), + params.group_norm_nums, params.initial_sigma, params.initial_fire_proba)), replay_buffer(std::make_shared(params.replay_buffer_size)), collector(std::make_shared(replay_buffer)), agent(std::make_shared(actor, device, collector)), @@ -26,7 +26,7 @@ namespace arenai::agent { params.alpha_learning_rate, params.hidden_size_sensors, params.hidden_size_actions, params.critic_hidden_sizes, params.vision_channels, params.group_norm_nums, device, params.metric_window_size, params.tau, params.gamma, params.train_every, - params.epochs, params.batch_size)) {} + params.epochs, params.batch_size, params.target_sigma, params.target_fire_proba)) {} std::shared_ptr SacTorchAgentFactory::get_agent() { return agent; } diff --git a/arenai_agent/src/agents/sac/sac_hyperparams.cpp b/arenai_agent/src/agents/sac/sac_hyperparams.cpp index d1a4fbab..4b12bf0a 100644 --- a/arenai_agent/src/agents/sac/sac_hyperparams.cpp +++ b/arenai_agent/src/agents/sac/sac_hyperparams.cpp @@ -8,22 +8,26 @@ namespace arenai::agent { std::vector> sac_cli_fields() { return { - {"--actor_learning_rate", &SacHyperParams::actor_learning_rate}, - {"--critic_learning_rate", &SacHyperParams::critic_learning_rate}, - {"--alpha_learning_rate", &SacHyperParams::alpha_learning_rate}, - {"--hidden_size_sensors", &SacHyperParams::hidden_size_sensors}, - {"--hidden_size_actions", &SacHyperParams::hidden_size_actions}, - {"--actor_hidden_sizes", &SacHyperParams::actor_hidden_sizes}, - {"--critic_hidden_sizes", &SacHyperParams::critic_hidden_sizes}, - {"--vision_channels", &SacHyperParams::vision_channels}, - {"--group_norm_nums", &SacHyperParams::group_norm_nums}, - {"--metric_window_size", &SacHyperParams::metric_window_size}, - {"--tau", &SacHyperParams::tau}, - {"--gamma", &SacHyperParams::gamma}, - {"--replay_buffer_size", &SacHyperParams::replay_buffer_size}, - {"--train_every", &SacHyperParams::train_every}, - {"--epochs", &SacHyperParams::epochs}, - {"--batch_size", &SacHyperParams::batch_size}, + {.name = "--actor_learning_rate", .member = &SacHyperParams::actor_learning_rate}, + {.name = "--critic_learning_rate", .member = &SacHyperParams::critic_learning_rate}, + {.name = "--alpha_learning_rate", .member = &SacHyperParams::alpha_learning_rate}, + {.name = "--hidden_size_sensors", .member = &SacHyperParams::hidden_size_sensors}, + {.name = "--hidden_size_actions", .member = &SacHyperParams::hidden_size_actions}, + {.name = "--actor_hidden_sizes", .member = &SacHyperParams::actor_hidden_sizes}, + {.name = "--critic_hidden_sizes", .member = &SacHyperParams::critic_hidden_sizes}, + {.name = "--vision_channels", .member = &SacHyperParams::vision_channels}, + {.name = "--group_norm_nums", .member = &SacHyperParams::group_norm_nums}, + {.name = "--initial_sigma", .member = &SacHyperParams::initial_sigma}, + {.name = "--initial_fire_proba", .member = &SacHyperParams::initial_fire_proba}, + {.name = "--target_sigma", .member = &SacHyperParams::target_sigma}, + {.name = "--target_fire_proba", .member = &SacHyperParams::target_fire_proba}, + {.name = "--metric_window_size", .member = &SacHyperParams::metric_window_size}, + {.name = "--tau", .member = &SacHyperParams::tau}, + {.name = "--gamma", .member = &SacHyperParams::gamma}, + {.name = "--replay_buffer_size", .member = &SacHyperParams::replay_buffer_size}, + {.name = "--train_every", .member = &SacHyperParams::train_every}, + {.name = "--epochs", .member = &SacHyperParams::epochs}, + {.name = "--batch_size", .member = &SacHyperParams::batch_size}, }; } diff --git a/arenai_agent/src/agents/sac/sac_hyperparams.h b/arenai_agent/src/agents/sac/sac_hyperparams.h index dc5fb840..02034e6f 100644 --- a/arenai_agent/src/agents/sac/sac_hyperparams.h +++ b/arenai_agent/src/agents/sac/sac_hyperparams.h @@ -24,9 +24,13 @@ namespace arenai::agent { std::vector> vision_channels = {{3, 8}, {8, 16}, {16, 24}, {24, 32}, {32, 48}, {48, 64}}; std::vector group_norm_nums = {1, 2, 3, 4, 6, 8}; + float initial_sigma = 0.4f; + float initial_fire_proba = 0.025f; + float target_sigma = 0.4f; + float target_fire_proba = 0.025f; int metric_window_size = 256; float tau = 0.005f; - float gamma = 0.995f; + float gamma = 0.99f; int replay_buffer_size = 300000; int train_every = 256; int epochs = 128; diff --git a/arenai_agent/src/agents/sac/sac_replay_buffer.cpp b/arenai_agent/src/agents/sac/sac_replay_buffer.cpp index 6a70556a..0a4b0942 100644 --- a/arenai_agent/src/agents/sac/sac_replay_buffer.cpp +++ b/arenai_agent/src/agents/sac/sac_replay_buffer.cpp @@ -52,21 +52,17 @@ namespace arenai::agent { const auto idx = static_cast(write_idx_); const auto done_bool = step.done.to(torch::kBool); - const auto truncated_bool = step.truncated.to(torch::kBool); store_vision_[idx].copy_(step.state.vision); store_proprioception_[idx].copy_(step.state.proprioception); store_cont_action_[idx].copy_(step.action.continuous_action.detach()); store_disc_action_[idx].copy_(step.action.discrete_action.detach()); store_reward_[idx].copy_(step.reward); - // a truncated step is not a real terminal: the critic must keep bootstrapping - // through it, only actual terminations (done && !truncated) cut the return - store_done_[idx].copy_(done_bool.logical_and(truncated_bool.logical_not())); + store_done_[idx].copy_(done_bool); // tanks already terminated before this step have no valid transition to store store_sampleable_[idx].copy_(already_terminated_.logical_not()); - already_terminated_.logical_or_( - done_bool.reshape({nb_tanks_}).logical_or(truncated_bool.reshape({nb_tanks_}))); + already_terminated_.logical_or_(done_bool.reshape({nb_tanks_})); advance_write_idx(); } diff --git a/arenai_agent/src/agents/sac/sac_replay_buffer.h b/arenai_agent/src/agents/sac/sac_replay_buffer.h index 9b1a57c4..9f86e96c 100644 --- a/arenai_agent/src/agents/sac/sac_replay_buffer.h +++ b/arenai_agent/src/agents/sac/sac_replay_buffer.h @@ -5,8 +5,6 @@ #ifndef ARENAI_AGENT_HOST_REPLAY_BUFFER_H #define ARENAI_AGENT_HOST_REPLAY_BUFFER_H -#include - #include #include "../torch_types.h" @@ -18,7 +16,6 @@ namespace arenai::agent { TorchAction action; torch::Tensor reward; torch::Tensor done; - torch::Tensor truncated; }; struct SacTrainStep { @@ -61,12 +58,11 @@ namespace arenai::agent { torch::Tensor store_cont_action_; torch::Tensor store_disc_action_; torch::Tensor store_reward_; - // real terminals only (done && !truncated): truncated steps stay bootstrappable torch::Tensor store_done_; // [mem, nb_tanks] whether the (step, tank) pair can start a sampled transition torch::Tensor store_sampleable_; - // [nb_tanks] tanks already done/truncated in the current episode + // [nb_tanks] tanks already done in the current episode torch::Tensor already_terminated_; }; diff --git a/arenai_agent/src/agents/sac/sac_trainer.cpp b/arenai_agent/src/agents/sac/sac_trainer.cpp index fa259620..133d970c 100644 --- a/arenai_agent/src/agents/sac/sac_trainer.cpp +++ b/arenai_agent/src/agents/sac/sac_trainer.cpp @@ -11,6 +11,7 @@ #include "../../metrics/last_metric.h" #include "../../metrics/mean_metric.h" #include "../../metrics/std_metric.h" +#include "../../networks/constants.h" #include "../../networks_utils/print_module.h" #include "../../networks_utils/target_update.h" #include "../../networks_utils/torch_loader.h" @@ -31,7 +32,8 @@ namespace arenai::agent { const std::vector> &vision_channels, const std::vector &group_norm_nums, const torch::Device device, const int metric_window_size, const float tau, const float gamma, const int train_every, - const int epochs, const int batch_size) + const int epochs, const int batch_size, const float target_sigma, + const float target_fire_proba) : actor(std::move(actor)), replay_buffer(std::move(replay_buffer)), critic_1(std::make_shared( vision_height, vision_width, nb_sensors, nb_continuous_actions, nb_discrete_actions, @@ -49,12 +51,9 @@ namespace arenai::agent { vision_height, vision_width, nb_sensors, nb_continuous_actions, nb_discrete_actions, hidden_size_sensors, hidden_size_actions, critic_hidden_sizes, vision_channels, group_norm_nums)), - alpha_continuous(std::make_shared(0.01f)), - alpha_discrete(std::make_shared(0.01f)), - continuous_target_entropy(std::make_shared( - nb_continuous_actions, 0.7f, 0.1f, 1000000)), - discrete_target_entropy( - std::make_shared(0.5f, 0.1f, 1000000)), + alpha_continuous( + std::make_shared(5e-2f, 1e-5f, 1e-1f, nb_continuous_actions)), + alpha_discrete(std::make_shared(5e-2f, 1e-5f, 1e-1f, 1)), actor_optim(std::make_unique( this->actor->parameters(), torch::optim::AdamOptions(actor_learning_rate))), critic_1_optim(std::make_unique( @@ -78,12 +77,18 @@ namespace arenai::agent { continuous_target_entropy_metric(std::make_shared("Hc_t")), discrete_target_entropy_metric(std::make_shared("Hd_t")), tau(tau), gamma(gamma), train_every(train_every), train_counter(0), epochs(epochs), - batch_size(batch_size) { + batch_size(batch_size), + target_sigma( + torch::tensor(std::vector(nb_continuous_actions, target_sigma)).unsqueeze(0)), + target_discrete_entropy( + torch::tensor({multinomial_target_entropy(target_fire_proba)}).unsqueeze(0)) { hard_update(target_critic_1, critic_1); hard_update(target_critic_2, critic_2); to(device); + + set_train(false); } void SacTrainer::step() { @@ -107,21 +112,24 @@ namespace arenai::agent { actor->act(next_state.vision, next_state.proprioception); const auto next_continuous_action = truncated_normal_sample(next_mu, next_sigma); - const auto next_continuous_entropy = - truncated_normal_entropy(next_mu, next_sigma).sum(-1, true); + const auto next_continuous_entropy = truncated_normal_entropy(next_mu, next_sigma); const auto next_discrete_entropy = multinomial_entropy(next_discrete_proba); - const auto next_target_q_value_1 = target_critic_1->value_expectation( - next_state.vision, next_state.proprioception, next_continuous_action, - next_discrete_proba); - const auto next_target_q_value_2 = target_critic_2->value_expectation( - next_state.vision, next_state.proprioception, next_continuous_action, - next_discrete_proba); + const auto next_target_q_values_1 = target_critic_1->value_per_discrete_action( + next_state.vision, next_state.proprioception, next_continuous_action); + const auto next_target_q_values_2 = target_critic_2->value_per_discrete_action( + next_state.vision, next_state.proprioception, next_continuous_action); + + const auto next_min_q_value = torch::sum( + next_discrete_proba + * torch::min(next_target_q_values_1, next_target_q_values_2), + -1, true); - const auto target_v_value = torch::min(next_target_q_value_1, next_target_q_value_2) - + alpha_continuous->alpha() * next_continuous_entropy - + alpha_discrete->alpha() * next_discrete_entropy; + const auto target_v_value = + next_min_q_value + + torch::sum(alpha_continuous->alpha() * next_continuous_entropy, -1, true) + + torch::sum(alpha_discrete->alpha() * next_discrete_entropy, -1, true); target_q_values = reward + (1.f - done.to(torch::kFloat)) * gamma * target_v_value; } @@ -159,20 +167,21 @@ namespace arenai::agent { actor->act(state.vision, state.proprioception); const auto curr_continuous_action = truncated_normal_sample(curr_mu, curr_sigma); - const auto curr_continuous_entropy = - truncated_normal_entropy(curr_mu, curr_sigma).sum(-1, true); + const auto curr_continuous_entropy = truncated_normal_entropy(curr_mu, curr_sigma); const auto curr_discrete_entropy = multinomial_entropy(curr_discrete_proba); - const auto curr_q_value_1 = critic_1->value_expectation( - state.vision, state.proprioception, curr_continuous_action, curr_discrete_proba); - const auto curr_q_value_2 = critic_2->value_expectation( - state.vision, state.proprioception, curr_continuous_action, curr_discrete_proba); - const auto q_value = torch::min(curr_q_value_1, curr_q_value_2); + const auto curr_q_values_1 = critic_1->value_per_discrete_action( + state.vision, state.proprioception, curr_continuous_action); + const auto curr_q_values_2 = critic_2->value_per_discrete_action( + state.vision, state.proprioception, curr_continuous_action); + const auto q_value = torch::sum( + curr_discrete_proba * torch::min(curr_q_values_1, curr_q_values_2), -1, true); const auto actor_loss = -torch::mean( - alpha_continuous->alpha().detach() * curr_continuous_entropy - + alpha_discrete->alpha().detach() * curr_discrete_entropy + q_value); + torch::sum(alpha_continuous->alpha().detach() * curr_continuous_entropy, -1, true) + + torch::sum(alpha_discrete->alpha().detach() * curr_discrete_entropy, -1, true) + + q_value); actor_optim->zero_grad(); actor_loss.backward(); @@ -180,18 +189,27 @@ namespace arenai::agent { actor_optim->step(); // continuous entropy - const auto alpha_continuous_loss = torch::mean( - alpha_continuous->log_alpha() - * (curr_continuous_entropy.detach() - continuous_target_entropy->target_entropy())); + const auto curr_continuous_target_entropy = + truncated_normal_entropy(curr_mu, target_sigma); + + const auto alpha_continuous_loss = + torch::sum( + alpha_continuous->log_alpha() + * torch::detach(curr_continuous_entropy - curr_continuous_target_entropy), + -1) + .mean(); alpha_continuous_optim->zero_grad(); alpha_continuous_loss.backward(); alpha_continuous_optim->step(); // discrete entropy - const auto alpha_discrete_loss = torch::mean( - alpha_discrete->log_alpha() - * (curr_discrete_entropy.detach() - discrete_target_entropy->target_entropy())); + const auto alpha_discrete_loss = + torch::sum( + alpha_discrete->log_alpha() + * torch::detach(curr_discrete_entropy - target_discrete_entropy), + -1) + .mean(); alpha_discrete_optim->zero_grad(); alpha_discrete_loss.backward(); @@ -204,19 +222,20 @@ namespace arenai::agent { continuous_entropy_metric->add(curr_continuous_entropy.mean().item()); discrete_entropy_metric->add(curr_discrete_entropy.mean().item()); + continuous_target_entropy_metric->add( + curr_continuous_target_entropy.mean().item()); + discrete_target_entropy_metric->add(target_discrete_entropy.item()); + critic_1_mean_loss_metric->add(critic_1_loss.cpu().item()); critic_1_std_loss_metric->add(critic_1_loss.cpu().item()); critic_2_mean_loss_metric->add(critic_2_loss.cpu().item()); critic_2_std_loss_metric->add(critic_2_loss.cpu().item()); - alpha_continuous_metric->add(alpha_continuous->alpha().item()); - alpha_discrete_metric->add(alpha_discrete->alpha().item()); + alpha_continuous_metric->add(alpha_continuous->alpha().mean().item()); + alpha_discrete_metric->add(alpha_discrete->alpha().mean().item()); } - continuous_target_entropy_metric->add( - continuous_target_entropy->target_entropy().item()); - discrete_target_entropy_metric->add( - discrete_target_entropy->target_entropy().item()); + set_train(false); } std::vector> SacTrainer::get_metrics() { @@ -272,15 +291,12 @@ namespace arenai::agent { alpha_continuous->train(train); alpha_discrete->train(train); - continuous_target_entropy->train(train); - discrete_target_entropy->train(train); - // force eval for target critics target_critic_1->train(false); target_critic_2->train(false); } - void SacTrainer::to(const torch::Device device) const { + void SacTrainer::to(const torch::Device device) { actor->to(device); critic_1->to(device); @@ -292,8 +308,8 @@ namespace arenai::agent { alpha_continuous->to(device); alpha_discrete->to(device); - continuous_target_entropy->to(device); - discrete_target_entropy->to(device); + target_sigma = target_sigma.to(device); + target_discrete_entropy = target_discrete_entropy.to(device); } int SacTrainer::count_parameters() { diff --git a/arenai_agent/src/agents/sac/sac_trainer.h b/arenai_agent/src/agents/sac/sac_trainer.h index 6dbd2e33..44b6b8e7 100644 --- a/arenai_agent/src/agents/sac/sac_trainer.h +++ b/arenai_agent/src/agents/sac/sac_trainer.h @@ -23,7 +23,8 @@ namespace arenai::agent { const std::vector &critic_hidden_sizes, const std::vector> &vision_channels, const std::vector &group_norm_nums, torch::Device device, int metric_window_size, - float tau, float gamma, int train_every, int epochs, int batch_size); + float tau, float gamma, int train_every, int epochs, int batch_size, float target_sigma, + float target_fire_proba); void step() override; @@ -48,11 +49,8 @@ namespace arenai::agent { std::shared_ptr target_critic_1; std::shared_ptr target_critic_2; - std::shared_ptr alpha_continuous; - std::shared_ptr alpha_discrete; - - std::shared_ptr continuous_target_entropy; - std::shared_ptr discrete_target_entropy; + std::shared_ptr alpha_continuous; + std::shared_ptr alpha_discrete; std::shared_ptr actor_optim; std::shared_ptr critic_1_optim; @@ -88,10 +86,13 @@ namespace arenai::agent { int epochs; int batch_size; + torch::Tensor target_sigma; + torch::Tensor target_discrete_entropy; + void train() const; void set_train(bool train) const; - void to(torch::Device device) const; + void to(torch::Device device); }; }// namespace arenai::agent diff --git a/arenai_agent/src/agents/step_collector.h b/arenai_agent/src/agents/step_collector.h index 118576fc..9ab8975e 100644 --- a/arenai_agent/src/agents/step_collector.h +++ b/arenai_agent/src/agents/step_collector.h @@ -5,8 +5,6 @@ #ifndef ARENAI_STEP_COLLECTOR_H #define ARENAI_STEP_COLLECTOR_H -#include - #include "./torch_types.h" namespace arenai::agent { @@ -18,9 +16,7 @@ namespace arenai::agent { public: virtual ~AbstractStepCollector() = default; - virtual void on_transition( - const torch::Tensor &rewards, const torch::Tensor &done, - const torch::Tensor &truncated) = 0; + virtual void on_transition(const torch::Tensor &rewards, const torch::Tensor &done) = 0; virtual void on_episode_end(const TorchState &final_state) = 0; }; diff --git a/arenai_agent/src/agents/torch_agent.h b/arenai_agent/src/agents/torch_agent.h index 2976e027..c30e55be 100644 --- a/arenai_agent/src/agents/torch_agent.h +++ b/arenai_agent/src/agents/torch_agent.h @@ -13,7 +13,7 @@ namespace arenai::agent { public: virtual ~AbstractTorchAgent() = default; - virtual TorchAction act(const TorchState &state) = 0; + virtual TorchAction act(const TorchState &state, bool sample) = 0; }; }// namespace arenai::agent diff --git a/arenai_agent/src/agents/torch_types.h b/arenai_agent/src/agents/torch_types.h index d18b902e..7b6dab83 100644 --- a/arenai_agent/src/agents/torch_types.h +++ b/arenai_agent/src/agents/torch_types.h @@ -22,7 +22,6 @@ namespace arenai::agent { TorchState states; torch::Tensor rewards; torch::Tensor is_done; - torch::Tensor is_truncated; }; }// namespace arenai::agent diff --git a/arenai_agent/src/core/train_environment.cpp b/arenai_agent/src/core/train_environment.cpp index 9b54abec..d1c8c014 100644 --- a/arenai_agent/src/core/train_environment.cpp +++ b/arenai_agent/src/core/train_environment.cpp @@ -4,6 +4,9 @@ #include "./train_environment.h" +#include +#include + #include #include #include @@ -22,7 +25,7 @@ namespace arenai::agent { const std::filesystem::path &android_assets_path, const float wanted_frequency, const int max_episode_steps, const int vision_height, const int vision_width, const int vision_num_threads) - : core::BaseTanksEnvironment( + : BaseTanksEnvironment( std::make_shared(android_assets_path), graphics_backend, nb_tanks, wanted_frequency, vision_height, vision_width, vision_num_threads, false), wanted_frequency(wanted_frequency), @@ -31,20 +34,22 @@ namespace arenai::agent { nb_frames_added_when_hit(static_cast(5.f / wanted_frequency)), nb_tanks(nb_tanks), nb_steps(0), done(nb_tanks, false), already_done(nb_tanks, false), max_episode_steps(max_episode_steps), + reward_metric( + std::make_shared("r", 4 * nb_tanks * max_episode_steps, 3, true)), episode_step_mean_nb_metric(std::make_shared("s_μ", 32, 1)), episode_step_std_nb_metric(std::make_shared("s_σ", 32)), fire_metric(std::make_shared("fire", 256, 2)), hit_metric(std::make_shared("hit", 256, 2, true)), kill_metric(std::make_shared("kill", 16, 1)), nb_kills_episode(0) {} - std::vector> + std::vector> TrainTankEnvironment::step(const float time_delta, const std::vector &actions) { // tanks flagged done on a previous step already emitted their terminal transition: // mark them so the caller can skip their post-mortem steps already_done = done; - auto step_result = core::BaseTanksEnvironment::step(time_delta, actions); + auto step_result = BaseTanksEnvironment::step(time_delta, actions); const auto has_hit = apply_on_factories>([&](const auto &factories) { std::vector has_hit_result; @@ -90,16 +95,16 @@ namespace arenai::agent { if (has_hit[i]) remaining_frames[i] += nb_frames_added_when_hit; - const auto &[state, reward, is_done, is_truncated] = step_result[i]; + const auto &[state, reward, is_done] = step_result[i]; if (is_done) { - step_result[i] = {state, reward, true, false}; if (!already_done[i] && !is_suicide[i]) nb_kills_episode++; done[i] = true; } + // starving out (no hit for too long) is a real death: penalized and terminal if (!done[i] && remaining_frames[i] <= 0) { - step_result[i] = {state, reward, true, true}; + step_result[i] = {state, reward - 1.f, true}; done[i] = true; } @@ -110,18 +115,22 @@ namespace arenai::agent { } } - // detect winner + // detect winner and log reward for (int i = 0; i < step_result.size(); i++) { if (done[i]) continue; + // detact winner if (const long nb_not_done = std::ranges::count(done, false); nb_not_done == 1) { - const auto &[state, reward, is_done, is_truncated] = step_result[i]; + const auto &[state, reward, is_done] = step_result[i]; if (only_one_tank_alive()) - step_result[i] = {state, reward + 2.f, true, is_truncated}; // winner réel - else step_result[i] = {state, reward + 1.f, true, is_truncated};// timeout winner + step_result[i] = {state, reward + 2.f, true}; // winner réel + else step_result[i] = {state, reward + 1.f, true};// timeout winner done[i] = true; } + + // log reward + reward_metric->add(std::get<1>(step_result[i])); } nb_steps++; @@ -129,16 +138,6 @@ namespace arenai::agent { return step_result; } - std::vector TrainTankEnvironment::get_phi_vector() { - return apply_on_factories>( - [](const std::vector> &tanks) { - std::vector phi_vector; - phi_vector.reserve(tanks.size()); - for (const auto &tank: tanks) phi_vector.emplace_back(tank->get_phi(tanks)); - return phi_vector; - }); - } - void TrainTankEnvironment::on_draw( const std::vector> &model_matrices) {} @@ -184,15 +183,12 @@ namespace arenai::agent { } std::vector> TrainTankEnvironment::get_metrics() const { - return { - episode_step_mean_nb_metric, episode_step_std_nb_metric, fire_metric, hit_metric, - kill_metric}; - } - - std::vector TrainTankEnvironment::get_valid_mask() const { - std::vector valid(nb_tanks); - for (int i = 0; i < nb_tanks; i++) valid[i] = !already_done[i]; - return valid; + return {reward_metric, + episode_step_mean_nb_metric, + episode_step_std_nb_metric, + fire_metric, + hit_metric, + kill_metric}; } }// namespace arenai::agent diff --git a/arenai_agent/src/core/train_environment.h b/arenai_agent/src/core/train_environment.h index 51c091b0..e52cda01 100644 --- a/arenai_agent/src/core/train_environment.h +++ b/arenai_agent/src/core/train_environment.h @@ -18,17 +18,11 @@ namespace arenai::agent { const std::filesystem::path &android_assets_path, float wanted_frequency, int max_episode_steps, int vision_height, int vision_width, int vision_num_threads); - std::vector> + std::vector> step(float time_delta, const std::vector &actions) override; - std::vector get_phi_vector(); - std::vector> get_metrics() const; - // tanks whose transition of the last step is a live one (not already done before - // the step) — mask for reward metrics, mirrors the rollout buffer's valid rows - std::vector get_valid_mask() const; - bool is_episode_terminated(); static void reset_singleton(); @@ -56,6 +50,8 @@ namespace arenai::agent { int max_episode_steps; + std::shared_ptr reward_metric; + std::shared_ptr episode_step_mean_nb_metric; std::shared_ptr episode_step_std_nb_metric; diff --git a/arenai_agent/src/distributions/beta_law.cpp b/arenai_agent/src/distributions/beta_law.cpp index 738b179b..9fce01e9 100644 --- a/arenai_agent/src/distributions/beta_law.cpp +++ b/arenai_agent/src/distributions/beta_law.cpp @@ -61,7 +61,8 @@ namespace arenai::agent { } float beta_law_target_entropy(const int &nb_actions) { - return beta_law_entropy(torch::tensor(1.f), torch::tensor(1.f)).item() * nb_actions; + return beta_law_entropy(torch::tensor(1.f), torch::tensor(1.f)).item() + * static_cast(nb_actions); } }// namespace arenai::agent diff --git a/arenai_agent/src/distributions/multinomial.cpp b/arenai_agent/src/distributions/multinomial.cpp index 7999f8cb..d7b80789 100644 --- a/arenai_agent/src/distributions/multinomial.cpp +++ b/arenai_agent/src/distributions/multinomial.cpp @@ -18,6 +18,13 @@ namespace arenai::agent { return one_hot; } + torch::Tensor multinomial_max_action(const torch::Tensor &probabilities) { + const auto clamped_proba = torch::clamp(probabilities, EPSILON, 1.0 - EPSILON); + const auto idx = torch::argmax(clamped_proba, 1, true); + const auto one_hot = torch::zeros_like(clamped_proba).scatter_(1, idx, 1.0); + return one_hot; + } + torch::Tensor multinomial_entropy(const torch::Tensor &probabilities) { const auto clamped_proba = torch::clamp(probabilities, EPSILON, 1.0 - EPSILON); return -torch::sum(clamped_proba * torch::log(clamped_proba), -1, true); diff --git a/arenai_agent/src/distributions/multinomial.h b/arenai_agent/src/distributions/multinomial.h index 62aeb1e6..d4b94877 100644 --- a/arenai_agent/src/distributions/multinomial.h +++ b/arenai_agent/src/distributions/multinomial.h @@ -10,6 +10,8 @@ namespace arenai::agent { torch::Tensor multinomial_sample(const torch::Tensor &probabilities); + torch::Tensor multinomial_max_action(const torch::Tensor &probabilities); + torch::Tensor multinomial_entropy(const torch::Tensor &probabilities); float multinomial_maximum_entropy(const int &nb_actions); diff --git a/arenai_agent/src/main.cpp b/arenai_agent/src/main.cpp index ed72db5f..22f358c9 100644 --- a/arenai_agent/src/main.cpp +++ b/arenai_agent/src/main.cpp @@ -24,9 +24,8 @@ int main(const int argc, char **argv) { parser.add_argument("--output_folder").required(); parser.add_argument("--resources_folder").required(); parser.add_argument("--max_episode_steps").scan<'i', int>().default_value(30 * 60 * 3); - parser.add_argument("--potential_reward_gamma").scan<'g', float>().default_value(0.997f); parser.add_argument("--nb_episodes").scan<'i', int>().default_value(2000); - parser.add_argument("--save_every").scan<'i', int>().default_value(30 * 60 * 3 * 5); + parser.add_argument("--save_every").scan<'i', int>().default_value(30 * 60 * 3 * 20); parser.add_argument("--cuda").default_value(false).implicit_value(true); // env @@ -37,18 +36,20 @@ int main(const int argc, char **argv) { parser.add_argument("--vision_width").scan<'i', int>().default_value(256); parser.add_argument("--initial_spawn_width").scan<'g', float>().default_value(500.f); parser.add_argument("--initial_spawn_height").scan<'g', float>().default_value(500.f); - parser.add_argument("--final_spawn_width").scan<'g', float>().default_value(2000.f); - parser.add_argument("--final_spawn_height").scan<'g', float>().default_value(2000.f); + parser.add_argument("--final_spawn_width").scan<'g', float>().default_value(500.f); + parser.add_argument("--final_spawn_height").scan<'g', float>().default_value(500.f); parser.add_argument("--vision_num_threads") .scan<'i', int>() .default_value(static_cast(std::thread::hardware_concurrency())); // one subcommand per algorithm, carrying its own hyper-parameter options - for (const auto &algorithm: algorithms) parser.add_subparser(*algorithm.parser); + for (const auto &[name, subparser, create_agent]: algorithms) parser.add_subparser(*subparser); parser.parse_args(argc, argv); - const AgentCli *selected_algorithm = nullptr; + // first algorithm by default: its subparser is not parsed, argparse falls back on + // the defaults carried by its arguments + const AgentCli *selected_algorithm = &algorithms.front(); for (const auto &algorithm: algorithms) if (parser.is_subcommand_used(algorithm.name)) selected_algorithm = &algorithm; @@ -59,23 +60,26 @@ int main(const int argc, char **argv) { const int vision_height = parser.get("--vision_height"); const int vision_width = parser.get("--vision_width"); - const auto create_factory_function = selected_algorithm == nullptr - ? get_default_agent_cli().create_factory - : selected_algorithm->create_factory; - - const auto agent_factory = create_factory_function( + const auto agent_factory = selected_algorithm->create_factory( vision_height, vision_width, cuda ? torch::Device(torch::kCUDA) : torch::Device(torch::kCPU)); train_main( - {parser.get("--wanted_frequency"), parser.get("--nb_tanks"), vision_height, - vision_width, parser.get("--initial_spawn_width"), - parser.get("--initial_spawn_height"), parser.get("--final_spawn_width"), - parser.get("--final_spawn_height"), parser.get("--vision_num_threads")}, - {std::filesystem::path(parser.get("--output_folder")), - std::filesystem::path(parser.get("--resources_folder")), - parser.get("--potential_reward_gamma"), parser.get("--max_episode_steps"), - parser.get("--nb_episodes"), parser.get("--save_every"), cuda}, + {.wanted_frequency = parser.get("--wanted_frequency"), + .nb_tanks = parser.get("--nb_tanks"), + .vision_height = vision_height, + .vision_width = vision_width, + .initial_spawn_width = parser.get("--initial_spawn_width"), + .initial_spawn_height = parser.get("--initial_spawn_height"), + .final_spawn_width = parser.get("--final_spawn_width"), + .final_spawn_height = parser.get("--final_spawn_height"), + .num_threads = parser.get("--vision_num_threads")}, + {.output_folder = std::filesystem::path(parser.get("--output_folder")), + .resources_folder = std::filesystem::path(parser.get("--resources_folder")), + .max_episode_steps = parser.get("--max_episode_steps"), + .nb_episodes = parser.get("--nb_episodes"), + .save_every = parser.get("--save_every"), + .cuda = cuda}, agent_factory); return 0; diff --git a/arenai_agent/src/metrics/last_metric.cpp b/arenai_agent/src/metrics/last_metric.cpp index 65469dd0..7823e0be 100644 --- a/arenai_agent/src/metrics/last_metric.cpp +++ b/arenai_agent/src/metrics/last_metric.cpp @@ -11,5 +11,5 @@ LastMetric::LastMetric(const std::string &name, const int precision, const bool : AbstractMetric(name, 1, precision, scientific) {} float LastMetric::compute_metric_impl(const std::vector &curr_values) { - return curr_values.size() > 0 ? curr_values.back() : 0.f; + return !curr_values.empty() ? curr_values.back() : 0.f; } diff --git a/arenai_agent/src/metrics/mean_metric.cpp b/arenai_agent/src/metrics/mean_metric.cpp index 54fcbd77..7910ce27 100644 --- a/arenai_agent/src/metrics/mean_metric.cpp +++ b/arenai_agent/src/metrics/mean_metric.cpp @@ -4,6 +4,8 @@ #include "./mean_metric.h" +#include + using namespace arenai; using namespace arenai::agent; diff --git a/arenai_agent/src/metrics/metric.cpp b/arenai_agent/src/metrics/metric.cpp index f0b32464..6a531064 100644 --- a/arenai_agent/src/metrics/metric.cpp +++ b/arenai_agent/src/metrics/metric.cpp @@ -4,7 +4,8 @@ #include "./metric.h" -#include +#include +#include #include using namespace arenai; diff --git a/arenai_agent/src/metrics/metric.h b/arenai_agent/src/metrics/metric.h index 700ac281..140421b6 100644 --- a/arenai_agent/src/metrics/metric.h +++ b/arenai_agent/src/metrics/metric.h @@ -5,11 +5,10 @@ #ifndef ARENAI_AGENT_HOST_METRIC_H #define ARENAI_AGENT_HOST_METRIC_H +#include #include #include -#include - namespace arenai::agent { class AbstractMetric { diff --git a/arenai_agent/src/metrics/std_metric.cpp b/arenai_agent/src/metrics/std_metric.cpp index 7d77f1cb..84d1e2ba 100644 --- a/arenai_agent/src/metrics/std_metric.cpp +++ b/arenai_agent/src/metrics/std_metric.cpp @@ -4,6 +4,9 @@ #include "./std_metric.h" +#include +#include + using namespace arenai; using namespace arenai::agent; diff --git a/arenai_agent/src/networks/actor.cpp b/arenai_agent/src/networks/actor.cpp index d2ce840a..32e83bc9 100644 --- a/arenai_agent/src/networks/actor.cpp +++ b/arenai_agent/src/networks/actor.cpp @@ -18,7 +18,8 @@ namespace arenai::agent { const int &nb_continuous_actions, const int &nb_discrete_actions, const int &hidden_size_sensors, const std::vector &hidden_sizes, const std::vector> &vision_channels, - const std::vector &group_norm_nums) + const std::vector &group_norm_nums, const float &initial_sigma, + const float &initial_fire_proba) : vision_encoder(register_module( "vision_encoder", std::make_shared( vision_height, vision_width, vision_channels, group_norm_nums))), @@ -64,9 +65,11 @@ namespace arenai::agent { head->apply(init_hidden_weights); mu->apply(init_mu_output_weights); - sigma->apply([](torch::nn::Module &m) { init_sigma_output_weights(m, 0.5f); }); + sigma->apply([initial_sigma](Module &m) { init_sigma_output_weights(m, initial_sigma); }); - discrete->apply(init_discrete_output_weights); + discrete->apply([initial_fire_proba](Module &m) { + init_discrete_output_weights(m, initial_fire_proba); + }); } ActorRawOutput Actor::act(const torch::Tensor &vision, const torch::Tensor &sensors) { diff --git a/arenai_agent/src/networks/actor.h b/arenai_agent/src/networks/actor.h index b1442531..549eddcf 100644 --- a/arenai_agent/src/networks/actor.h +++ b/arenai_agent/src/networks/actor.h @@ -26,7 +26,8 @@ namespace arenai::agent { const int &nb_continuous_actions, const int &nb_discrete_actions, const int &hidden_size_sensors, const std::vector &hidden_sizes, const std::vector> &vision_channels, - const std::vector &group_norm_nums); + const std::vector &group_norm_nums, const float &initial_sigma, + const float &initial_fire_proba); ActorRawOutput act(const torch::Tensor &vision, const torch::Tensor &sensors); private: diff --git a/arenai_agent/src/networks/constants.h b/arenai_agent/src/networks/constants.h index 91a2122e..5472d6c9 100644 --- a/arenai_agent/src/networks/constants.h +++ b/arenai_agent/src/networks/constants.h @@ -8,8 +8,8 @@ namespace arenai::agent { constexpr float EPSILON = 1e-8f; - constexpr float SIGMA_MIN = 1e-3f; - constexpr float SIGMA_MAX = 1.f; + constexpr float SIGMA_MIN = 1e-4f; + constexpr float SIGMA_MAX = 2.f; }// namespace arenai::agent #endif//ARENAI_CONSTANTS_H diff --git a/arenai_agent/src/networks/entropy.cpp b/arenai_agent/src/networks/entropy.cpp index 191a5d97..b6b3ec60 100644 --- a/arenai_agent/src/networks/entropy.cpp +++ b/arenai_agent/src/networks/entropy.cpp @@ -16,13 +16,39 @@ using namespace arenai::agent; namespace arenai::agent { - AlphaParameter::AlphaParameter(const float initial_alpha) - : log_alpha_tensor( - register_parameter("log_alpha", torch::tensor({std::log(initial_alpha)}))) {} + /* + * Alpha parameters [0; +inf[ + */ + + AlphaParameters::AlphaParameters(const float initial_alpha, const int nb_alphas) + : log_alpha_tensor(register_parameter( + "log_alpha", + torch::tensor(std::vector(nb_alphas, std::log(initial_alpha))).unsqueeze(0))) {} - torch::Tensor AlphaParameter::log_alpha() { return log_alpha_tensor; } + torch::Tensor AlphaParameters::log_alpha() { return log_alpha_tensor; } - torch::Tensor AlphaParameter::alpha() { return log_alpha().exp(); } + torch::Tensor AlphaParameters::alpha() { return log_alpha().exp(); } + + /* + * Clamped alpha parameters + */ + + ClampedAlphaParameters::ClampedAlphaParameters( + const float initial_alpha, const float min_alpha, const float max_alpha, + const int nb_alphas) + : AlphaParameters(std::clamp(initial_alpha, min_alpha, max_alpha), nb_alphas), + min_log_alpha(std::log(min_alpha)), max_log_alpha(std::log(max_alpha)) {} + + torch::Tensor ClampedAlphaParameters::log_alpha() { + auto curr_log_alpha = AlphaParameters::log_alpha(); + + { + const torch::NoGradGuard no_grad; + curr_log_alpha.data().clamp_(min_log_alpha, max_log_alpha); + } + + return curr_log_alpha; + } /* * Constant target entropy @@ -41,6 +67,10 @@ namespace arenai::agent { : ConstantTargetEntropy( truncated_normal_target_entropy(nb_continuous_action, target_sigma)) {} + ConstantContinuousPerActionTargetEntropy::ConstantContinuousPerActionTargetEntropy( + const float target_sigma) + : ConstantTargetEntropy(truncated_normal_target_entropy(1, target_sigma)) {} + /* * Target entropy warmup */ @@ -91,4 +121,45 @@ namespace arenai::agent { return truncated_normal_target_entropy(nb_actions, value); } + /* + * PID Lagrangian + */ + + PidLagrangianAlphaParameters::PidLagrangianAlphaParameters( + const float k_p, const float k_i, const float k_d, const float initial_alpha, + const int nb_alphas) + : k_p(k_p), k_i(k_i), k_d(k_d), + previous_entropy(register_buffer("previous_entropy", torch::zeros({1, nb_alphas}))), + has_previous(register_buffer("has_previous", torch::zeros({1}))), + integral(register_buffer( + "integral", + torch::full( + {1, nb_alphas}, std::log(std::clamp(initial_alpha, MIN_ALPHA, MAX_ALPHA))))), + log_alpha_tensor(register_buffer( + "log_alpha", + torch::full( + {1, nb_alphas}, std::log(std::clamp(initial_alpha, MIN_ALPHA, MAX_ALPHA))))) {} + + torch::Tensor PidLagrangianAlphaParameters::alpha() const { return log_alpha_tensor.exp(); } + + void PidLagrangianAlphaParameters::update( + const torch::Tensor &entropy, const torch::Tensor &target_entropy) const { + const torch::NoGradGuard no_grad; + + const auto mean_entropy = torch::mean(entropy.detach(), 0, true).view_as(integral); + const auto error = + torch::mean(target_entropy.detach() - entropy.detach(), 0, true).view_as(integral); + + integral.copy_( + torch::clamp(integral + k_i * error, std::log(MIN_ALPHA), std::log(MAX_ALPHA))); + + const auto derivative = + has_previous * torch::clamp_min(previous_entropy - mean_entropy, 0.f); + + previous_entropy.copy_(mean_entropy); + has_previous.fill_(1.f); + + log_alpha_tensor.copy_(k_p * error + integral + k_d * derivative); + } + }// namespace arenai::agent diff --git a/arenai_agent/src/networks/entropy.h b/arenai_agent/src/networks/entropy.h index 5d0367ce..d393e3ac 100644 --- a/arenai_agent/src/networks/entropy.h +++ b/arenai_agent/src/networks/entropy.h @@ -9,17 +9,33 @@ namespace arenai::agent { - class AlphaParameter final : public torch::nn::Module { + /* + * Base class + */ + + class AlphaParameters : public torch::nn::Module { public: - explicit AlphaParameter(float initial_alpha); + explicit AlphaParameters(float initial_alpha, int nb_alphas); - torch::Tensor log_alpha(); + virtual torch::Tensor log_alpha(); torch::Tensor alpha(); private: torch::Tensor log_alpha_tensor; }; + class ClampedAlphaParameters final : public AlphaParameters { + public: + explicit ClampedAlphaParameters( + float initial_alpha, float min_alpha, float max_alpha, int nb_alphas); + + torch::Tensor log_alpha() override; + + private: + float min_log_alpha; + float max_log_alpha; + }; + class AbstractTargetEntropy : public torch::nn::Module { public: virtual torch::Tensor target_entropy() = 0; @@ -49,6 +65,13 @@ namespace arenai::agent { explicit ConstantContinuousTargetEntropy(int nb_continuous_action, float target_sigma); }; + // target of a single action instead of the sum over them, to be broadcast against one + // alpha per dimension + class ConstantContinuousPerActionTargetEntropy : public ConstantTargetEntropy { + public: + explicit ConstantContinuousPerActionTargetEntropy(float target_sigma); + }; + /* * Warmup */ @@ -91,6 +114,36 @@ namespace arenai::agent { int nb_actions; }; + /* + * Lagrangian + */ + + // alpha is the output of the controller, not a parameter moved by gradient ascent: the + // integral term *is* the dual variable. It is driven on the log scale, where each gain + // acts multiplicatively on alpha whatever decade it currently sits in, and where the + // multiplier stays positive without a clamp on the output. + class PidLagrangianAlphaParameters final : public torch::nn::Module { + public: + PidLagrangianAlphaParameters( + float k_p, float k_i, float k_d, float initial_alpha, int nb_alphas); + + void update(const torch::Tensor &entropy, const torch::Tensor &target_entropy) const; + + torch::Tensor alpha() const; + + private: + static constexpr float MIN_ALPHA = 1e-8f; + static constexpr float MAX_ALPHA = 1.f; + + float k_p, k_i, k_d; + + torch::Tensor previous_entropy; + torch::Tensor has_previous; + + torch::Tensor integral; + torch::Tensor log_alpha_tensor; + }; + }// namespace arenai::agent #endif//ARENAI_AGENT_HOST_ENTROPY_H diff --git a/arenai_agent/src/networks/misc.cpp b/arenai_agent/src/networks/misc.cpp index 1c276cca..929aec17 100644 --- a/arenai_agent/src/networks/misc.cpp +++ b/arenai_agent/src/networks/misc.cpp @@ -15,7 +15,7 @@ namespace arenai::agent { torch::Tensor Exp::forward(const torch::Tensor &x) { return torch::exp(x); } - void Exp::pretty_print(std::ostream &stream) const { stream << name() << "()"; } + void Exp::pretty_print(std::ostream &stream) { stream << name() << "()"; } /* * Clamp @@ -28,7 +28,7 @@ namespace arenai::agent { return torch::clamp(x, lower_bound, upper_bound); } - void Clamp::pretty_print(std::ostream &stream) const { + void Clamp::pretty_print(std::ostream &stream) { stream << name() << "(min=" << lower_bound << ", max=" << upper_bound << ")"; } @@ -45,7 +45,7 @@ namespace arenai::agent { return torch::exp(log_sigma); } - void SigmaOutput::pretty_print(std::ostream &stream) const { + void SigmaOutput::pretty_print(std::ostream &stream) { stream << name() << "(min=" << std::exp(min_log_sigma) << ", max=" << std::exp(max_log_sigma) << ")"; } diff --git a/arenai_agent/src/networks/misc.h b/arenai_agent/src/networks/misc.h index f28bc258..70daf773 100644 --- a/arenai_agent/src/networks/misc.h +++ b/arenai_agent/src/networks/misc.h @@ -9,33 +9,40 @@ namespace arenai::agent { - class Clamp : public torch::nn::Module { + class AbstractFunctionModule : public torch::nn::Module { + public: + virtual torch::Tensor forward(const torch::Tensor &input) = 0; + + virtual void pretty_print(std::ostream &stream) = 0; + }; + + class Clamp : public AbstractFunctionModule { public: Clamp(float lower_bound, float upper_bound); - torch::Tensor forward(const torch::Tensor &x); + torch::Tensor forward(const torch::Tensor &x) override; - void pretty_print(std::ostream &stream) const override; + void pretty_print(std::ostream &stream) override; private: float lower_bound; float upper_bound; }; - class Exp : public torch::nn::Module { + class Exp : public AbstractFunctionModule { public: - torch::Tensor forward(const torch::Tensor &x); + torch::Tensor forward(const torch::Tensor &x) override; - void pretty_print(std::ostream &stream) const override; + void pretty_print(std::ostream &stream) override; }; - class SigmaOutput : public torch::nn::Module { + class SigmaOutput : public AbstractFunctionModule { public: SigmaOutput(float min_sigma, float max_sigma); - torch::Tensor forward(const torch::Tensor &input); + torch::Tensor forward(const torch::Tensor &input) override; - void pretty_print(std::ostream &stream) const override; + void pretty_print(std::ostream &stream) override; private: float min_log_sigma; diff --git a/arenai_agent/src/networks/q_function.cpp b/arenai_agent/src/networks/q_function.cpp index 9680021a..dcd91ee9 100644 --- a/arenai_agent/src/networks/q_function.cpp +++ b/arenai_agent/src/networks/q_function.cpp @@ -18,9 +18,11 @@ namespace arenai::agent { const std::vector &hidden_sizes, const std::vector> &vision_channels, const std::vector &group_norm_nums) - : vision_encoder(register_module( - "vision_encoder", std::make_shared( - vision_height, vision_width, vision_channels, group_norm_nums))), + : nb_discrete_actions(nb_discrete_actions), + vision_encoder(register_module( + "vision_encoder", + std::make_shared( + vision_height, vision_width, vision_channels, group_norm_nums))), sensors_encoder(register_module( "sensors_encoder", torch::nn::Sequential( @@ -79,28 +81,26 @@ namespace arenai::agent { return to_value->forward(head->forward(encoded_hidden)); } - torch::Tensor QFunction::value_expectation( + torch::Tensor QFunction::value_per_discrete_action( const torch::Tensor &vision, const torch::Tensor &sensors, - const torch::Tensor &continuous_actions, const torch::Tensor &discrete_actions_proba) { + const torch::Tensor &continuous_actions) { const auto batch_size = vision.size(0); - const auto nb_discrete_actions = discrete_actions_proba.size(1); const auto common_encoded = encode_common(vision, sensors, continuous_actions); - const auto one_hots = torch::eye(nb_discrete_actions, discrete_actions_proba.options()); + const auto one_hots = torch::eye(nb_discrete_actions, common_encoded.options()); - auto result = torch::zeros({batch_size, 1}, common_encoded.options()); + std::vector q_values; + q_values.reserve(nb_discrete_actions); for (int a = 0; a < nb_discrete_actions; a++) { const auto discrete_encoded = discrete_action_encoder->forward(one_hots[a].unsqueeze(0).expand({batch_size, -1})); - const auto q_a = - to_value->forward(head->forward(torch::cat({common_encoded, discrete_encoded}, 1))); - - result = result + discrete_actions_proba.select(1, a).unsqueeze(1) * q_a; + q_values.push_back(to_value->forward( + head->forward(torch::cat({common_encoded, discrete_encoded}, 1)))); } - return result; + return torch::cat(q_values, 1); } torch::Tensor QFunction::encode_common( diff --git a/arenai_agent/src/networks/q_function.h b/arenai_agent/src/networks/q_function.h index a5bf1336..fc7ac36a 100644 --- a/arenai_agent/src/networks/q_function.h +++ b/arenai_agent/src/networks/q_function.h @@ -26,11 +26,12 @@ namespace arenai::agent { const torch::Tensor &vision, const torch::Tensor &sensors, const torch::Tensor &continuous_actions, const torch::Tensor &discrete_action_ohe); - torch::Tensor value_expectation( + torch::Tensor value_per_discrete_action( const torch::Tensor &vision, const torch::Tensor &sensors, - const torch::Tensor &continuous_actions, const torch::Tensor &discrete_actions_proba); + const torch::Tensor &continuous_actions); private: + int nb_discrete_actions; std::shared_ptr vision_encoder; torch::nn::Sequential sensors_encoder; torch::nn::Sequential continuous_action_encoder; diff --git a/arenai_agent/src/networks/vision.cpp b/arenai_agent/src/networks/vision.cpp index edbec3c9..a51039fa 100644 --- a/arenai_agent/src/networks/vision.cpp +++ b/arenai_agent/src/networks/vision.cpp @@ -39,7 +39,7 @@ namespace arenai::agent { } torch::Tensor ConvolutionNetwork::forward(const torch::Tensor &input) { - if (input.dtype() != torch::kUInt8) throw std::runtime_error("Input must be UInt8"); + TORCH_CHECK(input.dtype() == torch::kUInt8, "Input must be UInt8"); return cnn->forward(input.to(torch::kFloat).mul_(2.0f / 255.0f).add_(-1.0f)); } diff --git a/arenai_agent/src/networks_utils/init.cpp b/arenai_agent/src/networks_utils/init.cpp index 25caef68..eb04e53f 100644 --- a/arenai_agent/src/networks_utils/init.cpp +++ b/arenai_agent/src/networks_utils/init.cpp @@ -53,10 +53,19 @@ namespace arenai::agent { } } - void init_discrete_output_weights(torch::nn::Module &module) { + void + init_discrete_output_weights(torch::nn::Module &module, const float initial_fire_probability) { if (auto *lin = module.as()) { torch::nn::init::orthogonal_(lin->weight, 0.01f); - if (lin->options.bias()) torch::nn::init::zeros_(lin->bias); + + if (lin->options.bias()) { + torch::nn::init::zeros_(lin->bias); + + lin->bias.data().index_fill_( + 0, torch::tensor({0}), std::log(initial_fire_probability)); + lin->bias.data().index_fill_( + 0, torch::tensor({1}), std::log(1.f - initial_fire_probability)); + } } } diff --git a/arenai_agent/src/networks_utils/init.h b/arenai_agent/src/networks_utils/init.h index b3af2fd6..612c234c 100644 --- a/arenai_agent/src/networks_utils/init.h +++ b/arenai_agent/src/networks_utils/init.h @@ -7,13 +7,15 @@ #include +#include "../networks/constants.h" + namespace arenai::agent { void init_hidden_weights(torch::nn::Module &module); void init_mu_output_weights(torch::nn::Module &module); - void init_sigma_output_weights(torch::nn::Module &module, float wanted_sigma = 0.5f); - void init_discrete_output_weights(torch::nn::Module &module); + void init_sigma_output_weights(torch::nn::Module &module, float wanted_sigma); + void init_discrete_output_weights(torch::nn::Module &module, float initial_fire_probability); void init_value_output_weights(torch::nn::Module &module); diff --git a/arenai_agent/src/networks_utils/torch_converter.cpp b/arenai_agent/src/networks_utils/torch_converter.cpp index 764f42c1..74e7061f 100644 --- a/arenai_agent/src/networks_utils/torch_converter.cpp +++ b/arenai_agent/src/networks_utils/torch_converter.cpp @@ -78,28 +78,23 @@ namespace arenai::agent { } TorchStep steps_to_tensor( - const std::vector> - &steps, + const std::vector> &steps, const int vision_height, const int vision_width) { std::vector states; std::vector rewards; std::vector are_done; - std::vector are_truncated; - for (const auto &[state, reward, is_done, is_truncated]: steps) { + for (const auto &[state, reward, is_done]: steps) { states.push_back(state); rewards.push_back(torch::tensor({reward}, torch::TensorOptions().dtype(torch::kFloat))); are_done.push_back( torch::tensor({is_done}, torch::TensorOptions().dtype(torch::kBool))); - are_truncated.push_back( - torch::tensor({is_truncated}, torch::TensorOptions().dtype(torch::kBool))); } return { .states = states_to_tensor(states, vision_height, vision_width), .rewards = torch::stack(rewards), - .is_done = torch::stack(are_done), - .is_truncated = torch::stack(are_truncated)}; + .is_done = torch::stack(are_done)}; } }// namespace arenai::agent diff --git a/arenai_agent/src/networks_utils/torch_converter.h b/arenai_agent/src/networks_utils/torch_converter.h index 53c93b03..3063e1dd 100644 --- a/arenai_agent/src/networks_utils/torch_converter.h +++ b/arenai_agent/src/networks_utils/torch_converter.h @@ -23,8 +23,7 @@ namespace arenai::agent { TorchState state_to_tensor(const core::State &state, int vision_height, int vision_width); TorchStep steps_to_tensor( - const std::vector> - &steps, + const std::vector> &steps, int vision_height, int vision_width); }// namespace arenai::agent diff --git a/arenai_agent/src/networks_utils/torch_loader.h b/arenai_agent/src/networks_utils/torch_loader.h index 6ad13577..3ad16973 100644 --- a/arenai_agent/src/networks_utils/torch_loader.h +++ b/arenai_agent/src/networks_utils/torch_loader.h @@ -26,7 +26,7 @@ namespace arenai::agent { torch::serialize::InputArchive archive; archive.load_from(file.string(), torch::kCPU); to_load->load(archive); - } catch (const std::exception &e) { + } catch (const std::exception &_) { std::throw_with_nested(utils::ModelLoadException(file)); } } diff --git a/arenai_agent/src/train.cpp b/arenai_agent/src/train.cpp index f3bcf7c2..64ff74b5 100644 --- a/arenai_agent/src/train.cpp +++ b/arenai_agent/src/train.cpp @@ -12,7 +12,6 @@ #include #include "./core/train_environment.h" -#include "./metrics/mean_metric.h" #include "./metrics/metric_saver.h" #include "./networks_utils/torch_converter.h" #include "./networks_utils/torch_saver.h" @@ -66,12 +65,11 @@ namespace arenai::agent { AgentSaver saver(trainer, train_options.output_folder, train_options.save_every); // metrics - auto reward_mean_metric = std::make_shared("r", 256, 2, true); const auto sac_metrics = trainer->get_metrics(); const auto env_metrics = env->get_metrics(); - std::vector> metrics = {reward_mean_metric}; + std::vector> metrics; metrics.insert(metrics.end(), env_metrics.begin(), env_metrics.end()); metrics.insert(metrics.end(), sac_metrics.begin(), sac_metrics.end()); @@ -110,15 +108,14 @@ namespace arenai::agent { env->reset(spawn_width, spawn_height), environment_options.vision_height, environment_options.vision_width); - auto last_phi_tensor = torch::tensor(env->get_phi_vector()).unsqueeze(1); - while (!is_done) { std::vector actions_for_env; const auto [continuous_action, discrete_action] = agent->act( {.vision = vision.to(torch_device), - .proprioception = proprioception.to(torch_device)}); + .proprioception = proprioception.to(torch_device)}, + true); TorchAction torch_action = { .continuous_action = continuous_action, .discrete_action = discrete_action}; @@ -127,27 +124,16 @@ namespace arenai::agent { // step environment const auto steps = env->step(environment_options.wanted_frequency, actions_for_env); - const auto phi_tensor = torch::tensor(env->get_phi_vector()).unsqueeze(1); - - const auto [torch_next_states, torch_rewards, torch_are_done, torch_are_truncated] = - steps_to_tensor( - steps, environment_options.vision_height, environment_options.vision_width); - - const auto terminal_mask = torch::logical_not( - torch::logical_and(torch_are_done, torch::logical_not(torch_are_truncated))); - const auto potential_reward = - terminal_mask * train_options.potential_reward_gamma * phi_tensor - - last_phi_tensor; - const auto torch_final_reward = torch_rewards + potential_reward; + const auto [torch_next_states, torch_rewards, torch_are_done] = steps_to_tensor( + steps, environment_options.vision_height, environment_options.vision_width); // complete the pending transition - maybe train - collector->on_transition(torch_final_reward, torch_are_done, torch_are_truncated); + collector->on_transition(torch_rewards, torch_are_done); trainer->step(); // step ending stuff is_done = env->is_episode_terminated(); - last_phi_tensor = phi_tensor; vision = torch_next_states.vision; proprioception = torch_next_states.proprioception; @@ -158,19 +144,6 @@ namespace arenai::agent { saver.attempt_save(); metric_csv_saver.attempt_append_to_csv(); - // metrics: mean reward over the tanks still in play, dead tanks emit a - // post-mortem -1 every frame that would drown the signal - const auto valid_mask = env->get_valid_mask(); - float valid_reward_sum = 0.f; - int nb_valid = 0; - for (size_t i = 0; i < steps.size(); i++) { - if (!valid_mask[i]) continue; - valid_reward_sum += std::get<1>(steps[i]); - nb_valid++; - } - if (nb_valid > 0) - reward_mean_metric->add(valid_reward_sum / static_cast(nb_valid)); - // progress bar metrics display if (print_counter == print_tqdm_bar_every - 1) { std::stringstream stream; diff --git a/arenai_agent/src/train.h b/arenai_agent/src/train.h index 213fabc9..639f9f7d 100644 --- a/arenai_agent/src/train.h +++ b/arenai_agent/src/train.h @@ -15,7 +15,6 @@ namespace arenai::agent { struct TrainOptions { std::filesystem::path output_folder; std::filesystem::path resources_folder; - float potential_reward_gamma; int max_episode_steps; int nb_episodes; int save_every; diff --git a/arenai_agent/src/utils/image_write.cpp b/arenai_agent/src/utils/image_write.cpp index 6e1a0b5d..0a0f1b03 100644 --- a/arenai_agent/src/utils/image_write.cpp +++ b/arenai_agent/src/utils/image_write.cpp @@ -8,6 +8,8 @@ #define STB_IMAGE_WRITE_STATIC #define STB_IMAGE_WRITE_IMPLEMENTATION +#include + #include "./image_writer.h" using namespace arenai; diff --git a/arenai_agent/src/utils/image_writer.h b/arenai_agent/src/utils/image_writer.h index 16b8062e..b854fecd 100644 --- a/arenai_agent/src/utils/image_writer.h +++ b/arenai_agent/src/utils/image_writer.h @@ -5,10 +5,8 @@ #ifndef ARENAI_AGENT_HOST_IMAGE_WRITER_H #define ARENAI_AGENT_HOST_IMAGE_WRITER_H -#include #include -#include #include namespace arenai::agent { diff --git a/arenai_agent/tests/include/arenai_agent_tests/tests_agents/tests_sac_training.h b/arenai_agent/tests/include/arenai_agent_tests/tests_agents/tests_sac_training.h index cd8bdc0e..4f36fd38 100644 --- a/arenai_agent/tests/include/arenai_agent_tests/tests_agents/tests_sac_training.h +++ b/arenai_agent/tests/include/arenai_agent_tests/tests_agents/tests_sac_training.h @@ -5,7 +5,6 @@ #ifndef ARENAI_TESTS_SAC_TRAINING_H #define ARENAI_TESTS_SAC_TRAINING_H -#include #include #include diff --git a/arenai_agent/tests/include/arenai_agent_tests/tests_distributions/tests_truncated_normal.h b/arenai_agent/tests/include/arenai_agent_tests/tests_distributions/tests_truncated_normal.h index 7e675690..af280dbb 100644 --- a/arenai_agent/tests/include/arenai_agent_tests/tests_distributions/tests_truncated_normal.h +++ b/arenai_agent/tests/include/arenai_agent_tests/tests_distributions/tests_truncated_normal.h @@ -7,8 +7,8 @@ #include -typedef int UpperBound; -typedef int LowerBound; +typedef float UpperBound; +typedef float LowerBound; typedef std::vector Shape; diff --git a/arenai_agent/tests/include/arenai_agent_tests/tests_e2e/interceptor_agent.h b/arenai_agent/tests/include/arenai_agent_tests/tests_e2e/interceptor_agent.h index 9f026908..d7a72b60 100644 --- a/arenai_agent/tests/include/arenai_agent_tests/tests_e2e/interceptor_agent.h +++ b/arenai_agent/tests/include/arenai_agent_tests/tests_e2e/interceptor_agent.h @@ -12,20 +12,23 @@ // Records every batch of states the host loop hands to the agent and answers // with neutral actions: what act() received is exactly what a real network // would have seen. -class InterceptorAgent final : public arenai::agent::AbstractAgent { +class InterceptorAgent final : public agent::AbstractAgent { public: - std::vector> received_states; + std::vector> received_states; int last_vision_height = -1; int last_vision_width = -1; - std::vector - act(const std::vector &states, const int vision_height, + std::vector + act(const std::vector &states, const int vision_height, const int vision_width) override { received_states.push_back(states); last_vision_height = vision_height; last_vision_width = vision_width; - return std::vector(states.size(), {{0.f, 0.f}, {0.f, 0.f}, {false}}); + return std::vector( + states.size(), {.left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {false}}); } void load(const std::filesystem::path &agent_folder) override {} diff --git a/arenai_agent/tests/include/arenai_agent_tests/tests_networks/tests_actor.h b/arenai_agent/tests/include/arenai_agent_tests/tests_networks/tests_actor.h index 26a253df..4e658e18 100644 --- a/arenai_agent/tests/include/arenai_agent_tests/tests_networks/tests_actor.h +++ b/arenai_agent/tests/include/arenai_agent_tests/tests_networks/tests_actor.h @@ -8,13 +8,13 @@ #include typedef std::vector HiddenLayers; -typedef uint32_t ContinuousActionsNb; -typedef uint32_t DiscreteActionsNb; +typedef int ContinuousActionsNb; +typedef int DiscreteActionsNb; -typedef uint32_t SensorsNb; -typedef uint32_t SensorsHiddenSize; +typedef int SensorsNb; +typedef int SensorsHiddenSize; -typedef uint32_t BatchSize; +typedef int BatchSize; class ActorTestParam : public testing::TestWithParam typedef std::vector HiddenLayers; -typedef uint32_t ContinuousActionsNb; -typedef uint32_t DiscreteActionsNb; +typedef int ContinuousActionsNb; +typedef int DiscreteActionsNb; -typedef uint32_t SensorsNb; -typedef uint32_t SensorsHiddenSize; +typedef int SensorsNb; +typedef int SensorsHiddenSize; -typedef uint32_t ActionsHiddenSize; +typedef int ActionsHiddenSize; -typedef uint32_t BatchSize; +typedef int BatchSize; class QFunctionTestParam : public testing::TestWithParam -typedef uint32_t VisionWidth; -typedef uint32_t VisionHeight; -typedef uint32_t VisionChannel; +typedef int VisionWidth; +typedef int VisionHeight; +typedef int VisionChannel; typedef std::vector OutputConvChannels; typedef std::vector GroupNormNums; -typedef uint32_t BatchSize; +typedef int BatchSize; class VisionTestParam : public testing::TestWithParam + +#include +#include +#include +#include + +using namespace arenai; +using namespace arenai::agent; + +namespace { + + TorchState probe_state(const int batch, const int h, const int w, const int nb_sensors) { + torch::manual_seed(42); + return { + .vision = torch::randint(0, 255, {batch, 3, h, w}, torch::kUInt8), + .proprioception = torch::randn({batch, nb_sensors})}; + } + +}// namespace + +TEST(ProbeActSample, SacStatsBiasedMu) { + torch::manual_seed(1234); + constexpr int h = 8, w = 8, nb_sensors = 4, nb_cont = 2, nb_disc = 2; + + const auto actor = std::make_shared( + h, w, nb_sensors, nb_cont, nb_disc, 8, std::vector{16}, + std::vector>{{3, 4}}, std::vector{2}, 0.3f, 0.1f); + + // push mu away from 0 : mu ~ tanh(bias) + { + torch::NoGradGuard guard; + for (auto &p: actor->named_parameters()) + if (p.key().find("mu") != std::string::npos + && p.key().find("bias") != std::string::npos) + p.value().copy_(torch::tensor({std::atanh(0.7f), std::atanh(-0.4f)})); + } + + const auto agent = std::make_shared(actor, torch::Device(torch::kCPU)); + const auto state = probe_state(1, h, w, nb_sensors); + + torch::NoGradGuard guard; + const auto [mu, sigma, disc] = actor->act(state.vision, state.proprioception); + std::cout << "mu: " << mu << "\nsigma: " << sigma << std::endl; + + constexpr int N = 4000; + std::vector conts; + for (int i = 0; i < N; i++) conts.push_back(agent->act(state, true).continuous_action); + const auto cont_all = torch::cat(conts, 0); + std::cout << "act(true) cont mean: " << cont_all.mean(0) + << "\nact(true) cont std: " << cont_all.std(0) << std::endl; +} + +TEST(ProbeActSample, PpoLogProbs) { + torch::manual_seed(99); + constexpr int h = 8, w = 8, nb_sensors = 4, nb_cont = 2, nb_disc = 2; + + const auto actor = std::make_shared( + h, w, nb_sensors, nb_cont, nb_disc, 8, std::vector{16}, + std::vector>{{3, 4}}, std::vector{2}, 0.3f, 0.2f); + const auto rollout_buffer = std::make_shared(); + const auto collector = std::make_shared(rollout_buffer); + const auto agent = + std::make_shared(actor, torch::Device(torch::kCPU), collector); + + const auto state = probe_state(3, h, w, nb_sensors); + + const auto action = agent->act(state, true); + collector->on_transition(torch::randn({3, 1}), torch::zeros({3, 1})); + collector->on_episode_end(state); + + torch::NoGradGuard guard; + const auto [mu, sigma, disc] = actor->act(state.vision, state.proprioception); + const auto expected_cont = + truncated_normal_log_pdf(action.continuous_action, mu, sigma).sum(-1, true); + const auto expected_disc = + (action.discrete_action * torch::log(torch::clamp(disc, 1e-8, 1.0 - 1e-8))).sum(-1, true); + + const auto rollout = rollout_buffer->get_rollout(); + std::cout << "stored cont lp: " << rollout.continuous_log_probs + << "\nexpected cont lp: " << expected_cont + << "\nstored disc lp: " << rollout.discrete_log_probs + << "\nexpected disc lp: " << expected_disc << std::endl; +} + +TEST(ProbeActSample, SacStats) { + torch::manual_seed(1234); + constexpr int h = 8, w = 8, nb_sensors = 4, nb_cont = 3, nb_disc = 2; + + const auto actor = std::make_shared( + h, w, nb_sensors, nb_cont, nb_disc, 8, std::vector{16}, + std::vector>{{3, 4}}, std::vector{2}, 0.4f, 0.1f); + const auto agent = std::make_shared(actor, torch::Device(torch::kCPU)); + + const auto state = probe_state(1, h, w, nb_sensors); + + // raw actor output + torch::NoGradGuard guard; + const auto [mu, sigma, disc] = actor->act(state.vision, state.proprioception); + std::cout << "mu: " << mu << "\nsigma: " << sigma << "\ndisc_proba: " << disc << std::endl; + + // deterministic + const auto det = agent->act(state, false); + std::cout << "act(false) cont: " << det.continuous_action + << "\nact(false) disc: " << det.discrete_action << std::endl; + std::cout << "cont == mu ? " << torch::allclose(det.continuous_action, mu) << std::endl; + + // stochastic stats + constexpr int N = 2000; + std::vector conts, discs; + for (int i = 0; i < N; i++) { + const auto a = agent->act(state, true); + conts.push_back(a.continuous_action); + discs.push_back(a.discrete_action); + } + const auto cont_all = torch::cat(conts, 0); + const auto disc_all = torch::cat(discs, 0); + std::cout << "act(true) cont mean: " << cont_all.mean(0) + << "\nact(true) cont std: " << cont_all.std(0) + << "\nact(true) cont min: " << std::get<0>(cont_all.min(0)) + << "\nact(true) cont max: " << std::get<0>(cont_all.max(0)) + << "\nact(true) disc freq: " << disc_all.mean(0) << std::endl; +} diff --git a/arenai_agent/tests/src/tests_agents/tests_ppo.cpp b/arenai_agent/tests/src/tests_agents/tests_ppo.cpp index f6b9d06e..dbf40e87 100644 --- a/arenai_agent/tests/src/tests_agents/tests_ppo.cpp +++ b/arenai_agent/tests/src/tests_agents/tests_ppo.cpp @@ -32,8 +32,6 @@ std::unique_ptr PpoAgentTest::make_factory(const PpoTestCo .gae_lambda = 0.95f, .clip_epsilon = 0.2f, .grad_norm_max = 1.f, - .continuous_entropy_coef = 0.01f, - .discrete_entropy_coef = 0.01f, .epochs = 1, .rollout_size = 8, .minibatch_size = 10}; @@ -45,8 +43,9 @@ std::unique_ptr PpoAgentTest::make_factory(const PpoTestCo TorchState PpoAgentTest::make_state(const PpoTestConfig &cfg, const int batch) { return { - torch::randint(0, 255, {batch, 3, cfg.vision_height, cfg.vision_width}, torch::kUInt8), - torch::randn({batch, cfg.nb_sensors})}; + .vision = + torch::randint(0, 255, {batch, 3, cfg.vision_height, cfg.vision_width}, torch::kUInt8), + .proprioception = torch::randn({batch, cfg.nb_sensors})}; } // ======================================================================== @@ -54,19 +53,29 @@ TorchState PpoAgentTest::make_state(const PpoTestConfig &cfg, const int batch) { // ======================================================================== TEST_F(PpoAgentTest, ParameterCountPositive) { - constexpr PpoTestConfig cfg{8, 8, 10, 4, 2}; + constexpr PpoTestConfig cfg{ + .vision_height = 8, + .vision_width = 8, + .nb_sensors = 10, + .nb_continuous_actions = 4, + .nb_discrete_actions = 2}; const auto factory = make_factory(cfg); ASSERT_GT(factory->get_trainer()->count_parameters(), 0); } TEST_F(PpoAgentTest, MetricsNotEmpty) { - constexpr PpoTestConfig cfg{8, 8, 10, 4, 2}; + constexpr PpoTestConfig cfg{ + .vision_height = 8, + .vision_width = 8, + .nb_sensors = 10, + .nb_continuous_actions = 4, + .nb_discrete_actions = 2}; const auto factory = make_factory(cfg); const auto metrics = factory->get_trainer()->get_metrics(); - ASSERT_EQ(metrics.size(), 9); + ASSERT_EQ(metrics.size(), 13); } // ======================================================================== @@ -79,7 +88,7 @@ TEST_P(PpoActShapeParamTest, ActOutputShapes) { constexpr int batch = 4; const auto [continuous_action, discrete_action] = - factory->get_agent()->act(make_state(cfg, batch)); + factory->get_agent()->act(make_state(cfg, batch), true); ASSERT_EQ(continuous_action.size(0), batch); ASSERT_EQ(continuous_action.size(1), cfg.nb_continuous_actions); @@ -94,7 +103,7 @@ TEST_P(PpoActShapeParamTest, ActContinuousFinite) { constexpr int batch = 4; const auto [continuous_action, discrete_action] = - factory->get_agent()->act(make_state(cfg, batch)); + factory->get_agent()->act(make_state(cfg, batch), true); ASSERT_TRUE(torch::all(torch::isfinite(continuous_action)).item()); } @@ -105,7 +114,7 @@ TEST_P(PpoActShapeParamTest, ActDiscreteIsOneHot) { constexpr int batch = 4; const auto [continuous_action, discrete_action] = - factory->get_agent()->act(make_state(cfg, batch)); + factory->get_agent()->act(make_state(cfg, batch), true); const auto row_sums = torch::sum(discrete_action, -1); ASSERT_TRUE(torch::allclose(row_sums, torch::ones({batch}))); diff --git a/arenai_agent/tests/src/tests_agents/tests_ppo_rollout_buffer.cpp b/arenai_agent/tests/src/tests_agents/tests_ppo_rollout_buffer.cpp index 97f431e0..e4029a07 100644 --- a/arenai_agent/tests/src/tests_agents/tests_ppo_rollout_buffer.cpp +++ b/arenai_agent/tests/src/tests_agents/tests_ppo_rollout_buffer.cpp @@ -13,8 +13,8 @@ using namespace arenai::agent; TorchState PpoRolloutBufferTest::make_state() { return { - torch::randn({NB_TANKS, 3, VISION_SIZE, VISION_SIZE}), - torch::randn({NB_TANKS, NB_SENSORS})}; + .vision = torch::randn({NB_TANKS, 3, VISION_SIZE, VISION_SIZE}), + .proprioception = torch::randn({NB_TANKS, NB_SENSORS})}; } PpoInputStep PpoRolloutBufferTest::make_step(const TorchState &state, const torch::Tensor &done) { @@ -30,8 +30,7 @@ PpoInputStep PpoRolloutBufferTest::make_step(const TorchState &state, const torc .continuous_log_prob = torch::randn({NB_TANKS, 1}), .discrete_log_prob = torch::randn({NB_TANKS, 1}), .reward = torch::randn({NB_TANKS, 1}), - .done = done, - .truncated = torch::zeros({NB_TANKS, 1})}; + .done = done}; } PpoInputStep PpoRolloutBufferTest::make_step(const TorchState &state) { diff --git a/arenai_agent/tests/src/tests_agents/tests_ppo_training.cpp b/arenai_agent/tests/src/tests_agents/tests_ppo_training.cpp index 69fb8c02..f433b5b0 100644 --- a/arenai_agent/tests/src/tests_agents/tests_ppo_training.cpp +++ b/arenai_agent/tests/src/tests_agents/tests_ppo_training.cpp @@ -21,8 +21,6 @@ PpoTrainingTest::make_factory(const PpoTrainingTestConfig &cfg) const { .gamma = 0.99f, .gae_lambda = 0.95f, .clip_epsilon = 0.2f, - .continuous_entropy_coef = 0.01f, - .discrete_entropy_coef = 0.01f, .epochs = 2, .rollout_size = ROLLOUT_SIZE, .minibatch_size = MINIBATCH_SIZE}; @@ -39,10 +37,16 @@ TorchState PpoTrainingTest::make_state(const PpoTrainingTestConfig &cfg, const i } TEST_F(PpoTrainingTest, ActProducesValidOutput) { - constexpr PpoTrainingTestConfig cfg{8, 8, 3, 2, 3}; + constexpr PpoTrainingTestConfig cfg{ + .vision_height = 8, + .vision_width = 8, + .nb_sensors = 3, + .nb_continuous_actions = 2, + .nb_discrete_actions = 3}; const auto factory = make_factory(cfg); - const auto [continuous_action, discrete_action] = factory->get_agent()->act(make_state(cfg, 1)); + const auto [continuous_action, discrete_action] = + factory->get_agent()->act(make_state(cfg, 1), true); ASSERT_EQ(continuous_action.size(0), 1); ASSERT_EQ(continuous_action.size(1), 2); @@ -54,7 +58,12 @@ TEST_F(PpoTrainingTest, ActProducesValidOutput) { } TEST_F(PpoTrainingTest, CountParametersPositive) { - constexpr PpoTrainingTestConfig cfg{8, 8, 3, 2, 3}; + constexpr PpoTrainingTestConfig cfg{ + .vision_height = 8, + .vision_width = 8, + .nb_sensors = 3, + .nb_continuous_actions = 2, + .nb_discrete_actions = 3}; const auto factory = make_factory(cfg); ASSERT_GT(factory->get_trainer()->count_parameters(), 0) @@ -63,23 +72,28 @@ TEST_F(PpoTrainingTest, CountParametersPositive) { TEST_F(PpoTrainingTest, TrainingUpdatesActorParameters) { // build the triad by hand to keep a handle on the actor's parameters - constexpr PpoTrainingTestConfig cfg{8, 8, 3, 2, 3}; - constexpr int nb_tanks = 2; + constexpr PpoTrainingTestConfig cfg{ + .vision_height = 8, + .vision_width = 8, + .nb_sensors = 3, + .nb_continuous_actions = 2, + .nb_discrete_actions = 3}; const std::vector> vision_channels{{3, 4}}; - const std::vector group_norm_nums{2}; + const std::vector group_norm_nums{2}; const auto actor = std::make_shared( cfg.vision_height, cfg.vision_width, cfg.nb_sensors, cfg.nb_continuous_actions, - cfg.nb_discrete_actions, 8, std::vector{16}, vision_channels, group_norm_nums); + cfg.nb_discrete_actions, 8, std::vector{16}, vision_channels, group_norm_nums, 0.1f, 0.2f); const auto rollout_buffer = std::make_shared(); const auto collector = std::make_shared(rollout_buffer); const auto agent = std::make_shared(actor, device, collector); // target_kl = 0 : early stop disabled so every minibatch applies its update const auto trainer = std::make_shared( - actor, rollout_buffer, cfg.vision_height, cfg.vision_width, cfg.nb_sensors, 1e-3f, 1e-3f, 8, - std::vector{16}, vision_channels, group_norm_nums, device, 10, 0.99f, 0.95f, 0.2f, 0.f, - 1.f, 0.01f, 0.01f, 2, ROLLOUT_SIZE, MINIBATCH_SIZE); + actor, rollout_buffer, cfg.vision_height, cfg.vision_width, cfg.nb_sensors, + cfg.nb_continuous_actions, 1e-3f, 1e-3f, 1e-3f, 8, std::vector{16}, vision_channels, + group_norm_nums, device, 10, 0.99f, 0.95f, 0.2f, 0.f, 1.f, 2e-3f, 2e-3f, 2, ROLLOUT_SIZE, + MINIBATCH_SIZE); std::vector initial_parameters; for (const auto ¶meter: actor->parameters()) @@ -88,9 +102,10 @@ TEST_F(PpoTrainingTest, TrainingUpdatesActorParameters) { // env loop: act -> transition -> maybe train, one more step than the rollout // horizon so that the batch is complete when the trainer checks for (int t = 0; t < ROLLOUT_SIZE + 2; t++) { - agent->act(make_state(cfg, nb_tanks)); - collector->on_transition( - torch::randn({nb_tanks, 1}), torch::zeros({nb_tanks, 1}), torch::zeros({nb_tanks, 1})); + constexpr int nb_tanks = 2; + + agent->act(make_state(cfg, nb_tanks), true); + collector->on_transition(torch::randn({nb_tanks, 1}), torch::zeros({nb_tanks, 1})); trainer->step(); } diff --git a/arenai_agent/tests/src/tests_agents/tests_sac.cpp b/arenai_agent/tests/src/tests_agents/tests_sac.cpp index b75ad29c..2a2905c7 100644 --- a/arenai_agent/tests/src/tests_agents/tests_sac.cpp +++ b/arenai_agent/tests/src/tests_agents/tests_sac.cpp @@ -44,8 +44,9 @@ std::unique_ptr SacAgentTest::make_factory(const SacTestCo TorchState SacAgentTest::make_state(const SacTestConfig &cfg, const int batch) { return { - torch::randint(0, 255, {batch, 3, cfg.vision_height, cfg.vision_width}, torch::kUInt8), - torch::randn({batch, cfg.nb_sensors})}; + .vision = + torch::randint(0, 255, {batch, 3, cfg.vision_height, cfg.vision_width}, torch::kUInt8), + .proprioception = torch::randn({batch, cfg.nb_sensors})}; } // ======================================================================== @@ -78,7 +79,7 @@ TEST_P(SacActShapeParamTest, ActOutputShapes) { constexpr int batch = 4; const auto [continuous_action, discrete_action] = - factory->get_agent()->act(make_state(cfg, batch)); + factory->get_agent()->act(make_state(cfg, batch), true); ASSERT_EQ(continuous_action.size(0), batch); ASSERT_EQ(continuous_action.size(1), cfg.nb_continuous_actions); @@ -93,7 +94,7 @@ TEST_P(SacActShapeParamTest, ActContinuousFinite) { constexpr int batch = 4; const auto [continuous_action, discrete_action] = - factory->get_agent()->act(make_state(cfg, batch)); + factory->get_agent()->act(make_state(cfg, batch), true); ASSERT_TRUE(torch::all(torch::isfinite(continuous_action)).item()); } @@ -104,7 +105,7 @@ TEST_P(SacActShapeParamTest, ActDiscreteIsOneHot) { constexpr int batch = 4; const auto [continuous_action, discrete_action] = - factory->get_agent()->act(make_state(cfg, batch)); + factory->get_agent()->act(make_state(cfg, batch), true); const auto row_sums = torch::sum(discrete_action, -1); ASSERT_TRUE(torch::allclose(row_sums, torch::ones({batch}))); diff --git a/arenai_agent/tests/src/tests_agents/tests_sac_training.cpp b/arenai_agent/tests/src/tests_agents/tests_sac_training.cpp index e4feb314..a7f259a0 100644 --- a/arenai_agent/tests/src/tests_agents/tests_sac_training.cpp +++ b/arenai_agent/tests/src/tests_agents/tests_sac_training.cpp @@ -34,15 +34,22 @@ SacTrainingTest::make_factory(const SacTrainingTestConfig &cfg) const { TorchState SacTrainingTest::make_state(const SacTrainingTestConfig &cfg) { return { - torch::randint(0, 255, {1, 3, cfg.vision_height, cfg.vision_width}, torch::kUInt8), - torch::randn({1, cfg.nb_sensors})}; + .vision = + torch::randint(0, 255, {1, 3, cfg.vision_height, cfg.vision_width}, torch::kUInt8), + .proprioception = torch::randn({1, cfg.nb_sensors})}; } TEST_F(SacTrainingTest, ActProducesValidOutput) { - constexpr SacTrainingTestConfig cfg{8, 8, 3, 2, 3}; + constexpr SacTrainingTestConfig cfg{ + .vision_height = 8, + .vision_width = 8, + .nb_sensors = 3, + .nb_continuous_actions = 2, + .nb_discrete_actions = 3}; const auto factory = make_factory(cfg); - const auto [continuous_action, discrete_action] = factory->get_agent()->act(make_state(cfg)); + const auto [continuous_action, discrete_action] = + factory->get_agent()->act(make_state(cfg), true); ASSERT_EQ(continuous_action.size(0), 1); ASSERT_EQ(continuous_action.size(1), 2); @@ -54,7 +61,12 @@ TEST_F(SacTrainingTest, ActProducesValidOutput) { } TEST_F(SacTrainingTest, CountParametersPositive) { - constexpr SacTrainingTestConfig cfg{8, 8, 3, 2, 3}; + constexpr SacTrainingTestConfig cfg{ + .vision_height = 8, + .vision_width = 8, + .nb_sensors = 3, + .nb_continuous_actions = 2, + .nb_discrete_actions = 3}; const auto factory = make_factory(cfg); ASSERT_GT(factory->get_trainer()->count_parameters(), 0) diff --git a/arenai_agent/tests/src/tests_distributions/tests_beta_law.cpp b/arenai_agent/tests/src/tests_distributions/tests_beta_law.cpp index 0085ac0c..bb534ecc 100644 --- a/arenai_agent/tests/src/tests_distributions/tests_beta_law.cpp +++ b/arenai_agent/tests/src/tests_distributions/tests_beta_law.cpp @@ -43,7 +43,7 @@ TEST_F(BetaLawTest, LogProbaConsistentWithSample) { // ======================================================================== TEST_P(BetaLawParamTest, SampleBounds) { - const auto shape = GetParam(); + const auto &shape = GetParam(); const auto alpha = torch::rand(shape) * 4.0f + 0.5f; const auto beta = torch::rand(shape) * 4.0f + 0.5f; @@ -56,7 +56,7 @@ TEST_P(BetaLawParamTest, SampleBounds) { } TEST_P(BetaLawParamTest, LogProbaShape) { - const auto shape = GetParam(); + const auto &shape = GetParam(); const auto alpha = torch::rand(shape) * 4.0f + 0.5f; const auto beta = torch::rand(shape) * 4.0f + 0.5f; @@ -69,7 +69,7 @@ TEST_P(BetaLawParamTest, LogProbaShape) { } TEST_P(BetaLawParamTest, EntropyShape) { - const auto shape = GetParam(); + const auto &shape = GetParam(); const auto alpha = torch::rand(shape) * 4.0f + 0.5f; const auto beta = torch::rand(shape) * 4.0f + 0.5f; @@ -81,7 +81,7 @@ TEST_P(BetaLawParamTest, EntropyShape) { } TEST_P(BetaLawParamTest, SampleNoNaNWithSmallParams) { - const auto shape = GetParam(); + const auto &shape = GetParam(); const auto alpha = torch::ones(shape) * 0.1f; const auto beta = torch::ones(shape) * 0.1f; @@ -94,7 +94,7 @@ TEST_P(BetaLawParamTest, SampleNoNaNWithSmallParams) { } TEST_P(BetaLawParamTest, SampleNoNaNWithLargeParams) { - const auto shape = GetParam(); + const auto &shape = GetParam(); const auto alpha = torch::ones(shape) * 50.0f; const auto beta = torch::ones(shape) * 50.0f; diff --git a/arenai_agent/tests/src/tests_distributions/tests_gaussian_tanh.cpp b/arenai_agent/tests/src/tests_distributions/tests_gaussian_tanh.cpp index f2967ecf..a6b4b9fb 100644 --- a/arenai_agent/tests/src/tests_distributions/tests_gaussian_tanh.cpp +++ b/arenai_agent/tests/src/tests_distributions/tests_gaussian_tanh.cpp @@ -52,7 +52,7 @@ TEST_F(GaussianTanhTest, SmallSigmaConcentratesAction) { // ======================================================================== TEST_P(GaussianTanhShapeParamTest, SampleShapeAndBounds) { - const auto shape = GetParam(); + const auto &shape = GetParam(); const auto mu = torch::randn(shape); const auto sigma = torch::rand(shape) + 0.1f; @@ -66,7 +66,7 @@ TEST_P(GaussianTanhShapeParamTest, SampleShapeAndBounds) { } TEST_P(GaussianTanhShapeParamTest, LogPdfShapeAndFinite) { - const auto shape = GetParam(); + const auto &shape = GetParam(); const auto mu = torch::randn(shape); const auto sigma = torch::rand(shape) + 0.1f; @@ -79,7 +79,7 @@ TEST_P(GaussianTanhShapeParamTest, LogPdfShapeAndFinite) { } TEST_P(GaussianTanhShapeParamTest, SampleFiniteWithZeroMu) { - const auto shape = GetParam(); + const auto &shape = GetParam(); const auto mu = torch::zeros(shape); const auto sigma = torch::ones(shape) * 0.5f; @@ -91,7 +91,7 @@ TEST_P(GaussianTanhShapeParamTest, SampleFiniteWithZeroMu) { } TEST_P(GaussianTanhShapeParamTest, SampleFiniteWithLargeMu) { - const auto shape = GetParam(); + const auto &shape = GetParam(); const auto mu = torch::ones(shape) * 10.0f; const auto sigma = torch::ones(shape) * 0.1f; diff --git a/arenai_agent/tests/src/tests_distributions/tests_gaussian_tanh_edge.cpp b/arenai_agent/tests/src/tests_distributions/tests_gaussian_tanh_edge.cpp index 5427a9d4..15de67e3 100644 --- a/arenai_agent/tests/src/tests_distributions/tests_gaussian_tanh_edge.cpp +++ b/arenai_agent/tests/src/tests_distributions/tests_gaussian_tanh_edge.cpp @@ -85,7 +85,7 @@ TEST_F(GaussianTanhEdgeTest, SampleWithZeroSigma) { // ======================================================================== TEST_F(GaussianTanhGradientTest, LogPdfGradientFlowsThroughMu) { - auto mu = torch::zeros({5}, torch::TensorOptions().requires_grad(true)); + const auto mu = torch::zeros({5}, torch::TensorOptions().requires_grad(true)); const auto sigma = torch::ones({5}) * 0.5f; const auto u = torch::tensor({0.1f, -0.2f, 0.3f, -0.1f, 0.0f}); @@ -103,7 +103,7 @@ TEST_F(GaussianTanhGradientTest, LogPdfGradientFlowsThroughMu) { TEST_F(GaussianTanhGradientTest, LogPdfGradientFlowsThroughSigma) { const auto mu = torch::zeros({5}); - auto sigma = torch::full({5}, 0.5f, torch::TensorOptions().requires_grad(true)); + const auto sigma = torch::full({5}, 0.5f, torch::TensorOptions().requires_grad(true)); const auto u = torch::tensor({0.1f, -0.2f, 0.3f, -0.1f, 0.0f}); const auto log_p = gaussian_tanh_log_pdf(u, mu, sigma); @@ -117,7 +117,7 @@ TEST_F(GaussianTanhGradientTest, LogPdfGradientFlowsThroughSigma) { } TEST_F(GaussianTanhGradientTest, LogPdfGradientFiniteWithLargeU) { - auto mu = torch::zeros({5}, torch::TensorOptions().requires_grad(true)); + const auto mu = torch::zeros({5}, torch::TensorOptions().requires_grad(true)); const auto sigma = torch::ones({5}) * 0.5f; const auto u = torch::ones({5}) * 10.0f; diff --git a/arenai_agent/tests/src/tests_distributions/tests_truncated_normal_edge.cpp b/arenai_agent/tests/src/tests_distributions/tests_truncated_normal_edge.cpp index 377e43c1..c348b690 100644 --- a/arenai_agent/tests/src/tests_distributions/tests_truncated_normal_edge.cpp +++ b/arenai_agent/tests/src/tests_distributions/tests_truncated_normal_edge.cpp @@ -71,7 +71,7 @@ TEST_F(TruncatedNormalEdgeTest, LogPdfAndPdfConsistent) { // ======================================================================== TEST_F(TruncatedNormalGradientTest, LogPdfGradientFlowsThroughMu) { - auto mu = torch::zeros({5}, torch::TensorOptions().requires_grad(true)); + const auto mu = torch::zeros({5}, torch::TensorOptions().requires_grad(true)); const auto sigma = torch::ones({5}) * 0.5f; const auto x = torch::tensor({0.1f, -0.2f, 0.3f, -0.1f, 0.0f}); @@ -89,7 +89,7 @@ TEST_F(TruncatedNormalGradientTest, LogPdfGradientFlowsThroughMu) { TEST_F(TruncatedNormalGradientTest, LogPdfGradientFlowsThroughSigma) { const auto mu = torch::zeros({5}); - auto sigma = torch::full({5}, 0.5f, torch::TensorOptions().requires_grad(true)); + const auto sigma = torch::full({5}, 0.5f, torch::TensorOptions().requires_grad(true)); const auto x = torch::tensor({0.1f, -0.2f, 0.3f, -0.1f, 0.0f}); const auto log_pdf = truncated_normal_log_pdf(x, mu, sigma); @@ -103,7 +103,7 @@ TEST_F(TruncatedNormalGradientTest, LogPdfGradientFlowsThroughSigma) { } TEST_F(TruncatedNormalGradientTest, EntropyGradientFlowsThroughMu) { - auto mu = torch::zeros({5}, torch::TensorOptions().requires_grad(true)); + const auto mu = torch::zeros({5}, torch::TensorOptions().requires_grad(true)); const auto sigma = torch::ones({5}) * 0.5f; const auto entropy = truncated_normal_entropy(mu, sigma); @@ -118,7 +118,7 @@ TEST_F(TruncatedNormalGradientTest, EntropyGradientFlowsThroughMu) { TEST_F(TruncatedNormalGradientTest, EntropyGradientFlowsThroughSigma) { const auto mu = torch::zeros({5}); - auto sigma = torch::full({5}, 0.5f, torch::TensorOptions().requires_grad(true)); + const auto sigma = torch::full({5}, 0.5f, torch::TensorOptions().requires_grad(true)); const auto entropy = truncated_normal_entropy(mu, sigma); const auto loss = entropy.sum(); diff --git a/arenai_agent/tests/src/tests_e2e/tests_e2e_agent_input.cpp b/arenai_agent/tests/src/tests_e2e/tests_e2e_agent_input.cpp index cb569fdb..3675f2a5 100644 --- a/arenai_agent/tests/src/tests_e2e/tests_e2e_agent_input.cpp +++ b/arenai_agent/tests/src/tests_e2e/tests_e2e_agent_input.cpp @@ -51,7 +51,7 @@ namespace { const auto steps = env.step(FREQUENCY, actions); states.clear(); - for (const auto &[state, reward, done, truncated]: steps) states.push_back(state); + for (const auto &[state, reward, done]: steps) states.push_back(state); actions = agent.act(states, VISION_HEIGHT, VISION_WIDTH); } diff --git a/arenai_agent/tests/src/tests_metrics/tests_abstract_metric.cpp b/arenai_agent/tests/src/tests_metrics/tests_abstract_metric.cpp index 2f9e2a32..a44f067d 100644 --- a/arenai_agent/tests/src/tests_metrics/tests_abstract_metric.cpp +++ b/arenai_agent/tests/src/tests_metrics/tests_abstract_metric.cpp @@ -39,7 +39,7 @@ TEST_F(AbstractMetricTest, ToStringFormat) { const auto str = metric.to_string(); ASSERT_TRUE(str.find("loss") != std::string::npos); - ASSERT_TRUE(str.find("=") != std::string::npos); + ASSERT_TRUE(str.find('=') != std::string::npos); ASSERT_TRUE(str.find("2.00") != std::string::npos); } @@ -50,7 +50,7 @@ TEST_F(AbstractMetricTest, ToStringScientific) { const auto str = metric.to_string(); - ASSERT_TRUE(str.find("e") != std::string::npos || str.find("E") != std::string::npos); + ASSERT_TRUE(str.find('e') != std::string::npos || str.find('E') != std::string::npos); } TEST_F(AbstractMetricTest, MetricsToStringMultiple) { @@ -62,9 +62,9 @@ TEST_F(AbstractMetricTest, MetricsToStringMultiple) { const auto str = AbstractMetric::metrics_to_string({m1, m2}); - ASSERT_TRUE(str.find("a") != std::string::npos); - ASSERT_TRUE(str.find("b") != std::string::npos); - ASSERT_TRUE(str.find(",") != std::string::npos); + ASSERT_TRUE(str.find('a') != std::string::npos); + ASSERT_TRUE(str.find('b') != std::string::npos); + ASSERT_TRUE(str.find(',') != std::string::npos); } // ======================================================================== diff --git a/arenai_agent/tests/src/tests_metrics/tests_metrics_edge.cpp b/arenai_agent/tests/src/tests_metrics/tests_metrics_edge.cpp index af076358..d1d084eb 100644 --- a/arenai_agent/tests/src/tests_metrics/tests_metrics_edge.cpp +++ b/arenai_agent/tests/src/tests_metrics/tests_metrics_edge.cpp @@ -2,6 +2,7 @@ // Created by claude on 01/07/2026. // +#include #include #include diff --git a/arenai_agent/tests/src/tests_metrics/tests_std_metric.cpp b/arenai_agent/tests/src/tests_metrics/tests_std_metric.cpp index 3734e3f9..adf9f8e3 100644 --- a/arenai_agent/tests/src/tests_metrics/tests_std_metric.cpp +++ b/arenai_agent/tests/src/tests_metrics/tests_std_metric.cpp @@ -2,6 +2,8 @@ // Created by samuel on 30/06/2026. // +#include + #include #include diff --git a/arenai_agent/tests/src/tests_networks/tests_actor.cpp b/arenai_agent/tests/src/tests_networks/tests_actor.cpp index e4f3c3f5..08c0b673 100644 --- a/arenai_agent/tests/src/tests_networks/tests_actor.cpp +++ b/arenai_agent/tests/src/tests_networks/tests_actor.cpp @@ -20,7 +20,7 @@ TEST_P(ActorTestParam, TestActorAct) { Actor actor( height, width, sensors_nb, cont_actions_nb, discrete_actions_nb, sensors_hidden_size, - layers, {{input_channels, 4}, {4, 8}}, {2, 4}); + layers, {{input_channels, 4}, {4, 8}}, {2, 4}, 0.1f, 0.2f); const auto image = torch::randint( 255, {batch_size, input_channels, height, width}, diff --git a/arenai_agent/tests/src/tests_networks/tests_entropy.cpp b/arenai_agent/tests/src/tests_networks/tests_entropy.cpp index e4a7c0f5..8ec8242d 100644 --- a/arenai_agent/tests/src/tests_networks/tests_entropy.cpp +++ b/arenai_agent/tests/src/tests_networks/tests_entropy.cpp @@ -11,7 +11,7 @@ using namespace arenai::agent; TEST_F(AlphaParameterTest, AlphaAlwaysPositive) { for (const float init: {0.01f, 0.1f, 1.0f, 10.0f}) { - AlphaParameter param(init); + AlphaParameters param(init, 1); ASSERT_GT(param.alpha().item(), 0.0f) << "alpha should be positive for initial_alpha=" << init; } @@ -19,22 +19,108 @@ TEST_F(AlphaParameterTest, AlphaAlwaysPositive) { TEST_F(AlphaParameterTest, InitialValueMatchesInput) { constexpr float initial = 0.2f; - AlphaParameter param(initial); + AlphaParameters param(initial, 1); ASSERT_NEAR(param.alpha().item(), initial, 1e-6f); } TEST_F(AlphaParameterTest, LogAlphaRequiresGrad) { - AlphaParameter param(1.0f); + AlphaParameters param(1.0f, 1); ASSERT_TRUE(param.log_alpha().requires_grad()); } TEST_F(AlphaParameterTest, LogAlphaConsistentWithAlpha) { - AlphaParameter param(0.5f); + AlphaParameters param(0.5f, 1); const auto log_a = param.log_alpha().item(); const auto a = param.alpha().item(); ASSERT_NEAR(std::exp(log_a), a, 1e-6f); } + +/* + * PID Lagrangian + */ + +TEST_F(PidLagrangianAlphaParameterTest, StartsAtInitialAlpha) { + const PidLagrangianAlphaParameters pid(2e-1f, 5e-3f, 1.f, 1e-3f, 1); + + ASSERT_NEAR(pid.alpha().item(), 1e-3f, 1e-7f); +} + +TEST_F(PidLagrangianAlphaParameterTest, NullErrorKeepsAlpha) { + const PidLagrangianAlphaParameters pid(2e-1f, 5e-3f, 1.f, 1e-3f, 1); + + const auto entropy = torch::full({8, 1}, 0.5f); + + for (int i = 0; i < 100; i++) pid.update(entropy, entropy); + + ASSERT_NEAR(pid.alpha().item(), 1e-3f, 1e-6f); +} + +TEST_F(PidLagrangianAlphaParameterTest, EntropyBelowTargetRaisesAlpha) { + const PidLagrangianAlphaParameters pid(2e-1f, 5e-3f, 1.f, 1e-3f, 1); + + const auto entropy = torch::full({8, 1}, 0.2f); + const auto target = torch::full({8, 1}, 0.7f); + + const auto before = pid.alpha().item(); + for (int i = 0; i < 10; i++) pid.update(entropy, target); + + ASSERT_GT(pid.alpha().item(), before); +} + +// a multiplier never goes negative, whichever side the error sits on: the entropy bonus can +// be switched off, never turned into a penalty. Running on log(alpha) gives this for free, +// and the bounded integral keeps the excursion finite on both sides +TEST_F(PidLagrangianAlphaParameterTest, StaysPositiveAndFinite) { + const PidLagrangianAlphaParameters pid(2e-1f, 5e-3f, 1.f, 1e-3f, 1); + + const auto target = torch::full({8, 1}, 0.7f); + + for (int i = 0; i < 10000; i++) { + pid.update(torch::full({8, 1}, -5.f), target); + const auto alpha = pid.alpha().item(); + ASSERT_GT(alpha, 0.f); + ASSERT_LT(alpha, 10.f); + } + + for (int i = 0; i < 10000; i++) { + pid.update(torch::full({8, 1}, 5.f), target); + const auto alpha = pid.alpha().item(); + ASSERT_GT(alpha, 0.f); + ASSERT_LT(alpha, 10.f); + } +} + +// the failure mode this controller replaces: a sustained negative error used to drive the +// integral arbitrarily low, so alpha could not come back once the error flipped sign +TEST_F(PidLagrangianAlphaParameterTest, RecoversFromSaturationWithoutWindup) { + const PidLagrangianAlphaParameters pid(2e-1f, 5e-3f, 1.f, 1e-3f, 1); + + const auto target = torch::full({8, 1}, 0.7f); + + // long stretch above target: alpha bottoms out + for (int i = 0; i < 5000; i++) pid.update(torch::full({8, 1}, 5.f), target); + ASSERT_LT(pid.alpha().item(), 1e-7f); + + // the integral is bounded, so coming back up takes (log range) / (k_i * error) updates + // and not the unbounded time an unclamped integral would need + for (int i = 0; i < 5000; i++) pid.update(torch::full({8, 1}, 0.2f), target); + ASSERT_GT(pid.alpha().item(), 1e-3f); +} + +TEST_F(PidLagrangianAlphaParameterTest, OneAlphaPerAction) { + const PidLagrangianAlphaParameters pid(2e-1f, 5e-3f, 1.f, 1e-3f, 3); + + const auto entropy = torch::tensor({0.2f, 0.7f, 1.2f}).unsqueeze(0).repeat({8, 1}); + const auto target = torch::full({8, 3}, 0.7f); + + for (int i = 0; i < 100; i++) pid.update(entropy, target); + + const auto alpha = pid.alpha().squeeze(0); + ASSERT_EQ(alpha.size(0), 3); + ASSERT_GT(alpha[0].item(), alpha[1].item()); + ASSERT_GT(alpha[1].item(), alpha[2].item()); +} diff --git a/arenai_agent/tests/src/tests_networks/tests_misc.cpp b/arenai_agent/tests/src/tests_networks/tests_misc.cpp index a682ea96..315a463b 100644 --- a/arenai_agent/tests/src/tests_networks/tests_misc.cpp +++ b/arenai_agent/tests/src/tests_networks/tests_misc.cpp @@ -50,7 +50,7 @@ INSTANTIATE_TEST_SUITE_P( // ======================================================================== TEST_P(ExpModuleParamTest, OutputAlwaysPositive) { - const auto shape = GetParam(); + const auto &shape = GetParam(); Exp exp_module; @@ -62,7 +62,7 @@ TEST_P(ExpModuleParamTest, OutputAlwaysPositive) { } TEST_P(ExpModuleParamTest, MatchesTorchExp) { - const auto shape = GetParam(); + const auto &shape = GetParam(); Exp exp_module; diff --git a/arenai_agent/tests/src/tests_networks/tests_q_function.cpp b/arenai_agent/tests/src/tests_networks/tests_q_function.cpp index 329b6b8d..27c726bd 100644 --- a/arenai_agent/tests/src/tests_networks/tests_q_function.cpp +++ b/arenai_agent/tests/src/tests_networks/tests_q_function.cpp @@ -9,7 +9,7 @@ using namespace arenai; using namespace arenai::agent; -TEST_P(QFunctionTestParam, TestQFunctionExpectation) { +TEST_P(QFunctionTestParam, TestQFunctionPerDiscreteAction) { const auto [layers, cont_actions_nb, discrete_actions_nb, sensors_nb, sensors_hidden_size, actions_hidden_size, batch_size] = GetParam(); @@ -28,15 +28,12 @@ TEST_P(QFunctionTestParam, TestQFunctionExpectation) { const auto sensors = torch::randn({batch_size, sensors_nb}); const auto continuous_actions = torch::rand({batch_size, cont_actions_nb}) * 2.f - 1.f; - const auto discrete_actions = - torch::softmax(torch::randn({batch_size, discrete_actions_nb}), -1); - const auto value = - q_function.value_expectation(image, sensors, continuous_actions, discrete_actions); + const auto value = q_function.value_per_discrete_action(image, sensors, continuous_actions); ASSERT_EQ(value.ndimension(), 2); ASSERT_EQ(value.size(0), batch_size); - ASSERT_EQ(value.size(1), 1); + ASSERT_EQ(value.size(1), discrete_actions_nb); } TEST_P(QFunctionTestParam, TestQFunctionOHE) { diff --git a/arenai_agent/tests/src/tests_networks/tests_q_function_consistency.cpp b/arenai_agent/tests/src/tests_networks/tests_q_function_consistency.cpp index 4bb1f52f..983873df 100644 --- a/arenai_agent/tests/src/tests_networks/tests_q_function_consistency.cpp +++ b/arenai_agent/tests/src/tests_networks/tests_q_function_consistency.cpp @@ -11,10 +11,10 @@ using namespace arenai; using namespace arenai::agent; // ======================================================================== -// value_expectation must equal the weighted sum of value_ohe +// value_per_discrete_action must match value_ohe action by action // ======================================================================== -TEST_F(QFunctionConsistencyTest, ExpectationMatchesManualWeightedSum) { +TEST_F(QFunctionConsistencyTest, PerDiscreteActionMatchesOhe) { constexpr int height = 8, width = 8; constexpr int nb_sensors = 5, nb_cont = 3, nb_disc = 4; constexpr int batch = 4; @@ -24,50 +24,23 @@ TEST_F(QFunctionConsistencyTest, ExpectationMatchesManualWeightedSum) { const auto vision = torch::randint(0, 255, {batch, 3, height, width}, torch::kUInt8); const auto sensors = torch::randn({batch, nb_sensors}); const auto cont_actions = torch::randn({batch, nb_cont}); - const auto disc_proba = torch::softmax(torch::randn({batch, nb_disc}), -1); torch::NoGradGuard no_grad; - const auto v_exp = q.value_expectation(vision, sensors, cont_actions, disc_proba); + const auto q_per_action = q.value_per_discrete_action(vision, sensors, cont_actions); + + ASSERT_EQ(q_per_action.size(0), batch); + ASSERT_EQ(q_per_action.size(1), nb_disc); - auto v_manual = torch::zeros({batch, 1}); const auto one_hots = torch::eye(nb_disc); for (int a = 0; a < nb_disc; a++) { const auto ohe = one_hots[a].unsqueeze(0).expand({batch, -1}); const auto q_a = q.value_ohe(vision, sensors, cont_actions, ohe); - v_manual = v_manual + disc_proba.select(1, a).unsqueeze(1) * q_a; - } - - ASSERT_TRUE(torch::allclose(v_exp, v_manual, 1e-4, 1e-4)) - << "value_expectation should equal sum of proba[a] * value_ohe(one_hot[a])"; -} - -TEST_F(QFunctionConsistencyTest, ExpectationWithUniformProbaIsMeanOfOhe) { - constexpr int height = 8, width = 8; - constexpr int nb_sensors = 3, nb_cont = 2, nb_disc = 3; - constexpr int batch = 2; - - QFunction q(height, width, nb_sensors, nb_cont, nb_disc, 8, 8, {16}, {{3, 4}}, {2}); - - const auto vision = torch::randint(0, 255, {batch, 3, height, width}, torch::kUInt8); - const auto sensors = torch::randn({batch, nb_sensors}); - const auto cont_actions = torch::randn({batch, nb_cont}); - const auto uniform_proba = torch::ones({batch, nb_disc}) / static_cast(nb_disc); - - torch::NoGradGuard no_grad; - const auto v_exp = q.value_expectation(vision, sensors, cont_actions, uniform_proba); - - auto sum_q = torch::zeros({batch, 1}); - const auto one_hots = torch::eye(nb_disc); - for (int a = 0; a < nb_disc; a++) { - const auto ohe = one_hots[a].unsqueeze(0).expand({batch, -1}); - sum_q = sum_q + q.value_ohe(vision, sensors, cont_actions, ohe); + ASSERT_TRUE(torch::allclose(q_per_action.select(1, a).unsqueeze(1), q_a, 1e-4, 1e-4)) + << "value_per_discrete_action column " << a << " should equal value_ohe(one_hot[" << a + << "])"; } - const auto mean_q = sum_q / static_cast(nb_disc); - - ASSERT_TRUE(torch::allclose(v_exp, mean_q, 1e-4, 1e-4)) - << "With uniform probabilities, expectation should be mean of Q values"; } TEST_F(QFunctionConsistencyTest, ValueOheOutputFinite) { @@ -80,7 +53,7 @@ TEST_F(QFunctionConsistencyTest, ValueOheOutputFinite) { const auto vision = torch::randint(0, 255, {batch, 3, height, width}, torch::kUInt8); const auto sensors = torch::randn({batch, nb_sensors}); const auto cont_actions = torch::randn({batch, nb_cont}); - auto disc_ohe = torch::zeros({batch, nb_disc}); + const auto disc_ohe = torch::zeros({batch, nb_disc}); disc_ohe.select(1, 0).fill_(1.0f); const auto value = q.value_ohe(vision, sensors, cont_actions, disc_ohe); @@ -102,9 +75,8 @@ TEST_F(QFunctionGradientTest, GradientFlowsThroughQFunction) { const auto vision = torch::randint(0, 255, {batch, 3, height, width}, torch::kUInt8); const auto sensors = torch::randn({batch, nb_sensors}); const auto cont_actions = torch::randn({batch, nb_cont}); - const auto disc_proba = torch::softmax(torch::randn({batch, nb_disc}), -1); - const auto value = q.value_expectation(vision, sensors, cont_actions, disc_proba); + const auto value = q.value_per_discrete_action(vision, sensors, cont_actions); const auto loss = value.sum(); loss.backward(); @@ -125,7 +97,7 @@ TEST_F(ActorGradientTest, GradientFlowsThroughActor) { constexpr int nb_sensors = 3, nb_cont = 2, nb_disc = 2; constexpr int batch = 2; - Actor actor(height, width, nb_sensors, nb_cont, nb_disc, 8, {16}, {{3, 4}}, {2}); + Actor actor(height, width, nb_sensors, nb_cont, nb_disc, 8, {16}, {{3, 4}}, {2}, 0.1f, 0.2f); const auto vision = torch::randint(0, 255, {batch, 3, height, width}, torch::kUInt8); const auto sensors = torch::randn({batch, nb_sensors}); @@ -151,7 +123,7 @@ TEST_F(ActorGradientTest, ActorWeightsChangeAfterOptimStep) { constexpr int nb_sensors = 3, nb_cont = 2, nb_disc = 2; constexpr int batch = 2; - Actor actor(height, width, nb_sensors, nb_cont, nb_disc, 8, {16}, {{3, 4}}, {2}); + Actor actor(height, width, nb_sensors, nb_cont, nb_disc, 8, {16}, {{3, 4}}, {2}, 0.1f, 0.2f); auto optimizer = torch::optim::Adam(actor.parameters(), 1e-3); diff --git a/arenai_agent/tests/src/tests_networks/tests_vision_edge.cpp b/arenai_agent/tests/src/tests_networks/tests_vision_edge.cpp index 87d17ff8..4f9041e5 100644 --- a/arenai_agent/tests/src/tests_networks/tests_vision_edge.cpp +++ b/arenai_agent/tests/src/tests_networks/tests_vision_edge.cpp @@ -14,8 +14,7 @@ TEST_F(VisionEdgeTest, RejectsNonUint8Input) { const auto float_input = torch::randn({1, 3, 8, 8}); - ASSERT_THROW(conv.forward(float_input), std::runtime_error) - << "Should throw when input is not UInt8"; + ASSERT_THROW(conv.forward(float_input), c10::Error) << "Should throw when input is not UInt8"; } TEST_F(VisionEdgeTest, NormalizesToExpectedRange) { @@ -33,13 +32,15 @@ TEST_F(VisionEdgeTest, NormalizesToExpectedRange) { TEST_F(VisionEdgeTest, OutputSizeMatchesGetOutputSize) { const std::vector> channels = {{3, 8}, {8, 16}}; - const std::vector gnums = {4, 4}; + const std::vector gnums = {4, 4}; constexpr int h = 16, w = 16; + constexpr int batch_size = 2; ConvolutionNetwork conv(h, w, channels, gnums); - const auto input = torch::randint(255, {2, 3, h, w}, torch::kUInt8); + const auto input = torch::randint(255, {batch_size, 3, h, w}, torch::kUInt8); const auto output = conv.forward(input); + ASSERT_EQ(output.size(0), batch_size); ASSERT_EQ(output.size(1), conv.get_output_size()); } diff --git a/arenai_agent/tests/src/tests_networks_utils/tests_init.cpp b/arenai_agent/tests/src/tests_networks_utils/tests_init.cpp index 6b2a1fae..8e7e3862 100644 --- a/arenai_agent/tests/src/tests_networks_utils/tests_init.cpp +++ b/arenai_agent/tests/src/tests_networks_utils/tests_init.cpp @@ -5,6 +5,7 @@ #include #include +#include #include "./networks/constants.h" #include "./networks/misc.h" @@ -59,15 +60,14 @@ TEST_F(InitWeightsTest, MuOutputBiasZero) { TEST_F(InitWeightsTest, SigmaOutputWeightsOrthogonal) { torch::nn::Linear linear(32, 4); - init_sigma_output_weights(*linear); + init_sigma_output_weights(*linear, 0.f); assert_orthogonal(linear->weight, 0.01f); } TEST_F(InitWeightsTest, SigmaOutputIsEqualToWantedOne) { torch::nn::Sequential sequential( - torch::nn::Linear(32, 4), - std::make_shared(agent::SIGMA_MIN, agent::SIGMA_MAX)); + torch::nn::Linear(32, 4), std::make_shared(SIGMA_MIN, SIGMA_MAX)); constexpr float wanted_sigma = 0.5f; @@ -81,11 +81,25 @@ TEST_F(InitWeightsTest, SigmaOutputIsEqualToWantedOne) { TEST_F(InitWeightsTest, DiscreteOutputWeightsOrthogonal) { torch::nn::Linear linear(32, 6); - init_discrete_output_weights(*linear); + init_discrete_output_weights(*linear, 0.f); assert_orthogonal(linear->weight, 0.01f); } +TEST_F(InitWeightsTest, DiscreteOutputFireProbaIsEqualToWantedOne) { + constexpr float wanted_fire_proba = 0.2f; + + torch::nn::Sequential seq( + torch::nn::Linear(32, model::ENEMY_NB_DISCRETE_ACTION), torch::nn::Softmax(-1)); + seq->apply([](torch::nn::Module &m) { init_discrete_output_weights(m, wanted_fire_proba); }); + + torch::Tensor x = torch::randn({1, 32}); + const auto out = seq->forward(x); + + ASSERT_NEAR(out[0][0].item(), wanted_fire_proba, 1e-2f); + ASSERT_NEAR(out[0][1].item(), 1.f - wanted_fire_proba, 1e-2f); +} + TEST_F(InitWeightsTest, ValueOutputWeightsOrthogonal) { torch::nn::Linear linear(32, 1); init_value_output_weights(*linear); diff --git a/arenai_agent/tests/src/tests_replay_buffer/create_random_step.cpp b/arenai_agent/tests/src/tests_replay_buffer/create_random_step.cpp index 625dd93c..dd7ceb33 100644 --- a/arenai_agent/tests/src/tests_replay_buffer/create_random_step.cpp +++ b/arenai_agent/tests/src/tests_replay_buffer/create_random_step.cpp @@ -8,20 +8,20 @@ using namespace arenai; using namespace arenai::agent; // single-tank state: every tensor carries a leading nb_tanks dimension of 1 -arenai::agent::TorchState -create_random_state(const int width, const int height, const int nb_sensors) { +TorchState create_random_state(const int width, const int height, const int nb_sensors) { return { - torch::randint(255, {1, 3, height, width}, torch::kUInt8), torch::randn({1, nb_sensors})}; + .vision = torch::randint(255, {1, 3, height, width}, torch::kUInt8), + .proprioception = torch::randn({1, nb_sensors})}; } -arenai::agent::SacInputStep create_random_step( +SacInputStep create_random_step( const int width, const int height, const int nb_cont_actions, const int nb_discrete_actions, const int nb_sensors, const bool done) { return { - create_random_state(width, height, nb_sensors), - {torch::rand({1, nb_cont_actions}) * 2.f - 1.f, - torch::softmax(torch::randn({1, nb_discrete_actions}), -1)}, - torch::randn({1, 1}), - torch::full({1, 1}, done, torch::kBool), - torch::full({1, 1}, false, torch::kBool)}; + .state = create_random_state(width, height, nb_sensors), + .action = + {.continuous_action = torch::rand({1, nb_cont_actions}) * 2.f - 1.f, + .discrete_action = torch::softmax(torch::randn({1, nb_discrete_actions}), -1)}, + .reward = torch::randn({1, 1}), + .done = torch::full({1, 1}, done, torch::kBool)}; } diff --git a/arenai_agent/tests/src/tests_replay_buffer/tests_replay_buffer_edge.cpp b/arenai_agent/tests/src/tests_replay_buffer/tests_replay_buffer_edge.cpp index c5dca4f6..72df747d 100644 --- a/arenai_agent/tests/src/tests_replay_buffer/tests_replay_buffer_edge.cpp +++ b/arenai_agent/tests/src/tests_replay_buffer/tests_replay_buffer_edge.cpp @@ -2,6 +2,8 @@ // Created by claude on 01/07/2026. // +#include + #include #include @@ -12,7 +14,7 @@ using namespace arenai; using namespace arenai::agent; TEST_F(ReplayBufferEdgeTest, SampleFromEmptyBufferDoesNotCrash) { - SacReplayBuffer buffer(10); + const SacReplayBuffer buffer(10); ASSERT_EQ(buffer.size(), 0u); @@ -48,41 +50,30 @@ TEST_F(ReplayBufferEdgeTest, SampleBatchLargerThanSingleElement) { << "Batch size should be clamped to the single available transition"; } -TEST_F(ReplayBufferEdgeTest, RewardUnchangedAtSample) { +namespace { + SacInputStep create_step_with_reward(const float reward) { + SacInputStep step; + step.state.vision = torch::randint(255, {1, 3, 8, 8}, torch::kUInt8); + step.state.proprioception = torch::randn({1, 5}); + step.action.continuous_action = torch::randn({1, 3}); + step.action.discrete_action = torch::zeros({1, 2}); + step.action.discrete_action[0][0] = 1.0f; + step.reward = torch::full({1, 1}, reward); + step.done = torch::zeros({1, 1}); + return step; + } +}// namespace + +TEST_F(ReplayBufferEdgeTest, ConstantRewardsAreNotRescaled) { SacReplayBuffer buffer(10); - SacInputStep step; - step.state.vision = torch::randint(255, {1, 3, 8, 8}, torch::kUInt8); - step.state.proprioception = torch::randn({1, 5}); - step.action.continuous_action = torch::randn({1, 3}); - step.action.discrete_action = torch::zeros({1, 2}); - step.action.discrete_action[0][0] = 1.0f; - step.reward = torch::full({1, 1}, 2.0f); - step.done = torch::zeros({1, 1}); - step.truncated = torch::zeros({1, 1}); - - buffer.add(step); - buffer.add(step); + buffer.add(create_step_with_reward(2.0f)); + buffer.add(create_step_with_reward(2.0f)); const auto output = buffer.sample(1, torch::kCPU); ASSERT_NEAR(output.reward.item(), 2.0f, 1e-5f) - << "Base ReplayBuffer should return the stored reward unchanged at sample time"; -} - -TEST_F(ReplayBufferEdgeTest, TruncatedStepIsNotStoredAsTerminal) { - SacReplayBuffer buffer(10); - - auto step = create_random_step(8, 8, 3, 2, 5, true); - step.truncated = torch::full({1, 1}, true, torch::kBool); - - buffer.add(step); - buffer.finish_episode(create_random_state(8, 8, 5)); - - const auto output = buffer.sample(1, torch::kCPU); - - ASSERT_FALSE(output.done.to(torch::kBool).item()) - << "A truncated step must sample done=false so the critic keeps bootstrapping"; + << "Zero reward variance must keep the scale at 1 (no division by ~0)"; } TEST_F(ReplayBufferEdgeTest, DeadStepIsStoredAsTerminal) { @@ -94,7 +85,7 @@ TEST_F(ReplayBufferEdgeTest, DeadStepIsStoredAsTerminal) { const auto output = buffer.sample(1, torch::kCPU); ASSERT_TRUE(output.done.to(torch::kBool).item()) - << "A real termination (done && !truncated) must sample done=true"; + << "A termination (done) must sample done=true"; } TEST_F(ReplayBufferEdgeTest, SampleWithZeroBatchSize) { diff --git a/arenai_agent/tests/src/tests_utils/tests_file_reader.cpp b/arenai_agent/tests/src/tests_utils/tests_file_reader.cpp index 6623d74b..a0e8cd98 100644 --- a/arenai_agent/tests/src/tests_utils/tests_file_reader.cpp +++ b/arenai_agent/tests/src/tests_utils/tests_file_reader.cpp @@ -34,7 +34,7 @@ TEST_F(DesktopAssetFileReaderTest, ReadTextReturnsContent) { } TEST_F(DesktopAssetFileReaderTest, ReadTextEmptyFile) { - std::ofstream(tmp_dir / "empty.txt"); + std::ofstream tmp(tmp_dir / "empty.txt"); DesktopAssetFileReader reader(tmp_dir); const auto content = reader.read_text("empty.txt"); @@ -92,17 +92,17 @@ TEST_F(DesktopAssetFileReaderTest, ReadPngValidImage) { << "Failed to create test PNG"; DesktopAssetFileReader reader(tmp_dir); - const auto img = reader.read_png("test.png"); + const auto [width, height, channels, out_pixels] = reader.read_png("test.png"); - ASSERT_EQ(img.width, W); - ASSERT_EQ(img.height, H); - ASSERT_EQ(img.channels, 4); - ASSERT_EQ(img.pixels.size(), static_cast(W * H * 4)); + ASSERT_EQ(width, W); + ASSERT_EQ(height, H); + ASSERT_EQ(channels, 4); + ASSERT_EQ(out_pixels.size(), static_cast(W * H * 4)); } TEST_F(DesktopAssetFileReaderTest, ReadPngPixelsNotAllZero) { constexpr int W = 2, H = 2, C = 4; - std::vector pixels(W * H * C, 200); + const std::vector pixels(W * H * C, 200); const auto png_path = (tmp_dir / "bright.png").string(); SOIL_save_image(png_path.c_str(), SOIL_SAVE_TYPE_PNG, W, H, C, pixels.data()); @@ -128,7 +128,7 @@ TEST_F(DesktopAssetFileReaderTest, ReadPngThrowsOnMissing) { TEST_F(DesktopAssetFileReaderTest, ReadPngAlwaysRGBA) { constexpr int W = 2, H = 2, C = 3; - std::vector pixels(W * H * C, 128); + const std::vector pixels(W * H * C, 128); const auto png_path = (tmp_dir / "rgb.png").string(); SOIL_save_image(png_path.c_str(), SOIL_SAVE_TYPE_PNG, W, H, C, pixels.data()); diff --git a/arenai_agent/tests/src/tests_utils/tests_torch_converter.cpp b/arenai_agent/tests/src/tests_utils/tests_torch_converter.cpp index a052cd2b..97415edb 100644 --- a/arenai_agent/tests/src/tests_utils/tests_torch_converter.cpp +++ b/arenai_agent/tests/src/tests_utils/tests_torch_converter.cpp @@ -178,9 +178,9 @@ TEST_P(StatesToTensorParamTest, OutputShapeCorrect) { const auto [batch, height, width] = GetParam(); std::vector states(batch); - for (auto &s: states) { - s.vision.pixels.resize(3 * height * width, 0); - s.proprioception.resize(model::ENEMY_PROPRIOCEPTION_SIZE, 0.0f); + for (auto &[vision, proprioception]: states) { + vision.pixels.resize(3 * height * width, 0); + proprioception.resize(model::ENEMY_PROPRIOCEPTION_SIZE, 0.0f); } const auto [vision, proprioception] = states_to_tensor(states, height, width); @@ -200,9 +200,9 @@ TEST_P(StatesToTensorParamTest, NoGradient) { const auto [batch, height, width] = GetParam(); std::vector states(batch); - for (auto &s: states) { - s.vision.pixels.resize(3 * height * width, 0); - s.proprioception.resize(model::ENEMY_PROPRIOCEPTION_SIZE, 0.0f); + for (auto &[vision, proprioception]: states) { + vision.pixels.resize(3 * height * width, 0); + proprioception.resize(model::ENEMY_PROPRIOCEPTION_SIZE, 0.0f); } const auto [vision, proprioception] = states_to_tensor(states, height, width); diff --git a/arenai_core/include/arenai_core/environment.h b/arenai_core/include/arenai_core/environment.h index 2148cca4..8d30b5b3 100644 --- a/arenai_core/include/arenai_core/environment.h +++ b/arenai_core/include/arenai_core/environment.h @@ -29,7 +29,7 @@ namespace arenai::core { float wanted_frequency, int vision_height, int vision_width, int vision_num_threads, bool vision_thread_sleep); - virtual std::vector> + virtual std::vector> step(float time_delta, const std::vector &actions); std::vector reset(float spawn_width, float spawn_height); diff --git a/arenai_core/include/arenai_core/types.h b/arenai_core/include/arenai_core/types.h index dc5f9594..164057e3 100644 --- a/arenai_core/include/arenai_core/types.h +++ b/arenai_core/include/arenai_core/types.h @@ -19,7 +19,6 @@ namespace arenai::core { typedef float Reward; typedef bool IsDone; - typedef bool IsTruncated; typedef controller::user_input Action; }// namespace arenai::core diff --git a/arenai_core/src/environment.cpp b/arenai_core/src/environment.cpp index 76e0f11b..5427795d 100644 --- a/arenai_core/src/environment.cpp +++ b/arenai_core/src/environment.cpp @@ -30,7 +30,7 @@ namespace arenai::core { graphics_backend(graphics_backend), gl_context(graphics_backend->render_context()), rng(dev()), file_reader(file_reader) {} - std::vector> + std::vector> BaseTanksEnvironment::step(const float time_delta, const std::vector &actions) { // 1. apply action @@ -51,13 +51,13 @@ namespace arenai::core { vision_pool_->loop_wait(); // 4. build State - std::vector> result; + std::vector> result; result.reserve(tanks.size()); for (int i = 0; i < tanks.size(); i++) { result.emplace_back( State(vision_pool_->read_vision(i), tanks[i]->get_proprioception()), - tanks[i]->get_reward(tanks), tanks[i]->is_dead(), false); + tanks[i]->get_reward(tanks), tanks[i]->is_dead()); } return result; diff --git a/arenai_core/tests/src/tests_environment.cpp b/arenai_core/tests/src/tests_environment.cpp index 599e33fa..c5c8ca9b 100644 --- a/arenai_core/tests/src/tests_environment.cpp +++ b/arenai_core/tests/src/tests_environment.cpp @@ -213,7 +213,7 @@ TEST_F(EnvironmentTest, StepRewardAndDoneAreValid) { const std::vector actions(nb_tanks, {{0.f, 0.f}, {0.f, 0.f}, {false}}); for (const auto results = env.step(frequency, actions); - const auto &[state, reward, is_done, is_truncated]: results) { + const auto &[state, reward, is_done]: results) { ASSERT_FALSE(std::isnan(reward)) << "reward should not be NaN"; ASSERT_FALSE(std::isinf(reward)) << "reward should not be Inf"; } diff --git a/arenai_desktop/src/controller/control_kind.h b/arenai_desktop/src/controller/control_kind.h index 3ea9a972..e083aaa6 100644 --- a/arenai_desktop/src/controller/control_kind.h +++ b/arenai_desktop/src/controller/control_kind.h @@ -7,6 +7,6 @@ namespace arenai::desktop { enum class ControllerKind { Keyboard, Gamepad }; -}; +} #endif//ARENAI_CONTROL_KIND_H diff --git a/arenai_desktop/src/controller/game_input_router.cpp b/arenai_desktop/src/controller/game_input_router.cpp index f138fd6c..2155d01b 100644 --- a/arenai_desktop/src/controller/game_input_router.cpp +++ b/arenai_desktop/src/controller/game_input_router.cpp @@ -9,10 +9,10 @@ namespace arenai::desktop { GameInputRouter::GameInputRouter( - std::shared_ptr game_keyboard, - std::shared_ptr game_gamepad, - std::shared_ptr pause_input, - std::shared_ptr pause_gamepad_input, + std::shared_ptr game_keyboard, + std::shared_ptr game_gamepad, + std::shared_ptr pause_input, + std::shared_ptr pause_gamepad_input, std::function on_pause_toggle) : game_keyboard_(std::move(game_keyboard)), game_gamepad_(std::move(game_gamepad)), pause_input_(std::move(pause_input)), diff --git a/arenai_desktop/src/controller/game_input_router.h b/arenai_desktop/src/controller/game_input_router.h index 216d8f5d..0ab77126 100644 --- a/arenai_desktop/src/controller/game_input_router.h +++ b/arenai_desktop/src/controller/game_input_router.h @@ -21,10 +21,10 @@ namespace arenai::desktop { public controller::AbstractGamepadCallback { public: GameInputRouter( - std::shared_ptr game_keyboard, - std::shared_ptr game_gamepad, - std::shared_ptr pause_input, - std::shared_ptr pause_gamepad_input, + std::shared_ptr game_keyboard, + std::shared_ptr game_gamepad, + std::shared_ptr pause_input, + std::shared_ptr pause_gamepad_input, std::function on_pause_toggle); void set_paused(bool paused); @@ -43,13 +43,13 @@ namespace arenai::desktop { void on_trigger(double z, controller::GamepadTrigger trigger) override; private: - const std::shared_ptr &keyboard_sink() const; - const std::shared_ptr &gamepad_sink() const; + const std::shared_ptr &keyboard_sink() const; + const std::shared_ptr &gamepad_sink() const; - std::shared_ptr game_keyboard_; - std::shared_ptr game_gamepad_; - std::shared_ptr pause_input_; - std::shared_ptr pause_gamepad_input_; + std::shared_ptr game_keyboard_; + std::shared_ptr game_gamepad_; + std::shared_ptr pause_input_; + std::shared_ptr pause_gamepad_input_; std::function on_pause_toggle_; bool paused_ = false; diff --git a/arenai_desktop/src/controller/gamepad.cpp b/arenai_desktop/src/controller/gamepad.cpp index b86d91f3..c8d898ef 100644 --- a/arenai_desktop/src/controller/gamepad.cpp +++ b/arenai_desktop/src/controller/gamepad.cpp @@ -8,19 +8,25 @@ namespace arenai::desktop { - namespace { - float apply_deadzone(const double value) { - constexpr double DEADZONE = 0.05; + float PlayerGamepadHandler::apply_dead_zone(const double value) { + constexpr double DEAD_ZONE = 0.05; - if (std::abs(value) < DEADZONE) return 0.f; + if (std::abs(value) < DEAD_ZONE) return 0.f; - // ramp from 0 at the deadzone edge up to ±1 at full deflection - const double sign = value > 0. ? 1. : -1.; - return static_cast(sign * (std::abs(value) - DEADZONE) / (1. - DEADZONE)); - } - }// namespace + // ramp from 0 at the deadzone edge up to ±1 at full deflection + const double sign = value > 0. ? 1. : -1.; + return static_cast(sign * (std::abs(value) - DEAD_ZONE) / (1. - DEAD_ZONE)); + } - PlayerGamepadHandler::PlayerGamepadHandler() : state{0., 0., 0., 0., 0., 0., std::nullopt} {} + PlayerGamepadHandler::PlayerGamepadHandler() + : state{ + .left_stick_x = 0., + .left_stick_y = 0., + .right_stick_x = 0., + .right_stick_y = 0., + .left_trigger = 0., + .right_trigger = 0., + .button = std::nullopt} {} void PlayerGamepadHandler::on_gamepad_button( const controller::GamepadButton button, const controller::InputAction action) { @@ -75,15 +81,19 @@ namespace arenai::desktop { // deflection is scaled into radians here (like the mouse handler) constexpr float factor = 0.02f * static_cast(M_PI); - turret_rotation = factor * apply_deadzone(event.right_stick_x); - canon_rotation = factor * apply_deadzone(event.right_stick_y); + turret_rotation = factor * apply_dead_zone(event.right_stick_x); + canon_rotation = factor * apply_dead_zone(event.right_stick_y); } - const float direction = apply_deadzone(event.left_stick_x); + const float direction = apply_dead_zone(event.left_stick_x); // triggers drive the tank: right forward, left backward (both in [0, 1]) const auto speed = static_cast(event.right_trigger - event.left_trigger); - return {true, {{direction, speed}, {turret_rotation, canon_rotation}, {need_fire}}}; + return { + true, + {.left_joystick = {.x = direction, .y = speed}, + .right_joystick = {.x = turret_rotation, .y = canon_rotation}, + .fire_button = {need_fire}}}; } }// namespace arenai::desktop diff --git a/arenai_desktop/src/controller/gamepad.h b/arenai_desktop/src/controller/gamepad.h index 5a5965d6..4f3ebd88 100644 --- a/arenai_desktop/src/controller/gamepad.h +++ b/arenai_desktop/src/controller/gamepad.h @@ -41,6 +41,8 @@ namespace arenai::desktop { private: PlayerGamepadInput state; + + static float apply_dead_zone(double value); }; }// namespace arenai::desktop diff --git a/arenai_desktop/src/controller/mouse_keyboard.cpp b/arenai_desktop/src/controller/mouse_keyboard.cpp index cecca95a..f3b0df95 100644 --- a/arenai_desktop/src/controller/mouse_keyboard.cpp +++ b/arenai_desktop/src/controller/mouse_keyboard.cpp @@ -4,7 +4,6 @@ #include "./mouse_keyboard.h" -#include #include namespace arenai::desktop { @@ -27,18 +26,26 @@ namespace arenai::desktop { void PlayerMouseKeyboardHandler::on_key( const controller::Key key, const controller::InputAction action) { - on_event({std::make_pair(key, action), std::nullopt, last_mouse_x, last_mouse_y}); + on_event( + {.key = std::make_pair(key, action), + .button = std::nullopt, + .mouse_x = last_mouse_x, + .mouse_y = last_mouse_y}); } void PlayerMouseKeyboardHandler::on_mouse_move(const double x, const double y) { last_mouse_x = x; last_mouse_y = y; - on_event({std::nullopt, std::nullopt, x, y}); + on_event({.key = std::nullopt, .button = std::nullopt, .mouse_x = x, .mouse_y = y}); } void PlayerMouseKeyboardHandler::on_mouse_button( const controller::MouseButton button, const controller::InputAction action) { - on_event({std::nullopt, std::make_pair(button, action), last_mouse_x, last_mouse_y}); + on_event( + {.key = std::nullopt, + .button = std::make_pair(button, action), + .mouse_x = last_mouse_x, + .mouse_y = last_mouse_y}); } std::tuple @@ -92,8 +99,8 @@ namespace arenai::desktop { // mouse buttons if (event.button) { - const auto [button, action] = *event.button; - if (button == controller::MouseButton::Left + if (const auto [button, action] = *event.button; + button == controller::MouseButton::Left && action == controller::InputAction::Press) { need_fire = true; cursor_captured = true; @@ -102,9 +109,9 @@ namespace arenai::desktop { return { true, - {{current_dir, current_speed}, - {current_turret_rotation, current_canon_rotation}, - {need_fire}}}; + {.left_joystick = {.x = current_dir, .y = current_speed}, + .right_joystick = {.x = current_turret_rotation, .y = current_canon_rotation}, + .fire_button = {need_fire}}}; } }// namespace arenai::desktop diff --git a/arenai_desktop/src/core/agent_loading_checker.cpp b/arenai_desktop/src/core/agent_loading_checker.cpp index 1c75338b..ca069915 100644 --- a/arenai_desktop/src/core/agent_loading_checker.cpp +++ b/arenai_desktop/src/core/agent_loading_checker.cpp @@ -29,7 +29,7 @@ namespace arenai::desktop { return "Missing file: " + e.missing_file().filename().string(); } catch (utils::ModelLoadException &e) { return "Error while loading file: " + e.wrong_state_dict_file().filename().string(); - } catch (const std::exception &e) { return "Unknow error while loading Model"; } + } catch (const std::exception &_) { return "Unknow error while loading Model"; } } }// namespace arenai::desktop diff --git a/arenai_desktop/src/core/game_environment.cpp b/arenai_desktop/src/core/game_environment.cpp index 2e92ad38..f7b575d5 100644 --- a/arenai_desktop/src/core/game_environment.cpp +++ b/arenai_desktop/src/core/game_environment.cpp @@ -24,7 +24,7 @@ namespace arenai::desktop { // The tank visions get their own headless backend (integrated GPU): their // synchronous readbacks are latency-bound on a discrete GPU, and this keeps // them off the window's GPU when the player view is offloaded (prime-run). - : core::BaseTanksEnvironment( + : BaseTanksEnvironment( std::make_shared(asset_folder_path), view::make_vulkan_backend(), nb_tanks, wanted_frequency, vision_height, vision_width, 8, true), @@ -119,7 +119,7 @@ namespace arenai::desktop { player_renderer->add_drawable( "cubemap", drawable_factory->make_cube_map(file_reader, "cubemap/1")); - std::uniform_real_distribution u_dist(0.f, 1.f); + std::uniform_real_distribution u_dist(0.f, 1.f); for (const auto &[name, shape]: player_tank->load_shell_shapes()) { const glm::vec4 color(u_dist(rng) * 0.8f, u_dist(rng) * 0.8f, u_dist(rng) * 0.8f, 1.f); diff --git a/arenai_desktop/src/game.cpp b/arenai_desktop/src/game.cpp index 2fa6a8ab..d74452aa 100644 --- a/arenai_desktop/src/game.cpp +++ b/arenai_desktop/src/game.cpp @@ -22,172 +22,152 @@ using namespace arenai; namespace arenai::desktop { - namespace { - - enum class InGameOutcome { MainMenu, ExitGame, Retry }; - - // One game session: steps the environment until the window closes or - // the pause menu asks to leave. While paused, the simulation and the - // agent are simply not stepped; the frozen frame is re-rendered with - // the pause popup composited on top. - InGameOutcome run_game( - const GameOptions &game_options, const ModelOptions &model_options, - const gui::GameSettings &settings, - const std::shared_ptr &graphics_backend, - const std::unique_ptr &gui) { - const auto window = graphics_backend->get_window(); - - const std::shared_ptr sac_agent = - agent::ActorAgentFactory(model_options.hyper_parameters) - .get_agent( - model_options.vision_height, model_options.vision_width, - model::ENEMY_PROPRIOCEPTION_SIZE, model::ENEMY_NB_CONTINUOUS_ACTION, - model::ENEMY_NB_DISCRETE_ACTION); - - sac_agent->load(settings.sac_folder); - - const auto env = std::make_shared( - game_options.resources_folder, graphics_backend, settings.nb_tanks, - model_options.vision_height, model_options.vision_width, - game_options.wanted_frequency, settings.controller_kind); - - auto states = env->reset( - static_cast(settings.spawn_side), static_cast(settings.spawn_side)); - - // the router owns the window's input slots for the whole session; - // Escape / Start flip the pause state through toggle_requested - bool paused = false; - bool game_over = false; - bool toggle_requested = false; - - const auto router = std::make_shared( - env->keyboard_handler(), env->gamepad_handler(), gui->pause_input(), - gui->pause_gamepad_input(), [&toggle_requested] { toggle_requested = true; }); - window->set_keyboard_callback(router); - window->set_gamepad_callback(router); - - // in keyboard mode the game handler captures (and hides) the cursor - // itself; the gamepad handler never touches the cursor, so the - // application hides it for the whole game and the pause popup - // restores it - if (settings.controller_kind == ControllerKind::Gamepad) - window->set_cursor_mode(controller::CursorMode::Disabled); - - // the window has a single resize slot: while in game it feeds both - // the player renderer and the gui overlay - window->set_resize_callback([&gui, env](const int width, const int height) { - gui->on_window_resized(width, height); - env->resize(width, height); - }); + InGameOutcome run_game( + const GameOptions &game_options, const ModelOptions &model_options, + const gui::GameSettings &settings, + const std::shared_ptr &graphics_backend, + const std::unique_ptr &gui) { + const auto window = graphics_backend->get_window(); - const auto set_paused = [&](const bool value) { - paused = value; - router->set_paused(value); - if (value) { - gui->open_pause(); - window->set_cursor_mode(controller::CursorMode::Normal); - } else { - gui->close_pause(); - // in keyboard mode the game handler re-captures the cursor - // on its next event - if (settings.controller_kind == ControllerKind::Gamepad) - window->set_cursor_mode(controller::CursorMode::Disabled); - } - }; - - // the frame the player dies on freezes under the game-over popup, - // exactly like the pause: same input routing, same overlay loop — - // only the popup (and its actions) differ, and it cannot be - // toggled away - const auto set_game_over = [&] { - game_over = true; - router->set_paused(true); - gui->open_game_over(env->get_score()); + const std::shared_ptr sac_agent = + agent::ActorAgentFactory(model_options.hyper_parameters) + .get_agent( + model_options.vision_height, model_options.vision_width, + model::ENEMY_PROPRIOCEPTION_SIZE, model::ENEMY_NB_CONTINUOUS_ACTION, + model::ENEMY_NB_DISCRETE_ACTION); + + sac_agent->load(settings.sac_folder); + + const auto env = std::make_shared( + game_options.resources_folder, graphics_backend, settings.nb_tanks, + model_options.vision_height, model_options.vision_width, game_options.wanted_frequency, + settings.controller_kind); + + auto states = env->reset( + static_cast(settings.spawn_side), static_cast(settings.spawn_side)); + + // the router owns the window's input slots for the whole session; + // Escape / Start flip the pause state through toggle_requested + bool paused = false; + bool game_over = false; + bool toggle_requested = false; + + const auto router = std::make_shared( + env->keyboard_handler(), env->gamepad_handler(), gui->pause_input(), + gui->pause_gamepad_input(), [&toggle_requested] { toggle_requested = true; }); + window->set_keyboard_callback(router); + window->set_gamepad_callback(router); + + // in keyboard mode the game handler captures (and hides) the cursor + // itself; the gamepad handler never touches the cursor, so the + // application hides it for the whole game and the pause popup + // restores it + if (settings.controller_kind == ControllerKind::Gamepad) + window->set_cursor_mode(controller::CursorMode::Disabled); + + // the window has a single resize slot: while in game it feeds both + // the player renderer and the gui overlay + window->set_resize_callback([&gui, env](const int width, const int height) { + gui->on_window_resized(width, height); + env->resize(width, height); + }); + + const auto set_paused = [&](const bool value) { + paused = value; + router->set_paused(value); + if (value) { + gui->open_pause(); window->set_cursor_mode(controller::CursorMode::Normal); - }; - - auto outcome = InGameOutcome::ExitGame; - - // [ARENAI-DBG] temporary auto-repro - int dbg_auto_frames = -1; - if (const char *dbg = std::getenv("ARENAI_DEBUG_AUTOFRAMES")) - dbg_auto_frames = std::atoi(dbg); - int dbg_frame_count = 0; - - const auto frame_dt = - std::chrono::milliseconds(static_cast(game_options.wanted_frequency * 1000.f)); + } else { + gui->close_pause(); + // in keyboard mode the game handler re-captures the cursor + // on its next event + if (settings.controller_kind == ControllerKind::Gamepad) + window->set_cursor_mode(controller::CursorMode::Disabled); + } + }; - while (!window->should_close()) { - if (dbg_auto_frames > 0 && dbg_frame_count++ >= dbg_auto_frames) break; - window->poll_events(); + // the frame the player dies on freezes under the game-over popup, + // exactly like the pause: same input routing, same overlay loop — + // only the popup (and its actions) differ, and it cannot be + // toggled away + const auto set_game_over = [&] { + game_over = true; + router->set_paused(true); + gui->open_game_over(env->get_score()); + window->set_cursor_mode(controller::CursorMode::Normal); + }; - if (toggle_requested) { - toggle_requested = false; - if (!game_over) set_paused(!paused); - } + auto outcome = InGameOutcome::ExitGame; - if (paused || game_over) { - // frozen scene + popup; pacing comes from the vsync - env->redraw(); - gui->render_pause_overlay(); - graphics_backend->present(); + const auto frame_dt = + std::chrono::milliseconds(static_cast(game_options.wanted_frequency * 1000.f)); - if (const auto action = gui->poll_pause_action(); - action == gui::PauseAction::Continue) - set_paused(false); - else if (action == gui::PauseAction::Retry) { - outcome = InGameOutcome::Retry; - break; - } else if (action == gui::PauseAction::MainMenu) { - outcome = InGameOutcome::MainMenu; - break; - } else if (action == gui::PauseAction::ExitGame) break; + while (!window->should_close()) { + window->poll_events(); - continue; - } + if (toggle_requested) { + toggle_requested = false; + if (!game_over) set_paused(!paused); + } - auto last_time = std::chrono::steady_clock::now(); + if (paused || game_over) { + // frozen scene + popup; pacing comes from the vsync + env->redraw(); + gui->render_pause_overlay(); + graphics_backend->present(); - const auto action = - sac_agent->act(states, model_options.vision_height, model_options.vision_width); + if (const auto action = gui->poll_pause_action(); + action == gui::PauseAction::Continue) + set_paused(false); + else if (action == gui::PauseAction::Retry) { + outcome = InGameOutcome::Retry; + break; + } else if (action == gui::PauseAction::MainMenu) { + outcome = InGameOutcome::MainMenu; + break; + } else if (action == gui::PauseAction::ExitGame) break; + + continue; + } - const auto steps = env->step(game_options.wanted_frequency, action); + auto last_time = std::chrono::steady_clock::now(); - graphics_backend->present(); + const auto action = + sac_agent->act(states, model_options.vision_height, model_options.vision_width); - if (env->is_player_dead()) set_game_over(); + const auto steps = env->step(game_options.wanted_frequency, action); - states.clear(); + graphics_backend->present(); - for (const auto &[state, reward, done, is_truncated]: steps) - states.push_back(state); + if (env->is_player_dead()) set_game_over(); - auto now = std::chrono::steady_clock::now(); - auto dt = now - last_time; + states.clear(); - std::this_thread::sleep_for( - std::max(frame_dt - dt, std::chrono::steady_clock::duration::zero())); - } + for (const auto &[state, reward, done]: steps) states.push_back(state); - gui->close_pause(); - gui->close_game_over(); - window->set_keyboard_callback(nullptr); - window->set_gamepad_callback(nullptr); - window->set_resize_callback([&gui](const int width, const int height) { - gui->on_window_resized(width, height); - }); + auto now = std::chrono::steady_clock::now(); + auto dt = now - last_time; - return outcome; + std::this_thread::sleep_for( + std::max(frame_dt - dt, std::chrono::steady_clock::duration::zero())); } - }// namespace + gui->close_pause(); + gui->close_game_over(); + window->set_keyboard_callback(nullptr); + window->set_gamepad_callback(nullptr); + window->set_resize_callback( + [&gui](const int width, const int height) { gui->on_window_resized(width, height); }); + + return outcome; + } - void game_loop(const GameOptions &game_options, const ModelOptions &model_options) { + void run_gui(const GameOptions &game_options, const ModelOptions &model_options) { // The view owns the window + GL context; the app only speaks the abstract // window/backend interface. - const std::shared_ptr graphics_backend = - view::make_glfw_vulkan_backend( - game_options.window_width, game_options.window_height, "ArenAI"); + const std::shared_ptr graphics_backend = view::make_glfw_vulkan_backend( + game_options.window_width, game_options.window_height, "ArenAI"); const auto window = graphics_backend->get_window(); std::cout << "Vulkan : " << graphics_backend->renderer_info() << std::endl; diff --git a/arenai_desktop/src/game.h b/arenai_desktop/src/game.h index a6795024..992ca8af 100644 --- a/arenai_desktop/src/game.h +++ b/arenai_desktop/src/game.h @@ -4,10 +4,13 @@ #ifndef ARENAI_DESKTOP_GAME_H #define ARENAI_DESKTOP_GAME_H + #include #include #include +#include "./gui/menu.h" + namespace arenai::desktop { struct ModelOptions { @@ -26,7 +29,15 @@ namespace arenai::desktop { std::filesystem::path resources_folder; }; - void game_loop(const GameOptions &game_options, const ModelOptions &model_options); + enum class InGameOutcome { MainMenu, ExitGame, Retry }; + + InGameOutcome run_game( + const GameOptions &game_options, const ModelOptions &model_options, + const gui::GameSettings &settings, + const std::shared_ptr &graphics_backend, + const std::unique_ptr &gui); + + void run_gui(const GameOptions &game_options, const ModelOptions &model_options); }// namespace arenai::desktop diff --git a/arenai_desktop/src/gui/rml/adapters.cpp b/arenai_desktop/src/gui/rml/adapters.cpp index 18302f16..09809ca1 100644 --- a/arenai_desktop/src/gui/rml/adapters.cpp +++ b/arenai_desktop/src/gui/rml/adapters.cpp @@ -21,7 +21,8 @@ namespace arenai::desktop::gui { Rml::FileHandle ReaderBackedFileInterface::Open(const Rml::String &path) { try { - auto file = std::make_unique(OpenedFile{reader_->read_text(path)}); + auto file = + std::make_unique(OpenedFile{.content = reader_->read_text(path)}); return reinterpret_cast(file.release()); } catch (const std::exception &e) { std::cerr << "RmlUi asset open failed: " << e.what() << std::endl; diff --git a/arenai_desktop/src/gui/rml/input.cpp b/arenai_desktop/src/gui/rml/input.cpp index 14a3d493..ec28eb4d 100644 --- a/arenai_desktop/src/gui/rml/input.cpp +++ b/arenai_desktop/src/gui/rml/input.cpp @@ -9,7 +9,8 @@ namespace arenai::desktop::gui { - std::vector ExplorerNavListener::visible_file_entries(Rml::Element *file_list) { + std::vector + ExplorerNavListener::visible_file_entries(const Rml::Element *file_list) { std::vector entries; for (int i = 0; i < file_list->GetNumChildren(); i++) if (Rml::Element *child = file_list->GetChild(i); child->IsVisible()) @@ -22,10 +23,10 @@ namespace arenai::desktop::gui { event.GetParameter("key_identifier", Rml::Input::KI_UNKNOWN)); if (key != Rml::Input::KI_UP && key != Rml::Input::KI_DOWN) return; - Rml::Element *focus = event.GetTargetElement(); + const Rml::Element *focus = event.GetTargetElement(); Rml::ElementDocument *document = focus->GetOwnerDocument(); if (document == nullptr) return; - Rml::Element *list = document->GetElementById("file-list"); + const Rml::Element *list = document->GetElementById("file-list"); Rml::Element *above = document->GetElementById("display-row"); Rml::Element *below = document->GetElementById("use-folder"); if (list == nullptr || above == nullptr || below == nullptr) return; diff --git a/arenai_desktop/src/gui/rml/input.h b/arenai_desktop/src/gui/rml/input.h index ffb14a2b..199fa532 100644 --- a/arenai_desktop/src/gui/rml/input.h +++ b/arenai_desktop/src/gui/rml/input.h @@ -30,7 +30,7 @@ namespace arenai::desktop::gui { public: // the data-for template stays in the list as a display:none // child; only its instantiated clones are real entries - static std::vector visible_file_entries(Rml::Element *file_list); + static std::vector visible_file_entries(const Rml::Element *file_list); void ProcessEvent(Rml::Event &event) override; }; diff --git a/arenai_desktop/src/gui/rml_menu.cpp b/arenai_desktop/src/gui/rml_menu.cpp index 3d4c89e1..c9fdf560 100644 --- a/arenai_desktop/src/gui/rml_menu.cpp +++ b/arenai_desktop/src/gui/rml_menu.cpp @@ -218,12 +218,18 @@ namespace arenai::desktop::gui { int weight; }; constexpr FontSpec MENU_FONTS[] = { - {"font/Sora-Regular.ttf", "Sora", 400}, - {"font/Sora-SemiBold.ttf", "Sora", 600}, - {"font/Sora-Bold.ttf", "Sora", 700}, - {"font/IBMPlexMono-Regular.ttf", "IBM Plex Mono", 400}, - {"font/IBMPlexMono-Medium.ttf", "IBM Plex Mono", 500}, - {"font/IBMPlexMono-SemiBold.ttf", "IBM Plex Mono", 600}, + {.path = "font/Sora-Regular.ttf", .family = "Sora", .weight = 400}, + {.path = "font/Sora-SemiBold.ttf", .family = "Sora", .weight = 600}, + {.path = "font/Sora-Bold.ttf", .family = "Sora", .weight = 700}, + {.path = "font/IBMPlexMono-Regular.ttf", + .family = "IBM Plex Mono", + .weight = 400}, + {.path = "font/IBMPlexMono-Medium.ttf", + .family = "IBM Plex Mono", + .weight = 500}, + {.path = "font/IBMPlexMono-SemiBold.ttf", + .family = "IBM Plex Mono", + .weight = 600}, }; font_buffers_.reserve(std::size(MENU_FONTS)); @@ -231,7 +237,7 @@ namespace arenai::desktop::gui { font_buffers_.push_back(asset_reader->read_text(path)); const auto &buffer = font_buffers_.back(); Rml::LoadFontFace( - Rml::Span( + Rml::Span( reinterpret_cast(buffer.data()), buffer.size()), family, Rml::Style::FontStyle::Normal, static_cast(weight)); @@ -393,7 +399,7 @@ namespace arenai::desktop::gui { } void focus_first_entry() const { - Rml::Element *list = params_document_->GetElementById("file-list"); + const Rml::Element *list = params_document_->GetElementById("file-list"); if (list == nullptr) return; const auto entries = ExplorerNavListener::visible_file_entries(list); if (entries.empty()) return; diff --git a/arenai_desktop/src/main.cpp b/arenai_desktop/src/main.cpp index 365fd353..8f3ed87d 100644 --- a/arenai_desktop/src/main.cpp +++ b/arenai_desktop/src/main.cpp @@ -19,7 +19,7 @@ using namespace arenai::desktop; // resources live next to the binary (copied there at build time, see // CMakeLists.txt), so they are resolved from the executable's folder — the // game can be launched from any working directory -std::filesystem::path executable_dir(const char *argv0) { +static std::filesystem::path executable_dir(const char *argv0) { #ifdef _WIN32 wchar_t buffer[MAX_PATH]; if (GetModuleFileNameW(nullptr, buffer, MAX_PATH) > 0) @@ -32,13 +32,13 @@ std::filesystem::path executable_dir(const char *argv0) { return std::filesystem::absolute(argv0).parent_path(); } -void trim_inplace(std::string &s) { +static void trim_inplace(std::string &s) { auto not_space = [](const unsigned char c) { return !std::isspace(c); }; s.erase(s.begin(), std::ranges::find_if(s, not_space)); s.erase(std::find_if(s.rbegin(), s.rend(), not_space).base(), s.end()); } -std::tuple parse_key_value(const std::string &value) { +static std::tuple parse_key_value(const std::string &value) { const std::regex regex_match(R"(^ *([^=]+)=\"?([^"]+)\"? *$)"); std::smatch match; @@ -80,12 +80,16 @@ int main(const int argc, char **argv) { const auto resources_folder = executable_dir(argv[0]) / "resources"; - game_loop( - {parser.get("--wanted_frequency"), parser.get("--window_width"), - parser.get("--window_height"), resources_folder}, - {parser.get("--vision_height"), parser.get("--vision_width"), hyper_params, - resources_folder / "trained_models" / "ppo-thin-256x128_save_6", - parser.get("--cuda")}); + run_gui( + {.wanted_frequency = parser.get("--wanted_frequency"), + .window_width = parser.get("--window_width"), + .window_height = parser.get("--window_height"), + .resources_folder = resources_folder}, + {.vision_height = parser.get("--vision_height"), + .vision_width = parser.get("--vision_width"), + .hyper_parameters = hyper_params, + .state_dict_folder = resources_folder / "trained_models" / "ppo_save_41", + .cuda = parser.get("--cuda")}); return 0; } diff --git a/arenai_desktop/tests/include/arenai_desktop_tests/headless_windowed_backend.h b/arenai_desktop/tests/include/arenai_desktop_tests/headless_windowed_backend.h index 865c1355..bb7e1474 100644 --- a/arenai_desktop/tests/include/arenai_desktop_tests/headless_windowed_backend.h +++ b/arenai_desktop/tests/include/arenai_desktop_tests/headless_windowed_backend.h @@ -16,33 +16,32 @@ // tests discard. The agents' visions keep their real offscreen backend, built // internally by the environment. -class NoopDrawable final : public arenai::view::AbstractDrawable { +class NoopDrawable final : public view::AbstractDrawable { public: void draw( glm::mat4 mvp_matrix, glm::mat4 mv_matrix, glm::vec3 light_pos_from_camera, glm::vec3 camera_pos) override {} }; -class NoopDrawableFactory final : public arenai::view::AbstractDrawableFactory { +class NoopDrawableFactory final : public view::AbstractDrawableFactory { public: - std::unique_ptr make_cube_map( - const std::shared_ptr &file_reader, + std::unique_ptr make_cube_map( + const std::shared_ptr &file_reader, const std::filesystem::path &pngs_root_path) override { return std::make_unique(); } - std::unique_ptr make_diffuse( - const std::shared_ptr &file_reader, + std::unique_ptr make_diffuse( + const std::shared_ptr &file_reader, const std::vector> &vertices, glm::vec4 color) override { return std::make_unique(); } }; -class NoopPlayerRenderer final : public arenai::view::AbstractPlayerRenderer { +class NoopPlayerRenderer final : public view::AbstractPlayerRenderer { public: void add_drawable( - const std::string &name, - std::unique_ptr drawable) override {} + const std::string &name, std::unique_ptr drawable) override {} void remove_drawable(const std::string &name) override {} void draw(const std::vector> &model_matrices) override {} @@ -53,37 +52,34 @@ class NoopPlayerRenderer final : public arenai::view::AbstractPlayerRenderer { void make_current() const override {} void release_current() const override {} - void - add_hud_drawable(std::unique_ptr hud_drawable) override {} + void add_hud_drawable(std::unique_ptr hud_drawable) override {} void set_window_size(int new_width, int new_height) override {} }; -class HeadlessWindowedBackend final : public arenai::view::AbstractWindowedGraphicBackend { +class HeadlessWindowedBackend final : public view::AbstractWindowedGraphicBackend { public: - std::shared_ptr render_context() override { - return nullptr; - } + std::shared_ptr render_context() override { return nullptr; } - std::unique_ptr make_offscreen_renderer( + std::unique_ptr make_offscreen_renderer( int width, int height, glm::vec3 light_pos, - const std::shared_ptr &camera) override { + const std::shared_ptr &camera) override { return nullptr; } - std::shared_ptr drawable_factory() override { + std::shared_ptr drawable_factory() override { return std::make_shared(); } - std::shared_ptr hud_factory() override { return nullptr; } + std::shared_ptr hud_factory() override { return nullptr; } std::string renderer_info() override { return "headless test backend"; } void release_thread() override {} - std::shared_ptr get_window() override { return nullptr; } + std::shared_ptr get_window() override { return nullptr; } - std::unique_ptr make_player_renderer( - glm::vec3 light_pos, const std::shared_ptr &camera) override { + std::unique_ptr make_player_renderer( + glm::vec3 light_pos, const std::shared_ptr &camera) override { return std::make_unique(); } diff --git a/arenai_desktop/tests/include/arenai_desktop_tests/interceptor_agent.h b/arenai_desktop/tests/include/arenai_desktop_tests/interceptor_agent.h index 29754151..addbd5c6 100644 --- a/arenai_desktop/tests/include/arenai_desktop_tests/interceptor_agent.h +++ b/arenai_desktop/tests/include/arenai_desktop_tests/interceptor_agent.h @@ -12,20 +12,23 @@ // Records every batch of states the game loop hands to the agent and answers // with neutral actions: what act() received is exactly what the SAC agent // would have seen in run_game(). -class InterceptorAgent final : public arenai::agent::AbstractAgent { +class InterceptorAgent final : public agent::AbstractAgent { public: - std::vector> received_states; + std::vector> received_states; int last_vision_height = -1; int last_vision_width = -1; - std::vector - act(const std::vector &states, const int vision_height, + std::vector + act(const std::vector &states, const int vision_height, const int vision_width) override { received_states.push_back(states); last_vision_height = vision_height; last_vision_width = vision_width; - return std::vector(states.size(), {{0.f, 0.f}, {0.f, 0.f}, {false}}); + return std::vector( + states.size(), {.left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {false}}); } void load(const std::filesystem::path &agent_folder) override {} diff --git a/arenai_desktop/tests/src/tests_e2e_agent_input.cpp b/arenai_desktop/tests/src/tests_e2e_agent_input.cpp index c55d6242..8a3dec7c 100644 --- a/arenai_desktop/tests/src/tests_e2e_agent_input.cpp +++ b/arenai_desktop/tests/src/tests_e2e_agent_input.cpp @@ -53,7 +53,7 @@ namespace { const auto steps = env.step(FREQUENCY, actions); states.clear(); - for (const auto &[state, reward, done, truncated]: steps) states.push_back(state); + for (const auto &[state, reward, done]: steps) states.push_back(state); actions = agent.act(states, VISION_HEIGHT, VISION_WIDTH); } @@ -91,7 +91,7 @@ namespace { void write_golden(const std::filesystem::path &golden_path, const std::vector &pixels) { std::filesystem::create_directories(golden_path.parent_path()); - nlohmann::json output_json(pixels); + const nlohmann::json output_json(pixels); std::ofstream output_file(golden_path); output_file << output_json; } diff --git a/arenai_model/include/arenai_model/constants.h b/arenai_model/include/arenai_model/constants.h index 08a14c27..46188a40 100644 --- a/arenai_model/include/arenai_model/constants.h +++ b/arenai_model/include/arenai_model/constants.h @@ -5,18 +5,18 @@ #ifndef ARENAI_MODEL_CONSTANTS_H #define ARENAI_MODEL_CONSTANTS_H -#include +#include namespace arenai::model { - constexpr float WHEEL_RADIAL_VELOCITY = static_cast(M_PI) * 5.f; + constexpr float WHEEL_RADIAL_VELOCITY = std::numbers::pi * 5.f; // (pos, vel, forward, up, ang_vel) * (9 items: 6 wheels, 1 chassis, 1 turret, 1 canon) - chassis pos + remaining shells ratio constexpr int ENEMY_PROPRIOCEPTION_SIZE = (3 + 3 + 3 + 3 + 3) * (6 + 3) - 3 + 1; constexpr int ENEMY_NB_CONTINUOUS_ACTION = 2 + 2; constexpr int ENEMY_NB_DISCRETE_ACTION = 2; - constexpr float ENEMY_TURRET_RADIAL_VELOCITY = static_cast(M_PI) * 1.f; + constexpr float ENEMY_TURRET_RADIAL_VELOCITY = std::numbers::pi * 1.f; }// namespace arenai::model diff --git a/arenai_model/include/arenai_model/item.h b/arenai_model/include/arenai_model/item.h index ebe401ea..a43fc525 100644 --- a/arenai_model/include/arenai_model/item.h +++ b/arenai_model/include/arenai_model/item.h @@ -59,6 +59,10 @@ namespace arenai::model { bool is_dead() const; bool is_already_dead(); + // zeroes the health and flags the item as already dead: a wreck pays + // neither hit nor kill to whoever keeps shooting at it + void kill(); + float receive_damages(float damages); int consume_hits_received(); diff --git a/arenai_model/include/arenai_model/tank.h b/arenai_model/include/arenai_model/tank.h index f78c63cb..355da2b9 100644 --- a/arenai_model/include/arenai_model/tank.h +++ b/arenai_model/include/arenai_model/tank.h @@ -39,7 +39,6 @@ namespace arenai::model { class EnemyTank : virtual public Tank { public: virtual float get_reward(const std::vector> &tanks) = 0; - virtual float get_phi(const std::vector> &tanks) = 0; virtual std::vector get_proprioception() = 0; virtual bool has_hit_other_tank() = 0; virtual bool has_fired_shell() = 0; diff --git a/arenai_model/src/item.cpp b/arenai_model/src/item.cpp index 297f0bf3..71300601 100644 --- a/arenai_model/src/item.cpp +++ b/arenai_model/src/item.cpp @@ -42,6 +42,11 @@ namespace arenai::model { return already_dead; } + void LifeItem::kill() { + health_points = 0.f; + already_dead = true; + } + float LifeItem::receive_damages(const float damages) { const float new_health_point = std::max(health_points - damages, 0.f); const float received_damages = health_points - new_health_point; diff --git a/arenai_model/src/items/convex.cpp b/arenai_model/src/items/convex.cpp index b12f5a95..13ef5d8c 100644 --- a/arenai_model/src/items/convex.cpp +++ b/arenai_model/src/items/convex.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include diff --git a/arenai_model/src/items/height_map.cpp b/arenai_model/src/items/height_map.cpp index a59e77e3..b3cb7f10 100644 --- a/arenai_model/src/items/height_map.cpp +++ b/arenai_model/src/items/height_map.cpp @@ -68,8 +68,8 @@ namespace arenai::model { auto push_triangle = [&](const glm::vec3 &p0, const glm::vec3 &p1, const glm::vec3 &p2) { - const glm::vec3 normal = glm::cross(p1 - p0, p2 - p0); - if (normal.y >= 0.f) triangles.emplace_back(to_jolt(p0), to_jolt(p1), to_jolt(p2)); + if (const glm::vec3 normal = glm::cross(p1 - p0, p2 - p0); normal.y >= 0.f) + triangles.emplace_back(to_jolt(p0), to_jolt(p1), to_jolt(p2)); else triangles.emplace_back(to_jolt(p0), to_jolt(p2), to_jolt(p1)); }; diff --git a/arenai_model/src/jolt_engine.cpp b/arenai_model/src/jolt_engine.cpp index 8020c0fa..71705390 100644 --- a/arenai_model/src/jolt_engine.cpp +++ b/arenai_model/src/jolt_engine.cpp @@ -44,8 +44,8 @@ namespace { JPH::uint GetNumBroadPhaseLayers() const override { return broad_phase_layers::NUM_LAYERS; } JPH::BroadPhaseLayer GetBroadPhaseLayer(const JPH::ObjectLayer layer) const override { - return layer == arenai::model::layers::NON_MOVING ? broad_phase_layers::NON_MOVING - : broad_phase_layers::MOVING; + return layer == layers::NON_MOVING ? broad_phase_layers::NON_MOVING + : broad_phase_layers::MOVING; } #if defined(JPH_EXTERNAL_PROFILE) || defined(JPH_PROFILE_ENABLED) @@ -59,8 +59,7 @@ namespace { public: bool ShouldCollide( const JPH::ObjectLayer layer1, const JPH::BroadPhaseLayer layer2) const override { - return layer1 != arenai::model::layers::NON_MOVING - || layer2 != broad_phase_layers::NON_MOVING; + return layer1 != layers::NON_MOVING || layer2 != broad_phase_layers::NON_MOVING; } }; @@ -68,8 +67,7 @@ namespace { public: bool ShouldCollide(const JPH::ObjectLayer layer1, const JPH::ObjectLayer layer2) const override { - return layer1 != arenai::model::layers::NON_MOVING - || layer2 != arenai::model::layers::NON_MOVING; + return layer1 != layers::NON_MOVING || layer2 != layers::NON_MOVING; } }; @@ -79,16 +77,20 @@ namespace arenai::model { void JoltPhysicEngine::BufferedContactListener::record( const JPH::Body &body1, const JPH::Body &body2, const JPH::ContactManifold &manifold) { - // Bullet fired on_contact once per penetrating manifold point + // speculative contacts are reported before they touch: only penetration counts if (manifold.mPenetrationDepth <= 0.f) return; const auto item_a = reinterpret_cast(body1.GetUserData()); const auto item_b = reinterpret_cast(body2.GetUserData()); if (item_a == nullptr || item_b == nullptr) return; + // one dispatch per touching body pair, never one per manifold point: Jolt + // builds the whole contact patch on the very first frame of an impact, where + // Bullet's persistent manifold grew one point per frame. Iterating the points + // would multiply on_contact — and the damages an impact deals — by however + // many points the clipping happened to produce std::lock_guard lock(contacts_mutex); - for (JPH::uint i = 0; i < manifold.mRelativeContactPointsOn1.size(); i++) - contacts.emplace_back(item_a, item_b); + contacts.emplace_back(item_a, item_b); } void JoltPhysicEngine::BufferedContactListener::OnContactAdded( diff --git a/arenai_model/src/jolt_engine.h b/arenai_model/src/jolt_engine.h index ed5c4ac6..7160a694 100644 --- a/arenai_model/src/jolt_engine.h +++ b/arenai_model/src/jolt_engine.h @@ -75,7 +75,7 @@ namespace arenai::model { float get_interpolation_delta() const; private: - // every step's penetrating contact points, kept until the next simulated + // every step's penetrating body pairs, kept until the next simulated // sub-step so that a step() call that simulates nothing (accumulated time // under the fixed timestep) re-fires the same contacts, like Bullet's // persistent manifolds did diff --git a/arenai_model/src/tank/jolt_enemy_tank.cpp b/arenai_model/src/tank/jolt_enemy_tank.cpp index 17b2f2e1..d6e010a2 100644 --- a/arenai_model/src/tank/jolt_enemy_tank.cpp +++ b/arenai_model/src/tank/jolt_enemy_tank.cpp @@ -21,6 +21,24 @@ using namespace arenai; using namespace arenai::model; +namespace { + + // closest point of segment [a, b] to p, and its distance + float distance_to_segment( + const glm::vec3 &a, const glm::vec3 &b, const glm::vec3 &p, glm::vec3 &closest) { + const glm::vec3 ab = b - a; + const float length_squared = glm::length2(ab); + + const float t = + length_squared > 0.f ? std::clamp(glm::dot(p - a, ab) / length_squared, 0.f, 1.f) : 0.f; + + closest = a + t * ab; + + return glm::length(p - closest); + } + +}// namespace + namespace arenai::model { JoltEnemyTank::JoltEnemyTank( @@ -30,31 +48,18 @@ namespace arenai::model { const float wanted_frame_frequency) : JoltTank( engine, file_reader, tank_prefix_name, chassis_pos, wanted_frame_frequency, - [this](const ShellContactInfo &info, Item *item) { on_shell_contact(info, item); }, + [this](const ShellItem *shell, const ShellContactInfo &info, Item *item) { + on_shell_contact(shell, info, item); + }, [this](const std::shared_ptr &shell) { on_shell_fired(shell); }, [this] { return nb_shells > 0; }), max_frames_upside_down(static_cast(4.f / wanted_frame_frequency)), - curr_frame_upside_down(0), distance_scale(250.f), miss_distance_scale(2.f), - hit_reward_scale(0.5f), optimal_distance(75.f), aim_angle_scale(glm::radians(10.f)), - hit_received_cost(0.1f), fire_cost(0.05f), initial_nb_shells(30), - nb_shells(initial_nb_shells), shells_recharged_per_hit(5), - is_dead_already_triggered(false), has_touch(false), has_fired(false), - fires_since_reward(0) {} - - float JoltEnemyTank::compute_aim_angle(const std::shared_ptr &other_tank) { - const auto canon_tr = get_canon()->get_model_matrix(); - const auto other_tr = other_tank->get_chassis()->get_model_matrix(); - - const auto canon_muzzle_pos = glm::vec3(canon_tr * glm::vec4(0.f, 0.f, 10.f, 1.f)); - const auto other_pos = glm::vec3(other_tr * glm::vec4(0.f, 0.f, 0.f, 1.f)); - - const glm::vec3 to_other = glm::normalize(other_pos - canon_muzzle_pos); - const auto forward = glm::normalize(glm::vec3(canon_tr * glm::vec4(0.f, 0.f, 1.f, 0.f))); - - const float d = std::clamp(glm::dot(forward, to_other), -1.f, 1.f); - - return std::acos(d); - } + curr_frame_upside_down(0), miss_distance_scale(1.5f), hit_reward_scale(0.5f), + hit_received_cost(0.1f), initial_nb_shells(10), nb_shells(initial_nb_shells), + shells_recharged_per_hit(5), + nb_frames_per_shell_regen(static_cast(1.5f / wanted_frame_frequency)), + curr_frame_shell_regen(0), is_dead_already_triggered(false), has_touch(false), + has_fired(false) {} float JoltEnemyTank::compute_hit_reward( const glm::vec3 &fire_pos, const glm::vec3 &enemy_pos, const glm::vec3 &shell_pos) const { @@ -65,7 +70,8 @@ namespace arenai::model { const float ideal_trajectory_distance = glm::length(ideal_trajectory); const float miss_distance = glm::length(miss_trajectory); - const float ratio = miss_distance_scale * miss_distance / ideal_trajectory_distance; + const float ratio = + miss_distance_scale * miss_distance / std::sqrt(ideal_trajectory_distance); return std::exp(-0.5f * std::pow(ratio, 2.f)); } @@ -73,20 +79,28 @@ namespace arenai::model { void JoltEnemyTank::update_closest_approach( TrackedShell &tracked, const glm::vec3 &shell_pos, const std::vector> &tanks) const { - const int nearest_index = get_nearest_enemy_index(tanks, shell_pos); - if (nearest_index == -1) return; - - const auto enemy_pos = glm::vec3( - tanks[nearest_index]->get_chassis()->get_model_matrix() - * glm::vec4(glm::vec3(0.f), 1.f)); - - if (const float distance = glm::length(shell_pos - enemy_pos); - distance < tracked.min_distance) { - tracked.min_distance = distance; - tracked.enemy_pos_at_t = enemy_pos; - tracked.shell_pos_at_t = shell_pos; - tracked.has_sample = true; + if (const int nearest_index = get_nearest_enemy_index(tanks, shell_pos); + nearest_index != -1) { + + const auto enemy_pos = glm::vec3( + tanks[nearest_index]->get_chassis()->get_model_matrix() + * glm::vec4(glm::vec3(0.f), 1.f)); + + // a shell covers ~8 m per frame, an order of magnitude more than the + // dispersion sigma: sampling positions alone aliases the miss distance, + // so measure against the segment actually travelled during the frame + glm::vec3 closest; + if (const float distance = + distance_to_segment(tracked.last_shell_pos, shell_pos, enemy_pos, closest); + distance < tracked.min_distance) { + tracked.min_distance = distance; + tracked.enemy_pos_at_t = enemy_pos; + tracked.shell_pos_at_t = closest; + tracked.has_sample = true; + } } + + tracked.last_shell_pos = shell_pos; } int JoltEnemyTank::get_nearest_enemy_index( @@ -112,46 +126,6 @@ namespace arenai::model { return best_i; } - float JoltEnemyTank::get_phi(const std::vector> &tanks) { - constexpr glm::vec4 world_center(glm::vec3(0.f), 1.f); - const glm::vec3 chassis_pos = get_chassis()->get_model_matrix() * world_center; - - std::vector scores; - std::vector logits; - - for (const auto &enemy: tanks) { - if (enemy.get() == this || enemy->is_dead()) continue; - - const glm::vec3 enemy_pos = enemy->get_chassis()->get_model_matrix() * world_center; - - const float distance = glm::length(enemy_pos - chassis_pos); - const float angle = compute_aim_angle(enemy); - - const float distance_score = - std::exp(-0.5f * std::pow((distance - optimal_distance) / distance_scale, 2.f)); - // sharp gaussian: pointing the canon at an enemy is the dense precursor of a - // hit; the previous (cos+1)/2 was still at 0.93 with 30 degrees of error - const float angle_score = std::exp(-0.5f * std::pow(angle / aim_angle_scale, 2.f)); - - scores.push_back(distance_score * angle_score); - logits.push_back(-distance / distance_scale); - } - - if (scores.empty()) return 0.f; - - const float max_logit = *std::ranges::max_element(logits); - float sum_exp = 0.f; - for (const float l: logits) sum_exp += std::exp(l - max_logit); - - float reward = 0.f; - for (std::size_t i = 0; i < scores.size(); ++i) { - const float weight = std::exp(logits[i] - max_logit) / sum_exp; - reward += weight * scores[i]; - } - - return reward; - } - float JoltEnemyTank::get_reward(const std::vector> &tanks) { // 1. flipped detection @@ -163,15 +137,20 @@ namespace arenai::model { curr_frame_upside_down++; else curr_frame_upside_down = 0; - // 2. dead / suicide penalty + // 2. passive shell regeneration, capped at the initial reserve: the hit recharge + // may push the reserve above it, regeneration never does + if (++curr_frame_shell_regen >= nb_frames_per_shell_regen) { + curr_frame_shell_regen = 0; + if (nb_shells < initial_nb_shells) nb_shells++; + } + + // 3. dead / suicide penalty const auto dead_penalty = is_dead() ? -1.f : 0.f; - // 3. fired shells: pay a small cost at fire time (a shot toward nobody must be - // net-negative, break-even at gaussian ~0.1), then sample the closest tank along - // the trajectory and pay the dispersion gaussian (plus hit/kill bonuses) once the - // shell dies - float shells_reward = -fire_cost * static_cast(fires_since_reward); - fires_since_reward = 0; + // 4. fired shells: sample the closest tank along the trajectory and pay the + // dispersion gaussian (plus hit/kill bonuses) once the shell dies; firing itself + // is free — the limited shell reserve (recharged on hit) taxes the spam + float shells_reward = 0.f; for (int i = static_cast(tracked_shells.size()) - 1; i >= 0; i--) { auto &tracked = tracked_shells[i]; @@ -186,8 +165,7 @@ namespace arenai::model { if (tracked.has_sample) { // the gaussian stays an order of magnitude under the hit bonus: a gradient - // toward the aim, not a farmable income; fire_cost and the shell reserve - // tax the spam + // toward the aim, not a farmable income; the shell reserve taxes the spam shells_reward += hit_reward_scale * compute_hit_reward( @@ -198,11 +176,11 @@ namespace arenai::model { tracked_shells.erase(tracked_shells.begin() + i); } - // 4. hits received penalty + // 5. hits received penalty const float hit_received_penalty = -hit_received_cost * static_cast(get_received_hits()); - // 5. total reward + // 6. total reward const float reward = dead_penalty + shells_reward + hit_received_penalty; return reward; @@ -211,11 +189,12 @@ namespace arenai::model { void JoltEnemyTank::on_shell_fired(const std::shared_ptr &shell) { nb_shells--; has_fired = true; - fires_since_reward++; tracked_shells.push_back( {.shell = shell, .fire_pos = shell->get_fire_position(), + // seeding the segment at the muzzle also covers the fire → first frame gap + .last_shell_pos = shell->get_fire_position(), .min_distance = std::numeric_limits::infinity(), .enemy_pos_at_t = glm::vec3(0.f), .shell_pos_at_t = glm::vec3(0.f), @@ -226,7 +205,8 @@ namespace arenai::model { .has_killed = false}); } - void JoltEnemyTank::on_shell_contact(const ShellContactInfo &shell_info, Item *item) { + void JoltEnemyTank::on_shell_contact( + const ShellItem *shell, const ShellContactInfo &shell_info, Item *item) { for (const auto &i: get_items()) if (i->get_name() == item->get_name()) return; @@ -248,7 +228,7 @@ namespace arenai::model { if (hit) nb_shells += shells_recharged_per_hit; for (auto &tracked: tracked_shells) { - if (tracked.has_final_pos || tracked.fire_pos != shell_info.fire_position) continue; + if (tracked.has_final_pos || tracked.shell.lock().get() != shell) continue; tracked.final_shell_pos = shell_info.current_position; tracked.has_final_pos = true; @@ -287,6 +267,9 @@ namespace arenai::model { if (is_dead() && !is_dead_already_triggered) { is_dead_already_triggered = true; remove_constraints_from_engine(); + // the wreck stays in the world as an obstacle, but its surviving parts must + // not pay hits, kills, shells nor survival frames to whoever keeps shooting it + kill_life_items(); } } diff --git a/arenai_model/src/tank/jolt_enemy_tank.h b/arenai_model/src/tank/jolt_enemy_tank.h index c75a3c68..7674f0bf 100644 --- a/arenai_model/src/tank/jolt_enemy_tank.h +++ b/arenai_model/src/tank/jolt_enemy_tank.h @@ -17,7 +17,9 @@ namespace arenai::model { std::weak_ptr shell; glm::vec3 fire_pos; - // closest approach over the whole trajectory + // closest approach over the whole trajectory, measured against the segment + // travelled each frame (last_shell_pos → current position) + glm::vec3 last_shell_pos; float min_distance; glm::vec3 enemy_pos_at_t; glm::vec3 shell_pos_at_t; @@ -40,7 +42,6 @@ namespace arenai::model { float wanted_frame_frequency); float get_reward(const std::vector> &tanks) override; - float get_phi(const std::vector> &tanks) override; bool is_dead() override; bool is_first_frame_dead() override; @@ -65,32 +66,26 @@ namespace arenai::model { int max_frames_upside_down; int curr_frame_upside_down; - float distance_scale; - float miss_distance_scale; float hit_reward_scale; - float optimal_distance; - float aim_angle_scale; - float hit_received_cost; - float fire_cost; int initial_nb_shells; int nb_shells; int shells_recharged_per_hit; + int nb_frames_per_shell_regen; + int curr_frame_shell_regen; bool is_dead_already_triggered; bool has_touch; bool has_fired; - int fires_since_reward; std::vector tracked_shells; void on_shell_fired(const std::shared_ptr &shell); - void on_shell_contact(const ShellContactInfo &shell_info, Item *item); - - float compute_aim_angle(const std::shared_ptr &other_tank); + void + on_shell_contact(const ShellItem *shell, const ShellContactInfo &shell_info, Item *item); int get_nearest_enemy_index( const std::vector> &tanks, const glm::vec3 &pos) const; diff --git a/arenai_model/src/tank/jolt_player_tank.cpp b/arenai_model/src/tank/jolt_player_tank.cpp index eb8d3938..51aac22e 100644 --- a/arenai_model/src/tank/jolt_player_tank.cpp +++ b/arenai_model/src/tank/jolt_player_tank.cpp @@ -18,7 +18,8 @@ namespace arenai::model { const float wanted_frame_frequency) : JoltTank( engine, file_reader, tank_prefix_name, chassis_pos, wanted_frame_frequency, - [this](const ShellContactInfo &info, Item *item) { + // the player only counts hits, it does not track individual shells + [this](const ShellItem *, const ShellContactInfo &info, Item *item) { on_fired_shell_contact(info, item); }), killed_nb(0), hit_nb(0) {} diff --git a/arenai_model/src/tank/jolt_tank.cpp b/arenai_model/src/tank/jolt_tank.cpp index 9d5e08f4..84673784 100644 --- a/arenai_model/src/tank/jolt_tank.cpp +++ b/arenai_model/src/tank/jolt_tank.cpp @@ -18,8 +18,6 @@ using namespace arenai; using namespace arenai::model; -using namespace arenai::view; -using namespace arenai::controller; namespace arenai::model { @@ -28,7 +26,8 @@ namespace arenai::model { const std::shared_ptr &file_reader, const std::string &tank_prefix_name, glm::vec3 chassis_pos, const float wanted_frame_frequency, - const std::function &on_contact_callback, + const std::function + &on_contact_callback, const std::function &)> &on_shell_fired_callback, const std::function &can_fire_callback) : engine(engine), name(tank_prefix_name), camera(std::nullptr_t()), @@ -60,9 +59,9 @@ namespace arenai::model { for (auto &[wheel_name, wheel_pos, angle_factor]: front_wheel_config) { auto wheel = std::make_shared( - tank_prefix_name + "_" + wheel_name, engine, file_reader, wheel_pos + chassis_pos, - wheel_pos, wheel_scale, wheel_mass, chassis_item->get_body(), front_axle_z, - angle_factor); + std::format("{}_{}", tank_prefix_name, wheel_name), engine, file_reader, + wheel_pos + chassis_pos, wheel_pos, wheel_scale, wheel_mass, + chassis_item->get_body(), front_axle_z, angle_factor); jolt_items.push_back(wheel); items.push_back(wheel); @@ -76,8 +75,9 @@ namespace arenai::model { for (auto &[wheel_name, wheel_pos]: wheel_config) { auto wheel = std::make_shared( - tank_prefix_name + "_" + wheel_name, engine, file_reader, wheel_pos + chassis_pos, - wheel_pos, wheel_scale, wheel_mass, chassis_item->get_body(), front_axle_z); + std::format("{}_{}", tank_prefix_name, wheel_name), engine, file_reader, + wheel_pos + chassis_pos, wheel_pos, wheel_scale, wheel_mass, + chassis_item->get_body(), front_axle_z); jolt_items.push_back(wheel); items.push_back(wheel); @@ -102,8 +102,11 @@ namespace arenai::model { auto canon_item = std::make_shared( tank_prefix_name, engine, file_reader, chassis_pos + turret_pos + canon_pos, canon_pos, scale * canon_scale, 100, turret->get_body(), wanted_frame_frequency, - [on_contact_callback](const glm::vec3 fire_pos, const glm::vec3 hit_pos, Item *item) { - on_contact_callback({fire_pos, hit_pos}, item); + [on_contact_callback]( + const ShellItem *shell, const glm::vec3 fire_pos, const glm::vec3 hit_pos, + Item *item) { + on_contact_callback( + shell, {.fire_position = fire_pos, .current_position = hit_pos}, item); }, on_shell_fired_callback, can_fire_callback); @@ -149,14 +152,16 @@ namespace arenai::model { // register with engine for (const auto &item: jolt_items) engine.add_jolt_item(item); - engine.add_jolt_item_producer([c = canon_item]() { return c->produce_jolt_items(); }); + engine.add_jolt_item_producer([c = canon_item] { return c->produce_jolt_items(); }); } - std::shared_ptr JoltTank::get_camera() { return camera; } + std::shared_ptr JoltTank::get_camera() { return camera; } std::vector> JoltTank::get_items() { return items; } - std::vector> JoltTank::get_controllers() { return controllers; } + std::vector> JoltTank::get_controllers() { + return controllers; + } std::map> JoltTank::load_shell_shapes() const { return {{ShellItem::NAME, ShellItem::load_shape(file_reader)}}; @@ -166,7 +171,7 @@ namespace arenai::model { return std::ranges::any_of(life_items, [](const LifeItem *li) { return li->is_dead(); }); } - int JoltTank::get_received_hits() { + int JoltTank::get_received_hits() const { int hits = 0; for (const auto life_item: life_items) hits += life_item->consume_hits_received(); return hits; @@ -176,6 +181,10 @@ namespace arenai::model { std::shared_ptr JoltTank::get_canon() { return canon; } + void JoltTank::kill_life_items() const { + for (const auto life_item: life_items) life_item->kill(); + } + void JoltTank::remove_constraints_from_engine() const { for (const auto &item: jolt_items) engine.remove_jolt_item_constraints(item); } diff --git a/arenai_model/src/tank/jolt_tank.h b/arenai_model/src/tank/jolt_tank.h index 7d79501e..7a9a010c 100644 --- a/arenai_model/src/tank/jolt_tank.h +++ b/arenai_model/src/tank/jolt_tank.h @@ -24,7 +24,8 @@ namespace arenai::model { const std::shared_ptr &file_reader, const std::string &tank_prefix_name, glm::vec3 chassis_pos, float wanted_frame_frequency, - const std::function &on_contact_callback, + const std::function + &on_contact_callback, const std::function &)> &on_shell_fired_callback = [](const std::shared_ptr &) {}, const std::function &can_fire_callback = [] { return true; }); @@ -34,7 +35,7 @@ namespace arenai::model { std::vector> get_controllers() override; std::map> load_shell_shapes() const override; bool is_dead() override; - int get_received_hits(); + int get_received_hits() const; std::shared_ptr get_chassis() override; std::shared_ptr get_canon() override; @@ -42,6 +43,7 @@ namespace arenai::model { protected: void remove_constraints_from_engine() const; + void kill_life_items() const; JoltPhysicEngine &engine; private: diff --git a/arenai_model/src/tank/parts/canon.cpp b/arenai_model/src/tank/parts/canon.cpp index 2624d4a2..0f527c11 100644 --- a/arenai_model/src/tank/parts/canon.cpp +++ b/arenai_model/src/tank/parts/canon.cpp @@ -10,7 +10,6 @@ using namespace arenai; using namespace arenai::model; -using namespace arenai::controller; namespace { @@ -32,7 +31,7 @@ namespace arenai::model { const std::shared_ptr &file_reader, glm::vec3 pos, glm::vec3 rel_pos, glm::vec3 scale, float mass, JPH::Body *turret, const float wanted_frame_frequency, - const std::function &on_contact, + const std::function &on_contact, const std::function &)> &on_shell_fired, const std::function &can_fire) : LifeItem(5), ConvexItem( @@ -55,8 +54,11 @@ namespace arenai::model { settings.mHingeAxis2 = JPH::Vec3::sAxisX(); settings.mNormalAxis2 = JPH::Vec3::sAxisY(); - hinge = - static_cast(settings.Create(*turret, *ConvexItem::get_body())); + auto *constraint = settings.Create(*turret, *ConvexItem::get_body()); + // Jolt is built without RTTI (-fno-rtti): dynamic_cast would not link. The dynamic + // type is guaranteed by the settings object the constraint was created from. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) + hinge = static_cast(constraint); hinge->SetMotorState(JPH::EMotorState::Position); hinge->SetTargetAngle(angle); @@ -98,7 +100,7 @@ namespace arenai::model { return {jolt_items.begin(), jolt_items.end()}; } - void CanonItem::apply_input(const user_input &input) { + void CanonItem::apply_input(const controller::user_input &input) { angle += input.right_joystick.y * 0.4f; angle = diff --git a/arenai_model/src/tank/parts/canon.h b/arenai_model/src/tank/parts/canon.h index 0fda787a..d6a8b2d5 100644 --- a/arenai_model/src/tank/parts/canon.h +++ b/arenai_model/src/tank/parts/canon.h @@ -31,7 +31,7 @@ namespace arenai::model { const std::shared_ptr &file_reader, glm::vec3 pos, glm::vec3 rel_pos, glm::vec3 scale, float mass, JPH::Body *turret, float wanted_frame_frequency, - const std::function &on_contact, + const std::function &on_contact, const std::function &)> &on_shell_fired, const std::function &can_fire); @@ -51,7 +51,7 @@ namespace arenai::model { JPH::Ref hinge; std::shared_ptr file_reader; bool will_fire; - std::function on_contact; + std::function on_contact; std::function &)> on_shell_fired; std::function can_fire; float wanted_frame_frequency; diff --git a/arenai_model/src/tank/parts/shell.cpp b/arenai_model/src/tank/parts/shell.cpp index 66f3658b..db375d79 100644 --- a/arenai_model/src/tank/parts/shell.cpp +++ b/arenai_model/src/tank/parts/shell.cpp @@ -19,17 +19,21 @@ namespace arenai::model { const std::shared_ptr &file_reader, const glm::vec3 pos, const glm::quat rot, const glm::vec3 scale, const float mass, const float wanted_frame_frequency, - const std::function &contact_callback) + const std::function + &contact_callback) : LifeItem(1), ConvexItem(NAME, engine, load_shape(file_reader), pos, scale, mass, rot), contact_callback(contact_callback), nb_frames_alive(static_cast(20.f / wanted_frame_frequency)), start_pos(pos) {} void ShellItem::on_contact(Item *other) { + if (is_dead()) return; + if (const auto t = dynamic_cast(other)) t->receive_damages(1); receive_damages(1); Item::on_contact(other); - contact_callback(get_fire_position(), get_current_position(), other); + + contact_callback(this, get_fire_position(), get_current_position(), other); if (is_dead()) destroy(); } diff --git a/arenai_model/src/tank/parts/shell.h b/arenai_model/src/tank/parts/shell.h index 1e6bc782..c5df60c5 100644 --- a/arenai_model/src/tank/parts/shell.h +++ b/arenai_model/src/tank/parts/shell.h @@ -25,12 +25,12 @@ namespace arenai::model { JoltPhysicEngine &engine, const std::shared_ptr &file_reader, glm::vec3 pos, glm::quat rot, glm::vec3 scale, float mass, float wanted_frame_frequency, - const std::function &contact_callback = - [](glm::vec3, glm::vec3, Item *) {}); + const std::function + &contact_callback = [](const ShellItem *, glm::vec3, glm::vec3, Item *) {}); void on_contact(Item *other) override; - inline const static std::string NAME = "shell_item"; + static constexpr std::string NAME = "shell_item"; void tick() override; @@ -38,7 +38,7 @@ namespace arenai::model { glm::vec3 get_current_position(); private: - std::function contact_callback; + std::function contact_callback; int nb_frames_alive; glm::vec3 start_pos; diff --git a/arenai_model/src/tank/parts/turret.cpp b/arenai_model/src/tank/parts/turret.cpp index 0e2170db..9f122d0e 100644 --- a/arenai_model/src/tank/parts/turret.cpp +++ b/arenai_model/src/tank/parts/turret.cpp @@ -6,7 +6,6 @@ #include #include -#include using namespace arenai; using namespace arenai::model; @@ -35,8 +34,11 @@ namespace arenai::model { settings.mHingeAxis2 = JPH::Vec3::sAxisY(); settings.mNormalAxis2 = JPH::Vec3::sAxisX(); - hinge = - static_cast(settings.Create(*chassis, *ConvexItem::get_body())); + auto *constraint = settings.Create(*chassis, *ConvexItem::get_body()); + // Jolt is built without RTTI (-fno-rtti): dynamic_cast would not link. The dynamic + // type is guaranteed by the settings object the constraint was created from. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) + hinge = static_cast(constraint); // like Bullet's limit-less hinge: free until the first input engages // the servo } diff --git a/arenai_model/src/tank/parts/wheel.cpp b/arenai_model/src/tank/parts/wheel.cpp index 00339b71..3d9d7f89 100644 --- a/arenai_model/src/tank/parts/wheel.cpp +++ b/arenai_model/src/tank/parts/wheel.cpp @@ -4,8 +4,6 @@ #include "./wheel.h" -#include - #include #include @@ -62,8 +60,11 @@ namespace arenai::model { settings.mMotorSettings[EAxis::RotationX].mMinTorqueLimit = -2e4f; settings.mMotorSettings[EAxis::RotationX].mMaxTorqueLimit = 2e4f; - hinge = static_cast( - settings.Create(*chassis, *ConvexItem::get_body())); + auto *constraint = settings.Create(*chassis, *ConvexItem::get_body()); + // Jolt is built without RTTI (-fno-rtti): dynamic_cast would not link. The dynamic + // type is guaranteed by the settings object the constraint was created from. + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-static-cast-downcast) + hinge = static_cast(constraint); hinge->SetMotorState(EAxis::TranslationY, JPH::EMotorState::Position); hinge->SetTargetPositionCS(JPH::Vec3(0.f, -0.2f, 0.f)); diff --git a/arenai_model/tests/include/arenai_model_tests/tests_enemy_tank/tests_enemy_tank.h b/arenai_model/tests/include/arenai_model_tests/tests_enemy_tank/tests_enemy_tank.h index 111e0973..dbd2cd1a 100644 --- a/arenai_model/tests/include/arenai_model_tests/tests_enemy_tank/tests_enemy_tank.h +++ b/arenai_model/tests/include/arenai_model_tests/tests_enemy_tank/tests_enemy_tank.h @@ -5,8 +5,6 @@ #ifndef ARENAI_MODEL_TESTS_ENEMY_TANK_H #define ARENAI_MODEL_TESTS_ENEMY_TANK_H -#include - #include "../utils/engine_test_fixture.h" class EnemyTankTest : public EngineTestFixture {}; diff --git a/arenai_model/tests/include/arenai_model_tests/tests_player_tank/tests_player_tank.h b/arenai_model/tests/include/arenai_model_tests/tests_player_tank/tests_player_tank.h index ce699b16..f34f7b86 100644 --- a/arenai_model/tests/include/arenai_model_tests/tests_player_tank/tests_player_tank.h +++ b/arenai_model/tests/include/arenai_model_tests/tests_player_tank/tests_player_tank.h @@ -5,8 +5,6 @@ #ifndef ARENAI_MODEL_TESTS_PLAYER_TANK_H #define ARENAI_MODEL_TESTS_PLAYER_TANK_H -#include - #include "../utils/engine_test_fixture.h" class PlayerTankTest : public EngineTestFixture {}; diff --git a/arenai_model/tests/include/arenai_model_tests/tests_proprioception/tests_proprioception.h b/arenai_model/tests/include/arenai_model_tests/tests_proprioception/tests_proprioception.h index f900bdef..3d3e7f11 100644 --- a/arenai_model/tests/include/arenai_model_tests/tests_proprioception/tests_proprioception.h +++ b/arenai_model/tests/include/arenai_model_tests/tests_proprioception/tests_proprioception.h @@ -5,8 +5,6 @@ #ifndef ARENAI_MODEL_TESTS_PROPRIOCEPTION_H #define ARENAI_MODEL_TESTS_PROPRIOCEPTION_H -#include - #include "../utils/engine_test_fixture.h" class ProprioceptionTest : public EngineTestFixture {}; diff --git a/arenai_model/tests/include/arenai_model_tests/tests_reward/tests_reward.h b/arenai_model/tests/include/arenai_model_tests/tests_reward/tests_reward.h index 8908d93d..7216b0a2 100644 --- a/arenai_model/tests/include/arenai_model_tests/tests_reward/tests_reward.h +++ b/arenai_model/tests/include/arenai_model_tests/tests_reward/tests_reward.h @@ -5,8 +5,6 @@ #ifndef ARENAI_MODEL_TESTS_REWARD_H #define ARENAI_MODEL_TESTS_REWARD_H -#include - #include "../utils/engine_test_fixture.h" class RewardTest : public EngineTestFixture {}; diff --git a/arenai_model/tests/include/arenai_model_tests/tests_shell/tests_shell.h b/arenai_model/tests/include/arenai_model_tests/tests_shell/tests_shell.h index bd060f33..3242cc7d 100644 --- a/arenai_model/tests/include/arenai_model_tests/tests_shell/tests_shell.h +++ b/arenai_model/tests/include/arenai_model_tests/tests_shell/tests_shell.h @@ -5,8 +5,6 @@ #ifndef ARENAI_MODEL_TESTS_SHELL_H #define ARENAI_MODEL_TESTS_SHELL_H -#include - #include "../utils/engine_test_fixture.h" class ShellTest : public EngineTestFixture {}; diff --git a/arenai_model/tests/include/arenai_model_tests/utils/engine_test_fixture.h b/arenai_model/tests/include/arenai_model_tests/utils/engine_test_fixture.h index ba9389fd..dd107629 100644 --- a/arenai_model/tests/include/arenai_model_tests/utils/engine_test_fixture.h +++ b/arenai_model/tests/include/arenai_model_tests/utils/engine_test_fixture.h @@ -11,7 +11,6 @@ #include #include -#include #include #include diff --git a/arenai_model/tests/src/tests_enemy_tank/tests_enemy_tank.cpp b/arenai_model/tests/src/tests_enemy_tank/tests_enemy_tank.cpp index 62fe0c00..4ac38bbb 100644 --- a/arenai_model/tests/src/tests_enemy_tank/tests_enemy_tank.cpp +++ b/arenai_model/tests/src/tests_enemy_tank/tests_enemy_tank.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include @@ -104,12 +103,15 @@ TEST_F(EnemyTankTest, RewardWhenAllEnemiesDeadAndShellFired) { ASSERT_TRUE(shared_b->is_dead()); // fire from tank_a — shell will hit the dead tank or ground - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: shared_a->get_controllers()) ctrl->apply_input(fire_input); for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); - const std::vector> tanks{shared_a, shared_b}; + const std::vector tanks{shared_a, shared_b}; // get_nearest_enemy_index should return -1 (all dead) // reward should not crash and should be 0 (no valid target) @@ -127,12 +129,15 @@ TEST_F(EnemyTankTest, RewardNoNaNWhenAloneInTankList) { const std::shared_ptr shared_tank(tank.release()); // fire a shell that will hit the ground - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: shared_tank->get_controllers()) ctrl->apply_input(fire_input); for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); - const std::vector> tanks{shared_tank}; + const std::vector tanks{shared_tank}; const float reward = shared_tank->get_reward(tanks); ASSERT_FALSE(std::isnan(reward)) << "reward should not be NaN when alone"; @@ -153,12 +158,18 @@ TEST_F(EnemyTankTest, ShellHitsGroundNoRewardNoCrash) { const std::shared_ptr shared_tank(tank.release()); // tilt canon downward to ensure it hits the ground - constexpr user_input aim_down{{0.f, 0.f}, {0.f, 1.f}, {false}}; + constexpr user_input aim_down{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 1.f}, + .fire_button = {false}}; for (const auto &ctrl: shared_tank->get_controllers()) ctrl->apply_input(aim_down); for (const auto &ctrl: shared_tank->get_controllers()) ctrl->apply_input(aim_down); for (const auto &ctrl: shared_tank->get_controllers()) ctrl->apply_input(aim_down); - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: shared_tank->get_controllers()) ctrl->apply_input(fire_input); for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); @@ -168,7 +179,7 @@ TEST_F(EnemyTankTest, ShellHitsGroundNoRewardNoCrash) { << "hitting the ground should not count as hitting another tank"; // but last_shoot_info should still be set — reward should not crash - std::vector> tanks{shared_tank}; + const std::vector tanks{shared_tank}; const float reward = shared_tank->get_reward(tanks); ASSERT_FALSE(std::isnan(reward)); } @@ -207,7 +218,10 @@ TEST_F(EnemyTankTest, HasHitOtherTankResetsAfterCall) { const std::shared_ptr shared_a(tank_a.release()); std::shared_ptr shared_b(tank_b.release()); - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: shared_a->get_controllers()) ctrl->apply_input(fire_input); for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); @@ -218,3 +232,64 @@ TEST_F(EnemyTankTest, HasHitOtherTankResetsAfterCall) { ASSERT_FALSE(shared_a->has_hit_other_tank()) << "has_hit_other_tank should reset to false after being queried"; } + +// ======================================================================== +// Shell reserve — passive regeneration +// ======================================================================== + +TEST_F(EnemyTankTest, ShellReserveRegeneratesOverTime) { + // the fixture engine runs at 60 Hz: one reward call per frame, one shell every 1.5 s. + // the bounds stay a frame away from the period so the test does not depend on how the + // period rounds to an integer number of frames + constexpr int nb_frames_one_second = 60; + + add_ground(); + auto tank = tank_factory->make_enemy_tank(file_reader, "tank_a", {0.f, 5.f, 0.f}); + + for (int i = 0; i < 300; i++) engine->step(1.f / 60.f); + + const std::shared_ptr shared_tank(tank.release()); + const std::vector tanks{shared_tank}; + + ASSERT_FLOAT_EQ(shared_tank->get_proprioception().back(), 1.f) + << "the reserve should start full"; + + // fire one shell: the reserve drops by one + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; + for (const auto &ctrl: shared_tank->get_controllers()) ctrl->apply_input(fire_input); + engine->step(1.f / 60.f); + + const float reserve_after_fire = shared_tank->get_proprioception().back(); + ASSERT_LT(reserve_after_fire, 1.f) << "firing should consume a shell"; + + // after 1 s the period is not over yet: still nothing + for (int i = 0; i < nb_frames_one_second; i++) shared_tank->get_reward(tanks); + ASSERT_FLOAT_EQ(shared_tank->get_proprioception().back(), reserve_after_fire) + << "the reserve should not regenerate before the full period has elapsed"; + + // after 2 s the shell is back + for (int i = 0; i < nb_frames_one_second; i++) shared_tank->get_reward(tanks); + ASSERT_FLOAT_EQ(shared_tank->get_proprioception().back(), 1.f) + << "one shell should be given back after 1.5 s"; +} + +TEST_F(EnemyTankTest, ShellReserveRegenerationIsCappedAtInitialReserve) { + constexpr int nb_frames_one_second = 60; + + add_ground(); + auto tank = tank_factory->make_enemy_tank(file_reader, "tank_a", {0.f, 5.f, 0.f}); + + for (int i = 0; i < 300; i++) engine->step(1.f / 60.f); + + const std::shared_ptr shared_tank(tank.release()); + const std::vector tanks{shared_tank}; + + // the reserve is already full: several periods must not push it above it + for (int i = 0; i < 5 * nb_frames_one_second; i++) shared_tank->get_reward(tanks); + + ASSERT_FLOAT_EQ(shared_tank->get_proprioception().back(), 1.f) + << "regeneration should never take the reserve above its initial value"; +} diff --git a/arenai_model/tests/src/tests_player_tank/tests_player_tank.cpp b/arenai_model/tests/src/tests_player_tank/tests_player_tank.cpp index d76a8ade..a1b1162f 100644 --- a/arenai_model/tests/src/tests_player_tank/tests_player_tank.cpp +++ b/arenai_model/tests/src/tests_player_tank/tests_player_tank.cpp @@ -33,7 +33,10 @@ TEST_F(PlayerTankTest, ScoreIncreasesOnHit) { for (int i = 0; i < 300; i++) engine->step(1.f / 60.f); - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: player->get_controllers()) ctrl->apply_input(fire_input); for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); @@ -53,7 +56,10 @@ TEST_F(PlayerTankTest, ScoreHigherOnKillThanHit) { if (auto *life = dynamic_cast(item.get())) { life->receive_damages(9.f); } } - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: player->get_controllers()) ctrl->apply_input(fire_input); for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); @@ -70,7 +76,10 @@ TEST_F(PlayerTankTest, ScoreDoesNotIncreaseOnSelfHit) { for (int i = 0; i < 300; i++) engine->step(1.f / 60.f); // fire without any enemy — shell should hit the ground or expire - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: player->get_controllers()) ctrl->apply_input(fire_input); for (int i = 0; i < 1300; i++) engine->step(1.f / 60.f); diff --git a/arenai_model/tests/src/tests_proprioception/tests_proprioception.cpp b/arenai_model/tests/src/tests_proprioception/tests_proprioception.cpp index d1d9edbe..3d6ae9d4 100644 --- a/arenai_model/tests/src/tests_proprioception/tests_proprioception.cpp +++ b/arenai_model/tests/src/tests_proprioception/tests_proprioception.cpp @@ -22,7 +22,7 @@ TEST_F(ProprioceptionTest, ProprioceptionSizeCorrect) { engine->step(1.f / 60.f); - std::shared_ptr shared_tank(tank.release()); + const std::shared_ptr shared_tank(tank.release()); const auto proprio = shared_tank->get_proprioception(); ASSERT_EQ(proprio.size(), ENEMY_PROPRIOCEPTION_SIZE); @@ -34,7 +34,7 @@ TEST_F(ProprioceptionTest, ProprioceptionNoNaN) { engine->step(1.f / 60.f); - std::shared_ptr shared_tank(tank.release()); + const std::shared_ptr shared_tank(tank.release()); const auto proprio = shared_tank->get_proprioception(); for (int i = 0; i < proprio.size(); i++) { @@ -49,7 +49,7 @@ TEST_F(ProprioceptionTest, ProprioceptionNoInfinity) { for (int i = 0; i < 10; i++) engine->step(1.f / 60.f); - std::shared_ptr shared_tank(tank.release()); + const std::shared_ptr shared_tank(tank.release()); const auto proprio = shared_tank->get_proprioception(); for (int i = 0; i < proprio.size(); i++) { @@ -63,7 +63,7 @@ TEST_F(ProprioceptionTest, ProprioceptionContainsSubItemRelativePositions) { engine->step(1.f / 60.f); - std::shared_ptr shared_tank(tank.release()); + const std::shared_ptr shared_tank(tank.release()); const auto proprio = shared_tank->get_proprioception(); // after the chassis data (12 floats), each sub-item contributes 15 floats: @@ -93,7 +93,7 @@ TEST_F(ProprioceptionTest, ProprioceptionConsistentSize) { engine->step(1.f / 60.f); - std::shared_ptr shared_tank(tank.release()); + const std::shared_ptr shared_tank(tank.release()); const auto proprio_1 = shared_tank->get_proprioception(); @@ -110,7 +110,7 @@ TEST_F(ProprioceptionTest, ProprioceptionForwardAndUpVectorsValid) { engine->step(1.f / 60.f); - std::shared_ptr shared_tank(tank.release()); + const std::shared_ptr shared_tank(tank.release()); const auto proprio = shared_tank->get_proprioception(); // elements [3..5] are chassis forward direction, [6..8] are chassis up direction diff --git a/arenai_model/tests/src/tests_reward/tests_reward.cpp b/arenai_model/tests/src/tests_reward/tests_reward.cpp index 1a2f0404..e3f5430d 100644 --- a/arenai_model/tests/src/tests_reward/tests_reward.cpp +++ b/arenai_model/tests/src/tests_reward/tests_reward.cpp @@ -7,7 +7,6 @@ #include #include -#include #include using namespace arenai; @@ -26,7 +25,7 @@ TEST_F(RewardTest, RewardZeroWhenAliveNoShot) { engine->step(1.f / 60.f); - const std::vector> tanks{ + const std::vector tanks{ std::shared_ptr(tank_a.release()), std::shared_ptr(tank_b.release())}; const float reward_a = tanks[0]->get_reward(tanks); @@ -45,7 +44,7 @@ TEST_F(RewardTest, RewardNegativeWhenDead) { engine->step(1.f / 60.f); - std::vector> tanks{ + const std::vector tanks{ std::shared_ptr(tank_a.release()), std::shared_ptr(tank_b.release())}; // damage chassis enough to kill it @@ -70,7 +69,7 @@ TEST_F(RewardTest, DeathPenaltyIsMinusOne) { engine->step(1.f / 60.f); - const std::vector> tanks{ + const std::vector tanks{ std::shared_ptr(tank_a.release()), std::shared_ptr(tank_b.release())}; // kill by damage → normal death penalty @@ -105,12 +104,15 @@ TEST_F(RewardTest, RewardPositiveOnHit) { const std::shared_ptr shared_b(tank_b.release()); // fire from tank_a toward tank_b (canon points +Z by default) - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: shared_a->get_controllers()) ctrl->apply_input(fire_input); for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); - const std::vector> tanks{shared_a, shared_b}; + const std::vector tanks{shared_a, shared_b}; const float reward = shared_a->get_reward(tanks); @@ -136,12 +138,15 @@ TEST_F(RewardTest, RewardUnderOneAfterHit) { const std::shared_ptr shared_b(tank_b.release()); // fire from tank_a toward tank_b (canon points +Z by default) - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: shared_a->get_controllers()) ctrl->apply_input(fire_input); for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); - const std::vector> tanks{shared_a, shared_b}; + const std::vector tanks{shared_a, shared_b}; const float reward_on_hit = shared_a->get_reward(tanks); @@ -154,7 +159,10 @@ TEST_F(RewardTest, RewardUnderOneAfterHit) { << "reward should be greater than or equal to 1.0 after hitting an enemy"; // no fire, reward under 1.0 - constexpr user_input no_fire_input{{0.f, 0.f}, {0.f, 0.f}, {false}}; + constexpr user_input no_fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {false}}; for (const auto &ctrl: shared_a->get_controllers()) ctrl->apply_input(no_fire_input); for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); @@ -169,6 +177,56 @@ TEST_F(RewardTest, RewardUnderOneAfterHit) { ASSERT_LE(reward_on_no_hit, 1.f) << "reward should be under 1.0 after no hitting an enemy"; } +// ======================================================================== +// get_reward — wrecks are not farmable +// ======================================================================== + +TEST_F(RewardTest, NoRewardWhenShootingAWreck) { + add_ground(); + // spawn tanks high enough so all parts start above ground and settle cleanly + auto tank_a = tank_factory->make_enemy_tank(file_reader, "tank_a", {0.f, 5.f, 0.f}); + auto tank_b = tank_factory->make_enemy_tank(file_reader, "tank_b", {0.f, 5.f, 30.f}); + + // settle on ground (300 frames = 5s at 60fps) + for (int i = 0; i < 300; i++) engine->step(1.f / 60.f); + + const std::shared_ptr shared_a(tank_a.release()); + const std::shared_ptr shared_b(tank_b.release()); + + // kill tank_b by destroying a single part, then close its death as the environment + // does: the 8 surviving parts must stop paying anything + for (const auto items_b = shared_b->get_items(); const auto &item: items_b) { + if (auto *life = dynamic_cast(item.get())) { + life->receive_damages(1e6f); + break; + } + } + ASSERT_TRUE(shared_b->is_dead()); + shared_b->on_death(); + + const auto shells_ratio_before = shared_a->get_proprioception().back(); + + // fire from tank_a toward the wreck (canon points +Z by default) + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; + for (const auto &ctrl: shared_a->get_controllers()) ctrl->apply_input(fire_input); + + for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); + + const std::vector tanks{shared_a, shared_b}; + + const float reward = shared_a->get_reward(tanks); + + ASSERT_FALSE(shared_a->has_hit_other_tank()) << "a wreck must not count as a hit"; + ASSERT_FLOAT_EQ(reward, 0.f) << "shooting a wreck must pay neither hit nor kill"; + + // the shell spent must not be given back: a wreck is not an ammo dump + ASSERT_LT(shared_a->get_proprioception().back(), shells_ratio_before) + << "a wreck must not recharge the shell reserve"; +} + // ======================================================================== // get_reward — NaN / Inf stability // ======================================================================== @@ -181,7 +239,10 @@ TEST_F(RewardTest, ZeroRewardWithEmptyTankList) { const std::shared_ptr shared_tank(tank.release()); - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: shared_tank->get_controllers()) ctrl->apply_input(fire_input); for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); @@ -193,6 +254,6 @@ TEST_F(RewardTest, ZeroRewardWithEmptyTankList) { ASSERT_FALSE(std::isnan(reward)) << "reward should not be NaN with empty tank list"; ASSERT_FALSE(std::isinf(reward)) << "reward should not be Inf with empty tank list"; - // only the fire cost remains: no enemy to sample, so no gaussian income - ASSERT_FLOAT_EQ(reward, -0.05f); + // firing is free and there is no enemy to sample, so no reward at all + ASSERT_FLOAT_EQ(reward, 0.f); } diff --git a/arenai_model/tests/src/tests_shell/tests_shell.cpp b/arenai_model/tests/src/tests_shell/tests_shell.cpp index e705363f..a02efcc3 100644 --- a/arenai_model/tests/src/tests_shell/tests_shell.cpp +++ b/arenai_model/tests/src/tests_shell/tests_shell.cpp @@ -25,7 +25,10 @@ TEST_F(ShellTest, FireCreatesShellItem) { const int count_before = static_cast(engine->get_items().size()); - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: tank->get_controllers()) ctrl->apply_input(fire_input); engine->step(1.f / 60.f); @@ -43,7 +46,10 @@ TEST_F(ShellTest, ShellDestroyedAfterLifetime) { const int count_before_fire = static_cast(engine->get_items().size()); - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: tank->get_controllers()) ctrl->apply_input(fire_input); engine->step(1.f / 60.f); @@ -70,7 +76,10 @@ TEST_F(ShellTest, ShellHitsEnemyTank) { const std::shared_ptr shared_a(tank_a.release()); std::shared_ptr shared_b(tank_b.release()); - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: shared_a->get_controllers()) ctrl->apply_input(fire_input); for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); @@ -90,7 +99,10 @@ TEST_F(ShellTest, ShellDestroyedOnContact) { const std::shared_ptr shared_a(tank_a.release()); std::shared_ptr shared_b(tank_b.release()); - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: shared_a->get_controllers()) ctrl->apply_input(fire_input); engine->step(1.f / 60.f); @@ -113,7 +125,10 @@ TEST_F(ShellTest, NoFireNoNewItems) { const int count_before = static_cast(engine->get_items().size()); - constexpr user_input no_fire{{0.f, 1.f}, {0.f, 0.f}, {false}}; + constexpr user_input no_fire{ + .left_joystick = {.x = 0.f, .y = 1.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {false}}; for (const auto &ctrl: tank->get_controllers()) ctrl->apply_input(no_fire); engine->step(1.f / 60.f); @@ -133,15 +148,138 @@ TEST_F(ShellTest, ShellContactCallbackSetsReward) { const std::shared_ptr shared_a(tank_a.release()); const std::shared_ptr shared_b(tank_b.release()); - constexpr user_input fire_input{{0.f, 0.f}, {0.f, 0.f}, {true}}; + constexpr user_input fire_input{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; for (const auto &ctrl: shared_a->get_controllers()) ctrl->apply_input(fire_input); for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); - const std::vector> tanks{shared_a, shared_b}; + const std::vector tanks{shared_a, shared_b}; ASSERT_TRUE(shared_a->has_hit_other_tank()) << "shell must hit for reward test"; const float reward = shared_a->get_reward(tanks); ASSERT_GT(reward, 0.f) << "reward should be positive after shell contact callback"; } + +// ======================================================================== +// ShellItem — damages dealt per impact +// ======================================================================== + +namespace { + + constexpr user_input FIRE_INPUT{ + .left_joystick = {.x = 0.f, .y = 0.f}, + .right_joystick = {.x = 0.f, .y = 0.f}, + .fire_button = {true}}; + + void fire_once(const std::shared_ptr &tank) { + for (const auto &ctrl: tank->get_controllers()) ctrl->apply_input(FIRE_INPUT); + } + + // a tank spreads its health over its parts: its damages are the sum over them + int consume_tank_hits(const std::shared_ptr &tank) { + int hits = 0; + for (const auto &item: tank->get_items()) + if (const auto life_item = dynamic_cast(item.get()); life_item) + hits += life_item->consume_hits_received(); + return hits; + } + + // ShellItem lives behind arenai_model's private Jolt headers: match it by name + std::shared_ptr find_shell(AbstractPhysicEngine &engine) { + for (const auto &item: engine.get_items()) + if (item->get_name() == "shell_item") return item; + return nullptr; + } + +}// namespace + +TEST_F(ShellTest, ShellImpactDealsExactlyOneDamage) { + add_ground(); + auto tank_a = tank_factory->make_enemy_tank(file_reader, "tank_a", {0.f, 5.f, 0.f}); + auto tank_b = tank_factory->make_enemy_tank(file_reader, "tank_b", {0.f, 5.f, 30.f}); + + for (int i = 0; i < 300; i++) engine->step(1.f / 60.f); + + const std::shared_ptr shared_a(tank_a.release()); + const std::shared_ptr shared_b(tank_b.release()); + + consume_tank_hits(shared_b);// drop whatever the settling produced + + fire_once(shared_a); + for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); + + ASSERT_TRUE(shared_a->has_hit_other_tank()) << "shell must hit for this test to mean anything"; + + ASSERT_EQ(consume_tank_hits(shared_b), 1) + << "one shell removes exactly one health point, whatever the number of contact " + "points the physics engine reports for the impact"; +} + +TEST_F(ShellTest, TankSurvivesUntilAPartsHealthPointsAreSpent) { + add_ground(); + auto tank_a = tank_factory->make_enemy_tank(file_reader, "tank_a", {0.f, 5.f, 0.f}); + auto tank_b = tank_factory->make_enemy_tank(file_reader, "tank_b", {0.f, 5.f, 30.f}); + + for (int i = 0; i < 300; i++) engine->step(1.f / 60.f); + + const std::shared_ptr shared_a(tank_a.release()); + const std::shared_ptr shared_b(tank_b.release()); + + consume_tank_hits(shared_b);// drop whatever the settling produced + + // the cheapest part (wheel, turret, canon) is worth 5 health points, and every + // shell removes at most one: no tank dies in fewer than 5 shells, whatever the + // parts they land on + constexpr int cheapest_part_health_points = 5; + constexpr int max_shots = 20; + + int hits = 0; + int shots = 0; + while (!shared_b->is_dead() && shots < max_shots) { + fire_once(shared_a); + for (int i = 0; i < 60; i++) engine->step(1.f / 60.f); + + hits += consume_tank_hits(shared_b); + shots++; + } + + ASSERT_TRUE(shared_b->is_dead()) << "tank should have died within " << max_shots << " shots"; + ASSERT_GE(shots, cheapest_part_health_points) + << "tank died in " << shots << " shells for " << hits + << " health points: a single impact removed more than one"; +} + +TEST_F(ShellTest, SpentShellDealsNoFurtherDamage) { + add_ground(); + auto tank_a = tank_factory->make_enemy_tank(file_reader, "tank_a", {0.f, 5.f, 0.f}); + auto tank_b = tank_factory->make_enemy_tank(file_reader, "tank_b", {0.f, 5.f, 30.f}); + + for (int i = 0; i < 300; i++) engine->step(1.f / 60.f); + + const std::shared_ptr shared_a(tank_a.release()); + const std::shared_ptr shared_b(tank_b.release()); + + fire_once(shared_a); + engine->step(1.f / 60.f); + + const auto shell = find_shell(*engine); + ASSERT_NE(shell, nullptr) << "shell must exist after fire"; + + const auto target = shared_b->get_chassis(); + const auto target_life = dynamic_cast(target.get()); + ASSERT_NE(target_life, nullptr) << "the chassis is a life item"; + + target_life->consume_hits_received(); + + // a shell colliding with several bodies in the same step is dispatched once per + // body: only the first contact may be paid for + shell->on_contact(target.get()); + shell->on_contact(target.get()); + + ASSERT_EQ(target_life->consume_hits_received(), 1) + << "a spent shell must not damage anything again"; +} diff --git a/resources/trained_models/ppo-thin-256x128_save_6/actor.pt b/resources/trained_models/ppo-thin-256x128_save_6/actor.pt deleted file mode 100644 index 5626d57c..00000000 Binary files a/resources/trained_models/ppo-thin-256x128_save_6/actor.pt and /dev/null differ diff --git a/resources/trained_models/ppo_save_41/actor.pt b/resources/trained_models/ppo_save_41/actor.pt new file mode 100644 index 00000000..a7d9e3e9 Binary files /dev/null and b/resources/trained_models/ppo_save_41/actor.pt differ