Fix training

.clang-tidy

@@ -0,0 +1,7 @@
+---
+# Checks désactivés pour tout le projet (clang-tidy n'est utilisé que par l'IDE :
+# ni le build CMake ni la CI ne le lancent).
+#
+# modernize-use-nodiscard : réclame [[nodiscard]] sur toute méthode const
+# renvoyant une valeur — trop bruyant pour les nombreux getters du projet.
+Checks: '-modernize-use-nodiscard'

README.md

@@ -4,9 +4,9 @@ A battle-royal game with trained agent which controls tanks in realistic physic
 
 ## Description
 
-Each agent receives the rendered frame of its camera as input, and it is trained to fire and hit enemies.
+Each tank receives the rendered frame of its camera as input, and the agent is trained to fire and hit enemies.
 
-When agent is trained (with SAC algorithm) you can fight against other tanks.
+When the agent is trained (with the PPO or SAC algorithm), you can fight it through the tanks it handles.
 
 ## Installation
 

arenai_agent/src/agents/agent_cli.cpp

@@ -47,7 +47,4 @@ namespace arenai::agent {
         return algorithms;
     }
 
-    AgentCli get_default_agent_cli() {
-        return make_agent_cli<SacHyperParams, SacTorchAgentFactory>("sac", sac_cli_fields());
-    }
 }// namespace arenai::agent

arenai_agent/src/agents/agent_cli.h

@@ -29,10 +29,9 @@ namespace arenai::agent {
             create_factory;
     };
 
+    // the first entry is the algorithm used when no subcommand is given
     std::vector<AgentCli> make_agent_clis();
 
-    AgentCli get_default_agent_cli();
-
 }// namespace arenai::agent
 
 #endif//ARENAI_ALGORITHMS_H

arenai_agent/src/agents/factory_set.cpp

@@ -31,7 +31,8 @@ namespace arenai::agent {
                     .channels,
                 get_value<group_norm_nums>(
                     "group_norm_nums", parse_cli_group_norms, {{{1, 2, 3, 4, 6, 8}}})
-                    .groups),
+                    .groups,
+                0.f, 0.f),
             get_value<bool>("cuda", false) ? torch::kCUDA : torch::kCPU);
     }
 

arenai_agent/src/agents/ppo/ppo_agent.cpp

@@ -29,11 +29,11 @@ namespace arenai::agent {
     std::vector<core::Action> TorchPpoAgent::act(
         const std::vector<core::State> &states, const int vision_height, const int vision_width) {
         const auto [continuous_action, discrete_action] =
-            act(states_to_tensor(states, vision_height, vision_width));
+            act(states_to_tensor(states, vision_height, vision_width), false);
         return tensor_to_actions(continuous_action, discrete_action);
     }
 
-    TorchAction TorchPpoAgent::act(const TorchState &state) {
+    TorchAction TorchPpoAgent::act(const TorchState &state, const bool sample) {
         TorchAction action;
         torch::Tensor continuous_log_prob;
         torch::Tensor discrete_log_prob;
@@ -44,8 +44,9 @@ namespace arenai::agent {
             const auto &[vision, sensors] = state;
             const auto &[mu, sigma, discrete_proba] = actor->act(vision, sensors);
 
-            action.continuous_action = truncated_normal_sample(mu, sigma);
-            action.discrete_action = multinomial_sample(discrete_proba);
+            action.continuous_action = sample ? truncated_normal_sample(mu, sigma) : mu;
+            action.discrete_action = sample ? multinomial_sample(discrete_proba)
+                                            : multinomial_max_action(discrete_proba);
 
             // old log-probabilities, kept for the PPO importance ratio
             continuous_log_prob =

arenai_agent/src/agents/ppo/ppo_agent.h

@@ -19,7 +19,7 @@ namespace arenai::agent {
             const std::shared_ptr<Actor> &actor, torch::Device device,
             std::optional<std::shared_ptr<PpoStepCollector>> collector = std::nullopt);
 
-        TorchAction act(const TorchState &state) override;
+        TorchAction act(const TorchState &state, bool sample) override;
 
         std::vector<core::Action>
         act(const std::vector<core::State> &states, int vision_height, int vision_width) override;

arenai_agent/src/agents/ppo/ppo_collector.cpp

@@ -21,16 +21,14 @@ namespace arenai::agent {
         last_discrete_log_prob = discrete_log_prob;
     }
 
-    void PpoStepCollector::on_transition(
-        const torch::Tensor &rewards, const torch::Tensor &done, const torch::Tensor &truncated) {
+    void PpoStepCollector::on_transition(const torch::Tensor &rewards, const torch::Tensor &done) {
         rollout_buffer->add(
             {.state = last_state,
              .action = last_action,
              .continuous_log_prob = last_continuous_log_prob,
              .discrete_log_prob = last_discrete_log_prob,
              .reward = rewards,
-             .done = done,
-             .truncated = truncated});
+             .done = done});
     }
 
     void PpoStepCollector::on_episode_end(const TorchState &final_state) {

arenai_agent/src/agents/ppo/ppo_collector.h

@@ -22,9 +22,7 @@ namespace arenai::agent {
             const TorchState &state, const TorchAction &action,
             const torch::Tensor &continuous_log_prob, const torch::Tensor &discrete_log_prob);
 
-        void on_transition(
-            const torch::Tensor &rewards, const torch::Tensor &done,
-            const torch::Tensor &truncated) override;
+        void on_transition(const torch::Tensor &rewards, const torch::Tensor &done) override;
 
         void on_episode_end(const TorchState &final_state) override;
 

arenai_agent/src/agents/ppo/ppo_factory.cpp

@@ -16,17 +16,17 @@ namespace arenai::agent {
         : actor(std::make_shared<Actor>(
             vision_height, vision_width, nb_sensors, nb_continuous_actions, nb_discrete_actions,
             params.hidden_size_sensors, params.actor_hidden_sizes, params.vision_channels,
-            params.group_norm_nums)),
+            params.group_norm_nums, params.initial_sigma, params.initial_fire_proba)),
           rollout_buffer(std::make_shared<PpoRolloutBuffer>()),
           collector(std::make_shared<PpoStepCollector>(rollout_buffer)),
           agent(std::make_shared<TorchPpoAgent>(actor, device, collector)),
           trainer(std::make_shared<PpoTrainer>(
-              actor, rollout_buffer, vision_height, vision_width, nb_sensors,
-              params.actor_learning_rate, params.critic_learning_rate, params.hidden_size_sensors,
-              params.critic_hidden_sizes, params.vision_channels, params.group_norm_nums, device,
-              params.metric_window_size, params.gamma, params.gae_lambda, params.clip_epsilon,
-              params.target_kl, params.grad_norm_max, params.continuous_entropy_coef,
-              params.discrete_entropy_coef, params.epochs, params.rollout_size,
+              actor, rollout_buffer, vision_height, vision_width, nb_sensors, nb_continuous_actions,
+              params.actor_learning_rate, params.critic_learning_rate, params.alpha_learning_rate,
+              params.hidden_size_sensors, params.critic_hidden_sizes, params.vision_channels,
+              params.group_norm_nums, device, params.metric_window_size, params.gamma,
+              params.gae_lambda, params.clip_epsilon, params.target_kl, params.grad_norm_max,
+              params.target_sigma, params.target_fire_proba, params.epochs, params.rollout_size,
               params.minibatch_size)) {}
 
     std::shared_ptr<AbstractTorchAgent> PpoTorchAgentFactory::get_agent() { return agent; }

arenai_agent/src/agents/ppo/ppo_hyperparams.cpp

@@ -8,24 +8,27 @@ namespace arenai::agent {
 
     std::vector<CliField<PpoHyperParams>> ppo_cli_fields() {
         return {
-            {"--actor_learning_rate", &PpoHyperParams::actor_learning_rate},
-            {"--critic_learning_rate", &PpoHyperParams::critic_learning_rate},
-            {"--hidden_size_sensors", &PpoHyperParams::hidden_size_sensors},
-            {"--actor_hidden_sizes", &PpoHyperParams::actor_hidden_sizes},
-            {"--critic_hidden_sizes", &PpoHyperParams::critic_hidden_sizes},
-            {"--vision_channels", &PpoHyperParams::vision_channels},
-            {"--group_norm_nums", &PpoHyperParams::group_norm_nums},
-            {"--metric_window_size", &PpoHyperParams::metric_window_size},
-            {"--gamma", &PpoHyperParams::gamma},
-            {"--gae_lambda", &PpoHyperParams::gae_lambda},
-            {"--clip_epsilon", &PpoHyperParams::clip_epsilon},
-            {"--target_kl", &PpoHyperParams::target_kl},
-            {"--grad_norm_max", &PpoHyperParams::grad_norm_max},
-            {"--continuous_entropy_coef", &PpoHyperParams::continuous_entropy_coef},
-            {"--discrete_entropy_coef", &PpoHyperParams::discrete_entropy_coef},
-            {"--epochs", &PpoHyperParams::epochs},
-            {"--rollout_size", &PpoHyperParams::rollout_size},
-            {"--minibatch_size", &PpoHyperParams::minibatch_size},
+            {.name = "--actor_learning_rate", .member = &PpoHyperParams::actor_learning_rate},
+            {.name = "--critic_learning_rate", .member = &PpoHyperParams::critic_learning_rate},
+            {.name = "--alpha_learning_rate", .member = &PpoHyperParams::alpha_learning_rate},
+            {.name = "--hidden_size_sensors", .member = &PpoHyperParams::hidden_size_sensors},
+            {.name = "--actor_hidden_sizes", .member = &PpoHyperParams::actor_hidden_sizes},
+            {.name = "--critic_hidden_sizes", .member = &PpoHyperParams::critic_hidden_sizes},
+            {.name = "--vision_channels", .member = &PpoHyperParams::vision_channels},
+            {.name = "--group_norm_nums", .member = &PpoHyperParams::group_norm_nums},
+            {.name = "--initial_sigma", .member = &PpoHyperParams::initial_sigma},
+            {.name = "--initial_fire_proba", .member = &PpoHyperParams::initial_fire_proba},
+            {.name = "--metric_window_size", .member = &PpoHyperParams::metric_window_size},
+            {.name = "--gamma", .member = &PpoHyperParams::gamma},
+            {.name = "--gae_lambda", .member = &PpoHyperParams::gae_lambda},
+            {.name = "--clip_epsilon", .member = &PpoHyperParams::clip_epsilon},
+            {.name = "--target_kl", .member = &PpoHyperParams::target_kl},
+            {.name = "--grad_norm_max", .member = &PpoHyperParams::grad_norm_max},
+            {.name = "--target_sigma", .member = &PpoHyperParams::target_sigma},
+            {.name = "--target_fire_proba", .member = &PpoHyperParams::target_fire_proba},
+            {.name = "--epochs", .member = &PpoHyperParams::epochs},
+            {.name = "--rollout_size", .member = &PpoHyperParams::rollout_size},
+            {.name = "--minibatch_size", .member = &PpoHyperParams::minibatch_size},
         };
     }
 

arenai_agent/src/agents/ppo/ppo_hyperparams.h

@@ -16,25 +16,24 @@ namespace arenai::agent {
     struct PpoHyperParams {
         float actor_learning_rate = 1e-4f;
         float critic_learning_rate = 3e-4f;
+        float alpha_learning_rate = 1e-3f;
         int hidden_size_sensors = 128;
         std::vector<int> actor_hidden_sizes = {1024, 512};
         std::vector<int> critic_hidden_sizes = {1024, 512};
         std::vector<std::tuple<int, int>> vision_channels = {{3, 8},   {8, 16},  {16, 24},
                                                              {24, 32}, {32, 48}, {48, 64}};
         std::vector<int> group_norm_nums = {1, 2, 3, 4, 6, 8};
+        float initial_sigma = 0.4f;
+        float initial_fire_proba = 0.025f;
         int metric_window_size = 256;
-        // 0.997 at 30 Hz -> ~11 s credit horizon (shell flight time + fights stay visible)
-        float gamma = 0.997f;
-        // 0.98: a shell resolving 30-60 steps after the fire still reaches the fire
-        // decision at x0.25-0.5 through GAE, instead of x0.04-0.2 with 0.95
+        float gamma = 0.99f;
         float gae_lambda = 0.98f;
         float clip_epsilon = 0.2f;
-        // early-stop of the epoch loop when approx KL > 1.5 * target_kl; <= 0 disables it
-        float target_kl = 0.02f;
+        float target_kl = 0.05f;
         float grad_norm_max = 0.5f;
-        float continuous_entropy_coef = 0.0005f;
-        float discrete_entropy_coef = 0.005f;
-        int epochs = 4;
+        float target_sigma = 0.4f;
+        float target_fire_proba = 0.025f;
+        int epochs = 2;
         int rollout_size = 30 * 30;
         int minibatch_size = 1024;
     };

arenai_agent/src/agents/ppo/ppo_rollout_buffer.cpp

@@ -27,11 +27,7 @@ namespace arenai::agent {
         // tanks already terminated before this step have no valid transition to store
         const auto valid = already_terminated_.logical_not();
         already_terminated_.logical_or_(
-            step.done.detach()
-                .cpu()
-                .to(torch::kBool)
-                .reshape({nb_tanks})
-                .logical_or(step.truncated.detach().cpu().to(torch::kBool).reshape({nb_tanks})));
+            step.done.detach().cpu().to(torch::kBool).reshape({nb_tanks}));
 
         steps_.push_back(
             {.step =
@@ -42,8 +38,7 @@ namespace arenai::agent {
                   .continuous_log_prob = step.continuous_log_prob.detach().cpu(),
                   .discrete_log_prob = step.discrete_log_prob.detach().cpu(),
                   .reward = step.reward.detach().cpu(),
-                  .done = step.done.detach().cpu(),
-                  .truncated = step.truncated.detach().cpu()},
+                  .done = step.done.detach().cpu()},
              .valid = valid});
 
         // the freshly added step is pending: its closing observation is not known yet
@@ -93,7 +88,6 @@ namespace arenai::agent {
                 stack([](const StoredStep &s) { return s.step.discrete_log_prob; }),
             .rewards = stack([](const StoredStep &s) { return s.step.reward; }),
             .dones = stack([](const StoredStep &s) { return s.step.done; }),
-            .truncateds = stack([](const StoredStep &s) { return s.step.truncated; }),
             .bootstrap_state = bootstrap_state,
             .valids = stack([](const StoredStep &s) { return s.valid; }).unsqueeze(-1)};
 

arenai_agent/src/agents/ppo/ppo_rollout_buffer.h

@@ -21,7 +21,6 @@ namespace arenai::agent {
         torch::Tensor discrete_log_prob;
         torch::Tensor reward;
         torch::Tensor done;
-        torch::Tensor truncated;
     };
 
     // On-policy rollout stacked on the time dimension: every tensor is [T, nb_tanks, ...]
@@ -32,7 +31,6 @@ namespace arenai::agent {
         torch::Tensor discrete_log_probs;
         torch::Tensor rewards;
         torch::Tensor dones;
-        torch::Tensor truncateds;
         // [nb_tanks, ...] observation closing the last step, for the value bootstrap
         TorchState bootstrap_state;
         // [T, nb_tanks, 1] whether the (step, tank) pair is a live transition
@@ -62,7 +60,7 @@ namespace arenai::agent {
         // observation closing the last stored step, set by finish_episode()
         std::optional<TorchState> final_state_;
 
-        // [nb_tanks] tanks already done/truncated in the current episode
+        // [nb_tanks] tanks already done in the current episode
         torch::Tensor already_terminated_;
     };