diff --git a/score/launch_manager/docs/user_guide/configuration.rst b/score/launch_manager/docs/user_guide/configuration.rst index b6499c423..623738ec2 100644 --- a/score/launch_manager/docs/user_guide/configuration.rst +++ b/score/launch_manager/docs/user_guide/configuration.rst @@ -357,6 +357,10 @@ run_targets (object, optional) * **Description:** Defines an individual **Run Target's** configuration, specifying the components and dependencies that constitute a particular operational mode. * **Reference:** This property refers to the ``run_target`` reusable type defined in this schema. +The name "Off" is reserved for a special **Run Target** that represents the system's shutdown state. +If a **Run Target** with this name is configured explicitly, the **Launch Manager** will transition to this **Run Target** when receiving a SIGTERM signal. +If not configured explicitly, the **Launch Manager** will automatically create a default **Run Target** named "Off" with no dependencies. + .. _lm_conf_initial_run_target_string_required_: initial_run_target (string, required) diff --git a/score/launch_manager/src/daemon/src/alive_monitor/details/ifexm/ProcessState.hpp b/score/launch_manager/src/daemon/src/alive_monitor/details/ifexm/ProcessState.hpp index 861735f7b..1cd0d2a73 100644 --- a/score/launch_manager/src/daemon/src/alive_monitor/details/ifexm/ProcessState.hpp +++ b/score/launch_manager/src/daemon/src/alive_monitor/details/ifexm/ProcessState.hpp @@ -77,7 +77,8 @@ class ProcessState : public saf::common::Observable starting = static_cast(score::lcm::ProcessState::kStarting), running = static_cast(score::lcm::ProcessState::kRunning), sigterm = static_cast(score::lcm::ProcessState::kTerminating), - off = static_cast(score::lcm::ProcessState::kTerminated)}; + off = static_cast(score::lcm::ProcessState::kTerminated), + failed = static_cast(score::lcm::ProcessState::kFailed)}; /// @brief Get Process State /// @return Returns Process State diff --git a/score/launch_manager/src/daemon/src/alive_monitor/details/ifexm/ProcessStateReader.cpp b/score/launch_manager/src/daemon/src/alive_monitor/details/ifexm/ProcessStateReader.cpp index 064c03dda..41ec4abd0 100644 --- a/score/launch_manager/src/daemon/src/alive_monitor/details/ifexm/ProcessStateReader.cpp +++ b/score/launch_manager/src/daemon/src/alive_monitor/details/ifexm/ProcessStateReader.cpp @@ -149,6 +149,9 @@ constexpr ProcessState::EProcState ProcessStateReader::translateProcessState( static_assert(static_cast(ProcessState::EProcState::off) == static_cast(score::lcm::ProcessState::kTerminated), "Lcm State Enum and ProcessState::EProcState Enum do not match."); + static_assert(static_cast(ProcessState::EProcState::failed) == + static_cast(score::lcm::ProcessState::kFailed), + "Lcm State Enum and ProcessState::EProcState Enum do not match."); return static_cast(f_processStateLcm); } diff --git a/score/launch_manager/src/daemon/src/alive_monitor/details/supervision/Alive_UT.cpp b/score/launch_manager/src/daemon/src/alive_monitor/details/supervision/Alive_UT.cpp index 902822638..14a001a5d 100644 --- a/score/launch_manager/src/daemon/src/alive_monitor/details/supervision/Alive_UT.cpp +++ b/score/launch_manager/src/daemon/src/alive_monitor/details/supervision/Alive_UT.cpp @@ -17,13 +17,13 @@ #include #include -#include "score/mw/launch_manager/common/identifier_hash.hpp" -#include "score/mw/launch_manager/recovery_client/irecovery_client.h" #include "score/mw/launch_manager/alive_monitor/details/ifappl/Checkpoint.hpp" #include "score/mw/launch_manager/alive_monitor/details/ifexm/ProcessCfg.hpp" #include "score/mw/launch_manager/alive_monitor/details/ifexm/ProcessState.hpp" #include "score/mw/launch_manager/alive_monitor/details/supervision/Alive.hpp" #include "score/mw/launch_manager/alive_monitor/details/supervision/SupervisionCfg.hpp" +#include "score/mw/launch_manager/common/identifier_hash.hpp" +#include "score/mw/launch_manager/recovery_client/irecovery_client.h" using namespace testing; @@ -36,12 +36,16 @@ namespace class MockRecoveryClient : public score::lcm::IRecoveryClient { public: - MOCK_METHOD(bool, - sendRecoveryRequest, - (const score::lcm::IdentifierHash& process_group_identifier), - (noexcept, override)); - MOCK_METHOD(std::optional, getNextRequest, (), (noexcept, override)); - MOCK_METHOD(bool, hasOverflow, (), (const, noexcept, override)); + MOCK_METHOD( + void, + setRecoveryRequestCallback, + (score::lcm::IRecoveryClient::RecoveryRequestCallback callback), + (noexcept, override)); + MOCK_METHOD( + bool, + sendRecoveryRequest, + (const score::lcm::IdentifierHash& process_group_identifier), + (noexcept, override)); }; /// Helper: build a minimal Alive under test. @@ -168,10 +172,11 @@ class AliveSupervisionTest : public ::testing::Test TEST_F(AliveSupervisionTest, AliveTransitionsOkToExpiredOnMissingHeartbeat) { - RecordProperty("Description", - "Verify that Alive transitions from deactivated -> ok -> expired when no heartbeats are reported " - "and failedCyclesTolerance == 0. sendRecoveryRequest must be called exactly once with the " - "configured recovery target hash."); + RecordProperty( + "Description", + "Verify that Alive transitions from deactivated -> ok -> expired when no heartbeats are reported " + "and failedCyclesTolerance == 0. sendRecoveryRequest must be called exactly once with the " + "configured recovery target hash."); AliveFixture fix = AliveFixture::Builder{}.build(); EXPECT_CALL(*fix.mockClient, sendRecoveryRequest(fix.kProcessIdentifier)).Times(1).WillOnce(Return(true)); @@ -189,8 +194,8 @@ TEST_F(AliveSupervisionTest, AliveTransitionsOkToExpiredOnMissingHeartbeat) TEST_F(AliveSupervisionTest, AliveStaysOkWithCorrectHeartbeats) { - RecordProperty("Description", - "Verify that sending at least minIndications heartbeats per cycle keeps Alive in ok."); + RecordProperty( + "Description", "Verify that sending at least minIndications heartbeats per cycle keeps Alive in ok."); AliveFixture fix = AliveFixture::Builder{}.build(); EXPECT_CALL(*fix.mockClient, sendRecoveryRequest(_)).Times(0); @@ -232,9 +237,10 @@ TEST_F(AliveSupervisionTest, AliveReportsEnqueueFailureWhenRingBufferFull) TEST_F(AliveSupervisionTest, AliveDebouncesThroughFailedBeforeExpired) { - RecordProperty("Description", - "Verify that failedCyclesTolerance debouncing works: with tolerance=1 the supervision passes " - "through failed before reaching expired."); + RecordProperty( + "Description", + "Verify that failedCyclesTolerance debouncing works: with tolerance=1 the supervision passes " + "through failed before reaching expired."); AliveFixture fix = AliveFixture::Builder{}.withFailedCyclesTolerance(1U).build(); EXPECT_CALL(*fix.mockClient, sendRecoveryRequest(fix.kProcessIdentifier)) @@ -256,8 +262,7 @@ TEST_F(AliveSupervisionTest, AliveDebouncesThroughFailedBeforeExpired) TEST_F(AliveSupervisionTest, DeactivatesOnProcessSigterm) { - RecordProperty("Description", - "Verify that a clean shutdown (sigterm) deactivates the supervision from ok."); + RecordProperty("Description", "Verify that a clean shutdown (sigterm) deactivates the supervision from ok."); AliveFixture fix = AliveFixture::Builder{}.build(); EXPECT_CALL(*fix.mockClient, sendRecoveryRequest(_)).Times(0); @@ -273,8 +278,8 @@ TEST_F(AliveSupervisionTest, DeactivatesOnProcessSigterm) TEST_F(AliveSupervisionTest, DeactivatesOnProcessCrash) { - RecordProperty("Description", - "Verify that a process crash (off without sigterm) also deactivates the supervision."); + RecordProperty( + "Description", "Verify that a process crash (off without sigterm) also deactivates the supervision."); AliveFixture fix = AliveFixture::Builder{}.build(); EXPECT_CALL(*fix.mockClient, sendRecoveryRequest(_)).Times(0); @@ -290,9 +295,10 @@ TEST_F(AliveSupervisionTest, DeactivatesOnProcessCrash) TEST_F(AliveSupervisionTest, ReactivatesAfterCrash) { - RecordProperty("Description", - "Verify that after a crash (off) the supervision can be reactivated when the process" - " reports running again, without any special recovery path."); + RecordProperty( + "Description", + "Verify that after a crash (off) the supervision can be reactivated when the process" + " reports running again, without any special recovery path."); AliveFixture fix = AliveFixture::Builder{}.build(); EXPECT_CALL(*fix.mockClient, sendRecoveryRequest(_)).Times(0); @@ -314,9 +320,10 @@ TEST_F(AliveSupervisionTest, ReactivatesAfterCrash) TEST_F(AliveSupervisionTest, IgnoresIrrelevantProcessStates) { - RecordProperty("Description", - "Verify that process states other than running/sigterm/off are ignored and do not" - " affect the supervision state."); + RecordProperty( + "Description", + "Verify that process states other than running/sigterm/off are ignored and do not" + " affect the supervision state."); AliveFixture fix = AliveFixture::Builder{}.build(); EXPECT_CALL(*fix.mockClient, sendRecoveryRequest(_)).Times(0); @@ -341,14 +348,11 @@ TEST_F(AliveSupervisionTest, IgnoresIrrelevantProcessStates) TEST_F(AliveSupervisionTest, MaxIndicationViolationExpires) { - RecordProperty("Description", - "Verify that exceeding the maximum allowed heartbeats per cycle leads to failure."); + RecordProperty("Description", "Verify that exceeding the maximum allowed heartbeats per cycle leads to failure."); // max=1, tolerance=0: more than 1 heartbeat per cycle expires immediately AliveFixture fix = AliveFixture::Builder{}.withMaxIndications(1U).build(); - EXPECT_CALL(*fix.mockClient, sendRecoveryRequest(fix.kProcessIdentifier)) - .Times(1) - .WillOnce(Return(true)); + EXPECT_CALL(*fix.mockClient, sendRecoveryRequest(fix.kProcessIdentifier)).Times(1).WillOnce(Return(true)); fix.activateProcess(10U); fix.alive->evaluate(11U); diff --git a/score/launch_manager/src/daemon/src/common/concurrency/BUILD b/score/launch_manager/src/daemon/src/common/concurrency/BUILD index efecb3822..c91cc6d5a 100644 --- a/score/launch_manager/src/daemon/src/common/concurrency/BUILD +++ b/score/launch_manager/src/daemon/src/common/concurrency/BUILD @@ -22,6 +22,7 @@ cc_library( ":mpmc_concurrent_queue", "//score/launch_manager/src/daemon/src/common:constants", "//score/launch_manager/src/daemon/src/common:log", + "//score/launch_manager/src/daemon/src/process_group_manager/details:icomponent_controller", ], ) diff --git a/score/launch_manager/src/daemon/src/common/concurrency/workerthread.hpp b/score/launch_manager/src/daemon/src/common/concurrency/workerthread.hpp index bf144c242..55249f074 100644 --- a/score/launch_manager/src/daemon/src/common/concurrency/workerthread.hpp +++ b/score/launch_manager/src/daemon/src/common/concurrency/workerthread.hpp @@ -17,6 +17,7 @@ #include "score/mw/launch_manager/common/concurrency/mpmc_concurrent_queue.hpp" #include "score/mw/launch_manager/common/constants.hpp" #include "score/mw/launch_manager/common/log.hpp" +#include "score/mw/launch_manager/process_group_manager/details/icomponent_controller.hpp" #include #include #include @@ -24,6 +25,8 @@ namespace score::lcm::internal { +using namespace score::mw::lifecycle::internal; + /// @brief Templated worker thread pool for executing jobs from a queue. /// This class manages a pool of worker threads that continuously retrieve and execute jobs /// from an MPMCConcurrentQueue until the pool is stopped or destructed. @@ -31,14 +34,16 @@ namespace score::lcm::internal template class WorkerThread final { - using Queue = MPMCConcurrentQueue, static_cast(ProcessLimits::kMaxProcesses)>; + using Queue = MPMCConcurrentQueue, static_cast(ProcessLimits::kMaxProcesses)>; public: /// @brief Constructs a WorkerThread pool with the specified number of threads. /// /// @param queue The MpmcQueue from which threads will take work items. /// @param num_threads Number of threads in the pool. - WorkerThread(std::shared_ptr queue, uint32_t num_threads) : the_job_queue_(queue) + /// @param component_controller_ The controller to delegate work to. + WorkerThread(std::shared_ptr queue, uint32_t num_threads, IComponentController& component_controller) + : the_job_queue_(queue), component_controller_(component_controller) { worker_threads_.reserve(num_threads); for (uint32_t i = 0U; i < num_threads; ++i) @@ -99,13 +104,15 @@ class WorkerThread final LM_LOG_ERROR() << "Got an error getting a job: " << job.error(); continue; } - (*job)->doWork(); + component_controller_.doWork(std::move(**job)); } } /// @brief The queue from which each thread takes work. std::shared_ptr the_job_queue_{}; + IComponentController& component_controller_; + /// @brief Vector of worker threads. std::vector> worker_threads_{}; }; diff --git a/score/launch_manager/src/daemon/src/configuration/configuration_adapter.cpp b/score/launch_manager/src/daemon/src/configuration/configuration_adapter.cpp index 4b2ecd3c3..f04f3f162 100644 --- a/score/launch_manager/src/daemon/src/configuration/configuration_adapter.cpp +++ b/score/launch_manager/src/daemon/src/configuration/configuration_adapter.cpp @@ -271,7 +271,6 @@ ProcessGroupState ConfigurationAdapter::buildProcessGroupState( std::vector ConfigurationAdapter::buildProcessGroupStates( const Config& config) const { std::vector states; - states.push_back(ProcessGroupState{IdentifierHash{"MainPG/Off"}, {}}); const auto& run_targets = config.runTargets(); DependsOnMap depends_on_by_name; @@ -287,6 +286,17 @@ std::vector ConfigurationAdapter::buildProcessGroupStates( } states.push_back(buildProcessGroupState("MainPG/fallback_run_target", fallback_cfg.depends_on, depends_on_by_name)); + + // Guarantee an Off state exists so it always has a RunTarget node. Add an empty one (no + // processes == everything stopped) unless a run target already produced it. + const IdentifierHash off_name{"MainPG/Off"}; // matches pg.off_state_ set in buildFromConfig + const bool has_off = std::any_of( + states.cbegin(), states.cend(), [&](const ProcessGroupState& s) { return s.name_ == off_name; }); + if (!has_off) + { + states.push_back(ProcessGroupState{off_name, {}}); + } + return states; } @@ -404,6 +414,19 @@ std::optional*> ConfigurationAdapter::getProcessInde return std::nullopt; } +std::optional*> ConfigurationAdapter::getListOfProcessGroupStates( + const IdentifierHash& pg_name) const +{ + std::optional*> result = std::nullopt; + + if (const auto* pg = getProcessGroupByID(pg_name)) + { + result = &pg->states_; + } + + return result; +} + std::optional ConfigurationAdapter::getOsProcessConfiguration( const IdentifierHash& pg_name, const uint32_t index) const { if (auto pg = getProcessGroupByNameAndIndex(pg_name, index)) { diff --git a/score/launch_manager/src/daemon/src/configuration/configuration_adapter.hpp b/score/launch_manager/src/daemon/src/configuration/configuration_adapter.hpp index 1fdcc563a..4a07a8dcf 100644 --- a/score/launch_manager/src/daemon/src/configuration/configuration_adapter.hpp +++ b/score/launch_manager/src/daemon/src/configuration/configuration_adapter.hpp @@ -14,59 +14,65 @@ #ifndef CONFIGURATIONADAPTER_HPP_INCLUDED #define CONFIGURATIONADAPTER_HPP_INCLUDED +#include "score/mw/launch_manager/common/constants.hpp" +#include "score/mw/launch_manager/common/identifier_hash.hpp" +#include "score/mw/launch_manager/common/process_group_state_id.hpp" +#include "score/mw/launch_manager/configuration/config.hpp" +#include "score/mw/launch_manager/process_group_manager/iprocess.hpp" +#include "score/mw/launch_manager/process_state_client/posix_process.hpp" #include #include #include #include #include #include -#include "score/mw/launch_manager/configuration/config.hpp" -#include "score/mw/launch_manager/common/identifier_hash.hpp" -#include "score/mw/launch_manager/common/constants.hpp" -#include "score/mw/launch_manager/common/process_group_state_id.hpp" -#include "score/mw/launch_manager/process_group_manager/iprocess.hpp" -#include "score/mw/launch_manager/process_state_client/posix_process.hpp" -namespace score::mw::launch_manager::configuration { +namespace score::mw::launch_manager::configuration +{ using IdentifierHash = score::lcm::IdentifierHash; -struct PgManagerConfig final { - bool is_self_terminating_; - std::chrono::milliseconds startup_timeout_ms_; - std::chrono::milliseconds termination_timeout_ms_; - uint32_t number_of_restart_attempts; - uint32_t execution_error_code_; +struct PgManagerConfig final +{ + bool is_self_terminating_{}; + std::chrono::milliseconds startup_timeout_ms_{}; + std::chrono::milliseconds termination_timeout_ms_{}; + uint32_t number_of_restart_attempts{}; + uint32_t execution_error_code_{}; }; -struct Dependency final { - score::lcm::ProcessState process_state_; - IdentifierHash target_process_id_; - uint32_t os_process_index_; +struct Dependency final +{ + score::lcm::ProcessState process_state_{}; + IdentifierHash target_process_id_{}; + uint32_t os_process_index_{}; }; using DependencyList = std::vector; -struct OsProcess final { - IdentifierHash process_id_; - uint32_t process_number_; +struct OsProcess final +{ + IdentifierHash process_id_{}; + uint32_t process_number_{}; score::lcm::internal::osal::OsalConfig startup_config_{}; - PgManagerConfig pgm_config_; - DependencyList dependencies_; + PgManagerConfig pgm_config_{}; + DependencyList dependencies_{}; }; -struct ProcessGroupState final { - IdentifierHash name_; - std::vector process_indexes_; +struct ProcessGroupState final +{ + IdentifierHash name_{}; + std::vector process_indexes_{}; }; -struct ProcessGroup final { - IdentifierHash name_; - IdentifierHash sw_cluster_; - IdentifierHash off_state_; - IdentifierHash recovery_state_; - std::vector states_; - std::vector processes_; +struct ProcessGroup final +{ + IdentifierHash name_{}; + IdentifierHash sw_cluster_{}; + IdentifierHash off_state_{}; + IdentifierHash recovery_state_{}; + std::vector states_{}; + std::vector processes_{}; }; /// @brief Temporary bridge that translates the new Config model (RunTargets, Components) into the @@ -74,7 +80,8 @@ struct ProcessGroup final { /// and related code. It exists only to decouple the config-format migration from the broader code /// migration and should be removed once ProcessGroupManager and its dependents are adapted to work /// directly with RunTarget/Component concepts. -class ConfigurationAdapter final { +class ConfigurationAdapter final +{ public: /// @brief Initialize from a pre-loaded Config object (preferred — avoids a second file read). bool initialize(const Config& config); @@ -88,22 +95,22 @@ class ConfigurationAdapter final { std::optional getMainPGStartupState() const; std::optional*> getProcessIndexesList( const score::lcm::internal::ProcessGroupStateID& process_group_state_id) const; + std::optional*> getListOfProcessGroupStates( + const IdentifierHash& pg_name) const; std::optional getOsProcessConfiguration(const IdentifierHash& pg_name_, - const uint32_t index) const; + const uint32_t index) const; std::optional getOsProcessDependencies(const IdentifierHash& process_group_name, - const uint32_t index) const; + const uint32_t index) const; private: using DependsOnMap = std::map*>; bool buildFromConfig(const Config& config); - OsProcess buildOsProcess(const ComponentConfig& comp, - uint32_t process_index) const; + OsProcess buildOsProcess(const ComponentConfig& comp, uint32_t process_index) const; void fillStartupConfigFromDeployment(const ComponentConfig& comp, score::lcm::internal::osal::OsalConfig& startup) const; - void fillStartupArguments(const ComponentProperties& props, - score::lcm::internal::osal::OsalConfig& startup) const; + void fillStartupArguments(const ComponentProperties& props, score::lcm::internal::osal::OsalConfig& startup) const; size_t fillStartupEnvironment(const DeploymentConfig& deploy, score::lcm::internal::osal::OsalConfig& startup) const; void appendAliveInterfaceEnvironment(const ComponentConfig& comp, @@ -112,8 +119,7 @@ class ConfigurationAdapter final { PgManagerConfig buildPgManagerConfig(const ComponentConfig& comp) const; DependencyList buildDependencyList(const ComponentProperties& props) const; - std::vector buildProcessGroupStates( - const Config& config) const; + std::vector buildProcessGroupStates(const Config& config) const; ProcessGroupState buildProcessGroupState(const std::string& state_name, const std::vector& depends_on, const DependsOnMap& depends_on_by_name) const; @@ -129,27 +135,28 @@ class ConfigurationAdapter final { ProcessGroup* getProcessGroupByID(const IdentifierHash& pg_name) const; ProcessGroupState* getProcessGroupStateByID(const score::lcm::internal::ProcessGroupStateID& pg_id) const; std::optional getProcessGroupByNameAndIndex(const IdentifierHash& pg_name, - const uint32_t index) const; + const uint32_t index) const; std::map component_by_name_{}; std::map component_to_process_index_{}; std::vector process_groups_{}; std::vector process_group_names_{}; score::lcm::internal::ProcessGroupStateID main_pg_startup_state_{static_cast("MainPG"), - static_cast("MainPG/Startup")}; + static_cast("MainPG/Startup")}; }; } // namespace score::mw::launch_manager::configuration // Aliases for backward compatibility with score::lcm::internal consumers -namespace score::lcm::internal { +namespace score::lcm::internal +{ using ConfigurationAdapter = score::mw::launch_manager::configuration::ConfigurationAdapter; -using OsProcess = score::mw::launch_manager::configuration::OsProcess; -using DependencyList = score::mw::launch_manager::configuration::DependencyList; -using ProcessGroup = score::mw::launch_manager::configuration::ProcessGroup; -using ProcessGroupState = score::mw::launch_manager::configuration::ProcessGroupState; -using PgManagerConfig = score::mw::launch_manager::configuration::PgManagerConfig; -using Dependency = score::mw::launch_manager::configuration::Dependency; +using OsProcess = score::mw::launch_manager::configuration::OsProcess; +using DependencyList = score::mw::launch_manager::configuration::DependencyList; +using ProcessGroup = score::mw::launch_manager::configuration::ProcessGroup; +using ProcessGroupState = score::mw::launch_manager::configuration::ProcessGroupState; +using PgManagerConfig = score::mw::launch_manager::configuration::PgManagerConfig; +using Dependency = score::mw::launch_manager::configuration::Dependency; } // namespace score::lcm::internal #endif // CONFIGURATIONADAPTER_HPP_INCLUDED diff --git a/score/launch_manager/src/daemon/src/configuration/configuration_manager.cpp b/score/launch_manager/src/daemon/src/configuration/configuration_manager.cpp index bc538918c..08c0e7f65 100644 --- a/score/launch_manager/src/daemon/src/configuration/configuration_manager.cpp +++ b/score/launch_manager/src/daemon/src/configuration/configuration_manager.cpp @@ -19,6 +19,7 @@ #include "score/mw/launch_manager/configuration/configuration_manager.hpp" #include "score/mw/launch_manager/osal/num_cores.hpp" +#include #include using namespace std; @@ -186,6 +187,19 @@ std::optional*> ConfigurationManager::getProcessInde return result; } +std::optional*> ConfigurationManager::getListOfProcessGroupStates( + const IdentifierHash& pg_name) const +{ + std::optional*> result = std::nullopt; + + if (const auto* pg = getProcessGroupByID(pg_name)) + { + result = &pg->states_; + } + + return result; +} + std::optional ConfigurationManager::getOsProcessConfiguration(const IdentifierHash& pg_name, const uint32_t index) const { @@ -390,11 +404,13 @@ bool ConfigurationManager::parseModeGroups(const ModeGroup* node, ProcessGroup& bool result = false; const auto* mode_declaration_list = node ? node->mode_declaration() : nullptr; - if (mode_declaration_list && (mode_declaration_list->size())) { + if (mode_declaration_list && (mode_declaration_list->size())) + { process_group_data.off_state_ = IdentifierHash("Off"); // default value if no other path is defined const flatbuffers::String* recovery_state_name = node->recovery_mode_name(); - if (recovery_state_name) { + if (recovery_state_name) + { process_group_data.recovery_state_ = getStringViewFromFlatBuffer(recovery_state_name); } else @@ -423,6 +439,20 @@ bool ConfigurationManager::parseModeGroups(const ModeGroup* node, ProcessGroup& } } + // Guarantee an Off state exists so it always has a RunTarget node. When no state name + // ends in "/Off", off_state_ stays the default "Off" with no matching entry above; add an + // empty one (no processes == everything stopped). + const bool has_off_state = + std::any_of(process_group_data.states_.cbegin(), + process_group_data.states_.cend(), + [&](const ProcessGroupState& s) { return s.name_ == process_group_data.off_state_; }); + if (!has_off_state) + { + ProcessGroupState off_state; + off_state.name_ = process_group_data.off_state_; // process_indexes_ left empty + process_group_data.states_.push_back(off_state); + } + // Successfully parsed machine configurations process_groups_.push_back(process_group_data); result = true; @@ -468,8 +498,10 @@ static void setSchedulingParameters(const Process& node, const ProcessStartupCon instance.startup_config_.scheduling_policy_ = ConfigurationManager::kDefaultSchedulingPolicy; instance.startup_config_.scheduling_priority_ = ConfigurationManager::kDefaultNormalSchedulingPriority; auto attribute = config.scheduling_policy(); - if (attribute != nullptr) { - if (strcasecmp("SCHED_FIFO", attribute->c_str()) == 0) { + if (attribute != nullptr) + { + if (strcasecmp("SCHED_FIFO", attribute->c_str()) == 0) + { instance.startup_config_.scheduling_policy_ = SCHED_FIFO; instance.startup_config_.scheduling_priority_ = ConfigurationManager::kDefaultRealtimeSchedulingPriority; } @@ -489,7 +521,8 @@ static void setSchedulingParameters(const Process& node, const ProcessStartupCon } } attribute = config.scheduling_priority(); - if (attribute != nullptr) { + if (attribute != nullptr) + { instance.startup_config_.scheduling_priority_ = std::stoi(attribute->c_str()); } attribute = node.coremask(); @@ -560,7 +593,8 @@ bool ConfigurationManager::parseProcessConfigurations(const Process* node) std::chrono::milliseconds(startup_config_node->exit_timeout_value()); auto execution_error_string = startup_config_node->execution_error(); - if (execution_error_string) { + if (execution_error_string) + { instance.pgm_config_.execution_error_code_ = static_cast(std::stoi(execution_error_string->c_str())); } @@ -739,10 +773,12 @@ void ConfigurationManager::parseExecutionDependency( Dependency dep{}; auto state_name = getStringViewFromFlatBuffer(process_dependency_node->state_name()); dep.process_state_ = getProcessState(state_name); - dep.target_process_id_ = getStringViewFromFlatBuffer(process_dependency_node->target_process_identifier()); + dep.target_process_id_ = + getStringViewFromFlatBuffer(process_dependency_node->target_process_identifier()); LM_LOG_DEBUG() << "ParseProcessExecutionDependency: target process path:" - << std::string_view{getStringFromFlatBuffer(process_dependency_node->target_process_identifier())} - << "ID:" << dep.target_process_id_; + << std::string_view{getStringFromFlatBuffer( + process_dependency_node->target_process_identifier())} + << "ID:" << dep.target_process_id_; process_instance.dependencies_.push_back(dep); } } @@ -965,11 +1001,6 @@ osal::CommsType ConfigurationManager::getfunctionClusterAffiliation(osal::CommsT // TODO - example introduce PHM enum. LM_LOG_DEBUG() << "Process is PLATFORM_HEALTH_MANAGEMENT function Cluster Affiliation"; } - else if (attribute && std::string_view(attribute) == "LAUNCH_MANAGEMENT") - { - comms_type = osal::CommsType::kLaunchManager; - LM_LOG_DEBUG() << "Process is LAUNCH_MANAGEMENT function Cluster Affiliation"; - } else { LM_LOG_DEBUG() << "Process is NOT associated with any function Cluster Affiliation"; diff --git a/score/launch_manager/src/daemon/src/configuration/configuration_manager.hpp b/score/launch_manager/src/daemon/src/configuration/configuration_manager.hpp index 2320b6ca0..cd7e28a87 100644 --- a/score/launch_manager/src/daemon/src/configuration/configuration_manager.hpp +++ b/score/launch_manager/src/daemon/src/configuration/configuration_manager.hpp @@ -46,13 +46,13 @@ using IdentifierHash = score::lcm::IdentifierHash; ///< Defines a type alias 'I // have user-declared constructor. The rule doesn’t apply.", false) struct PgManagerConfig final { - bool is_self_terminating_; ///< true if the adaptive application may terminate without being first asked + bool is_self_terminating_{}; ///< true if the adaptive application may terminate without being first asked std::chrono::milliseconds - startup_timeout_ms_; ///< Number of milliseconds to wait for kRunning before flagging an error - std::chrono::milliseconds termination_timeout_ms_; ///< Number of milliseconds to wait for process to terminate - ///< after requesting termination - uint32_t number_of_restart_attempts; ///< Number of times to attempt restart if the initial attempt fails - uint32_t execution_error_code_; ///< Code to report if this process fails + startup_timeout_ms_{}; ///< Number of milliseconds to wait for kRunning before flagging an error + std::chrono::milliseconds termination_timeout_ms_{}; ///< Number of milliseconds to wait for process to terminate + ///< after requesting termination + uint32_t number_of_restart_attempts{}; ///< Number of times to attempt restart if the initial attempt fails + uint32_t execution_error_code_{}; ///< Code to report if this process fails }; /// @brief Represents process dependency in a particular process group associated process. @@ -61,9 +61,9 @@ struct PgManagerConfig final struct Dependency final { score::lcm::ProcessState - process_state_; ///< The state of the other process upon which starting of this process depends. - IdentifierHash target_process_id_; ///< The ID of the target process this dependency is associated with. - uint32_t os_process_index_; ///< The index of the OS process in the target process list. + process_state_{}; ///< The state of the other process upon which starting of this process depends. + IdentifierHash target_process_id_{}; ///< The ID of the target process this dependency is associated with. + uint32_t os_process_index_{}; ///< The index of the OS process in the target process list. }; using DependencyList = std::vector; @@ -74,11 +74,11 @@ using DependencyList = std::vector; // have user-declared constructor. The rule doesn’t apply.", false) struct OsProcess final { - IdentifierHash process_id_; ///< id of a Process. - uint32_t process_number_; ///< unique number for this process & startup configuration combination + IdentifierHash process_id_{}; ///< id of a Process. + uint32_t process_number_{}; ///< unique number for this process & startup configuration combination osal::OsalConfig startup_config_{}; ///< Startup configuration. - PgManagerConfig pgm_config_; ///< Process group manager operations configuration. - DependencyList dependencies_; ///< List of dependencies for each OS process in a specific process group. + PgManagerConfig pgm_config_{}; ///< Process group manager operations configuration. + DependencyList dependencies_{}; ///< List of dependencies for each OS process in a specific process group. }; /// @brief Represents configuration of a process group state. @@ -86,9 +86,9 @@ struct OsProcess final // have user-declared constructor. The rule doesn’t apply.", false) struct ProcessGroupState final { - IdentifierHash name_; ///< Name of a process group state. + IdentifierHash name_{}; ///< Name of a process group state. std::vector - process_indexes_; ///< Processes that should be started / run in this process group state. It is an array of + process_indexes_{}; ///< Processes that should be started / run in this process group state. It is an array of ///< indexes (aka pointers) to the processes managed by a process group. }; @@ -97,12 +97,12 @@ struct ProcessGroupState final // have user-declared constructor. The rule doesn’t apply.", false) struct ProcessGroup final { - IdentifierHash name_; ///< Name of a process group. - IdentifierHash sw_cluster_; ///< Software cluster to which this process group belongs - IdentifierHash off_state_; ///< ID of the "Off" state for this process group - IdentifierHash recovery_state_; ///< ID of the recovery state for this process group - std::vector states_; ///< States configured for this process group. - std::vector processes_; ///< Processes that are managed (started / stopped) by this process group. + IdentifierHash name_{}; ///< Name of a process group. + IdentifierHash sw_cluster_{}; ///< Software cluster to which this process group belongs + IdentifierHash off_state_{}; ///< ID of the "Off" state for this process group + IdentifierHash recovery_state_{}; ///< ID of the recovery state for this process group + std::vector states_{}; ///< States configured for this process group. + std::vector processes_{}; ///< Processes that are managed (started / stopped) by this process group. }; /// @@ -186,6 +186,13 @@ class ConfigurationManager final std::optional*> getProcessIndexesList( const ProcessGroupStateID& process_group_state_id) const; + /// @brief Get the names of all states configured for a particular process group. + /// @param[in] pg_name The name of the process group. + /// @return Returns a pointer to a vector of the process group's configured state names, or no + /// value if the process group does not exist. + std::optional*> getListOfProcessGroupStates( + const IdentifierHash& pg_name) const; + /// @brief Get startup configuration for a given process. /// @param[in] pg_name_ The name of the process group for which to retrieve the OS Configurations. . /// @param[in] index The index of the OS process. diff --git a/score/launch_manager/src/daemon/src/main.cpp b/score/launch_manager/src/daemon/src/main.cpp index e2c8aece8..6fd0ecb19 100644 --- a/score/launch_manager/src/daemon/src/main.cpp +++ b/score/launch_manager/src/daemon/src/main.cpp @@ -179,7 +179,6 @@ int main([[maybe_unused]] int argc, [[maybe_unused]] const char* argv[]) // osal::IpcCommsSync::control_client_handler_nudge_fd (fd4) for communication tpyes: kNoComms !fd3 & !fd4 // kReporting fd3 & !fd4 // kControlClient fd3 & fd4 - // kLaunchManager does not matter // the file descriptors are closed inside the handleComms function. reserveFD(osal::IpcCommsSync::sync_fd); reserveFD(osal::IpcCommsSync::control_client_handler_nudge_fd); @@ -199,7 +198,7 @@ int main([[maybe_unused]] int argc, [[maybe_unused]] const char* argv[]) score::mw::launch_manager::configuration::FlatbufferConfigLoader config_loader; auto config_result = config_loader.load(config_path); if (!config_result.has_value()) { - LM_LOG_FATAL() << "Failed to load config from: " << config_path; + LM_LOG_FATAL() << "Failed to load config from: " << std::string_view(config_path); return EXIT_FAILURE; } #endif diff --git a/score/launch_manager/src/daemon/src/osal/return_types.hpp b/score/launch_manager/src/daemon/src/osal/return_types.hpp index 5c34f0452..7660b7151 100644 --- a/score/launch_manager/src/daemon/src/osal/return_types.hpp +++ b/score/launch_manager/src/daemon/src/osal/return_types.hpp @@ -42,7 +42,6 @@ enum class CommsType : std::uint_least8_t { kNoComms = 0, // Do not create any communications channel kReporting = 1, // Create an osal::Comms object only kControlClient = 2, // Create an osal::Comms object and reserve space for a ControlClientChannel - kLaunchManager = 3 // Do not create any comms chanel because this is us }; ///@brief This enum class likely represents the return status or outcome of an operating system abstraction layer (OSAL) diff --git a/score/launch_manager/src/daemon/src/process_group_manager/BUILD b/score/launch_manager/src/daemon/src/process_group_manager/BUILD index 8fd583c3c..b97bc0394 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/BUILD +++ b/score/launch_manager/src/daemon/src/process_group_manager/BUILD @@ -24,6 +24,19 @@ cc_library( ], ) +cc_library( + name = "mock_iprocess", + testonly = True, + hdrs = ["mock_iprocess.hpp"], + include_prefix = "score/mw/launch_manager/process_group_manager", + strip_include_prefix = "/score/launch_manager/src/daemon/src/process_group_manager", + visibility = ["//score:__subpackages__"], + deps = [ + ":iprocess", + "@googletest//:gtest_main", + ], +) + cc_library( name = "ialive_monitor_thread", hdrs = ["ialive_monitor_thread.hpp"], @@ -68,6 +81,7 @@ cc_library( "//score/launch_manager/src/daemon/src/process_group_manager/details:graph", "//score/launch_manager/src/daemon/src/process_group_manager/details:os_handler", "//score/launch_manager/src/daemon/src/process_group_manager/details:process_info_node", + "//score/launch_manager/src/daemon/src/process_group_manager/details:process_launcher", "//score/launch_manager/src/daemon/src/process_group_manager/details:safe_process_map", "//score/launch_manager/src/daemon/src/process_state_client:iprocess_state_notifier", "//score/launch_manager/src/daemon/src/recovery_client", diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/BUILD b/score/launch_manager/src/daemon/src/process_group_manager/details/BUILD index 6b0f00e10..4dee835f2 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/BUILD +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/BUILD @@ -73,9 +73,9 @@ cc_library( ) cc_library( - name = "mock_termination_callback", + name = "mock_component_controller", testonly = True, - hdrs = ["mock_termination_callback.hpp"], + hdrs = ["mock_component_controller.hpp"], include_prefix = "score/mw/launch_manager/process_group_manager/details", strip_include_prefix = "/score/launch_manager/src/daemon/src/process_group_manager/details", visibility = ["//score:__subpackages__"], @@ -135,6 +135,7 @@ cc_test( cc_library( name = "process_info_node", + srcs = ["process_info_node.cpp"], hdrs = ["process_info_node.hpp"], defines = select({ "//config:lm_use_new_configuration": ["USE_NEW_CONFIGURATION"], @@ -161,6 +162,17 @@ cc_library( }), ) +cc_test( + name = "process_info_node_UT", + srcs = ["process_info_node_UT.cpp"], + deps = [ + ":process_info_node", + ":safe_process_map", + "//score/launch_manager/src/daemon/src/process_group_manager:mock_iprocess", + "@googletest//:gtest_main", + ], +) + cc_library( name = "run_target", hdrs = ["run_target.hpp"], @@ -180,10 +192,13 @@ cc_library( visibility = ["//score/launch_manager/src/daemon/src/process_group_manager:__pkg__"], deps = [ ":component_event_queue", + ":component_of", ":component_task", + ":dependency_graph", ":process_info_node", ":run_target", ":safe_process_map", + ":transition", "//score/launch_manager/src/daemon/src/common:identifier_hash", "//score/launch_manager/src/daemon/src/control:control_client_channel", "//score/launch_manager/src/daemon/src/osal:semaphore", @@ -191,6 +206,32 @@ cc_library( ], ) +cc_library( + name = "transition", + hdrs = ["transition.hpp"], + include_prefix = "score/mw/launch_manager/process_group_manager/details", + strip_include_prefix = "/score/launch_manager/src/daemon/src/process_group_manager/details", + visibility = ["//score/launch_manager/src/daemon/src/process_group_manager:__pkg__"], + deps = [ + ":dependency_graph", + "//score/launch_manager/src/daemon/src/common/concurrency:fixed_size_queue", + "@score_baselibs//score/language/futurecpp", + ], +) + +cc_library( + name = "component_of", + hdrs = ["component_of.hpp"], + include_prefix = "score/mw/launch_manager/process_group_manager/details", + strip_include_prefix = "/score/launch_manager/src/daemon/src/process_group_manager/details", + visibility = ["//score/launch_manager/src/daemon/src/process_group_manager:__pkg__"], + deps = [ + ":icomponent", + ":process_info_node", + ":run_target", + ], +) + cc_library( name = "safe_process_map", srcs = ["safe_process_map.cpp"], @@ -209,7 +250,7 @@ cc_test( timeout = "long", srcs = ["safeprocessmap_UT.cpp"], deps = [ - ":mock_termination_callback", + ":mock_component_controller", ":safe_process_map", "@googletest//:gtest_main", ], @@ -263,7 +304,8 @@ cc_test( name = "oshandler_UT", srcs = ["oshandler_UT.cpp"], deps = [ - ":mock_termination_callback", + ":mock_component", + ":mock_component_controller", ":os_handler", ":safe_process_map", "@googletest//:gtest_main", @@ -274,6 +316,9 @@ cc_test( cc_library( name = "process_launcher", srcs = ["process_launcher.cpp"], + hdrs = ["process_launcher.hpp"], + include_prefix = "score/mw/launch_manager/process_group_manager/details", + strip_include_prefix = "/score/launch_manager/src/daemon/src/process_group_manager/details", visibility = ["//score/launch_manager/src/daemon/src/process_group_manager:__pkg__"], deps = [ "//score/launch_manager/src/daemon/src/common:log", @@ -287,6 +332,40 @@ cc_library( ], ) +cc_library( + name = "dependency_graph", + hdrs = [ + "dependency_graph.hpp", + ], + include_prefix = "score/mw/launch_manager/process_group_manager/details", + strip_include_prefix = "/score/launch_manager/src/daemon/src/process_group_manager/details", + visibility = ["//score/launch_manager/src/daemon/src/process_group_manager:__pkg__"], + deps = [ + "//score/launch_manager/src/daemon/src/common/concurrency:fixed_size_queue", + ], +) + +cc_test( + name = "dependency_graph_UT", + srcs = ["dependency_graph_UT.cpp"], + deps = [ + ":dependency_graph", + "//score/launch_manager/src/daemon/src/common:identifier_hash", + "@googletest//:gtest_main", + ], +) + +cc_test( + name = "transition_UT", + srcs = ["transition_UT.cpp"], + deps = [ + ":dependency_graph", + ":icomponent", + ":transition", + "@googletest//:gtest_main", + ], +) + # graph.cpp, process_info_node.cpp, and process_group_manager.cpp include # process_group_manager.hpp which in turn includes graph.hpp and process_info_node.hpp — # circular at link time, so all three must share a target. diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/component_of.hpp b/score/launch_manager/src/daemon/src/process_group_manager/details/component_of.hpp new file mode 100644 index 000000000..80f5e4241 --- /dev/null +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/component_of.hpp @@ -0,0 +1,40 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#ifndef SCORE_LCM_COMPONENT_OF_HPP_INCLUDED +#define SCORE_LCM_COMPONENT_OF_HPP_INCLUDED + +#include "score/mw/launch_manager/process_group_manager/details/icomponent.hpp" +#include "score/mw/launch_manager/process_group_manager/details/process_info_node.hpp" +#include "score/mw/launch_manager/process_group_manager/details/run_target.hpp" + +#include + +namespace score::mw::lifecycle::internal +{ + +using namespace score::lcm::internal; + +/// @brief Returns the IComponent reference from a variant type +/// @details All types in the variant must implement the IComponent interface. +inline IComponent& componentOf(std::variant& node) +{ + return std::visit( + [](auto& component) -> IComponent& { + return component; + }, + node); +} + +} // namespace score::mw::lifecycle::internal + +#endif // SCORE_LCM_COMPONENT_OF_HPP_INCLUDED diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/dependency_graph.hpp b/score/launch_manager/src/daemon/src/process_group_manager/details/dependency_graph.hpp new file mode 100644 index 000000000..854d43bc6 --- /dev/null +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/dependency_graph.hpp @@ -0,0 +1,183 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#ifndef SCORE_LCM_DEPENDENCY_GRAPH_HPP +#define SCORE_LCM_DEPENDENCY_GRAPH_HPP + +#include "score/assert.hpp" +#include "score/mw/launch_manager/common/concurrency/fixed_size_queue.hpp" + +#include +#include +#include + +namespace score::mw::lifecycle +{ + +using namespace score::lcm::internal; + +/// @brief Index type used to identify nodes in the graph. +using GraphIndex = std::size_t; + +/// @brief Stores a set of nodes as a directed acyclic graph (DAG) with edges representing dependencies between nodes. +/// @details The class provides methods to create and traverse the graph. +template +class DependencyGraph +{ + private: + /// @brief Wrapper around objects in the graph to store information about dependencies. + struct GraphNode + { + T value; + std::vector depends_on; + std::vector dependents; + + /// @brief Constructor to allow in-place construction of T. + template + GraphNode(Args&&... args) : value(std::forward(args)...) + { + } + }; + + public: + /// @param count The exact number of nodes that will be added. + DependencyGraph(std::size_t count) : traversal_queue(std::max(count, std::size_t(2)) - 1) + { + nodes.reserve(count); + visited.resize(count); + } + + /// @brief Construct a new node in-place. Returns the node's index, which equals the current size + /// before insertion (i.e. the first node is 0, second is 1, etc.). + template + GraphIndex emplace(Args&&... args) + { + nodes.emplace_back(std::forward(args)...); + return nodes.size() - 1; + } + + /// @brief Add an edge: @p node depends on @p depends_on. + /// During activation, depends_on will be started before node. + /// During deactivation, node will be stopped before depends_on. + void addDependency(const GraphIndex node, const GraphIndex depends_on) + { + nodes[node].depends_on.push_back(depends_on); + nodes[depends_on].dependents.push_back(node); + } + + /// @return The number of nodes in the graph. + std::size_t size() const + { + return nodes.size(); + } + + /// @return The number of nodes this graph can hold without reallocating (the @c count + /// reserved at construction). + std::size_t capacity() const + { + return nodes.capacity(); + } + + T& operator[](GraphIndex index) + { + return nodes[index].value; + } + + /// @return The nodes that @p index depends on. + const std::vector& dependsOn(GraphIndex index) const + { + return nodes[index].depends_on; + } + + /// @return The nodes that depend on @p index. + const std::vector& dependents(GraphIndex index) const + { + return nodes[index].dependents; + } + + /// @brief Traverse the graph, starting at @p start, performing @p per_node on each node and moving to the nodes + /// provided by the return value from @p per_node. Only nodes such that @c filter(node) is true are traversed. + /// Nodes are visited at most once. + template + void traverse(const GraphIndex start, PerNodeFn per_node, FilterFn filter) + { + visited.assign(visited.size(), false); + auto push_res = traversal_queue.push(start); + SCORE_LANGUAGE_FUTURECPP_ASSERT_MESSAGE(push_res, "Traversal queue was already full"); + visited[start] = true; + while (!traversal_queue.empty()) + { + const auto pop_res = traversal_queue.tryPop(); + SCORE_LANGUAGE_FUTURECPP_ASSERT_PRD_MESSAGE( + pop_res.has_value(), "Pop failed even though queue was not empty"); + const auto current = pop_res.value(); + + const auto& neighbors = per_node(current); + + for (const auto neighbor : neighbors) + { + if (visited[neighbor] || !filter(neighbor)) + { + continue; + } + push_res = traversal_queue.push(neighbor); + SCORE_LANGUAGE_FUTURECPP_ASSERT_MESSAGE(push_res, "Failed to push to traversal queue"); + visited[neighbor] = true; + } + } + } + + /// @brief Iterator over node values. + struct ValueIterator + { + typename std::vector::iterator it; + T& operator*() + { + return it->value; + } + + ValueIterator& operator++() + { + ++it; + return *this; + } + + bool operator!=(const ValueIterator& other) const + { + return it != other.it; + } + }; + + /// @returns Iterator at the beginning of the nodes store. + ValueIterator begin() + { + return ValueIterator{nodes.begin()}; + } + + /// @returns Iterator at the end of the nodes store. + ValueIterator end() + { + return ValueIterator{nodes.end()}; + } + + private: + std::vector nodes; + + /// @brief Presized queue reused by single-threaded traversals. + FixedSizeQueue traversal_queue; + /// @brief Presized visited set reused by single-threaded traversals. + std::vector visited; +}; + +} // namespace score::mw::lifecycle + +#endif // SCORE_LCM_DEPENDENCY_GRAPH_HPP diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/dependency_graph_UT.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/dependency_graph_UT.cpp new file mode 100644 index 000000000..ec0c68060 --- /dev/null +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/dependency_graph_UT.cpp @@ -0,0 +1,151 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#include +#include + +#include "score/mw/launch_manager/common/identifier_hash.hpp" +#include "score/mw/launch_manager/process_group_manager/details/dependency_graph.hpp" + +#include + +namespace score::mw::lifecycle +{ + +using namespace score::lcm; + +TEST(DependencyGraphTest, EmplaceAndAccessByIndex) +{ + const std::string_view text = "AAAAA"; + DependencyGraph graph(1); + const auto res = graph.emplace(text); + + auto& hash = graph[res]; + EXPECT_EQ(hash, IdentifierHash{text}); +} + +TEST(DependencyGraphTest, EmplaceReturnsSequentialIndices) +{ + DependencyGraph graph(3); + const auto first = graph.emplace("a"); + const auto second = graph.emplace("b"); + const auto third = graph.emplace("c"); + + EXPECT_EQ(first, 0U); + EXPECT_EQ(second, 1U); + EXPECT_EQ(third, 2U); +} + +TEST(DependencyGraphTest, AddDependencyWiresDependsOnAndDependents) +{ + DependencyGraph graph(2); + const auto dep = graph.emplace("dep"); + const auto root = graph.emplace("root"); + graph.addDependency(root, dep); + + EXPECT_THAT(graph.dependsOn(root), ::testing::ElementsAre(dep)); + EXPECT_THAT(graph.dependents(dep), ::testing::ElementsAre(root)); + EXPECT_TRUE(graph.dependsOn(dep).empty()); + EXPECT_TRUE(graph.dependents(root).empty()); +} + +TEST(DependencyGraphTest, SizeReflectsNumberOfEmplacedNodes) +{ + DependencyGraph graph(2); + EXPECT_EQ(graph.size(), 0U); + graph.emplace("a"); + EXPECT_EQ(graph.size(), 1U); + graph.emplace("b"); + EXPECT_EQ(graph.size(), 2U); +} + +TEST(DependencyGraphTest, TraverseVisitsWholeChainThroughDependsOn) +{ + // root -> mid -> leaf (X -> Y means X depends_on Y) + DependencyGraph graph(3); + const auto leaf = graph.emplace("leaf"); + const auto mid = graph.emplace("mid"); + const auto root = graph.emplace("root"); + graph.addDependency(root, mid); + graph.addDependency(mid, leaf); + + std::vector visited; + graph.traverse( + root, + [&](GraphIndex i) -> const std::vector& { + visited.push_back(graph[i]); + return graph.dependsOn(i); + }, + [](GraphIndex) { + return true; + }); + + EXPECT_THAT( + visited, + ::testing::UnorderedElementsAre(IdentifierHash{"root"}, IdentifierHash{"mid"}, IdentifierHash{"leaf"})); +} + +TEST(DependencyGraphTest, TraverseVisitsSharedDependencyExactlyOnce) +{ + // Diamond: both a and b depend on shared; root depends on both a and b. + DependencyGraph graph(4); + const auto shared = graph.emplace("shared"); + const auto a = graph.emplace("a"); + const auto b = graph.emplace("b"); + const auto root = graph.emplace("root"); + graph.addDependency(a, shared); + graph.addDependency(b, shared); + graph.addDependency(root, a); + graph.addDependency(root, b); + + std::size_t shared_visits = 0; + graph.traverse( + root, + [&](GraphIndex i) -> const std::vector& { + if (i == shared) + { + ++shared_visits; + } + return graph.dependsOn(i); + }, + [](GraphIndex) { + return true; + }); + + EXPECT_EQ(shared_visits, 1U); +} + +TEST(DependencyGraphTest, TraverseFilterBoundsWhichNodesAreVisited) +{ + DependencyGraph graph(3); + const auto excluded = graph.emplace("excluded"); + const auto included = graph.emplace("included"); + const auto root = graph.emplace("root"); + graph.addDependency(root, included); + graph.addDependency(root, excluded); + + std::vector visited; + graph.traverse( + root, + [&](GraphIndex i) -> const std::vector& { + visited.push_back(i); + return graph.dependsOn(i); + }, + [excluded](GraphIndex neighbor) { + return neighbor != excluded; + }); + + EXPECT_THAT(visited, ::testing::UnorderedElementsAre(root, included)); +} + +} // namespace score::mw::lifecycle diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp index 93c32af77..012f30483 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp @@ -11,9 +11,13 @@ * SPDX-License-Identifier: Apache-2.0 ********************************************************************************/ +#include #include #include +#include +#include +#include #include "score/mw/launch_manager/common/log.hpp" #include "score/mw/launch_manager/process_group_manager/details/graph.hpp" @@ -21,6 +25,8 @@ #include "score/mw/launch_manager/process_group_manager/process_group_manager.hpp" +#include "score/assert.hpp" + namespace score { @@ -32,10 +38,8 @@ namespace internal Graph::Graph(uint32_t max_num_nodes, ProcessGroupManager* pgm) : pg_index_(0U), - nodes_(), - nodes_to_execute_(0U), - nodes_in_flight_(0U), - starting_(false), + nodes_(max_num_nodes), + transition_builder_(nodes_), state_(GraphState::kSuccess), semaphore_(), requested_state_(), @@ -49,7 +53,6 @@ Graph::Graph(uint32_t max_num_nodes, ProcessGroupManager* pgm) request_start_time_() { LM_LOG_DEBUG() << "Creating graph with" << max_num_nodes << "nodes"; - nodes_.reserve(max_num_nodes); last_state_manager_.process_index_ = 0xFFFFU; // an invalid state manager last_state_manager_.process_group_index_ = 0xFFFFU; cancel_message_.request_or_response_ = ControlClientCode::kNotSet; @@ -57,7 +60,6 @@ Graph::Graph(uint32_t max_num_nodes, ProcessGroupManager* pgm) Graph::~Graph() { - nodes_.clear(); LM_LOG_DEBUG() << "Graph destroyed"; } @@ -76,43 +78,120 @@ void Graph::initProcessGroupNodes(IdentifierHash pg_name, uint32_t num_processes if (nodes_.size() == num_processes) { createSuccessorLists(pg_name); + createRunTargetNodes(pg_name); } } inline void Graph::createProcessInfoNodes(uint32_t num_processes) { - nodes_.reserve(num_processes); // Reserve space for efficiency - for (uint32_t process_id = 0U; process_id < num_processes; ++process_id) { LM_LOG_DEBUG() << "Creating process node with id:" << process_id; - nodes_.push_back(std::make_shared()); - nodes_.back()->initNode(this, process_id); + auto ready_condition = nodeHasTerminatedDeps(getProcessGroupName(), process_id) + ? ProcessInfoNode::ReadyCondition::kTerminated + : ProcessInfoNode::ReadyCondition::kRunning; + + auto report_state_lambda = [this](IdentifierHash id, ProcessState state, timespec timestamp) { + score::lcm::PosixProcess process_info; + process_info.id = id; + process_info.processStateId = state; + process_info.processGroupStateId = getProcessGroupState(); + process_info.systemClockTimestamp = timestamp; + return getProcessGroupManager()->queuePosixProcess(process_info); + }; + + const auto* config = + pgm_->getConfiguration()->getOsProcessConfiguration(getProcessGroupName(), process_id).value_or(nullptr); + if (!config) + { + LM_LOG_ERROR() << "No configuration for process" << process_id << "of process group" + << getProcessGroupName(); + } + + const auto index = nodes_.emplace( + std::in_place_type, + config, + process_id, + ready_condition, + report_state_lambda, + pgm_->getProcessInterface(), + pgm_->getProcessMap()); + assert(index == process_id && "Graph indicies must line up with os process indices"); } LM_LOG_DEBUG() << "Created" << nodes_.size() << "process nodes"; } +inline void Graph::createRunTargetNodes(IdentifierHash pg_name) +{ + const auto num_processes = nodes_.size(); + const auto* states = pgm_->getConfiguration()->getListOfProcessGroupStates(pg_name).value_or(nullptr); + + SCORE_LANGUAGE_FUTURECPP_ASSERT_PRD_MESSAGE(states != nullptr, "Process group states not found for process group"); + + for (const auto& state : *states) + { + const auto node_index = static_cast(nodes_.size()); + const auto emplaced_index = nodes_.emplace(std::in_place_type, node_index); + SCORE_LANGUAGE_FUTURECPP_ASSERT_PRD_MESSAGE( + emplaced_index == node_index, "RunTarget index must match its position in the graph"); + run_targets_.emplace_back(state.name_, node_index); + + for (const auto process_index : state.process_indexes_) + { + SCORE_LANGUAGE_FUTURECPP_PRECONDITION_MESSAGE( + process_index < num_processes, + "Process index is out of range for the dependency graph of process group"); + nodes_.addDependency(node_index, process_index); + LM_LOG_DEBUG() << "Added RunTarget dependency:" << process_index << "->" << node_index; + } + } +} + +int32_t Graph::getRunTargetIndex(IdentifierHash pg_state) const +{ + for (const auto& [state_name, index] : run_targets_) + { + if (state_name == pg_state) + { + return static_cast(index); + } + } + return -1; +} + +bool Graph::nodeHasTerminatedDeps(IdentifierHash pg_name, uint32_t node_index) +{ + const DependencyList* dep_list = + pgm_->getConfiguration()->getOsProcessDependencies(pg_name, node_index).value_or(nullptr); + + if (dep_list && dep_list->size() > 0) + { + return (*dep_list)[0].process_state_ == ProcessState::kTerminated; + } + + return false; +} + inline void Graph::createSuccessorLists(IdentifierHash pg_name) { LM_LOG_DEBUG() << "Creating successor lists for process group" << pg_name; // Now create the successor lists for each process in this process group - for (auto& node : nodes_) + for (std::size_t i = 0; i < nodes_.size(); i++) { - // If the other process has a dependency on this one, put it on the correct list - auto node_index = node->getNodeIndex(); const DependencyList* dep_list = - pgm_->getConfiguration()->getOsProcessDependencies(pg_name, node_index).value_or(nullptr); + pgm_->getConfiguration()->getOsProcessDependencies(pg_name, i).value_or(nullptr); if (dep_list) { for (const Dependency& dep : *dep_list) { - if (dep.os_process_index_ < nodes_.size()) - { - nodes_[dep.os_process_index_]->addSuccessorNode(node, dep.process_state_); - LM_LOG_DEBUG() << "Added successor node dependency:" << dep.os_process_index_ << "->" << node_index; - } + SCORE_LANGUAGE_FUTURECPP_PRECONDITION_MESSAGE( + dep.os_process_index_ < nodes_.size(), + "Process index is out of range for the dependency graph of process group"); + + nodes_.addDependency(i, dep.os_process_index_); + LM_LOG_DEBUG() << "Added successor node dependency:" << dep.os_process_index_ << "->" << i; } } } @@ -140,82 +219,108 @@ void Graph::setState(GraphState new_state) target_state = line.data()[static_cast(old_state)]; // score::cpp::span does not implement operator[] - if (state_.compare_exchange_strong(old_state, target_state)) + state_ = target_state; + + if (target_state == GraphState::kInTransition && old_state != GraphState::kInTransition) + { + stop_source_ = score::cpp::stop_source{}; + } + else if (target_state != GraphState::kInTransition && old_state == GraphState::kInTransition) { - LM_LOG_DEBUG() << "Graph::setState changes from" << toString(old_state) << "to" - << toString(target_state) << "for PG" << pg_index_ << "(" << requested_state_.pg_name_ - << ")"; - old_state = target_state; + // If we've left the transition state, we should stop any continuing jobs + static_cast(stop_source_.request_stop()); + } - if (new_state == GraphState::kSuccess) - { - // get state transition end time stamp - auto request_end_time = std::chrono::steady_clock::now(); + old_state = target_state; - // log state transition duration - auto timeDiff = - std::chrono::duration_cast(request_end_time - getRequestStartTime()); + if (new_state == GraphState::kSuccess) + { + // get state transition end time stamp + auto request_end_time = std::chrono::steady_clock::now(); - LM_LOG_INFO() << "Completed the request for PG" << getProcessGroupName() << "to State" - << getProcessGroupState() << "in" << timeDiff.count() << "ms"; - } + // log state transition duration + auto timeDiff = + std::chrono::duration_cast(request_end_time - getRequestStartTime()); + + LM_LOG_INFO() << "Completed the request for PG" << getProcessGroupName() << "to State" + << getProcessGroupState() << "in" << timeDiff.count() << "ms"; } } } } -bool Graph::queueHeadNodes(bool start) +void Graph::updateRunTargetInPlace(RunTarget& run_target, ComponentTaskType task_type) { - // Count the number of nodes in this graph - starting_ = start; - - uint32_t executing_nodes = countExecutableNodes(start); - - nodes_to_execute_.store(executing_nodes); - nodes_in_flight_.store(0); - - if (executing_nodes > 0U) + // RunTargets are updated in place, no need to queue them on the thread pool. + if (task_type == ComponentTaskType::kActivate) { - queueHeadNodesForExecution(); + run_target.activate(stop_source_.get_token()); } - - return (executing_nodes > 0); + else + { + run_target.deactivate(stop_source_.get_token()); + } + current_transition_->onNodeFinished(run_target.getIndex()); } -inline uint32_t Graph::countExecutableNodes(bool start) +void Graph::queueReadyNodes() { - uint32_t executable_nodes = 0U; - - for (const auto& node : nodes_) + // Range-for consumes the frontier via the transition's iterator (nextReady()); RunTarget + // completions reported inside the loop append successors that this same iteration picks up. + for (const auto [node, action] : *current_transition_) { - if (node->constructGraphNode(start)) - { - ++executable_nodes; - } + const ComponentTaskType task_type = + action == Action::Start ? ComponentTaskType::kActivate : ComponentTaskType::kDeactivate; + LM_LOG_DEBUG() << "Node" << node << "is ready for" + << (task_type == ComponentTaskType::kActivate ? std::string_view("activation") + : std::string_view("deactivation")); + std::visit( + [this, task_type](auto& component) { + using ComponentT = std::decay_t; + if constexpr (std::is_same_v) + { + updateRunTargetInPlace(component, task_type); + } + else + { + // Queue ProcessInfoNode for execution on worker thread; completion arrives later + // via a ComponentEvent, draining into nodeExecuted() -> onNodeFinished(). + tryQueueNode(ComponentTask{task_type, component, stop_source_.get_token()}); + } + }, + nodes_[node]); } - - return executable_nodes; } -inline void Graph::queueHeadNodesForExecution() +void Graph::finalizeTransitionSuccess() { - for (const auto& node : nodes_) + if (is_initial_state_transition_) { - if (node->isHeadNode()) - { - tryQueueNode(node); - } + is_initial_state_transition_ = false; + pgm_->setInitialStateTransitionResult(ControlClientCode::kInitialMachineStateSuccess); + + // RULECHECKER_comment(1, 3, check_c_style_cast, "This is the definition provided by the OS and does + // a C-style cast.", true) + LM_LOG_DEBUG() << "clock() at successful initial state transition:" + // coverity[cert_err33_c_violation:INTENTIONAL] Does not matter if clock() gives a + // weird value in debug messages. + << (static_cast(clock()) / (static_cast(CLOCKS_PER_SEC) / 1000.0)) << "ms"; } + setState(GraphState::kSuccess); + setPendingEvent(ControlClientCode::kSetStateSuccess); } -inline void Graph::tryQueueNode(const std::shared_ptr& node) +inline void Graph::tryQueueNode(ComponentTask task) { while (GraphState::kInTransition == getState()) { - auto push_res = pgm_->getWorkerJobs()->push(node, kMaxQueueDelay); + auto push_res = pgm_->getWorkerJobs()->push(task, kMaxQueueDelay); if (push_res) { - markNodeInFlight(); + jobs_in_progress_++; + // LM_LOG_DEBUG() << "Queued node " << task.component.get().getIndex() << " for " + // << (task.type == ComponentTaskType::kDeactivate ? "deactivation" : "activation") + // << " execution, jobs in progress:" << jobs_in_progress_; break; } else if (push_res.error() == ConcurrencyErrc::kTimeout) @@ -228,8 +333,7 @@ inline void Graph::tryQueueNode(const std::shared_ptr& node) // here LM_LOG_ERROR() << "Failed to queue node for execution " << push_res.error(); - abort(getLastExecutionError(), ControlClientCode::kSetStateFailed); - markNodeInFlight(); // This will be decremented below, avoid it going negative + abort(getLastExecutionError(), IComponent::ComponentError::kErrorBeforeReady); // Also, we need to be careful not to recurse or deadlock here. The below function does not lock any mutex // nor call this function handleNonTransitionExecution(GraphState::kAborting); @@ -238,39 +342,6 @@ inline void Graph::tryQueueNode(const std::shared_ptr& node) } } -void Graph::queueStopJobs(const std::vector* process_index_list) -{ - // p is not nullptr - guaranteed by caller. - // First mark all processes as being not in the requested state - for (auto node : nodes_) - { - node->markRequested(false); - } - - // Then go through the processes in the requested state and mark them true - for (uint32_t index : *process_index_list) - { - if (index < nodes_.size()) - { - nodes_[index]->markRequested(true); - } - } - - if (!queueHeadNodes(false)) - { - queueStartJobs(); - } -} - -void Graph::queueStartJobs() -{ - if (!queueHeadNodes(true)) - { - setState(GraphState::kSuccess); // nothing to do, done nothing, success! - setPendingEvent(ControlClientCode::kSetStateSuccess); - } -} - bool Graph::startTransition(ProcessGroupStateID pg_state) { IdentifierHash old_state_name; @@ -284,11 +355,22 @@ bool Graph::startTransition(ProcessGroupStateID pg_state) if (nullptr != process_index_list) { + const int32_t target_node = getRunTargetIndex(requested_state_.pg_state_name_); + + SCORE_LANGUAGE_FUTURECPP_ASSERT_PRD_MESSAGE( + target_node >= 0, "RunTarget node not found for requested process group state"); + setState(GraphState::kInTransition); if (GraphState::kInTransition == getState()) { - queueStopJobs(process_index_list); + const auto target = static_cast(target_node); + current_transition_ = &transition_builder_.createTransition(target); + queueReadyNodes(); + if (current_transition_->isFinished()) + { + finalizeTransitionSuccess(); + } return true; } } @@ -316,118 +398,151 @@ bool Graph::startInitialTransition(ProcessGroupStateID pg_state) bool Graph::startTransitionToOffState() { - // Guaranteed to transition to all off even if there - // is no configured "Off" state. + // The Off state always has a RunTarget node (guaranteed by the configuration layer), so this + // is an ordinary transition to that node: everything the Off target doesn't need is stopped, + // and the (dependency-less) Off node is activated. setRequestStartTime(); { std::lock_guard lock(requested_state_mutex_); requested_state_.pg_state_name_ = off_state_; } - bool result = false; - { - std::shared_lock lock(transition_completion_mutex_); - setState(GraphState::kInTransition); - } + setState(GraphState::kInTransition); if (GraphState::kInTransition == getState()) { - std::vector empty_list{}; - queueStopJobs(&empty_list); - result = true; + const int32_t off_index = getRunTargetIndex(off_state_); + SCORE_LANGUAGE_FUTURECPP_ASSERT_PRD_MESSAGE(off_index >= 0, "Off RunTarget node not found"); + current_transition_ = &transition_builder_.createTransition(static_cast(off_index)); + queueReadyNodes(); + if (current_transition_->isFinished()) + { + finalizeTransitionSuccess(); + } + return true; } - return result; + return false; +} + +bool Graph::isTransitioningToOff() const +{ + std::lock_guard lock(requested_state_mutex_); + return (getState() == GraphState::kInTransition) && (requested_state_.pg_state_name_ == off_state_); +} + +void Graph::handleComponentEvent(const ComponentEvent& event) +{ + std::visit( + [this](const auto& data) { + using T = std::decay_t; + if constexpr (std::is_same_v || std::is_same_v) + { + LM_LOG_DEBUG() << "Component " << data.node_index << " finished " + << (std::is_same_v ? std::string_view("activation") + : std::string_view("deactivation")) + << " successfully"; + nodeExecuted(data.node_index, {}); + } + else if constexpr (std::is_same_v) + { + nodeExecuted(data.node_index, score::cpp::make_unexpected(data.reason)); + } + else if constexpr (std::is_same_v) + { + // ProcessMonitor::terminated() only pushes UnexpectedTermination from the final + // (already-ready) branch of ProcessInfoNode::tryHandleTermination; the "still + // starting" and "termination was requested" branches both resolve with kWaiting and + // never reach here. So this is always a post-ready crash. + abort(1U, IComponent::ComponentError::kErrorAfterReady); + + // If there are no jobs in progress, we need to trigger cleanup immediately + // Otherwise the graph will stay stuck in kAborting state forever + // ??? + // if (jobs_in_progress_ == 0) + // { + // handleNonTransitionExecution(GraphState::kAborting); + // } + } + }, + event); } -void Graph::nodeExecuted() +void Graph::nodeExecuted(uint32_t node, score::cpp::expected_blank error) { - std::unique_lock lock(transition_completion_mutex_); + bool was_last_in_queue = --jobs_in_progress_ == 0; + + if (!error.has_value()) + { + abort(1, error.error()); + } GraphState current_state = getState(); if (current_state == GraphState::kInTransition) { - handleTransitionExecution(); + current_transition_->onNodeFinished(node); + queueReadyNodes(); + if (current_transition_->isFinished()) + { + finalizeTransitionSuccess(); + } } - else + else if (was_last_in_queue) { handleNonTransitionExecution(current_state); } } -inline void Graph::handleTransitionExecution() +inline void Graph::handleNonTransitionExecution(GraphState current_state) { - if (nodes_to_execute_.load() > 0U) + if (is_initial_state_transition_) { - --nodes_in_flight_; + is_initial_state_transition_ = false; + pgm_->setInitialStateTransitionResult(ControlClientCode::kInitialMachineStateFailed); - if (0U == --nodes_to_execute_) - { - if (starting_) - { - if (is_initial_state_transition_) - { - is_initial_state_transition_ = false; - pgm_->setInitialStateTransitionResult(ControlClientCode::kInitialMachineStateSuccess); - - // RULECHECKER_comment(1, 3, check_c_style_cast, "This is the definition provided by the OS and does - // a C-style cast.", true) - LM_LOG_DEBUG() << "clock() at successful initial state transition:" - // coverity[cert_err33_c_violation:INTENTIONAL] Does not matter if clock() gives a - // weird value in debug messages. - << (static_cast(clock()) / (static_cast(CLOCKS_PER_SEC) / 1000.0)) - << "ms"; - } - setState(GraphState::kSuccess); - setPendingEvent(ControlClientCode::kSetStateSuccess); - } - else - { - queueStartJobs(); - } - } + // RULECHECKER_comment(1, 3, check_c_style_cast, "This is the definition provided by the OS and does a + // C-style cast.", true) + LM_LOG_FATAL() << "clock() at failed initial state transition:" + // coverity[cert_err33_c_violation:INTENTIONAL] Does not matter if clock() gives a weird + // value in debug messages. + << (static_cast(clock()) / (static_cast(CLOCKS_PER_SEC) / 1000.0)) << "ms"; + } + setState(GraphState::kUndefinedState); + if (current_state == GraphState::kAborting) + { + setPendingEvent(abort_code_); + } + else + { + ControlClientChannel::nudgeControlClientHandler(); } } -inline void Graph::handleNonTransitionExecution(GraphState current_state) +void Graph::abort(uint32_t code, IComponent::ComponentError reason) { - if (0 >= --nodes_in_flight_) + auto from_state = getState(); + if (from_state < GraphState::kAborting) { - if (is_initial_state_transition_) - { - is_initial_state_transition_ = false; - pgm_->setInitialStateTransitionResult(ControlClientCode::kInitialMachineStateFailed); - - // RULECHECKER_comment(1, 3, check_c_style_cast, "This is the definition provided by the OS and does a - // C-style cast.", true) - LM_LOG_FATAL() << "clock() at failed initial state transition:" - // coverity[cert_err33_c_violation:INTENTIONAL] Does not matter if clock() gives a weird - // value in debug messages. - << (static_cast(clock()) / (static_cast(CLOCKS_PER_SEC) / 1000.0)) << "ms"; - } - setState(GraphState::kUndefinedState); - if (current_state == GraphState::kAborting) + setState(GraphState::kAborting); + last_execution_error_ = code; + if (from_state != GraphState::kInTransition || reason == IComponent::ComponentError::kErrorAfterReady) { - setPendingEvent(abort_code_); + abort_code_ = ControlClientCode::kFailedUnexpectedTermination; } else { - ControlClientChannel::nudgeControlClientHandler(); + if (reason == IComponent::ComponentError::kErrorBeforeReady) + { + abort_code_ = ControlClientCode::kFailedUnexpectedTerminationOnEnter; + } + else + { + abort_code_ = ControlClientCode::kSetStateFailed; + } } } } -void Graph::abort(uint32_t code, ControlClientCode reason) -{ - if (getState() < GraphState::kAborting) - { - setState(GraphState::kAborting); - last_execution_error_.store(code); - abort_code_.store(reason); - } -} - void Graph::cancel() { - std::shared_lock lock(transition_completion_mutex_); setState(GraphState::kCancelled); if (getState() == GraphState::kCancelled) @@ -435,26 +550,44 @@ void Graph::cancel() setPendingEvent(ControlClientCode::kSetStateCancelled); } - if (0 == nodes_in_flight_) + if (jobs_in_progress_ > 0) + { + return; + } + + if (is_initial_state_transition_) { - if (is_initial_state_transition_) + is_initial_state_transition_ = false; + pgm_->setInitialStateTransitionResult(ControlClientCode::kInitialMachineStateFailed); + // Some may argue that not finishing MachineGF.Startup state transition, is a critical problem. + // Essentially, controller SM is requesting MachineGF.Startup transition, on an action list assigned to its + // initial state. RULECHECKER_comment(1, 3, check_c_style_cast, "This is the definition provided by the OS + // and does a C-style cast.", true) + LM_LOG_DEBUG() << "clock() at canceled initial state transition:" + // coverity[cert_err33_c_violation:INTENTIONAL] Does not matter if clock() gives a weird + // value in debug messages. + << (static_cast(clock()) / (static_cast(CLOCKS_PER_SEC) / 1000.0)) << "ms"; + } + setState(GraphState::kUndefinedState); +} + +void Graph::forceKillProcesses() +{ + for (const auto& component : nodes_) + { + if (const ProcessInfoNode* process = std::get_if(&component)) { - is_initial_state_transition_ = false; - pgm_->setInitialStateTransitionResult(ControlClientCode::kInitialMachineStateFailed); - // Some may argue that not finishing MachineGF.Startup state transition, is a critical problem. - // Essentially, controller SM is requesting MachineGF.Startup transition, on an action list assigned to its - // initial state. RULECHECKER_comment(1, 3, check_c_style_cast, "This is the definition provided by the OS - // and does a C-style cast.", true) - LM_LOG_DEBUG() << "clock() at canceled initial state transition:" - // coverity[cert_err33_c_violation:INTENTIONAL] Does not matter if clock() gives a weird - // value in debug messages. - << (static_cast(clock()) / (static_cast(CLOCKS_PER_SEC) / 1000.0)) << "ms"; + osal::ProcessID pid = process->getPid(); + if (pid > 0) + { + // forceTermination already handles errors appropriately, so we can ignore its result. + static_cast(pgm_->getProcessInterface()->forceTermination(pid)); + } } - setState(GraphState::kUndefinedState); } } -void Graph::setStateManager(ControlClientID& control_client_id) +void Graph::updateCancelMessage() { ControlClientCode code = getPendingEvent(); @@ -465,19 +598,21 @@ void Graph::setStateManager(ControlClientID& control_client_id) cancel_message_.request_or_response_ = code; clearPendingEvent(code); } - last_state_manager_ = control_client_id; } -std::shared_ptr Graph::getProcessInfoNode(uint32_t process_index) +void Graph::setStateManager(ControlClientID& control_client_id) { - std::shared_ptr node{}; + last_state_manager_ = control_client_id; +} - if (process_index < nodes_.size()) +ProcessInfoNode* Graph::getProcessInfoNode(uint32_t process_index) +{ + if (process_index >= nodes_.size()) { - node = nodes_[process_index]; + return nullptr; } - return node; + return std::get_if(&nodes_[process_index]); } ProcessGroupManager* Graph::getProcessGroupManager() @@ -490,19 +625,9 @@ IdentifierHash Graph::getProcessGroupName() return requested_state_.pg_name_; } -bool Graph::isStarting() const -{ - return starting_; -} - -void Graph::markNodeInFlight() -{ - ++nodes_in_flight_; -} - GraphState Graph::getState() const { - return state_.load(); + return state_; } IdentifierHash Graph::getProcessGroupState() @@ -516,9 +641,23 @@ uint32_t Graph::getProcessGroupIndex() return pg_index_; } -NodeList& Graph::getNodes() +const ProcessInfoNode* Graph::findControlClient() { - return nodes_; + auto* pin = getProcessInfoNode(getStateManager().process_index_); + if (pin && pin->getControlClientChannel()) + { + return pin; + } + + for (std::size_t i = 0; i < nodes_.size(); ++i) + { + if (const auto* node = std::get_if(&nodes_[i]); node && node->getControlClientChannel()) + { + return node; + } + } + + return nullptr; } ControlClientID Graph::getStateManager() @@ -528,18 +667,16 @@ ControlClientID Graph::getStateManager() uint32_t Graph::getLastExecutionError() { - return last_execution_error_.load(); + return last_execution_error_; } void Graph::setLastExecutionError(uint32_t code) { - last_execution_error_.store(code); + last_execution_error_ = code; } IdentifierHash Graph::setPendingState(IdentifierHash new_state) { - std::shared_lock lock(transition_completion_mutex_); - IdentifierHash result_state = pending_state_; pending_state_ = new_state; @@ -560,17 +697,20 @@ IdentifierHash Graph::getPendingState() ControlClientCode Graph::getPendingEvent() { - return event_.load(); + return event_; } void Graph::clearPendingEvent(ControlClientCode expected) { - (void)event_.compare_exchange_strong(expected, ControlClientCode::kNotSet); + if (event_ == expected) + { + event_ = ControlClientCode::kNotSet; + } } void Graph::setPendingEvent(ControlClientCode event) { - event_.store(event); + event_ = event; ControlClientChannel::nudgeControlClientHandler(); } diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp index bbcaf845e..197933fb1 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.hpp @@ -11,33 +11,42 @@ * SPDX-License-Identifier: Apache-2.0 ********************************************************************************/ - #ifndef GRAPH_HPP_INCLUDED #define GRAPH_HPP_INCLUDED +#include #include #include -#include #include -#include #include +#include +#include #include #include "score/mw/launch_manager/common/identifier_hash.hpp" -#include "score/mw/launch_manager/osal/semaphore.hpp" #include "score/mw/launch_manager/control/control_client_channel.hpp" +#include "score/mw/launch_manager/osal/semaphore.hpp" +#include "score/mw/launch_manager/process_group_manager/details/component_event.hpp" +#include "score/mw/launch_manager/process_group_manager/details/component_of.hpp" +#include "score/mw/launch_manager/process_group_manager/details/component_task.hpp" +#include "score/mw/launch_manager/process_group_manager/details/dependency_graph.hpp" #include "score/mw/launch_manager/process_group_manager/details/process_info_node.hpp" +#include "score/mw/launch_manager/process_group_manager/details/run_target.hpp" +#include "score/mw/launch_manager/process_group_manager/details/transition.hpp" #include "score/mw/launch_manager/process_group_manager/iprocess.hpp" -namespace score { +#include +namespace score +{ -namespace lcm { +namespace lcm +{ -namespace internal { +namespace internal +{ -class ProcessGroupManager; +using namespace score::mw::lifecycle; -/// Alias for NodeList using std::vector. -using NodeList = std::vector>; +class ProcessGroupManager; /// @brief GraphState - the graph/process group state. /// @details Enumeration representing the state of the graph. @@ -73,7 +82,8 @@ using NodeList = std::vector>; /// kUndefinedState | kAborting | kUndefinedState /// kUndefinedState | kCancelled | kUndefinedState /// @endverbatim -enum class GraphState : std::uint_least8_t { +enum class GraphState : std::uint_least8_t +{ ///@brief Graph is not running and process group state is known kSuccess = 0U, @@ -90,14 +100,6 @@ enum class GraphState : std::uint_least8_t { kUndefinedState = 4U }; -/// @brief StateTransition - state transitions -/// @deprecated Please see current POC for a faster and more obvious method of calculating the state result based upon a 25-byte table. -// RULECHECKER_comment(1, 1, check_incomplete_data_member_construction, "wi 45913 - This struct is POD, which doesn't have user-declared constructor. The rule doesn’t apply.", false) -struct StateTransition { - GraphState old_state; - GraphState new_state; - GraphState target_state; -}; /// @details Allowed transitions: /// ------------------- /// kSuccess -> kInTransition @@ -118,35 +120,48 @@ struct StateTransition { /// kUndefinedState -> kSuccess kUndefinedState /// kUndefinedState -> kAborting kUndefinedState // coverity[autosar_cpp14_m3_4_1_violation:INTENTIONAL] The value is used in a global context. +// clang-format off static constexpr GraphState state_results[][static_cast(GraphState::kUndefinedState) + 1U] = { - //from kSuccess kInTransition kAborting kCancelled kUndefinedState to new_state - {GraphState::kSuccess, GraphState::kSuccess, GraphState::kUndefinedState, GraphState::kUndefinedState, + // from kSuccess kInTransition kAborting kCancelled + // kUndefinedState to new_state + {GraphState::kSuccess, + GraphState::kSuccess, + GraphState::kUndefinedState, + GraphState::kUndefinedState, GraphState::kUndefinedState}, // kSuccess - {GraphState::kInTransition, GraphState::kInTransition, GraphState::kAborting, GraphState::kCancelled, + {GraphState::kInTransition, + GraphState::kInTransition, + GraphState::kAborting, + GraphState::kCancelled, GraphState::kInTransition}, // kInTransition - {GraphState::kUndefinedState, GraphState::kAborting, GraphState::kAborting, GraphState::kCancelled, + {GraphState::kUndefinedState, + GraphState::kAborting, + GraphState::kAborting, + GraphState::kCancelled, GraphState::kUndefinedState}, // kAborting - {GraphState::kUndefinedState, GraphState::kCancelled, GraphState::kCancelled, GraphState::kCancelled, + {GraphState::kUndefinedState, + GraphState::kCancelled, + GraphState::kCancelled, + GraphState::kCancelled, GraphState::kUndefinedState}, // kCancelled - {GraphState::kUndefinedState, GraphState::kAborting, GraphState::kUndefinedState, GraphState::kUndefinedState, + {GraphState::kUndefinedState, + GraphState::kAborting, + GraphState::kUndefinedState, + GraphState::kUndefinedState, GraphState::kUndefinedState} // kUndefinedState }; +// clang-format on -/// @brief Represents a graph of a process group. +/// @brief Manages the processes and state transitions for a single process group. /// -/// The Graph class manages processes and their states within a process group, providing methods -/// for state management, node execution, initialization, and transitioning between process group states. -/// Each Graph instance corresponds to a process group and maintains a collection -/// of ProcessInfoNode instances representing individual processes. -/// The graph manages transitions between different states of the process group, ensuring orderly -/// execution of processes and handling error conditions. -/// The execution of the graph involves connecting processing flow nodes together, -/// facilitating the stopping and starting of processes as required during state transitions. -/// If the transition completes successfully, the graph enters a new process group state; -/// otherwise, it remains in an undefined state. -/// -class Graph final { - public: +/// Each Graph holds a set of ProcessInfoNode instances (one per process) arranged in a +/// dependency graph. During a state transition the Graph stops processes that are no longer +/// needed and starts the ones required for the new state, respecting dependency order. If +/// the transition completes without errors the graph enters kSuccess. Otherwise it enters +/// kUndefinedState. +class Graph final +{ + public: /// @brief Constructor to initialize a Graph object. /// @param max_num_nodes Maximum number of nodes this graph can hold. /// @param pgm Pointer to the ProcessGroupManager managing this graph. @@ -173,131 +188,108 @@ class Graph final { /// @param index The index of the process group in the vector of process groups void initProcessGroupNodes(IdentifierHash pg, uint32_t num_processes, uint32_t index); - /// @brief Return whether this is a graph to start or stop processes - /// @return bool true if this graph is starting processes, false if it is stopping processes - /// @todo this is a trivial function and should be inlined - bool isStarting() const; - - /// @brief Notifies the graph that a node has been executed. - /// Decrements the count of nodes waiting to execute, and updates states if the graph has - /// completed. - void nodeExecuted(); - - /// @brief Inform the graph that a node is being queued for execution. - /// This method increments the count of nodes that have been queued for execution - /// but have not yet started execution. It is typically called when a node is - /// added to a worker job queue for processing. - /// @todo this is a trivial function and should be inlined - void markNodeInFlight(); - - /// @brief Abort graph execution (because a process has mis-behaved) - /// @param code The error code defined for the process that caused graph abort - /// @param reason The ControlClientCode describing the reason for abort (e.g. kSetStateFailed) - void abort(uint32_t code, ControlClientCode reason); - - /// @brief Cancel the graph execution (because a new state was requested) - /// Set the graph state to kCancelled; if the resulting state is kCancelled, then - /// set a pending event of kSetStateCancelled. - /// If there are no nodes in flight set the graph state to kUndefinedState; also process - /// initial state transition result if applicable. + /// @brief Applies a ComponentEvent — produced by ProcessMonitor from worker/OS-handler thread + /// callbacks and drained on the main thread — to this graph. + /// @details Dispatches on the event's variant: + /// - ActivationSuccessful / DeactivationComplete: `nodeExecuted(node_index, {})` + /// - ActivationFailed: `nodeExecuted(node_index, make_unexpected(reason))` + /// - UnexpectedTermination: `abort(1, kErrorAfterReady)` — ProcessMonitor::terminated() only + /// pushes this event once a process has already reached its ready condition, so it is + /// always a post-ready crash. + /// @param event The event to process. + void handleComponentEvent(const ComponentEvent& event); + + /// @brief Cancel the current transition because a new state has been requested. + /// Sets the graph state to kCancelled and posts a kSetStateCancelled pending event. + /// If no jobs are in progress, transitions immediately to kUndefinedState. void cancel(); - /// @brief Start a transition of this process group to the given state; return false if it was not possible - /// This function will return false either if the process group state name was not found or if the Graph - /// would not enter GraphState::kInTransition - /// @param pg_state The process group state to move to - /// @return bool true if the transition was started, false if it was not possible + /// @brief Begin transitioning this process group to the given state. + /// Returns false if the state name was not found in the configuration or if the graph + /// could not enter kInTransition (for example, because a cancellation is in progress). + /// @param pg_state The target process group state. + /// @return True if the transition was started. bool startTransition(ProcessGroupStateID pg_state); - /// @brief Same as startTransition, but this is the initial machine group startup state - /// @param pg_state Initial machine group startup state - /// @returns true if the transition was started, false otherwise + /// @brief Begin the initial machine group startup transition. + /// Behaves like startTransition but also reports the initial state transition result + /// to the ProcessGroupManager on failure. + /// @param pg_state The initial machine group startup state. + /// @return True if the transition was started. bool startInitialTransition(ProcessGroupStateID pg_state); - /// @brief Start a transition to the "Off" state for this process group - /// Even if there is no configured "Off" state for this process group, this - /// method will attempt to start a transition that shuts down all processes in - /// the process group. - /// This method will return false if the Graph would not enter GraphState::kInTransition - /// @return bool true if the transition was started, false if it was not possible + /// @brief Begin transitioning this process group to the "Off" state. + /// Stops all processes in the group even if no explicit "Off" state is configured. + /// @return True if the transition was started. False if the graph could not enter kInTransition. bool startTransitionToOffState(); - /// @brief Gets the current state of the graph. - /// @return The current state of the graph. - /// @todo this is a trivial function and should be inlined + /// @return True if the graph is currently transitioning to the Off state. + bool isTransitioningToOff() const; + + /// @return The current graph state. GraphState getState() const; - /// @brief Retrieves a ProcessInfoNode by its index. - /// Retrieves the ProcessInfoNode located at the specified `process_index`. /// @param process_index Index of the process node to retrieve. - /// @return Pointer to the ProcessInfoNode at the specified index, - /// or nullptr if the index is out of bounds. - std::shared_ptr getProcessInfoNode(uint32_t process_index); + /// @return The ProcessInfoNode at the given index, or nullptr if out of bounds or if the node + /// at that index is a RunTarget rather than a ProcessInfoNode. + ProcessInfoNode* getProcessInfoNode(uint32_t process_index); - /// @brief Retrieves the ProcessGroupManager associated with this graph. - /// @return Pointer to the ProcessGroupManager instance. - /// @todo this is a trivial method and should be inlined + /// @brief Helper function to identify a node with ready state "Terminated" from the legacy configuration + bool nodeHasTerminatedDeps(IdentifierHash pg_name, uint32_t node_index); + + /// @return The ProcessGroupManager that owns this graph. ProcessGroupManager* getProcessGroupManager(); - /// @brief Retrieves the ID of the process group managed by this graph. - /// @return The ID of the process group stored when the graph was created - /// @todo this is a trivial method and should be inlined + /// @return The identifier of the process group managed by this graph. IdentifierHash getProcessGroupName(); - /// @brief return the current target state of the process group represented by the graph object - /// @return IdentifierHash the last set value of the process group state; it is valid only if getState() returns GraphState::kSuccess + /// @return The current target state of the process group. Only meaningful when + /// getState() returns GraphState::kSuccess. IdentifierHash getProcessGroupState(); - /// @brief get the index of this graph in the vector of graphs held by the process group manager - /// @return uint32 index + /// @return The index of this graph within the ProcessGroupManager's graph list. uint32_t getProcessGroupIndex(); - /// @brief Return the entire vector of nodes - /// @return NodeList - NodeList& getNodes(); + /// @return The ProcessInfoNode that has a ControlClientChannel, or nullptr if none exists. + const ProcessInfoNode* findControlClient(); - /// @brief Set the state manager for this process group - /// If there was a pending event, set the cancel message so the event - /// will be sent to the previous state manager, and clear the pending event. - /// @param control_client_id + /// @brief Sets the control client that is managing state transitions for this process group. + /// @param control_client_id The identifier of the new state manager. void setStateManager(ControlClientID& control_client_id); - /// @brief Get the state manager for this process group as a single uint64 - /// @return uint64 identifying the Control Client + /// @brief Update the details for the cancel message to match the current state. + void updateCancelMessage(); + + /// @return Information about the control client managing this process group's state. ControlClientID getStateManager(); - /// @brief get the error code for the last process to cause an issue - /// @return uint32 + /// @return The error code set by the last process that caused an unexpected termination. uint32_t getLastExecutionError(); - /// @brief set the last execution error - /// @param code + /// @brief Stores an error code representing the last execution failure. + /// @param code The error code to store. void setLastExecutionError(uint32_t code); - /// @brief set a new pending state for the process group, and return the old one - /// @param new_state to set in pending_state_ - /// @return the previous value of pending_state_ + /// @brief Replaces the pending state with new_state and returns the previous pending state. + /// @param new_state The new pending state to set. + /// @return The previous pending state. IdentifierHash setPendingState(IdentifierHash new_state); - /// @brief get the value of the pending_state_ - /// @return current value of pending_state_ + /// @return The pending state, or an empty hash if no state is pending. IdentifierHash getPendingState(); - /// @brief get any pending event - /// @return the event code + /// @return The pending event code, or kNotSet if there is none. ControlClientCode getPendingEvent(); - /// @brief clear any pending event - /// Clear the pending event, but only if it was the expected value - /// @param expected The value of ControlClientCode expected in the event + /// @brief Clears the pending event, but only if its current value matches expected. + /// @param expected The event code to compare against. void clearPendingEvent(ControlClientCode expected); - /// @brief set a pending event code & nudge the process group manager - /// @param event code to store + /// @brief Stores a pending event code and notifies the ProcessGroupManager to process it. + /// @param event The event code to store. void setPendingEvent(ControlClientCode event); - /// @brief get the message constructed when (if) the graph was cancelled - /// @return a ControlClientMessage to send + /// @return The cancel message prepared when updateCancelMessage() was called. ControlClientMessage& getCancelMessage(); /// @brief A utility function that converts codes to strings for logging purposes @@ -305,106 +297,91 @@ class Graph final { /// @return A string representing the state static std::string_view toString(GraphState state); - /// @brief Sets the state transition request start time + /// @brief Records the current time as the start of a state transition request. void setRequestStartTime(); - /// @brief Gets the state transition request start time - /// @return Timestamp based on the system clock when starting a state transition request + /// @return The timestamp recorded at the start of the current state transition request. std::chrono::time_point getRequestStartTime(); - private: + /// @brief For forced shutdown, kill all leftover processes + void forceKillProcesses(); + + private: + /// @brief Reports that a node has finished executing, enqueuing successors or updating the graph state if a + /// transition has finished. + void nodeExecuted(uint32_t node, score::cpp::expected_blank error); + + /// @brief Abort the current transition due to a process error. + /// @deprecated @param code The execution error for the process that caused the abort. + /// @param reason The process error that triggered the abort. + void abort(uint32_t code, IComponent::ComponentError reason); + /// @brief Sets the current state of the graph. /// @param new_state The new state to set for the graph. void setState(GraphState new_state); - /// @brief Queue head nodes to start a graph. Return false if there were no head nodes - /// @param start true if this run of the graph is to start processes, false it it is to stop them - /// @return true if one or more head nodes were found - bool queueHeadNodes(bool start); - - /// @brief Queue the jobs to kick-off a stop process graph - /// If there are no head nodes in the stop graph, queue the start jobs - /// @param p pointer to vector of processes in the requested state; caller guarantees that this is not null - void queueStopJobs(const std::vector* p); + /// @brief Creates one ProcessInfoNode per process and adds it to the dependency graph. + /// @param num_processes The number of processes in this process group. + inline void createProcessInfoNodes(uint32_t num_processes); - /// @brief Queue the jobs to kick-off a start process graph - /// Queue the head nodes; if there are none, graph is complete so set state and pending event accordingly - void queueStartJobs(); + /// @brief Creates one RunTarget node per configured ProcessGroupState and wires it to depend + /// on the processes listed for that state. + /// @param pg_name The identifier of the process group. + inline void createRunTargetNodes(IdentifierHash pg_name); - /// @brief Initializes ProcessInfoNodes for a process group. - /// Creates ProcessInfoNode instances for the specified number of processes - /// and initializes them. - /// @param num_processes Number of processes to initialize nodes for. - inline void createProcessInfoNodes(uint32_t num_processes); + /// @return The index of the RunTarget node for @p pg_state, or -1 if not found. + int32_t getRunTargetIndex(IdentifierHash pg_state) const; - /// @brief Creates successor lists for nodes based on dependencies. - /// For each node in the graph, creates successor lists based on dependencies - /// retrieved from the ProcessGroupManager. - /// @param pg_name The name of the process group. + /// @brief Reads process dependencies from the configuration and adds the corresponding + /// edges to the dependency graph. + /// @param pg_name The identifier of the process group. inline void createSuccessorLists(IdentifierHash pg_name); - /// @brief Counts executable nodes based on the start condition. - /// Counts nodes that are executable based on the `start` flag. - /// @param start Indicates whether to count executable nodes for starting (true) or stopping (false). - /// @return Number of executable nodes. - inline uint32_t countExecutableNodes(bool start); - - /// @brief Queues the head nodes for execution. - /// - /// This method initiates the process of queuing head nodes for execution. - /// It identifies nodes that are eligible to be executed based on their - /// current state and adds them to the execution queue for further processing. - inline void queueHeadNodesForExecution(); - - /// @brief Attempts to queue a specified node for execution. - /// - /// This method tries to queue the provided ProcessInfoNode for execution. - /// If the node meets the necessary conditions (e.g., state, dependencies), - /// it will be added to the execution queue. If the node cannot be queued, - /// appropriate actions or logging may occur. - /// - /// @param node A shared pointer to the ProcessInfoNode to be queued. - inline void tryQueueNode(const std::shared_ptr& node); - - /// @brief Handles the execution of transitions within the graph. - /// - /// This method is responsible for managing the execution of transitions - /// between different states in the graph. It may involve updating states, - /// processing nodes that are transitioning, and ensuring that transitions - /// occur smoothly according to the defined rules. - /// - /// This function should be called when a transition in the graph is detected - /// and needs to be executed. - inline void handleTransitionExecution(); - - /// @brief Handles the execution when the graph is not in transition. - /// - /// This method deals with the execution of nodes when the graph is in a - /// stable state (i.e., not transitioning). It processes the nodes that are - /// ready to be executed without changing the state of the graph. The - /// current_state parameter indicates the state of the graph before the - /// execution begins, which may be used to determine appropriate actions. - /// - /// @param current_state The current state of the graph before execution. + /// @brief Pushes the given task onto the worker queue while the graph is in transition. + /// Retries on timeout. + /// @param task The task to enqueue. + inline void tryQueueNode(ComponentTask task); + + /// @brief Every node that is ready to execute is either executed in place (RunTarget) or queued for execution + /// (ProcessInfoNode). + void queueReadyNodes(); + + /// @brief Executes a RunTarget's activation/deactivation in place + /// @details Since a RunTarget is a virtual node with no work to do + /// and reports its completion to the current transition immediately. + void updateRunTargetInPlace(RunTarget& run_target, ComponentTaskType task_type); + + /// @brief Common tail of a transition that finished without error: moves the graph to + /// kSuccess, posts kSetStateSuccess, and reports initial-state-transition success if this + /// was the initial transition. + void finalizeTransitionSuccess(); + + /// @brief Finalizes a failed or cancelled transition after the last in-flight job + /// completes. Moves the graph state to kUndefinedState and posts the appropriate event. + /// @param current_state The graph state when the last job completed (not kInTransition). inline void handleNonTransitionExecution(GraphState current_state); /// @brief The process group index uint32_t pg_index_; - /// @brief Nodes for all unique processes in this process group. - NodeList nodes_; + /// @brief Number of jobs that have been queued but are not yet executed + int32_t jobs_in_progress_{0}; - /// @brief Number of nodes left to execute. - std::atomic_uint32_t nodes_to_execute_{0}; + /// @brief Nodes for all unique processes in this process group, plus a virtual RunTarget node + /// per configured ProcessGroupState. + DependencyGraph> nodes_; - /// @brief Number of nodes that have been queued but are not yet executed - std::atomic_int32_t nodes_in_flight_{0}; + /// @brief Maps a ProcessGroupState name to the index of its RunTarget node in @c nodes_. + std::vector> run_targets_; - /// @brief Indicates whether the graph is starting. - std::atomic_bool starting_{false}; + /// @brief Builder for creating the transition object for the current state transition. + TransitionBuilder> transition_builder_; + + /// @brief The currently active transition or nullptr before the first one starts. + Transition>* current_transition_{nullptr}; /// @brief Current state of the graph. - std::atomic state_{GraphState::kSuccess}; + GraphState state_{GraphState::kSuccess}; /// @brief Graph semaphore for synchronization. /// @deprecated Not required when Control Client handler is implemented, to be removed @@ -412,21 +389,17 @@ class Graph final { /// @brief the requested (target) Process Group State ProcessGroupStateID requested_state_{}; - /// @brief Mutex protecting concurrent access to requested_state_.pg_state_name_ + /// @brief Mutex protecting concurrent access to requested_state_.pg_state_name_. mutable std::mutex requested_state_mutex_{}; - /// @brief Mutex protecting new transitions from interfering with concluding transitions. - /// This enforces that, when a transition has completed successfully, it cannot then be cancelled. - std::shared_mutex transition_completion_mutex_; - /// @brief Pointer to the ProcessGroupManager. ProcessGroupManager* pgm_; - /// @brief The last state manager to control this process group + /// @brief The state manager node for this process group ControlClientID last_state_manager_; /// @brief The last execution error set on an unexpected termination - std::atomic_uint32_t last_execution_error_; + uint32_t last_execution_error_; /// @brief Set the true if this is the MainPG and this is the initial state transition bool is_initial_state_transition_{false}; @@ -435,10 +408,10 @@ class Graph final { IdentifierHash pending_state_{""}; /// @brief Any pending event to report - std::atomic event_{ControlClientCode::kNotSet}; + ControlClientCode event_{ControlClientCode::kNotSet}; /// @brief Reason that tha graph was aborted - std::atomic abort_code_{ControlClientCode::kNotSet}; + ControlClientCode abort_code_{ControlClientCode::kNotSet}; /// @brief The message to send when a transition is cancelled ControlClientMessage cancel_message_; @@ -448,12 +421,14 @@ class Graph final { /// @brief Stores the timestamp based on the system clock when starting a request std::chrono::time_point request_start_time_; -}; -} // namespace lcm + score::cpp::stop_source stop_source_; +}; } // namespace internal +} // namespace lcm + } // namespace score #endif /// GRAPH_HPP_INCLUDED diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/mock_termination_callback.hpp b/score/launch_manager/src/daemon/src/process_group_manager/details/mock_component_controller.hpp similarity index 61% rename from score/launch_manager/src/daemon/src/process_group_manager/details/mock_termination_callback.hpp rename to score/launch_manager/src/daemon/src/process_group_manager/details/mock_component_controller.hpp index 904ffe040..501e1b759 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/mock_termination_callback.hpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/mock_component_controller.hpp @@ -10,21 +10,22 @@ * * SPDX-License-Identifier: Apache-2.0 ********************************************************************************/ -#ifndef MOCK_TERMINATION_CALLBACK_HPP_INCLUDED -#define MOCK_TERMINATION_CALLBACK_HPP_INCLUDED +#ifndef MOCK_COMPONENT_CONTROLLER_HPP_INCLUDED +#define MOCK_COMPONENT_CONTROLLER_HPP_INCLUDED #include "score/mw/launch_manager/process_group_manager/details/safe_process_map.hpp" #include -namespace score::lcm::internal +namespace score::mw::lifecycle::internal { -class MockTerminationCallback : public ITerminationCallback +class MockComponentController : public IComponentController { public: - MOCK_METHOD(void, terminated, (int32_t process_status), (override)); + MOCK_METHOD(void, doWork, (ComponentTask && task), (override)); + MOCK_METHOD(void, terminated, (IComponent & component, int32_t status), (override)); }; -} // namespace score::lcm::internal +} // namespace score::mw::lifecycle::internal -#endif // MOCK_TERMINATION_CALLBACK_HPP_INCLUDED +#endif // MOCK_COMPONENT_CONTROLLER_HPP_INCLUDED diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/os_handler.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/os_handler.cpp index f57b1dc5d..096f3ae1d 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/os_handler.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/os_handler.cpp @@ -31,7 +31,7 @@ void OsHandler::run(void) if (result.has_value() && result.value() > 0) { - if (score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kInsertionError == + if (score::lcm::internal::SafeProcessMapReturnType::kInsertionError == safe_process_map_.findTerminated(result.value(), wait_status)) { LM_LOG_ERROR() << "No more resources available to track process with PID " << result.value() diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/oshandler_UT.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/oshandler_UT.cpp index c9c0735ca..5dc150889 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/oshandler_UT.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/oshandler_UT.cpp @@ -21,8 +21,9 @@ #include "score/mw/launch_manager/process_group_manager/details/os_handler.hpp" #include "score/mw/launch_manager/common/constants.hpp" +#include "score/mw/launch_manager/process_group_manager/details/mock_component_controller.hpp" +#include "score/mw/launch_manager/process_group_manager/details/mock_component.hpp" #include "score/mw/launch_manager/process_group_manager/details/safe_process_map.hpp" -#include "score/mw/launch_manager/process_group_manager/details/mock_termination_callback.hpp" #include "score/os/mocklib/sys_wait_mock.h" using namespace testing; @@ -42,23 +43,25 @@ class OsHandlerTest : public ::testing::Test static constexpr uint32_t kCapacity = 32U; - SafeProcessMap process_map_{kCapacity}; + MockComponentController ccontroller_; + MockComponent component_; + SafeProcessMap process_map_{kCapacity, ccontroller_}; score::os::MockGuard sys_wait_mock_; std::unique_ptr sut_; }; TEST_F(OsHandlerTest, WaitReturnsProcessId_FindTerminatedIsCalled) { - RecordProperty("Description", - "When sys_wait returns a valid pid, OsHandler calls findTerminated and the termination callback " - "is invoked."); + RecordProperty( + "Description", + "When sys_wait returns a valid pid, OsHandler calls findTerminated and the termination callback " + "is invoked."); // given — insert a callback for pid 1000 - NiceMock callback; - process_map_.insertIfNotTerminated(1000, &callback); + process_map_.insertIfNotTerminated(1000, &component_); constexpr int32_t kExitStatus = 42; - EXPECT_CALL(callback, terminated(kExitStatus)).Times(AtLeast(1)); + EXPECT_CALL(ccontroller_, terminated(_, kExitStatus)).Times(AtLeast(1)); // sys_wait returns pid 1000 once, then blocks with error EXPECT_CALL(*sys_wait_mock_, wait(_)) @@ -79,8 +82,7 @@ TEST_F(OsHandlerTest, WaitReturnsError_OsHandlerSleepsAndDoesNotCallFindTerminat RecordProperty("Description", "When sys_wait returns an error, OsHandler sleeps and does not call findTerminated."); // given — insert a callback that should NOT be invoked - StrictMock callback; - process_map_.insertIfNotTerminated(2000, &callback); + process_map_.insertIfNotTerminated(2000, &component_); EXPECT_CALL(*sys_wait_mock_, wait(_)) .WillRepeatedly(Return(score::cpp::unexpected(score::os::Error::createFromErrno(ECHILD)))); @@ -98,7 +100,7 @@ TEST_F(OsHandlerTest, WaitReturnsZeroPid_OsHandlerSleepsAndDoesNotCallFindTermin RecordProperty("Description", "When sys_wait returns pid 0, OsHandler sleeps and does not call findTerminated."); // given - StrictMock callback; + StrictMock callback; process_map_.insertIfNotTerminated(3000, &callback); EXPECT_CALL(*sys_wait_mock_, wait(_)).WillRepeatedly(Return(score::cpp::expected{0})); @@ -113,10 +115,11 @@ TEST_F(OsHandlerTest, WaitReturnsZeroPid_OsHandlerSleepsAndDoesNotCallFindTermin TEST_F(OsHandlerTest, WaitReturnsProcessIdBeforeRegistration_LaterRegistrationReceivesStoredTermination) { - RecordProperty("Description", - "Covers the race where a child terminates before the process has been registered in " - "SafeProcessMap. OsHandler sees the pid first, stores the terminated state, and a later " - "insertIfNotTerminated call must immediately deliver the stored exit status to the callback."); + RecordProperty( + "Description", + "Covers the race where a child terminates before the process has been registered in " + "SafeProcessMap. OsHandler sees the pid first, stores the terminated state, and a later " + "insertIfNotTerminated call must immediately deliver the stored exit status to the callback."); // given // Simulate the OS reporting that pid 4000 has already exited. @@ -147,26 +150,28 @@ TEST_F(OsHandlerTest, WaitReturnsProcessIdBeforeRegistration_LaterRegistrationRe // Register the process after the termination has already been observed. // insertIfNotTerminated must detect the previously stored termination, return 1, and invoke the // callback immediately with the saved exit status instead of creating a new live entry. - StrictMock callback; - EXPECT_CALL(callback, terminated(99)).Times(1); - EXPECT_EQ(process_map_.insertIfNotTerminated(4000, &callback), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); + EXPECT_CALL(ccontroller_, terminated(_, 99)).Times(1); + EXPECT_EQ( + process_map_.insertIfNotTerminated(4000, &component_), + score::lcm::internal::SafeProcessMapReturnType::kYield); sut_.reset(); } TEST_F(OsHandlerTest, WaitReturnsUnknownPidWhenMapIsFull_OutOfResourcesPathDoesNotNotifyCallbacks) { - RecordProperty("Description", - "When sys_wait reports an unknown pid and the map is full, OsHandler takes the out-of-resources " - "path without notifying tracked callbacks."); + RecordProperty( + "Description", + "When sys_wait reports an unknown pid and the map is full, OsHandler takes the out-of-resources " + "path without notifying tracked callbacks."); // given - StrictMock callbacks[kCapacity]; + StrictMock callbacks[kCapacity]; for (uint32_t i = 0; i < kCapacity; ++i) { - ASSERT_EQ(process_map_.insertIfNotTerminated(static_cast(i + 1U), &callbacks[i]), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kOk); + ASSERT_EQ( + process_map_.insertIfNotTerminated(static_cast(i + 1U), &callbacks[i]), + score::lcm::internal::SafeProcessMapReturnType::kOk); } EXPECT_CALL(*sys_wait_mock_, wait(_)) @@ -186,15 +191,15 @@ TEST_F(OsHandlerTest, WaitReturnsUnknownPidWhenMapIsFull_OutOfResourcesPathDoesN TEST_F(OsHandlerTest, WaitReturnsErrorThenProcessId_HandlerRecoversAndInvokesCallback) { - RecordProperty("Description", - "When sys_wait first returns an error and then a valid pid, OsHandler resumes processing and " - "invokes the callback."); + RecordProperty( + "Description", + "When sys_wait first returns an error and then a valid pid, OsHandler resumes processing and " + "invokes the callback."); // given - NiceMock callback; - process_map_.insertIfNotTerminated(5000, &callback); + process_map_.insertIfNotTerminated(5000, &component_); - EXPECT_CALL(callback, terminated(7)).Times(AtLeast(1)); + EXPECT_CALL(ccontroller_, terminated(_, 7)).Times(AtLeast(1)); EXPECT_CALL(*sys_wait_mock_, wait(_)) .WillOnce(Return(score::cpp::unexpected(score::os::Error::createFromErrno(ECHILD)))) diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/process_group_manager.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/process_group_manager.cpp index 40c94ba54..38be63e8d 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/process_group_manager.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/process_group_manager.cpp @@ -17,6 +17,7 @@ #include #include "score/mw/launch_manager/common/log.hpp" +#include "score/mw/launch_manager/process_group_manager/details/process_monitor.hpp" #include "score/mw/launch_manager/process_group_manager/ialive_monitor_thread.hpp" #include "score/mw/launch_manager/process_group_manager/process_group_manager.hpp" @@ -37,9 +38,10 @@ void ProcessGroupManager::cancel() my_signal_handler(SIGTERM); } -ProcessGroupManager::ProcessGroupManager(std::unique_ptr alive_monitor_thread, - std::shared_ptr recovery_client, - std::unique_ptr process_state_notifier) +ProcessGroupManager::ProcessGroupManager( + std::unique_ptr alive_monitor_thread, + std::shared_ptr recovery_client, + std::unique_ptr process_state_notifier) : configuration_(), process_interface_(), process_map_(nullptr), @@ -56,19 +58,6 @@ ProcessGroupManager::ProcessGroupManager(std::unique_ptr al // { return reloadConfiguration(act, updateCtx, IdentifierHash(swc.c_str())); }) { } - -void ProcessGroupManager::setLaunchManagerConfiguration(const OsProcess* launch_manager_configuration) -{ - if (launch_manager_config_) - { - LM_LOG_WARN() << "More than one launch manager configured! (ignoring)"; - } - else - { - launch_manager_config_ = launch_manager_configuration; - } -} - #ifdef USE_NEW_CONFIGURATION bool ProcessGroupManager::initialize(const Config& config) #else @@ -112,18 +101,18 @@ bool ProcessGroupManager::initialize() return false; } - if (launch_manager_config_ && - OsalReturnType::kFail == IProcess::setSchedulingAndSecurity(launch_manager_config_->startup_config_)) - { - return false; - } - return true; } void ProcessGroupManager::deinitialize() { // ucm_polling_thread_.stopPolling(); + if (event_queue_) + { + event_queue_->stop(); + } + os_handler_.reset(); + process_monitor_.reset(); alive_monitor_thread_->stop(); configuration_.deinitialize(); process_groups_.clear(); @@ -144,9 +133,10 @@ inline bool ProcessGroupManager::initializeControlClientHandler() ControlClientChannel::nudgeControlClientHandler_ = nullptr; char shm_name[static_cast(score::lcm::internal::ProcessLimits::maxLocalBuffSize)]; - static_cast(snprintf(shm_name, - static_cast(score::lcm::internal::ProcessLimits::maxLocalBuffSize), - "/_nudge~._.~me_")); // random name + static_cast(snprintf( + shm_name, + static_cast(score::lcm::internal::ProcessLimits::maxLocalBuffSize), + "/_nudge~._.~me_")); // random name int fd = shm_open(shm_name, O_CREAT | O_EXCL | O_RDWR, 0U); if (fd >= 0) @@ -166,7 +156,6 @@ inline bool ProcessGroupManager::initializeControlClientHandler() return false; } - if (osal::IpcCommsSync::control_client_handler_nudge_fd == fd2) { void* buf = mmap(NULL, sizeof(osal::Semaphore), PROT_WRITE, MAP_SHARED, fd2, 0); @@ -179,8 +168,8 @@ inline bool ProcessGroupManager::initializeControlClientHandler() // coverity[cert_mem52_cpp_violation:FALSE] The allocated memory is checked by the containing if // statement. const auto osal_result = ControlClientChannel::nudgeControlClientHandler_->init(0U, true); - SCORE_LANGUAGE_FUTURECPP_ASSERT_MESSAGE(osal_result == OsalReturnType::kSuccess, - "ControlClientChannel semaphore init failed"); + SCORE_LANGUAGE_FUTURECPP_ASSERT_MESSAGE( + osal_result == OsalReturnType::kSuccess, "ControlClientChannel semaphore init failed"); result = true; } @@ -217,11 +206,13 @@ inline bool ProcessGroupManager::initializeProcessGroups() for (const auto& pg_name : *pg_list) { uint32_t num_processes = configuration_.getNumberOfOsProcesses(pg_name).value_or(0); + const auto* states = configuration_.getListOfProcessGroupStates(pg_name).value_or(nullptr); + const uint32_t num_run_targets = states ? static_cast(states->size()) : 0U; if (static_cast(total_processes_) + num_processes <= static_cast(score::lcm::internal::ProcessLimits::kMaxProcesses)) { - process_groups_.push_back(std::make_shared(num_processes, this)); + process_groups_.push_back(std::make_shared(num_processes + num_run_targets, this)); total_processes_ += num_processes; } else @@ -256,15 +247,31 @@ inline bool ProcessGroupManager::initializeProcessGroups() inline void ProcessGroupManager::createProcessComponentsObjects() { + LM_LOG_DEBUG() << "Creating component event queue..."; + event_queue_ = std::make_unique(total_processes_); + + if (recovery_client_) + { + recovery_client_->setRecoveryRequestCallback([this](const IdentifierHash& process_identifier) { + static_cast(event_queue_->push(SupervisionFailure{process_identifier})); + }); + } + + LM_LOG_DEBUG() << "Creating process monitor..."; + process_monitor_ = std::make_unique(*event_queue_); + LM_LOG_DEBUG() << "Creating Safe Process Map with" << total_processes_ << "entries"; - process_map_ = std::make_shared(total_processes_); + process_map_ = std::make_shared(total_processes_, *process_monitor_); + + LM_LOG_DEBUG() << "Creating OS handler..."; + os_handler_ = std::make_unique(*process_map_); LM_LOG_DEBUG() << "Creating job queue with capacity" << static_cast(ProcessLimits::kMaxProcesses); worker_jobs_ = std::make_shared(); LM_LOG_DEBUG() << "Creating worker threads..."; - worker_threads_ = std::make_unique>( - worker_jobs_, static_cast(ProcessLimits::kNumWorkerThreads)); + worker_threads_ = std::make_unique>( + worker_jobs_, static_cast(ProcessLimits::kNumWorkerThreads), *process_monitor_); } inline void ProcessGroupManager::initializeGraphNodes() @@ -291,27 +298,32 @@ bool ProcessGroupManager::run() // debug messages. << (static_cast(clock()) / (static_cast(CLOCKS_PER_SEC) / 1000.0)) << "ms"; - OsHandler os_handler(*process_map_); - bool result = startInitialTransition(); + bool overflow_logged = false; if (result) while (!em_cancelled.load()) { - // Wait for something to happen... - const auto osal_result = - ControlClientChannel::nudgeControlClientHandler_->timedWait(std::chrono::milliseconds(100)); + // Wait for a graph-relevant event (activation/deactivation completion or + // unexpected termination). All Graph state mutations happen here, on the main thread. + if (event_queue_->waitForEvents(std::chrono::milliseconds(50))) + { + processComponentEvents(); + } - SCORE_LANGUAGE_FUTURECPP_ASSERT_MESSAGE( - osal_result == OsalReturnType::kSuccess || osal_result == OsalReturnType::kTimeout, - "ControlClientChannel semaphore wait failed"); + if (event_queue_->getOverflow() && !overflow_logged) + { + LM_LOG_FATAL() << "ComponentEventQueue overflow - one or more events were lost"; + overflow_logged = true; + + // fire the watchdog here once available ... + } for (auto pg : process_groups_) { controlClientHandler(*pg); processGroupHandler(*pg); } - recoveryActionHandler(); } allProcessGroupsOff(); @@ -319,6 +331,25 @@ bool ProcessGroupManager::run() return result; } +void ProcessGroupManager::processComponentEvents() +{ + // Single-graph assumption: PGM creates one ProcessMonitor bound to the first (only) graph, so + // every event always applies to it. Multi-graph routing is deferred to a future revision. + Graph& graph = *process_groups_.front(); + + while (auto event = event_queue_->getNextEvent()) + { + if (const auto* supervision_failure = std::get_if(&*event)) + { + handleRecoveryRequest(supervision_failure->process_identifier); + } + else + { + graph.handleComponentEvent(*event); + } + } +} + inline bool ProcessGroupManager::startInitialTransition() { bool result = false; @@ -345,67 +376,70 @@ inline bool ProcessGroupManager::startInitialTransition() inline void ProcessGroupManager::allProcessGroupsOff() { - // Lambda to wait for process group transitions with a timeout - auto waitForStateCompletion = - [](auto& process_groups, GraphState state_to_be_completed, int32_t max_wait_ms) -> bool { - int32_t counter = max_wait_ms; - constexpr int32_t sleep_interval_ms = 10; - for (auto& pg : process_groups) + // Wait for process group states to change while actively draining shutdown events. + // SupervisionFailure is intentionally ignored here so recovery transitions do not + // fight the forced transition to Off. + auto waitForStateCompletion = [this](GraphState state_to_be_completed, int32_t max_wait_ms) -> bool { + constexpr int32_t kSleepIntervalMs = 10; + + Graph& graph = *process_groups_.front(); + auto has_state = [&graph, state_to_be_completed]() { + return graph.getState() == state_to_be_completed; + }; + + int32_t remaining_ms = max_wait_ms; + while (has_state() && (remaining_ms > 0)) { - while (pg->getState() == state_to_be_completed && counter > 1) + static_cast(event_queue_->waitForEvents(std::chrono::milliseconds(kSleepIntervalMs))); + while (auto event = event_queue_->getNextEvent()) { - counter -= sleep_interval_ms; - std::this_thread::sleep_for(std::chrono::milliseconds(sleep_interval_ms)); + if (std::holds_alternative(*event)) + { + continue; + } + graph.handleComponentEvent(*event); } + + remaining_ms -= kSleepIntervalMs; } - return counter > 0; - }; - // First, cancel any pending transitions - LM_LOG_DEBUG() << "Cancel all process group transitions"; + return !has_state(); + }; - for (auto& pg : process_groups_) + Graph& graph = *process_groups_.front(); + // First, check if we're already transitioning to Off - if so, no need to cancel + if (!graph.isTransitioningToOff()) { - pg->cancel(); - } + // Cancel any pending transitions that are not going to Off + LM_LOG_DEBUG() << "Cancel all process group transitions"; + graph.cancel(); - // Wait for cancellation to complete (max 2 seconds) - LM_LOG_DEBUG() << "Wait for process group cancellations"; + // Wait for cancellation to complete + LM_LOG_DEBUG() << "Wait for process group cancellations"; + if (!waitForStateCompletion(GraphState::kCancelled, 2000)) + { + LM_LOG_ERROR() << "NOTE: Cancellation timed out"; + } - if (!waitForStateCompletion(process_groups_, GraphState::kCancelled, 2000)) - { - LM_LOG_ERROR() << "NOTE: Cancellation timed out"; + // Start transitioning all process groups to the "Off" state + LM_LOG_DEBUG() << "Start transitioning process groups to Off state"; + (void)graph.startTransitionToOffState(); } - - // Start transitioning all process groups to the "Off" state - LM_LOG_DEBUG() << "Start transitioning process groups to Off state"; - - for (auto& pg : process_groups_) + else { - (void)pg->startTransitionToOffState(); + LM_LOG_DEBUG() << "Already transitioning to Off state, skipping cancellation"; } - // Wait for the transition to complete (max 1 second) LM_LOG_DEBUG() << "Wait for all process groups to complete the transition"; - - if (!waitForStateCompletion(process_groups_, GraphState::kInTransition, 1000)) + if (!waitForStateCompletion(GraphState::kInTransition, 1000)) { LM_LOG_ERROR() << "NOTE: Transition to Off state timed out"; worker_threads_->stop(); + for (auto& pg : process_groups_) { - for (auto& node : pg->getNodes()) - { - osal::ProcessID pid = node->getPid(); - if (pid > 0) - { - // forceTermination already handles errors appropriately, so we can ignore its result. - static_cast(process_interface_.forceTermination(pid)); - } - } + pg->forceKillProcesses(); } - while (wait(NULL) != -1 || errno == EINTR) - ; } } @@ -449,8 +483,8 @@ inline void ProcessGroupManager::controlClientResponses(Graph& pg) bool ProcessGroupManager::sendResponse(ControlClientMessage msg) { - auto pin = getProcessInfoNode(msg.originating_control_client_.process_group_index_, - msg.originating_control_client_.process_index_); + auto pin = getProcessInfoNode( + msg.originating_control_client_.process_group_index_, msg.originating_control_client_.process_index_); bool ret = true; if (pin) @@ -476,131 +510,121 @@ bool ProcessGroupManager::sendResponse(ControlClientMessage msg) inline void ProcessGroupManager::controlClientRequests(Graph& pg) { - // Perform the function of Control Client handler by polling for state transition requests - // from any State Managers in this process group. - if (pg.getNodes().empty()) + const auto* control_client = pg.findControlClient(); + + if (!control_client) { return; } - for (auto node = pg.getNodes()[0U]; node; node = node->getNextStateManager()) + + ControlClientChannelP scc = control_client->getControlClientChannel(); + + if (!scc) { - ControlClientChannelP scc = node->getControlClientChannel(); + return; + } - if (scc) + if (scc->getRequest()) + { + // Fill in some routing details + scc->request().originating_control_client_.process_group_index_ = + static_cast(pg.getProcessGroupIndex() & 0xFFFFU); + scc->request().originating_control_client_.process_index_ = + static_cast(control_client->getIndex() & 0xFFFFU); + + LM_LOG_DEBUG() << "ProcessGroupManager::ControlClientHandler: got request" + << scc->toString(scc->request().request_or_response_) << "(" + << static_cast(scc->request().request_or_response_) << ") re state" + << scc->request().process_group_state_.pg_state_name_ << "of PG" + << scc->request().process_group_state_.pg_name_; + + // Now process the request + switch (scc->request().request_or_response_) { - if (scc->getRequest()) - { - // Fill in some routing details - scc->request().originating_control_client_.process_group_index_ = - static_cast(pg.getProcessGroupIndex() & 0xFFFFU); - scc->request().originating_control_client_.process_index_ = - static_cast(node->getNodeIndex() & 0xFFFFU); - - LM_LOG_DEBUG() << "ProcessGroupManager::ControlClientHandler: got request" - << scc->toString(scc->request().request_or_response_) << "(" - << static_cast(scc->request().request_or_response_) << ") re state" - << scc->request().process_group_state_.pg_state_name_ << "of PG" - << scc->request().process_group_state_.pg_name_; - - // Now process the request - switch (scc->request().request_or_response_) - { - case ControlClientCode::kSetStateRequest: - processStateTransition(scc); - break; + case ControlClientCode::kSetStateRequest: + processStateTransition(scc); + break; - case ControlClientCode::kGetExecutionErrorRequest: - processGetExecutionError(scc); - break; + case ControlClientCode::kGetExecutionErrorRequest: + processGetExecutionError(scc); + break; - case ControlClientCode::kGetInitialMachineStateRequest: - processGetInitialMachineStateTransitionResult(scc); - break; + case ControlClientCode::kGetInitialMachineStateRequest: + processGetInitialMachineStateTransitionResult(scc); + break; - case ControlClientCode::kValidateProcessGroupState: - processValidateFunctionStateID(scc); - break; + case ControlClientCode::kValidateProcessGroupState: + processValidateFunctionStateID(scc); + break; - default: // Error, this is not a recognised request! - scc->request().request_or_response_ = ControlClientCode::kInvalidRequest; - break; - } - scc->acknowledgeRequest(); - } + default: // Error, this is not a recognised request! + scc->request().request_or_response_ = ControlClientCode::kInvalidRequest; + break; + } + scc->acknowledgeRequest(); + } - // now process deferred requests for initial state transition results - if (ControlClientCode::kInitialMachineStateNotSet != initial_state_transition_result_ && - scc->initial_result_count_) - { - ControlClientMessage msg; - msg.request_or_response_ = initial_state_transition_result_; - msg.originating_control_client_ = scc->request().originating_control_client_; - if (scc->sendResponse(msg)) - { - scc->initial_result_count_--; - } - else - { - ControlClientChannel::nudgeControlClientHandler(); // will need to try again - } - } + // now process deferred requests for initial state transition results + if (ControlClientCode::kInitialMachineStateNotSet != initial_state_transition_result_ && scc->initial_result_count_) + { + ControlClientMessage msg; + msg.request_or_response_ = initial_state_transition_result_; + msg.originating_control_client_ = scc->request().originating_control_client_; + if (scc->sendResponse(msg)) + { + scc->initial_result_count_--; + } + else + { + ControlClientChannel::nudgeControlClientHandler(); // will need to try again } } } -inline void ProcessGroupManager::recoveryActionHandler() +inline void ProcessGroupManager::handleRecoveryRequest(const IdentifierHash& process_identifier) { - if (!recovery_client_) + auto pg = getProcessGroupByProcessId(process_identifier); + + if (nullptr == pg) { + LM_LOG_ERROR() << "handleRecoveryRequest: Unknown process " << process_identifier; return; } - for (auto recovery_request = recovery_client_->getNextRequest(); recovery_request.has_value(); - recovery_request = recovery_client_->getNextRequest()) - { - auto pg = getProcessGroupByProcessId(*recovery_request); - - if (nullptr == pg) - { - LM_LOG_ERROR() << "recoveryActionHandler: Unknown process " << *recovery_request; - continue; - } - - const IdentifierHash old_state = pg->getProcessGroupState(); - const IdentifierHash recovery_state = configuration_.getNameOfRecoveryState(pg->getProcessGroupName()); - const GraphState graph_state = pg->getState(); + const IdentifierHash old_state = pg->getProcessGroupState(); + const IdentifierHash recovery_state = configuration_.getNameOfRecoveryState(pg->getProcessGroupName()); + const GraphState graph_state = pg->getState(); - LM_LOG_DEBUG() << "recoveryActionHandler: Processing recovery request for PG " << *recovery_request - << " to state " << recovery_state; + LM_LOG_DEBUG() << "handleRecoveryRequest: Processing recovery request for PG " << process_identifier << " to state " + << recovery_state; - if (GraphState::kInTransition == graph_state) - { - if (old_state != recovery_state) - { - // Cancel current transition and start new one - (void)pg->setPendingState(recovery_state); - pg->setRequestStartTime(); - pg->cancel(); - controlClientResponses(*pg); - } - else - { - // Already in transition to the requested state - LM_LOG_DEBUG() << "recoveryActionHandler: Already transitioning to same state"; - } - } - else if (GraphState::kSuccess == graph_state && old_state == recovery_state) + if (GraphState::kInTransition == graph_state) + { + if (old_state != recovery_state) { - // Already in the requested state - LM_LOG_DEBUG() << "recoveryActionHandler: Already in requested state"; + // Cancel current transition and start new one + (void)pg->setPendingState(recovery_state); + pg->setRequestStartTime(); + pg->cancel(); + controlClientResponses(*pg); } else { - // Start new state transition - (void)pg->setPendingState(recovery_state); - pg->setRequestStartTime(); + // Already in transition to the requested state + LM_LOG_DEBUG() << "handleRecoveryRequest: Already transitioning to same state"; } } + else if (GraphState::kSuccess == graph_state && old_state == recovery_state) + { + // Already in the requested state + LM_LOG_DEBUG() << "handleRecoveryRequest: Already in requested state"; + } + else + { + // Start new state transition + (void)pg->setPendingState(recovery_state); + pg->setRequestStartTime(); + } } inline void ProcessGroupManager::processStateTransition(ControlClientChannelP scc) @@ -651,6 +675,7 @@ inline void ProcessGroupManager::processStateTransition(ControlClientChannelP sc // get state transition start time stamp pg->setRequestStartTime(); } + pg->updateCancelMessage(); pg->setStateManager(scc->request().originating_control_client_); } } @@ -765,16 +790,14 @@ void ProcessGroupManager::setInitialStateTransitionResult(ControlClientCode resu ControlClientChannel::nudgeControlClientHandler(); } -std::shared_ptr ProcessGroupManager::getProcessInfoNode(uint32_t pg_index, uint32_t process_index) +ProcessInfoNode* ProcessGroupManager::getProcessInfoNode(uint32_t pg_index, uint32_t process_index) { - std::shared_ptr result = nullptr; - if (pg_index < process_groups_.size()) { - result = process_groups_[pg_index]->getProcessInfoNode(process_index); + return process_groups_[pg_index]->getProcessInfoNode(process_index); } - return result; + return nullptr; } std::shared_ptr ProcessGroupManager::getProcessGroupByProcessId(const IdentifierHash& process_id) diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.cpp index ca51d3faa..9547d611d 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.cpp @@ -11,13 +11,12 @@ * SPDX-License-Identifier: Apache-2.0 ********************************************************************************/ -#include - #include "process_info_node.hpp" -#include "graph.hpp" #include "score/mw/launch_manager/common/log.hpp" #include "score/mw/launch_manager/osal/ipc_comms.hpp" -#include "score/mw/launch_manager/process_group_manager/process_group_manager.hpp" +#include "score/mw/launch_manager/process_group_manager/details/safe_process_map.hpp" +#include +#include namespace score { @@ -28,116 +27,76 @@ namespace lcm namespace internal { -void ProcessInfoNode::initNode(Graph* graph, uint32_t index) +ProcessInfoNode::ProcessInfoNode( + const OsProcess* config, + uint32_t index, + ReadyCondition ready_condition, + ReportStateFn report_function, + osal::IProcess* process_interface, + std::shared_ptr process_map) + : terminator_(), + has_semaphore_(false), + process_index_(index), + pid_(0), + status_(0), + process_state_(score::lcm::ProcessState::kIdle), + ready_condition_(ready_condition), + config_(config), + report_state_(std::move(report_function)), + process_interface_(process_interface), + process_map_(std::move(process_map)) { - if (graph) - { - IdentifierHash pg = graph->getProcessGroupName(); - graph_ = graph; - process_index_ = index; - pid_ = 0; - status_ = 0; - process_state_.store(score::lcm::ProcessState::kIdle); - dependencies_ = 0U; - dependency_list_ = nullptr; - dependent_on_running_.clear(); - dependent_on_terminating_.clear(); - start_dependencies_ = 0U; - auto cfg_mgr = graph_->getProcessGroupManager()->getConfiguration(); - config_ = cfg_mgr->getOsProcessConfiguration(pg, index).value_or(nullptr); - if (config_) - { - if (osal::CommsType::kLaunchManager == config_->startup_config_.comms_type_) - { - graph_->getProcessGroupManager()->setLaunchManagerConfiguration(config_); - } - dependency_list_ = cfg_mgr->getOsProcessDependencies(pg, process_index_).value_or(nullptr); - if (dependency_list_) - { - start_dependencies_ = static_cast(dependency_list_->size() & 0xFFFFFFFFUL); - LM_LOG_DEBUG() << "Process" << process_index_ << "has" << start_dependencies_ << "start dependencies"; - } - } - else - { - LM_LOG_ERROR() << "No configuration for process" << process_index_ << "of process group" << pg; - } - } } -bool ProcessInfoNode::constructGraphNode(bool starting) +IComponent::RequestResult ProcessInfoNode::tryReportCompletion(score::lcm::ProcessState new_state) { - bool included = false; - - if (!starting) + std::optional desired_state = std::nullopt; + switch (ready_condition_) { - std::ptrdiff_t count = - std::count_if(dependent_on_running_.begin(), dependent_on_running_.end(), [](auto& d) -> bool { - return d->process_state_ == score::lcm::ProcessState::kRunning; - }); - - if (count > 0L) - stop_dependencies_ = static_cast(count & 0xFFFFFFFFL); - else - stop_dependencies_ = 0U; - - LM_LOG_DEBUG() << "Stop Dependencies:" << stop_dependencies_; - - dependencies_ = stop_dependencies_; - // A stop node should be inserted for processes not in the idle or terminated state where: - // The process is not listed in the requested state - included = !((getState() == score::lcm::ProcessState::kIdle) || - (getState() == score::lcm::ProcessState::kTerminated)) && - !in_requested_state_; + case ReadyCondition::kRunning: + desired_state = ProcessState::kRunning; + break; + case ReadyCondition::kTerminated: + desired_state = ProcessState::kTerminated; + break; + default: + break; } - else + assert(desired_state.has_value() && "Ready condition is not implemented"); + if (!desired_state.has_value()) { - LM_LOG_DEBUG() << "Start Dependencies:" << start_dependencies_; - dependencies_ = start_dependencies_; - // The process should be started (node inserted) if - // - it is listed, and - // - it isn't already running - included = in_requested_state_ && (getState() != score::lcm::ProcessState::kRunning); - - // Go through the predecessors nodes to check if those are already in the ExecutionState - // that has been configured as part of the execution dependency - if (dependency_list_) - { - for (const auto& dep : *dependency_list_) - { - if (dep.process_state_ == graph_->getNodes()[dep.os_process_index_]->getState()) - { - --dependencies_; - } - } - } + return {IComponent::RequestState::kWaiting}; } - is_included_ = included; - is_head_node_ = included && (dependencies_ == 0U); - - return included; + if (new_state == ProcessState::kFailed) + { + // Didn't reach running or startup + return tryReportError(ComponentError::kErrorBeforeReady); + } + if (new_state == desired_state.value()) + { + return tryReportSuccess(); + } + return {IComponent::RequestState::kWaiting}; } -void ProcessInfoNode::addSuccessorNode(std::shared_ptr& successor_node, - score::lcm::ProcessState dependency) +IComponent::RequestResult ProcessInfoNode::tryReportSuccess() { - if (dependency == score::lcm::ProcessState::kTerminated) + if (!success_returned_.test_and_set()) { - LM_LOG_DEBUG() << "Adding kTerminated for process" << process_index_ << ":" << successor_node->process_index_; - dependent_on_terminating_.push_back(successor_node); + reached_ready_.store(true); + return {RequestState::kSuccess}; } - else if (dependency == score::lcm::ProcessState::kRunning) - { - dependent_on_running_.push_back(successor_node); - LM_LOG_DEBUG() << "Adding kRunning successor for process" << process_index_ << ":" - << successor_node->process_index_; - } - else + return {IComponent::RequestState::kWaiting}; +} + +IComponent::RequestResult ProcessInfoNode::tryReportError(ComponentError error) +{ + if (!success_returned_.test_and_set()) { - // all other dependency types are forbidden! - LM_LOG_ERROR() << "Invalid dependency type for process" << process_index_ << ":" - << static_cast(dependency); + // Activation failed to reach its ready condition. + return score::cpp::make_unexpected(error); } + return {IComponent::RequestState::kWaiting}; } bool ProcessInfoNode::setState(score::lcm::ProcessState new_state) @@ -145,14 +104,15 @@ bool ProcessInfoNode::setState(score::lcm::ProcessState new_state) bool success = true; score::lcm::ProcessState old_state = getState(); - if (score::lcm::ProcessState::kTerminated == new_state || - (new_state == score::lcm::ProcessState::kIdle && old_state == score::lcm::ProcessState::kTerminated)) + if (new_state > old_state || (new_state == old_state && new_state == ProcessState::kIdle)) { - process_state_.store(new_state); + success = process_state_.compare_exchange_strong(old_state, new_state); } - else if (new_state >= old_state) + else if ( + new_state == score::lcm::ProcessState::kIdle && + (old_state == score::lcm::ProcessState::kTerminated || old_state == ProcessState::kFailed)) { - success = process_state_.compare_exchange_strong(old_state, new_state); + process_state_.store(new_state); } else { @@ -163,381 +123,255 @@ bool ProcessInfoNode::setState(score::lcm::ProcessState new_state) score::lcm::ProcessState::kIdle != new_state) { // for a reporting process, report a process state change to PHM - score::lcm::PosixProcess process_info; - process_info.id = config_->process_id_; - process_info.processStateId = new_state; - process_info.processGroupStateId = graph_->getProcessGroupState(); // Note the following system call will not fail by design. // Possible failure modes would be: // a) CLOCK_MONOTONIC is not supported, but we assert that in all systems it is supported - // b) &p.systemClockTimeStamp points outside the accessible address space, but it does not - static_cast(clock_gettime(CLOCK_MONOTONIC, &process_info.systemClockTimestamp)); - // Note that we ignore the return value from QueuePosixProcess. + // b) ×tamp points outside the accessible address space, but it does not + timespec timestamp{}; + static_cast(clock_gettime(CLOCK_MONOTONIC, ×tamp)); + // Note that we ignore the return value. // An error would indicate that PHM is not reading values fast enough from the shared memory; the buffer // over-run should be visible at the PHM side and handled there. If PHM is not responding do we need to handle // this? If PHM terminates state manager will be informed in any case. - static_cast(graph_->getProcessGroupManager()->queuePosixProcess(process_info)); + static_cast(report_state_(config_->process_id_, new_state, timestamp)); } return success; } -void ProcessInfoNode::queueTerminationSuccessorJobs() +void ProcessInfoNode::unblockSync() { - auto processJob = [&](std::shared_ptr successor_node) { - if (successor_node->is_included_ && successor_node->dependencies_ > 0U && --successor_node->dependencies_ == 0U) - { - while (graph_->getState() == GraphState::kInTransition) - { - if (graph_->getProcessGroupManager()->getWorkerJobs()->push(successor_node, kMaxQueueDelay)) - { - graph_->markNodeInFlight(); - break; - } - } - } - }; - - if (graph_->isStarting()) + auto sync = sync_; // take a copy as the pointer otherwise may become invalidated + if (sync) { - for (auto& successor_node : dependent_on_terminating_) - { - processJob(successor_node); - } - } - else if (dependency_list_) // Successors given by our dependencies - { - for (const auto& dependency : *dependency_list_) - { - auto successorNode = graph_->getProcessInfoNode(dependency.os_process_index_); - - if (successorNode->getState() != score::lcm::ProcessState::kTerminated) - { - processJob(successorNode); - } - } + // note that we ignore the return code. The semaphore operation may fail because it could + // be destroyed by another thread + static_cast(sync->send_sync_.post()); } } -void ProcessInfoNode::unexpectedTermination() +IComponent::RequestResult ProcessInfoNode::tryHandleTermination(int32_t process_status) { - LM_LOG_WARN() << "unexpected termination of process" << process_index_ << "pid" << pid_ << "(" - << config_->startup_config_.short_name_ << ")"; - - uint32_t execution_error_code = config_->pgm_config_.execution_error_code_; - auto graph_state = graph_->getState(); - if (GraphState::kSuccess == graph_state) - { - // We were in a defined state, this error needs to be reported to SM - graph_->abort(execution_error_code, ControlClientCode::kFailedUnexpectedTermination); - } - else if (score::lcm::ProcessState::kStarting == getState()) - { - // for graph in any other state, the error will be found elsewhere. But if the graph is in - // transition, and the process status is not yet kRunning, we may want to post on the semaphore - // to save a little waiting time. - auto sync = sync_; // take a copy as the pointer otherwise may become invalidated - if (sync) - { - // note that we ignore the return code. The semaphore operation may fail because it could - // be destroyed by another thread - static_cast(sync->send_sync_.post()); - } - } - else if (score::lcm::ProcessState::kTerminating == getState()) + LM_LOG_DEBUG() << "Process" << process_index_ << "pid" << pid_ << "(" << config_->startup_config_.short_name_ + << ") for node" << this << "terminated with status" << process_status; + status_ = process_status; + IComponent::RequestResult res = {IComponent::RequestState::kWaiting}; + if (has_semaphore_.exchange(false)) { - // prevents a spurious graph_->abort() by recognizing that terminateProcess() - // already owns the retry decision when the state is kTerminating. - // explanation: - // terminateProcess() is already active (kRunning timeout path). The do-while retry loop - // in startProcess() owns the restart/abort decision. Calling graph_->abort() here would - // race with that loop and trigger a spurious abort before retries are exhausted. - // terminated() will post on terminator_, unblocking terminateProcess() — nothing else needed. + // Termination was requested, we don't care if status is not 0 (a SIGKILL will set status to 9) + setState(ProcessState::kTerminated); + unblockSync(); + static_cast(terminator_.post()); } - else if (GraphState::kInTransition == graph_state) + else if (getState() < ProcessState::kRunning) { - // process has started, but graph is still in transition - graph_->abort(execution_error_code, ControlClientCode::kFailedUnexpectedTerminationOnEnter); - } -} + // Defer to the startup thread to handle this + setState(ProcessState::kTerminated); -void ProcessInfoNode::terminated(int32_t process_status) -{ - LM_LOG_DEBUG() << "Child process" << process_index_ << "of" << graph_->getProcessGroupName() << "pid" << pid_ << "(" - << config_->startup_config_.short_name_ << ") for node" << this << "terminated with status" - << process_status; - status_ = process_status; - if (!config_->pgm_config_.is_self_terminating_ || (process_status != 0)) + unblockSync(); + } + else { - // fudge the status if the process is not self-terminating but has still exited - // with zero status: - if (0 == status_) + setState(ProcessState::kTerminated); + if (config_->pgm_config_.is_self_terminating_ && process_status == 0) { - status_ = -1; + // Only valid case for a process to terminate without it being requested + res = tryReportCompletion(ProcessState::kTerminated); } - if (graph_->isStarting()) + else { - unexpectedTermination(); + LM_LOG_WARN() << "unexpected termination of process" << process_index_ << "pid" << pid_ << "(" + << config_->startup_config_.short_name_ << ")" << "( status" << status_ << ")"; + res = score::cpp::make_unexpected(IComponent::ComponentError::kErrorAfterReady); } } - static_cast(setState(score::lcm::ProcessState::kTerminated)); // Cannot fail by design + if (control_client_channel_) { control_client_channel_->releaseParentMapping(); control_client_channel_.reset(); } - // Handle the situation where the graph is stalled waiting for a process to terminate - if (config_->pgm_config_.is_self_terminating_ && dependent_on_terminating_.size()) - { - queueTerminationSuccessorJobs(); - } - // handle the situation where a worker thread is waiting for a process to terminate - if (has_semaphore_.exchange(false)) - { - static_cast(terminator_.post()); - } + + return res; } -void ProcessInfoNode::startProcess() +IComponent::RequestResult ProcessInfoNode::startProcess(score::cpp::stop_token stop_token) { LM_LOG_DEBUG() << "Starting process" << process_index_ << "(" << config_->startup_config_.short_name_ << ") from executable" << config_->startup_config_.executable_path_; - restart_counter_ = config_->pgm_config_.number_of_restart_attempts; - do - { + uint32_t restart_counter = config_->pgm_config_.number_of_restart_attempts; + std::optional error; + for (auto attempts = static_cast(restart_counter); attempts >= 0; attempts--) + { + // setState(kIdle) will fail if the state is: + // - Starting: this would mean we did not set state to kFailed on failure or exit when we successfully launched + assert(getState() != ProcessState::kStarting && "Process state is invalid"); + // - Running: we should already have exited the loop + assert(getState() != ProcessState::kRunning && "Restart attempted even though process is running"); + // - Terminating: A termination is in progress (allowed) + if (!setState(score::lcm::ProcessState::kIdle)) + { + LM_LOG_WARN() << "Starting process" << this << "failed: termination in progress"; + error = ComponentError::kErrorBeforeReady; + break; + } + + pid_ = 0; status_ = 0; - if (setState(score::lcm::ProcessState::kIdle)) + error = std::nullopt; + static_cast(setState(score::lcm::ProcessState::kStarting)); // Cannot fail by design + + if (osal::OsalReturnType::kSuccess == + process_interface_->startProcess(&pid_, &sync_, &config_->startup_config_)) { - uint32_t execution_error_code = config_->pgm_config_.execution_error_code_; - auto pg_mgr = graph_->getProcessGroupManager(); - pid_ = 0; - status_ = 0; - static_cast(setState(score::lcm::ProcessState::kStarting)); // Cannot fail by design + LM_LOG_DEBUG() << "startProcess pid" << pid_ + << "received for process:" << config_->startup_config_.short_name_; - if (osal::CommsType::kLaunchManager == config_->startup_config_.comms_type_) + if (osal::CommsType::kControlClient == config_->startup_config_.comms_type_) { - // Don't start launch manager, we're already running - LM_LOG_DEBUG() << "Found myself (" << std::string_view{config_->startup_config_.argv_[0U]} - << ") in a process group to start, not starting, reporting kRunning"; - pid_ = getpid(); - static_cast(setState(score::lcm::ProcessState::kRunning)); // Cannot fail by design - processSuccessorNodes(); - return; + setupControlClientChannel(); } - - if (osal::OsalReturnType::kSuccess == - pg_mgr->getProcessInterface()->startProcess(&pid_, &sync_, &config_->startup_config_)) + auto res = handleProcessStarted(stop_token); + if (!res.has_value()) { - LM_LOG_DEBUG() << "startProcess pid" << pid_ - << "received for process:" << config_->startup_config_.short_name_; - - if (osal::CommsType::kControlClient == config_->startup_config_.comms_type_) - { - setupControlClientChannel(); - } - handleProcessStarted(execution_error_code); + // Fatal error, do not retry + setState(score::lcm::ProcessState::kFailed); + error = res.error(); + break; } - else + if (res.value().has_value()) { - setState(score::lcm::ProcessState::kTerminated); - graph_->abort(execution_error_code, ControlClientCode::kSetStateFailed); + // No error + break; } + // Ordinary failure happened after the process started, e.g. kRunning timeout + assert(getState() == ProcessState::kTerminated && "Process was not terminated after failed startup"); + error = res.value().error(); } + else + { + setState(score::lcm::ProcessState::kFailed); + error = ComponentError::kErrorBeforeReady; + break; + } + sync_.reset(); - } while ((status_ != 0) && (restart_counter_-- != 0U)); - LM_LOG_DEBUG() << "startProcess for" << graph_->getProcessGroupName() << "process" << process_index_ << "(" - << config_->startup_config_.short_name_ << ") done"; + } + LM_LOG_DEBUG() << "startProcess for process" << process_index_ << "(" << config_->startup_config_.short_name_ + << ") done"; + + if (error.has_value()) + { + return tryReportError(error.value()); + } + + setState(ProcessState::kRunning); // Can fail if we've terminated already + return tryReportCompletion(ProcessState::kRunning); } inline void ProcessInfoNode::setupControlClientChannel() { // Make sure we store the control_client_channel before waiting for kRunning std::atomic_store(&control_client_channel_, ControlClientChannel::getControlClientChannel(sync_)); +} - if (control_client_channel_) - { // Put it at the front of the list if it's not there already - auto node0 = graph_->getNodes()[0U]; +score::cpp::expected_blank ProcessInfoNode::handleProcessStillStarting( + const score::cpp::stop_token& stop_token) +{ + static_cast(stop_token); // Not yet supported - if (this != node0.get()) - { - std::atomic_store(&next_state_manager_, node0->next_state_manager_); - std::atomic_store(&node0->next_state_manager_, graph_->getNodes()[process_index_]); - } + if (((osal::CommsType::kNoComms == config_->startup_config_.comms_type_) || + (process_interface_->waitForkRunning(sync_, config_->pgm_config_.startup_timeout_ms_) == + osal::OsalReturnType::kSuccess)) && + (0 == status_)) + { + handleProcessRunning(); + return {}; } -} -void ProcessInfoNode::handleProcessStillStarting(uint32_t execution_error_code) -{ - if (graph_->getState() == GraphState::kInTransition) + if (getState() == ProcessState::kTerminated) { - if (((osal::CommsType::kNoComms == config_->startup_config_.comms_type_) || - (graph_->getProcessGroupManager()->getProcessInterface()->waitForkRunning( - sync_, config_->pgm_config_.startup_timeout_ms_) == osal::OsalReturnType::kSuccess)) && - (0 == status_)) - { - handleProcessRunning(execution_error_code); - } - else // process is reporting, result is kFail or status is not zero (indicating the process has exited badly) - { - LM_LOG_WARN() << "Got kRunning timeout for process" << process_index_ << "(" - << config_->startup_config_.short_name_ << ")"; - ControlClientCode errcode = - status_ ? ControlClientCode::kFailedUnexpectedTerminationOnEnter : ControlClientCode::kSetStateFailed; - terminateProcess(); - if (0U == restart_counter_) - { - graph_->abort(execution_error_code, errcode); - } - } + return score::cpp::make_unexpected(ComponentError::kErrorBeforeReady); } + + LM_LOG_WARN() << "Got kRunning timeout for process" << process_index_ << "(" << config_->startup_config_.short_name_ + << ")"; + terminateProcess(stop_token); + return score::cpp::make_unexpected(ComponentError::kActivationTimedOut); } -void ProcessInfoNode::handleProcessAlreadyTerminated(uint32_t execution_error_code) +score::cpp::expected_blank ProcessInfoNode::handleProcessAlreadyTerminated() { if ((0 != status_) || (osal::CommsType::kNoComms != config_->startup_config_.comms_type_)) { // Error. To get a legal terminated before kRunning the process must be self-terminating, non-reporting // and to have exited with zero status LM_LOG_WARN() << "Got process termination before kRunning for pid" << pid_ << "(" - << config_->startup_config_.short_name_ << ") process" << process_index_ << "of group" - << graph_->getProcessGroupName(); - // This will cause the graph to fail, we must report to SM (unless we have restart attempts left) - if (0U == restart_counter_) - { - graph_->abort(execution_error_code, ControlClientCode::kFailedUnexpectedTerminationOnEnter); - } + << config_->startup_config_.short_name_ << ") process" << process_index_; + // This will cause the graph to fail unless we have restart attempts left + return score::cpp::make_unexpected(ComponentError::kErrorBeforeReady); } else { // case of a self-terminating, non-reporting process exiting nicely before we've had a chance to put an // entry in the map - queueTerminationSuccessorJobs(); + return {}; } } -void ProcessInfoNode::handleProcessStarted(uint32_t execution_error_code) +score::cpp::expected, IComponent::ComponentError> +ProcessInfoNode::handleProcessStarted(const score::cpp::stop_token& stop_token) { - switch (graph_->getProcessGroupManager()->getProcessMap()->insertIfNotTerminated(pid_, this)) + switch (process_map_->insertIfNotTerminated(pid_, this)) { - case score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kOk: // Normal case, entry was put in + case score::lcm::internal::SafeProcessMapReturnType::kOk: // Normal case, entry was put in // the map, process still running - handleProcessStillStarting(execution_error_code); - break; - case score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield: // Process has already exited - handleProcessAlreadyTerminated(execution_error_code); - break; + return handleProcessStillStarting(stop_token); + case score::lcm::internal::SafeProcessMapReturnType::kYield: // Process has already exited + return handleProcessAlreadyTerminated(); default: // Error case when pn == -1 // really bad fatal error, should not happen, treat as a failure to set the state & kill the process LM_LOG_ERROR() << "Could not add PID to map!"; - restart_counter_ = 0U; - terminateProcess(); - graph_->abort(execution_error_code, ControlClientCode::kSetStateFailed); - break; + terminateProcess(stop_token); + return score::cpp::make_unexpected(ComponentError::kErrorBeforeReady); } } -void ProcessInfoNode::handleProcessRunning(uint32_t execution_error_code) +void ProcessInfoNode::handleProcessRunning() { if (osal::CommsType::kNoComms == config_->startup_config_.comms_type_) { LM_LOG_DEBUG() << "Considered kRunning for Non Reporting Process pid" << pid_ << "(" - << config_->startup_config_.short_name_ << ") process" << process_index_ << "of group" - << graph_->getProcessGroupName(); + << config_->startup_config_.short_name_ << ") process" << process_index_; } else { LM_LOG_DEBUG() << "Got kRunning for pid" << pid_ << "(" << config_->startup_config_.short_name_ << ") process" - << process_index_ << "of group" << graph_->getProcessGroupName(); - } - - // kRunning has already been reported (or assumed) - // Therefore, a process in the terminated state is a new error not related to process - // starting (and so not eligible for a restart), or it's OK because its a self- - // terminating process. - if (setState(score::lcm::ProcessState::kRunning) || (config_->pgm_config_.is_self_terminating_ && (0 == status_))) - { - processSuccessorNodes(); - } - else if (restart_counter_ == 0U) - { - graph_->abort(execution_error_code, ControlClientCode::kSetStateFailed); - } - - // Note that if there is any process dependent upon this one terminating, that event will be handled in the - // OSHandler thread, when terminated() is called -} - -void ProcessInfoNode::processSuccessorNodes() -{ - for (auto& successor_node : dependent_on_running_) - { - if (successor_node->is_included_ && successor_node->dependencies_ > 0U) - { - checkForEmptyDependencies(successor_node); - } - } -} - -void ProcessInfoNode::checkForEmptyDependencies(std::shared_ptr& successor_node) -{ - if (0U == --successor_node->dependencies_) - { - while (graph_->getState() == GraphState::kInTransition) - { - auto push_res = graph_->getProcessGroupManager()->getWorkerJobs()->push(successor_node, kMaxQueueDelay); - if (push_res) - { - graph_->markNodeInFlight(); - break; - } - else if (push_res.error() == ConcurrencyErrc::kTimeout) - { - continue; - } - else - { - break; - } - } + << process_index_; } } -void ProcessInfoNode::terminateProcess() +void ProcessInfoNode::terminateProcess(const score::cpp::stop_token& stop_token) { LM_LOG_DEBUG() << "terminating process" << process_index_ << "(" << config_->startup_config_.short_name_ << ")"; if (setState(score::lcm::ProcessState::kTerminating)) { - if (osal::CommsType::kLaunchManager == config_->startup_config_.comms_type_) - { - LM_LOG_DEBUG() << "Found myself (" << std::string_view{config_->startup_config_.argv_[0U]} - << ") in a process group to terminate, not terminating, reporting kTerminated"; - static_cast(setState(score::lcm::ProcessState::kTerminated)); // Cannot fail by design - } - else - { - handleTerminationProcess(); - } - } - if (!graph_->isStarting() || (0 == status_)) - { - queueTerminationSuccessorJobs(); + handleTerminationProcess(stop_token); } - LM_LOG_DEBUG() << "terminateProcess for" << graph_->getProcessGroupName() << "process" << process_index_ << "(" - << config_->startup_config_.short_name_ << ") done"; + LM_LOG_DEBUG() << "terminateProcess for process" << process_index_ << "(" << config_->startup_config_.short_name_ + << ") done"; } -inline void ProcessInfoNode::handleTerminationProcess() +inline void ProcessInfoNode::handleTerminationProcess(const score::cpp::stop_token& stop_token) { - auto pg_mgr = graph_->getProcessGroupManager(); - static_cast(terminator_.init(0U, false)); has_semaphore_.store(true); - LM_LOG_DEBUG() << "Requesting termination of process" << process_index_ << "of" << graph_->getProcessGroupName() - << "pid" << pid_ << "(" << config_->startup_config_.short_name_ << ")"; + LM_LOG_DEBUG() << "Requesting termination of process" << process_index_ << "pid" << pid_ << "(" + << config_->startup_config_.short_name_ << ")"; // handle request termination - if ((pg_mgr->getProcessInterface()->requestTermination(pid_) == osal::OsalReturnType::kFail) || + if ((process_interface_->requestTermination(pid_) == osal::OsalReturnType::kFail) || (terminator_.timedWait(config_->pgm_config_.termination_timeout_ms_) == osal::OsalReturnType::kSuccess)) { LM_LOG_DEBUG() << "Queuing jobs after regular termination of process wait" << process_index_ << "(" @@ -546,21 +380,21 @@ inline void ProcessInfoNode::handleTerminationProcess() else { // handle forced termination - handleForcedTermination(); + handleForcedTermination(stop_token); } has_semaphore_.store(false); static_cast(terminator_.deinit()); } -inline void ProcessInfoNode::handleForcedTermination() +inline void ProcessInfoNode::handleForcedTermination(const score::cpp::stop_token& stop_token) { + static_cast(stop_token); // Not yet supported + LM_LOG_WARN() << "Process" << process_index_ << "(" << config_->startup_config_.short_name_ << ") did not respond to SIGTERM, sending SIGKILL"; - while ((osal::OsalReturnType::kSuccess == - graph_->getProcessGroupManager()->getProcessInterface()->forceTermination(pid_)) && - (graph_->getState() == GraphState::kInTransition) && + while ((osal::OsalReturnType::kSuccess == process_interface_->forceTermination(pid_)) && (terminator_.timedWait(score::lcm::internal::kMaxSigKillDelay) != osal::OsalReturnType::kSuccess)) { LM_LOG_FATAL() << "Process" << process_index_ << "(" << config_->startup_config_.short_name_ @@ -568,31 +402,30 @@ inline void ProcessInfoNode::handleForcedTermination() } } -void ProcessInfoNode::doWork() +IComponent::RequestResult ProcessInfoNode::activate(score::cpp::stop_token stop_token) { - if (graph_->getState() == GraphState::kInTransition) - { - if (graph_->isStarting()) - { - startProcess(); - } - else - { - terminateProcess(); - } - } - graph_->nodeExecuted(); + success_returned_.clear(); + if (reached_ready_.load()) + { // Already activated (still active — even if the process has since self-terminated), + // nothing to do. A component is only restarted after it has been deactivated. + return tryReportSuccess(); + } + auto res = startProcess(std::move(stop_token)); + return res; } -std::shared_ptr ProcessInfoNode::getNextStateManager() +IComponent::RequestResult ProcessInfoNode::deactivate(score::cpp::stop_token stop_token) { - // Remove dead state managers from the list on the fly - while (std::atomic_load(&next_state_manager_) && !std::atomic_load(&next_state_manager_)->control_client_channel_) - { - std::atomic_store(&next_state_manager_, next_state_manager_->next_state_manager_); - } + success_returned_.clear(); + reached_ready_.store(false); + terminateProcess(stop_token); + setState(ProcessState::kIdle); + return IComponent::RequestState::kSuccess; +} - return std::atomic_load(&next_state_manager_); +bool ProcessInfoNode::active() const +{ + return reached_ready_.load(); } osal::ProcessID ProcessInfoNode::getPid() const @@ -605,22 +438,12 @@ score::lcm::ProcessState ProcessInfoNode::getState() const return process_state_.load(); } -void ProcessInfoNode::markRequested(bool requested) -{ - in_requested_state_ = requested; -} - -bool ProcessInfoNode::isHeadNode() const -{ - return is_head_node_; -} - -uint32_t ProcessInfoNode::getNodeIndex() const +uint32_t ProcessInfoNode::getIndex() const { return process_index_; } -ControlClientChannelP ProcessInfoNode::getControlClientChannel() +ControlClientChannelP ProcessInfoNode::getControlClientChannel() const { return std::atomic_load(&control_client_channel_); } diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.hpp b/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.hpp index 845906932..016766813 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.hpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node.hpp @@ -14,219 +14,168 @@ #ifndef _INCLUDED_PROCESSINFONODE_ #define _INCLUDED_PROCESSINFONODE_ -#include -#include "score/mw/launch_manager/process_state_client/posix_process.hpp" #ifdef USE_NEW_CONFIGURATION #include "score/mw/launch_manager/configuration/configuration_adapter.hpp" #else #include "score/mw/launch_manager/configuration/configuration_manager.hpp" #endif -#include "score/mw/launch_manager/process_group_manager/details/safe_process_map.hpp" #include "score/mw/launch_manager/control/control_client_channel.hpp" +#include "score/mw/launch_manager/process_group_manager/details/icomponent.hpp" +#include "score/mw/launch_manager/process_group_manager/details/safe_process_map.hpp" +#include +#include -namespace score { +namespace score +{ + +namespace lcm +{ + +namespace internal +{ + +using namespace score::mw::lifecycle::internal; + +using ReportStateFn = std::function; + +/// @brief Represents both a process and a component in the graph. +/// @details A ProcessInfoNode is a node in the dependency graph that represents an OS process and its associated +/// component. It manages the lifecycle of the process, including activation, deactivation, and state reporting. The +/// node tracks the process's PID, status, and state, and provides methods to activate or deactivate the process, as +/// well as to report its completion or error state. At the same time it implements the IComponent interface, allowing +/// it to be treated as a component in the graph's transition engine. +/// @note A Component and a Process have distinct state machines. The component may be in kActive state while the +/// process is in kTerminated state, for example, if the process self-terminates after reaching its ready condition. +/// In the future, this class shall be split up to properly separate Component and Process lifecycle. +class ProcessInfoNode final : public IComponent +{ + public: + /// @brief The criteria for when a process is considered "ready" + enum class ReadyCondition : uint8_t + { + kRunning, // Running reported in the case of a reporting process, process launched if non-reporting + kTerminated, // Process has terminated with status 0 + }; + + /// @brief Constructs a ProcessInfoNode. + /// @param config Configuration for the OS process. + /// @param index The process index within its process group. + /// @param ready_condition Whether this process is considered ready when running or when terminated. + /// @param report_function Callback used to report state changes to the platform health manager. + /// @param process_interface The OS process interface used to start and stop the process. + /// @param process_map The shared process map used to track process pids. + ProcessInfoNode( + const OsProcess* config, + uint32_t index, + ReadyCondition ready_condition, + ReportStateFn report_function, + osal::IProcess* process_interface, + std::shared_ptr process_map); + + /// @brief Explicit move constructor required due to atomics. PIN must be moveable to exist in the graph + ProcessInfoNode(ProcessInfoNode&& other) noexcept + : terminator_(), + has_semaphore_(other.has_semaphore_.load()), + process_index_(other.process_index_), + pid_(other.pid_), + status_(other.status_.load()), + process_state_(other.process_state_.load()), + reached_ready_(other.reached_ready_.load()), + ready_condition_(other.ready_condition_), + config_(other.config_), + control_client_channel_(std::move(other.control_client_channel_)), + sync_(std::move(other.sync_)), + process_interface_(other.process_interface_), + process_map_(std::move(other.process_map_)) + { + } -namespace lcm { + ProcessInfoNode(ProcessInfoNode& other) = delete; + ProcessInfoNode& operator=(const ProcessInfoNode& other) = delete; + ProcessInfoNode& operator=(ProcessInfoNode&& other) = delete; + ~ProcessInfoNode() = default; -namespace internal { + uint32_t getIndex() const override; -/// @brief Forward declaration of the ProcessInfoNode class. -class ProcessInfoNode; + RequestResult activate(score::cpp::stop_token stop_token) override; -/// @brief Forward declaration of the Graph class. -class Graph; + RequestResult deactivate(score::cpp::stop_token stop_token) override; -/// @brief Type alias for a vector of shared pointers to ProcessInfoNode objects. -/// Used to maintain a list of successor nodes in a directed graph of process dependencies. -using SuccessorList = std::vector>; + RequestResult tryHandleTermination(int32_t process_status) override; -/// @brief ProcessInfoNode holds the current state of the process and is responsible for performing the actions required to start and stop processes -/// During initialisation it calculates the reverse dependencies of a process -/// At the start of a state transition it evaluates if a process should be included in the operation, and how -/// During state transition it queues new jobs in response to events so that each node participates in a directed graph with -/// the events kRunning received or expected process termination forming connecting edges. -/// Unexpected termination of a process or reception of a timeout result in the failure of this graph. -/// The ProcessInfoNode thus represents an adaptive process with specific startup configuration and dependencies -class ProcessInfoNode final : public ITerminationCallback { - public: - /// Constructor for Process Infonode - ProcessInfoNode() - : terminator_(), - has_semaphore_(false), - process_index_(0), - pid_(0), - status_(0), - process_state_(score::lcm::ProcessState::kIdle), - dependencies_(0), - start_dependencies_(0), - stop_dependencies_(0), - dependent_on_running_(), - dependent_on_terminating_(), - graph_(nullptr), - in_requested_state_(false), - is_included_(false), - is_head_node_(false), - config_(nullptr), - dependency_list_(nullptr){}; - /// @brief Initialise this node. - /// The member variables are initialised. The configuration manager is called to retrieve the configuration details and - /// dependencies for the process given by pg, idx. - /// The successor lists (dependent_on_running_, dependent_on_termination_) are initialised ready for the graph object to add items. - /// @param graph Pointer to the graph of which this node is part - /// @param index The process index in the process group - void initNode(Graph* graph, uint32_t index); - - /// @brief Notify the ProcessInfoNode that the associated adaptive application has terminated - /// The process enters the kTerminated state, and if the termination was expected, any successor nodes - /// defined for this function state transition are queued on the job queue - /// An unexpected termination will result in graph failure and an undefined process group state - /// The process status is saved in the ProcessInfoNode - /// This method will be called by the OsHander. - /// @param process_status - the value returned by the OS for the process termination status - void terminated(int32_t process_status) override; - - /// @brief Called by a worker thread to perform the expected action for this node - /// The action will either be to start the process, if graph_->isStarting() returns true, - /// or otherwise to stop the process. The action will result in process termination, kRunning received, - /// or a timeout. - void doWork(); - - /// @brief Get the operating system process identifier (pid) last stored for this process - /// @return A non-zero positive integer, or zero if the process has never run. + bool active() const override; + + /// @return The OS process ID, or zero if the process has never been started. osal::ProcessID getPid() const; - /// @brief Get the current state of the process recorded in the ProcessInfoNode - /// @return a value from the ProcessState enumeration + /// @return The current state of this process. score::lcm::ProcessState getState() const; - /// @brief Participate in graph initialisation - /// If starting== false, the graph is in the first (stopping) stage, and the number of dependencies for the starting phase - /// are counted in preparation. The number of termination dependencies is constant and was calculated during initialisation. - /// In this phase, a node should be included in the graph if the process state is not kIdle, and the process is not in the - /// requested state. - /// If starting == true, the graph is in the second (process starting) phase, and the a node should be included if the process is in - /// the requested state and it's not running. - /// - /// Initialise dependencies_ appropriately (start_dependencies in start phase, stop_dependencies_ in stop phase). - /// Record whether or not this node will be a head node for the current graph. A head node is one that is included in the graph, and has no - /// dependencies. - /// @param starting False is the graph is in the first phase (process stopping). True if the graph is in the second phase (process starting) - /// @return True if the node should be included in the graph, false otherwise. - bool constructGraphNode(bool starting); - - /// @brief Mark the node as being included or not included in the requested state - /// @param requested true for inclusion, false otherwise - void markRequested(bool requested); - - /// @brief Return true if the node is a head node in the current graph - /// @return true for head node, false otherwise - bool isHeadNode() const; - - /// @brief Add a successor item. The ProcessState parameter determines which list the successor is added to - /// @param successor_node The ProcessInfoNode to succeed this one - /// @param dependency The dependency relation (kRunning or kTerminated) - void addSuccessorNode(std::shared_ptr& successor_node, score::lcm::ProcessState dependency); - - /// @brief Return the index of this process in the process group. This method is used by the Graph object during - /// calculation of the successor lists. - /// @return index of process in process group as set by initNode() - uint32_t getNodeIndex() const; - - /// @brief get the ControlClientChannel pointer for this process - /// @return the value (including possibly nullptr) of the ControlClientChannel for this process - ControlClientChannelP getControlClientChannel(); - - /// @brief Return a pointer to the next active state manager in this process group - /// This assumes that this process was an active state manager; Any ControlClientChannel pointer retrieved - /// for the node must still be checked for validity. To get the first state manager, simply check the first - /// node in the vector, if this does not have a control_client_channel_, call getNextStateManager(). - /// @note this method will remove inactive (not running) state managers "on the fly". We do this rather than - /// removing them when a process exits as it is simpler and more efficient. - /// @return Pointer to the node that's next in the list as a state manager, or nullptr if there isn't one - std::shared_ptr getNextStateManager(); - - private: - /// @brief Indivisibly set the state of the process and report. Only valid transitions are allowed - /// @details If a valid process state transition was made the process state change is also reported - /// to the platform health manager using the process state notifier mechanism, but only if the - /// process is marked as a reporting process. If no PHM is running, then the shared buffer will - /// simply overrun. This design means that PHM is able to see buffered state changes for processes - /// that were started before PHM was started. - /// @param new_state - /// @return true if the state was set to the provided value, false otherwise + /// @return The ControlClientChannel for this process, or nullptr if none exists. + ControlClientChannelP getControlClientChannel() const; + + private: + /// @brief Atomically transitions to new_state if the transition is valid. For reporting + /// processes, also notifies the platform health manager of the state change. + /// @param new_state The desired process state. + /// @return True if the state was changed, false if the transition was not valid. bool setState(score::lcm::ProcessState new_state); - /// @brief Request process termination - /// Set the process state to terminating, and if this was successful, start the timeout and request termination of - /// the process. - /// If the state could not be set, then the process must have terminated, so process the termination by notifying the - /// graph that a node executed and queuing any successor nodes - void terminateProcess(); - - /// @brief handle the case of unexpected termination - void unexpectedTermination(); - - /// @brief Start the process - /// Set the process state to kStarting and attempt to start the process using the startProcess method of the process interface. - /// If this was successful, if the process is a state manager add it to the list of state managers then wait for kRunning using - /// the kRunningReceived() method, and then for a self-terminating process that has successors in this process group state, wait - /// for up to kMaxTerminationDelay for the process to end. - /// If waiting for kRunning failed, call the timeoutReceived() method. - /// If starting the process failed, call graph_->abort(error_code, kSetStateFailed) where error_code if the configured error code - /// for this process. - void startProcess(); - - /// @brief Handle actions when the process has started successfully. - /// This method is called when the process has started successfully. It performs necessary actions such as synchronization - /// with external components. - inline void handleProcessStarted(uint32_t execution_error_code); - - /// @brief Handle actions when process is still starting - /// called when SafeProcessMap::insertIfNotTerminated returns 0 - inline void handleProcessStillStarting(uint32_t execution_error_code); - - /// @brief Handle actions when a process has already terminated - /// called when SafeProcessMap::insertIfNotTerminated returns 1 - inline void handleProcessAlreadyTerminated(uint32_t execution_error_code); - - /// @brief Handle actions when the process is in the running state. - /// This method is called when the process is running. It may perform ongoing monitoring or state-dependent actions. - inline void handleProcessRunning(uint32_t execution_error_code); - - /// @brief Handle actions when the process terminates. - /// This method is called when the process has terminated. It performs cleanup tasks or initiates further actions based on - /// the termination status. - inline void handleTerminationProcess(); - - /// @brief Handle actions when the process needs to be forcibly terminated. - /// This method is called when there is a need to forcibly terminate the process. It ensures that the process is forcefully - /// stopped and any associated resources are properly released. - inline void handleForcedTermination(); - - /// @brief Queue the nodes that follow this one. - /// For a starting graph, for all the nodes in dependent_on_terminating_ that are included in the graph and have a positive dependency - /// count, decrement that count and if zero add the node to the job queue. - /// For a stop graph, perform a similar operation for all the process upon which we have a termination dependency. - inline void queueTerminationSuccessorJobs(); - - /// @brief Initialize and configure the Control Client communication channel. - /// This method sets up the communication channel used by the Control Client to synchronize with other processes. - /// It maps the shared memory region for the Control Client communication and ensures proper initialization of semaphores - /// used for synchronization between processes. If any part of the setup fails, appropriate fallback actions are taken. - inline void setupControlClientChannel(); + /// @brief Helper method to post on the semaphore waiting for kRunning if it exists + void unblockSync(); + + /// @brief Get the request result corresponding to the new state reached. For example, if the ready state is + /// terminated, the function will only return kSuccess if the new state is kTerminated. + /// @return Success if the ready condition is satisfied and completion is not already reported, an error if the + /// state is unrecoverable, waiting otherwise. + RequestResult tryReportCompletion(score::lcm::ProcessState new_state); + + /// @return The provided error if the result has not been reported yet. A waiting result otherwise. + RequestResult tryReportError(ComponentError error); + + /// @return Success state if success has not been returned yet, waiting result otherwise. + RequestResult tryReportSuccess(); + + /// @brief Requests the OS to terminate this process and waits for it to exit. + /// This operation cannot fail. If the process does not terminate, the function does not return. + void terminateProcess(const score::cpp::stop_token& stop_token); + + /// @brief Starts the OS process, retrying up to the configured restart count on failure. + /// @return A success if the process reached its ready conditon, an error if startup failed, waiting otherwise. + RequestResult startProcess(score::cpp::stop_token stop_token); + + /// @brief Handles the result of inserting the process into the process map after the OS + /// process has been created. Routes to the appropriate handler based on whether the + /// process is still running or has already terminated. + /// @returns An error if the process map error is unrecoverable, the result from the relevant handler otherwise. + inline score::cpp::expected, ComponentError> handleProcessStarted( + const score::cpp::stop_token& stop_token); + + /// @brief Waits for the process to report kRunning, terminating the process if it times out. + /// @post Process state is either running or terminated. + /// @returns an error if the startup times out. + inline score::cpp::expected_blank handleProcessStillStarting( + const score::cpp::stop_token& stop_token); + + /// @brief Handles the case where the process exited before the map insertion completed. + /// @returns success only for self-terminating, non-reporting processes with exit status 0. + inline score::cpp::expected_blank handleProcessAlreadyTerminated(); + + /// @brief Logs that the process has reached the running state. + inline void handleProcessRunning(); - /// @brief Process the Sucessor nodes - /// This method is called when there is a need to process the sucessor nodes - /// calls the checkForEmptyDependencies method to check if the dependencies are empty. - void processSuccessorNodes(); + /// @brief Sends SIGTERM to the process and waits up to the configured timeout for it to exit. If the timeout is + /// reached, force the termination. + inline void handleTerminationProcess(const score::cpp::stop_token& stop_token); - /// @brief check for the empty dependencies - /// This method is called when there is a need to check the empty dependencies and add successor nodes to graph - /// successor node attached to the graph. - void checkForEmptyDependencies(std::shared_ptr& successor_node); + /// @brief Sends SIGKILL repeatedly until the process exits or the stop token is triggered. + inline void handleForcedTermination(const score::cpp::stop_token& stop_token); + + /// @brief Creates the ControlClientChannel from the process's IPC comms handle. + inline void setupControlClientChannel(); /// @brief semaphore used to check termination with timeout - osal::Semaphore terminator_; + osal::Semaphore terminator_{}; /// @brief True if semaphore is being used std::atomic_bool has_semaphore_{false}; @@ -243,56 +192,39 @@ class ProcessInfoNode final : public ITerminationCallback { /// @brief The current state of the OS process std::atomic process_state_{score::lcm::ProcessState::kIdle}; - /// @brief Number of nodes still to process before this one can be processed - std::atomic_uint32_t dependencies_{0}; - - /// @brief initial dependencies for a start graph - uint32_t start_dependencies_ = 0; - - /// @brief initial dependencies for a stop graph - uint32_t stop_dependencies_ = 0; - - /// @brief List of nodes dependent upon kRunning reported for pid - SuccessorList dependent_on_running_; - - /// @brief List of nodes dependent upon termination received for pid - SuccessorList dependent_on_terminating_; - - /// @brief Graph to which this node belongs - Graph* graph_{nullptr}; - - /// @brief True during transition if this process runs in the requested state - bool in_requested_state_ = false; - - /// @brief true if this node is included in the graph - bool is_included_ = false; + /// @brief Flag indicating whether the Ready Condition has been satisfied. + /// The flag is reset when deactivate() is called. + std::atomic_bool reached_ready_{false}; - /// @brief True if this is a head node in a graph - std::atomic_bool is_head_node_{false}; + /// @brief Enum representing the criteria for this process to be considered "ready" + ReadyCondition ready_condition_; /// @brief Pointer to config for this process const OsProcess* config_{nullptr}; - /// @brief Pointer to the list of dependencies for this process - const DependencyList* dependency_list_{nullptr}; - /// @brief Pointer to the ControlClientChannel object if it exists ControlClientChannelP control_client_channel_{nullptr}; - /// @brief Pointer to the next node that is a state manager - std::shared_ptr next_state_manager_{nullptr}; - /// @brief Pointer to the comms for this process osal::IpcCommsP sync_{nullptr}; - /// @brief Restart counter; determines how errors are handled - uint32_t restart_counter_ = 0; -}; + /// @brief Callback for reporting process state to health monitor + ReportStateFn report_state_; -} // namespace lcm + /// @brief True if we have returned a success or failure for the current activation/deactivation + std::atomic_flag success_returned_{false}; + + /// @brief Handle to manage the underlying posix process + osal::IProcess* process_interface_{nullptr}; + + /// @brief Map this node will be stored in + std::shared_ptr process_map_; +}; } // namespace internal +} // namespace lcm + } // namespace score #endif diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node_UT.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node_UT.cpp new file mode 100644 index 000000000..d61a47332 --- /dev/null +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/process_info_node_UT.cpp @@ -0,0 +1,575 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#include "score/mw/launch_manager/process_group_manager/details/process_info_node.hpp" +#include "score/mw/launch_manager/process_group_manager/details/safe_process_map.hpp" +#include "score/mw/launch_manager/process_group_manager/mock_iprocess.hpp" +#include +#include +#include +#include + +using namespace testing; +using namespace score::lcm::internal; + +// Default ProcessIndex for testing +constexpr uint32_t kProcessIndex = 111; + +class MockSafeProcessMapInserter : public SafeProcessMapInserter +{ + public: + MOCK_METHOD(SafeProcessMapReturnType, insertIfNotTerminated, (osal::ProcessID key, IComponent* object), (override)); +}; + +class ProcessInfoNodeFixture : public ::testing::Test +{ + protected: + void SetUp() override + { + RecordProperty("TestType", "interface-test"); + RecordProperty("DerivationTechnique", "equivalence-classes"); + } + + /// @brief Helper method to create a ProcessInfoNode with the given parameters. + std::unique_ptr createProcessInfoNode( + osal::CommsType comms_type = osal::CommsType::kReporting, + int restart_attempts = 0, + bool self_terminating = false, + ProcessInfoNode::ReadyCondition ready_condition = ProcessInfoNode::ReadyCondition::kRunning) + { + config_.startup_config_.comms_type_ = comms_type; + + PgManagerConfig pgm_config; + pgm_config.number_of_restart_attempts = restart_attempts; + pgm_config.is_self_terminating_ = self_terminating; + config_.pgm_config_ = pgm_config; + + return std::make_unique( + &config_, kProcessIndex, ready_condition, report_fn_, &mock_processIf_, process_map_); + } + + /// @brief Helper method to create a ProcessInfoNode that is self-terminating. + std::unique_ptr createSelfTerminatingProcessInfoNode( + osal::CommsType comms_type = osal::CommsType::kReporting, + int restart_attempts = 0) + { + return createProcessInfoNode(comms_type, restart_attempts, true); + } + + /// @brief Helper method to create a ProcessInfoNode that is already in Running state + std::unique_ptr createRunningProcessInfoNode( + osal::CommsType comms_type = osal::CommsType::kReporting, + std::chrono::milliseconds termination_timeout = std::chrono::milliseconds{1000}) + { + auto node = createProcessInfoNode(comms_type); + config_.pgm_config_.termination_timeout_ms_ = termination_timeout; + + expectSuccessfulProcessLaunch(); + + node->activate(score::cpp::stop_token{}); + return node; + } + + /// @brief Helper method to create a Running ProcessInfoNode with a specific termination timeout. + std::unique_ptr createRunningProcessInfoNode_TermTimeout( + std::chrono::milliseconds termination_timeout) + { + return createRunningProcessInfoNode(osal::CommsType::kReporting, termination_timeout); + } + + /// @brief Asserts that mock_report_fn_ is called with each of the given states, in the given order. + void expectStateTransitions(const std::vector& states) + { + Sequence seq; + for (const auto state : states) + { + EXPECT_CALL(mock_report_fn_, Call(_, state, _)).InSequence(seq).WillOnce(Return(true)); + } + } + + /// @brief Sets up expectations for the OS process being launched and successfully added to the process map. + void expectSuccessfulProcessLaunch() + { + EXPECT_CALL(mock_processIf_, startProcess(_, _, _)).WillOnce(Return(osal::OsalReturnType::kSuccess)); + EXPECT_CALL(*process_map_, insertIfNotTerminated(_, _)) + .WillOnce(Return(score::lcm::internal::SafeProcessMapReturnType::kOk)); + } + + /// @brief Sets up requestTermination to synchronously deliver the OS exit notification. + void expectOsAcknowledgesTermination(ProcessInfoNode* node, int32_t exit_status = 0) + { + EXPECT_CALL(mock_processIf_, requestTermination(_)) + .WillOnce(DoAll( + InvokeWithoutArgs([node, exit_status] { + node->tryHandleTermination(exit_status); + }), + Return(osal::OsalReturnType::kSuccess))); + } + + OsProcess config_{}; + score::cpp::stop_source stop_source_{}; + std::shared_ptr process_map_{std::make_shared()}; + StrictMock mock_processIf_{}; + MockFunction mock_report_fn_{}; + ReportStateFn report_fn_{mock_report_fn_.AsStdFunction()}; +}; + +// Bundles different cases for activate() that occur during startup, before the ready condition is reached. +class ProcessInfoNodeStartupTest : public ProcessInfoNodeFixture +{ +}; + +TEST_F(ProcessInfoNodeStartupTest, CanConstructIdleProcessInfoNode) +{ + RecordProperty( + "Description", "Construct an idle ProcessInfoNode and check the initial state and index are correct."); + + auto node = createProcessInfoNode(); + + ASSERT_THAT(node->getIndex(), Eq(kProcessIndex)); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kIdle)); + ASSERT_THAT(node->getPid(), Eq(0)); + ASSERT_THAT(node->active(), IsFalse()); + ASSERT_THAT(node->getControlClientChannel(), IsNull()); +} + +TEST_F(ProcessInfoNodeStartupTest, CanStartNonReportingProcess) +{ + RecordProperty( + "Description", "Can start a non-reporting process and check that the state transitions to kRunning without waiting for kRunning report."); + + auto node = createProcessInfoNode(osal::CommsType::kNoComms); + expectSuccessfulProcessLaunch(); + + auto result = node->activate(score::cpp::stop_token{}); + + ASSERT_THAT(result.has_value(), IsTrue()); + ASSERT_THAT(result.value(), Eq(IComponent::RequestState::kSuccess)); + ASSERT_THAT(node->getControlClientChannel(), IsNull()); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kRunning)); +} + +TEST_F(ProcessInfoNodeStartupTest, CanStartReportingProcess_ReportsRunningInTime) +{ + RecordProperty("Description", "Can start a reporting process and check that the state transitions to kRunning."); + + auto node = createProcessInfoNode(osal::CommsType::kReporting); + expectSuccessfulProcessLaunch(); + EXPECT_CALL(mock_processIf_, waitForkRunning(_, _)).WillOnce(Return(osal::OsalReturnType::kSuccess)); + expectStateTransitions({score::lcm::ProcessState::kStarting, score::lcm::ProcessState::kRunning}); + + auto result = node->activate(score::cpp::stop_token{}); + + ASSERT_THAT(result.has_value(), IsTrue()); + ASSERT_THAT(result.value(), Eq(IComponent::RequestState::kSuccess)); + ASSERT_THAT(node->getControlClientChannel(), IsNull()); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kRunning)); +} + +TEST_F(ProcessInfoNodeStartupTest, OsForkFails_ReturnsErrorBeforeReady) +{ + RecordProperty( + "Description", + "If the OS fails to fork the process, activate() returns kErrorBeforeReady and the node ends up in kFailed."); + + auto node = createProcessInfoNode(osal::CommsType::kNoComms); + EXPECT_CALL(mock_processIf_, startProcess(_, _, _)).WillOnce(Return(osal::OsalReturnType::kFail)); + + auto result = node->activate(score::cpp::stop_token{}); + + ASSERT_THAT(result.has_value(), IsFalse()); + ASSERT_THAT(result.error(), Eq(IComponent::ComponentError::kErrorBeforeReady)); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kFailed)); +} + +TEST_F(ProcessInfoNodeStartupTest, MapInsertError_ReturnsErrorBeforeReady) +{ + RecordProperty( + "Description", + "If the process map insertion fails with an error, activate() returns kErrorBeforeReady and the " + "node ends up in kFailed."); + + auto node = createProcessInfoNode(osal::CommsType::kNoComms); + EXPECT_CALL(mock_processIf_, startProcess(_, _, _)).WillOnce(Return(osal::OsalReturnType::kSuccess)); + EXPECT_CALL(*process_map_, insertIfNotTerminated(_, _)) + .WillOnce(Return(score::lcm::internal::SafeProcessMapReturnType::kInsertionError)); + // The error handler calls terminateProcess(), which sends SIGTERM; simulate the OS ack. + expectOsAcknowledgesTermination(node.get()); + + auto result = node->activate(score::cpp::stop_token{}); + + ASSERT_THAT(result.has_value(), IsFalse()); + ASSERT_THAT(result.error(), Eq(IComponent::ComponentError::kErrorBeforeReady)); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kFailed)); +} + +TEST_F(ProcessInfoNodeStartupTest, SelfTerminating_ExitsBeforeMapInsert_ReturnsSuccess) +{ + RecordProperty( + "Description", + "A self-terminating non-reporting process that exits with status 0 before the map insertion completes is " + "treated as a successful startup."); + + auto node = createSelfTerminatingProcessInfoNode(osal::CommsType::kNoComms); + // Simulate the process exiting before the map insertion happens. + EXPECT_CALL(mock_processIf_, startProcess(_, _, _)) + .WillOnce(DoAll( + InvokeWithoutArgs([node = node.get()] { + node->tryHandleTermination(0); + }), + Return(osal::OsalReturnType::kSuccess))); + EXPECT_CALL(*process_map_, insertIfNotTerminated(_, _)) + .WillOnce(Return(score::lcm::internal::SafeProcessMapReturnType::kYield)); + + auto result = node->activate(score::cpp::stop_token{}); + + ASSERT_THAT(result.has_value(), IsTrue()); + ASSERT_THAT(result.value(), Eq(IComponent::RequestState::kSuccess)); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kTerminated)); +} + +TEST_F(ProcessInfoNodeStartupTest, ActivateAlreadyActiveNode_ReturnsSuccess) +{ + RecordProperty( + "Description", + "Calling activate() on a node that is already active returns kSuccess without re-launching the process."); + + auto node = createRunningProcessInfoNode(osal::CommsType::kNoComms); + + auto result = node->activate(score::cpp::stop_token{}); + + ASSERT_THAT(result.has_value(), IsTrue()); + ASSERT_THAT(result.value(), Eq(IComponent::RequestState::kSuccess)); + ASSERT_THAT(node->active(), IsTrue()); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kRunning)); +} + +// Bundles process crashes and timeouts that occur during activate(), before the ready condition is reached. +class ProcessInfoNodeStartupCrashTest : public ProcessInfoNodeFixture +{ +}; + +TEST_F(ProcessInfoNodeStartupCrashTest, ProcesssTerminated_OnWaitForkRunningTimeout) +{ + RecordProperty( + "Description", + "If waitForkRunning times out, the process reports kActivationTimedOut and ends up in state kTerminated."); + + auto node = createProcessInfoNode(osal::CommsType::kReporting); + expectSuccessfulProcessLaunch(); + EXPECT_CALL(mock_processIf_, waitForkRunning(_, _)).WillOnce(Return(osal::OsalReturnType::kFail)); + // Simulate the OS handler reporting the killed process's exit once termination is requested. + expectOsAcknowledgesTermination(node.get()); + expectStateTransitions( + {score::lcm::ProcessState::kStarting, + score::lcm::ProcessState::kTerminating, + score::lcm::ProcessState::kTerminated}); + + auto result = node->activate(score::cpp::stop_token{}); + + ASSERT_THAT(result.has_value(), IsFalse()); + ASSERT_THAT(result.error(), Eq(IComponent::ComponentError::kActivationTimedOut)); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kTerminated)); +} + +TEST_F(ProcessInfoNodeStartupCrashTest, ReportingProcess_CrashesBeforeReady_NoRestarts) +{ + RecordProperty( + "Description", + "Process returns kErrorBeforeReady when crashing before reaching its ready condition (kRunning) with 0 restart " + "attempts"); + + auto node = createProcessInfoNode(osal::CommsType::kReporting); + expectSuccessfulProcessLaunch(); + // Simulate the OS handler detecting the crash while the process is still waiting to reach kRunning. + EXPECT_CALL(mock_processIf_, waitForkRunning(_, _)) + .WillOnce(DoAll( + InvokeWithoutArgs([node = node.get()] { + node->tryHandleTermination(-1); + }), + Return(osal::OsalReturnType::kFail))); + expectStateTransitions({score::lcm::ProcessState::kStarting, score::lcm::ProcessState::kTerminated}); + + auto result = node->activate(score::cpp::stop_token{}); + + ASSERT_THAT(result.has_value(), IsFalse()); + ASSERT_THAT(result.error(), Eq(IComponent::ComponentError::kErrorBeforeReady)); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kTerminated)); +} + +TEST_F(ProcessInfoNodeStartupCrashTest, ReportingProcess_CrashesBeforeReady_WithRestartAttempts) +{ + RecordProperty( + "Description", + "Process returns kErrorBeforeReady when crashing before reaching its ready condition (kRunning) with 3 restart " + "attempts"); + + constexpr uint32_t kRestartAttempts = 3; + constexpr uint32_t kTotalAttempts = kRestartAttempts + 1; + auto node = createProcessInfoNode(osal::CommsType::kReporting, kRestartAttempts); + + EXPECT_CALL(mock_processIf_, startProcess(_, _, _)) + .Times(kTotalAttempts) + .WillRepeatedly(Return(osal::OsalReturnType::kSuccess)); + EXPECT_CALL(*process_map_, insertIfNotTerminated(_, _)) + .Times(kTotalAttempts) + .WillRepeatedly(Return(score::lcm::internal::SafeProcessMapReturnType::kOk)); + // Simulate the OS handler detecting the crash on every attempt, while the process is still waiting to reach + // kRunning. + EXPECT_CALL(mock_processIf_, waitForkRunning(_, _)) + .Times(kTotalAttempts) + .WillRepeatedly(DoAll( + InvokeWithoutArgs([node = node.get()] { + node->tryHandleTermination(-1); + }), + Return(osal::OsalReturnType::kFail))); + expectStateTransitions( + {score::lcm::ProcessState::kStarting, + score::lcm::ProcessState::kTerminated, + score::lcm::ProcessState::kStarting, + score::lcm::ProcessState::kTerminated, + score::lcm::ProcessState::kStarting, + score::lcm::ProcessState::kTerminated, + score::lcm::ProcessState::kStarting, + score::lcm::ProcessState::kTerminated}); + + auto result = node->activate(score::cpp::stop_token{}); + + ASSERT_THAT(result.has_value(), IsFalse()); + ASSERT_THAT(result.error(), Eq(IComponent::ComponentError::kErrorBeforeReady)); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kTerminated)); +} + +TEST_F(ProcessInfoNodeStartupCrashTest, NonReportingProcess_CrashesBeforeReady_NoRestarts) +{ + RecordProperty( + "Description", + "A non-reporting process that crashes (non-zero status) between map insertion and the startup thread's status " + "check returns kErrorBeforeReady."); + + auto node = createProcessInfoNode(osal::CommsType::kNoComms); + EXPECT_CALL(mock_processIf_, startProcess(_, _, _)).WillOnce(Return(osal::OsalReturnType::kSuccess)); + // Simulate the process crashing after the map insertion. + EXPECT_CALL(*process_map_, insertIfNotTerminated(_, _)) + .WillOnce(DoAll( + InvokeWithoutArgs([node = node.get()] { + node->tryHandleTermination(-1); + }), + Return(score::lcm::internal::SafeProcessMapReturnType::kOk))); + + auto result = node->activate(score::cpp::stop_token{}); + + ASSERT_THAT(result.has_value(), IsFalse()); + ASSERT_THAT(result.error(), Eq(IComponent::ComponentError::kErrorBeforeReady)); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kTerminated)); +} + +TEST_F(ProcessInfoNodeStartupCrashTest, NonReportingProcess_CrashesBeforeReady_WithRestartAttempts) +{ + RecordProperty( + "Description", + "A non-reporting process that crashes before ready on every attempt exhausts all restart attempts and returns " + "kErrorBeforeReady."); + + constexpr uint32_t kRestartAttempts = 2; + constexpr uint32_t kTotalAttempts = kRestartAttempts + 1; + auto node = createProcessInfoNode(osal::CommsType::kNoComms, kRestartAttempts); + + EXPECT_CALL(mock_processIf_, startProcess(_, _, _)) + .Times(kTotalAttempts) + .WillRepeatedly(Return(osal::OsalReturnType::kSuccess)); + EXPECT_CALL(*process_map_, insertIfNotTerminated(_, _)) + .Times(kTotalAttempts) + .WillRepeatedly(DoAll( + InvokeWithoutArgs([node = node.get()] { + node->tryHandleTermination(-1); + }), + Return(score::lcm::internal::SafeProcessMapReturnType::kOk))); + + auto result = node->activate(score::cpp::stop_token{}); + + ASSERT_THAT(result.has_value(), IsFalse()); + ASSERT_THAT(result.error(), Eq(IComponent::ComponentError::kErrorBeforeReady)); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kTerminated)); +} + +TEST_F(ProcessInfoNodeStartupCrashTest, TimeoutThenSuccess_WithRestarts) +{ + RecordProperty( + "Description", + "A reporting process that times out on the first attempt but reports kRunning on the retry returns kSuccess."); + + constexpr uint32_t kRestartAttempts = 1; + auto node = createProcessInfoNode(osal::CommsType::kReporting, kRestartAttempts); + + EXPECT_CALL(mock_processIf_, startProcess(_, _, _)).Times(2).WillRepeatedly(Return(osal::OsalReturnType::kSuccess)); + EXPECT_CALL(*process_map_, insertIfNotTerminated(_, _)) + .Times(2) + .WillRepeatedly(Return(score::lcm::internal::SafeProcessMapReturnType::kOk)); + EXPECT_CALL(mock_processIf_, waitForkRunning(_, _)) + .WillOnce(Return(osal::OsalReturnType::kFail)) + .WillOnce(Return(osal::OsalReturnType::kSuccess)); + // Simulate the OS handler reporting the killed process's exit on the first (timed-out) attempt. + expectOsAcknowledgesTermination(node.get()); + expectStateTransitions( + {score::lcm::ProcessState::kStarting, + score::lcm::ProcessState::kTerminating, + score::lcm::ProcessState::kTerminated, + score::lcm::ProcessState::kStarting, + score::lcm::ProcessState::kRunning}); + + auto result = node->activate(score::cpp::stop_token{}); + + ASSERT_THAT(result.has_value(), IsTrue()); + ASSERT_THAT(result.value(), Eq(IComponent::RequestState::kSuccess)); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kRunning)); +} + +// Bundles unexpected terminations that occur after the ready condition has been reached. +class ProcessInfoNodeUnexpectedTerminationTest : public ProcessInfoNodeFixture +{ +}; +TEST_F(ProcessInfoNodeUnexpectedTerminationTest, ProcesssCrashed_AfterReadyCondition) +{ + RecordProperty( + "Description", "Process returns kErrorAfterReady when crashing after reaching its ready condition (kRunning)."); + + auto node = createRunningProcessInfoNode(osal::CommsType::kNoComms); + + auto result = node->tryHandleTermination(-1); + + ASSERT_THAT(result.has_value(), IsFalse()); + ASSERT_THAT(result.error(), Eq(IComponent::ComponentError::kErrorAfterReady)); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kTerminated)); +} + +TEST_F(ProcessInfoNodeUnexpectedTerminationTest, SelfTerminatingProcess_ExitsWithoutTerminationRequest) +{ + RecordProperty( + "Description", + "A self-terminating process exits without an explicit termination request and ends up in state kTerminated."); + + auto node = createSelfTerminatingProcessInfoNode(osal::CommsType::kNoComms); + expectSuccessfulProcessLaunch(); + node->activate(score::cpp::stop_token{}); + + auto result = node->tryHandleTermination(0); + + ASSERT_THAT(result.has_value(), IsTrue()); + ASSERT_THAT(result.value(), Eq(IComponent::RequestState::kWaiting)); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kTerminated)); +} + +TEST_F(ProcessInfoNodeUnexpectedTerminationTest, SelfTerminating_TerminatedReadyCondition_CleanExit_ReturnsSuccess) +{ + RecordProperty( + "Description", + "A self-terminating process with ReadyCondition::kTerminated returns kSuccess from tryHandleTermination() when " + "it exits cleanly, since its exit is the event that satisfies the ready condition."); + + auto node = createProcessInfoNode(osal::CommsType::kNoComms, 0 /*restart_attempts*/, true /*self terminating*/, + ProcessInfoNode::ReadyCondition::kTerminated /*ready condition*/); + expectSuccessfulProcessLaunch(); + // activate() returns kWaiting because kRunning != kTerminated (the ready condition). + auto activate_result = node->activate(score::cpp::stop_token{}); + ASSERT_THAT(activate_result.has_value(), IsTrue()); + ASSERT_THAT(activate_result.value(), Eq(IComponent::RequestState::kWaiting)); + + auto result = node->tryHandleTermination(0); + + ASSERT_THAT(result.has_value(), IsTrue()); + ASSERT_THAT(result.value(), Eq(IComponent::RequestState::kSuccess)); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kTerminated)); +} + +TEST_F(ProcessInfoNodeUnexpectedTerminationTest, SelfTerminating_CrashAfterReady_ReturnsErrorAfterReady) +{ + RecordProperty( + "Description", + "A self-terminating process that crashes (non-zero exit status) after reaching its ready condition returns " + "kErrorAfterReady, just like a non-self-terminating process crash."); + + auto node = createSelfTerminatingProcessInfoNode(osal::CommsType::kNoComms); + expectSuccessfulProcessLaunch(); + node->activate(score::cpp::stop_token{}); + + auto result = node->tryHandleTermination(-1); + + ASSERT_THAT(result.has_value(), IsFalse()); + ASSERT_THAT(result.error(), Eq(IComponent::ComponentError::kErrorAfterReady)); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kTerminated)); +} + +// Bundles succeess and failures cases when deactivating a process +class ProcessInfoNodeDeactivationTest : public ProcessInfoNodeFixture +{ +}; + +TEST_F(ProcessInfoNodeDeactivationTest, CanTerminateNonSelfTerminatingProcess) +{ + RecordProperty( + "Description", + "Can terminate a non-self-terminating process by calling `deactivate()` and check that the state transitions " + "to kTerminated."); + + EXPECT_CALL(mock_processIf_, waitForkRunning(_, _)).WillOnce(Return(osal::OsalReturnType::kSuccess)); + expectStateTransitions( + {score::lcm::ProcessState::kStarting, + score::lcm::ProcessState::kRunning, + score::lcm::ProcessState::kTerminating, + score::lcm::ProcessState::kTerminated}); + + auto node = createRunningProcessInfoNode(osal::CommsType::kReporting); + // Simulate the OS handler reporting the process's exit once termination is requested. + expectOsAcknowledgesTermination(node.get()); + + auto result = node->deactivate(score::cpp::stop_token{}); + + ASSERT_THAT(result.has_value(), IsTrue()); + ASSERT_THAT(result.value(), Eq(IComponent::RequestState::kSuccess)); + ASSERT_THAT(node->active(), IsFalse()); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kIdle)); +} + +TEST_F(ProcessInfoNodeDeactivationTest, ProcessIgnoresSigterm_ForcedWithSigkill) +{ + RecordProperty( + "Description", + "If a process does not exit within the termination timeout after receiving SIGTERM, it is forcibly killed with " + "SIGKILL."); + + EXPECT_CALL(mock_processIf_, waitForkRunning(_, _)).WillOnce(Return(osal::OsalReturnType::kSuccess)); + expectStateTransitions( + {score::lcm::ProcessState::kStarting, + score::lcm::ProcessState::kRunning, + score::lcm::ProcessState::kTerminating, + score::lcm::ProcessState::kTerminated}); + + auto node = createRunningProcessInfoNode_TermTimeout(std::chrono::milliseconds{0}); + EXPECT_CALL(mock_processIf_, requestTermination(_)).WillOnce(Return(osal::OsalReturnType::kSuccess)); + // Simulate the OS handler reporting the exit in response to SIGKILL. + EXPECT_CALL(mock_processIf_, forceTermination(_)) + .WillOnce(DoAll( + InvokeWithoutArgs([node = node.get()] { + node->tryHandleTermination(0); + }), + Return(osal::OsalReturnType::kSuccess))); + + auto result = node->deactivate(score::cpp::stop_token{}); + + ASSERT_THAT(result.has_value(), IsTrue()); + ASSERT_THAT(result.value(), Eq(IComponent::RequestState::kSuccess)); + ASSERT_THAT(node->active(), IsFalse()); + ASSERT_THAT(node->getState(), Eq(score::lcm::ProcessState::kIdle)); +} + diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/process_launcher.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/process_launcher.cpp index 1b7369578..cef0f7396 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/process_launcher.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/process_launcher.cpp @@ -22,7 +22,7 @@ #include #include -#include "score/mw/launch_manager/process_group_manager/iprocess.hpp" +#include "score/mw/launch_manager/process_group_manager/details/process_launcher.hpp" #include "score/mw/launch_manager/control/control_client_channel.hpp" #include "score/mw/launch_manager/common/log.hpp" #include "score/mw/launch_manager/osal/ipc_comms.hpp" @@ -80,7 +80,6 @@ void handleComms(score::lcm::internal::osal::ChildProcessConfig& param) // kNoComms !fd3 & !fd4 // kReporting fd3 & !fd4 // kControlClient fd3 & fd4 - // kLaunchManager does not matter if (!param.shared_block) { // kNoComms, fds are CLOEXEC @@ -120,9 +119,6 @@ void handleComms(score::lcm::internal::osal::ChildProcessConfig& param) << "failed:" << score::lcm::internal::errno_message(errno); } break; - case CommsType::kLaunchManager: - // nothing to do here - break; default: LM_LOG_ERROR() << "[New process] at line" << __LINE__ << "unknown CommsType" << static_cast(param.shared_block->comms_type_); @@ -194,7 +190,7 @@ namespace internal namespace osal { -OsalReturnType IProcess::startProcess(ProcessID* pid, IpcCommsP* block, const OsalConfig* config) +OsalReturnType ProcessLauncher::startProcess(ProcessID* pid, IpcCommsP* block, const OsalConfig* config) { OsalReturnType result = OsalReturnType::kFail; @@ -260,7 +256,7 @@ OsalReturnType IProcess::startProcess(ProcessID* pid, IpcCommsP* block, const Os return result; } -inline bool IProcess::setupComms(IpcCommsP& block, int& fd, const OsalConfig& config) +inline bool ProcessLauncher::setupComms(IpcCommsP& block, int& fd, const OsalConfig& config) { bool comms_result = true; char shm_name[static_cast(score::lcm::internal::ProcessLimits::maxLocalBuffSize)]; @@ -323,7 +319,7 @@ inline bool IProcess::setupComms(IpcCommsP& block, int& fd, const OsalConfig& co return comms_result; } -inline IpcCommsP IProcess::initializeControlClient(int& fd, const OsalConfig& config) +inline IpcCommsP ProcessLauncher::initializeControlClient(int& fd, const OsalConfig& config) { LM_LOG_DEBUG() << "Initialize the control client for" << config.short_name_ << " process"; /* Initialise the control client communications */ @@ -339,7 +335,7 @@ inline IpcCommsP IProcess::initializeControlClient(int& fd, const OsalConfig& co return shared_block; } -inline bool IProcess::initializeSemaphores(IpcCommsP shared_block) +inline bool ProcessLauncher::initializeSemaphores(IpcCommsP shared_block) { bool result = true; @@ -353,13 +349,12 @@ inline bool IProcess::initializeSemaphores(IpcCommsP shared_block) return result; } -OsalReturnType IProcess::setSchedulingAndSecurity(const OsalConfig& config) +OsalReturnType ProcessLauncher::setSchedulingAndSecurity(const OsalConfig& config) { OsalReturnType retval = OsalReturnType::kSuccess; // Set process group id to be equal to the pid - // setpgid will fail if called by a session lader (which LCMd is), so skip - if (config.comms_type_ != osal::CommsType::kLaunchManager && 0 != setpgid(0, getpid())) + if (0 != setpgid(0, getpid())) { LM_LOG_ERROR() << "setpgid() failed:" << score::lcm::internal::errno_message(errno); retval = OsalReturnType::kFail; @@ -424,7 +419,7 @@ OsalReturnType IProcess::setSchedulingAndSecurity(const OsalConfig& config) return retval; } -inline void IProcess::handleChildProcess(ChildProcessConfig& param) +inline void ProcessLauncher::handleChildProcess(ChildProcessConfig& param) { handleComms(param); @@ -449,7 +444,7 @@ inline void IProcess::handleChildProcess(ChildProcessConfig& param) } } -OsalReturnType IProcess::requestTermination(ProcessID pid) +OsalReturnType ProcessLauncher::requestTermination(ProcessID pid) { LM_LOG_DEBUG() << "Request termination received for" << pid; @@ -475,7 +470,7 @@ OsalReturnType IProcess::requestTermination(ProcessID pid) return result; } -OsalReturnType IProcess::forceTermination(ProcessID pid) +OsalReturnType ProcessLauncher::forceTermination(ProcessID pid) { LM_LOG_DEBUG() << "Forced termination received for pid" << pid; @@ -504,7 +499,7 @@ OsalReturnType IProcess::forceTermination(ProcessID pid) return result; } -OsalReturnType IProcess::waitForTermination(osal::ProcessID& pid, int32_t& status) +OsalReturnType ProcessLauncher::waitForTermination(osal::ProcessID& pid, int32_t& status) { int32_t wait_status; osal::OsalReturnType result = osal::OsalReturnType::kFail; @@ -527,7 +522,7 @@ OsalReturnType IProcess::waitForTermination(osal::ProcessID& pid, int32_t& statu return result; } -OsalReturnType IProcess::waitForkRunning(IpcCommsP sync, std::chrono::milliseconds timeout) +OsalReturnType ProcessLauncher::waitForkRunning(IpcCommsP sync, std::chrono::milliseconds timeout) { OsalReturnType result = OsalReturnType::kSuccess; diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/process_launcher.hpp b/score/launch_manager/src/daemon/src/process_group_manager/details/process_launcher.hpp new file mode 100644 index 000000000..f143fb1dd --- /dev/null +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/process_launcher.hpp @@ -0,0 +1,95 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +#ifndef PROCESS_LAUNCHER_HPP_INCLUDED +#define PROCESS_LAUNCHER_HPP_INCLUDED + +#include "score/mw/launch_manager/process_group_manager/iprocess.hpp" +#include + +namespace score +{ + +namespace lcm +{ + +namespace internal +{ + +namespace osal +{ + +/// @brief POSIX implementation of IProcess, managing child processes via fork/exec. +class ProcessLauncher final : public IProcess +{ + public: + /// @see IProcess::startProcess() for details + OsalReturnType startProcess(ProcessID* pid, IpcCommsP* sync, const osal::OsalConfig* config) override; + + /// @see IProcess::requestTermination() for details + OsalReturnType requestTermination(ProcessID pid) override; + + /// @see IProcess::forceTermination() for details + OsalReturnType forceTermination(ProcessID pid) override; + + /// @see IProcess::waitForTermination() for details + OsalReturnType waitForTermination(ProcessID& pid, int32_t& status) override; + + /// @see IProcess::waitForkRunning() for details + OsalReturnType waitForkRunning(IpcCommsP sync, std::chrono::milliseconds timeout) override; + + private: + /// @brief Creates shared memory for communication between processes. + /// @param[in,out] sync Pointer to a location to store a pointer to a structure containing + /// information about the communication channel. + /// @param[in,out] fd Reference to an integer where the file descriptor of the shared memory + /// segment will be stored. + /// @param[in,out] block Reference to a pointer that will be set to point to the shared memory block. + /// @param[in] config Pointer to the configuration for initializing the communication. + /// @return True if shared memory creation and initialization are successful, false otherwise. + inline bool setupComms(IpcCommsP& sync, int& fd, const OsalConfig& config); + + /// @brief Initializes semaphores within a given shared memory block. + /// @param[in] block Pointer to the shared memory block where semaphores will be initialized. + /// @return True if semaphore initialization is successful, false otherwise. + inline bool initializeSemaphores(IpcCommsP block); + + /// @brief Initializes the Control Client for communication using the shared memory block. + /// @param[in] shared_block Pointer to the shared memory block. + /// @param[in,out] fd Reference to store the file descriptor of the shared memory. + /// @param[in] config Pointer to the configuration for initializing the Control Client. + /// @return None. + inline IpcCommsP initializeControlClient(int& fd, const OsalConfig& config); + + /// @brief Handles the execution of the child process after forking. + /// @param[in] param Reference to child process configuration. + inline void handleChildProcess(ChildProcessConfig& param); + + /// @brief Sets up all the scheduling and security parameters described in the config, for the current process. + /// @param config the configuration to use + /// @return kFail if any operation fails, kSuccess otherwise + static OsalReturnType setSchedulingAndSecurity(const osal::OsalConfig& config); + + ///@brief Atomic counter for shared memory names + std::atomic_uint32_t shm_name_counter = {0}; +}; + +} // namespace osal + +} // namespace internal + +} // namespace lcm + +} // namespace score + +#endif // PROCESS_LAUNCHER_HPP_INCLUDED diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/safe_process_map.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/safe_process_map.cpp index 85620404f..b51360843 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/safe_process_map.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/safe_process_map.cpp @@ -26,7 +26,8 @@ namespace lcm namespace internal { -SafeProcessMap::SafeProcessMap(uint32_t capacity) : items_(std::make_unique(capacity)) +SafeProcessMap::SafeProcessMap(uint32_t capacity, IComponentController& termination_handler) + : items_(std::make_unique(capacity)), termination_handler_(termination_handler) { if (capacity) { @@ -294,7 +295,7 @@ int32_t SafeProcessMap::search(osal::ProcessID key, ProcessInfoData data) } else if (target.pin_) { - target.pin_->terminated(target.status_); + termination_handler_.terminated(*target.pin_, target.status_); } } } @@ -302,13 +303,12 @@ int32_t SafeProcessMap::search(osal::ProcessID key, ProcessInfoData data) return ret_value; } -SafeProcessMap::SafeProcessMapReturnType SafeProcessMap::findTerminated(osal::ProcessID key, int32_t status) +SafeProcessMapReturnType SafeProcessMap::findTerminated(osal::ProcessID key, int32_t status) { return static_cast(search(key, {status, nullptr})); } -SafeProcessMap::SafeProcessMapReturnType SafeProcessMap::insertIfNotTerminated(osal::ProcessID key, - ITerminationCallback* object) +SafeProcessMapReturnType SafeProcessMap::insertIfNotTerminated(osal::ProcessID key, IComponent* object) { return static_cast(search(key, {0, object})); } diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/safe_process_map.hpp b/score/launch_manager/src/daemon/src/process_group_manager/details/safe_process_map.hpp index 00d5fdf80..67b8c881e 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/safe_process_map.hpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/safe_process_map.hpp @@ -14,9 +14,10 @@ #ifndef SAFE_PROCESS_MAP_HPP_INCLUDED #define SAFE_PROCESS_MAP_HPP_INCLUDED -#include "score/mw/launch_manager/process_group_manager/iprocess.hpp" #include #include +#include "score/mw/launch_manager/process_group_manager/details/icomponent_controller.hpp" +#include "score/mw/launch_manager/process_group_manager/iprocess.hpp" namespace score { @@ -27,32 +28,13 @@ namespace lcm namespace internal { -/// @brief Callback interface for process termination notification. -/// -/// Decouples SafeProcessMap from concrete node types. Any object that needs to -/// be notified when a tracked process terminates implements this interface. -class ITerminationCallback -{ - public: - virtual ~ITerminationCallback() = default; - - /// @brief Called when the associated process has terminated. - /// @param process_status The exit status reported by the operating system. - virtual void terminated(int32_t process_status) = 0; - - protected: - ITerminationCallback() = default; - ITerminationCallback(const ITerminationCallback&) = default; - ITerminationCallback& operator=(const ITerminationCallback&) = default; - ITerminationCallback(ITerminationCallback&&) = default; - ITerminationCallback& operator=(ITerminationCallback&&) = default; -}; +using namespace score::mw::lifecycle::internal; /// @brief Struct representing data in a map item struct ProcessInfoData { int32_t status_ = -1; ///< Exit status for process - ITerminationCallback* pin_ = nullptr; ///< Pointer to the termination callback associated with this item. + IComponent* pin_ = nullptr; ///< Pointer to the termination callback associated with this item. }; /// @brief Struct representing an item in the map. struct ProcessTreeNode @@ -63,34 +45,55 @@ struct ProcessTreeNode ProcessInfoData data_; }; -/// @brief The SafeProcessMap class provides a thread-safe mapping of unique process IDs (ProcessID) to -/// ITerminationCallback pointers. It ensures safe concurrent access and modification of the mapping, using atomic -/// operations. -class SafeProcessMap final +/// @brief Enum type used for public methods retrun value. +enum class SafeProcessMapReturnType : std::int32_t { - public: - /// @brief Enum type used for public methods retrun value. - enum class SafeProcessMapReturnType : std::int32_t - { - /// @brief Method successfully executed. - kOk = 0, + /// @brief Method successfully executed. + kOk = 0, - /// @brief Clash due to PID re-use, method yields until the situation is resolved. - kYield = 1, + /// @brief Clash due to PID re-use, method yields until the situation is resolved. + kYield = 1, - /// @brief An error occurred during insertion (e.g., out of memory). - kInsertionError = -1, + /// @brief An error occurred during insertion (e.g., out of memory). + kInsertionError = -1, - /// @brief The provided process ID (`key`) is not valid ( < 0). - kInvalidIdError = -2, + /// @brief The provided process ID (`key`) is not valid ( < 0). + kInvalidIdError = -2, + + /// @brief The state is not defined. + kUndefined = 2, +}; - /// @brief The state is not defined. - kUndefined = 2, - }; - /// @brief Constructs a SafeProcessMap with a specified capacity. - /// This constructor initializes the SafeProcessMap with the given capacity. - /// @param capacity The maximum number of entries that the SafeProcessMap can hold. - explicit SafeProcessMap(uint32_t capacity); +/// @brief Interface for inserting a process into a process map if it has not already terminated. +class SafeProcessMapInserter +{ + public: + virtual ~SafeProcessMapInserter() = default; + + /// @brief Inserts a process into the map if it has not already terminated. + /// This method is called by a worker thread after starting a process. It attempts to insert the given process ID + /// (key) and its associated ITerminationCallback pointer into the map, ensuring that the process is not already + /// marked as terminated. In the case of a clash due to PID re-use, this method yields until the situation is + /// resolved. + /// @param key The process ID to insert into the map. + /// @param object A pointer to the ITerminationCallback associated with the process. + /// @return kOk if the key (Process ID) was not found and a new entry was made, + /// kYield if the key was found (indicating the process has terminated), and updated with the provided + /// object, kInsertionError if an error occurred during insertion (e.g., out of memory), or kInvalidIdError + /// if the provided process ID (`key`) is not valid ( < 0). + virtual SafeProcessMapReturnType insertIfNotTerminated(osal::ProcessID key, IComponent* object) = 0; +}; + +/// @brief The SafeProcessMap class provides a thread-safe mapping of unique process IDs (ProcessID) to +/// ITerminationCallback pointers. It ensures safe concurrent access and modification of the mapping, using atomic +/// operations. +class SafeProcessMap final : public SafeProcessMapInserter +{ + public: + /// @brief Constructs a SafeProcessMap. + /// @param capacity The maximum number of entries the map can hold. + /// @param termination_handler Called when a terminated process is matched with its component. + SafeProcessMap(uint32_t capacity, IComponentController& termination_handler); /// @brief Destructor to clean up resources used by the SafeProcessMap object. ~SafeProcessMap() = default; @@ -108,17 +111,8 @@ class SafeProcessMap final SafeProcessMapReturnType findTerminated(osal::ProcessID key, int32_t status); /// @brief Inserts a process into the map if it has not already terminated. - /// This method is called by a worker thread after starting a process. It attempts to insert the given process ID - /// (key) and its associated ITerminationCallback pointer into the map, ensuring that the process is not already - /// marked as terminated. In the case of a clash due to PID re-use, this method yields until the situation is - /// resolved. - /// @param key The process ID to insert into the map. - /// @param object A pointer to the ITerminationCallback associated with the process. - /// @return kOk if the key (Process ID) was not found and a new entry was made, - /// kYield if the key was found (indicating the process has terminated), and updated with the provided - /// object, kInsertionError if an error occurred during insertion (e.g., out of memory), or kInvalidIdError - /// if the provided process ID (`key`) is not valid ( < 0). - SafeProcessMapReturnType insertIfNotTerminated(osal::ProcessID key, ITerminationCallback* object); + /// @see SafeProcessMapInserter::insertIfNotTerminated() for details + SafeProcessMapReturnType insertIfNotTerminated(osal::ProcessID key, IComponent* object) override; private: /// @brief Searches for a process with the given process ID (key) in the map. @@ -224,6 +218,8 @@ class SafeProcessMap final /// This variable represents the current index used for traversal within the SafeProcessMap. /// It initially starts with LINK_NO_VALUE, indicating no valid position. uint32_t rover_{LINK_NO_VALUE}; + + IComponentController& termination_handler_; }; } // namespace internal diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/safeprocessmap_UT.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/safeprocessmap_UT.cpp index 63d2bd6dd..6d47f3d58 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/safeprocessmap_UT.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/safeprocessmap_UT.cpp @@ -20,8 +20,8 @@ #include #include "score/mw/launch_manager/common/constants.hpp" +#include "score/mw/launch_manager/process_group_manager/details/mock_component_controller.hpp" #include "score/mw/launch_manager/process_group_manager/details/safe_process_map.hpp" -#include "score/mw/launch_manager/process_group_manager/details/mock_termination_callback.hpp" using namespace testing; using namespace score::lcm::internal; @@ -42,6 +42,17 @@ constexpr int kPidsPerThread = 256; constexpr uint32_t kCapacity = static_cast(ProcessLimits::kMaxProcesses); +class MockComponent : public IComponent +{ + public: + MOCK_METHOD(RequestResult, activate, (score::cpp::stop_token stop_token), (override)); + MOCK_METHOD(RequestResult, deactivate, (score::cpp::stop_token stop_token), (override)); + MOCK_METHOD(RequestResult, tryHandleTermination, (int32_t status), (override)); + MOCK_METHOD(bool, active, (), (const override)); + MOCK_METHOD(bool, stopped, (), (const override)); + MOCK_METHOD(uint32_t, getIndex, (), (const override)); +}; + class SafeProcessMapTest : public ::testing::Test { protected: @@ -51,8 +62,9 @@ class SafeProcessMapTest : public ::testing::Test RecordProperty("DerivationTechnique", "explorative-testing"); } - SafeProcessMap sut_{kCapacity}; - MockTerminationCallback callback_; + NiceMock controller; + SafeProcessMap sut_{kCapacity, controller}; + MockComponent callback_; }; // --- Construction --- @@ -62,51 +74,52 @@ TEST_F(SafeProcessMapTest, ConstructWithZeroCapacity) RecordProperty("Description", "SafeProcessMap can be constructed with zero capacity."); // when - SafeProcessMap map(0); + SafeProcessMap map(0, controller); } // --- findTerminated --- TEST_F(SafeProcessMapTest, FindTerminatedWithNegativePidReturnsInvalid) { - RecordProperty("Description", - "findTerminated returns -score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kUndefined " - "for a negative " - "process ID."); + RecordProperty( + "Description", + "findTerminated returns -score::lcm::internal::SafeProcessMapReturnType::kUndefined " + "for a negative " + "process ID."); // when - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType result = sut_.findTerminated(-1, 1000); + score::lcm::internal::SafeProcessMapReturnType result = sut_.findTerminated(-1, 1000); // then - EXPECT_EQ(result, score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kInvalidIdError); + EXPECT_EQ(result, score::lcm::internal::SafeProcessMapReturnType::kInvalidIdError); } TEST_F(SafeProcessMapTest, FindTerminatedInsertsEntryWhenPidNotPresent) { - RecordProperty("Description", - "findTerminated inserts an entry and returns kYield (1) when the PID is not in the map."); + RecordProperty( + "Description", "findTerminated inserts an entry and returns kYield (1) when the PID is not in the map."); // when - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType result = sut_.findTerminated(1000, 0); + score::lcm::internal::SafeProcessMapReturnType result = sut_.findTerminated(1000, 0); // then - EXPECT_EQ(result, score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); + EXPECT_EQ(result, score::lcm::internal::SafeProcessMapReturnType::kYield); } TEST_F(SafeProcessMapTest, FindTerminatedMatchesExistingInsertAndCallsCallback) { - RecordProperty("Description", - "findTerminated matches an existing insert entry and invokes the termination callback."); + RecordProperty( + "Description", "findTerminated matches an existing insert entry and invokes the termination callback."); // given sut_.insertIfNotTerminated(1000, &callback_); // then - EXPECT_CALL(callback_, terminated(42)); + EXPECT_CALL(controller, terminated(Ref(callback_), 42)); // when - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType result = sut_.findTerminated(1000, 42); - EXPECT_EQ(result, score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kOk); + score::lcm::internal::SafeProcessMapReturnType result = sut_.findTerminated(1000, 42); + EXPECT_EQ(result, score::lcm::internal::SafeProcessMapReturnType::kOk); } // --- insertIfNotTerminated --- @@ -116,11 +129,11 @@ TEST_F(SafeProcessMapTest, InsertIntoEmptyTreeReturnsZero) RecordProperty("Description", "insertIfNotTerminated returns kOk (0) when inserting into an empty tree."); // when - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType result = + score::lcm::internal::SafeProcessMapReturnType result = sut_.insertIfNotTerminated(2000, &callback_); // then - EXPECT_EQ(result, score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kOk); + EXPECT_EQ(result, score::lcm::internal::SafeProcessMapReturnType::kOk); } TEST_F(SafeProcessMapTest, InsertMatchesExistingFindTerminatedEntry) @@ -132,23 +145,23 @@ TEST_F(SafeProcessMapTest, InsertMatchesExistingFindTerminatedEntry) // given sut_.findTerminated(1000, 0); - EXPECT_CALL(callback_, terminated(0)); + EXPECT_CALL(controller, terminated(Ref(callback_), 0)); // when - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType result = + score::lcm::internal::SafeProcessMapReturnType result = sut_.insertIfNotTerminated(1000, &callback_); // then - EXPECT_EQ(result, score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); + EXPECT_EQ(result, score::lcm::internal::SafeProcessMapReturnType::kYield); } TEST_F(SafeProcessMapTest, InsertMultipleNodesThenFindTerminatedRemovesAll) { - RecordProperty("Description", - "Inserting kMaxProcesses nodes and then calling findTerminated for each returns kOk (0)."); + RecordProperty( + "Description", "Inserting kMaxProcesses nodes and then calling findTerminated for each returns kOk (0)."); // given - NiceMock callbacks[kCapacity]; + NiceMock callbacks[kCapacity]; for (uint32_t i = 1; i <= kCapacity; ++i) { sut_.insertIfNotTerminated(static_cast(i), &callbacks[i - 1]); @@ -157,8 +170,9 @@ TEST_F(SafeProcessMapTest, InsertMultipleNodesThenFindTerminatedRemovesAll) // when / then for (uint32_t j = 1; j <= kCapacity; ++j) { - EXPECT_EQ(sut_.findTerminated(static_cast(j), 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kOk); + EXPECT_EQ( + sut_.findTerminated(static_cast(j), 0), + score::lcm::internal::SafeProcessMapReturnType::kOk); } } @@ -169,36 +183,38 @@ TEST_F(SafeProcessMapTest, InsertBeyondCapacityReturnsOutOfMemory) "insertIfNotTerminated returns kInsertionError (-1) when the map is full and a new entry is attempted."); // given - NiceMock callbacks[kCapacity]; + NiceMock callbacks[kCapacity]; for (uint32_t i = 0; i < kCapacity; ++i) { - EXPECT_EQ(sut_.insertIfNotTerminated(static_cast(i), &callbacks[i]), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kOk); + EXPECT_EQ( + sut_.insertIfNotTerminated(static_cast(i), &callbacks[i]), + score::lcm::internal::SafeProcessMapReturnType::kOk); } // when - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType result = + score::lcm::internal::SafeProcessMapReturnType result = sut_.insertIfNotTerminated(static_cast(kCapacity + 1), &callback_); // then - EXPECT_EQ(result, score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kInsertionError); + EXPECT_EQ(result, score::lcm::internal::SafeProcessMapReturnType::kInsertionError); } // --- Anomalous (PID reuse) cases --- TEST_F(SafeProcessMapTest, InsertSamePidTwiceYieldsUntilFindTerminatedResolves) { - RecordProperty("Description", - "Inserting the same PID twice causes the second insert to yield until findTerminated resolves it."); + RecordProperty( + "Description", + "Inserting the same PID twice causes the second insert to yield until findTerminated resolves it."); // given std::atomic_bool first_done{false}; - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType ret1 = - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kUndefined; - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType ret2 = - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kUndefined; + score::lcm::internal::SafeProcessMapReturnType ret1 = + score::lcm::internal::SafeProcessMapReturnType::kUndefined; + score::lcm::internal::SafeProcessMapReturnType ret2 = + score::lcm::internal::SafeProcessMapReturnType::kUndefined; - NiceMock cb; + NiceMock cb; std::thread inserter([&]() { ret1 = sut_.insertIfNotTerminated(42, &cb); @@ -212,31 +228,32 @@ TEST_F(SafeProcessMapTest, InsertSamePidTwiceYieldsUntilFindTerminatedResolves) std::this_thread::sleep_for(std::chrono::milliseconds(10)); // then — first succeeded, second is still blocked - EXPECT_EQ(ret1, score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kOk); - EXPECT_EQ(ret2, score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kUndefined); + EXPECT_EQ(ret1, score::lcm::internal::SafeProcessMapReturnType::kOk); + EXPECT_EQ(ret2, score::lcm::internal::SafeProcessMapReturnType::kUndefined); // when — resolve the anomaly - EXPECT_EQ(sut_.findTerminated(42, 0), score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kOk); + EXPECT_EQ(sut_.findTerminated(42, 0), score::lcm::internal::SafeProcessMapReturnType::kOk); inserter.join(); // then - EXPECT_EQ(ret2, score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kOk); + EXPECT_EQ(ret2, score::lcm::internal::SafeProcessMapReturnType::kOk); } TEST_F(SafeProcessMapTest, FindTerminatedSamePidTwiceYieldsUntilInsertResolves) { - RecordProperty("Description", - "Calling findTerminated twice with the same PID causes the second call to yield " - "until insertIfNotTerminated resolves it."); + RecordProperty( + "Description", + "Calling findTerminated twice with the same PID causes the second call to yield " + "until insertIfNotTerminated resolves it."); // given std::atomic_bool first_done{false}; - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType ret1 = - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kUndefined; - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType ret2 = - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kUndefined; + score::lcm::internal::SafeProcessMapReturnType ret1 = + score::lcm::internal::SafeProcessMapReturnType::kUndefined; + score::lcm::internal::SafeProcessMapReturnType ret2 = + score::lcm::internal::SafeProcessMapReturnType::kUndefined; - NiceMock cb; + NiceMock cb; std::thread finder([&]() { ret1 = sut_.findTerminated(42, 0); @@ -250,16 +267,16 @@ TEST_F(SafeProcessMapTest, FindTerminatedSamePidTwiceYieldsUntilInsertResolves) std::this_thread::sleep_for(std::chrono::milliseconds(10)); // then — first succeeded, second is still blocked - EXPECT_EQ(ret1, score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(ret2, score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kUndefined); + EXPECT_EQ(ret1, score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ(ret2, score::lcm::internal::SafeProcessMapReturnType::kUndefined); // when — resolve the anomaly - EXPECT_EQ(sut_.insertIfNotTerminated(42, &cb), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.insertIfNotTerminated(42, &cb), score::lcm::internal::SafeProcessMapReturnType::kYield); finder.join(); // then - EXPECT_EQ(ret2, score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); + EXPECT_EQ(ret2, score::lcm::internal::SafeProcessMapReturnType::kYield); } // --- Max depth tree --- @@ -269,102 +286,109 @@ TEST_F(SafeProcessMapTest, FindTerminatedWorksAtMaxTreeDepth) RecordProperty("Description", "The binary tree handles maximum depth correctly."); // given — build a deep tree using bit patterns that always branch one way - EXPECT_EQ(sut_.findTerminated(0x00000000, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x00000001, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x00000002, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x00000003, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x00000007, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x0000000F, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x0000001F, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x0000003F, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x0000007F, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x000000FF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x000001FF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x000003FF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x000007FF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x00000FFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x00001FFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x00003FFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x00007FFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x0000FFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x0000FFFE, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x0001FFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x0003FFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x0007FFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x000FFFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x001FFFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x003FFFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x007FFFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x00FFFFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x01FFFFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x03FFFFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x07FFFFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x0FFFFFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x1FFFFFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x3FFFFFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.findTerminated(0x7FFFFFFF, 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x00000000, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x00000001, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x00000002, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x00000003, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x00000007, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x0000000F, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x0000001F, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x0000003F, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x0000007F, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x000000FF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x000001FF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x000003FF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x000007FF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x00000FFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x00001FFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x00003FFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x00007FFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x0000FFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x0000FFFE, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x0001FFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x0003FFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x0007FFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x000FFFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x001FFFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x003FFFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x007FFFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x00FFFFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x01FFFFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x03FFFFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x07FFFFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x0FFFFFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x1FFFFFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x3FFFFFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.findTerminated(0x7FFFFFFF, 0), score::lcm::internal::SafeProcessMapReturnType::kYield); // when / then — boundary values - EXPECT_EQ(sut_.findTerminated(static_cast(0xFFFFFFFF), 0), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kInvalidIdError); - EXPECT_EQ(sut_.insertIfNotTerminated(static_cast(0xFFFFFFFF), &callback_), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kInvalidIdError); + EXPECT_EQ( + sut_.findTerminated(static_cast(0xFFFFFFFF), 0), + score::lcm::internal::SafeProcessMapReturnType::kInvalidIdError); + EXPECT_EQ( + sut_.insertIfNotTerminated(static_cast(0xFFFFFFFF), &callback_), + score::lcm::internal::SafeProcessMapReturnType::kInvalidIdError); // when / then — retrieve entries using insertIfNotTerminated - NiceMock cb; - EXPECT_EQ(sut_.insertIfNotTerminated(0x0000FFFE, &cb), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.insertIfNotTerminated(0x00010000, &cb), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kOk); - EXPECT_EQ(sut_.insertIfNotTerminated(0x0001FFFF, &cb), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); - EXPECT_EQ(sut_.insertIfNotTerminated(0x00000002, &cb), - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); + NiceMock cb; + EXPECT_EQ( + sut_.insertIfNotTerminated(0x0000FFFE, &cb), + score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.insertIfNotTerminated(0x00010000, &cb), + score::lcm::internal::SafeProcessMapReturnType::kOk); + EXPECT_EQ( + sut_.insertIfNotTerminated(0x0001FFFF, &cb), + score::lcm::internal::SafeProcessMapReturnType::kYield); + EXPECT_EQ( + sut_.insertIfNotTerminated(0x00000002, &cb), + score::lcm::internal::SafeProcessMapReturnType::kYield); } // --- Multi-threaded stress tests --- TEST_F(SafeProcessMapTest, ConcurrentInsertAndFindFromMultipleThreads) { - RecordProperty("Description", - "Multiple threads concurrently inserting and finding terminated processes completes without error."); + RecordProperty( + "Description", + "Multiple threads concurrently inserting and finding terminated processes completes without error."); - NiceMock stubs[kNumThreads]; - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType results[kNumThreads] = {}; + NiceMock stubs[kNumThreads]; + score::lcm::internal::SafeProcessMapReturnType results[kNumThreads] = {}; // when std::vector threads; @@ -395,17 +419,17 @@ TEST_F(SafeProcessMapTest, ConcurrentInsertAndFindFromMultipleThreads) // then for (int t = 0; t < kNumThreads; ++t) { - EXPECT_EQ(results[t], score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kOk); + EXPECT_EQ(results[t], score::lcm::internal::SafeProcessMapReturnType::kOk); } } TEST_F(SafeProcessMapTest, ConcurrentFindAndInsertFromMultipleThreads) { - RecordProperty("Description", - "Multiple threads concurrently finding and inserting processes completes without error."); + RecordProperty( + "Description", "Multiple threads concurrently finding and inserting processes completes without error."); - NiceMock stubs[kNumThreads]; - score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType results[kNumThreads] = {}; + NiceMock stubs[kNumThreads]; + score::lcm::internal::SafeProcessMapReturnType results[kNumThreads] = {}; // when std::vector threads; @@ -436,7 +460,7 @@ TEST_F(SafeProcessMapTest, ConcurrentFindAndInsertFromMultipleThreads) // then for (int t = 0; t < kNumThreads; ++t) { - EXPECT_EQ(results[t], score::lcm::internal::SafeProcessMap::SafeProcessMapReturnType::kYield); + EXPECT_EQ(results[t], score::lcm::internal::SafeProcessMapReturnType::kYield); } } diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/transition.hpp b/score/launch_manager/src/daemon/src/process_group_manager/details/transition.hpp new file mode 100644 index 000000000..0b5504896 --- /dev/null +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/transition.hpp @@ -0,0 +1,476 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#ifndef SCORE_LCM_TRANSITION_HPP +#define SCORE_LCM_TRANSITION_HPP + +#include "score/mw/launch_manager/common/concurrency/fixed_size_queue.hpp" +#include "score/mw/launch_manager/process_group_manager/details/dependency_graph.hpp" +#include "score/mw/launch_manager/process_group_manager/details/icomponent.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace score::mw::lifecycle +{ +using namespace score::lcm::internal; + +/// @brief What should happen to a ready node right now. +enum class Action : std::uint8_t +{ + Stop, ///< deactivate the node + Start, ///< activate the node +}; + +/// @brief Contains a node that is ready to be activated/deactivated +struct ReadyNode +{ + GraphIndex node; + Action action; +}; + +inline bool operator==(const ReadyNode& lhs, const ReadyNode& rhs) +{ + return lhs.node == rhs.node && lhs.action == rhs.action; +} +inline bool operator!=(const ReadyNode& lhs, const ReadyNode& rhs) +{ + return !(lhs == rhs); +} + +template +class TransitionBuilder; + +/// @brief The Transition computes the nodes to activate/deactivate when transitioning between states in the graph. +/// @details When transitioning to a target state, the Transition computes the nodes that need to be +/// deactivated (those nodes not reachable from the target state) and those that need to be activated (those reachable +/// from the target that are not yet active). This list is then used to drive the transition by iterating over the ready +/// nodes (@ref nextReady()) and marking them as finished once activated/deactivated (@ref onNodeFinished()). The +/// transition is split into two phases: Stopping phase and Starting phase. Stopping Phase: Every node that is currently +/// running (@ref stopped() is false) and is not part of the target subgraph is deactivated. The stop set is derived +/// from live component state across the whole graph rather than from a source subgraph, so it also captures nodes left +/// running by a previously aborted transition. Once all nodes in the Stopping phase are finished, the transition moves +/// to the Starting phase. Starting Phase: The nodes that need to be activated are processed next. Once all nodes in the +/// Starting phase are finished, the transition is complete. +/// +/// The template parameter is expected to be a type that can be used to retrieve the corresponding IComponent instance +/// via the componentOf() function, which is expected to be defined for the type T. +/// +/// Example Usage: +/// +/// TransitionBuilder> builder(graph); +/// auto& transition = builder.createTransition(target); +/// for (const auto [node, action] : current_transition_) +/// { +/// if (action == Action::Start) { +/// // queue activation of the node +/// } else { +/// // queue deactivation of the node +/// } +/// } +/// // ... later on, when a node finishes activation/deactivation: +/// transition.onNodeFinished(node); +/// +/// Note: It is also supported to call @ref onNodeFinished() while iterating the ready nodes. +/// This may be needed in case of node types that can be activated/deactivated synchronously, so that their +/// successors can be dispatched immediately. +/// +namespace detail +{ +/// @brief True iff componentOf(U&) is callable (via ADL) and yields a reference convertible to +/// IComponent&. Used to enforce the requirement of the Transition template parameter. +template +struct is_component_type : std::false_type +{ +}; + +template +struct is_component_type()))>> + : std::is_convertible())), internal::IComponent&> +{ +}; +} // namespace detail + +template +class Transition +{ + static_assert( + detail::is_component_type::value, + "Transition requires an ADL-findable componentOf(T&) that returns a reference " + "to IComponent&."); + + friend class TransitionBuilder; + + public: + /// @brief Pop the next ready node, or std::nullopt if none is ready right now + /// @details Unlike a getter, this CONSUMES: a node is returned at most once and + /// is gone from the frontier the moment it's returned. Safe to interleave + /// with onNodeFinished() — nodes onNodeFinished() appends are queued behind + /// whatever's already pending, never lost, regardless of consumption order. + std::optional nextReady() + { + if (state_.next_nodes.empty()) + { + return std::nullopt; + } + return ReadyNode{state_.next_nodes.tryPop().value(), currentAction()}; + } + + /// @brief Input iterator that drains the transition via nextReady(). + /// @details Holds only a pointer + a cached ReadyNode — no allocation. + /// CONSUMING: advancing pops from the shared frontier, so only one traversal (e.g. one range-for) + /// should be in flight at a time; a second `begin()` continues where the + /// first left off, it does not restart. + class Iterator + { + public: + using value_type = ReadyNode; + using reference = ReadyNode; + using difference_type = std::ptrdiff_t; + using iterator_category = std::input_iterator_tag; + using pointer = void; + + Iterator() = default; // end sentinel: owner_ == nullptr + explicit Iterator(Transition* owner) : owner_(owner) + { + advance(); + } + + ReadyNode operator*() const + { + return *current_; + } + Iterator& operator++() + { + advance(); + return *this; + } + bool operator==(const Iterator& other) const + { + return current_.has_value() == other.current_.has_value(); + } + bool operator!=(const Iterator& other) const + { + return !(*this == other); + } + + private: + void advance() + { + current_ = owner_ ? owner_->nextReady() : std::nullopt; + } + + Transition* owner_ = nullptr; + std::optional current_; + }; + + Iterator begin() + { + return Iterator{this}; + } + Iterator end() + { + return Iterator{}; + } + + /// @brief Mark @p node finished and refresh the list of ready nodes + /// @details A successor becomes ready when: + /// activation: it is part of the target subgraph, and every dependency it + /// has is active() + /// deactivation: every dependent it has is stopped() + void onNodeFinished(GraphIndex node) + { + SCORE_LANGUAGE_FUTURECPP_ASSERT(isValidNode(node)); + + --state_.pending; // completion counter; only detects done-ness, never gates order + + if (state_.pending == 0) // active phase drained + { + if (state_.phase == Phase::Stopping) + { + advanceToStarting(); // sets up the starting-phase heads + } + else + { + state_.phase = Phase::Done; + } + return; + } + + // Still within the phase: append the finished node's neighbours in the + // phase's direction, filtered by readiness, behind whatever's already + // waiting to be dispatched. + const auto& successors = state_.phase == Phase::Starting ? graph_.dependents(node) : graph_.dependsOn(node); + for (const GraphIndex s : successors) + { + if (isReady(s)) + { + SCORE_LANGUAGE_FUTURECPP_ASSERT_MESSAGE( + state_.next_nodes.push(s), "Transition queue should never exceed capacity"); + } + } + } + + /// @return True once every node participating in this transition has reached + /// its terminal state. + bool isFinished() const + { + return state_.phase == Phase::Done; + } + + private: + /// @brief True iff @p node is a valid index into the underlying graph, i.e. in [0, size()). + bool isValidNode(GraphIndex node) const + { + return node < graph_.size(); + } + + /// @brief Construct a reusable Transition for the given graph. + /// @details All the memory needed for a transition is allocated here, so that no further allocations are + /// needed while the transition is in flight. The same transition object is then reused for multiple transitions by + /// calling @ref setupTransition() with a new target node. + explicit Transition(DependencyGraph& graph) : state_(graph.capacity()), graph_(graph) + { + state_.in_target_subgraph.assign(graph.capacity(), false); + } + + /// @brief Set up a fresh transition to @p target. + /// @details Starts in the Stopping Phase: every node currently running and not needed by @p target is + /// deactivated (derived from live component state across the whole graph, so nodes left running by a previous + /// aborted transition are captured too). Then moves to the Starting Phase to bring up @p target. + void setupTransition(GraphIndex target) + { + beginSetup(target); + setupDeactivation(target); + if (state_.pending == 0) + { + advanceToStarting(); + } + } + + enum class Phase : std::uint8_t + { + Stopping, ///< Deactivating nodes exclusive to the `source` state + Starting, ///< Activating new nodes that are exclusive to the `target` state + Done, ///< every participating node has reached its terminal state + }; + + struct State + { + /// @brief Per-node membership mask + /// @details in_target_subgraph[i] is true iff node + /// i is reachable from target_root (i.e. belongs to the subgraph about to + /// be running). Recomputed at every setup. Serves two purposes: + /// stopping: the nodes excluded from the whole-graph stop scan, and the + /// nodes onNodeFinished must never (re)stop. + /// starting: nodes that are directly or indirectly depended on by + /// the target_root. + std::vector in_target_subgraph; + + /// @brief The destination subgraph's root (the `target` endpoint) + GraphIndex target_root{}; + + /// @brief The nodes that are ready to be activated/deactivated in the current phase, in the order they were + /// discovered. + FixedSizeQueue next_nodes; + std::size_t pending = 0; // nodes still to reach terminal state in this phase + Phase phase = Phase::Done; // active vs deactivation vs finished + + State(std::size_t nodes) : next_nodes(nodes) + { + } + }; + + /// @brief Check if the node is active + bool active(GraphIndex i) + { + return componentOf(graph_[i]).active(); + } + + /// @brief Check if the node is stopped + bool stopped(GraphIndex i) + { + return !componentOf(graph_[i]).active(); + } + + /// @brief Check if all dependencies of the given node are active + bool allDepsActive(GraphIndex i) + { + const auto& d = graph_.dependsOn(i); + return std::all_of(d.begin(), d.end(), [this](GraphIndex dep) { + return active(dep); + }); + } + + /// @brief Check if all dependents of the given node are stopped + bool allDependentsStopped(GraphIndex i) + { + const auto& d = graph_.dependents(i); + return std::all_of(d.begin(), d.end(), [this](GraphIndex dep) { + return stopped(dep); + }); + } + + State state_; + DependencyGraph& graph_; + + /// @brief The action based on whether the transition is in the Stopping or Starting phase + Action currentAction() const + { + return state_.phase == Phase::Starting ? Action::Start : Action::Stop; + } + + /// @brief Check if the node is ready to be activated/deactivated in the current phase. + bool isReady(GraphIndex s) + { + return state_.phase == Phase::Starting + ? (state_.in_target_subgraph[s] && !active(s) && allDepsActive(s)) + : (!state_.in_target_subgraph[s] && !stopped(s) && allDependentsStopped(s)); + } + + void clearNextNodes() + { + while (!state_.next_nodes.empty()) + { + static_cast(state_.next_nodes.tryPop()); + } + } + + /// @brief Reset the internal state before setting up the next phase + void beginSetup(GraphIndex target) + { + state_.target_root = target; + clearNextNodes(); + state_.pending = 0; + state_.phase = Phase::Stopping; + } + + /// @brief Leave the Stopping phase for the Starting phase. + /// @details Sets up activation of the `target` subgraph; if there is nothing to start (everything is + /// already active()), the transition is finished. + void advanceToStarting() + { + state_.phase = Phase::Starting; + state_.pending = 0; + clearNextNodes(); + + setupActivation(state_.target_root); + if (state_.pending == 0) + { + state_.phase = Phase::Done; + } + } + + /// @brief Setup the starting phase of the transition + /// @details This will traverse the target subgraph and initialize the following bookkeeping structures: + /// - in_target_subgraph: marks the nodes that are part of the target subgraph + /// - next_nodes: the list of nodes that are ready to be activated (those whose dependencies are all active) + /// - pending: the count of nodes that are still to be activated + void setupActivation(GraphIndex root) + { + std::fill(state_.in_target_subgraph.begin(), state_.in_target_subgraph.end(), false); + graph_.traverse( + root, + [this](GraphIndex i) -> const std::vector& { + state_.in_target_subgraph[i] = true; + if (!active(i)) + { + ++state_.pending; + if (allDepsActive(i)) + { + state_.next_nodes.push(i); + } + } + return graph_.dependsOn(i); + }, + [](GraphIndex) { + return true; + }); + } + + /// @brief Setup the Stopping phase of the transition + /// @details First traverses the target subgraph to initialize: + /// - in_target_subgraph: marks the nodes that are part of the target subgraph (must stay running) + /// Then scans the complete graph to initialize: + /// - pending: the count of running nodes that must be deactivated + /// - next_nodes: those already ready to be deactivated (dependents all stopped) + /// The stop set is every node that is currently running (@ref stopped() is false) and is not in the target + /// subgraph. Deriving it from live component state rather than from a source root makes it independent of how the + /// previous transition ended, so nodes left running by an aborted transition — even ones outside any assumed + /// source subgraph — are still stopped. + void setupDeactivation(GraphIndex target) + { + std::fill(state_.in_target_subgraph.begin(), state_.in_target_subgraph.end(), false); + graph_.traverse( + target, + [this](GraphIndex i) -> const std::vector& { + state_.in_target_subgraph[i] = true; + return graph_.dependsOn(i); + }, + [](GraphIndex) { + return true; + }); + for (GraphIndex i = 0; i < graph_.size(); ++i) + { + if (!state_.in_target_subgraph[i] && !stopped(i)) + { + ++state_.pending; + if (allDependentsStopped(i)) + { + state_.next_nodes.push(i); + } + } + } + } +}; + +/// @brief The TransitionBuilder owns the single Transition object for a given graph, which is reused for each +/// Transition. +/// @details The builder only supports a single transition at a time. It is +/// expected that whenever a new transition is created, the previous one is no longer in use. +/// The reason is that Memory is only allocated during initialization and then reused for each transition. +template +class TransitionBuilder final +{ + public: + explicit TransitionBuilder(DependencyGraph& graph) : transition_(graph) + { + } + + /// @brief A transition to @p target + /// @details First deactivates every node currently running that is not needed by @p target (keeping anything + /// shared with @p target active), then activates all nodes reachable from @p target. The stop set is derived from + /// live component state, so this recovers correctly even when a previous transition was aborted mid-flight. + Transition& createTransition(GraphIndex target) + { + SCORE_LANGUAGE_FUTURECPP_ASSERT(transition_.isValidNode(target)); + transition_.setupTransition(target); + return transition_; + } + + private: + /// @brief The single reusable transition + Transition transition_; +}; + +} // namespace score::mw::lifecycle + +#endif // SCORE_LCM_TRANSITION_HPP diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/transition_UT.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/transition_UT.cpp new file mode 100644 index 000000000..445f1209b --- /dev/null +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/transition_UT.cpp @@ -0,0 +1,660 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include "score/mw/launch_manager/process_group_manager/details/dependency_graph.hpp" +#include "score/mw/launch_manager/process_group_manager/details/icomponent.hpp" +#include "score/mw/launch_manager/process_group_manager/details/transition.hpp" + +#include +#include + +#include +#include + +namespace score::mw::lifecycle::internal +{ + +/// @brief Mock IComponent backed by two flags. Transition only ever reads active()/stopped(); +/// the rest of the interface is mocked purely to satisfy IComponent and is never exercised. +class MockComponent : public IComponent +{ + public: + MockComponent() + { + ON_CALL(*this, active()).WillByDefault(::testing::ReturnPointee(&active_)); + } + + MOCK_METHOD(RequestResult, activate, (score::cpp::stop_token), (override)); + MOCK_METHOD(RequestResult, deactivate, (score::cpp::stop_token), (override)); + MOCK_METHOD(RequestResult, tryHandleTermination, (int32_t), (override)); + MOCK_METHOD(uint32_t, getIndex, (), (const, override)); + MOCK_METHOD(bool, active, (), (const, override)); + + /// @brief Flip both flags together, mirroring a real component reaching a terminal state. + void setActive(bool is_active) + { + active_ = is_active; + stopped_ = !is_active; + } + + // Default: inactive and fully stopped. + bool active_ = false; + bool stopped_ = true; +}; + +/// @brief Test-only projection for Transition, mirroring component_of.hpp's +/// production overload. Declared in this namespace so Transition finds it via ADL. +inline IComponent& componentOf(IComponent* node) +{ + return *node; +} + +using ComponentType = internal::IComponent*; + +/// @brief Base fixture: owns a DependencyGraph plus the address-stable mocks its nodes point to, +/// and offers small helpers shared by every test. +class TransitionTest : public ::testing::Test +{ + protected: + void SetUp() override + { + RecordProperty("TestType", "interface-test"); + RecordProperty("DerivationTechnique", "explorative-testing"); + } + + /// @brief (Re)create the underlying graph sized for exactly @p node_count nodes, plus a builder + /// over it. The graph's capacity is fixed here, so the builder can be constructed before any + /// nodes are added. + void makeGraph(std::size_t node_count) + { + graph_ = std::make_unique>(node_count); + components_.clear(); + builder_ = std::make_unique>(*graph_); + } + + /// @brief Add a fresh mock-backed node and return its index. + GraphIndex addNode() + { + components_.push_back(std::make_unique<::testing::NiceMock>()); + return graph_->emplace(components_.back().get()); + } + + internal::MockComponent& componentAt(GraphIndex i) + { + return *components_[i]; + } + + /// @brief Mark @p node active and report it finished — mimics an activation completing. + void activate(Transition& t, GraphIndex node) + { + componentAt(node).setActive(true); + t.onNodeFinished(node); + } + + /// @brief Mark @p node stopped and report it finished — mimics a deactivation completing. + void deactivate(Transition& t, GraphIndex node) + { + componentAt(node).setActive(false); + t.onNodeFinished(node); + } + + /// @brief Drain everything ready right now into a vector so it can be matched. + /// @details Iterating CONSUMES the frontier, so call this once per step (after each + /// onNodeFinished()), not repeatedly for the same step. + static std::vector collectReady(Transition& t) + { + std::vector out; + for (const ReadyNode rn : t) + { + out.push_back(rn); + } + return out; + } + + std::unique_ptr> graph_; + std::vector>> components_; + std::unique_ptr> builder_; +}; + +// --------------------------------------------------------------------------- +// Empty graph +// --------------------------------------------------------------------------- + +/// @brief Fixture for the degenerate empty graph (no nodes). Every possible node index is out of +/// range, so the builder's create*() methods have no valid argument. +class EmptyGraphTest : public TransitionTest +{ + protected: + void SetUp() override + { + TransitionTest::SetUp(); + makeGraph(0); + } +}; + +using EmptyGraphDeathTest = EmptyGraphTest; + +TEST_F(EmptyGraphTest, BuilderConstructsButGraphIsEmpty) +{ + RecordProperty( + "Description", + "A TransitionBuilder can be constructed over an empty graph (no nodes). There are no node " + "indices, so no transition can be created or driven."); + + EXPECT_EQ(graph_->size(), 0U); +} + +TEST_F(EmptyGraphDeathTest, CreateTransitionAssertsOnOutOfRangeTarget) +{ + RecordProperty( + "Description", "createTransition() with an out-of-range target index aborts via a futurecpp assert."); + + EXPECT_DEATH(builder_->createTransition(0), ""); +} + +// --------------------------------------------------------------------------- +// Single node +// --------------------------------------------------------------------------- + +/// @brief Fixture for a graph with exactly one node and no edges. That node is both a leaf (no +/// dependencies) and a root (no dependents), so it is always ready the instant a transition needs +/// it — nothing gates it. +class SingleNodeGraphTest : public TransitionTest +{ + protected: + void SetUp() override + { + TransitionTest::SetUp(); + makeGraph(1); + node_ = addNode(); + } + + GraphIndex node_{}; +}; + +TEST_F(SingleNodeGraphTest, TransitionStartsTheNode) +{ + RecordProperty( + "Description", + "A single inactive node with no dependencies is immediately ready to start; the transition " + "finishes once the node reports active."); + + auto& transition = builder_->createTransition(node_); + + EXPECT_FALSE(transition.isFinished()); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{node_, Action::Start})); + + activate(transition, node_); + + EXPECT_THAT(collectReady(transition), ::testing::IsEmpty()); + EXPECT_TRUE(transition.isFinished()); +} + +TEST_F(SingleNodeGraphTest, TransitionToAlreadyActiveNodeIsImmediatelyFinished) +{ + RecordProperty( + "Description", + "A node that is already active is not started again; the transition is finished " + "immediately with nothing ready."); + + componentAt(node_).setActive(true); + + auto& transition = builder_->createTransition(node_); + + EXPECT_THAT(collectReady(transition), ::testing::IsEmpty()); + EXPECT_TRUE(transition.isFinished()); +} + +TEST_F(SingleNodeGraphTest, TransitionToOffStopsTheRunningNodeThenStartsOff) +{ + RecordProperty( + "Description", + "Transitioning to the Off target stops a running node not needed by Off, then activates the " + "dependency-less Off node; the transition finishes once both reach their terminal state."); + + makeGraph(2); + const GraphIndex node = addNode(); + const GraphIndex off = addNode(); + componentAt(node).setActive(true); // node running; Off node stopped (default) + + auto& transition = builder_->createTransition(off); + + // Stopping phase: the running node is stopped first (Off does not depend on it). + EXPECT_FALSE(transition.isFinished()); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{node, Action::Stop})); + deactivate(transition, node); + + // Starting phase: the Off node is now ready to activate. + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{off, Action::Start})); + activate(transition, off); + + EXPECT_THAT(collectReady(transition), ::testing::IsEmpty()); + EXPECT_TRUE(transition.isFinished()); +} + +TEST_F(SingleNodeGraphTest, TransitionToOffFromAllStoppedStartsTheOffNode) +{ + RecordProperty( + "Description", + "With nothing running, transitioning to the Off target skips the stopping phase and simply " + "activates the dependency-less Off node, leaving the stopped application node untouched."); + + makeGraph(2); + const GraphIndex node = addNode(); // an application node, left stopped + const GraphIndex off = addNode(); + + auto& transition = builder_->createTransition(off); + + EXPECT_FALSE(transition.isFinished()); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{off, Action::Start})); + activate(transition, off); + + EXPECT_THAT(collectReady(transition), ::testing::IsEmpty()); + EXPECT_TRUE(transition.isFinished()); + EXPECT_TRUE(componentAt(node).stopped_); + EXPECT_TRUE(componentAt(off).active_); +} + +// --------------------------------------------------------------------------- +// Two RunTargets sharing a node +// --------------------------------------------------------------------------- + +/// @brief Fixture for the graph from the original transition_UT.cpp: two RunTargets that share a +/// common dependency, plus the standalone Off RunTarget. +/// +/// ┌────────┐ ┌────────┐ ┌────────┐ +/// │ RT2 │ │ RT1 │ │ Off │ +/// └───┬─┬──┘ └──┬─┬───┘ └────────┘ +/// │ └───────┐ ┌───────┘ │ (no dependencies) +/// │ │ │ │ +/// ┌──▼──┐ ┌──▼────▼───┐ ┌──▼──┐ +/// │ A │ │ B │ │ C │ +/// └─────┘ └───────────┘ └─────┘ +/// +/// (X -> Y means X depends on Y) +/// RT1 -> C, B +/// RT2 -> A, B (B is shared by both RunTargets) +/// Off (no dependencies: reaching it stops everything else) +class SharedNodeGraphTest : public TransitionTest +{ + protected: + void SetUp() override + { + TransitionTest::SetUp(); + makeGraph(6); + a_ = addNode(); + b_ = addNode(); + c_ = addNode(); + rt1_ = addNode(); + rt2_ = addNode(); + off_ = addNode(); // Off RunTarget: no dependencies, so reaching it stops everything else + graph_->addDependency(rt1_, c_); + graph_->addDependency(rt1_, b_); + graph_->addDependency(rt2_, a_); + graph_->addDependency(rt2_, b_); + } + + GraphIndex a_{}; + GraphIndex b_{}; + GraphIndex c_{}; + GraphIndex rt1_{}; + GraphIndex rt2_{}; + GraphIndex off_{}; +}; + +TEST_F(SharedNodeGraphTest, TransitionStartsDependenciesBeforeDependent) +{ + RecordProperty( + "Description", + "Bringing up RT1 from scratch starts its dependencies B and C first; RT1 only becomes ready " + "once both of them are active."); + + auto& transition = builder_->createTransition(rt1_); + EXPECT_THAT( + collectReady(transition), + ::testing::UnorderedElementsAre(ReadyNode{c_, Action::Start}, ReadyNode{b_, Action::Start})); + + activate(transition, c_); + EXPECT_THAT(collectReady(transition), ::testing::IsEmpty()); // still blocked on B + + activate(transition, b_); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{rt1_, Action::Start})); + + activate(transition, rt1_); + EXPECT_THAT(collectReady(transition), ::testing::IsEmpty()); + EXPECT_TRUE(transition.isFinished()); +} + +TEST_F(SharedNodeGraphTest, NextReadyInterleavedWithOnNodeFinishedKeepsPendingSibling) +{ + RecordProperty( + "Description", + "Popping one of two simultaneously-ready siblings via nextReady() and finishing it before the " + "other is popped must not discard the sibling still in the frontier."); + + auto& transition = builder_->createTransition(rt1_); + + const auto first = transition.nextReady(); + ASSERT_TRUE(first.has_value()); + ASSERT_THAT(first->node, ::testing::AnyOf(c_, b_)); + EXPECT_EQ(first->action, Action::Start); + activate(transition, first->node); // must not discard the sibling + + const GraphIndex sibling = (first->node == c_) ? b_ : c_; + const auto second = transition.nextReady(); + ASSERT_TRUE(second.has_value()); + EXPECT_EQ(second->node, sibling); + activate(transition, second->node); + + // Only now that both of RT1's dependencies are active does RT1 itself become ready. + const auto third = transition.nextReady(); + ASSERT_TRUE(third.has_value()); + EXPECT_EQ(third->node, rt1_); + EXPECT_FALSE(transition.nextReady().has_value()); + + activate(transition, rt1_); + EXPECT_FALSE(transition.nextReady().has_value()); + EXPECT_TRUE(transition.isFinished()); +} + +TEST_F(SharedNodeGraphTest, TransitionBetweenRunTargetsKeepsSharedNodeActive) +{ + RecordProperty( + "Description", + "Transitioning RT1 -> RT2 stops RT1 then its exclusive node C, then starts A; the shared node " + "B stays active throughout because RT2 still needs it."); + + componentAt(b_).setActive(true); + componentAt(c_).setActive(true); + componentAt(rt1_).setActive(true); + + auto& transition = builder_->createTransition(rt2_); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{rt1_, Action::Stop})); + + deactivate(transition, rt1_); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{c_, Action::Stop})); + + // Finishing the last stop-set node auto-advances into the starting phase. + deactivate(transition, c_); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{a_, Action::Start})); + + // B was shared and stayed up the whole time. + EXPECT_TRUE(componentAt(b_).active_); + EXPECT_FALSE(componentAt(b_).stopped_); + + activate(transition, a_); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{rt2_, Action::Start})); + + activate(transition, rt2_); + EXPECT_THAT(collectReady(transition), ::testing::IsEmpty()); + EXPECT_TRUE(transition.isFinished()); +} + +TEST_F(SharedNodeGraphTest, OnNodeFinishedDuringIterationDrivesWholeTransitionInOneLoop) +{ + RecordProperty( + "Description", + "A single range-for loop that reports each node finished mid-iteration (as a synchronous " + "driver would) discovers all newly-ready successors until the transition completes."); + + componentAt(b_).setActive(true); + componentAt(c_).setActive(true); + componentAt(rt1_).setActive(true); + + auto& transition = builder_->createTransition(rt2_); + + std::vector visited; + for (const auto rn : transition) + { + visited.push_back(rn); + // Report completion immediately, mid-iteration, as a synchronously completing node would. + componentAt(rn.node).setActive(rn.action == Action::Start); + transition.onNodeFinished(rn.node); + } + + EXPECT_THAT( + visited, + ::testing::ElementsAre( + ReadyNode{rt1_, Action::Stop}, + ReadyNode{c_, Action::Stop}, + ReadyNode{a_, Action::Start}, + ReadyNode{rt2_, Action::Start})); + EXPECT_TRUE(transition.isFinished()); + + // B was shared and never touched. + EXPECT_TRUE(componentAt(b_).active_); + EXPECT_FALSE(componentAt(b_).stopped_); +} + +TEST_F(SharedNodeGraphTest, FailedTransitionIsRecoveredByAFreshFallbackTransition) +{ + RecordProperty( + "Description", + "When activating C never completes, the RT2 -> RT1 transition is stuck; a fresh RT1 -> RT2 " + "transition on the same builder recovers from the inconsistent state and brings RT2 back up."); + + componentAt(a_).setActive(true); + componentAt(b_).setActive(true); + componentAt(rt2_).setActive(true); + + auto& toRt1 = builder_->createTransition(rt1_); + EXPECT_THAT(collectReady(toRt1), ::testing::ElementsAre(ReadyNode{rt2_, Action::Stop})); + + deactivate(toRt1, rt2_); + EXPECT_THAT(collectReady(toRt1), ::testing::ElementsAre(ReadyNode{a_, Action::Stop})); + + // Finishing the last stop-set node auto-advances into the starting phase. + deactivate(toRt1, a_); + EXPECT_THAT(collectReady(toRt1), ::testing::ElementsAre(ReadyNode{c_, Action::Start})); + + // C fails to activate: it never reports finished, so the transition is stuck forever. + EXPECT_FALSE(componentAt(c_).active_); + EXPECT_FALSE(componentAt(rt1_).active_); + EXPECT_FALSE(toRt1.isFinished()); + + // Fallback RT1 -> RT2: RT1's exclusive nodes are already stopped, so the stopping phase is a + // no-op and the transition starts straight in the starting phase. + auto& toRt2 = builder_->createTransition(rt2_); + EXPECT_THAT(collectReady(toRt2), ::testing::ElementsAre(ReadyNode{a_, Action::Start})); + + activate(toRt2, a_); + EXPECT_THAT(collectReady(toRt2), ::testing::ElementsAre(ReadyNode{rt2_, Action::Start})); + + activate(toRt2, rt2_); + EXPECT_THAT(collectReady(toRt2), ::testing::IsEmpty()); + EXPECT_TRUE(toRt2.isFinished()); + + EXPECT_TRUE(componentAt(b_).active_); + EXPECT_FALSE(componentAt(b_).stopped_); +} + +TEST_F(SharedNodeGraphTest, StopsOrphanLeftRunningOutsideTheLastTargetSubgraph) +{ + RecordProperty( + "Description", + "A node left running outside the current target's subgraph — here A, orphaned by an earlier " + "aborted RT2 attempt while RT1 is up — is still stopped. The stop set is every running node " + "derived from live component state, not a traversal of one target's subgraph, so an orphan that " + "is unreachable from RT1 is not missed."); + + // RT1 is up (needs B and C). A is a stray still running from an aborted RT2 attempt; nothing + // reachable from RT1 leads to it, so a source-subgraph traversal from RT1 would leave it running. + componentAt(a_).setActive(true); + componentAt(b_).setActive(true); + componentAt(c_).setActive(true); + componentAt(rt1_).setActive(true); + + auto& transition = builder_->createTransition(off_); + + // Drive the whole transition in one loop, reporting each node's terminal state as it comes up. + // Everything running is stopped (the orphan A included), then the Off node is activated. + std::vector stopped_nodes; + bool off_started = false; + for (const auto rn : transition) + { + if (rn.node == off_) + { + EXPECT_EQ(rn.action, Action::Start); + off_started = true; + componentAt(rn.node).setActive(true); + } + else + { + EXPECT_EQ(rn.action, Action::Stop); + stopped_nodes.push_back(rn); + componentAt(rn.node).setActive(false); + } + transition.onNodeFinished(rn.node); + } + + EXPECT_TRUE(transition.isFinished()); + EXPECT_TRUE(off_started); + EXPECT_TRUE(componentAt(off_).active_); + // The orphan A is the key assertion: it was included in the stop set and is now stopped. + EXPECT_TRUE(componentAt(a_).stopped_); + EXPECT_TRUE(componentAt(b_).stopped_); + EXPECT_TRUE(componentAt(c_).stopped_); + EXPECT_TRUE(componentAt(rt1_).stopped_); + EXPECT_THAT( + stopped_nodes, + ::testing::UnorderedElementsAre( + ReadyNode{a_, Action::Stop}, + ReadyNode{b_, Action::Stop}, + ReadyNode{c_, Action::Stop}, + ReadyNode{rt1_, Action::Stop})); +} + +// --------------------------------------------------------------------------- +// Linear chain A -> B -> C -> D +// --------------------------------------------------------------------------- + +/// @brief Fixture for a linear dependency chain, plus the standalone Off RunTarget. +/// +/// A -> B -> C -> D (X -> Y means X depends on Y) +/// Off (no dependencies: reaching it stops the whole chain) +/// +/// A is the top root (nothing depends on it); D is the deepest leaf (depends on nothing). +/// Activation flows bottom-up (D, C, B, A); deactivation flows top-down (A, B, C, D). +/// Off stands alone, so a transition to it stops A..D and then activates Off. +class LinearGraphTest : public TransitionTest +{ + protected: + void SetUp() override + { + TransitionTest::SetUp(); + makeGraph(5); + a_ = addNode(); + b_ = addNode(); + c_ = addNode(); + d_ = addNode(); + off_ = addNode(); // Off RunTarget: no dependencies, so reaching it stops the whole chain + graph_->addDependency(a_, b_); + graph_->addDependency(b_, c_); + graph_->addDependency(c_, d_); + } + + /// @brief Bring the whole chain up front — a common precondition for the tests below. + void activateWholeChain() + { + componentAt(a_).setActive(true); + componentAt(b_).setActive(true); + componentAt(c_).setActive(true); + componentAt(d_).setActive(true); + } + + GraphIndex a_{}; + GraphIndex b_{}; + GraphIndex c_{}; + GraphIndex d_{}; + GraphIndex off_{}; +}; + +TEST_F(LinearGraphTest, TransitionToAStartsChainBottomUp) +{ + RecordProperty( + "Description", + "Bringing up A from scratch starts the whole chain in dependency order: D, then C, then B, " + "then A. Each node becomes ready only once its single dependency is active."); + + auto& transition = builder_->createTransition(a_); + + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{d_, Action::Start})); + activate(transition, d_); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{c_, Action::Start})); + activate(transition, c_); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{b_, Action::Start})); + activate(transition, b_); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{a_, Action::Start})); + activate(transition, a_); + + EXPECT_THAT(collectReady(transition), ::testing::IsEmpty()); + EXPECT_TRUE(transition.isFinished()); +} + +TEST_F(LinearGraphTest, TransitionToOffStopsChainTopDownThenStartsOff) +{ + RecordProperty( + "Description", + "Transitioning to Off from a fully-active chain stops it in reverse dependency order: A, then B, " + "then C, then D (each ready only once its single dependent is stopped), then activates the " + "dependency-less Off node."); + + activateWholeChain(); + + auto& transition = builder_->createTransition(off_); + + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{a_, Action::Stop})); + deactivate(transition, a_); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{b_, Action::Stop})); + deactivate(transition, b_); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{c_, Action::Stop})); + deactivate(transition, c_); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{d_, Action::Stop})); + deactivate(transition, d_); + + // Chain fully stopped: the Off node is now ready to activate. + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{off_, Action::Start})); + activate(transition, off_); + + EXPECT_THAT(collectReady(transition), ::testing::IsEmpty()); + EXPECT_TRUE(transition.isFinished()); +} + +TEST_F(LinearGraphTest, TransitionToCStopsNodesNotNeededByC) +{ + RecordProperty( + "Description", + "Transitioning to C while the whole chain A..D is active stops the higher-level nodes A and B " + "that C does not need: the stop set is every running node outside the target subgraph, drained " + "top-down (A, then B once A is stopped). C and its dependency D stay active throughout."); + + activateWholeChain(); + + auto& transition = builder_->createTransition(c_); + + // A has no dependents, so it is ready to stop first; B becomes ready once A is stopped. + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{a_, Action::Stop})); + deactivate(transition, a_); + EXPECT_THAT(collectReady(transition), ::testing::ElementsAre(ReadyNode{b_, Action::Stop})); + deactivate(transition, b_); + + // C and D are already active, so there is nothing to start: the transition is done. + EXPECT_THAT(collectReady(transition), ::testing::IsEmpty()); + EXPECT_TRUE(transition.isFinished()); + + EXPECT_FALSE(componentAt(a_).active_); + EXPECT_FALSE(componentAt(b_).active_); + EXPECT_TRUE(componentAt(c_).active_); + EXPECT_TRUE(componentAt(d_).active_); +} + +} // namespace score::mw::lifecycle::internal diff --git a/score/launch_manager/src/daemon/src/process_group_manager/iprocess.hpp b/score/launch_manager/src/daemon/src/process_group_manager/iprocess.hpp index e50738951..c79d35842 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/iprocess.hpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/iprocess.hpp @@ -19,7 +19,6 @@ #include "score/mw/launch_manager/common/constants.hpp" #include "score/mw/launch_manager/osal/ipc_comms.hpp" -#include #include #include @@ -79,13 +78,15 @@ struct ChildProcessConfig IpcCommsP shared_block; ///< sync Pointer to the shared memory block. }; -///@brief This class provides functionality that is needed to manage child processes. -/// The `IProcess` class provides functionality for child process management, which is required by Launch Manager. -/// As a part of OSAL it also provides porting interface for LCM. +///@brief This interface provides functionality that is needed to manage child processes. +/// The `IProcess` interface provides functionality for child process management, which is required by Launch +/// Manager. As a part of OSAL it also provides porting interface for LCM. class IProcess { public: + virtual ~IProcess() = default; + /// @brief The startProcess function initiates the execution of a new process, /// providing the necessary parameters such as the executable path, command-line arguments, and environment /// variables. The process ID of the newly started process is stored in the ProcessID object pointed to by pid. @@ -113,7 +114,7 @@ class IProcess /// number shall be returned as the function return value KFail to indicate the error. If the pid or config argument /// is NULL then simply KFail returned as the function return. - OsalReturnType startProcess(ProcessID* pid, IpcCommsP* sync, const osal::OsalConfig* config); + virtual OsalReturnType startProcess(ProcessID* pid, IpcCommsP* sync, const osal::OsalConfig* config) = 0; ///@brief This function request graceful termination by sending SIGTERM signal to a specified process. /// Requesting the group of processes for graceful termination is not supported. @@ -121,7 +122,7 @@ class IProcess ///@return Upon successful child process termination request, KSuccess shall be returned. Otherwise, KFail shall be /// returned. - OsalReturnType requestTermination(ProcessID pid); + virtual OsalReturnType requestTermination(ProcessID pid) = 0; ///@brief This function forcibly terminates a specified process. On Posix based system this can be implemented by /// sending SIGKILL. @@ -129,7 +130,7 @@ class IProcess ///@param[in] pid Child process identifier that should be terminated. ///@return When SIGKILL was successfully sent, KSuccess shall be returned. Otherwise, KFail shall be returned. - OsalReturnType forceTermination(ProcessID pid); + virtual OsalReturnType forceTermination(ProcessID pid) = 0; ///@brief This method waits until one of child processes of the caller terminates and retrieve its exit status (aka /// exit code). @@ -142,50 +143,14 @@ class IProcess /// is available. /// - `OsalReturnType::KFail` otherwise, the value stored in pid and status is undefined. - OsalReturnType waitForTermination(ProcessID& pid, int32_t& status); + virtual OsalReturnType waitForTermination(ProcessID& pid, int32_t& status) = 0; /// @brief This method wait for kRunning to be received from the process that was started /// @param sync The valid pointer returned from startProcess. Must not be NULL /// @param timeout How long to wait for kRunning /// @return kFail if sync is NULL or a timeout occurs, kSuccess otherwise - OsalReturnType waitForkRunning(IpcCommsP sync, std::chrono::milliseconds timeout); - - /// @brief This method will set up all the scheduling and security parameters described in the config, for the - /// current process - /// @param config the configuration to use - /// @return kFail if any operation fails, kSuccess otherwise - static OsalReturnType setSchedulingAndSecurity(const osal::OsalConfig& config); - - private: - /// @brief Creates shared memory for communication between processes. - /// @param[in,out] sync Pointer to a location to store a pointer to a structure containing - /// information about the communication channel. - /// @param[in,out] fd Reference to an integer where the file descriptor of the shared memory - /// segment will be stored. - /// @param[in,out] block Reference to a pointer that will be set to point to the shared memory block. - /// @param[in] config Pointer to the configuration for initializing the communication. - /// @return True if shared memory creation and initialization are successful, false otherwise. - inline bool setupComms(IpcCommsP& sync, int& fd, const OsalConfig& config); - - /// @brief Initializes semaphores within a given shared memory block. - /// @param[in] block Pointer to the shared memory block where semaphores will be initialized. - /// @return True if semaphore initialization is successful, false otherwise. - inline bool initializeSemaphores(IpcCommsP block); - - /// @brief Initializes the Control Client for communication using the shared memory block. - /// @param[in] shared_block Pointer to the shared memory block. - /// @param[in,out] fd Reference to store the file descriptor of the shared memory. - /// @param[in] config Pointer to the configuration for initializing the Control Client. - /// @return None. - inline IpcCommsP initializeControlClient(int& fd, const OsalConfig& config); - - /// @brief Handles the execution of the child process after forking. - /// @param[in] param Reference to child process configuration. - inline void handleChildProcess(ChildProcessConfig& param); - - ///@brief Atomic counter for shared memory names - std::atomic_uint32_t shm_name_counter = {0}; + virtual OsalReturnType waitForkRunning(IpcCommsP sync, std::chrono::milliseconds timeout) = 0; }; } // namespace osal diff --git a/score/launch_manager/src/daemon/src/process_group_manager/mock_iprocess.hpp b/score/launch_manager/src/daemon/src/process_group_manager/mock_iprocess.hpp new file mode 100644 index 000000000..8f08d18b5 --- /dev/null +++ b/score/launch_manager/src/daemon/src/process_group_manager/mock_iprocess.hpp @@ -0,0 +1,37 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#ifndef MOCK_IPROCESS_HPP_INCLUDED +#define MOCK_IPROCESS_HPP_INCLUDED + +#include "score/mw/launch_manager/process_group_manager/iprocess.hpp" +#include + +namespace score::lcm::internal::osal +{ + +class MockIProcess : public IProcess +{ + public: + MOCK_METHOD(OsalReturnType, + startProcess, + (ProcessID* pid, IpcCommsP* sync, const OsalConfig* config), + (override)); + MOCK_METHOD(OsalReturnType, requestTermination, (ProcessID pid), (override)); + MOCK_METHOD(OsalReturnType, forceTermination, (ProcessID pid), (override)); + MOCK_METHOD(OsalReturnType, waitForTermination, (ProcessID& pid, int32_t& status), (override)); + MOCK_METHOD(OsalReturnType, waitForkRunning, (IpcCommsP sync, std::chrono::milliseconds timeout), (override)); +}; + +} // namespace score::lcm::internal::osal + +#endif // MOCK_IPROCESS_HPP_INCLUDED diff --git a/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.hpp b/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.hpp index 9b2c57c68..2f7d15834 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.hpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/process_group_manager.hpp @@ -26,17 +26,22 @@ #else #include "score/mw/launch_manager/configuration/configuration_manager.hpp" #endif -#include "score/mw/launch_manager/process_group_manager/iprocess.hpp" -#include "score/mw/launch_manager/process_group_manager/details/graph.hpp" #include "score/mw/launch_manager/common/concurrency/mpmc_concurrent_queue.hpp" +#include "score/mw/launch_manager/common/concurrency/workerthread.hpp" +#include "score/mw/launch_manager/common/constants.hpp" +#include "score/mw/launch_manager/common/identifier_hash.hpp" +#include "score/mw/launch_manager/control/control_client_channel.hpp" +#include "score/mw/launch_manager/process_group_manager/details/component_event_queue.hpp" +#include "score/mw/launch_manager/process_group_manager/details/graph.hpp" #include "score/mw/launch_manager/process_group_manager/details/os_handler.hpp" -#include "score/mw/launch_manager/process_state_client/iprocess_state_notifier.hpp" #include "score/mw/launch_manager/process_group_manager/details/process_info_node.hpp" +#include "score/mw/launch_manager/process_group_manager/details/process_launcher.hpp" +#include "score/mw/launch_manager/process_group_manager/details/process_monitor.hpp" #include "score/mw/launch_manager/process_group_manager/details/safe_process_map.hpp" -#include "score/mw/launch_manager/common/concurrency/workerthread.hpp" #include "score/mw/launch_manager/process_group_manager/ialive_monitor_thread.hpp" +#include "score/mw/launch_manager/process_group_manager/iprocess.hpp" +#include "score/mw/launch_manager/process_state_client/iprocess_state_notifier.hpp" #include "score/mw/launch_manager/recovery_client/recovery_client.hpp" -#include "score/mw/launch_manager/common/constants.hpp" namespace score::lcm::internal { @@ -64,7 +69,7 @@ using ConfigurationType = ConfigurationManager; class ProcessGroupManager final { using WorkerQueue = - MPMCConcurrentQueue, static_cast(ProcessLimits::kMaxProcesses)>; + MPMCConcurrentQueue, static_cast(ProcessLimits::kMaxProcesses)>; public: /// @brief Constructs a new ProcessGroupManager object. @@ -76,9 +81,10 @@ class ProcessGroupManager final /// @param recovery_client A shared pointer to an IRecoveryClient instance for handling recovery operations. /// @param process_state_notifier A unique pointer to an IProcessStateNotifier instance for notifying the Alive /// Monitor thread of process state changes. - ProcessGroupManager(std::unique_ptr alive_monitor_thread, - std::shared_ptr recovery_client, - std::unique_ptr process_state_notifier); + ProcessGroupManager( + std::unique_ptr alive_monitor_thread, + std::shared_ptr recovery_client, + std::unique_ptr process_state_notifier); /// @brief Initializes the process group manager. /// Loads the flat configuration through ConfigurationManager. @@ -121,7 +127,7 @@ class ProcessGroupManager final /// @param pg_index The index of the process group in the list of groups managed by this manager /// @param process_index The index of the process in the list of processes in the process group /// @return nullptr if the node does not exist, otherwise a pointer to the corresponding node. - std::shared_ptr getProcessInfoNode(uint32_t pg_index, uint32_t process_index); + ProcessInfoNode* getProcessInfoNode(uint32_t pg_index, uint32_t process_index); /// @brief set the initial machine group state change result, called by graph when the transition completes /// @param result the result to save; it can only be saved once @@ -164,9 +170,6 @@ class ProcessGroupManager final /// @brief Cancels processGroupManager main routine as though SIGTERM had been sent void cancel(); - /// @brief Set the internal pointer for the Launch Manager ProcessInfoNode - void setLaunchManagerConfiguration(const OsProcess* launch_manager_config); - private: /// @brief Perform the function of Control Client handler /// @details (a) check for requests from any state manager processes in this process group\n @@ -197,8 +200,16 @@ class ProcessGroupManager final /// @param pg Reference of the process group (Graph) to check for pending responses void controlClientResponses(Graph& pg); - /// @brief Handle recovery actions requested by the Alive Monitor - void recoveryActionHandler(); + /// @brief Handle a single recovery request emitted by Alive supervision. + void handleRecoveryRequest(const IdentifierHash& process_identifier); + + /// @brief Drains every ComponentEvent currently queued to the (single) graph managed by this + /// ProcessGroupManager. + /// @details Single-graph assumption: PGM creates one ProcessMonitor bound to the first (only) + /// graph. SupervisionFailure is routed by process identifier through handleRecoveryRequest(), + /// while all other events are forwarded to Graph::handleComponentEvent(). Multi-graph routing + /// is deferred to a future revision. + void processComponentEvents(); /// @brief Manage the process group by starting any pending transitions that were requested /// @details If the Graph is in the correct state to start a transition (i.e. `kSuccess` or `kUndefined`) @@ -282,7 +293,6 @@ class ProcessGroupManager final inline bool initializeProcessGroups(); #endif - /// @brief Creates process component objects, including the job queue and worker threads. inline void createProcessComponentsObjects(); @@ -296,13 +306,13 @@ class ProcessGroupManager final ConfigurationType configuration_; /// @brief The process interface object associated with the ProcessGroupManager. - osal::IProcess process_interface_; + osal::ProcessLauncher process_interface_; /// @brief Shared pointer to the SafeProcessMap object. std::shared_ptr process_map_; /// @brief Unique pointer to the worker threads handling ProcessInfoNode jobs. - std::unique_ptr> worker_threads_; + std::unique_ptr> worker_threads_; /// @brief Shared pointer to the job queue for ProcessInfoNode jobs. std::shared_ptr worker_jobs_; @@ -330,11 +340,16 @@ class ProcessGroupManager final /// @brief Process state notifier object used to send data to PHM std::unique_ptr process_state_notifier_; - /// @brief pointer to the configuration for Launch Manager - const OsProcess* launch_manager_config_{nullptr}; - std::unique_ptr alive_monitor_thread_; + std::unique_ptr process_monitor_; + + std::unique_ptr os_handler_; + + /// @brief Queue of ComponentEvents produced by worker/OS-handler threads and drained by run() + /// on the main thread, so all Graph state mutations happen from a single thread. + std::unique_ptr event_queue_; + std::shared_ptr recovery_client_{}; }; diff --git a/score/launch_manager/src/daemon/src/process_state_client/posix_process.hpp b/score/launch_manager/src/daemon/src/process_state_client/posix_process.hpp index 273d53119..950b7c6a8 100644 --- a/score/launch_manager/src/daemon/src/process_state_client/posix_process.hpp +++ b/score/launch_manager/src/daemon/src/process_state_client/posix_process.hpp @@ -29,7 +29,8 @@ enum class ProcessState : std::uint8_t { kStarting = 1, ///< process in starting state. kRunning = 2, ///< process in running state. kTerminating = 3, ///< process in terminating state. - kTerminated = 4 ///< process in terminated state. + kTerminated = 4, ///< process in terminated state. + kFailed = 5, ///< process failed to start }; /// @brief Structure containing the Process's current state, its mapped ProcessGroupState and the timestamp when the process state changed. diff --git a/score/launch_manager/src/daemon/src/recovery_client/BUILD b/score/launch_manager/src/daemon/src/recovery_client/BUILD index 0944da271..eaf087981 100644 --- a/score/launch_manager/src/daemon/src/recovery_client/BUILD +++ b/score/launch_manager/src/daemon/src/recovery_client/BUILD @@ -27,7 +27,6 @@ cc_library( "//score:__subpackages__", ], deps = [ - "//externals/ipc_dropin", "//score/launch_manager/src/daemon/src/common:identifier_hash", ], ) diff --git a/score/launch_manager/src/daemon/src/recovery_client/irecovery_client.h b/score/launch_manager/src/daemon/src/recovery_client/irecovery_client.h index 810fcf278..a44715040 100644 --- a/score/launch_manager/src/daemon/src/recovery_client/irecovery_client.h +++ b/score/launch_manager/src/daemon/src/recovery_client/irecovery_client.h @@ -14,7 +14,7 @@ #define SCORE_LCM_IRECOVERYCLIENT_H_ #include "score/mw/launch_manager/common/identifier_hash.hpp" -#include +#include namespace score { @@ -22,12 +22,13 @@ namespace lcm { /// @brief The RecoveryClient allows the AliveMonitor component to report supervision failures to the -/// ProcessGroupManager thus requesting recovery for a specific process group. The requests are queued and periodically -/// processed by the ProcessGroupManager. In case the buffer is full and request cannot be queued, the overflow flag is -/// set. A detected overflow shall be handled as a critical failure by the ProcessGroupManager. +/// ProcessGroupManager. Requests are forwarded through a registered callback, which is expected +/// to hand off work to the ProcessGroupManager's main-thread event queue. class IRecoveryClient { public: + using RecoveryRequestCallback = std::function; + IRecoveryClient() noexcept = default; virtual ~IRecoveryClient() noexcept = default; IRecoveryClient(const IRecoveryClient&) = delete; @@ -35,20 +36,15 @@ class IRecoveryClient IRecoveryClient(IRecoveryClient&&) = delete; IRecoveryClient& operator=(IRecoveryClient&&) = delete; + /// @brief Registers the callback invoked by sendRecoveryRequest(). + /// @details Must be called before the Alive monitor thread can emit recovery requests. + virtual void setRecoveryRequestCallback(RecoveryRequestCallback callback) noexcept = 0; + /// @brief Send recovery request for a specific process. - /// @details If the internal buffer is full, the request is discarded and the overflow flag is set. + /// @details Invokes the registered callback with the provided process identifier. /// @param process_identifier The process that requires recovery. - /// @return true if the request was queued, false if the buffer was full. + /// @return true if a callback was registered and invoked, false otherwise. virtual bool sendRecoveryRequest(const score::lcm::IdentifierHash& process_identifier) noexcept = 0; - /// @brief Retrieve the latest request from the queue, removing it from the queue - /// @return The request, or std::nullopt if no request is available - virtual std::optional getNextRequest() noexcept = 0; - - /// @brief Checks if overflow has been set, by previously calling `sendRecoveryRequest` while the queue was already - /// full - /// @details Since overflow is a critical error, the flag is never reset - /// @return True if overflow has occurred, else false. - virtual bool hasOverflow() const noexcept = 0; }; } // namespace lcm } // namespace score diff --git a/score/launch_manager/src/daemon/src/recovery_client/recovery_client.cpp b/score/launch_manager/src/daemon/src/recovery_client/recovery_client.cpp index bd19dfaa3..bce0094e0 100644 --- a/score/launch_manager/src/daemon/src/recovery_client/recovery_client.cpp +++ b/score/launch_manager/src/daemon/src/recovery_client/recovery_client.cpp @@ -10,34 +10,30 @@ * * SPDX-License-Identifier: Apache-2.0 ********************************************************************************/ -#include #include "score/mw/launch_manager/recovery_client/recovery_client.hpp" -namespace score { -namespace lcm { +#include -RecoveryClient::RecoveryClient() noexcept : ringBuffer_{} { - ringBuffer_.initialize(); +namespace score +{ +namespace lcm +{ + +void RecoveryClient::setRecoveryRequestCallback(RecoveryRequestCallback callback) noexcept +{ + std::lock_guard lock(callback_mutex_); + callback_ = std::move(callback); } -bool RecoveryClient::sendRecoveryRequest(const score::lcm::IdentifierHash& process_identifier) noexcept { - if (!ringBuffer_.tryEnqueue(process_identifier)) { - overflow_flag_ = true; +bool RecoveryClient::sendRecoveryRequest(const score::lcm::IdentifierHash& process_identifier) noexcept +{ + std::lock_guard lock(callback_mutex_); + if (!callback_) + { return false; } + callback_(process_identifier); return true; } - -bool RecoveryClient::hasOverflow() const noexcept { - return overflow_flag_.load(); -} - -std::optional RecoveryClient::getNextRequest() noexcept { - score::lcm::IdentifierHash req; - if(ringBuffer_.tryDequeue(req)) { - return req; - } - return std::nullopt; -} -} -} +} // namespace lcm +} // namespace score diff --git a/score/launch_manager/src/daemon/src/recovery_client/recovery_client.hpp b/score/launch_manager/src/daemon/src/recovery_client/recovery_client.hpp index 5fed222fc..4c413deeb 100644 --- a/score/launch_manager/src/daemon/src/recovery_client/recovery_client.hpp +++ b/score/launch_manager/src/daemon/src/recovery_client/recovery_client.hpp @@ -13,37 +13,33 @@ #ifndef SCORE_LCM_RECOVERYCLIENT_H_ #define SCORE_LCM_RECOVERYCLIENT_H_ -#include -#include +#include -#include "ipc_dropin/ringbuffer.hpp" #include "score/mw/launch_manager/recovery_client/irecovery_client.h" -namespace score { -namespace lcm { +namespace score +{ +namespace lcm +{ -class RecoveryClient final : public IRecoveryClient { -public: - /// @brief Number of requests that can be stored before overflow occurs - static constexpr std::size_t kBufferCapacity = 128; - - RecoveryClient() noexcept; +class RecoveryClient final : public IRecoveryClient +{ + public: + RecoveryClient() noexcept = default; ~RecoveryClient() noexcept = default; RecoveryClient(const RecoveryClient&) = delete; RecoveryClient& operator=(const RecoveryClient&) = delete; RecoveryClient(RecoveryClient&&) = delete; RecoveryClient& operator=(RecoveryClient&&) = delete; + void setRecoveryRequestCallback(RecoveryRequestCallback callback) noexcept override; bool sendRecoveryRequest(const score::lcm::IdentifierHash& process_identifier) noexcept override; - std::optional getNextRequest() noexcept override; - bool hasOverflow() const noexcept override; -private: - static const std::size_t element_size_ = sizeof(score::lcm::IdentifierHash); - ipc_dropin::RingBuffer ringBuffer_; ///< Ring buffer to store recovery requests - std::atomic_bool overflow_flag_{false}; + private: + mutable std::mutex callback_mutex_; + RecoveryRequestCallback callback_; }; -} // namespace lcm -} // namespace score +} // namespace lcm +} // namespace score #endif diff --git a/score/launch_manager/src/daemon/src/recovery_client/recovery_client_UT.cpp b/score/launch_manager/src/daemon/src/recovery_client/recovery_client_UT.cpp index 1ee888551..0bac4bbb9 100644 --- a/score/launch_manager/src/daemon/src/recovery_client/recovery_client_UT.cpp +++ b/score/launch_manager/src/daemon/src/recovery_client/recovery_client_UT.cpp @@ -12,10 +12,14 @@ ********************************************************************************/ #include +#include + #include "score/mw/launch_manager/recovery_client/recovery_client.hpp" -namespace score { -namespace lcm { +namespace score +{ +namespace lcm +{ class RecoveryClientTest : public ::testing::Test { @@ -27,87 +31,77 @@ class RecoveryClientTest : public ::testing::Test } }; -TEST_F(RecoveryClientTest, SendSingleRequest) +TEST_F(RecoveryClientTest, SendRecoveryRequestInvokesRegisteredCallback) { RecordProperty("Description", - "RecoveryClient can send single request successfully."); + "RecoveryClient invokes the registered callback with the provided process identifier."); RecoveryClient client; + IdentifierHash received{""}; + client.setRecoveryRequestCallback([&received](const IdentifierHash& process_identifier) { + received = process_identifier; + }); + const bool result = client.sendRecoveryRequest(IdentifierHash("proc_a")); EXPECT_TRUE(result); - EXPECT_FALSE(client.hasOverflow()); + EXPECT_EQ(received, IdentifierHash("proc_a")); } -TEST_F(RecoveryClientTest, GetNextRequest) +TEST_F(RecoveryClientTest, SendRecoveryRequestReturnsFalseWithoutRegisteredCallback) { - RecordProperty("Description", - "RecoveryClient can send and retrieve single request successfully."); + RecordProperty("Description", "RecoveryClient returns false and does not crash when no callback is registered."); RecoveryClient client; - const IdentifierHash expected_proc("proc_b"); - client.sendRecoveryRequest(expected_proc); - - const auto req = client.getNextRequest(); - ASSERT_TRUE(req.has_value()); - EXPECT_EQ(*req, expected_proc); + EXPECT_FALSE(client.sendRecoveryRequest(IdentifierHash("proc_b"))); } -TEST_F(RecoveryClientTest, GetNextRequestEmpty) +TEST_F(RecoveryClientTest, MultipleRequestsInvokeCallbackInOrder) { RecordProperty("Description", - "RecoveryClient returns no request when buffer is empty."); + "RecoveryClient invokes the callback once per request in the same order requests are sent."); RecoveryClient client; - EXPECT_FALSE(client.getNextRequest().has_value()); -} + std::vector received; + client.setRecoveryRequestCallback([&received](const IdentifierHash& process_identifier) { + received.push_back(process_identifier); + }); -TEST_F(RecoveryClientTest, RingBufferFull) -{ - RecordProperty("Description", - "RecoveryClient sets overflow flag if buffer is full."); - - RecoveryClient client; - const IdentifierHash proc("proc_c"); + const IdentifierHash proc_first("proc_first"); + const IdentifierHash proc_second("proc_second"); + const IdentifierHash proc_third("proc_third"); - // Fill the ring buffer - for (std::size_t i = 0U; i < RecoveryClient::kBufferCapacity; ++i) - { - EXPECT_TRUE(client.sendRecoveryRequest(proc)); - } + EXPECT_TRUE(client.sendRecoveryRequest(proc_first)); + EXPECT_TRUE(client.sendRecoveryRequest(proc_second)); + EXPECT_TRUE(client.sendRecoveryRequest(proc_third)); - // One more should fail and set overflow - EXPECT_FALSE(client.sendRecoveryRequest(proc)); - EXPECT_TRUE(client.hasOverflow()); + ASSERT_EQ(received.size(), 3U); + EXPECT_EQ(received[0], proc_first); + EXPECT_EQ(received[1], proc_second); + EXPECT_EQ(received[2], proc_third); } -TEST_F(RecoveryClientTest, FIFOOrdering) +TEST_F(RecoveryClientTest, ReRegisteringCallbackReplacesPreviousCallback) { RecordProperty("Description", - "RecoveryClient maintains the order of inserted requests"); + "RecoveryClient uses the latest callback when setRecoveryRequestCallback is called again."); RecoveryClient client; - const IdentifierHash proc_first("proc_first"); - const IdentifierHash proc_second("proc_second"); - const IdentifierHash proc_third("proc_third"); - - client.sendRecoveryRequest(proc_first); - client.sendRecoveryRequest(proc_second); - client.sendRecoveryRequest(proc_third); - - const auto req1 = client.getNextRequest(); - ASSERT_TRUE(req1.has_value()); - EXPECT_EQ(req1.value(), proc_first); + std::size_t callback1_calls = 0U; + std::size_t callback2_calls = 0U; - const auto req2 = client.getNextRequest(); - ASSERT_TRUE(req2.has_value()); - EXPECT_EQ(req2.value(), proc_second); + client.setRecoveryRequestCallback([&callback1_calls](const IdentifierHash&) { + ++callback1_calls; + }); + ASSERT_TRUE(client.sendRecoveryRequest(IdentifierHash("proc_before_replace"))); - const auto req3 = client.getNextRequest(); - ASSERT_TRUE(req3.has_value()); - EXPECT_EQ(req3.value(), proc_third); + client.setRecoveryRequestCallback([&callback2_calls](const IdentifierHash&) { + ++callback2_calls; + }); + ASSERT_TRUE(client.sendRecoveryRequest(IdentifierHash("proc_after_replace"))); - EXPECT_FALSE(client.getNextRequest().has_value()); + EXPECT_EQ(callback1_calls, 1U); + EXPECT_EQ(callback2_calls, 1U); } -} // namespace lcm -} // namespace score +} // namespace lcm +} // namespace score