diff --git a/README.md b/README.md index 7fa9c15e..1bcefdfd 100644 --- a/README.md +++ b/README.md @@ -612,7 +612,9 @@ Set `"lazy_load": true` to register configured model ids at startup while loadin Set top-level `"backend"` to `"cuda"`, `"cpu"`, `"vulkan"`, `"metal"`, or `"hip"`. CUDA is the optimized path for audio.cpp; CPU, Vulkan, Metal, and HIP are intended for portability and testing when the binary is built with that backend, but performance and model coverage may be lower. > [!WARNING] -> Lazy loading does not unload models after a request. Once a model is first used, the server keeps that model and session in memory for reuse until the server exits. +> Lazy loading does not unload models after a request. Once a model is first used, the server keeps that model and session in memory for reuse until the server exits, unless `max_loaded_models` limits residency. + +Set top-level `"max_loaded_models"` (or start with `--max-loaded-models `) to bound how many models stay resident in memory at once: loading one more past the limit first unloads the least recently used idle model, and `1` enforces a single loaded model at a time. The default `0` keeps every used model in memory. See [app/server/README.md](app/server/README.md) for details. Start: diff --git a/app/server/README.md b/app/server/README.md index 959278a7..0361a7e7 100644 --- a/app/server/README.md +++ b/app/server/README.md @@ -93,7 +93,9 @@ Set top-level `"backend"` to `"cuda"`, `"cpu"`, `"vulkan"`, or `"metal"`. CUDA i Set top-level `"lazy_load": true` to register all configured model ids at startup but defer each model's framework load and session creation until its first request. A model can override the default with `"lazy": true` or `"lazy": false`. > [!WARNING] -> Lazy loading does not unload models after a request. Once a model is first used, the server keeps that model and session in memory for reuse until the server exits. +> Lazy loading does not unload models after a request. Once a model is first used, the server keeps that model and session in memory for reuse until the server exits, unless `max_loaded_models` limits residency. + +Set top-level `"max_loaded_models"` to bound how many models are resident in memory at once. When a request needs a model that is not loaded and the limit is already reached, the server first unloads the least recently used idle model (freeing VRAM on GPU backends) and reloads it on its own next request. `1` enforces a single loaded model at a time, which is the practical choice when each model alone nearly fills the device. Higher values keep that many most recently used models warm. The default `0` disables the limit. A model that is mid-inference is never unloaded; if the limit is reached and every loaded model is busy, the request fails with `503` so the client can retry. With more non-lazy models configured than the limit allows, startup loads the first `max_loaded_models` of them and defers the rest to their first request. The equivalent command-line option is `--max-loaded-models `. Set per-model `"default_request_options"` to apply request-option defaults to every request for that model. Values supplied by the actual request body override these defaults. diff --git a/app/server/busy_guard.h b/app/server/busy_guard.h index ac72193a..2c130ff6 100644 --- a/app/server/busy_guard.h +++ b/app/server/busy_guard.h @@ -118,6 +118,19 @@ class BusyGuard { return Lock(*this, std::move(lock)); } + // Non-blocking variant: the lock only if the model is idle right now, nullopt + // otherwise. Eviction sweeps use this so a caller that already holds one + // model's guard never waits on another's -- two loads evicting each other's + // target would otherwise deadlock. + std::optional try_acquire() { + std::unique_lock lock(mutex_, std::try_to_lock); + if (!lock.owns_lock()) { + return std::nullopt; + } + busy_since_ms_.store(steady_now_ms(), std::memory_order_release); + return Lock(*this, std::move(lock)); + } + private: std::timed_mutex mutex_; std::atomic busy_since_ms_{0}; diff --git a/app/server/config.cpp b/app/server/config.cpp index 62ef3ae8..af219842 100644 --- a/app/server/config.cpp +++ b/app/server/config.cpp @@ -234,6 +234,7 @@ ServerConfig load_server_config(const std::filesystem::path & path) { config.max_request_body_bytes = parse_max_request_body_bytes(*value); } config.busy_timeout_ms = engine::io::json::optional_i32(root, "busy_timeout_ms", config.busy_timeout_ms); + config.max_loaded_models = engine::io::json::optional_i32(root, "max_loaded_models", config.max_loaded_models); if (const auto * value = root.find("live_ingest")) { config.live_ingest = parse_live_ingest_limits(*value, config.live_ingest, "server live_ingest"); } @@ -252,6 +253,9 @@ ServerConfig load_server_config(const std::filesystem::path & path) { if (config.busy_timeout_ms < 0) { throw std::runtime_error("server busy_timeout_ms must be >= 0 (0 disables the guard)"); } + if (config.max_loaded_models < 0) { + throw std::runtime_error("server max_loaded_models must be >= 0 (0 disables the limit)"); + } if (config.threads <= 0) { throw std::runtime_error("server threads must be positive"); } diff --git a/app/server/config.h b/app/server/config.h index 70c70b3b..3aee4ead 100644 --- a/app/server/config.h +++ b/app/server/config.h @@ -83,6 +83,13 @@ struct ServerConfig { // single inference (music generation can take minutes). 0 disables the guard and // restores unbounded waiting. int busy_timeout_ms = 300000; + // Upper bound on how many models may be resident in memory at once. Loading a + // model past the limit first unloads the least recently used idle model (its + // next request reloads it), so a multi-model config can run on a device that + // only fits a few of them. 1 enforces a single resident model. 0 disables the + // limit and keeps the original behavior: once loaded, a model stays in memory + // until it is unloaded explicitly or the server exits. + int max_loaded_models = 0; // Fleet-wide bounds for incrementally delivered request bodies. The defaults are // in LiveIngestLimits; a model entry may override any subset of them. LiveIngestLimits live_ingest; diff --git a/app/server/main.cpp b/app/server/main.cpp index 61fb777e..348988d6 100644 --- a/app/server/main.cpp +++ b/app/server/main.cpp @@ -60,7 +60,7 @@ std::filesystem::path executable_directory(const char * argv0) { void print_help() { std::cout << "audiocpp_server [--config ] [--ui] [--host ] [--port ] [--backend ]\n" - << " [--device ] [--threads ] [--busy-timeout-ms ]\n" + << " [--device ] [--threads ] [--busy-timeout-ms ] [--max-loaded-models ]\n" << " [--model-spec-override ] [--voice-dir ]\n" << " [--log] [--log-file ]\n" << " [--cors-origins ]\n" @@ -71,6 +71,9 @@ void print_help() { << " --backend cpu|cuda|hip|rocm|vulkan|metal default cuda (rocm is an alias for hip)\n" << " --busy-timeout-ms fail a request with 503 when the model has been\n" << " busy this long; default 300000, 0 disables\n" + << " --max-loaded-models keep at most n models resident in memory, unloading\n" + << " the least recently used idle model first; 1 enforces\n" + << " a single loaded model, default 0 (no limit)\n" << " --voice-dir override the shared reference voice library directory\n" << " --cors-origins \"*\" experimental; disabled by default. Allows browser\n" << " requests from any origin for trusted local demos only\n" @@ -175,6 +178,9 @@ int main(int argc, char ** argv) { if (const auto busy_timeout = arg_value(argc, argv, "--busy-timeout-ms")) { config.busy_timeout_ms = std::stoi(*busy_timeout); } + if (const auto max_loaded_models = arg_value(argc, argv, "--max-loaded-models")) { + config.max_loaded_models = std::stoi(*max_loaded_models); + } if (const auto model_spec = arg_value(argc, argv, "--model-spec-override")) { config.model_spec_override = std::filesystem::path(*model_spec); } @@ -190,6 +196,9 @@ int main(int argc, char ** argv) { if (config.busy_timeout_ms < 0) { throw std::runtime_error("--busy-timeout-ms must be >= 0 (0 disables the guard)"); } + if (config.max_loaded_models < 0) { + throw std::runtime_error("--max-loaded-models must be >= 0 (0 disables the limit)"); + } const auto ui_resource_anchor = executable_directory(argc > 0 ? argv[0] : nullptr); minitts::server::ServerState state( diff --git a/app/server/runtime.cpp b/app/server/runtime.cpp index 6772784b..9ec08011 100644 --- a/app/server/runtime.cpp +++ b/app/server/runtime.cpp @@ -1112,13 +1112,24 @@ HttpResponse ServerState::handle(const HttpRequest & request) { } void ServerState::load_models() { + int eager_loaded = 0; for (auto & config : config_.models) { auto loaded = make_model(std::move(config)); if (!model_index_.emplace(loaded->config.id, models_.size()).second) { throw std::runtime_error("duplicate server model id: " + loaded->config.id); } if (!loaded->config.lazy) { - ensure_model_loaded_locked(*loaded); + if (config_.max_loaded_models > 0 && eager_loaded >= config_.max_loaded_models) { + // Loading it now would immediately evict an earlier entry, so an + // over-limit config would churn through doomed loads at startup. + // Register the id and defer the load to the model's first request. + std::cerr << "model '" << loaded->config.id + << "' registered but not loaded (max_loaded_models=" + << config_.max_loaded_models << " reached); it loads on first use\n"; + } else { + ensure_model_loaded_locked(*loaded); + ++eager_loaded; + } } models_.push_back(std::move(loaded)); } @@ -1164,11 +1175,7 @@ HttpResponse ServerState::handle_model_load(const std::string & body_text) { existing->config.session_options != requested.session_options || existing->config.model_spec_override != requested.model_spec_override; if (changed) { - existing->streaming = nullptr; - existing->offline = nullptr; - existing->session.reset(); - existing->model.reset(); - existing->loaded.store(false); + existing->unload(); existing->voice_presets.clear(); existing->default_voice_preset.reset(); existing->config = std::move(requested); @@ -1228,11 +1235,7 @@ HttpResponse ServerState::handle_model_unload(const std::string & body_text) { model = models_.at(found->second).get(); } BusyGuard::Lock run_lock = acquire_model_run(*model, std::nullopt); - model->streaming = nullptr; - model->offline = nullptr; - model->session.reset(); - model->model.reset(); - model->loaded.store(false); + model->unload(); return json_response("{\"id\":" + json_quote(id) + ",\"loaded\":false}"); } @@ -1581,10 +1584,57 @@ void ServerState::load_voice_presets(LoadedModel & model) const { } } +void ServerState::evict_for_model_limit(const LoadedModel & loading) { + const int limit = config_.max_loaded_models; + std::vector resident; + { + std::lock_guard state_lock(models_mutex_); + for (const auto & model : models_) { + if (model.get() != &loading && model->session != nullptr) { + resident.push_back(model.get()); + } + } + } + // `loading` itself will occupy one slot, so at most limit - 1 others may stay. + int to_evict = static_cast(resident.size()) - (limit - 1); + if (to_evict <= 0) { + return; + } + std::sort(resident.begin(), resident.end(), [](const LoadedModel * a, const LoadedModel * b) { + return a->last_used_ms.load(std::memory_order_relaxed) < + b->last_used_ms.load(std::memory_order_relaxed); + }); + for (LoadedModel * victim : resident) { + if (to_evict == 0) { + return; + } + // A victim mid-inference is only try-acquired: a blocking wait here could + // deadlock against that run evicting in return, and a busy model is in + // active use anyway -- prefer the next-oldest instead. + const auto lock = victim->busy.try_acquire(); + if (!lock.has_value()) { + continue; + } + victim->unload(); + --to_evict; + } + if (to_evict > 0) { + throw ServerBusyError( + "cannot load model '" + loading.config.id + "': max_loaded_models=" + + std::to_string(limit) + " and every loaded model is busy; retry later"); + } +} + void ServerState::ensure_model_loaded_locked(LoadedModel & model) { + model.last_used_ms.store(steady_now_ms(), std::memory_order_relaxed); if (model.session != nullptr) { return; } + std::unique_lock load_lock; + if (config_.max_loaded_models > 0) { + load_lock = std::unique_lock(model_load_mutex_); + evict_for_model_limit(model); + } auto registry = engine::runtime::make_default_registry(); engine::runtime::ModelLoadRequest load_request; @@ -2690,10 +2740,11 @@ std::string ServerState::get_allowed_origin(const HttpRequest & request) const { } void ServerState::LoadedModel::unload() { - offline = nullptr; + offline = nullptr; streaming = nullptr; session.reset(); model.reset(); + loaded.store(false); } HttpResponse ServerState::handle_unload_models(const std::string & body_text) { diff --git a/app/server/runtime.h b/app/server/runtime.h index 7b2ed784..45af1c55 100644 --- a/app/server/runtime.h +++ b/app/server/runtime.h @@ -56,6 +56,10 @@ class ServerState final : public IHttpHandler { engine::runtime::IOfflineVoiceTaskSession * offline = nullptr; engine::runtime::IStreamingVoiceTaskSession * streaming = nullptr; std::atomic loaded{false}; + // Steady-clock ms of the most recent load or run of this model. Orders + // eviction when max_loaded_models forces an unload: the least recently + // used idle model goes first. + std::atomic last_used_ms{0}; mutable std::shared_mutex metadata_mutex; std::unordered_map voice_presets; std::optional default_voice_preset; @@ -99,6 +103,10 @@ class ServerState final : public IHttpHandler { LoadedModel::RuntimeVoicePreset load_runtime_voice_preset(const ServerModelConfig::VoicePreset & preset) const; void load_voice_presets(LoadedModel & model) const; void ensure_model_loaded_locked(LoadedModel & model); + // With max_loaded_models set, unload least recently used idle models until + // `loading` fits within the limit. A model mid-inference is never a victim; + // when nothing can be evicted this throws ServerBusyError (-> HTTP 503). + void evict_for_model_limit(const LoadedModel & loading); LoadedModel & require_model(const engine::io::json::Value & body); const LoadedModel::RuntimeVoicePreset * select_voice_preset( const LoadedModel & model, @@ -168,6 +176,10 @@ class ServerState final : public IHttpHandler { std::vector> models_; std::unordered_map model_index_; mutable std::mutex models_mutex_; + // Serializes framework loads while max_loaded_models is active, so two + // concurrent lazy loads cannot both pass the eviction check and overshoot + // the limit. Not taken when the limit is 0: loads stay concurrent there. + std::mutex model_load_mutex_; std::filesystem::path upload_root_; std::filesystem::path repository_root_; #if defined(AUDIOCPP_HAS_NATIVE_MODEL_MANAGER) diff --git a/tests/unittests/test_server_config.cpp b/tests/unittests/test_server_config.cpp index e56e9764..8c830802 100644 --- a/tests/unittests/test_server_config.cpp +++ b/tests/unittests/test_server_config.cpp @@ -294,6 +294,42 @@ void test_negative_busy_timeout_is_rejected() { require(rejected, "negative busy_timeout_ms is rejected"); } +void test_max_loaded_models_defaults_and_overrides() { + const auto root = make_temp_root(); + + const auto default_path = write_config( + root, "max_loaded_default.json", std::string("{") + kMinimalModel + "}"); + require( + minitts::server::load_server_config(default_path).max_loaded_models == 0, + "max_loaded_models defaults to 0 (no limit) when omitted"); + + const auto single_path = write_config( + root, "max_loaded_single.json", std::string(R"JSON({"max_loaded_models": 1,)JSON") + kMinimalModel + "}"); + require( + minitts::server::load_server_config(single_path).max_loaded_models == 1, + "max_loaded_models accepts 1 to enforce a single resident model"); + + const auto multi_path = write_config( + root, "max_loaded_multi.json", std::string(R"JSON({"max_loaded_models": 3,)JSON") + kMinimalModel + "}"); + require( + minitts::server::load_server_config(multi_path).max_loaded_models == 3, + "max_loaded_models is read from the config"); +} + +void test_negative_max_loaded_models_is_rejected() { + const auto root = make_temp_root(); + const auto config_path = write_config( + root, "max_loaded_negative.json", std::string(R"JSON({"max_loaded_models": -1,)JSON") + kMinimalModel + "}"); + + bool rejected = false; + try { + (void) minitts::server::load_server_config(config_path); + } catch (const std::runtime_error & error) { + rejected = std::string(error.what()).find("max_loaded_models") != std::string::npos; + } + require(rejected, "negative max_loaded_models is rejected"); +} + void test_per_model_busy_timeout() { const auto root = make_temp_root(); const auto config_path = write_config( @@ -421,6 +457,8 @@ int main() { test_negative_max_request_body_is_rejected(); test_unsafe_numeric_max_request_body_is_rejected(); test_negative_busy_timeout_is_rejected(); + test_max_loaded_models_defaults_and_overrides(); + test_negative_max_loaded_models_is_rejected(); test_per_model_busy_timeout(); test_negative_per_model_busy_timeout_is_rejected(); test_ui_configuration();