Add F5-TTS community model (M0 scaffolding; Habibi Arabic aliases)

CMakeLists.txt

@@ -541,6 +541,23 @@ audiocpp_add_model(inflect_v2
         engine::models::inflect_v2::make_inflect_v2_loader
 )
 
+audiocpp_add_model(f5_tts
+    SOURCES
+        src/community_models/f5_tts/session.cpp
+        src/community_models/f5_tts/runtime.cpp
+        src/community_models/f5_tts/weights.cpp
+        src/community_models/f5_tts/dit_modules.cpp
+        src/community_models/f5_tts/synthesize.cpp
+    INCLUDES
+        engine/community_models/f5_tts/session.h
+        engine/community_models/f5_tts/runtime.h
+    LOADERS
+        engine::models::f5_tts::make_f5_tts_loader
+    ALIASES
+        habibi
+        habibi_tts
+)
+
 audiocpp_add_model(minimax_h3
     SOURCES
         src/community_models/minimax_h3/assets.cpp
@@ -1502,6 +1519,24 @@ if (vibevoice IN_LIST AUDIOCPP_LINKED_MODELS)
     endif()
 endif()
 
+# F5/Habibi tests: parity harnesses + e2e sample generator. Only when the
+# f5_tts model is linked (they call model-internal symbols) AND test hooks
+# are explicitly enabled — F5_MEL_TEST must never leak into production
+# model libraries (PR #275 review feedback).
+option(ENGINE_F5_TEST_HOOKS "Build F5-TTS test hooks (f5_test_* symbols) and parity binaries" OFF)
+if (f5_tts IN_LIST AUDIOCPP_LINKED_MODELS AND ENGINE_F5_TEST_HOOKS)
+    target_compile_definitions(engine_model_f5_tts PRIVATE F5_MEL_TEST=1)
+    foreach(f5_test IN ITEMS f5_e2e f5_parity f5_cfg_parity f5_tokenizer)
+        add_executable(${f5_test} tests/${f5_test}_main.cpp)
+        target_compile_definitions(${f5_test} PRIVATE F5_MEL_TEST=1)
+        target_link_libraries(${f5_test} PRIVATE engine_runtime ggml)
+        target_include_directories(${f5_test} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+        if (ENGINE_ENABLE_OPENMP)
+            target_link_libraries(${f5_test} PRIVATE OpenMP::OpenMP_CXX)
+        endif()
+    endforeach()
+endif()
+
 if (ENGINE_BUILD_TESTS)
     add_executable(moss_tts_local_smoke
         tests/moss_tts_local/moss_tts_local_smoke.cpp

docs/community_models/f5_tts.md

@@ -0,0 +1,105 @@
+# F5-TTS (community model)
+
+[F5-TTS](https://github.com/SWivid/F5-TTS) is an open-source zero-shot voice-cloning TTS built on a
+flow-matching diffusion transformer (DiT) with a ConvNeXt text conditioner and a Vocos vocoder.
+This community port also targets the [Habibi-TTS](https://github.com/SWivid/Habibi-TTS) finetune —
+a multi-dialect Arabic checkpoint suite (MSA, SAU, UAE, ALG, IRQ, EGY, MAR, OMN, TUN, LEV, SDN, LBY)
+from the same authors — which uses the identical architecture, giving Arabic support through the
+same family (`habibi` / `habibi_tts` are registered as aliases).
+
+**Status: M4 — inference wired end to end.** The session (`src/community_models/f5_tts/session.cpp`)
+runs full synthesis via `f5_synthesize` (text pipeline with Habibi dialect tokens → batched-CFG DiT
+Euler sampler → Vocos vocoder) and is served by `audiocpp_server`. Parity vs the reference PyTorch
+implementation is covered by golden harnesses (DiT stage taps, batched CFG incl. the null branch,
+tokenizer ids) plus a whisper.cpp ASR pronunciation check; see `/mnt/ai/f5-parity/run_all.sh`.
+
+## Milestones
+
+Each milestone is gated on parity against the reference PyTorch implementation (cosine similarity
+≥ 0.999 on fixed inputs) plus a listening check, matching the evidence bar described in #54 and
+PR #180.
+
+| Milestone | Scope | Status |
+|---|---|---|
+| M0 | Family registration, model spec, stub session, this doc | done |
+| M1 | Weight loading + mel-Vocos decode path (ConvNeXt + iSTFT) | done (mel-corr 0.9963) |
+| M2 | DiT forward (RoPE, adaLN) + ConvNeXt text conditioner | done (all stages cosine 1.000000) |
+| M3 | CFM sampler (Euler, sway + EPSS, CFG null-branch parity), inference wiring, En/Ar samples | done |
+| M4 | Long-form chunking, RTF/VRAM evidence, server wiring | done (0.51x RTF on RTX 3090; GGUF packages hosted at trklou/audio.cpp) |
+
+## Server usage
+
+`audiocpp_server.json` entry: family `f5_tts`, model path = checkpoint directory (a `*.gguf`
+package, or a safetensors checkpoint + `vocab.txt` for development). Session options:
+`f5_tts.vocos_path` (only needed with safetensors checkpoints lacking a bundled vocoder),
+`f5_tts.dialect` (default UNK), `f5_tts.frame_budget` (mel frames per CFM pass, 0 = 2048).
+Requests take `reference_text` (required), `dialect`, `speed`, `seed`, `num_inference_steps`,
+`cfg_strength` (alias `guidance_scale`), `sway_sampling_coef`, `strip_diacritics`.
+
+**Diacritics (harakat):** Habibi was trained on ASR transcripts, which are undiacritized —
+the harakat/tanwin/shadda tokens exist in the vocab but are severely undertrained, and raw
+diacritized input degrades to garbled speech with character repetitions (identical in the
+Python reference; not a port issue). By default the frontend strips combining marks
+(U+0640, U+064B–U+065F, U+0670) before synthesis, so `أَيْنَ اللَّوْنُ الأَحْمَر؟` reads as
+`أين اللون الأحمر؟`. Disable per request with `strip_diacritics=false`.
+
+## Quickstart (from a fresh clone)
+
+Everything needed is installable from this repository — no Python inference stack required:
+
+```bash
+# 1. Build (f5_tts is included in AUDIOCPP_MODEL_SET=full, or select it explicitly)
+cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DAUDIOCPP_MODEL_SET=custom \
+      -DAUDIOCPP_MODELS=f5_tts -DENGINE_ENABLE_CUDA=ON
+cmake --build build --parallel --target audiocpp_cli audiocpp_server
+
+# 2. Download the model package (GGUF: DiT + Vocos vocoder in one file + vocab.txt)
+python3 tools/model_manager_v2.py install habibi_unified
+# Per-dialect specialized checkpoints (stronger accent): habibi_alg, habibi_egy,
+# habibi_irq, habibi_mar, habibi_msa, habibi_sau, habibi_uae
+# Standalone vocoder for the original safetensors checkpoints: vocos_mel_24khz
+
+# 3. Synthesize (zero-shot: any short reference WAV + its transcript)
+build/bin/audiocpp_cli --task tts --family habibi \
+    --model <models>/Habibi-TTS/Unified \
+    --voice-ref /path/to/reference.wav \
+    --reference-text 'transcript of the reference audio' \
+    --text 'أهلا، هذا نص عربي تجريبي.' \
+    --request-option dialect=UNK --out out.wav
+```
+
+The GGUF package is self-contained: the DiT lives under the `transformer` namespace and the
+Vocos vocoder under `vocos`, so no separate vocoder download or `f5_tts.vocos_path` option is
+needed (the option and safetensors fallbacks still work for the original HF checkpoints).
+Dialects: `UNK MSA SAU UAE ALG IRQ EGY MAR OMN TUN LEV SDN LBY`.
+Reference audio longer than ~10.9s is truncated (with a warning) — keep refs under that
+and make sure the transcript matches, otherwise the transcript tail leaks into the output.
+
+### Converting checkpoints to GGUF
+
+`tools/convert_f5_tts.py` wraps `audiocpp_gguf` and produces one self-contained GGUF per
+checkpoint (`transformer` + `vocos` namespaces) plus the standalone vocoder package:
+
+```bash
+python3 tools/convert_f5_tts.py --checkpoint-root /models/Habibi-TTS \
+    --vocos /models/vocos-mel-24khz/vocos.safetensors \
+    --converter build/bin/audiocpp_gguf --output-dir gguf-packages
+```
+
+The safetensors source layout stays supported for development; the packaged/default format is GGUF.
+
+## Relevant building blocks already in-tree
+
+- Vocos vocoder: `src/models/vevo2/components.cpp`, `src/models/index_tts2/`
+- iSTFT: `src/models/miocodec/`, `src/models/seed_vc/`
+- Flow matching: `src/models/vevo2/fm.cpp`
+- RoPE DiT / adaLN: `src/models/stable_audio/foundation/rf_dit.cpp`
+
+## Checkpoints
+
+| Model | Source | License |
+|---|---|---|
+| F5-TTS Base (en/zh) | `SWivid/F5-TTS` | cc-by-nc-4.0 |
+| Habibi Unified (ar) | `SWivid/Habibi-TTS` | cc-by-nc-sa-4.0 |
+
+Both are non-commercial licenses; keep that in mind before shipping anything built on them.

docs/community_models/models.md

@@ -16,6 +16,7 @@ Practical expectations:
 
 | Family | Task | Supported language(s) | Contributor | What They Added |
 |---|---|---|---|---|
+| **f5_tts** | TTS, voice cloning | en, ar (Habibi) | Community | [F5-TTS](f5_tts.md) flow-matching DiT — M0 scaffolding, aliases `habibi`/`habibi_tts` |
 | **glm_tts** | TTS, voice cloning | zh, en | Mirek [@mirek190](https://github.com/mirek190) | [GLM-TTS](glm_tts.md) zero-shot synthesis and voice cloning support |
 | **inflect_v2** | TTS | en | Community | [Inflect Micro v2 and Nano v2](inflect_v2.md) native FP32 offline synthesis |
 | **kroko_asr** | ASR | de, en, es, fr, it, he, nl, pt, sv, tr | Mirek [@mirek190](https://github.com/mirek190) | [Kroko Community ASR](kroko_asr.md) native offline/streaming Zipformer2/RNN-T transcription with word timestamps |

include/engine/community_models/f5_tts/dit_modules.h

@@ -0,0 +1,53 @@
+#pragma once
+
+#include "engine/community_models/f5_tts/runtime.h"
+#include "engine/community_models/f5_tts/weights.h"
+
+#include "engine/framework/core/module.h"
+
+namespace engine::models::f5_tts {
+
+struct F5DiTGraphBuild {
+    core::TensorValue x;          // leaf [1, N, 100]
+    core::TensorValue cond;       // leaf [1, N, 100]
+    core::TensorValue text_ids;   // leaf [NT] (i32)
+    core::TensorValue time_input; // leaf [1, 256]
+    core::TensorValue output;     // [1, N, 100]
+};
+
+// Builds the DiT velocity graph from framework modules (see dit_modules.cpp).
+F5DiTGraphBuild build_dit_modules_graph(
+    ggml_context * ggml,
+    const F5DiTWeights & w,
+    const F5Architecture & arch,
+    int frames,
+    int text_len,
+    core::BackendType backend_type);
+
+}  // namespace engine::models::f5_tts
+
+namespace engine::models::f5_tts {
+// CUDA build-time constant staging (internal; used by runtime.cpp)
+struct ConstStage;
+std::vector<ConstStage> * const_stage_begin();
+void const_stage_bind(std::vector<ConstStage> * stage, ggml_backend_t backend);
+void const_stage_upload(std::vector<ConstStage> * stage, ggml_backend_t backend);
+void const_stage_end(std::vector<ConstStage> * stage);
+}  // namespace engine::models::f5_tts
+
+namespace engine::models::f5_tts {
+
+// Batched-CFG (B=2) variant of the module-composed DiT graph.
+F5DiTGraphBuild build_dit_cfg_modules_graph(
+    ggml_context * ggml,
+    const F5DiTWeights & w,
+    const F5Architecture & arch,
+    int frames,
+    int text_len,
+    core::BackendType backend_type);
+
+// debug: registered stage taps from the last CFG graph build (F5_DUMP_STAGES=1)
+std::vector<std::pair<std::string, ggml_tensor *>> & stage_taps();
+
+}  // namespace engine::models::f5_tts
+

include/engine/community_models/f5_tts/runtime.h

@@ -0,0 +1,94 @@
+#pragma once
+
+#include "engine/framework/assets/resource_bundle.h"
+#include "engine/framework/runtime/session.h"
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace engine::models::f5_tts {
+
+// F5TTS_v1_Base / Habibi architecture (validated against the checkpoint:
+// 22 blocks, dim 1024, 16 heads x 64, ff_mult 2, text_dim 512, conv_layers 4,
+// 100 mel channels, embedding rows 2731).
+struct F5Architecture {
+    int dim = 1024;
+    int depth = 22;
+    int heads = 16;
+    int head_dim = 64;
+    int ff_mult = 2;
+    int text_dim = 512;
+    int conv_layers = 4;
+    int mel_dim = 100;
+    int vocab_rows = 2731;
+    int sample_rate = 24000;
+    int hop_length = 256;
+    int n_fft = 1024;
+};
+
+struct F5SampleOptions {
+    float cfg_strength = 2.0F;
+    float sway_sampling_coef = -1.0F;
+    float speed = 1.0F;
+    int steps = 32;
+    uint32_t seed = 0;
+};
+
+// Compute device for the DiT forward: CUDA device index or CPU threads.
+struct F5ComputeDevice {
+    bool use_cuda = false;
+    int device = 0;      // CUDA device index
+    int threads = 0;     // CPU threads; 0 = hardware concurrency
+    // FP16 linear weights: GEMMs get ~3x faster on tensor cores but each
+    // mul_mat converts the F32 activations to F16 first; at F5's GEMM sizes
+    // (K=1024/2048, N~1022) the conversion overhead outweighs the gain on an
+    // RTX 3090 (measured 4.0s -> 5.0s per clip). Off by default.
+    bool fp16_weights = false;
+};
+
+// Debug taps for parity testing: when non-null, intermediate stage outputs are
+// appended (column layout, [features, T] flattened feature-major).
+struct F5DebugTaps {
+    std::vector<float> * text_embed = nullptr;      // after lookup + pos (01)
+    std::vector<float> * text_convnext = nullptr;   // after 4 ConvNeXt (02)
+    std::vector<float> * text_padded = nullptr;     // after pad/curtail (03)
+    std::vector<float> * input_embed = nullptr;     // after proj + cpe (04)
+    std::vector<float> * time_embed = nullptr;      // (05)
+    std::vector<float> * block0 = nullptr;          // (07_block0)
+    std::vector<float> * block21 = nullptr;         // (07_block21)
+};
+
+// Full DiT velocity forward for one Euler step inputs.
+// x/cond: [seq_len * mel_dim] row-major (seq-major), text: ids (0 = filler),
+// returns [seq_len * mel_dim] column layout (out[m * N + n]).
+std::vector<float> f5_dit_forward(
+    const std::string & weights_path,
+    const std::vector<float> & x,
+    const std::vector<float> & cond,
+    const std::vector<int32_t> & text,
+    float time_value,
+    int seq_len,
+    const F5Architecture & arch,
+    bool drop_audio_cond,
+    bool drop_text,
+    const F5DebugTaps * taps = nullptr,
+    const F5ComputeDevice * device = nullptr);
+
+// Batched CFG: one ne3=2 graph compute returning {conditioned, unconditioned}
+// velocities. Matches python cfg_infer: the uncond half runs with
+// drop_audio_cond (zeroed cond) and drop_text (filler text id 0); the host
+// upload in runtime.cpp prepares both halves accordingly.
+std::pair<std::vector<float>, std::vector<float>> f5_dit_forward_cfg(
+    const std::string & weights_path,
+    const std::vector<float> & x,
+    const std::vector<float> & cond,
+    const std::vector<int32_t> & text,
+    float time_value,
+    int seq_len,
+    const F5Architecture & arch,
+    const F5ComputeDevice * device = nullptr);
+
+}  // namespace engine::models::f5_tts

include/engine/community_models/f5_tts/session.h

@@ -0,0 +1,54 @@
+#pragma once
+
+#include "engine/framework/assets/resource_bundle.h"
+#include "engine/framework/runtime/model.h"
+#include "engine/framework/runtime/session.h"
+#include "engine/framework/runtime/spec_backed_model.h"
+
+#include <filesystem>
+#include <memory>
+#include <string>
+
+namespace engine::models::f5_tts {
+
+// F5-TTS community model assets: resource bundle + resolved checkpoint path.
+// Weights are loaded lazily by the runtime on first synthesis (graph cache).
+struct F5TTSAssets {
+    assets::ResourceBundle resources;
+    std::filesystem::path checkpoint;  // DiT *.safetensors (ema weights)
+};
+
+class F5TTSSession final : public runtime::IOfflineVoiceTaskSession {
+public:
+    F5TTSSession(
+        const runtime::TaskSpec & task,
+        const runtime::SessionOptions & options,
+        std::shared_ptr<const F5TTSAssets> assets,
+        std::shared_ptr<const engine::model_spec::ModelContract> contract);
+
+    std::string family() const noexcept override;
+    runtime::VoiceTaskKind task_kind() const noexcept override;
+    runtime::RunMode run_mode() const noexcept override;
+    void prepare(const runtime::SessionPreparationRequest & request) override;
+
+    runtime::TaskResult run(const runtime::TaskRequest & request) override;
+
+private:
+    runtime::VoiceTaskKind task_kind_;
+    runtime::RunMode run_mode_;
+    std::shared_ptr<const F5TTSAssets> assets_;
+    std::shared_ptr<const engine::model_spec::ModelContract> contract_;
+    std::string vocos_path_;
+    std::string dialect_ = "UNK";
+    int frame_budget_ = 0;  // 0 = default 2048
+    bool use_cuda_ = false;
+    int cuda_device_ = 0;
+    int threads_ = 0;
+};
+
+std::shared_ptr<const F5TTSAssets> load_f5_tts_assets(
+    const std::filesystem::path & model_path);
+
+std::shared_ptr<runtime::IVoiceModelLoader> make_f5_tts_loader();
+
+}  // namespace engine::models::f5_tts