feat: true streaming acoustic decode + vv_capi_tts_stream (realtime-0.5B)

include/vibevoice_capi.h

@@ -70,6 +70,31 @@ int vv_capi_tts(const char*        text,
                 int                max_speech_frames,
                 uint32_t           seed);
 
+// Streaming counterpart of vv_capi_tts for the realtime-0.5B model. Each
+// decoded audio window is converted to 24 kHz mono signed-16-bit PCM and handed
+// to `on_pcm` as soon as it is produced. Concatenating every callback's samples
+// yields exactly the PCM that vv_capi_tts writes to a WAV for the same
+// text/voice/seed (same generate path, same float→int16 conversion).
+//
+//   text              - sentence or speaker-tagged dialog (see vv_capi_tts).
+//   voice_path        - voice gguf; may be NULL if supplied to vv_capi_load.
+//   n_diffusion_steps - 0 → 20. cfg_scale 0 → 1.3. max_speech_frames 0 → 200.
+//   seed              - 0 → random.
+//   on_pcm            - called per window; return non-zero to abort cleanly.
+//   user              - opaque pointer forwarded to on_pcm.
+//
+// Realtime-0.5B only: returns -20 if a 1.5B model is loaded (unsupported).
+// Returns 0 on success, non-zero error code otherwise.
+typedef int (*vv_pcm_cb)(const int16_t* samples, int n_samples, void* user);
+int vv_capi_tts_stream(const char* text,
+                       const char* voice_path,
+                       int         n_diffusion_steps,
+                       float       cfg_scale,
+                       int         max_speech_frames,
+                       uint32_t    seed,
+                       vv_pcm_cb   on_pcm,
+                       void*       user);
+
 // Transcribe `src_wav_path` into a JSON string written into the caller-
 // owned `out_json` buffer of size `out_capacity`. The JSON is the same
 // shape the model produces, e.g.

src/acoustic_tokenizer.cpp

@@ -351,4 +351,44 @@ struct ggml_tensor* decoder_forward(struct ggml_context*  ctx,
     return y;
 }
 
+struct ggml_tensor* decoder_forward_streaming(struct ggml_context*    ctx,
+                                              struct ggml_tensor*     z,
+                                              const DecoderWeights&   w,
+                                              const AcousticConfig&   cfg,
+                                              StreamingCache&         cache) {
+    char buf[64];
+
+    struct ggml_tensor* h = sconv1d_causal_streaming(
+        ctx, z, w.stem.kernel, w.stem.bias, w.stem.stride, /*dilation=*/1, /*groups=*/1,
+        cache, "dec.stem");
+    for (size_t j = 0; j < w.stages[0].size(); ++j) {
+        std::snprintf(buf, sizeof(buf), "dec.s0.b%zu", j);
+        h = block1d_forward_streaming(ctx, h, w.stages[0][j], cfg.eps, cache, buf);
+    }
+    for (size_t i = 1; i < cfg.depths.size(); ++i) {
+        std::snprintf(buf, sizeof(buf), "dec.up%zu", i - 1);
+        h = sconv_transpose1d_causal_streaming(
+            ctx, h, w.ups[i - 1].kernel, w.ups[i - 1].bias, w.ups[i - 1].stride,
+            cache, buf);
+        for (size_t j = 0; j < w.stages[i].size(); ++j) {
+            std::snprintf(buf, sizeof(buf), "dec.s%zu.b%zu", i, j);
+            h = block1d_forward_streaming(ctx, h, w.stages[i][j], cfg.eps, cache, buf);
+        }
+    }
+
+    struct ggml_tensor* y = h;
+    if (w.final_norm) {
+        struct ggml_tensor* p = ggml_permute(ctx, y, 1, 0, 2, 3);
+        p = ggml_cont(ctx, p);
+        p = ggml_rms_norm(ctx, p, cfg.eps);
+        p = ggml_mul(ctx, p, w.final_norm);
+        p = ggml_permute(ctx, p, 1, 0, 2, 3);
+        y = ggml_cont(ctx, p);
+    }
+    y = sconv1d_causal_streaming(
+        ctx, y, w.head.kernel, w.head.bias, w.head.stride, /*dilation=*/1, /*groups=*/1,
+        cache, "dec.head");
+    return y;
+}
+
 }  // namespace vv

src/acoustic_tokenizer.hpp

@@ -120,6 +120,16 @@ struct ggml_tensor* encoder_forward_streaming(struct ggml_context*    ctx,
                                               const AcousticConfig&   cfg,
                                               StreamingCache&         cache);
 
+// Streaming decoder. Same forward math as decoder_forward, but every causal
+// conv (stem, per-stage residual blocks, transpose upsamplers, head) reads /
+// writes its left context through `cache` under a unique layer_id so chunked
+// decoding is bit-exact vs a single-shot pass. z: [n_frames, vae_dim, B].
+struct ggml_tensor* decoder_forward_streaming(struct ggml_context*    ctx,
+                                              struct ggml_tensor*     z,
+                                              const DecoderWeights&   w,
+                                              const AcousticConfig&   cfg,
+                                              StreamingCache&         cache);
+
 }  // namespace vv
 
 #endif  // VIBEVOICE_ACOUSTIC_TOKENIZER_HPP

src/conv1d.cpp

@@ -175,4 +175,60 @@ struct ggml_tensor* sconv_transpose1d_causal(struct ggml_context* ctx,
     return maybe_add_bias_t(ctx, y, bias);
 }
 
+struct ggml_tensor* sconv_transpose1d_causal_streaming(struct ggml_context* ctx,
+                                                       struct ggml_tensor*  x,
+                                                       struct ggml_tensor*  kernel,
+                                                       struct ggml_tensor*  bias,
+                                                       int stride,
+                                                       StreamingCache&    cache,
+                                                       const std::string& layer_id) {
+    const int K    = static_cast<int>(kernel->ne[0]);
+    const int C_in = static_cast<int>(x->ne[1]);
+    const int B    = static_cast<int>(x->ne[2]);
+    const int T_in = static_cast<int>(x->ne[0]);
+    const int L    = (K - 1 + stride - 1) / stride;  // ceil((K-1)/stride) input frames
+
+    auto& entry = cache[layer_id];
+    entry.T = L;
+    entry.C = C_in;
+
+    struct ggml_tensor* prefix = nullptr;
+    if (L > 0) {
+        prefix = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, L, C_in, B);
+        ggml_set_name(prefix, ("cache_prefix_" + layer_id).c_str());
+    }
+    entry.prefix = prefix;
+
+    struct ggml_tensor* xp = x;
+    if (prefix) xp = ggml_concat(ctx, prefix, x, /*dim=*/0);
+
+    // conv_transpose_1d requires F32 kernels (matches sconv_transpose1d_causal).
+    struct ggml_tensor* k = (kernel->type == GGML_TYPE_F32)
+                              ? kernel : ggml_cast(ctx, kernel, GGML_TYPE_F32);
+    struct ggml_tensor* y_full = ggml_conv_transpose_1d(ctx, k, xp, stride, /*p0=*/0, /*d0=*/1);
+
+    // Emit exactly the new frames' outputs: [L*stride, L*stride + T_in*stride).
+    const int64_t emit_off = static_cast<int64_t>(L) * stride;
+    const int64_t emit_len = static_cast<int64_t>(T_in) * stride;
+    struct ggml_tensor* y = ggml_view_3d(ctx, y_full,
+        /*ne0=*/emit_len, /*ne1=*/y_full->ne[1], /*ne2=*/y_full->ne[2],
+        /*nb1=*/y_full->nb[1], /*nb2=*/y_full->nb[2],
+        /*offset=*/static_cast<size_t>(emit_off) * y_full->nb[0]);
+    y = ggml_cont(ctx, y);
+
+    // Cache the last L input frames for the next chunk's left context.
+    if (L > 0) {
+        struct ggml_tensor* base = prefix ? ggml_concat(ctx, prefix, x, /*dim=*/0) : x;
+        const int T_avail = T_in + L;
+        const int start   = std::max(0, T_avail - L);
+        struct ggml_tensor* view = ggml_view_3d(ctx, base,
+            /*ne0=*/L, /*ne1=*/C_in, /*ne2=*/B,
+            /*nb1=*/base->nb[1], /*nb2=*/base->nb[2],
+            /*offset=*/static_cast<size_t>(start) * base->nb[0]);
+        entry.next_view = ggml_cont(ctx, view);
+    }
+
+    return maybe_add_bias_t(ctx, y, bias);
+}
+
 }  // namespace vv

src/conv1d.hpp

@@ -110,6 +110,18 @@ struct ggml_tensor* sconv_transpose1d_causal(struct ggml_context* ctx,
                                              struct ggml_tensor*  bias,   // [C_out] or null
                                              int stride);
 
+// Streaming variant of sconv_transpose1d_causal. Keeps L = ceil((K-1)/stride)
+// input frames of left context in `cache[layer_id]`; emits exactly the new
+// frames' output samples [L*stride : L*stride + T_in*stride). Bit-exact vs a
+// single-shot pass. Ignores is_final_chunk (all-causal, no right-pad needed).
+struct ggml_tensor* sconv_transpose1d_causal_streaming(struct ggml_context* ctx,
+                                                       struct ggml_tensor*  x,
+                                                       struct ggml_tensor*  kernel,
+                                                       struct ggml_tensor*  bias,
+                                                       int stride,
+                                                       StreamingCache&    cache,
+                                                       const std::string& layer_id);
+
 }  // namespace vv
 
 #endif  // VIBEVOICE_CONV1D_HPP

src/vibevoice_capi.cpp

@@ -11,11 +11,14 @@
 #include "vibevoice_asr.hpp"
 #include "vibevoice_tts.hpp"
 
+#include <algorithm>
+#include <cmath>
 #include <cstdio>
 #include <cstring>
 #include <memory>
 #include <mutex>
 #include <string>
+#include <vector>
 
 namespace {
 
@@ -184,6 +187,66 @@ int vv_capi_tts(const char*        text,
     return vv::save_wav_pcm16(dst_wav_path, audio_out);
 }
 
+int vv_capi_tts_stream(const char* text,
+                       const char* voice_path,
+                       int         n_diffusion_steps,
+                       float       cfg_scale,
+                       int         max_speech_frames,
+                       uint32_t    seed,
+                       vv_pcm_cb   on_pcm,
+                       void*       user) {
+    auto& g = engine();
+    std::lock_guard<std::mutex> lk(g.mu);
+    if (!g.tts)          return -3;
+    if (!text || !on_pcm) return -2;
+
+    // Realtime-0.5B only: the streaming decoder is not wired for the 1.5B
+    // variant (see vibevoice_tts_generate_streaming). Report a distinct code.
+    if (g.tts->variant == "1.5b") {
+        VV_LOG_ERROR("vv_capi_tts_stream: streaming unsupported for 1.5b model");
+        return -20;
+    }
+
+    vv::VibeVoiceTTSParams p;
+    p.cfg_scale          = cfg_scale > 0.0f ? cfg_scale : 1.3f;
+    p.n_diffusion_steps  = n_diffusion_steps > 0 ? n_diffusion_steps : 20;
+    p.max_speech_frames  = max_speech_frames > 0 ? max_speech_frames : 200;
+    p.seed               = seed;
+    p.verbose            = false;
+
+    if (voice_path && voice_path[0]) {
+        if (!ensure_voice_loaded(g, voice_path)) return -3;
+    }
+    if (!g.voice) {
+        VV_LOG_ERROR("vv_capi_tts_stream: no voice loaded — pass voice_path "
+                     "here or to vv_capi_load");
+        return -2;
+    }
+    p.voice = g.voice.get();
+
+    // Reused int16 buffer. Convert each float window with the EXACT same
+    // clamp + round as save_wav_pcm16 (audio_io.cpp) so the concatenated
+    // callback stream is byte-identical to the WAV vv_capi_tts writes.
+    std::vector<int16_t> buf;
+    auto on_chunk = [&](const float* s, int n) -> bool {
+        buf.resize(static_cast<size_t>(n));
+        for (int i = 0; i < n; ++i) {
+            float x = std::max(-1.0f, std::min(1.0f, s[i]));
+            buf[static_cast<size_t>(i)] =
+                static_cast<int16_t>(std::lround(x * 32767.0f));
+        }
+        // Non-zero from the caller means abort → stop generation cleanly.
+        return on_pcm(buf.data(), n, user) == 0;
+    };
+
+    int rc = vv::vibevoice_tts_generate_streaming(g.tts.get(), text, p, on_chunk);
+    if (rc != 0) {
+        VV_LOG_ERROR("vv_capi_tts_stream: generate rc=%d", rc);
+        return rc;
+    }
+    return 0;
+}
+
 int vv_capi_asr(const char* src_wav_path,
                 char*       out_json,
                 size_t      out_capacity,