From a111c7505908bad301a08b5ec805b40d284192f2 Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Thu, 10 Sep 2026 08:33:18 +0000 Subject: [PATCH] docs: clarify SenseVoice word timestamp alignment Signed-off-by: zhifu gao --- docs/python_api.md | 23 +++++++++++++++++ docs/python_api_zh.md | 23 +++++++++++++++++ tests/test_python_api_docs_contract.py | 34 +++++++++++++++++++++++++- 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/docs/python_api.md b/docs/python_api.md index 97ad49d96..26fd06e81 100644 --- a/docs/python_api.md +++ b/docs/python_api.md @@ -135,6 +135,29 @@ For `sentence_timestamp=True` without speakers, the current wrapper can return V Additional fields such as `words`, `ctc_timestamps`, or speaker embeddings are model-specific. Do not relabel SDK milliseconds as service seconds without conversion. See [timestamp regression tests](../tests/test_paraformer_timestamp_contract.py) and the model sources below. +### Consuming SenseVoice Alignment + +For this toolkit's SenseVoice path, request `output_timestamp=True` in `generate()`. Pair the returned `words` with `timestamp` in **milliseconds**. These are model alignment units: English subwords can merge into words, while other languages can retain character units. Neither the raw rich-tagged text nor `rich_transcription_postprocess(text)` is a character-by-character index into the timestamps. Display postprocessing and text replacements do not realign audio. + +Use this helper on each result that has alignment fields. The length check prevents silent truncation; missing fields require explicit application handling, not invented timestamps. + +```python +def sensevoice_word_intervals(result): + words, timestamps = result.get("words"), result.get("timestamp") + if words is None or timestamps is None: + raise ValueError("This result has no word/timestamp alignment") + if len(words) != len(timestamps): + raise ValueError("Word/timestamp count mismatch") + return [ + {"word": word, "start_ms": start_ms, "end_ms": end_ms} + for word, (start_ms, end_ms) in zip(words, timestamps) + ] +``` + +Call `sensevoice_word_intervals(result)` for a dictionary from `model.generate(..., output_timestamp=True)`. Empty aligned lists return an empty list; a no-speech result without `words` raises instead. Keep display-text cleanup separate from the aligned sequence. + +A fixed-checkpoint check on FunASR 1.4.15 produced 242 paired units for the public Chinese sample with ITN and 217 without ITN. An English sample had 84 display characters but 16 paired units. These results illustrate the consumption contract, not timestamp precision, transcription accuracy or a fix for every special-symbol case. See the [reproducible inputs and boundaries](https://github.com/QwenAudio/SenseVoice/issues/215#issuecomment-5615363096). This is not native Nano Transformers output and does not add speaker identity. + ## Language, Hotwords, and Alignment Are Model-Specific | Implementation | Runtime controls in this checkout | diff --git a/docs/python_api_zh.md b/docs/python_api_zh.md index 1a35a628b..400510d17 100644 --- a/docs/python_api_zh.md +++ b/docs/python_api_zh.md @@ -135,6 +135,29 @@ for result in model.generate(input=str(paths["audio"]), return_spk_res=True): `words`、`ctc_timestamps`、说话人特征等额外字段属于模型特有输出。不能不经换算就把 SDK 毫秒值当作服务端秒值。参见[时间戳回归测试](../tests/test_paraformer_timestamp_contract.py) 及下列模型源码。 +### 使用 SenseVoice 对齐结果 + +在本工具包的 SenseVoice 路径中，通过 `generate()` 的 `output_timestamp=True` 请求对齐。应将返回的 `words` 与**毫秒**单位的 `timestamp` 配对。这些是模型的对齐单元：英文子词可能合并成词，其他语言则可能保留字符单元。原始富标签文本和 `rich_transcription_postprocess(text)` 都不能作为时间戳的逐字符索引；显示后处理和文本替换不会重新对齐音频。 + +对含对齐字段的每个结果使用下面的辅助函数。长度检查避免静默截断；缺少字段时应由应用显式处理，不能编造时间戳。 + +```python +def sensevoice_word_intervals(result): + words, timestamps = result.get("words"), result.get("timestamp") + if words is None or timestamps is None: + raise ValueError("This result has no word/timestamp alignment") + if len(words) != len(timestamps): + raise ValueError("Word/timestamp count mismatch") + return [ + {"word": word, "start_ms": start_ms, "end_ms": end_ms} + for word, (start_ms, end_ms) in zip(words, timestamps) + ] +``` + +将 `model.generate(..., output_timestamp=True)` 返回的结果字典传给 `sensevoice_word_intervals(result)`。两份对齐列表均为空时返回空列表；无语音结果若缺少 `words`，则明确报错。显示文本清理应与对齐序列分开处理。 + +在 FunASR 1.4.15 的固定 checkpoint 检查中，公开中文样例开启 ITN 得到 242 组配对单元，关闭 ITN 得到 217 组；英文样例有 84 个显示字符，但只有 16 组配对单元。这些结果用于说明消费契约，不是时间戳精度、转写准确率评测，也不代表所有特殊符号问题已修复。参见[可复现输入与验证边界](https://github.com/QwenAudio/SenseVoice/issues/215#issuecomment-5615363096)。这不是原生 Nano Transformers 的输出，也不提供说话人身份识别。 + ## 语言、热词和对齐取决于模型 | 实现 | 当前代码版本的运行参数 | diff --git a/tests/test_python_api_docs_contract.py b/tests/test_python_api_docs_contract.py index 2f4d51025..cf6703343 100644 --- a/tests/test_python_api_docs_contract.py +++ b/tests/test_python_api_docs_contract.py @@ -46,7 +46,7 @@ def test_examples_compile_and_use_source_backed_keywords(self): "funasr/models/paraformer_streaming/model.py"))) for path in DOCS: blocks = python_blocks(path) - self.assertEqual(len(blocks), 3, path.name) + self.assertEqual(len(blocks), 4, path.name) for index, block in enumerate(blocks): with self.subTest(doc=path.name, block=index): compile(block, str(path), "exec") @@ -63,6 +63,38 @@ def test_examples_compile_and_use_source_backed_keywords(self): def test_translations_keep_identical_executable_examples(self): self.assertEqual(python_blocks(DOCS[0]), python_blocks(DOCS[1])) + def alignment_helper(self, path): + blocks = [block for block in python_blocks(path) + if "def sensevoice_word_intervals(" in block] + self.assertEqual(len(blocks), 1, path.name) + namespace = {} + exec(compile(blocks[0], str(path), "exec"), namespace) + return namespace["sensevoice_word_intervals"] + + def test_sensevoice_example_pairs_words_not_display_characters(self): + # SenseVoice215: display characters are not the alignment units. + result = {"text": "<|en|>hello world", "words": ["hello", "world"], + "timestamp": [[0, 500], [700, 1000]]} + for path in DOCS: + with self.subTest(doc=path.name): + helper = self.alignment_helper(path) + self.assertEqual(helper(result), [ + {"word": "hello", "start_ms": 0, "end_ms": 500}, + {"word": "world", "start_ms": 700, "end_ms": 1000}, + ]) + self.assertEqual(helper({"words": [], "timestamp": []}), []) + self.assertEqual(result["words"], ["hello", "world"]) + + def test_sensevoice_example_rejects_missing_or_truncated_alignment(self): + for path in DOCS: + helper = self.alignment_helper(path) + for result in ({"text": ""}, {"words": [], "timestamp": None}, + {"words": ["hello"], "timestamp": []}, + {"words": [], "timestamp": [[0, 500]]}): + with self.subTest(doc=path.name, result=result): + with self.assertRaises(ValueError): + helper(result) + def test_local_links_resolve_without_repository_scan(self): for path in DOCS: text = path.read_text(encoding="utf-8")