diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..cde9a881 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,5 @@ +self-hosted-runner: + labels: + - voicelife-hil + - sparkbot + - pcb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 898ad437..8dfce642 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,8 @@ jobs: --journey im-gateway-strong-reminder \ --profile host \ --artifact-dir artifacts/im-gateway-e2e \ - --timeout 60 + --timeout 60 \ + --retries 0 - name: 运行 Host 快速恢复 E2E run: | E2E_RECOVERY_SUITE=quick python3 scripts/run_e2e.py \ @@ -101,13 +102,36 @@ jobs: --artifact-dir artifacts/im-gateway-e2e \ --timeout 120 \ --retries 0 - - name: 上传 Host E2E 脱敏证据 + - name: 验证故意失败 evidence + if: always() + shell: bash + run: | + set +e + VOICELIFE_E2E_CONTRACT_FAILURE=1 python3 scripts/run_e2e.py \ + --layer host \ + --journey lifecycle-example \ + --profile host \ + --artifact-dir artifacts/im-gateway-e2e/contract-failure \ + --timeout 60 \ + --retries 0 + failure_status=$? + set -e + test "$failure_status" -eq 20 + - name: 校验 Host E2E 脱敏 evidence + id: validate-host-evidence if: always() + run: python3 scripts/check_e2e_artifacts.py artifacts/im-gateway-e2e + - name: 写入 Host E2E job summary + if: always() + run: python3 scripts/render_e2e_summary.py artifacts/im-gateway-e2e >> "$GITHUB_STEP_SUMMARY" + - name: 上传 Host E2E 脱敏证据 + if: always() && steps.validate-host-evidence.outcome == 'success' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: im-gateway-host-e2e-evidence path: artifacts/im-gateway-e2e if-no-files-found: ignore + retention-days: 14 coverage: name: 覆盖率（Codecov） diff --git a/.github/workflows/hil-nightly.yml b/.github/workflows/hil-nightly.yml new file mode 100644 index 00000000..df1240f0 --- /dev/null +++ b/.github/workflows/hil-nightly.yml @@ -0,0 +1,154 @@ +name: HIL Manual + +on: + workflow_dispatch: + inputs: + profile: + description: Device profile to run (the default runs both profiles) + required: false + type: choice + default: all + options: + - all + - sparkbot + - pcb + journey: + description: HIL journey + required: false + type: choice + default: im-pairing + options: + - im-pairing + - voice + tts_provider: + description: Host TTS fixture provider used by the voice journey + required: false + type: choice + default: dashscope + options: + - dashscope + - aliyun-nls + device: + description: Optional descriptor name; requires a single profile selection + required: false + type: string + default: "" + +permissions: + contents: read + +concurrency: + group: hil-nightly-${{ github.ref }}-${{ inputs.profile || 'all' }} + cancel-in-progress: false + +jobs: + hil: + name: HIL / ${{ matrix.profile }} + runs-on: [self-hosted, voicelife-hil, "${{ matrix.profile }}"] + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + profile: ${{ fromJSON(inputs.profile == 'sparkbot' && '["sparkbot"]' || inputs.profile == 'pcb' && '["pcb"]' || '["sparkbot", "pcb"]') }} + env: + HIL_DEVICE_ROOT: ${{ vars.VOICELIFE_HIL_DEVICE_ROOT || '/opt/voicelife/hil/devices' }} + HIL_LEASE_ROOT: ${{ vars.VOICELIFE_HIL_LEASE_ROOT || '/opt/voicelife/hil/leases' }} + VOICELIFE_HIL_SERVER: ${{ secrets.VOICELIFE_HIL_SERVER }} + VOICELIFE_HIL_SERVER_DIR: ${{ secrets.VOICELIFE_HIL_SERVER_DIR }} + VOICELIFE_HIL_GATEWAY_ORIGIN: ${{ secrets.VOICELIFE_HIL_GATEWAY_ORIGIN }} + VOICELIFE_HIL_USER_ID: ${{ secrets.VOICELIFE_HIL_USER_ID }} + DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }} + ALIYUN_NLS_APPKEY: ${{ secrets.ALIYUN_NLS_APPKEY }} + ALIYUN_NLS_TOKEN: ${{ secrets.ALIYUN_NLS_TOKEN }} + ALIYUN_NLS_URL: ${{ vars.ALIYUN_NLS_URL }} + HIL_DEVICE_NAME: ${{ inputs.device || '' }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - name: 检查受控 Runner 配置 + run: | + set -euo pipefail + device_name="$HIL_DEVICE_NAME" + if [[ -n "$device_name" ]]; then + [[ "${{ inputs.profile || 'all' }}" != "all" ]] + [[ "$device_name" =~ ^[a-z0-9][a-z0-9_-]{0,63}$ ]] + descriptor="$HIL_DEVICE_ROOT/$device_name.json" + else + descriptor="$HIL_DEVICE_ROOT/${{ matrix.profile }}.json" + fi + test -r "$descriptor" + test -n "${VOICELIFE_HIL_SERVER:-}" + test -n "${VOICELIFE_HIL_SERVER_DIR:-}" + test -n "${VOICELIFE_HIL_GATEWAY_ORIGIN:-}" + test -n "${VOICELIFE_HIL_USER_ID:-}" + python3 -c 'import serial, esptool' >/dev/null + if [[ "${{ inputs.journey || 'im-pairing' }}" == "voice" ]]; then + if [[ "${{ inputs.tts_provider || 'dashscope' }}" == "dashscope" ]]; then + test -n "${DASHSCOPE_API_KEY:-}" + python3 -c 'import dashscope' >/dev/null + else + test -n "${ALIYUN_NLS_APPKEY:-}" + test -n "${ALIYUN_NLS_TOKEN:-}" + python3 -c 'import nls' >/dev/null + fi + command -v ffmpeg >/dev/null + fi + - name: 运行 HIL journey + id: journey + run: | + set +e + mkdir -p "artifacts/hil-${{ matrix.profile }}" + device_name="$HIL_DEVICE_NAME" + if [[ -n "$device_name" ]]; then + device_path="$HIL_DEVICE_ROOT/$device_name.json" + else + device_path="$HIL_DEVICE_ROOT/${{ matrix.profile }}.json" + fi + run_args=( + python3 scripts/run_e2e.py \ + --layer hil \ + --journey "${{ inputs.journey || 'im-pairing' }}" \ + --profile "${{ matrix.profile }}" \ + --artifact-dir "artifacts/hil-${{ matrix.profile }}" \ + --timeout 900 \ + --retries 0 \ + --device "$device_path" \ + --lease-dir "$HIL_LEASE_ROOT" \ + --server "$VOICELIFE_HIL_SERVER" \ + --server-dir "$VOICELIFE_HIL_SERVER_DIR" \ + --gateway-origin "$VOICELIFE_HIL_GATEWAY_ORIGIN" \ + --user-id "$VOICELIFE_HIL_USER_ID" + ) + if [[ "${{ inputs.journey || 'im-pairing' }}" == "voice" ]]; then + run_args+=( + --input-tts "${{ inputs.tts_provider || 'dashscope' }}" + --tts-model "${VOICELIFE_TTS_MODEL:-qwen-audio-3.0-tts-flash}" + --voice "${VOICELIFE_TTS_VOICE:-longanlingxi}" + --text '你好牛牛，请介绍一下你自己。' + --text '把刚才的回答再简短一点。' + --text '请用一句话总结我们刚才的对话。' + --expect-terminal + ) + fi + "${run_args[@]}" + run_status=$? + exit "$run_status" + - name: 校验 HIL 脱敏 evidence + id: validate-hil-evidence + if: always() + run: python3 scripts/check_e2e_artifacts.py "artifacts/hil-${{ matrix.profile }}" + - name: 写入 HIL job summary + if: always() + run: python3 scripts/render_e2e_summary.py "artifacts/hil-${{ matrix.profile }}" >> "$GITHUB_STEP_SUMMARY" + - name: 上传 HIL 脱敏 evidence + if: always() && steps.validate-hil-evidence.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: hil-${{ matrix.profile }}-evidence + path: artifacts/hil-${{ matrix.profile }} + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/im-recovery-nightly.yml b/.github/workflows/im-recovery-nightly.yml index b71e2bcb..edfa2ce4 100644 --- a/.github/workflows/im-recovery-nightly.yml +++ b/.github/workflows/im-recovery-nightly.yml @@ -59,10 +59,18 @@ jobs: --artifact-dir artifacts/im-gateway-recovery \ --timeout 300 \ --retries 0 - - name: 上传恢复矩阵脱敏证据 + - name: 校验恢复矩阵脱敏 evidence + id: validate-recovery-evidence + if: always() + run: python3 scripts/check_e2e_artifacts.py artifacts/im-gateway-recovery + - name: 写入恢复矩阵 job summary if: always() + run: python3 scripts/render_e2e_summary.py artifacts/im-gateway-recovery >> "$GITHUB_STEP_SUMMARY" + - name: 上传恢复矩阵脱敏证据 + if: always() && steps.validate-recovery-evidence.outcome == 'success' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: im-gateway-recovery-nightly-evidence path: artifacts/im-gateway-recovery if-no-files-found: ignore + retention-days: 14 diff --git a/components/voicelife_mcp/test/mcp_server_test.cc b/components/voicelife_mcp/test/mcp_server_test.cc index 7d95a421..701169a1 100644 --- a/components/voicelife_mcp/test/mcp_server_test.cc +++ b/components/voicelife_mcp/test/mcp_server_test.cc @@ -463,9 +463,6 @@ void TestToolListing() { Check(document != nullptr, "tools/list 应序列化为合法 JSON"); yyjson_val* tools = yyjson_obj_get(yyjson_doc_get_root(document), "tools"); Check(yyjson_is_arr(tools) && yyjson_arr_size(tools) == 3, "tools/list JSON 应包含全部工具"); - Check(yyjson_is_null(yyjson_obj_get(yyjson_doc_get_root(document), "nextCursor")), - "tools/list JSON 应包含空的 nextCursor 分页字段"); - yyjson_val* configure = yyjson_arr_get(tools, 0); yyjson_val* configure_schema = yyjson_obj_get(configure, "inputSchema"); yyjson_val* properties = yyjson_obj_get(configure_schema, "properties"); diff --git a/config/profiles/esp32s3-esp-sparkbot-serial-voice.json b/config/profiles/esp32s3-esp-sparkbot-serial-voice.json index 9cbcb250..f46fb95a 100644 --- a/config/profiles/esp32s3-esp-sparkbot-serial-voice.json +++ b/config/profiles/esp32s3-esp-sparkbot-serial-voice.json @@ -6,7 +6,11 @@ "audio": { "driver": "esp32s3-es8311-duplex", "capabilities": ["es8311-duplex"], "configRef": "env://VOICELIFE_AUDIO_PROFILE" }, "speech": { "driver": "xrobot-websocket", "capabilities": ["streaming-asr", "tts", "cancel-generation", "pcm"], "configRef": "nvs://linx/websocket_url" }, "storage": { "driver": "fatfs-sqlite", "capabilities": ["persistent-sqlite", "atomic-calendar-write", "durable-calendar"] }, - "im": { "driver": "disabled", "capabilities": [] } + "im": { + "driver": "voicelife-gateway", + "capabilities": ["https", "secure-credentials"], + "configRef": "nvs://im" + } }, "sdkconfig": [ "CONFIG_LOG_DEFAULT_LEVEL_INFO=y", @@ -30,6 +34,10 @@ "CONFIG_NVS_ENCRYPTION=y", "CONFIG_NVS_SEC_KEY_PROTECT_USING_HMAC=y", "CONFIG_NVS_SEC_HMAC_EFUSE_KEY_ID=1", + "CONFIG_VOICELIFE_IM_GATEWAY=y", + "CONFIG_LWIP_DHCP_GET_NTP_SRV=y", + "CONFIG_LWIP_SNTP_MAX_SERVERS=2", + "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_CROSS_SIGNED_VERIFY=y", "CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y", "CONFIG_ESP_CONSOLE_SECONDARY_NONE=y", "CONFIG_LV_USE_CLIB_MALLOC=y", diff --git a/config/profiles/esp32s3-voicelife-pcb-serial-voice.json b/config/profiles/esp32s3-voicelife-pcb-serial-voice.json new file mode 100644 index 00000000..6fdb00b5 --- /dev/null +++ b/config/profiles/esp32s3-voicelife-pcb-serial-voice.json @@ -0,0 +1,62 @@ +{ + "schemaVersion": 1, + "id": "esp32s3-voicelife-pcb-serial-voice", + "target": "esp32s3", + "adapters": { + "audio": { + "driver": "esp32s3-pcm-port", + "capabilities": [ + "pcm-port", + "bounded-input", + "bounded-output", + "direct-i2s-simplex", + "transport-frame-assembly" + ], + "configRef": "env://VOICELIFE_AUDIO_PROFILE" + }, + "speech": { + "driver": "xrobot-websocket", + "capabilities": ["streaming-asr", "tts", "cancel-generation", "pcm"], + "configRef": "nvs://linx/websocket_url" + }, + "storage": { + "driver": "fatfs-sqlite", + "capabilities": ["persistent-sqlite"] + }, + "im": { + "driver": "voicelife-gateway", + "capabilities": ["https", "secure-credentials"], + "configRef": "nvs://im" + } + }, + "sdkconfig": [ + "CONFIG_LOG_DEFAULT_LEVEL_INFO=y", + "CONFIG_NVS_ENCRYPTION=y", + "CONFIG_NVS_SEC_KEY_PROTECT_USING_HMAC=y", + "CONFIG_NVS_SEC_HMAC_EFUSE_KEY_ID=0", + "CONFIG_VOICELIFE_IM_GATEWAY=y", + "CONFIG_LWIP_DHCP_GET_NTP_SRV=y", + "CONFIG_LWIP_SNTP_MAX_SERVERS=2", + "CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_CROSS_SIGNED_VERIFY=y", + "CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=y", + "CONFIG_ESP_CONSOLE_SECONDARY_NONE=y", + "CONFIG_ESP_WIFI_NVS_ENABLED=n", + "CONFIG_ESP_MAIN_TASK_STACK_SIZE=16384", + "CONFIG_FREERTOS_CHECK_STACKOVERFLOW=2", + "CONFIG_PARTITION_TABLE_CUSTOM_FILENAME=\"config/partitions/voicelife-pcb.csv\"", + "CONFIG_SPIRAM=y", + "CONFIG_SPIRAM_MODE_OCT=y", + "CONFIG_SPIRAM_SPEED_80M=y", + "CONFIG_SPIRAM_USE_MALLOC=y", + "CONFIG_FREERTOS_TASK_CREATE_ALLOW_EXT_MEM=y", + "CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=2048", + "CONFIG_SPIRAM_MALLOC_RESERVE_INTERNAL=98304", + "CONFIG_SPIRAM_MEMTEST=n", + "CONFIG_SR_MN_CN_MULTINET7_QUANT=y", + "CONFIG_FATFS_SECTOR_4096=y", + "CONFIG_WL_SECTOR_SIZE_4096=y", + "CONFIG_VOICELIFE_STORAGE_FATFS=y", + "CONFIG_VOICELIFE_STORAGE_SQLITE=y", + "CONFIG_VOICELIFE_SERIAL_VOICE_TEST=y" + ] +} diff --git a/docs/README.md b/docs/README.md index 9fd3def8..6fcce60c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,7 +9,7 @@ | 语音日程联调 | [语音日程联调测试矩阵](engineering/schedule-voice-integration-test-matrix.md)、[语音日程边界调研与本期取舍](engineering/schedule-voice-industry-boundaries.md) | | 架构维护者 | [架构与适配器设计](architecture/design-guidelines.md)、[语音子架构](architecture/voice-subarchitecture.md)、[SQLite 存储子架构](architecture/storage-subarchitecture.md)、[ADR](adr/) | | 组件与服务维护者 | [SparkBot 显示组件](components/sparkbot-display.md)、[IM Gateway](services/im-gateway.md) | -| 协作与交付 | [协同开发](engineering/collaboration.md)、[质量门禁](engineering/ci-quality-gates.md)、[契约版本](engineering/contract-versioning.md)、[提交规范](engineering/commit-convention.md) | +| 协作与交付 | [协同开发](engineering/collaboration.md)、[质量门禁](engineering/ci-quality-gates.md)、[E2E 分层门禁与真机 HIL](engineering/e2e-layered-gates.md)、[发布清单](engineering/release-checklist.md)、[契约版本](engineering/contract-versioning.md)、[提交规范](engineering/commit-convention.md) | | 文档归档判断 | [文档放置规则](engineering/document-placement.md)、[历史归档](archive/README.md) | Issue [#264](https://github.com/1024XEngineer/VoiceLife/issues/264) 将旧小智能力迁移计划和旧 ESP32-S3 验证记录移入[历史归档](archive/README.md)，并删除没有独立长期价值的 SparkBot 阶段执行草稿。语音原始研究材料继续归档在 Issue [#150](https://github.com/1024XEngineer/VoiceLife/issues/150)。 diff --git a/docs/engineering/e2e-journey-template.md b/docs/engineering/e2e-journey-template.md new file mode 100644 index 00000000..3bb2a526 --- /dev/null +++ b/docs/engineering/e2e-journey-template.md @@ -0,0 +1,42 @@ +# E2E Journey 模板 + +新增 journey 前复制本页到对应 Issue/PR，并在代码中复用 `scripts/run_e2e.py` 的统一生命周期。 + +## 标识 + +- Journey：`` +- 层级：`host` / `hil` +- Profile：`host` / `sparkbot` / `pcb` +- 负责人和依赖：``, `` + +## 生命周期 + +| 阶段 | 资源与动作 | 超时 | 清理 | 失败分类 | +| --- | --- | --- | --- | --- | +| prepare | 租约、临时目录、进程或设备检查 | `` | `` | configuration/lease/device/infrastructure | +| run | 真实请求或串口旅程 | `` | `` | external/product/device | +| assert | 有序状态和终态断言 | `` | `` | product | +| collect | 只采集 allowlist 字段 | `` | `` | infrastructure | +| cleanup | 撤销凭据、复位设备、释放租约 | `` | `` | cleanup | + +## Evidence + +- 公开字段：`` +- 明确禁止：原始串口、token、密码、SSID、个人数据、完整 URL 或命令输出 +- 故意失败测试：`` +- 脱敏校验：`python3 scripts/check_e2e_artifacts.py ` + +## 门禁入口 + +- 本地：`` +- PR / nightly / manual：`` +- artifact retention：`` +- required check 计划：先观察，达到稳定门槛后再提议 + +## 验收 + +- [ ] Host 和 HIL（如适用）使用同一 runner/evidence schema +- [ ] 硬超时、默认 retries=0、并发隔离已测试 +- [ ] 无设备、外部服务和产品断言能分别分类 +- [ ] 失败与清理 evidence 可上传且无敏感字段 +- [ ] 本地命令、workflow 命令和文档一致 diff --git a/docs/engineering/e2e-layered-gates.md b/docs/engineering/e2e-layered-gates.md new file mode 100644 index 00000000..de432544 --- /dev/null +++ b/docs/engineering/e2e-layered-gates.md @@ -0,0 +1,116 @@ +# E2E 分层门禁与真机 HIL + +本文是 Host E2E、ESP32-S3 HIL 和发布验收的共同入口。工作流只负责编排；旅程语义、断言和 evidence schema 由 `scripts/run_e2e.py` 与 adapter 维护。 + +## 测试层级 + +| 层级 | 运行位置 | 入口 | 作用 | 门禁策略 | +| --- | --- | --- | --- | --- | +| unit | 主机 | `./scripts/run_checks.sh` | 领域、契约和适配器边界 | PR required | +| integration | 主机 + Postgres | Gateway `pnpm run ci` | 服务边界、持久化和 SSE | PR required | +| Host E2E | GitHub-hosted runner | `python3 scripts/run_e2e.py --layer host ...` | 真实 Gateway 进程和最小强提醒旅程 | PR required，硬超时，retries=0 | +| HIL smoke | 受控 self-hosted + 一台设备 | `--layer hil --journey im-pairing` | 烧录、配网、就绪和配对主链路 | manual，非 required | +| HIL E2E | 受控 self-hosted + 设备池 | 同一 runner CLI，按 profile 矩阵 | SparkBot/PCB 真实串口旅程 | 稳定门槛达成后再升级 | +| 手工验收 | 真实平台和物理场景 | [release checklist](release-checklist.md) | 微信、声学、显示、掉电和真实 Linx/ASR/TTS | 发布阻断，不由 Host/HIL 替代 | + +## 本地命令 + +Host E2E 需要 Node 24、pnpm lockfile 依赖和本地 Postgres： + +```bash +pnpm --dir services/im-gateway install --frozen-lockfile +DATABASE_URL=postgres://voicelife:voicelife@127.0.0.1:5432/voicelife \ + python3 scripts/run_e2e.py --layer host \ + --journey im-gateway-strong-reminder --profile host \ + --artifact-dir artifacts/host-e2e --timeout 60 --retries 0 +python3 scripts/check_e2e_artifacts.py artifacts/host-e2e +python3 scripts/render_e2e_summary.py artifacts/host-e2e +``` + +HIL 只能在已批准的设备和私有 Gateway 上运行。设备描述文件只含 `name`、`port`、`profile`，放在设备池的受控目录，不提交仓库： + +```bash +python3 -m pip install pyserial esptool +python3 scripts/run_e2e.py --layer hil --journey im-pairing \ + --profile sparkbot --artifact-dir artifacts/hil-sparkbot \ + --timeout 900 --retries 0 \ + --device "$VOICELIFE_HIL_DEVICE_ROOT/sparkbot.json" \ + --lease-dir "$VOICELIFE_HIL_LEASE_ROOT" \ + --server "$VOICELIFE_HIL_SERVER" \ + --server-dir "$VOICELIFE_HIL_SERVER_DIR" \ + --gateway-origin "$VOICELIFE_HIL_GATEWAY_ORIGIN" \ + --user-id "$VOICELIFE_HIL_USER_ID" +python3 scripts/check_e2e_artifacts.py artifacts/hil-sparkbot +``` + +SparkBot 与 PCB 真实语音 HIL 分别使用 `esp32s3-esp-sparkbot-serial-voice` 和 +`esp32s3-voicelife-pcb-serial-voice` 测试 Profile。它需要受控环境中的 +`DASHSCOPE_API_KEY`、`dashscope` Python 包和 `ffmpeg`；API Key 不进入命令行参数或 evidence： + +```bash +DASHSCOPE_API_KEY="$DASHSCOPE_API_KEY" \ +python3 scripts/run_e2e.py --layer hil --journey voice \ + --profile sparkbot --artifact-dir artifacts/hil-voice \ + --timeout 900 --retries 0 \ + --device "$VOICELIFE_HIL_DEVICE_ROOT/sparkbot.json" \ + --lease-dir "$VOICELIFE_HIL_LEASE_ROOT" \ + --server "$VOICELIFE_HIL_SERVER" \ + --server-dir "$VOICELIFE_HIL_SERVER_DIR" \ + --gateway-origin "$VOICELIFE_HIL_GATEWAY_ORIGIN" \ + --user-id "$VOICELIFE_HIL_USER_ID" \ + --input-tts dashscope \ + --tts-model qwen-audio-3.0-tts-flash \ + --voice longanlingxi \ + --text '你好牛牛，请介绍一下你自己。' \ + --text '把刚才的回答再简短一点。' +python3 scripts/check_e2e_artifacts.py artifacts/hil-voice +``` + +`voice` journey 会先独立执行输入 TTS preflight，再完成 application-only flash、临时设备注册、USB 配网和 +readiness。SparkBot 与 PCB 都通过真实唤醒词 PCM 验证 `wake_detected -> listen.detect -> 确认 TTS -> +capture_started`，复位回 readiness 后才运行多轮旅程；两种 Profile 的 evidence 都必须记录 +`wake_stage=passed`。多轮脚本将输入 TTS 结果转为 16 kHz mono S16LE PCM 注入真实设备。 +SparkBot 验证 LVGL 文本轨迹与滚动；PCB 验证 SSD1306 在每轮中实际绘制。结果证明真实云端语音和 +设备数字播放链路，不替代麦克风声学、AEC 或全双工验收。 + +输入夹具支持 `dashscope` 和 `aliyun-nls`。DashScope 使用 `DASHSCOPE_API_KEY`；NLS 使用 +`ALIYUN_NLS_APPKEY`、短期 `ALIYUN_NLS_TOKEN` 和可选 `ALIYUN_NLS_URL`。两者只负责生成用户输入， +Linx 下行 TTS 仍通过 `tts_started/tts_first_audio/tts_stopped`、I2S 帧和显示证据验证，禁止把二者混称为 +同一个 TTS Provider。完整接线和次日跑板步骤见 [语音 HIL 实板操作手册](hil-voice-device-runbook.md)。 + +Gateway host、目录、origin 和 user id 通过环境变量或受控 secret 注入；token 不进入 shell 参数、日志或 evidence。HIL 默认 `retries=0`，只允许基础设施层在 workflow 外显式重试，并且必须在 summary 中可见。 + +## workflow 与设备矩阵 + +- `.github/workflows/ci.yml` 仅运行稳定 Host E2E，并上传 14 天的 JSON evidence。 +- PR Host job 另运行一次受控的 `lifecycle-example` 故意失败，验证失败 evidence 和 `product` 分类仍能通过脱敏校验。 +- `.github/workflows/hil-nightly.yml` 当前仅开放 `workflow_dispatch`，只接受 `self-hosted, voicelife-hil` 及 profile 标签；`fail-fast=false`，SparkBot 和 PCB 独立汇总、独立上传 artifact。`voice` 需要受控 `DASHSCOPE_API_KEY`。 +- 手工触发可选 `sparkbot`、`pcb` 或 `all`，并可在选择单一 Profile 后指定 device descriptor 名称，用于隔离故障设备；配置受控 Runner 后再执行。 +- public GitHub-hosted Runner 不接触硬件或长期平台凭据。受控 Runner 只保存原始串口日志；公开 artifact 只包含通过 `scripts/check_e2e_artifacts.py` 校验的脱敏 JSON。 +- artifact 保留 14 天，访问权限跟随仓库 Actions 权限；发现凭据或个人数据时立即删除公开 artifact，保留受控私有原始日志并按 [SECURITY.md](../../SECURITY.md) 上报。 + +## 失败分类与重试 + +每个 evidence 都带 `failure_category`、`failed_phase` 和稳定 `message_code`，job summary 按 profile 展示： + +| 分类 | 例子 | 处理 | +| --- | --- | --- | +| `configuration` | 缺 descriptor、缺 pyserial、Profile 参数错误 | 修 Runner 配置，不重试 | +| `lease` | 设备或串口租约冲突 | 等待或释放租约后重跑，不归因于产品 | +| `device` | 串口打开失败、分区/Profile 不匹配 | 隔离该设备，修复或替换后手动重跑 | +| `infrastructure` | 构建工具、SSH/进程或 artifact 写入失败 | 修 Runner 基础设施；不把产品判定为失败 | +| `external` | Gateway/微信/ASR/TTS 服务不可用 | 记录外部依赖和时间窗口，允许一次可见的基础设施重试 | +| `product` | readiness 或 pairing 断言失败 | 保留 evidence，按产品缺陷处理，不自动重跑掩盖 | +| `timeout` / `cleanup` | 硬超时或资源回收失败 | 标记为失败并阻止设备继续进入池中 | + +没有设备租约、串口异常、外部服务故障和产品断言必须在 summary 中保持不同类别。设备 workflow 默认 retries=0；只有 `infrastructure`/`external` 经批准后才能有限重试。 + +## Evidence 与新增 journey + +公共 evidence 只允许 run/correlation id、时间、Profile、阶段状态、断言、数值指标和 HIL 的固件/commit/fingerprint。禁止原始串口、token、密码、SSID、用户/设备 ID、URL 凭据和任意命令输出。新增旅程按 [journey template](e2e-journey-template.md) 补齐：准备/运行/断言/采集/清理、失败类别、清理动作、超时预算、脱敏字段、Host/HIL 入口和一条故意失败测试。 + +## 稳定门槛 + +HIL 连续 14 次手工执行、每个 Profile 至少 10 次通过，且没有未分类失败、泄露扫描失败或设备租约泄露，才能提议升级为 required check。任意设备连续两次 `device`/`infrastructure` 失败即隔离并暂停该 Profile；不得通过增加 retries 把红灯隐藏。 + +真实微信公众号、H5 推迟、SSE 重连、声学、显示、物理输入和掉电验收仍见 [release checklist](release-checklist.md) 与 [Issue #132](https://github.com/1024XEngineer/VoiceLife/issues/132)。DashScope PCM 注入只能作为真实语音数字链路证据，不能替代这些声学和体验验收。 diff --git a/docs/engineering/hil-voice-device-runbook.md b/docs/engineering/hil-voice-device-runbook.md new file mode 100644 index 00000000..76cfdc64 --- /dev/null +++ b/docs/engineering/hil-voice-device-runbook.md @@ -0,0 +1,69 @@ +# 语音 HIL 实板操作手册 + +本文用于 SparkBot/PCB 接入受控 HIL Runner 后的首次验证。没有真实串口、设备和 Linx 凭据时，只能完成 +Host/contract 验证，不得宣称 HIL 通过。 + +## 1. Runner 准备 + +- 安装 `pyserial`、`esptool`、`ffmpeg` 和 ESP-IDF 构建依赖。 +- DashScope 路线安装 `dashscope` 并设置 `DASHSCOPE_API_KEY`。 +- 阿里云 NLS 路线安装官方 SDK（`pip install alibabacloud-nls-python-sdk`），设置 + `ALIYUN_NLS_APPKEY`、短期 + `ALIYUN_NLS_TOKEN`，按地域需要设置 `ALIYUN_NLS_URL`。 +- 准备项目外 descriptor：`{"schema_version":1,"name":"bench-a","port":"/dev/cu...","profile":"sparkbot"}`。 +- 设置 Gateway、设备池和租约目录相关的 `VOICELIFE_HIL_*` 环境变量；任何密钥、token、SSID 和原始串口 + 日志不得进入 Git 或公开 artifact。 + +## 2. 首次执行 + +先确认串口名称且没有其它串口监视器占用，然后执行： + +```bash +python3 scripts/run_e2e.py --layer hil --journey voice \ + --profile sparkbot \ + --artifact-dir artifacts/hil-voice-sparkbot \ + --timeout 900 --retries 0 \ + --device "$VOICELIFE_HIL_DEVICE_ROOT/sparkbot.json" \ + --lease-dir "$VOICELIFE_HIL_LEASE_ROOT" \ + --server "$VOICELIFE_HIL_SERVER" \ + --server-dir "$VOICELIFE_HIL_SERVER_DIR" \ + --gateway-origin "$VOICELIFE_HIL_GATEWAY_ORIGIN" \ + --user-id "$VOICELIFE_HIL_USER_ID" \ + --input-tts dashscope \ + --tts-model qwen-audio-3.0-tts-flash \ + --voice longanlingxi \ + --text '你好牛牛，请介绍一下你自己。' \ + --text '把刚才的回答再简短一点。' \ + --text '请用一句话总结我们刚才的对话。' \ + --expect-terminal +``` + +改用 NLS 时只替换 provider、模型标签和 NLS 音色；模型与音色必须是在该 NLS 项目中真实可用的配置： + +```bash +--input-tts aliyun-nls --tts-model aliyun-nls-v1 --voice xiaoyun +``` + +## 3. 必须通过的证据 + +- TTS preflight：有音频字节、有 16 kHz PCM 帧，并记录请求数、首包和总时延。 +- SparkBot 与 PCB 唤醒：严格出现 `wake_detected`、`local_wake_ack_requested`、`tts_started`、 + `tts_first_audio`、`tts_stopped`、`capture_started`；确认音播放完之前不得开物理采集。 +- 每轮对话：ASR 严格匹配，出现下行 TTS start/first-audio/stop，显示状态完整。 +- 音频与队列：`in_drop=0`、`out_reject=0`、`short_write=0`、I2S error 为零、交互队列无 drop。 +- 终结回合：8 秒 wake guard 内无重新唤醒。 +- evidence：`scripts/check_e2e_artifacts.py` 校验通过；公开 JSON 不含原始话术、设备 ID 或凭据。 + +## 4. 失败处理 + +- `configuration`：补依赖、凭据或 descriptor，不重试掩盖。 +- `external`：记录 TTS/Linx 服务时间窗口；只允许一次显式基础设施重试。 +- `device`：关闭串口占用，核对板型和分区；连续两次失败先隔离设备。 +- `product`：保留私有串口日志和脱敏 evidence，不改成通过。 +- `Connection reset by peer`：记录发生在唤醒确认、长播报或普通回合哪个阶段；重连后再跑一轮，当前不得宣称 + 长播报重连闭环通过。 + +## 5. 跑板后回填 + +记录固件 commit、Gateway commit、板卡、串口、输入 TTS provider/model/voice、三轮完成数、ASR 精确匹配数、 +音频错误计数、唤醒结果、终结 guard、Linx RST/重连情况和脱敏 artifact 链接。SparkBot 与 PCB 分别保存结果。 diff --git a/docs/engineering/release-checklist.md b/docs/engineering/release-checklist.md new file mode 100644 index 00000000..0d673ea5 --- /dev/null +++ b/docs/engineering/release-checklist.md @@ -0,0 +1,30 @@ +# Release Checklist + +Host E2E 和 HIL 结果只能证明软件旅程的有限边界，不能替代真实平台和体验验收。发布 PR 必须记录 firmware commit、Gateway commit、Profile、测试账号类型和脱敏 evidence 位置。 + +## 自动门禁 + +- [ ] PR 的 Host unit/integration/Host E2E 通过，失败 evidence 已上传并完成敏感字段扫描。 +- [ ] 最近一次手工 HIL 对 SparkBot、PCB 分别有结果；HIL 仍为非 required 时记录原因。 +- [ ] workflow、设备标签、artifact retention 和 `retries=0` 配置未漂移。 +- [ ] 失败按 product / infrastructure / device / external / configuration 分类，没有用重试掩盖。 + +## 真实平台验收（#132） + +- [ ] 真实微信公众号绑定、强提醒通知和动作回执通过。 +- [ ] H5 推迟/确认链接在真实 HTTPS origin 下通过，过期、重复点击和错误 scope 有记录。 +- [ ] SSE 断线重连、Gateway 重启和 PostgreSQL 恢复通过。 +- [ ] 真实 Linx/ASR/TTS 凭据在受控环境使用；凭据未进入 PR、artifact 或设备镜像。 +- [ ] SparkBot、PCB voice HIL 通过：DashScope TTS -> PCM 注入 -> ASR -> Linx/WSS -> 下行 TTS/I2S/显示；记录 model、voice、Profile、固件 commit 和脱敏 evidence。 +- [ ] 声学：唤醒、AEC、全双工抢话和误唤醒在真实扬声器/麦克风下观察并记录。 +- [ ] 显示：SparkBot 状态、表情、字幕和物理输入在实板观察通过。 +- [ ] 掉电、重启、配网和旧凭据撤销/恢复通过。 + +## 证据与回退 + +- [ ] PR 只附最小连续非敏感摘录；原始串口和音频保留在受控私有目录。 +- [ ] 记录测试时间窗口、板型、固件/Gateway commit、Profile、账号类型和 evidence URL。 +- [ ] 已知 flaky 或外部故障有分类和后续负责人，不标记为产品通过。 +- [ ] 发布后发现回归时可暂停 HIL 手工 workflow 或将 Host E2E 降为非 required，但不得删除 journey、失败 evidence 或本清单。 + +关联：[#132](https://github.com/1024XEngineer/VoiceLife/issues/132)、[#288](https://github.com/1024XEngineer/VoiceLife/issues/288)。 diff --git a/main/Kconfig.projbuild b/main/Kconfig.projbuild index 4ee42003..713ebee4 100644 --- a/main/Kconfig.projbuild +++ b/main/Kconfig.projbuild @@ -50,15 +50,16 @@ config VOICELIFE_STATE_FLOW_TEST is a test-only build option and must remain disabled in production. config VOICELIFE_SERIAL_VOICE_TEST - bool "Enable SparkBot serial PCM voice regression harness" + bool "Enable serial PCM voice regression harness" default n - depends on VOICELIFE_BOARD_ESP_SPARKBOT && ESP_CONSOLE_USB_SERIAL_JTAG && !VOICELIFE_STATE_FLOW_TEST + depends on ESP_CONSOLE_USB_SERIAL_JTAG && !VOICELIFE_STATE_FLOW_TEST help Start a test-only USB Serial/JTAG PCM input reader. It accepts a fixed, bounded 16 kHz S16LE mono protocol and routes frames through the normal - audio queue, VoiceSession, Linx transport and ES8311 output. The test - build emits unredacted voice text and detailed hardware telemetry for - board diagnosis. It must remain disabled in production profiles. + audio queue, VoiceSession, Linx transport and board-specific output. + The test build emits unredacted voice text and detailed hardware + telemetry for board diagnosis. It must remain disabled in production + profiles. if VOICELIFE_AUDIO_PROBE diff --git a/scripts/check_e2e_artifacts.py b/scripts/check_e2e_artifacts.py new file mode 100644 index 00000000..ce76f66e --- /dev/null +++ b/scripts/check_e2e_artifacts.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Validate public E2E artifacts without printing their contents.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from e2e_evidence import EvidenceValidationError, scan_sensitive, validate_evidence + + +def validate_directory(root: Path) -> tuple[int, int]: + """Return (file_count, evidence_count) after validating an artifact directory.""" + if not root.is_dir(): + raise ValueError("artifact directory is missing") + files = sorted(path for path in root.rglob("*") if path.is_file()) + if not files: + raise ValueError("artifact directory is empty") + evidence_count = 0 + for path in files: + if path.suffix != ".json": + raise ValueError("public artifacts may only contain JSON") + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise ValueError("artifact JSON is unreadable") from error + if scan_sensitive(document): + raise ValueError("artifact contains sensitive data") + if path.name.startswith("evidence-"): + if not isinstance(document, dict): + raise ValueError("E2E evidence must be a JSON object") + try: + validate_evidence(document) + except EvidenceValidationError as error: + raise ValueError("E2E evidence does not match the public schema") from error + evidence_count += 1 + if evidence_count == 0: + raise ValueError("artifact directory has no E2E evidence") + return len(files), evidence_count + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifact_dir", type=Path) + args = parser.parse_args(argv) + try: + file_count, evidence_count = validate_directory(args.artifact_dir) + except ValueError as error: + print(str(error), file=sys.stderr) + return 1 + print(f"validated {evidence_count} E2E evidence file(s) in {file_count} public artifact file(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/e2e_evidence.py b/scripts/e2e_evidence.py index 1a17f3e0..d804e970 100644 --- a/scripts/e2e_evidence.py +++ b/scripts/e2e_evidence.py @@ -16,6 +16,7 @@ HEX_COMMIT = re.compile(r"^[0-9a-f]{40}$") HEX_FINGERPRINT = re.compile(r"^[0-9a-f]{16}$") SAFE_NAME = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") +SAFE_MODEL = re.compile(r"^[a-z0-9][a-z0-9_.-]{0,63}$") UTC_TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3,6})?Z$") JWT_PATTERN = re.compile(r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}\b") SENSITIVE_KEY_PATTERN = re.compile( @@ -58,9 +59,54 @@ HIL_KEYS = frozenset( {"firmware_sha256", "gateway_commit", "device_fingerprint", "readiness_markers", "pairing_markers"} ) -METRIC_KEYS = frozenset({"resource_count", "bound_port_count", "namespace_count"}) +VOICE_HIL_KEYS = frozenset( + { + "firmware_sha256", + "gateway_commit", + "device_fingerprint", + "readiness_markers", + "input_tts_provider", + "input_tts_model", + "input_tts_voice", + "wake_stage", + } +) +METRIC_KEYS = frozenset( + { + "resource_count", + "bound_port_count", + "namespace_count", + "requested_turns", + "completed_turns", + "asr_exact_matches", + "input_tts_requests", + "input_tts_audio_bytes", + "input_tts_first_packet_ms_max", + "input_tts_total_ms_max", + "wake_completed", + "test_in_frames", + "out_frames", + "in_drop", + "out_reject", + "short_write", + "in_i2s_err", + "out_i2s_err", + "serial_pcm_rejections", + "display_content_snapshots", + } +) FAILURE_CATEGORIES = frozenset( - {"configuration", "infrastructure", "product", "device", "external", "timeout", "interrupted", "cleanup"} + { + "configuration", + "infrastructure", + "product", + "device", + "lease", + "external", + "timeout", + "interrupted", + "cleanup", + } ) PHASE_ORDER = ("prepare", "run", "assert", "collect", "cleanup") PHASES = frozenset(PHASE_ORDER) @@ -133,7 +179,7 @@ def _validate_assertion(assertion: Any) -> None: def _validate_metrics(metrics: Any) -> None: _reject(not isinstance(metrics, dict) or not set(metrics).issubset(METRIC_KEYS)) for value in metrics.values(): - _reject(type(value) is not int or not 0 <= value <= 1_000_000) + _reject(type(value) is not int or not 0 <= value <= 100_000_000) def _validate_hil(hil: Any) -> None: @@ -148,13 +194,27 @@ def _validate_hil(hil: Any) -> None: _reject(value["pairing_markers"] != ["scope_matched", "code_valid", "pending", "expired"]) +def _validate_voice_hil(hil: Any) -> None: + value = _exact_keys(hil, VOICE_HIL_KEYS) + _reject(not isinstance(value["firmware_sha256"], str) or HEX_SHA256.fullmatch(value["firmware_sha256"]) is None) + _reject(not isinstance(value["gateway_commit"], str) or HEX_COMMIT.fullmatch(value["gateway_commit"]) is None) + _reject( + not isinstance(value["device_fingerprint"], str) + or HEX_FINGERPRINT.fullmatch(value["device_fingerprint"]) is None + ) + _reject(value["readiness_markers"] != ["provisioned", "wifi_ready", "sntp_synced", "ready"]) + _reject(value["input_tts_provider"] not in {"dashscope", "aliyun-nls"}) + _reject(SAFE_MODEL.fullmatch(value["input_tts_model"]) is None or not _safe_name(value["input_tts_voice"])) + _reject(value["wake_stage"] != "passed") + + def validate_evidence(document: dict[str, object]) -> None: """Validate the complete evidence allowlist and cross-field relations.""" value = _exact_keys(document, TOP_LEVEL_KEYS) _reject(value["schema_version"] != 1) _reject(not isinstance(value["run_id"], str) or HEX_ID.fullmatch(value["run_id"]) is None) _reject(not isinstance(value["correlation_id"], str) or HEX_ID.fullmatch(value["correlation_id"]) is None) - _reject(value["scope"] not in {"runner_contract_only", "hil_im_pairing"}) + _reject(value["scope"] not in {"runner_contract_only", "hil_im_pairing", "hil_voice"}) _reject(value["layer"] not in {"host", "hil"}) _reject(not _safe_name(value["journey"]) or not _safe_name(value["profile"])) _reject(not isinstance(value["started_at"], str) or UTC_TIMESTAMP.fullmatch(value["started_at"]) is None) @@ -165,7 +225,7 @@ def validate_evidence(document: dict[str, object]) -> None: _reject(type(value["hardware_verified"]) is not bool) if value["scope"] == "runner_contract_only": _reject(value["hardware_verified"] or value["hil"] is not None) - else: + elif value["scope"] == "hil_im_pairing": _reject( value["layer"] != "hil" or value["journey"] != "im-pairing" @@ -173,6 +233,15 @@ def validate_evidence(document: dict[str, object]) -> None: or value["hardware_verified"] is not True ) _validate_hil(value["hil"]) + else: + _reject( + value["layer"] != "hil" + or value["journey"] != "voice" + or value["profile"] not in {"sparkbot", "pcb"} + or value["status"] != "passed" + or value["hardware_verified"] is not True + ) + _validate_voice_hil(value["hil"]) failure_category = value["failure_category"] failed_phase = value["failed_phase"] @@ -216,6 +285,10 @@ def validate_evidence(document: dict[str, object]) -> None: _validate_assertion(assertion) _validate_metrics(value["metrics"]) + if value["scope"] == "hil_voice": + voice_hil = value["hil"] + voice_metrics = value["metrics"] + _reject(voice_hil["wake_stage"] != "passed" or voice_metrics.get("wake_completed") != 1) cleanup = _exact_keys(value["cleanup"], CLEANUP_KEYS) _reject(cleanup["status"] not in {"passed", "failed"}) error_codes = cleanup["error_codes"] diff --git a/scripts/e2e_example_adapters.py b/scripts/e2e_example_adapters.py index 68947560..4cc07af6 100644 --- a/scripts/e2e_example_adapters.py +++ b/scripts/e2e_example_adapters.py @@ -43,6 +43,8 @@ def run(self, context: RunContext) -> dict[str, bool]: def assert_result(self, context: RunContext, result: object) -> list[AssertionResult]: values = result if isinstance(result, dict) else {} + if os.environ.get("VOICELIFE_E2E_CONTRACT_FAILURE") == "1": + return [AssertionResult(name="lifecycle_complete", passed=False, code="contract_failure")] passed = all(values.get(name) is True for name in values) return [AssertionResult(name="lifecycle_complete", passed=passed, code="ok" if passed else "incomplete")] @@ -159,8 +161,9 @@ def __init__(self, artifact_directory: Path) -> None: self._process: subprocess.Popen[str] | None = None def prepare(self, context: RunContext) -> None: - self.artifact_directory.mkdir(parents=True, exist_ok=True) - detail_path = self.artifact_directory / f"recovery-{context.run_id}.json" + # Detailed recovery snapshots may contain internal database fields; keep them + # in the runner-owned temporary directory instead of the public artifact tree. + detail_path = context.temporary_directory / "recovery-details" / f"recovery-{context.run_id}.json" environment = { **os.environ, "E2E_RUN_ID": context.run_id, diff --git a/scripts/e2e_hil_adapters.py b/scripts/e2e_hil_adapters.py index 535fe9b1..3e4cf52e 100644 --- a/scripts/e2e_hil_adapters.py +++ b/scripts/e2e_hil_adapters.py @@ -5,9 +5,12 @@ import hashlib import json +import os +import shutil import subprocess +import sys import time -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from pathlib import Path from typing import Protocol @@ -26,10 +29,22 @@ load_device_descriptor, validate_device_layout, ) -from e2e_runner import AssertionResult, FailureCategory, RunContext, RunnerDeadlineExceeded, RunnerFailure +from e2e_runner import ( + AssertionResult, + FailureCategory, + RunContext, + RunnerDeadlineExceeded, + RunnerFailure, +) from start_im_pairing import PairingLifecycle, PairingLifecycleError +from voice_tts_fixture import TtsFixtureError, validate_provider_environment ROOT = Path(__file__).resolve().parents[1] +SQLITE_COMPONENT_FILES = ( + ROOT / "third_party" / "sqlite3" / "sqlite3.c", + ROOT / "third_party" / "sqlite3" / "sqlite3.h", + ROOT / "third_party" / "sqlite3" / "CMakeLists.txt", +) @dataclass @@ -76,12 +91,31 @@ def _runner_failure(error: Exception) -> RunnerFailure: if isinstance(error, HilConfigurationError): return RunnerFailure(FailureCategory.CONFIGURATION, "hil_configuration_invalid") if isinstance(error, HilLeaseUnavailable): - return RunnerFailure(FailureCategory.DEVICE, "device_lease_unavailable") + return RunnerFailure(FailureCategory.LEASE, "device_lease_unavailable") if isinstance(error, HilProfileMismatch): return RunnerFailure(FailureCategory.DEVICE, "device_profile_mismatch") raise error +def ensure_sqlite_component() -> None: + """Prepare the checked-in SQLite component before an ESP-IDF build.""" + if all(path.is_file() for path in SQLITE_COMPONENT_FILES): + return + try: + subprocess.run( + [sys.executable, str(ROOT / "scripts" / "prepare_sqlite.py")], + cwd=ROOT, + capture_output=True, + text=True, + timeout=180, + check=True, + ) + except subprocess.TimeoutExpired as error: + raise RunnerDeadlineExceeded from error + except (OSError, subprocess.CalledProcessError) as error: + raise RunnerFailure(FailureCategory.INFRASTRUCTURE, "sqlite_component_prepare_failed") from error + + class HilPairingAdapter: """Execute the real HIL flow through injected hardware boundary adapters.""" @@ -190,6 +224,383 @@ def collect(self, context: RunContext, result: object, assertions: list[Assertio } +class HilVoiceAdapter: + """Run provider-neutral input TTS, wake, and multi-turn device voice HIL.""" + + VOICE_FIRMWARE_PROFILES = { + "sparkbot": "esp32s3-esp-sparkbot-serial-voice", + "pcb": "esp32s3-voicelife-pcb-serial-voice", + } + CONVERSATION_ACCEPTANCE_KEYS = frozenset( + { + "input_tts_preflight", + "audio_stats_present", + "audio_flow_observed", + "zero_loss", + "zero_serial_pcm_rejections", + "asr_text_matches_input", + "input_not_endpoint_truncated", + "interaction_queue_clean", + "no_interaction_rejection", + "no_provider_error", + "state_flow_complete", + "display_flow_complete", + "display_text_trace_complete", + "display_scroll_observed", + "terminal_guard_clean", + } + ) + WAKE_ACCEPTANCE_KEYS = frozenset( + { + "input_tts_preflight", + "wake_detected", + "wake_ack_requested", + "wake_ack_tts_started", + "wake_ack_tts_first_audio", + "wake_ack_tts_stopped", + "capture_started_after_ack", + } + ) + PREFLIGHT_ACCEPTANCE_KEYS = frozenset({"audio_present", "pcm_frames_present"}) + + def __init__( + self, + descriptor_path: Path, + lease_root: Path, + *, + hardware: HilHardware, + input_tts: str, + tts_model: str, + voice: str, + texts: list[str] | None, + expect_terminal: bool, + response_timeout: float, + ) -> None: + self._descriptor_path = descriptor_path + self._lease_root = lease_root + self._hardware = hardware + self._input_tts = input_tts + self._tts_model = tts_model + self._voice = voice + self._texts = texts or [] + self._expect_terminal = expect_terminal + self._response_timeout = response_timeout + self._descriptor: DeviceDescriptor | None = None + self._lease: DeviceLease | None = None + self._partitions: list[Partition] = [] + self._image: ApplicationImage | None = None + self._identity: TemporaryIdentity | None = None + self._pending_device_id = "" + self._readiness: list[dict[str, object]] = [] + self._device_fingerprint = "" + self._result: dict[str, object] = {} + self.lease_held = False + + def prepare(self, context: RunContext) -> None: + if shutil.which("ffmpeg") is None: + raise RunnerFailure(FailureCategory.CONFIGURATION, "ffmpeg_unavailable") + try: + validate_provider_environment(self._input_tts) + except TtsFixtureError as error: + raise RunnerFailure(FailureCategory.CONFIGURATION, error.code) from error + try: + descriptor = load_device_descriptor(self._descriptor_path, context.config.profile) + if descriptor.profile not in self.VOICE_FIRMWARE_PROFILES: + raise HilProfileMismatch("voice journey requires a supported profile") + lease = DeviceLease(descriptor, self._lease_root) + lease.acquire() + self.lease_held = True + self._descriptor = descriptor + self._lease = lease + context.cleanup.push("hil-voice-device-lease", self._release_lease, timeout_required=False) + self._partitions = self._hardware.inspect(self._descriptor, context.temporary_directory) + validate_device_layout(self._descriptor, self._partitions) + except (HilConfigurationError, HilLeaseUnavailable, HilProfileMismatch) as error: + raise _runner_failure(error) from error + + def _release_lease(self) -> None: + if self._lease is not None: + self._lease.release() + self.lease_held = False + + def _recover(self) -> None: + if self._descriptor is not None: + self._hardware.recover(self._descriptor) + + def _revoke(self) -> None: + if self._identity is not None: + self._hardware.revoke(self._identity) + elif self._pending_device_id: + self._hardware.revoke_device_id(self._pending_device_id) + + @staticmethod + def _classify_voice_exit(returncode: int, stderr: str) -> RunnerFailure: + if returncode == 1: + return RunnerFailure(FailureCategory.PRODUCT, "voice_acceptance_failed") + if "cannot open serial port" in stderr: + return RunnerFailure(FailureCategory.DEVICE, "voice_serial_unavailable") + if any(marker in stderr for marker in ("_missing", "_unavailable", "pyserial is required")): + return RunnerFailure(FailureCategory.CONFIGURATION, "voice_dependency_missing") + if "input_preparation_failed" in stderr: + return RunnerFailure(FailureCategory.EXTERNAL, "voice_tts_failed") + return RunnerFailure(FailureCategory.EXTERNAL, "voice_harness_failed") + + @staticmethod + def _fixture_valid(value: object) -> bool: + if not isinstance(value, dict) or set(value) != { + "provider", + "model", + "voice", + "requests", + "audio_bytes", + "first_packet_ms_max", + "total_ms_max", + }: + return False + return ( + isinstance(value.get("provider"), str) + and isinstance(value.get("model"), str) + and isinstance(value.get("voice"), str) + and all( + type(value.get(key)) is int and value.get(key, -1) >= 0 + for key in ( + "requests", + "audio_bytes", + "first_packet_ms_max", + "total_ms_max", + ) + ) + and value.get("requests", 0) > 0 + and value.get("audio_bytes", 0) > 0 + ) + + @classmethod + def _acceptance_valid(cls, value: object, expected: frozenset[str]) -> bool: + return isinstance(value, dict) and set(value) == expected and all(item is True for item in value.values()) + + def _run_harness( + self, context: RunContext, script_name: str, command: list[str], result_name: str + ) -> dict[str, object]: + result_path = context.temporary_directory / result_name + command.extend(("--result-json", str(result_path))) + try: + completed = subprocess.run( + command, + cwd=ROOT, + env=os.environ.copy(), + capture_output=True, + text=True, + timeout=max(0.1, context.remaining()), + check=False, + ) + except subprocess.TimeoutExpired as error: + raise RunnerDeadlineExceeded from error + except OSError as error: + raise RunnerFailure(FailureCategory.INFRASTRUCTURE, f"{script_name}_harness_unavailable") from error + if completed.returncode != 0: + raise self._classify_voice_exit(completed.returncode, completed.stderr) + try: + value = json.loads(result_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError, TypeError) as error: + raise RunnerFailure(FailureCategory.PRODUCT, f"{script_name}_result_invalid") from error + if not isinstance(value, dict): + raise RunnerFailure(FailureCategory.PRODUCT, f"{script_name}_result_invalid") + return value + + def _run_wake_script(self, context: RunContext) -> dict[str, object]: + if self._descriptor is None: + raise RunnerFailure(FailureCategory.INFRASTRUCTURE, "voice_device_not_prepared") + command = [ + sys.executable, + str(ROOT / "scripts" / "voice_linx_wake_injection_test.py"), + "--port", + str(self._descriptor.port), + "--input-tts", + self._input_tts, + "--tts-model", + self._tts_model, + "--voice", + self._voice, + "--timeout", + str(self._response_timeout), + "--serial-log", + str(context.temporary_directory / "voice-wake-serial.log"), + ] + return self._run_harness(context, "voice_wake", command, "voice-wake-result.json") + + def _run_tts_preflight(self, context: RunContext) -> dict[str, object]: + command = [ + sys.executable, + str(ROOT / "scripts" / "voice_tts_preflight.py"), + "--input-tts", + self._input_tts, + "--tts-model", + self._tts_model, + "--voice", + self._voice, + ] + return self._run_harness(context, "voice_tts_preflight", command, "voice-tts-preflight-result.json") + + def _run_voice_script(self, context: RunContext) -> dict[str, object]: + if self._descriptor is None: + raise RunnerFailure(FailureCategory.INFRASTRUCTURE, "voice_device_not_prepared") + script = ROOT / "scripts" / "voice_linx_serial_multiturn_test.py" + serial_log = context.temporary_directory / "voice-serial.log" + command = [ + sys.executable, + str(script), + "--port", + str(self._descriptor.port), + "--input-tts", + self._input_tts, + "--tts-model", + self._tts_model, + "--voice", + self._voice, + "--display-profile", + self._descriptor.profile, + "--response-timeout", + str(self._response_timeout), + "--serial-log", + str(serial_log), + ] + if self._expect_terminal: + command.append("--expect-terminal") + for text in self._texts: + command.extend(("--text", text)) + return self._run_harness(context, "voice", command, "voice-result.json") + + def run(self, context: RunContext) -> dict[str, object]: + if self._descriptor is None: + raise RunnerFailure(FailureCategory.INFRASTRUCTURE, "hil_device_not_prepared") + try: + preflight_result = self._run_tts_preflight(context) + voice_descriptor = replace( + self._descriptor, + firmware_profile=self.VOICE_FIRMWARE_PROFILES[self._descriptor.profile], + ) + build_directory = self._hardware.build(voice_descriptor) + self._image = self._hardware.image(build_directory, self._descriptor, self._partitions) + self._hardware.flash(voice_descriptor, self._image, context.remaining()) + self._pending_device_id = f"e2e-{context.run_id}" + context.cleanup.push("voice-gateway-device-revoke", self._revoke) + self._identity = self._hardware.register(context.run_id) + self._device_fingerprint = device_fingerprint(self._identity.device_id, context.run_id) + context.cleanup.push("voice-hil-device-recovery", self._recover) + self._hardware.provision(self._descriptor, self._identity, context.remaining()) + self._readiness = self._hardware.reboot_and_readiness(self._descriptor, context.remaining()) + ready, _ = hil_readiness_status(self._readiness) + if not ready: + raise RunnerFailure(FailureCategory.PRODUCT, "voice_readiness_incomplete") + # Both official voice profiles expose the same local MultiNet wake + # detector and VLVT serial wake-input protocol. Keep the wake gate + # mandatory for every board so PCB cannot pass on multi-turn audio + # evidence while its local wake path remains unverified. + wake_result = self._run_wake_script(context) + self._readiness = self._hardware.reboot_and_readiness(self._descriptor, context.remaining()) + ready, _ = hil_readiness_status(self._readiness) + if not ready: + raise RunnerFailure(FailureCategory.PRODUCT, "voice_post_wake_readiness_incomplete") + conversation_result = self._run_voice_script(context) + self._result = { + "schema_version": 1, + "preflight": preflight_result, + "wake": wake_result, + "conversation": conversation_result, + } + return self._result + except (HilConfigurationError, HilLeaseUnavailable, HilProfileMismatch) as error: + raise _runner_failure(error) from error + + def assert_result(self, context: RunContext, result: object) -> list[AssertionResult]: + values = result if isinstance(result, dict) and result.get("schema_version") == 1 else {} + conversation = values.get("conversation") if isinstance(values.get("conversation"), dict) else {} + preflight = values.get("preflight") if isinstance(values.get("preflight"), dict) else {} + acceptance = conversation.get("acceptance") + acceptance_values = acceptance if isinstance(acceptance, dict) else {} + wake = values.get("wake") + wake_clean = ( + isinstance(wake, dict) + and wake.get("schema_version") == 1 + and self._fixture_valid(wake.get("fixture")) + and self._acceptance_valid(wake.get("acceptance"), self.WAKE_ACCEPTANCE_KEYS) + ) + checks = { + "voice_input_tts_preflight_clean": preflight.get("schema_version") == 1 + and self._fixture_valid(preflight.get("fixture")) + and self._acceptance_valid(preflight.get("acceptance"), self.PREFLIGHT_ACCEPTANCE_KEYS), + "voice_input_fixture_clean": self._fixture_valid(conversation.get("fixture")), + "voice_wake_sequence_clean": wake_clean, + "voice_turns_complete": conversation.get("completed_turns") == conversation.get("requested_turns") + and isinstance(conversation.get("requested_turns"), int) + and conversation.get("requested_turns", 0) > 0, + "voice_state_flow_clean": acceptance_values.get("state_flow_complete") is True, + "voice_display_flow_clean": acceptance_values.get("display_flow_complete") is True + and acceptance_values.get("display_text_trace_complete") is True, + "voice_wake_guard_clean": acceptance_values.get("terminal_guard_clean") is True, + "voice_acceptance_clean": self._acceptance_valid(acceptance, self.CONVERSATION_ACCEPTANCE_KEYS), + } + return [ + AssertionResult(name=name, passed=passed, code="ok" if passed else "mismatch") + for name, passed in checks.items() + ] + + def collect(self, context: RunContext, result: object, assertions: list[AssertionResult]) -> dict[str, object]: + if self._descriptor is None or self._image is None or self._identity is None: + raise RunnerFailure(FailureCategory.INFRASTRUCTURE, "voice_evidence_incomplete") + values = result if isinstance(result, dict) else {} + conversation = values.get("conversation") if isinstance(values.get("conversation"), dict) else {} + preflight = values.get("preflight") if isinstance(values.get("preflight"), dict) else {} + preflight_fixture = preflight.get("fixture") if isinstance(preflight.get("fixture"), dict) else {} + fixture = conversation.get("fixture") if isinstance(conversation.get("fixture"), dict) else {} + audio = conversation.get("audio_stats") if isinstance(conversation.get("audio_stats"), dict) else {} + display = conversation.get("display") if isinstance(conversation.get("display"), dict) else {} + serial_rejections = conversation.get("serial_pcm_rejections") + wake = values.get("wake") if isinstance(values.get("wake"), dict) else {} + wake_fixture = wake.get("fixture") if isinstance(wake.get("fixture"), dict) else {} + fixture_sources = (preflight_fixture, wake_fixture, fixture) + + def fixture_metric(item: dict[str, object], key: str) -> int: + value = item.get(key, 0) + return value if type(value) is int and value >= 0 else 0 + + return { + "scope": "hil_voice", + "hardware_verified": all(assertion.passed for assertion in assertions), + "firmware_sha256": self._image.sha256, + "gateway_commit": self._identity.gateway_commit, + "device_fingerprint": self._device_fingerprint, + "readiness_markers": hil_readiness_markers(self._readiness), + "input_tts_provider": self._input_tts, + "input_tts_model": self._tts_model, + "input_tts_voice": self._voice, + "wake_stage": "passed", + "metrics": { + "input_tts_requests": sum(fixture_metric(item, "requests") for item in fixture_sources), + "input_tts_audio_bytes": sum(fixture_metric(item, "audio_bytes") for item in fixture_sources), + "input_tts_first_packet_ms_max": max( + (fixture_metric(item, "first_packet_ms_max") for item in fixture_sources), default=0 + ), + "input_tts_total_ms_max": max( + (fixture_metric(item, "total_ms_max") for item in fixture_sources), default=0 + ), + "wake_completed": 1 if wake else 0, + "requested_turns": conversation.get("requested_turns", 0), + "completed_turns": conversation.get("completed_turns", 0), + "asr_exact_matches": conversation.get("asr_exact_matches", 0), + "test_in_frames": audio.get("test_in_frames", 0), + "out_frames": audio.get("out_frames", 0), + "in_drop": audio.get("in_drop", 0), + "out_reject": audio.get("out_reject", 0), + "short_write": audio.get("short_write", 0), + "in_i2s_err": audio.get("in_i2s_err", 0), + "out_i2s_err": audio.get("out_i2s_err", 0), + "serial_pcm_rejections": len(serial_rejections) if isinstance(serial_rejections, list) else 0, + "display_content_snapshots": display.get("content_snapshots", 0), + }, + } + + class RealHilHardware: """Production adapters that reuse existing build, flash, provisioning and pairing scripts.""" @@ -247,6 +658,7 @@ def build(self, descriptor: DeviceDescriptor) -> Path: from firmware import build try: + ensure_sqlite_component() return build(descriptor.firmware_profile) except RunnerDeadlineExceeded: raise @@ -267,7 +679,12 @@ def flash(self, descriptor: DeviceDescriptor, image: ApplicationImage, timeout_s self._run(list(operation.argv), timeout_s) def _remote(self, script: str, timeout_s: float = 180.0) -> str: - return self._run(["ssh", "-o", "BatchMode=yes", self._server, "bash", "-s"], timeout_s, input_text=script) + try: + return self._run(["ssh", "-o", "BatchMode=yes", self._server, "bash", "-s"], timeout_s, input_text=script) + except RunnerFailure as error: + if error.message_code == "hil_command_failed": + raise RunnerFailure(FailureCategory.EXTERNAL, "external_service_unavailable") from error + raise def register(self, run_id: str) -> TemporaryIdentity: from provision_device import server_register_script, validate_credential diff --git a/scripts/e2e_hil_device.py b/scripts/e2e_hil_device.py index 3fdbc568..58135db7 100644 --- a/scripts/e2e_hil_device.py +++ b/scripts/e2e_hil_device.py @@ -20,9 +20,9 @@ fcntl = None try: - from sqlite_board_probe_protocol import Partition, partition_by_label + from sqlite_board_probe_protocol import Partition, ProbeError, parse_partition_table, partition_by_label except ModuleNotFoundError: - from scripts.sqlite_board_probe_protocol import Partition, partition_by_label + from scripts.sqlite_board_probe_protocol import Partition, ProbeError, parse_partition_table, partition_by_label SAFE_NAME = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") HEX_SHA256 = re.compile(r"^[0-9a-f]{64}$") @@ -66,6 +66,7 @@ class OfficialProfile: ("linx_secrets", 1, 2, 0x2E0000, 0x10000), ("assets", 1, 0x82, 0x300000, 0x100000), ("model", 1, 0x82, 0x400000, 0x300000), + ("voicelife", 1, 0x81, 0x700000, 0x900000), ), ), "pcb": OfficialProfile( @@ -267,6 +268,16 @@ def load_application_image( build_directory: Path, descriptor: DeviceDescriptor, partitions: list[Partition] ) -> ApplicationImage: application = validate_device_layout(descriptor, partitions) + built_partition_table = build_directory / "partition_table" / "partition-table.bin" + if built_partition_table.is_file(): + try: + built_partitions = parse_partition_table(built_partition_table.read_bytes()) + except (OSError, ProbeError, ValueError) as error: + raise HilConfigurationError("application build partition table is invalid") from error + if tuple(_partition_tuple(partition) for partition in built_partitions) != tuple( + _partition_tuple(partition) for partition in partitions + ): + raise HilProfileMismatch("application build partition layout does not match board") binary = build_directory / "voicelife.bin" flasher = build_directory / "flasher_args.json" try: diff --git a/scripts/e2e_runner.py b/scripts/e2e_runner.py index b6629793..08b9c2db 100644 --- a/scripts/e2e_runner.py +++ b/scripts/e2e_runner.py @@ -29,6 +29,7 @@ class FailureCategory(str, Enum): INFRASTRUCTURE = "infrastructure" PRODUCT = "product" DEVICE = "device" + LEASE = "lease" EXTERNAL = "external" TIMEOUT = "timeout" INTERRUPTED = "interrupted" @@ -43,6 +44,7 @@ class ExitCode(IntEnum): INFRASTRUCTURE = 10 PRODUCT = 20 DEVICE = 30 + LEASE = 31 EXTERNAL = 40 TIMEOUT = 60 INTERRUPTED = 70 @@ -59,6 +61,7 @@ class RunStatus(str, Enum): FailureCategory.INFRASTRUCTURE: ExitCode.INFRASTRUCTURE, FailureCategory.PRODUCT: ExitCode.PRODUCT, FailureCategory.DEVICE: ExitCode.DEVICE, + FailureCategory.LEASE: ExitCode.LEASE, FailureCategory.EXTERNAL: ExitCode.EXTERNAL, FailureCategory.TIMEOUT: ExitCode.TIMEOUT, FailureCategory.INTERRUPTED: ExitCode.INTERRUPTED, diff --git a/scripts/render_e2e_summary.py b/scripts/render_e2e_summary.py new file mode 100644 index 00000000..be83c5fb --- /dev/null +++ b/scripts/render_e2e_summary.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Render a safe GitHub Actions job summary from E2E evidence.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from e2e_evidence import EvidenceValidationError, validate_evidence + + +def render(root: Path) -> str: + rows: list[str] = [] + paths = sorted(root.rglob("evidence-*.json")) + for path in paths: + try: + document = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(document, dict): + raise ValueError + validate_evidence(document) + except (OSError, UnicodeError, json.JSONDecodeError, ValueError, EvidenceValidationError) as error: + raise ValueError("cannot render invalid E2E evidence") from error + status = str(document["status"]).upper() + category = document["failure_category"] or "none" + phase = document["failed_phase"] or "none" + rows.append( + f"| `{document['profile']}` | `{document['journey']}` | {status} | `{category}` | " + f"`{phase}` | `{document['message_code']}` |" + ) + if not rows: + raise ValueError("no E2E evidence found") + return ( + "## E2E result\n\n" + "| Profile | Journey | Status | Failure category | Failed phase | Message |\n" + "| --- | --- | --- | --- | --- | --- |\n" + "\n".join(rows) + "\n" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifact_dir", type=Path) + parser.add_argument("--output", type=Path, default=None) + args = parser.parse_args(argv) + try: + summary = render(args.artifact_dir) + if args.output is None: + print(summary, end="") + else: + args.output.write_text(summary, encoding="utf-8") + except (OSError, ValueError) as error: + print(str(error), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_e2e.py b/scripts/run_e2e.py index 1846392d..3612f7e0 100644 --- a/scripts/run_e2e.py +++ b/scripts/run_e2e.py @@ -16,8 +16,15 @@ HostImGatewayRecoveryE2EAdapter, HostLifecycleExampleAdapter, ) -from e2e_hil_adapters import HilPairingAdapter, RealHilHardware -from e2e_runner import ExitCode, FailureCategory, RunnerConfig, RunnerResult, exit_code_for, run_e2e +from e2e_hil_adapters import HilPairingAdapter, HilVoiceAdapter, RealHilHardware +from e2e_runner import ( + ExitCode, + FailureCategory, + RunnerConfig, + RunnerResult, + exit_code_for, + run_e2e, +) PROFILES = {"host": frozenset({"host"}), "hil": frozenset({"sparkbot", "pcb"})} @@ -43,6 +50,12 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.add_argument("--server-dir") parser.add_argument("--gateway-origin") parser.add_argument("--user-id") + parser.add_argument("--input-tts", choices=("dashscope", "aliyun-nls"), default="dashscope") + parser.add_argument("--tts-model", default="qwen-audio-3.0-tts-flash") + parser.add_argument("--voice", default="longanlingxi") + parser.add_argument("--text", action="append") + parser.add_argument("--expect-terminal", action="store_true") + parser.add_argument("--response-timeout", type=float, default=45.0) return parser.parse_args(argv) @@ -63,7 +76,7 @@ def build_adapter(layer: str, journey: str, args: argparse.Namespace | Path | No artifact_directory = args.artifact_dir if isinstance(args, argparse.Namespace) else args hil_options = ("device", "lease_dir", "server", "server_dir", "gateway_origin", "user_id") if ( - journey != "im-pairing" + journey not in {"im-pairing", "voice"} and isinstance(args, argparse.Namespace) and any(getattr(args, name) is not None for name in hil_options) ): @@ -79,6 +92,29 @@ def build_adapter(layer: str, journey: str, args: argparse.Namespace | Path | No lease_directory = args.lease_dir or Path.home() / ".voicelife" / "hil-leases" hardware = RealHilHardware(str(server), str(server_directory), str(gateway_origin), str(user_id)) return HilPairingAdapter(Path(device), lease_directory, hardware=hardware) + if journey == "voice": + if layer != "hil" or not isinstance(args, argparse.Namespace) or args.profile not in {"sparkbot", "pcb"}: + raise ValueError("voice journey requires a supported HIL profile") + device = _required_hil_option(args, "device") + server = _required_hil_option(args, "server") + server_directory = _required_hil_option(args, "server_dir") + gateway_origin = _required_hil_option(args, "gateway_origin") + user_id = _required_hil_option(args, "user_id") + if args.response_timeout <= 0: + raise ValueError("response-timeout must be positive") + lease_directory = args.lease_dir or Path.home() / ".voicelife" / "hil-leases" + hardware = RealHilHardware(str(server), str(server_directory), str(gateway_origin), str(user_id)) + return HilVoiceAdapter( + Path(device), + lease_directory, + hardware=hardware, + input_tts=args.input_tts, + tts_model=args.tts_model, + voice=args.voice, + texts=args.text, + expect_terminal=args.expect_terminal, + response_timeout=args.response_timeout, + ) if journey == "im-gateway-strong-reminder" and layer == "host": return HostImGatewayE2EAdapter() if journey == "im-gateway-recovery" and layer == "host": @@ -135,14 +171,24 @@ def build_evidence(result: RunnerResult, config: RunnerConfig) -> dict[str, obje else False ) hil = None - if scope == "hil_im_pairing" and result.status.value == "passed": + if scope in {"hil_im_pairing", "hil_voice"} and result.status.value == "passed": hil = { "firmware_sha256": collected.get("firmware_sha256"), "gateway_commit": collected.get("gateway_commit"), "device_fingerprint": collected.get("device_fingerprint"), "readiness_markers": collected.get("readiness_markers"), - "pairing_markers": collected.get("pairing_markers"), } + if scope == "hil_im_pairing": + hil["pairing_markers"] = collected.get("pairing_markers") + else: + hil.update( + { + "input_tts_provider": collected.get("input_tts_provider"), + "input_tts_model": collected.get("input_tts_model"), + "input_tts_voice": collected.get("input_tts_voice"), + "wake_stage": collected.get("wake_stage"), + } + ) return { "schema_version": 1, "run_id": result.run_id, diff --git a/scripts/voice_linx_serial_multiturn_test.py b/scripts/voice_linx_serial_multiturn_test.py index 79d4f48e..b0d113fb 100644 --- a/scripts/voice_linx_serial_multiturn_test.py +++ b/scripts/voice_linx_serial_multiturn_test.py @@ -5,24 +5,20 @@ import argparse import json -import os import re import subprocess import sys -import tempfile import threading import time from dataclasses import dataclass from pathlib import Path +from voice_tts_fixture import TtsFixtureError, synthesize_fixture, validate_provider_environment + try: - import dashscope import serial - from dashscope.audio.tts_v2 import SpeechSynthesizer except ImportError: - dashscope = None serial = None - SpeechSynthesizer = None MAGIC, VERSION, BEGIN, PCM, END = b"VLVT", 1, 1, 2, 3 @@ -53,6 +49,8 @@ class TurnResult: class PreparedTurn: input_text: str tts_ms: int + first_packet_ms: int + audio_bytes: int frames: list[bytes] @@ -165,68 +163,38 @@ def reset_usb_serial_jtag(device: serial.Serial) -> None: time.sleep(0.2) -def synthesize(text: str, model: str, voice: str) -> bytes: - synthesizer = SpeechSynthesizer(model=model, voice=voice) - audio = synthesizer.call(text) - if not isinstance(audio, (bytes, bytearray)) or not audio: - raise RuntimeError("empty_tts_audio") - return bytes(audio) - - -def synthesize_macos_say(text: str, voice: str) -> bytes: - """Generate local AIFF only for board-harness fallback input. - - This path is intentionally opt-in. It lets the serial state-machine test - remain reproducible when the external TTS provider is unavailable; it is - not reported as a DashScope model result. - """ - path: Path | None = None - try: - with tempfile.NamedTemporaryFile(prefix="voicelife-serial-input-", suffix=".aiff", delete=False) as output: - path = Path(output.name) - subprocess.run(["say", "-v", voice, "-o", str(path), text], check=True, capture_output=True) - audio = path.read_bytes() - if not audio: - raise RuntimeError("empty_local_tts_audio") - return audio - except FileNotFoundError as error: - raise RuntimeError("macos_say_unavailable") from error - except subprocess.CalledProcessError as error: - raise RuntimeError("macos_say_failed") from error - finally: - if path is not None: - path.unlink(missing_ok=True) - - -def to_pcm_frames(audio: bytes) -> list[bytes]: - result = subprocess.run( - [ - "ffmpeg", - "-hide_banner", - "-loglevel", - "error", - "-i", - "pipe:0", - "-f", - "s16le", - "-acodec", - "pcm_s16le", - "-ac", - "1", - "-ar", - "16000", - "pipe:1", - ], - input=audio, - capture_output=True, - check=False, - ) - if result.returncode != 0 or not result.stdout: - raise RuntimeError("ffmpeg_pcm_decode_failed") +def to_pcm_frames(audio: bytes, audio_format: str = "encoded") -> list[bytes]: + if audio_format == "pcm_s16le_16000_mono": + pcm = audio + else: + result = subprocess.run( + [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-i", + "pipe:0", + "-f", + "s16le", + "-acodec", + "pcm_s16le", + "-ac", + "1", + "-ar", + "16000", + "pipe:1", + ], + input=audio, + capture_output=True, + check=False, + ) + if result.returncode != 0 or not result.stdout: + raise RuntimeError("ffmpeg_pcm_decode_failed") + pcm = result.stdout # The explicit serial end packet is the endpoint for this harness. Adding # trailing silence makes local VAD stop early, then incorrectly turns the # remaining test frames into late-frame rejection noise. - pcm = result.stdout remainder = len(pcm) % PCM_FRAME_BYTES if remainder: pcm += bytes(PCM_FRAME_BYTES - remainder) @@ -251,6 +219,41 @@ def parse_audio_stats(line: str) -> dict[str, int]: return {key: int(value) for key, value in re.findall(r"(\w+)=([0-9]+)", line)} +def display_evidence( + display_profile: str, raw_lines: list[str], results: list[TurnResult], requested_turns: int +) -> tuple[bool, int, bool]: + """Return (complete, content_snapshots, scroll_observed) without retaining display text.""" + if display_profile == "sparkbot": + rendered_text_lines = [line for line in raw_lines if "SPARKBOT_TEXT_RENDER" in line] + content_render_lines = [line for line in rendered_text_lines if "content_visible=1" in line] + complete = ( + bool(rendered_text_lines) + and all( + "generation=" in line + and "revision=" in line + and "content_height=" in line + and "viewport_height=" in line + and "overflow_width=" in line + and "manual_line_breaks=" in line + and " status=" in line + and " content=" in line + for line in rendered_text_lines + ) + and all("viewport_height=120" in line and "content_height=0" not in line for line in content_render_lines) + ) + return ( + complete, + len(content_render_lines), + any(re.search(r"overflow_width=[1-9][0-9]*", line) for line in content_render_lines), + ) + + display_draws_by_turn = [ + sum("DISPLAY_DRAW=1" in line for line in raw_lines[result.log_start : result.log_end]) for result in results + ] + complete = len(results) == requested_turns and all(count > 0 for count in display_draws_by_turn) + return complete, sum(display_draws_by_turn), False + + def prepare_turns(texts: list[str], input_tts: str, model: str, voice: str, say_voice: str) -> list[PreparedTurn]: """Generate every host utterance before opening the first device capture. @@ -260,13 +263,20 @@ def prepare_turns(texts: list[str], input_tts: str, model: str, voice: str, say_ """ prepared: list[PreparedTurn] = [] for index, text in enumerate(texts, start=1): - started = time.monotonic() - audio = synthesize(text, model, voice) if input_tts == "dashscope" else synthesize_macos_say(text, say_voice) - frames = to_pcm_frames(audio) - tts_ms = round((time.monotonic() - started) * 1000) + fixture_voice = say_voice if input_tts == "macos-say" else voice + fixture = synthesize_fixture(text, input_tts, model, fixture_voice) + frames = to_pcm_frames(fixture.audio, fixture.audio_format) if not frames: raise RuntimeError(f"turn_{index}_has_no_pcm_frames") - prepared.append(PreparedTurn(input_text=text, tts_ms=tts_ms, frames=frames)) + prepared.append( + PreparedTurn( + input_text=text, + tts_ms=fixture.total_ms, + first_packet_ms=fixture.first_packet_ms, + audio_bytes=len(fixture.audio), + frames=frames, + ) + ) return prepared @@ -399,6 +409,11 @@ def run_turn( # Wait for that existing capture instead of sending a second PressDown. log.wait_for("SERIAL_VOICE_CAPTURE_READY", cursor, 12) result.completed = True + # The capture-ready evidence and its phase-4 display snapshot are + # emitted by separate runtime tasks. Give the logger one scheduling + # slice to collect the snapshot before taking the turn boundary; + # otherwise the next turn's range can accidentally omit phase=4. + time.sleep(0.1) except (RuntimeError, TimeoutError, serial.SerialException) as error: result.error = str(error) result.log_end = log.mark() @@ -419,9 +434,9 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--voice", default="longanhuan_v3") parser.add_argument( "--input-tts", - choices=("dashscope", "macos-say"), + choices=("dashscope", "aliyun-nls", "macos-say"), default="dashscope", - help="Source for injected audio. macos-say is a local fallback when external TTS is unavailable.", + help="Host TTS fixture provider. macos-say is local-only and never proves cloud availability.", ) parser.add_argument("--say-voice", default="Ting-Ting", help="macOS voice used with --input-tts macos-say.") parser.add_argument("--response-timeout", type=float, default=45) @@ -441,6 +456,12 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Require at least one subtitle wider than the safe one-line viewport to enter horizontal scrolling.", ) + parser.add_argument( + "--display-profile", + choices=("sparkbot", "pcb"), + default="sparkbot", + help="Board-specific display evidence contract used by the test firmware.", + ) parser.add_argument( "--allow-asr-mismatch", action="store_true", @@ -458,23 +479,20 @@ def parse_args() -> argparse.Namespace: def main() -> int: args = parse_args() - if args.input_tts == "dashscope" and not os.environ.get("DASHSCOPE_API_KEY"): - print("DASHSCOPE_API_KEY is required", file=sys.stderr) - return 2 if serial is None: print("pyserial is required", file=sys.stderr) return 2 - if args.input_tts == "dashscope" and (dashscope is None or SpeechSynthesizer is None): - print("dashscope is required for --input-tts dashscope", file=sys.stderr) + try: + validate_provider_environment(args.input_tts) + except TtsFixtureError as error: + print(f"input_preparation_failed:{error.code}", file=sys.stderr) return 2 - if args.input_tts == "dashscope": - dashscope.api_key = os.environ["DASHSCOPE_API_KEY"] texts = args.text or DEFAULT_TURNS try: # Prepare every utterance before the serial endpoint is opened. This is # part of the harness contract, not a production latency measurement. prepared_turns = prepare_turns(texts, args.input_tts, args.tts_model, args.voice, args.say_voice) - except (RuntimeError, subprocess.SubprocessError, TimeoutError) as error: + except (TtsFixtureError, RuntimeError, subprocess.SubprocessError, TimeoutError) as error: print(f"input_preparation_failed:{error}", file=sys.stderr) return 2 try: @@ -530,36 +548,32 @@ def main() -> int: interaction_stats = [parse_audio_stats(line) for line in raw_lines if "INTERACTION_QUEUE_STATS" in line] interaction_keys = ("control_dropped", "best_effort_dropped", "board_dropped") serial_pcm_rejections = [line for line in raw_lines if "SERIAL_VOICE_PCM=reject" in line] - required_active_phases = (3, 4, 5, 6) + # Auto mode may receive Linx's server-VAD STT event before the local + # capture-stopped callback. In that valid path the runtime transitions + # directly from listening (4) to thinking (6), so phase 5 is optional + # only when the segment contains an authoritative STT event. + required_active_phases = (3, 4, 6, 7) rendered_text_lines = [line for line in raw_lines if "SPARKBOT_TEXT_RENDER" in line] - content_render_lines = [line for line in rendered_text_lines if "content_visible=1" in line] - display_text_trace_complete = ( - bool(rendered_text_lines) - and all( - "generation=" in line - and "revision=" in line - and "content_height=" in line - and "viewport_height=" in line - and "overflow_width=" in line - and "manual_line_breaks=" in line - and " status=" in line - and " content=" in line - for line in rendered_text_lines - ) - and all("viewport_height=120" in line and "content_height=0" not in line for line in content_render_lines) + display_text_trace_complete, display_content_snapshots, display_scroll_observed = display_evidence( + args.display_profile, raw_lines, results, len(texts) ) - display_scroll_observed = any(re.search(r"overflow_width=[1-9][0-9]*", line) for line in content_render_lines) def turn_phases_complete(marker: str) -> bool: - return len(results) == len(texts) and all( - all( - any(f"{marker}={phase} " in line for line in raw_lines[result.log_start : result.log_end]) - for phase in required_active_phases - ) - for result in results - ) + if len(results) != len(texts): + return False + for result in results: + segment = raw_lines[result.log_start : result.log_end] + if not all(any(f"{marker}={phase} " in line for line in segment) for phase in required_active_phases): + return False + if not any(f"{marker}=5 " in line for line in segment): + has_server_stt = any("SERIAL_VOICE_EVIDENCE event=stt_text_received " in line for line in segment) + if not has_server_stt: + return False + return True acceptance = { + "input_tts_preflight": bool(prepared_turns) + and all(turn.audio_bytes > 0 and len(turn.frames) > 0 for turn in prepared_turns), "audio_stats_present": all(key in audio_stats for key in required_present), "audio_flow_observed": audio_stats.get("test_in_frames", 0) > 0 and audio_stats.get("out_frames", 0) > 0, "zero_loss": all(audio_stats.get(key, -1) == 0 for key in required_zero), @@ -581,6 +595,16 @@ def turn_phases_complete(marker: str) -> bool: or (bool(results) and results[-1].terminal_guard_armed and results[-1].terminal_guard_clean), } report = { + "schema_version": 2, + "fixture": { + "provider": args.input_tts, + "model": "macos-say" if args.input_tts == "macos-say" else args.tts_model, + "voice": args.say_voice if args.input_tts == "macos-say" else args.voice, + "requests": len(prepared_turns), + "audio_bytes": sum(turn.audio_bytes for turn in prepared_turns), + "first_packet_ms_max": max((turn.first_packet_ms for turn in prepared_turns), default=0), + "total_ms_max": max((turn.tts_ms for turn in prepared_turns), default=0), + }, "requested_turns": len(texts), "completed_turns": sum(result.completed for result in results), "asr_exact_matches": sum(result.asr_matches_input for result in results), @@ -590,7 +614,7 @@ def turn_phases_complete(marker: str) -> bool: "serial_pcm_rejections": serial_pcm_rejections, "display": { "rendered_text_snapshots": len(rendered_text_lines), - "content_snapshots": len(content_render_lines), + "content_snapshots": display_content_snapshots, "scroll_observed": display_scroll_observed, }, "acceptance": acceptance, diff --git a/scripts/voice_linx_wake_injection_test.py b/scripts/voice_linx_wake_injection_test.py index bf7726e5..00afe123 100644 --- a/scripts/voice_linx_wake_injection_test.py +++ b/scripts/voice_linx_wake_injection_test.py @@ -4,7 +4,7 @@ from __future__ import annotations import argparse -import os +import json import sys import time from pathlib import Path @@ -16,18 +16,14 @@ packet, reset_usb_serial_jtag, run_turn, - synthesize, to_pcm_frames, ) +from voice_tts_fixture import TtsFixtureError, synthesize_fixture, validate_provider_environment try: - import dashscope import serial - from dashscope.audio.tts_v2 import SpeechSynthesizer except ImportError: - dashscope = None serial = None - SpeechSynthesizer = None TURN_BEGIN, PCM, TURN_END = 1, 2, 3 WAKE_BEGIN, WAKE_END = 4, 5 @@ -45,6 +41,7 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("--tts-model", default="cosyvoice-v3-flash") parser.add_argument("--voice", default="longanhuan_v3") + parser.add_argument("--input-tts", choices=("dashscope", "aliyun-nls"), default="dashscope") parser.add_argument("--timeout", type=float, default=30) parser.add_argument( "--reset-before-run", @@ -52,6 +49,7 @@ def parse_args() -> argparse.Namespace: help="Hard-reset the board after opening the serial port, then wait for its ready sequence.", ) parser.add_argument("--serial-log", type=Path) + parser.add_argument("--result-json", type=Path) args = parser.parse_args() if args.baud <= 0 or args.timeout <= 0: parser.error("baud 和 timeout 必须为正数") @@ -108,23 +106,26 @@ def request_wake_begin(log: SerialLog, device: serial.Serial, timeout: float) -> def main() -> int: args = parse_args() - api_key = os.environ.get("DASHSCOPE_API_KEY") - if not api_key: - print("DASHSCOPE_API_KEY is required", file=sys.stderr) + if serial is None: + print("pyserial is required", file=sys.stderr) return 2 - if serial is None or dashscope is None or SpeechSynthesizer is None: - print("pyserial and dashscope are required", file=sys.stderr) - return 2 - - dashscope.api_key = api_key try: - audio = synthesize(args.text, args.tts_model, args.voice) - frames = to_pcm_frames(audio) + validate_provider_environment(args.input_tts) + wake_fixture = synthesize_fixture(args.text, args.input_tts, args.tts_model, args.voice) + frames = to_pcm_frames(wake_fixture.audio, wake_fixture.audio_format) followups = [] for text in args.followup_text or []: - followup_audio = synthesize(text, args.tts_model, args.voice) - followups.append(PreparedTurn(input_text=text, tts_ms=0, frames=to_pcm_frames(followup_audio))) - except (RuntimeError, OSError) as error: + fixture = synthesize_fixture(text, args.input_tts, args.tts_model, args.voice) + followups.append( + PreparedTurn( + input_text=text, + tts_ms=fixture.total_ms, + first_packet_ms=fixture.first_packet_ms, + audio_bytes=len(fixture.audio), + frames=to_pcm_frames(fixture.audio, fixture.audio_format), + ) + ) + except (TtsFixtureError, RuntimeError, OSError) as error: print(f"input_preparation_failed:{error}", file=sys.stderr) return 2 if not frames: @@ -140,6 +141,14 @@ def main() -> int: log = SerialLog(device) log.start() cursor = 0 + wake_detected = False + ack_requested = False + ack_tts_started = False + ack_tts_first_audio = False + ack_tts_stopped = False + capture_started = False + completed_followups = 0 + exit_code = 1 try: if args.reset_before_run: reset_usb_serial_jtag(device) @@ -157,6 +166,7 @@ def main() -> int: device.flush() cursor = wait_for(log, "SERIAL_VOICE_WAKE_END=ok", cursor, 5) cursor = wait_for(log, "WAKE_DETECTED word=你好牛牛", cursor, args.timeout) + wake_detected = True cursor = wait_for(log, "SERIAL_VOICE_EVIDENCE event=wake_detected ", cursor, 5) # Both Linx-compatible wake paths are valid: a silent detect opens the # capture immediately, while a deliberate confirmation speech first @@ -172,11 +182,19 @@ def main() -> int: args.timeout, ) if "local_wake_ack_requested" in wake_protocol: + ack_requested = True + cursor = wait_for(log, "SERIAL_VOICE_EVIDENCE event=tts_started ", cursor, args.timeout) + ack_tts_started = True + cursor = wait_for(log, "SERIAL_VOICE_EVIDENCE event=tts_first_audio ", cursor, args.timeout) + ack_tts_first_audio = True cursor = wait_for(log, "SERIAL_VOICE_EVIDENCE event=tts_stopped ", cursor, args.timeout) + ack_tts_stopped = True cursor = wait_for(log, "SERIAL_VOICE_EVIDENCE event=capture_started ", cursor, args.timeout) + capture_started = True if not followups: print(f"wake_injection_success text={args.text} frames={len(frames)}") - return 0 + exit_code = 0 + return exit_code for index, prepared in enumerate(followups, start=1): result = run_turn( device, @@ -190,11 +208,13 @@ def main() -> int: ) if result.error: raise TimeoutError(f"followup_{index}:{result.error}") + completed_followups += 1 print( f"wake_followup_success index={index} input={prepared.input_text} " f"asr={result.asr_text} reply={result.reply_text}" ) - return 0 + exit_code = 0 + return exit_code except TimeoutError as error: print(f"wake_injection_timeout:{error}", file=sys.stderr) return 1 @@ -207,6 +227,34 @@ def main() -> int: device.close() if args.serial_log: args.serial_log.write_text("\n".join(log.all_lines()) + "\n", encoding="utf-8") + if args.result_json: + report = { + "schema_version": 1, + "fixture": { + "provider": args.input_tts, + "model": args.tts_model, + "voice": args.voice, + "requests": 1 + len(followups), + "audio_bytes": len(wake_fixture.audio) + sum(turn.audio_bytes for turn in followups), + "first_packet_ms_max": max( + [wake_fixture.first_packet_ms, *(turn.first_packet_ms for turn in followups)] + ), + "total_ms_max": max([wake_fixture.total_ms, *(turn.tts_ms for turn in followups)]), + }, + "requested_followups": len(followups), + "completed_followups": completed_followups, + "acceptance": { + "input_tts_preflight": bool(frames), + "wake_detected": wake_detected, + "wake_ack_requested": ack_requested, + "wake_ack_tts_started": ack_tts_started, + "wake_ack_tts_first_audio": ack_tts_first_audio, + "wake_ack_tts_stopped": ack_tts_stopped, + "capture_started_after_ack": capture_started and ack_tts_stopped, + }, + "exit_code": exit_code, + } + args.result_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") if __name__ == "__main__": diff --git a/scripts/voice_tts_fixture.py b/scripts/voice_tts_fixture.py new file mode 100644 index 00000000..e72d7fab --- /dev/null +++ b/scripts/voice_tts_fixture.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Provider-neutral host TTS fixtures for voice HIL input generation.""" + +from __future__ import annotations + +import importlib +import os +import shutil +import subprocess +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + + +class TtsFixtureError(RuntimeError): + """Stable, non-sensitive TTS fixture failure.""" + + def __init__(self, code: str) -> None: + super().__init__(code) + self.code = code + + +@dataclass(frozen=True) +class TtsFixtureResult: + audio: bytes + audio_format: str + provider: str + model: str + voice: str + total_ms: int + first_packet_ms: int + + +SUPPORTED_PROVIDERS = frozenset({"dashscope", "aliyun-nls", "macos-say"}) + + +def validate_provider_environment(provider: str) -> None: + if provider not in SUPPORTED_PROVIDERS: + raise TtsFixtureError("input_tts_provider_unsupported") + if provider == "dashscope": + if not os.environ.get("DASHSCOPE_API_KEY"): + raise TtsFixtureError("dashscope_api_key_missing") + try: + importlib.import_module("dashscope.audio.tts_v2") + except ImportError as error: + raise TtsFixtureError("dashscope_unavailable") from error + elif provider == "aliyun-nls": + if not os.environ.get("ALIYUN_NLS_APPKEY") or not os.environ.get("ALIYUN_NLS_TOKEN"): + raise TtsFixtureError("aliyun_nls_credentials_missing") + try: + importlib.import_module("nls") + except ImportError as error: + raise TtsFixtureError("aliyun_nls_unavailable") from error + elif shutil.which("say") is None: + raise TtsFixtureError("macos_say_unavailable") + + +def _dashscope_synthesize(text: str, model: str, voice: str) -> TtsFixtureResult: + dashscope = importlib.import_module("dashscope") + tts_v2 = importlib.import_module("dashscope.audio.tts_v2") + dashscope.api_key = os.environ["DASHSCOPE_API_KEY"] + endpoint = os.environ.get("DASHSCOPE_BASE_WEBSOCKET_API_URL") + if endpoint: + dashscope.base_websocket_api_url = endpoint + started = time.monotonic() + synthesizer = tts_v2.SpeechSynthesizer(model=model, voice=voice) + audio = synthesizer.call(text) + total_ms = round((time.monotonic() - started) * 1000) + if not isinstance(audio, (bytes, bytearray)) or not audio: + raise TtsFixtureError("empty_tts_audio") + first_packet = getattr(synthesizer, "get_first_package_delay", None) + first_packet_ms = first_packet() if callable(first_packet) else 0 + if not isinstance(first_packet_ms, (int, float)) or first_packet_ms < 0: + first_packet_ms = 0 + return TtsFixtureResult( + audio=bytes(audio), + audio_format="encoded", + provider="dashscope", + model=model, + voice=voice, + total_ms=total_ms, + first_packet_ms=round(first_packet_ms), + ) + + +def _aliyun_nls_synthesize(text: str, model: str, voice: str) -> TtsFixtureResult: + """Use the official NLS callback SDK and request raw 16 kHz PCM.""" + nls = importlib.import_module("nls") + chunks: list[bytes] = [] + failure: list[str] = [] + first_packet_ms = 0 + started = time.monotonic() + + def on_data(data: bytes, *_: Any) -> None: + nonlocal first_packet_ms + if not chunks: + first_packet_ms = round((time.monotonic() - started) * 1000) + if isinstance(data, (bytes, bytearray)): + chunks.append(bytes(data)) + + def on_error(message: object, *_: Any) -> None: + failure.append(type(message).__name__) + + synthesizer = nls.NlsSpeechSynthesizer( + url=os.environ.get("ALIYUN_NLS_URL", "wss://nls-gateway.cn-shanghai.aliyuncs.com/ws/v1"), + token=os.environ["ALIYUN_NLS_TOKEN"], + appkey=os.environ["ALIYUN_NLS_APPKEY"], + on_data=on_data, + on_error=on_error, + ) + try: + completed = synthesizer.start(text, voice=voice, aformat="pcm", sample_rate=16000) + except Exception as error: + raise TtsFixtureError("aliyun_nls_request_failed") from error + if completed is False or failure: + raise TtsFixtureError("aliyun_nls_request_failed") + audio = b"".join(chunks) + if not audio: + raise TtsFixtureError("empty_tts_audio") + return TtsFixtureResult( + audio=audio, + audio_format="pcm_s16le_16000_mono", + provider="aliyun-nls", + model=model, + voice=voice, + total_ms=round((time.monotonic() - started) * 1000), + first_packet_ms=first_packet_ms, + ) + + +def _macos_say_synthesize(text: str, voice: str) -> TtsFixtureResult: + path: Path | None = None + started = time.monotonic() + try: + with tempfile.NamedTemporaryFile(prefix="voicelife-serial-input-", suffix=".aiff", delete=False) as output: + path = Path(output.name) + subprocess.run(["say", "-v", voice, "-o", str(path), text], check=True, capture_output=True) + audio = path.read_bytes() + if not audio: + raise TtsFixtureError("empty_local_tts_audio") + return TtsFixtureResult( + audio=audio, + audio_format="encoded", + provider="macos-say", + model="macos-say", + voice=voice, + total_ms=round((time.monotonic() - started) * 1000), + first_packet_ms=0, + ) + except FileNotFoundError as error: + raise TtsFixtureError("macos_say_unavailable") from error + except subprocess.CalledProcessError as error: + raise TtsFixtureError("macos_say_failed") from error + finally: + if path is not None: + path.unlink(missing_ok=True) + + +def synthesize_fixture(text: str, provider: str, model: str, voice: str) -> TtsFixtureResult: + validate_provider_environment(provider) + if provider == "dashscope": + return _dashscope_synthesize(text, model, voice) + if provider == "aliyun-nls": + return _aliyun_nls_synthesize(text, model, voice) + return _macos_say_synthesize(text, voice) diff --git a/scripts/voice_tts_preflight.py b/scripts/voice_tts_preflight.py new file mode 100644 index 00000000..40e61bee --- /dev/null +++ b/scripts/voice_tts_preflight.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Verify a host TTS fixture before reserving time on a real voice device.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from voice_linx_serial_multiturn_test import to_pcm_frames +from voice_tts_fixture import TtsFixtureError, synthesize_fixture, validate_provider_environment + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-tts", choices=("dashscope", "aliyun-nls"), required=True) + parser.add_argument("--tts-model", required=True) + parser.add_argument("--voice", required=True) + parser.add_argument("--text", default="这是 VoiceLife HIL TTS 预检。") + parser.add_argument("--result-json", type=Path, required=True) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + validate_provider_environment(args.input_tts) + fixture = synthesize_fixture(args.text, args.input_tts, args.tts_model, args.voice) + frames = to_pcm_frames(fixture.audio, fixture.audio_format) + except (TtsFixtureError, RuntimeError, OSError) as error: + print(f"input_preparation_failed:{error}", file=sys.stderr) + return 2 + report = { + "schema_version": 1, + "fixture": { + "provider": fixture.provider, + "model": fixture.model, + "voice": fixture.voice, + "requests": 1, + "audio_bytes": len(fixture.audio), + "first_packet_ms_max": fixture.first_packet_ms, + "total_ms_max": fixture.total_ms, + }, + "acceptance": { + "audio_present": bool(fixture.audio), + "pcm_frames_present": bool(frames), + }, + } + args.result_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return 0 if all(report["acceptance"].values()) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/python/test_e2e_artifact_tools.py b/tests/python/test_e2e_artifact_tools.py new file mode 100644 index 00000000..25f468d8 --- /dev/null +++ b/tests/python/test_e2e_artifact_tools.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" + + +def load(name: str, path: Path) -> object: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +EVIDENCE = load("e2e_evidence", SCRIPTS / "e2e_evidence.py") +CHECK = load("check_e2e_artifacts", SCRIPTS / "check_e2e_artifacts.py") +SUMMARY = load("render_e2e_summary", SCRIPTS / "render_e2e_summary.py") + + +class E2EArtifactToolsTest(unittest.TestCase): + def evidence(self) -> dict[str, object]: + return { + "schema_version": 1, + "run_id": "a" * 32, + "correlation_id": "b" * 32, + "scope": "runner_contract_only", + "layer": "host", + "journey": "lifecycle-example", + "profile": "host", + "started_at": "2026-08-21T00:00:00.000Z", + "finished_at": "2026-08-21T00:00:01.000Z", + "duration_ms": 1000, + "status": "passed", + "failure_category": None, + "failed_phase": None, + "message_code": "run_passed", + "stages": [ + {"name": phase, "status": "passed", "code": "phase_passed"} + for phase in ("prepare", "run", "assert", "collect", "cleanup") + ], + "assertions": [], + "metrics": {"resource_count": 1}, + "cleanup": {"status": "passed", "error_codes": []}, + "hardware_verified": False, + "hil": None, + } + + def test_validates_evidence_and_renders_only_public_fields(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "evidence-a.json").write_text(json.dumps(self.evidence()), encoding="utf-8") + self.assertEqual(CHECK.validate_directory(root), (1, 1)) + summary = SUMMARY.render(root) + self.assertIn("lifecycle-example", summary) + self.assertNotIn("run_id", summary) + + def test_rejects_raw_logs_and_sensitive_json(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "serial.log").write_text("raw", encoding="utf-8") + with self.assertRaises(ValueError): + CHECK.validate_directory(root) + (root / "serial.log").unlink() + (root / "other.json").write_text(json.dumps({"token": "canary"}), encoding="utf-8") + with self.assertRaises(ValueError): + CHECK.validate_directory(root) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_e2e_evidence.py b/tests/python/test_e2e_evidence.py index f058f606..348c4e7b 100644 --- a/tests/python/test_e2e_evidence.py +++ b/tests/python/test_e2e_evidence.py @@ -178,6 +178,79 @@ def test_accepts_strict_hil_pairing_evidence(self) -> None: EVIDENCE.validate_evidence(document) self.assertEqual(json.loads(EVIDENCE.canonical_json(document)), document) + def voice_document(self) -> dict[str, object]: + document = self.document() + document.update( + { + "scope": "hil_voice", + "layer": "hil", + "journey": "voice", + "profile": "sparkbot", + "hardware_verified": True, + "assertions": [ + {"name": "voice_turns_complete", "passed": True, "code": "ok"}, + {"name": "voice_acceptance_clean", "passed": True, "code": "ok"}, + ], + "metrics": { + "input_tts_requests": 3, + "input_tts_audio_bytes": 32000, + "input_tts_first_packet_ms_max": 180, + "input_tts_total_ms_max": 500, + "wake_completed": 1, + "requested_turns": 3, + "completed_turns": 3, + "asr_exact_matches": 3, + "test_in_frames": 120, + "out_frames": 240, + "in_drop": 0, + "out_reject": 0, + "short_write": 0, + "in_i2s_err": 0, + "out_i2s_err": 0, + "serial_pcm_rejections": 0, + "display_content_snapshots": 6, + }, + "hil": { + "firmware_sha256": "a" * 64, + "gateway_commit": "b" * 40, + "device_fingerprint": "c" * 16, + "readiness_markers": ["provisioned", "wifi_ready", "sntp_synced", "ready"], + "input_tts_provider": "dashscope", + "input_tts_model": "qwen-audio-3.0-tts-flash", + "input_tts_voice": "longanlingxi", + "wake_stage": "passed", + }, + } + ) + return document + + def test_accepts_strict_hil_voice_evidence_without_raw_conversation(self) -> None: + document = self.voice_document() + EVIDENCE.validate_evidence(document) + self.assertEqual(json.loads(EVIDENCE.canonical_json(document)), document) + + def test_accepts_pcb_hil_voice_evidence_with_wake_gate(self) -> None: + document = self.voice_document() + document["profile"] = "pcb" + EVIDENCE.validate_evidence(document) + + def test_rejects_pcb_hil_voice_evidence_without_wake_gate(self) -> None: + document = self.voice_document() + document["profile"] = "pcb" + document["hil"]["wake_stage"] = "not_applicable" + document["metrics"]["wake_completed"] = 0 + with self.assertRaises(EVIDENCE.EvidenceValidationError): + EVIDENCE.validate_evidence(document) + + def test_voice_evidence_rejects_pairing_fields_and_unsafe_model(self) -> None: + pairing_fields = self.voice_document() + pairing_fields["hil"]["pairing_markers"] = ["pending"] + unsafe_model = self.voice_document() + unsafe_model["hil"]["input_tts_model"] = "model with spaces" + for document in (pairing_fields, unsafe_model): + with self.subTest(document=document), self.assertRaises(EVIDENCE.EvidenceValidationError): + EVIDENCE.validate_evidence(document) + def test_hil_evidence_rejects_missing_gates_false_success_claim_and_sensitive_values(self) -> None: missing = self.hil_document() missing["hil"]["readiness_markers"] = ["ready"] diff --git a/tests/python/test_e2e_hil_adapter.py b/tests/python/test_e2e_hil_adapter.py index 5986e560..7c1514a0 100644 --- a/tests/python/test_e2e_hil_adapter.py +++ b/tests/python/test_e2e_hil_adapter.py @@ -2,6 +2,7 @@ import hashlib import json +import os import sys import tempfile import types @@ -38,9 +39,21 @@ def __init__( self.device_revoked = False self.serial_open = False self.reset_count = 0 + self.build_profiles: list[str] = [] def inspect(self, descriptor: object, temporary_directory: Path) -> object: self.calls.append("inspect") + if getattr(descriptor, "profile", "") == "pcb": + return [ + HIL.Partition("nvs", 1, 2, 0x9000, 0x6000, 0), + HIL.Partition("otadata", 1, 0, 0xF000, 0x2000, 0), + HIL.Partition("phy_init", 1, 1, 0x11000, 0x1000, 0), + HIL.Partition("ota_0", 0, 0x10, 0x20000, 0x3E0000, 0), + HIL.Partition("ota_1", 0, 0x11, 0x400000, 0x3E0000, 0), + HIL.Partition("voicelife", 1, 0x82, 0x7E0000, 0x200000, 0), + HIL.Partition("linx_secrets", 1, 2, 0xA00000, 0x10000, 0), + HIL.Partition("model", 1, 0x82, 0xA10000, 0x300000, 0), + ] return [ HIL.Partition("nvs", 1, 2, 0x9000, 0x4000, 0), HIL.Partition("otadata", 1, 0, 0xD000, 0x2000, 0), @@ -49,15 +62,18 @@ def inspect(self, descriptor: object, temporary_directory: Path) -> object: HIL.Partition("linx_secrets", 1, 2, 0x2E0000, 0x10000, 0), HIL.Partition("assets", 1, 0x82, 0x300000, 0x100000, 0), HIL.Partition("model", 1, 0x82, 0x400000, 0x300000, 0), + HIL.Partition("voicelife", 1, 0x81, 0x700000, 0x900000, 0), ] def build(self, descriptor: object) -> Path: self.calls.append("build") + self.build_profiles.append(str(getattr(descriptor, "firmware_profile", ""))) return Path("/safe/build") def image(self, build_directory: Path, descriptor: object, partitions: object) -> object: self.calls.append("image") - return HIL.ApplicationImage(Path("/safe/voicelife.bin"), 0x10000, 8, "a" * 64) + offset = 0x20000 if getattr(descriptor, "profile", "") == "pcb" else 0x10000 + return HIL.ApplicationImage(Path("/safe/voicelife.bin"), offset, 8, "a" * 64) def flash(self, descriptor: object, image: object, timeout_s: float) -> None: self.calls.append("flash") @@ -95,10 +111,20 @@ def recover(self, descriptor: object) -> None: class HilPairingAdapterTest(unittest.TestCase): - def descriptor_file(self, directory: str) -> Path: + def test_real_hardware_prepares_sqlite_component_before_build(self) -> None: + with ( + mock.patch.object(HIL, "SQLITE_COMPONENT_FILES", (Path("/definitely-missing-sqlite.c"),)), + mock.patch.object(HIL.subprocess, "run") as run, + ): + HIL.ensure_sqlite_component() + run.assert_called_once() + command = run.call_args.args[0] + self.assertEqual(command[-1], str(HIL.ROOT / "scripts" / "prepare_sqlite.py")) + + def descriptor_file(self, directory: str, profile: str = "sparkbot") -> Path: path = Path(directory) / "device.json" path.write_text( - json.dumps({"schema_version": 1, "name": "bench-a", "port": "/dev/cu.test-a", "profile": "sparkbot"}), + json.dumps({"schema_version": 1, "name": "bench-a", "port": "/dev/cu.test-a", "profile": profile}), encoding="utf-8", ) return path @@ -175,6 +201,14 @@ def test_profile_mismatch_prevents_flash(self) -> None: self.assertNotIn("flash", hardware.calls) self.assertFalse(adapter.lease_held) + def test_lease_conflict_is_classified_separately_from_device_failure(self) -> None: + hardware = FakeHardware() + with mock.patch.object(HIL.DeviceLease, "acquire", side_effect=HIL.HilLeaseUnavailable): + adapter, result = self.execute(hardware) + self.assertEqual(result.failure_category, RUNNER.FailureCategory.LEASE) + self.assertEqual(result.message_code, "device_lease_unavailable") + self.assertFalse(adapter.lease_held) + def test_registration_failure_revokes_run_scoped_device_and_releases_lease(self) -> None: hardware = FakeHardware() hardware.register = mock.Mock( @@ -229,6 +263,22 @@ def test_real_hardware_classifies_missing_command_as_infrastructure(self) -> Non self.assertEqual(raised.exception.category, RUNNER.FailureCategory.INFRASTRUCTURE) self.assertEqual(raised.exception.message_code, "hil_command_unavailable") + def test_real_hardware_classifies_remote_service_failure_as_external(self) -> None: + hardware = HIL.RealHilHardware( + "runner@example.test", "/srv/voicelife", "https://gateway.example.test", "user-test" + ) + with ( + mock.patch.object( + hardware, + "_run", + side_effect=RUNNER.RunnerFailure(RUNNER.FailureCategory.INFRASTRUCTURE, "hil_command_failed"), + ), + self.assertRaises(RUNNER.RunnerFailure) as raised, + ): + hardware._remote("safe script") + self.assertEqual(raised.exception.category, RUNNER.FailureCategory.EXTERNAL) + self.assertEqual(raised.exception.message_code, "external_service_unavailable") + def test_real_hardware_passes_serial_path_as_string(self) -> None: serial_port = mock.Mock() serial_port.__enter__ = mock.Mock(return_value=serial_port) @@ -247,5 +297,149 @@ def test_real_hardware_passes_serial_path_as_string(self) -> None: serial_module.Serial.assert_called_once_with("/dev/cu.test", 115200, timeout=0.2, write_timeout=2) +class HilVoiceAdapterTest(HilPairingAdapterTest): + def voice_config(self, profile: str = "sparkbot") -> object: + return RUNNER.RunnerConfig( + layer="hil", + journey="voice", + profile=profile, + hard_timeout_s=5.0, + phase_timeout_s=3.0, + cleanup_timeout_s=0.5, + ) + + def execute_voice(self, hardware: FakeHardware, profile: str = "sparkbot") -> tuple[HIL.HilVoiceAdapter, object]: + with tempfile.TemporaryDirectory() as directory: + adapter = HIL.HilVoiceAdapter( + self.descriptor_file(directory, profile), + Path(directory) / "leases", + hardware=hardware, + input_tts="dashscope", + tts_model="qwen-audio-3.0-tts-flash", + voice="longanlingxi", + texts=["你好"], + expect_terminal=False, + response_timeout=5.0, + ) + result = RUNNER.run_e2e(self.voice_config(profile), adapter) + return adapter, result + + def test_dashscope_voice_journey_uses_voice_profile_and_sanitized_metrics(self) -> None: + hardware = FakeHardware() + fixture = { + "provider": "dashscope", + "model": "qwen-audio-3.0-tts-flash", + "voice": "longanlingxi", + "requests": 1, + "audio_bytes": 4096, + "first_packet_ms_max": 120, + "total_ms_max": 360, + } + voice_result = { + "schema_version": 2, + "fixture": fixture, + "requested_turns": 1, + "completed_turns": 1, + "asr_exact_matches": 1, + "audio_stats": { + "test_in_frames": 4, + "out_frames": 8, + "in_drop": 0, + "out_reject": 0, + "short_write": 0, + "in_i2s_err": 0, + "out_i2s_err": 0, + }, + "serial_pcm_rejections": [], + "display": {"content_snapshots": 2}, + "acceptance": {key: True for key in HIL.HilVoiceAdapter.CONVERSATION_ACCEPTANCE_KEYS}, + } + wake_result = { + "schema_version": 1, + "fixture": fixture, + "requested_followups": 0, + "completed_followups": 0, + "acceptance": {key: True for key in HIL.HilVoiceAdapter.WAKE_ACCEPTANCE_KEYS}, + "exit_code": 0, + } + preflight_result = { + "schema_version": 1, + "fixture": fixture, + "acceptance": {key: True for key in HIL.HilVoiceAdapter.PREFLIGHT_ACCEPTANCE_KEYS}, + } + + commands: list[list[str]] = [] + + def run_voice(command: list[str], **kwargs: object) -> object: + commands.append(command) + result_path = Path(command[command.index("--result-json") + 1]) + if "voice_tts_preflight.py" in command[1]: + payload = preflight_result + elif "voice_linx_wake_injection_test.py" in command[1]: + payload = wake_result + else: + payload = voice_result + result_path.write_text(json.dumps(payload), encoding="utf-8") + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + + with ( + mock.patch.dict(os.environ, {"DASHSCOPE_API_KEY": "test-key"}, clear=False), + mock.patch.object(HIL, "validate_provider_environment"), + mock.patch.object(HIL.shutil, "which", return_value="/usr/bin/ffmpeg"), + mock.patch.object(HIL.subprocess, "run", side_effect=run_voice), + ): + for profile, firmware_profile in ( + ("sparkbot", "esp32s3-esp-sparkbot-serial-voice"), + ("pcb", "esp32s3-voicelife-pcb-serial-voice"), + ): + with self.subTest(profile=profile): + adapter, result = self.execute_voice(hardware, profile) + self.assertEqual(result.exit_code, RUNNER.ExitCode.SUCCESS) + self.assertEqual(hardware.build_profiles[-1], firmware_profile) + self.assertEqual(commands[-1][commands[-1].index("--display-profile") + 1], profile) + self.assertEqual(result.collected["scope"], "hil_voice") + self.assertTrue(result.collected["hardware_verified"]) + self.assertEqual(result.collected["metrics"]["asr_exact_matches"], 1) + self.assertEqual(result.collected["input_tts_model"], "qwen-audio-3.0-tts-flash") + self.assertEqual(result.collected["wake_stage"], "passed") + self.assertNotIn("你好", repr(result.collected)) + self.assertFalse(adapter.lease_held) + + self.assertEqual( + [assertion.name for assertion in result.assertions], + [ + "voice_input_tts_preflight_clean", + "voice_input_fixture_clean", + "voice_wake_sequence_clean", + "voice_turns_complete", + "voice_state_flow_clean", + "voice_display_flow_clean", + "voice_wake_guard_clean", + "voice_acceptance_clean", + ], + ) + + def test_voice_acceptance_contract_fails_closed_when_a_check_disappears(self) -> None: + incomplete = {key: True for key in HIL.HilVoiceAdapter.CONVERSATION_ACCEPTANCE_KEYS} + incomplete.pop("zero_loss") + self.assertFalse( + HIL.HilVoiceAdapter._acceptance_valid(incomplete, HIL.HilVoiceAdapter.CONVERSATION_ACCEPTANCE_KEYS) + ) + + def test_voice_script_exit_categories_are_stable(self) -> None: + self.assertEqual( + HIL.HilVoiceAdapter._classify_voice_exit(1, "").category, + RUNNER.FailureCategory.PRODUCT, + ) + self.assertEqual( + HIL.HilVoiceAdapter._classify_voice_exit(2, "cannot open serial port").category, + RUNNER.FailureCategory.DEVICE, + ) + self.assertEqual( + HIL.HilVoiceAdapter._classify_voice_exit(2, "input_preparation_failed:timeout").category, + RUNNER.FailureCategory.EXTERNAL, + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/python/test_e2e_hil_device.py b/tests/python/test_e2e_hil_device.py index f5d9df12..cfc22a53 100644 --- a/tests/python/test_e2e_hil_device.py +++ b/tests/python/test_e2e_hil_device.py @@ -6,6 +6,7 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "scripts")) @@ -69,6 +70,7 @@ def sparkbot_layout(self) -> list[object]: self.partition("linx_secrets", 1, 2, 0x2E0000, 0x10000), self.partition("assets", 1, 0x82, 0x300000, 0x100000), self.partition("model", 1, 0x82, 0x400000, 0x300000), + self.partition("voicelife", 1, 0x81, 0x700000, 0x900000), ] def pcb_layout(self) -> list[object]: @@ -119,6 +121,23 @@ def test_application_image_must_match_profile_offset_and_partition_size(self) -> with self.assertRaises(HIL.HilProfileMismatch): HIL.load_application_image(build, descriptor, self.sparkbot_layout()) + def test_application_image_rejects_build_layout_different_from_board(self) -> None: + descriptor = self.descriptor("sparkbot") + with tempfile.TemporaryDirectory() as directory: + build = Path(directory) + (build / "voicelife.bin").write_bytes(b"firmware") + (build / "flasher_args.json").write_text( + json.dumps({"flash_files": {"0x10000": "voicelife.bin"}}), encoding="utf-8" + ) + table = build / "partition_table" / "partition-table.bin" + table.parent.mkdir() + table.write_bytes(b"partition-table") + with ( + patch.object(HIL, "parse_partition_table", return_value=self.pcb_layout()), + self.assertRaises(HIL.HilProfileMismatch), + ): + HIL.load_application_image(build, descriptor, self.sparkbot_layout()) + def test_flash_plan_writes_and_verifies_only_the_validated_application(self) -> None: with tempfile.TemporaryDirectory() as directory: image_path = Path(directory) / "voicelife.bin" diff --git a/tests/python/test_e2e_runner.py b/tests/python/test_e2e_runner.py index 36bec39b..147addb6 100644 --- a/tests/python/test_e2e_runner.py +++ b/tests/python/test_e2e_runner.py @@ -115,6 +115,7 @@ def test_failure_categories_have_stable_exit_codes(self) -> None: self.assertEqual(RUNNER.exit_code_for(RUNNER.FailureCategory.INFRASTRUCTURE), 10) self.assertEqual(RUNNER.exit_code_for(RUNNER.FailureCategory.PRODUCT), 20) self.assertEqual(RUNNER.exit_code_for(RUNNER.FailureCategory.DEVICE), 30) + self.assertEqual(RUNNER.exit_code_for(RUNNER.FailureCategory.LEASE), 31) self.assertEqual(RUNNER.exit_code_for(RUNNER.FailureCategory.EXTERNAL), 40) self.assertEqual(RUNNER.exit_code_for(RUNNER.FailureCategory.TIMEOUT), 60) self.assertEqual(RUNNER.exit_code_for(RUNNER.FailureCategory.INTERRUPTED), 70) diff --git a/tests/python/test_firmware.py b/tests/python/test_firmware.py index 77cb4d56..ffc8e673 100644 --- a/tests/python/test_firmware.py +++ b/tests/python/test_firmware.py @@ -162,6 +162,17 @@ def test_im_pcb_profile_verifies_cross_signed_cloudflare_chain(self) -> None: self.assertIn("CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_CROSS_SIGNED_VERIFY=y", profile["sdkconfig"]) + def test_pcb_serial_voice_profile_keeps_im_and_enables_test_harness(self) -> None: + profile_path = ROOT / "config" / "profiles" / "esp32s3-voicelife-pcb-serial-voice.json" + profile = json.loads(profile_path.read_text(encoding="utf-8")) + + self.assertEqual(profile["adapters"]["im"]["driver"], "voicelife-gateway") + self.assertIn("CONFIG_VOICELIFE_SERIAL_VOICE_TEST=y", profile["sdkconfig"]) + self.assertIn( + 'CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="config/partitions/voicelife-pcb.csv"', profile["sdkconfig"] + ) + self.assertNotIn("CONFIG_VOICELIFE_BOARD_ESP_SPARKBOT=y", profile["sdkconfig"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/test_run_e2e.py b/tests/python/test_run_e2e.py index 7e6b5447..b59d303b 100644 --- a/tests/python/test_run_e2e.py +++ b/tests/python/test_run_e2e.py @@ -8,6 +8,7 @@ import tempfile import unittest from pathlib import Path +from unittest import mock ROOT = Path(__file__).resolve().parents[2] SCRIPTS = ROOT / "scripts" @@ -107,6 +108,23 @@ def remaining(self) -> float: self.assertEqual(raised.exception.category, RUNNER.FailureCategory.INFRASTRUCTURE) self.assertEqual(raised.exception.message_code, "host_recovery_journey_failed") + def test_recovery_details_stay_out_of_public_artifacts(self) -> None: + class Context: + run_id = "a" * 32 + temporary_directory = Path("/tmp/voicelife-e2e-test") + cleanup = mock.Mock() + + with tempfile.TemporaryDirectory() as directory, mock.patch.object(ADAPTERS.subprocess, "Popen") as popen: + artifact_directory = Path(directory) / "artifacts" + adapter = ADAPTERS.HostImGatewayRecoveryE2EAdapter(artifact_directory) + adapter.prepare(Context()) + + environment = popen.call_args.kwargs["env"] + detail_path = Path(environment["E2E_RECOVERY_EVIDENCE"]) + self.assertEqual(detail_path.parent.parent, Context.temporary_directory) + self.assertNotEqual(detail_path.parent, artifact_directory) + self.assertFalse(artifact_directory.exists()) + class RunE2eCliTest(unittest.TestCase): def run_cli(self, *arguments: str, env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]: @@ -153,6 +171,17 @@ def test_host_and_hil_cli_write_valid_sanitized_evidence(self) -> None: self.assertFalse(document["hardware_verified"]) self.assertNotIn(str(evidence_paths[0]), result.stdout) + def test_contract_failure_writes_failed_evidence(self) -> None: + with tempfile.TemporaryDirectory() as directory: + result = self.run_cli( + *self.base_args(Path(directory)), env={**os.environ, "VOICELIFE_E2E_CONTRACT_FAILURE": "1"} + ) + self.assertEqual(result.returncode, 20, result.stderr) + document = json.loads(next(Path(directory).glob("evidence-*.json")).read_text(encoding="utf-8")) + EVIDENCE.validate_evidence(document) + self.assertEqual(document["failure_category"], "product") + self.assertEqual(document["failed_phase"], "assert") + def test_cli_rejects_nonzero_retries_and_hil_host_profile(self) -> None: with tempfile.TemporaryDirectory() as directory: retry_args = self.base_args(Path(directory)) @@ -374,6 +403,43 @@ def test_build_adapter_registers_real_hil_without_host_fallback(self) -> None: with self.assertRaises(ValueError): RUN_E2E.build_adapter("host", "im-pairing", args) + def test_build_adapter_registers_voice_for_supported_profiles(self) -> None: + with tempfile.TemporaryDirectory() as directory: + descriptor = Path(directory) / "device.json" + for profile in ("sparkbot", "pcb"): + descriptor.write_text( + json.dumps({"schema_version": 1, "name": "bench-a", "port": "/dev/cu.test", "profile": profile}), + encoding="utf-8", + ) + args = RUN_E2E.parse_args( + [ + "--layer", + "hil", + "--journey", + "voice", + "--profile", + profile, + "--artifact-dir", + directory, + "--timeout", + "2", + "--device", + str(descriptor), + "--server", + "runner@example.test", + "--server-dir", + "/srv/voicelife", + "--gateway-origin", + "https://gateway.example.test", + "--user-id", + "user-test", + ] + ) + with self.subTest(profile=profile): + self.assertEqual( + type(RUN_E2E.build_adapter(args.layer, args.journey, args)).__name__, "HilVoiceAdapter" + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/python/test_voice_linx_serial_multiturn_test.py b/tests/python/test_voice_linx_serial_multiturn_test.py new file mode 100644 index 00000000..460710a9 --- /dev/null +++ b/tests/python/test_voice_linx_serial_multiturn_test.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import importlib.util +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "voice_linx_serial_multiturn_test.py" +sys.path.insert(0, str(ROOT / "scripts")) + + +def load() -> object: + spec = importlib.util.spec_from_file_location("voice_linx_serial_multiturn_test", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +MODULE = load() + + +class DisplayEvidenceTest(unittest.TestCase): + def test_sparkbot_requires_complete_text_trace_and_observes_scroll(self) -> None: + line = ( + "SPARKBOT_TEXT_RENDER generation=1 revision=2 content_height=16 viewport_height=120 " + "overflow_width=4 manual_line_breaks=0 status=说话 content=你好 content_visible=1" + ) + self.assertEqual(MODULE.display_evidence("sparkbot", [line], [], 0), (True, 1, True)) + + def test_pcb_requires_a_draw_for_every_turn_after_the_turn_starts(self) -> None: + results = [ + MODULE.TurnResult(index=1, input_text="", log_start=1, log_end=3), + MODULE.TurnResult(index=2, input_text="", log_start=3, log_end=5), + ] + lines = ["DISPLAY_DRAW=1 boot", "state", "DISPLAY_DRAW=1 first", "state", "DISPLAY_DRAW=1 second"] + self.assertEqual(MODULE.display_evidence("pcb", lines, results, 2), (True, 2, False)) + self.assertEqual(MODULE.display_evidence("pcb", lines, results, 3), (False, 2, False)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/python/test_voice_tts_fixture.py b/tests/python/test_voice_tts_fixture.py new file mode 100644 index 00000000..fadb640e --- /dev/null +++ b/tests/python/test_voice_tts_fixture.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import os +import sys +import types +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts")) + +import voice_linx_serial_multiturn_test as MULTITURN # noqa: E402 +import voice_tts_fixture as FIXTURE # noqa: E402 + + +class TtsFixtureTest(unittest.TestCase): + def test_dashscope_returns_sanitized_metrics(self) -> None: + synthesizer = mock.Mock() + synthesizer.call.return_value = b"encoded-audio" + synthesizer.get_first_package_delay.return_value = 123.4 + tts_v2 = types.SimpleNamespace(SpeechSynthesizer=mock.Mock(return_value=synthesizer)) + dashscope = types.SimpleNamespace(api_key=None) + + def import_module(name: str) -> object: + return tts_v2 if name == "dashscope.audio.tts_v2" else dashscope + + with ( + mock.patch.dict(os.environ, {"DASHSCOPE_API_KEY": "secret"}, clear=False), + mock.patch.object(FIXTURE.importlib, "import_module", side_effect=import_module), + ): + result = FIXTURE.synthesize_fixture("你好", "dashscope", "model-a", "voice-a") + self.assertEqual(result.audio, b"encoded-audio") + self.assertEqual(result.provider, "dashscope") + self.assertEqual(result.first_packet_ms, 123) + self.assertNotIn("secret", repr(result)) + + def test_aliyun_nls_collects_raw_pcm_without_ffmpeg(self) -> None: + constructor: dict[str, object] = {} + + class FakeSynthesizer: + def __init__(self, **kwargs: object) -> None: + constructor.update(kwargs) + self.on_data = kwargs["on_data"] + + def start(self, text: str, **kwargs: object) -> bool: + self.on_data(bytes(MULTITURN.PCM_FRAME_BYTES)) + return True + + module = types.SimpleNamespace(NlsSpeechSynthesizer=FakeSynthesizer) + with ( + mock.patch.dict( + os.environ, + {"ALIYUN_NLS_APPKEY": "app", "ALIYUN_NLS_TOKEN": "token"}, + clear=False, + ), + mock.patch.object(FIXTURE.importlib, "import_module", return_value=module), + mock.patch.object(MULTITURN.subprocess, "run") as ffmpeg, + ): + result = FIXTURE.synthesize_fixture("你好", "aliyun-nls", "nls-v1", "xiaoyun") + frames = MULTITURN.to_pcm_frames(result.audio, result.audio_format) + self.assertEqual(len(frames), 1) + self.assertEqual(constructor["url"], "wss://nls-gateway.cn-shanghai.aliyuncs.com/ws/v1") + ffmpeg.assert_not_called() + + def test_missing_provider_credentials_have_stable_codes(self) -> None: + with mock.patch.dict(os.environ, {}, clear=True), self.assertRaises(FIXTURE.TtsFixtureError) as raised: + FIXTURE.validate_provider_environment("aliyun-nls") + self.assertEqual(raised.exception.code, "aliyun_nls_credentials_missing") + + +if __name__ == "__main__": + unittest.main()