Skip to content

Commit 44acff9

Browse files
Merge pull request #5 from openlayer-ai/vini/open-12792-azure-speech-redact-errors-audio-encoding
fix(OPEN-12792): redact credentials from Azure Speech errors; label audio by its real encoding
2 parents 44f048e + 4e41a62 commit 44acff9

2 files changed

Lines changed: 343 additions & 35 deletions

File tree

‎src/openlayer/lib/integrations/azure_speech_tracer.py‎

Lines changed: 187 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,13 @@
1414
import contextvars
1515
import logging
1616
import mimetypes
17+
import re
18+
import struct
1719
import time
1820
from functools import wraps
1921
from pathlib import Path
20-
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
22+
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, Union
23+
from urllib.parse import quote, unquote, urlsplit, urlunsplit, parse_qsl
2124

2225
try:
2326
import azure.cognitiveservices.speech as speechsdk
@@ -244,6 +247,7 @@ def make_step(
244247
inference_id: Optional[str],
245248
) -> None:
246249
config = get_model_parameters(client)
250+
secrets = _collect_secrets(client)
247251
inputs: Dict[str, Any] = {"language": config.get("language")}
248252
if is_translation:
249253
inputs["targetLanguages"] = list(getattr(client, "target_languages", None) or [])
@@ -268,8 +272,8 @@ def make_step(
268272
output=output,
269273
model=config.get("endpoint_id") or "speech-to-text",
270274
model_parameters=config,
271-
raw_output=getattr(result, "json", None) or None,
272-
metadata=get_result_metadata(result),
275+
raw_output=_redact_secrets(getattr(result, "json", None) or "", secrets) or None,
276+
metadata=get_result_metadata(result, secrets),
273277
id=inference_id,
274278
)
275279
)
@@ -296,11 +300,11 @@ def make_step(
296300
"audioSizeBytes": len(audio_data),
297301
}
298302
if audio_data and _audio_upload_enabled():
299-
attachment = Attachment.from_bytes(
300-
audio_data,
301-
name=f"synthesis.{_extension(config.get('output_format'))}",
302-
media_type=_synthesis_media_type(config.get("output_format")),
303+
audio_bytes, media_type, extension, audio_metadata = _describe_synthesis_audio(
304+
audio_data, config.get("output_format")
303305
)
306+
attachment = Attachment.from_bytes(audio_bytes, name=f"synthesis.{extension}", media_type=media_type)
307+
attachment.metadata.update(audio_metadata)
304308
output["audio"] = attachment
305309

306310
add_to_trace(
@@ -314,7 +318,7 @@ def make_step(
314318
model_parameters=config,
315319
# The result carries the audio bytes; never copy it into the trace.
316320
raw_output=None,
317-
metadata=get_result_metadata(result),
321+
metadata=get_result_metadata(result, _collect_secrets(client)),
318322
id=inference_id,
319323
)
320324
)
@@ -356,8 +360,12 @@ def get_model_parameters(client: Any) -> Dict[str, Any]:
356360
return parameters
357361

358362

359-
def get_result_metadata(result: Any) -> Dict[str, Any]:
360-
"""Extract the result ID, reason, timings and failure details."""
363+
def get_result_metadata(result: Any, secrets: Sequence[str] = ()) -> Dict[str, Any]:
364+
"""Extract the result ID, reason, timings and failure details.
365+
366+
``errorDetails`` is passed through ``_redact_secrets``: the Speech SDK can
367+
embed the endpoint URL (and any credential in its query string) in it.
368+
"""
361369
metadata: Dict[str, Any] = {}
362370

363371
result_id = getattr(result, "result_id", None)
@@ -388,7 +396,7 @@ def get_result_metadata(result: Any) -> Dict[str, Any]:
388396
metadata["cancellation"] = {
389397
"reason": _enum_name(getattr(cancellation, "reason", None)),
390398
"errorCode": _enum_name(code),
391-
"errorDetails": getattr(cancellation, "error_details", None),
399+
"errorDetails": _redact_secrets(getattr(cancellation, "error_details", None) or "", secrets) or None,
392400
}
393401

394402
return metadata
@@ -451,28 +459,174 @@ def _audio_input_attachment(audio: Any) -> Optional[Attachment]:
451459
return None
452460

453461

454-
def _synthesis_media_type(output_format: Optional[str]) -> str:
455-
"""Map a Speech output format (e.g. ``audio-24khz-48kbitrate-mono-mp3``) to a
456-
MIME type. The default format is RIFF (WAV)."""
457-
fmt = (output_format or "").lower()
458-
if "mp3" in fmt:
459-
return "audio/mpeg"
460-
if "ogg" in fmt:
461-
return "audio/ogg"
462-
if "webm" in fmt:
463-
return "audio/webm"
464-
if fmt.startswith("raw"):
465-
return "audio/pcm"
466-
return "audio/wav"
467-
468-
469-
def _extension(output_format: Optional[str]) -> str:
470-
return {
471-
"audio/mpeg": "mp3",
472-
"audio/ogg": "ogg",
473-
"audio/webm": "webm",
474-
"audio/pcm": "pcm",
475-
}.get(_synthesis_media_type(output_format), "wav")
462+
# The WAVE format tags for headerless output that can be wrapped in a WAV container.
463+
_WAVE_FORMAT_TAGS = {"pcm": 1, "alaw": 6, "mulaw": 7}
464+
465+
466+
def _sniff_container(data: bytes) -> Optional[tuple]:
467+
"""(media type, extension) from the audio's own header, if recognizable.
468+
469+
No bare MPEG frame-sync check: headerless PCM samples can start with 0xFFEx.
470+
"""
471+
if data[:4] == b"RIFF" and data[8:12] == b"WAVE":
472+
return "audio/wav", "wav"
473+
if data[:4] == b"OggS":
474+
return "audio/ogg", "ogg"
475+
if data[:4] == b"\x1a\x45\xdf\xa3":
476+
return "audio/webm", "webm"
477+
if data[:9] == b"#!AMR-WB\n":
478+
return "audio/amr-wb", "amr"
479+
if data[:3] == b"ID3":
480+
return "audio/mpeg", "mp3"
481+
return None
482+
483+
484+
def _parse_raw_format(normalized: str) -> Optional[tuple]:
485+
"""(sample rate Hz, bits per sample) from e.g. ``raw22050hz16bitmonopcm``."""
486+
khz = re.search(r"(\d+)khz", normalized)
487+
hz = re.search(r"(\d+)hz", normalized)
488+
bits = re.search(r"(\d+)bit", normalized)
489+
sample_rate = int(khz.group(1)) * 1000 if khz else int(hz.group(1)) if hz else None
490+
if sample_rate is None or bits is None:
491+
return None
492+
return sample_rate, int(bits.group(1))
493+
494+
495+
def _wrap_in_wav(samples: bytes, format_tag: int, sample_rate: int, bits_per_sample: int, channels: int = 1) -> bytes:
496+
"""Wrap headerless samples in a WAV (RIFF) container."""
497+
block_align = channels * bits_per_sample // 8
498+
header = b"RIFF" + struct.pack("<I", 36 + len(samples)) + b"WAVE"
499+
header += b"fmt " + struct.pack(
500+
"<IHHIIHH", 16, format_tag, channels, sample_rate, sample_rate * block_align, block_align, bits_per_sample
501+
)
502+
header += b"data" + struct.pack("<I", len(samples))
503+
return header + samples
504+
505+
506+
def _describe_synthesis_audio(data: bytes, output_format: Optional[str]) -> tuple:
507+
"""Type the synthesized audio accurately: ``(bytes, media_type, extension, metadata)``.
508+
509+
The container is detected from the audio's own header (WAV, Ogg, WebM,
510+
AMR-WB, ID3-tagged MP3); untagged MP3 is recognized by the format name.
511+
Headerless PCM, mu-law and A-law (``raw-*`` formats) are wrapped in a WAV
512+
container so they play, using the sample rate and bit depth in the format
513+
name. Anything else (raw Opus frames, TrueSilk, Siren, G.722, unknown) is
514+
labeled ``application/octet-stream`` rather than guessed.
515+
516+
``output_format`` is the Python SDK's kebab-case name
517+
(``raw-8khz-8bit-mono-mulaw``); the JS SDK's enum names normalize the same.
518+
"""
519+
normalized = (output_format or "").lower().replace("-", "").replace("_", "")
520+
metadata: Dict[str, Any] = {"outputFormat": output_format} if output_format else {}
521+
522+
container = _sniff_container(data)
523+
if container:
524+
return data, container[0], container[1], metadata
525+
if normalized.endswith("mp3"):
526+
# Azure's MP3 output is a bare frame stream (no ID3 tag).
527+
return data, "audio/mpeg", "mp3", metadata
528+
529+
if normalized.startswith("raw"):
530+
encoding = next((name for name in ("pcm", "mulaw", "alaw") if normalized.endswith(name)), None)
531+
raw = _parse_raw_format(normalized)
532+
if encoding and raw:
533+
sample_rate, bits = raw
534+
wav = _wrap_in_wav(data, _WAVE_FORMAT_TAGS[encoding], sample_rate, bits)
535+
metadata.update({"encoding": encoding, "sampleRateHz": sample_rate, "wrappedInWav": True})
536+
return wav, "audio/wav", "wav", metadata
537+
538+
return data, "application/octet-stream", "bin", metadata
539+
540+
541+
# ----------------------------- Redaction ----------------------------- #
542+
543+
_REDACTED = "[REDACTED]"
544+
_SECRET_PROPERTY_NAMES = (
545+
"SpeechServiceConnection_Key",
546+
"SpeechServiceAuthorization_Token",
547+
"SpeechServiceConnection_ProxyPassword",
548+
)
549+
_URL_PROPERTY_NAMES = ("SpeechServiceConnection_Endpoint", "SpeechServiceConnection_Host")
550+
# Values shorter than this are not scrubbed verbatim (too likely to collide with ordinary text).
551+
_MIN_SECRET_LENGTH = 6
552+
553+
_URL_PATTERN = re.compile(r"\b(?:wss?|https?)://[^\s'\"<>`]+", re.IGNORECASE)
554+
_BEARER_PATTERN = re.compile(r"\b(Bearer|Basic)\s+[A-Za-z0-9\-._~+/]+=*", re.IGNORECASE)
555+
_SENSITIVE_PAIR_PATTERN = re.compile(
556+
r"\b((?:ocp-apim-)?subscription[-_]?key|api[-_]?key|access[-_]?token|auth(?:orization)?|token|sig|signature"
557+
r"|password|secret)(\s*[=:]\s*)(?!Bearer\b|Basic\b|\[REDACTED\])[^\s&,;'\")\]}>]+",
558+
re.IGNORECASE,
559+
)
560+
561+
562+
def _collect_secrets(client: Any) -> List[str]:
563+
"""Secret values configured on a Speech client: the subscription key, auth
564+
token and proxy password, plus every query value and userinfo credential in
565+
its custom endpoint/host URL. Used only to scrub text; never recorded."""
566+
properties = getattr(client, "properties", None)
567+
if properties is None:
568+
return []
569+
570+
def read(name: str) -> str:
571+
try:
572+
return properties.get_property(getattr(speechsdk.PropertyId, name)) or ""
573+
# pylint: disable=broad-except
574+
except Exception:
575+
return ""
576+
577+
secrets = [read(name) for name in _SECRET_PROPERTY_NAMES]
578+
for name in _URL_PROPERTY_NAMES:
579+
raw = read(name)
580+
if not raw:
581+
continue
582+
try:
583+
parts = urlsplit(raw)
584+
secrets.extend(value for _, value in parse_qsl(parts.query, keep_blank_values=True))
585+
secrets.extend(unquote(part) for part in (parts.username or "", parts.password or ""))
586+
except ValueError:
587+
continue
588+
return sorted({secret for secret in secrets if len(secret) >= _MIN_SECRET_LENGTH}, key=len, reverse=True)
589+
590+
591+
def _redact_url(match: "re.Match[str]") -> str:
592+
url = match.group(0)
593+
try:
594+
parts = urlsplit(url)
595+
netloc = parts.netloc
596+
if "@" in netloc:
597+
userinfo, host = netloc.rsplit("@", 1)
598+
netloc = f"{_REDACTED}:{_REDACTED}@{host}" if ":" in userinfo else f"{_REDACTED}@{host}"
599+
keys = list(dict.fromkeys(key for key, _ in parse_qsl(parts.query, keep_blank_values=True)))
600+
query = "&".join(f"{quote(key)}={_REDACTED}" for key in keys)
601+
return urlunsplit((parts.scheme, netloc, parts.path, query, ""))
602+
except ValueError:
603+
cut = min((i for i in (url.find("?"), url.find("#")) if i != -1), default=-1)
604+
return url if cut == -1 else f"{url[:cut]}?{_REDACTED}"
605+
606+
607+
def _redact_secrets(text: Optional[str], secrets: Sequence[str] = ()) -> str:
608+
"""Remove credentials from free text the Speech SDK produces (error details).
609+
610+
The SDK can embed the full endpoint URL in connection errors, so a custom
611+
endpoint carrying a token or signature in its query string would otherwise
612+
be published. URLs keep scheme, host, path and parameter names (values and
613+
userinfo are replaced); ``Bearer``/``Basic`` tokens and ``key=``/``token=``/
614+
``sig=``-style pairs are replaced; and every value in ``secrets`` (see
615+
``_collect_secrets``) is replaced wherever it appears, raw or URL-encoded.
616+
"""
617+
if not text:
618+
return ""
619+
result = _URL_PATTERN.sub(_redact_url, text)
620+
result = _BEARER_PATTERN.sub(lambda m: f"{m.group(1)} {_REDACTED}", result)
621+
result = _SENSITIVE_PAIR_PATTERN.sub(lambda m: f"{m.group(1)}{m.group(2)}{_REDACTED}", result)
622+
for secret in sorted(secrets, key=len, reverse=True):
623+
if len(secret) < _MIN_SECRET_LENGTH:
624+
continue
625+
result = result.replace(secret, _REDACTED)
626+
encoded = quote(secret, safe="")
627+
if encoded != secret:
628+
result = result.replace(encoded, _REDACTED)
629+
return result
476630

477631

478632
# ----------------------------- Trace ----------------------------- #

0 commit comments

Comments
 (0)