Add livekit-plugins-rnnoise (OSS self-hosted noise cancellation)

livekit-plugins/livekit-plugins-rnnoise/README.md

@@ -0,0 +1,61 @@
+# RNNoise Plugin for LiveKit Agents
+
+OSS, self-hosted noise cancellation for LiveKit Agents (RNNoise) — a free, fully-local alternative
+to the Cloud-gated Krisp plugin.
+
+## Features
+
+- **`RNNoise`**: A `FrameProcessor` that runs [RNNoise](https://github.com/xiph/rnnoise) noise
+  suppression on the agent's incoming audio, entirely on-device.
+
+## Installation
+
+```bash
+pip install livekit-plugins-rnnoise
+```
+
+No API key, license, or separate model download is required — the RNNoise model is bundled in the
+wheel via [`pyrnnoise`](https://pypi.org/project/pyrnnoise/).
+
+## Quick Start
+
+```python
+from livekit.agents import Agent, AgentServer, AgentSession, JobContext, cli, room_io
+from livekit.plugins import rnnoise
+
+
+server = AgentServer()
+
+
+@server.rtc_session()
+async def entrypoint(ctx: JobContext) -> None:
+    session = AgentSession(...)
+
+    await session.start(
+        agent=Agent(instructions="..."),
+        room=ctx.room,
+        room_options=room_io.RoomOptions(
+            audio_input=room_io.AudioInputOptions(
+                noise_cancellation=rnnoise.RNNoise(),
+            ),
+        ),
+    )
+
+
+if __name__ == "__main__":
+    cli.run_app(server)
+```
+
+**Audio Pipeline:** `Room → RoomIO (with RNNoise) → VAD → STT → LLM`
+
+See `examples/rnnoise_agent_example.py` for a complete, runnable agent.
+
+## Notes
+
+- **Mono only (v1):** `RNNoise` currently supports single-channel audio. A frame with more than one
+  channel raises `ValueError`.
+- **Latency:** RNNoise operates on fixed 48kHz/10ms frames internally, so incoming audio is resampled
+  in and out. Expect roughly 10-70ms of warm-up latency (padded with silence) before denoised audio
+  starts flowing, depending on the source sample rate.
+- **CPU cost:** small — a resample pass plus RNNoise inference per frame. No GPU required.
+- **License:** Apache-2.0.

livekit-plugins/livekit-plugins-rnnoise/examples/rnnoise_agent_example.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Example: Voice Agent with RNNoise Noise Cancellation
+
+This example demonstrates how to integrate RNNoise noise cancellation
+into a LiveKit voice agent for human-to-bot conversations. RNNoise runs
+entirely on-device -- no API key, license, or model download required.
+
+The audio pipeline:
+    Room → RoomIO (with RNNoise) → VAD → STT → LLM → TTS → Room
+
+Prerequisites:
+    Install required packages:
+       - livekit-agents
+       - livekit-plugins-rnnoise
+       - livekit-plugins-openai (or your preferred STT/LLM/TTS)
+
+Usage:
+    python rnnoise_agent_example.py dev
+"""
+
+import logging
+
+from dotenv import load_dotenv
+
+from livekit.agents import (
+    Agent,
+    AgentServer,
+    AgentSession,
+    JobContext,
+    cli,
+    inference,
+    room_io,
+)
+from livekit.plugins import openai, rnnoise
+
+logger = logging.getLogger("rnnoise-agent-example")
+load_dotenv()
+
+
+class RNNoiseAgent(Agent):
+    """Voice agent that uses RNNoise for noise cancellation."""
+
+    def __init__(self) -> None:
+        super().__init__(
+            instructions=(
+                "You are a helpful voice assistant. "
+                "Keep your responses concise and conversational. "
+                "Do not use emojis or special characters in your responses."
+            ),
+        )
+
+    async def on_enter(self) -> None:
+        """Called when the agent enters the session."""
+        logger.info("RNNoise agent entered session")
+        self.session.generate_reply(allow_interruptions=False)
+
+
+server = AgentServer()
+
+
+@server.rtc_session()
+async def entrypoint(ctx: JobContext) -> None:
+    """Main entrypoint for the agent session."""
+
+    session = AgentSession(
+        vad=inference.VAD(),
+        stt=openai.STT(model="whisper-1"),
+        llm=openai.LLM(model="gpt-4o-mini"),
+        tts=openai.TTS(voice="alloy"),
+        allow_interruptions=True,
+        min_endpointing_delay=0.5,
+        max_endpointing_delay=3.0,
+    )
+
+    logger.info("Starting agent session with RoomIO and RNNoise noise cancellation")
+
+    # Start the session with RoomIO configuration
+    await session.start(
+        agent=RNNoiseAgent(),
+        room=ctx.room,
+        room_options=room_io.RoomOptions(
+            audio_input=room_io.AudioInputOptions(
+                noise_cancellation=rnnoise.RNNoise(),
+            ),
+        ),
+    )
+
+    logger.info("RNNoise noise cancellation active")
+
+
+if __name__ == "__main__":
+    cli.run_app(server)

livekit-plugins/livekit-plugins-rnnoise/livekit/plugins/rnnoise/__init__.py

@@ -0,0 +1,31 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""OSS, self-hosted noise cancellation for LiveKit Agents, powered by RNNoise."""
+
+from livekit.agents import Plugin
+
+from .log import logger
+from .rnnoise_filter import RNNoise, RNNoiseFrameProcessor
+from .version import __version__
+
+__all__ = ["RNNoise", "RNNoiseFrameProcessor", "logger", "__version__"]
+
+
+class RNNoisePlugin(Plugin):
+    def __init__(self) -> None:
+        super().__init__(__name__, __version__, __package__, logger)
+
+
+Plugin.register_plugin(RNNoisePlugin())

livekit-plugins/livekit-plugins-rnnoise/livekit/plugins/rnnoise/_rnnoise_native.py

@@ -0,0 +1,128 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Thin ctypes binding to the RNNoise C library bundled inside pyrnnoise's
+site-packages directory.
+
+This deliberately never imports the `pyrnnoise` Python package -- only its
+package directory is located (via `importlib.util.find_spec`) so we can load
+the native `librnnoise` shared library it ships. Importing `pyrnnoise` itself
+pulls in `audiolab` (and transitively `matplotlib`) and forces an upper pin on
+`av`, none of which this plugin needs.
+"""
+
+from __future__ import annotations
+
+import ctypes
+import importlib.util
+from pathlib import Path
+
+import numpy as np
+
+_RNNOISE_FRAME_SAMPLES = 480
+_NATIVE_LIB_EXTENSIONS = (".dylib", ".so", ".dll")
+
+_lib: ctypes.CDLL | None = None
+
+
+def _find_bundled_lib() -> Path:
+    """Locate the RNNoise native library inside pyrnnoise's package directory."""
+    spec = importlib.util.find_spec("pyrnnoise")
+    if spec is None or not spec.submodule_search_locations:
+        raise RuntimeError(
+            "Could not locate the pyrnnoise package, which bundles the native "
+            "RNNoise library required by this plugin. Install it with "
+            "`pip install pyrnnoise`."
+        )
+    pkg_dir = Path(next(iter(spec.submodule_search_locations)))
+
+    candidates = sorted(
+        path
+        for ext in _NATIVE_LIB_EXTENSIONS
+        for path in pkg_dir.glob(f"*{ext}")
+        if "rnnoise" in path.name.lower()
+    )
+    if not candidates:
+        raise RuntimeError(
+            f"No bundled RNNoise native library found in {pkg_dir}. Install/reinstall "
+            "pyrnnoise with `pip install pyrnnoise` to provide it."
+        )
+    return candidates[0]
+
+
+def _load_lib() -> ctypes.CDLL:
+    global _lib
+    if _lib is not None:
+        return _lib
+
+    lib = ctypes.CDLL(str(_find_bundled_lib()))
+
+    lib.rnnoise_get_frame_size.argtypes = []
+    lib.rnnoise_get_frame_size.restype = ctypes.c_int
+
+    lib.rnnoise_create.argtypes = [ctypes.c_void_p]
+    lib.rnnoise_create.restype = ctypes.c_void_p
+
+    lib.rnnoise_process_frame.argtypes = [
+        ctypes.c_void_p,
+        ctypes.POINTER(ctypes.c_float),
+        ctypes.POINTER(ctypes.c_float),
+    ]
+    lib.rnnoise_process_frame.restype = ctypes.c_float
+
+    lib.rnnoise_destroy.argtypes = [ctypes.c_void_p]
+    lib.rnnoise_destroy.restype = None
+
+    frame_size = lib.rnnoise_get_frame_size()
+    if frame_size != _RNNOISE_FRAME_SAMPLES:
+        raise RuntimeError(
+            f"Unexpected RNNoise frame size {frame_size}, expected {_RNNOISE_FRAME_SAMPLES}"
+        )
+
+    _lib = lib
+    return lib
+
+
+class _RNNoiseDenoiser:
+    """Owns one native RNNoise DenoiseState and processes 480-sample frames."""
+
+    def __init__(self) -> None:
+        self._state: int | None = None
+        self._lib = _load_lib()
+        state = self._lib.rnnoise_create(None)  # NULL model -> baked-in default RNNModel
+        if not state:
+            raise RuntimeError("rnnoise_create(None) returned NULL")
+        self._state = state
+
+    def process_frame(self, frame_480_int16: np.ndarray) -> np.ndarray:
+        """Denoise exactly one 480-sample mono int16 frame, in place semantics aside."""
+        if self._state is None:
+            raise RuntimeError("process_frame() called after close()")
+        assert frame_480_int16.dtype == np.int16
+        assert len(frame_480_int16) == _RNNOISE_FRAME_SAMPLES
+
+        # RNNoise's C ABI operates on float32 samples in the same numeric range as
+        # int16 (i.e. not normalized to [-1, 1]).
+        buf = frame_480_int16.astype(np.float32)
+        ptr = buf.ctypes.data_as(ctypes.POINTER(ctypes.c_float))
+        self._lib.rnnoise_process_frame(self._state, ptr, ptr)
+        return buf.astype(np.int16)
+
+    def close(self) -> None:
+        if self._state is not None:
+            self._lib.rnnoise_destroy(self._state)
+            self._state = None
+
+    def __del__(self) -> None:
+        self.close()

livekit-plugins/livekit-plugins-rnnoise/livekit/plugins/rnnoise/log.py

@@ -0,0 +1,17 @@
+# Copyright 2023 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+logger = logging.getLogger("livekit.plugins.rnnoise")