From 5a2998cacd6ff5948b7a5b97183711c32ab456a4 Mon Sep 17 00:00:00 2001 From: Greg Holmes Date: Tue, 18 Aug 2026 10:23:37 +0100 Subject: [PATCH] fix(core): keep stdout pure when -o json/yaml/csv is requested MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #98. Commands print status lines and human tables for people, while the framework writes the serialised result to stdout. With a machine-readable format those two collided on the same stream, so the JSON arrived after the human output: $ dg -o json projects | jq . Fetching projects... <- stdout Found 1 project(s): <- stdout { ... } <- stdout, too late for jq The payload was never missing, just buried — in `dg -o json models` the JSON started 37KB into stdout, behind a Rich table. `listen` and `speak` were unaffected because they gate their own display on the format; the other eight commands did not. Status routing previously keyed off `agentic` (a TTY/env heuristic) rather than "does something else own stdout", so an explicit `-o json` on a terminal still sent status to stdout. - Add `StatusConsole`, which resolves its target at write time: stderr when the active format is in MACHINE_FORMATS (json/yaml/csv), stdout otherwise. The shared `console` becomes one, so print_info/success/warning/panel/separator follow automatically. - Add `stdout_console` for the payload and point `print_output` at it, so the result is never diverted along with the status text. - Add `get_status_console()` and use it in read, models, projects, keys, billing, usage, requests and members in place of a private `Console()`. `table` is deliberately not a machine format: it is a human rendering and keeps stdout. Deliberately avoided redirecting sys.stdout around handle(): `dg speak` streams audio via `sys.stdout.buffer.write()` and branches on `sys.stdout.isatty()`, so a blanket redirect would corrupt piped audio. Verified: stdout is parseable JSON for all eight commands (plus valid YAML/CSV); `dg speak | ffplay` still receives a RIFF/WAV stream with no JSON mixed in; `dg -o json listen | jq` unchanged; human TTY output byte-for-byte as before, with status now on stderr under -o json. 1059 unit tests pass (7 new regression tests for the routing) and the 68-check live API smoke suite is green. --- .../src/deepctl_cmd_billing/command.py | 5 +- .../src/deepctl_cmd_keys/command.py | 5 +- .../src/deepctl_cmd_members/command.py | 5 +- .../src/deepctl_cmd_models/command.py | 5 +- .../src/deepctl_cmd_projects/command.py | 5 +- .../src/deepctl_cmd_read/command.py | 5 +- .../src/deepctl_cmd_requests/command.py | 5 +- .../src/deepctl_cmd_usage/command.py | 5 +- .../deepctl-core/src/deepctl_core/output.py | 61 ++++++++++++++++--- .../deepctl-core/tests/unit/test_output.py | 57 +++++++++++++++-- 10 files changed, 127 insertions(+), 31 deletions(-) diff --git a/packages/deepctl-cmd-billing/src/deepctl_cmd_billing/command.py b/packages/deepctl-cmd-billing/src/deepctl_cmd_billing/command.py index 6d8b8b8..2d7bef2 100644 --- a/packages/deepctl-cmd-billing/src/deepctl_cmd_billing/command.py +++ b/packages/deepctl-cmd-billing/src/deepctl_cmd_billing/command.py @@ -11,12 +11,13 @@ Config, DeepgramClient, ) -from rich.console import Console +from deepctl_core.output import get_status_console from rich.table import Table from .models import BalanceInfo, BillingResult -console = Console() +# Human/status output — routed to stderr when -o json/yaml/csv owns stdout +console = get_status_console() class BillingCommand(BaseCommand): diff --git a/packages/deepctl-cmd-keys/src/deepctl_cmd_keys/command.py b/packages/deepctl-cmd-keys/src/deepctl_cmd_keys/command.py index fee0e50..1afbf62 100644 --- a/packages/deepctl-cmd-keys/src/deepctl_cmd_keys/command.py +++ b/packages/deepctl-cmd-keys/src/deepctl_cmd_keys/command.py @@ -11,12 +11,13 @@ Config, DeepgramClient, ) -from rich.console import Console +from deepctl_core.output import get_status_console from rich.table import Table from .models import KeyCreatedInfo, KeyInfo, KeysResult -console = Console() +# Human/status output — routed to stderr when -o json/yaml/csv owns stdout +console = get_status_console() class KeysCommand(BaseCommand): diff --git a/packages/deepctl-cmd-members/src/deepctl_cmd_members/command.py b/packages/deepctl-cmd-members/src/deepctl_cmd_members/command.py index 643c4f2..221f8bc 100644 --- a/packages/deepctl-cmd-members/src/deepctl_cmd_members/command.py +++ b/packages/deepctl-cmd-members/src/deepctl_cmd_members/command.py @@ -11,12 +11,13 @@ Config, DeepgramClient, ) -from rich.console import Console +from deepctl_core.output import get_status_console from rich.table import Table from .models import InviteInfo, MemberInfo, MembersResult -console = Console() +# Human/status output — routed to stderr when -o json/yaml/csv owns stdout +console = get_status_console() class MembersCommand(BaseCommand): diff --git a/packages/deepctl-cmd-models/src/deepctl_cmd_models/command.py b/packages/deepctl-cmd-models/src/deepctl_cmd_models/command.py index d400dd7..34903bb 100644 --- a/packages/deepctl-cmd-models/src/deepctl_cmd_models/command.py +++ b/packages/deepctl-cmd-models/src/deepctl_cmd_models/command.py @@ -11,12 +11,13 @@ Config, DeepgramClient, ) -from rich.console import Console +from deepctl_core.output import get_status_console from rich.table import Table from .models import ModelInfo, ModelsResult -console = Console() +# Human/status output — routed to stderr when -o json/yaml/csv owns stdout +console = get_status_console() class ModelsCommand(BaseCommand): diff --git a/packages/deepctl-cmd-projects/src/deepctl_cmd_projects/command.py b/packages/deepctl-cmd-projects/src/deepctl_cmd_projects/command.py index 71ccaed..7bbbf7e 100644 --- a/packages/deepctl-cmd-projects/src/deepctl_cmd_projects/command.py +++ b/packages/deepctl-cmd-projects/src/deepctl_cmd_projects/command.py @@ -9,11 +9,12 @@ Config, DeepgramClient, ) -from rich.console import Console +from deepctl_core.output import get_status_console from .models import ProjectInfo, ProjectsResult -console = Console() +# Human/status output — routed to stderr when -o json/yaml/csv owns stdout +console = get_status_console() class ProjectsCommand(BaseCommand): diff --git a/packages/deepctl-cmd-read/src/deepctl_cmd_read/command.py b/packages/deepctl-cmd-read/src/deepctl_cmd_read/command.py index 428d9c3..44f78da 100644 --- a/packages/deepctl-cmd-read/src/deepctl_cmd_read/command.py +++ b/packages/deepctl-cmd-read/src/deepctl_cmd_read/command.py @@ -13,11 +13,12 @@ Config, DeepgramClient, ) -from rich.console import Console +from deepctl_core.output import get_status_console from .models import ReadResult -console = Console() +# Human/status output — routed to stderr when -o json/yaml/csv owns stdout +console = get_status_console() class ReadCommand(BaseCommand): diff --git a/packages/deepctl-cmd-requests/src/deepctl_cmd_requests/command.py b/packages/deepctl-cmd-requests/src/deepctl_cmd_requests/command.py index 1d207a6..547bc73 100644 --- a/packages/deepctl-cmd-requests/src/deepctl_cmd_requests/command.py +++ b/packages/deepctl-cmd-requests/src/deepctl_cmd_requests/command.py @@ -12,12 +12,13 @@ Config, DeepgramClient, ) -from rich.console import Console +from deepctl_core.output import get_status_console from rich.table import Table from .models import RequestInfo, RequestsResult -console = Console() +# Human/status output — routed to stderr when -o json/yaml/csv owns stdout +console = get_status_console() class RequestsCommand(BaseCommand): diff --git a/packages/deepctl-cmd-usage/src/deepctl_cmd_usage/command.py b/packages/deepctl-cmd-usage/src/deepctl_cmd_usage/command.py index b02dea5..ceeed75 100644 --- a/packages/deepctl-cmd-usage/src/deepctl_cmd_usage/command.py +++ b/packages/deepctl-cmd-usage/src/deepctl_cmd_usage/command.py @@ -10,12 +10,13 @@ Config, DeepgramClient, ) +from deepctl_core.output import get_status_console from deepctl_shared_utils import validate_date_format -from rich.console import Console from .models import UsageBucket, UsageResult -console = Console() +# Human/status output — routed to stderr when -o json/yaml/csv owns stdout +console = get_status_console() class UsageCommand(BaseCommand): diff --git a/packages/deepctl-core/src/deepctl_core/output.py b/packages/deepctl-core/src/deepctl_core/output.py index 685cb72..77bcc41 100644 --- a/packages/deepctl-core/src/deepctl_core/output.py +++ b/packages/deepctl-core/src/deepctl_core/output.py @@ -70,10 +70,6 @@ def is_agentic() -> bool: _agentic = is_agentic() -# Global console instance — plain when agentic, rich when interactive -console = Console(no_color=_agentic, highlight=not _agentic) -stderr_console = Console(stderr=True, no_color=_agentic, highlight=not _agentic) - # Global output configuration _output_config: dict[str, Any] = { "format": "default", @@ -84,6 +80,51 @@ def is_agentic() -> bool: } +# Formats where stdout carries a machine-readable payload and must not be +# polluted by human-facing status text, tables or progress. +MACHINE_FORMATS = frozenset({"json", "yaml", "csv"}) + + +class StatusConsole(Console): + """Console for human-facing output that yields stdout to the payload. + + Commands print status lines and tables for humans, while the framework + writes the serialised result to stdout. When a machine-readable format is + requested those two collide: `dg -o json projects | jq` sees + "Fetching projects..." before the JSON and fails. Resolving the target at + write time (rather than at construction) lets status output move to stderr + for exactly those formats, without commands having to ask. + """ + + @property + def file(self) -> Any: + if _output_config["format"] in MACHINE_FORMATS: + return sys.stderr + return self._file or sys.stdout + + @file.setter + def file(self, new_file: Any) -> None: + self._file = new_file + + +# Global console instance — plain when agentic, rich when interactive. +# `console` is the human/status channel: it steps aside to stderr whenever a +# machine-readable format owns stdout. `stdout_console` is the payload channel +# and always writes to stdout. +console = StatusConsole(no_color=_agentic, highlight=not _agentic) +stdout_console = Console(no_color=_agentic, highlight=not _agentic) +stderr_console = Console(stderr=True, no_color=_agentic, highlight=not _agentic) + + +def get_status_console() -> Console: + """Get the shared human/status console. + + Prefer this over a module-level ``Console()`` so status output is routed to + stderr automatically when ``-o json``/``yaml``/``csv`` owns stdout. + """ + return console + + def setup_output( format_type: str = "json", quiet: bool = False, verbose: bool = False ) -> None: @@ -309,23 +350,23 @@ def print_output(data: Any, format_type: str | None = None) -> None: if isinstance(data, str): try: parsed = json.loads(data) - console.print(JSON.from_data(parsed)) + stdout_console.print(JSON.from_data(parsed)) except json.JSONDecodeError: - console.print(data) + stdout_console.print(data) else: - console.print(JSON.from_data(data)) + stdout_console.print(JSON.from_data(data)) elif format_type == "yaml": # Use Rich's syntax highlighting for YAML yaml_str = formatter.format(data) syntax = Syntax(yaml_str, "yaml", theme="monokai", line_numbers=False) - console.print(syntax) + stdout_console.print(syntax) elif format_type == "table": # Table is already formatted for Rich formatted = formatter.format(data) - console.print(formatted, end="") + stdout_console.print(formatted, end="") else: # CSV and other formats - console.print(formatter.format(data)) + stdout_console.print(formatter.format(data)) def print_success(message: str) -> None: diff --git a/packages/deepctl-core/tests/unit/test_output.py b/packages/deepctl-core/tests/unit/test_output.py index ef9d45d..62d5426 100644 --- a/packages/deepctl-core/tests/unit/test_output.py +++ b/packages/deepctl-core/tests/unit/test_output.py @@ -1,13 +1,16 @@ """Tests for the output utilities.""" import json +import sys from unittest.mock import MagicMock, Mock, patch import pytest import yaml from deepctl_core.output import ( + MACHINE_FORMATS, OutputFormatter, get_console, + get_status_console, is_agentic, print_error, print_info, @@ -387,7 +390,7 @@ def test_print_info(self, mock_console): class TestPrintOutput: """Test print_output function.""" - @patch("deepctl_core.output.console") + @patch("deepctl_core.output.stdout_console") @patch( "deepctl_core.output._output_config", {"quiet": False, "format": "json"}, @@ -400,7 +403,7 @@ def test_print_output_json_dict(self, mock_console): # Should use Rich's JSON display mock_console.print.assert_called_once() - @patch("deepctl_core.output.console") + @patch("deepctl_core.output.stdout_console") @patch( "deepctl_core.output._output_config", {"quiet": True, "format": "json"} ) @@ -411,7 +414,7 @@ def test_print_output_quiet(self, mock_console): # Should not print in quiet mode mock_console.print.assert_not_called() - @patch("deepctl_core.output.console") + @patch("deepctl_core.output.stdout_console") @patch( "deepctl_core.output._output_config", {"quiet": False, "format": "yaml"}, @@ -424,7 +427,7 @@ def test_print_output_yaml(self, mock_console): # Should use syntax highlighting mock_console.print.assert_called_once() - @patch("deepctl_core.output.console") + @patch("deepctl_core.output.stdout_console") @patch( "deepctl_core.output._output_config", {"quiet": False, "format": "table"}, @@ -437,7 +440,7 @@ def test_print_output_table(self, mock_console): # Table formatting uses capture which results in multiple print calls assert mock_console.print.call_count >= 1 - @patch("deepctl_core.output.console") + @patch("deepctl_core.output.stdout_console") @patch( "deepctl_core.output._output_config", {"quiet": False, "format": "csv"} ) @@ -460,3 +463,47 @@ def test_get_console(self): # Should return Rich Console instance assert console is not None assert hasattr(console, "print") + + +class TestStatusConsoleRouting: + """Status output must yield stdout to machine-readable payloads. + + Regression tests for the `-o json` pollution bug: commands printed status + lines and tables to stdout, so `dg -o json projects | jq` received + "Fetching projects..." ahead of the JSON and failed to parse. + """ + + @pytest.mark.parametrize("fmt", sorted(MACHINE_FORMATS)) + def test_status_console_writes_to_stderr_for_machine_formats(self, fmt): + with patch( + "deepctl_core.output._output_config", + {"format": fmt, "quiet": False, "agentic": False}, + ): + assert get_status_console().file is sys.stderr + + @pytest.mark.parametrize("fmt", ["default", "table"]) + def test_status_console_writes_to_stdout_for_human_formats(self, fmt): + with patch( + "deepctl_core.output._output_config", + {"format": fmt, "quiet": False, "agentic": False}, + ): + assert get_status_console().file is sys.stdout + + def test_machine_formats_membership(self): + # `table` is a human rendering and must keep stdout + assert "table" not in MACHINE_FORMATS + assert "default" not in MACHINE_FORMATS + assert {"json", "yaml", "csv"} == set(MACHINE_FORMATS) + + def test_payload_console_is_not_the_status_console(self): + """The payload must never be diverted along with status output.""" + from deepctl_core.output import stdout_console + + assert stdout_console is not get_status_console() + with patch( + "deepctl_core.output._output_config", + {"format": "json", "quiet": False, "agentic": False}, + ): + # status steps aside, payload keeps stdout + assert get_status_console().file is sys.stderr + assert stdout_console.file is sys.stdout