diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 98475fd..b8743f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,6 +17,8 @@ jobs: outputs: release_created: ${{ steps.release.outputs.release_created }} tag_name: ${{ steps.release.outputs.tag_name }} + prs_created: ${{ steps.release.outputs.prs_created }} + pr: ${{ steps.release.outputs.pr }} steps: - uses: googleapis/release-please-action@16a9c90856f42705d54a6fda1823352bdc62cf38 # v4 id: release @@ -25,6 +27,64 @@ jobs: config-file: .github/release-please-config.json manifest-file: .github/.release-please-manifest.json + sync-release-pr: + name: Sync lockfile and floors on the release PR + needs: release-please + if: ${{ needs.release-please.outputs.prs_created == 'true' && needs.release-please.outputs.pr }} + runs-on: ubuntu-latest + # This job pushes to the release branch. Two commits landing on main in + # quick succession would otherwise run two of these against the same + # branch and the loser fails on a non-fast-forward push, reddening the + # release workflow for no real reason. Serialize instead of cancelling: + # the second run still has work to do once the first one's commit lands. + concurrency: + group: sync-release-pr + cancel-in-progress: false + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ fromJSON(needs.release-please.outputs.pr).headBranchName }} + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + + # release-please bumps every pyproject version but updates neither + # uv.lock (CI's `uv sync --locked` then fails all jobs at the install + # step -- the 0.2.28/0.3.0 wall, twice) nor the root dependency floors + # (without which `dg update` on pip delivers the wrapper and skips the + # sub-packages the changelog advertises). Both are mechanical + # consequences of the version bumps, so regenerate them here. + # + # Caveat: this push uses GITHUB_TOKEN, which does not retrigger the + # PR's checks. The routine hand-edit of the release notes retriggers + # them; for an untouched release PR, re-run checks from the UI. If + # releases ever need to go out unattended, switch this push to a + # dedicated PAT or GitHub App token. + - name: Regenerate uv.lock and pin root floors + run: | + set -euo pipefail + python3 scripts/check_dependency_floors.py --fix + uv lock + python3 scripts/check_dependency_floors.py + + - name: Commit and push if changed + run: | + set -euo pipefail + if git diff --quiet; then + echo "uv.lock and floors already in sync" + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add uv.lock pyproject.toml + git commit -m "chore: sync uv.lock and root dependency floors with release-please bumps" + git push + build: name: Build packages needs: release-please @@ -77,9 +137,70 @@ jobs: pip install --find-links "$DIST_DIR" dist/deepctl-*.whl deepctl --version + verify-published: + name: Verify release is installable from PyPI + needs: [release-please, publish] + # `pip install deepctl==X` must resolve the full dependency closure from + # PyPI before anything advertises the release. Two real failure modes: + # 0.2.27 published partially (root uninstallable, and pip *silently + # backtracks* to the previous version, exit 0), and the 0.3.0 rollout + # showed a fresh install backtracking to 0.2.26 during the CDN + # propagation window. Poll the actual resolver, not just the JSON + # endpoint, so mark-latest / deploy-web / brew never point at a version + # a user cannot install. + if: | + needs.release-please.outputs.release_created == 'true' && + startsWith(needs.release-please.outputs.tag_name, 'v') + runs-on: ubuntu-latest + steps: + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + # Poll with --dry-run: it stops after resolution, which is cheap and is + # exactly what distinguishes "not propagated yet" from "broken". + - name: Wait until pip can resolve the full release + run: | + set -euo pipefail + VERSION="${TAG_NAME#v}" + python -m venv /tmp/verify + /tmp/verify/bin/pip install --quiet --upgrade pip + # The loop exits the moment resolution succeeds, so the ceiling only + # costs anything when it is actually hit. Propagation is normally + # seconds; 20 minutes is insurance, not an expected wait. + for i in $(seq 1 60); do + if /tmp/verify/bin/pip install --dry-run --no-cache-dir \ + "deepctl==${VERSION}" >/dev/null 2>&1; then + echo "deepctl==${VERSION} resolves with its full closure" + exit 0 + fi + echo "Waiting for deepctl==${VERSION} to resolve (${i}/60)..." + sleep 20 + done + echo "::error::deepctl==${VERSION} did not resolve from PyPI after 20 minutes -- a dependency is missing or the index has not propagated. Re-run this job once PyPI has propagated; mark-latest, deploy-web and bump-brew-formula re-evaluate and will then run." + exit 1 + env: + TAG_NAME: ${{ needs.release-please.outputs.tag_name }} + + # Resolving is not installing. --dry-run is satisfied by PyPI metadata + # (often via PEP 658, without fetching a single wheel), so a corrupt + # artifact or an entry point that cannot import still passes it. The + # build job smoke-tests `deepctl --version`, but against the local + # dist/ artifacts -- this is the only check against what PyPI serves, + # and it is the last gate before three jobs advertise the release. + - name: Install for real and run the CLI + run: | + set -euo pipefail + VERSION="${TAG_NAME#v}" + /tmp/verify/bin/pip install --quiet --no-cache-dir "deepctl==${VERSION}" + /tmp/verify/bin/deepctl --version + env: + TAG_NAME: ${{ needs.release-please.outputs.tag_name }} + mark-latest: name: Mark root release as latest - needs: [release-please, publish] + needs: [release-please, publish, verify-published] # Re-assert latest on the root package tag (vX.Y.Z) after PyPI publish so # users clicking "latest" land on a tag whose artifact is actually # installable. Also re-asserts after all sub-package releases since @@ -97,7 +218,7 @@ jobs: deploy-web: name: Deploy web to production - needs: [release-please, publish] + needs: [release-please, publish, verify-published] # Only fire on root-package releases (v0.2.4, v1.0.0, …) and only after # PyPI publish so cli.deepgram.com never advertises a version that isn't # installable yet. Sub-package tags look like deepctl-cmd-listen-v0.0.3 — @@ -128,7 +249,7 @@ jobs: bump-brew-formula: name: Bump Homebrew formula - needs: [release-please, publish] + needs: [release-please, publish, verify-published] # Only fire on root-package releases (v0.2.4, v1.0.0, …). # Sub-package tags look like deepctl-cmd-listen-v0.0.3 — skip those. if: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 78b465b..746ff55 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,14 +56,31 @@ jobs: run: uv sync --group testing --locked - name: Check formatting - run: uv run ruff format --check src/ packages/*/src + run: uv run ruff format --check src/ packages/*/src scripts/ - name: Lint - run: uv run ruff check src/ packages/*/src + run: uv run ruff check src/ packages/*/src scripts/ - name: Type check run: uv run mypy src/ packages/*/src + floors: + name: Dependency floors + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + # Root floors must equal workspace versions or `pip install --upgrade + # deepctl` (what `dg update` runs) silently skips the packages whose + # fixes the release advertises. See scripts/check_dependency_floors.py. + - name: Check dependency floors + run: make floors-check + build-test: name: Test Build Process runs-on: ubuntu-latest @@ -86,3 +103,31 @@ jobs: - name: Verify packages run: make verify-packages + + # Single rollup so branch protection needs exactly one required check. + # Requiring the matrix contexts directly means editing repo settings every + # time a Python version or OS is added or dropped, and a required context + # that stops reporting blocks every merge until someone notices. + # + # `if: always()` is load-bearing: without it this job is *skipped* when a + # prerequisite fails, and a skipped required check never reports failure -- + # it just stalls. Run always, then fail on anything that is not success, so + # cancelled and skipped prerequisites are failures here too. + all-checks: + name: All checks + needs: [test, lint, floors, build-test] + if: always() + runs-on: ubuntu-latest + steps: + - name: Report prerequisite results + run: | + echo "test: ${{ needs.test.result }}" + echo "lint: ${{ needs.lint.result }}" + echo "floors: ${{ needs.floors.result }}" + echo "build-test: ${{ needs.build-test.result }}" + + - name: Fail unless every prerequisite succeeded + if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }} + run: | + echo "::error::One or more required jobs did not succeed" + exit 1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b625b65..492c985 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,6 +64,34 @@ Or manually create a package under `packages/` following the existing pattern. E Then run `make readmes` to update all READMEs. +The root `pyproject.toml` dependency is not optional bookkeeping — it is the +delivery manifest. `pip install --upgrade deepctl` (what `dg update` runs) +only installs what root depends on, so a published package missing from that +list never reaches anyone. `make floors-check` fails on both that omission and +a floor left below the workspace version; run `make floors-fix` to pin floors. + +### Releasing + +Releases are driven by release-please. Two things are worth knowing before you +run one: + +**Land your release-notes edits as a commit, not a PR description edit.** +`sync-release-pr` regenerates `uv.lock` and the root dependency floors on the +release branch automatically, but it pushes with `GITHUB_TOKEN`, which by +design retriggers nothing — so the PR's checks still reflect the bot's first +commit and stay red on content that is now correct. Editing `CHANGELOG.md` on +the branch is a commit and retriggers them. Editing the PR *description* does +not: that fires `pull_request: edited`, which is outside the default trigger +types. If a release ever needs to go out unattended, switch that push to a +dedicated PAT or GitHub App token. + +**Nothing advertises a release until it is installable.** `verify-published` +polls PyPI until `pip install deepctl==X` resolves its full closure, then +installs it for real and runs the CLI. `mark-latest`, `deploy-web`, and the +Homebrew bump all wait on it. If it times out, PyPI has not propagated or a +sub-package failed to publish — re-run that one job once PyPI has caught up +and the three downstream jobs re-evaluate. + ### Testing - Unit tests: `packages/*/tests/unit/` diff --git a/Makefile b/Makefile index 54e1dba..0073dd2 100644 --- a/Makefile +++ b/Makefile @@ -1,201 +1,207 @@ -# =================================================================== -# deepctl Makefile - Development Tools -# =================================================================== -# -# Quick Start: -# make dev-setup # First time setup -# make dev # Daily development (format + lint + test) -# make help # Show organized help -# -# For new contributors: see README.md -# =================================================================== - -.PHONY: help -.DEFAULT_GOAL := help - -# =================================================================== -# HELP & INFO -# =================================================================== - -help: ## Show this help message - @echo "🔧 deepctl - Deepgram CLI Development Tools" - @echo "" - @echo "Usage: make [target]" - @echo "" - @echo "🚀 \033[1mQuick Start:\033[0m" - @echo " \033[36mdev-setup\033[0m Set up development environment" - @echo " \033[36mdev\033[0m Format, lint, and test (full dev cycle)" - @echo " \033[36mtest\033[0m Run tests" - @echo "" - @echo "🧪 \033[1mTesting:\033[0m" - @echo " \033[36mtest\033[0m Run tests (development)" - @echo " \033[36mcheck\033[0m Quick quality check (no tests)" - @echo "" - @echo "🔧 \033[1mCode Quality:\033[0m" - @echo " \033[36mformat\033[0m Auto-format code" - @echo " \033[36mlint\033[0m Run all linters" - @echo " \033[36mtypecheck\033[0m Run mypy type checker" - @echo "" - @echo "🧹 \033[1mUtilities:\033[0m" - @echo " \033[36mclean\033[0m Clean build artifacts" - @echo " \033[36minfo\033[0m Show project information" - @echo " \033[36mhelp-all\033[0m Show all available targets" - @echo "" - @echo "For more targets, run: \033[36mmake help-all\033[0m" - -help-all: ## Show all available targets - @echo "🔧 deepctl - All Available Targets" - @echo "" - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | grep -v '^\.' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-25s\033[0m %s\n", $$1, $$2}' - -info: ## Show project information - @echo "🔧 deepctl - Deepgram CLI" - @echo "📁 $(shell pwd)" - @echo "🐍 Python: $(shell python --version 2>/dev/null || echo 'Not found')" - @echo "📦 uv: $(shell uv --version 2>/dev/null || echo 'Not found')" - @echo "🎯 Virtual env: $(shell echo $$VIRTUAL_ENV || echo 'Not activated')" - -# =================================================================== -# DEVELOPMENT SETUP -# =================================================================== - -dev-setup: ## Set up complete development environment - uv venv - uv pip install -e ".[dev]" - @echo "✅ Development environment ready!" - @echo "Activate with: source .venv/bin/activate (Linux/macOS) or .venv\\Scripts\\activate (Windows)" - -install: ## Install runtime dependencies only - uv pip install -e . - -install-dev: ## Install all development dependencies (includes testing) - uv pip install -e ".[dev]" - -# =================================================================== -# QUICK DEVELOPMENT WORKFLOWS -# =================================================================== - -dev: format lint-fix test ## Run full development cycle: format, fix lints, test - @echo "✅ Development cycle complete!" - -check: format-check lint-check typecheck ## Quick quality check (no tests) - @echo "✅ Quick check complete!" - -# =================================================================== -# TESTING -# =================================================================== - -test: ## Run tests with pytest - uv run pytest - -test-quick: ## Run tests quickly (no coverage) - uv run pytest -x - -test-verbose: ## Run tests with verbose output - uv run pytest -xvs - -test-watch: ## Run tests in watch mode (requires pytest-watch) - uv run ptw - -# =================================================================== -# CODE QUALITY -# =================================================================== - -## Formatting -format: ## Auto-format code with ruff - uv run ruff format src/ packages/**/src - -format-check: ## Check code formatting (no changes) - uv run ruff format --check src/ packages/**/src - -## Linting -lint: format-check lint-check typecheck ## Run all linters - @echo "✅ All linters passed!" - -lint-fix: ## Run ruff with auto-fix - uv run ruff check --fix src/ packages/**/src - -lint-check: ## Run ruff without fixes - uv run ruff check src/ packages/**/src - -## Type Checking -typecheck: ## Run mypy type checker - uv run mypy src/ packages/**/src - -## All Checks -quality: format-check lint-check typecheck ## Run all quality checks - -# =================================================================== -# RELEASE MANAGEMENT -# =================================================================== - -build: clean ## Build all packages into dist/ - @pip install build - @for pkg in . packages/*; do \ - if [ -f "$$pkg/pyproject.toml" ]; then \ - echo " Building $$pkg..."; \ - python -m build "$$pkg" --outdir dist/; \ - fi; \ - done - -verify-packages: ## Verify built packages with twine - @pip install twine - twine check dist/* - -readmes: ## Generate sub-package READMEs from pyproject.toml metadata - python3 scripts/generate_readmes.py - -readmes-check: ## Check sub-package READMEs are up to date - python3 scripts/generate_readmes.py --check - -# =================================================================== -# RUNNING THE CLI -# =================================================================== - -run: ## Run the CLI (show help) - uv run python -m deepctl --help - -run-version: ## Show CLI version - uv run python -m deepctl --version - -# =================================================================== -# CLEANUP -# =================================================================== - -clean: ## Clean all build artifacts and caches - rm -rf build/ - rm -rf dist/ - rm -rf *.egg-info/ - rm -rf packages/**/*.egg-info/ - find . -type d -name __pycache__ -exec rm -rf {} + - find . -type f -name "*.pyc" -delete - rm -rf .pytest_cache/ - rm -rf .coverage - rm -rf htmlcov/ - rm -rf .mypy_cache/ - rm -rf .ruff_cache/ - -clean-env: ## Remove virtual environment - rm -rf .venv/ - -# =================================================================== -# PRE-COMMIT HOOKS -# =================================================================== - -pre-commit-install: ## Install pre-commit hooks - uv run pre-commit install - -pre-commit-run: ## Run pre-commit on all files - uv run pre-commit run --all-files - -# =================================================================== -# ALIASES (for convenience) -# =================================================================== - -.PHONY: t tl q f l - -t: test ## Alias for test -tl: lint ## Alias for lint -q: check ## Alias for check (quick) -f: format ## Alias for format -l: lint-fix ## Alias for lint-fix +# =================================================================== +# deepctl Makefile - Development Tools +# =================================================================== +# +# Quick Start: +# make dev-setup # First time setup +# make dev # Daily development (format + lint + test) +# make help # Show organized help +# +# For new contributors: see README.md +# =================================================================== + +.PHONY: help +.DEFAULT_GOAL := help + +# =================================================================== +# HELP & INFO +# =================================================================== + +help: ## Show this help message + @echo "🔧 deepctl - Deepgram CLI Development Tools" + @echo "" + @echo "Usage: make [target]" + @echo "" + @echo "🚀 \033[1mQuick Start:\033[0m" + @echo " \033[36mdev-setup\033[0m Set up development environment" + @echo " \033[36mdev\033[0m Format, lint, and test (full dev cycle)" + @echo " \033[36mtest\033[0m Run tests" + @echo "" + @echo "🧪 \033[1mTesting:\033[0m" + @echo " \033[36mtest\033[0m Run tests (development)" + @echo " \033[36mcheck\033[0m Quick quality check (no tests)" + @echo "" + @echo "🔧 \033[1mCode Quality:\033[0m" + @echo " \033[36mformat\033[0m Auto-format code" + @echo " \033[36mlint\033[0m Run all linters" + @echo " \033[36mtypecheck\033[0m Run mypy type checker" + @echo "" + @echo "🧹 \033[1mUtilities:\033[0m" + @echo " \033[36mclean\033[0m Clean build artifacts" + @echo " \033[36minfo\033[0m Show project information" + @echo " \033[36mhelp-all\033[0m Show all available targets" + @echo "" + @echo "For more targets, run: \033[36mmake help-all\033[0m" + +help-all: ## Show all available targets + @echo "🔧 deepctl - All Available Targets" + @echo "" + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | grep -v '^\.' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-25s\033[0m %s\n", $$1, $$2}' + +info: ## Show project information + @echo "🔧 deepctl - Deepgram CLI" + @echo "📁 $(shell pwd)" + @echo "🐍 Python: $(shell python --version 2>/dev/null || echo 'Not found')" + @echo "📦 uv: $(shell uv --version 2>/dev/null || echo 'Not found')" + @echo "🎯 Virtual env: $(shell echo $$VIRTUAL_ENV || echo 'Not activated')" + +# =================================================================== +# DEVELOPMENT SETUP +# =================================================================== + +dev-setup: ## Set up complete development environment + uv venv + uv pip install -e ".[dev]" + @echo "✅ Development environment ready!" + @echo "Activate with: source .venv/bin/activate (Linux/macOS) or .venv\\Scripts\\activate (Windows)" + +install: ## Install runtime dependencies only + uv pip install -e . + +install-dev: ## Install all development dependencies (includes testing) + uv pip install -e ".[dev]" + +# =================================================================== +# QUICK DEVELOPMENT WORKFLOWS +# =================================================================== + +dev: format lint-fix test ## Run full development cycle: format, fix lints, test + @echo "✅ Development cycle complete!" + +check: format-check lint-check typecheck ## Quick quality check (no tests) + @echo "✅ Quick check complete!" + +# =================================================================== +# TESTING +# =================================================================== + +test: ## Run tests with pytest + uv run pytest + +test-quick: ## Run tests quickly (no coverage) + uv run pytest -x + +test-verbose: ## Run tests with verbose output + uv run pytest -xvs + +test-watch: ## Run tests in watch mode (requires pytest-watch) + uv run ptw + +# =================================================================== +# CODE QUALITY +# =================================================================== + +## Formatting +format: ## Auto-format code with ruff + uv run ruff format src/ packages/**/src scripts/ + +format-check: ## Check code formatting (no changes) + uv run ruff format --check src/ packages/**/src scripts/ + +## Linting +lint: format-check lint-check typecheck ## Run all linters + @echo "✅ All linters passed!" + +lint-fix: ## Run ruff with auto-fix + uv run ruff check --fix src/ packages/**/src scripts/ + +lint-check: ## Run ruff without fixes + uv run ruff check src/ packages/**/src scripts/ + +## Type Checking +typecheck: ## Run mypy type checker + uv run mypy src/ packages/**/src + +## All Checks +quality: format-check lint-check typecheck ## Run all quality checks + +# =================================================================== +# RELEASE MANAGEMENT +# =================================================================== + +build: clean ## Build all packages into dist/ + @pip install build + @for pkg in . packages/*; do \ + if [ -f "$$pkg/pyproject.toml" ]; then \ + echo " Building $$pkg..."; \ + python -m build "$$pkg" --outdir dist/; \ + fi; \ + done + +verify-packages: ## Verify built packages with twine + @pip install twine + twine check dist/* + +readmes: ## Generate sub-package READMEs from pyproject.toml metadata + python3 scripts/generate_readmes.py + +readmes-check: ## Check sub-package READMEs are up to date + python3 scripts/generate_readmes.py --check + +floors-check: ## Check intra-workspace dependency floors (root must pin workspace versions) + python3 scripts/check_dependency_floors.py + +floors-fix: ## Pin root dependency floors to the current workspace versions + python3 scripts/check_dependency_floors.py --fix + +# =================================================================== +# RUNNING THE CLI +# =================================================================== + +run: ## Run the CLI (show help) + uv run python -m deepctl --help + +run-version: ## Show CLI version + uv run python -m deepctl --version + +# =================================================================== +# CLEANUP +# =================================================================== + +clean: ## Clean all build artifacts and caches + rm -rf build/ + rm -rf dist/ + rm -rf *.egg-info/ + rm -rf packages/**/*.egg-info/ + find . -type d -name __pycache__ -exec rm -rf {} + + find . -type f -name "*.pyc" -delete + rm -rf .pytest_cache/ + rm -rf .coverage + rm -rf htmlcov/ + rm -rf .mypy_cache/ + rm -rf .ruff_cache/ + +clean-env: ## Remove virtual environment + rm -rf .venv/ + +# =================================================================== +# PRE-COMMIT HOOKS +# =================================================================== + +pre-commit-install: ## Install pre-commit hooks + uv run pre-commit install + +pre-commit-run: ## Run pre-commit on all files + uv run pre-commit run --all-files + +# =================================================================== +# ALIASES (for convenience) +# =================================================================== + +.PHONY: t tl q f l + +t: test ## Alias for test +tl: lint ## Alias for lint +q: check ## Alias for check (quick) +f: format ## Alias for format +l: lint-fix ## Alias for lint-fix diff --git a/README.md b/README.md index 0be41eb..8ff3363 100644 --- a/README.md +++ b/README.md @@ -319,6 +319,27 @@ dg usage --last-week -o yaml When running in a non-TTY environment (pipes, CI, or AI coding tools), the CLI automatically switches to structured JSON output with plain-text status messages. +### Exit codes + +Since 0.3.0, `dg` exits non-zero when a command fails — branch on the exit +code, not on parsing output: + +| Code | Meaning | +| --- | --- | +| `0` | Success | +| `1` | Error — a failed command, a crash, or a usage error (bad flag, unknown command) | +| `2` | Cancelled by the user (Ctrl-C, or declining a confirmation prompt) | + +Note that `dg` reports `2` for an interrupt rather than the shell's +conventional `130`, so the code is the same whether the cancellation came from +Ctrl-C or from declining a prompt. + +Human-readable status and error messages go to stderr; stdout carries only the +result, so `dg ... -o json | jq` stays parseable even when a command fails — a +failure arrives there as a payload with `"status": "error"`. If a CI step +relied on `dg` always exiting `0` (every command did, before 0.3.0), it will +now fail where it previously passed silently. + ### Forcing non-interactive mode Three explicit ways to skip every prompt and run with defaults — useful from a diff --git a/packages/deepctl-cmd-plugin/src/deepctl_cmd_plugin/command.py b/packages/deepctl-cmd-plugin/src/deepctl_cmd_plugin/command.py index 17fd9f7..8321a48 100644 --- a/packages/deepctl-cmd-plugin/src/deepctl_cmd_plugin/command.py +++ b/packages/deepctl-cmd-plugin/src/deepctl_cmd_plugin/command.py @@ -475,7 +475,11 @@ def _handle_remove( yes = kwargs.get("yes", False) if not yes and not click.confirm(f"Are you sure you want to remove {package}?"): - return + # Abort rather than return: this is a group subcommand, so there is + # no result for BaseCommand.EXIT_CODES to map, and a bare return + # exits 0 -- indistinguishable from a successful removal. main.py + # turns Abort into the documented exit 2 for user cancellation. + raise click.Abort() result = self.remove_plugin(config, auth_manager, client, package) diff --git a/packages/deepctl-cmd-plugin/tests/unit/test_plugin_command.py b/packages/deepctl-cmd-plugin/tests/unit/test_plugin_command.py index 486040e..5c4cfa4 100644 --- a/packages/deepctl-cmd-plugin/tests/unit/test_plugin_command.py +++ b/packages/deepctl-cmd-plugin/tests/unit/test_plugin_command.py @@ -4,6 +4,7 @@ from pathlib import Path from unittest.mock import MagicMock, patch +import click import pytest from click.testing import CliRunner from deepctl_cmd_plugin.command import PluginCommand @@ -234,6 +235,54 @@ def test_remove_plugin_not_installed(self) -> None: assert result.success is False assert "not installed" in result.message + def test_remove_declined_aborts_instead_of_exiting_zero(self) -> None: + """Declining the prompt must exit 2, not 0. + + `_handle_remove` is a group subcommand returning None, so there is no + result for BaseCommand.EXIT_CODES to map to an exit code. A bare + return made a declined removal indistinguishable from a successful + one for any script branching on the exit code, contradicting the + contract published in the README. Abort is what main.py turns into 2. + """ + with ( + patch("click.confirm", return_value=False), + patch.object(self.command, "remove_plugin") as mock_remove, + ): + with pytest.raises(click.Abort): + self.command._handle_remove( + self.config, + self.auth_manager, + self.client, + package="test-plugin", + ) + + mock_remove.assert_not_called() + + def test_remove_with_yes_skips_the_prompt(self) -> None: + """Positive control: --yes must not prompt and must not abort.""" + with ( + patch("click.confirm") as mock_confirm, + patch.object(self.command, "remove_plugin") as mock_remove, + patch.object(self.command, "_maybe_update_skills"), + ): + mock_remove.return_value = PluginOperationResult( + success=True, + action="remove", + package="test-plugin", + message="Successfully removed test-plugin", + ) + + self.command._handle_remove( + self.config, + self.auth_manager, + self.client, + package="test-plugin", + yes=True, + ) + + mock_confirm.assert_not_called() + mock_remove.assert_called_once() + @patch("deepctl_cmd_plugin.command.subprocess.run") def test_discover_from_environment(self, mock_run: MagicMock) -> None: """Test discovering plugins from a specific environment.""" diff --git a/packages/deepctl-core/src/deepctl_core/plugin_manager.py b/packages/deepctl-core/src/deepctl_core/plugin_manager.py index e25628c..854bb02 100644 --- a/packages/deepctl-core/src/deepctl_core/plugin_manager.py +++ b/packages/deepctl-core/src/deepctl_core/plugin_manager.py @@ -7,12 +7,11 @@ from typing import Any, cast import click -from rich.console import Console from .base_command import BaseCommand from .base_group_command import BaseGroupCommand from .models import ErrorResult, PluginInfo -from .output import print_warning +from .output import print_warning, stderr_console from .plugin_env import ( PLUGIN_VENV, get_plugin_state, @@ -21,7 +20,9 @@ ) from .timing import TimingContext -console = Console() +# Load errors are diagnostics: they go to stderr so a broken plugin can't +# corrupt `dg ... -o json` payloads on stdout. +console = stderr_console class PluginManager: diff --git a/packages/deepctl-core/tests/unit/test_plugin_manager.py b/packages/deepctl-core/tests/unit/test_plugin_manager.py index ad5d16d..e40ec06 100644 --- a/packages/deepctl-core/tests/unit/test_plugin_manager.py +++ b/packages/deepctl-core/tests/unit/test_plugin_manager.py @@ -575,3 +575,39 @@ def test_warn_if_plugin_venv_python_mismatch_warns_on_major_diff( ) as mock_warn: plugin_manager._warn_if_plugin_venv_python_mismatch() mock_warn.assert_called_once() + + +class TestLoadErrorStream: + """Plugin-load diagnostics must go to stderr, never stdout. + + A broken plugin's ImportError used to print through a stdout Console, + corrupting `dg ... -o json` payloads for every remaining command (the + amplifier in the 0.2.x core-floor incident). The module console is the + shared stderr console so that cannot recur. + """ + + def test_module_console_is_the_shared_stderr_console(self): + from deepctl_core import plugin_manager + from deepctl_core.output import stderr_console + + assert plugin_manager.console is stderr_console + assert plugin_manager.console.stderr is True + + def test_load_error_writes_to_stderr_not_stdout(self, capsys): + from deepctl_core import plugin_manager + + mock_entry_point = Mock() + mock_entry_point.name = "broken-command" + mock_entry_point.load.side_effect = ImportError("Module not found") + + with patch( + "deepctl_core.plugin_manager.metadata.entry_points" + ) as mock_eps: + mock_eps.return_value.select.return_value = [mock_entry_point] + plugin_manager.PluginManager()._load_builtin_commands( + click.Group("dg") + ) + + captured = capsys.readouterr() + assert "broken-command" in captured.err + assert captured.out == "" diff --git a/pyproject.toml b/pyproject.toml index 0f0a29c..ad16a44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,21 +38,21 @@ dependencies = [ "deepctl-core>=0.2.16", "deepctl-cmd-login>=0.1.17", "deepctl-cmd-projects>=0.2.0", - "deepctl-cmd-transcribe>=0.1.10", + "deepctl-cmd-transcribe>=0.1.12", "deepctl-cmd-usage>=0.2.0", "deepctl-cmd-mcp>=0.1.15", - "deepctl-cmd-api>=0.0.1", - "deepctl-cmd-debug>=0.1.10", - "deepctl-cmd-debug-browser>=0.1.10", - "deepctl-cmd-debug-network>=0.1.10", - "deepctl-cmd-debug-audio>=0.1.10", - "deepctl-cmd-debug-probe>=0.0.1", - "deepctl-cmd-debug-toolkit>=0.0.1", - "deepctl-cmd-ffprobe>=0.0.1", + "deepctl-cmd-api>=0.0.2", + "deepctl-cmd-debug>=0.1.12", + "deepctl-cmd-debug-browser>=0.1.12", + "deepctl-cmd-debug-network>=0.1.12", + "deepctl-cmd-debug-audio>=0.1.13", + "deepctl-cmd-debug-probe>=0.0.2", + "deepctl-cmd-debug-toolkit>=0.1.0", + "deepctl-cmd-ffprobe>=0.0.2", "deepctl-cmd-update>=0.2.6", - "deepctl-cmd-plugin>=0.1.10", + "deepctl-cmd-plugin>=0.1.12", "deepctl-cmd-skills>=0.0.7", - "deepctl-cmd-init>=0.0.1", + "deepctl-cmd-init>=0.0.4", "deepctl-cmd-models>=0.1.0", "deepctl-cmd-speak>=0.0.4", "deepctl-cmd-keys>=0.1.0", @@ -61,8 +61,8 @@ dependencies = [ "deepctl-cmd-requests>=0.1.0", "deepctl-cmd-billing>=0.1.0", "deepctl-cmd-members>=0.1.0", - "deepctl-cmd-completion>=0.0.1", - "deepctl-shared-utils>=0.1.10", + "deepctl-cmd-completion>=0.0.3", + "deepctl-shared-utils>=0.1.12", "deepctl-telemetry>=0.0.6", "pydantic>=2.0.0", "rich>=13.0.0", diff --git a/scripts/build_standalone.py b/scripts/build_standalone.py index a2a0449..5a8941d 100644 --- a/scripts/build_standalone.py +++ b/scripts/build_standalone.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Build a standalone deepctl binary for testing system installations.""" -import os +import importlib.util import shutil import subprocess import sys @@ -18,9 +18,7 @@ def build_standalone(): print("🔨 Building standalone deepctl binary...") # Install PyInstaller if needed - try: - import PyInstaller - except ImportError: + if importlib.util.find_spec("PyInstaller") is None: print("📦 Installing PyInstaller...") subprocess.run( [sys.executable, "-m", "pip", "install", "pyinstaller"], check=True @@ -132,9 +130,7 @@ def build_standalone(): print(" ./dist_standalone/deepctl plugin search") print(" ./dist_standalone/deepctl plugin install deepctl-plugin-example") print("\n💡 The standalone binary should detect as 'system' installation") - print( - " and create an isolated plugin environment at ~/.deepctl/plugins/" - ) + print(" and create an isolated plugin environment at ~/.deepctl/plugins/") if __name__ == "__main__": diff --git a/scripts/check_dependency_floors.py b/scripts/check_dependency_floors.py new file mode 100644 index 0000000..5609bcb --- /dev/null +++ b/scripts/check_dependency_floors.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Check (or fix) intra-workspace dependency floors. + +Three rules, learned from the 0.3.0 release (PRs #100/#102): + +1. **Root floors equal workspace versions.** `dg update` runs + `pip install --upgrade deepctl`, and pip's default `only-if-needed` + strategy upgrades a sub-package only when the root floor forces it. Any + root floor below the current version means that package's fixes are + published but never delivered on upgrade — `dg --version` reports the new + release while 13 of 17 packages stay stale, which is what happened before + 0.3.0. Root's dependency list is the delivery manifest, so each + `deepctl-*` floor must equal that package's current workspace version. + +2. **Sub-package floors stay satisfiable.** Sub-package floors are API + contracts (e.g. deepctl-cmd-keys needs the deepctl-core that provides + `get_status_console`), hand-raised when a package starts using a newer + sibling API. They must never exceed the sibling's current version. + Keeping them *accurate* is still on the developer: raise the floor in the + same PR that starts importing the new API. + +3. **Root's dependency list covers every published package.** Rule 1 only + validates the floors that are already listed. A package that release-please + versions and publishes but that nobody added to root's `dependencies` is + never installed by `pip install --upgrade deepctl` at all — the same + delivery gap as rule 1, through the door rule 1 leaves open. Anything + deliberately not shipped as part of the CLI goes in NOT_SHIPPED, so that + intent is stated in the diff rather than inferred from an omission. + +Run with --fix to rewrite root floors in place (used by the release +workflow's sync job so rule 1 holds automatically on every release PR). +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +try: + import tomllib +except ModuleNotFoundError: + try: + import tomli as tomllib # type: ignore[no-redef] + except ModuleNotFoundError: + print("Python 3.11+ required (tomllib), or install tomli: pip install tomli") + sys.exit(1) + +REPO = Path(__file__).resolve().parent.parent +MANIFEST = REPO / ".github" / ".release-please-manifest.json" + +# Published packages that are deliberately not part of what `dg` installs. +# Everything else in the manifest must be a root dependency (rule 3). +NOT_SHIPPED = { + "deepctl", # the root package itself + "deepctl-plugin-example", # sample plugin, installed on demand +} + + +def workspace_versions() -> dict[str, str]: + """Map package name -> current workspace version, per the manifest.""" + versions: dict[str, str] = {} + for path in json.loads(MANIFEST.read_text(encoding="utf-8")): + pyproject = REPO / ( + "pyproject.toml" if path == "." else f"{path}/pyproject.toml" + ) + project = tomllib.loads(pyproject.read_text(encoding="utf-8"))["project"] + versions[project["name"]] = project["version"] + return versions + + +def floors(pyproject: Path) -> list[tuple[str, str, str]]: + """Yield (name, floor, raw-spec) for each intra-workspace dependency.""" + deps = tomllib.loads(pyproject.read_text(encoding="utf-8"))["project"].get( + "dependencies", [] + ) + out = [] + for dep in deps: + m = re.match(r"^(deepctl[\w-]*)>=([0-9][0-9.]*)", dep) + if m: + out.append((m.group(1), m.group(2), dep)) + return out + + +def vkey(version: str) -> list[int]: + """Sortable key from the numeric release segments only. + + PEP 440 suffixes (0.4.0rc1, 1.0.0.dev1, 0.3.0.post1) and local versions + are ignored rather than crashing the comparison -- a single hand-set + pre-release anywhere in the workspace used to turn `make floors-check` + into a bare ValueError traceback naming no package. + """ + release = re.match(r"\d+(?:\.\d+)*", version) + return [int(part) for part in release.group().split(".")] if release else [0] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--fix", + action="store_true", + help="rewrite root pyproject.toml floors to the workspace versions", + ) + args = parser.parse_args() + + versions = workspace_versions() + problems: list[str] = [] + + # Rule 1: root floors == workspace versions. + root = REPO / "pyproject.toml" + root_text = root.read_text(encoding="utf-8") + fixed = root_text + root_floors = floors(root) + for name, floor, raw in root_floors: + current = versions.get(name) + if current is None: + problems.append(f"root depends on {name}, which is not in the manifest") + continue + if floor == current: + continue + if args.fix: + # Rewrite the version inside the spec we matched, so any upper + # bound, extra, or environment marker survives. A whole-spec + # literal replace silently no-ops on those, and an unfixable + # floor has to fall through to `problems` -- reporting "OK" + # after failing to fix is worse than not fixing. + new_raw = raw.replace(f">={floor}", f">={current}", 1) + if new_raw != raw and f'"{raw}"' in fixed: + fixed = fixed.replace(f'"{raw}"', f'"{new_raw}"', 1) + continue + problems.append( + f"root floor {name}>={floor} != workspace version {current}" + " (published fixes will not be delivered by pip upgrades)" + ) + if args.fix and fixed != root_text: + root.write_text(fixed, encoding="utf-8") + print(f"fixed: root floors pinned to workspace versions in {root}") + + # Rule 2: every sub-package floor must be satisfiable at co-release. + for pkg_dir in sorted((REPO / "packages").iterdir()): + pyproject = pkg_dir / "pyproject.toml" + if not pyproject.is_file(): + continue + for name, floor, _ in floors(pyproject): + current = versions.get(name) + if current is None: + problems.append( + f"{pkg_dir.name} depends on {name}, which is not in the manifest" + ) + elif vkey(floor) > vkey(current): + problems.append( + f"{pkg_dir.name}: floor {name}>={floor} exceeds" + f" workspace version {current} (unsatisfiable)" + ) + + # Rule 3: every published package is a root dependency. + listed = {name for name, _, _ in root_floors} + for name in sorted(set(versions) - listed - NOT_SHIPPED): + problems.append( + f"{name} is published but is not a root dependency" + " (pip upgrades will never install it; add it to root" + " pyproject.toml or to NOT_SHIPPED in this script)" + ) + + if problems: + print("dependency floor check FAILED:", file=sys.stderr) + for problem in problems: + print(f" - {problem}", file=sys.stderr) + print( + "\nRun `python3 scripts/check_dependency_floors.py --fix` to pin" + " stale root floors. Sub-package floors and missing root" + " dependencies are hand-maintained.", + file=sys.stderr, + ) + return 1 + + print("dependency floors OK") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/generate_readmes.py b/scripts/generate_readmes.py index 63262ef..37e58fe 100644 --- a/scripts/generate_readmes.py +++ b/scripts/generate_readmes.py @@ -19,10 +19,7 @@ try: import tomli as tomllib # type: ignore[no-redef] except ModuleNotFoundError: - print( - "Python 3.11+ required (tomllib), " - "or install tomli: pip install tomli" - ) + print("Python 3.11+ required (tomllib), or install tomli: pip install tomli") sys.exit(1) ROOT = Path(__file__).resolve().parent.parent @@ -130,8 +127,7 @@ def render_command_readme( lines = [ f"# {name}", "", - "> Part of [deepctl](https://github.com/deepgram/cli)" - " — Official Deepgram CLI", + "> Part of [deepctl](https://github.com/deepgram/cli) — Official Deepgram CLI", "", description, "", @@ -159,9 +155,7 @@ def render_command_readme( else: lines.append("No external dependencies.") - lines.extend( - ["", "## License", "", "MIT — see [LICENSE](../../LICENSE)", ""] - ) + lines.extend(["", "## License", "", "MIT — see [LICENSE](../../LICENSE)", ""]) return "\n".join(lines) @@ -176,8 +170,7 @@ def render_debug_subcommand_readme( lines = [ f"# {name}", "", - "> Part of [deepctl](https://github.com/deepgram/cli)" - " — Official Deepgram CLI", + "> Part of [deepctl](https://github.com/deepgram/cli) — Official Deepgram CLI", "", description, "", @@ -207,9 +200,7 @@ def render_debug_subcommand_readme( else: lines.append("No external dependencies.") - lines.extend( - ["", "## License", "", "MIT — see [LICENSE](../../LICENSE)", ""] - ) + lines.extend(["", "## License", "", "MIT — see [LICENSE](../../LICENSE)", ""]) return "\n".join(lines) @@ -223,8 +214,7 @@ def render_core_readme( lines = [ f"# {name}", "", - "> Part of [deepctl](https://github.com/deepgram/cli)" - " — Official Deepgram CLI", + "> Part of [deepctl](https://github.com/deepgram/cli) — Official Deepgram CLI", "", description, "", @@ -243,9 +233,7 @@ def render_core_readme( else: lines.append("No external dependencies.") - lines.extend( - ["", "## License", "", "MIT — see [LICENSE](../../LICENSE)", ""] - ) + lines.extend(["", "## License", "", "MIT — see [LICENSE](../../LICENSE)", ""]) return "\n".join(lines) @@ -269,9 +257,7 @@ def generate_readme(package_dir: Path) -> str: name, description, entry_points, external_deps, package_dir ) else: - return render_core_readme( - name, description, external_deps, package_dir - ) + return render_core_readme(name, description, external_deps, package_dir) def get_package_dirs(single: str | None = None) -> list[Path]: @@ -307,13 +293,9 @@ def get_all_packages() -> list[dict]: { "dir_name": pkg_dir.name, "name": project["name"], - "description": project.get( - "description", project["name"] - ), + "description": project.get("description", project["name"]), "commands": eps.get("deepctl.commands", {}), - "debug_subcommands": eps.get( - "deepctl.subcommands.debug", {} - ), + "debug_subcommands": eps.get("deepctl.subcommands.debug", {}), } ) return packages @@ -328,9 +310,7 @@ def generate_commands_section(packages: list[dict]) -> str: for cmd_name in pkg["commands"]: rows.append((f"`deepctl {cmd_name}`", pkg["description"])) for cmd_name in pkg["debug_subcommands"]: - rows.append( - (f"`deepctl debug {cmd_name}`", pkg["description"]) - ) + rows.append((f"`deepctl debug {cmd_name}`", pkg["description"])) rows.sort(key=lambda r: r[0]) lines = [ "| Command | Description |", @@ -349,8 +329,7 @@ def generate_packages_section(packages: list[dict]) -> str: ] for pkg in sorted(packages, key=lambda p: p["name"]): lines.append( - f"| [`{pkg['name']}`](packages/{pkg['dir_name']})" - f" | {pkg['description']} |" + f"| [`{pkg['name']}`](packages/{pkg['dir_name']}) | {pkg['description']} |" ) return "\n".join(lines) @@ -365,9 +344,7 @@ def tree_line(prefix: str, name: str, desc: str) -> str: return f"{line}{' ' * padding}# {desc}" lines = ["```", "cli/"] - lines.append( - tree_line("├── ", "src/deepctl/", "Main CLI entry point") - ) + lines.append(tree_line("├── ", "src/deepctl/", "Main CLI entry point")) lines.append("├── packages/") sorted_pkgs = sorted(packages, key=lambda p: p["dir_name"]) @@ -382,19 +359,13 @@ def tree_line(prefix: str, name: str, desc: str) -> str: ) ) - lines.append( - tree_line("├── ", "tests/", "Integration tests") - ) - lines.append( - tree_line("└── ", "Makefile", "Development tasks") - ) + lines.append(tree_line("├── ", "tests/", "Integration tests")) + lines.append(tree_line("└── ", "Makefile", "Development tasks")) lines.append("```") return "\n".join(lines) -def replace_section( - content: str, section: str, replacement: str -) -> str: +def replace_section(content: str, section: str, replacement: str) -> str: """Replace content between BEGIN:section and END:section markers.""" pattern = re.compile( rf"(\n)" @@ -409,9 +380,7 @@ def repl(m: re.Match) -> str: return pattern.sub(repl, content) -def update_root_readme( - *, dry_run: bool = False, check: bool = False -) -> bool: +def update_root_readme(*, dry_run: bool = False, check: bool = False) -> bool: """Update auto-generated sections in the root README.md. Returns True if the file was (or needs to be) changed. @@ -506,9 +475,7 @@ def main(): # Update root README sections (skip when targeting a single package) if not args.package: - root_changed = update_root_readme( - dry_run=args.dry_run, check=args.check - ) + root_changed = update_root_readme(dry_run=args.dry_run, check=args.check) if root_changed: stale.append("README.md") @@ -522,10 +489,7 @@ def main(): if not args.check and not args.dry_run: total = len(package_dirs) + (0 if args.package else 1) updated = len(stale) - print( - f"\nDone: {updated} updated, " - f"{total - updated} already current" - ) + print(f"\nDone: {updated} updated, {total - updated} already current") if __name__ == "__main__":