test: layered test strategy — unit, command, container

.github/workflows/verify.yml

@@ -32,12 +32,22 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
       - name: Build container
+        id: build
         uses: docker/build-push-action@v7
         env:
           DOCKER_BUILD_RECORD_UPLOAD: 'false'
         with:
           context: ./
           load: true
-          tags: steadybit/cli:latest
+      # By image id rather than a tag. `docker run` pulls a tag it cannot find locally,
+      # so a name like steadybit/cli:latest would quietly fall back to the last release
+      # if this build ever stopped being loaded. An id is never looked up in a registry.
       - name: Test container
-        run: docker run --rm steadybit/cli:latest -V
+        run: docker run --rm ${{ steps.build.outputs.imageid }} -V
+      # Everything these cover needs a real process: an exit status, a terminal, or the
+      # spawn of a subcommand. They run against the image users install, built from the
+      # npm tarball, so the packaging is exercised too.
+      - name: Run container smoke tests
+        run: |
+          docker run --rm -v "$PWD/e2e:/e2e" --entrypoint sh \
+            ${{ steps.build.outputs.imageid }} /e2e/run.sh

CONTRIBUTING.md

@@ -13,6 +13,45 @@ npm ci
 Run `npm run ci` before pushing. It type-checks, tests, lints and builds, and is the
 same script CI runs.
 
+## Tests
+
+Tests sit at three levels. Put a test at the lowest one that can hold it — the
+levels get slower and harder to debug as you go down this list.
+
+| Level     | Tool                               | Covers                                                 |
+| --------- | ---------------------------------- | ------------------------------------------------------ |
+| Unit      | vitest                             | A single function or class, no I/O                     |
+| Command   | vitest + msw + `@inquirer/testing` | A command end to end in process, including its prompts |
+| Container | `e2e/run.sh` + expect              | Only what needs a real process                         |
+
+Prompts are driven through `@inquirer/testing`. Mock the prompt package with
+`wrapPrompt` so the application's own call is intercepted, and use the helpers in
+`src/mocks/prompts.ts` rather than writing to the screen directly:
+
+```ts
+vi.mock('@inquirer/input', async importOriginal => {
+  const actual = await importOriginal<typeof import('@inquirer/input')>();
+  return { ...actual, default: wrapPrompt(actual.default) };
+});
+
+await answerPrompt('Profile name:', 'my-profile');
+```
+
+The container tests are deliberately thin. They exist for the four things no in-process
+test can reach &mdash; real exit codes, a real terminal, the spawn of a subcommand, and
+the packaged artifact &mdash; and they assert exit status and a line of output, never
+content. Anything checking structure belongs at the command level.
+
+```sh
+docker build -t steadybit/cli:under-test .
+docker run --rm -v "$PWD/e2e:/e2e" --entrypoint sh steadybit/cli:under-test /e2e/run.sh
+```
+
+The scripts are mounted into the image rather than baked into a derived one, and CI runs
+the image by id rather than by name. Both avoid the same mistake: a name is resolved
+against a registry when it cannot be found locally, so the suite can end up exercising
+the last release while reporting success.
+
 ### Local CLI Execution
 
 ```sh

e2e/add-profile.exp

@@ -0,0 +1,10 @@
+# Drives `config profile add` through a real pty.
+log_user 0
+set timeout 20
+spawn steadybit config profile add
+expect "Profile name:"     { send "e2e\r" }
+expect "Base URL"          { send "https://platform.example.com\r" }
+expect "API access token:" { send "s3cr3t\r" }
+expect eof
+catch wait result
+exit [lindex $result 3]

e2e/cancel-profile.exp

@@ -0,0 +1,9 @@
+# Ctrl+C at the first question must exit quietly with the SIGINT status.
+log_user 0
+set timeout 20
+spawn steadybit config profile add
+expect "Profile name:" { puts "REACHED_PROMPT" }
+send "\003"
+expect eof
+catch wait result
+exit [lindex $result 3]

e2e/colour-on-tty.exp

@@ -0,0 +1,13 @@
+# Colour is gated on stdout being a terminal, so the CLI has to be given one. Piping
+# its output into a matcher would remove the very thing under test.
+#
+# The pattern is the escape character alone: expect matches globs by default, in which
+# a "[" opens a character class rather than matching itself.
+log_user 0
+set timeout 20
+spawn env STEADYBIT_TOKEN= steadybit experiment get -k ADM-1
+expect {
+  "\033" { catch { exp_close }; exit 0 }
+  eof      { exit 1 }
+  timeout  { exit 1 }
+}

e2e/run.sh

@@ -0,0 +1,101 @@
+#!/bin/sh
+# SPDX-License-Identifier: MIT
+# SPDX-FileCopyrightText: 2026 Steadybit GmbH
+
+# Smoke tests for the packaged CLI. Everything here needs a real process: an exit
+# status, a terminal, or the spawn of a subcommand. Anything that can be asserted
+# in-process belongs in the vitest suite instead, so these stay at the level of
+# "did it exit correctly" rather than checking output in detail.
+
+set -u
+failures=0
+
+# Driving the prompts needs a pty allocator, which the shipped image has no reason to
+# carry. Installed here rather than baked into a second image: a derived image has to
+# name its base by tag, and a tag can silently resolve to something from a registry
+# instead of the build under test.
+if ! command -v expect >/dev/null 2>&1; then
+  apk add --no-cache expect >/dev/null 2>&1 || {
+    echo "cannot install expect, which the interactive checks need"
+    exit 1
+  }
+fi
+
+check() {
+  description=$1
+  shift
+  if "$@"; then
+    echo "  ok    $description"
+  else
+    echo "  FAIL  $description"
+    failures=$((failures + 1))
+  fi
+}
+
+exits_with() {
+  expected=$1
+  shift
+  "$@" >/dev/null 2>&1
+  actual=$?
+  [ "$actual" -eq "$expected" ] || {
+    echo "        expected exit $expected, got $actual"
+    return 1
+  }
+}
+
+echo "steadybit CLI container smoke tests"
+
+check "--version succeeds" exits_with 0 steadybit --version
+check "--help succeeds" exits_with 0 steadybit --help
+check "a subcommand is spawned and runs" exits_with 0 steadybit experiment --help
+check "an unknown command fails" exits_with 1 steadybit definitely-not-a-command
+
+# The access token is resolved before anything else, so this is the guard on every
+# platform-touching command.
+check "a missing access token fails" exits_with 1 env STEADYBIT_TOKEN= steadybit experiment get -k ADM-1
+check "an unreachable platform fails" exits_with 1 \
+  env STEADYBIT_TOKEN=t STEADYBIT_URL=http://127.0.0.1:1 steadybit experiment get -k ADM-1
+
+# Colour is gated on stdout being a terminal. Both halves are checked here rather than
+# depending on how the container was started: the pipe below is genuinely not a
+# terminal, and expect genuinely provides one.
+# Asserting only the absence of escapes would also hold if nothing were printed at all,
+# so the output has to be there first.
+check "output is clean when piped" sh -c '
+  out=$(env STEADYBIT_TOKEN= steadybit experiment get -k ADM-1 2>&1)
+  # Something the CLI itself prints. Merely having output would also be satisfied by the
+  # shell reporting that there is no such command.
+  echo "$out" | grep -q "No API access token" || exit 1
+  ! printf "%s" "$out" | grep -q "$(printf "\033")"
+'
+check "output is coloured on a terminal" expect /e2e/colour-on-tty.exp
+
+# The interactive flow, driven through a pty. Writes into the container's own home.
+check "profile add stores what was typed" sh -c '
+  expect /e2e/add-profile.exp >/dev/null 2>&1 || exit 1
+  grep -q "\"name\": \"e2e\"" "$HOME/.steadybit/profiles.json" || exit 1
+  steadybit config profile list | grep -q "e2e"
+'
+
+check "ctrl-c during a prompt exits 130" sh -c '
+  expect /e2e/cancel-profile.exp >/dev/null 2>&1
+  [ $? -eq 130 ]
+'
+
+# Same trap: no stack trace and no profile are both true of a CLI that never ran, so
+# the prompt has to be shown to have been reached.
+check "ctrl-c leaves no stack trace and no profile" sh -c '
+  rm -rf "$HOME/.steadybit"
+  out=$(expect /e2e/cancel-profile.exp 2>&1)
+  echo "$out" | grep -q "REACHED_PROMPT" || exit 1
+  echo "$out" | grep -q "ExitPromptError" && exit 1
+  [ ! -f "$HOME/.steadybit/profiles.json" ]
+'
+
+echo
+if [ "$failures" -eq 0 ]; then
+  echo "all container smoke tests passed"
+else
+  echo "$failures container smoke test(s) failed"
+fi
+exit "$failures"

package.json

@@ -54,6 +54,7 @@
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
+    "@inquirer/testing": "^3.3.9",
     "@types/node": "^22.19.4",
     "@types/semver": "^7.3.9",
     "eslint": "^10.8.0",