diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6867935..63c1e4d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,7 +46,7 @@ jobs: persist-credentials: false # Errors only, for now: the engine predates this check. Raise to warning # once the existing findings are cleared. - - run: shellcheck -x --source-path=SCRIPTDIR -S error src/deployctl/scripts/*.sh src/deployctl/scripts/common/*.sh src/deployctl/server/*.sh + - run: shellcheck -x --source-path=SCRIPTDIR -S error src/deployctl/scripts/*.sh src/deployctl/scripts/common/*.sh package: name: the wheel works on its own diff --git a/CHANGELOG.md b/CHANGELOG.md index c81f679..3a3ffef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,56 @@ Versions are git tags (`vX.Y.Z`); projects install one with `uv tool install git+ssh://git@github.com/hybridinteract/deployctl@vX.Y.Z`. +## 0.10.0 + +Continuous deployment by default, and fleets that are never left split. + +**Deploys** +- **A failed release is reverted across the fleet.** When a host fails its health gate or + the service watch, it is restored, and so is every host the run already moved, newest + first, each from its own snapshot (`REVERT_SCOPE=fleet`, the default; `host` reverts + only the one that failed). A reverted release is not recorded as a release. +- **`SSH_JUMP_HOST`** reaches hosts on a private network through a bastion (ssh + `ProxyJump`) — every connection, rsync and the panel's terminal included. +- **The image tag is not configuration any more.** Each command decides it: `--tag` (or + CI's `$IMAGE_TAG`); for commands that act on the hosts, the tag the primary is + running — `deploy update` without `--tag` redeploys it, which is how a config change goes + out; for local ones, the tag last used on this machine. An `IMAGE_TAG` left in + `config/` still works, with a deprecation warning. New: `--tag` on `setup`, + `deploy init|update|migrate|doctor`. `image push` no longer writes the tag anywhere. +- **The history lives on the primary**: every release appends when, which tag, what kind + and who — an Actions run URL, or `user@machine`. `deploy history` and `deploy rollback` + read it, so a laptop and CI agree on "the previous release". `.deployctl-state` records + `DEPLOYED_BY`. The laptop's `generated/.history` is no longer used. +- `deploy status --json`: every host's tag, who deployed it, and every service. +- `deploy rollback` refuses a tag that is already running (now read from the primary). + +**CI/CD, one command per step** (`deployctl ci …`) +- `connect` — detects the repository and plan, records `CI_SCOPE` (repository secrets on + GitHub Free with a private repository, environment secrets otherwise) and `DEPLOY_BRANCH`. +- `init` — writes `build-image.yml` (hands its tag to the deploy) and `deploy.yml` + (installs this deployctl version, pinned; timeout sized to the fleet), both managed and + regenerated with `--force` after a diff; `ci.yml` only when there is none. Actions pinned + to commit SHAs, all on Node 24. +- `setup-key` / `--rotate` — a CI-only key on every host and the jump host, proven, then + uploaded; the local copy is deleted and only the fingerprint is printed. +- `pin-hosts` — the host keys this machine trusts, into `DEPLOY_KNOWN_HOSTS`. +- `doctor` / `--json` — every piece checked, with the command that fills each gap. +- `deploy` (`--tag`, `--rollback`, `--allow-config-change`) — runs the deploy workflow and + follows it; `runs` — recent deploys, manual and on merge; `auto-deploy on|off`. + +**Servers and backups** +- `server bootstrap-script` renders the root script that prepares a fresh Ubuntu server — + for cloud-init user data or `| ssh root@host 'bash -s'`. +- Fetched backups go to `~/.deployctl/backups///` (or `DEPLOYCTL_BACKUP_DIR`), + never into a repository. +- `migrate-config` removes settings the tool no longer reads (today `IMAGE_TAG`), keeping + its value as this machine's last-used tag. + +**Upgrading from 0.9.0:** `deployctl migrate-config --apply`, then `deployctl ci init +--force` for the new workflows. Rollback needs one release recorded on the primary before +it can pick "the previous" one by itself; until then, pass `--to`. + ## 0.9.0 The first packaged release — imported from the application repository deployctl was diff --git a/README.md b/README.md index a6304b9..9a50599 100644 --- a/README.md +++ b/README.md @@ -168,10 +168,10 @@ deployctl ssl setup --env production # single-server TLS only Then day to day: ```bash -deployctl image tags # what can I deploy? -$EDITOR config/production.env # bump IMAGE_TAG -deployctl deploy update --env production # rolling, health-gated -deployctl deploy rollback --env production # back to the previous tag +deployctl image tags # what can I deploy? +deployctl deploy update --env production --tag # rolling, health-gated, reverted on failure +deployctl deploy rollback --env production # back to the previous release +deployctl ci deploy --env production --tag # the same, through GitHub Actions ``` --- @@ -219,8 +219,10 @@ and production with `cluster` is just the common case, not a rule. | `config --env E [--key K]` | the resolved configuration, secrets masked | | `envs` | list configured environments | | `image push [--tag T]` / `image tags` | build+push from this machine / list registry tags | -| `ci init [--branch B]` | write `.github/workflows/build-image.yml` | -| `ci sync-config\|status\|unpack` | hand CI this environment's config — see [35-CONTINUOUS-DEPLOYMENT.md](docs/35-CONTINUOUS-DEPLOYMENT.md) | +| `ci connect\|init\|setup-key\|pin-hosts\|sync-config\|doctor` | set up deploys from GitHub Actions, one step each — see [35-CONTINUOUS-DEPLOYMENT.md](docs/35-CONTINUOUS-DEPLOYMENT.md) | +| `ci deploy [--tag T\|--rollback]` / `ci runs` / `ci auto-deploy on\|off` | deploy through GitHub, see what it did, deploy on merge | +| `server bootstrap-script` | the root script that prepares a fresh server | +| `adopt` / `migrate-config` | move a copied-in deployctl to `deploy/`; drop settings the tool no longer reads | | `deploy doctor\|init\|update\|migrate\|rollback\|restart\|stop\|status\|logs\|shell\|history` | operate the stack | | `deploy doctor --fix` | repair the permission problems that do not need root | | `ssl setup\|renew\|check` | Let's Encrypt (single host only) | @@ -240,10 +242,9 @@ deployctl (this repository — the installed tool) ├── src/deployctl/ │ ├── cli/ Python: config resolution, validation, rendering, commands │ ├── scripts/ bash: ssh, rsync, docker compose — the deploy engine -│ ├── templates/ Jinja2 → generated/ +│ ├── templates/ Jinja2 → generated/, the CI workflows, the server bootstrap │ ├── profiles/ defaults for each shape (single, cluster) │ ├── webui/ the control panel: routes, action whitelist, static files -│ └── server/ preparing a fresh server (cloud-init user data) ├── examples/demo/project/ what a project's committed part looks like ├── tests/ pytest — run with: uv run pytest └── docs/ start with 05-QUICKSTART.md diff --git a/docs/05-QUICKSTART.md b/docs/05-QUICKSTART.md index 5d1f609..ce5be82 100644 --- a/docs/05-QUICKSTART.md +++ b/docs/05-QUICKSTART.md @@ -152,9 +152,8 @@ deployctl ssl setup --env production # single-server TLS only Then for each release: ```bash -deployctl image tags # what can I deploy? -$EDITOR config/production.env # bump IMAGE_TAG -deployctl deploy update --env production # rolling, health-gated +deployctl image tags # what can I deploy? +deployctl deploy update --env production --tag # rolling, health-gated, reverted on failure ``` Add `--dry-run` to any deploy command to print every ssh/rsync/compose command it would diff --git a/docs/10-SINGLE-SERVER.md b/docs/10-SINGLE-SERVER.md index 389b58c..66b1550 100644 --- a/docs/10-SINGLE-SERVER.md +++ b/docs/10-SINGLE-SERVER.md @@ -96,7 +96,6 @@ REGISTRY_TOKEN=ghp_… # classic PAT, read:packages only ```sh MODE=single API_SUBDOMAIN=api -IMAGE_TAG= # filled in after the first build HOSTS="203.0.113.10" SSH_USER=deploy REMOTE_DIR=/opt/myapp @@ -134,10 +133,11 @@ The run's Summary prints the tag. Or build from your machine: deployctl image push # tags with the current git short SHA, pins it for you ``` -Then put the tag in `config/production.env` (`image push` does this automatically): +Then name that tag on the first deploy — the hosts run nothing yet, so there is no running +tag to default to. Every later deploy defaults to the tag the primary runs: ```sh -IMAGE_TAG=fb31c25 +deployctl deploy init --env production --tag fb31c25 ``` --- @@ -211,8 +211,7 @@ An unverified backup is a hypothesis, not a backup. ```bash # deploy a new build deployctl image tags # what is available -$EDITOR config/production.env # bump IMAGE_TAG -deployctl deploy update --env production +deployctl deploy update --env production --tag # or through GitHub: deployctl ci deploy --tag # something wrong deployctl deploy rollback --env production diff --git a/docs/20-CLUSTER.md b/docs/20-CLUSTER.md index 9652c7a..0f7577f 100644 --- a/docs/20-CLUSTER.md +++ b/docs/20-CLUSTER.md @@ -68,9 +68,13 @@ manage them from the control panel). - Note the load balancer's public IP. ### Firewall -- **Inbound:** `80` from the load balancer only; `22` from your IP only. Nothing else. +- **Inbound:** `80` from the load balancer only; `22` from whoever deploys. Nothing else. In particular, do not expose `80` to the internet — clients should only ever arrive through the LB. +- **Deploying from CI:** GitHub's runners connect from a large, changing set of addresses, + so "22 from your IP only" shuts them out. Either put the hosts on a tailnet the job joins, + or keep :22 private and set `SSH_JUMP_HOST` to a bastion that can reach them — see + [35-CONTINUOUS-DEPLOYMENT.md § Reaching the hosts](35-CONTINUOUS-DEPLOYMENT.md#reaching-the-hosts). - **Outbound:** the VPC (databases) and `443` (registry, object storage). - Use the cloud firewall, and leave the host firewall (ufw) inactive. Running both is a well-known way to lock yourself out of your own servers. @@ -88,10 +92,17 @@ A api.example.com → ← the LB, not a host ## 2. Bootstrap each host -Identical on every host, once. See -**[10-SINGLE-SERVER.md](10-SINGLE-SERVER.md#1-bootstrap-the-host-once-by-hand)** — the same -steps: Docker, a `deploy` user in the `docker` group with your key, `REMOTE_DIR` owned by -it, swap, unattended upgrades. Keep every host on **UTC**. +Identical on every host, once — the one step that needs root, rendered for this +environment: + +```bash +deployctl server bootstrap-script --env production > bootstrap.sh +``` + +Paste it into each droplet's user data when you create it, or run it: +`ssh root@ 'bash -s' < bootstrap.sh`. Docker, a `deploy` user in the `docker` +group with your key, `REMOTE_DIR` owned by it, swap, key-only ssh, UTC — the steps of +[10-SINGLE-SERVER.md](10-SINGLE-SERVER.md#1-bootstrap-the-host-once-by-hand), as one script. Verify each one: @@ -127,7 +138,6 @@ REGISTRY_TOKEN=ghp_… # classic PAT, read:packages only ```sh MODE=cluster API_SUBDOMAIN=api -IMAGE_TAG=fb31c25 HOSTS="203.0.113.10 203.0.113.11" # ssh-reachable addresses PRIMARY_HOST=203.0.113.10 # scheduler + migrations; must be in HOSTS @@ -166,12 +176,12 @@ if you mount the provider's CA into your image. ```bash deployctl ci init --branch main # then enable read/write workflow permissions -git push # → the run Summary prints IMAGE_TAG +git push # → the run Summary prints the image tag deployctl setup --env production deployctl validate --env production deployctl doctor --env production # every host: ssh, docker, permissions, image, arch -deployctl deploy init --env production +deployctl deploy init --env production --tag ``` Verify through the load balancer, not a host: @@ -205,14 +215,19 @@ scoped, because trusting it from anywhere lets a client spoof its way past rate ## 6. Rolling updates ```bash -$EDITOR config/production.env # bump IMAGE_TAG -deployctl deploy update --env production +deployctl ci deploy --env production --tag # through GitHub — or a merge, with AUTO_DEPLOY on +deployctl deploy update --env production --tag # straight from this machine ``` What happens: migrations run once on the primary, then each host in turn is synced, -pulled, restarted, nginx-reloaded and **health-gated**. The roll stops at the first host -that does not come back healthy — so the remaining hosts are still serving the previous -release while you investigate. +pulled, restarted, nginx-reloaded, **health-gated**, and watched for a minute +(`DEPLOY_SETTLE_SECONDS`) for any service that restarts or turns unhealthy. + +When a host fails, **the whole fleet goes back** (`REVERT_SCOPE=fleet`, the default): the +failed host is restored from its snapshot, then every host this run already moved, newest +first — so the load balancer is never left in front of two releases, even when nobody is +watching a deploy that ran from CI. Hosts not yet reached were never touched. +`REVERT_SCOPE=host` reverts only the failed host and tells you the fleet is split. Prove it, from another terminal: @@ -232,7 +247,8 @@ load balancer's API — neither is built in. If a release turns out to be bad after it fully rolled: ```bash -deployctl deploy rollback --env production # back to the previous tag +deployctl ci deploy --env production --rollback # back to the previous release, through GitHub +deployctl deploy rollback --env production # the same, from this machine ``` Rollback moves the image only — **migrations are not reverted**. Prefer additive, @@ -249,7 +265,10 @@ Append-only: 2. Add it to the load balancer's backends. 3. Add it to **both** databases' trusted sources. 4. Add its address to `HOSTS` in `config/production.env`. -5. `deployctl setup --env production && deployctl deploy update --env production` +5. With CI: `deployctl ci setup-key --env production --rotate` (the key onto the new host), + `deployctl ci pin-hosts` (its host key — connect once first), `deployctl ci sync-config` + (the new `HOSTS`), `deployctl ci init --force` (the job's timeout grows with the fleet). +6. `deployctl ci deploy --env production` — or from this machine, `deployctl deploy update --env production`. No re-architecture. Do not change `PRIMARY_HOST` unless you mean to move the scheduler. diff --git a/docs/30-OPERATIONS.md b/docs/30-OPERATIONS.md index 4a5094e..4c6b786 100644 --- a/docs/30-OPERATIONS.md +++ b/docs/30-OPERATIONS.md @@ -12,11 +12,9 @@ top of the Deploy tab, drawn in this order; from the CLI it is the same six step ### What you edit -On a routine release, **exactly one value changes**: - -| Value | Where | Set it to | -|---|---|---| -| `IMAGE_TAG` | `config/.env`, or *Application image* in the panel | The tag CI published — a git short SHA. Never `latest`. | +On a routine release **nothing in `config/` changes**: the image tag is passed to the +deploy (`--tag`, or by CI), never written down — a tag in a file is whatever that machine +deployed last. Pick the tag CI published, a git short SHA; never `latest`. Everything else stays as it is. If you are *also* changing configuration — worker counts, a rate limit, a new application setting — edit it in the same pass, before step 1, so one @@ -24,7 +22,7 @@ release carries both. Which file: | Changing | Edit | Notes | |---|---|---| -| Image tag | `config/.env` | The only edit a plain release needs. | +| Image tag | nowhere — `--tag` on the deploy | Without it, a deploy redeploys the running tag. | | Project name, domain, registry | `config/common.env` | Shared by every environment. | | Hosts, TLS, database, limits | `config/.env` | This environment only. | | Your app's own env keys | *Application settings* in the panel (writes `config/app..env`) | Declared in `project/fields.toml`. A value set there beats the template's default. | @@ -45,15 +43,14 @@ values from the host's live `.env.` instead. ```bash # 1. pick the tag deployctl image tags # what is in the registry -$EDITOR config/.env # set IMAGE_TAG # 2-4. checks — none of these change a running deployment -deployctl setup --env --force # re-render generated/ from the config +deployctl setup --env --tag --force # re-render generated/ with that tag deployctl validate --env # lint the config and the artifacts deployctl doctor --env # every host: ssh, docker, dir, image, arch # 5. the release itself -deployctl deploy update --env # rolling, one host at a time, health-gated +deployctl deploy update --env --tag # rolling, health-gated, reverted on failure # 6. confirm deployctl deploy status --env @@ -152,7 +149,7 @@ deployctl backup restore --env --file .sql.gz --db _restore installs by hand once, to see it work. - Those dumps sit on the same host as the data — enough for a bad migration or a deleted row, not for losing the host. Turn on your provider's backups or volume snapshots too, and - pull copies off it with `backup run` (it fetches to `deploy/backups/`). + pull copies off it with `backup run` (it fetches to `~/.deployctl/backups///`). - `--keep N` controls how many stay on the host. - With a **managed** database the provider also has snapshots; these dumps are still worth having, because they are portable and restore into anything. diff --git a/docs/35-CONTINUOUS-DEPLOYMENT.md b/docs/35-CONTINUOUS-DEPLOYMENT.md index b339d9a..880599a 100644 --- a/docs/35-CONTINUOUS-DEPLOYMENT.md +++ b/docs/35-CONTINUOUS-DEPLOYMENT.md @@ -1,192 +1,89 @@ # 35 — Continuous deployment (GitHub Actions) -A merge to the deploy branch builds the image, and — once switched on — deploys it. The -deploy is deployctl itself, run by a GitHub Actions job instead of your laptop: the same -render, doctor, migrate-once, host-by-host health gate and deploy lock. Your laptop and -the control panel keep working, and share that lock. +A merge to the deploy branch builds the image and — once switched on — deploys it. The +deploy is deployctl itself, installed in a GitHub Actions job instead of run from your +laptop: the same render, doctor, migrate-once, host-by-host health gate, service watch, +fleet-wide revert and deploy lock. Your laptop and the control panel keep working beside it +and share that lock. ``` -PR ─► prod ─merge─► ci.yml: tests · migrations · lint · merges - └─► publish (build-image.yml) ──tag──► deploy (deploy.yml) - environment: production - ssh with the CI key, pinned host keys - config from the DEPLOYCTL_CONFIG secret - deployctl deploy update - -Manual: Actions → deploy → Run workflow (a tag · update | rollback) -Emergency: your laptop or the panel — same engine, same lock on the primary +PR ─► deploy branch ─merge─► ci.yml: your checks + └─► build-image.yml ──tag──► deploy.yml (with AUTO_DEPLOY on) + installs deployctl vX.Y.Z (pinned) + ssh with the CI key, pinned host keys + config from the DEPLOYCTL_CONFIG secret + deployctl deploy update --tag + +By hand: deployctl ci deploy [--tag T | --rollback] (the same workflow, from your terminal) +Emergency: deployctl deploy update --tag T (straight over ssh, same lock) ``` ---- - -## 1. What you set up once - -| What | Where | Step | -|---|---|---| -| A CI-only ssh key pair | private half → GitHub secret, public half → the server | 2, 5 | -| The server's host key | GitHub variable, so CI never trusts on first use | 3 | -| A GitHub environment `production` | locked to the `prod` branch (Team / Pro plans) | 4 | -| Your `config/` | one GitHub secret, uploaded by `deployctl ci sync-config` | 6 | -| `AUTO_DEPLOY=true` | repository variable — the on switch | 8 | - -Nothing on the server changes except one line in `authorized_keys`. - -> **Your GitHub plan decides where secrets live.** A **private repository on GitHub Free** -> has no environment secrets, no environment variables, no deployment-branch rules and no -> branch protection. Everything then goes at **repository** level (`--repo-level`, and -> `gh secret set` without `--env`), and two things weaken: -> -> - repository secrets are readable by a workflow on **any** branch, so everyone with write -> access can reach the ssh key — root on your servers; -> - nothing stops a direct push to `prod`, which now deploys. -> -> The workflow still refuses runs from any branch but `prod`, but that is a guard against -> accidents, not against someone with write access. **GitHub Team** (per user, per month) -> gives both back: environment secrets readable only from `prod`, and a protected `prod`. -> For client production, that is worth it. The workflow is the same either way — -> `secrets.X` reads the environment's value first, then the repository's. - -> **A repository "Deploy key" is not this.** Settings → Deploy keys lets a *machine* clone -> or push the *repository*. CI needs the opposite direction — to log in to the *server* — -> which is an ssh key pair whose private half is an Actions secret. The server never needs -> the repository at all: it pulls images. - ---- - -## 2. Make the CI key — on your laptop - -Generate it on your laptop, not the server: the private half is going to GitHub, and a -private key that also sits on the server it unlocks protects nothing. Use a new key rather -than your own — it can then be revoked without locking you out, and the server's -`authorized_keys` says which logins are CI. - -```bash -ssh-keygen -t ed25519 -N '' -C 'github-actions@' -f ~/.ssh/_ci_deploy -``` - -`-N ''` means no passphrase: nobody is there to type one. The file's protection is that it -leaves your laptop for GitHub in step 5 and is then deleted. - -Install the public half with the access you already have. Prefix it with `restrict`, which -turns off port, agent and X11 forwarding and PTYs — none of which a deploy uses: - -```bash -ssh @ "cat >> ~/.ssh/authorized_keys" <<< "restrict $(cat ~/.ssh/_ci_deploy.pub)" -``` - -Prove it works as CI will use it — only that key, no agent: - -```bash -ssh -i ~/.ssh/_ci_deploy -o IdentitiesOnly=yes -o IdentityAgent=none @ 'docker ps >/dev/null && echo ok' -``` - -> The deploy user is in the `docker` group, which is root-equivalent on that host. Treat -> this key as root: it lives only in the GitHub environment secret, scoped to one branch. - ---- - -## 3. Pin the server's host key - -Your laptop has connected before, so its `known_hosts` already holds the key you trusted. -Copy that entry, for every address in `HOSTS`: - -```bash -ssh-keygen -F | grep -v '^#' -``` - -That output is the `DEPLOY_KNOWN_HOSTS` variable. The workflow sets -`DEPLOYCTL_SSH_STRICT=1`, so a host missing from it fails with *Host key verification -failed* instead of being trusted on sight — on a fresh runner, "accept new" would mean -trusting whatever answers on :22, every run. - ---- - -## 4. Create the GitHub environment (Team / Pro / public repositories) - -On GitHub Free with a private repository, skip this step: the workflow still records -Deployments under the name `production`, but there is nothing to configure. - -**Settings → Environments → New environment → `production`** (the name must match -`config/.env`). - -- **Deployment branches and tags → Selected branches → `prod`.** Only runs from `prod` can - read this environment's secrets — a workflow started from any other branch is refused. -- **Required reviewers** are optional: a button someone presses before each deploy. Merging - to `prod` is already the approval when the branch is protected (below). Availability on - private repositories depends on your plan. - -And protect the branch, because merging to it now deploys: -**Settings → Rules → Rulesets → New branch ruleset**, target `prod`: require a pull -request, require the status checks `Tests (no database)`, `Migrations on a fresh Postgres`, -`Lint` and `Merges keep both sides`, block force pushes and deletions. +**Where the tag comes from.** The build publishes `` and hands exactly that tag +to the deploy; nothing recomputes it. A manual deploy without `--tag` redeploys what the +primary is running — which is how a config change goes out. The history of what was +deployed, by whom (an Actions run, or `user@machine`), lives on the primary, so a laptop and +CI always agree on "the previous release". --- -## 5. Secrets and variables +## Setting it up — one command per step -On the **`production` environment** where your plan has one — environment scope is what -the branch rule protects — otherwise on the **repository**: +Run these from the application's repository, with `gh` logged in (`gh auth login`). After +each, `deployctl ci doctor` shows what is left. -| Name | Kind | Value | +| # | Command | What it does | |---|---|---| -| `DEPLOY_SSH_KEY` | secret | the private key from step 2 | -| `DEPLOY_KNOWN_HOSTS` | variable | the lines from step 3 | -| `APP_URL` | variable | optional — e.g. `https://api.`; linked from each Deployment | -| `DEPLOYCTL_CONFIG` + `DEPLOYCTL_CONFIG_DIGEST` | secret + variable | step 6 sets both | - -```bash -# with an environment (Team / Pro): add --env production to each gh command -gh secret set DEPLOY_SSH_KEY < ~/.ssh/_ci_deploy -gh variable set DEPLOY_KNOWN_HOSTS --body "$(ssh-keygen -F | grep -v '^#')" -rm ~/.ssh/_ci_deploy # GitHub holds the only copy now; lost = make a new one -``` - -Optional, at repository or organisation level: `SLACK_WEBHOOK_URL` (a failed deploy posts -there; otherwise GitHub's failure email is the alert). - ---- - -## 6. Upload the config - -CI has no `config/`. `ci sync-config` packs this environment's four files into one secret, -with a digest beside it: - -```bash -deployctl ci sync-config --env production # GitHub Free, private repo: add --repo-level -deployctl ci status --env production # "matches" — the laptop and CI agree -``` - -**Re-run it after every config change made on this machine** — in the panel or by hand. -If you forget, nothing is silently reverted: the workflow sets `DEPLOYCTL_CONFIG_STRICT=1`, -which refuses to change any value the servers are running with, and names the keys. - -What is inside, and what it cannot contain: only `common.env`, `.env`, `app..env` -and `secrets..env`. The job masks every credential in them in the log, and refuses a -bundle for another environment or one that does not match its digest. - ---- - -## 7. First deploy — by hand, with the tag already running - -Before automating, run it once where the outcome is known. Deploy the tag production is -already on: it proves the key, the host key, the config and the registry login end to end, -and changes nothing. - -```bash -deployctl deploy status --env production # "deployed tag: " -gh workflow run deploy.yml --ref prod -f environment=production -f tag= -f action=update -gh run watch -``` - -(Or Actions → **deploy** → Run workflow, branch `prod`.) - ---- - -## 8. Switch it on - -**Settings → Secrets and variables → Actions → Variables → New repository variable:** -`AUTO_DEPLOY` = `true`. From the next merge to `prod`, `ci.yml` hands the tag it just -published to `deploy.yml`. Set it to anything else to go back to deploying by hand. +| 1 | `deployctl ci connect --env production --branch prod` | Finds the repository and its plan, decides where secrets live (`CI_SCOPE`), saves the deploy branch | +| 2 | `deployctl ci init --env production` | Writes `build-image.yml` and `deploy.yml` (managed), and `ci.yml` if there is none (yours) | +| 3 | `deployctl ci setup-key --env production` | A CI-only ssh key: installed on every host and the jump host, proven, private half into GitHub, local copy deleted | +| 4 | `deployctl ci pin-hosts --env production` | The host keys your machine already trusts, into GitHub — CI refuses any other | +| 5 | `deployctl ci sync-config --env production` | Your `config/` into GitHub as one secret plus its digest | +| 6 | `deployctl ci doctor --env production` | Everything above, checked | +| 7 | commit and push the workflows | the deploy workflow must be on the default branch and the deploy branch | +| 8 | `deployctl ci deploy --env production` | A first deploy by hand — the running tag, so nothing changes but the proof | +| 9 | `deployctl ci auto-deploy on --env production` | From now on a merge deploys itself | + +Two things only you can do, once, on GitHub: + +- **Workflow permissions** — Settings → Actions → General → "Read and write permissions", or + the image push fails with `denied: permission_denied`. +- **While deployctl's repository is private**, the deploy job needs a token to install it: a + fine-grained personal access token with *Contents: read* on that repository, stored as + `DEPLOYCTL_INSTALL_TOKEN` (`gh secret set DEPLOYCTL_INSTALL_TOKEN`). `ci doctor` says when + it is missing, and stops asking once the repository is public. + +### Your GitHub plan decides where secrets live + +A **private repository on GitHub Free** has no environment secrets, no deployment-branch +rules and no branch protection, so `ci connect` sets `CI_SCOPE=repository`: everything goes +on the repository. Two things weaken there — repository secrets are readable by a workflow on +**any** branch, so everyone with write access can reach the ssh key (root on your hosts), and +nothing stops a direct push to the deploy branch, which now deploys. The workflow still +refuses runs from any other branch, but that guards against accidents, not against someone +with write access. **GitHub Team** gives both back, and nothing else changes: `ci connect` +switches to `environment` scope, and the same workflow reads the environment's secrets first. + +### The CI key + +`ci setup-key` never lets the private key touch disk outside a private temporary directory, +and deletes it: GitHub holds the only copy — lost means `ci setup-key --rotate`, which +replaces it on every host. The public half is marked `deployctl-ci@/` and +installed with `restrict` (no forwarding, no PTY). The deploy user is in the `docker` group, +which is root-equivalent on the host: treat the key as root. + +### Reaching the hosts + +GitHub's runners connect from a large, changing set of addresses. Three ways in: + +- **Port 22 open** to the internet, key-only (`server bootstrap-script` turns passwords + off). Common and acceptable for a single server. +- **A tailnet** — the job joins your Tailscale network for its duration, and :22 need not be + public at all. Set the variable `TAILSCALE_TAGS` (e.g. `tag:ci`) and the secrets + `TS_OAUTH_CLIENT_ID` / `TS_OAUTH_SECRET` (an OAuth client with the writable `auth_keys` + scope); point `HOSTS` at the tailnet addresses and re-run `ci pin-hosts`. +- **A jump host** — `SSH_JUMP_HOST=deploy@bastion.example.com` in `config/.env`. Every + connection goes through it (ssh `ProxyJump`); `ci setup-key` installs the CI key there too + and `ci pin-hosts` pins its host key. The usual shape for a fleet on a private network. --- @@ -194,29 +91,19 @@ published to `deploy.yml`. Set it to anything else to go back to deploying by ha | You want to | Do | |---|---| -| Ship code | Merge the PR into `prod`. | -| Change config (a key, a worker count) | Panel → Save → `deployctl ci sync-config --env production` → Actions → deploy, the **running** tag, `allow_config_change` ticked. | +| Ship code | Merge into the deploy branch. | +| Change config (a key, a worker count) | Panel → Save, or edit `config/` → `deployctl ci sync-config` → `deployctl ci deploy --allow-config-change` (redeploys the running tag with the new config). | | Ship code that needs a new key | The config change first (row above), then merge. | -| Roll back | Actions → deploy, the previous tag from **Code → Deployments**, action `rollback`. Never migrates. | -| Deploy from the laptop anyway | As before. Pick the tag production runs or a newer one — `deploy update` refuses a tag older than the running one, which is what a laptop's stale `IMAGE_TAG` would be. | - ---- - -## Reaching the server: open :22 or a tailnet +| Roll back | `deployctl ci deploy --rollback` — the release before the running one; never migrates. `--tag T` for a specific one. | +| See what happened | `deployctl ci runs`, and `deployctl deploy history` for what each release was and who shipped it. | +| Deploy with GitHub down | `deployctl deploy update --tag T` from a laptop with the config — same engine, same lock. It refuses a tag older than the running one. | -GitHub's runners connect from a large, changing set of addresses, so the workflow needs -:22 reachable from the internet — check your cloud firewall allows it. With key-only -login (`PasswordAuthentication no`, `PermitRootLogin no`) that is a common, acceptable -setup. +**If you forget `sync-config`** after a config change, nothing is silently reverted: the +workflow refuses to change any value the hosts run with and names the keys. -The stronger one closes :22 to the internet entirely: install Tailscale on the server, and -the job joins your tailnet for its duration. Set the variable `TAILSCALE_TAGS` (e.g. -`tag:ci`) and the secrets `TS_OAUTH_CLIENT_ID` / `TS_OAUTH_SECRET` (an OAuth client with the -writable `auth_keys` scope and that tag); the workflow's Tailscale step runs only when -`TAILSCALE_TAGS` is set. Point `HOSTS` at the tailnet names — for your laptop too — and -re-pin `DEPLOY_KNOWN_HOSTS` for them. Tailscale's access rules can then let `tag:ci` reach -only port 22 on that client's servers, and `authorized_keys` can add -`from="100.64.0.0/10"` to accept the CI key only from the tailnet. +**A failed release** is reverted on the host that failed and on every host the run already +moved (`REVERT_SCOPE=fleet`), so the fleet is never left split between two releases, and the +run goes red. Migrations are not reverted; the older code runs on the newer schema. --- @@ -224,13 +111,14 @@ only port 22 on that client's servers, and `authorized_keys` can add | The job says | Meaning | |---|---| -| `deploys run only from prod` | The run was started from another branch. Run it from `prod`. | -| `DEPLOYCTL_CONFIG is empty` | Never uploaded, or uploaded to the other scope — environment vs `--repo-level`. | -| `does not match DEPLOYCTL_CONFIG_DIGEST` | Secret and variable were set separately. `ci sync-config --force`. | -| `Host key verification failed` | An address in `HOSTS` is missing from `DEPLOY_KNOWN_HOSTS` (step 3). | -| `Permission denied (publickey)` | The public key is not in the deploy user's `authorized_keys`, or the secret holds the wrong half. | -| `would CHANGE config values … running with` | GitHub's config copy differs from the live one. Stale? `ci sync-config`. Intended? Re-run with `allow_config_change`. | -| `refusing to generate new ones (DEPLOYCTL_NO_MINT=1)` | The uploaded bundle has no `secrets..env` values. Re-sync from the machine that has them. | -| `… is running X, which is NEWER than Y` | An update to an older tag. That is a rollback. | -| `another run holds the deploy lock` | Shows who — a laptop, or a link to the Actions run. | -| Job waits and never starts | Concurrency: another deploy to this environment is still running. | +| `deploys run only from ` | The run was started from another branch. | +| `DEPLOYCTL_CONFIG is empty` | Never uploaded, or uploaded to the other scope — `ci doctor`. | +| `does not match DEPLOYCTL_CONFIG_DIGEST` | Secret and variable were set separately: `ci sync-config --force`. | +| `Host key verification failed` | A host (or the jump host) is missing from `DEPLOY_KNOWN_HOSTS`: `ci pin-hosts`. | +| `Permission denied (publickey)` | The CI key is not on that host: `ci setup-key --rotate`. | +| Installing deployctl fails with 404 / auth | `DEPLOYCTL_INSTALL_TOKEN` missing or without access to deployctl's repository. | +| `would CHANGE config values … running with` | GitHub's config differs from the hosts'. Stale? `ci sync-config`. Intended? `ci deploy --allow-config-change`. | +| `refusing to generate new ones (DEPLOYCTL_NO_MINT=1)` | The uploaded bundle has no `secrets..env` values — re-sync from the machine that has them. | +| `… is running X, which is NEWER than Y` | An update to an older tag — that is a rollback. | +| `another run holds the deploy lock` | Shows whose — a laptop, or a link to the Actions run. | +| Job waits and never starts | Another deploy to this environment is still running. | diff --git a/docs/60-COMMAND-REFERENCE.md b/docs/60-COMMAND-REFERENCE.md index 95e4074..6b7f5c9 100644 --- a/docs/60-COMMAND-REFERENCE.md +++ b/docs/60-COMMAND-REFERENCE.md @@ -14,6 +14,13 @@ Every command takes `--env ` (`-e`). It is inferred when only one environm configured, and may come from `DEPLOYCTL_ENV`. With several configured and no name given, the command stops and lists them rather than guessing. +**Image tags are not configuration.** Once CI deploys, a tag written in `config/` is whatever +that machine deployed last — stale by design. Each command decides it instead: `--tag` (or +`$IMAGE_TAG`, which CI sets) first; then, for commands that act on the hosts, the tag the +primary is **running** — so `deploy update` without `--tag` redeploys the running release, which +is how a config change goes out; for local commands, the tag last used on this machine. An +`IMAGE_TAG` still in `config/` is honoured with a deprecation warning; `migrate-config` removes it. + Every `deploy` command takes `--dry-run`, which prints each ssh, rsync and compose command instead of running it — and opens no connection at all, so it is safe against production. @@ -34,10 +41,12 @@ Two options go before the command, and apply to all of them: picks the deployment shape; `--env` names the environment (default `production`). Run it again with a different `--env` to add another. Does not overwrite an existing config. -### `deployctl setup --env E [--force] [--rotate-secrets]` +### `deployctl setup --env E [--tag T] [--force] [--rotate-secrets]` **Touches: local.** Renders `generated//` — the compose files, nginx config, env file -and Redis config — from the resolved configuration. `--force` overwrites an existing -render. `--rotate-secrets` re-mints the generated secrets. +and Redis config — from the resolved configuration. `--tag` names the image to render; without +it, the tag last used on this machine (see *Image tags* above). A deploy re-renders with its own +tag, so this is for inspecting and validating. `--force` overwrites an existing render. +`--rotate-secrets` re-mints the generated secrets. > **`--rotate-secrets` logs every user out.** `JWT_SECRET_KEY` signs sessions; changing it > invalidates all of them. Without the flag, secrets are minted once and reused forever, so @@ -59,6 +68,12 @@ when `deploy/` exists, or when the copied code has uncommitted edits that would untracked files are never deleted — they are listed. `--from` names the copy when it is not the one found from the current directory. +### `deployctl migrate-config [--apply]` +**Touches: local.** Brings `config/` up to the current tool. Today: removes `IMAGE_TAG` from +every `config/.env` — the tag is decided per command now (see *Image tags*) — and keeps +its value as this machine's last-used tag, so `setup` still works. Prints the plan unless +`--apply` is given; comments and every other line are left as they were. + ### `deployctl selftest [--skip-docker]` **Touches: local.** Renders every supported deployment shape into a temporary directory and asserts the output. Needs no configuration and no servers — this is the command that tells @@ -89,21 +104,56 @@ everything in full; `--key K` prints one value and nothing else, for scripting. short SHAs are shown — a moving pointer like `latest` is exactly what should not be pinned. Implemented for ghcr.io; needs `REGISTRY_TOKEN` with `read:packages`. -### `deployctl image push --env E [--tag T] [--no-pin] [--allow-dirty]` +### `deployctl image push --env E [--tag T] [--allow-dirty]` **Touches: local + registry.** Builds the image on this machine and pushes it. The tag -defaults to the current git short SHA, and is written into this environment's config -afterwards — `--no-pin` skips that. `--allow-dirty` builds despite uncommitted changes; +defaults to the current git short SHA; deploy it with `deploy update --tag T`. `--allow-dirty` +builds despite uncommitted changes; without it the command refuses, because a tag derived from a SHA that does not describe the built bits makes "what is running?" unanswerable. Useful before CI exists; CI is the better long-term answer. -### `deployctl ci init --env E [--branch B] [--force]` -**Touches: local.** Writes `.github/workflows/build-image.yml`, which builds and pushes on -every push to `--branch` (default `main`). `--force` overwrites an existing workflow. +## Continuous deployment — `ci` + +The steps of setting up deploys from GitHub Actions, one command each, in the order a +project goes through them. Every `ci` command needs `gh`, logged in, run from inside the +application's repository. The whole path, explained: [35-CONTINUOUS-DEPLOYMENT.md](35-CONTINUOUS-DEPLOYMENT.md). -> The workflow bakes `IMAGE_REPO` in at generation time. Change `IMAGE_REPO` later and the -> two silently disagree — `validate` checks for this. +### `deployctl ci connect --env E [--branch B]` +**Touches: GitHub (read), local.** Checks `gh`, reads the repository and its owner's plan, +and records where deploy secrets can live as `CI_SCOPE` in `config/common.env`: +`environment` (the GitHub environment named like this one — needs GitHub Team/Pro for a +private repository) or `repository` (every plan). A private repository on GitHub Free, or a +plan this account cannot see, gets `repository`, with a warning that repository secrets are +readable from any branch. `--branch` saves the deploy branch as `DEPLOY_BRANCH`. + +### `deployctl ci init --env E [--branch B] [--force]` +**Touches: local.** Writes `.github/workflows/`: `build-image.yml` (builds and pushes the +image, and hands its tag on as the `tag` output) and `deploy.yml` (installs this version of +deployctl, pinned, and deploys), both **managed** — regenerated from this version's +templates, with every third-party action pinned to a commit SHA and the deploy job's timeout +sized to the number of hosts. A managed file that differs is shown as a diff and only +replaced with `--force`. `ci.yml` is written only when there is none — it is the project's +own (its checks go there); an existing one is left alone, and if it never calls the two +workflows, the jobs to add are printed. `--branch` saves `DEPLOY_BRANCH` first. + +> The build workflow bakes `IMAGE_REPO` in at generation time. Change `IMAGE_REPO` later and +> the two silently disagree — `validate` checks for this. + +### `deployctl ci setup-key --env E [--rotate] [--repo-level]` +**Touches: hosts (write), GitHub.** Makes an ssh key only CI uses and puts it where CI needs +it: generated in a private temporary directory; its public half installed on every host — +and on `SSH_JUMP_HOST` — prefixed `restrict`, using your own ssh access; then proven to log +in on its own (no agent) before its private half is uploaded as `DEPLOY_SSH_KEY`. The local +copy is deleted; only the fingerprint is printed. The key is marked +`deployctl-ci@/`: an installed one is only replaced with `--rotate`, which +removes it from every host and installs the new one. + +### `deployctl ci pin-hosts --env E [--repo-level]` +**Touches: GitHub.** Uploads the host keys this machine already trusts — every host and the +jump host, from your `known_hosts` — as `DEPLOY_KNOWN_HOSTS`. CI refuses any host key it was +not given. A host you have never connected to is refused rather than looked up: connect +once, checking the fingerprint, then run it again. ### `deployctl ci sync-config --env E [--force] [--repo-level]` **Touches: GitHub.** Uploads this environment's `config/` files (`common.env`, `.env`, @@ -130,11 +180,51 @@ its digest or holds files of another environment. Under GitHub Actions it first every credential in the bundle with `::add-mask::`. `--env` is required, not inferred. Existing files with different contents are only replaced with `--force`. + +### `deployctl ci doctor --env E [--json] [--repo-level]` +**Touches: GitHub (read).** Checks everything continuous deployment needs — `gh`, the +secrets' scope, the workflows (present, current for this version, `ci.yml` calling +`deploy.yml`), `DEPLOY_SSH_KEY`, `DEPLOY_KNOWN_HOSTS` covering every host and the jump host, +the config in GitHub matching this machine's, the install token while deployctl's repository +is private, and whether `AUTO_DEPLOY` is on — and prints the command that fills each gap. +`--json` is what the control panel reads. Exit `0` when nothing is missing, `1` otherwise. + +### `deployctl ci deploy --env E [--tag T] [--rollback] [--allow-config-change] [--watch]` +**Touches: GitHub → hosts (write).** Runs the deploy workflow — the same one a merge runs, +with GitHub's copy of the config and the CI key — and follows it to the end (`--no-watch` +returns once started). Without `--tag`, the tag the primary is running: a redeploy, which is +how a config change goes out (`--allow-config-change` lets it change values the hosts run). +`--rollback` rolls back instead, by default to the release before the running one, and never +migrates. Runs on `DEPLOY_BRANCH`. + +### `deployctl ci runs --env E [--limit N] [--json]` +**Touches: GitHub (read).** Recent deploys through GitHub, newest first — manual runs and the +deploys merges triggered (skipped ones, with `AUTO_DEPLOY` off, are left out) — with action, +tag, outcome and a link. + +### `deployctl ci auto-deploy --env E` +**Touches: GitHub.** Sets the repository variable `AUTO_DEPLOY`: on, every push to +`DEPLOY_BRANCH` that passes its checks deploys itself; off, deploys are `ci deploy`. Always +at repository level — `ci.yml` reads it before any environment is entered. + +--- + +## Servers + +### `deployctl server bootstrap-script --env E [--swap SIZE] [--harden-ssh] [--reboot]` +**Touches: nothing — prints.** The root script that prepares a fresh Ubuntu server for this +environment: updates, Docker from Docker's repository, the deploy user (`SSH_USER`, in the +`docker` group, with root's authorised keys), `REMOTE_DIR` owned by it, swap (`--swap`, +default `2G`), key-only ssh (`--no-harden-ssh` skips it), and a reboot when the updates +asked for one (`--no-reboot`). Safe to run again. Only the script is printed on stdout, so +it can be pasted into a cloud provider's user data or piped: +`deployctl server bootstrap-script --env E | ssh root@IP 'bash -s'`. + --- ## Deploying -### `deployctl deploy doctor --env E [--fix]` +### `deployctl deploy doctor --env E [--tag T] [--fix]` **Touches: hosts (read).** Per host: ssh reachable, docker usable without sudo, `REMOTE_DIR` writable, permissions on everything a deploy writes or mounts, CPU architecture matches `IMAGE_PLATFORM`, and the image tag resolves from that host. It also compares the `.env` @@ -146,19 +236,28 @@ what a lost or re-created `config/` produces. See `deploy update --allow-secret- needing root is printed as an exact command instead — a deploy user that can escalate defeats the point of having one. -### `deployctl deploy init --env E` -**Touches: hosts (write).** First bring-up. Runs doctor, stages artifacts and pulls the +### `deployctl deploy init --env E [--tag T]` +**Touches: hosts (write).** First bring-up — new hosts run nothing yet, so pass `--tag` the +first time. Runs doctor, stages artifacts and pulls the image everywhere, migrates once on the primary, then starts hosts primary-first behind a health gate. Run once per environment; use `update` after that. -### `deployctl deploy update --env E [--host H] [--allow-secret-change]` +### `deployctl deploy update --env E [--tag T] [--host H] [--allow-secret-change]` **Touches: hosts (write).** The rolling release. Re-renders artifacts, runs doctor, migrates once on the primary, then per host: set the running release aside → push → pull → `up -d` → reload nginx → health gate → watch every service for `DEPLOY_SETTLE_SECONDS` -(default 60) → record tag. A host that fails the gate or the watch is reverted to its -previous release and the roll stops, leaving the rest on the previous release -(`DEPLOYCTL_NO_REVERT=1` keeps the failed release for inspection). `--host` rolls one host only (no migration, no doctor — but still the -`.env` comparison), and is recorded in history as a one-host roll, not a release. +(default 60) → record the tag and who deployed it. Without `--tag`, the running tag is +redeployed. + +A host that fails the gate or the watch is reverted to its previous release and the roll +stops — and with `REVERT_SCOPE=fleet` (the default) **every host this run already moved is +put back too**, newest first, each from its own snapshot, so the fleet is never left split +between two releases. `REVERT_SCOPE=host` reverts only the failed host. +`DEPLOYCTL_NO_REVERT=1` keeps the failed release for inspection. A completed release is +appended to the history on the primary; a reverted one is not. + +`--host` rolls one host only (no migration, no doctor — but still the `.env` comparison), +and is recorded as a one-host roll, not a release. `--allow-secret-change` ships a `.env` that re-keys a generated secret or blanks a key the hosts are running with. Without it doctor refuses — the flag is for a deliberate rotation @@ -166,28 +265,27 @@ hosts are running with. Without it doctor refuses — the flag is for a delibera (the deploy workflow sets it) a changed value of **any** key counts, not only the generated secrets; the flag is how a deliberate config change gets through there too. -An update refuses a tag that is an ancestor of the one the primary is running: once CI -deploys, `IMAGE_TAG` in a laptop's config is stale, and deploying it would take production -backwards while migrating with the older image. Going back on purpose is `deploy rollback`. +An update refuses a tag that is an ancestor of the one the primary is running: deploying +it would take production backwards while migrating with the older image. Going back on purpose is `deploy rollback`. Tags that are not commits this checkout knows are not compared. Full walkthrough: [30-OPERATIONS.md § Rolling out an update](30-OPERATIONS.md#rolling-out-an-update). ### `deployctl deploy rollback --env E [--to TAG]` -**Touches: hosts (write).** Re-deploys a previous image tag with the same health-gated -roll, then pins that tag in `config/.env` so the next routine deploy does not silently -re-deploy the bad build. Defaults to the previous release from local history (one-host rolls, -rollbacks and another image repository's tags are skipped). An explicit `--to` naming the tag -config already holds rolls anyway, with a warning: CI deploys do not update config, so the -hosts may be running something newer. +**Touches: hosts (write).** Re-deploys a previous image tag with the same health-gated roll +(and the same fleet-wide revert if it fails). Defaults to the release before the one the +primary runs, from the history kept on the primary — which CI and every laptop write, so they +agree (one-host rolls and earlier rollbacks are skipped). Refuses a tag that is already +running. > **Nothing is migrated, and nothing is reverted** — only the image moves back. The older > image's migrations stop short of the revision the database is already at, so running them > fails ("Can't locate revision"); rollback does not try. The older code has to tolerate the > newer schema, which is what additive migrations guarantee. -### `deployctl deploy migrate --env E` -**Touches: hosts (write).** Runs `MIGRATE_CMD` once, on the primary. `update` already does +### `deployctl deploy migrate --env E [--tag T]` +**Touches: hosts (write).** Runs `MIGRATE_CMD` once, on the primary, with the running image +unless `--tag` names another. `update` already does this; use it alone only to re-run a migration that failed. ### `deployctl deploy restart --env E [--host H]` @@ -197,8 +295,11 @@ this; use it alone only to re-run a migration that failed. **Touches: hosts (write).** `compose down` on every host. **The site goes down and stays down** until `init` or `update`. Confirms first unless `ASSUME_YES=1`. -### `deployctl deploy status --env E` -**Touches: hosts (read).** Service status and the deployed tag on every host. +### `deployctl deploy status --env E [--json]` +**Touches: hosts (read).** Service status and the deployed tag on every host. `--json` prints +each host's tag, when and by whom it was deployed (an Actions run URL, or `user@machine`), +and every service's state, health and restart count — plus `split` when hosts disagree on +the tag. ### `deployctl deploy logs --env E [--host H]` **Touches: hosts (read).** Follows `docker compose logs -f`. Defaults to the primary. @@ -207,8 +308,9 @@ down** until `init` or `update`. Confirms first unless `ASSUME_YES=1`. **Touches: hosts (read).** Interactive shell in `REMOTE_DIR`, using your own ssh agent. ### `deployctl deploy history --env E` -**Touches: local.** The locally recorded deploys for this environment — what `rollback` -consults when no `--to` is given. +**Touches: hosts (read).** The environment's releases, from the history kept on the +primary: when, which tag, what kind (release, rollback, first bring-up, one-host roll) and +who — the record `rollback` consults when no `--to` is given. ### `deployctl deploy unlock --env E` **Touches: hosts (write).** Every host-changing command (init, update, rollback, migrate, @@ -252,7 +354,7 @@ bootstrap one. ### `deployctl backup run --env E [--keep N] [--no-fetch]` **Touches: hosts (write).** `pg_dump` on the primary, prune to the last `N` (default 7), -then fetch a copy to `deploy/backups/`. A host that dies must not take its own backups +then fetch a copy to `~/.deployctl/backups///` on this machine — never into a repository (`DEPLOYCTL_BACKUP_DIR` moves it). A host that dies must not take its own backups with it — `--no-fetch` leaves the dump on the host only. ### `deployctl backup schedule --env E [--at HH:MM] [--keep N] [--off]` diff --git a/docs/README.md b/docs/README.md index c103543..78fc800 100644 --- a/docs/README.md +++ b/docs/README.md @@ -31,23 +31,21 @@ deployctl ci init # publish images from CI deployctl setup --env production --force deployctl validate --env production deployctl doctor --env production -deployctl deploy init --env production +deployctl deploy init --env production --tag deployctl ssl setup --env production # single-server TLS only ``` -**Every release after that** — the only edit is `IMAGE_TAG`: +**Every release after that** — merge, with CI/CD set up +([35-CONTINUOUS-DEPLOYMENT.md](35-CONTINUOUS-DEPLOYMENT.md)); or by hand, naming the tag: ```bash deployctl image tags # what can I deploy? -$EDITOR config/production.env # set IMAGE_TAG -deployctl setup --env production --force -deployctl validate --env production -deployctl doctor --env production -deployctl deploy update --env production +deployctl ci deploy --env production --tag # through GitHub Actions +deployctl deploy update --env production --tag # or straight from this machine deployctl deploy status --env production ``` -Both are the two rails on the panel's **Deploy** tab, in the same order. +The panel's **Deploy** tab walks the same first deployment, one card per command. Something unclear or wrong in these docs is worth fixing — the deployment steps that are easy to get wrong are exactly the ones that produce confusing outages. diff --git a/pyproject.toml b/pyproject.toml index c2faf28..5410a48 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ deployctl = "deployctl.cli.main:main" dev = [ "pytest>=8.0", "httpx>=0.27", # FastAPI's TestClient, for the panel tests + "pyyaml>=6.0", # parsing the generated workflows in tests ] [tool.hatch.version] diff --git a/src/deployctl/__init__.py b/src/deployctl/__init__.py index 2143b0d..dfec050 100644 --- a/src/deployctl/__init__.py +++ b/src/deployctl/__init__.py @@ -1,3 +1,3 @@ """deployctl — deploy containerized apps to one server or a fleet over SSH.""" -__version__ = "0.9.0" +__version__ = "0.10.0" diff --git a/src/deployctl/cli/commands/backup.py b/src/deployctl/cli/commands/backup.py index 7b6385d..6a24f5d 100644 --- a/src/deployctl/cli/commands/backup.py +++ b/src/deployctl/cli/commands/backup.py @@ -32,7 +32,7 @@ def run_backup( env: str = env_option(), keep: int = typer.Option(7, "--keep", help="Host-side dumps to keep (older ones are pruned)."), fetch: bool = typer.Option( - True, "--fetch/--no-fetch", help="Also copy the dump to this machine (backups/)." + True, "--fetch/--no-fetch", help="Also copy the dump to this machine (~/.deployctl/backups///)." ), dry_run: bool = typer.Option(False, "--dry-run", help="Print the commands instead of running them."), ) -> None: diff --git a/src/deployctl/cli/commands/ci.py b/src/deployctl/cli/commands/ci.py index 860aadb..9d5dbdb 100644 --- a/src/deployctl/cli/commands/ci.py +++ b/src/deployctl/cli/commands/ci.py @@ -1,175 +1,414 @@ -"""``deployctl ci`` — generate the CI pipeline, and hand CI the configuration it deploys with.""" +"""``deployctl ci`` — CI/CD: the workflows, GitHub's secrets, and deploying through it. + +Every manual step of setting up continuous deployment is a command here, in the +order a project goes through them: + + ci connect gh logged in, which repository, which plan → CI_SCOPE + ci init .github/workflows: build-image.yml, deploy.yml (+ ci.yml once) + ci setup-key a CI-only ssh key on every host, its private half in GitHub + ci pin-hosts the hosts' keys from this machine's known_hosts, into GitHub + ci sync-config this machine's config/ into GitHub, as one secret + ci doctor all of the above, checked + ci deploy run the deploy workflow (a tag, a config change, a rollback) + ci runs what the workflow deployed, and how it went + ci auto-deploy merges to the deploy branch deploy themselves, or not +""" from __future__ import annotations +import datetime +import difflib +import json import os -import shutil +import re import subprocess import sys +import tempfile +import time import jinja2 import typer -from .. import bundle, paths, secrets, ui +from ... import __version__ +from .. import bundle, github, paths, secrets, tags, ui +from ..config import Config from ..context import env_option, load_config, resolve_env -from ..envfile import parse_env_text, read_env_file +from ..envfile import parse_env_text, patch_env_file, read_env_file +from ..runner import run +from .deploy import parse_history, previous_release ci_app = typer.Typer( name="ci", - help="Generate CI configuration (GitHub Actions → registry).", + help="CI/CD: generate the workflows, connect GitHub, and deploy through it.", no_args_is_help=True, ) +#: Where the deploy workflow installs deployctl from — pinned to this version. +TOOL_REPO = "hybridinteract/deployctl" + +#: Third-party actions, pinned to commit SHAs: the deploy job holds the production +#: ssh key. Bumping one is a change here, then `ci init --force` in each project. +ACTIONS = { + "checkout": "3d3c42e5aac5ba805825da76410c181273ba90b1", "checkout_version": "v7.0.1", + "setup_uv": "c18668ad3cf93ea998bef934396af7bb5c839dc7", "setup_uv_version": "v10.2.0", + "login": "dbcb813823bdd20940b903addbd779551569679f", "login_version": "v4.6.0", + "buildx": "f87e5991a6d7451dcb8d9637bfbc97413f497069", "buildx_version": "v4.4.1", + "build_push": "c3c9e263c25d99ce0380d002d59b67737d91b0dc", "build_push_version": "v7.4.0", + "tailscale": "d1b6cd204f8dceda5b3eaad7f1f767be390056cd", "tailscale_version": "v4.2.0", +} + +#: Regenerated by `ci init` — edits are overwritten, so they carry a header saying so. +MANAGED = ("build-image.yml", "deploy.yml") +#: Written once and then the project's own: its checks are its business. +OWNED = "ci.yml" +_MANAGED_HEADER = re.compile(r"^# Managed by deployctl (\S+)") + +_repo_level_option = typer.Option( + False, + "--repo-level", + help="Store at repository level this once. CI_SCOPE=repository (set by `ci connect` on " + "GitHub Free) does it every time.", +) + + +# ---- shared ------------------------------------------------------------------------ + + +def _github() -> None: + try: + github.require() + except github.GitHubError as exc: + ui.error(str(exc)) + raise typer.Exit(2) from None + + +def scope_env(cfg: Config, repo_level: bool = False) -> str | None: + """The GitHub environment the secrets live on, or None for the repository.""" + if repo_level or cfg.raw["CI_SCOPE"] == "repository": + return None + return cfg.env + + +def _where(cfg: Config, scope: str | None) -> str: + return f"GitHub environment '{scope}'" if scope else "the repository's Actions secrets" + + +def workflows_dir(): + return paths.REPO_ROOT / ".github" / "workflows" -def _render_workflow(ctx: dict) -> str: - """Render the workflow with «…» delimiters. - A GitHub Actions file is full of ``${{ … }}`` expressions that must reach - GitHub verbatim; switching Jinja2's delimiters for this one template beats - wrapping half the file in ``{% raw %}``. +def deploy_timeout(hosts: int) -> int: + """Minutes for the deploy job: a few per host, so a fleet is never cut off mid-roll.""" + return max(20, min(360, 10 + 4 * max(1, hosts))) + + +def _render(name: str, ctx: dict) -> str: + """Render a workflow with «…» delimiters. + + A GitHub Actions file is full of ``${{ … }}`` that must reach GitHub verbatim; + moving Jinja's variable delimiters beats wrapping half the file in raw blocks. + Statements keep ``{% … %}``, which GitHub never uses. """ env = jinja2.Environment( loader=jinja2.FileSystemLoader(str(paths.TEMPLATES_DIR / "ci")), undefined=jinja2.StrictUndefined, keep_trailing_newline=True, + trim_blocks=True, + lstrip_blocks=True, variable_start_string="«", variable_end_string="»", autoescape=False, ) - return env.get_template("build-image.yml.j2").render(**ctx) + return env.get_template(f"{name}.j2").render(**ctx) + + +def _context(cfg: Config) -> dict: + return { + "VERSION": __version__, + "TOOL_REPO": TOOL_REPO, + "TOOL_REF": f"v{__version__}", + "ENV": cfg.env, + "DEPLOY_BRANCH": cfg.raw["DEPLOY_BRANCH"], + "IMAGE_REPO": cfg.raw["IMAGE_REPO"], + "REGISTRY_HOST": cfg.derived["REGISTRY_HOST"] or "ghcr.io", + "CI_BUILD_CONTEXT": cfg.derived["CI_BUILD_CONTEXT"], + "CI_DOCKERFILE": cfg.derived["CI_DOCKERFILE"], + "BUILD_TARGET": cfg.raw["BUILD_TARGET"], + "WORKER_BUILD_TARGET": cfg.raw["WORKER_BUILD_TARGET"], + "IMAGE_PLATFORM": cfg.raw["IMAGE_PLATFORM"], + "HOST_COUNT": len(cfg.hosts), + "TIMEOUT_MINUTES": deploy_timeout(len(cfg.hosts)), + "ACTIONS": ACTIONS, + } -def _worker_build_step(cfg, branch: str) -> str: - """The second build-push step, when the project publishes a worker image. +def _jump_target(jump: str) -> tuple[str, str]: + """The first hop of SSH_JUMP_HOST as ``(host, port)``, user dropped.""" + first = jump.split(",", 1)[0].split("@")[-1] + host, _, port = first.partition(":") + return host, port - Emitted as text rather than a ``{% if %}`` because this template renders with - ``«…»`` delimiters and no statement tags — see :func:`_render_workflow`. + +# ---- ci connect ---------------------------------------------------------------------- + + +@ci_app.command("connect") +def connect( + env: str = env_option(), + branch: str = typer.Option(None, "--branch", "-b", help="The deploy branch, saved as DEPLOY_BRANCH."), +) -> None: + """Check gh, find the repository and its plan, and record where secrets go (CI_SCOPE). + + A private repository on GitHub Free has no environment secrets, so they go on + the repository. Anything else gets them on the GitHub environment, readable + only from the branches its rules allow. """ - target = cfg.raw["WORKER_BUILD_TARGET"] - if not target: - return "" - repo = cfg.raw["IMAGE_REPO"] - tag = "${{ steps.meta.outputs.tag }}" - return f""" - # The Celery image: the same commit and the same tag, built from a stage - # carrying tools the api deliberately does not. Pushed in this run so the - # two images can never be a commit apart. - - name: Build and push the worker image - uses: docker/build-push-action@v6 - with: - context: {cfg.derived["CI_BUILD_CONTEXT"]} - file: {cfg.derived["CI_DOCKERFILE"]} - target: {target} - platforms: {cfg.raw["IMAGE_PLATFORM"]} - push: true - tags: | - {repo}:{tag}-worker - ${{{{ github.ref == 'refs/heads/{branch}' && '{repo}:latest-worker' || '' }}}} - cache-from: type=gha - cache-to: type=gha,mode=max - provenance: false -""" + env_name = resolve_env(env) + _github() + try: + info = github.repo() + except github.GitHubError as exc: + ui.error(str(exc)) + ui.hint("run this inside the application's git repository, with a GitHub remote") + raise typer.Exit(2) from None + plan = github.plan(info["owner"]) + + if not info["private"]: + scope, why = "environment", "a public repository has environment secrets on every plan" + elif plan and plan != "free": + scope, why = "environment", f"GitHub {plan} gives a private repository environment secrets" + elif plan == "free": + scope, why = "repository", "GitHub Free has no environment secrets for a private repository" + else: + scope, why = "repository", "this account cannot see the plan; repository secrets work on every plan" + + values = {"CI_SCOPE": scope} + if branch: + values["DEPLOY_BRANCH"] = branch + patch_env_file(paths.COMMON_CONFIG, values) + cfg = load_config(env_name, require_valid=False) + + ui.header("CI connect") + ui.kv("repository", f"{info['name']} ({'private' if info['private'] else 'public'})") + ui.kv("plan", plan or "not visible to this account") + ui.kv("secrets", f"{scope} — {why}") + ui.kv("deploy branch", cfg.raw["DEPLOY_BRANCH"]) + ui.ok(f"saved CI_SCOPE={scope} in config/common.env") + if scope == "repository" and info["private"]: + ui.warn("repository secrets are readable from ANY branch by anyone with write access — including " + "the ssh key, which is root on the hosts. GitHub Team scopes them to the deploy branch.") + print() + ui.info(f"Next: deployctl ci init --env {env_name}") + + +# ---- ci init ------------------------------------------------------------------------- @ci_app.command("init") def init( env: str = env_option(), branch: str = typer.Option( - "main", "--branch", "-b", help="Branch whose pushes trigger a build (your deploy branch)." + None, "--branch", "-b", help="The deploy branch, saved as DEPLOY_BRANCH. Default: DEPLOY_BRANCH (main)." ), - force: bool = typer.Option(False, "--force", "-f", help="Overwrite an existing workflow file."), + force: bool = typer.Option(False, "--force", "-f", help="Overwrite workflows that differ from the templates."), ) -> None: - """Write .github/workflows/build-image.yml for this repository.""" - cfg = load_config(env, require_valid=False) + """Write .github/workflows/build-image.yml and deploy.yml, and ci.yml if there is none. + build-image.yml and deploy.yml are managed: regenerated from this version's + templates, and a file that differs is shown as a diff and needs --force. + ci.yml is written once and is then the project's own — its checks. + """ + cfg = load_config(env, require_valid=False) if not cfg.raw["IMAGE_REPO"] or "your-org" in cfg.raw["IMAGE_REPO"]: ui.error("IMAGE_REPO is not set — fill it in config/common.env first") raise typer.Exit(1) - - target = paths.REPO_ROOT / ".github" / "workflows" / "build-image.yml" - if target.exists() and not force: - ui.error(f"{target.relative_to(paths.REPO_ROOT)} already exists (use --force to overwrite)") + if branch: + patch_env_file(paths.COMMON_CONFIG, {"DEPLOY_BRANCH": branch}) + cfg.raw["DEPLOY_BRANCH"] = branch + + ctx = _context(cfg) + target = workflows_dir() + target.mkdir(parents=True, exist_ok=True) + blocked = [] + for name in MANAGED: + path = target / name + rel = path.relative_to(paths.REPO_ROOT) + content = _render(name, ctx) + if path.is_file(): + current = path.read_text() + if current == content: + ui.info(f"{rel} is current") + continue + if not force: + blocked.append(str(rel)) + diff = list(difflib.unified_diff(current.splitlines(), content.splitlines(), + f"{rel} (now)", f"{rel} (deployctl {__version__})", lineterm="", n=1)) + ui.warn(f"{rel} differs from the template ({sum(l.startswith('+') for l in diff[2:])} added, " + f"{sum(l.startswith('-') for l in diff[2:])} removed lines):") + for line in diff[:40]: + print(f" {line}") + if len(diff) > 40: + print(f" … {len(diff) - 40} more lines") + continue + path.write_text(content) + ui.ok(f"wrote {rel}") + + owned = target / OWNED + rel_owned = owned.relative_to(paths.REPO_ROOT) + if not owned.is_file(): + owned.write_text(_render(OWNED, ctx)) + ui.ok(f"wrote {rel_owned} — yours from now on: put your checks in its `checks` job") + else: + text = owned.read_text() + if "build-image.yml" not in text or "deploy.yml" not in text: + ui.warn(f"{rel_owned} is your own and was left alone — but it does not call " + f"{'build-image.yml' if 'build-image.yml' not in text else 'deploy.yml'}.") + ui.hint("add these two jobs to it, after your checks (and list your checks in `needs`):") + skeleton = _render(OWNED, ctx) + print(skeleton[skeleton.index(" publish:"):]) + + if blocked: + ui.error(f"not overwritten: {', '.join(blocked)} — review the diff above, then: deployctl ci init --force") raise typer.Exit(1) - - build_target = cfg.raw["BUILD_TARGET"] - content = _render_workflow( - { - "DEPLOY_BRANCH": branch, - "IMAGE_REPO": cfg.raw["IMAGE_REPO"], - "REGISTRY_HOST": cfg.derived["REGISTRY_HOST"] or "ghcr.io", - "CI_BUILD_CONTEXT": cfg.derived["CI_BUILD_CONTEXT"], - "CI_DOCKERFILE": cfg.derived["CI_DOCKERFILE"], - "IMAGE_PLATFORM": cfg.raw["IMAGE_PLATFORM"], - "BUILD_TARGET_LINE": f" target: {build_target}" if build_target else "", - "WORKER_BUILD_STEP": _worker_build_step(cfg, branch), - } - ) - - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(content) - ui.ok(f"wrote {target.relative_to(paths.REPO_ROOT)}") - - ui.separator() - ui.info("One repository setting is REQUIRED before the first run:") - print(" GitHub → Settings → Actions → General → Workflow permissions") - print(' → select "Read and write permissions" → Save') - ui.hint('without it, the image push fails with "denied: permission_denied"') print() - ui.info("Then:") - print(f" 1. Commit and push the workflow to '{branch}'") - print(" 2. Watch Actions — the run's Summary prints the IMAGE_TAG") - print(f" 3. Put that tag in config/{cfg.env}.env and run: deployctl setup --env {cfg.env}") - print() - if cfg.derived["REGISTRY_HOST"] == "ghcr.io": - ui.info("Keep the package private; hosts pull with a read:packages token (REGISTRY_TOKEN).") - + ui.info("Next, once: Settings → Actions → General → Workflow permissions → \"Read and write\" (image push)") + ui.info(f"then: deployctl ci setup-key --env {cfg.env} && deployctl ci pin-hosts --env {cfg.env} " + f"&& deployctl ci sync-config --env {cfg.env}") -# ---- the config bundle: this machine's config/ → GitHub → the deploy job ---------- +# ---- ci setup-key -------------------------------------------------------------------- -def _gh(args: list[str], *, stdin: str | None = None) -> subprocess.CompletedProcess: - """Run ``gh`` from the repository, so it resolves owner/repo from the git remote.""" - return subprocess.run( - ["gh", *args], cwd=str(paths.REPO_ROOT), input=stdin, text=True, capture_output=True - ) - -def _require_gh() -> None: - if shutil.which("gh") is None: - ui.error("the GitHub CLI (gh) is required — https://cli.github.com, then: gh auth login") - raise typer.Exit(2) +def _ci_key(cfg: Config, mode: str, **extra: str) -> list[tuple[str, str]]: + """``deploy.sh ci-key`` → ``[(verdict, where), …]``.""" + proc = run(cfg, "deploy.sh", ["ci-key"], check=False, capture=True, + extra_env={"CI_KEY_MODE": mode, **extra}) + results = [] + for line in proc.stdout.splitlines(): + verdict, _, where = line.partition(" ") + if verdict in ("PRESENT", "ABSENT", "INSTALLED", "OK", "FAILED", "UNREACHABLE"): + results.append((verdict, where)) + if not results and proc.returncode != 0: + ui.error((proc.stderr or proc.stdout).strip() or "deploy.sh ci-key failed") + raise typer.Exit(1) + return results -_repo_level_option = typer.Option( - False, - "--repo-level", - help="Store at repository level instead of on the GitHub environment — for private " - "repositories on GitHub Free, which has no environment secrets. One CI-deployed " - "environment per repository.", -) - +@ci_app.command("setup-key") +def setup_key( + env: str = env_option(), + rotate: bool = typer.Option(False, "--rotate", help="Replace the CI key already installed."), + repo_level: bool = _repo_level_option, +) -> None: + """Make a CI-only ssh key, install it on every host, and hand its private half to GitHub. -def _scope(env: str, repo_level: bool) -> list[str]: - """The ``gh secret|variable set`` flags for where the bundle lives.""" - return [] if repo_level else ["--env", env] + The key is generated in a private temporary directory and deleted when this + returns: GitHub holds the only copy. Its public half goes on every host (and + the jump host) with `restrict`, installed with your own ssh access, and it is + proven to log in on its own before it is uploaded. Only its fingerprint is + ever printed. + """ + env_name = resolve_env(env) + cfg = load_config(env_name) + _github() + info = github.repo() + scope = scope_env(cfg, repo_level) + comment = f"deployctl-ci@{info['name']}/{env_name}" + + ui.header(f"CI deploy key — {env_name}") + with tempfile.TemporaryDirectory(prefix="deployctl-ci-key-") as tmp: + key = os.path.join(tmp, "id_ed25519") + subprocess.run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", comment, "-f", key], check=True) + pub = open(key + ".pub").read().strip() + fingerprint = subprocess.run(["ssh-keygen", "-lf", key + ".pub"], capture_output=True, text=True).stdout.strip() + + now = _ci_key(cfg, "check", CI_KEY_COMMENT=comment) + installed = [where for verdict, where in now if verdict == "PRESENT"] + unreachable = [where for verdict, where in now if verdict == "UNREACHABLE"] + if unreachable: + ui.error(f"cannot reach {', '.join(unreachable)} with your own ssh access") + raise typer.Exit(1) + if installed and not rotate: + ui.error(f"a CI key for {info['name']}/{env_name} is already on {', '.join(installed)}") + ui.hint("replace it: deployctl ci setup-key --rotate (the old one stops working everywhere)") + raise typer.Exit(1) + + results = _ci_key(cfg, "rotate" if rotate else "install", CI_KEY_COMMENT=comment, CI_PUBKEY=pub) + failed = [where for verdict, where in results if verdict != "INSTALLED"] + for verdict, where in results: + (ui.ok if verdict == "INSTALLED" else ui.error)(f"{where}: {verdict.lower()}") + if failed: + ui.error("not installed everywhere — nothing was uploaded; fix the above and run again with --rotate") + raise typer.Exit(1) + + proof = _ci_key(cfg, "verify", CI_KEY_COMMENT=comment, CI_KEY_FILE=key) + refused = [where for verdict, where in proof if verdict != "OK"] + if refused: + ui.error(f"the new key does not log in on its own to {', '.join(refused)} — nothing was uploaded") + ui.hint("check the deploy user's ~/.ssh permissions there (700 for the directory, 600 for the file)") + raise typer.Exit(1) + ui.ok("the key logs in on its own to every host") + + try: + with open(key) as handle: + github.set_secret("DEPLOY_SSH_KEY", handle.read(), scope) + except github.GitHubError as exc: + ui.error(str(exc)) + raise typer.Exit(1) from None + + ui.ok(f"DEPLOY_SSH_KEY → {_where(cfg, scope)}; the local copy is deleted") + ui.kv("fingerprint", fingerprint) + ui.kv("comment", f"{comment} (what --rotate finds on the hosts)") + + +# ---- ci pin-hosts -------------------------------------------------------------------- + + +def known_hosts_lines(host: str, port: str = "") -> list[str]: + """This machine's trusted keys for a host, as known_hosts lines (hashed or not).""" + name = f"[{host}]:{port}" if port and port != "22" else host + proc = subprocess.run(["ssh-keygen", "-F", name], capture_output=True, text=True) + return [line for line in proc.stdout.splitlines() if line and not line.startswith("#")] + + +@ci_app.command("pin-hosts") +def pin_hosts(env: str = env_option(), repo_level: bool = _repo_level_option) -> None: + """Give GitHub the host keys this machine already trusts (DEPLOY_KNOWN_HOSTS). + + CI refuses any host key it was not given (DEPLOYCTL_SSH_STRICT), so every host + — and the jump host — must be here. Keys come from this machine's known_hosts, + which learned them the first time you connected: a host you have never + connected to is refused rather than looked up. + """ + env_name = resolve_env(env) + cfg = load_config(env_name) + _github() + scope = scope_env(cfg, repo_level) + targets = [(host, "") for host in cfg.hosts] + if cfg.raw["SSH_JUMP_HOST"]: + targets.append(_jump_target(cfg.raw["SSH_JUMP_HOST"])) + + lines, missing = [], [] + for host, port in targets: + found = known_hosts_lines(host, port) + if found: + lines += found + else: + missing.append(f"{host}:{port}" if port else host) + if missing: + ui.error(f"this machine has never connected to {', '.join(missing)}, so it has no key to pin") + ui.hint("connect once, checking the fingerprint it shows against the host's console: " + f"ssh {cfg.raw['SSH_USER']}@{missing[0]} true") + raise typer.Exit(1) + try: + github.set_variable("DEPLOY_KNOWN_HOSTS", "\n".join(lines), scope) + except github.GitHubError as exc: + ui.error(str(exc)) + raise typer.Exit(1) from None + ui.ok(f"DEPLOY_KNOWN_HOSTS → {_where(cfg, scope)}: {len(lines)} key(s) for {len(targets)} host(s)") -def _where(env: str, repo_level: bool) -> str: - return "the repository's Actions secrets" if repo_level else f"GitHub environment '{env}'" - - -def _remote_digest(env: str, repo_level: bool) -> tuple[str | None, str]: - """``(digest, "")`` from GitHub, or ``(None, why not)``.""" - path = ( - f"repos/{{owner}}/{{repo}}/actions/variables/{bundle.DIGEST_NAME}" if repo_level - else f"repos/{{owner}}/{{repo}}/environments/{env}/variables/{bundle.DIGEST_NAME}" - ) - proc = _gh(["api", path, "--jq", ".value"]) - if proc.returncode == 0: - return proc.stdout.strip() or None, "the digest variable is empty" - if "404" in proc.stderr: - return None, "nothing uploaded yet" + ("" if repo_level else f" (or no GitHub environment named '{env}')") - return None, proc.stderr.strip().splitlines()[-1] if proc.stderr.strip() else "gh api failed" +# ---- ci sync-config / status / unpack: this machine's config/ → GitHub → the runner ---- def _collect(env: str) -> dict[str, str]: @@ -180,13 +419,24 @@ def _collect(env: str) -> dict[str, str]: raise typer.Exit(1) from None +def _remote_digest(scope: str | None) -> tuple[str | None, str]: + """``(digest, "")`` from GitHub, or ``(None, why not)``.""" + try: + value = github.variables(scope).get(bundle.DIGEST_NAME) + except github.GitHubError as exc: + if "404" in str(exc) or "not found" in str(exc).lower(): + return None, "nothing uploaded yet" + (f" (or no GitHub environment named '{scope}')" if scope else "") + return None, str(exc) + return (value, "") if value else (None, "nothing uploaded yet") + + @ci_app.command("sync-config") def sync_config( env: str = env_option(), force: bool = typer.Option(False, "--force", "-f", help="Upload even when GitHub's copy already matches."), repo_level: bool = _repo_level_option, ) -> None: - """Upload this environment's config/ to its GitHub environment, for the deploy workflow. + """Upload this environment's config/ to GitHub, for the deploy workflow. Run it after every config change made on this machine — in the panel or by hand. CI deploys with the uploaded copy, and refuses to change a value the @@ -194,11 +444,11 @@ def sync_config( rather than quietly revert your change. """ env_name = resolve_env(env) - _require_gh() - + _github() # Upload only what would deploy: an invalid config would fail in CI later, and # a missing generated secret would fail there too (CI never mints one). cfg = load_config(env_name) + scope = scope_env(cfg, repo_level) stored = read_env_file(paths.secrets_file(env_name)) unminted = [k for k in secrets.pinned(cfg) if not stored.get(k) and not cfg.raw_input.get(k)] if unminted: @@ -208,29 +458,28 @@ def sync_config( files = _collect(env_name) local = bundle.digest(files) - remote, _ = _remote_digest(env_name, repo_level) + remote, _ = _remote_digest(scope) if remote == local and not force: ui.ok(f"GitHub's copy of '{env_name}' is already current ({local[:19]}…)") return # Secret first, digest second: if the second call fails, CI holds a bundle it # cannot verify and refuses it — failing closed rather than deploying either copy. - scope = _scope(env_name, repo_level) - proc = _gh(["secret", "set", bundle.SECRET_NAME, *scope], stdin=bundle.encode(files)) - if proc.returncode != 0: - ui.error(f"could not set the {bundle.SECRET_NAME} secret: {proc.stderr.strip()}") - if not repo_level: - ui.hint(f"no GitHub environment '{env_name}'? Create it in Settings → Environments. A private " - "repository on GitHub Free has no environment secrets: use --repo-level " - "(docs/35-CONTINUOUS-DEPLOYMENT.md)") - raise typer.Exit(1) - proc = _gh(["variable", "set", bundle.DIGEST_NAME, *scope, "--body", local]) - if proc.returncode != 0: - ui.error(f"uploaded the bundle but could not set {bundle.DIGEST_NAME}: {proc.stderr.strip()}") + try: + github.set_secret(bundle.SECRET_NAME, bundle.encode(files), scope) + except github.GitHubError as exc: + ui.error(str(exc)) + if scope: + ui.hint(f"no GitHub environment '{env_name}'? Create it, or on GitHub Free: deployctl ci connect") + raise typer.Exit(1) from None + try: + github.set_variable(bundle.DIGEST_NAME, local, scope) + except github.GitHubError as exc: + ui.error(f"uploaded the bundle but not its digest: {exc}") ui.hint("CI refuses the bundle until they match — re-run this command") - raise typer.Exit(1) + raise typer.Exit(1) from None - ui.ok(f"uploaded config for '{env_name}' → {_where(env_name, repo_level)} ({local[:19]}…)") + ui.ok(f"uploaded config for '{env_name}' → {_where(cfg, scope)} ({local[:19]}…)") for name in sorted(files): print(f" {name:<28} {len(parse_env_text(files[name]))} keys") @@ -243,18 +492,19 @@ def status(env: str = env_option(), repo_level: bool = _repo_level_option) -> No be checked. """ env_name = resolve_env(env) - _require_gh() + _github() + cfg = load_config(env_name, require_valid=False) + scope = scope_env(cfg, repo_level) local = bundle.digest(_collect(env_name)) - remote, why = _remote_digest(env_name, repo_level) - flag = " --repo-level" if repo_level else "" + remote, why = _remote_digest(scope) if remote is None: ui.warn(f"GitHub has no config for '{env_name}': {why}") - ui.hint(f"deployctl ci sync-config --env {env_name}{flag}") + ui.hint(f"deployctl ci sync-config --env {env_name}") raise typer.Exit(1 if "nothing uploaded" in why else 2) if remote != local: ui.warn(f"GitHub's copy of '{env_name}' differs from this machine's config/") ui.hint(f"this machine: {local[:19]}… GitHub: {remote[:19]}…") - ui.hint(f"if this machine has the newer config: deployctl ci sync-config --env {env_name}{flag}") + ui.hint(f"if this machine has the newer config: deployctl ci sync-config --env {env_name}") raise typer.Exit(1) ui.ok(f"GitHub's copy of '{env_name}' matches this machine's config/ ({local[:19]}…)") @@ -313,3 +563,291 @@ def unpack( ui.error(str(exc)) raise typer.Exit(1) from None ui.ok(f"config for '{env}' written ({actual[:19]}…): {', '.join(written)}") + + +# ---- ci doctor ----------------------------------------------------------------------- + + +def _check(items: list[dict], key: str, status: str, title: str, detail: str = "", fix: str = "") -> None: + items.append({"id": key, "status": status, "title": title, "detail": detail, "fix": fix}) + + +def doctor_items(cfg: Config, repo_level: bool = False) -> list[dict]: + """Every piece CI/CD needs, checked: status ok | warn | todo | fail.""" + items: list[dict] = [] + env = cfg.env + try: + github.require() + info = github.repo() + except github.GitHubError as exc: + _check(items, "github", "fail", "GitHub CLI", str(exc), "gh auth login") + return items + _check(items, "github", "ok", "GitHub", f"{info['name']} ({'private' if info['private'] else 'public'})") + + if cfg.raw["CI_SCOPE"]: + _check(items, "scope", "ok", "Where secrets live", cfg.raw["CI_SCOPE"]) + else: + _check(items, "scope", "todo", "Where secrets live", "not decided yet", f"deployctl ci connect --env {env}") + scope = scope_env(cfg, repo_level) + + wf = workflows_dir() + for name in MANAGED: + path = wf / name + if not path.is_file(): + _check(items, f"workflow:{name}", "todo", f".github/workflows/{name}", "missing", + f"deployctl ci init --env {env}") + continue + header = _MANAGED_HEADER.match(path.read_text()) + if header and header.group(1) != __version__: + _check(items, f"workflow:{name}", "warn", f".github/workflows/{name}", + f"from deployctl {header.group(1)}, this is {__version__}", f"deployctl ci init --env {env} --force") + elif not header: + _check(items, f"workflow:{name}", "warn", f".github/workflows/{name}", + "hand-written, not generated", f"deployctl ci init --env {env} --force") + else: + _check(items, f"workflow:{name}", "ok", f".github/workflows/{name}", "current") + ci_text = (wf / OWNED).read_text() if (wf / OWNED).is_file() else "" + if "deploy.yml" in ci_text: + _check(items, "workflow:ci", "ok", f".github/workflows/{OWNED}", "calls the deploy workflow") + else: + _check(items, "workflow:ci", "todo", f".github/workflows/{OWNED}", + "missing" if not ci_text else "never calls deploy.yml", f"deployctl ci init --env {env}") + + try: + names = github.secret_names(scope) + values = github.variables(scope) + except github.GitHubError as exc: + _check(items, "secrets", "fail", f"Secrets on {_where(cfg, scope)}", str(exc), + f"deployctl ci connect --env {env}") + return items + repo_values = values if scope is None else _safe_variables(None) + + if "DEPLOY_SSH_KEY" in names: + _check(items, "ssh-key", "ok", "CI ssh key (DEPLOY_SSH_KEY)", "set") + else: + _check(items, "ssh-key", "todo", "CI ssh key (DEPLOY_SSH_KEY)", "not set", f"deployctl ci setup-key --env {env}") + + pinned = values.get("DEPLOY_KNOWN_HOSTS", "") + wanted = list(cfg.hosts) + ([_jump_target(cfg.raw["SSH_JUMP_HOST"])[0]] if cfg.raw["SSH_JUMP_HOST"] else []) + uncovered = [host for host in wanted if not _covers(pinned, host)] + if not pinned: + _check(items, "host-keys", "todo", "Pinned host keys", "not set", f"deployctl ci pin-hosts --env {env}") + elif uncovered: + _check(items, "host-keys", "todo", "Pinned host keys", f"missing for {', '.join(uncovered)}", + f"deployctl ci pin-hosts --env {env}") + else: + _check(items, "host-keys", "ok", "Pinned host keys", f"every host ({len(wanted)})") + + digest = values.get(bundle.DIGEST_NAME) + if bundle.SECRET_NAME not in names or not digest: + _check(items, "config", "todo", "Deploy config in GitHub", "not uploaded", f"deployctl ci sync-config --env {env}") + else: + try: + local = bundle.digest(bundle.collect(env)) + except bundle.BundleError as exc: + _check(items, "config", "warn", "Deploy config in GitHub", str(exc)) + else: + if local == digest: + _check(items, "config", "ok", "Deploy config in GitHub", "matches this machine") + else: + _check(items, "config", "warn", "Deploy config in GitHub", "differs from this machine's config/", + f"deployctl ci sync-config --env {env}") + + tool_private = github.gh(["repo", "view", TOOL_REPO, "--json", "visibility", "--jq", ".visibility"]).stdout.strip() + if tool_private and tool_private != "PUBLIC" and "DEPLOYCTL_INSTALL_TOKEN" not in names: + _check(items, "install-token", "todo", "Installing deployctl in CI", + f"{TOOL_REPO} is private and DEPLOYCTL_INSTALL_TOKEN is not set", + "a fine-grained token, Contents: read on that repository → gh secret set DEPLOYCTL_INSTALL_TOKEN") + else: + _check(items, "install-token", "ok", "Installing deployctl in CI", + "token set" if "DEPLOYCTL_INSTALL_TOKEN" in names else f"{TOOL_REPO} is public") + + auto = repo_values.get("AUTO_DEPLOY", "") + _check(items, "auto-deploy", "ok", "Automatic deploys", + f"on — every push to {cfg.raw['DEPLOY_BRANCH']} deploys" if auto == "true" else "off — deploy with `ci deploy`") + return items + + +def _safe_variables(scope: str | None) -> dict[str, str]: + try: + return github.variables(scope) + except github.GitHubError: + return {} + + +def _covers(known_hosts: str, host: str) -> bool: + """Does a known_hosts text hold a key for host? Hashed entries are checked by ssh-keygen.""" + if not known_hosts: + return False + with tempfile.NamedTemporaryFile("w", suffix=".known_hosts", delete=False) as handle: + handle.write(known_hosts + "\n") + path = handle.name + try: + proc = subprocess.run(["ssh-keygen", "-F", host, "-f", path], capture_output=True, text=True) + return proc.returncode == 0 and bool(proc.stdout.strip()) + finally: + os.unlink(path) + + +_MARK = {"ok": "✓", "warn": "!", "todo": "·", "fail": "✗"} + + +@ci_app.command("doctor") +def doctor( + env: str = env_option(), + as_json: bool = typer.Option(False, "--json", help="Machine-readable, for the control panel."), + repo_level: bool = _repo_level_option, +) -> None: + """Check everything continuous deployment needs, and say what to run for each gap. + + Exit 0 when nothing is missing, 1 otherwise. + """ + cfg = load_config(env, require_valid=False) + items = doctor_items(cfg, repo_level) + if as_json: + print(json.dumps({"env": cfg.env, "items": items}, indent=2)) + else: + ui.header(f"CI doctor — {cfg.env}") + for item in items: + print(f" {_MARK[item['status']]} {item['title']:<32} {item['detail']}") + if item["fix"] and item["status"] != "ok": + ui.hint(item["fix"]) + if any(item["status"] in ("todo", "fail") for item in items): + raise typer.Exit(1) + + +# ---- ci deploy / runs / auto-deploy -------------------------------------------------- + + +@ci_app.command("deploy") +def deploy( + env: str = env_option(), + tag: str = typer.Option(None, "--tag", "-t", help="Image tag. Default: the running one (to ship a config " + "change), or with --rollback the release before it."), + rollback: bool = typer.Option(False, "--rollback", help="Roll back instead: never migrates."), + allow_config_change: bool = typer.Option( + False, "--allow-config-change", help="Ship config values that differ from what the hosts run."), + watch: bool = typer.Option(True, "--watch/--no-watch", help="Follow the run until it finishes."), +) -> None: + """Run the deploy workflow on GitHub — the everyday way to deploy by hand. + + Deploys through the same workflow merges use, with GitHub's copy of the config + and the CI key, so every deploy lands in one history. + """ + env_name = resolve_env(env) + cfg = load_config(env_name) + _github() + if not tag: + running = tags.running(cfg) + if rollback: + history = run(cfg, "deploy.sh", ["history"], check=False, capture=True).stdout + tag = previous_release(parse_history(history, env_name), running) or "" + else: + tag = running + if not tag: + ui.error("no tag to deploy: pass --tag" + ("" if not rollback else " (no earlier release is recorded)")) + raise typer.Exit(2) + try: + tags.check(tag) + except tags.NoTag as exc: + ui.error(str(exc)) + raise typer.Exit(2) from None + + action = "rollback" if rollback else "update" + started = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(seconds=5) + try: + github.dispatch("deploy.yml", cfg.raw["DEPLOY_BRANCH"], { + "environment": env_name, "tag": tag, "action": action, + "allow_config_change": "true" if allow_config_change else "false", + }) + except github.GitHubError as exc: + ui.error(str(exc)) + ui.hint(f"is deploy.yml on {cfg.raw['DEPLOY_BRANCH']} and the default branch? deployctl ci doctor") + raise typer.Exit(1) from None + ui.ok(f"started: {action} {tag} → {env_name} (from {cfg.raw['DEPLOY_BRANCH']})") + if not watch: + return + + run_id = None + for _ in range(20): # the run appears a few seconds after the dispatch + for item in github.runs("deploy.yml", limit=5, event="workflow_dispatch"): + created = datetime.datetime.fromisoformat(item["createdAt"].replace("Z", "+00:00")) + if created >= started: + run_id = item["databaseId"] + ui.info(item["url"]) + break + if run_id: + break + time.sleep(3) + if not run_id: + ui.warn("started, but the run did not show up to follow — see: deployctl ci runs") + return + raise typer.Exit(github.watch(run_id)) + + +def deploy_runs(cfg: Config, limit: int = 10) -> list[dict]: + """Deploys through GitHub, newest first: manual ones, and those a push to the deploy branch ran.""" + out = [] + for item in github.runs("deploy.yml", limit=limit, event="workflow_dispatch"): + title = item.get("displayTitle", "") + parts = title.split() + out.append({ + "at": item["createdAt"], "trigger": "manual", + "action": parts[0] if parts else "update", "tag": parts[1] if len(parts) > 1 else "", + "status": item.get("conclusion") or item.get("status", ""), "url": item["url"], + }) + for item in github.runs(OWNED, limit=limit, branch=cfg.raw["DEPLOY_BRANCH"], event="push"): + job = next((j for j in github.run_jobs(item["databaseId"]) + if j.get("name", "").lower().startswith("deploy")), None) + if job is None or job.get("conclusion") == "skipped": + continue + out.append({ + "at": item["createdAt"], "trigger": "merge", "action": "update", "tag": item["headSha"][:7], + "status": job.get("conclusion") or job.get("status", ""), "url": item["url"], + }) + return sorted(out, key=lambda r: r["at"], reverse=True)[:limit] + + +@ci_app.command("runs") +def runs( + env: str = env_option(), + limit: int = typer.Option(10, "--limit", "-n", help="How many."), + as_json: bool = typer.Option(False, "--json", help="Machine-readable, for the control panel."), +) -> None: + """Recent deploys through GitHub — manual and on merge — with their outcome.""" + cfg = load_config(env, require_valid=False) + _github() + found = deploy_runs(cfg, limit) + if as_json: + print(json.dumps(found, indent=2)) + return + if not found: + ui.warn("no deploys through GitHub yet") + return + for item in found: + print(f" {item['at']} {item['status'] or '…':<10} {item['action']:<9} {item['tag']:<10} " + f"{item['trigger']:<7} {item['url']}") + + +@ci_app.command("auto-deploy") +def auto_deploy( + state: str = typer.Argument(..., help="on or off"), + env: str = env_option(), +) -> None: + """Turn deploy-on-merge on or off (the repository variable AUTO_DEPLOY).""" + if state not in ("on", "off"): + ui.error("say on or off") + raise typer.Exit(2) + cfg = load_config(env, require_valid=False) + _github() + try: + # Repository level whatever CI_SCOPE says: ci.yml reads it in a job's `if`, + # before any environment is entered. + github.set_variable("AUTO_DEPLOY", "true" if state == "on" else "false", None) + except github.GitHubError as exc: + ui.error(str(exc)) + raise typer.Exit(1) from None + if state == "on": + ui.ok(f"on — every push to {cfg.raw['DEPLOY_BRANCH']} that passes its checks deploys itself") + else: + ui.ok("off — deploy with: deployctl ci deploy") diff --git a/src/deployctl/cli/commands/deploy.py b/src/deployctl/cli/commands/deploy.py index a5704ae..a5b8750 100644 --- a/src/deployctl/cli/commands/deploy.py +++ b/src/deployctl/cli/commands/deploy.py @@ -2,16 +2,14 @@ from __future__ import annotations -import datetime -import os +import json import subprocess import typer -from .. import locks, paths, render, secrets, ui -from ..config import Config, load +from .. import locks, render, secrets, tags, ui +from ..config import Config from ..context import env_option, exclusive, load_config, resolve_env -from ..envfile import patch_env_file from ..runner import run deploy_app = typer.Typer( @@ -24,6 +22,12 @@ False, "--dry-run", help="Print every ssh/rsync/compose command instead of executing it." ) _host_option = typer.Option(None, "--host", help="Act on one host instead of all of them.") +_tag_option = typer.Option( + None, "--tag", "-t", + help="Image tag to deploy. Default: the tag the primary is running (a redeploy — how a config " + "change goes out).", + show_default=False, +) _allow_secret_change_option = typer.Option( False, "--allow-secret-change", @@ -32,14 +36,28 @@ ) -def _prepare(env: str | None) -> Config: - """Load config and re-render artifacts so a deploy never ships stale ones. +def _resolve(cfg: Config, tag: str | None) -> str: + """Decide the tag (cli/tags.py), say where it came from, or stop.""" + try: + chosen, source = tags.resolve(cfg, tag, from_hosts=True) + except tags.NoTag as exc: + ui.error(str(exc)) + raise typer.Exit(2) from None + ui.info(f"image tag {chosen} ({source})") + if tags.deprecated_in_config(cfg): + ui.hint("IMAGE_TAG in config/ is deprecated — remove it with: deployctl migrate-config --apply") + return chosen + + +def _prepare(env: str | None, tag: str | None = None) -> Config: + """Load config, decide the tag and re-render, so a deploy never ships stale artifacts. Rendering is deterministic and secrets are minted once, so regenerating here is free — and it removes the entire "edited config but forgot to run setup" failure class. """ cfg = load_config(env) + _resolve(cfg, tag) try: secrets.ensure(cfg) except secrets.MintRefused as exc: @@ -61,66 +79,57 @@ def _engine( raise typer.Exit(exc.returncode) from exc -def _record_history(cfg: Config, *, kind: str = "deploy") -> None: - """Append one line to the local deploy log. - - ``kind`` distinguishes a release from a rollback, and both from a roll of a - single host (``host:
``). Without it, rolling back appends the older - tag as though it were the newest release, and the history reads A, B, A — so - a second rollback picks B, the tag that was just rolled away from. A - one-host roll is not a release of the environment either. +def _engine_output(cfg: Config, command: str) -> str: + """A read-only engine command's output, captured.""" + proc = run(cfg, "deploy.sh", [command], check=False, capture=True) + if proc.returncode != 0: + ui.error((proc.stderr or proc.stdout).strip() or f"deploy.sh {command} failed") + raise typer.Exit(proc.returncode or 1) + return proc.stdout - The image repository is recorded too. The log is local, untracked state, and - a deployctl directory copied from another project brings its log along — - tags from a different image, which rollback would otherwise offer. - """ - stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - with paths.history_file().open("a") as handle: - handle.write(f"{stamp} {cfg.env} {cfg.raw['IMAGE_TAG']} {kind} {cfg.raw['IMAGE_REPO']}\n") +# ---- the history, kept on the primary --------------------------------------------- -def _history_entries(env: str, repo: str = "") -> list[tuple[str, str, str]]: - """``(stamp, tag, kind)`` for one environment, oldest first. - Lines written before ``kind`` existed have three fields and are read as - releases, which is what they were. Lines that name a different image - repository than ``repo`` belong to another project and are skipped; lines - older than the repository field cannot be told apart and are kept. - """ - if not paths.history_file().is_file(): - return [] - out: list[tuple[str, str, str]] = [] - for line in paths.history_file().read_text().splitlines(): - parts = line.split() - if len(parts) not in (3, 4, 5) or parts[1] != env: - continue - if len(parts) == 5 and repo and parts[4] != repo: +def parse_history(text: str, env: str) -> list[dict[str, str]]: + """The primary's ``.deployctl-history``, oldest first (see append_history in remote.sh).""" + entries = [] + for line in text.splitlines(): + parts = line.split("\t") + if len(parts) != 5 or parts[1] != env: continue - out.append((parts[0], parts[2], parts[3] if len(parts) >= 4 else "deploy")) - return out + stamp, _env, tag, kind, by = parts + entries.append({"at": stamp, "tag": tag, "kind": kind, "by": by}) + return entries -def _previous_tag(env: str, current: str, repo: str = "") -> str | None: - """The release before the one running now, or None if there isn't one. +def previous_release(entries: list[dict[str, str]], current: str) -> str | None: + """The release before the one running now, or None. - Rollbacks and one-host rolls are skipped: the question is "what was released - to this environment before this", and neither is a new point in that history. + Rollbacks and one-host rolls are skipped: the question is "what was released to + this environment before this", and neither is a new point in that history. + Without skipping them the history reads A, B, A after a rollback, and a second + rollback would pick B — the tag just rolled away from. """ releases: list[str] = [] - for _stamp, tag, kind in _history_entries(env, repo): - if kind != "deploy": + for entry in entries: + if entry["kind"] not in ("deploy", "init"): continue - if not releases or releases[-1] != tag: - releases.append(tag) + if not releases or releases[-1] != entry["tag"]: + releases.append(entry["tag"]) if current in releases: index = len(releases) - 1 - releases[::-1].index(current) return releases[index - 1] if index > 0 else None return releases[-1] if releases else None +# ---- commands ---------------------------------------------------------------------- + + @deploy_app.command("doctor") def doctor( env: str = env_option(), + tag: str = _tag_option, fix: bool = typer.Option( False, "--fix", help="Repair the permission problems that do not need root, then re-report." ), @@ -136,41 +145,45 @@ def doctor( args = ["--fix"] if fix else [] try: with locks.env_lock(env_name, "deploy doctor"): - _engine(_prepare(env_name), "doctor", *args) + _engine(_prepare(env_name, tag), "doctor", *args) return except locks.LockHeld as held: # Re-rendering now would rewrite the files that run is rsyncing. The # artifacts on disk are exactly what it is shipping, so check those. ui.warn(f"a run is in progress ({held.holder}) — checking the artifacts it is shipping, not re-rendering") - _engine(load_config(env_name), "doctor", *args) + cfg = load_config(env_name) + _resolve(cfg, tag) + _engine(cfg, "doctor", *args) @deploy_app.command("init") -def init(env: str = env_option(), dry_run: bool = _dry_run_option) -> None: +def init(env: str = env_option(), tag: str = _tag_option, dry_run: bool = _dry_run_option) -> None: """First-time bring-up: push, pull, migrate once, then start primary → rest.""" env_name = resolve_env(env) with exclusive(env_name, "deploy init"): - cfg = _prepare(env_name) + cfg = _prepare(env_name, tag) _engine(cfg, "init", dry_run=dry_run) if not dry_run: - _record_history(cfg) + tags.remember(env_name, cfg.raw["IMAGE_TAG"]) @deploy_app.command("update") def update( env: str = env_option(), + tag: str = _tag_option, host: str = _host_option, dry_run: bool = _dry_run_option, allow_secret_change: bool = _allow_secret_change_option, ) -> None: """Rolling release: migrate once, then per host pull + up, health-gated. - Stops at the first host that fails its health gate, leaving the remaining - hosts on the previous release. + A host that fails its health gate or the service watch is reverted, and so + is every host already rolled (REVERT_SCOPE=fleet), leaving the whole fleet on + its previous release. Without --tag, the running tag is redeployed. """ env_name = resolve_env(env) with exclusive(env_name, "deploy update"): - cfg = _prepare(env_name) + cfg = _prepare(env_name, tag) if host: if host not in cfg.hosts: ui.error(f"{host} is not in HOSTS ({' '.join(cfg.hosts)})") @@ -179,21 +192,21 @@ def update( else: _engine(cfg, "update", dry_run=dry_run, allow_secret_change=allow_secret_change) if not dry_run: - _record_history(cfg, kind=f"host:{host}" if host else "deploy") + tags.remember(env_name, cfg.raw["IMAGE_TAG"]) @deploy_app.command("migrate") -def migrate(env: str = env_option(), dry_run: bool = _dry_run_option) -> None: - """Run the project's MIGRATE_CMD once, on the primary host.""" +def migrate(env: str = env_option(), tag: str = _tag_option, dry_run: bool = _dry_run_option) -> None: + """Run the project's MIGRATE_CMD once, on the primary host (with the running image by default).""" env_name = resolve_env(env) with exclusive(env_name, "deploy migrate"): - _engine(_prepare(env_name), "migrate", dry_run=dry_run) + _engine(_prepare(env_name, tag), "migrate", dry_run=dry_run) @deploy_app.command("rollback") def rollback( env: str = env_option(), - to: str = typer.Option(None, "--to", help="Tag to roll back to. Default: the previously deployed one."), + to: str = typer.Option(None, "--to", help="Tag to roll back to. Default: the release before the running one."), dry_run: bool = _dry_run_option, ) -> None: """Re-deploy a previous image tag, with the same health-gated roll. @@ -201,47 +214,30 @@ def rollback( Only the application image moves back, and nothing is migrated: the older image cannot run its migrations against a database that is already past them. Migrations are NOT reverted either — the older code has to tolerate the - newer schema, which is what additive migrations guarantee. + newer schema, which is what additive migrations guarantee. The running tag and + the history both come from the primary, so a laptop and CI agree on them. """ env_name = resolve_env(env) - loaded = load(env_name) - current = loaded.raw.get("IMAGE_TAG", "") + cfg = load_config(env_name) + current = tags.running(cfg) target = to if not target: - target = _previous_tag(env_name, current, loaded.raw.get("IMAGE_REPO", "")) + target = previous_release(parse_history(_engine_output(cfg, "history"), env_name), current) if not target: - ui.error("no earlier release recorded for this environment") - ui.hint("pass one explicitly: deployctl deploy rollback --to " - "(see deployctl image tags, or deployctl deploy history)") + ui.error("no earlier release is recorded on the primary for this environment") + ui.hint("pass one explicitly: deployctl deploy rollback --to (see deployctl image tags)") raise typer.Exit(2) - if target == current: - if not to: - ui.error(f"{target} is already the deployed tag") - raise typer.Exit(2) - # IMAGE_TAG in config is what THIS machine last deployed. Once CI deploys, - # the hosts can be on something newer, and a rollback to the tag this file - # happens to hold is exactly the one that must not be refused. Re-rolling - # a tag that really is running is harmless: the same health-gated roll. - ui.warn(f"config/{env_name}.env already names {target}, but it is not updated by CI " - "deploys — rolling anyway") + ui.error(f"{target} is already running on {cfg.primary_host}") + raise typer.Exit(2) ui.header(f"Rollback — {env_name}: {current or '?'} → {target}") ui.warn("only the image moves back: nothing is migrated, and no migration is reverted") - - # IMAGE_TAG is env-introducible, so the override flows through config - # resolution, re-rendering and the bash layer as one consistent value. - os.environ["IMAGE_TAG"] = target with exclusive(env_name, f"deploy rollback → {target}"): - cfg = _prepare(env_name) + cfg = _prepare(env_name, target) _engine(cfg, "rollback", dry_run=dry_run) - if not dry_run: - # Pin the config so the on-disk record matches what is now running — - # otherwise the next routine deploy would silently re-deploy the bad tag. - patch_env_file(paths.config_file(env_name), {"IMAGE_TAG": target}) - ui.ok(f"pinned IMAGE_TAG={target} in config/{env_name}.env") - _record_history(cfg, kind="rollback") + tags.remember(env_name, target) @deploy_app.command("restart") @@ -254,6 +250,8 @@ def restart(env: str = env_option(), host: str = _host_option, dry_run: bool = _ @deploy_app.command("stop") def stop(env: str = env_option(), dry_run: bool = _dry_run_option) -> None: """docker compose down on every host. The site goes down.""" + import os + cfg = load_config(env) # ASSUME_YES covers non-interactive callers (the control panel confirms in # the browser before it ever invokes this). @@ -276,10 +274,47 @@ def unlock(env: str = env_option()) -> None: _engine(load_config(env), "unlock") +def parse_state(text: str) -> dict: + """``deploy.sh state`` output as the structure ``status --json`` prints.""" + hosts: dict[str, dict] = {} + for line in text.splitlines(): + parts = line.split("\t") + if parts[0] == "HOST" and len(parts) == 4: + hosts[parts[1]] = {"host": parts[1], "role": parts[2], "reachable": parts[3] == "reachable", + "tag": "", "deployed_at": "", "deployed_by": "", "services": []} + elif parts[0] == "STATE" and len(parts) == 3 and parts[1] in hosts and "=" in parts[2]: + key, value = parts[2].split("=", 1) + field = {"IMAGE_TAG": "tag", "DEPLOYED_AT": "deployed_at", "DEPLOYED_BY": "deployed_by"}.get(key) + if field: + hosts[parts[1]][field] = value + elif parts[0] == "SVC" and len(parts) == 3 and parts[1] in hosts: + fields = parts[2].split() + if len(fields) == 6: + name, restarts, state, health, code, policy = fields + hosts[parts[1]]["services"].append({ + "name": name, "state": state, "health": health, "restarts": int(restarts) + if restarts.isdigit() else None, "exit_code": code, "restart_policy": policy, + }) + tags_running = {h["tag"] for h in hosts.values() if h["tag"]} + return { + "hosts": list(hosts.values()), + # One tag everywhere, or the fleet is split (a revert that could not finish). + "tag": next(iter(tags_running)) if len(tags_running) == 1 else None, + "split": len(tags_running) > 1, + } + + @deploy_app.command("status") -def status(env: str = env_option()) -> None: +def status( + env: str = env_option(), + as_json: bool = typer.Option(False, "--json", help="Machine-readable: tags, who deployed, every service."), +) -> None: """Service status and the deployed tag on every host.""" - _engine(load_config(env), "status") + cfg = load_config(env) + if as_json: + print(json.dumps({"env": cfg.env, **parse_state(_engine_output(cfg, "state"))}, indent=2)) + return + _engine(cfg, "status") @deploy_app.command("logs") @@ -299,17 +334,14 @@ def shell( @deploy_app.command("history") def history(env: str = env_option()) -> None: - """Locally recorded deploys for this environment (what rollback consults).""" + """The environment's releases, from the primary — the same record CI and laptops write.""" env_name = resolve_env(env) - if not paths.history_file().is_file(): - ui.warn("no deploy history recorded yet") + entries = parse_history(_engine_output(load_config(env_name), "history"), env_name) + if not entries: + ui.warn("no releases recorded on the primary yet") raise typer.Exit(2) - repo = load(env_name).raw.get("IMAGE_REPO", "") - for stamp, tag, kind in _history_entries(env_name, repo): - if kind == "rollback": - marker = " (rollback)" - elif kind.startswith("host:"): - marker = f" (one host: {kind[5:]})" - else: - marker = "" - print(f" {stamp} {tag}{marker}") + for entry in entries: + marker = {"rollback": " (rollback)", "init": " (first bring-up)"}.get(entry["kind"], "") + if entry["kind"].startswith("host:"): + marker = f" (one host: {entry['kind'][5:]})" + print(f" {entry['at']} {entry['tag']:<14}{marker} {entry['by']}") diff --git a/src/deployctl/cli/commands/image.py b/src/deployctl/cli/commands/image.py index 88d06a1..63fefdf 100644 --- a/src/deployctl/cli/commands/image.py +++ b/src/deployctl/cli/commands/image.py @@ -7,10 +7,9 @@ import typer -from .. import paths, registry, runner, ui +from .. import paths, registry, runner, tags, ui from ..config import Config from ..context import env_option, load_config -from ..envfile import patch_env_file image_app = typer.Typer( name="image", @@ -42,7 +41,6 @@ def _resolve_tag(cfg: Config, explicit: str | None) -> str: def push( env: str = env_option(), tag: str = typer.Option(None, "--tag", "-t", help="Image tag. Defaults to the current git short SHA."), - pin: bool = typer.Option(True, "--pin/--no-pin", help="Write the tag into this environment's config."), allow_dirty: bool = typer.Option(False, "--allow-dirty", help="Build even with uncommitted changes."), ) -> None: """Build the image on this machine and push it to the registry. @@ -105,10 +103,8 @@ def push( raise typer.Exit(exc.returncode) from exc ui.ok(f"pushed {ref}") - if pin: - patch_env_file(paths.config_file(cfg.env), {"IMAGE_TAG": tag}) - ui.ok(f"pinned IMAGE_TAG={tag} in config/{cfg.env}.env") - ui.info(f"Next: deployctl setup --env {cfg.env} && deployctl deploy update --env {cfg.env}") + # Not written into config: the tag is chosen per deploy (cli/tags.py). + ui.info(f"Next: deployctl deploy update --env {cfg.env} --tag {tag}") def _build(reference: str, target: str, dockerfile: str, context: str, platform: str) -> None: @@ -183,7 +179,8 @@ def tags( ui.warn("no git-sha tags found for this image") raise typer.Exit(2) - current = cfg.raw["IMAGE_TAG"] + # Registry only — no ssh here; "deployed" is the tag this machine last deployed. + current = tags.cached(cfg.env) for index, item in enumerate(found): marks = [] if index == 0: diff --git a/src/deployctl/cli/commands/migrate_config.py b/src/deployctl/cli/commands/migrate_config.py new file mode 100644 index 0000000..9a0d4ba --- /dev/null +++ b/src/deployctl/cli/commands/migrate_config.py @@ -0,0 +1,56 @@ +"""``deployctl migrate-config`` — bring a project's config/ up to the current tool. + +Today it does one thing: take ``IMAGE_TAG`` out of ``config/.env``. A tag +written there is whatever that machine deployed last, and once CI deploys it is +stale by design (``cli/tags.py``). The value is not lost: it becomes this machine's +"last used" tag, so ``setup`` and ``validate`` keep working until the next deploy. +""" + +from __future__ import annotations + +import typer + +from .. import paths, tags, ui +from ..envfile import assignment_key, read_env_file, write_env_file + +#: Keys the tool no longer reads from config files. +RETIRED = ("IMAGE_TAG",) + + +def migrate_config( + apply: bool = typer.Option(False, "--apply", help="Make the changes. Without it, only print the plan."), +) -> None: + """Remove settings the tool no longer reads from config/ (today: IMAGE_TAG). + + Prints the plan unless --apply is given. A removed IMAGE_TAG becomes this + machine's last-used tag, so local commands keep working. + """ + plan: list[tuple[str, str, str]] = [] # (env, key, value) + for env in paths.known_environments(): + stored = read_env_file(paths.config_file(env)) + for key in RETIRED: + if key in stored: + plan.append((env, key, stored[key])) + + ui.header("Migrate config") + if not plan: + ui.ok("nothing to migrate — config/ holds no retired settings") + return + for env, key, value in plan: + keep = f"; {value} becomes this machine's last-used tag" if key == "IMAGE_TAG" and value else "" + print(f" remove {key}={value} from config/{env}.env{keep}") + + if not apply: + ui.separator() + ui.info("nothing changed — this was the plan. Run again with --apply to make it.") + return + + for env in sorted({env for env, _, _ in plan}): + path = paths.config_file(env) + lines = path.read_text().splitlines() + kept = [line for line in lines if assignment_key(line) not in RETIRED] + write_env_file(path, "\n".join(kept) + "\n") + for env, key, value in plan: + if key == "IMAGE_TAG" and value and not tags.cached(env): + tags.remember(env, value) + ui.ok(f"migrated {len(plan)} setting(s) — deploys now take the tag from --tag or the running hosts") diff --git a/src/deployctl/cli/commands/server.py b/src/deployctl/cli/commands/server.py new file mode 100644 index 0000000..2af9fb6 --- /dev/null +++ b/src/deployctl/cli/commands/server.py @@ -0,0 +1,73 @@ +"""``deployctl server`` — preparing a server before deployctl can use it.""" + +from __future__ import annotations + +import re +import sys + +import jinja2 +import typer + +from ... import __version__ +from .. import paths +from ..context import env_option, load_config + +server_app = typer.Typer( + name="server", + help="Prepare servers: the one step that needs root, rendered for this environment.", + no_args_is_help=True, +) + + +def render_bootstrap(cfg, *, swap: str = "2G", harden_ssh: bool = True, reboot: bool = True) -> str: + """The root script, with this environment's user, directory and firewall rules in it.""" + if cfg.derived["TLS_LB"]: + firewall = "80 from the load balancer only; 22 from you and CI (or a tailnet / jump host)" + else: + firewall = "80 and 443 from anywhere; 22 from you and CI (or a tailnet / jump host)" + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(str(paths.TEMPLATES_DIR / "server")), + undefined=jinja2.StrictUndefined, + keep_trailing_newline=True, + # «…» like the CI templates: the script is full of ${…}, and Jinja's own + # {{ }} would be one typo away from a template error on a live server. + variable_start_string="«", + variable_end_string="»", + autoescape=False, + ) + return env.get_template("bootstrap.sh.j2").render( + VERSION=__version__, + PROJECT=cfg.raw.get("PROJECT_NAME") or "app", + ENV=cfg.env, + SSH_USER=cfg.raw["SSH_USER"], + REMOTE_DIR=cfg.remote_dir, + SWAP_SIZE=swap, + HARDEN_SSH="true" if harden_ssh else "false", + REBOOT="true" if reboot else "false", + FIREWALL=firewall, + ) + + +@server_app.command("bootstrap-script") +def bootstrap_script( + env: str = env_option(), + swap: str = typer.Option("2G", "--swap", help="Swap file size, e.g. 2G or 1024M."), + harden_ssh: bool = typer.Option(True, "--harden-ssh/--no-harden-ssh", + help="Key-only ssh: no passwords, root by key only."), + reboot: bool = typer.Option(True, "--reboot/--no-reboot", + help="Reboot at the end when the updates asked for it."), +) -> None: + """Print the root script that prepares a fresh Ubuntu server for this environment. + + Docker, the deploy user (in the docker group, with root's authorised keys), + the deploy directory owned by it, swap, and key-only ssh. Safe to run again. + Only the script goes to stdout, so it can be piped: + deployctl server bootstrap-script --env E | ssh root@IP 'bash -s' + """ + if not re.fullmatch(r"\d+[MG]", swap): + print("deployctl: --swap takes a size like 2G or 1024M", file=sys.stderr) + raise typer.Exit(2) + cfg = load_config(env, require_valid=False) + sys.stdout.write(render_bootstrap(cfg, swap=swap, harden_ssh=harden_ssh, reboot=reboot)) + print(f"# ↑ run once as root on each new server for '{cfg.env}'; then: ssh {cfg.raw['SSH_USER']}@ docker ps", + file=sys.stderr) diff --git a/src/deployctl/cli/commands/setup.py b/src/deployctl/cli/commands/setup.py index 0c9eef2..a05238f 100644 --- a/src/deployctl/cli/commands/setup.py +++ b/src/deployctl/cli/commands/setup.py @@ -4,12 +4,17 @@ import typer -from .. import paths, render, secrets, ui +from .. import paths, render, secrets, tags, ui from ..context import env_option, exclusive, load_config, resolve_env def setup( env: str = env_option(), + tag: str = typer.Option( + None, "--tag", "-t", + help="Image tag to render. Default: the tag last used on this machine (a deploy ships its own).", + show_default=False, + ), force: bool = typer.Option(False, "--force", "-f", help="Rebuild artifacts even if they already exist."), rotate_secrets: bool = typer.Option( False, @@ -21,12 +26,20 @@ def setup( env_name = resolve_env(env) # A render mid-deploy rewrites the files that deploy is rsyncing. with exclusive(env_name, "setup"): - _setup(env_name, force=force, rotate_secrets=rotate_secrets) + _setup(env_name, tag=tag, force=force, rotate_secrets=rotate_secrets) -def _setup(env: str, *, force: bool, rotate_secrets: bool) -> None: +def _setup(env: str, *, tag: str | None, force: bool, rotate_secrets: bool) -> None: cfg = load_config(env) ui.header(f"Generate artifacts — {cfg.env}") + # Local only: no ssh to ask the hosts what they run. A deploy re-renders with + # its own tag anyway; this is for looking at, and validating, the artifacts. + try: + chosen, source = tags.resolve(cfg, tag, from_hosts=False) + except tags.NoTag as exc: + ui.error(str(exc)) + raise typer.Exit(2) from None + ui.info(f"image tag {chosen} ({source})") minted = secrets.ensure(cfg, rotate=rotate_secrets) for key in minted: @@ -43,6 +56,7 @@ def _setup(env: str, *, force: bool, rotate_secrets: bool) -> None: ui.debug(f"removed stale {path}") written = render.render_all(cfg) + tags.remember(cfg.env, chosen) for path in written: ui.ok(f"wrote {path.relative_to(paths.ROOT)}") diff --git a/src/deployctl/cli/config.py b/src/deployctl/cli/config.py index a014e5a..5349238 100644 --- a/src/deployctl/cli/config.py +++ b/src/deployctl/cli/config.py @@ -57,6 +57,9 @@ "MODE": "single", "API_SUBDOMAIN": "api", "SSH_USER": "deploy", + # A bastion (ssh ProxyJump) for hosts that are only on a private network — the + # usual shape of a fleet behind a load balancer. Empty: connect directly. + "SSH_JUMP_HOST": "", "TLS_MODE": "letsencrypt", "POSTGRES_MODE": "container", "REDIS_MODE": "container", @@ -96,6 +99,16 @@ # a restart or an unhealthy turn before the release counts as good. The gate # probes only the api; a worker that crash-loops passes it. 0 skips the watch. "DEPLOY_SETTLE_SECONDS": "60", + # When a host fails mid-roll: `fleet` puts every host this run already moved + # back on its previous release too, so the fleet is never split between two; + # `host` reverts only the host that failed. + "REVERT_SCOPE": "fleet", + # CI/CD. Where GitHub holds this project's deploy secrets: `environment` (the + # GitHub environment named like this one — Team/Pro for a private repository) + # or `repository` (every plan). `deployctl ci connect` detects and writes it. + "CI_SCOPE": "", + # The branch whose pushes build and, with AUTO_DEPLOY on, deploy. + "DEPLOY_BRANCH": "main", "WITH_BEAT": "true", "ENABLE_DOCS": "false", "LOG_LEVEL": "WARNING", @@ -338,7 +351,8 @@ def warn(message: str, hint: str = "") -> None: err(f"{key} must be one of {VALID_SERVICE_MODES} (got {raw[key]!r})") # -- identity and image - for key in ("PROJECT_NAME", "BASE_DOMAIN", "IMAGE_REPO", "IMAGE_TAG"): + # IMAGE_TAG is not among them: it is decided per command (cli/tags.py). + for key in ("PROJECT_NAME", "BASE_DOMAIN", "IMAGE_REPO"): if not raw.get(key): err(f"{key} is required", f"set it in config/{self.env}.env or config/common.env") elif any(marker in raw[key] for marker in ("your-org", "your-app", "example.com", "CHANGE_THIS")): @@ -359,12 +373,33 @@ def warn(message: str, hint: str = "") -> None: "docker login needs both; the token alone cannot authenticate", ) + if self.raw_input.get("IMAGE_TAG") and not os.environ.get("IMAGE_TAG"): + warn( + f"IMAGE_TAG is set in config/ ({self.raw_input['IMAGE_TAG']}) — deprecated", + "once CI deploys, a tag in config is whatever this machine deployed last, and deploying it " + "can take the hosts backwards. The hosts' running tag is used instead; pass --tag to " + "choose one. Remove it with: deployctl migrate-config --apply", + ) if raw.get("IMAGE_TAG") == "latest": warn( "IMAGE_TAG=latest is a moving pointer", "pin an immutable tag (a git short sha) so every host runs identical bits and rollback works", ) + # -- how deploys reach and treat the hosts + if raw["CI_SCOPE"] not in ("", "environment", "repository"): + err(f"CI_SCOPE must be 'environment' or 'repository' (got {raw['CI_SCOPE']!r})", + "deployctl ci connect detects the right one") + if not re.fullmatch(r"[A-Za-z0-9._/-]+", raw["DEPLOY_BRANCH"]): + err(f"DEPLOY_BRANCH {raw['DEPLOY_BRANCH']!r} is not a branch name") + if raw["REVERT_SCOPE"] not in ("fleet", "host"): + err(f"REVERT_SCOPE must be 'fleet' or 'host' (got {raw['REVERT_SCOPE']!r})") + if raw["SSH_JUMP_HOST"] and not re.fullmatch(r"[A-Za-z0-9@._:\[\]-]+(,[A-Za-z0-9@._:\[\]-]+)*", raw["SSH_JUMP_HOST"]): + err( + f"SSH_JUMP_HOST {raw['SSH_JUMP_HOST']!r} is not a [user@]host[:port] (commas chain several)", + "it is passed to ssh as ProxyJump; spaces and quotes are not allowed", + ) + # -- hosts if not self.hosts: err("HOSTS is empty", "list the SSH-reachable address of every target, space-separated") diff --git a/src/deployctl/cli/github.py b/src/deployctl/cli/github.py new file mode 100644 index 0000000..c97cbca --- /dev/null +++ b/src/deployctl/cli/github.py @@ -0,0 +1,129 @@ +""" +GitHub, through the ``gh`` CLI — the one place deployctl talks to it. + +``gh`` rather than the REST API directly: it already holds the operator's login +(keyring, SSO, 2FA), resolves ``owner/repo`` from the checkout's git remote, and +encrypts secrets for upload. Everything here runs in the application repository, +so ``{owner}/{repo}`` placeholders resolve to it. + +Where secrets and variables live depends on the plan: environment-scoped ones +need GitHub Team/Pro for a private repository. ``scope_env`` is the environment +name when they live on the GitHub environment, and ``None`` for the repository. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess + +from . import paths + + +class GitHubError(Exception): + """A gh call failed; the message is gh's own, never a secret's value.""" + + +def gh(args: list[str], *, stdin: str | None = None) -> subprocess.CompletedProcess: + """Run ``gh`` from the application repository. Never raises on a non-zero exit.""" + return subprocess.run(["gh", *args], cwd=str(paths.workdir()), input=stdin, text=True, capture_output=True) + + +def _ok(proc: subprocess.CompletedProcess, what: str) -> str: + if proc.returncode != 0: + detail = (proc.stderr or proc.stdout).strip().splitlines() + raise GitHubError(f"{what}: {detail[-1] if detail else 'gh failed'}") + return proc.stdout + + +def require() -> None: + """gh is installed and logged in.""" + if shutil.which("gh") is None: + raise GitHubError("the GitHub CLI (gh) is not installed — https://cli.github.com, then: gh auth login") + if gh(["auth", "status"]).returncode != 0: + raise GitHubError("gh is not logged in — run: gh auth login") + + +def repo() -> dict: + """``{"name": "owner/repo", "owner": ..., "private": bool, "default_branch": ...}``.""" + out = _ok(gh(["repo", "view", "--json", "nameWithOwner,owner,visibility,defaultBranchRef"]), + "reading the repository") + data = json.loads(out) + return { + "name": data["nameWithOwner"], + "owner": data["owner"]["login"], + "private": data["visibility"] != "PUBLIC", + "default_branch": (data.get("defaultBranchRef") or {}).get("name", ""), + } + + +def plan(owner: str) -> str | None: + """The owner's plan (``free``, ``team``, …), or None when it cannot be seen. + + An organisation's plan is visible to its members; a personal account's only + to itself. None is common and fine — the caller then picks the scope that + works on every plan. + """ + proc = gh(["api", f"orgs/{owner}", "--jq", ".plan.name // empty"]) + if proc.returncode == 0 and proc.stdout.strip(): + return proc.stdout.strip().lower() + proc = gh(["api", "user", "--jq", 'if .login == "' + owner + '" then (.plan.name // empty) else empty end']) + if proc.returncode == 0 and proc.stdout.strip(): + return proc.stdout.strip().lower() + return None + + +def _scope_args(scope_env: str | None) -> list[str]: + return ["--env", scope_env] if scope_env else [] + + +def secret_names(scope_env: str | None) -> set[str]: + out = _ok(gh(["secret", "list", *_scope_args(scope_env), "--json", "name"]), "listing secrets") + return {item["name"] for item in json.loads(out or "[]")} + + +def variables(scope_env: str | None) -> dict[str, str]: + out = _ok(gh(["variable", "list", *_scope_args(scope_env), "--json", "name,value"]), "listing variables") + return {item["name"]: item["value"] for item in json.loads(out or "[]")} + + +def set_secret(name: str, value: str, scope_env: str | None) -> None: + """The value goes over stdin — never argv, which other processes can read.""" + _ok(gh(["secret", "set", name, *_scope_args(scope_env)], stdin=value), f"setting the {name} secret") + + +def set_variable(name: str, value: str, scope_env: str | None) -> None: + _ok(gh(["variable", "set", name, *_scope_args(scope_env), "--body", value]), f"setting the {name} variable") + + +def dispatch(workflow: str, ref: str, fields: dict[str, str]) -> None: + args = ["workflow", "run", workflow, "--ref", ref] + for key, value in fields.items(): + args += ["-f", f"{key}={value}"] + _ok(gh(args), f"starting {workflow}") + + +def runs(workflow: str, *, limit: int = 10, branch: str | None = None, event: str | None = None) -> list[dict]: + args = ["run", "list", "--workflow", workflow, "--limit", str(limit), + "--json", "databaseId,displayTitle,status,conclusion,createdAt,event,headBranch,headSha,url"] + if branch: + args += ["--branch", branch] + if event: + args += ["--event", event] + proc = gh(args) + if proc.returncode != 0: + return [] # a workflow that does not exist yet has no runs + return json.loads(proc.stdout or "[]") + + +def run_jobs(run_id: int) -> list[dict]: + proc = gh(["run", "view", str(run_id), "--json", "jobs"]) + if proc.returncode != 0: + return [] + return json.loads(proc.stdout or "{}").get("jobs", []) + + +def watch(run_id: int) -> int: + """Follow a run to the end, output straight through. Returns its exit status.""" + return subprocess.run(["gh", "run", "watch", str(run_id), "--exit-status", "--interval", "5"], + cwd=str(paths.workdir())).returncode diff --git a/src/deployctl/cli/main.py b/src/deployctl/cli/main.py index 2e43829..b2dae2e 100644 --- a/src/deployctl/cli/main.py +++ b/src/deployctl/cli/main.py @@ -20,7 +20,9 @@ from .commands import deploy as deploy_cmd from .commands import image as image_cmd from .commands import init as init_cmd +from .commands import migrate_config as migrate_config_cmd from .commands import selftest as selftest_cmd +from .commands import server as server_cmd from .commands import setup as setup_cmd from .commands import show as show_cmd from .commands import ssl as ssl_cmd @@ -76,11 +78,13 @@ def _global_options( app.command("selftest")(selftest_cmd.selftest) app.command("webui")(webui_cmd.webui) app.command("adopt")(adopt_cmd.adopt) +app.command("migrate-config")(migrate_config_cmd.migrate_config) app.add_typer(deploy_cmd.deploy_app) app.add_typer(image_cmd.image_app) app.add_typer(ci_cmd.ci_app) app.add_typer(ssl_cmd.ssl_app) app.add_typer(backup_cmd.backup_app) +app.add_typer(server_cmd.server_app) def main() -> None: diff --git a/src/deployctl/cli/paths.py b/src/deployctl/cli/paths.py index 49f6a90..867990e 100644 --- a/src/deployctl/cli/paths.py +++ b/src/deployctl/cli/paths.py @@ -28,7 +28,6 @@ TEMPLATES_DIR = TOOL_ROOT / "templates" SCRIPTS_DIR = TOOL_ROOT / "scripts" WEBUI_DIR = TOOL_ROOT / "webui" -SERVER_DIR = TOOL_ROOT / "server" # ---- the project -------------------------------------------------------------- @@ -216,8 +215,3 @@ def compose_artifact(env: str, role: str) -> pathlib.Path: def compose_extra_artifact(env: str) -> pathlib.Path: return env_root(env) / "compose.extra.yml" - - -def history_file() -> pathlib.Path: - """Local record of deployed image tags, used by `deploy rollback`.""" - return GENERATED_DIR / ".history" diff --git a/src/deployctl/cli/scaffold.py b/src/deployctl/cli/scaffold.py index 0b03542..52900a7 100644 --- a/src/deployctl/cli/scaffold.py +++ b/src/deployctl/cli/scaffold.py @@ -59,11 +59,6 @@ # DNS: A {api_subdomain}. → this server's public IP. API_SUBDOMAIN={api_subdomain} -# ---------- Image ---------- -# An immutable tag (a git short sha). Avoid 'latest': it breaks rollback and -# makes "what is actually running?" unanswerable. -IMAGE_TAG= - # ---------- Server ---------- # The address your control machine reaches over SSH. HOSTS="" @@ -114,10 +109,6 @@ # DNS: A {api_subdomain}. → the LOAD BALANCER's IP, not a host. API_SUBDOMAIN={api_subdomain} -# ---------- Image ---------- -# An immutable tag (a git short sha), so every host runs identical bits. -IMAGE_TAG= - # ---------- Hosts ---------- # Every app server, space-separated, at the address your control machine reaches # over SSH (public IPs from a laptop; private IPs from an in-VPC bastion). diff --git a/src/deployctl/cli/tags.py b/src/deployctl/cli/tags.py new file mode 100644 index 0000000..772b085 --- /dev/null +++ b/src/deployctl/cli/tags.py @@ -0,0 +1,103 @@ +""" +Which image tag a command works with — no longer a value in ``config/``. + +Once CI deploys, a tag written in ``config/.env`` is whatever that machine +deployed last: stale by design, and a laptop "Update" with it would take +production backwards. So the tag is decided per command, in this order: + +1. ``--tag`` / ``$IMAGE_TAG`` — what CI and an explicit operator pass; +2. ``IMAGE_TAG`` in a config file — still honoured, with a deprecation warning + (``deployctl migrate-config`` removes it); +3. for commands that act on the hosts: **what the primary is running**, read from + its ``.deployctl-state`` — so ``deploy update`` with no tag redeploys the + running release, which is how a config change goes out; +4. for local commands (``setup``, ``validate``): the tag this machine last + rendered or deployed, cached in ``generated/.tag-``. + +Nothing found is an error that says to pass ``--tag`` — never a silent guess. +""" + +from __future__ import annotations + +import os +import re + +from . import paths +from .config import Config + +#: What a tag may look like. Registry tags allow more, but a tag is also written +#: into shell commands on the hosts, so the alphabet is kept to what is needed. +TAG_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}") + + +class NoTag(Exception): + """No tag could be resolved for a command that needs one.""" + + +def cache_file(env: str): + return paths.GENERATED_DIR / f".tag-{env}" + + +def cached(env: str) -> str: + path = cache_file(env) + return path.read_text().strip() if path.is_file() else "" + + +def remember(env: str, tag: str) -> None: + """The tag this machine last rendered or deployed, for later local commands.""" + path = cache_file(env) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(tag + "\n") + + +def running(cfg: Config) -> str: + """The tag the primary runs, over ssh. Empty when it cannot be read.""" + from .runner import run # deferred: runner imports config, config nothing of this + + try: + proc = run(cfg, "deploy.sh", ["running-tag"], check=False, capture=True) + except FileNotFoundError: + return "" + return proc.stdout.strip().splitlines()[-1] if proc.returncode == 0 and proc.stdout.strip() else "" + + +def check(tag: str) -> str: + if not TAG_RE.fullmatch(tag): + raise NoTag(f"{tag!r} is not a usable image tag") + return tag + + +def resolve(cfg: Config, explicit: str | None = None, *, from_hosts: bool) -> tuple[str, str]: + """``(tag, where it came from)`` for this command; applied to ``cfg``. + + ``from_hosts`` is for commands that talk to the hosts anyway — reading the + running tag costs one ssh round trip there, and nothing a local command + should pay for. + """ + tag, source = "", "" + if explicit: + tag, source = explicit, "--tag" + elif os.environ.get("IMAGE_TAG"): + tag, source = os.environ["IMAGE_TAG"], "$IMAGE_TAG" + elif cfg.raw_input.get("IMAGE_TAG"): + tag, source = cfg.raw_input["IMAGE_TAG"], f"config/{cfg.env}.env (deprecated)" + elif from_hosts and (found := running(cfg)): + tag, source = found, f"running on {cfg.primary_host}" + elif found := cached(cfg.env): + tag, source = found, "last used on this machine" + + if not tag: + raise NoTag( + f"no image tag for '{cfg.env}' — pass --tag " + f"(see: deployctl image tags --env {cfg.env})" + ) + check(tag) + cfg.raw["IMAGE_TAG"] = tag + cfg.refresh_derived() + return tag, source + + +def deprecated_in_config(cfg: Config) -> bool: + """IMAGE_TAG written in a config file, rather than passed or read from the hosts.""" + return bool(cfg.raw_input.get("IMAGE_TAG")) and not os.environ.get("IMAGE_TAG") + diff --git a/src/deployctl/scripts/backup.sh b/src/deployctl/scripts/backup.sh index 6004e97..a08c570 100755 --- a/src/deployctl/scripts/backup.sh +++ b/src/deployctl/scripts/backup.sh @@ -9,7 +9,7 @@ # client tools) # # Dumps land in $REMOTE_DIR/backups/ on the host (excluded from deploy syncs) -# and, by default, are also fetched to backups/ in the deploy directory — a dead host +# and, by default, are also fetched to ~/.deployctl/backups/// — a dead host # must not take its own backups with it. # # Usage: backup.sh @@ -39,7 +39,14 @@ source "$SCRIPT_DIR/common/remote.sh" COMMAND="${1:-}" readonly HOST="$PRIMARY_HOST" readonly BACKUP_DIR="${REMOTE_DIR}/backups" -readonly LOCAL_DIR="${DEPLOYCTL_PROJECT}/backups" +# Never inside a repository: a copied deploy directory once carried another +# client's production dump into a different client's project. DEPLOYCTL_BACKUP_DIR +# moves it elsewhere (an encrypted volume, a synced folder). +# An unquoted `~`, not $HOME: cron and stripped CI environments have no HOME (and +# set -u would abort on it), while bash resolves a bare ~ from the passwd entry. +# It must stay outside quotes — "~" is never expanded. +_home=~ +readonly LOCAL_DIR="${DEPLOYCTL_BACKUP_DIR:-${_home}/.deployctl/backups/${COMPOSE_PROJECT}/${DEPLOYCTL_ENV}}" readonly KEEP="${BACKUP_KEEP:-7}" readonly FETCH="${BACKUP_FETCH:-true}" # Must be at least the server's major version: pg_dump refuses to dump a newer @@ -95,8 +102,8 @@ cmd_run() { remote_maybe "$HOST" "cd '${BACKUP_DIR}' && ls -1t *.sql.gz 2>/dev/null | tail -n +$((KEEP + 1)) | xargs -r rm -f" if [[ "$FETCH" == "true" ]]; then - mkdir -p "$LOCAL_DIR" - print_info "fetching a local copy → backups/${file}" + mkdir -p "$LOCAL_DIR" && chmod 700 "$LOCAL_DIR" + print_info "fetching a local copy → ${LOCAL_DIR}/${file}" # shellcheck disable=SC2086 maybe rsync -az -e "ssh $SSH_OPTS" "${SSH_USER}@${HOST}:${BACKUP_DIR}/${file}" "${LOCAL_DIR}/${file}" [[ "${DEPLOYCTL_DRY_RUN:-}" != "1" ]] && chmod 600 "${LOCAL_DIR}/${file}" 2>/dev/null @@ -118,7 +125,7 @@ cmd_list() { fi print_info "on ${HOST}:${BACKUP_DIR}" remote "$HOST" "ls -lht '${BACKUP_DIR}' 2>/dev/null | grep -v '^total' | sed 's/^/ /'" || print_warning " none yet" - print_info "local (${LOCAL_DIR#"${DEPLOYCTL_PROJECT}"/}/)" + print_info "local (${LOCAL_DIR}/)" local found=false f for f in "$LOCAL_DIR"/*.sql.gz; do [[ -e "$f" ]] || break diff --git a/src/deployctl/scripts/common/remote.sh b/src/deployctl/scripts/common/remote.sh index 86ec418..854b8a4 100755 --- a/src/deployctl/scripts/common/remote.sh +++ b/src/deployctl/scripts/common/remote.sh @@ -32,7 +32,17 @@ if [[ "${DEPLOYCTL_SSH_STRICT:-}" == "1" ]]; then else _HOST_KEY_POLICY="accept-new" fi -readonly SSH_OPTS="-o StrictHostKeyChecking=${_HOST_KEY_POLICY} -o ConnectTimeout=10 -o ServerAliveInterval=15 -o ServerAliveCountMax=4" +# SSH_JUMP_HOST reaches hosts that are only on a private network through a +# bastion (ssh's ProxyJump, e.g. deploy@bastion.example.com). Everything that +# connects — ssh, rsync, the lock, the panel's terminal — goes through SSH_OPTS, so +# setting it once covers all of them. Its own host key must be pinned too. +_JUMP_OPT="" +if [[ -n "${SSH_JUMP_HOST:-}" ]]; then + _JUMP_OPT=" -o ProxyJump=${SSH_JUMP_HOST}" +fi +# SSH_BASE_OPTS reaches the jump host itself; ProxyJump through it would loop. +readonly SSH_BASE_OPTS="-o StrictHostKeyChecking=${_HOST_KEY_POLICY} -o ConnectTimeout=10 -o ServerAliveInterval=15 -o ServerAliveCountMax=4" +readonly SSH_OPTS="${SSH_BASE_OPTS}${_JUMP_OPT}" # Artifacts are scoped per environment, and generated// mirrors REMOTE_DIR # exactly — which is what makes the compose file's ../ mounts resolve the same @@ -504,16 +514,24 @@ revert_host() { readonly LOCK_DIR="${REMOTE_DIR}/.deployctl.lock" DEPLOY_LOCK_HELD=0 +# Who is changing the hosts, in one word: the Actions run's URL under CI — the +# runner's own name is a throwaway VM, the URL shows the log — else user@machine. +# Used by the lock, the state file and the history, so all three agree. +deployer_identity() { + if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then + echo "${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY:-?}/actions/runs/${GITHUB_RUN_ID:-?}" + else + echo "$(id -un)@$(hostname -s 2>/dev/null || hostname)" + fi +} + acquire_deploy_lock() { local purpose="$1" me verdict [[ "${DEPLOYCTL_DRY_RUN:-}" == "1" ]] && return 0 if [[ "${GITHUB_ACTIONS:-}" == "true" ]]; then - # "runner@fv-az123" names a throwaway VM. The run's URL is what someone - # locked out from their laptop needs: it shows the log, and whether it is - # still going. - me="GitHub Actions (${GITHUB_ACTOR:-?}) · ${purpose} · since $(date -u +%Y-%m-%dT%H:%M:%SZ) · ${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY:-?}/actions/runs/${GITHUB_RUN_ID:-?}" + me="GitHub Actions (${GITHUB_ACTOR:-?}) · ${purpose} · since $(date -u +%Y-%m-%dT%H:%M:%SZ) · $(deployer_identity)" else - me="$(id -un)@$(hostname -s 2>/dev/null || hostname) · ${purpose} · since $(date -u +%Y-%m-%dT%H:%M:%SZ) · pid $$" + me="$(deployer_identity) · ${purpose} · since $(date -u +%Y-%m-%dT%H:%M:%SZ) · pid $$" fi verdict="$(remote "$PRIMARY_HOST" " mkdir -p '${REMOTE_DIR}' 2>/dev/null @@ -660,10 +678,24 @@ check_live_env() { return 1 } -# Record which tag runs on a host — what `deploy rollback` reads back. +# Record which tag runs on a host — what `deploy rollback` reads back. The tag is +# IMAGE_TAG unless given: a fleet revert records the one each host went back to. record_state() { - local host="$1" - remote_maybe "$host" "printf 'IMAGE_TAG=%s\nDEPLOYED_AT=%s\nENV=%s\n' '${IMAGE_TAG}' \"\$(date -u +%Y-%m-%dT%H:%M:%SZ)\" '${DEPLOYCTL_ENV}' > '${REMOTE_DIR}/${STATE_FILE}'" + local host="$1" tag="${2:-$IMAGE_TAG}" + remote_maybe "$host" "printf 'IMAGE_TAG=%s\nDEPLOYED_AT=%s\nDEPLOYED_BY=%s\nENV=%s\n' '${tag}' \"\$(date -u +%Y-%m-%dT%H:%M:%SZ)\" '$(deployer_identity)' '${DEPLOYCTL_ENV}' > '${REMOTE_DIR}/${STATE_FILE}'" +} + +# ---- the history, kept on the primary --------------------------------------------- +# One line per completed release of the environment, appended where every deploy +# can see it — a laptop's own log never saw what CI deployed, so rollback picked +# its "previous" tag from half the story. Tab-separated: +# +# A release that failed and was reverted is not a release, and is not recorded. +readonly HISTORY_FILE=".deployctl-history" + +append_history() { + local kind="$1" + remote_maybe "$PRIMARY_HOST" "printf '%s\t%s\t%s\t%s\t%s\n' \"\$(date -u +%Y-%m-%dT%H:%M:%SZ)\" '${DEPLOYCTL_ENV}' '${IMAGE_TAG}' '${kind}' '$(deployer_identity)' >> '${REMOTE_DIR}/${HISTORY_FILE}'" } read_state_tag() { diff --git a/src/deployctl/scripts/deploy.sh b/src/deployctl/scripts/deploy.sh index e1bfe7d..1d718ad 100755 --- a/src/deployctl/scripts/deploy.sh +++ b/src/deployctl/scripts/deploy.sh @@ -46,7 +46,7 @@ fi require_env() { local missing=0 key - for key in DEPLOYCTL_ENV HOSTS PRIMARY_HOST SSH_USER REMOTE_DIR COMPOSE_PROJECT IMAGE_REF; do + for key in DEPLOYCTL_ENV HOSTS PRIMARY_HOST SSH_USER REMOTE_DIR COMPOSE_PROJECT; do if [[ -z "${!key:-}" ]]; then print_error "missing required environment: $key (deploy.sh is driven by the deployctl CLI)" missing=1 @@ -55,6 +55,15 @@ require_env() { [[ $missing -eq 0 ]] || exit 2 } +# Only the commands that put an image on the hosts need to know which one. +# Reading — status, logs, the running tag, the history — does not. +require_image() { + if [[ -z "${IMAGE_TAG:-}" || -z "${IMAGE_REF:-}" || "$IMAGE_REF" == *: ]]; then + print_error "no image tag for this command — pass --tag (the CLI resolves one from the hosts otherwise)" + exit 2 + fi +} + # ---- permissions ------------------------------------------------------------- # The rendered artifacts, before any of them are shipped. @@ -344,32 +353,76 @@ start_release() { verify_services "$host" "$role" "$baseline" || return 1 } +# Hosts this run has moved to the new release, in order, and the tag each ran +# before it — "host=tag" pairs. Strings, not arrays: macOS ships bash 3.2, which +# has no associative arrays. +ROLLED="" +PREVIOUS_TAGS="" + +previous_tag_of() { + local pair + for pair in $PREVIOUS_TAGS; do + [[ "${pair%%=*}" == "$1" ]] && { echo "${pair#*=}"; return 0; } + done + return 0 +} + # Bring one host to the new release and verify it before touching the next. If -# any step fails, the previous release is put back on this host and the roll -# stops. Both steps run inside `if`, where bash suspends set -e, which is why -# every function they call returns its own status. +# any step fails, the previous release is put back on this host and it returns 1; +# the caller decides what happens to the hosts rolled before it. Both steps run +# inside `if`, where bash suspends set -e, which is why every function they call +# returns its own status. roll_host() { local host="$1" role previous="" role="$(host_role "$host")" print_separator print_info "rolling ${host} (${role})" [[ "${DEPLOYCTL_DRY_RUN:-}" == "1" ]] || previous="$(read_state_tag "$host")" + PREVIOUS_TAGS="${PREVIOUS_TAGS} ${host}=${previous}" take_snapshot "$host" "$role" if stage_release "$host" "$role" && start_release "$host" "$role"; then record_state "$host" + ROLLED="${ROLLED} ${host}" return 0 fi print_error "[$host] ${IMAGE_TAG} failed on this host — stopping the roll" revert_host "$host" "$role" "$previous" || true - local -a fleet - read -r -a fleet <<< "$HOSTS" - if (( ${#fleet[@]} > 1 )); then - print_error " hosts rolled before this one stay on ${IMAGE_TAG}; the rest were not touched." + return 1 +} + +# A host failed mid-roll. Unless told otherwise, every host this run already +# moved goes back too, newest first, each from its own snapshot — so an unattended +# deploy never leaves the fleet split between two releases behind the load +# balancer. REVERT_SCOPE=host keeps them on the new release (the failed host is +# still reverted) and says how to converge by hand. +revert_fleet() { + local failed="$1" h role previous reversed="" all_back=1 + [[ -n "${ROLLED// /}" ]] || return 0 + if [[ "${REVERT_SCOPE:-fleet}" != "fleet" || "${DEPLOYCTL_NO_REVERT:-}" == "1" ]]; then + print_error " hosts already on ${IMAGE_TAG}:${ROLLED} — the fleet is split between two releases" print_error " → all back to one release: deployctl deploy rollback --env ${DEPLOYCTL_ENV}" + return 0 + fi + print_separator + print_warning "[fleet] ${failed} failed — putting back the hosts this run already moved to ${IMAGE_TAG}:${ROLLED}" + for h in $ROLLED; do reversed="${h} ${reversed}"; done + for h in $reversed; do + role="$(host_role "$h")" + previous="$(previous_tag_of "$h")" + if revert_host "$h" "$role" "$previous"; then + # It recorded the new tag when it passed; the record follows the revert. + [[ -n "$previous" ]] && record_state "$h" "$previous" + else + all_back=0 + fi + done + if [[ $all_back -eq 1 ]]; then + print_warning "[fleet] every host is back on the release it ran before" + else + print_error "[fleet] not every host could be put back — deployctl deploy status --env ${DEPLOYCTL_ENV} shows what each runs" fi - exit 1 } # ---- commands ----------------------------------------------------------------- @@ -417,6 +470,7 @@ cmd_init() { record_state "$h" done + append_history init print_separator print_success "initial deployment complete" if [[ "${TLS_LE:-false}" == "true" ]]; then @@ -463,13 +517,17 @@ cmd_update() { fi fi - # One host at a time, health-gated: a bad release stops the roll while the - # remaining hosts still serve the previous one. + # One host at a time, health-gated. A host that fails is reverted, and so is + # every host already rolled (revert_fleet), so the command is all or nothing. local h for h in $(ordered_hosts); do - roll_host "$h" + if ! roll_host "$h"; then + revert_fleet "$h" + exit 1 + fi done + append_history "$( [[ "$kind" == rollback ]] && echo rollback || echo deploy )" print_separator print_success "${kind} complete — every host is on ${IMAGE_TAG}" } @@ -480,7 +538,8 @@ cmd_roll_one() { acquire_deploy_lock "deploy update --host ${host} → ${IMAGE_TAG}" || exit 1 # This path skips doctor, but not the one check that stops an outage. check_live_env "$host" || exit 1 - roll_host "$host" + roll_host "$host" || exit 1 + append_history "host:${host}" } cmd_migrate() { @@ -600,6 +659,99 @@ cmd_unlock() { print_success "lock cleared" } +# ---- the CI deploy key ------------------------------------------------------------ +# `deployctl ci setup-key` drives this: a key only CI uses, on every host — and on +# the jump host, whose own login CI needs too. The key is recognised by its comment +# (CI_KEY_COMMENT), which is how a rotation finds the one it replaces. +# CI_KEY_MODE=check print "PRESENT " or "ABSENT " for each +# CI_KEY_MODE=install append CI_PUBKEY, prefixed `restrict` (no forwarding, no pty) +# CI_KEY_MODE=rotate remove every line with CI_KEY_COMMENT, then append +# CI_KEY_MODE=verify log in with CI_KEY_FILE alone, no agent: what CI will do +ci_key_on() { + local target="$1" opts="$2" script + # shellcheck disable=SC2016 # $HOME and friends expand on the host + script='umask 077; mkdir -p ~/.ssh; touch ~/.ssh/authorized_keys; f=~/.ssh/authorized_keys' + case "$CI_KEY_MODE" in + check) + script="$script; if grep -q -F -- '${CI_KEY_COMMENT}' \"\$f\"; then echo PRESENT; else echo ABSENT; fi" ;; + install) + script="$script; grep -q -x -F -- 'restrict ${CI_PUBKEY}' \"\$f\" || printf '%s\n' 'restrict ${CI_PUBKEY}' >> \"\$f\"; echo INSTALLED" ;; + rotate) + script="$script; grep -v -F -- '${CI_KEY_COMMENT}' \"\$f\" > \"\$f.tmp\" || true; mv \"\$f.tmp\" \"\$f\"; printf '%s\n' 'restrict ${CI_PUBKEY}' >> \"\$f\"; echo INSTALLED" ;; + esac + # shellcheck disable=SC2086 # opts word-splits into flags + ssh $opts "$target" "$script" +} + +cmd_ci_key() { + local h out bad=0 + [[ "${CI_KEY_MODE:-}" =~ ^(check|install|rotate|verify)$ ]] || { print_error "CI_KEY_MODE must be check, install, rotate or verify"; exit 2; } + [[ -n "${CI_KEY_COMMENT:-}" ]] || { print_error "CI_KEY_COMMENT is not set"; exit 2; } + local targets=() opts=() + for h in $(ordered_hosts); do targets+=("${SSH_USER}@${h}"); opts+=("$SSH_OPTS"); done + if [[ -n "${SSH_JUMP_HOST:-}" ]]; then + # The first hop of the chain. ProxyJump takes host:port; an ssh target does + # not, so a port becomes -p. + local jump="${SSH_JUMP_HOST%%,*}" jump_opts="$SSH_BASE_OPTS" + if [[ "$jump" =~ :([0-9]+)$ ]]; then + jump_opts="$jump_opts -p ${BASH_REMATCH[1]}" + jump="${jump%:*}" + fi + targets+=("$jump"); opts+=("$jump_opts") + fi + + local i + for i in "${!targets[@]}"; do + if [[ "$CI_KEY_MODE" == "verify" ]]; then + # shellcheck disable=SC2086 + if ssh ${opts[$i]} -i "$CI_KEY_FILE" -o IdentitiesOnly=yes -o IdentityAgent=none -o BatchMode=yes "${targets[$i]}" true; then + echo "OK ${targets[$i]}" + else + echo "FAILED ${targets[$i]}"; bad=1 + fi + continue + fi + if out="$(ci_key_on "${targets[$i]}" "${opts[$i]}")"; then + echo "${out##*$'\n'} ${targets[$i]}" + else + echo "UNREACHABLE ${targets[$i]}"; bad=1 + fi + done + return $bad +} + +# The tag the primary runs — what `deploy update` redeploys when none is given. +cmd_running_tag() { + read_state_tag "$PRIMARY_HOST" +} + +# The environment's release history, from the primary. Empty when there is none. +cmd_history() { + remote "$PRIMARY_HOST" "cat '${REMOTE_DIR}/${HISTORY_FILE}' 2>/dev/null" 2>/dev/null || true +} + +# Everything `deploy status --json` needs, tab-separated, one fact per line: +# HOST +# STATE — the host's .deployctl-state +# SVC — see container_report +cmd_state() { + local h role line + for h in $(ordered_hosts); do + role="$(host_role "$h")" + if ! remote "$h" true 2>/dev/null; then + printf 'HOST\t%s\t%s\tunreachable\n' "$h" "$role" + continue + fi + printf 'HOST\t%s\t%s\treachable\n' "$h" "$role" + while IFS= read -r line; do + [[ -n "$line" ]] && printf 'STATE\t%s\t%s\n' "$h" "$line" + done < <(remote "$h" "cat '${REMOTE_DIR}/${STATE_FILE}' 2>/dev/null" 2>/dev/null || true) + while IFS= read -r line; do + [[ -n "$line" ]] && printf 'SVC\t%s\t%s\n' "$h" "$line" + done < <(container_report "$h") + done +} + hosts_or_arg() { if [[ -n "$ARG_HOST" ]]; then echo "$ARG_HOST"; else ordered_hosts; fi } @@ -607,6 +759,9 @@ hosts_or_arg() { # ---- dispatch ------------------------------------------------------------------- require_env +case "$COMMAND" in + doctor|init|update|rollback|roll-one|migrate) require_image ;; +esac case "$COMMAND" in doctor) doctor ;; init) cmd_init ;; @@ -614,6 +769,10 @@ case "$COMMAND" in rollback) cmd_update rollback ;; roll-one) cmd_roll_one ;; migrate) cmd_migrate ;; + running-tag) cmd_running_tag ;; + history) cmd_history ;; + state) cmd_state ;; + ci-key) cmd_ci_key ;; restart) cmd_restart ;; stop) cmd_stop ;; status) cmd_status ;; diff --git a/src/deployctl/templates/ci/build-image.yml.j2 b/src/deployctl/templates/ci/build-image.yml.j2 index b6d638a..0cbfbe7 100644 --- a/src/deployctl/templates/ci/build-image.yml.j2 +++ b/src/deployctl/templates/ci/build-image.yml.j2 @@ -1,93 +1,121 @@ -# Build and publish the application image — generated by `deployctl ci init`. +# Managed by deployctl «VERSION» — regenerate with: deployctl ci init --force # -# On every push to «DEPLOY_BRANCH» this builds «CI_DOCKERFILE» and pushes -# «IMAGE_REPO»: (plus :latest as a convenience pointer) -# using the workflow's own GITHUB_TOKEN — no secrets to configure. +# Build and publish the application image. ci.yml calls this after its checks +# pass on a push to «DEPLOY_BRANCH»; a manual run or a v* tag builds without them. # -# When the project sets WORKER_BUILD_TARGET, a second image -# «IMAGE_REPO»:-worker -# is built from that stage and pushed in the same run, so the two can never be -# published a commit apart. +# «IMAGE_REPO»: the api image +{% if WORKER_BUILD_TARGET %} +# «IMAGE_REPO»:-worker the «WORKER_BUILD_TARGET» stage, for the Celery containers — +# built in the same run, so the two are never a commit apart +{% endif %} +# +# The tag it pushed is this workflow's `tag` output. The deploy job uses exactly +# that: recomputing it (`git rev-parse --short=7` prints more than seven +# characters when seven are ambiguous) can name an image that was never pushed. # # ONE repository setting is required or the push fails with # "denied: permission_denied": # Settings → Actions → General → Workflow permissions → "Read and write permissions" -# -# After a green run, the Summary shows the exact IMAGE_TAG to deploy. name: build-image on: + workflow_call: + outputs: + tag: + description: The immutable image tag this run pushed (short commit SHA) + value: ${{ jobs.build.outputs.tag }} push: - branches: ["«DEPLOY_BRANCH»"] + tags: ["v*"] workflow_dispatch: {} +# Two pushes in quick succession would race, and both would write `latest`. A +# build that was going to publish an immutable tag is worth finishing. concurrency: group: build-image-${{ github.ref }} cancel-in-progress: false -permissions: - contents: read - packages: write - jobs: build: runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + outputs: + tag: ${{ steps.sha.outputs.short }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@«ACTIONS.checkout» # «ACTIONS.checkout_version» + with: + persist-credentials: false - - name: Compute the image tag (git short SHA) - id: meta - # The deploy branch publishes the plain short SHA — the only form the - # control panel's tag picker offers, newest first. A manual run from any - # other branch gets -dev appended: pullable by name, never offered. + # Pushes to «DEPLOY_BRANCH» and v* tags publish the plain short SHA — the form + # `deployctl image tags` lists. Any other branch gets -dev appended: pullable + # by name, never offered as a release. + - name: Compute the image tag + id: sha run: | - sha="$(git rev-parse --short HEAD)" - if [ "$GITHUB_REF" = "refs/heads/«DEPLOY_BRANCH»" ]; then - echo "tag=$sha" >> "$GITHUB_OUTPUT" - else - echo "tag=$sha-dev" >> "$GITHUB_OUTPUT" - fi + short="$(git rev-parse --short=7 HEAD)" + case "$GITHUB_REF" in + refs/heads/«DEPLOY_BRANCH»|refs/tags/*) echo "short=$short" >> "$GITHUB_OUTPUT" ;; + *) echo "short=$short-dev" >> "$GITHUB_OUTPUT" ;; + esac - - uses: docker/setup-buildx-action@v3 - - - name: Log in to «REGISTRY_HOST» - uses: docker/login-action@v3 + - uses: docker/login-action@«ACTIONS.login» # «ACTIONS.login_version» with: registry: «REGISTRY_HOST» username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push - uses: docker/build-push-action@v6 + - uses: docker/setup-buildx-action@«ACTIONS.buildx» # «ACTIONS.buildx_version» + + - name: Build & push — api + uses: docker/build-push-action@«ACTIONS.build_push» # «ACTIONS.build_push_version» with: context: «CI_BUILD_CONTEXT» file: «CI_DOCKERFILE» -«BUILD_TARGET_LINE» +{% if BUILD_TARGET %} + target: «BUILD_TARGET» +{% endif %} platforms: «IMAGE_PLATFORM» push: true tags: | - «IMAGE_REPO»:${{ steps.meta.outputs.tag }} + «IMAGE_REPO»:${{ steps.sha.outputs.short }} ${{ github.ref == 'refs/heads/«DEPLOY_BRANCH»' && '«IMAGE_REPO»:latest' || '' }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=api + cache-to: type=gha,mode=max,scope=api + # A provenance attestation turns the tag into a manifest index with an + # `unknown/unknown` entry, which `docker manifest inspect` — how doctor + # checks an image is pullable — reports confusingly. provenance: false -«WORKER_BUILD_STEP» +{% if WORKER_BUILD_TARGET %} - - name: Deployment summary + # Reuses everything the api build just made; reads its cache as well as + # its own, so a cache-only run stays fast. + - name: Build & push — worker + uses: docker/build-push-action@«ACTIONS.build_push» # «ACTIONS.build_push_version» + with: + context: «CI_BUILD_CONTEXT» + file: «CI_DOCKERFILE» + target: «WORKER_BUILD_TARGET» + platforms: «IMAGE_PLATFORM» + push: true + tags: | + «IMAGE_REPO»:${{ steps.sha.outputs.short }}-worker + ${{ github.ref == 'refs/heads/«DEPLOY_BRANCH»' && '«IMAGE_REPO»:latest-worker' || '' }} + cache-from: | + type=gha,scope=worker + type=gha,scope=api + cache-to: type=gha,mode=max,scope=worker + provenance: false +{% endif %} + + - name: Summary + env: + TAG: ${{ steps.sha.outputs.short }} run: | { - echo "## Image published" - echo "" - echo '```' - echo "IMAGE_REPO=«IMAGE_REPO»" - echo "IMAGE_TAG=${{ steps.meta.outputs.tag }}" - echo '```' + echo "### Image published: \`${TAG}\`" echo "" - echo "Deploy it:" - echo '```' - echo "deployctl image tags # confirm it is visible" - echo "# set IMAGE_TAG=${{ steps.meta.outputs.tag }} in deploy/config/.env, then" - echo "deployctl setup --env && deployctl deploy update --env " - echo '```' + echo "Deploy it: \`deployctl ci deploy --env «ENV» --tag ${TAG}\`" } >> "$GITHUB_STEP_SUMMARY" diff --git a/src/deployctl/templates/ci/ci.yml.j2 b/src/deployctl/templates/ci/ci.yml.j2 new file mode 100644 index 0000000..9eccfc3 --- /dev/null +++ b/src/deployctl/templates/ci/ci.yml.j2 @@ -0,0 +1,52 @@ +# Written once by deployctl «VERSION» (`deployctl ci init`) — this file is YOURS: +# ci init never overwrites it. Put your project's checks in the `checks` job. +# +# A push to «DEPLOY_BRANCH» runs the checks, then builds and publishes the image +# (build-image.yml), then — with the repository variable AUTO_DEPLOY set to +# 'true' — deploys that exact image (deploy.yml). Nothing is deployed that did +# not pass the checks. + +name: CI + +on: + pull_request: + push: + branches: ["«DEPLOY_BRANCH»"] + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + checks: + name: Checks + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@«ACTIONS.checkout» # «ACTIONS.checkout_version» + with: + persist-credentials: false + # Replace with your tests, lint, migration checks. + - run: echo "No checks yet — add your test command to .github/workflows/ci.yml" + + publish: + name: Build & push image + needs: [checks] + if: github.event_name == 'push' && github.ref == 'refs/heads/«DEPLOY_BRANCH»' + permissions: + contents: read + packages: write + uses: ./.github/workflows/build-image.yml + + deploy: + name: Deploy to «ENV» + needs: publish + if: github.event_name == 'push' && github.ref == 'refs/heads/«DEPLOY_BRANCH»' && vars.AUTO_DEPLOY == 'true' + uses: ./.github/workflows/deploy.yml + with: + environment: «ENV» + tag: ${{ needs.publish.outputs.tag }} + secrets: inherit diff --git a/src/deployctl/templates/ci/deploy.yml.j2 b/src/deployctl/templates/ci/deploy.yml.j2 new file mode 100644 index 0000000..3f16e72 --- /dev/null +++ b/src/deployctl/templates/ci/deploy.yml.j2 @@ -0,0 +1,192 @@ +# Managed by deployctl «VERSION» — regenerate with: deployctl ci init --force +# +# Deploys an already-published image with deployctl — the same engine, deploy +# lock and guards as a deploy from a laptop, run from here. +# +# automatic ci.yml calls this after a push to «DEPLOY_BRANCH» passed its checks +# and published, when the repository variable AUTO_DEPLOY is 'true' +# manual Actions → deploy → Run workflow from «DEPLOY_BRANCH», or +# `deployctl ci deploy`: a tag, a config change, or a rollback +# +# Nothing here builds, and nothing here needs the application's dependencies: +# the job installs deployctl «TOOL_REF» and tells the hosts which image to run. +# Setup — keys, secrets, variables: `deployctl ci doctor` lists what is missing. + +name: deploy +run-name: ${{ inputs.action || 'update' }} ${{ inputs.tag }} → ${{ inputs.environment }} + +on: + workflow_call: + inputs: + environment: { type: string, required: true } + tag: { type: string, required: true } + action: { type: string, default: update } + allow_config_change: { type: boolean, default: false } + workflow_dispatch: + inputs: + # A plain string, not `type: environment`: that dropdown lists configured + # GitHub environments, and a private repository on GitHub Free has none. + environment: + description: Environment to deploy to (a deploy/config/.env) + type: string + default: «ENV» + required: true + tag: + description: Image tag, a short commit SHA + type: string + required: true + action: + description: update migrates then rolls; rollback rolls an older tag and never migrates + type: choice + options: [update, rollback] + default: update + allow_config_change: + description: Ship config values that differ from the servers' (after deployctl ci sync-config) + type: boolean + default: false + +permissions: + contents: read + +jobs: + deploy: + name: ${{ inputs.action }} ${{ inputs.tag }} → ${{ inputs.environment }} + runs-on: ubuntu-latest + # Each host takes a few minutes (pull, restart, health gate, service watch); + # sized for «HOST_COUNT» host(s). `deployctl ci init --force` recomputes it. + timeout-minutes: «TIMEOUT_MINUTES» + # Records a Deployment. On GitHub Team/Pro it also unlocks the environment's + # own secrets, readable only from its allowed branches; on GitHub Free with a + # private repository, secrets.* and vars.* resolve to the repository's. + environment: + name: ${{ inputs.environment }} + url: ${{ vars.APP_URL }} + # One deploy per environment at a time, never cancelled mid-roll. The lock on + # the primary also keeps a laptop deploy from interleaving with this one. + concurrency: + group: deploy-${{ inputs.environment }} + cancel-in-progress: false + env: + # Prefixed so they cannot collide with a deployctl config key: the process + # environment overrides any key the config defines. + DEPLOY_ENV: ${{ inputs.environment }} + DEPLOY_TAG: ${{ inputs.tag }} + DEPLOY_ACTION: ${{ inputs.action }} + DEPLOY_ALLOW_CONFIG_CHANGE: ${{ inputs.allow_config_change }} + steps: + # Inputs reach scripts through env, never pasted in with ${{ }}, and are + # checked first. The branch check matters most: the workflow that runs is + # the one on the branch the run was started from. + - name: Check the inputs + run: | + [[ "$GITHUB_REF" == "refs/heads/«DEPLOY_BRANCH»" ]] \ + || { echo "::error::deploys run only from «DEPLOY_BRANCH» — this run was started from $GITHUB_REF"; exit 1; } + [[ "$DEPLOY_TAG" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]] \ + || { echo "::error::not an image tag: '$DEPLOY_TAG'"; exit 1; } + [[ "$DEPLOY_ACTION" == update || "$DEPLOY_ACTION" == rollback ]] \ + || { echo "::error::action must be update or rollback, got '$DEPLOY_ACTION'"; exit 1; } + + # Actions are pinned to commit SHAs: this job holds the production ssh key. + - uses: actions/checkout@«ACTIONS.checkout» # «ACTIONS.checkout_version» + with: + # Full history: `deploy update` refuses a tag that is an ancestor of the + # running one, and needs both commits to tell. + fetch-depth: 0 + persist-credentials: false + + - uses: astral-sh/setup-uv@«ACTIONS.setup_uv» # «ACTIONS.setup_uv_version» + + # The pinned version, not whatever is newest: a tool upgrade is a reviewed + # change to this file. DEPLOYCTL_INSTALL_TOKEN (read-only, contents:read on + # «TOOL_REPO») is needed only while that repository is private. + - name: Install deployctl «TOOL_REF» + env: + TOKEN: ${{ secrets.DEPLOYCTL_INSTALL_TOKEN }} + run: | + if [[ -n "$TOKEN" ]]; then + uv tool install "git+https://x-access-token:${TOKEN}@github.com/«TOOL_REPO»@«TOOL_REF»" + else + uv tool install "git+https://github.com/«TOOL_REPO»@«TOOL_REF»" + fi + deployctl --version + + # Optional: reach the servers over a tailnet, so :22 need not be open to the + # internet. Skipped until the TAILSCALE_TAGS variable is set. + - name: Join the tailnet + if: vars.TAILSCALE_TAGS != '' + uses: tailscale/github-action@«ACTIONS.tailscale» # «ACTIONS.tailscale_version» + with: + oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} + oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} + tags: ${{ vars.TAILSCALE_TAGS }} + + - name: SSH identity and pinned host keys + env: + SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + KNOWN_HOSTS: ${{ vars.DEPLOY_KNOWN_HOSTS }} + run: | + [[ -n "$SSH_KEY" ]] || { echo "::error::DEPLOY_SSH_KEY is not set — run: deployctl ci setup-key"; exit 1; } + [[ -n "$KNOWN_HOSTS" ]] || { echo "::error::DEPLOY_KNOWN_HOSTS is not set — run: deployctl ci pin-hosts"; exit 1; } + install -d -m 700 ~/.ssh + printf '%s\n' "$SSH_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + printf '%s\n' "$KNOWN_HOSTS" > ~/.ssh/known_hosts + + # Registers every credential in the bundle with ::add-mask:: before anything + # can print one, and refuses a bundle that does not match its digest. + - name: Load the deploy config + env: + DEPLOYCTL_CONFIG: ${{ secrets.DEPLOYCTL_CONFIG }} + DEPLOYCTL_CONFIG_DIGEST: ${{ vars.DEPLOYCTL_CONFIG_DIGEST }} + run: deployctl ci unpack --env "$DEPLOY_ENV" + + - name: Deploy + env: + DEPLOYCTL_SSH_STRICT: "1" # only the pinned host keys, never trust-on-first-use + DEPLOYCTL_CONFIG_STRICT: "1" # a value the servers run with changes only on purpose + DEPLOYCTL_NO_MINT: "1" # a secret missing from the bundle fails, never regenerates + run: | + if [[ "$DEPLOY_ALLOW_CONFIG_CHANGE" == "true" ]]; then + export DEPLOYCTL_ALLOW_SECRET_CHANGE=1 + fi + if [[ "$DEPLOY_ACTION" == rollback ]]; then + deployctl deploy rollback --env "$DEPLOY_ENV" --to "$DEPLOY_TAG" + else + deployctl deploy update --env "$DEPLOY_ENV" --tag "$DEPLOY_TAG" + fi + + - name: Summary + if: always() + env: + RESULT: ${{ job.status }} + run: | + { + echo "### ${DEPLOY_ACTION} → ${DEPLOY_ENV}: ${RESULT}" + echo "" + echo "| | |" + echo "|---|---|" + echo "| image tag | \`${DEPLOY_TAG}\` |" + echo "| started by | ${GITHUB_ACTOR} (${GITHUB_EVENT_NAME}) |" + if [[ "$RESULT" != success ]]; then + echo "" + echo "A failed release is reverted on every host it reached. To go back further:" + echo "\`deployctl ci deploy --env ${DEPLOY_ENV} --rollback\`" + fi + } >> "$GITHUB_STEP_SUMMARY" + + # GitHub already emails whoever triggered a failed run. For a team channel, + # add a SLACK_WEBHOOK_URL secret. + - name: Alert on failure + if: failure() + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + run: | + [[ -n "$SLACK_WEBHOOK_URL" ]] || exit 0 + run_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + text="Deploy failed: ${DEPLOY_ACTION} ${DEPLOY_TAG} → ${DEPLOY_ENV} (${GITHUB_REPOSITORY}) ${run_url}" + jq -n --arg text "$text" '{text: $text}' \ + | curl -fsS -X POST -H 'Content-Type: application/json' --data @- "$SLACK_WEBHOOK_URL" + + - name: Remove credentials + if: always() + run: rm -rf ~/.ssh/id_ed25519 deploy/config deploy/generated deployctl/config deployctl/generated diff --git a/src/deployctl/server/droplet-user-data.sh b/src/deployctl/templates/server/bootstrap.sh.j2 old mode 100755 new mode 100644 similarity index 87% rename from src/deployctl/server/droplet-user-data.sh rename to src/deployctl/templates/server/bootstrap.sh.j2 index 91a1fa2..f4f8247 --- a/src/deployctl/server/droplet-user-data.sh +++ b/src/deployctl/templates/server/bootstrap.sh.j2 @@ -1,20 +1,24 @@ #!/bin/bash ############################################################################## -# droplet-user-data.sh — prepare a fresh Ubuntu droplet for deployctl. +# Prepare a fresh Ubuntu server for deployctl — «PROJECT» / «ENV». +# Rendered by deployctl «VERSION»: deployctl server bootstrap-script --env «ENV» # -# Paste this into DigitalOcean's "Add Initialization scripts (user data)" box when -# creating the droplet, or pass it with doctl --user-data-file. cloud-init runs it -# once, as root, on first boot. It does Part B of the Tutorial (B1–B6) so the -# first thing you do by hand is the B7 check from your own machine: +# Run it once, as root, on each new server — either of: +# • DigitalOcean: paste it into "Add Initialization scripts (user data)" when +# creating the droplet (cloud-init runs it on first boot), or +# doctl compute droplet create … --user-data-file bootstrap.sh +# • any server: deployctl server bootstrap-script --env «ENV» | ssh root@ 'bash -s' # -# ssh deploy@ docker ps +# Then, from your own machine: +# +# ssh «SSH_USER»@ docker ps # # Unlike the rest of deployctl this runs ON the server, not on your machine. # # Progress: ssh root@ 'grep userdata /var/log/cloud-init-output.log' # Finished: /var/lib/deployctl-bootstrap.done exists # -# Safe to re-run by hand on an existing server (sudo bash droplet-user-data.sh): +# Safe to re-run on an existing server (sudo bash bootstrap.sh): # every step checks before it changes anything. ############################################################################## @@ -23,23 +27,22 @@ set -euo pipefail # --------------------------------------------------------------------------- # Settings. The first two must match config/.env: SSH_USER and REMOTE_DIR. # --------------------------------------------------------------------------- -DEPLOY_USER="deploy" -REMOTE_DIR="/opt/myapp" # /opt/; staging: /opt/-staging -SWAP_SIZE="2G" +DEPLOY_USER="«SSH_USER»" +REMOTE_DIR="«REMOTE_DIR»" +SWAP_SIZE="«SWAP_SIZE»" -# Public keys the deploy user accepts in addition to the droplet's own (the ones -# you picked in the DigitalOcean panel). Put the CI deploy key here — the public -# half of the DEPLOY_SSH_KEY secret — so deploy.yml can reach the new server. -# One key per line. +# Public keys the deploy user accepts besides root's own (the ones you picked when +# creating the server). The CI key does not belong here: `deployctl ci setup-key` +# installs it on every host, and rotates it, over ssh. One key per line. EXTRA_AUTHORIZED_KEYS="" # Key-only SSH: no passwords for anyone, root by key only. Root stays reachable # because the permission fixes in "When it breaks" need it. -HARDEN_SSH=true +HARDEN_SSH=«HARDEN_SSH» # A kernel upgrade in B1 only takes effect after a reboot. First boot is the one # moment nothing is running yet, so take it now rather than mid-deploy later. -REBOOT_IF_REQUIRED=true +REBOOT_IF_REQUIRED=«REBOOT» # --------------------------------------------------------------------------- @@ -174,7 +177,7 @@ sysctl -q -p /etc/sysctl.d/60-deployctl-swap.conf if ufw status 2>/dev/null | grep -q 'Status: active'; then log "B6: WARNING ufw is active — the Tutorial expects it inactive (use the cloud firewall)" else - log "B6: ufw inactive, as expected — attach a DigitalOcean cloud firewall (22, 80, 443)" + log "B6: ufw inactive, as expected — attach a cloud firewall: «FIREWALL»" fi # --------------------------------------------------------------------------- @@ -194,7 +197,7 @@ if [[ "$HARDEN_SSH" == "true" ]]; then # sshd keeps the FIRST value it reads, and the drop-ins are read in name # order — 10- wins over cloud-init's own 50-cloud-init.conf. cat > /etc/ssh/sshd_config.d/10-deployctl.conf <<'EOF' -# Written by deployctl/host/droplet-user-data.sh +# Written by deployctl server bootstrap-script PasswordAuthentication no KbdInteractiveAuthentication no PermitRootLogin prohibit-password @@ -218,9 +221,10 @@ touch /var/lib/deployctl-bootstrap.done public_ip="$(curl -fsS --max-time 5 http://169.254.169.254/metadata/v1/interfaces/public/0/ipv4/address 2>/dev/null || hostname -I | awk '{print $1}')" log "done — $(uname -m), $(free -h | awk '/^Mem:/{print $2}') RAM, $(free -h | awk '/^Swap:/{print $2}') swap" -log "next, from your own machine (Tutorial B7):" +log "next, from your own machine:" log " ssh $DEPLOY_USER@$public_ip docker ps" -log "known_hosts lines for this server — for DEPLOY_KNOWN_HOSTS in GitHub:" +log "this server's host keys — compare with the fingerprint your first ssh shows;" +log "then \`deployctl ci pin-hosts\` gives them to GitHub:" for pub in /etc/ssh/ssh_host_ed25519_key.pub /etc/ssh/ssh_host_ecdsa_key.pub /etc/ssh/ssh_host_rsa_key.pub; do if [[ -f "$pub" ]]; then log " $public_ip $(cut -d' ' -f1,2 "$pub")"; fi done diff --git a/src/deployctl/webui/fields/cluster.toml b/src/deployctl/webui/fields/cluster.toml index 1b5070e..00e4f15 100644 --- a/src/deployctl/webui/fields/cluster.toml +++ b/src/deployctl/webui/fields/cluster.toml @@ -14,6 +14,13 @@ target = "env" default = "deploy" help = "Unprivileged user in the docker group. Not root." +[[section.field]] +key = "SSH_JUMP_HOST" +label = "Jump host (bastion)" +target = "env" +placeholder = "deploy@bastion.example.com" +help = "Only when the hosts have no SSH reachable from you or from CI — deploys go through this host (ssh ProxyJump). Leave empty to connect directly." + [[section.field]] key = "REMOTE_DIR" label = "Remote directory" diff --git a/src/deployctl/webui/fields/single.toml b/src/deployctl/webui/fields/single.toml index cd4cad1..2f940e2 100644 --- a/src/deployctl/webui/fields/single.toml +++ b/src/deployctl/webui/fields/single.toml @@ -20,6 +20,13 @@ target = "env" default = "deploy" help = "Unprivileged user in the docker group. Not root." +[[section.field]] +key = "SSH_JUMP_HOST" +label = "Jump host (bastion)" +target = "env" +placeholder = "deploy@bastion.example.com" +help = "Only when the hosts have no SSH reachable from you or from CI — deploys go through this host (ssh ProxyJump). Leave empty to connect directly." + [[section.field]] key = "REMOTE_DIR" label = "Remote directory" diff --git a/src/deployctl/webui/panel/actions.py b/src/deployctl/webui/panel/actions.py index 0a3fa7d..0aa7169 100644 --- a/src/deployctl/webui/panel/actions.py +++ b/src/deployctl/webui/panel/actions.py @@ -97,7 +97,8 @@ def exclusive(self) -> bool: danger=True, touches_hosts=True, needs_migrate=True), Action("rollback", "Rollback", ("deploy", "rollback", "--env", "{env}"), "Re-deploy the previously recorded image tag, health-gated.", - effect="Moves the image back and pins that tag in config. Runs no migrations and reverts none.", + effect="Moves the image back to the release before the running one, from the history on the " + "primary. Runs no migrations and reverts none.", danger=True, touches_hosts=True), Action("restart", "Restart", ("deploy", "restart", "--env", "{env}"), "Restart services on every host, health-gated.", @@ -113,7 +114,7 @@ def exclusive(self) -> bool: danger=True, touches_hosts=True, tls_modes=("letsencrypt",)), Action("backup-run", "Backup now", ("backup", "run", "--env", "{env}"), "pg_dump on the primary, prune old dumps, fetch a local copy.", - effect="Writes a dump on the host and a copy in backups/ of the deploy directory.", + effect="Writes a dump on the host and a copy in ~/.deployctl/backups/ on this machine.", danger=True, touches_hosts=True), ) } diff --git a/src/deployctl/webui/panel/routes.py b/src/deployctl/webui/panel/routes.py index ffda2e1..e073d0b 100644 --- a/src/deployctl/webui/panel/routes.py +++ b/src/deployctl/webui/panel/routes.py @@ -237,6 +237,8 @@ def open_terminal_ssh(host: str, env: str): if host not in cfg.hosts: raise HTTPException(status_code=400, detail="unknown host") user = cfg.raw.get("SSH_USER") or "deploy" - command = f"ssh -o StrictHostKeyChecking=accept-new {shlex.quote(f'{user}@{host}')}" + jump = cfg.raw.get("SSH_JUMP_HOST", "") + via = f"-J {shlex.quote(jump)} " if jump else "" + command = f"ssh -o StrictHostKeyChecking=accept-new {via}{shlex.quote(f'{user}@{host}')}" ok, message = open_native_terminal(command, f"ssh {host}") return term_result(ok, message) diff --git a/tests/test_backup_restore.py b/tests/test_backup_restore.py index 8c0a22d..73562f6 100644 --- a/tests/test_backup_restore.py +++ b/tests/test_backup_restore.py @@ -105,3 +105,17 @@ def test_schedule_rejects_a_time_that_is_not_one(calls): result = CliRunner().invoke(app, ["backup", "schedule", "--env", "production", "--at", "25:00"]) assert result.exit_code == 2, result.output assert calls == [] + + +def test_local_copies_go_to_the_users_home_even_without_home_set(tmp_path): + """Never into a repository, and never into a directory literally named ~.""" + script = f''' +set -euo pipefail +source "{DEPLOYCTL}/scripts/common/common.sh" +eval "$(sed -n '/^_home=~/,/^readonly LOCAL_DIR=/p' "{DEPLOYCTL}/scripts/backup.sh")" +echo "$LOCAL_DIR" +''' + env = {"PATH": os.environ["PATH"], "COMPOSE_PROJECT": "demo", "DEPLOYCTL_ENV": "production"} + out = subprocess.run(["bash", "-c", script], env=env, capture_output=True, text=True, timeout=30).stdout.strip() + assert out.endswith("/.deployctl/backups/demo/production") + assert out.startswith("/") and "~" not in out diff --git a/tests/test_ci_commands.py b/tests/test_ci_commands.py new file mode 100644 index 0000000..a46018f --- /dev/null +++ b/tests/test_ci_commands.py @@ -0,0 +1,399 @@ +"""The `ci` commands: every manual step of setting up continuous deployment, as a command. + +GitHub is replaced by stand-ins for the functions in cli/github.py; the ssh side +of `setup-key` runs the real engine against directories standing in for hosts. +""" + +from __future__ import annotations + +import os +import pathlib +import subprocess + +import pytest +import yaml +from typer.testing import CliRunner + +from deployctl import __version__ +from deployctl.cli import checks, config, github, paths +from deployctl.cli.commands import ci +from deployctl.cli.main import app + +CONFIG = """\ +MODE=single +PROJECT_NAME=demo +BASE_DOMAIN=demo.test +IMAGE_REPO=ghcr.io/acme/demo +HOSTS=203.0.113.10 +ACME_EMAIL=ops@demo.test +POSTGRES_DB=demo +POSTGRES_USER=demo +""" +SECRETS = "SECRET_KEY=a\nJWT_SECRET_KEY=b\nREDIS_PASSWORD=c\nPOSTGRES_PASSWORD=d\n" + + +@pytest.fixture +def repo(project, write_config, monkeypatch): + """A project inside a git repository, with GitHub stood in for.""" + write_config("production", CONFIG) + paths.COMMON_CONFIG.write_text("REGISTRY_USER=acme\n") + paths.secrets_file("production").write_text(SECRETS) + (project / ".git").mkdir() + monkeypatch.setattr(paths, "REPO_ROOT", project) + monkeypatch.setattr(github, "require", lambda: None) + monkeypatch.setattr(github, "repo", lambda: {"name": "acme/demo", "owner": "acme", "private": True, + "default_branch": "main"}) + return project + + +def _cli(*args): + return CliRunner().invoke(app, ["ci", *args]) + + +# ---- ci connect ------------------------------------------------------------------------ + + +@pytest.mark.parametrize("private, plan, scope", [ + (True, "free", "repository"), + (True, "team", "environment"), + (False, "free", "environment"), + (True, None, "repository"), +]) +def test_connect_records_where_secrets_can_live(repo, monkeypatch, private, plan, scope): + monkeypatch.setattr(github, "repo", lambda: {"name": "acme/demo", "owner": "acme", "private": private, + "default_branch": "main"}) + monkeypatch.setattr(github, "plan", lambda owner: plan) + result = _cli("connect", "--env", "production", "--branch", "prod") + assert result.exit_code == 0, result.output + loaded = config.load("production") + assert loaded.raw["CI_SCOPE"] == scope + assert loaded.raw["DEPLOY_BRANCH"] == "prod" + if scope == "repository" and private: + assert "readable from ANY branch" in result.output + + +# ---- ci init --------------------------------------------------------------------------- + + +def _workflow(repo, name) -> str: + return (repo / ".github" / "workflows" / name).read_text() + + +def test_init_writes_valid_workflows_pinned_to_this_version(repo): + result = _cli("init", "--env", "production", "--branch", "prod") + assert result.exit_code == 0, result.output + for name in ("build-image.yml", "deploy.yml", "ci.yml"): + yaml.safe_load(_workflow(repo, name)) + deploy = _workflow(repo, "deploy.yml") + assert deploy.startswith(f"# Managed by deployctl {__version__}") + assert f"github.com/{ci.TOOL_REPO}@v{__version__}" in deploy + assert 'refs/heads/prod"' in deploy + assert "«" not in deploy and "{%" not in deploy + + +def test_the_build_workflow_hands_its_tag_to_the_deploy(repo): + _cli("init", "--env", "production") + build = yaml.safe_load(_workflow(repo, "build-image.yml")) + assert build[True]["workflow_call"]["outputs"]["tag"]["value"] == "${{ jobs.build.outputs.tag }}" + pipeline = yaml.safe_load(_workflow(repo, "ci.yml")) + assert pipeline["jobs"]["deploy"]["with"]["tag"] == "${{ needs.publish.outputs.tag }}" + assert "vars.AUTO_DEPLOY == 'true'" in pipeline["jobs"]["deploy"]["if"] + + +def test_validate_accepts_the_generated_build_workflow(repo, monkeypatch): + """validate checks the workflow publishes IMAGE_REPO; the template must satisfy it.""" + _cli("init", "--env", "production") + monkeypatch.setattr(paths, "CI_WORKFLOW", repo / ".github" / "workflows" / "build-image.yml") + problems = checks.ci_workflow_checks(config.load("production")) + assert [p.message for p in problems if p.level == "error"] == [] + + +def test_a_worker_image_is_built_in_the_same_run(repo): + (repo / "project" / "project.env").write_text((repo / "project" / "project.env").read_text() + + "WORKER_BUILD_TARGET=worker\n") + _cli("init", "--env", "production") + assert "Build & push — worker" in _workflow(repo, "build-image.yml") + + +@pytest.mark.parametrize("hosts, minutes", [("203.0.113.10", 20), (" ".join(f"10.0.0.{n}" for n in range(1, 11)), 50)]) +def test_the_deploy_timeout_grows_with_the_fleet(repo, write_config, hosts, minutes): + write_config("production", CONFIG.replace("HOSTS=203.0.113.10", f'HOSTS="{hosts}"')) + _cli("init", "--env", "production") + job = yaml.safe_load(_workflow(repo, "deploy.yml"))["jobs"]["deploy"] + assert job["timeout-minutes"] == minutes + + +def test_a_changed_managed_workflow_needs_force_and_the_diff_is_shown(repo): + _cli("init", "--env", "production") + path = repo / ".github" / "workflows" / "deploy.yml" + path.write_text(path.read_text().replace("timeout-minutes: 20", "timeout-minutes: 99")) + result = _cli("init", "--env", "production") + assert result.exit_code == 1 + assert "timeout-minutes: 99" in result.output and "--force" in result.output + assert "timeout-minutes: 99" in path.read_text() + assert _cli("init", "--env", "production", "--force").exit_code == 0 + assert "timeout-minutes: 20" in path.read_text() + + +def test_the_projects_own_ci_yml_is_never_overwritten(repo): + wf = repo / ".github" / "workflows" + wf.mkdir(parents=True) + (wf / "ci.yml").write_text("name: mine\njobs: {}\n") + result = _cli("init", "--env", "production") + assert result.exit_code == 0, result.output + assert (wf / "ci.yml").read_text() == "name: mine\njobs: {}\n" + assert "does not call" in result.output and "uses: ./.github/workflows/deploy.yml" in result.output + + +# ---- ci pin-hosts ---------------------------------------------------------------------- + + +def test_pin_hosts_uploads_what_this_machine_trusts(repo, monkeypatch): + uploaded = {} + monkeypatch.setattr(ci, "known_hosts_lines", lambda host, port="": [f"{host} ssh-ed25519 AAAAkey"]) + monkeypatch.setattr(github, "set_variable", lambda name, value, scope: uploaded.update({name: (value, scope)})) + result = _cli("pin-hosts", "--env", "production") + assert result.exit_code == 0, result.output + assert uploaded["DEPLOY_KNOWN_HOSTS"] == ("203.0.113.10 ssh-ed25519 AAAAkey", "production") + + +def test_the_jump_host_is_pinned_too(repo, write_config, monkeypatch): + write_config("production", CONFIG + "SSH_JUMP_HOST=ops@bastion.demo.test:2222\n") + asked = [] + monkeypatch.setattr(ci, "known_hosts_lines", lambda host, port="": asked.append((host, port)) or ["x"]) + monkeypatch.setattr(github, "set_variable", lambda *a: None) + assert _cli("pin-hosts", "--env", "production").exit_code == 0 + assert ("bastion.demo.test", "2222") in asked + + +def test_a_host_never_connected_to_is_refused_not_looked_up(repo, monkeypatch): + monkeypatch.setattr(ci, "known_hosts_lines", lambda host, port="": []) + monkeypatch.setattr(github, "set_variable", lambda *a: pytest.fail("nothing may be uploaded")) + result = _cli("pin-hosts", "--env", "production") + assert result.exit_code == 1 + assert "never connected to 203.0.113.10" in result.output + + +# ---- ci setup-key ---------------------------------------------------------------------- + + +@pytest.fixture +def key_steps(repo, monkeypatch): + """What setup-key asks the engine, and what it uploads.""" + seen = {"modes": [], "uploaded": None, "verify": "OK", "present": False} + + def fake_ci_key(cfg, mode, **extra): + seen["modes"].append(mode) + where = "deploy@203.0.113.10" + if mode == "check": + return [("PRESENT" if seen["present"] else "ABSENT", where)] + if mode == "verify": + seen["key_file"] = extra["CI_KEY_FILE"] + return [(seen["verify"], where)] + seen["pubkey"] = extra["CI_PUBKEY"] + return [("INSTALLED", where)] + + monkeypatch.setattr(ci, "_ci_key", fake_ci_key) + monkeypatch.setattr(github, "set_secret", lambda name, value, scope: seen.update(uploaded=(name, value, scope))) + return seen + + +def test_setup_key_installs_proves_then_uploads_and_keeps_no_copy(key_steps): + result = _cli("setup-key", "--env", "production") + assert result.exit_code == 0, result.output + assert key_steps["modes"] == ["check", "install", "verify"] + name, private_key, scope = key_steps["uploaded"] + assert name == "DEPLOY_SSH_KEY" and "OPENSSH PRIVATE KEY" in private_key and scope == "production" + assert "deployctl-ci@acme/demo/production" in key_steps["pubkey"] + assert not os.path.exists(key_steps["key_file"]), "the private key must not outlive the command" + assert "PRIVATE KEY" not in result.output + + +def test_a_key_that_cannot_log_in_is_never_uploaded(key_steps): + key_steps["verify"] = "FAILED" + result = _cli("setup-key", "--env", "production") + assert result.exit_code == 1 + assert key_steps["uploaded"] is None + + +def test_an_installed_key_is_only_replaced_on_purpose(key_steps): + key_steps["present"] = True + assert _cli("setup-key", "--env", "production").exit_code == 1 + assert key_steps["modes"] == ["check"] + assert _cli("setup-key", "--env", "production", "--rotate").exit_code == 0 + assert "rotate" in key_steps["modes"] + + +# The engine half: the real `deploy.sh ci-key` against a directory standing in for +# each host's home. +_FAKE_SSH = """#!/usr/bin/env bash +while [[ $# -gt 0 ]]; do case "$1" in -o|-i|-p) shift 2 ;; *@*) host="${1#*@}"; shift; break ;; *) shift ;; esac; done +mkdir -p "$FAKE_HOMES/$host" +HOME="$FAKE_HOMES/$host" exec bash -c "$*" +""" + + +def _engine_key(tmp_path, mode, **extra): + bin_dir = tmp_path / "bin" + bin_dir.mkdir(exist_ok=True) + (bin_dir / "ssh").write_text(_FAKE_SSH) + (bin_dir / "ssh").chmod(0o755) + env = {"PATH": f"{bin_dir}:{os.environ['PATH']}", "NO_COLOR": "1", "DEPLOYCTL_PROJECT": str(tmp_path), + "DEPLOYCTL_ENV": "production", "HOSTS": "10.0.0.1 10.0.0.2", "PRIMARY_HOST": "10.0.0.1", + "SSH_USER": "deploy", "REMOTE_DIR": "/opt/demo", "COMPOSE_PROJECT": "demo", + "FAKE_HOMES": str(tmp_path / "homes"), "CI_KEY_MODE": mode, + "CI_KEY_COMMENT": "deployctl-ci@acme/demo/production", **extra} + tool = pathlib.Path(__file__).resolve().parent.parent / "src" / "deployctl" + proc = subprocess.run(["bash", str(tool / "scripts" / "deploy.sh"), "ci-key"], env=env, + capture_output=True, text=True, timeout=30) + return proc.returncode, proc.stdout + + +def _keys(tmp_path, host): + return (tmp_path / "homes" / host / ".ssh" / "authorized_keys").read_text() + + +def test_the_engine_installs_on_every_host_restricted(tmp_path): + rc, out = _engine_key(tmp_path, "install", CI_PUBKEY="ssh-ed25519 AAAAnew deployctl-ci@acme/demo/production") + assert rc == 0, out + assert out.split() == ["INSTALLED", "deploy@10.0.0.1", "INSTALLED", "deploy@10.0.0.2"] + for host in ("10.0.0.1", "10.0.0.2"): + assert _keys(tmp_path, host) == "restrict ssh-ed25519 AAAAnew deployctl-ci@acme/demo/production\n" + + +def test_rotation_replaces_only_the_ci_key(tmp_path): + home = tmp_path / "homes" / "10.0.0.1" / ".ssh" + home.mkdir(parents=True) + (home / "authorized_keys").write_text("ssh-ed25519 AAAAmine amal@laptop\n" + "restrict ssh-ed25519 AAAAold deployctl-ci@acme/demo/production\n") + _engine_key(tmp_path, "rotate", CI_PUBKEY="ssh-ed25519 AAAAnew deployctl-ci@acme/demo/production") + assert _keys(tmp_path, "10.0.0.1") == ("ssh-ed25519 AAAAmine amal@laptop\n" + "restrict ssh-ed25519 AAAAnew deployctl-ci@acme/demo/production\n") + + +def test_check_reports_each_host(tmp_path): + _engine_key(tmp_path, "install", CI_PUBKEY="ssh-ed25519 AAAAk deployctl-ci@acme/demo/production") + (tmp_path / "homes" / "10.0.0.2" / ".ssh" / "authorized_keys").write_text("") + rc, out = _engine_key(tmp_path, "check") + assert out.split() == ["PRESENT", "deploy@10.0.0.1", "ABSENT", "deploy@10.0.0.2"] + + +# ---- ci doctor ------------------------------------------------------------------------- + + +def test_doctor_lists_every_gap_with_its_command(repo, monkeypatch): + monkeypatch.setattr(github, "secret_names", lambda scope: set()) + monkeypatch.setattr(github, "variables", lambda scope: {}) + monkeypatch.setattr(github, "gh", lambda args, stdin=None: subprocess.CompletedProcess(args, 0, "PUBLIC\n", "")) + result = _cli("doctor", "--env", "production") + assert result.exit_code == 1 + for fix in ("ci connect", "ci init", "ci setup-key", "ci pin-hosts", "ci sync-config"): + assert fix in result.output + + +def test_doctor_is_green_when_everything_is_in_place(repo, monkeypatch): + from deployctl.cli import bundle + + _cli("init", "--env", "production") + paths.COMMON_CONFIG.write_text("REGISTRY_USER=acme\nCI_SCOPE=repository\n") + digest = bundle.digest(bundle.collect("production")) + monkeypatch.setattr(github, "secret_names", lambda scope: {"DEPLOY_SSH_KEY", "DEPLOYCTL_CONFIG"}) + monkeypatch.setattr(github, "variables", lambda scope: { + "DEPLOY_KNOWN_HOSTS": "203.0.113.10 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl", + "DEPLOYCTL_CONFIG_DIGEST": digest, "AUTO_DEPLOY": "true"}) + monkeypatch.setattr(github, "gh", lambda args, stdin=None: subprocess.CompletedProcess(args, 0, "PUBLIC\n", "")) + result = _cli("doctor", "--env", "production") + assert result.exit_code == 0, result.output + assert "every push to main deploys" in result.output + + +# ---- ci deploy / runs / auto-deploy ---------------------------------------------------- + + +def test_deploy_without_a_tag_redeploys_what_is_running(repo, monkeypatch): + from deployctl.cli import tags + + sent = {} + monkeypatch.setattr(tags, "running", lambda cfg: "run1234") + monkeypatch.setattr(github, "dispatch", lambda wf, ref, fields: sent.update(wf=wf, ref=ref, **fields)) + result = _cli("deploy", "--env", "production", "--allow-config-change", "--no-watch") + assert result.exit_code == 0, result.output + assert sent == {"wf": "deploy.yml", "ref": "main", "environment": "production", "tag": "run1234", + "action": "update", "allow_config_change": "true"} + + +def test_deploy_rollback_picks_the_release_before_the_running_one(repo, monkeypatch): + from deployctl.cli import tags + + sent = {} + history = ("2026-09-01T00:00:00Z\tproduction\taaa1111\tdeploy\tx\n" + "2026-09-02T00:00:00Z\tproduction\tbbb2222\tdeploy\tx\n") + monkeypatch.setattr(tags, "running", lambda cfg: "bbb2222") + monkeypatch.setattr(ci, "run", lambda *a, **k: subprocess.CompletedProcess(a, 0, history, "")) + monkeypatch.setattr(github, "dispatch", lambda wf, ref, fields: sent.update(fields)) + assert _cli("deploy", "--env", "production", "--rollback", "--no-watch").exit_code == 0 + assert (sent["action"], sent["tag"]) == ("rollback", "aaa1111") + + +def test_deploy_follows_the_run_it_started(repo, monkeypatch): + monkeypatch.setattr(github, "dispatch", lambda *a: None) + monkeypatch.setattr(github, "runs", lambda wf, **k: [ + {"databaseId": 7, "createdAt": "2999-01-01T00:00:00Z", "url": "https://github.com/acme/demo/actions/runs/7"}]) + watched = [] + monkeypatch.setattr(github, "watch", lambda run_id: watched.append(run_id) or 0) + result = _cli("deploy", "--env", "production", "--tag", "abc1234") + assert result.exit_code == 0, result.output + assert watched == [7] + + +def test_runs_merges_manual_deploys_and_deploys_on_merge(repo, monkeypatch): + def fake_runs(workflow, **kwargs): + if workflow == "deploy.yml": + return [{"databaseId": 1, "displayTitle": "rollback aaa1111 → production", "status": "completed", + "conclusion": "success", "createdAt": "2026-09-25T10:00:00Z", "url": "u1"}] + return [{"databaseId": 2, "headSha": "bbb2222ffff", "createdAt": "2026-09-25T11:00:00Z", "url": "u2"}, + {"databaseId": 3, "headSha": "ccc3333ffff", "createdAt": "2026-09-25T12:00:00Z", "url": "u3"}] + + jobs = {2: [{"name": "Deploy to production / update bbb2222 → production", "conclusion": "failure"}], + 3: [{"name": "Deploy to production", "conclusion": "skipped"}]} # AUTO_DEPLOY was off + monkeypatch.setattr(github, "runs", fake_runs) + monkeypatch.setattr(github, "run_jobs", lambda run_id: jobs[run_id]) + found = ci.deploy_runs(config.load("production")) + assert [(r["trigger"], r["action"], r["tag"], r["status"]) for r in found] == [ + ("merge", "update", "bbb2222", "failure"), + ("manual", "rollback", "aaa1111", "success"), + ] + + +def test_auto_deploy_is_a_repository_variable(repo, monkeypatch): + """ci.yml reads it in a job's `if`, before any environment is entered.""" + set_ = [] + monkeypatch.setattr(github, "set_variable", lambda name, value, scope: set_.append((name, value, scope))) + assert _cli("auto-deploy", "on", "--env", "production").exit_code == 0 + assert set_ == [("AUTO_DEPLOY", "true", None)] + + +# ---- server bootstrap-script ----------------------------------------------------------- + + +def test_the_bootstrap_script_is_this_environments_and_valid_bash(repo, tmp_path): + result = CliRunner().invoke(app, ["server", "bootstrap-script", "--env", "production", "--swap", "4G"]) + assert result.exit_code == 0, result.output + script = result.stdout + assert script.startswith("#!/bin/bash") + assert 'DEPLOY_USER="deploy"' in script and 'REMOTE_DIR="/opt/demo"' in script and 'SWAP_SIZE="4G"' in script + assert "«" not in script + path = tmp_path / "bootstrap.sh" + path.write_text(script) + assert subprocess.run(["bash", "-n", str(path)], capture_output=True).returncode == 0 + + +def test_a_fleet_behind_a_load_balancer_gets_its_own_firewall_advice(repo, write_config): + write_config("production", CONFIG.replace("MODE=single", "MODE=cluster") + + "TLS_MODE=loadbalancer\nPOSTGRES_MODE=external\nREDIS_MODE=external\n") + result = CliRunner().invoke(app, ["server", "bootstrap-script", "--env", "production"]) + assert "80 from the load balancer only" in result.stdout + + +def test_a_nonsense_swap_size_is_refused(repo): + assert CliRunner().invoke(app, ["server", "bootstrap-script", "--env", "production", "--swap", "lots"]).exit_code == 2 diff --git a/tests/test_ci_deploy.py b/tests/test_ci_deploy.py index 2a79571..55e25a7 100644 --- a/tests/test_ci_deploy.py +++ b/tests/test_ci_deploy.py @@ -9,6 +9,7 @@ from __future__ import annotations +import json import os import pathlib import stat @@ -17,8 +18,7 @@ import pytest from typer.testing import CliRunner -from deployctl.cli import bundle, paths, secrets -from deployctl.cli.commands import ci +from deployctl.cli import bundle, github, paths, secrets from deployctl.cli.config import load from deployctl.cli.main import app @@ -201,14 +201,13 @@ def gh(monkeypatch): def fake(args, *, stdin=None): calls.append((args, stdin)) - if args[0] == "api": - if remote["digest"] is None: - return subprocess.CompletedProcess(args, 1, "", "gh: Not Found (HTTP 404)") - return subprocess.CompletedProcess(args, 0, remote["digest"] + "\n", "") + if args[:2] == ["variable", "list"]: + listed = [{"name": "DEPLOYCTL_CONFIG_DIGEST", "value": remote["digest"]}] if remote["digest"] else [] + return subprocess.CompletedProcess(args, 0, json.dumps(listed), "") return subprocess.CompletedProcess(args, 0, "", "") - monkeypatch.setattr(ci, "_gh", fake) - monkeypatch.setattr(ci, "_require_gh", lambda: None) + monkeypatch.setattr(github, "gh", fake) + monkeypatch.setattr(github, "require", lambda: None) return calls, remote @@ -216,24 +215,34 @@ def _sync(*args: str): return CliRunner().invoke(app, ["ci", "sync-config", "--env", "production", *args]) +def _writes(calls): + return [(args, stdin) for args, stdin in calls if args[1] == "set"] + + class TestSyncConfig: def test_uploads_the_bundle_then_its_digest_to_the_environment(self, config, gh): calls, _ = gh files = bundle.collect("production") result = _sync() assert result.exit_code == 0, result.output - sets = [(args, stdin) for args, stdin in calls if args[0] != "api"] - assert sets == [ + assert _writes(calls) == [ (["secret", "set", "DEPLOYCTL_CONFIG", "--env", "production"], bundle.encode(files)), (["variable", "set", "DEPLOYCTL_CONFIG_DIGEST", "--env", "production", "--body", bundle.digest(files)], None), ], "the digest goes second: a failure between the two must leave CI refusing, not deploying" - def test_repo_level_for_github_free(self, config, gh): + def test_repo_level_once_with_the_flag(self, config, gh): calls, _ = gh result = _sync("--repo-level") assert result.exit_code == 0, result.output - assert "actions/variables/DEPLOYCTL_CONFIG_DIGEST" in calls[0][0][1] + assert all("--env" not in args for args, _ in calls) + + def test_repo_level_every_time_on_github_free(self, config, gh): + """What `ci connect` records for a private repository on GitHub Free.""" + calls, _ = gh + paths.COMMON_CONFIG.write_text(paths.COMMON_CONFIG.read_text() + "CI_SCOPE=repository\n") + result = _sync() + assert result.exit_code == 0, result.output assert all("--env" not in args for args, _ in calls) def test_nothing_is_uploaded_when_github_is_current(self, config, gh): @@ -242,7 +251,7 @@ def test_nothing_is_uploaded_when_github_is_current(self, config, gh): result = _sync() assert result.exit_code == 0, result.output assert "already current" in result.output - assert [args for args, _ in calls if args[0] != "api"] == [] + assert _writes(calls) == [] def test_a_config_missing_a_generated_secret_is_not_uploaded(self, config, gh): """CI never mints: uploading this would only move the failure to the deploy.""" @@ -304,6 +313,16 @@ def test_ci_trusts_only_the_pinned_keys(self): assert rc == 0, out assert "StrictHostKeyChecking=yes" in out + def test_a_jump_host_is_used_for_every_connection(self): + """ssh, rsync (-e "ssh $SSH_OPTS"), the lock: everything goes through SSH_OPTS.""" + rc, out = _bash('echo "$SSH_OPTS"', SSH_JUMP_HOST="deploy@bastion.example.com") + assert rc == 0, out + assert "-o ProxyJump=deploy@bastion.example.com" in out + + def test_no_jump_host_means_a_direct_connection(self): + rc, out = _bash('echo "$SSH_OPTS"') + assert "ProxyJump" not in out + @pytest.fixture def history(tmp_path): diff --git a/tests/test_config.py b/tests/test_config.py index b11ece7..75b2aef 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -317,3 +317,28 @@ def test_memory_limits_are_read_the_way_docker_writes_them(limit, mib): def test_an_unreadable_limit_is_not_guessed(): assert config.memory_mib("lots") is None + + +# ---- how deploys reach and treat the hosts --------------------------------------- + + +def test_the_fleet_is_reverted_as_a_whole_by_default(write_config): + write_config("production", CLUSTER) + assert config.load("production").raw["REVERT_SCOPE"] == "fleet" + + +def test_an_unknown_revert_scope_is_an_error(write_config): + write_config("production", CLUSTER + "REVERT_SCOPE=some\n") + assert any("REVERT_SCOPE must be" in e for e in errors(config.load("production"))) + + +@pytest.mark.parametrize("jump", ["bastion", "deploy@bastion.example.com", "deploy@10.0.0.5:2222", "a@one,b@two"]) +def test_a_jump_host_in_ssh_form_is_accepted(write_config, jump): + write_config("production", CLUSTER + f"SSH_JUMP_HOST={jump}\n") + assert not any("SSH_JUMP_HOST" in e for e in errors(config.load("production"))) + + +@pytest.mark.parametrize("jump", ["bastion -o ProxyCommand=evil", "a;b", "a'b"]) +def test_a_jump_host_that_would_inject_ssh_options_is_refused(write_config, jump): + write_config("production", CLUSTER + f'SSH_JUMP_HOST="{jump}"\n') + assert any("SSH_JUMP_HOST" in e for e in errors(config.load("production"))) diff --git a/tests/test_deploy_lock.py b/tests/test_deploy_lock.py index e8f03ab..f4f92fa 100644 --- a/tests/test_deploy_lock.py +++ b/tests/test_deploy_lock.py @@ -72,8 +72,12 @@ def test_released_when_the_run_ends_even_on_failure(self, tmp_path): assert not (tmp_path / ".deployctl.lock").exists(), out def test_released_when_the_run_is_cancelled(self, tmp_path): - # Signalled the way the panel's Cancel does it: the whole process group. - proc = _bash('acquire_deploy_lock "deploy update"; echo HELD; sleep 30 & wait', tmp_path, + # Signalled the way the panel's Cancel does it: the whole process group, + # mid-step. HELD comes from the step itself, after its exec: until then a + # child forked by bash still runs bash's TERM trap handler, which swallows a + # TERM landing in that gap, and the step would then run its full 30s. (And + # not `sleep & wait`: bash 3.2 can crash on a TERM arriving as wait starts.) + proc = _bash('acquire_deploy_lock "deploy update"; sh -c "echo HELD; exec sleep 30"', tmp_path, start_new_session=True) assert proc.stdout.readline().strip() == "HELD" assert (tmp_path / ".deployctl.lock").is_dir() diff --git a/tests/test_fleet_revert.py b/tests/test_fleet_revert.py new file mode 100644 index 0000000..9b018d0 --- /dev/null +++ b/tests/test_fleet_revert.py @@ -0,0 +1,220 @@ +"""A release across a fleet is all or nothing. + +When the third host of three fails, reverting only that host leaves the first +two on the new release and the third on the old one — two releases behind one +load balancer, with nobody watching when the deploy ran from CI. So by default +every host the run already moved goes back as well (REVERT_SCOPE=fleet). + +These run the real deploy.sh `update` — doctor, lock, snapshot, roll, revert — +against three directories standing in for three hosts: the stand-in ssh and rsync +send each host's commands to its own directory. +""" + +from __future__ import annotations + +import os +import pathlib +import subprocess + +import pytest + +TOOL = pathlib.Path(__file__).resolve().parent.parent / "src" / "deployctl" +PRIMARY, SECOND, THIRD = "203.0.113.10", "203.0.113.11", "203.0.113.12" + +# ssh / rsync: every path under FAKE_BASE becomes FAKE_BASE.; FAKE_HOST tells +# the stand-ins below which host they are running on. +_FAKE_SSH = """#!/usr/bin/env bash +while [[ $# -gt 0 ]]; do case "$1" in -o) shift 2 ;; -t) shift ;; *@*) host="${1#*@}"; shift; break ;; *) shift ;; esac; done +cmd="$*" +FAKE_HOST="$host" exec bash -c "${cmd//$FAKE_BASE/$FAKE_BASE.$host}" +""" +_FAKE_RSYNC = """#!/usr/bin/env bash +args=("$@"); n=${#args[@]} +src="${args[n-2]}"; target="${args[n-1]}"; host="${target#*@}"; host="${host%%:*}" +dest="${target#*:}"; dest="${dest//$FAKE_BASE/$FAKE_BASE.$host}" +if [[ -d "$src" ]]; then mkdir -p "$dest" && cp -R "$src"/. "$dest"/ +else mkdir -p "$(dirname "$dest")" && cp "$src" "$dest"; fi +""" +# docker: `up` logs which release it started where; `inspect` reports the worker +# restarting on a FAKE_CRASH_HOSTS host once the NEW release is running there. +_FAKE_DOCKER = """#!/usr/bin/env bash +here="$FAKE_BASE.$FAKE_HOST" +case "$*" in + *" up -d"*) + echo "up $FAKE_HOST: $(cat "$here"/*/docker-compose.production.yml)" >> "$FAKE_LOG" + echo 0 > "$FAKE_DIR/n.$FAKE_HOST" ;; + *"config --services"*) printf 'api\\ncelery_worker\\n' ;; + "ps -aq"*) echo c1 ;; + inspect*) + n=$(cat "$FAKE_DIR/n.$FAKE_HOST" 2>/dev/null || echo 0); echo $((n + 1)) > "$FAKE_DIR/n.$FAKE_HOST" + restarts=0 + if [[ " $FAKE_CRASH_HOSTS " == *" $FAKE_HOST "* ]] && grep -q NEW "$here"/*/docker-compose.production.yml && (( n > 0 )); then restarts=1; fi + echo "api 0 running healthy 0 unless-stopped" + echo "celery_worker $restarts running starting 0 unless-stopped" ;; +esac +exit 0 +""" +_FAKE_CURL = """#!/usr/bin/env bash +if [[ " $FAKE_UNHEALTHY_HOSTS " == *" $FAKE_HOST "* ]] && grep -q NEW "$FAKE_BASE.$FAKE_HOST"/*/docker-compose.production.yml; then exit 7; fi +exit 0 +""" + + +@pytest.fixture +def fleet(tmp_path): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + for name, body in (("ssh", _FAKE_SSH), ("rsync", _FAKE_RSYNC), ("docker", _FAKE_DOCKER), + ("curl", _FAKE_CURL), ("sleep", "#!/usr/bin/env bash\nexit 0\n")): + (bin_dir / name).write_text(body) + (bin_dir / name).chmod(0o755) + + # The control machine: the NEW release, rendered. + rendered = tmp_path / "project" / "generated" / "production" + for role in ("primary", "secondary"): + (rendered / role).mkdir(parents=True) + (rendered / role / "docker-compose.production.yml").write_text(f"image: demo:NEW ({role})") + (rendered / "nginx").mkdir() + (rendered / "nginx" / "site.conf").write_text("new nginx") + (rendered / ".env.production").write_text("SETTING=new\n") + (rendered / ".env.production").chmod(0o600) + + # Three hosts running the OLD release. + base = tmp_path / "srv" + for host in (PRIMARY, SECOND, THIRD): + role = "primary" if host == PRIMARY else "secondary" + home = pathlib.Path(f"{base}.{host}") + (home / role).mkdir(parents=True) + (home / role / "docker-compose.production.yml").write_text(f"image: demo:OLD ({role})") + (home / "nginx" / "auth").mkdir(parents=True) + (home / "nginx" / "site.conf").write_text("old nginx") + (home / ".env.production").write_text("SETTING=old\n") + (home / ".deployctl-state").write_text(f"IMAGE_TAG=old-{host[-2:]}\nENV=production\n") + + fakes = tmp_path / "fakes" + fakes.mkdir() + (fakes / "log").write_text("") + return {"tmp": tmp_path, "bin": bin_dir, "project": tmp_path / "project", "base": base, "fakes": fakes} + + +def _update(fleet, **extra: str) -> tuple[int, str]: + env = { + "PATH": f"{fleet['bin']}:{os.environ['PATH']}", + "NO_COLOR": "1", + "DEPLOYCTL_PROJECT": str(fleet["project"]), + "DEPLOYCTL_ENV": "production", + "HOSTS": f"{PRIMARY} {SECOND} {THIRD}", + "PRIMARY_HOST": PRIMARY, + "SSH_USER": "deploy", + "REMOTE_DIR": str(fleet["base"]), + "COMPOSE_PROJECT": "demo", + "CONTAINER_PREFIX": "demo", + "IMAGE_TAG": "abc1234", + "IMAGE_REF": "ghcr.io/acme/demo:abc1234", + "IMAGE_PLATFORM": "", + "API_DOMAIN": "api.example.com", + "HEALTH_PATH": "/health", + "MIGRATE_CMD": "", + "TLS_LE": "false", + "DEPLOY_SETTLE_SECONDS": "10", + "FAKE_BASE": str(fleet["base"]), + "FAKE_LOG": str(fleet["fakes"] / "log"), + "FAKE_DIR": str(fleet["fakes"]), + **extra, + } + proc = subprocess.run(["bash", str(TOOL / "scripts" / "deploy.sh"), "update"], + env=env, capture_output=True, text=True, timeout=120, cwd=fleet["tmp"]) + return proc.returncode, proc.stdout + proc.stderr + + +def _running(fleet, host: str) -> str: + home = pathlib.Path(f"{fleet['base']}.{host}") + role = "primary" if host == PRIMARY else "secondary" + return (home / role / "docker-compose.production.yml").read_text() + + +def _recorded(fleet, host: str) -> str: + return pathlib.Path(f"{fleet['base']}.{host}", ".deployctl-state").read_text().splitlines()[0] + + +def _ups(fleet) -> list[str]: + return (fleet["fakes"] / "log").read_text().splitlines() + + +class TestAHealthyFleet: + def test_every_host_moves_and_records_the_tag(self, fleet): + rc, out = _update(fleet) + assert rc == 0, out + assert "every host is on abc1234" in out + for host in (PRIMARY, SECOND, THIRD): + assert "NEW" in _running(fleet, host) + assert _recorded(fleet, host) == "IMAGE_TAG=abc1234" + + def test_the_release_is_recorded_once_on_the_primary_with_who_deployed_it(self, fleet): + rc, out = _update(fleet) + assert rc == 0, out + history = pathlib.Path(f"{fleet['base']}.{PRIMARY}", ".deployctl-history").read_text().splitlines() + assert len(history) == 1 + when, env, tag, kind, by = history[0].split("\t") + assert (env, tag, kind) == ("production", "abc1234", "deploy") + assert "@" in by, "user@machine off CI" + state = pathlib.Path(f"{fleet['base']}.{PRIMARY}", ".deployctl-state").read_text() + assert f"DEPLOYED_BY={by}" in state + + def test_under_ci_the_run_is_who(self, fleet): + rc, out = _update(fleet, GITHUB_ACTIONS="true", GITHUB_SERVER_URL="https://github.com", + GITHUB_REPOSITORY="acme/app", GITHUB_RUN_ID="42") + assert rc == 0, out + history = pathlib.Path(f"{fleet['base']}.{PRIMARY}", ".deployctl-history").read_text() + assert history.rstrip().endswith("\thttps://github.com/acme/app/actions/runs/42") + + +class TestTheLastHostFailing: + def test_puts_the_whole_fleet_back(self, fleet): + rc, out = _update(fleet, FAKE_CRASH_HOSTS=THIRD) + assert rc == 1, out + assert f"[{THIRD}] abc1234 failed on this host" in out + assert "every host is back on the release it ran before" in out + for host in (PRIMARY, SECOND, THIRD): + assert "OLD" in _running(fleet, host), f"{host} was left on the new release" + assert not pathlib.Path(f"{fleet['base']}.{PRIMARY}", ".deployctl-history").exists(), \ + "a reverted release is not a release" + + def test_reverts_newest_first_and_restores_each_hosts_own_record(self, fleet): + _update(fleet, FAKE_CRASH_HOSTS=THIRD) + assert _ups(fleet) == [ + f"up {PRIMARY}: image: demo:NEW (primary)", + f"up {SECOND}: image: demo:NEW (secondary)", + f"up {THIRD}: image: demo:NEW (secondary)", + f"up {THIRD}: image: demo:OLD (secondary)", # the host that failed + f"up {SECOND}: image: demo:OLD (secondary)", # then the others, newest first + f"up {PRIMARY}: image: demo:OLD (primary)", + ] + assert _recorded(fleet, PRIMARY) == "IMAGE_TAG=old-10" + assert _recorded(fleet, SECOND) == "IMAGE_TAG=old-11" + assert _recorded(fleet, THIRD) == "IMAGE_TAG=old-12" + + def test_a_failed_health_gate_does_the_same(self, fleet): + rc, out = _update(fleet, FAKE_UNHEALTHY_HOSTS=SECOND) + assert rc == 1, out + assert "OLD" in _running(fleet, PRIMARY) and "OLD" in _running(fleet, SECOND) + assert "OLD" in _running(fleet, THIRD), "the third host must never have been touched" + assert not any(THIRD in line for line in _ups(fleet)) + + +class TestTheFirstHostFailing: + def test_touches_no_other_host(self, fleet): + rc, out = _update(fleet, FAKE_CRASH_HOSTS=PRIMARY) + assert rc == 1, out + assert _ups(fleet) == [f"up {PRIMARY}: image: demo:NEW (primary)", f"up {PRIMARY}: image: demo:OLD (primary)"] + assert "[fleet]" not in out, "nothing else was moved, so there is nothing else to put back" + + +class TestHostScope: + def test_reverts_only_the_failed_host_and_says_the_fleet_is_split(self, fleet): + rc, out = _update(fleet, FAKE_CRASH_HOSTS=THIRD, REVERT_SCOPE="host") + assert rc == 1, out + assert "NEW" in _running(fleet, PRIMARY) and "NEW" in _running(fleet, SECOND) + assert "OLD" in _running(fleet, THIRD) + assert "the fleet is split between two releases" in out + assert "deployctl deploy rollback --env production" in out diff --git a/tests/test_history.py b/tests/test_history.py index 33bea40..2b7f929 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -1,52 +1,87 @@ -"""The local deploy log that `deploy rollback` consults to find "the previous release".""" +"""The release history kept on the primary, which `deploy rollback` consults for "the previous release". + +It lives on the host, not the laptop: a laptop's own log never saw what CI +deployed, so "the previous release" came from half the story. +""" from __future__ import annotations -from deployctl.cli import paths -from deployctl.cli.commands.deploy import _history_entries, _previous_tag +from deployctl.cli.commands.deploy import parse_history, parse_state, previous_release + + +def _history(*lines: tuple[str, str, str]) -> list[dict[str, str]]: + """(tag, kind, env) triples → the primary's file, parsed.""" + text = "".join(f"2026-09-{n + 1:02d}T00:00:00Z\t{env}\t{tag}\t{kind}\tamal@laptop\n" + for n, (tag, kind, env) in enumerate(lines)) + return parse_history(text, "production") + + +def test_releases_step_back_one_at_a_time(): + entries = _history(("aaa1111", "deploy", "production"), ("bbb2222", "deploy", "production")) + assert previous_release(entries, "bbb2222") == "aaa1111" + + +def test_the_first_bring_up_counts_as_a_release(): + entries = _history(("aaa1111", "init", "production"), ("bbb2222", "deploy", "production")) + assert previous_release(entries, "bbb2222") == "aaa1111" + + +def test_another_environments_lines_are_never_offered(): + entries = _history(("sss0000", "deploy", "staging"), ("bbb2222", "deploy", "production")) + assert [e["tag"] for e in entries] == ["bbb2222"] + assert previous_release(entries, "bbb2222") is None + + +def test_a_one_host_roll_is_not_a_release(): + entries = _history(("aaa1111", "deploy", "production"), ("ccc3333", "host:10.0.0.2", "production"), + ("bbb2222", "deploy", "production")) + assert previous_release(entries, "bbb2222") == "aaa1111" + -REPO = "ghcr.io/acme/backend" +def test_a_rollback_is_not_a_release(): + """Otherwise the history reads A, B, A and a second rollback picks B — the tag just left.""" + entries = _history(("aaa1111", "deploy", "production"), ("bbb2222", "deploy", "production"), + ("aaa1111", "rollback", "production")) + assert previous_release(entries, "aaa1111") is None -def _log(project, *lines: str) -> None: - paths.history_file().write_text("".join(line + "\n" for line in lines)) +def test_a_running_tag_the_history_never_saw_rolls_back_to_the_last_release(): + entries = _history(("aaa1111", "deploy", "production"), ("bbb2222", "deploy", "production")) + assert previous_release(entries, "zzz9999") == "bbb2222" -def test_releases_step_back_one_at_a_time(project): - _log(project, - f"2026-09-01T00:00:00Z production aaa1111 deploy {REPO}", - f"2026-09-02T00:00:00Z production bbb2222 deploy {REPO}") - assert _previous_tag("production", "bbb2222", REPO) == "aaa1111" +def test_who_deployed_is_kept(): + text = "2026-09-25T05:03:25Z\tproduction\t8e3e648\tdeploy\thttps://github.com/acme/app/actions/runs/1\n" + assert parse_history(text, "production")[0]["by"] == "https://github.com/acme/app/actions/runs/1" -def test_another_projects_log_is_never_offered(project): - """A deployctl directory copied between projects carries its untracked log along.""" - _log(project, - "2026-07-31T00:00:00Z production 6806988 deploy ghcr.io/acme/other-project", - f"2026-09-22T00:00:00Z production 2916a2f deploy {REPO}") - assert _previous_tag("production", "2916a2f", REPO) is None - assert [tag for _, tag, _ in _history_entries("production", REPO)] == ["2916a2f"] +def test_malformed_lines_are_skipped(): + assert parse_history("garbage\n2026 production only three\n", "production") == [] -def test_a_one_host_roll_is_not_a_release(project): - _log(project, - f"2026-09-01T00:00:00Z production aaa1111 deploy {REPO}", - f"2026-09-02T00:00:00Z production ccc3333 host:10.0.0.2 {REPO}", - f"2026-09-03T00:00:00Z production bbb2222 deploy {REPO}") - assert _previous_tag("production", "bbb2222", REPO) == "aaa1111" +# ---- deploy status --json --------------------------------------------------------- -def test_a_rollback_is_not_a_release(project): - _log(project, - f"2026-09-01T00:00:00Z production aaa1111 deploy {REPO}", - f"2026-09-02T00:00:00Z production bbb2222 deploy {REPO}", - f"2026-09-03T00:00:00Z production aaa1111 rollback {REPO}") - assert _previous_tag("production", "aaa1111", REPO) is None +def test_state_becomes_one_record_per_host(): + text = ( + "HOST\t10.0.0.1\tprimary\treachable\n" + "STATE\t10.0.0.1\tIMAGE_TAG=abc1234\n" + "STATE\t10.0.0.1\tDEPLOYED_BY=https://github.com/acme/app/actions/runs/7\n" + "SVC\t10.0.0.1\tapi 0 running healthy 0 unless-stopped\n" + "SVC\t10.0.0.1\tcelery_worker 3 running starting 0 unless-stopped\n" + "HOST\t10.0.0.2\tsecondary\tunreachable\n" + ) + state = parse_state(text) + primary, secondary = state["hosts"] + assert primary["tag"] == "abc1234" and primary["deployed_by"].endswith("/runs/7") + assert primary["services"][1] == {"name": "celery_worker", "state": "running", "health": "starting", + "restarts": 3, "exit_code": "0", "restart_policy": "unless-stopped"} + assert secondary["reachable"] is False and secondary["services"] == [] + assert state["tag"] == "abc1234" and state["split"] is False -def test_lines_from_before_the_repository_field_still_count(project): - _log(project, - "2026-07-31T09:11:34Z production 3b9bd38", - "2026-09-22T09:44:57Z production 2916a2f deploy", - f"2026-09-22T10:43:57Z production 41881fe deploy {REPO}") - assert _previous_tag("production", "41881fe", REPO) == "2916a2f" +def test_two_tags_across_the_fleet_are_reported_as_split(): + text = ("HOST\ta\tprimary\treachable\nSTATE\ta\tIMAGE_TAG=new1111\n" + "HOST\tb\tsecondary\treachable\nSTATE\tb\tIMAGE_TAG=old0000\n") + state = parse_state(text) + assert state["split"] is True and state["tag"] is None diff --git a/tests/test_migrate_config.py b/tests/test_migrate_config.py new file mode 100644 index 0000000..1bf9b1e --- /dev/null +++ b/tests/test_migrate_config.py @@ -0,0 +1,52 @@ +"""`deployctl migrate-config`: IMAGE_TAG leaves config/ without anything else changing.""" + +from __future__ import annotations + +from typer.testing import CliRunner + +from deployctl.cli import paths, tags +from deployctl.cli.main import app + +CONFIG = """\ +# ---------- Server ---------- +HOSTS=203.0.113.10 +# ---------- Image ---------- +# An immutable tag. +IMAGE_TAG=8e3e648 +REMOTE_DIR=/opt/demo +""" + + +def _run(*args): + return CliRunner().invoke(app, ["migrate-config", *args]) + + +def test_without_apply_it_only_prints_the_plan(project, write_config): + path = write_config("production", CONFIG) + result = _run() + assert result.exit_code == 0, result.output + assert "remove IMAGE_TAG=8e3e648 from config/production.env" in result.output + assert path.read_text() == CONFIG + + +def test_apply_removes_only_the_tag_and_keeps_it_as_the_last_used_one(project, write_config): + path = write_config("production", CONFIG) + result = _run("--apply") + assert result.exit_code == 0, result.output + assert "IMAGE_TAG" not in path.read_text().replace("# An immutable tag.", "") + assert "HOSTS=203.0.113.10" in path.read_text() and "# ---------- Image ----------" in path.read_text() + assert tags.cached("production") == "8e3e648" + + +def test_a_tag_used_since_is_not_overwritten(project, write_config): + write_config("production", CONFIG) + tags.remember("production", "newer99") + _run("--apply") + assert tags.cached("production") == "newer99" + + +def test_nothing_to_do_says_so(project, write_config): + write_config("production", "HOSTS=203.0.113.10\n") + result = _run("--apply") + assert result.exit_code == 0 + assert "nothing to migrate" in result.output diff --git a/tests/test_tags.py b/tests/test_tags.py new file mode 100644 index 0000000..f6e42a0 --- /dev/null +++ b/tests/test_tags.py @@ -0,0 +1,93 @@ +"""Which image tag a command works with, now that it is not a value in config/.""" + +from __future__ import annotations + +import pytest + +from deployctl.cli import config, tags + +SINGLE = """\ +MODE=single +PROJECT_NAME=demo +BASE_DOMAIN=demo.test +IMAGE_REPO=ghcr.io/acme/demo +HOSTS=203.0.113.10 +ACME_EMAIL=ops@demo.test +POSTGRES_DB=demo +POSTGRES_USER=demo +""" + + +@pytest.fixture +def cfg(write_config, monkeypatch): + monkeypatch.delenv("IMAGE_TAG", raising=False) + write_config("production", SINGLE) + return lambda: config.load("production") + + +@pytest.fixture +def hosts_run(monkeypatch): + """What the primary reports running (tags.running talks ssh; here, a stand-in).""" + running = {"tag": ""} + monkeypatch.setattr(tags, "running", lambda cfg: running["tag"]) + return running + + +def test_a_config_without_a_tag_is_valid(cfg): + assert [p.message for p in cfg().validate() if p.level == "error"] == [] + + +def test_an_explicit_tag_wins(cfg, hosts_run, monkeypatch): + hosts_run["tag"] = "run1111" + monkeypatch.setenv("IMAGE_TAG", "env2222") + c = cfg() + assert tags.resolve(c, "cli3333", from_hosts=True) == ("cli3333", "--tag") + assert c.derived["IMAGE_REF"] == "ghcr.io/acme/demo:cli3333" + + +def test_ci_passes_it_in_the_environment(cfg, hosts_run, monkeypatch): + hosts_run["tag"] = "run1111" + monkeypatch.setenv("IMAGE_TAG", "env2222") + assert tags.resolve(cfg(), from_hosts=True) == ("env2222", "$IMAGE_TAG") + + +def test_without_one_a_deploy_redeploys_what_is_running(cfg, hosts_run): + hosts_run["tag"] = "run1111" + tags.remember("production", "old0000") + tag, source = tags.resolve(cfg(), from_hosts=True) + assert tag == "run1111" and "running on 203.0.113.10" in source + + +def test_a_local_command_uses_the_tag_last_used_here(cfg, hosts_run): + hosts_run["tag"] = "run1111" + tags.remember("production", "old0000") + assert tags.resolve(cfg(), from_hosts=False) == ("old0000", "last used on this machine") + + +def test_a_tag_left_in_config_still_works_and_is_flagged(write_config, hosts_run, monkeypatch): + monkeypatch.delenv("IMAGE_TAG", raising=False) + write_config("production", SINGLE + "IMAGE_TAG=cfg4444\n") + hosts_run["tag"] = "run1111" + c = config.load("production") + tag, source = tags.resolve(c, from_hosts=True) + assert tag == "cfg4444" and "deprecated" in source + assert tags.deprecated_in_config(c) + assert any("IMAGE_TAG is set in config/" in p.message for p in c.validate() if p.level == "warn") + + +def test_nothing_found_is_an_error_that_says_what_to_do(cfg, hosts_run): + with pytest.raises(tags.NoTag, match="pass --tag"): + tags.resolve(cfg(), from_hosts=True) + + +@pytest.mark.parametrize("bad", ["abc 123", "abc;rm", "-rf", "a" * 200, "$(x)"]) +def test_a_tag_that_is_not_a_tag_is_refused(cfg, bad): + with pytest.raises(tags.NoTag): + tags.resolve(cfg(), bad, from_hosts=False) + + +def test_the_cache_survives_a_forced_render(cfg): + """render.clean(--force) empties generated//; the cache lives beside it, not in it.""" + tags.remember("production", "abc1234") + assert tags.cache_file("production").parent.name == "generated" + assert tags.cached("production") == "abc1234" diff --git a/uv.lock b/uv.lock index bbe10f0..254ca0a 100644 --- a/uv.lock +++ b/uv.lock @@ -75,6 +75,7 @@ dependencies = [ dev = [ { name = "httpx" }, { name = "pytest" }, + { name = "pyyaml" }, ] [package.metadata] @@ -90,6 +91,7 @@ requires-dist = [ dev = [ { name = "httpx", specifier = ">=0.27" }, { name = "pytest", specifier = ">=8.0" }, + { name = "pyyaml", specifier = ">=6.0" }, ] [[package]]