From 0973dad6245539ea1ded529c9513570bcc957187 Mon Sep 17 00:00:00 2001 From: Jeremy Daer Date: Thu, 20 Aug 2026 15:17:26 -0700 Subject: [PATCH] Repair and validate the AUR key before using it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v0.1.1 AUR publish failed all three attempts with ssh's 'invalid format' / 'Permission denied (publickey)' — a secret that did not survive paste, not an AUR outage — and filed the outage issue for it. This has recurred across our CLIs (CRLF, dropped trailing newline, flattened lines). scripts/install-ssh-key.sh repairs those shapes, proves the key loads with ssh-keygen -y before anything authenticates, and on refusal diagnoses the shape without printing key material. Both the release job and the recovery dispatch install the key through it in a named step, and release.yml's outage issue is now scoped to the publish step like aur-publish.yml's already was. --- .github/workflows/aur-publish.yml | 18 +++--- .github/workflows/release.yml | 22 +++++-- RELEASING.md | 2 +- scripts/install-ssh-key.sh | 70 ++++++++++++++++++++++ tests/e2e/install_ssh_key.bats | 98 +++++++++++++++++++++++++++++++ 5 files changed, 195 insertions(+), 15 deletions(-) create mode 100755 scripts/install-ssh-key.sh create mode 100644 tests/e2e/install_ssh_key.bats diff --git a/.github/workflows/aur-publish.yml b/.github/workflows/aur-publish.yml index 964c9f51..b181356d 100644 --- a/.github/workflows/aur-publish.yml +++ b/.github/workflows/aur-publish.yml @@ -125,25 +125,27 @@ jobs: fi echo "AUR is at ${current}; publishing ${VERSION}" - - name: Verify AUR key is configured + # Fails here, named, when the secret is missing or does not load as a + # key — the install script repairs the usual paste damage (CRLF, dropped + # trailing newline, flattened lines) and diagnoses the rest without + # printing key material. + - name: Install AUR key env: - AUR_KEY: ${{ secrets.AUR_KEY }} + SSH_KEY: ${{ secrets.AUR_KEY }} run: | - if [ -z "$AUR_KEY" ]; then + if [ -z "$SSH_KEY" ]; then echo "::error::AUR_KEY is not configured for the release environment" exit 1 fi + mkdir -p ~/.ssh + scripts/install-ssh-key.sh ~/.ssh/aur + echo -e "Host aur.archlinux.org\n IdentityFile ~/.ssh/aur\n User aur\n StrictHostKeyChecking accept-new" >> ~/.ssh/config - name: Publish to AUR id: publish env: - AUR_KEY: ${{ secrets.AUR_KEY }} VERSION: ${{ inputs.version }} run: | - mkdir -p ~/.ssh - echo "$AUR_KEY" > ~/.ssh/aur - chmod 600 ~/.ssh/aur - echo -e "Host aur.archlinux.org\n IdentityFile ~/.ssh/aur\n User aur\n StrictHostKeyChecking accept-new" >> ~/.ssh/config git config --global user.name "37signals" git config --global user.email "dev@37signals.com" for attempt in 1 2 3; do diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 741cb959..ea59449c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -312,16 +312,23 @@ jobs: echo "enabled=false" >> "$GITHUB_OUTPUT" fi - - name: Publish to AUR + # Its own step so a key that does not load fails here, named, instead of + # surfacing three attempts later as "Permission denied (publickey)" and + # filing an AUR-outage issue for what is a secrets problem. + - name: Install AUR key if: steps.aur-config.outputs.enabled == 'true' env: - AUR_KEY: ${{ secrets.AUR_KEY }} + SSH_KEY: ${{ secrets.AUR_KEY }} run: | - VERSION="${GITHUB_REF_NAME#v}" mkdir -p ~/.ssh - echo "$AUR_KEY" > ~/.ssh/aur - chmod 600 ~/.ssh/aur + scripts/install-ssh-key.sh ~/.ssh/aur echo -e "Host aur.archlinux.org\n IdentityFile ~/.ssh/aur\n User aur\n StrictHostKeyChecking accept-new" >> ~/.ssh/config + + - name: Publish to AUR + id: publish + if: steps.aur-config.outputs.enabled == 'true' + run: | + VERSION="${GITHUB_REF_NAME#v}" git config --global user.name "37signals" git config --global user.email "dev@37signals.com" # publish-aur.sh derives everything from the published release assets @@ -338,8 +345,11 @@ jobs: echo "AUR publish failed after 3 attempts" exit 1 + # Scoped to the publish step, as in aur-publish.yml: a key that fails to + # load is an operator problem the run annotates itself, not the outage + # this issue's "retry once the AUR is reachable" advice is written for. - name: Notify on AUR publish failure - if: failure() + if: failure() && steps.publish.outcome == 'failure' env: GH_TOKEN: ${{ github.token }} REPO_SLUG: ${{ github.repository }} diff --git a/RELEASING.md b/RELEASING.md index 05a64509..96d174e2 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -102,7 +102,7 @@ converges the environment across the CLI repos and copies secrets in from | `SM_API_KEY` | secret | DigiCert ONE API key for KeyLocker | | `SM_CLIENT_CERT_FILE_B64` | secret | Base64 (single line) of the DigiCert ONE mTLS client certificate `.p12` attachment | | `SM_CLIENT_CERT_PASSWORD` | secret | Client certificate password | -| `AUR_KEY` | secret | ed25519 SSH private key for the AUR (optional; publish skips without it) | +| `AUR_KEY` | secret | ed25519 SSH private key for the AUR (optional; publish skips without it) — set it from the key file, never a paste: `gh secret set AUR_KEY --env release < ~/.ssh/aur_hey_cli` | The `SM_*` values come from the **DigiCert CodeSigning Cert** item (Development vault): the two text fields byte-exact with no trailing newline, and the `.p12` diff --git a/scripts/install-ssh-key.sh b/scripts/install-ssh-key.sh new file mode 100755 index 00000000..5fc7d529 --- /dev/null +++ b/scripts/install-ssh-key.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Usage: SSH_KEY="$SECRET" scripts/install-ssh-key.sh DEST +# +# Writes an OpenSSH private key held in a secret to DEST (mode 0600) and proves +# it loads before anything tries to authenticate with it. +# +# A private key is the one secret that has to survive copy-paste byte for byte, +# and a secrets UI is where it gets mangled: CRLF line endings from a Windows +# clipboard, the trailing newline dropped by a form field, the whole key +# flattened onto one line with literal "\n" sequences, blank lines or spaces +# around it. Every one of those makes ssh report the useless +# `Load key "...": invalid format` and then `Permission denied (publickey)`, +# which reads as a credentials problem when it is a transport one — a mistake +# this team has made repeatedly. The mangling is repaired here, so a key that +# was pasted in any of those shapes still works, and a key that is genuinely +# wrong is reported with a shape diagnosis that never prints key material. + +set -euo pipefail + +dest="${1:?usage: SSH_KEY=... install-ssh-key.sh DEST}" +raw="${SSH_KEY:?SSH_KEY is not set}" + +die() { + echo "::error::$*" >&2 + exit 1 +} + +# A flattened paste carries the newlines as the two characters '\' 'n'. Expand +# them only when the key actually arrived on one line: a well-formed key has no +# backslashes in it (PEM armor plus base64), so the expansion is safe there, and +# a multi-line key is left untouched rather than risk rewriting anything. +if [[ "$raw" == *'\n'* ]] && [[ "$(printf '%s' "$raw" | tr -d '\r' | wc -l)" -le 1 ]]; then + raw="${raw//\\n/$'\n'}" +fi + +tmp="$(mktemp "${dest}.XXXXXX")" +trap 'rm -f "$tmp"' EXIT +chmod 600 "$tmp" + +# Drop carriage returns, surrounding blank lines and trailing whitespace, and +# end the file with exactly one newline — OpenSSH refuses a key without it. +printf '%s\n' "$raw" \ + | tr -d '\r' \ + | sed -e 's/[[:space:]]*$//' \ + | awk 'NF { if (blank && started) printf "%s", blank; blank = ""; started = 1; print; next } started { blank = blank "\n" }' \ + > "$tmp" + +if ! err="$(ssh-keygen -y -f "$tmp" 2>&1 >/dev/null)"; then + lines=$(wc -l < "$tmp") + first=$(head -n1 "$tmp") + last=$(tail -n1 "$tmp") + shape="lines=${lines}" + case "$first" in + "-----BEGIN "*" PRIVATE KEY-----") shape="${shape} header=ok" ;; + "ssh-"*|"ecdsa-"*) shape="${shape} header=PUBLIC-KEY" ;; + *) shape="${shape} header=missing" ;; + esac + case "$last" in + "-----END "*" PRIVATE KEY-----") shape="${shape} footer=ok" ;; + *) shape="${shape} footer=missing" ;; + esac + [[ "$SSH_KEY" == *$'\r'* ]] && shape="${shape} had-CR" + [[ "$SSH_KEY" == *'\n'* ]] && shape="${shape} had-literal-backslash-n" + [[ "$SSH_KEY" == *$'\n' ]] || shape="${shape} no-trailing-newline" + die "SSH key does not load after normalisation (${shape}): ${err}. Re-save the secret from the key file itself (gh secret set NAME < key), not from a paste." +fi + +mv "$tmp" "$dest" +trap - EXIT +chmod 600 "$dest" diff --git a/tests/e2e/install_ssh_key.bats b/tests/e2e/install_ssh_key.bats new file mode 100644 index 00000000..7955cc33 --- /dev/null +++ b/tests/e2e/install_ssh_key.bats @@ -0,0 +1,98 @@ +#!/usr/bin/env bats +# +# scripts/install-ssh-key.sh repairs the ways a private key gets mangled on its +# way through a secrets UI, and refuses — with a shape diagnosis and without +# printing key material — anything that still does not load. The AUR publish +# failed its first live run on exactly this: a key that ssh reported only as +# "invalid format". Each mangling that has bitten in practice gets a case, and +# the refusal cases pin that a bad key never reaches the destination. + +setup() { + REPO_ROOT="$(cd "${BATS_TEST_DIRNAME}/../.." && pwd)" + INSTALL="$REPO_ROOT/scripts/install-ssh-key.sh" + WORK="$(mktemp -d)" + ssh-keygen -q -t ed25519 -N '' -C 'aur@example.com' -f "$WORK/id" >/dev/null + KEY="$(cat "$WORK/id")" + PUB="$(cut -d' ' -f1,2 "$WORK/id.pub")" + DEST="$WORK/installed" +} + +teardown() { + rm -rf "$WORK" +} + +installed_pubkey() { + ssh-keygen -y -f "$DEST" | cut -d' ' -f1,2 +} + +assert_installed() { + [[ "$status" -eq 0 ]] + [[ -f "$DEST" ]] + [[ "$(stat -c %a "$DEST")" == "600" ]] + [[ "$(installed_pubkey)" == "$PUB" ]] +} + +@test "installs a pristine key" { + SSH_KEY="$KEY"$'\n' run "$INSTALL" "$DEST" + assert_installed +} + +@test "repairs a key whose trailing newline was dropped" { + SSH_KEY="$KEY" run "$INSTALL" "$DEST" + assert_installed + # Substitution strips a trailing newline, so an empty result is the newline. + [[ -z "$(tail -c1 "$DEST")" ]] +} + +@test "repairs CRLF line endings" { + SSH_KEY="$(printf '%s\n' "$KEY" | sed 's/$/\r/')" run "$INSTALL" "$DEST" + assert_installed + ! grep -q $'\r' "$DEST" +} + +@test "repairs a key flattened onto one line with literal backslash-n" { + flat="$(printf '%s\n' "$KEY" | awk '{ printf "%s\\n", $0 }')" + [[ "$flat" != *$'\n'* ]] + SSH_KEY="$flat" run "$INSTALL" "$DEST" + assert_installed +} + +@test "repairs surrounding blank lines and trailing spaces" { + SSH_KEY=$'\n\n'"$(printf '%s\n' "$KEY" | sed 's/$/ /')"$'\n\n \n' run "$INSTALL" "$DEST" + assert_installed +} + +@test "refuses a public key with a diagnosis and installs nothing" { + SSH_KEY="$(cat "$WORK/id.pub")" run "$INSTALL" "$DEST" + [[ "$status" -ne 0 ]] + [[ ! -e "$DEST" ]] + [[ "$output" == *"header=PUBLIC-KEY"* ]] + [[ "$output" == *"gh secret set"* ]] +} + +@test "refuses a truncated key and names the missing footer" { + SSH_KEY="$(printf '%s\n' "$KEY" | head -n 3)" run "$INSTALL" "$DEST" + [[ "$status" -ne 0 ]] + [[ ! -e "$DEST" ]] + [[ "$output" == *"footer=missing"* ]] +} + +@test "the diagnosis never prints key material" { + body="$(printf '%s\n' "$KEY" | sed -n 2p)" + SSH_KEY="$(printf '%s\n' "$KEY" | head -n 3)" run "$INSTALL" "$DEST" + [[ "$status" -ne 0 ]] + [[ "$output" != *"$body"* ]] +} + +@test "leaves no temp file behind on refusal" { + SSH_KEY="not a key" run "$INSTALL" "$DEST" + [[ "$status" -ne 0 ]] + [[ -z "$(ls "$WORK" | grep '^installed')" ]] +} + +@test "requires SSH_KEY and a destination" { + run "$INSTALL" + [[ "$status" -ne 0 ]] + SSH_KEY= run "$INSTALL" "$DEST" + [[ "$status" -ne 0 ]] +}