From 700ccf09b95c7a5c374ffdce1178356f75750786 Mon Sep 17 00:00:00 2001 From: Marc Bekhuis Date: Thu, 3 Sep 2026 13:31:10 +0200 Subject: [PATCH] Changes for version 0.8.1 --- .github/scripts/smoke.sh | 49 ++++ CHANGELOG.md | 36 ++- README.md | 25 +- bin/roll | 1 + commands/config.cmd | 236 ++++++++++++++++-- commands/config.help | 20 ++ commands/magento2-init.help | 2 - commands/usage.help | 27 --- docs/configuration/version-pins.md | 63 ++++- docs/machine-interface.md | 13 + utils/config.sh | 109 +++++++++ utils/images.sh | 375 +++++++++++++++++++++++++++++ version | 2 +- 13 files changed, 905 insertions(+), 53 deletions(-) create mode 100644 utils/images.sh diff --git a/.github/scripts/smoke.sh b/.github/scripts/smoke.sh index 11cc4af..0b7b400 100755 --- a/.github/scripts/smoke.sh +++ b/.github/scripts/smoke.sh @@ -53,6 +53,55 @@ trap cleanup EXIT cd -- "${TMP_DIR}" "${ROLL_BIN[@]}" env-init smoketest magento2 "${ROLL_BIN[@]}" config validate + + # The version overview reads the catalog and the project config only, so it needs no network + echo "+ config versions" + "${ROLL_BIN[@]}" config versions >/dev/null + + # Setting a version non-interactively must rewrite the pin in place. The registry lookup that + # would warn about an unknown tag is best-effort, so this passes with or without network access. + echo "+ config version php 8.2" + "${ROLL_BIN[@]}" config version php 8.2 >/dev/null 2>&1 + if ! grep -q '^PHP_VERSION=8.2$' .env.roll; then + >&2 echo "FAIL: roll config version did not write PHP_VERSION=8.2" + exit 1 + fi + if [[ "$(grep -c '^PHP_VERSION=' .env.roll)" != "1" ]]; then + >&2 echo "FAIL: roll config version left more than one PHP_VERSION line" + exit 1 + fi + + # Same again, so the "nothing to change" path is exercised and stays a success + "${ROLL_BIN[@]}" config version php 8.2 >/dev/null 2>&1 + + # A commented pin is replaced rather than duplicated (magento2's init.env ships one) + echo "+ config version opensearch 2.19" + "${ROLL_BIN[@]}" config version opensearch 2.19 >/dev/null 2>&1 + if [[ "$(grep -c '^#*[[:space:]]*OPENSEARCH_VERSION=' .env.roll)" != "1" ]]; then + >&2 echo "FAIL: roll config version duplicated the commented OPENSEARCH_VERSION line" + exit 1 + fi + + # With no terminal, the service prompt must fail naming the non-interactive form, and write + # nothing - this is the contract automation depends on + echo "+ config version (no tty)" + if out="$("${ROLL_BIN[@]}" config version 2>&1 &2 echo "FAIL: roll config version succeeded with no terminal and no arguments" + exit 1 + fi + case "${out}" in + *"roll config version "*) ;; + *) + >&2 echo "FAIL: roll config version did not name its non-interactive form" + exit 1 + ;; + esac + + echo "+ config version nosuchservice" + if "${ROLL_BIN[@]}" config version nosuchservice 1.0 >/dev/null 2>&1; then + >&2 echo "FAIL: roll config version accepted an unknown service" + exit 1 + fi ) "$(dirname "$0")/test-syntax.sh" diff --git a/CHANGELOG.md b/CHANGELOG.md index 669a415..052311e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,41 @@ range. Versions before 0.3.0 predate the GitHub releases and are reconstructed f Merge commits, automated `Tagged ` commits and version bumps are omitted, as are bullets that only restate the release note above them. -## Unreleased +## Unreleased — 0.8.1 + +### Added + +- **`roll config version` — change a service version interactively.** Prompts for the service, then + for one of the versions that service's image actually has, and writes the pin into `.env.roll` + after taking a timestamped backup. The version list is read from the registry the environment + pulls from (`$ROLL_IMAGE_REPOSITORY`, `ghcr.io/epartment/roll` by default) over the Docker + Registry v2 API and cached for an hour, so it cannot drift from what is publishable. Supplying + both arguments (`roll config version php 8.3`) skips the prompts, and with no terminal the prompt + fails naming that form rather than hanging. +- **`roll config versions`** — the version this project pins for every service, with the key that + holds it and whether the service is enabled. With a service name it lists that service's + published versions, one per line on stdout, for scripting. + +### Changed + +- **`roll config set` no longer edits `.env.roll` with `sed`.** Both it and the new version picker + go through one writer, which rewrites an existing pin in place, replaces a commented-out one + rather than appending a second line, and passes the value through no pattern expansion — so a + value containing `/` or `&` is written literally instead of corrupting the line. The reported + backup path is now the file that was actually created; it was recomputed from a second `date` + call, so it named a non-existent path whenever the two landed in different seconds. + +- **`roll --help` no longer prints the ASCII-art banner.** The usage output now opens on the + version line. + +### Fixed + +- **`roll --help` and `roll usage` printed the whole help twice.** `commands/usage.help` echoed + `ROLL_USAGE` itself and `commands/usage.cmd` echoed it again after sourcing the file. A `.help` + file only defines `ROLL_USAGE`; rendering belongs to `usage.cmd`. The same stray echo was removed + from `commands/magento2-init.help`. + +## [0.8.0](https://github.com/epartment/rolldev/releases/tag/0.8.0) — 2026-09-03 The 0.8.0 development work. Two changes need reading before upgrading — unpinned service versions now warn (and become an error in 0.9.0), and gum becomes a dependency. Both are covered in diff --git a/README.md b/README.md index ea9faf0..3e0a6c6 100644 --- a/README.md +++ b/README.md @@ -142,8 +142,9 @@ environment can produce a fragment set that does not match either copy. ### Dispatch -`bin/roll` is the single entry point. It sources four libraries in a fixed order — `utils/core.sh`, -`utils/config.sh`, `utils/registry.sh`, `utils/env.sh` — verifies Docker and the compose version, then +`bin/roll` is the single entry point. It sources its libraries in a fixed order — `utils/core.sh`, +`utils/interact.sh`, `utils/config.sh`, `utils/images.sh`, `utils/registry.sh`, `utils/env.sh`, +`utils/backup.sh` — verifies Docker and the compose version, then resolves the first argument through the command registry and **sources** the matching `.cmd` file (`bin/roll:98`). Sourcing rather than executing is deliberate: every command runs in the same shell and can use all globals and helper functions without re-deriving them. @@ -284,7 +285,8 @@ responsibility is still describable. |---|---|---|---| | `bin/roll` | 98 lines | Entry point, dispatch, argument parsing | **Coherent** | | `utils/core.sh` | 184 lines | Messaging helpers, array/version utilities, network peering | **Coherent** — the three box-drawing functions are one-line wrappers around a shared `box` helper | -| `utils/config.sh` | 606 lines | Config schema, loading, validation, post-processing | **Coherent**; sole owner of configuration defaults | +| `utils/config.sh` | 946 lines | Config schema, loading, validation, post-processing, `.env.roll` writes | **Coherent**; sole owner of configuration defaults | +| `utils/images.sh` | 375 lines | Service catalog (version key, toggle and image per service) and image-tag discovery from the registry | **Coherent** | | `utils/registry.sh` | 461 lines | Command discovery and priority resolution | **Oversized for what it delivers** — ~200 lines serve `roll registry`'s reporting subcommands; the metadata layer they report on is a stub (**M6**) | | `utils/env.sh` | 108 lines | Env path location, partial precedence, env-type validation | **Coherent** | | `utils/install.sh` | 63 lines | Host install assertion, SSH config | **Coherent** | @@ -477,6 +479,18 @@ Inspect the resolved configuration for the current project: roll config show ``` +See which version this project pins for every service, and whether that service is enabled: + +```bash +roll config versions +``` + +Change one of them, picking from the versions the image actually has: + +```bash +roll config version +``` + List every command the registry resolved, including user and third-party packs: ```bash @@ -513,6 +527,8 @@ exists and where it lives. | `roll env up --wait` | Compose native, made meaningful by the healthchecks in `environments/includes/*.base.yml` | | `roll env doctor` | `commands/doctor.cmd` | | `roll config check-pins` / `fix-pins` | `commands/config.cmd` | +| `roll config versions [service]` — pinned versions, or a service's published versions one per line on stdout | `commands/config.cmd`, `utils/images.sh` | +| `roll config version ` — set a pin without prompting | `commands/config.cmd` | | Flag-first prompts | `utils/interact.sh` | Two rules bind anything added here: @@ -562,6 +578,9 @@ These are enforced by review rather than by tooling, except where noted. up with no central list to edit. If it must accept arbitrary pass-through flags, add it to `ROLL_CMD_ANYARGS` in `bin/roll:43`. Add it to `commands/usage.help` as well — that file is maintained by hand for visibility. +- **Help files define, they do not print.** A `commands/*.help` file only assigns `ROLL_USAGE`; + `commands/usage.cmd` sources it and does the single `echo -e`. A `.help` file that echoes + `ROLL_USAGE` itself makes the help render twice. - **Adding a service:** add the fragment(s) under `environments/includes/` (plus per-type overrides), wire a `ROLL_` toggle into the assembly block in `commands/env.cmd`, and register the variable and its default in `initConfigSchema` (`utils/config.sh:58`). Do not rely on a diff --git a/bin/roll b/bin/roll index 5081476..ab82f4b 100755 --- a/bin/roll +++ b/bin/roll @@ -15,6 +15,7 @@ readonly ROLL_DIR="$( source "${ROLL_DIR}/utils/core.sh" source "${ROLL_DIR}/utils/interact.sh" source "${ROLL_DIR}/utils/config.sh" +source "${ROLL_DIR}/utils/images.sh" source "${ROLL_DIR}/utils/registry.sh" source "${ROLL_DIR}/utils/env.sh" source "${ROLL_DIR}/utils/backup.sh" diff --git a/commands/config.cmd b/commands/config.cmd index 33e7ffc..7bd9a37 100644 --- a/commands/config.cmd +++ b/commands/config.cmd @@ -190,24 +190,12 @@ case "${ROLL_PARAMS[0]}" in exit 1 fi - # Create backup - cp "$config_file" "${config_file}.backup.$(date +%Y%m%d_%H%M%S)" - - # Update or add the configuration - if grep -q "^${key}=" "$config_file"; then - # Update existing key - use different approach for macOS compatibility - if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' "s/^${key}=.*/${key}=${value}/" "$config_file" - else - sed -i "s/^${key}=.*/${key}=${value}/" "$config_file" - fi - else - # Add new key - echo "${key}=${value}" >> "$config_file" - fi - + ## Backs up, then rewrites the key in place, uncomments a commented one, or appends it - + ## see writeEnvRollValue in utils/config.sh + writeEnvRollValue "$config_file" "$key" "$value" || exit 1 + success "Configuration updated: ${key}=${value}" - info "Backup created: ${config_file}.backup.$(date +%Y%m%d_%H%M%S)" + info "Backup created: ${ROLL_CONFIG_WRITE_BACKUP}" ;; get) @@ -286,9 +274,221 @@ case "${ROLL_PARAMS[0]}" in success "Wrote ${#ROLL_MISSING_PINS[@]} pin(s) to ${config_file}" ;; + versions) + ## Read-only companion to `version`: with no argument it reports what this project pins, + ## which needs no network; with a service it lists that service's published versions on + ## stdout, one per line, so it can be piped. Everything explanatory goes to stderr via the + ## messaging helpers, keeping stdout parseable. + if ROLL_ENV_PATH="$(locateEnvPath 2>/dev/null)"; then + config_file="${ROLL_ENV_PATH}/.env.roll" + else + error "Not in a Roll project directory" + exit 1 + fi + + loadRollConfig "${ROLL_ENV_PATH}" >/dev/null 2>&1 || { + error "Failed to load configuration from ${config_file}" + exit 1 + } + + service="${ROLL_PARAMS[1]:-}" + + if [[ -z "${service}" ]]; then + echo -e "\033[33mService versions pinned by ${config_file}:\033[0m" + echo "" + showServiceVersions + echo "" + info "\`roll config version\` changes one; \`roll config versions \` lists what is available." + exit 0 + fi + + if ! isImageService "${service}"; then + error "Unknown service: ${service}" + echo "Available services: $(imageServiceSlugs | tr '\n' ' ')" + exit 1 + fi + + version_key="$(versionKeyForService "${service}")" + current="$(getConfig "${version_key}" "")" + + case "$(imageCatalogField "${service}" source)" in + phpnode) + ## Node ships as a suffix on the PHP image's tag, so the list depends on the PHP + ## version in effect - another PHP version's Node builds are tags this project + ## cannot pull. + php_version="$(getConfig PHP_VERSION "")" + if [[ -z "${php_version}" ]]; then + error "NODE_VERSION depends on PHP_VERSION, which this project does not pin." + info "Pin it first with \`roll config version php \`." + exit 1 + fi + [[ -n "${current}" ]] && info "${version_key} is currently ${current} (Node builds of PHP ${php_version})" + availableNodeVersions "${php_version}" || exit 1 + ;; + registry) + [[ -n "${current}" ]] && info "${version_key} is currently ${current}" + availableServiceVersions "${service}" || exit 1 + ;; + *) + error "RollDev cannot list versions of $(imageCatalogField "${service}" image)." + info "It is not published under \${ROLL_IMAGE_REPOSITORY}; set ${version_key} with \`roll config version ${service} \`." + exit 1 + ;; + esac + ;; + + version) + ## Pick a service, pick one of the versions its image actually has, write it to .env.roll. + ## The version list comes from the registry the environment pulls from, so it cannot go + ## stale the way a list maintained in this repository would - see utils/images.sh. + ## + ## Positional arguments only: `config` is not on bin/roll's ROLL_CMD_ANYARGS list, so a + ## dash-prefixed argument never reaches this file. The non-interactive form is + ## `roll config version `. + if ROLL_ENV_PATH="$(locateEnvPath 2>/dev/null)"; then + config_file="${ROLL_ENV_PATH}/.env.roll" + else + error "Not in a Roll project directory" + exit 1 + fi + + loadRollConfig "${ROLL_ENV_PATH}" >/dev/null 2>&1 || { + error "Failed to load configuration from ${config_file}" + exit 1 + } + + service="${ROLL_PARAMS[1]:-}" + + ## Choosing the service touches no network, so a machine that cannot reach the registry + ## still gets this far and then fails about the registry rather than about the prompt. + if [[ -z "${service}" ]]; then + echo -e "\033[33mService versions pinned by ${config_file}:\033[0m" + echo "" + showServiceVersions + echo "" + # shellcheck disable=SC2046 + promptChoose service "roll config version " \ + "Which service's version do you want to change?" $(imageServiceSlugs) + fi + + if ! isImageService "${service}"; then + error "Unknown service: ${service}" + echo "Available services: $(imageServiceSlugs | tr '\n' ' ')" + exit 1 + fi + + version_key="$(versionKeyForService "${service}")" + version_source="$(imageCatalogField "${service}" source)" + current="$(getConfig "${version_key}" "")" + selected="${ROLL_PARAMS[2]:-}" + + if [[ -z "${selected}" ]]; then + case "${version_source}" in + phpnode) + php_version="$(getConfig PHP_VERSION "")" + if [[ -z "${php_version}" ]]; then + error "NODE_VERSION depends on PHP_VERSION, which this project does not pin." + info "Pin it first with \`roll config version php \`." + exit 1 + fi + available="$(availableNodeVersions "${php_version}")" || exit 1 + ;; + registry) + available="$(availableServiceVersions "${service}")" || exit 1 + ;; + *) + ## Selenium's tags are not enumerable (see utils/images.sh), so ask for one + info "RollDev cannot list versions of $(imageCatalogField "${service}" image); enter one yourself." + promptInput selected "roll config version ${service} " \ + "${version_key} (currently ${current:-unpinned}):" "${current}" + available="" + ;; + esac + + if [[ -z "${selected}" ]]; then + if [[ -z "${available}" ]]; then + error "No published versions found for ${service}." + exit 1 + fi + + [[ -n "${current}" ]] && info "${version_key} is currently ${current}" + # shellcheck disable=SC2046 + promptChoose selected "roll config version ${service} " \ + "Which version of ${service} should ${ROLL_ENV_NAME} run?" $(printf '%s\n' "${available}") + fi + fi + + ## The value becomes an image tag, so hold it to what a tag may look like. Checked before + ## the registry lookup below so a malformed value is rejected on its own terms rather than + ## first drawing a puzzling "not a published version" warning. + if [[ ! "${selected}" =~ ^[0-9][0-9a-zA-Z._-]*$ ]]; then + error "\"${selected}\" is not a valid version for ${version_key}." + exit 1 + fi + + ## A version given on the command line is not checked the way a picked one is, so say when + ## it is not a tag we know of - it may be brand new, or a typo, and only the person running + ## the command can tell which. + if [[ -n "${ROLL_PARAMS[2]:-}" && "${version_source}" == "registry" ]] \ + && available="$(availableServiceVersions "${service}" 2>/dev/null)" \ + && [[ -n "${available}" ]] + then + if ! containsElement "${selected}" ${available}; then + warning "${selected} is not a published version of $(imageForService "${service}"); \`roll env up\` will fail if that tag does not exist." + fi + fi + + ## Switching distribution and switching version are one act here: the image name comes from + ## DB_DISTRIBUTION and the tag from its version, so picking mysql on a mariadb project has + ## to move both, or the project would ask for a mysql version of the mariadb image. The two + ## database slugs also share one version key, so an unchanged version still means a change + ## when the distribution moves - which is why this is settled before the no-op check. + db_distribution="" + switch_distribution=0 + if [[ "${service}" == "mysql" || "${service}" == "mariadb" ]]; then + db_distribution="$(getConfig DB_DISTRIBUTION mariadb)" + [[ "${service}" != "${db_distribution}" ]] && switch_distribution=1 + fi + + if [[ "${selected}" == "${current}" && ${switch_distribution} -eq 0 ]]; then + success "${version_key} is already ${selected}; nothing to change." + exit 0 + fi + + if [[ ${switch_distribution} -eq 1 ]]; then + warning "This project runs ${db_distribution}; choosing ${service} switches DB_DISTRIBUTION as well." + warning "The existing database volume was written by ${db_distribution} and ${service} will not read it." + warning "Dump the database first, then recreate the volume (\`roll env down -v\`) and import it again." + if ! promptConfirm "roll config set DB_DISTRIBUTION ${service}" "Switch DB_DISTRIBUTION to ${service}?"; then + error "Cancelled; nothing was written." + exit 1 + fi + writeEnvRollValue "${config_file}" DB_DISTRIBUTION "${service}" || exit 1 + success "DB_DISTRIBUTION=${service}" + fi + + writeEnvRollValue "${config_file}" "${version_key}" "${selected}" || exit 1 + + success "${version_key}=${selected} (was ${current:-unpinned})" + info "Backup: ${ROLL_CONFIG_WRITE_BACKUP}" + + toggle="$(imageCatalogField "${service}" toggle)" + if [[ -n "${toggle}" ]] && [[ "$(getConfig "${toggle}" 0)" != "1" ]]; then + warning "${service} is not enabled in this project; switch it on with \`roll config set ${toggle} 1\`." + exit 0 + fi + + info "Apply it with \`roll env down && roll env up\`." + case "${service}" in + mariadb|mysql|elasticsearch|opensearch|mongodb) + info "The ${service} data volume was written by the previous version; check that engine's upgrade path before starting it." + ;; + esac + ;; + *) error "Unknown config command: ${ROLL_PARAMS[0]}" - echo "Available commands: show, validate, conflicts, schema, set, get, check-pins, fix-pins" + echo "Available commands: show, validate, conflicts, schema, set, get, version, versions, check-pins, fix-pins" exit 1 ;; esac \ No newline at end of file diff --git a/commands/config.help b/commands/config.help index a0b5ed4..b25bdcd 100644 --- a/commands/config.help +++ b/commands/config.help @@ -24,6 +24,16 @@ ROLL_USAGE=$(cat < [default] Get configuration value from current environment Returns default if key is not set + version [service] [version] + Change a service version in .env.roll interactively. + Prompts for the service, then for one of the versions its + image actually has, read live from the image registry. + Supply both arguments to skip the prompts. + + versions [service] Without a service: show the version this project pins for + every service, and whether that service is enabled. + With a service: list its published versions, one per line. + check-pins List enabled services with no version pin. Exits 1 if any are missing, so scripts and CI can gate on it. @@ -39,6 +49,11 @@ ROLL_USAGE=$(cat < not selected (DB_DISTRIBUTION=mariadb) + elasticsearch ELASTICSEARCH_VERSION 8.11 enabled + opensearch OPENSEARCH_VERSION disabled (ROLL_OPENSEARCH=0) + ... +``` + +One row per service RollDev can run: the key that holds its version, the version this project +pins, and whether the service is switched on. An `` row for an enabled service is what +`check-pins` reports. + ## Upgrading a version deliberately -Edit the value in `.env.roll` and bring the environment back up: +```bash +roll config version +``` + +This asks which service to change, then offers the versions that service's image actually has, and +writes your choice into `.env.roll` after taking a timestamped backup. The version list is read +from the registry RollDev pulls from, so it is never a list someone has to keep up to date — a +version that exists is offered, and one that does not is not. + +Skip either prompt by naming the service, or the service and the version: + +```bash +roll config version php # choose from the PHP versions the image is built for +roll config version php 8.3 # set PHP_VERSION=8.3 without prompting +``` + +To see the choices without changing anything: + +```bash +roll config versions php +``` + +The list comes back newest first, one version per line, so it can be piped. It is cached for an +hour per image; `ROLL_TAG_CACHE_TTL=0 roll config versions php` refetches. + +Three services need a word of explanation: + +- **Node** ships inside the PHP image, as a `-nodeNN` suffix on its tag, so the Node versions on + offer are the ones built for the PHP version you have pinned. `NODE_VERSION=0` is offered too, + and means an image with no Node at all. +- **The database** has one version key for two engines. Picking `mysql` on a MariaDB project (or + the reverse) switches `DB_DISTRIBUTION` as well, and asks first — the existing database volume + was written by the other engine and will not be readable. Dump the database, recreate the volume + with `roll env down -v`, then import it again. +- **Selenium** is the one image not published under `$ROLL_IMAGE_REPOSITORY`, and Docker Hub's + `selenium/standalone-chrome` carries thousands of tags in several shapes, so RollDev asks you to + type the version rather than offering a list. + +Either way the command only edits `.env.roll`. Bring the environment back up to apply it: ```bash roll env down @@ -99,3 +158,5 @@ roll env up Changing a database or search-engine version usually means the existing data volume was written by the old version. Check that engine's upgrade path first — RollDev only selects the image, it does not migrate data. + +You can still edit `.env.roll` by hand; nothing about the file has changed. diff --git a/docs/machine-interface.md b/docs/machine-interface.md index 9a80c6f..cd96198 100644 --- a/docs/machine-interface.md +++ b/docs/machine-interface.md @@ -137,6 +137,19 @@ roll config fix-pins # writes the versions the project runs today `check-pins` is the one to put in CI. See [Service version pins](configuration/version-pins.md). +Changing a pin needs no terminal either. `roll config version ` writes it +directly, and `roll config versions ` lists the versions that service's image has, one per +line on stdout, so a script can pick one: + +```bash +roll config versions elasticsearch # 9.5, 9.4, 9.3, ... newest first +roll config version elasticsearch 8.19 # write ELASTICSEARCH_VERSION=8.19 +``` + +Both write nothing but the pin, and `roll config versions` with no service reports what the project +pins today without touching the network. Called with no arguments, `roll config version` needs a +terminal to prompt on and fails naming the non-interactive form rather than hanging. + ## Running many environments on one host Two settings exist for build servers running several environments at once: diff --git a/utils/config.sh b/utils/config.sh index 9a067b4..0b99744 100644 --- a/utils/config.sh +++ b/utils/config.sh @@ -447,6 +447,115 @@ function setConfigValue() { export "$key"="$value" } +## The backup path written by the last writeEnvRollValue call, so callers can name it accurately - +## recomputing a timestamp afterwards can land a second later and report a file that does not exist. +ROLL_CONFIG_WRITE_BACKUP="" + +## writeEnvRollValue +## +## Persist KEY=value in a .env.roll, after taking a timestamped backup. +## +## Three cases, in this order of preference: +## 1. an active `KEY=` line is rewritten in place, so the value keeps its position in the file and +## whatever comment sits above it; +## 2. otherwise a commented `#KEY=` line is replaced, because the environment type templates ship +## keys that way (`#OPENSEARCH_VERSION=2.19` in magento2's init.env) and appending a second +## line would leave the file carrying two answers for one key; +## 3. otherwise the line is appended. +## Any further active line for the same key is dropped, since it would override the one just +## written - loadConfigFromFile takes the last occurrence. +## +## Done with a read/write loop rather than `sed -i`: the value goes through no pattern expansion, so +## one containing `/` or `&` cannot corrupt the file, and there is no BSD/GNU `-i` difference to +## work around. +function writeEnvRollValue() { + local config_file="$1" key="$2" value="$3" + local line="" tmp_file="" target=-1 i=0 + local lines=() + + if [[ ! -f "${config_file}" ]]; then + error "Configuration file not found: ${config_file}" + return 1 + fi + + ## The key is interpolated into the line-matching regex below, so anything that is not a plain + ## variable name would match lines it has no business matching + if [[ ! "${key}" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then + error "\"${key}\" is not a valid configuration key name." + return 1 + fi + + ## One backup per roll invocation, not per write. A command that writes twice - the database + ## distribution and then its version - would otherwise overwrite the pristine backup with the + ## half-changed file, since both writes land in the same second and so produce the same name. + ## The variable is process-local, so a value naming a backup of THIS file means the backup for + ## this invocation has already been taken. + if [[ "${ROLL_CONFIG_WRITE_BACKUP}" != "${config_file}.backup."* ]] \ + || [[ ! -f "${ROLL_CONFIG_WRITE_BACKUP}" ]] + then + ROLL_CONFIG_WRITE_BACKUP="${config_file}.backup.$(date +%Y%m%d_%H%M%S)" + if ! cp -- "${config_file}" "${ROLL_CONFIG_WRITE_BACKUP}"; then + error "Could not write a backup to ${ROLL_CONFIG_WRITE_BACKUP}" + return 1 + fi + fi + + while IFS= read -r line || [[ -n "${line}" ]]; do + lines[${#lines[@]}]="${line}" + done < "${config_file}" + + while [[ ${i} -lt ${#lines[@]} ]]; do + if [[ "${lines[$i]}" =~ ^[[:space:]]*${key}= ]]; then + target=${i} + break + fi + i=$((i + 1)) + done + + if [[ ${target} -lt 0 ]]; then + i=0 + while [[ ${i} -lt ${#lines[@]} ]]; do + if [[ "${lines[$i]}" =~ ^[[:space:]]*#[[:space:]]*${key}= ]]; then + target=${i} + break + fi + i=$((i + 1)) + done + fi + + tmp_file="$(mktemp "${TMPDIR:-/tmp}/roll-env-roll.XXXXXX")" + + ## One redirect around the whole loop rather than one append per line. The info below still + ## reaches the terminal because the messaging helpers write to stderr. + { + i=0 + while [[ ${i} -lt ${#lines[@]} ]]; do + if [[ ${i} -eq ${target} ]]; then + printf '%s=%s\n' "${key}" "${value}" + elif [[ "${lines[$i]}" =~ ^[[:space:]]*${key}= ]]; then + info "Dropped a later duplicate ${key} line, which would have overridden this one" + else + printf '%s\n' "${lines[$i]}" + fi + i=$((i + 1)) + done + + if [[ ${target} -lt 0 ]]; then + printf '%s=%s\n' "${key}" "${value}" + fi + } > "${tmp_file}" + + ## cp rather than mv: the destination keeps its inode, owner and permissions + if ! cp -- "${tmp_file}" "${config_file}"; then + error "Could not write ${config_file}; its previous content is in ${ROLL_CONFIG_WRITE_BACKUP}" + rm -f -- "${tmp_file}" + return 1 + fi + + rm -f -- "${tmp_file}" + return 0 +} + ## Apply an environment-type default. Unlike setConfigDefault, which fills in the schema literal, ## this expresses "what this environment type needs unless the project said otherwise", so it must ## run BEFORE the schema loop and must never override a value the user actually wrote in diff --git a/utils/images.sh b/utils/images.sh new file mode 100644 index 0000000..ff1d043 --- /dev/null +++ b/utils/images.sh @@ -0,0 +1,375 @@ +#!/usr/bin/env bash +[[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 + +## Service catalog and image-tag discovery. +## +## Everything a user can pin a version for is described once, here, as a record per service. The +## catalog answers three questions that were previously answered by hand in three different places: +## which .env.roll key holds a service's version, which toggle decides whether that service runs, +## and which image tag list the version has to come from. +## +## The version lists are NOT maintained here. They are read from the registry the environment +## actually pulls from (${ROLL_IMAGE_REPOSITORY}, ghcr.io/epartment/roll by default) using the +## Docker Registry v2 API, so a version that exists is offered and one that does not is not - no +## curated list in this repository can drift out of date. Responses are cached under +## ${ROLL_HOME_DIR}/tmp/image-tags so a picker that lists several services costs one round trip per +## image per hour rather than one per invocation. + +## How long a cached tag list stays fresh, in minutes. ROLL_TAG_CACHE_TTL=0 forces a refetch. +ROLL_TAG_CACHE_TTL="${ROLL_TAG_CACHE_TTL:-60}" + +## Service catalog, one colon-delimited record per pinnable service: +## +## :::: +## +## slug what the user types and picks from +## version key the .env.roll key holding the version (may be remapped, see versionKeyForService) +## toggle key the .env.roll key that switches the service on; empty means "always running" +## image repository name under ${ROLL_IMAGE_REPOSITORY}; "php-fpm" is resolved per env type +## source registry = tags come from the image repository +## phpnode = tags come from the PHP image's -nodeNN suffixes +## manual = not enumerable, the version is typed in +## +## Selenium is `manual` on purpose: it is the one image not published under +## ${ROLL_IMAGE_REPOSITORY}, and Docker Hub's selenium/standalone-chrome carries thousands of tags +## in four different shapes, so a picker over them would be less usable than typing the version. +ROLL_IMAGE_CATALOG=( + "php:PHP_VERSION::php-fpm:registry" + "node:NODE_VERSION::php-fpm:phpnode" + "mariadb:DB_DISTRIBUTION_VERSION:ROLL_DB:mariadb:registry" + "mysql:DB_DISTRIBUTION_VERSION:ROLL_DB:mysql:registry" + "nginx:NGINX_VERSION:ROLL_NGINX:nginx:registry" + "redis:REDIS_VERSION:ROLL_REDIS:redis:registry" + "dragonfly:DRAGONFLY_VERSION:ROLL_DRAGONFLY:dragonfly:registry" + "varnish:VARNISH_VERSION:ROLL_VARNISH:varnish:registry" + "elasticsearch:ELASTICSEARCH_VERSION:ROLL_ELASTICSEARCH:elasticsearch:registry" + "opensearch:OPENSEARCH_VERSION:ROLL_OPENSEARCH:opensearch:registry" + "rabbitmq:RABBITMQ_VERSION:ROLL_RABBITMQ:rabbitmq:registry" + "mongodb:MONGO_VERSION:ROLL_MONGODB:mongo:registry" + "magepack:MAGEPACK_VERSION:ROLL_MAGEPACK:magepack:registry" + "selenium:ROLL_SELENIUM_VERSION:ROLL_SELENIUM:selenium/standalone-chrome:manual" +) + +## Every slug in catalog order, one per line. +function imageServiceSlugs() { + local record="" + + for record in "${ROLL_IMAGE_CATALOG[@]}"; do + echo "${record%%:*}" + done + + return 0 +} + +## imageCatalogField +## Empty output when the slug is unknown, so callers test the slug with isImageService first. +function imageCatalogField() { + local slug="$1" field="$2" + local record="" rest="" + + for record in "${ROLL_IMAGE_CATALOG[@]}"; do + [[ "${record%%:*}" == "${slug}" ]] || continue + + rest="${record#*:}" + case "${field}" in + key) echo "${rest%%:*}" ;; + toggle) rest="${rest#*:}"; echo "${rest%%:*}" ;; + image) rest="${rest#*:}"; rest="${rest#*:}"; echo "${rest%%:*}" ;; + source) echo "${record##*:}" ;; + *) echo "" ;; + esac + return 0 + done + + echo "" + return 0 +} + +function isImageService() { + local slug="$1" + local record="" + + for record in "${ROLL_IMAGE_CATALOG[@]}"; do + [[ "${record%%:*}" == "${slug}" ]] && return 0 + done + + return 1 +} + +## The .env.roll key to write for a service, which is not always the catalog's key. +## +## The database version has three spellings: DB_DISTRIBUTION_VERSION is what the image tag is built +## from, while MYSQL_VERSION and MARIADB_VERSION are the older per-distribution names that +## applyEnvTypeDefaults turns into it. A project already using the older spelling must keep using +## it - writing DB_DISTRIBUTION_VERSION alongside it would leave two version lines in .env.roll +## saying different things, with the reader having to know which one wins. +function versionKeyForService() { + local slug="$1" + local key="" + + key="$(imageCatalogField "${slug}" key)" + + if [[ "${key}" == "DB_DISTRIBUTION_VERSION" ]]; then + local legacy="" + case "${slug}" in + mysql) legacy="MYSQL_VERSION" ;; + mariadb) legacy="MARIADB_VERSION" ;; + esac + + if [[ -n "${legacy}" ]] && containsElement "${legacy}" "${ROLL_PROJECT_CONFIG_KEYS[@]:-}"; then + echo "${legacy}" + return 0 + fi + fi + + echo "${key}" + return 0 +} + +## The image repository a service's tags come from, relative to ${ROLL_IMAGE_REPOSITORY}. +## +## php-fpm is per environment type: postProcessConfig appends -magento1/-magento2/-wordpress to the +## image name for those types, so the tag list has to come from the same variant the environment +## will actually pull. Getting this wrong would offer a PHP version that exists for plain php-fpm +## but was never built for, say, the magento2 variant. +function imageForService() { + local slug="$1" + local image="" + + image="$(imageCatalogField "${slug}" image)" + + if [[ "${image}" == "php-fpm" ]]; then + case "${ROLL_ENV_TYPE:-}" in + magento1|magento2|wordpress) image="php-fpm-${ROLL_ENV_TYPE}" ;; + esac + fi + + echo "${image}" + return 0 +} + +## Split ${ROLL_IMAGE_REPOSITORY} into its registry host and namespace path. A repository without a +## host component (a bare "epartment/roll") is a Docker Hub reference, which the v2 API serves from +## registry-1.docker.io under the library/ namespace for single-segment names. +function imageRegistryHost() { + local repository="${ROLL_IMAGE_REPOSITORY:-ghcr.io/epartment/roll}" + local first="${repository%%/*}" + + ## A host has a dot, a colon, or is exactly "localhost"; anything else is a Docker Hub namespace + if [[ "${first}" == *.* || "${first}" == *:* || "${first}" == "localhost" ]]; then + echo "${first}" + else + echo "registry-1.docker.io" + fi + + return 0 +} + +function imageRegistryPath() { + local repository="${ROLL_IMAGE_REPOSITORY:-ghcr.io/epartment/roll}" + local first="${repository%%/*}" + + if [[ "${first}" == *.* || "${first}" == *:* || "${first}" == "localhost" ]]; then + echo "${repository#*/}" + else + echo "${repository}" + fi + + return 0 +} + +## Pull the "tags" array out of a v2 tags/list response without needing jq. The body is a single +## line, so cutting at the key and then at the closing bracket is enough; an empty repository +## answers `"tags":null`, which yields nothing. +function parseRegistryTags() { + local body="$1" + + case "${body}" in + *'"tags"'*) ;; + *) return 0 ;; + esac + + printf '%s' "${body}" \ + | sed -e 's/.*"tags"[[:space:]]*:[[:space:]]*\[//' -e 's/\].*//' \ + | tr ',' '\n' \ + | tr -d '"' \ + | sed -e 's/[[:space:]]//g' \ + | grep -v '^$' || true + + return 0 +} + +## fetchRegistryTags +## +## Every tag of ${ROLL_IMAGE_REPOSITORY}/, one per line, newest-first ordering NOT applied. +## Cached for ROLL_TAG_CACHE_TTL minutes. Returns 1 - with nothing on stdout - when the registry +## could not be read, so a caller can fall back to asking for the version instead of pretending the +## image has no tags. +## +## Anonymous pulls are authenticated in two steps: the unauthenticated request answers 401 with a +## WWW-Authenticate header naming a token endpoint, and the token it hands back authorises the +## retry. Reading the realm from the header rather than hardcoding ghcr.io's keeps this working for +## any v2 registry a ROLL_IMAGE_REPOSITORY override might point at. +function fetchRegistryTags() { + local image="$1" + local host="" path="" url="" cache_dir="" cache_file="" headers="" body="" + local challenge="" realm="" service="" scope="" token="" status=0 + + host="$(imageRegistryHost)" + path="$(imageRegistryPath)" + + cache_dir="${ROLL_HOME_DIR:-${HOME}/.roll}/tmp/image-tags" + cache_file="${cache_dir}/$(printf '%s' "${host}/${path}/${image}" | tr -c '[:alnum:]._-' '_')" + + ## `find -mmin` is the one file-age test that behaves identically on BSD and GNU find + if [[ -s "${cache_file}" ]] && [[ "${ROLL_TAG_CACHE_TTL}" != "0" ]] \ + && [[ -z "$(find "${cache_file}" -mmin +"${ROLL_TAG_CACHE_TTL}" 2>/dev/null)" ]] + then + cat "${cache_file}" + return 0 + fi + + if ! command -v curl >/dev/null 2>&1; then + error "Reading available versions needs \`curl\`, which is not installed." + return 1 + fi + + url="https://${host}/v2/${path}/${image}/tags/list?n=1000" + headers="$(mktemp "${TMPDIR:-/tmp}/roll-tags.XXXXXX")" + + body="$(curl -sS -m 15 -D "${headers}" "${url}" 2>/dev/null)" || status=$? + + if [[ ${status} -ne 0 ]]; then + rm -f "${headers}" + error "Could not reach ${host} to list versions of ${path}/${image}." + return 1 + fi + + ## 401 is the normal first answer for an anonymous pull; take the challenge and come back + challenge="$(grep -i '^www-authenticate:' "${headers}" | head -n1)" || true + rm -f "${headers}" + + if [[ -n "${challenge}" ]]; then + realm="$(printf '%s' "${challenge}" | sed -n 's/.*realm="\([^"]*\)".*/\1/p')" + service="$(printf '%s' "${challenge}" | sed -n 's/.*service="\([^"]*\)".*/\1/p')" + scope="$(printf '%s' "${challenge}" | sed -n 's/.*scope="\([^"]*\)".*/\1/p')" + [[ -z "${scope}" ]] && scope="repository:${path}/${image}:pull" + + if [[ -n "${realm}" ]]; then + token="$(curl -sS -m 15 --get \ + --data-urlencode "service=${service}" \ + --data-urlencode "scope=${scope}" \ + "${realm}" 2>/dev/null \ + | sed -n 's/.*"token"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p')" || true + fi + + if [[ -z "${token}" ]]; then + error "Could not obtain a pull token for ${path}/${image} from ${host}." + return 1 + fi + + body="$(curl -sS -m 15 -H "Authorization: Bearer ${token}" "${url}" 2>/dev/null)" || status=$? + if [[ ${status} -ne 0 ]]; then + error "Could not reach ${host} to list versions of ${path}/${image}." + return 1 + fi + fi + + local tags="" + tags="$(parseRegistryTags "${body}")" + + if [[ -z "${tags}" ]]; then + error "${host}/${path}/${image} reported no tags." + return 1 + fi + + mkdir -p "${cache_dir}" 2>/dev/null || true + printf '%s\n' "${tags}" > "${cache_file}" 2>/dev/null || true + + printf '%s\n' "${tags}" + return 0 +} + +## Newest first, on stdin. Field-wise numeric sort rather than `sort -V`: BSD and GNU sort agree on +## this form, and it puts a non-numeric tag such as `latest` last instead of first. +function sortVersionsDesc() { + sort -t. -k1,1nr -k2,2nr -k3,3nr -k4,4nr -u + return 0 +} + +## The version tags of an image: the build-cache and floating tags dropped, and for php-fpm the +## -nodeNN suffixes stripped, since the suffix is NODE_VERSION's business and not PHP's. +function availableServiceVersions() { + local slug="$1" + local image="" tags="" + + image="$(imageForService "${slug}")" + tags="$(fetchRegistryTags "${image}")" || return 1 + + printf '%s\n' "${tags}" \ + | grep -v -- '-buildcache$' \ + | sed -e 's/-node[0-9x]*$//' \ + | grep -E '^[0-9]+(\.[0-9]+)*$' \ + | sortVersionsDesc + + return 0 +} + +## The Node versions built into a given PHP image tag, newest first, plus 0 for "no Node at all" - +## which is a real choice: NODE_VERSION=0 makes postProcessConfig leave the -nodeNN suffix off the +## image tag entirely. +function availableNodeVersions() { + local php_version="$1" + local image="" tags="" + + image="$(imageForService node)" + tags="$(fetchRegistryTags "${image}")" || return 1 + + printf '%s\n' "${tags}" \ + | grep -v -- '-buildcache$' \ + | grep -E "^${php_version}-node[0-9]+$" \ + | sed -e 's/.*-node//' \ + | sort -nr -u + + echo "0" + return 0 +} + +## The version this project pins for every catalogued service, and whether that service actually +## runs. Rendered here rather than in the command because it is a view of the catalog, and because +## the version picker shows the same table before asking - a command re-invoking `roll` for it +## would render whatever roll happens to be first on PATH instead of the running one. +## +## Needs loadRollConfig to have run, so the values are the project's own. +function showServiceVersions() { + local slug="" version_key="" current="" toggle="" status="" distribution="" + + distribution="$(getConfig DB_DISTRIBUTION mariadb)" + + printf " %-14s %-26s %-12s %s\n" "SERVICE" "KEY" "VERSION" "STATUS" + + for slug in $(imageServiceSlugs); do + version_key="$(versionKeyForService "${slug}")" + current="$(getConfig "${version_key}" "")" + toggle="$(imageCatalogField "${slug}" toggle)" + status="enabled" + + if [[ -n "${toggle}" ]] && [[ "$(getConfig "${toggle}" 0)" != "1" ]]; then + status="disabled (${toggle}=0)" + elif [[ -z "${toggle}" && "${ROLL_ENV_TYPE:-}" == "local" ]]; then + status="unused (local env type)" + fi + + ## Both database slugs read the same key, but only the selected distribution runs, so + ## showing the version against the other one would misreport what the project uses + if [[ "${slug}" == "mysql" || "${slug}" == "mariadb" ]] && [[ "${slug}" != "${distribution}" ]]; then + status="not selected (DB_DISTRIBUTION=${distribution})" + current="" + fi + + printf " %-14s %-26s %-12s %s\n" \ + "${slug}" "${version_key}" "${current:-}" "${status}" + done + + return 0 +} diff --git a/version b/version index 8adc70f..c18d72b 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.8.0 \ No newline at end of file +0.8.1 \ No newline at end of file