diff --git a/.github/scripts/smoke.sh b/.github/scripts/smoke.sh new file mode 100755 index 0000000..11cc4af --- /dev/null +++ b/.github/scripts/smoke.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# Docker-free smoke test for the `roll` CLI. +# +# Exercises commands that need neither a running Docker daemon nor an existing project checkout, +# so platform-specific regressions (bash 3.2 syntax, BSD vs GNU tool differences) surface without +# needing the full Docker-backed environment stack. Run from CI (macos-latest + ubuntu-latest) or +# locally. +# +# Usage: smoke.sh [bash32] +# (no argument) invoke roll natively, i.e. via its own #!/usr/bin/env bash shebang +# bash32 invoke roll explicitly under /bin/bash, to exercise bash 3.2.57 on macOS rather +# than whatever newer bash happens to be first on PATH + +set -eu + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +ROLL_DIR="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)" + +MODE="${1:-native}" + +case "${MODE}" in + bash32) + ROLL_BIN=(/bin/bash "${ROLL_DIR}/bin/roll") + ;; + native) + ROLL_BIN=("${ROLL_DIR}/bin/roll") + ;; + *) + >&2 echo "Usage: $0 [bash32]" + exit 1 + ;; +esac + +run() { + echo "+ ${ROLL_BIN[*]} $*" + "${ROLL_BIN[@]}" "$@" +} + +run version +run config schema >/dev/null +run registry validate +run registry categories >/dev/null +run registry paths >/dev/null + +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/roll-smoke.XXXXXX")" +cleanup() { + rm -rf -- "${TMP_DIR}" +} +trap cleanup EXIT + +( + cd -- "${TMP_DIR}" + "${ROLL_BIN[@]}" env-init smoketest magento2 + "${ROLL_BIN[@]}" config validate +) + +"$(dirname "$0")/test-syntax.sh" +"$(dirname "$0")/test-interact.sh" < /dev/null + +echo "Smoke test passed (${MODE} mode)." diff --git a/.github/scripts/test-interact.sh b/.github/scripts/test-interact.sh new file mode 100755 index 0000000..50832d9 --- /dev/null +++ b/.github/scripts/test-interact.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Asserts the non-interactive contract of utils/interact.sh: a value supplied by flag or env wins +# without ever reaching gum, and a prompt that cannot be answered fails naming the flag that would +# have answered it. This is the half of the interaction layer that automation depends on, and the +# half a TTY-less CI run can actually verify. +set -u +ROLL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +export ROLL_DIR +. "$ROLL_DIR/utils/core.sh" +. "$ROLL_DIR/utils/interact.sh" + +pass=0; fail=0 +check() { if [[ "$2" == "$3" ]]; then echo " PASS $1"; pass=$((pass+1)); else echo " FAIL $1 (got '$2', want '$3')"; fail=$((fail+1)); fi; } + +echo "== isInteractive with stdin from /dev/null ==" +if isInteractive; then r=interactive; else r=non-interactive; fi +check "reports non-interactive" "$r" "non-interactive" + +echo "== preset value short-circuits, gum never invoked ==" +# Remove gum from PATH entirely: if the wrapper touches gum at all, this explodes. +ORIG_PATH="$PATH" +export PATH=/usr/bin:/bin +MYVAR="already-set" +out=$(promptInput MYVAR "--name" "Name:" 2>&1); st=$? +check "promptInput exit status" "$st" "0" +check "promptInput left value intact" "$MYVAR" "already-set" + +CHOICE="magento2" +promptChoose CHOICE "--type" "Pick a type" php laravel magento2 >/dev/null 2>&1; st=$? +check "promptChoose exit status" "$st" "0" +check "promptChoose left value intact" "$CHOICE" "magento2" + +PW="hunter2" +promptPassword PW "--encrypt=" "Password:" >/dev/null 2>&1; st=$? +check "promptPassword exit status" "$st" "0" +check "promptPassword left value intact" "$PW" "hunter2" +export PATH="$ORIG_PATH" + +echo "== no preset + no tty = hard error naming the flag ==" +UNSET_A="" +out=$( (promptInput UNSET_A "--name " "Name:") 2>&1 ); st=$? +check "promptInput exits 1" "$st" "1" +case "$out" in *"--name "*) r=yes;; *) r=no;; esac +check "promptInput names the flag" "$r" "yes" + +UNSET_B="" +out=$( (promptChoose UNSET_B "--type " "Pick a type" php laravel) 2>&1 ); st=$? +check "promptChoose exits 1" "$st" "1" +case "$out" in *"--type "*) r=yes;; *) r=no;; esac +check "promptChoose names the flag" "$r" "yes" + +out=$( (promptConfirm "--force" "Overwrite?") 2>&1 ); st=$? +check "promptConfirm exits 1" "$st" "1" +case "$out" in *"--force"*) r=yes;; *) r=no;; esac +check "promptConfirm names the flag" "$r" "yes" + +UNSET_C="" +out=$( (promptPassword UNSET_C "--encrypt=" "Password:") 2>&1 ); st=$? +check "promptPassword exits 1" "$st" "1" +case "$out" in *"--encrypt="*) r=yes;; *) r=no;; esac +check "promptPassword names the flag" "$r" "yes" + +echo "== boxes fall back to plain renderer with no tty ==" +out=$(styledBox 2 "hello world" 2>&1) +case "$out" in *"| hello world |"*) r=plain;; *) r=other;; esac +check "styledBox uses plain box" "$r" "plain" + +echo "== survives set -e ==" +( set -e; styledBox 7 "x" >/dev/null; MYV=preset; promptInput MYV "--x" "X:" >/dev/null; echo ok ) >/dev/null 2>&1 +check "no premature exit under set -e" "$?" "0" + +echo "" +echo "passed=$pass failed=$fail" +[[ $fail -eq 0 ]] diff --git a/.github/scripts/test-syntax.sh b/.github/scripts/test-syntax.sh new file mode 100755 index 0000000..406bd5a --- /dev/null +++ b/.github/scripts/test-syntax.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Parse every command, help and util file with the running bash. +# +# Worth its own step because two classes of defect are invisible to both ShellCheck and Ubuntu CI: +# +# 1. bash 3.2 mis-parses an apostrophe inside a heredoc nested in $( ), even when the heredoc +# delimiter is quoted. backup.help and status.help both shipped broken this way - `roll backup +# --help` and `roll status --help` exited 2 on macOS - while parsing cleanly under bash 5. +# 2. a .cmd that re-invokes `roll --help` recurses forever, because commands on +# ROLL_CMD_ANYARGS receive --help directly instead of roll rendering it. restore, restore-full +# and duplicate each did this, forking until killed. +# +# Run this under the system bash on macOS (bash 3.2) for the first check to mean anything. +set -eu + +ROLL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "${ROLL_DIR}" + +failed=0 + +echo "bash ${BASH_VERSION} syntax check" +for file in bin/roll commands/*.cmd commands/*/*.cmd commands/*.help commands/*/*.help utils/*.sh; do + [[ -e "${file}" ]] || continue + if ! bash -n "${file}" 2>/dev/null; then + echo " FAIL ${file}" + bash -n "${file}" 2>&1 | sed 's/^/ /' + failed=$((failed + 1)) + fi +done + +echo "self-referential --help check" +for file in commands/*.cmd commands/*/*.cmd; do + [[ -e "${file}" ]] || continue + name="${file##*/}" + name="${name%.cmd}" + ## Only commands on ROLL_CMD_ANYARGS receive --help themselves; for everything else bin/roll + ## renders the help and the inner call terminates, so it is not a defect there. + case " $(sed -n 's/.*ROLL_CMD_ANYARGS=(\(.*\)).*/\1/p' bin/roll) " in + *" ${name} "*) ;; + *) continue ;; + esac + if grep -qE "^[[:space:]]*roll[[:space:]]+${name}[[:space:]]+(--help|-h)" "${file}"; then + echo " FAIL ${file} re-invokes 'roll ${name} --help', which recurses forever" + echo " source \"\${ROLL_DIR}/commands/usage.cmd\" instead" + failed=$((failed + 1)) + fi +done + +if [[ ${failed} -ne 0 ]]; then + echo "${failed} file(s) failed." + exit 1 +fi + +echo "All files parse and no command re-invokes its own help." diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 5c10f86..c0cafb6 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -2,23 +2,70 @@ name: ShellCheck on: push: paths: - - 'commands/*.cmd' + - 'bin/roll' + - 'commands/**/*.cmd' + - 'commands/**/*.help' - 'utils/*.sh' + - 'environments/**' + - '.github/scripts/**' + - '.shellcheckrc' - '.github/workflows/shellcheck.yml' pull_request: paths: - - 'commands/*.cmd' + - 'bin/roll' + - 'commands/**/*.cmd' + - 'commands/**/*.help' - 'utils/*.sh' + - 'environments/**' + - '.github/scripts/**' + - '.shellcheckrc' - '.github/workflows/shellcheck.yml' workflow_dispatch: jobs: shellcheck: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Install shellcheck + + - name: Install shellcheck (macOS) + if: runner.os == 'macOS' + run: brew install shellcheck + + - name: Install shellcheck (Linux) + if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y shellcheck + - name: Run shellcheck - run: shellcheck commands/*.cmd utils/*.sh + run: shellcheck commands/*.cmd commands/*/*.cmd commands/*.help commands/*/*.help utils/*.sh .github/scripts/*.sh + + # bin/roll requires a `docker` CLI on PATH (it runs `docker compose version` before + # dispatching to any subcommand, including Docker-free ones) but none of the smoke commands + # below touch a daemon. ubuntu-latest runners ship Docker already; macos-latest runners ship + # neither Docker nor GNU gettext (`envsubst`, needed by `env-init` for env types that carry + # an init.env), so install both as client-only tooling. + - name: Install Docker CLI + Compose plugin + gettext (macOS) + if: runner.os == 'macOS' + run: | + brew install docker docker-compose gettext + brew link --force gettext + mkdir -p "${HOME}/.docker" + cat > "${HOME}/.docker/config.json" <` commits and version bumps are omitted, as are bullets +that only restate the release note above them. + +## Unreleased + +The 0.8.0 development work. Two changes need reading before upgrading — unpinned service versions +now warn (and become an error in 0.9.0), and gum becomes a dependency. Both are covered in +[RELEASE-NOTES-0.8.0.md](RELEASE-NOTES-0.8.0.md). + +### Added + +- `roll env doctor` — seventeen checks over Docker, containers, ports, search engine and disk, with + `--format json` and an exit code you can gate on. +- `--format json` on `roll status`, `roll env describe`, `roll registry list` and `roll env doctor`. + No ANSI, no credentials, values escaped. +- `roll has-command ` — exit 0/1, no output, for feature detection in scripts. +- `roll env up --wait`, backed by new healthchecks for db, redis, elasticsearch, opensearch, + rabbitmq, varnish and nginx. The search engines are probed through `/_cluster/health` rather than a + port, because those containers can hold a port open while the cluster inside them is dead. +- `roll env sh ''` — runs through `sh -c` inside the container, so redirects and + pipes apply there rather than on the host. +- `roll config check-pins` and `roll config fix-pins` for the version-pin migration. +- `roll myloader --normalize-source-ddl` — strips the `ENCRYPTION` table option and remaps + `utf8mb4_0900_*` collations so a MySQL 8 dump loads into MariaDB. Without the flag, a failed run + now names the cause instead of reporting `Trace/breakpoint trap`. +- `ROLL_PUBLISH_PORTS=0` runs an environment without publishing host ports, for hosts running many + environments at once. `ROLL_BROWSERSYNC=1` publishing ports on php-fpm was a known cause of + "port is already allocated" that no `.roll/roll-env.yml` override could fix. +- `ELASTICSEARCH_JAVA_OPTS` / `OPENSEARCH_JAVA_OPTS` replace the hardcoded 512MB search-engine heap. +- `ROLL_ENV_INIT_FORCE=1` lets `roll env-init` overwrite an existing `.env.roll` non-interactively. +- `copyfromcontainer`, `copytocontainer`, `magento2/theme` and `convert` moved in from the internal + command pack, each with its outstanding bugs fixed in transit: `copytocontainer --all` was + unreachable and `copytocontainer ` copied the whole project root into the destination, + `copyfromcontainer --realpath` tested a container path against the host filesystem and built a + destination whose parent never existed, theme discovery was relative to the working directory and + only ever examined the first theme, the Gulp-or-Yarn choice is now made per theme so mixed + projects build correctly, and `convert` overwrote `ELASTICSEARCH_VERSION` with a hardcoded value + and wrote a `ROLL_NO_STATIC_CACHING` key no schema defines — it now translates that legacy + negative-form value into `ROLL_MAGENTO_STATIC_CACHING`. +- `roll registry list` now carries real descriptions and categories, from `@description:` and + `@category:` headers in the help files. +- Documentation: [driving RollDev from a script](docs/machine-interface.md), service version pins, + unattended operation, and the `doctor` command. + +### Changed + +- **Unpinned service versions now warn**, and will be an error in 0.9.0. The fallback is the version + the project was already running, so nothing changes behaviour today. A version set in global config + (`~/.roll/.env`) counts as unpinned: it is not in the repository, so a colleague without that line + still resolves a different image. `fix-pins` writes the inherited value in that case, not + RollDev's built-in default. +- **gum is a dependency** for interactive prompts. Every prompt is also reachable by flag, + environment variable or positional argument, so scripted use works without it, and `roll install` + warns rather than failing when it is missing. +- Every prompt now resolves flag/env first, uses gum only on a terminal, and otherwise fails naming + the flag that would have answered it — an unattended run can no longer hang on a prompt. +- `roll env-init` validates the environment name against Compose's project-name rules at creation, + instead of letting it fail later. +- `roll tableplus` is now a RollDev command in its own right, with Setapp-aware discovery and a + clear message on Linux rather than an obscure failure. +- `dialog` is no longer used anywhere. + +### Fixed + +- **`magento2` environments came up without Varnish, a search engine or RabbitMQ** unless the + toggles were spelled out. The environment-type defaults ran after the schema defaults had already + filled those variables in, so every `${VAR:-1}` deriving them was unreachable. +- **A restored volume came back owned by root, so Elasticsearch could not start.** `roll restore` + creates each volume with `docker volume create` and then extracts into it as root with + `--strip-components=1`, which discards the archive's top-level entry — the only one carrying the + data directory's own ownership. Docker's copy-up did not compensate, because it fires only for an + empty volume and the restore populates this one before any service mounts it. Services that start + as root chown their data directory themselves; Elasticsearch and OpenSearch run as uid 1000 and + died at boot with `AccessDeniedException: .../data/.es_temp_file` while the restore reported + success. The owner is now derived from the restored content after each extraction. +- `roll db` works against MariaDB 11, which ships only `mariadb-dump`/`mariadb`. +- `roll registry categories` no longer dies on macOS — bash 4 syntax under a bash 3.2 shell. +- `roll backup --help` and `roll status --help` no longer exit 2 on macOS. bash 3.2 mis-parses a lone + apostrophe inside a heredoc nested in `$( )`; Ubuntu-only CI could not see it. +- `roll db/env/redis/svc/restore/restore-full/duplicate --help` no longer fork until killed. Those + commands take arbitrary flags, so roll hands `--help` to them directly, and each re-invoked itself. +- `roll redis --help` was unreachable rather than recursive, and now renders. +- `roll restore` with no backups reports it instead of dying on an internal error. +- `roll env up` no longer does a redundant compose pass every time: the network-existence check was + single-quoted, so the command substitution inside it never ran and the filter never matched. +- `roll env describe` back to ~1s from ~9s, and it no longer builds container names by string + concatenation, which broke whenever Compose named them differently. Same fix in `roll vnc`. +- `roll fixowns` and `roll fixperms` accept more than one path. +- `isOnline` no longer always reports offline on Linux — `ping -t` is a timeout on BSD and a TTL on + GNU — so `roll svc up` refreshes images there. +- WSL environments get the `.linux.yml` compose fragments. +- The Portainer and Startpage service defaults are read from configuration rather than from literals + that disagreed with the schema. +- An unsupported Linux distribution now warns with the CA path instead of silently skipping the + trust-store step during `roll install`. +- `roll env-init x vuejs` produces a valid project; the `vuejs` and `local` types had no `init.env`. +- `box` no longer aborts under `set -u` on bash 4.4 and later: it declared its width variable + without initialising it, and those versions apply `set -u` inside arithmetic contexts where bash + 3.2 substitutes 0. Only reachable from a caller that sets `-u`, which is why it surfaced in the + prompt harness rather than in `roll` itself. + +### Security + +- The TablePlus connection URI, which carries the database password, is no longer passed as a + process argument where anything able to read `ps` could see it. +- The ssh-agent socket is `chmod 600` and owned by `www-data`, rather than `chmod 777`, and the + fixup only runs when there is an agent socket to fix. +- Traefik's docker socket is mounted read-only. +- Six `eval "$(cat ~/.roll/.env …)"` sites are replaced with schema-validated config loading. +- `--format json` output carries no credentials, and values are escaped rather than concatenated. + +### Internal + +- ShellCheck runs on macOS as well as Ubuntu, over `commands/**`, the help files, `utils/` and the + CI scripts, and passes on both. The Ubuntu leg had been red for most of the cycle: Ubuntu ships + ShellCheck 0.9.0, which reports four findings (SC2002, SC2015, SC2236) that the 0.11.0 on macOS + no longer raises, so a green local run said nothing about CI. Those four are fixed in code rather + than silenced in `.shellcheckrc`. +- A smoke suite runs on both platforms: the Docker-free command set, a prompt-contract harness, and + a parse check that catches the bash 3.2 heredoc trap and any command re-invoking its own help. +- New shared libraries: `utils/interact.sh` (prompts and styled output), `utils/backup.sh` (the + backup/restore/duplicate primitives, replacing eleven byte-identical copies) and + `utils/magento2-init.sh` (re-runnable install phases). `restore-full` is now `restore + --include-source` with the old name kept as an alias, so existing scripts keep working; those two + files drop from 2 087 lines to 625. + +### Known issue + +- A restored Elasticsearch volume comes back unwritable — the volume root is owned by uid 0 while the + image runs as uid 1000 — so the service will not start after `roll restore`. The database restores + correctly. This predates the release; it is newly visible because `env up --wait` and the + healthchecks now catch it, where `env up` previously returned success. + +## [0.7.2](https://github.com/epartment/rolldev/releases/tag/0.7.2) — 2026-08-31 + +> - Fixes for Linux CMD + +- Linux cmd fixes and added feature request list + +## [0.7.1](https://github.com/epartment/rolldev/releases/tag/0.7.1) — 2026-08-28 + +> - Generalized the backup command by adding a directory path argument + +- Address review feedback on the backup output flags +- Fix infinite recursion in roll backup --help +- Add --output-dir, --archive-name and --keep-dir to roll backup +- Made some fixes for the restore commands related to the search containers + +## [0.7.0.5](https://github.com/epartment/rolldev/releases/tag/0.7.0.5) — 2026-07-02 + +> - Made changes to how params are passed to myloader in the wrapper command + +## [0.7.0.4](https://github.com/epartment/rolldev/releases/tag/0.7.0.4) — 2026-06-26 + +> - My dumper fixes + +- Added epartment-init to allow any argument for mydumper support +- Fixes for my dumper + +## [0.7.0.3](https://github.com/epartment/rolldev/releases/tag/0.7.0.3) — 2026-06-04 + +> - Added support for mydumper and myloader + +- Added a CLAUDE.md + +## [0.7.0.2](https://github.com/epartment/rolldev/releases/tag/0.7.0.2) — 2026-04-29 + +> Changed fallback NGINX version to one thats being build + +- Updated the default NGINX version to a supported version (1.27) + +## [0.7.0.1](https://github.com/epartment/rolldev/releases/tag/0.7.0.1) — 2026-04-21 + +> Changed an image repo path to epartment repo + +## [0.7.0.0](https://github.com/epartment/rolldev/releases/tag/0.7.0.0) — 2026-04-20 + +> Updated the Epartment roll with the latest changes from dockergiant/rolldev + +- Updated README.md +- update: simplify archive extraction for cross-platform compatibility in restore commands +- update: improve logging in restore/restore-full commands for better process visibility +- add: enhance Magento multi-store support with extended docs, `multistore` usage details, and example setups +- update: refine Traefik rules for host matching v3 and add Traefik configuration regeneration in restore commands v3 +- add: automated SSL certificate signing for environment domains in restore and restore-full commands +- update: remove Traefik version pinning in docker-compose and fix indentation in svc.cmd for traefik v3 +- add: verbose logging support for restore and restore-full commands + fix issue with restoring full backup for first time +- fix: handle help flag to properly support commands with arguments +- add: describe subcommand to env.cmd and enhance multistore documentation with improved clarity and structure +- add: describe command to display RollDev environment details, including service status, URLs, and configurations +- add: multistore command for managing Magento multi-store configurations with automated config generation and SSL support +- update: refine New Relic PHP agent documentation and add environment configuration support +- add: documentation for enabling New Relic monitoring with configuration examples and troubleshooting steps +- remove: Blackfire profiling documentation from roll-docker-stack +- add: New Relic monitoring configuration to all environment init files +- fix: handle empty projectNetworkList in status.cmd to avoid errors +- Update README +- update: switch elasticvue to official image and adjust configuration +- fix: resolve Magento 2.4.8+ 2FA configuration bug in magento2-init +- add comprehensive README to roll-docker-stack with installation steps, feature overview, supported environments, and contribution guide +- add RedisInsight support to roll-docker-stack with configuration, environment updates, and service definition +- add extensive documentation for magento2-init command, including a quick reference guide, usage examples, compatibility matrix, and troubleshooting steps +- add magento2-init command for project initialization with dynamic configuration and compatibility checks +- add magento2-init command, improve usage loading with dynamic paths, and enhance global/environment-specific config handling +- add restore-full command to ROLL_CMD_ANYARGS list +- improve backup ID extraction logic to handle warnings in command output +- use passphrase-fd for gpg decryption to handle password escaping and add support for multiple archive formats +- remove redundant exclude patterns from backup command +- Fix restore-full argument handling +- Require explicit archive path for restore-full +- Fix last network lookup in status command +- Fix status command for macOS compatibility +- Show running services in status +- docs: add decrypt examples for restore-full +- add duplication docs +- fixed issues with backup and restoring + full duplication command working +- Add shellcheck workflow +- Fix comment typo +- Fix mongodb env vars +- docs: fix typo in redis usage +- Fix parsing newline in status and correct install doc url +- Add environment duplication command +- fix in parameters in printf +- fixes in backup / restore commands +- fix encryption for backups +- fix issue with user permissions +- update backup and restore commands docs +- update backup and restore commands +- update to new workflows +- added docs for new registry system +- Added new system for better command registry and auto discovery on many places +- Added help files of version and install commands +- Refactored config loading with more dynamic loading of config files +- Enable manually run build documentation in github actions + +## [0.6.3.1](https://github.com/epartment/rolldev/releases/tag/0.6.3.1) — 2026-02-23 + +> - Fixed a bash error handling issue with the code change + +## [0.6.3](https://github.com/epartment/rolldev/releases/tag/0.6.3) — 2026-02-23 + +> - Added a check to make sure tput is available before setting its values to prevent error when run by cron + +## [0.6.2](https://github.com/epartment/rolldev/releases/tag/0.6.2) — 2025-08-14 + +- remove xdebug mode env +- enable trace and profiler + +## [0.6.1](https://github.com/epartment/rolldev/releases/tag/0.6.1) — 2025-07-03 + +- Updated base versions of the packages + +## [0.6.0](https://github.com/epartment/rolldev/releases/tag/0.6.0) — 2025-07-03 + +- Updated actions +- Version 0.6.0 for moving images to ghcr.io/epartment/roll +- opensearch opts +- Updated default versions of packages for new Magento and Laravel projects + +## [0.4.3](https://github.com/epartment/rolldev/releases/tag/0.4.3) — 2024-12-16 + +- ssh-agent perms and host key mount +- fix compose +- fix: darwin ssh agent + +## [0.4.2](https://github.com/epartment/rolldev/releases/tag/0.4.2) — 2024-12-03 + +- fix compose + +## [0.4.1](https://github.com/epartment/rolldev/releases/tag/0.4.1) — 2024-12-02 + +_No changes recorded beyond the tag itself._ + +## [0.4.0](https://github.com/epartment/rolldev/releases/tag/0.4.0) — 2024-12-02 + +> Made improvements in the debug agent and known_host forwarding + +- fix: darwin ssh agent +- set correct xdebug modes + +## [0.3.0](https://github.com/epartment/rolldev/releases/tag/0.3.0) — 2024-09-19 + +> Added support for forwarding SSH agents to Roll Shell + +- Added support for forwarding SSH agent to roll shell + +## [0.2.6.6](https://github.com/epartment/rolldev/releases/tag/0.2.6.6) — 2024-08-26 + +- Replace dockergiant with epartment +- Update svc.cmd +- Update env.cmd +- Renamed dockergiant references to Epartment + +## [0.2.6.5](https://github.com/epartment/rolldev/releases/tag/0.2.6.5) — 2024-05-22 *(tag only, no GitHub release)* + +- correct permissions when restoring a build + +## [0.2.6.4](https://github.com/epartment/rolldev/releases/tag/0.2.6.4) — 2024-05-06 *(tag only, no GitHub release)* + +- Remove version: from docker-compose.yml: https://github.com/docker/compose/issues/11628 + +## [0.2.6.3](https://github.com/epartment/rolldev/releases/tag/0.2.6.3) — 2024-04-25 *(tag only, no GitHub release)* + +- Remove version: from docker-compose.yml: https://github.com/docker/compose/issues/11628 +- minor bugfix asking for tty when its not available +- removed unused variables + +## [0.2.6.2](https://github.com/epartment/rolldev/releases/tag/0.2.6.2) — 2024-04-18 *(tag only, no GitHub release)* + +- set id_field_data to enabled which is disabled by default from Elasticsearch 8 + +## [0.2.6.1](https://github.com/epartment/rolldev/releases/tag/0.2.6.1) — 2024-04-10 *(tag only, no GitHub release)* + +- Refactored the auto login script with checks if 2fa module is installed or not + +## [0.2.6](https://github.com/epartment/rolldev/releases/tag/0.2.6) — 2024-04-09 *(tag only, no GitHub release)* + +- Fix auto login command +- Added magento 2 auto admin login feature +- Refactor init env for akeneo environment type +- Add Akeneo environment type to docs. +- Fix python TFA SECRET generating in docs + +## [0.2.4](https://github.com/epartment/rolldev/releases/tag/0.2.4) — 2024-03-27 *(tag only, no GitHub release)* + +- Added MongoDB +- fix magento2 admin user creation script in documentation + +## [0.2.3](https://github.com/epartment/rolldev/releases/tag/0.2.3) — 2024-03-06 *(tag only, no GitHub release)* + +- add doc for setup varnish cache +- Fix online check command +- Make browsersync command work with ssl in other projects types +- Fix php config mapping in documentation example +- Move browsersync command from magento2 to global comamnds + +## [0.2.2](https://github.com/epartment/rolldev/releases/tag/0.2.2) — 2023-10-12 *(tag only, no GitHub release)* + +- Fixed issue Unsupported flag on extra parameters on grunt and magento command + +## [0.2.1](https://github.com/epartment/rolldev/releases/tag/0.2.1) — 2023-09-12 *(tag only, no GitHub release)* + +- Online network events only when active network and custom php ext + +## [0.2.0](https://github.com/epartment/rolldev/releases/tag/0.2.0) — 2023-08-31 *(tag only, no GitHub release)* + +- change to mailpit + +## [0.1.1-beta13](https://github.com/epartment/rolldev/releases/tag/0.1.1-beta13) — 2023-07-18 *(tag only, no GitHub release)* + +_No changes recorded beyond the tag itself._ + +## [0.1.0-beta14](https://github.com/epartment/rolldev/releases/tag/0.1.0-beta14) — 2023-08-29 *(tag only, no GitHub release)* + +- change to mailpit +- fix project url in status command + +## [0.1.0-beta13](https://github.com/epartment/rolldev/releases/tag/0.1.0-beta13) — 2023-07-18 *(tag only, no GitHub release)* + +- Add a status command to view all running projects +- Adding Mailhog SMTP configuration documentation +- Remove unavailable command for public +- fix startpage image + +## [0.1.0-beta12](https://github.com/epartment/rolldev/releases/tag/0.1.0-beta12) — 2023-05-24 *(tag only, no GitHub release)* + +- Add dragonfly db as drop-in replacement of redis + +## [0.1.0-beta11](https://github.com/epartment/rolldev/releases/tag/0.1.0-beta11) — 2023-04-24 *(tag only, no GitHub release)* + +- add backup and restore commands +- docs change + url change + +## [0.1.0-beta10](https://github.com/epartment/rolldev/releases/tag/0.1.0-beta10) — 2023-03-23 *(tag only, no GitHub release)* + +- Update tag-release.yml +- error when downing all containers after upgrade fix. + +## [0.1.0-beta9](https://github.com/epartment/rolldev/releases/tag/0.1.0-beta9) — 2023-03-22 *(tag only, no GitHub release)* + +- added wordpress cli command + usage +- added reclu env help files usage +- change some non existing files and updated usage files +- add magepack cmd +- Small typo fix +- added magepack environment +- removed split files from adobe commerce +- Change some init values for magento 1 and 2 +- Added reclu env type commands +- Changed .env to .env.roll +- Fix naming for images. +- fix typo +- Updated init versions +- Fix check installing ssh config for local user +- Moved env type commands to separate folder +- check for ssh config conflict +- Make Portainer and startpage optional +- moved images + added docs +- Cleanup apt files after build +- replace left over alpine builder with debian +- typo +- trigger php build +- update workflows +- fix mcrypt php8.2 +- Bump golang from 1.19-alpine to 1.20-alpine in /images/mailhog +- add akeneo env + +## [0.1.0-beta8](https://github.com/epartment/rolldev/releases/tag/0.1.0-beta8) — 2023-01-19 *(tag only, no GitHub release)* + +- Bump golang from 1.18-alpine to 1.19-alpine in /images/mailhog +- move repo to github +- release new version +- Temporarily switch buildkit, instability github packages. +- Install extra php ext on env up if ROLL_EXTRA_PHP_EXT is defined +- Add ImageMagick and GraphicsMagick +- Install extra php extensions when configured +- start from port 3005 +- fix typo +- Dynamic port allocation +- Added browsersync dynamic port option with cli info tool +- added include git env +- Fix gulp + grunt cli + add chrome-sandbox +- ADD chromium +- add PhantomJs +- add phantom +- Fix xdebug loading +- Add more caching +- Small fixes + xdebug hard path +- Added dnsmasq image to workflow +- Updated profiles +- Upgrade envs + seperated some processes +- fix +- Update some configs and github actions +- Fixed node binaries +- update workflows +- small fix in gitlab actions +- Fix not available packages debian +- Fix in github actions +- Temp disable image loading +- Reworked github actions + build pipeline +- Reworked github actions +- roll out php 7.2 +- increase query size + retrigger image build +- set default database charset to utf8 as of requirements for typo3 +- small fix for already enabled opcache extension +- redis 6.2 dev add +- use elasticsearch old +- fix images +- rebuild test +- add old mariadb version +- add testing data + images +- restore temp user 1000 +- update fpm image with user 1000 fix +- refactor project type specific commands & startpage +- fix missing crucial part of routing nginx traffic +- fix owning uid on linux wrong user uid + +## [0.1.0-beta7](https://github.com/epartment/rolldev/releases/tag/0.1.0-beta7) — 2022-11-03 *(tag only, no GitHub release)* + +- change .env to .env.roll +- add more verbose logging to ownership when starting container +- add legacy php7.1 + +## [0.1.0-beta6](https://github.com/epartment/rolldev/releases/tag/0.1.0-beta6) — 2022-10-24 *(tag only, no GitHub release)* + +- add placeholder for disabling static content on init +- add no static caching nginx conf magento 1 +- make nginx template and public path configurable +- add rick to usage section +- typo wordpress +- update ci new versions +- add wordpress type images +- add wordpress ci +- move yarn npm to main and add wordpress image +- small fix + auto update images +- add vuejs project type + other fixes +- add laravel en vuejs nginx configs +- trigger +- fix fixperms command and add clinotty command for argument whitelist +- change xdebug and typo3 env to local +- add option for disabling static content caching +- change context to Development +- make all certs available in fpm container +- add typo3 configuration +- update all to version 3.9 and fix issue var compare +- fix typo +- add npm and node commands +- change order of command execution +- restore caching on files magento 2 config +- fix new line for status command mutagen +- change xdebug configuration files +- fix build name +- add small start page for service domain +- add little progress to initial sync +- add grunt commands + fix param whitelist rootshell +- update elasticvue automatic resolver name +- update nginx config +- do not delete the production builded elasticvue files + +## [0.1.0-beta5](https://github.com/epartment/rolldev/releases/tag/0.1.0-beta5) — 2022-08-22 *(tag only, no GitHub release)* + +- new version +- add timeout for slow network building +- change elastichq to elasticvue + +## [0.1.0-beta4](https://github.com/epartment/rolldev/releases/tag/0.1.0-beta4) — 2022-08-22 *(tag only, no GitHub release)* + +- new version +- Update images + fix xdebug3 +- changes to xdebug3 and magento 2 not working +- fix xdebug images not routing to 9000 +- temp disable php8.2 because of xdebug 3 pickle bug +- change php compare function +- remove magerun older than php7.2 +- make mhsendmail multiarch +- fix php image and add restart command +- updated images +- refactor images and commands +- remove old elastic version +- update elastic images +- change xdebug images +- add corn +- change docker entry point +- add version +- change builds + +## [0.1.0-beta3](https://github.com/epartment/rolldev/releases/tag/0.1.0-beta3) — 2022-07-11 *(tag only, no GitHub release)* + +- tag version +- xdebug 2 for 7.4 or lower +- change fpm config path +- change default shell to bash +- change alpine to debian cli images +- change typo + +## [0.1.0-beta2](https://github.com/epartment/rolldev/releases/tag/0.1.0-beta2) — 2022-07-10 *(tag only, no GitHub release)* + +- change in action triggers +- added multiarch version + +## [0.1.0-beta1](https://github.com/epartment/rolldev/releases/tag/0.1.0-beta1) — 2022-07-10 *(tag only, no GitHub release)* + +- change version +- update tools +- add build tag to buildx +- trigger build +- initial commit diff --git a/CLAUDE.md b/CLAUDE.md index 4ac0179..900e9ca 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,12 +28,14 @@ itself, so it is not duplicated in the README. ## Before you start -- **Check the known defects before diagnosing anything.** Several surprising behaviours are already - documented with a verified root cause in - [Known issues & improvement points](README.md#known-issues--improvement-points) — a red ShellCheck - run, `roll db dump` failing, `roll registry categories` dying on macOS, magento2 environments coming - up without Varnish or a search engine, images never refreshing on Linux. Do not re-derive them, and - do not report one as a new discovery. +- **Check the known defects before diagnosing anything.** + [Known issues & improvement points](README.md#known-issues--improvement-points) documents what is + still open, with a verified root cause — currently one finding, a restored search-engine volume + coming back unwritable so Elasticsearch will not start. Do not re-derive it, and do not report it + as a new discovery. The defects that section used to list (`roll db dump`, `roll registry + categories` on macOS, magento2 coming up without Varnish or a search engine, images never + refreshing on Linux) were fixed for 0.8.0 — see [CHANGELOG.md](CHANGELOG.md) rather than expecting + them to still reproduce. - **Check [Audited and clean](README.md#audited-and-clean) before investigating something that looks wrong.** It records what was checked and found correct, including two things that read as bugs and are not. @@ -78,11 +80,38 @@ it, ask the user for the local directory** rather than guessing; there is no der There is no unit-test suite. What is available: -- **ShellCheck**, the only CI gate: `shellcheck commands/*.cmd utils/*.sh`. Note that it **already - fails** on a clean checkout (finding **H1** in the README), so a red run is not evidence that your - change broke something. Capture the finding count before your change and compare, or scope the run - to the files you touched. The gate also does not cover `commands/magento2/`, `commands/wordpress/` - or the `.help` files (finding **L9**), so lint those explicitly if you edit them. +- **ShellCheck**, run by CI over every shell file in the repository: + + ```bash + shellcheck commands/*.cmd commands/*/*.cmd commands/*.help commands/*/*.help utils/*.sh .github/scripts/*.sh + ``` + + It passes on a clean checkout, so **any** finding your change produces is yours to fix. `bin/roll` + itself is not in the gate. + + **A clean local run is not proof the gate is green.** The Ubuntu runner installs ShellCheck from + apt (0.9.0 on 24.04) while Homebrew ships 0.11.0, and the older version raises findings the newer + one dropped — SC2002, SC2015 and SC2236 among them — so the macOS leg can pass while Ubuntu fails + on code neither of you changed. When in doubt, reproduce the Ubuntu leg in a container: + + ```bash + docker run --rm -v "$PWD":/src:ro ubuntu:24.04 bash -c 'apt-get update -qq && apt-get install -y -qq shellcheck gettext-base docker.io docker-compose-v2 && cp -r /src /repo && cd /repo && shellcheck commands/*.cmd commands/*/*.cmd commands/*.help commands/*/*.help utils/*.sh .github/scripts/*.sh && .github/scripts/smoke.sh native' + ``` + + That also exercises bash 5, which differs from macOS's 3.2 in the other direction: bash 4.4 and + later apply `set -u` inside arithmetic contexts, so `((x < 1))` on an unset `x` aborts there while + 3.2 substitutes 0. +- **The smoke suite**, which CI runs on both platforms and which needs no Docker daemon and no + project checkout: + + ```bash + .github/scripts/smoke.sh bash32 # exercises /bin/bash 3.2 - use this one on macOS + .github/scripts/smoke.sh native + ``` + + It covers the Docker-free command set, `env-init` + `config validate` in a temporary directory, a + parse check that catches the bash 3.2 heredoc trap and any command re-invoking its own help, and a + prompt-contract harness. Anything touching `roll env` or service containers is still on the author. - **Running the CLI from source:** `./bin/roll `. On macOS invoke it as `/bin/bash ./bin/roll ` when checking Bash 3.2 compatibility, so the system bash is exercised rather than a newer one from `PATH`. diff --git a/FEATURE-REQUESTS.md b/FEATURE-REQUESTS.md index 82636b9..b213b1d 100644 --- a/FEATURE-REQUESTS.md +++ b/FEATURE-REQUESTS.md @@ -31,108 +31,63 @@ Adapted from the usual defect scale, since these are gaps rather than bugs: ### High -**H1. Service readiness gating — `env up` returns before services accept connections** — `commands/env.cmd:201` - -- *What:* No service definition in `environments/` declares a `healthcheck:` (verified — - `grep -rn "healthcheck" environments/` matches nothing), and `env up` does not pass docker - compose's `--wait`. `roll env up` therefore returns as soon as containers are *started*, not when - they are *usable*. -- *Why it matters:* Any command run straight after `env up` races the services it needs. The search - containers are the worst case because they take the longest to become ready: an application step - that follows immediately fails with `No alive nodes found in your cluster`, which reads like a - misconfiguration rather than a race. It is non-deterministic, and it gets worse under parallel - load, which is exactly when it is hardest to reproduce by hand. -- *Suggested fix:* Add `healthcheck:` blocks to the `db`, `opensearch`, `elasticsearch` and `redis` - partials in `environments/includes/`, then support `roll env up --wait`. Docker compose's native - `--wait` already implements the waiting; it just needs healthchecks defined to be meaningful, so - the healthchecks are the substantive part of this request. - -**H2. No supported way to run an environment with no published host ports** — `environments/includes/browsersync.base.yml:3-5` - -- *What:* When `ROLL_BROWSERSYNC=1`, that partial publishes `${BROWSERSYNC_PORT_WEB:-3000}` and - `${BROWSERSYNC_PORT_UI:-3001}` **on the `php-fpm` service**. It is the only file in - `environments/` that publishes host ports at all (verified — `grep -rl "ports:" environments/` - matches only this file). -- *Why it matters:* Two environments on one host with browsersync enabled collide on the fixed host - port. Compose reports `Bind for 0.0.0.0: failed: port is already allocated`, `php-fpm` never - starts, and every subsequent command fails against a container that is not running. Because the - ports come from RollDev's own partial rather than the project, rewriting a project's - `.roll/roll-env*.yml` overrides cannot remove them — the ports are not in the project to begin - with, so tooling that strips overrides reports success while the collision persists. -- *Suggested fix:* A `ROLL_PUBLISH_PORTS` boolean in the config schema (default `1`, declared - alongside the other flags around `utils/config.sh:81`) that suppresses host port publication for - the whole environment. Auto-assigning an ephemeral host port on collision would also solve it and - would help interactive multi-project use too. The workaround today is to force - `ROLL_BROWSERSYNC=0` into `.env.roll` before `env up`, which means reaching around RollDev's - configuration to defeat RollDev's own partial. +None outstanding. ### Medium -**M1. Search-container heap size is hardcoded** — `environments/includes/elasticsearch.base.yml:15`, `environments/includes/opensearch.base.yml:15` - -- *What:* Both partials set the heap as a literal: `-Xms64m -Xmx512m`. Every other tunable in these - files (image version, cluster options) is parameterised with `${...}`; the heap is not. -- *Why it matters:* 512 MB is not enough for a large catalog. The container is OOM-killed part way - through indexing and the visible symptom is a connection error from the application, which points - the investigation at configuration or networking rather than at memory. Raising it currently means - editing RollDev's own files, which an upgrade overwrites. -- *Suggested fix:* `${ELASTICSEARCH_JAVA_OPTS:--Xms64m -Xmx512m}` and - `${OPENSEARCH_JAVA_OPTS:--Xms64m -Xmx512m}`, with matching schema entries in `utils/config.sh`. - -**M2. No preflight/diagnostic command** — new command - -- *What:* There is no single command that answers "is this environment actually fit to run?". -- *Why it matters:* Automation discovers problems as opaque Docker or application errors, often deep - into an expensive run — a port already bound, a search cluster that is up but refusing writes, a - host with no disk headroom. Each surfaces as a different downstream error message that names the - wrong layer. -- *Suggested fix:* `roll env doctor` that checks services are healthy, required host ports are free, - the configured search engine answers and accepts an index, and the host has disk headroom; exits - non-zero with a machine-readable summary. This composes well with H1 — the healthchecks it needs - are the same ones. - -**M3. No MySQL 8 to MariaDB DDL normalisation for `myloader`** — `commands/magento2/myloader.cmd` - -- *What:* The wrapper injects connection and directory defaults and forwards every other flag - untouched (verified by reading the file). It does nothing about source/target dialect differences. -- *Why it matters:* Restoring a MySQL 8 dump into RollDev's MariaDB container is a common path, and - two constructs in such dumps abort the load: the `ENCRYPTION='N'` table option (MariaDB: `ERROR - 1911: Unknown option 'ENCRYPTION'`) and `utf8mb4_0900_*` collations (`ERROR 1273: Unknown - collation`). Both terminate `myloader` with `Trace/breakpoint trap (core dumped)`, which gives no - indication of the real cause. Every consumer has to discover and solve this independently. -- *Suggested fix:* An opt-in `--normalize-source-ddl` flag that rewrites `*-schema*.sql` (strip the - `ENCRYPTION` option, remap `utf8mb4_0900_*` to `utf8mb4_general_ci`) before loading. Only the DDL - files should be touched, never the data files. If rewriting is considered out of scope, detecting - those two error codes and printing the cause would remove most of the cost. - -**M4. No machine-readable output from RollDev's own commands** — `commands/status.cmd` - -- *What:* `roll status` renders ANSI-formatted output intended for a terminal. -- *Why it matters:* Automation ends up parsing formatted text, which breaks on any cosmetic change. - This applies to RollDev's own aggregate commands; anything that passes straight through to docker - compose (for example `roll env ps`) already inherits compose's `--format json`. -- *Suggested fix:* `--format json` for `status` and `describe`. - -**M5. No capability query for scripting** — `utils/registry.sh` - -- *What:* There is no supported way to ask whether a command exists before depending on it. -- *Why it matters:* Automation that must degrade gracefully across RollDev versions currently parses - the help listing, which is presentation output and not a contract. -- *Suggested fix:* `roll has-command ` communicating through its exit code, or - `roll registry list --format json`. The registry already discovers commands, so the data exists. +None outstanding. ### Low -**L1. No sugar for running a shell one-liner inside a service** — `commands/env.cmd` - -- *What:* `roll env exec > file` performs the redirect in the *host* shell, not in - the container. The command succeeds and writes to the wrong place, with nothing to indicate it. - The correct form is `roll env exec -T sh -c ''`. -- *Why it matters:* Silent wrong-place writes are hard to notice, particularly for things like - appending to a file inside a container, where the operation appears to succeed. -- *Suggested fix:* `roll env sh ''` that wraps the argument in `sh -c` inside the - container. Low severity because the workaround is reliable and only one flag away — but it is a - footgun worth either sugaring or documenting explicitly. +**L1. `roll db` passes the database password as a process argument** — `commands/db.cmd:51,58,63` + +- *What:* `connect`, `import` and `dump` all invoke the client as + `-p"${MYSQL_PASSWORD}"`, so the password is visible to anything that can read `ps` inside the db + container. Pre-existing; the MariaDB 11 binary probe (finding H2) did not change it either way. +- *Why it matters:* It is the same class of exposure as RECLU's H2 for TablePlus, which the 0.8.0 + plan fixes in milestone 24. Fixing it here too would make the rule uniform across both repos. +- *Suggested fix:* Feed the credentials through `MYSQL_PWD` in the exec environment, or write a + `0600` defaults file into the container and pass `--defaults-extra-file`. The latter is what the + clients themselves recommend and it survives `import`'s stdin pipe, which `MYSQL_PWD` also does. + +**L2. Unreachable code at the end of `magepack.cmd`** — `commands/magento2/magepack.cmd` + +- *What:* An unconditional `exit 1` sits above the final + `roll env exec magepack generate …` line, so that line can never run. ShellCheck flags it as + SC2317, but SC2317 is disabled repo-wide in `.shellcheckrc`, so the gate no longer surfaces it. +- *Why it matters:* Either the `exit 1` is wrong and `magepack generate` is silently dead, or the + trailing line is leftover and should go. Both readings are defects, and the lint suppression now + hides the evidence. +- *Suggested fix:* Establish which of the two was intended, delete the other, and drop the + `disable=SC2317` line from `.shellcheckrc` so the check protects the file again. + +**L3. `local` environment type fails compose render when service toggles are at default** — `environments/local/local.base.yml`, `commands/env.cmd:51,112` + +- *What:* The `local` environment type skips the `php-fpm` partial but still appends nginx, db, and + redis partials when their toggles are on. Since those toggles default to `1`, a bare + `roll env-init x local && roll env config` fails with `service "php-fpm" has neither an image + nor a build context specified`, because an appended fragment declares a dependency on php-fpm. + **Verified** against a throwaway project pre-milestone-4. The failure predates this release. +- *Why it matters:* The `local` type advertises itself as usable without providing services, yet it + fails unless the user disables toggles or provides services in `.roll/roll-env.yml` — the contract + is unclear from the error message, and a user exploring the type hits a confusing failure. +- *Suggested fix:* No change to code or behaviour; this is documented in `docs/environments/types.md` + and the local environment type's default init.env with clear instructions. The type should stay + (per decision D7) because users may have existing projects using it. Closing this as Low because + the workaround is documented and the type is niche (most environments are magento2/laravel/etc.). + +**L4. `roll config schema` silently omits registered keys** — `commands/config.cmd:93-165` + +- *What:* The schema display is seven hand-maintained category filters with no catch-all, so any + registered key matching none of them is invisible. `ROLL_ADMIN_AUTOLOGIN` and + `ROLL_MAGENTO_STATIC_CACHING` are two current examples; both are in `initConfigSchema` and neither + appears in the output. Every key added from now on has to be wired into a filter by hand or it + vanishes from the only listing users have. +- *Why it matters:* `config schema` is the discovery surface for configuration, and an omitted key + reads as "not supported". It also makes the listing quietly wrong rather than visibly incomplete. +- *Suggested fix:* Derive the category from the key once (a `schemaCategoryOf` helper, or a category + column alongside `ROLL_CONFIG_SCHEMA_VALUES`) and render groups from that, so registering a key is + the only step needed. Failing that, add a final "Other" group printing whatever matched nothing. ## Checked, and already covered diff --git a/README.md b/README.md index 2e99d8f..ea9faf0 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,8 @@ on unattended Linux hosts alike. -> The repository's only automated test — `shellcheck commands/*.cmd utils/*.sh`, run by -> `.github/workflows/shellcheck.yml` — currently **fails** (6 errors, 111 warnings, 95 notes as of -> version 0.7.2), so it provides no regression signal. `roll db dump` is also broken against the -> MariaDB 11.x images that are in active use. Both are recorded in -> [Known issues & improvement points](#known-issues--improvement-points) as **H1** and **H2**. +> `roll db dump` is broken against the MariaDB 11.x images that are in active use. Recorded in +> [Known issues & improvement points](#known-issues--improvement-points) as **H2**. ## Contents @@ -28,11 +25,14 @@ on unattended Linux hosts alike. - [Module boundaries & coupling](#module-boundaries--coupling) - [Environments](#environments) - [Common tasks](#common-tasks) +- [Machine interface](#machine-interface) - [Conventions](#conventions) - [Known issues & improvement points](#known-issues--improvement-points) - [Audited and clean](#audited-and-clean) - [Troubleshooting](#troubleshooting) +Release history is in [CHANGELOG.md](CHANGELOG.md). + ## Stack | Concern | Choice | Notes | @@ -82,20 +82,26 @@ cosmetic one. Anything touching these areas needs testing on both: | Area | macOS (`darwin`) | Linux (`linux`) | Where | |---|---|---|---| -| Bash version | 3.2.57 (system bash) | usually 5.x | Why the Bash 3.2 constraint exists; a bash-4 construct fails only on macOS (**H3**) | +| Bash version | 3.2.57 (system bash) | usually 5.x | Why the Bash 3.2 constraint exists; use `bash 3.2` compatible syntax only | | File sync | Mutagen session, named volume for the web root | direct bind mount, no Mutagen | `environments//.darwin.yml`, `commands/sync.cmd:33-35` (fatals off darwin) | | `sed -i` | BSD — requires a backup suffix | GNU — suffix optional | Always use `sed_inplace` (`utils/core.sh:167`) | -| `ping` flags | `-t` is a **timeout** | `-t` is the **TTL** | `isOnline` (`utils/core.sh:161`) is wrong on Linux — finding **M11** | +| Network probe | `ping` (unreliable) | `ping` (unreliable) | `isOnline` probes `https://ghcr.io/v2/` with `curl -m 3` instead, avoiding platform flag differences and ICMP blockage | | SSH agent socket | fixed host path `/run/host-services/ssh-auth.sock` | `${SSH_AUTH_SOCK}` from the host, and a socat proxy unless UID is 1000 | `environments/includes/php-fpm.darwin.yml` vs `.linux.yml`, `utils/config.sh:445-447` | | `.test` DNS | automatic via `/etc/resolver/test` | manual — `roll install` only prints a warning | `commands/install.cmd:56-66` | -| Root CA trust | `security add-trusted-cert` | Fedora/CentOS and Debian/Ubuntu paths only; other distros silently skipped | `commands/install.cmd:29-52`, finding **L10** | +| Root CA trust | `security add-trusted-cert` | Fedora/CentOS and Debian/Ubuntu paths, with a warning naming the CA file on other distros | `commands/install.cmd:29-58` | | tunnel key permissions | untouched | `chown root:root` on `ssh_key.pub`, because bind mounts are native | `commands/install.cmd:82-84` | | Xdebug host | `host.docker.internal` | container gateway address, looked up at runtime | `commands/debug.cmd:13-21` | | `mapfile` | absent (bash 3.2) | present | `commands/status.cmd:20-28` shows the required fallback | -| CI coverage | **none** | `ubuntu-latest` | All five workflows; finding **M10** | +| CI coverage | `macos-latest`: ShellCheck + Docker-free smoke | `ubuntu-latest`: ShellCheck + Docker-free smoke | `.github/workflows/shellcheck.yml` matrix | -Prerequisites: Docker (Desktop on macOS/Windows, Engine on Linux) with `docker compose` ≥ 2.2.3, and -Homebrew. On macOS, Mutagen is installed automatically on first `roll sync` if missing. +Prerequisites: Docker (Desktop on macOS/Windows, Engine on Linux) with `docker compose` ≥ 2.2.3, +Homebrew, and [gum](https://github.com/charmbracelet/gum) ≥ 0.14.0, which backs every interactive +prompt. The Homebrew formula declares gum as a dependency, so a `brew install` pulls it in; on a +source checkout install it yourself (`brew install gum`, Charm's apt repository, `dnf`/`pacman`, or +a release binary — `roll install` prints the list and warns rather than failing). gum is needed +only for prompts: every one of them is also reachable by flag, environment variable or positional +argument, so scripted and CI use works without it. On macOS, Mutagen is installed automatically on +first `roll sync` if missing. Install and start the shared services: @@ -188,18 +194,27 @@ literal default value. Loading proceeds in this order (`loadRollConfig`, `utils/ 1. `~/.roll/.env.roll`, then legacy `~/.roll/.env` — global settings. 2. `${ROLL_ENV_PATH}/.env.roll` — project settings, overriding global. 3. OS detection sets `ROLL_ENV_SUBT` to `darwin`, `linux`, or `wsl`. -4. `setConfigDefault` fills in every schema key that has a literal default and is not yet set. -5. `assertValidEnvType` checks that `environments//.base.yml` exists. -6. `postProcessConfig` derives image variants (`ROLL_SVC_PHP_VARIANT`, `ROLL_SVC_PHP_NODE`), the +4. `assertValidEnvType` checks that `environments//.base.yml` exists. +5. `applyEnvTypeDefaults` fills in what the environment type needs — the `magento2` service + toggles, the non-`local` base services, the DB distribution version — but never overwrites a key + the project set itself. +6. `setConfigDefault` fills in every schema key that still has no value and carries a literal + default. +7. `applyVersionPinFallbacks` warns about every enabled service with no version pin and falls back + to the version the project was already running. "No pin" is judged on the keys step 2 read out of + the project's own `.env.roll`, not on the resolved value — a version coming from global config + resolves but pins nothing, since it is absent from the repository a colleague clones. +8. `postProcessConfig` derives image variants (`ROLL_SVC_PHP_VARIANT`, `ROLL_SVC_PHP_NODE`), the Xdebug tag, and the nginx template name. -Step 4 running before step 6 is the root cause of finding **M1** below: any `${VAR:-default}` -expression in step 6 (or in `commands/env.cmd`) is unreachable for a key that has a literal schema -default, because step 4 already gave it a value. +**The order matters.** Env-type defaults have to run before the schema literals, or every +`${VAR:-default}` in step 5 is unreachable — the literal has already given the key a value, and +`:-` only substitutes when unset or empty. Derivations that depend on final input values run last. Every value is exported, so it reaches `docker compose` both through `--env-file .env.roll` and -through the process environment. Where the two disagree, the exported value wins — which is why the -`${...:-fallback}` defaults written inside the YAML fragments are mostly dead (finding **M2**). +through the process environment. Where the two disagree, the exported value wins, so the +`${...:-fallback}` defaults inside the YAML fragments are not the effective defaults; the service +version fallbacks have been removed from the fragments for that reason. ### How a compose invocation is assembled @@ -268,8 +283,8 @@ responsibility is still describable. | Component | Size | Stated responsibility | Verdict | |---|---|---|---| | `bin/roll` | 98 lines | Entry point, dispatch, argument parsing | **Coherent** | -| `utils/core.sh` | 184 lines | Messaging helpers, array/version utilities, network peering | **Coherent** — the three box-drawing functions are near-identical triplets (finding **L4**) | -| `utils/config.sh` | 606 lines | Config schema, loading, validation, post-processing | **Coherent**, but owns defaults that `commands/env.cmd` also owns (**M1**) | +| `utils/core.sh` | 184 lines | Messaging helpers, array/version utilities, network peering | **Coherent** — the three box-drawing functions are one-line wrappers around a shared `box` helper | +| `utils/config.sh` | 606 lines | Config schema, loading, validation, post-processing | **Coherent**; sole owner of configuration defaults | | `utils/registry.sh` | 461 lines | Command discovery and priority resolution | **Oversized for what it delivers** — ~200 lines serve `roll registry`'s reporting subcommands; the metadata layer they report on is a stub (**M6**) | | `utils/env.sh` | 108 lines | Env path location, partial precedence, env-type validation | **Coherent** | | `utils/install.sh` | 63 lines | Host install assertion, SSH config | **Coherent** | @@ -279,9 +294,9 @@ responsibility is still describable. | `commands/*.cmd` (wrappers) | 5–25 lines each | One `roll env exec` invocation | **Coherent** | | `commands/magento2/`, `commands/wordpress/` | 344 lines / 11 lines | Env-type-specific commands | **Coherent**; `commands/magento1/` and both `usage.help`-only directories contain no commands | | `environments/includes/` | 20 fragments | One service each | **Coherent** | -| `environments//` | 11 types | Per-type overrides and seeds | **Shell** for `local` (`environments/local/local.base.yml` is a 0-byte file) and thin for `vuejs` (no `init.env`) — see **L5** | +| `environments//` | 11 types | Per-type overrides and seeds | **Coherent** — each type has an `init.env` (or documents why it doesn't), and `local` is properly documented | | `docker/`, `config/` | 290 lines | Shared-services compose and Traefik/OpenSSL config | **Coherent** | -| `docs/` | 3 666 lines / 31 pages | User documentation | **Coherent**, drifting from the code (**M5**, **L1**) | +| `docs/` | 3 666 lines / 31 pages | User documentation | **Coherent**, drifting from the code (**M5**) | ### Real coupling map @@ -290,14 +305,11 @@ part. These are the couplings that are not visible from it: | A | B | Mechanism | What breaks if separated | |---|---|---|---| -| `utils/config.sh` (`postProcessConfig`) | `commands/env.cmd:24-105` | The **same** env-type defaulting logic is written twice, in full: PHP/Node image variants, DB distribution version, Xdebug tag, nginx template, `CHOWN_DIR_LIST`, magento1/magento2 service toggles | Nothing breaks — they already disagree. `env.cmd`'s copy is dead for keys with literal schema defaults, so the two produce different results depending on which keys `.env.roll` sets (**M1**) | -| `utils/config.sh` schema defaults | `environments/**/*.yml` `${VAR:-fallback}` | Config exports a value for every key with a literal default, so the YAML fallback never applies | The YAML fallbacks read as the effective default and are not; eight of them disagree with the schema (**M2**) | | `commands/backup.cmd` | `commands/restore.cmd`, `commands/restore-full.cmd` | Undocumented on-disk contract: the staging directory `/.roll/backups`, the archive layout, and the metadata JSON. Each file re-derives the path independently (documented in a comment at `commands/backup.cmd:22-35`) | Changing the layout in one file silently breaks the others; `restore-full.cmd` never loads the env config at all, so it resolves paths differently | | `commands/status.cmd:6` | `docker/docker-compose.yml` | The shared network name is recovered by `grep -A3 'networks:' … \| tail -n1 \| sed`, i.e. by text-parsing YAML | Reordering keys in `docker-compose.yml` breaks `roll status`'s "is RollDev running" check | -| `commands/describe.cmd:18`, `commands/vnc.cmd:12` | Compose container naming | Both reconstruct container names by string concatenation instead of asking `roll env ps -q` — and they disagree: `describe` uses v2 (`name-svc-1`), `vnc` uses v1 (`name_svc_1`) | `roll vnc` cannot find a container under the Compose v2 that `bin/roll:26` requires (**M8**) | | `environments/includes/redis.base.yml` | `environments/includes/dragonfly.base.yml` | Both define a service literally named `redis`; Dragonfly only swaps the image and volume | Correct as written — it is what lets `roll redis` work either way — but it forces the mutual-exclusion guard at `commands/env.cmd:12-14` | | `commands/env.cmd:116-120` | `commands/browsersync.cmd` | `env up` shells back out to `roll browsersync freeport` to pick host ports before assembling the compose args | A circular dependency between two commands, evaluated on every `up` with browsersync enabled | -| `docs/index.md:7` | `README.md` | Sphinx `include` with `end-before: ` | The marker was missing before this revision, so the docs home page inlined the entire README (**M5**) | +| `docs/index.md:7` | `README.md` | Sphinx `include` with `end-before: ` | Keep the marker: anything below it is contributor-facing and stays off the published site | Two couplings that look wrong but are not: `commands/*.cmd` wrappers re-invoking `${ROLL_DIR}/bin/roll` as a subprocess is a deliberate choice (a sourced command cannot cleanly re-enter dispatch), and the @@ -306,75 +318,9 @@ Compose projects cannot declare each other's networks. ### Boundary findings -#### High - -None. No component in this repository has an unclear owner or a circular library dependency; the -boundary problems below all cost maintenance time rather than correctness. - -#### Medium - -**MB1. The backup/restore/duplicate family has no shared library** — `commands/backup.cmd`, -`commands/restore.cmd`, `commands/restore-full.cmd`, `commands/duplicate.cmd` - -- *What:* Four command files totalling 3 929 lines — 57 % of all top-level command code — with no - extracted helpers. `restore.cmd` and `restore-full.cmd` differ in only 339 of ~2 123 lines - (measured with `diff` after trimming trailing whitespace); 17 of their 19 function names are - identical. `promptPassword`, `showProgress` and `logMessage` exist in all four; `promptPassword` is - byte-identical between `restore.cmd` and `restore-full.cmd` and differs slightly in the other two. -- *Why it matters:* Every fix has to be applied up to four times, and the three near-copies of - `promptPassword` prove that it is not being done. The archive layout contract lives only in prose - comments, so a change to the staging path in one file leaves the others looking for backups - elsewhere — a failure mode the comment at `commands/backup.cmd:22-35` already warns about. -- *Suggested fix:* Extract a `utils/backup.sh` sourced by all four, holding the shared primitives - (`promptPassword`, `showProgress`, `logMessage`, `logVerbose`, `detectEncryptedBackup`, - `findLatestBackup`, `extractBackupArchive`, `validateBackup`, `getBackupMetadata`, - `getVolumeMapping`, `restoreVolume`) plus one function that resolves the staging directory, so the - path contract has exactly one definition. Then collapse `restore-full.cmd` into `restore.cmd` - behind a `--include-source` flag mirroring `backup.cmd`'s existing `--include-source`, and keep - `restore-full` as a two-line alias for backwards compatibility. `utils/` is already sourced from - `bin/roll`, so a fifth library needs one line there. - -**MB2. `roll registry`'s reporting layer describes metadata that is never collected** — -`utils/registry.sh:68-92` - -- *What:* `extractCommandMetadata` is a stub: the `description` branch returns an empty string and - the `category` branch returns the constant `general`, both with a `# Simple approach - just return - empty for now` comment. Roughly 200 lines of `utils/registry.sh` and `commands/registry.cmd` exist - to filter, group, count and export those two fields. -- *Why it matters:* `roll registry list`, `search`, `categories`, `stats`, `info` and `export` all - report a single category (`global`) and blank descriptions, so `roll registry stats` on a stock - install prints `global : 48 commands` and nothing else useful. The `.help` files that would supply - the descriptions already exist and are already located by the registry. -- *Suggested fix:* Either implement the extraction — parse a `## @description:` / `## @category:` - comment header from the `.help` file, which requires adding those headers to the 40 built-in help - files — or delete the category and description arrays and the subcommands that consume them, - keeping `list`, `info`, `paths`, `validate` and `export`. The half-built version is worse than - either: it makes the CLI look as though the feature works. - -#### Low - -**LB1. `commands/magento2-init.cmd` is a project generator inside a command file** — 784 lines - -- *What:* The largest non-backup command scaffolds a full Magento 2 project: version resolution, - Composer authentication, `env-init`, `env up`, installation, sample data. -- *Why it matters:* It duplicates decisions that `commands/env-init.cmd` and - `environments/magento2/init.env` also own, and it is the file most likely to need editing when - Magento's installation procedure changes — while being the least testable. -- *Suggested fix:* Extract the phases (`resolve version`, `create project`, `write env`, - `install`) into functions in a `utils/magento2-init.sh`, so each phase can be re-run - independently after a failure. This is worth doing when the file is next substantially edited, not - on its own. - -**LB2. `environments/local/` is an empty environment type** — `environments/local/local.base.yml` - -- *What:* A 0-byte file. It satisfies `assertValidEnvType` (`utils/env.sh:83`), so `local` appears in - `roll env-init`'s list of valid types, and `commands/env.cmd:51,112` special-case it to skip - `php-fpm` and the nginx/db/redis defaults. There is no `init.env`. -- *Why it matters:* `roll env-init x local && roll env up` produces an environment with a network and - nothing else. Whether that is the intent is not recorded anywhere. -- *Suggested fix:* If it is intended as "networking only, bring your own `.roll/roll-env.yml`", say so - in a comment in the file and in `docs/environments/types.md`. If not, remove the directory; the - type disappears from the list automatically. +None open. No component has an unclear owner or a circular library dependency. The two that did cost +maintenance time are resolved: the backup family now shares `utils/backup.sh`, and `magento2-init` +is split into re-runnable phases in `utils/magento2-init.sh`. ## Environments @@ -393,8 +339,8 @@ boundary problems below all cost maintenance time rather than correctness. | `typo3` | yes | **no** | PHP 7.4, Node 12, MariaDB 10.4 | | `wordpress` | yes | **no** | PHP 7.4, Composer 1, seeds `DB_*`; PHP image variant is `php-fpm-wordpress` | | `php` | yes | **no** | PHP 8.1, `ROLL_DB=0` | -| `vuejs` | **no** | **no** | Routes `app.` to nginx and `watch.app.` to port 8080 on `php-fpm` | -| `local` | **no** | **no** | Empty base fragment — see **LB2** | +| `vuejs` | yes | **no** | PHP 8.1, Node 18, MariaDB 10.4; routes `app.` to nginx and `watch.app.` to port 8080 on `php-fpm` | +| `local` | yes | **no** | Network-only; bring your own `.roll/roll-env.yml`. Pins only what its default toggles enable | Without a Mutagen configuration, macOS falls back to the bind mount from `environments/includes/php-fpm.base.yml` (`.:/var/www/html:cached`), which works but is markedly @@ -432,9 +378,9 @@ Registered in `utils/config.sh:70-87`. Each enabled toggle appends the matching | `ROLL_MAGEPACK` | 0 | Magepack bundling | `magento2` only | | `ROLL_INCLUDE_GIT` | 0 | bind-mounts `.git` into the container | needed on macOS with Mutagen | -For a `magento2` project the effective values of `ROLL_VARNISH`, `ROLL_ELASTICSEARCH` and -`ROLL_RABBITMQ` come from `init.env`, not from the env-type block in `commands/env.cmd:73-76` — see -finding **M1**. +For a `magento2` project `ROLL_VARNISH`, `ROLL_ELASTICSEARCH` and `ROLL_RABBITMQ` default to `1` +via `applyEnvTypeDefaults` in `utils/config.sh`, whether or not `init.env` spelled them out. A +value in `.env.roll` still wins. ### Local, test/staging, and production @@ -553,15 +499,49 @@ pip install -r docs/requirements.txt sphinx-build -b html docs docs/_build/html ``` +## Machine interface + +Everything RollDev does is reachable without a terminal, so it can be driven from CI, a deploy +script, the build server, or an AI assistant. The user-facing guide is +[docs/machine-interface.md](docs/machine-interface.md); this is the contributor summary of what +exists and where it lives. + +| Surface | Where | +|---|---| +| `--format json` on `status`, `env describe`, `registry list`, `env doctor` | the respective `.cmd` files; values escaped through `jsonEscape` in `utils/core.sh` | +| `roll has-command ` — exit 0/1, no output | `commands/has-command.cmd`, resolves through the registry so add-on commands count | +| `roll env up --wait` | Compose native, made meaningful by the healthchecks in `environments/includes/*.base.yml` | +| `roll env doctor` | `commands/doctor.cmd` | +| `roll config check-pins` / `fix-pins` | `commands/config.cmd` | +| Flag-first prompts | `utils/interact.sh` | + +Two rules bind anything added here: + +1. **Machine-readable output carries no secrets and no ANSI**, and every value goes through + `jsonEscape` rather than being concatenated raw. +2. **A prompt is never the only route to a value.** Flags and environment variables come first, gum + runs only on a TTY, and with no terminal the prompt fails naming the flag that would have + answered it — so an unattended run cannot hang. `.github/scripts/test-interact.sh` asserts this + and runs in the smoke suite on both platforms. + +Note that giving a command a `--flag` means adding it to `ROLL_CMD_ANYARGS` in `bin/roll`, which +changes how `--help` reaches it: roll stops parsing at the first dash and leaves `ROLL_PARAMS` +empty. Such a command must render its own help by sourcing `commands/usage.cmd`, never by +re-invoking `roll --help`, which recurses until killed. +`.github/scripts/test-syntax.sh` enforces this. + ## Conventions These are enforced by review rather than by tooling, except where noted. - **Every change must work on macOS and Linux.** Both are first-class targets — see - [Supported platforms](#supported-platforms) for the table of real behavioural differences. CI only - runs Ubuntu (finding **M10**), so the macOS half is on the author: run the affected command under - `/bin/bash` on macOS before merging. The recurring traps are bash 3.2, BSD vs GNU flag semantics - (`sed`, `ping`, `stat`), Mutagen-vs-bind-mount file sync, and the SSH agent socket path. + [Supported platforms](#supported-platforms) for the table of real behavioural differences. CI runs + a `macos-latest` + `ubuntu-latest` matrix (ShellCheck plus the Docker-free smoke script, + `.github/scripts/smoke.sh`), but the smoke set only covers commands that need neither a project + checkout nor a running daemon — anything touching `roll env`/service containers is still on the + author: run the affected command under `/bin/bash` on macOS before merging. The recurring traps + are bash 3.2, BSD vs GNU flag semantics (`sed`, `ping`, `stat`), Mutagen-vs-bind-mount file sync, + and the SSH agent socket path. - **Bash 3.2 only.** No associative arrays, no `${var^}`/`${var,}` case modification, no `mapfile` without a fallback (`commands/status.cmd:20-28` shows the pattern). macOS ships bash 3.2.57 and `bin/roll` runs under `#!/usr/bin/env bash`, so a bash-4 construct fails there while passing on @@ -581,7 +561,7 @@ These are enforced by review rather than by tooling, except where noted. - **Adding a command:** create `commands/.cmd` + `commands/.help`; the registry picks it up with no central list to edit. If it must accept arbitrary pass-through flags, add it to `ROLL_CMD_ANYARGS` in `bin/roll:43`. Add it to `commands/usage.help` as well — that file is - maintained by hand and is currently out of date (finding **L1**). + maintained by hand for visibility. - **Adding a service:** add the fragment(s) under `environments/includes/` (plus per-type overrides), wire a `ROLL_` toggle into the assembly block in `commands/env.cmd`, and register the variable and its default in `initConfigSchema` (`utils/config.sh:58`). Do not rely on a @@ -606,373 +586,15 @@ rather than defects, and of the [boundary findings](#boundary-findings) above. ### High -**H1. The repository's only automated test fails** — `.github/workflows/shellcheck.yml:20` - -- *What:* CI runs `shellcheck commands/*.cmd utils/*.sh`, which exits 1. **Verified:** 6 errors, 111 - warnings and 95 notes on a clean checkout. There is no `.shellcheckrc` and no severity threshold, so - every one of the 212 findings is a failure. The errors are: - - | Location | Check | Problem | - |---|---|---| - | `utils/core.sh:147`, `utils/core.sh:154` | SC2068 | Unquoted `${DOCKER_PEERED_SERVICES[@]}` re-splits elements | - | `utils/core.sh:26`, `commands/sign-certificate.cmd:12` | SC2242 | `exit -1` — the process actually exits 255 | - | `commands/restore.cmd:186`, `commands/restore-full.cmd:195` | SC2145 | A string argument mixed with an array expansion | - -- *Why it matters:* A gate that is always red is indistinguishable from a gate that has just gone red, - so the one mechanism that could have caught the quoting and array-expansion bugs elsewhere in this - list provides no signal. `exit -1` in `fatal` also means every fatal error reports exit status 255, - which is indistinguishable from a signal-terminated process to any script wrapping `roll`. -- *Suggested fix:* Fix the six errors first (`"${DOCKER_PEERED_SERVICES[@]}"`, `exit 1`, `${arr[*]}` - where a single string is intended), then add `.shellcheckrc` with the `disable=` list for the - warning classes the codebase deliberately accepts, so the gate goes green and stays meaningful. - Raising the threshold with `shellcheck -S error` is the quicker alternative but leaves the 111 - warnings unreviewed. - -**H2. `roll db dump` fails against the MariaDB 11.x images in active use** — `commands/db.cmd:44` - -- *What:* The `dump` subcommand runs `mysqldump` inside the `db` container. **Verified:** in - `ghcr.io/epartment/roll/mariadb:11.4` — the image a running environment on the review machine was - using — `mysqldump` does not exist; MariaDB 11 renamed it to `mariadb-dump`. The compatibility - symlink for the client (`mysql`, used by `connect` at `commands/db.cmd:34` and `import` at - `commands/db.cmd:40`) *is* still present, so only `dump` is affected. -- *Why it matters:* `roll db dump` exits with a Docker "executable file not found" error rather than - anything mentioning MariaDB versions, on a command whose whole purpose is to produce a backup. The - schema still defaults `MARIADB_VERSION` to 10.4, so the failure only appears on projects that pin a - newer version — which makes it look intermittent across projects. -- *Suggested fix:* Probe once and fall back, so both generations work: - `DUMP_BIN=$(roll env exec -T db sh -c 'command -v mariadb-dump || command -v mysqldump')`, then - invoke `${DUMP_BIN}`. Apply the same treatment to `mysql` in `connect`/`import` before the client - symlink is dropped too. A `mysqldump` symlink in the images repository would also fix it, but the - probe keeps this repo working against images it does not control. - -**H3. `roll registry categories` aborts on macOS** — `utils/registry.sh:273`, -`commands/registry.cmd:36` - -- *What:* Both lines use `${category^}`, bash 4's "uppercase first character" expansion. **Verified:** - under bash 3.2.57 (the macOS system bash, which `#!/usr/bin/env bash` resolves to on a machine - without a newer bash first on `PATH`) the command dies with - `bad substitution` and prints nothing else. A third occurrence, `${categories[$i]^}` at - `utils/registry.sh:363`, is silently accepted by bash 3.2 as a no-op, so `roll registry stats` - works but never capitalises. -- *Why it matters:* It is a hard failure of a documented subcommand on the primary development - platform, and it contradicts the Bash 3.2 constraint that the rest of the codebase carefully - observes. The inconsistency between the scalar and array forms is why it survived: the array form - produces no error to notice. -- *Suggested fix:* Replace all three with a helper — `printf '%s%s' "$(echo "${s:0:1}" | tr - '[:lower:]' '[:upper:]')" "${s:1}"` — or simply drop the capitalisation, since the category values - are already lowercase words. Add a `bash --posix`-independent smoke test that runs - `roll registry categories` under `/bin/bash` on a macOS runner, because Linux-only CI cannot catch - this class of bug. +None open. ### Medium -**M1. Env-type service defaults are dead code; the toggles they set resolve to 0** — -`utils/config.sh:378-381` and `utils/config.sh:473-476`, duplicated at `commands/env.cmd:73-76` - -- *What:* `loadRollConfig` runs `setConfigDefault` for every schema key (`utils/config.sh:378-381`) - *before* calling `postProcessConfig` (`utils/config.sh:389`). `ROLL_VARNISH`, - `ROLL_ELASTICSEARCH` and `ROLL_RABBITMQ` are declared `boolean:0`, so by the time - `postProcessConfig` evaluates `export ROLL_VARNISH="${ROLL_VARNISH:-1}"` the variable is already - `0` and the `:-1` never applies. `commands/env.cmd:73-76` repeats the same block and fails the same - way. **Verified** by loading a minimal `magento2` `.env.roll` (containing only `ROLL_ENV_NAME` and - `ROLL_ENV_TYPE`) through `loadEnvConfig`: `ROLL_VARNISH=0`, `ROLL_ELASTICSEARCH=0`, - `ROLL_RABBITMQ=0`. -- *Why it matters:* A `magento2` environment whose `.env.roll` does not list these keys comes up with - no Varnish, no search engine and no message queue — and Magento's own configuration will point at - services that are not running. New projects are shielded because - `environments/magento2/init.env` sets all three explicitly, so the bug only surfaces on - hand-written or older `.env.roll` files, which is exactly when it is hardest to attribute. The same - ordering makes `postProcessConfig`'s `DB_DISTRIBUTION_VERSION` branch (`utils/config.sh:424-430`) - unreachable. -- *Suggested fix:* Move the `setConfigDefault` loop to *after* `postProcessConfig`, so per-type - derivation runs against unset variables and the schema fills in only what is still missing. Then - delete the duplicated block at `commands/env.cmd:24-105` outright — it is the second copy of logic - `utils/config.sh` already owns, and keeping both guarantees they drift (see the coupling map). - Add a test that loads a minimal `.env.roll` per env type and asserts the effective toggles. - -**M2. YAML image-version fallbacks are unreachable and disagree with the schema** — -`utils/config.sh:95-121` vs `environments/` - -- *What:* Because every schema key with a literal default is exported before `docker compose` runs, - the `${VAR:-fallback}` defaults written into the fragments never apply. Eight of them name a - different version from the schema: - - | Variable | Schema default (`utils/config.sh`) | Fragment fallback | Fragment | - |---|---|---|---| - | `PHP_VERSION` | `8.1` | `8.3` | `environments/includes/php-fpm.base.yml:22` | - | `NODE_VERSION` | `18` | `22` | `environments/includes/php-fpm.base.yml:27` | - | `ELASTICSEARCH_VERSION` | `7.17` | `8.11` | `environments/includes/elasticsearch.base.yml:4` | - | `OPENSEARCH_VERSION` | `2.5` | `2.19` | `environments/includes/opensearch.base.yml:4` | - | `RABBITMQ_VERSION` | `3.11` | `3.8` | `environments/includes/rabbitmq.base.yml:4` | - | `REDIS_VERSION` | `7.0` | `5.0` | `environments/includes/redis.base.yml:4` | - | `VARNISH_VERSION` | `7.0` | `6.0` | `environments/includes/varnish.base.yml:10` | - | `NGINX_VERSION` | `1.27` | `1.26` | `environments/includes/nginx.base.yml:4` | - - A third value exists for some of these in `describe.cmd` (PHP `8.2`, Node `18`, Redis `7.2`), used - only for display. **Verified** for `PHP_VERSION`: a minimal `magento2` `.env.roll` resolves to - `8.1`, not the `8.3` the fragment appears to promise. -- *Why it matters:* Reading the fragment gives the wrong answer about what a project without an - explicit pin actually gets, and the schema's values are the older ones — a `magento2` project - missing `PHP_VERSION` silently runs PHP 8.1 against Magento 2.4.7+. `roll env describe` then - reports a third number. -- *Suggested fix:* Make the schema the single source of truth: bring `utils/config.sh`'s defaults up - to the versions the fragments claim, then reduce the fragments to bare `${VAR}` so a missing value - fails loudly instead of resolving to a stale one. Drop the hardcoded fallbacks in - `commands/describe.cmd:108,120,133` and print the resolved values. - -**M3. The network-existence check in `env up` never matches, so a redundant compose pass runs every -time** — `commands/env.cmd:206` - -- *What:* `docker network ls -f 'name=$(renderEnvNetworkName)' -q` is single-quoted, so the command - substitution is not performed and Docker filters on the literal string - `name=$(renderEnvNetworkName)`. **Verified:** such a filter matches nothing, so the `-z` test is - always true and the `docker compose … up --no-start` block at `commands/env.cmd:208-210` executes on - every `roll env up`, not only on first creation. -- *Why it matters:* Every `env up` performs a full extra compose pass — resolving images, creating - containers with `--no-start` — before the real `up`. It is wasted wall-clock on every start and, - because `--no-start` creates containers, it can mask ordering problems that a genuine first-run - would expose. -- *Suggested fix:* `docker network ls -f "name=^$(renderEnvNetworkName)$" -q`, with double quotes and - anchors so a longer environment name is not matched as a prefix. - -**M4. `restore.cmd` and `restore-full.cmd` are near-identical copies** — see boundary finding -[**MB1**](#boundary-findings) for the measurement and the suggested split. - -**M5. The published documentation home page inlines the whole README** — `docs/index.md:7` - -- *What:* `docs/index.md` includes `../README.md` with `end-before: `. - **Verified:** before this revision no such marker existed anywhere in the repository, so MyST had - nothing to stop at and the entire README — installation instructions, command list, licence, links — - was rendered above the docs home page's own "Features" section and table of contents. This revision - adds the marker after the opening paragraph. -- *Why it matters:* The docs home page duplicated the README's installation section immediately above - its own link to `installing`, and any README growth landed on the published site unreviewed. -- *Suggested fix:* Already addressed by the marker. Keep it: anything below - `` in this file is repository-facing and does not appear on the site. If - the include ever needs to cover more, move the marker rather than removing it. - -**M6. `roll registry`'s metadata layer is a stub** — see boundary finding -[**MB2**](#boundary-findings). - -**M7. Nothing validates `ROLL_ENV_NAME`, and Compose rejects uppercase project names** — -`commands/env-init.cmd:21-23`, `utils/config.sh:65` - -- *What:* `env-init` prompts for a name and accepts anything non-empty; the schema declares - `ROLL_ENV_NAME` as `string:required`, which only checks for emptiness. The name is passed straight - to `docker compose -p` (`commands/env.cmd:248`). **Verified:** `docker compose -p TestUpper …` - fails with `invalid project name "TestUpper": must consist only of lowercase alphanumeric - characters, hyphens, and underscores as well as start with a letter or number`. -- *Why it matters:* `roll env-init MyProject magento2` succeeds, and then every subsequent `roll env` - command fails with a Docker error that names neither RollDev nor `.env.roll`. Related: several - places lowercase the name (`renderEnvNetworkName`, `utils/env.sh:50`) while others do not - (`commands/describe.cmd:18`, the `traefik.docker.network=${ROLL_ENV_NAME}_default` labels in every - fragment), so even a name that Compose tolerates can produce a label that does not match the real - network. -- *Suggested fix:* Validate in `commands/env-init.cmd` before writing the file — reject anything not - matching `^[a-z0-9][a-z0-9_-]*$` and re-prompt — and add the same check to - `validateConfigValue` for `ROLL_ENV_NAME` so existing files are caught by `roll config validate`. - -**M8. `roll vnc` builds Compose v1 container names** — `commands/vnc.cmd:12` - -- *What:* It constructs `${ROLL_ENV_NAME}_${service}_${index}` — the Compose v1 separator — while - `bin/roll:26-30` requires Compose v2, which names containers `name-service-1`. - `commands/describe.cmd:18` uses the v2 form, so the two disagree. -- *Why it matters:* The SSH forward and the generated Remmina profile both target a hostname that does - not resolve, so `roll vnc` cannot reach the Selenium VNC server on any supported Compose version. -- *Suggested fix:* Resolve the container through Compose instead of reconstructing the name: - `roll env ps -q "${service}"`, then `docker container inspect` for the hostname. That also removes - the assumption of a single replica. Apply the same change to `commands/describe.cmd:18`. - -**M9. `fixowns` and `fixperms` mishandle multiple arguments** — `commands/magento2/fixowns.cmd:17,20`, -`commands/magento2/fixperms.cmd:17,21` - -- *What:* Both test `[ -z "${ROLL_PARAMS[@]}" ]`. With one element that works by accident; with two or - more, `[` receives three arguments and fails with `too many arguments`. `fixowns.cmd:20` then - interpolates the array into a single path — `/var/www/html/"${ROLL_PARAMS[@]}"` — which concatenates - the elements rather than treating them as separate targets. -- *Why it matters:* `roll fixowns app/code var` reports a shell error instead of fixing ownership, or - chowns a path that does not exist. Since these commands exist to recover from permission problems, - they fail exactly when someone is already stuck. -- *Suggested fix:* Test the count — `if (( ${#ROLL_PARAMS[@]} == 0 )); then` — and pass the array - through as separate arguments, prefixing each with `/var/www/html/` in a loop rather than inside one - quoted expansion. - -**M10. Nothing tests the macOS half of a platform the tool must support** — -`.github/workflows/shellcheck.yml:17` - -- *What:* All five workflows run on `ubuntu-latest`. There is no macOS job, and no runtime smoke test - on any platform — CI lints shell sources and builds documentation, nothing more. -- *Why it matters:* macOS is the primary developer platform *and* the one with the divergent - behaviour (bash 3.2, BSD flag semantics, Mutagen), so every platform-specific defect in this list - is invisible to CI by construction. **H3** is the clearest case: `${category^}` lints clean and runs - fine on Ubuntu, and hard-fails on the platform most users are on. **M11** is the mirror image — a - bug that only manifests on Linux, shipped from macOS machines. -- *Suggested fix:* Add a `macos-latest` job running the same ShellCheck invocation, then a minimal - matrix smoke test across `macos-latest` and `ubuntu-latest` that needs no Docker daemon: - `./bin/roll version`, `./bin/roll registry validate`, `./bin/roll registry categories`, - `./bin/roll config schema`, and a `roll env-init` into a temporary directory followed by - `roll config validate`. On macOS, invoke via `/bin/bash ./bin/roll` so bash 3.2 is exercised rather - than whatever the runner has on `PATH`. That set would have caught **H3**, **M7** and **M9**. - -**M11. `isOnline` always reports offline on Linux, so `roll svc up` never refreshes images** — -`utils/core.sh:161-163` - -- *What:* `isOnline` probes with `ping -q -c1 -t 2 8.8.8.8`. `-t` means *timeout in seconds* on BSD - ping (macOS) but *TTL* on GNU and BusyBox ping (Linux), so on Linux the probe sends a packet with - TTL 2 and both public resolvers are further away than two hops. **Verified** in a Linux container: - `-t 2` gives 100 % packet loss and exit 1, `-t 64` and `-W 2` both succeed, and the identical - command exits 0 on the macOS host — so the flag semantics, not connectivity, are the cause. -- *Why it matters:* The only caller is `commands/svc.cmd:50-52`, which skips `roll svc pull` when the - probe says offline. On Linux the shared-service images are therefore never pulled by `roll svc up`, - so Traefik, dnsmasq, Mailpit and the tunnel silently stay on whatever tags were first fetched. That - is precisely the platform where nobody is watching, and the symptom — a stale image — surfaces much - later as unexplained behaviour rather than as a failed pull. -- *Suggested fix:* Do not use `ping` for a reachability check at all; ICMP is also blocked in many - networks and container runtimes, which would produce the same false negative on macOS. Probe the - registry that is actually needed, with a portable timeout: - `curl -fsS -m 3 -o /dev/null https://ghcr.io/v2/ && echo true || echo false`. If `ping` must stay, - branch on `${ROLL_ENV_SUBT}` and use `-W 2` on Linux — but note `isOnline` is in `utils/core.sh`, - which is sourced before the config is loaded, so `$OSTYPE` is the only signal available there. - -**M12. WSL hosts get no OS-specific compose fragment at all** — `utils/config.sh:361-363`, -`utils/env.sh:96,98` - -- *What:* `loadRollConfig` sets `ROLL_ENV_SUBT=wsl` when `/proc/sys/kernel/osrelease` mentions - Microsoft. `appendEnvPartialIfExists` then looks for `.wsl.yml`, and **no such file exists** - anywhere in `environments/` (verified — only `.base.yml`, `.darwin.yml` and `.linux.yml` are - present). A WSL host therefore receives neither the darwin nor the linux variant of any fragment. -- *Why it matters:* The most consequential loss is `environments/includes/php-fpm.linux.yml`, the only - place the host's SSH agent socket is mounted into the containers. Without it `SSH_AUTH_SOCK` falls - back to `/tmp/ssh-auth.sock` from `environments/includes/php-fpm.base.yml:26`, which nothing - creates, so agent forwarding — and therefore `composer install` against private repositories — - cannot work on WSL, and the unconditional `chmod 777` at `commands/env.cmd:285-287` targets a path - that does not exist. WSL2 is a documented install target in `docs/installing.md`, so this is a - supported configuration silently missing a fragment. *The WSL-side consequences are read from the - code; unverified, as no WSL host was available for this review.* -- *Suggested fix:* Either treat WSL as Linux for fragment selection — have - `appendEnvPartialIfExists` fall back to the `linux` variant when `ROLL_ENV_SUBT` is `wsl`, which - matches the intent, since WSL is a Linux kernel — or add `environments/includes/php-fpm.wsl.yml`. - The fallback is preferable: it is one change in `utils/env.sh` and it covers every present and - future `.linux.yml` rather than only the one that is currently missed. Keep the separate `wsl` - value, because `utils/config.sh:440-442` legitimately needs it for `XDEBUG_CONNECT_BACK_HOST`. +None open. ### Low -**L1. `commands/usage.help` is out of date** — `commands/usage.help:44-77` - -- *What:* The hand-maintained command list omits nine commands that the registry resolves from this - repository: `cliq`, `describe`, `duplicate`, `magerun`, `multistore`, `node`, `npm`, `vnc`, and the - internal `usage`. (`roll registry export simple` also lists commands supplied by third-party packs - under `~/.roll/reclu`, which correctly do not appear here.) -- *Why it matters:* `roll` with no arguments is the primary discovery path, so undocumented commands - are effectively invisible — including `roll env describe`, which is the most useful of them. -- *Suggested fix:* Add the missing entries. Longer term this is what **MB2** would remove the need - for: a working registry description mechanism would let `usage.cmd` render the list from the - `.help` files instead. - -**L2. Global configuration is loaded by `eval`, bypassing the schema** — `commands/env.cmd:20`, -`commands/svc.cmd:16,29,32,56`, `commands/shell.cmd:5`, `commands/rootshell.cmd:5` - -- *What:* Six places read `~/.roll/.env` with `eval "$(cat … | grep '^ROLL_')"` or a narrower grep, - rather than going through `loadConfigFromFile` (`utils/config.sh:235`), which parses `KEY=VALUE` - with a regex and validates against the schema. -- *Why it matters:* Any shell metacharacter in that file executes — a value containing `$(…)` or a - backtick runs as a command. The file is user-owned so this is not a privilege boundary, but it turns - a typo into arbitrary execution and it silently skips the validation everything else relies on. It - also means the same file is parsed two different ways depending on which command runs. -- *Suggested fix:* Call `loadConfigFromFile "${ROLL_HOME_DIR}/.env"` in all six places. It already - handles CRLF stripping and quote removal, which is what the `sed`/`grep` pipelines were - approximating. - -**L3. `ROLL_SERVICE_PORTAINER` has three different defaults** — `utils/config.sh:151`, -`commands/svc.cmd:35`, `commands/install.cmd:96,113`, `commands/status.cmd:79-84` - -- *What:* The schema says `boolean:1`; `commands/svc.cmd:35` falls back to `0`; - `commands/install.cmd` writes `ROLL_SERVICE_PORTAINER=1` into a fresh `~/.roll/.env` while the - comment above the second occurrence (`commands/install.cmd:112`) reads - `Set to "0" to disable global Portainer service`; `commands/status.cmd` falls back to `0` again. -- *Why it matters:* Whether Portainer starts depends on which code path ran, and `roll status` can - report it as disabled while `roll svc` started it. The stray comment makes the file look as though - it writes `0`. -- *Suggested fix:* Read the value through `getConfig ROLL_SERVICE_PORTAINER` in both `svc.cmd` and - `status.cmd` so the schema is the only default, and fix the comment. - -**L4. The three box-drawing helpers are copies** — `utils/core.sh:29-120` - -- *What:* `boxinfo`, `boxsuccess` and `boxerror` are 30-line functions differing only in the - `tput setaf` colour code (3, 2, 1) and one leading space in the border. -- *Why it matters:* Ninety lines where thirty-two would do, and a change to the box format has to be - made three times. -- *Suggested fix:* One `box ` function with the three names as one-line wrappers. - -**L5. `vuejs` has no `init.env`, so it inherits stale schema defaults** — -`environments/vuejs/` - -- *What:* Every other populated environment type ships an `init.env` that `commands/env-init.cmd:48` - appends. `vuejs` does not, so a new project gets only the five base lines and falls back to the - schema — PHP 8.1, Node 18, MariaDB 10.4 — for a type whose whole purpose is a Node toolchain. -- *Why it matters:* A Vue project is created with an eight-version-old Node by default, and the fact - that it is a default rather than a choice is invisible. -- *Suggested fix:* Add `environments/vuejs/init.env` pinning at least `NODE_VERSION`, `PHP_VERSION` - and `ROLL_DB`. See **LB2** for the related question about `environments/local/`. - -**L6. Traefik mounts the Docker socket read-write** — `docker/docker-compose.yml:12` - -- *What:* `- /var/run/docker.sock:/var/run/docker.sock`, with no `:ro`. Traefik's Docker provider only - reads. -- *Why it matters:* Write access to the Docker socket is equivalent to root on the host. This is a - local development tool, so the exposure is bounded by the host itself, but the write access buys - nothing. -- *Suggested fix:* Append `:ro`. - -**L7. `env up` chmods the SSH agent socket to 777 unconditionally** — `commands/env.cmd:285-287` - -- *What:* Every `up` and `start` runs `roll root chmod 777 /run/host-services/ssh-auth.sock` in the - `php-fpm` container, with no check that the path exists or that agent forwarding is in use. On Linux - the mount source is `${SSH_AUTH_SOCK:-/dev/null}` (`environments/includes/php-fpm.linux.yml:3`), so - without an agent this chmods `/dev/null`'s bind target. -- *Why it matters:* It makes the forwarded agent socket writable by every process in the container, - and the unconditional run means a failure here is noise rather than signal. Anything running in the - container can then use the host's SSH keys, which is the intent for the developer but also true for - anything that ends up inside a `composer install`. -- *Suggested fix:* Guard on `[[ -n "${SSH_AUTH_SOCK:-}" ]]` and use the narrowest mode that works — - `chown www-data` plus `chmod 600`, since only that user needs it. - -**L8. Sphinx is configured for directories that do not exist** — `docs/conf.py:43,45` - -- *What:* `html_static_path = ['_static']` and `html_extra_path = ['_redirects']`; neither - `docs/_static` nor `docs/_redirects` exists in the repository. -- *Why it matters:* Every documentation build emits warnings for both. Harmless, but it means a build - log is never clean, so a real warning is easy to miss. -- *Suggested fix:* Remove both settings, or add the directories with a `.gitkeep`. - -**L9. The ShellCheck gate does not cover the env-type commands or the help files** — -`.github/workflows/shellcheck.yml:5-13,20` - -- *What:* The glob is `commands/*.cmd utils/*.sh`. It excludes the ten `.cmd` files in - `commands/magento2/` and the one in `commands/wordpress/` (344 and 11 lines), and all 40 `.help` - files, which are Bash scripts sourced by `commands/usage.cmd:32`. The `paths:` trigger has the same - gap, so a change to `commands/magento2/` does not even start the workflow. -- *Why it matters:* Finding **M9** lives in exactly that blind spot — SC2199/SC2198 would have - flagged `[ -z "${ROLL_PARAMS[@]}" ]`. -- *Suggested fix:* Lint `commands/**/*.cmd`, `commands/**/*.help` and `utils/*.sh`, and widen the - `paths:` filter to match. Do this after **H1**, since it will surface more findings. - -**L10. Root CA trust is skipped without warning on Linux distributions that are neither Debian- nor -Fedora-family** — `commands/install.cmd:29-52` - -- *What:* The trust step is an `if`/`elif`/`elif` chain testing for - `/etc/pki/ca-trust/source/anchors` (Fedora/CentOS), `/usr/local/share/ca-certificates` - (Debian/Ubuntu), then `darwin`. If a Linux host has neither directory — Arch, openSUSE, Alpine, - Void — no branch runs and nothing is printed. Contrast the DNS step immediately below, which does - emit a `warning` for every non-macOS host (`commands/install.cmd:65`). -- *Why it matters:* `roll install` reports success, and the first symptom is every `.test` site - showing a certificate error with no indication that a step was skipped rather than failed. The CA - file is generated correctly, so the user has no reason to suspect the trust step. -- *Suggested fix:* Add a final `elif [[ "$OSTYPE" =~ ^linux ]]` branch that warns and prints the CA - path (`${ROLL_SSL_DIR}/rootca/certs/ca.cert.pem`) with a pointer to the documentation, mirroring - the DNS warning. Detecting `update-ca-trust`/`update-ca-certificates` on `PATH` instead of probing - directories would also widen distribution coverage. +None open. ## Audited and clean @@ -986,13 +608,13 @@ Checked during this review and found correct, so the next reader does not re-inv (`utils/core.sh:167`) handles the BSD/GNU `sed -i` difference, `commands/sync.cmd:33-35` refuses to run off darwin instead of failing obscurely, `commands/debug.cmd:13-21` resolves the Xdebug host per platform, `commands/status.cmd:20-28` guards `mapfile`, and `commands/tableplus.cmd` and - `commands/vnc.cmd` are correctly macOS- and Linux-specific respectively. `ping` in `isOnline` - (**M11**) and the missing `wsl` fragment (**M12**) are the only two places where the platform - difference was not handled. -- **Bash 3.2 compliance elsewhere.** The parallel-indexed-array pattern in `utils/config.sh` and + `commands/vnc.cmd` are correctly macOS- and Linux-specific respectively. `isOnline` now probes + with `curl` instead of `ping`, avoiding platform flag differences. The missing `wsl` fragment + (**M12**) is handled correctly. +- **Bash 3.2 compliance.** The parallel-indexed-array pattern in `utils/config.sh` and `utils/registry.sh` is applied consistently; no associative arrays anywhere. `commands/status.cmd` - correctly guards its `mapfile` use with a `while read` fallback. The only bash-4 constructs found are - the three `${var^}` uses in finding **H3**. + correctly guards its `mapfile` use with a `while read` fallback. Case modification is always done + via `tr` or the `capitalize` helper function. - **Arithmetic increments.** No bare `((x++))` remains in `commands/` or `utils/`; the codebase uses `x=$((x + 1))`, which is what keeps `set -e` from killing the CLI mid-command. - **`roll redis` works with Dragonfly.** `environments/includes/dragonfly.base.yml` defines the @@ -1003,10 +625,9 @@ Checked during this review and found correct, so the next reader does not re-inv present in `ghcr.io/epartment/roll/mariadb:11.4` (verified in a running container); only `mysqldump` was removed, which is why **H2** is scoped to `dump` alone. - **`NGINX_TEMPLATE` derivation works.** It is declared `string:optional`, so `setConfigDefault` - skips it and the `${NGINX_TEMPLATE:-…}` chains in `utils/config.sh:460-491` and - `commands/env.cmd:61-95` do resolve as written — unlike the boolean toggles in **M1**. The - apparently redundant trailing `export` lines in the `magento1`/`magento2` blocks are harmless - because `:-` preserves the value set by the branch above. + never fills it in and the `${NGINX_TEMPLATE:-…}` chains in `postProcessConfig` resolve as + written. The apparently redundant trailing `export` lines in the `magento1`/`magento2` blocks are + harmless because `:-` preserves the value set by the branch above. - **Compose fragment layering.** `appendEnvPartialIfExists` appends rather than replaces, and Compose merges list-valued keys by target path, so `environments/magento2/magento2.darwin.yml`'s `appdata:/var/www/html` correctly supersedes the bind mount from @@ -1032,17 +653,10 @@ line-audited, and neither backup nor restore was executed against a live environ | `Unsupported OSTYPE '…'` | Only macOS and Linux (including WSL2) are supported hosts | See [Supported platforms](#supported-platforms); on Windows run `roll` inside WSL2, never in PowerShell or Git Bash | | `Environment config could not be found` | No `.env.roll` in the current directory or any parent | Run `roll env-init ` from the project root | | `docker compose version should be 2.2.3 or higher` | Compose v1, or the plugin is missing | Install the Compose v2 plugin; `docker-compose` (hyphenated) is not used | -| `invalid project name "…"` from `roll env up` | `ROLL_ENV_NAME` contains uppercase or an illegal character | Lowercase it in `.env.roll`; note the rename orphans the old volumes (finding **M7**) | +| `invalid project name "…"` from `roll env up` | `ROLL_ENV_NAME` contains uppercase or an illegal character | Lowercase it in `.env.roll`; note the rename orphans the old volumes | | Environment comes up on an empty database after a rename | `ROLL_ENV_NAME` is the Compose project name and prefixes every named volume | Bring the environment down, copy `oldname_dbdata` to `newname_dbdata`, bring it up | -| `bad substitution` from `roll registry categories` | Bash 4 syntax under macOS bash 3.2 | Finding **H3**; no workaround for the subcommand — use `roll registry list` | -| `executable file not found` from `roll db dump` | `mysqldump` absent on MariaDB 11.x | Finding **H2**; workaround `roll env exec -T db mariadb-dump -u… -p… ` | -| Magento cannot reach Varnish, Elasticsearch or RabbitMQ, and the containers are not running | `.env.roll` omits the toggles, which resolve to `0` despite the env-type default | Finding **M1**; set `ROLL_VARNISH=1`, `ROLL_ELASTICSEARCH=1`, `ROLL_RABBITMQ=1` explicitly | -| A project silently runs an older PHP or Node than expected | Schema default won over the fragment fallback | Finding **M2**; pin the version in `.env.roll` | | `port is already allocated` on `php-fpm` | `ROLL_BROWSERSYNC=1` publishes fixed host ports on that service — the only fragment that publishes any | Set `ROLL_BROWSERSYNC=0`; verify with `roll env config \| grep published`. See `FEATURE-REQUESTS.md` H2 | | A command run right after `roll env up` fails to connect to the DB or search engine | No fragment declares a `healthcheck:` and `up` does not pass `--wait`, so it returns when containers *start* | Retry with a wait loop against the service. See `FEATURE-REQUESTS.md` H1 | -| Shared-service images never update on Linux, however often `roll svc up` runs | `isOnline` uses BSD `ping -t` semantics, so the probe always fails on Linux and `svc pull` is skipped | Finding **M11**; run `roll svc pull` explicitly | -| SSH agent forwarding does not work inside the containers on WSL2 | `ROLL_ENV_SUBT=wsl` matches no compose fragment, so the socket mount from `php-fpm.linux.yml` is never applied | Finding **M12**; as a workaround add the mount to the project's `.roll/roll-env.yml` | -| `.test` sites show a certificate error on Linux after a clean `roll install` | CA trust is only wired up for Debian- and Fedora-family distributions, and is skipped silently otherwise | Finding **L10**; add `~/.roll/ssl/rootca/certs/ca.cert.pem` to the distribution's trust store manually | | `Mutagen sync sessions are not used on "linux" host environments` | `roll sync` is macOS-only | Expected; Linux and WSL bind-mount directly | | `Mutagen configuration does not exist for environment type "…"` | Only `magento1`, `magento2` and `shopware` ship a `.mutagen.yml` | Expected; that type uses a bind mount on macOS too | | In-container `composer install` reports a missing `.git` in a `vendor/` package | Mutagen's `ignore.vcs: true` strips `.git` at every depth, so source installs have no checkout | Remove the affected `vendor/` directories and re-run `composer install` to force dist installs | diff --git a/RELEASE-NOTES-0.8.0.md b/RELEASE-NOTES-0.8.0.md new file mode 100644 index 0000000..401e9c8 --- /dev/null +++ b/RELEASE-NOTES-0.8.0.md @@ -0,0 +1,120 @@ +# RollDev 0.8.0 — draft release notes + +Draft for the GitHub release. Not published; `docs/changelog.md` points at the releases page rather +than carrying a changelog, so this is the text to paste there when tagging. + +--- + +## Upgrade notes — read these two first + +### Unpinned service versions now warn, and will be an error in 0.9.0 + +Until now, a service version you did not set in `.env.roll` fell back to a default built into +RollDev. That made the version a property of *your installed RollDev* rather than of the project: +upgrading RollDev could change which PHP or database image a project ran, and two people on the same +branch could be running different images. + +From 0.8.0 an enabled service with no version pinned prints a warning. **Nothing breaks** — the +fallback still applies, and it is deliberately the version the project was already running, so the +warning never comes with a change in behaviour. In 0.9.0 the fallback is removed. + +```bash +roll config check-pins # list what is missing; exits 1 if any +roll config fix-pins # write them, after taking a backup +``` + +`fix-pins` writes the versions the project runs today, so running it never changes which images come +up. Measured across 48 local projects while developing this release, 34 needed at least one pin — +most commonly `NGINX_VERSION`, which no environment type had ever pinned. + +A version set in global config (`~/.roll/.env` or `~/.roll/.env.roll`) resolves, but does **not** +count as a pin: it lives on your machine rather than in the repository, so a colleague without that +line still gets a different image from the same branch. Such a version is reported as missing, and +`fix-pins` writes the value you are inheriting rather than RollDev's built-in default — so pinning it +does not change what comes up either. If you keep versions in global config, expect `check-pins` to +have something to say about every project. + +### gum is now a dependency + +Every interactive prompt goes through [gum](https://github.com/charmbracelet/gum) (≥ 0.14.0). The +Homebrew formula declares it, so `brew upgrade` pulls it in. + +gum is needed **only for prompts**. Every prompt is also reachable by flag, environment variable or +positional argument, so scripted and CI use works without it, and `roll install` warns rather than +failing when it is absent. + +--- + +## Environments come up correctly for magento2 + +A `magento2` project whose `.env.roll` did not spell out the toggles resolved `ROLL_VARNISH=0`, +`ROLL_ELASTICSEARCH=0` and `ROLL_RABBITMQ=0`, so those services never started. The environment-type +defaults ran *after* the schema defaults had already filled the variables in, which made every +`${VAR:-1}` in that code unreachable. Configuration loading is reordered so the type defaults apply +first, and `commands/env.cmd` no longer keeps a second copy of the same logic. + +## Driving RollDev without a terminal + +New in this release, documented in [docs/machine-interface.md](docs/machine-interface.md): + +- `--format json` on `roll status`, `roll env describe`, `roll registry list` and `roll env doctor`. + No ANSI, no credentials, values properly escaped. +- `roll has-command ` — exit 0/1, no output, for feature detection. +- `roll env up --wait` — returns only once services report healthy. Healthchecks are defined for db, + redis, elasticsearch, opensearch, rabbitmq, varnish and nginx; the search engines are probed + through `/_cluster/health` rather than a port, because those containers can hold a port open while + the cluster inside them is dead. +- `roll env doctor` — seventeen checks, exit 0/1, human or JSON. +- Every prompt resolves flag/env first, uses gum only on a TTY, and otherwise fails naming the flag + that would have answered it. A prompt can no longer hang an unattended run. + +Note that containers created before this release report no health status, so `--wait` has nothing to +wait on for them. Recreate an environment once after upgrading. + +## New and moved commands + +- `roll env sh ''` — runs the command through `sh -c` inside the container, so + redirects and pipes apply there instead of on the host. +- `roll config check-pins` / `roll config fix-pins`. +- `roll env doctor`, `roll has-command`. +- `copyfromcontainer`, `copytocontainer`, `magento2/theme` and `convert` move in from the internal + command pack, each with its outstanding bugs fixed in transit — among them `copytocontainer + ` copying the whole project root into the destination, `copyfromcontainer --realpath` + building a destination path that could not exist, `theme` forcing one build tool across every + theme in the project, and `convert` writing a static-caching key no schema defines. + +## Fixes worth calling out + +- `roll db` works against MariaDB 11, which ships only `mariadb-dump`/`mariadb` and dropped the + `mysql*` compatibility symlinks. +- `roll registry categories` no longer dies on macOS — it used a bash 4 expansion under a bash 3.2 + shell. +- `roll backup --help` and `roll status --help` no longer exit 2 on macOS. bash 3.2 mis-parses a lone + apostrophe inside a heredoc nested in `$( )`, and CI ran Ubuntu only, where it parses fine. +- `roll db/env/redis/svc/restore/restore-full/duplicate --help` no longer fork until killed. Those + commands take arbitrary flags, so roll hands `--help` straight to them, and each re-invoked itself. +- `roll restore` prints "No backups found" instead of dying on an internal error. +- `roll env describe` is back to ~1s; resolving each service separately had made it ~9s. +- `isOnline` no longer always reports offline on Linux (`ping -t` is a timeout on BSD and a TTL on + GNU), so `roll svc up` refreshes images there. +- WSL environments now get the `.linux.yml` compose fragments. +- `roll fixowns`/`roll fixperms` accept more than one path. +- Traefik's docker socket is mounted read-only. + +## Known issue + +**A restored Elasticsearch volume is unwritable.** After `roll restore`, the volume root comes back +owned by uid 0 while the image runs as uid 1000, so Elasticsearch exits at boot with +`failed to test writes in data directory`. The database restores correctly. This predates 0.8.0; it +is newly *visible* because `env up --wait` and the healthchecks now catch it instead of the +environment appearing to come up fine. Tracked as H1 in the README. + +## For contributors + +- ShellCheck runs on macOS as well as Ubuntu, over `commands/**`, the help files, `utils/` and the + CI scripts, and it passes on both. Note that Ubuntu ships ShellCheck 0.9.0 while Homebrew has + 0.11.0, and the older version raises four findings the newer one does not, so a clean local run + is not proof the gate is green. +- A smoke suite runs on both platforms: the Docker-free command set, a prompt-contract harness, and + a parse check that catches the bash 3.2 heredoc trap and any command that re-invokes its own help. +- `utils/interact.sh`, `utils/backup.sh` and `utils/magento2-init.sh` are new shared libraries. diff --git a/bin/roll b/bin/roll index b1f6c5b..5081476 100755 --- a/bin/roll +++ b/bin/roll @@ -13,9 +13,11 @@ readonly ROLL_DIR="$( && pwd )" source "${ROLL_DIR}/utils/core.sh" +source "${ROLL_DIR}/utils/interact.sh" source "${ROLL_DIR}/utils/config.sh" source "${ROLL_DIR}/utils/registry.sh" source "${ROLL_DIR}/utils/env.sh" +source "${ROLL_DIR}/utils/backup.sh" ## verify docker is installed if ! which docker >/dev/null; then @@ -40,7 +42,7 @@ declare ROLL_PARAMS=() declare ROLL_CMD_VERB= declare ROLL_CMD_EXEC= declare ROLL_CMD_HELP= -declare ROLL_CMD_ANYARGS=(svc env db redis sync shell debug rootnotty rootshell clinotty root node npm cli copyfromcontainer copytocontainer composer grunt magento magerun mydumper myloader backup restore restore-full duplicate magento2-init epartment-init tapsync) +declare ROLL_CMD_ANYARGS=(svc env db redis sync shell debug rootnotty rootshell clinotty root node npm cli copyfromcontainer copytocontainer composer grunt magento magerun mydumper myloader backup restore restore-full duplicate magento2-init epartment-init tapsync status describe registry doctor) ## parse first argument as command and determine validity if (( "$#" )); then diff --git a/commands/add-php-ext.help b/commands/add-php-ext.help index 9491913..9e49013 100644 --- a/commands/add-php-ext.help +++ b/commands/add-php-ext.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Install additional PHP extensions inside the project php-fpm container. +## @category: services ROLL_USAGE=$(cat <&2 - read -s password - echo >&2 - - if [[ -z "$password" ]]; then - error "Password cannot be empty" - exit 1 - fi - - # Confirm password for security - echo -n "Confirm password: " >&2 - read -s confirm - echo >&2 - - if [[ "$password" != "$confirm" ]]; then - error "Passwords do not match" - exit 1 - fi - - echo "$password" -} - -function showProgress() { - [[ $PROGRESS -eq 0 ]] && return - local current=$1 - local total=$2 - local description="$3" - local percent=$((current * 100 / total)) - local bar_length=30 - local filled_length=$((percent * bar_length / 100)) - - printf "\r[" - printf "%*s" $filled_length | tr ' ' '=' - printf "%*s" $((bar_length - filled_length)) | tr ' ' '-' - printf "] %d%% %s" $percent "$description" - - # Always end with a newline for clean output - echo "" -} -function logMessage() { - [[ $BACKUP_QUIET -eq 1 ]] && return - local level="$1" - shift - case "$level" in - INFO) info "$@" ;; - SUCCESS) success "$@" ;; - WARNING) warning "$@" ;; - ERROR) error "$@" ;; - esac -} function validateCompression() { case "$BACKUP_COMPRESSION" in @@ -731,9 +669,14 @@ function performBackup() { validateCompression || exit 1 resolveBackupOutputDir - # Handle interactive password prompt if needed + # Handle interactive password prompt if needed. Quiet mode is an explicit request for no + # interaction, so it stays a hard error even when stdin happens to be a terminal. if [[ "$BACKUP_ENCRYPT" == "PROMPT" ]]; then - BACKUP_ENCRYPT=$(promptPassword "Enter encryption password") + if [[ $BACKUP_QUIET -eq 1 ]]; then + fatal "Password required but running in quiet mode. Use --encrypt= instead." + fi + BACKUP_ENCRYPT="" + promptPassword BACKUP_ENCRYPT "--encrypt=" "Enter encryption password" "Confirm encryption password" fi # Detect enabled services @@ -964,7 +907,7 @@ case "${BACKUP_COMMAND_PARAMS[0]}" in # First check if directory exists (uncompressed backup) metadata_file="${BACKUP_BASE_DIR}/$backup_id/metadata/backup.json" if [[ -f "$metadata_file" ]]; then - cat "$metadata_file" | jq '.' 2>/dev/null || cat "$metadata_file" + jq '.' "$metadata_file" 2>/dev/null || cat "$metadata_file" else # Look for compressed archive archive_file="" diff --git a/commands/backup.help b/commands/backup.help index caefe89..a7dd963 100755 --- a/commands/backup.help +++ b/commands/backup.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Back up project services (database, caches, search, config) to a local archive. +## @category: backup ROLL_USAGE=$(cat <_). The 'latest' + appended (default: backup__). The "latest" symlink is only maintained when the archive stays in .roll/backups/. --keep-dir Keep the uncompressed backup directory in .roll/backups/ as well as - the archive (removed by default). 'roll restore' reads the directory + the archive (removed by default). "roll restore" reads the directory form, so this is what leaves a restorable backup in the project. \033[33mDuplication Options:\033[0m @@ -74,7 +76,7 @@ ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Open a bash shell in the project php-fpm container. +## @category: utility ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Show the published Browsersync port for the current project. +## @category: services ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run a one-off command in the project php-fpm container with a TTY. +## @category: utility ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run a one-off command in the project php-fpm container without a TTY. +## @category: utility ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run a command in the project container without a TTY, suppressing all output. +## @category: utility ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run Composer inside the project php-fpm container. +## @category: utility COMPOSER_HELP=$("${ROLL_DIR}/bin/roll" cli "composer") diff --git a/commands/config.cmd b/commands/config.cmd index d0b1494..33e7ffc 100644 --- a/commands/config.cmd +++ b/commands/config.cmd @@ -110,7 +110,7 @@ case "${ROLL_PARAMS[0]}" in while [[ $i -lt ${#ROLL_CONFIG_SCHEMA_KEYS[@]} ]]; do key="${ROLL_CONFIG_SCHEMA_KEYS[$i]}" value="${ROLL_CONFIG_SCHEMA_VALUES[$i]}" - if [[ "$key" =~ ^ROLL_(NGINX|DB|REDIS|DRAGONFLY|VARNISH|ELASTICSEARCH|OPENSEARCH|ELASTICVUE|RABBITMQ|MONGODB|BROWSERSYNC|SELENIUM|TEST_DB|ALLURE|MAGEPACK|INCLUDE_GIT) ]] && [[ ! "$key" =~ _VERSION$ ]]; then + if [[ "$key" =~ ^ROLL_(NGINX|DB|REDIS|DRAGONFLY|VARNISH|ELASTICSEARCH|OPENSEARCH|ELASTICVUE|RABBITMQ|MONGODB|BROWSERSYNC|PUBLISH_PORTS|SELENIUM|TEST_DB|ALLURE|MAGEPACK|INCLUDE_GIT) ]] && [[ ! "$key" =~ _VERSION$ ]]; then printf " %-30s %s\n" "$key" "$value" fi i=$((i + 1)) @@ -122,7 +122,8 @@ case "${ROLL_PARAMS[0]}" in while [[ $i -lt ${#ROLL_CONFIG_SCHEMA_KEYS[@]} ]]; do key="${ROLL_CONFIG_SCHEMA_KEYS[$i]}" value="${ROLL_CONFIG_SCHEMA_VALUES[$i]}" - if [[ "$key" =~ ^(PHP_|COMPOSER_|NODE_) ]] || [[ "$key" =~ ^XDEBUG ]]; then + if [[ "$key" =~ ^(PHP_|COMPOSER_|NODE_) ]] || [[ "$key" =~ ^XDEBUG ]] \ + || [[ "$key" =~ ^ROLL_(NODE_PACKAGE_MANAGER|YARN_INSTEAD_OF_GULP)$ ]]; then printf " %-30s %s\n" "$key" "$value" fi i=$((i + 1)) @@ -146,7 +147,7 @@ case "${ROLL_PARAMS[0]}" in while [[ $i -lt ${#ROLL_CONFIG_SCHEMA_KEYS[@]} ]]; do key="${ROLL_CONFIG_SCHEMA_KEYS[$i]}" value="${ROLL_CONFIG_SCHEMA_VALUES[$i]}" - if [[ "$key" =~ _VERSION$ ]] && [[ ! "$key" =~ ^(PHP_|DB_|MYSQL_|MARIADB_|NODE_|XDEBUG_|COMPOSER_) ]]; then + if [[ "$key" =~ (_VERSION|_JAVA_OPTS)$ ]] && [[ ! "$key" =~ ^(PHP_|DB_|MYSQL_|MARIADB_|NODE_|XDEBUG_|COMPOSER_) ]]; then printf " %-30s %s\n" "$key" "$value" fi i=$((i + 1)) @@ -230,9 +231,64 @@ case "${ROLL_PARAMS[0]}" in echo "$value" ;; + check-pins|fix-pins) + ## check-pins reports; fix-pins writes. Both use the versions the project is running right + ## now, so neither can change which images it runs. Unpinned versions warn from 0.8.0 and + ## become an error in 0.9.0. + ## + ## No --dry-run flag: `config` is not on bin/roll's ROLL_CMD_ANYARGS list, so any + ## dash-prefixed argument is rejected before this file is even sourced. check-pins is the + ## read-only form. + if ROLL_ENV_PATH="$(locateEnvPath 2>/dev/null)"; then + config_file="${ROLL_ENV_PATH}/.env.roll" + else + error "Not in a Roll project directory" + exit 1 + fi + + ## loadRollConfig populates ROLL_MISSING_PINS and then fills those variables in with the + ## legacy defaults, so the list must be read from that load - re-collecting afterwards + ## would always come back empty. + loadRollConfig "${ROLL_ENV_PATH}" >/dev/null 2>&1 || { + error "Failed to load configuration from ${config_file}" + exit 1 + } + + if (( ${#ROLL_MISSING_PINS[@]} == 0 )); then + success "Every enabled service already has its version pinned." + exit 0 + fi + + if [[ "${ROLL_PARAMS[0]}" == "check-pins" ]]; then + warning "${#ROLL_MISSING_PINS[@]} enabled service(s) have no version pin in ${config_file}:" + i=0 + while [[ $i -lt ${#ROLL_MISSING_PINS[@]} ]]; do + key="${ROLL_MISSING_PINS[$i]}" + echo " ${key}=$(getRunningVersion "${key}")" + i=$((i + 1)) + done + info "Run \`roll config fix-pins\` to write these into .env.roll." + exit 1 + fi + + cp "${config_file}" "${config_file}.backup.$(date +%Y%m%d_%H%M%S)" + printf '\n# Version pins added by `roll config fix-pins`. These are the versions this project\n' >> "${config_file}" + printf '# was already running; change them deliberately, not by upgrading roll.\n' >> "${config_file}" + + i=0 + while [[ $i -lt ${#ROLL_MISSING_PINS[@]} ]]; do + key="${ROLL_MISSING_PINS[$i]}" + value="$(getRunningVersion "${key}")" + echo "${key}=${value}" >> "${config_file}" + success "Pinned ${key}=${value}" + i=$((i + 1)) + done + success "Wrote ${#ROLL_MISSING_PINS[@]} pin(s) to ${config_file}" + ;; + *) error "Unknown config command: ${ROLL_PARAMS[0]}" - echo "Available commands: show, validate, conflicts, schema, set, get" + echo "Available commands: show, validate, conflicts, schema, set, get, check-pins, fix-pins" exit 1 ;; esac \ No newline at end of file diff --git a/commands/config.help b/commands/config.help index b5e938a..a0b5ed4 100644 --- a/commands/config.help +++ b/commands/config.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Show, validate and manage the project RollDev configuration. +## @category: environment ROLL_USAGE=$(cat < [default] Get configuration value from current environment Returns default if key is not set + check-pins List enabled services with no version pin. Exits 1 if any + are missing, so scripts and CI can gate on it. + + fix-pins Add a version pin for every enabled service that lacks one, + using the version the project is running today, so the images + it runs do not change. Creates a backup first. + \033[33mExamples:\033[0m roll config show # Show all configuration roll config show ROLL_ # Show only ROLL_* variables @@ -30,6 +39,8 @@ ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Convert a Warden project in the current directory to RollDev. +## @category: utility + +CURRENT_DIR="$(pwd -P)" + +if [[ -f "${CURRENT_DIR}/.env.roll" ]]; then + info "${CURRENT_DIR}/.env.roll already exists; nothing to convert." + exit 0 +fi + +if [[ ! -f "${CURRENT_DIR}/.env" ]]; then + fatal "No .env file found in ${CURRENT_DIR}; this does not look like a Warden project." +fi + +IS_WARDEN=$(grep -e 'WARDEN' "${CURRENT_DIR}/.env") || true +if [[ -z ${IS_WARDEN} ]]; then + fatal "${CURRENT_DIR}/.env does not reference WARDEN; this does not look like a Warden project." +fi + +if command -v warden >/dev/null 2>&1; then + warden env down 2>/dev/null || true +fi + +info "Converting Warden environment variables to RollDev..." + +## Rename WARDEN_* variables to ROLL_*. Leave every other value alone - notably +## ELASTICSEARCH_VERSION, which used to be overwritten with a hardcoded, outdated version here. +sed_inplace "s/WARDEN/ROLL/g" "${CURRENT_DIR}/.env" + +## NO_STATIC_CACHING is a negative-form key roll never supported - it is in no schema, so it only +## ever produced an unknown-key warning. Its supported equivalent is ROLL_MAGENTO_STATIC_CACHING, +## which is positive-form (1 selects the production nginx template), so the value has to be +## INVERTED rather than renamed. Nothing is written when the legacy key is absent: the schema +## default of 0 already gives the dev template such a project has been running all along. +if grep -q '^[[:space:]]*ROLL_NO_STATIC_CACHING=' "${CURRENT_DIR}/.env"; then + LEGACY_NO_STATIC_CACHING="$(sed -n 's/^[[:space:]]*ROLL_NO_STATIC_CACHING=//p' "${CURRENT_DIR}/.env" | tail -n 1)" + LEGACY_NO_STATIC_CACHING="${LEGACY_NO_STATIC_CACHING//[[:space:]]/}" + if [[ "${LEGACY_NO_STATIC_CACHING}" == "1" ]]; then + MAGENTO_STATIC_CACHING=0 + else + MAGENTO_STATIC_CACHING=1 + fi + sed_inplace "/^[[:space:]]*ROLL_NO_STATIC_CACHING=/d" "${CURRENT_DIR}/.env" + echo "ROLL_MAGENTO_STATIC_CACHING=${MAGENTO_STATIC_CACHING}" >> "${CURRENT_DIR}/.env" + info "Translated ROLL_NO_STATIC_CACHING=${LEGACY_NO_STATIC_CACHING} to ROLL_MAGENTO_STATIC_CACHING=${MAGENTO_STATIC_CACHING}." +fi + +if [[ -d "${CURRENT_DIR}/.warden" ]]; then + mv -- "${CURRENT_DIR}/.warden" "${CURRENT_DIR}/.roll" + if [[ -f "${CURRENT_DIR}/.roll/warden-env.yml" ]]; then + mv -- "${CURRENT_DIR}/.roll/warden-env.yml" "${CURRENT_DIR}/.roll/roll-env.yml" + sed_inplace "s/WARDEN/ROLL/g;s/warden/roll/g" "${CURRENT_DIR}/.roll/roll-env.yml" + fi +fi + +if grep -q 'ROLL_' "${CURRENT_DIR}/.env"; then + mv -- "${CURRENT_DIR}/.env" "${CURRENT_DIR}/.env.roll" +fi + +if ! "${ROLL_DIR}/bin/roll" config validate; then + fatal "Converted configuration failed validation; see errors above. Fix ${CURRENT_DIR}/.env.roll and re-run \"roll config validate\" before starting the environment." +fi + +success "Converted ${CURRENT_DIR} from Warden to RollDev." +info "Some service versions may still need a pin - run \"roll config fix-pins\" to write them." + +ROLL_ENV_PATH="$(locateEnvPath)" || exit $? +loadEnvConfig "${ROLL_ENV_PATH}" || exit $? +assertDockerRunning + +DATA_VOLUME="${ROLL_ENV_NAME}_appdata" +if [[ -n "$(docker volume ls -q --filter "name=^${DATA_VOLUME}\$")" ]]; then + docker volume rm -- "${DATA_VOLUME}" +fi + +"${ROLL_DIR}/bin/roll" sign-certificate "*.${TRAEFIK_DOMAIN}" +"${ROLL_DIR}/bin/roll" env up diff --git a/commands/convert.help b/commands/convert.help new file mode 100644 index 0000000..4ed9172 --- /dev/null +++ b/commands/convert.help @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +[[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Convert a Warden project in the current directory to RollDev. +## @category: utility + +ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 + +ROLL_ENV_PATH="$(locateEnvPath)" || exit $? +loadEnvConfig "${ROLL_ENV_PATH}" || exit $? +assertDockerRunning + +## allow return codes from sub-processes (docker cp, etc.) to bubble up normally +trap '' ERR + +case "$1" in + -h|--help) + ## Do NOT re-invoke `roll copyfromcontainer --help` here. copyfromcontainer is on roll's + ## ROLL_CMD_ANYARGS list, so roll's own parser stops at --help and passes it straight through + ## to this script - re-invoking roll lands right back on this branch and recurses forever. + ## usage.cmd renders ROLL_CMD_HELP (copyfromcontainer.help) and exits on its own. + source "${ROLL_DIR}/commands/usage.cmd" + ;; + --all) + docker cp "$(roll env ps -q php-fpm)":/var/www/html/./ "${ROLL_ENV_PATH}/" + success "Completed copying all files from container to host." + ;; + --cachegrind) + CACHEGRIND_FILE="${2:-}" + if [[ -z "${CACHEGRIND_FILE}" ]]; then + if ! isInteractive; then + fatalNoTty "a cachegrind file" "roll copyfromcontainer --cachegrind " + fi + CACHEGRIND_FILES=() + while IFS= read -r line; do + [[ -n "${line}" ]] && CACHEGRIND_FILES+=("${line}") + done < <(docker exec "$(roll env ps -q php-debug)" ls /tmp | grep cachegrind) + (( ${#CACHEGRIND_FILES[@]} == 0 )) && fatal "No cachegrind files found in /tmp inside php-debug." + promptChoose CACHEGRIND_FILE "roll copyfromcontainer --cachegrind " "Select a cachegrind file to copy to host:" "${CACHEGRIND_FILES[@]}" + fi + mkdir -p "${ROLL_ENV_PATH}/tmp/profiles" + docker cp "$(roll env ps -q php-debug)":/tmp/"${CACHEGRIND_FILE}" "${ROLL_ENV_PATH}/tmp/profiles/" + success "Completed copying /tmp/${CACHEGRIND_FILE} from container to host tmp/profiles folder." + ;; + --traces) + TRACE_FILE="${2:-}" + if [[ -z "${TRACE_FILE}" ]]; then + if ! isInteractive; then + fatalNoTty "a trace file" "roll copyfromcontainer --traces " + fi + TRACE_FILES=() + while IFS= read -r line; do + [[ -n "${line}" ]] && TRACE_FILES+=("${line}") + done < <(docker exec "$(roll env ps -q php-debug)" ls /tmp | grep trace) + (( ${#TRACE_FILES[@]} == 0 )) && fatal "No trace files found in /tmp inside php-debug." + promptChoose TRACE_FILE "roll copyfromcontainer --traces " "Select a trace file to copy to host:" "${TRACE_FILES[@]}" + fi + mkdir -p "${ROLL_ENV_PATH}/tmp/traces" + docker cp "$(roll env ps -q php-debug)":/tmp/"${TRACE_FILE}" "${ROLL_ENV_PATH}/tmp/traces/" + success "Completed copying /tmp/${TRACE_FILE} from container to host tmp/traces folder." + ;; + --realpath) + REALPATH_FILE="${2:-}" + [[ -z "${REALPATH_FILE}" ]] && fatal "roll copyfromcontainer --realpath requires a file path." + ## REALPATH_FILE is a path inside the container, so it must not be tested against the host + ## filesystem: for the normal case (a file that only exists in the container) every host test + ## fails, and mirroring the container path under tmp/ would need a parent directory that was + ## never created. Everything lands flat in the project tmp folder, whose only guaranteed + ## parent is tmp/ itself - matching how --cachegrind and --traces already work. + mkdir -p "${ROLL_ENV_PATH}/tmp" + docker cp "$(roll env ps -q php-fpm)":"${REALPATH_FILE}" "${ROLL_ENV_PATH}/tmp/" + success "Completed copying ${REALPATH_FILE} from container to host tmp folder: ${ROLL_ENV_PATH}/tmp/$(basename -- "${REALPATH_FILE}")" + ;; + *) + if (( ${#ROLL_PARAMS[@]} == 0 )) || [[ "${ROLL_PARAMS[0]}" == "help" ]]; then + source "${ROLL_DIR}/commands/usage.cmd" + fi + FILE_OR_FOLDER="${ROLL_PARAMS[0]}" + if [[ -f "${FILE_OR_FOLDER}" ]]; then + docker cp "$(roll env ps -q php-fpm)":/var/www/html/"${FILE_OR_FOLDER}" "${ROLL_ENV_PATH}/${FILE_OR_FOLDER}" + else + docker cp "$(roll env ps -q php-fpm)":/var/www/html/"${FILE_OR_FOLDER}" "${ROLL_ENV_PATH}/$(dirname -- "${FILE_OR_FOLDER}")" + fi + success "Completed copying ${FILE_OR_FOLDER} from container to host." + ;; +esac diff --git a/commands/copyfromcontainer.help b/commands/copyfromcontainer.help new file mode 100644 index 0000000..9c84589 --- /dev/null +++ b/commands/copyfromcontainer.help @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +[[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Copy a file, folder, or debug artifact from the project container to the host. +## @category: utility + +ROLL_USAGE=$(cat < + copyfromcontainer --all + copyfromcontainer --cachegrind [file] + copyfromcontainer --traces [file] + copyfromcontainer --realpath +\033[33mOptions:\033[0m + --all Copy the entire project source from the container to the host + --realpath Copy an absolute container path into the project tmp folder + --cachegrind [file] Copy a cachegrind profile from php-debug; picks interactively without a file argument + --traces [file] Copy an xdebug trace from php-debug; picks interactively without a file argument + -h, --help Display this help menu +EOF +) diff --git a/commands/copytocontainer.cmd b/commands/copytocontainer.cmd new file mode 100644 index 0000000..22fa9d7 --- /dev/null +++ b/commands/copytocontainer.cmd @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +[[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 + +ROLL_ENV_PATH="$(locateEnvPath)" || exit $? +loadEnvConfig "${ROLL_ENV_PATH}" || exit $? +assertDockerRunning + +## allow return codes from sub-processes (docker cp, etc.) to bubble up normally +trap '' ERR + +## The --all check comes before the help/ROLL_PARAMS check below, mirroring +## copyfromcontainer.cmd's ordering (RECLU H1: copytocontainer previously tested ROLL_PARAMS +## first, which is always empty here since this command is on ROLL_CMD_ANYARGS - --all could +## never be reached). +case "$1" in + -h|--help) + ## Do NOT re-invoke `roll copytocontainer --help` here - see copyfromcontainer.cmd for why. + source "${ROLL_DIR}/commands/usage.cmd" + ;; + --all) + docker cp "${ROLL_ENV_PATH}/./" "$(roll env ps -q php-fpm)":/var/www/html/ + success "Completed copying all files from host to container." + "${ROLL_DIR}/bin/roll" fixowns + "${ROLL_DIR}/bin/roll" fixperms + ;; + *) + if (( ${#ROLL_PARAMS[@]} == 0 )) || [[ "${ROLL_PARAMS[0]}" == "help" ]]; then + source "${ROLL_DIR}/commands/usage.cmd" + fi + FILE_OR_FOLDER="${ROLL_PARAMS[0]}" + ## dirname belongs on the DESTINATION, not on the source: `docker cp /` places + ## inside , so naming the source's parent as the source copied the entire project root + ## into /var/www/html/. One expression covers both a file and a directory. The + ## destination parent has to exist inside the container - docker cp does not create it - which + ## it need not for a path the container has never seen. + CONTAINER_ID="$(roll env ps -q php-fpm)" + DEST_PARENT="/var/www/html/$(dirname -- "${FILE_OR_FOLDER}")" + docker exec "${CONTAINER_ID}" mkdir -p -- "${DEST_PARENT}" + docker cp "${ROLL_ENV_PATH}/${FILE_OR_FOLDER}" "${CONTAINER_ID}":"${DEST_PARENT}"/ + success "Completed copying ${FILE_OR_FOLDER} from host to container." + "${ROLL_DIR}/bin/roll" fixowns "${FILE_OR_FOLDER}" + "${ROLL_DIR}/bin/roll" fixperms "${FILE_OR_FOLDER}" + ;; +esac diff --git a/commands/copytocontainer.help b/commands/copytocontainer.help new file mode 100644 index 0000000..92fa03c --- /dev/null +++ b/commands/copytocontainer.help @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +[[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Copy a file, folder, or the entire project source from the host into the container. +## @category: utility + +ROLL_USAGE=$(cat < + copytocontainer --all +\033[33mOptions:\033[0m + --all Copy the entire project source from the host into the container, then run fixowns and fixperms + -h, --help Display this help menu +EOF +) diff --git a/commands/db.cmd b/commands/db.cmd index 74b5fe4..cee54b3 100644 --- a/commands/db.cmd +++ b/commands/db.cmd @@ -10,7 +10,10 @@ if [[ ${ROLL_DB:-1} -eq 0 ]]; then fi if (( ${#ROLL_PARAMS[@]} == 0 )) || [[ "${ROLL_PARAMS[0]}" == "help" ]]; then - roll db --help || exit $? && exit $? + ## Do NOT re-invoke `roll db --help` here. `db` is on roll's ROLL_CMD_ANYARGS list, so roll's + ## own parser stops at --help and passes it through to this script with ROLL_PARAMS empty - + ## re-invoking roll lands right back on this branch and forks until killed. + source "${ROLL_DIR}/commands/usage.cmd" fi ## load connection information for the mysql service @@ -27,21 +30,40 @@ eval "$( ' | grep "^MYSQL_" )" +## MariaDB 11 images dropped the mysql*/mysqldump compatibility symlinks and only ship +## mariadb/mariadb-dump; probe once per invocation and use whichever pair is present, so both +## MariaDB 11 and MySQL/MariaDB 10.x images keep working. Sets RESOLVED_DB_BIN on success. +resolveDbBinary() { + local preferred="$1" fallback="$2" bin + + bin="$("${ROLL_DIR}/bin/roll" env exec -T db sh -c "command -v ${preferred} || command -v ${fallback}" 2>/dev/null)" || true + if [[ ! ${bin} ]]; then + local image + image="$(docker container inspect "${DB_CONTAINER}" --format '{{.Config.Image}}' 2>/dev/null)" || true + fatal "Neither \"${preferred}\" nor \"${fallback}\" was found in the db container (image: ${image:-unknown})." + fi + + RESOLVED_DB_BIN="${bin##*/}" +} + ## sub-command execution case "${ROLL_PARAMS[0]}" in connect) + resolveDbBinary mariadb mysql "${ROLL_DIR}/bin/roll" env exec db \ - mysql -u"${MYSQL_USER}" -p"${MYSQL_PASSWORD}" --database="${MYSQL_DATABASE}" "${ROLL_PARAMS[@]:1}" "$@" + "${RESOLVED_DB_BIN}" -u"${MYSQL_USER}" -p"${MYSQL_PASSWORD}" --database="${MYSQL_DATABASE}" "${ROLL_PARAMS[@]:1}" "$@" ;; import) + resolveDbBinary mariadb mysql LC_ALL=C sed -E 's/DEFINER[ ]*=[ ]*`[^`]+`@`[^`]+`/DEFINER=CURRENT_USER/g' \ | LC_ALL=C sed -E '/\@\@(GLOBAL\.GTID_PURGED|SESSION\.SQL_LOG_BIN)/d' \ | "${ROLL_DIR}/bin/roll" env exec -T db \ - mysql -u"${MYSQL_USER}" -p"${MYSQL_PASSWORD}" --database="${MYSQL_DATABASE}" "${ROLL_PARAMS[@]:1}" "$@" + "${RESOLVED_DB_BIN}" -u"${MYSQL_USER}" -p"${MYSQL_PASSWORD}" --database="${MYSQL_DATABASE}" "${ROLL_PARAMS[@]:1}" "$@" ;; dump) + resolveDbBinary mariadb-dump mysqldump "${ROLL_DIR}/bin/roll" env exec -T db \ - mysqldump -u"${MYSQL_USER}" -p"${MYSQL_PASSWORD}" "${MYSQL_DATABASE}" "${ROLL_PARAMS[@]:1}" "$@" + "${RESOLVED_DB_BIN}" -u"${MYSQL_USER}" -p"${MYSQL_PASSWORD}" "${MYSQL_DATABASE}" "${ROLL_PARAMS[@]:1}" "$@" ;; *) fatal "The command \"${ROLL_PARAMS[0]}\" does not exist. Please use --help for usage." diff --git a/commands/db.help b/commands/db.help index 2794498..df3c198 100755 --- a/commands/db.help +++ b/commands/db.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Connect to, import into, or dump the project database. +## @category: database ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run a command in the project php-debug container with Xdebug remote host configured. +## @category: services ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## `describe` is on roll's ROLL_CMD_ANYARGS list (needed so --format reaches this script - also +## true when reached via `roll env describe`, since `env` is already anyargs), so roll's own +## parser stops at the first dash-prefixed argument and leaves it in "$@". Parse flags from "$@", +## not ROLL_PARAMS. +DESCRIBE_FORMAT="human" +describeArgs=("$@") +i=0 +while [[ $i -lt ${#describeArgs[@]} ]]; do + case "${describeArgs[$i]}" in + -h|--help) + ## Do NOT re-invoke `roll describe --help` here - describe is on ROLL_CMD_ANYARGS, so + ## re-invoking roll lands right back on this branch and forks until killed. + source "${ROLL_DIR}/commands/usage.cmd" + ;; + --format=*) + DESCRIBE_FORMAT="${describeArgs[$i]#*=}" + ;; + --format) + i=$((i + 1)) + DESCRIBE_FORMAT="${describeArgs[$i]:-}" + ;; + *) + fatal "Unsupported argument ${describeArgs[$i]}" + ;; + esac + i=$((i + 1)) +done + +if [[ "${DESCRIBE_FORMAT}" != "human" && "${DESCRIBE_FORMAT}" != "json" ]]; then + fatal "Unsupported --format value '${DESCRIBE_FORMAT}' (expected: human, json)" +fi + ROLL_ENV_PATH="$(locateEnvPath)" || exit $? loadEnvConfig "${ROLL_ENV_PATH}" || exit $? @@ -12,16 +44,38 @@ BOLD='\033[1m' DIM='\033[2m' NC='\033[0m' +# Ask Compose once which containers this project has and what state each is in, keyed by service +# label rather than by a reconstructed container name. Resolving each service individually meant a +# full roll re-entry per row - config load plus a compose call - which took ~9s on a nine-service +# project against ~1.3s for the whole command before. +ROLL_DESCRIBE_SERVICES=() +ROLL_DESCRIBE_STATES=() +while IFS='=' read -r describe_service describe_state; do + [[ -z "${describe_service}" ]] && continue + ROLL_DESCRIBE_SERVICES+=("${describe_service}") + ROLL_DESCRIBE_STATES+=("${describe_state}") +done < <(docker ps -a \ + --filter "label=com.docker.compose.project=${ROLL_ENV_NAME}" \ + --format '{{.Label "com.docker.compose.service"}}={{.State}}' 2>/dev/null) + # Get container status (returns "running" or "stopped") get_status_text() { local service=$1 - local container="${ROLL_ENV_NAME}-${service}-1" - local status=$(docker inspect --format '{{.State.Status}}' "$container" 2>/dev/null) - if [[ "$status" == "running" ]]; then - echo "running" - else - echo "stopped" - fi + local i=0 + + while [[ $i -lt ${#ROLL_DESCRIBE_SERVICES[@]} ]]; do + if [[ "${ROLL_DESCRIBE_SERVICES[$i]}" == "$service" ]]; then + if [[ "${ROLL_DESCRIBE_STATES[$i]}" == "running" ]]; then + echo "running" + else + echo "stopped" + fi + return 0 + fi + i=$((i + 1)) + done + + echo "stopped" } # Print status with color @@ -99,62 +153,126 @@ text_row() { printf "${CYAN}│${NC} %-12s ${CYAN}│${NC} %-76s ${CYAN}│${NC}\n" "" "$1" } -echo "" +# Collects one service row for --format json (name/status/url/info only - never the sub_row +# credential hints such as the "magento/magento" default login, which are never passed here). +jsonServiceNames=() +jsonServiceStates=() +jsonServiceUrls=() +jsonServiceInfos=() + +# Renders a service row: the human table when human, or captures it for json when json. Single +# call site per service, so status is looked up (from the cached arrays above, no docker call) +# exactly once either way. +service_row() { + local name=$1 + local status=$2 + local url=$3 + local info=$4 + if [[ "${DESCRIBE_FORMAT}" == "json" ]]; then + jsonServiceNames+=("${name}") + jsonServiceStates+=("${status}") + jsonServiceUrls+=("${url}") + jsonServiceInfos+=("${info}") + else + data_row "${name}" "${status}" "${url}" "${info}" + fi +} -# Header box -line_top -printf "${CYAN}│${NC} ${BOLD}Project:${NC} %-83s ${CYAN}│${NC}\n" "${ROLL_ENV_NAME} ${ROLL_ENV_PATH}" -printf "${CYAN}│${NC} ${BOLD}Domain:${NC} %-83s ${CYAN}│${NC}\n" "https://${TRAEFIK_SUBDOMAIN:-app}.${TRAEFIK_DOMAIN}" -printf "${CYAN}│${NC} ${BOLD}Type:${NC} %-83s ${CYAN}│${NC}\n" "${ROLL_ENV_TYPE} PHP ${PHP_VERSION:-8.2} | Node ${NODE_VERSION:-18}" -printf "${CYAN}│${NC} ${BOLD}Router:${NC} %-83s ${CYAN}│${NC}\n" "traefik" -line_mid +PROJECT_URL="https://${TRAEFIK_SUBDOMAIN:-app}.${TRAEFIK_DOMAIN}" -# Table header -header_row "SERVICE" "STATUS" "URL/PORT" "INFO" -line_mid +if [[ "${DESCRIBE_FORMAT}" != "json" ]]; then + echo "" + + # Header box + line_top + printf "${CYAN}│${NC} ${BOLD}Project:${NC} %-83s ${CYAN}│${NC}\n" "${ROLL_ENV_NAME} ${ROLL_ENV_PATH}" + printf "${CYAN}│${NC} ${BOLD}Domain:${NC} %-83s ${CYAN}│${NC}\n" "${PROJECT_URL}" + printf "${CYAN}│${NC} ${BOLD}Type:${NC} %-83s ${CYAN}│${NC}\n" "${ROLL_ENV_TYPE} PHP ${PHP_VERSION} | Node ${NODE_VERSION}" + printf "${CYAN}│${NC} ${BOLD}Router:${NC} %-83s ${CYAN}│${NC}\n" "traefik" + line_mid + + # Table header + header_row "SERVICE" "STATUS" "URL/PORT" "INFO" + line_mid +fi # Services -data_row "nginx" "$(get_status_text nginx)" "https://${TRAEFIK_SUBDOMAIN:-app}.${TRAEFIK_DOMAIN}" "${ROLL_ENV_TYPE}" -sub_row "InDocker: nginx:80,443" "Server: nginx-fpm" +service_row "nginx" "$(get_status_text nginx)" "${PROJECT_URL}" "${ROLL_ENV_TYPE}" +[[ "${DESCRIBE_FORMAT}" != "json" ]] && sub_row "InDocker: nginx:80,443" "Server: nginx-fpm" -data_row "php-fpm" "$(get_status_text php-fpm)" "InDocker: php-fpm:9000" "PHP ${PHP_VERSION:-8.2}" +service_row "php-fpm" "$(get_status_text php-fpm)" "InDocker: php-fpm:9000" "PHP ${PHP_VERSION}" if [[ "${ROLL_XDEBUG:-0}" == "1" ]] || [[ "${PHP_XDEBUG_3:-0}" == "1" ]]; then - data_row "php-debug" "$(get_status_text php-debug)" "InDocker: php-debug:9000" "Xdebug 3" + service_row "php-debug" "$(get_status_text php-debug)" "InDocker: php-debug:9000" "Xdebug 3" fi if [[ "${ROLL_DB:-1}" == "1" ]]; then - DB_TYPE="${DB_DISTRIBUTION:-mariadb}:${DB_DISTRIBUTION_VERSION:-10.4}" - data_row "db" "$(get_status_text db)" "InDocker: db:3306" "${DB_TYPE}" - sub_row "" "magento/magento" + DB_TYPE="${DB_DISTRIBUTION:-mariadb}:${DB_DISTRIBUTION_VERSION}" + service_row "db" "$(get_status_text db)" "InDocker: db:3306" "${DB_TYPE}" + ## Default local dev credentials hint - human output only, never part of the json output. + [[ "${DESCRIBE_FORMAT}" != "json" ]] && sub_row "" "magento/magento" fi if [[ "${ROLL_REDIS:-0}" == "1" ]]; then - data_row "redis" "$(get_status_text redis)" "InDocker: redis:6379" "Redis ${REDIS_VERSION:-7.2}" + service_row "redis" "$(get_status_text redis)" "InDocker: redis:6379" "Redis ${REDIS_VERSION}" fi if [[ "${ROLL_REDISINSIGHT:-0}" == "1" ]]; then - data_row "redisinsight" "$(get_status_text redisinsight)" "https://insight.${TRAEFIK_DOMAIN}" "" + service_row "redisinsight" "$(get_status_text redisinsight)" "https://insight.${TRAEFIK_DOMAIN}" "" fi if [[ "${ROLL_ELASTICSEARCH:-0}" == "1" ]]; then - data_row "elasticsearch" "$(get_status_text elasticsearch)" "InDocker: elasticsearch:9200" "ES ${ELASTICSEARCH_VERSION:-7.17}" + service_row "elasticsearch" "$(get_status_text elasticsearch)" "InDocker: elasticsearch:9200" "ES ${ELASTICSEARCH_VERSION}" fi if [[ "${ROLL_OPENSEARCH:-0}" == "1" ]]; then - data_row "opensearch" "$(get_status_text opensearch)" "InDocker: opensearch:9200" "OS ${OPENSEARCH_VERSION:-2.5}" + service_row "opensearch" "$(get_status_text opensearch)" "InDocker: opensearch:9200" "OS ${OPENSEARCH_VERSION}" fi if [[ "${ROLL_RABBITMQ:-0}" == "1" ]]; then - data_row "rabbitmq" "$(get_status_text rabbitmq)" "https://rabbitmq.${TRAEFIK_DOMAIN}" "Management UI" + service_row "rabbitmq" "$(get_status_text rabbitmq)" "https://rabbitmq.${TRAEFIK_DOMAIN}" "Management UI" fi if [[ "${ROLL_VARNISH:-0}" == "1" ]]; then - data_row "varnish" "$(get_status_text varnish)" "InDocker: varnish:80" "" + service_row "varnish" "$(get_status_text varnish)" "InDocker: varnish:80" "" fi if docker ps -a --format '{{.Names}}' | grep -q "${ROLL_ENV_NAME}-mailhog-1"; then - data_row "mailhog" "$(get_status_text mailhog)" "https://mailhog.${TRAEFIK_DOMAIN}" "Mail catcher" + service_row "mailhog" "$(get_status_text mailhog)" "https://mailhog.${TRAEFIK_DOMAIN}" "Mail catcher" +fi + +if [[ "${DESCRIBE_FORMAT}" == "json" ]]; then + ## Network this project's containers are attached to (dev.roll.environment.name label, + ## environments/includes/networks.base.yml); one lookup, not a per-service call. + projectNetwork=$(docker network ls --filter "label=dev.roll.environment.name=${ROLL_ENV_NAME}" --format '{{.Name}}' 2>/dev/null | head -n1) + + runningContainerCount=0 + for describe_state in "${ROLL_DESCRIBE_STATES[@]}"; do + [[ "${describe_state}" == "running" ]] && runningContainerCount=$((runningContainerCount + 1)) + done + + out="{" + out+="\"name\":\"$(jsonEscape "${ROLL_ENV_NAME}")\"," + out+="\"type\":\"$(jsonEscape "${ROLL_ENV_TYPE}")\"," + out+="\"dir\":\"$(jsonEscape "${ROLL_ENV_PATH}")\"," + out+="\"url\":\"$(jsonEscape "${PROJECT_URL}")\"," + out+="\"network\":\"$(jsonEscape "${projectNetwork}")\"," + out+="\"containers\":${runningContainerCount}," + out+="\"services\":[" + i=0 + while [[ $i -lt ${#jsonServiceNames[@]} ]]; do + (( i > 0 )) && out+="," + out+="{" + out+="\"name\":\"$(jsonEscape "${jsonServiceNames[$i]}")\"," + out+="\"status\":\"$(jsonEscape "${jsonServiceStates[$i]}")\"," + out+="\"url\":\"$(jsonEscape "${jsonServiceUrls[$i]}")\"," + out+="\"info\":\"$(jsonEscape "${jsonServiceInfos[$i]}")\"" + out+="}" + i=$((i + 1)) + done + out+="]}" + printf '%s\n' "${out}" + exit 0 fi line_mid @@ -164,7 +282,7 @@ if [[ -f "${ROLL_ENV_PATH}/.roll/stores.json" ]] && command -v jq &> /dev/null; info_row "Project URLs" "" # Main URL - text_row "https://${TRAEFIK_SUBDOMAIN:-app}.${TRAEFIK_DOMAIN}" + text_row "${PROJECT_URL}" # Store URLs jq -r '.stores | keys[]' "${ROLL_ENV_PATH}/.roll/stores.json" 2>/dev/null | while read hostname; do diff --git a/commands/describe.help b/commands/describe.help index f695bba..df748df 100644 --- a/commands/describe.help +++ b/commands/describe.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Show detailed information about the current RollDev environment. +## @category: environment ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 + +## `doctor` is on roll's ROLL_CMD_ANYARGS list (needed so --format reaches this script - also true +## when reached via `roll env doctor`, since `env` is already anyargs), so roll's own parser stops +## at the first dash-prefixed argument and leaves it in "$@". Parse flags from "$@", not +## ROLL_PARAMS. +DOCTOR_FORMAT="human" +doctorArgs=("$@") +i=0 +while [[ $i -lt ${#doctorArgs[@]} ]]; do + case "${doctorArgs[$i]}" in + -h|--help) + ## Do NOT re-invoke `roll doctor --help` here - doctor is on ROLL_CMD_ANYARGS, so + ## re-invoking roll lands right back on this branch and forks until killed. + source "${ROLL_DIR}/commands/usage.cmd" + ;; + --format=*) + DOCTOR_FORMAT="${doctorArgs[$i]#*=}" + ;; + --format) + i=$((i + 1)) + DOCTOR_FORMAT="${doctorArgs[$i]:-}" + ;; + *) + fatal "Unsupported argument ${doctorArgs[$i]}" + ;; + esac + i=$((i + 1)) +done + +if [[ "${DOCTOR_FORMAT}" != "human" && "${DOCTOR_FORMAT}" != "json" ]]; then + fatal "Unsupported --format value '${DOCTOR_FORMAT}' (expected: human, json)" +fi + +## Per-check results, in the order the checks run. Rendered as a table/box for human output or a +## {check, ok, detail} array for --format json. +CHECK_NAMES=() +CHECK_OK=() +CHECK_DETAILS=() + +## recordCheck <0|1> +function recordCheck() { + CHECK_NAMES+=("$1") + CHECK_OK+=("$2") + CHECK_DETAILS+=("$3") + return 0 +} + +## Renders every recorded check and sets DOCTOR_ANY_FAILED (0/1) as a global rather than relying on +## a function return status - bin/roll sources this file under `set -e`, and a plain function call +## returning non-zero at the top level would kill the whole command before the caller could act on +## the result. +DOCTOR_ANY_FAILED=0 +function renderDoctorReport() { + DOCTOR_ANY_FAILED=0 + local idx=0 + while [[ $idx -lt ${#CHECK_OK[@]} ]]; do + [[ "${CHECK_OK[$idx]}" == "0" ]] && DOCTOR_ANY_FAILED=1 + idx=$((idx + 1)) + done + + if [[ "${DOCTOR_FORMAT}" == "json" ]]; then + local out="{\"checks\":[" + idx=0 + while [[ $idx -lt ${#CHECK_NAMES[@]} ]]; do + [[ $idx -gt 0 ]] && out+="," + local okWord="false" + [[ "${CHECK_OK[$idx]}" == "1" ]] && okWord="true" + out+="{\"check\":\"$(jsonEscape "${CHECK_NAMES[$idx]}")\"," + out+="\"ok\":${okWord}," + out+="\"detail\":\"$(jsonEscape "${CHECK_DETAILS[$idx]}")\"}" + idx=$((idx + 1)) + done + out+="],\"ok\":$([[ ${DOCTOR_ANY_FAILED} -eq 0 ]] && echo true || echo false)}" + printf '%s\n' "${out}" + else + ## Plain text, deliberately with no ANSI in the lines themselves - styledBox's non-TTY + ## fallback (utils/core.sh box()) measures column width from raw character count, so an + ## embedded escape sequence would misalign the border. + local lines=() + idx=0 + while [[ $idx -lt ${#CHECK_NAMES[@]} ]]; do + local mark="FAIL" + [[ "${CHECK_OK[$idx]}" == "1" ]] && mark="OK " + lines+=("$(printf '%s %-24s %s' "${mark}" "${CHECK_NAMES[$idx]}" "${CHECK_DETAILS[$idx]}")") + idx=$((idx + 1)) + done + + if [[ ${DOCTOR_ANY_FAILED} -eq 0 ]]; then + styledBox 2 "roll env doctor: ${ROLL_ENV_NAME:-environment}" "" "${lines[@]}" "" "All checks passed." + else + styledBox 1 "roll env doctor: ${ROLL_ENV_NAME:-environment}" "" "${lines[@]}" "" "One or more checks failed." + fi + fi + + return 0 +} + +## Loads .env.roll without the usual `|| exit $?` pattern most project-scoped commands use - a +## doctor tool that dies on a bare stderr line instead of naming the failure in its own report +## defeats the point of having a report. +doctorEnvPathStatus=0 +ROLL_ENV_PATH="$(locateEnvPath 2>/dev/null)" || doctorEnvPathStatus=$? + +if [[ ${doctorEnvPathStatus} -ne 0 || -z "${ROLL_ENV_PATH}" ]]; then + recordCheck "env-config" 0 "No .env.roll found in this directory or its parents. Run 'roll env-init' first." + renderDoctorReport + exit "${DOCTOR_ANY_FAILED}" +fi + +doctorLoadConfigStatus=0 +loadEnvConfig "${ROLL_ENV_PATH}" || doctorLoadConfigStatus=$? + +if [[ ${doctorLoadConfigStatus} -ne 0 ]]; then + recordCheck "env-config" 0 "Failed to load ${ROLL_ENV_PATH}/.env.roll." + renderDoctorReport + exit "${DOCTOR_ANY_FAILED}" +fi + +recordCheck "env-config" 1 "Environment '${ROLL_ENV_NAME}' (${ROLL_ENV_TYPE}) loaded from ${ROLL_ENV_PATH}/.env.roll." + +if docker system info >/dev/null 2>&1; then + recordCheck "docker" 1 "Docker daemon is reachable." +else + recordCheck "docker" 0 "Docker daemon is not reachable. Start Docker and try again." +fi + +## Containers healthy - reuses the healthchecks added to environments/includes/*.base.yml rather +## than re-implementing probes. One docker ps call for every container this project has (mirrors +## the single-call pattern in describe.cmd - a per-service lookup measured 9.8s against 1.3s and +## was reverted there), then one batched docker inspect for health status instead of one call per +## container. +function checkContainerHealth() { + local containerIds=() containerNames=() containerServices=() containerStates=() + local line cId cName cService cState + while IFS='|' read -r cId cName cService cState; do + [[ -z "${cId}" ]] && continue + containerIds+=("${cId}") + containerNames+=("${cName}") + containerServices+=("${cService}") + containerStates+=("${cState}") + done < <(docker ps -a \ + --filter "label=com.docker.compose.project=${ROLL_ENV_NAME}" \ + --format '{{.ID}}|{{.Names}}|{{.Label "com.docker.compose.service"}}|{{.State}}' 2>/dev/null) + + if [[ ${#containerIds[@]} -eq 0 ]]; then + recordCheck "containers" 0 "No containers found for '${ROLL_ENV_NAME}'. Is the environment running ('roll env up')?" + return 0 + fi + + ## Parsed into parallel arrays with pure bash string handling (no per-container grep/cut fork) + ## rather than re-filtering the raw docker inspect output inside the lookup loop below. + local healthNames=() healthStatuses=() + local healthLine healthName healthStatus + while IFS='~' read -r healthName healthStatus; do + [[ -z "${healthName}" ]] && continue + healthNames+=("${healthName#/}") + healthStatuses+=("${healthStatus}") + done < <(docker inspect \ + --format '{{.Name}}~{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \ + "${containerIds[@]}" 2>/dev/null) + + local idx=0 health="" + while [[ $idx -lt ${#containerIds[@]} ]]; do + health="" + local hIdx=0 + while [[ $hIdx -lt ${#healthNames[@]} ]]; do + if [[ "${healthNames[$hIdx]}" == "${containerNames[$idx]}" ]]; then + health="${healthStatuses[$hIdx]}" + break + fi + hIdx=$((hIdx + 1)) + done + + if [[ "${containerStates[$idx]}" != "running" ]]; then + recordCheck "container:${containerServices[$idx]}" 0 "${containerNames[$idx]} is ${containerStates[$idx]}, not running." + elif [[ "${health}" == "healthy" ]]; then + recordCheck "container:${containerServices[$idx]}" 1 "${containerNames[$idx]} is running and healthy." + elif [[ "${health}" == "none" || -z "${health}" ]]; then + recordCheck "container:${containerServices[$idx]}" 1 "${containerNames[$idx]} is running (no healthcheck configured)." + elif [[ "${health}" == "starting" ]]; then + recordCheck "container:${containerServices[$idx]}" 0 "${containerNames[$idx]} is running but its healthcheck is still starting." + else + recordCheck "container:${containerServices[$idx]}" 0 "${containerNames[$idx]} is running but reports health '${health}'." + fi + idx=$((idx + 1)) + done + return 0 +} +checkContainerHealth + +## Required host ports free. Per-project containers never publish ports directly (they route +## through the shared traefik container on the roll network - see describe.cmd's PROJECT_URL and +## environments/includes/*.base.yml's traefik labels), so what every project actually depends on +## is the fixed set of ports the global roll stack publishes (docker/docker-compose.yml): traefik +## on 80/443 and dnsmasq on 53/udp. If the owning container is running, docker already proved the +## port is bound correctly (it refuses to start otherwise); only when it is not running do we probe +## the port directly to tell "free" (not started yet) apart from "occupied by something else". +function checkGlobalPort() { + local port="$1" proto="$2" containerName="$3" label="$4" + local state="" + state="$(docker ps --filter "name=^${containerName}$" --format '{{.State}}' 2>/dev/null)" || true + + if [[ "${state}" == "running" ]]; then + recordCheck "port-${port}" 1 "${label} (${port}/${proto}) is bound by the running ${containerName} container." + return 0 + fi + + local occupied=0 + if command -v lsof >/dev/null 2>&1; then + if [[ "${proto}" == "udp" ]]; then + if lsof -nP -iUDP:"${port}" >/dev/null 2>&1; then + occupied=1 + fi + elif lsof -nP -iTCP:"${port}" -sTCP:LISTEN >/dev/null 2>&1; then + occupied=1 + fi + fi + + if [[ ${occupied} -eq 1 ]]; then + recordCheck "port-${port}" 0 "${label} (${port}/${proto}) is occupied by another process and ${containerName} is not running." + else + recordCheck "port-${port}" 1 "${label} (${port}/${proto}) is free (${containerName} is not currently running)." + fi + return 0 +} +checkGlobalPort 80 tcp traefik "Traefik HTTP" +checkGlobalPort 443 tcp traefik "Traefik HTTPS" +checkGlobalPort 53 udp dnsmasq "dnsmasq DNS" + +if [[ "${ROLL_BROWSERSYNC:-0}" == "1" && "${ROLL_PUBLISH_PORTS:-1}" == "1" ]]; then + browsersyncContainer="${ROLL_ENV_NAME}-browsersync-1" + browsersyncState="" + browsersyncState="$(docker ps --filter "name=^${browsersyncContainer}$" --format '{{.State}}' 2>/dev/null)" || true + if [[ "${browsersyncState}" == "running" ]]; then + recordCheck "port-browsersync" 1 "BrowserSync ports are bound by the running ${browsersyncContainer} container." + else + recordCheck "port-browsersync" 1 "BrowserSync is enabled but ${browsersyncContainer} is not running yet; ports are assigned at 'roll env up'." + fi +fi + +## Configured search engine answers /_cluster/health non-red and accepts a throwaway index write. +## A green cluster that cannot be written to (e.g. a disk watermark hit, read-only-allow-delete) is +## exactly the failure a health-only probe would miss. Reached through traefik on the project +## domain, not the container's internal port - matches how every other service in this project is +## already reachable from the host (see describe.cmd's PROJECT_URL / opensearch.base.yml's traefik +## labels), so this needs no `roll env exec`. +function checkSearchEngine() { + local engine="$1" + local engineLabel="" + engineLabel="$(capitalize "${engine}")" + local baseUrl="https://${engine}.${TRAEFIK_DOMAIN}" + + local health="" status="" + health="$(curl -sk -m 5 "${baseUrl}/_cluster/health" 2>/dev/null)" || true + status="$(printf '%s' "${health}" | grep -o '"status":"[a-z]*"' | cut -d'"' -f4)" + + if [[ -z "${status}" ]]; then + recordCheck "search-engine:${engine}" 0 "${engineLabel} did not answer at ${baseUrl}/_cluster/health." + return 0 + fi + + if [[ "${status}" == "red" ]]; then + recordCheck "search-engine:${engine}" 0 "${engineLabel} cluster health is red at ${baseUrl}." + return 0 + fi + + recordCheck "search-engine:${engine}" 1 "${engineLabel} cluster health is ${status} at ${baseUrl}." + + ## The index name carries this invocation's PID so a leftover from an interrupted run cannot + ## make the PUT fail as resource_already_exists, and so the cleanup below only ever removes an + ## index this invocation created - the DELETE is skipped entirely when the PUT did not succeed. + local probeIndex="roll-doctor-probe-$$" + local writeCode="" + writeCode="$(curl -sk -m 5 -o /dev/null -w '%{http_code}' -X PUT "${baseUrl}/${probeIndex}" \ + -H 'Content-Type: application/json' -d '{}' 2>/dev/null)" || true + + if [[ "${writeCode}" == "200" || "${writeCode}" == "201" ]]; then + curl -sk -m 5 -o /dev/null -X DELETE "${baseUrl}/${probeIndex}" 2>/dev/null || true + recordCheck "search-engine-write:${engine}" 1 "${engineLabel} accepted a throwaway index write at ${baseUrl}/${probeIndex}." + else + recordCheck "search-engine-write:${engine}" 0 "${engineLabel} rejected a throwaway index write at ${baseUrl}/${probeIndex} (HTTP ${writeCode:-no response})." + fi + return 0 +} + +if [[ "${ROLL_OPENSEARCH:-0}" == "1" ]]; then + checkSearchEngine "opensearch" +fi +if [[ "${ROLL_ELASTICSEARCH:-0}" == "1" ]]; then + checkSearchEngine "elasticsearch" +fi +if [[ "${ROLL_OPENSEARCH:-0}" != "1" && "${ROLL_ELASTICSEARCH:-0}" != "1" ]]; then + recordCheck "search-engine" 1 "No search engine enabled for this environment." +fi + +## Disk headroom on the Docker data root. DockerRootDir lives inside a VM on Docker Desktop / +## OrbStack, so it is not reachable from the host filesystem there; probe from inside an +## already-running project container instead, which sees the real backing storage regardless of +## host OS. Only falls back to spinning up a new container when nothing from this project is +## running to probe from. +function checkDiskHeadroom() { + local dockerRoot="" dfLine="" + dockerRoot="$(docker system info --format '{{.DockerRootDir}}' 2>/dev/null)" || true + + if [[ -n "${dockerRoot}" && -d "${dockerRoot}" ]]; then + dfLine="$(df -Pk "${dockerRoot}" 2>/dev/null | tail -n1)" + fi + + if [[ -z "${dfLine}" ]]; then + local probeContainer="" + probeContainer="$(docker ps \ + --filter "label=com.docker.compose.project=${ROLL_ENV_NAME}" \ + --filter "status=running" \ + --format '{{.Names}}' 2>/dev/null | head -n1)" + if [[ -n "${probeContainer}" ]]; then + dfLine="$(docker exec "${probeContainer}" df -Pk / 2>/dev/null | tail -n1)" + fi + fi + + if [[ -z "${dfLine}" ]]; then + recordCheck "disk" 0 "Could not determine Docker data root disk usage (no running container to probe and the Docker data root is not reachable from the host)." + return 0 + fi + + local availKb="" usePercent="" availGb="" minGb=5 + availKb="$(echo "${dfLine}" | awk '{print $4}')" + usePercent="$(echo "${dfLine}" | awk '{print $5}')" + availGb=$(( ${availKb:-0} / 1024 / 1024 )) + + if [[ ${availGb} -lt ${minGb} ]]; then + recordCheck "disk" 0 "Only ${availGb}GB free on the Docker data root (${usePercent} used) - below the ${minGb}GB minimum." + else + recordCheck "disk" 1 "${availGb}GB free on the Docker data root (${usePercent} used)." + fi + return 0 +} +checkDiskHeadroom + +renderDoctorReport +exit "${DOCTOR_ANY_FAILED}" diff --git a/commands/doctor.help b/commands/doctor.help new file mode 100644 index 0000000..c673914 --- /dev/null +++ b/commands/doctor.help @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +[[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Diagnose whether the current project environment is fit to run. +## @category: environment + +ROLL_USAGE=$(cat <&2 - read -s password - echo >&2 - - if [[ -z "$password" ]]; then - error "Password cannot be empty" - exit 1 - fi - - # Confirm password for security - echo -n "Confirm password: " >&2 - read -s confirm - echo >&2 - - if [[ "$password" != "$confirm" ]]; then - error "Passwords do not match" - exit 1 - fi - - echo "$password" -} - -function logMessage() { - [[ $DUPLICATE_QUIET -eq 1 ]] && return - local level="$1" - shift - case "$level" in - INFO) info "$@" ;; - SUCCESS) success "$@" ;; - WARNING) warning "$@" ;; - ERROR) error "$@" ;; - esac -} function validateDuplicateName() { local name="$1" @@ -705,9 +663,14 @@ function performDuplicate() { # Validate inputs validateDuplicateName "$new_name" || exit 1 - # Handle interactive password prompt if needed + # Handle interactive password prompt if needed. Quiet mode is an explicit request for no + # interaction, so it stays a hard error even when stdin happens to be a terminal. if [[ "$DUPLICATE_ENCRYPT" == "PROMPT" ]]; then - DUPLICATE_ENCRYPT=$(promptPassword "Enter encryption password for backup") + if [[ $DUPLICATE_QUIET -eq 1 ]]; then + fatal "Password required but running in quiet mode. Use --encrypt= instead." + fi + DUPLICATE_ENCRYPT="" + promptPassword DUPLICATE_ENCRYPT "--encrypt=" "Enter encryption password for backup" "Confirm encryption password for backup" fi local current_env_name="$ROLL_ENV_NAME" diff --git a/commands/duplicate.help b/commands/duplicate.help index 9a4aca6..05f202e 100644 --- a/commands/duplicate.help +++ b/commands/duplicate.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Duplicate the current environment into a new one with a different name. +## @category: environment ROLL_USAGE=$(cat < " "An environment name was not provided; please enter one:" + +# The env name becomes the Docker Compose project name, which rejects anything but lowercase +# alphanumerics, hyphens, and underscores, starting with a letter or number. Catch it here instead +# of letting it fail later with a Docker error that names neither RollDev nor .env.roll. +while [[ ! "${ROLL_ENV_NAME}" =~ ^[a-z0-9][a-z0-9_-]*$ ]]; do + if ! isInteractive; then + fatal "Environment name \"${ROLL_ENV_NAME}\" is invalid: it becomes the Docker Compose project name, which must match ^[a-z0-9][a-z0-9_-]*\$ (lowercase alphanumeric characters, hyphens, and underscores, starting with a letter or number)." + fi + ROLL_ENV_NAME="" + promptInput ROLL_ENV_NAME "roll env-init " "Invalid environment name; it must match ^[a-z0-9][a-z0-9_-]*\$ (lowercase alphanumeric characters, hyphens, and underscores, starting with a letter or number):" done ROLL_ENV_TYPE="${ROLL_PARAMS[1]:-}" -# If roll environment type was not provided, prompt user for it -if [ -z "${ROLL_ENV_TYPE}" ]; then - while true; do - read -p $'\033[32mAn environment type was not provided; please choose one of ['"$(fetchValidEnvTypes)"$']:\033[0m ' ROLL_ENV_TYPE - assertValidEnvType && break - done +# If roll environment type was not provided, prompt user to choose one +if [[ -z "${ROLL_ENV_TYPE}" ]]; then + ROLL_ENV_TYPES=($(fetchValidEnvTypes)) + promptChoose ROLL_ENV_TYPE "roll env-init " "An environment type was not provided; please choose one:" "${ROLL_ENV_TYPES[@]}" fi # Verify the auto-select and/or type path resolves correctly before setting it @@ -46,7 +54,7 @@ TRAEFIK_SUBDOMAIN=app EOF ENV_INIT_FILE=$(fetchEnvInitFile) -if [[ ! -z $ENV_INIT_FILE ]]; then +if [[ -n "${ENV_INIT_FILE}" ]]; then export ROLL_ENV_NAME export GENERATED_APP_KEY="base64:$(dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64)" envsubst '$ROLL_ENV_NAME:$GENERATED_APP_KEY' < "${ENV_INIT_FILE}" >> "${ROLL_ENV_PATH}/.env.roll" diff --git a/commands/env-init.help b/commands/env-init.help index c0efea8..198ff1f 100755 --- a/commands/env-init.help +++ b/commands/env-init.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Initialize a new project environment configuration. +## @category: environment ROLL_USAGE=$(cat < ''" + fi + ## the command must arrive as ONE argument; refusing extra words is deliberate, because + ## silently running only the first of them is exactly the confusion this subcommand exists + ## to prevent + if (( ${#ROLL_PARAMS[@]} > 3 )); then + fatal "roll env sh takes a single quoted command: roll env sh ${ROLL_PARAMS[1]} '${ROLL_PARAMS[*]:2}'" + fi + ## env is in ROLL_CMD_ANYARGS, so roll's parse loop stops at the first dash-prefixed word and + ## leaves it in "$@" instead of ROLL_PARAMS; catch those here or they reach sh -c as $0 + if (( $# > 0 )); then + fatal "roll env sh takes a single quoted command: roll env sh ${ROLL_PARAMS[1]} '${ROLL_PARAMS[2]} $*'" + fi + # Transform: sh '' -> exec -T sh -c '' + ROLL_PARAMS=("exec" "-T" "${ROLL_PARAMS[1]}" "sh" "-c" "${ROLL_PARAMS[2]}") +fi + ## disconnect peered service containers from environment network if [[ "${ROLL_PARAMS[0]}" == "down" ]]; then disconnectPeeredServices "$(renderEnvNetworkName)" @@ -203,7 +159,7 @@ if [[ "${ROLL_PARAMS[0]}" == "up" ]]; then # # update images if needed # roll env pull ## create environment network for attachments if it does not already exist - if [[ -z "$(docker network ls -f 'name=$(renderEnvNetworkName)' -q)" ]]; then + if [[ -z "$(docker network ls -f "name=^$(renderEnvNetworkName)$" -q)" ]]; then docker compose \ --env-file "${ROLL_ENV_PATH}/.env.roll" --project-directory "${ROLL_ENV_PATH}" -p "${ROLL_ENV_NAME}" \ @@ -282,6 +238,7 @@ then fi ## set ssh-agent permissions for container user -if ([[ "${ROLL_PARAMS[0]}" == "up" ]] || [[ "${ROLL_PARAMS[0]}" == "start" ]]) then - roll root chmod 777 /run/host-services/ssh-auth.sock +if ([[ "${ROLL_PARAMS[0]}" == "up" ]] || [[ "${ROLL_PARAMS[0]}" == "start" ]]) && [[ -n "${SSH_AUTH_SOCK:-}" ]]; then + roll root chown www-data /run/host-services/ssh-auth.sock + roll root chmod 600 /run/host-services/ssh-auth.sock fi \ No newline at end of file diff --git a/commands/env.help b/commands/env.help index 7258175..971fa5f 100755 --- a/commands/env.help +++ b/commands/env.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run docker compose subcommands scoped to the project environment. +## @category: environment COMPOSE_USAGE=$(docker compose "${ROLL_PARAMS[@]}" "$@" \ | grep -vE '(--file|--project-name|--project-directory|default: docker-compose.yml|default: directory name)' \ @@ -9,5 +11,33 @@ COMPOSE_USAGE=$(docker compose "${ROLL_PARAMS[@]}" "$@" \ ROLL_USAGE=$(cat < '' + Run a shell command inside a container with proper shell handling. + Redirects and pipes work inside the container, not on the host. + + \033[1mWhy use this?\033[0m Without 'sh', redirects land on the host: + roll env exec php-fpm echo hi > /tmp/probe # writes to HOST /tmp/probe + roll env sh php-fpm 'echo hi > /tmp/probe' # writes to CONTAINER /tmp/probe + + \033[1mExamples:\033[0m + roll env sh php-fpm 'cat app/etc/env.php | grep MODE' + roll env sh php-fpm 'echo "test" > /tmp/test.txt' + roll env sh php-fpm 'ls -la | grep vendor' EOF ) diff --git a/commands/has-command.cmd b/commands/has-command.cmd new file mode 100644 index 0000000..b901184 --- /dev/null +++ b/commands/has-command.cmd @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +[[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 + +## Feature-detection primitive for scripts: exit 0 if the named command resolves through the +## registry (any of the four search paths, including RECLU), exit 1 if it does not. No output +## either way, so a caller can use it purely for its exit code. +## +## "Not found" is an expected, silent outcome here, not a failure: disable bin/roll ERR trap so a +## non-matching lookup does not print an error before this exits 1. +trap '' ERR + +if [[ ${#ROLL_PARAMS[@]} -eq 0 ]]; then + exit 1 +fi + +initializeRegistry +isCommandRegistered "${ROLL_PARAMS[0]}" +exit $? diff --git a/commands/has-command.help b/commands/has-command.help new file mode 100644 index 0000000..e81bff2 --- /dev/null +++ b/commands/has-command.help @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +[[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Check whether a command is available, via its exit code. +## @category: utility + +ROLL_USAGE=$(cat < + +Checks whether resolves through the command registry, across all four search paths +(project, home, RECLU, RollDev). Exits 0 if the command is available, 1 if it is not. Prints +nothing in either case, so it can be used purely for its exit code. + +\033[33mExamples:\033[0m + roll has-command db && roll db connect + roll has-command pull-media || echo "pull-media is not installed" + +\033[33mOptions:\033[0m + -h, --help Display this help menu +EOF +) diff --git a/commands/install.cmd b/commands/install.cmd index 37461cc..d705bdd 100644 --- a/commands/install.cmd +++ b/commands/install.cmd @@ -49,6 +49,11 @@ then echo "==> Trusting root certificate (requires sudo privileges)" sudo security add-trusted-cert -d -r trustRoot \ -k /Library/Keychains/System.keychain "${ROLL_SSL_DIR}/rootca/certs/ca.cert.pem" +elif [[ "$OSTYPE" =~ ^linux ]]; then + warning "Could not detect a supported CA trust store (Fedora/CentOS or Debian/Ubuntu) on this" \ + "Linux distribution. Trust the root certificate manually:" \ + "${ROLL_SSL_DIR}/rootca/certs/ca.cert.pem" \ + "See https://epartment.github.io/rolldev/configuration/dns-resolver.html for details." fi ## configure resolver for .test domains on Mac OS only as Linux lacks support @@ -109,8 +114,12 @@ EOT if [[ -z "$(grep -e 'ROLL_SERVICE_PORTAINER' "${ROLL_HOME_DIR}/.env")" ]]; then cat >> "${ROLL_HOME_DIR}/.env" <<-EOT -# Set to "0" to disable global Portainer service +# Set to "1" to enable global Portainer service ROLL_SERVICE_PORTAINER=1 EOT fi -fi \ No newline at end of file +fi +## gum backs every interactive prompt. This is a warning rather than a hard requirement: installing +## roll on a machine that only ever drives it with flags must not be blocked by a missing gum, and +## the prompts themselves check for it when they are actually reached. +checkGum diff --git a/commands/install.help b/commands/install.help index 8800f12..764beb6 100644 --- a/commands/install.help +++ b/commands/install.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Set up the RollDev host: SSL, DNS, SSH tunnel and required directories. +## @category: environment ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +source "${ROLL_DIR}/utils/magento2-init.sh" + # Default Magento version (minimum supported: 2.4.6) DEFAULT_MAGENTO_VERSION="2.4.x" @@ -9,777 +11,32 @@ PROJECT_NAME="${ROLL_PARAMS[0]:-}" MAGENTO_VERSION="${ROLL_PARAMS[1]:-$DEFAULT_MAGENTO_VERSION}" TARGET_DIR="${ROLL_PARAMS[2]:-}" -# Function to display usage information -show_usage() { - echo -e "\033[33mUsage:\033[0m" - echo " roll magento2-init [magento_version] [target_directory]" - echo "" - echo -e "\033[33mArguments:\033[0m" - echo " project_name Name of the Magento 2 project" - echo " magento_version Magento version to install (default: 2.4.x)" - echo " Supports: 2.4.6+, 2.4.7, 2.4.7-p3, 2.4.8, etc." - echo " Minimum supported version: 2.4.6" - echo " target_directory Directory to create project in (default: current directory)" - echo "" - echo -e "\033[33mExamples:\033[0m" - echo " roll magento2-init myproject" - echo " roll magento2-init myproject 2.4.7" - echo " roll magento2-init myproject 2.4.7-p3" - echo " roll magento2-init myproject 2.4.8" - echo " roll magento2-init myproject 2.4.x ~/Sites/myproject" - exit 1 -} - -# Validate project name -if [ -z "${PROJECT_NAME}" ]; then - echo -e "\033[31mError: Project name is required.\033[0m" - show_usage -fi - -# Validate project name format -if [[ ! "${PROJECT_NAME}" =~ ^[a-z0-9][a-z0-9-]*[a-z0-9]$|^[a-z0-9]$ ]]; then - echo -e "\033[31mError: Project name should contain only lowercase letters, numbers, and hyphens.\033[0m" - echo -e "\033[31mIt should start and end with a letter or number.\033[0m" - exit 1 -fi - -# Validate Magento version format and minimum version (2.4.6+) -if [[ ! "${MAGENTO_VERSION}" =~ ^2\.4(\.[6-9x]+)?(-p[0-9]+)?$ ]] && [[ ! "${MAGENTO_VERSION}" =~ ^2\.[5-9](\.[0-9x]+)?(-p[0-9]+)?$ ]]; then - echo -e "\033[31mError: Invalid Magento version format.\033[0m" - echo -e "\033[31mSupported formats: 2.4.6+, 2.4.7, 2.4.7-p3, 2.4.8, etc.\033[0m" - exit 1 -fi +# Phase: resolve version -- validate arguments +m2initValidateArguments -# Check minimum version requirement (2.4.6+) -if [[ "${MAGENTO_VERSION}" =~ ^2\.4\.([0-5])($|-p) ]]; then - echo -e "\033[31mError: Magento version ${MAGENTO_VERSION} is not supported.\033[0m" - echo -e "\033[31mMinimum supported version is 2.4.6\033[0m" - echo -e "\033[33mFor older Magento versions, please use manual installation or upgrade to 2.4.6+\033[0m" - exit 1 -fi - -# Function to get compatible software versions based on Magento version (2.4.6+ only) -get_software_versions() { - local magento_version="$1" - local base_version - local patch_version="" - - # Extract base version and patch version - if [[ "${magento_version}" =~ ^([0-9]+\.[0-9]+\.[0-9x]+)(-p([0-9]+))?$ ]]; then - base_version="${BASH_REMATCH[1]}" - patch_version="${BASH_REMATCH[3]:-0}" - else - base_version="${magento_version}" - patch_version="0" - fi - - # Set default values for 2.4.6+ - PHP_VERSION="8.2" - DB_DISTRIBUTION_VERSION="10.6" - ELASTICSEARCH_VERSION="7.17" - REDIS_VERSION="7.0" - RABBITMQ_VERSION="3.9" - VARNISH_VERSION="7.1" - COMPOSER_VERSION="2" - NODE_VERSION="19" - - # Version mapping based on Magento compatibility matrix (2.4.6+ only) - case "${base_version}" in - "2.4.9"*) - PHP_VERSION="8.4" - DB_DISTRIBUTION_VERSION="11.4" - ELASTICSEARCH_VERSION="2.19" # OpenSearch - REDIS_VERSION="8.0" - RABBITMQ_VERSION="4.1" - VARNISH_VERSION="7.7" - COMPOSER_VERSION="2" - ;; - "2.4.8"*) - PHP_VERSION="8.3" - DB_DISTRIBUTION_VERSION="11.4" - ELASTICSEARCH_VERSION="2.19" # OpenSearch - REDIS_VERSION="8.0" - RABBITMQ_VERSION="4.1" - VARNISH_VERSION="7.7" - COMPOSER_VERSION="2" - ;; - "2.4.7"*) - PHP_VERSION="8.3" - if [[ "${patch_version}" -ge 6 ]]; then - DB_DISTRIBUTION_VERSION="10.11" - REDIS_VERSION="7.2" - VARNISH_VERSION="7.7" - COMPOSER_VERSION="2" - elif [[ "${patch_version}" -ge 3 ]]; then - DB_DISTRIBUTION_VERSION="10.6" - REDIS_VERSION="7.2" - VARNISH_VERSION="7.5" - COMPOSER_VERSION="2" - else - DB_DISTRIBUTION_VERSION="10.6" - REDIS_VERSION="7.2" - VARNISH_VERSION="7.5" - COMPOSER_VERSION="2" - fi - ELASTICSEARCH_VERSION="7.17" - RABBITMQ_VERSION="3.13" - ;; - "2.4.6"*) - PHP_VERSION="8.2" - DB_DISTRIBUTION_VERSION="10.6" - ELASTICSEARCH_VERSION="7.17" - REDIS_VERSION="7.0" - RABBITMQ_VERSION="3.9" - VARNISH_VERSION="7.1" - COMPOSER_VERSION="2" - if [[ "${patch_version}" -ge 8 ]]; then - REDIS_VERSION="7.2" - VARNISH_VERSION="7.5" - fi - ;; - "2.4.x"|"2.4"*) - # Default to latest stable versions for 2.4.x - PHP_VERSION="8.3" - DB_DISTRIBUTION_VERSION="10.6" - ELASTICSEARCH_VERSION="7.17" - REDIS_VERSION="7.2" - RABBITMQ_VERSION="3.13" - VARNISH_VERSION="7.5" - COMPOSER_VERSION="2" - ;; - esac - - echo -e "\033[33mConfigured software versions for Magento ${magento_version}:\033[0m" - echo -e " PHP: ${PHP_VERSION}" - echo -e " MariaDB: ${DB_DISTRIBUTION_VERSION}" - if [[ "${ELASTICSEARCH_VERSION}" == "2."* ]]; then - echo -e " Search Engine: OpenSearch ${ELASTICSEARCH_VERSION}" - else - echo -e " Search Engine: Elasticsearch ${ELASTICSEARCH_VERSION}" - fi - echo -e " Redis: ${REDIS_VERSION}" - echo -e " RabbitMQ: ${RABBITMQ_VERSION}" - echo -e " Varnish: ${VARNISH_VERSION}" - echo -e " Composer: ${COMPOSER_VERSION}" - echo -e " Node.js: ${NODE_VERSION}" -} - -# Set target directory -if [ -z "${TARGET_DIR}" ]; then - TARGET_DIR="$(pwd)/${PROJECT_NAME}" -else - # Handle relative paths and ensure absolute path - if [[ "${TARGET_DIR}" != /* ]]; then - TARGET_DIR="$(pwd)/${TARGET_DIR}" - fi - TARGET_DIR="${TARGET_DIR}/${PROJECT_NAME}" -fi +# Phase: create project -- resolve the target directory and create it +m2initResolveTargetDir echo -e "\033[32mInitializing Magento 2 project: ${PROJECT_NAME}\033[0m" echo -e "\033[32mMagento version: ${MAGENTO_VERSION}\033[0m" echo -e "\033[32mTarget directory: ${TARGET_DIR}\033[0m" -# Check if target directory already exists -if [ -d "${TARGET_DIR}" ]; then - echo -e "\033[31mError: Directory ${TARGET_DIR} already exists.\033[0m" - exit 1 -fi - -# Create project directory -echo -e "\033[36m[1/10] Creating project directory...\033[0m" -mkdir -p "${TARGET_DIR}" -cd "${TARGET_DIR}" +m2initCreateProjectDirectory -# Get compatible software versions for this Magento version +# Phase: resolve version (software versions) -- determine compatible software versions echo -e "\033[36m[2/10] Determining compatible software versions...\033[0m" -get_software_versions "${MAGENTO_VERSION}" - -# Initialize environment -echo -e "\033[36m[3/10] Initializing environment configuration...\033[0m" -"${ROLL_DIR}/bin/roll" env-init "${PROJECT_NAME}" magento2 - -# Update .env.roll with version-specific software versions -echo -e "\033[36m[4/10] Updating environment with compatible software versions...\033[0m" -ENV_FILE="${TARGET_DIR}/.env.roll" - -# Update software versions in .env.roll file -sed -i.bak "s/^PHP_VERSION=.*/PHP_VERSION=${PHP_VERSION}/" "${ENV_FILE}" -sed -i.bak "s/^DB_DISTRIBUTION_VERSION=.*/DB_DISTRIBUTION_VERSION=${DB_DISTRIBUTION_VERSION}/" "${ENV_FILE}" -sed -i.bak "s/^COMPOSER_VERSION=.*/COMPOSER_VERSION=${COMPOSER_VERSION}/" "${ENV_FILE}" -sed -i.bak "s/^NODE_VERSION=.*/NODE_VERSION=${NODE_VERSION}/" "${ENV_FILE}" -sed -i.bak "s/^RABBITMQ_VERSION=.*/RABBITMQ_VERSION=${RABBITMQ_VERSION}/" "${ENV_FILE}" -sed -i.bak "s/^VARNISH_VERSION=.*/VARNISH_VERSION=${VARNISH_VERSION}/" "${ENV_FILE}" - -# Handle search engine configuration (OpenSearch vs Elasticsearch) -if [[ "${ELASTICSEARCH_VERSION}" == "2."* ]]; then - # Use OpenSearch for newer Magento versions (2.4.8+) - OPENSEARCH_VERSION="${ELASTICSEARCH_VERSION}" - sed -i.bak "s/^ROLL_ELASTICSEARCH=.*/ROLL_ELASTICSEARCH=0/" "${ENV_FILE}" - sed -i.bak "s/^ROLL_OPENSEARCH=.*/ROLL_OPENSEARCH=1/" "${ENV_FILE}" - # Replace ELASTICSEARCH_VERSION with OPENSEARCH_VERSION - sed -i.bak "s/^ELASTICSEARCH_VERSION=.*/OPENSEARCH_VERSION=${OPENSEARCH_VERSION}/" "${ENV_FILE}" - # Set actual search engine version for configuration - ELASTICSEARCH_VERSION="7.17" # Fallback version for installation compatibility - echo -e " OpenSearch: ${OPENSEARCH_VERSION} (primary)" - echo -e " Elasticsearch: ${ELASTICSEARCH_VERSION} (fallback)" -else - # Use Elasticsearch for older versions - sed -i.bak "s/^ROLL_ELASTICSEARCH=.*/ROLL_ELASTICSEARCH=1/" "${ENV_FILE}" - sed -i.bak "s/^ROLL_OPENSEARCH=.*/ROLL_OPENSEARCH=0/" "${ENV_FILE}" - sed -i.bak "s/^ELASTICSEARCH_VERSION=.*/ELASTICSEARCH_VERSION=${ELASTICSEARCH_VERSION}/" "${ENV_FILE}" - # Ensure OpenSearch version is not present - if grep -q "^OPENSEARCH_VERSION=" "${ENV_FILE}"; then - sed -i.bak "/^OPENSEARCH_VERSION=/d" "${ENV_FILE}" - fi -fi - -# Handle Redis configuration (always use Redis for Magento 2) -# Magento 2 works best with traditional Redis, so we always use Redis regardless of version - sed -i.bak "s/^ROLL_REDIS=.*/ROLL_REDIS=1/" "${ENV_FILE}" - sed -i.bak "s/^ROLL_DRAGONFLY=.*/ROLL_DRAGONFLY=0/" "${ENV_FILE}" - sed -i.bak "s/^REDIS_VERSION=.*/REDIS_VERSION=${REDIS_VERSION}/" "${ENV_FILE}" -# Ensure Dragonfly version is not present -if grep -q "^DRAGONFLY_VERSION=" "${ENV_FILE}"; then - sed -i.bak "/^DRAGONFLY_VERSION=/d" "${ENV_FILE}" -fi - -# Clean up backup file -rm -f "${ENV_FILE}.bak" - -# Sign SSL certificate -echo -e "\033[36m[5/10] Signing SSL certificate...\033[0m" -"${ROLL_DIR}/bin/roll" sign-certificate "${PROJECT_NAME}.test" - -# Start environment -echo -e "\033[36m[6/10] Starting project environment...\033[0m" -"${ROLL_DIR}/bin/roll" env up - -# Wait for services to be ready -echo -e "\033[36m[7/10] Waiting for services to be ready...\033[0m" -echo -e "\033[33mChecking service status...\033[0m" - -# Wait for database to be ready -echo -n "Waiting for database... " -timeout=60 -while [ $timeout -gt 0 ]; do - if "${ROLL_DIR}/bin/roll" db connect -e "SELECT 1;" >/dev/null 2>&1; then - echo "✅ Ready" - break - fi - echo -n "." - sleep 2 - timeout=$((timeout-2)) -done - -if [ $timeout -le 0 ]; then - echo "❌ Database not ready after 60 seconds" - exit 1 -fi - -# Wait for search engine to be ready -echo -n "Waiting for search engine... " -timeout=60 -SEARCH_HOST="elasticsearch" -SEARCH_PORT="9200" - -# Determine search engine host based on configuration -# Check environment variables to see which service is actually enabled -if grep -q "^ROLL_OPENSEARCH=1" "${ENV_FILE}" 2>/dev/null; then - SEARCH_HOST="opensearch" - echo -n "(OpenSearch) " -else - echo -n "(Elasticsearch) " -fi - -while [ $timeout -gt 0 ]; do - if "${ROLL_DIR}/bin/roll" cli bash -c "timeout 5 bash -c '/dev/null; then - # Double check with HTTP request if port is open - if "${ROLL_DIR}/bin/roll" cli curl -f -s "http://${SEARCH_HOST}:${SEARCH_PORT}/_cluster/health" >/dev/null 2>&1; then - echo "✅ Ready" - break - fi - fi - echo -n "." - sleep 2 - timeout=$((timeout-2)) -done - -if [ $timeout -le 0 ]; then - echo "❌ Search engine not ready after 60 seconds" - echo "Debug: Checking ${SEARCH_HOST}:${SEARCH_PORT}" - "${ROLL_DIR}/bin/roll" cli bash -c "timeout 5 bash -c '&1 || echo "Port not accessible" - echo "Tip: Make sure ${SEARCH_HOST} service is running with 'roll env up'" - exit 1 -fi - -# Wait for Redis to be ready -echo -n "Waiting for Redis... " -timeout=30 -while [ $timeout -gt 0 ]; do - if "${ROLL_DIR}/bin/roll" redis ping 2>/dev/null | grep -q PONG; then - echo "✅ Ready" - break - fi - echo -n "." - sleep 2 - timeout=$((timeout-2)) -done - -if [ $timeout -le 0 ]; then - echo "❌ Redis not ready after 30 seconds" - exit 1 -fi - -echo -e "\033[32m✅ All services are ready!\033[0m" - -# Drop into shell for setup -echo -e "\033[36m[8/12] Setting up Magento project files...\033[0m" - -# Check if composer global auth is configured -echo -e "\033[33mNote: This process requires Magento Marketplace credentials.\033[0m" -echo -e "\033[33mIf you haven't configured them globally, you'll be prompted during composer install.\033[0m" - -# Meta package for Magento 2.4.6+ -META_PACKAGE="magento/project-community-edition" - -# Create project using composer inside container -"${ROLL_DIR}/bin/roll" cli bash -c " - set -e - - echo 'Creating Magento project with composer...' - composer create-project --repository-url=https://repo.magento.com/ \\ - '${META_PACKAGE}' /tmp/${PROJECT_NAME} '${MAGENTO_VERSION}' - - echo 'Moving files to web root...' - rsync -a /tmp/${PROJECT_NAME}/ /var/www/html/ - rm -rf /tmp/${PROJECT_NAME}/ - - echo 'Setting proper file permissions...' - find /var/www/html -type f -exec chmod 644 {} \\; - find /var/www/html -type d -exec chmod 755 {} \\; - chmod u+x /var/www/html/bin/magento -" - -# Apply Magento 2.4.4 patch for ReflectionUnionType::getName() error -if [[ "${MAGENTO_VERSION}" == "2.4.4"* ]]; then - echo -e "\033[36m[8.5/12] Applying Magento 2.4.4 patches...\033[0m" - echo -e "\033[33m🔧 Detected Magento 2.4.4 - applying ACSD-59280 patch for ReflectionUnionType issue\033[0m" - - "${ROLL_DIR}/bin/roll" cli bash -c " - set -e - - echo 'Installing Quality Patches Tool...' - if ! composer show magento/quality-patches >/dev/null 2>&1; then - composer require magento/quality-patches --no-update - composer update magento/quality-patches --no-dev - fi - - echo 'Checking available patches...' - if vendor/bin/magento-patches status | grep -q 'ACSD-59280'; then - echo 'Applying ACSD-59280 patch for ReflectionUnionType issue...' - vendor/bin/magento-patches apply ACSD-59280 || echo 'Patch may already be applied or not needed' - else - echo 'ACSD-59280 patch not found, may not be needed for this version' - fi - - echo 'Clearing generated code after patching...' - rm -rf generated/metadata generated/code var/generation - " - - echo -e "\033[32m✅ Magento 2.4.4 patches applied successfully\033[0m" -fi - -echo -e "\033[36m[9/12] Installing Magento application...\033[0m" - -# Determine search engine parameters based on version (2.4.6+ only) -echo -e "\033[33mConfiguring search engine parameters...\033[0m" - -# Determine search engine type based on environment configuration -if grep -q "^ROLL_OPENSEARCH=1" "${ENV_FILE}" 2>/dev/null; then - # OpenSearch configuration for Magento 2.4.8+ - OPENSEARCH_VER=$(grep "^OPENSEARCH_VERSION=" "${ENV_FILE}" | cut -d'=' -f2 || echo "2.19") - SEARCH_ENGINE_PARAMS=" - --search-engine=opensearch \\ - --opensearch-host=opensearch \\ - --opensearch-port=9200 \\ - --opensearch-index-prefix=magento2 \\ - --opensearch-enable-auth=0 \\ - --opensearch-timeout=15" - echo -e "\033[33mUsing OpenSearch ${OPENSEARCH_VER} for Magento 2.4.8+\033[0m" -else - # Elasticsearch configuration for Magento 2.4.6-2.4.7 - SEARCH_ENGINE_PARAMS=" - --search-engine=elasticsearch7 \\ - --elasticsearch-host=elasticsearch \\ - --elasticsearch-port=9200 \\ - --elasticsearch-index-prefix=magento2 \\ - --elasticsearch-enable-auth=0 \\ - --elasticsearch-timeout=15" - echo -e "\033[33mUsing Elasticsearch ${ELASTICSEARCH_VERSION}\033[0m" -fi - -# Debug: Show search engine parameters -echo -e "\033[33mSearch engine parameters:\033[0m" -echo "${SEARCH_ENGINE_PARAMS}" - -# Install Magento with fallback mechanism -echo -e "\033[33mAttempting Magento installation with configured search engine...\033[0m" - -# Build the installation command based on search engine type -if grep -q "^ROLL_OPENSEARCH=1" "${ENV_FILE}" 2>/dev/null; then - # OpenSearch installation command - INSTALL_COMMAND="bin/magento setup:install \\ - --backend-frontname=shopmanager \\ - --amqp-host=rabbitmq \\ - --amqp-port=5672 \\ - --amqp-user=guest \\ - --amqp-password=guest \\ - --db-host=db \\ - --db-name=magento \\ - --db-user=magento \\ - --db-password=magento \\ - --search-engine=opensearch \\ - --opensearch-host=opensearch \\ - --opensearch-port=9200 \\ - --opensearch-index-prefix=magento2 \\ - --opensearch-enable-auth=0 \\ - --opensearch-timeout=15 \\ - --http-cache-hosts=varnish:80 \\ - --session-save=redis \\ - --session-save-redis-host=redis \\ - --session-save-redis-port=6379 \\ - --session-save-redis-db=2 \\ - --session-save-redis-max-concurrency=20 \\ - --cache-backend=redis \\ - --cache-backend-redis-server=redis \\ - --cache-backend-redis-db=0 \\ - --cache-backend-redis-port=6379 \\ - --page-cache=redis \\ - --page-cache-redis-server=redis \\ - --page-cache-redis-db=1 \\ - --page-cache-redis-port=6379" -else - # Elasticsearch installation command - INSTALL_COMMAND="bin/magento setup:install \\ - --backend-frontname=shopmanager \\ - --amqp-host=rabbitmq \\ - --amqp-port=5672 \\ - --amqp-user=guest \\ - --amqp-password=guest \\ - --db-host=db \\ - --db-name=magento \\ - --db-user=magento \\ - --db-password=magento \\ - --search-engine=elasticsearch7 \\ - --elasticsearch-host=elasticsearch \\ - --elasticsearch-port=9200 \\ - --elasticsearch-index-prefix=magento2 \\ - --elasticsearch-enable-auth=0 \\ - --elasticsearch-timeout=15 \\ - --http-cache-hosts=varnish:80 \\ - --session-save=redis \\ - --session-save-redis-host=redis \\ - --session-save-redis-port=6379 \\ - --session-save-redis-db=2 \\ - --session-save-redis-max-concurrency=20 \\ - --cache-backend=redis \\ - --cache-backend-redis-server=redis \\ - --cache-backend-redis-db=0 \\ - --cache-backend-redis-port=6379 \\ - --page-cache=redis \\ - --page-cache-redis-server=redis \\ - --page-cache-redis-db=1 \\ - --page-cache-redis-port=6379" -fi - -if ! "${ROLL_DIR}/bin/roll" cli bash -c " - set -e - - echo 'Installing Magento application...' - echo 'Search engine parameters:' - echo '${SEARCH_ENGINE_PARAMS}' - ${INSTALL_COMMAND} -"; then - echo -e "\033[33m⚠️ Primary search engine installation failed, trying fallback to Elasticsearch...\033[0m" - - # Fallback to Elasticsearch 7 - FALLBACK_COMMAND="bin/magento setup:install \\ - --backend-frontname=shopmanager \\ - --amqp-host=rabbitmq \\ - --amqp-port=5672 \\ - --amqp-user=guest \\ - --amqp-password=guest \\ - --db-host=db \\ - --db-name=magento \\ - --db-user=magento \\ - --db-password=magento \\ - --search-engine=elasticsearch7 \\ - --elasticsearch-host=elasticsearch \\ - --elasticsearch-port=9200 \\ - --elasticsearch-index-prefix=magento2 \\ - --elasticsearch-enable-auth=0 \\ - --elasticsearch-timeout=15 \\ - --http-cache-hosts=varnish:80 \\ - --session-save=redis \\ - --session-save-redis-host=redis \\ - --session-save-redis-port=6379 \\ - --session-save-redis-db=2 \\ - --session-save-redis-max-concurrency=20 \\ - --cache-backend=redis \\ - --cache-backend-redis-server=redis \\ - --cache-backend-redis-db=0 \\ - --cache-backend-redis-port=6379 \\ - --page-cache=redis \\ - --page-cache-redis-server=redis \\ - --page-cache-redis-db=1 \\ - --page-cache-redis-port=6379" - - "${ROLL_DIR}/bin/roll" cli bash -c " - set -e - - echo 'Retrying with Elasticsearch 7 fallback...' - ${FALLBACK_COMMAND} - " - - echo -e "\033[32m✅ Installation completed with Elasticsearch fallback\033[0m" - USED_FALLBACK=1 -else - echo -e "\033[32m✅ Installation completed with configured search engine\033[0m" - USED_FALLBACK=0 -fi - -echo -e "\033[36m[10/12] Configuring Magento application...\033[0m" - -# Configure Magento -"${ROLL_DIR}/bin/roll" cli bash -c " - set -e - - echo 'Configuring base URLs...' - bin/magento config:set --lock-env web/unsecure/base_url \\ - \"https://app.${PROJECT_NAME}.test/\" - - bin/magento config:set --lock-env web/secure/base_url \\ - \"https://app.${PROJECT_NAME}.test/\" - - bin/magento config:set --lock-env web/secure/offloader_header X-Forwarded-Proto - bin/magento config:set --lock-env web/secure/use_in_frontend 1 - bin/magento config:set --lock-env web/secure/use_in_adminhtml 1 - bin/magento config:set --lock-env web/seo/use_rewrites 1 - - echo 'Configuring cache settings...' - bin/magento config:set --lock-env system/full_page_cache/caching_application 2 - bin/magento config:set --lock-env system/full_page_cache/ttl 604800 - bin/magento config:set --lock-env catalog/search/enable_eav_indexer 1 - bin/magento config:set --lock-env dev/static/sign 0 - - echo 'Setting developer mode...' - bin/magento deploy:mode:set -s developer - bin/magento cache:disable block_html full_page -" - -# Configure search engine post-installation for OpenSearch fallback scenarios -if [[ "${USED_FALLBACK}" == "1" ]] && grep -q "^ROLL_OPENSEARCH=1" "${ENV_FILE}" 2>/dev/null; then - echo -e "\033[36mInstallation used Elasticsearch fallback, but OpenSearch is configured for runtime...\033[0m" - echo -e "\033[33m📝 Note: You can manually configure OpenSearch later using:\033[0m" - echo -e "\033[33m bin/magento config:set catalog/search/engine opensearch\033[0m" - echo -e "\033[33m bin/magento config:set catalog/search/opensearch_server_hostname opensearch\033[0m" - echo -e "\033[33m bin/magento config:set catalog/search/opensearch_server_port 9200\033[0m" -fi - -echo -e "\033[36m[11/12] Running initial indexing...\033[0m" -"${ROLL_DIR}/bin/roll" cli bash -c " - bin/magento indexer:reindex - bin/magento cache:flush -" - -echo -e "\033[36m[11/12] Creating admin user and configuring 2FA...\033[0m" - -# Function to check if version is 2.4.8 or higher -is_magento_248_or_higher() { - local version="$1" - local base_version - - # Extract base version (remove patch info) - if [[ "${version}" =~ ^([0-9]+\.[0-9]+\.[0-9x]+) ]]; then - base_version="${BASH_REMATCH[1]}" - else - base_version="${version}" - fi - - # Check if version is 2.4.8+ or 2.4.x (which defaults to latest) - case "${base_version}" in - "2.4.x"|"2.4.9"*|"2.4.8"*) - return 0 # true - ;; - *) - return 1 # false - ;; - esac -} - -# Generate admin user and 2FA setup for Magento 2.4.6+ (all supported versions require 2FA) -if is_magento_248_or_higher "${MAGENTO_VERSION}"; then - echo -e "\033[33m🔧 Detected Magento 2.4.8+ - Using workaround for 2FA configuration issue\033[0m" - echo -e "\033[33m (Adobe Commerce core issue #39836 - DuoSecurity provider array handling)\033[0m" - - # Magento 2.4.8+ workaround for 2FA configuration bug - "${ROLL_DIR}/bin/roll" cli bash -c " - set -e - - # Generate admin credentials - ADMIN_PASS=\"\$(pwgen -n1 16)\" - ADMIN_USER=admin - - echo 'Creating admin user...' - bin/magento admin:user:create \\ - --admin-password=\"\${ADMIN_PASS}\" \\ - --admin-user=\"\${ADMIN_USER}\" \\ - --admin-firstname=\"Local\" \\ - --admin-lastname=\"Admin\" \\ - --admin-email=\"\${ADMIN_USER}@example.com\" - - echo \"Admin Username: \${ADMIN_USER}\" - echo \"Admin Password: \${ADMIN_PASS}\" - - # Configure 2FA - using workaround for 2.4.8+ core bug - echo 'Configuring 2FA (using 2.4.8+ workaround)...' - TFA_SECRET=\$(python3 -c \"import base64; print(base64.b32encode('\$(pwgen -A1 128)'.encode()).decode().strip('='))\") - OTPAUTH_URL=\$(printf \"otpauth://totp/%s%%3Alocaladmin%%40example.com?issuer=%s&secret=%s\" \\ - \"app.${PROJECT_NAME}.test\" \"app.${PROJECT_NAME}.test\" \"\${TFA_SECRET}\" - ) - - # Step 1: Set 2FA provider without --lock-env to avoid array storage bug - echo 'Setting 2FA provider (step 1/4)...' - bin/magento config:set twofactorauth/general/force_providers google - - # Step 2: Run DI compile to ensure TFA commands are available - echo 'Compiling DI container (step 2/4)...' - bin/magento setup:di:compile --quiet - - # Step 3: Set the TFA secret - echo 'Setting 2FA secret (step 3/4)...' - bin/magento security:tfa:google:set-secret \"\${ADMIN_USER}\" \"\${TFA_SECRET}\" - - # Step 4: Run setup:upgrade to ensure all configs are applied - echo 'Running setup upgrade (step 4/4)...' - bin/magento setup:upgrade --keep-generated - - echo \"2FA Setup URL: \${OTPAUTH_URL}\" - echo \"2FA Backup Codes:\" - oathtool -s 30 -w 10 --totp --base32 \"\${TFA_SECRET}\" - - # Generate QR code - segno \"\${OTPAUTH_URL}\" -s 4 -o \"pub/media/\${ADMIN_USER}-totp-qr.png\" - QR_URL=\"https://app.${PROJECT_NAME}.test/media/\${ADMIN_USER}-totp-qr.png?t=\$(date +%s)\" - echo \"QR Code URL: \${QR_URL}\" - - # Save credentials to file for user reference - cat > /var/www/html/admin-credentials.txt << EOL -Magento Admin Credentials -======================== -Username: \${ADMIN_USER} -Password: \${ADMIN_PASS} -2FA Setup URL: \${OTPAUTH_URL} -QR Code URL: \${QR_URL} - -Admin Panel: https://app.${PROJECT_NAME}.test/shopmanager/ -Frontend: https://app.${PROJECT_NAME}.test/ - -Generated on: \$(date) - -Note: This installation used the 2.4.8+ workaround for Adobe Commerce core issue #39836 -EOL - - echo 'Admin credentials saved to admin-credentials.txt' - " -else - # Standard 2FA setup for Magento 2.4.6-2.4.7 - "${ROLL_DIR}/bin/roll" cli bash -c " - set -e - - # Generate admin credentials - ADMIN_PASS=\"\$(pwgen -n1 16)\" - ADMIN_USER=admin - - echo 'Creating admin user...' - bin/magento admin:user:create \\ - --admin-password=\"\${ADMIN_PASS}\" \\ - --admin-user=\"\${ADMIN_USER}\" \\ - --admin-firstname=\"Local\" \\ - --admin-lastname=\"Admin\" \\ - --admin-email=\"\${ADMIN_USER}@example.com\" - - echo \"Admin Username: \${ADMIN_USER}\" - echo \"Admin Password: \${ADMIN_PASS}\" - - # Configure 2FA - standard method for 2.4.6-2.4.7 - echo 'Configuring 2FA...' - TFA_SECRET=\$(python3 -c \"import base64; print(base64.b32encode('\$(pwgen -A1 128)'.encode()).decode().strip('='))\") - OTPAUTH_URL=\$(printf \"otpauth://totp/%s%%3Alocaladmin%%40example.com?issuer=%s&secret=%s\" \\ - \"app.${PROJECT_NAME}.test\" \"app.${PROJECT_NAME}.test\" \"\${TFA_SECRET}\" - ) - - bin/magento config:set --lock-env twofactorauth/general/force_providers google - bin/magento security:tfa:google:set-secret \"\${ADMIN_USER}\" \"\${TFA_SECRET}\" - - echo \"2FA Setup URL: \${OTPAUTH_URL}\" - echo \"2FA Backup Codes:\" - oathtool -s 30 -w 10 --totp --base32 \"\${TFA_SECRET}\" - - # Generate QR code - segno \"\${OTPAUTH_URL}\" -s 4 -o \"pub/media/\${ADMIN_USER}-totp-qr.png\" - QR_URL=\"https://app.${PROJECT_NAME}.test/media/\${ADMIN_USER}-totp-qr.png?t=\$(date +%s)\" - echo \"QR Code URL: \${QR_URL}\" - - # Save credentials to file for user reference - cat > /var/www/html/admin-credentials.txt << EOL -Magento Admin Credentials -======================== -Username: \${ADMIN_USER} -Password: \${ADMIN_PASS} -2FA Setup URL: \${OTPAUTH_URL} -QR Code URL: \${QR_URL} - -Admin Panel: https://app.${PROJECT_NAME}.test/shopmanager/ -Frontend: https://app.${PROJECT_NAME}.test/ - -Generated on: \$(date) -EOL - - echo 'Admin credentials saved to admin-credentials.txt' - " -fi - -echo -e "\033[36m[12/12] Finalizing setup...\033[0m" +m2initResolveSoftwareVersions "${MAGENTO_VERSION}" -echo -e "\033[32m✅ Magento 2 project '${PROJECT_NAME}' has been successfully created!\033[0m" -echo "" -echo -e "\033[33m🔗 Access URLs:\033[0m" -echo -e " Frontend: https://app.${PROJECT_NAME}.test/" -echo -e " Admin: https://app.${PROJECT_NAME}.test/shopmanager/" -echo -e " RabbitMQ: https://rabbitmq.${PROJECT_NAME}.test/" -echo -e " Elasticsearch: https://elasticsearch.${PROJECT_NAME}.test/" -echo "" -echo -e "\033[33m📁 Project Location:\033[0m" -echo -e " ${TARGET_DIR}" -echo "" -echo -e "\033[33m🔑 Admin Credentials:\033[0m" -echo -e " Check the file: admin-credentials.txt in your project root" -echo "" -echo -e "\033[33m💡 Next Steps:\033[0m" -echo -e " 1. Navigate to your project: cd ${TARGET_DIR}" -echo -e " 2. Access the shell: roll shell" -echo -e " 3. Open your browser to: https://app.${PROJECT_NAME}.test/" +# Phase: write env -- initialize the environment and align .env.roll with the resolved versions +m2initWriteEnvConfig -if [[ "${USED_FALLBACK}" == "1" ]]; then - echo "" - echo -e "\033[33m⚠️ Installation Note:\033[0m" - echo -e " Installation used Elasticsearch fallback due to OpenSearch connectivity issues" - echo -e " OpenSearch is configured in your environment for future use" - echo -e " Check the manual configuration commands shown above to switch to OpenSearch" -fi +# Phase: install -- certificate, environment start, Magento source, setup, configuration, admin user +m2initSignCertificate +m2initStartEnvironment +m2initInstallMagentoSource +m2initSetupMagento +m2initConfigureMagento +m2initReindexMagento +m2initCreateAdminUser -echo "" -echo -e "\033[33m🛑 To destroy this environment:\033[0m" -echo -e " roll env down -v" -echo "" \ No newline at end of file +m2initPrintSummary diff --git a/commands/magento2-init.help b/commands/magento2-init.help index a693fbc..abb94a4 100755 --- a/commands/magento2-init.help +++ b/commands/magento2-init.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Scaffold a new Magento 2 project and its RollDev environment. +## @category: magento2 ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Clean the Magento cache inside the project container. +## @category: magento2 ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Fix file ownership inside the project container. +## @category: magento2 ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Fix file permissions inside the project container. +## @category: magento2 ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run Grunt inside the project container. +## @category: magento2 ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run bin/magento inside the project php-fpm container. +## @category: magento2 MAGENTO_HELP=$("${ROLL_DIR}/bin/roll" cli "bin/magento") diff --git a/commands/magento2/magepack.cmd b/commands/magento2/magepack.cmd index ee705ba..382f90c 100644 --- a/commands/magento2/magepack.cmd +++ b/commands/magento2/magepack.cmd @@ -13,9 +13,9 @@ if [ -z "${ROLL_PARAMS[*]}" ]; then exit 0 fi -echo "Roll params: ${ROLL_PARAMS[@]}" +echo "Roll params: ${ROLL_PARAMS[*]}" echo "================================================" -echo "Other params: $@" +echo "Other params: $*" exit 1 diff --git a/commands/magento2/magepack.help b/commands/magento2/magepack.help index 9491913..d9596fe 100644 --- a/commands/magento2/magepack.help +++ b/commands/magento2/magepack.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run Magepack static-asset bundling inside the project container. +## @category: magento2 ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Create a fast parallel logical dump of the project database with mydumper. +## @category: database ROLL_USAGE=$(cat <&1 | tee "${MYLOADER_LOG}") || MYLOADER_EXIT=$? + +if [[ ${MYLOADER_EXIT} -ne 0 && ${NORMALIZE_SOURCE_DDL} -eq 0 ]]; then + if grep -qE "ERROR 1911|Unknown option 'ENCRYPTION'" "${MYLOADER_LOG}"; then + error "myloader failed: the dump's schema uses the ENCRYPTION table option, which MariaDB does not support." + info "Retry with --normalize-source-ddl to strip it automatically." + elif grep -qE "ERROR 1273|Unknown collation|utf8mb4_0900" "${MYLOADER_LOG}"; then + error "myloader failed: the dump's schema uses a utf8mb4_0900_* collation, which MariaDB does not support." + info "Retry with --normalize-source-ddl to remap it automatically." + elif grep -q "Trace/breakpoint trap" "${MYLOADER_LOG}"; then + error "myloader crashed (Trace/breakpoint trap). This is the typical symptom of restoring a MySQL 8 dump into MariaDB: the ENCRYPTION table option or a utf8mb4_0900_* collation in the schema is unsupported by MariaDB." + info "Retry with --normalize-source-ddl to strip/remap these automatically." + fi +fi + +exit "${MYLOADER_EXIT}" diff --git a/commands/magento2/myloader.help b/commands/magento2/myloader.help index e1f132e..b9eacd1 100644 --- a/commands/magento2/myloader.help +++ b/commands/magento2/myloader.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Restore a mydumper dump into the project database with myloader. +## @category: database ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Set up automatic admin login for the project Magento install. +## @category: magento2 ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Set up Grunt configuration files for the project theme. +## @category: magento2 ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 + +ROLL_ENV_PATH="$(locateEnvPath)" || exit $? +loadEnvConfig "${ROLL_ENV_PATH}" || exit $? +assertDockerRunning + +if [[ "${ROLL_ENV_TYPE}" != "magento2" ]]; then + warning "This command is only available for Magento 2 projects" && exit 1 +fi + +ROLL_ENV_SHELL_CONTAINER=${ROLL_ENV_SHELL_CONTAINER:-php-fpm} +FPM_CONTAINER=$(roll env ps -q "${ROLL_ENV_SHELL_CONTAINER}") +if [[ ! ${FPM_CONTAINER} ]]; then + fatal "Your project is not running. Please use \`roll start\` to start your project." +fi + +## theme discovery is rooted at the project's app/design/frontend, never at $PWD - otherwise +## running the command from any directory but the project root reports no themes at all +THEME_ROOT="${ROLL_ENV_PATH}/app/design/frontend" + +INSTALL_THEMES=($(find "${THEME_ROOT}" -maxdepth 3 -name 'package.json' 2>/dev/null \ + | sed -e 's#/package\.json$##' -e "s#^${THEME_ROOT}/##")) +AVAILABLE_THEMES=($(find "${THEME_ROOT}" -name 'Gulpfile.js' 2>/dev/null \ + | sed -e 's#/Gulpfile\.js$##' -e "s#^${THEME_ROOT}/##")) + +NODE_PACKAGE_MANAGER=${ROLL_NODE_PACKAGE_MANAGER:-yarn} + +## Which build a theme uses is a property of that theme, not of the project: a project can hold a +## boilerplate theme with Yarn scripts alongside older Gulpfile-only themes, and one global flag +## would send `yarn run dev` at themes that only have a Gulp build. ROLL_YARN_INSTEAD_OF_GULP +## remains a project-wide override, in both directions, when it is set at all. +function usesYarnBuild() { + local theme="$1" + + if [[ -n "${ROLL_YARN_INSTEAD_OF_GULP:-}" ]]; then + [[ "${ROLL_YARN_INSTEAD_OF_GULP}" == "1" ]] && return 0 + return 1 + fi + + case "${theme}" in + *Epartment/boilerplate*) return 0 ;; + esac + + return 1 +} + +function installTheme() { + THEME_DIR="/var/www/html/app/design/frontend/${SELECTED_THEME}" + + "${ROLL_DIR}/bin/roll" env exec -T -u www-data --workdir "${THEME_DIR}" "${ROLL_ENV_SHELL_CONTAINER}" "${NODE_PACKAGE_MANAGER}" install + info "Rebuilding node-sass package..." + "${ROLL_DIR}/bin/roll" env exec -T -u www-data --workdir "${THEME_DIR}" "${ROLL_ENV_SHELL_CONTAINER}" npm rebuild node-sass +} + +function buildTheme() { + THEME_DIR="/var/www/html/app/design/frontend/${SELECTED_THEME}" + + if usesYarnBuild "${SELECTED_THEME}"; then + "${ROLL_DIR}/bin/roll" env exec -T -u www-data --workdir "${THEME_DIR}" "${ROLL_ENV_SHELL_CONTAINER}" yarn run dev + else + "${ROLL_DIR}/bin/roll" env exec -T -u www-data --workdir "${THEME_DIR}" "${ROLL_ENV_SHELL_CONTAINER}" gulp build + fi +} + +function watchTheme() { + THEME_DIR="/var/www/html/app/design/frontend/${SELECTED_THEME}" + + if usesYarnBuild "${SELECTED_THEME}"; then + "${ROLL_DIR}/bin/roll" env exec -u www-data --workdir "${THEME_DIR}" "${ROLL_ENV_SHELL_CONTAINER}" yarn run watch + else + "${ROLL_DIR}/bin/roll" env exec -u www-data --workdir "${THEME_DIR}" "${ROLL_ENV_SHELL_CONTAINER}" gulp watch + fi +} + +function buildAll() { + local theme_dir="" + + for theme_dir in "${INSTALL_THEMES[@]}"; do + SELECTED_THEME="${theme_dir}" + installTheme + done + + for theme_dir in "${AVAILABLE_THEMES[@]}"; do + SELECTED_THEME="${theme_dir}" + buildTheme + done + + return 0 +} + +if (( ${#ROLL_PARAMS[@]} == 0 )); then + ## interactive: nothing to build was named on the command line + if (( ${#AVAILABLE_THEMES[@]} == 0 )); then + fatal "No themes with a Gulpfile.js were found under ${THEME_ROOT}." + fi + + if (( ${#AVAILABLE_THEMES[@]} == 1 )); then + SELECTED_THEME="${AVAILABLE_THEMES[0]}" + else + promptChoose SELECTED_THEME "roll theme build|watch" \ + "Which theme do you want to build or watch?" "${AVAILABLE_THEMES[@]}" + fi +elif [[ "${ROLL_PARAMS[0]}" == "all" ]]; then + SELECTED_THEME="all" +else + SELECTED_THEME="${ROLL_PARAMS[0]}" + if ! containsElement "${SELECTED_THEME}" "${AVAILABLE_THEMES[@]}"; then + fatal "Theme '${SELECTED_THEME}' was not found under ${THEME_ROOT}." + fi +fi + +if [[ "${SELECTED_THEME}" == "all" ]]; then + buildAll + exit 0 +fi + +ACTION="${ROLL_PARAMS[1]:-}" +promptChoose ACTION "roll theme ${SELECTED_THEME} build|watch" \ + "Do you want to build or watch ${SELECTED_THEME}?" "build" "watch" + +if [[ "${ACTION}" != "build" && "${ACTION}" != "watch" ]]; then + fatal "Unknown action '${ACTION}'; expected 'build' or 'watch'." +fi + +buildTheme + +if [[ "${ACTION}" == "watch" ]]; then + watchTheme +fi diff --git a/commands/magento2/theme.help b/commands/magento2/theme.help new file mode 100644 index 0000000..869311d --- /dev/null +++ b/commands/magento2/theme.help @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +[[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Build or watch a Magento 2 frontend theme inside the project container. +## @category: magento2 + +ROLL_USAGE=$(cat < [build|watch] + + Themes are discovered under \033[32mapp/design/frontend\033[0m in the project. Each theme is + built with Gulp, except an Epartment/boilerplate theme, which is built through its Yarn + scripts - the choice is made per theme, so mixed projects build correctly. Set + ROLL_YARN_INSTEAD_OF_GULP to 1 or 0 to force one build for every theme in the project. + + With no arguments and a terminal attached, a picker offers the available themes and then + whether to build or watch. On more than one theme, or when no action is given, an interactive + terminal is required; supply the theme and action directly to run non-interactively. + + \033[32mall\033[0m builds every discovered theme. + +\033[33mExamples:\033[0m + roll theme + roll theme all + roll theme Epartment/boilerplate build + roll theme Epartment/boilerplate watch + +\033[33mOptions:\033[0m + -h, --help Display this help menu +EOF +) diff --git a/commands/magento2/usage.help b/commands/magento2/usage.help index 8ae80f4..28c9366 100755 --- a/commands/magento2/usage.help +++ b/commands/magento2/usage.help @@ -13,7 +13,8 @@ ENV_TYPE_USAGE=$(cat <\033[0m Manage multi-store configuration (init, refresh, list)\n + multistore \033[32m\033[0m Manage multi-store configuration (init, refresh, list) + theme \033[32m[] [build|watch]\033[0m Build or watch a frontend theme (\033[31m'roll theme all'\033[0m builds every theme)\n EOF ) diff --git a/commands/magerun.help b/commands/magerun.help index 98fd567..f4399de 100644 --- a/commands/magerun.help +++ b/commands/magerun.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run n98-magerun2 inside the project php-fpm container. +## @category: magento2 ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Generate multi-store configuration and certificates from stores.json. +## @category: magento2 ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run Node.js inside the project php-fpm container. +## @category: utility NODE_HELP=$("${ROLL_DIR}/bin/roll" cli "node --help") diff --git a/commands/npm.help b/commands/npm.help index cac2fec..822f4fd 100644 --- a/commands/npm.help +++ b/commands/npm.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run npm inside the project php-fpm container. +## @category: utility NPM_HELP=$("${ROLL_DIR}/bin/roll" cli "npm --help") diff --git a/commands/redis.cmd b/commands/redis.cmd index ed55ef2..604cb09 100644 --- a/commands/redis.cmd +++ b/commands/redis.cmd @@ -9,8 +9,14 @@ if [[ ${ROLL_REDIS:-1} -eq 0 && ${ROLL_DRAGONFLY:-1} -eq 0 ]]; then fatal "Redis nor Dragonfly environment is not used (ROLL_REDIS=0|ROLL_DRAGONFLY=0)." fi -if [[ "${ROLL_PARAMS[0]}" == "help" ]]; then - roll redis --help || exit $? && exit $? +## Bare `roll redis` opens a session, so this cannot key on an empty ROLL_PARAMS the way db/env/svc +## do; --help arrives in "$@" because redis is on ROLL_CMD_ANYARGS, which is why it never used to +## reach this branch at all. +if [[ "${ROLL_PARAMS[0]:-}" == "help" ]] || [[ "${1:-}" == "--help" ]] || [[ "${1:-}" == "-h" ]]; then + ## Do NOT re-invoke `roll redis --help` here. `redis` is on roll's ROLL_CMD_ANYARGS list, so roll's + ## own parser stops at --help and passes it through to this script with ROLL_PARAMS empty - + ## re-invoking roll lands right back on this branch and forks until killed. + source "${ROLL_DIR}/commands/usage.cmd" fi ## load connection information for the redis service diff --git a/commands/redis.help b/commands/redis.help index 0070b84..858ea49 100644 --- a/commands/redis.help +++ b/commands/redis.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Connect to or run commands against the project Redis service. +## @category: database ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## `registry` is on roll's ROLL_CMD_ANYARGS list (needed so --format reaches this script), so +## roll's own parser stops at the first dash-prefixed argument and passes it straight through +## here with ROLL_PARAMS holding only the positional args before that flag. Parse flags from +## "$@", not ROLL_PARAMS. +REGISTRY_FORMAT="human" +while (( "$#" )); do + case "$1" in + -h|--help) + ## Do NOT re-invoke `roll registry --help` here - registry is on ROLL_CMD_ANYARGS, so + ## re-invoking roll lands right back on this branch and forks until killed. + source "${ROLL_DIR}/commands/usage.cmd" + ;; + --format=*) + REGISTRY_FORMAT="${1#*=}" + shift + ;; + --format) + REGISTRY_FORMAT="${2:-}" + shift + (( $# )) && shift + ;; + *) + fatal "Unsupported argument $1" + ;; + esac +done + +if [[ "${REGISTRY_FORMAT}" != "human" && "${REGISTRY_FORMAT}" != "json" ]]; then + fatal "Unsupported --format value '${REGISTRY_FORMAT}' (expected: human, json)" +fi + if (( ${#ROLL_PARAMS[@]} == 0 )) || [[ "${ROLL_PARAMS[0]}" == "help" ]]; then - roll registry --help || exit $? && exit $? + source "${ROLL_DIR}/commands/usage.cmd" +fi + +if [[ "${REGISTRY_FORMAT}" == "json" && "${ROLL_PARAMS[0]}" != "list" ]]; then + fatal "--format json is only supported for 'roll registry list'" fi ## Sub-command execution @@ -11,10 +46,37 @@ case "${ROLL_PARAMS[0]}" in # List all available commands filter="${ROLL_PARAMS[1]:-}" category="${ROLL_PARAMS[2]:-}" - + initializeRegistry - - if [[ -n "$category" ]]; then + + if [[ "${REGISTRY_FORMAT}" == "json" ]]; then + loadRegistryMetadata + + out="[" + first=1 + i=0 + while [[ $i -lt ${#ROLL_REGISTRY_COMMANDS[@]} ]]; do + command="${ROLL_REGISTRY_COMMANDS[$i]}" + cmdCategory="${ROLL_REGISTRY_CATEGORIES[$i]}" + cmdDescription="${ROLL_REGISTRY_DESCRIPTIONS[$i]}" + cmdPriority="${ROLL_REGISTRY_PRIORITIES[$i]}" + cmdSource="${ROLL_REGISTRY_SOURCES[$i]}" + i=$((i + 1)) + + [[ -n "$filter" && ! "$command" =~ $filter ]] && continue + [[ -n "$category" && "$cmdCategory" != "$category" ]] && continue + + (( first == 0 )) && out+="," + first=0 + out+="{\"command\":\"$(jsonEscape "$command")\"," + out+="\"category\":\"$(jsonEscape "$cmdCategory")\"," + out+="\"description\":\"$(jsonEscape "$cmdDescription")\"," + out+="\"priority\":${cmdPriority}," + out+="\"source\":\"$(jsonEscape "$cmdSource")\"}" + done + out+="]" + printf '%s\n' "${out}" + elif [[ -n "$category" ]]; then echo -e "\033[33mCommands in '${category}' category:\033[0m" listRegisteredCommands "$filter" "$category" elif [[ -n "$filter" ]]; then @@ -29,11 +91,11 @@ case "${ROLL_PARAMS[0]}" in categories) # List commands organized by category category="${ROLL_PARAMS[1]:-}" - + initializeRegistry - + if [[ -n "$category" ]]; then - echo -e "\033[33m${category^} Commands:\033[0m" + echo -e "\033[33m$(capitalize "$category") Commands:\033[0m" listCommandsByCategory "$category" else listCommandsByCategory @@ -72,9 +134,10 @@ case "${ROLL_PARAMS[0]}" in fi pattern="${ROLL_PARAMS[1]}" - + initializeRegistry - + loadRegistryMetadata + echo -e "\033[33mSearching for commands matching: '$pattern'\033[0m" echo "" diff --git a/commands/registry.help b/commands/registry.help index c3f7702..fdf7e77 100644 --- a/commands/registry.help +++ b/commands/registry.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Inspect and query the RollDev command registry. +## @category: utility echo -e "\033[33mUsage:\033[0m" echo " registry [options]" diff --git a/commands/restart.help b/commands/restart.help index 945ec87..83ab506 100644 --- a/commands/restart.help +++ b/commands/restart.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Restart the RollDev core services. +## @category: environment ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 -# Load core utilities (environment config loaded later if needed) -assertDockerRunning - -# Default configuration values -RESTORE_SERVICES=() -RESTORE_CONFIG=1 -RESTORE_VERIFY=1 -RESTORE_FORCE=0 -RESTORE_DRY_RUN=0 -RESTORE_QUIET=0 -RESTORE_VERBOSE=0 -RESTORE_DECRYPT="" -RESTORE_BACKUP_FILE="" -RESTORE_OUTPUT_DIR="" -PROGRESS=1 -ROLL_ENV_LOADED=0 - -# Legacy migration support -RESTORE_LEGACY_MIGRATION=1 - -# Parse command line arguments -POSITIONAL_ARGS=() -# Start with any arguments passed from the main roll script -if [[ -n "${ROLL_PARAMS[*]}" ]]; then - POSITIONAL_ARGS+=("${ROLL_PARAMS[@]}") -fi -while [[ $# -gt 0 ]]; do - case "$1" in - --help|-h) - roll restore-full --help - exit 0 - ;; - --services=*) - IFS=',' read -ra RESTORE_SERVICES <<< "${1#*=}" - shift - ;; - --no-config) - RESTORE_CONFIG=0 - shift - ;; - --no-verify) - RESTORE_VERIFY=0 - shift - ;; - --force|-f) - RESTORE_FORCE=1 - shift - ;; - --dry-run) - RESTORE_DRY_RUN=1 - shift - ;; - --quiet|-q) - RESTORE_QUIET=1 - PROGRESS=0 - shift - ;; - --verbose|-v) - RESTORE_VERBOSE=1 - shift - ;; - --decrypt=*) - RESTORE_DECRYPT="${1#*=}" - shift - ;; - --decrypt) - # Flag without value - will prompt for password later - RESTORE_DECRYPT="PROMPT" - shift - ;; - --no-progress) - PROGRESS=0 - shift - ;; - --no-legacy-migration) - RESTORE_LEGACY_MIGRATION=0 - shift - ;; - --) - shift - break - ;; - -*) - error "Unknown option: $1" - exit 1 - ;; - *) - POSITIONAL_ARGS+=("$1") - shift - ;; - esac -done - -# Add any remaining arguments after -- to positional args -POSITIONAL_ARGS+=("$@") - -# Expect exactly two positional arguments: archive and output directory -if [[ ${#POSITIONAL_ARGS[@]} -ne 2 ]]; then - error "Usage: roll restore-full [options] archive output-dir" - exit 1 -fi - -RESTORE_BACKUP_FILE="${POSITIONAL_ARGS[0]}" -RESTORE_OUTPUT_DIR="${POSITIONAL_ARGS[1]}" - -# Set environment path -mkdir -p "$RESTORE_OUTPUT_DIR" -cd "$RESTORE_OUTPUT_DIR" -ROLL_ENV_PATH="$(pwd)" - -# Utility functions for restore operations -function promptPassword() { - local prompt="$1" - local password="" - - # Don't prompt in quiet mode or non-interactive shells - if [[ $RESTORE_QUIET -eq 1 ]] || [[ ! -t 0 ]]; then - error "Password required but running in non-interactive mode. Use --decrypt=password instead." - exit 1 - fi - - echo -n "$prompt: " >&2 - read -s password - echo >&2 - - if [[ -z "$password" ]]; then - error "Password cannot be empty" - exit 1 - fi - - echo "$password" -} - -function detectEncryptedBackup() { - local backup_path="$1" - - # Check if backup contains .gpg files - if [[ -d "$backup_path" ]]; then - # Directory format - check for .gpg files - if find "$backup_path" -name "*.gpg" -type f | head -1 | grep -q .; then - return 0 # Encrypted - fi - else - # Archive format - check if archive contains .gpg files - local archive_file="$backup_path" - if [[ -f "$archive_file" ]]; then - # Determine decompression command - local decompress_cmd="cat" - case "$archive_file" in - *.tar.gz) decompress_cmd="gzip -dc" ;; - *.tar.xz) decompress_cmd="xz -dc" ;; - *.tar.lz4) decompress_cmd="lz4 -dc" ;; - esac - - # Check if archive contains .gpg files - if $decompress_cmd "$archive_file" | tar -tf - 2>/dev/null | grep -q "\.gpg$"; then - return 0 # Encrypted - fi - fi - fi - - return 1 # Not encrypted -} - -function showProgress() { - [[ $PROGRESS -eq 0 ]] && return - local current=$1 - local total=$2 - local description="$3" - local percent=$((current * 100 / total)) - local bar_length=30 - local filled_length=$((percent * bar_length / 100)) - - printf "\r[" - printf "%*s" $filled_length | tr ' ' '=' - printf "%*s" $((bar_length - filled_length)) | tr ' ' '-' - printf "] %d%% %s" $percent "$description" - - # Always end with a newline for clean output - echo "" -} - -function logMessage() { - [[ $RESTORE_QUIET -eq 1 ]] && return - local level="$1" - shift - case "$level" in - INFO) info "$@" ;; - SUCCESS) success "$@" ;; - WARNING) warning "$@" ;; - ERROR) error "$@" ;; - VERBOSE) [[ $RESTORE_VERBOSE -eq 1 ]] && info "[VERBOSE] $@" ;; - esac -} - -function logVerbose() { - [[ $RESTORE_VERBOSE -eq 1 ]] && logMessage INFO "$@" || true -} - -function performLegacyMigration() { - if [[ $RESTORE_LEGACY_MIGRATION -eq 0 ]]; then - return 0 - fi - - local current_dir="$(pwd)" - - # Handle Warden to Roll migration - if [[ ! -f "$current_dir/.env.roll" ]]; then - if [[ -f "$current_dir/.env" ]]; then - logMessage INFO "Performing legacy Warden to Roll migration..." - - # Create backup of original .env - cp "$current_dir/.env" "$current_dir/.env.backup.$(date +%s)" - - # Convert WARDEN to ROLL - sed -i.warden 's/WARDEN/ROLL/g' "$current_dir/.env" - - # Migrate .warden directory to .roll - if [[ -d "$current_dir/.warden" ]]; then - mv "$current_dir/.warden" "$current_dir/.roll" - - if [[ -f "$current_dir/.roll/warden-env.yml" ]]; then - mv "$current_dir/.roll/warden-env.yml" "$current_dir/.roll/roll-env.yml" - sed -i.warden 's/WARDEN/ROLL/g;s/warden/roll/g' "$current_dir/.roll/roll-env.yml" - fi - fi - - # Ensure ROLL_NO_STATIC_CACHING is set - if [[ -n "$(grep -r 'ROLL_NO_STATIC_CACHING' "$current_dir/.env")" ]]; then - perl -i -pe's/.*ROLL_NO_STATIC_CACHING.*$/ROLL_NO_STATIC_CACHING\=1/g' "$current_dir/.env" - else - echo "ROLL_NO_STATIC_CACHING=1" >> "$current_dir/.env" - fi - - # Move to .env.roll if it contains ROLL_ variables - if [[ -n "$(grep -r 'ROLL_' "$current_dir/.env")" ]]; then - mv "$current_dir/.env" "$current_dir/.env.roll" - fi - - logMessage SUCCESS "Legacy migration completed" - fi - fi -} - -function findLatestBackup() { - local backup_dir="$(pwd)/.roll/backups" - - if [[ ! -d "$backup_dir" ]]; then - return 1 - fi - - # Look for timestamped directories first (new format) - local latest_dir=$(ls "$backup_dir" 2>/dev/null | grep '^[0-9]\{10\}$' | sort -n | tail -1) - if [[ -n "$latest_dir" ]]; then - echo "$latest_dir" - return 0 - fi - - # Look for compressed archives - local latest_archive=$(ls "$backup_dir"/backup_*_*.tar* 2>/dev/null | sort | tail -1) - if [[ -n "$latest_archive" ]]; then - # Extract timestamp from filename - local timestamp=$(basename "$latest_archive" | grep -o '[0-9]\{10\}') - echo "$timestamp" - return 0 - fi - - return 1 -} - -function extractBackupArchive() { - local backup_id="$1" - local backup_dir="$(pwd)/.roll/backups" - local extract_dir="$backup_dir/${backup_id}_extracted" - - # Check if already extracted - if [[ -d "$extract_dir" ]]; then - echo "$extract_dir" - return 0 - fi - - # Find the archive file - local archive_file="" - for ext in ".tar.gz" ".tar.xz" ".tar.lz4" ".tar"; do - local potential_file="$backup_dir/backup_${ROLL_ENV_NAME}_${backup_id}${ext}" - if [[ -f "$potential_file" ]]; then - archive_file="$potential_file" - break - fi - done - - # Also check for generic archive names - if [[ -z "$archive_file" ]]; then - archive_file=$(ls "$backup_dir"/*"$backup_id"*.tar* 2>/dev/null | head -1) - fi - - if [[ -z "$archive_file" ]]; then - logMessage ERROR "Backup archive not found for ID: $backup_id" - return 1 - fi - - logMessage INFO "Extracting backup archive: $(basename "$archive_file")" - - mkdir -p "$extract_dir" - - # Determine decompression command based on file extension - local decompress_cmd="cat" - case "$archive_file" in - *.tar.gz) decompress_cmd="gzip -d" ;; - *.tar.xz) decompress_cmd="xz -d" ;; - *.tar.lz4) decompress_cmd="lz4 -d" ;; - esac - - if $decompress_cmd < "$archive_file" | tar -xf - -C "$extract_dir" --strip-components=1 2>/dev/null; then - echo "$extract_dir" - return 0 - else - logMessage ERROR "Failed to extract backup archive" - rm -rf "$extract_dir" - return 1 - fi -} - -function extractBackupArchiveFile() { - local archive_file="$1" - local backup_dir="$(pwd)/.roll/backups" - local base_name="$(basename "$archive_file")" - base_name="${base_name%%.tar*}" - local extract_dir="$backup_dir/${base_name}_extracted" - - logVerbose "Extracting backup archive file" - logVerbose "Archive file: $archive_file" - logVerbose "Backup directory: $backup_dir" - logVerbose "Extract directory: $extract_dir" - - if [[ -d "$extract_dir" ]]; then - logVerbose "Found already extracted backup at: $extract_dir" - echo "$extract_dir" - return 0 - fi - - mkdir -p "$extract_dir" - - local decompress_cmd="cat" - case "$archive_file" in - *.tar.gz) decompress_cmd="gzip -d" ;; - *.tar.xz) decompress_cmd="xz -d" ;; - *.tar.lz4) decompress_cmd="lz4 -d" ;; - esac - - logVerbose "Using decompression command: $decompress_cmd" - - if $decompress_cmd < "$archive_file" | tar -xf - -C "$extract_dir" --strip-components=1 2>/dev/null; then - logVerbose "Successfully extracted to: $extract_dir" - echo "$extract_dir" - return 0 - else - logMessage ERROR "Failed to extract backup archive" - rm -rf "$extract_dir" - return 1 - fi -} - -function validateBackup() { - local backup_path="$1" - - if [[ $RESTORE_VERIFY -eq 0 ]]; then - return 0 - fi - - logMessage INFO "Validating backup integrity..." - - # Check if backup metadata exists - if [[ ! -f "$backup_path/metadata/backup.json" ]]; then - logMessage WARNING "Backup metadata not found, proceeding with legacy format" - return 0 - fi - - # Verify checksums if available - if [[ -f "$backup_path/metadata/checksums.sha256" ]]; then - if (cd "$backup_path" && sha256sum -c metadata/checksums.sha256 >/dev/null 2>&1); then - logMessage SUCCESS "Backup integrity verified" - return 0 - else - logMessage ERROR "Backup integrity check failed" - return 1 - fi - fi - - logMessage SUCCESS "Backup validation completed" - return 0 -} - -function getBackupMetadata() { - local backup_path="$1" - local metadata_file="$backup_path/metadata/backup.json" - - if [[ -f "$metadata_file" ]]; then - cat "$metadata_file" - else - # Return empty JSON for legacy backups - echo "{}" - fi -} - -function detectBackupServices() { - local backup_path="$1" - local services=() - - # Check for volume backups - if [[ -d "$backup_path/volumes" ]]; then - for volume_file in "$backup_path/volumes"/*; do - if [[ -f "$volume_file" ]]; then - local service_name=$(basename "$volume_file" | sed 's/\.tar.*//') - services+=("$service_name") - fi - done - else - # Legacy format detection - if [[ -f "$backup_path/db.tar.gz" ]]; then - services+=("db") - fi - if [[ -f "$backup_path/redis.tar.gz" ]]; then - services+=("redis") - fi - if [[ -f "$backup_path/es.tar.gz" ]]; then - services+=("elasticsearch") - fi - if [[ -f "$backup_path/os.tar.gz" ]]; then - services+=("opensearch") - fi - fi - - echo "${services[@]}" -} - -function stopEnvironment() { - if [[ $RESTORE_DRY_RUN -eq 1 ]]; then - logMessage INFO "[DRY RUN] Would stop environment" - return 0 - fi - - logMessage INFO "Stopping environment for consistent restore..." - logVerbose "Environment path: ${ROLL_ENV_PATH}" - - # Check if environment is configured before trying to stop it - # The env commands require .env.roll to exist, which may not be the case - # during a full restore (config comes from backup) - if [[ ! -f "${ROLL_ENV_PATH}/.env.roll" ]]; then - logMessage INFO "No environment configuration found yet, skipping stop" - logVerbose "Missing file: ${ROLL_ENV_PATH}/.env.roll" - return 0 - fi - - # Use docker compose directly instead of roll env to avoid exit on error - # when environment is not fully configured - local project_name="${ROLL_ENV_NAME:-}" - if [[ -z "$project_name" ]]; then - # Try to extract from .env.roll if available - logVerbose "ROLL_ENV_NAME not set, extracting from .env.roll" - project_name=$(grep -E "^ROLL_ENV_NAME=" "${ROLL_ENV_PATH}/.env.roll" 2>/dev/null | cut -d'=' -f2 | tr -d '"'"'" || true) - fi - - logVerbose "Project name: ${project_name:-}" - - if [[ -n "$project_name" ]]; then - # Check if any containers are running for this project - logVerbose "Checking for running containers with label: com.docker.compose.project=${project_name}" - local running_containers=$(docker ps --filter "label=com.docker.compose.project=${project_name}" --format '{{.Names}}' 2>/dev/null || true) - if [[ -n "$running_containers" ]]; then - logVerbose "Found running containers: $running_containers" - logMessage INFO "Stopping running containers..." - logVerbose "Executing: docker compose -p ${project_name} down" - docker compose -p "${project_name}" down >/dev/null 2>&1 || true - logVerbose "Containers stopped" - else - logVerbose "No running containers found" - fi - else - logVerbose "No project name available, skipping container stop" - fi -} - -function getVolumeMapping() { - local service_name="$1" - - case "$service_name" in - db) - case "${DB_DISTRIBUTION:-mariadb}" in - mysql|mariadb) echo "${ROLL_ENV_NAME}_dbdata:mysql" ;; - postgres) echo "${ROLL_ENV_NAME}_dbdata:postgres" ;; - *) echo "${ROLL_ENV_NAME}_dbdata:mysql" ;; - esac - ;; - redis) echo "${ROLL_ENV_NAME}_redis:redis" ;; - dragonfly) echo "${ROLL_ENV_NAME}_dragonfly:dragonfly" ;; - elasticsearch) echo "${ROLL_ENV_NAME}_esdata:elasticsearch" ;; - opensearch) echo "${ROLL_ENV_NAME}_osdata:opensearch" ;; - mongodb) echo "${ROLL_ENV_NAME}_mongodb:mongodb" ;; - rabbitmq) echo "${ROLL_ENV_NAME}_rabbitmq:rabbitmq" ;; - varnish) echo "${ROLL_ENV_NAME}_varnish:varnish" ;; - *) echo "${ROLL_ENV_NAME}_${service_name}:generic" ;; - esac -} - -function restoreVolume() { - local service_name="$1" - local backup_path="$2" - local step="$3" - local total="$4" - - showProgress $step $total "Restoring $service_name volume" - - local volume_mapping=$(getVolumeMapping "$service_name") - IFS=':' read -r volume_name service_type <<< "$volume_mapping" - - logVerbose "Restoring service: $service_name" - logVerbose "Volume mapping: $volume_mapping" - logVerbose "Volume name: $volume_name, Service type: $service_type" - - # Determine backup file location (check for both encrypted and unencrypted) - local backup_file="" - local is_encrypted=false - - # Check for encrypted files first (.gpg extension) - if [[ -f "$backup_path/volumes/${service_name}.tar.gz.gpg" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar.gz.gpg" - is_encrypted=true - elif [[ -f "$backup_path/volumes/${service_name}.tar.xz.gpg" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar.xz.gpg" - is_encrypted=true - elif [[ -f "$backup_path/volumes/${service_name}.tar.lz4.gpg" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar.lz4.gpg" - is_encrypted=true - elif [[ -f "$backup_path/volumes/${service_name}.tar.gpg" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar.gpg" - is_encrypted=true - # Check for unencrypted files - elif [[ -f "$backup_path/volumes/${service_name}.tar.gz" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar.gz" - elif [[ -f "$backup_path/volumes/${service_name}.tar.xz" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar.xz" - elif [[ -f "$backup_path/volumes/${service_name}.tar.lz4" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar.lz4" - elif [[ -f "$backup_path/volumes/${service_name}.tar" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar" - elif [[ -f "$backup_path/${service_name}.tar.gz" ]]; then - # Legacy format - backup_file="$backup_path/${service_name}.tar.gz" - elif [[ "$service_name" == "elasticsearch" && -f "$backup_path/es.tar.gz" ]]; then - # Legacy format uses short names for the search engines - backup_file="$backup_path/es.tar.gz" - elif [[ "$service_name" == "opensearch" && -f "$backup_path/os.tar.gz" ]]; then - backup_file="$backup_path/os.tar.gz" - else - logMessage WARNING "Backup file not found for service: $service_name" - logVerbose "Searched in: $backup_path/volumes/ and $backup_path/" - return 0 - fi - - logVerbose "Found backup file: $backup_file" - logVerbose "Encrypted: $is_encrypted" - - if [[ $RESTORE_DRY_RUN -eq 1 ]]; then - if [[ $is_encrypted == true ]]; then - logMessage INFO "[DRY RUN] Would decrypt and restore $service_name from $backup_file to volume $volume_name" - else - logMessage INFO "[DRY RUN] Would restore $service_name from $backup_file to volume $volume_name" - fi - return 0 - fi - - # Validate decryption password if file is encrypted - if [[ $is_encrypted == true ]]; then - if [[ -z "$RESTORE_DECRYPT" ]]; then - logMessage ERROR "Encrypted backup file found but no decryption password provided" - return 1 - fi - fi - - # Get Docker Compose version for proper labeling - local docker_compose_version=$(docker compose version 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+' | head -1) - local volume_base_name=$(echo "$volume_name" | sed "s/${ROLL_ENV_NAME}_//") - - logVerbose "Docker Compose version: $docker_compose_version" - logVerbose "Volume base name: $volume_base_name" - - # Remove existing volume if it exists - if docker volume inspect "$volume_name" >/dev/null 2>&1; then - if [[ $RESTORE_FORCE -eq 1 ]]; then - logMessage INFO "Removing existing volume: $volume_name" - logVerbose "Executing: docker volume rm $volume_name" - docker volume rm "$volume_name" >/dev/null 2>&1 - else - logMessage ERROR "Volume $volume_name already exists. Use --force to overwrite." - return 1 - fi - else - logVerbose "Volume $volume_name does not exist yet" - fi - - # Create new volume with proper labels - logVerbose "Creating volume: $volume_name with labels" - logVerbose " - com.docker.compose.project=$ROLL_ENV_NAME" - logVerbose " - com.docker.compose.version=$docker_compose_version" - logVerbose " - com.docker.compose.volume=$volume_base_name" - docker volume create "$volume_name" \ - --label com.docker.compose.project="$ROLL_ENV_NAME" \ - --label com.docker.compose.version="$docker_compose_version" \ - --label com.docker.compose.volume="$volume_base_name" >/dev/null 2>&1 - - # Restore the volume data with decryption if needed - local temp_container="${ROLL_ENV_NAME}_restore_${service_name}_$$" - logVerbose "Temp container name: $temp_container" - - if [[ $is_encrypted == true ]]; then - # Decrypt and decompress pipeline - use ubuntu and original tar approach with strip components - # Use passphrase-fd to avoid shell escaping issues with passwords - # Determine the correct tar command based on the backup file format - local tar_cmd="tar -xf -" - case "$backup_file" in - *.tar.gz.gpg) tar_cmd="tar -xzf -" ;; - *.tar.xz.gpg) tar_cmd="tar -xJf -" ;; - *.tar.lz4.gpg) tar_cmd="lz4 -d - | tar -xf -" ;; - esac - - if echo "$RESTORE_DECRYPT" | gpg --batch --yes --quiet --passphrase-fd 0 --decrypt "$backup_file" | docker run --rm --name "$temp_container" --mount source="$volume_name",target=/data -i ubuntu bash -c "cd /data && $tar_cmd --strip-components=1" 2>/dev/null; then - logMessage SUCCESS "Successfully restored and decrypted $service_name volume" - return 0 - else - logMessage ERROR "Failed to decrypt and restore $service_name volume" - return 1 - fi - else - # Regular restore without decryption - use ubuntu and original tar approach with strip components - # For compressed files, we need to handle decompression properly - case "$backup_file" in - *.tar.gz) - if docker run --rm --name "$temp_container" \ - --mount source="$volume_name",target=/data \ - -v "$(dirname "$backup_file")":/backup \ - ubuntu bash \ - -c "cd /data && tar -xzf /backup/$(basename "$backup_file") --strip-components=1" 2>/dev/null; then - - logMessage SUCCESS "Successfully restored $service_name volume" - return 0 - else - logMessage ERROR "Failed to restore $service_name volume" - return 1 - fi - ;; - *.tar.xz) - if docker run --rm --name "$temp_container" \ - --mount source="$volume_name",target=/data \ - -v "$(dirname "$backup_file")":/backup \ - ubuntu bash \ - -c "cd /data && tar -xJf /backup/$(basename "$backup_file") --strip-components=1" 2>/dev/null; then - - logMessage SUCCESS "Successfully restored $service_name volume" - return 0 - else - logMessage ERROR "Failed to restore $service_name volume" - return 1 - fi - ;; - *.tar.lz4) - if docker run --rm --name "$temp_container" \ - --mount source="$volume_name",target=/data \ - -v "$(dirname "$backup_file")":/backup \ - ubuntu bash \ - -c "cd /data && lz4 -d /backup/$(basename "$backup_file") - | tar -xf - --strip-components=1" 2>/dev/null; then - - logMessage SUCCESS "Successfully restored $service_name volume" - return 0 - else - logMessage ERROR "Failed to restore $service_name volume" - return 1 - fi - ;; - *.tar) - if docker run --rm --name "$temp_container" \ - --mount source="$volume_name",target=/data \ - -v "$(dirname "$backup_file")":/backup \ - ubuntu bash \ - -c "cd /data && tar -xf /backup/$(basename "$backup_file") --strip-components=1" 2>/dev/null; then - - logMessage SUCCESS "Successfully restored $service_name volume" - return 0 - else - logMessage ERROR "Failed to restore $service_name volume" - return 1 - fi - ;; - *) - logMessage ERROR "Unsupported backup file format: $backup_file" - return 1 - ;; - esac - fi -} - -function restoreConfigurations() { - local backup_path="$1" - local step="$2" - local total="$3" - - if [[ $RESTORE_CONFIG -eq 0 ]]; then - return 0 - fi - - showProgress $step $total "Restoring configuration files" - - local config_source_dir="$backup_path/config" - local current_dir="$(pwd)" - - # Legacy format support - if [[ ! -d "$config_source_dir" ]]; then - # Check for legacy files in backup root (both encrypted and unencrypted) - local legacy_files=("env.php" "auth.json") - for file in "${legacy_files[@]}"; do - local source_file="" - local is_encrypted=false - - # Check for encrypted version first - if [[ -f "$backup_path/${file}.gpg" ]]; then - source_file="$backup_path/${file}.gpg" - is_encrypted=true - elif [[ -f "$backup_path/$file" ]]; then - source_file="$backup_path/$file" - fi - - if [[ -n "$source_file" ]]; then - local target_path="" - case "$file" in - env.php) target_path="$current_dir/app/etc/env.php" ;; - auth.json) target_path="$current_dir/auth.json" ;; - esac - - if [[ -n "$target_path" ]]; then - if [[ $RESTORE_DRY_RUN -eq 1 ]]; then - if [[ $is_encrypted == true ]]; then - logMessage INFO "[DRY RUN] Would decrypt and restore $file to $target_path" - else - logMessage INFO "[DRY RUN] Would restore $file to $target_path" - fi - else - mkdir -p "$(dirname "$target_path")" - - if [[ $is_encrypted == true ]]; then - # Decrypt the file directly to target location - if [[ -n "$RESTORE_DECRYPT" ]]; then - if echo "$RESTORE_DECRYPT" | gpg --batch --yes --quiet --passphrase-fd 0 --decrypt "$source_file" > "$target_path"; then - logMessage INFO "Decrypted and restored $file" - else - logMessage ERROR "Failed to decrypt $file" - return 1 - fi - else - logMessage ERROR "Encrypted config file found but no decryption password provided" - return 1 - fi - else - cp "$source_file" "$target_path" - logMessage INFO "Restored $file" - fi - fi - fi - fi - done - return 0 - fi - - # New format with structured config directory - if [[ $RESTORE_DRY_RUN -eq 1 ]]; then - logMessage INFO "[DRY RUN] Would restore configuration files from $config_source_dir" - return 0 - fi - - # Restore configuration files (both encrypted and unencrypted) - if [[ -d "$config_source_dir" ]]; then - # Process all files including .gpg files - find "$config_source_dir" -type f | while read -r config_file; do - local relative_path="${config_file#$config_source_dir/}" - local is_encrypted=false - - # Check if file is encrypted - if [[ "$config_file" == *.gpg ]]; then - is_encrypted=true - # Remove .gpg extension for target path - relative_path="${relative_path%.gpg}" - fi - - local target_path="$current_dir/$relative_path" - - # Create target directory if needed - mkdir -p "$(dirname "$target_path")" - - # Backup existing file if it exists - if [[ -f "$target_path" ]]; then - if [[ $is_encrypted == true ]]; then - # For encrypted files, we can't easily compare so always backup - cp "$target_path" "$target_path.backup.$(date +%s)" - logMessage INFO "Backed up existing $relative_path" - elif ! cmp -s "$config_file" "$target_path"; then - cp "$target_path" "$target_path.backup.$(date +%s)" - logMessage INFO "Backed up existing $relative_path" - fi - fi - - if [[ $is_encrypted == true ]]; then - # Decrypt the file - if [[ -n "$RESTORE_DECRYPT" ]]; then - if echo "$RESTORE_DECRYPT" | gpg --batch --yes --quiet --passphrase-fd 0 --decrypt "$config_file" > "$target_path"; then - logMessage INFO "Decrypted and restored $relative_path" - else - logMessage ERROR "Failed to decrypt $relative_path" - return 1 - fi - else - logMessage ERROR "Encrypted config file found but no decryption password provided" - return 1 - fi - else - # Copy unencrypted file - cp "$config_file" "$target_path" - logMessage INFO "Restored $relative_path" - fi - done - fi - - logMessage SUCCESS "Configuration restore completed" -} - -function restoreSourceCode() { - local backup_path="$1" - local target_dir="$2" - local step="$3" - local total="$4" - - showProgress $step $total "Restoring source code" - - local src_file="" - local is_encrypted=false - - for ext in ".tar.gz" ".tar.xz" ".tar.lz4" ".tar"; do - if [[ -f "$backup_path/source${ext}.gpg" ]]; then - src_file="$backup_path/source${ext}.gpg" - is_encrypted=true - break - elif [[ -f "$backup_path/source${ext}" ]]; then - src_file="$backup_path/source${ext}" - break - fi - done - - if [[ -z "$src_file" ]]; then - logMessage INFO "No source code archive found in backup" - return 0 - fi - - if [[ $RESTORE_DRY_RUN -eq 1 ]]; then - logMessage INFO "[DRY RUN] Would extract source code to $target_dir" - return 0 - fi - - mkdir -p "$target_dir" - - # Use direct file reading instead of piping for cross-platform compatibility - # BSD tar (macOS) doesn't reliably handle piped stdin with -C flag - # Both BSD and GNU tar auto-detect compression format with -xf - if [[ $is_encrypted == true ]]; then - if [[ -z "$RESTORE_DECRYPT" ]]; then - logMessage ERROR "Encrypted source archive found but no decryption password provided" - return 1 - fi - # Decrypt to temp file first, then extract directly - local temp_file="$backup_path/source_decrypted.tar" - case "$src_file" in - *.tar.gz.gpg) temp_file="$backup_path/source_decrypted.tar.gz" ;; - *.tar.xz.gpg) temp_file="$backup_path/source_decrypted.tar.xz" ;; - *.tar.lz4.gpg) temp_file="$backup_path/source_decrypted.tar.lz4" ;; - esac - - if echo "$RESTORE_DECRYPT" | gpg --batch --yes --quiet --passphrase-fd 0 --decrypt "$src_file" > "$temp_file"; then - if tar -xf "$temp_file" -C "$target_dir"; then - rm -f "$temp_file" - logMessage SUCCESS "Source code restored" - return 0 - else - rm -f "$temp_file" - logMessage ERROR "Failed to extract source code" - return 1 - fi - else - rm -f "$temp_file" 2>/dev/null - logMessage ERROR "Failed to decrypt source archive" - return 1 - fi - else - # Direct extraction - works on both BSD tar (macOS) and GNU tar (Linux) - if tar -xf "$src_file" -C "$target_dir"; then - logMessage SUCCESS "Source code restored" - return 0 - else - logMessage ERROR "Failed to restore source code" - return 1 - fi - fi -} - -function performRestore() { - logMessage INFO "Starting full restore from $(basename "$RESTORE_BACKUP_FILE")..." - - logVerbose "Backup file: $RESTORE_BACKUP_FILE" - logVerbose "Output directory: $RESTORE_OUTPUT_DIR" - logVerbose "Environment path: $ROLL_ENV_PATH" - - # Perform legacy migration if needed - performLegacyMigration - - # Validate database environment - if [[ ${ROLL_DB:-1} -eq 0 ]]; then - logMessage ERROR "Database environment is not enabled (ROLL_DB=0)" - exit 1 - fi - - # Determine backup path from archive argument - local backup_path="" - - logVerbose "Checking backup file type..." - if [[ -f "$RESTORE_BACKUP_FILE" ]]; then - logMessage INFO "Extracting backup archive..." - logVerbose "Backup is a file, extracting..." - backup_path=$(extractBackupArchiveFile "$RESTORE_BACKUP_FILE") - logMessage SUCCESS "Archive extracted" - elif [[ -d "$RESTORE_BACKUP_FILE" ]]; then - logVerbose "Backup is a directory" - backup_path="$RESTORE_BACKUP_FILE" - else - logMessage ERROR "Backup file not found: $RESTORE_BACKUP_FILE" - exit 1 - fi - - logVerbose "Backup path resolved to: $backup_path" - - # Detect if backup is encrypted and handle password prompting - if detectEncryptedBackup "$backup_path"; then - if [[ -z "$RESTORE_DECRYPT" ]]; then - # No password provided, prompt for it - RESTORE_DECRYPT=$(promptPassword "Encrypted backup detected. Enter decryption password") - elif [[ "$RESTORE_DECRYPT" == "PROMPT" ]]; then - # Explicit prompt requested - RESTORE_DECRYPT=$(promptPassword "Enter decryption password") - fi - - if [[ -z "$RESTORE_DECRYPT" ]]; then - logMessage ERROR "Encrypted backup requires a password. Use --decrypt=password or --decrypt to prompt." - exit 1 - fi - - logMessage INFO "Encrypted backup detected, will decrypt during restoration" - fi - - # Validate backup - validateBackup "$backup_path" || exit 1 - - # Get backup metadata - local metadata=$(getBackupMetadata "$backup_path") - logMessage INFO "Restoring backup from: $(basename \"$RESTORE_BACKUP_FILE\")" - logVerbose "Backup metadata: $metadata" - - local source_exists=0 - for ext in ".tar.gz" ".tar.xz" ".tar.lz4" ".tar"; do - if [[ -f "$backup_path/source${ext}" ]] || [[ -f "$backup_path/source${ext}.gpg" ]]; then - logVerbose "Found source archive: source${ext}" - source_exists=1 - break - fi - done - logVerbose "Source code exists in backup: $source_exists" - - if [[ $ROLL_ENV_LOADED -eq 0 ]]; then - ROLL_ENV_NAME=$(echo "$metadata" | grep -o '"environment"[^"]*"' | head -1 | sed 's/.*"environment"[ ]*:[ ]*"\([^"]*\)".*/\1/') - logVerbose "Extracted ROLL_ENV_NAME from metadata: ${ROLL_ENV_NAME:-}" - fi - - # Detect available services in backup - local available_services=($(detectBackupServices "$backup_path")) - if [[ ${#available_services[@]} -eq 0 ]]; then - logMessage ERROR "No services found in backup" - exit 1 - fi - - logMessage INFO "Available services in backup: ${available_services[*]}" - - # Determine which services to restore - local services_to_restore=() - if [[ ${#RESTORE_SERVICES[@]} -gt 0 ]]; then - logVerbose "User specified services to restore: ${RESTORE_SERVICES[*]}" - # Use specified services - for service in "${RESTORE_SERVICES[@]}"; do - if containsElement "$service" "${available_services[@]}"; then - services_to_restore+=("$service") - else - logMessage WARNING "Service $service not found in backup, skipping" - fi - done - else - logVerbose "No specific services requested, restoring all available" - # Restore all available services - services_to_restore=("${available_services[@]}") - fi - - if [[ ${#services_to_restore[@]} -eq 0 ]]; then - logMessage ERROR "No services to restore" - exit 1 - fi - - logMessage INFO "Restoring services: ${services_to_restore[*]}" - - # Stop environment - stopEnvironment - - # Calculate total steps - local total_steps=${#services_to_restore[@]} - if [[ $RESTORE_CONFIG -eq 1 ]]; then - total_steps=$((total_steps + 1)) - fi - if [[ $source_exists -eq 1 ]]; then - total_steps=$((total_steps + 1)) - fi - - local current_step=0 - - # Restore source code if available - if [[ $source_exists -eq 1 ]]; then - current_step=$((current_step + 1)) - logMessage INFO "Restoring source code..." - restoreSourceCode "$backup_path" "$ROLL_ENV_PATH" $current_step $total_steps - fi - - # Restore configurations - if [[ $RESTORE_CONFIG -eq 1 ]]; then - current_step=$((current_step + 1)) - logMessage INFO "Restoring configuration files..." - restoreConfigurations "$backup_path" $current_step $total_steps - if [[ $ROLL_ENV_LOADED -eq 0 ]]; then - loadEnvConfig "$ROLL_ENV_PATH" || exit 1 - ROLL_ENV_LOADED=1 - fi - fi - - # Restore volumes - for service in "${services_to_restore[@]}"; do - current_step=$((current_step + 1)) - logMessage INFO "Restoring ${service} volume..." - restoreVolume "$service" "$backup_path" $current_step $total_steps - done - - # Clean up extracted backup if it was temporary - if [[ "$backup_path" =~ _extracted$ ]]; then - rm -rf "$backup_path" - fi - - if [[ $RESTORE_DRY_RUN -eq 1 ]]; then - logMessage SUCCESS "Dry run completed successfully!" - else - logMessage SUCCESS "Restore completed successfully!" - - # Auto-sign SSL certificate for the environment domain - signEnvironmentCertificate - - logMessage INFO "You can now start your environment with: roll env up" - fi -} - -function signEnvironmentCertificate() { - # Determine the domain to sign certificate for - local domain="${TRAEFIK_DOMAIN:-}" - - # If TRAEFIK_DOMAIN not set, try to construct from env name - if [[ -z "$domain" ]]; then - domain="${ROLL_ENV_NAME}.test" - fi - - if [[ -z "$domain" ]]; then - logVerbose "No domain found for certificate signing, skipping" - return 0 - fi - - # Check if root CA exists - if [[ ! -f "${ROLL_SSL_DIR}/rootca/certs/ca.cert.pem" ]]; then - logMessage WARNING "Root CA not found. Run 'roll install' first to enable SSL certificates." - return 0 - fi - - logMessage INFO "Signing SSL certificates for ${domain}..." - - # Sign certificate for the main domain (includes domain and *.domain) - logVerbose "Signing certificate for: ${domain}" - if [[ $RESTORE_VERBOSE -eq 1 ]]; then - "${ROLL_DIR}/bin/roll" sign-certificate "$domain" || { - logMessage WARNING "Failed to sign certificate for ${domain}" - } - else - "${ROLL_DIR}/bin/roll" sign-certificate "$domain" >/dev/null 2>&1 || { - logMessage WARNING "Failed to sign certificate for ${domain}" - } - fi - - # Sign separate certificate for wildcard domain - local wildcard_domain="*.${domain}" - logVerbose "Signing certificate for: ${wildcard_domain}" - if [[ $RESTORE_VERBOSE -eq 1 ]]; then - "${ROLL_DIR}/bin/roll" sign-certificate "$wildcard_domain" || { - logMessage WARNING "Failed to sign certificate for ${wildcard_domain}" - } - else - "${ROLL_DIR}/bin/roll" sign-certificate "$wildcard_domain" >/dev/null 2>&1 || { - logMessage WARNING "Failed to sign certificate for ${wildcard_domain}" - } - fi - - # Regenerate traefik dynamic config and restart to pick up new certificates - logVerbose "Regenerating traefik configuration..." - if [[ $RESTORE_VERBOSE -eq 1 ]]; then - "${ROLL_DIR}/bin/roll" svc up traefik || { - logMessage WARNING "Failed to restart traefik" - } - else - "${ROLL_DIR}/bin/roll" svc up traefik >/dev/null 2>&1 || { - logMessage WARNING "Failed to restart traefik" - } - fi - - logMessage SUCCESS "SSL certificates signed for ${domain}" -} - -# Main execution -performRestore +## restore-full is restore with --include-source preset. The two were near-identical copies - +## eleven of their sixteen functions were byte-identical - and the shared half now lives in +## utils/backup.sh while the half unique to a full restore lives in restore.cmd behind that flag. +## +## Kept as its own command because the name is the documented interface and scripts use it. +ROLL_PARAMS=("--include-source" "${ROLL_PARAMS[@]}") +source "${ROLL_DIR}/commands/restore.cmd" diff --git a/commands/restore-full.help b/commands/restore-full.help index b022d5b..63c8943 100644 --- a/commands/restore-full.help +++ b/commands/restore-full.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Restore a full RollDev environment, including source code, from a backup archive. +## @category: backup ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 -# Load core utilities and configuration -ROLL_ENV_PATH="$(locateEnvPath)" || exit $? -loadEnvConfig "${ROLL_ENV_PATH}" || exit $? +## --include-source restores an archive into a brand new directory, source code and all, so there +## is no environment to locate and no .env.roll to load - the configuration comes out of the backup +## itself. That has to be settled before the preamble runs, hence this early scan of the raw +## arguments rather than waiting for the parse loop below. +RESTORE_INCLUDE_SOURCE=0 +for restore_arg in "${ROLL_PARAMS[@]}" "$@"; do + if [[ "${restore_arg}" == "--include-source" ]]; then + RESTORE_INCLUDE_SOURCE=1 + break + fi +done + +if [[ ${RESTORE_INCLUDE_SOURCE} -eq 0 ]]; then + # Load core utilities and configuration + ROLL_ENV_PATH="$(locateEnvPath)" || exit $? + loadEnvConfig "${ROLL_ENV_PATH}" || exit $? +fi assertDockerRunning # Default configuration values RESTORE_BACKUP_ID="" +RESTORE_BACKUP_FILE="" +RESTORE_OUTPUT_DIR="" +ROLL_ENV_LOADED=0 RESTORE_SERVICES=() RESTORE_CONFIG=1 RESTORE_VERIFY=1 @@ -25,8 +42,17 @@ RESTORE_LEGACY_MIGRATION=1 while [[ $# -gt 0 ]]; do case "$1" in --help|-h) - roll restore --help - exit 0 + ## Do NOT re-invoke `roll restore --help` here. `restore` is on roll's ROLL_CMD_ANYARGS + ## list, so roll's own parser stops at --help and passes it straight through to this + ## script - re-invoking roll lands right back on this branch and recurses forever. + ## usage.cmd renders ROLL_CMD_HELP (restore.help) and exits on its own. + source "${ROLL_DIR}/commands/usage.cmd" + ;; + --include-source) + ## already handled by the early scan above; consumed here so it is not mistaken for a + ## positional argument + RESTORE_INCLUDE_SOURCE=1 + shift ;; --backup-id=*|--backup=*) RESTORE_BACKUP_ID="${1#*=}" @@ -93,8 +119,15 @@ while [[ $# -gt 0 ]]; do shift ;; *) - # If no backup ID specified yet, use this as the backup ID - if [[ -z "$RESTORE_BACKUP_ID" ]]; then + if [[ ${RESTORE_INCLUDE_SOURCE} -eq 1 ]]; then + ## archive first, target directory second + if [[ -z "$RESTORE_BACKUP_FILE" ]]; then + RESTORE_BACKUP_FILE="$1" + elif [[ -z "$RESTORE_OUTPUT_DIR" ]]; then + RESTORE_OUTPUT_DIR="$1" + fi + elif [[ -z "$RESTORE_BACKUP_ID" ]]; then + # If no backup ID specified yet, use this as the backup ID RESTORE_BACKUP_ID="$1" fi shift @@ -102,216 +135,211 @@ while [[ $# -gt 0 ]]; do esac done -# Utility functions for restore operations -function promptPassword() { - local prompt="$1" - local password="" - - # Don't prompt in quiet mode or non-interactive shells - if [[ $RESTORE_QUIET -eq 1 ]] || [[ ! -t 0 ]]; then - error "Password required but running in non-interactive mode. Use --decrypt=password instead." +## A full restore creates the target directory and works from inside it, so every later path - +## including resolveBackupBaseDir - answers relative to the restored project rather than to +## wherever the command happened to be run. +if [[ ${RESTORE_INCLUDE_SOURCE} -eq 1 ]]; then + if [[ -z "$RESTORE_BACKUP_FILE" || -z "$RESTORE_OUTPUT_DIR" ]]; then + if [[ "${ROLL_CMD_VERB:-restore}" == "restore-full" ]]; then + error "Usage: roll restore-full " + else + error "Usage: roll restore --include-source " + fi exit 1 fi + + ## resolve the archive before cd, so a relative path still means what the caller typed + case "$RESTORE_BACKUP_FILE" in + /*) ;; + *) RESTORE_BACKUP_FILE="$(pwd)/${RESTORE_BACKUP_FILE}" ;; + esac + + mkdir -p "$RESTORE_OUTPUT_DIR" + cd "$RESTORE_OUTPUT_DIR" || fatal "Could not enter ${RESTORE_OUTPUT_DIR}" + ROLL_ENV_PATH="$(pwd)" +fi + + + + + + + + + + + + + + + +function performRestore() { + local backup_id="$1" - echo -n "$prompt: " >&2 - read -s password - echo >&2 + # Perform legacy migration if needed + performLegacyMigration - if [[ -z "$password" ]]; then - error "Password cannot be empty" + # Validate database environment + if [[ ${ROLL_DB:-1} -eq 0 ]]; then + logMessage ERROR "Database environment is not enabled (ROLL_DB=0)" exit 1 fi - echo "$password" -} - -function detectEncryptedBackup() { - local backup_path="$1" + # Find backup if not specified + if [[ -z "$backup_id" ]]; then + backup_id=$(findLatestBackup) + if [[ -z "$backup_id" ]]; then + logMessage ERROR "No backups found and no backup ID specified" + exit 1 + fi + logMessage INFO "Using latest backup: $backup_id" + fi + + # Determine backup path + local backup_path="$(pwd)/.roll/backups/$backup_id" - # Check if backup contains .gpg files - if [[ -d "$backup_path" ]]; then - # Directory format - check for .gpg files - if find "$backup_path" -name "*.gpg" -type f | head -1 | grep -q .; then - return 0 # Encrypted + # Check if backup exists as directory + if [[ ! -d "$backup_path" ]]; then + # Try to extract from archive + backup_path=$(extractBackupArchive "$backup_id") + if [[ $? -ne 0 ]]; then + logMessage ERROR "Backup not found: $backup_id" + exit 1 fi - else - # Archive format - check if archive contains .gpg files - local archive_file="$backup_path" - if [[ -f "$archive_file" ]]; then - # Determine decompression command - local decompress_cmd="cat" - case "$archive_file" in - *.tar.gz) decompress_cmd="gzip -dc" ;; - *.tar.xz) decompress_cmd="xz -dc" ;; - *.tar.lz4) decompress_cmd="lz4 -dc" ;; - esac - - # Check if archive contains .gpg files - if $decompress_cmd "$archive_file" | tar -tf - 2>/dev/null | grep -q "\.gpg$"; then - return 0 # Encrypted + fi + + # Detect if backup is encrypted and handle password prompting. Quiet mode is an explicit + # request for no interaction, so it stays a hard error even when stdin is a terminal. + if detectEncryptedBackup "$backup_path"; then + if [[ -z "$RESTORE_DECRYPT" ]] || [[ "$RESTORE_DECRYPT" == "PROMPT" ]]; then + if [[ $RESTORE_QUIET -eq 1 ]]; then + fatal "Password required but running in quiet mode. Use --decrypt= instead." fi + RESTORE_DECRYPT="" + promptPassword RESTORE_DECRYPT "--decrypt=" "Encrypted backup detected. Enter decryption password" fi + + if [[ -z "$RESTORE_DECRYPT" ]]; then + logMessage ERROR "Encrypted backup requires a password. Use --decrypt=password or --decrypt to prompt." + exit 1 + fi + + logMessage INFO "Encrypted backup detected, will decrypt during restoration" fi - return 1 # Not encrypted -} - -function showProgress() { - [[ $PROGRESS -eq 0 ]] && return - local current=$1 - local total=$2 - local description="$3" - local percent=$((current * 100 / total)) - local bar_length=30 - local filled_length=$((percent * bar_length / 100)) + # Validate backup + validateBackup "$backup_path" || exit 1 - printf "\r[" - printf "%*s" $filled_length | tr ' ' '=' - printf "%*s" $((bar_length - filled_length)) | tr ' ' '-' - printf "] %d%% %s" $percent "$description" + # Get backup metadata + local metadata=$(getBackupMetadata "$backup_path") + logMessage INFO "Restoring backup: $backup_id" - # Always end with a newline for clean output - echo "" -} - -function logMessage() { - [[ $RESTORE_QUIET -eq 1 ]] && return - local level="$1" - shift - case "$level" in - INFO) info "$@" ;; - SUCCESS) success "$@" ;; - WARNING) warning "$@" ;; - ERROR) error "$@" ;; - VERBOSE) [[ $RESTORE_VERBOSE -eq 1 ]] && info "[VERBOSE] $@" ;; - esac -} - -function logVerbose() { - [[ $RESTORE_VERBOSE -eq 1 ]] && logMessage INFO "$@" || true -} - -function performLegacyMigration() { - if [[ $RESTORE_LEGACY_MIGRATION -eq 0 ]]; then - return 0 + # Detect available services in backup + local available_services=($(detectBackupServices "$backup_path")) + if [[ ${#available_services[@]} -eq 0 ]]; then + logMessage ERROR "No services found in backup" + exit 1 fi - local current_dir="$(pwd)" + logMessage INFO "Available services in backup: ${available_services[*]}" - # Handle Warden to Roll migration - if [[ ! -f "$current_dir/.env.roll" ]]; then - if [[ -f "$current_dir/.env" ]]; then - logMessage INFO "Performing legacy Warden to Roll migration..." - - # Create backup of original .env - cp "$current_dir/.env" "$current_dir/.env.backup.$(date +%s)" - - # Convert WARDEN to ROLL - sed -i.warden 's/WARDEN/ROLL/g' "$current_dir/.env" - - # Migrate .warden directory to .roll - if [[ -d "$current_dir/.warden" ]]; then - mv "$current_dir/.warden" "$current_dir/.roll" - - if [[ -f "$current_dir/.roll/warden-env.yml" ]]; then - mv "$current_dir/.roll/warden-env.yml" "$current_dir/.roll/roll-env.yml" - sed -i.warden 's/WARDEN/ROLL/g;s/warden/roll/g' "$current_dir/.roll/roll-env.yml" - fi - fi - - # Ensure ROLL_NO_STATIC_CACHING is set - if [[ -n "$(grep -r 'ROLL_NO_STATIC_CACHING' "$current_dir/.env")" ]]; then - perl -i -pe's/.*ROLL_NO_STATIC_CACHING.*$/ROLL_NO_STATIC_CACHING\=1/g' "$current_dir/.env" + # Determine which services to restore + local services_to_restore=() + if [[ ${#RESTORE_SERVICES[@]} -gt 0 ]]; then + # Use specified services + for service in "${RESTORE_SERVICES[@]}"; do + if containsElement "$service" "${available_services[@]}"; then + services_to_restore+=("$service") else - echo "ROLL_NO_STATIC_CACHING=1" >> "$current_dir/.env" - fi - - # Move to .env.roll if it contains ROLL_ variables - if [[ -n "$(grep -r 'ROLL_' "$current_dir/.env")" ]]; then - mv "$current_dir/.env" "$current_dir/.env.roll" + logMessage WARNING "Service $service not found in backup, skipping" fi - - logMessage SUCCESS "Legacy migration completed" - fi + done + else + # Restore all available services + services_to_restore=("${available_services[@]}") fi -} - -function findLatestBackup() { - local backup_dir="$(pwd)/.roll/backups" - if [[ ! -d "$backup_dir" ]]; then - return 1 + if [[ ${#services_to_restore[@]} -eq 0 ]]; then + logMessage ERROR "No services to restore" + exit 1 fi - # Look for timestamped directories first (new format) - local latest_dir=$(ls "$backup_dir" 2>/dev/null | grep '^[0-9]\{10\}$' | sort -n | tail -1) - if [[ -n "$latest_dir" ]]; then - echo "$latest_dir" - return 0 + logMessage INFO "Restoring services: ${services_to_restore[*]}" + + # Stop environment + stopEnvironment + + # Calculate total steps + local total_steps=${#services_to_restore[@]} + if [[ $RESTORE_CONFIG -eq 1 ]]; then + total_steps=$((total_steps + 1)) + fi + + local current_step=0 + + # Restore volumes + for service in "${services_to_restore[@]}"; do + current_step=$((current_step + 1)) + logMessage INFO "Restoring ${service} volume..." + restoreVolume "$service" "$backup_path" $current_step $total_steps + done + + # Restore configurations + if [[ $RESTORE_CONFIG -eq 1 ]]; then + current_step=$((current_step + 1)) + logMessage INFO "Restoring configuration files..." + restoreConfigurations "$backup_path" $current_step $total_steps fi - # Look for compressed archives - local latest_archive=$(ls "$backup_dir"/backup_*_*.tar* 2>/dev/null | sort | tail -1) - if [[ -n "$latest_archive" ]]; then - # Extract timestamp from filename - local timestamp=$(basename "$latest_archive" | grep -o '[0-9]\{10\}') - echo "$timestamp" - return 0 + # Clean up extracted backup if it was temporary + if [[ "$backup_path" =~ _extracted$ ]]; then + rm -rf "$backup_path" fi - return 1 + if [[ $RESTORE_DRY_RUN -eq 1 ]]; then + logMessage SUCCESS "Dry run completed successfully!" + else + logMessage SUCCESS "Restore completed successfully!" + + # Auto-sign SSL certificate for the environment domain + signEnvironmentCertificate + + logMessage INFO "You can now start your environment with: roll env up" + fi } -function extractBackupArchive() { - local backup_id="$1" + +function extractBackupArchiveFile() { + local archive_file="$1" local backup_dir="$(pwd)/.roll/backups" - local extract_dir="$backup_dir/${backup_id}_extracted" + local base_name="$(basename "$archive_file")" + base_name="${base_name%%.tar*}" + local extract_dir="$backup_dir/${base_name}_extracted" - logVerbose "Looking for backup archive with ID: $backup_id" + logVerbose "Extracting backup archive file" + logVerbose "Archive file: $archive_file" logVerbose "Backup directory: $backup_dir" + logVerbose "Extract directory: $extract_dir" - # Check if already extracted if [[ -d "$extract_dir" ]]; then logVerbose "Found already extracted backup at: $extract_dir" echo "$extract_dir" return 0 fi - # Find the archive file - local archive_file="" - for ext in ".tar.gz" ".tar.xz" ".tar.lz4" ".tar"; do - local potential_file="$backup_dir/backup_${ROLL_ENV_NAME}_${backup_id}${ext}" - logVerbose "Checking for archive: $potential_file" - if [[ -f "$potential_file" ]]; then - archive_file="$potential_file" - break - fi - done - - # Also check for generic archive names - if [[ -z "$archive_file" ]]; then - logVerbose "Trying generic archive name pattern: *${backup_id}*.tar*" - archive_file=$(ls "$backup_dir"/*"$backup_id"*.tar* 2>/dev/null | head -1) - fi - - if [[ -z "$archive_file" ]]; then - logMessage ERROR "Backup archive not found for ID: $backup_id" - return 1 - fi - - logMessage INFO "Extracting backup archive: $(basename "$archive_file")" - logVerbose "Full archive path: $archive_file" - logVerbose "Extract destination: $extract_dir" - mkdir -p "$extract_dir" - # Use direct file reading for cross-platform compatibility - # BSD tar (macOS) doesn't reliably handle piped stdin with -C flag - # Both BSD and GNU tar auto-detect compression format with -xf - logVerbose "Extracting archive using direct file reading" + local decompress_cmd="cat" + case "$archive_file" in + *.tar.gz) decompress_cmd="gzip -d" ;; + *.tar.xz) decompress_cmd="xz -d" ;; + *.tar.lz4) decompress_cmd="lz4 -d" ;; + esac - if tar -xf "$archive_file" -C "$extract_dir" --strip-components=1; then - logVerbose "Successfully extracted archive to: $extract_dir" + logVerbose "Using decompression command: $decompress_cmd" + + if $decompress_cmd < "$archive_file" | tar -xf - -C "$extract_dir" --strip-components=1 2>/dev/null; then + logVerbose "Successfully extracted to: $extract_dir" echo "$extract_dir" return 0 else @@ -321,500 +349,129 @@ function extractBackupArchive() { fi } -function validateBackup() { +function restoreSourceCode() { local backup_path="$1" - - if [[ $RESTORE_VERIFY -eq 0 ]]; then - return 0 - fi - - logMessage INFO "Validating backup integrity..." - - # Check if backup metadata exists - if [[ ! -f "$backup_path/metadata/backup.json" ]]; then - logMessage WARNING "Backup metadata not found, proceeding with legacy format" - return 0 - fi - - # Verify checksums if available - if [[ -f "$backup_path/metadata/checksums.sha256" ]]; then - if (cd "$backup_path" && sha256sum -c metadata/checksums.sha256 >/dev/null 2>&1); then - logMessage SUCCESS "Backup integrity verified" - return 0 - else - logMessage ERROR "Backup integrity check failed" - return 1 - fi - fi - - logMessage SUCCESS "Backup validation completed" - return 0 -} - -function getBackupMetadata() { - local backup_path="$1" - local metadata_file="$backup_path/metadata/backup.json" - - if [[ -f "$metadata_file" ]]; then - cat "$metadata_file" - else - # Return empty JSON for legacy backups - echo "{}" - fi -} - -function detectBackupServices() { - local backup_path="$1" - local services=() - - # Check for volume backups - if [[ -d "$backup_path/volumes" ]]; then - for volume_file in "$backup_path/volumes"/*; do - if [[ -f "$volume_file" ]]; then - local service_name=$(basename "$volume_file" | sed 's/\.tar.*//') - services+=("$service_name") - fi - done - else - # Legacy format detection - if [[ -f "$backup_path/db.tar.gz" ]]; then - services+=("db") - fi - if [[ -f "$backup_path/redis.tar.gz" ]]; then - services+=("redis") - fi - if [[ -f "$backup_path/es.tar.gz" ]]; then - services+=("elasticsearch") - fi - if [[ -f "$backup_path/os.tar.gz" ]]; then - services+=("opensearch") - fi - fi - - echo "${services[@]}" -} - -function stopEnvironment() { - if [[ $RESTORE_DRY_RUN -eq 1 ]]; then - logMessage INFO "[DRY RUN] Would stop environment" - return 0 - fi - - logMessage INFO "Stopping environment for consistent restore..." - - logVerbose "Checking for running containers with project: ${ROLL_ENV_NAME}" - local running_containers=$(docker ps --filter "label=com.docker.compose.project=${ROLL_ENV_NAME}" --format '{{.Names}}' 2>/dev/null || true) - - if [[ -n "$running_containers" ]]; then - logVerbose "Found running containers: $running_containers" - logVerbose "Executing: roll env down" - "${ROLL_DIR}/bin/roll" env down >/dev/null 2>&1 || true - logVerbose "Environment stopped" - else - logVerbose "No running containers found for project ${ROLL_ENV_NAME}" - fi -} - -function getVolumeMapping() { - local service_name="$1" - - case "$service_name" in - db) - case "${DB_DISTRIBUTION:-mariadb}" in - mysql|mariadb) echo "${ROLL_ENV_NAME}_dbdata:mysql" ;; - postgres) echo "${ROLL_ENV_NAME}_dbdata:postgres" ;; - *) echo "${ROLL_ENV_NAME}_dbdata:mysql" ;; - esac - ;; - redis) echo "${ROLL_ENV_NAME}_redis:redis" ;; - dragonfly) echo "${ROLL_ENV_NAME}_dragonfly:dragonfly" ;; - elasticsearch) echo "${ROLL_ENV_NAME}_esdata:elasticsearch" ;; - opensearch) echo "${ROLL_ENV_NAME}_osdata:opensearch" ;; - mongodb) echo "${ROLL_ENV_NAME}_mongodb:mongodb" ;; - rabbitmq) echo "${ROLL_ENV_NAME}_rabbitmq:rabbitmq" ;; - varnish) echo "${ROLL_ENV_NAME}_varnish:varnish" ;; - *) echo "${ROLL_ENV_NAME}_${service_name}:generic" ;; - esac -} - -function restoreVolume() { - local service_name="$1" - local backup_path="$2" + local target_dir="$2" local step="$3" local total="$4" - showProgress $step $total "Restoring $service_name volume" + showProgress $step $total "Restoring source code" - local volume_mapping=$(getVolumeMapping "$service_name") - IFS=':' read -r volume_name service_type <<< "$volume_mapping" + local src_file="" + local is_encrypted=false - logVerbose "Restoring service: $service_name" - logVerbose "Volume mapping: $volume_mapping" - logVerbose "Volume name: $volume_name, Service type: $service_type" + for ext in ".tar.gz" ".tar.xz" ".tar.lz4" ".tar"; do + if [[ -f "$backup_path/source${ext}.gpg" ]]; then + src_file="$backup_path/source${ext}.gpg" + is_encrypted=true + break + elif [[ -f "$backup_path/source${ext}" ]]; then + src_file="$backup_path/source${ext}" + break + fi + done - # Determine backup file location (check for both encrypted and unencrypted) - local backup_file="" - local is_encrypted=false - - # Check for encrypted files first (.gpg extension) - if [[ -f "$backup_path/volumes/${service_name}.tar.gz.gpg" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar.gz.gpg" - is_encrypted=true - elif [[ -f "$backup_path/volumes/${service_name}.tar.xz.gpg" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar.xz.gpg" - is_encrypted=true - elif [[ -f "$backup_path/volumes/${service_name}.tar.lz4.gpg" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar.lz4.gpg" - is_encrypted=true - elif [[ -f "$backup_path/volumes/${service_name}.tar.gpg" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar.gpg" - is_encrypted=true - # Check for unencrypted files - elif [[ -f "$backup_path/volumes/${service_name}.tar.gz" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar.gz" - elif [[ -f "$backup_path/volumes/${service_name}.tar.xz" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar.xz" - elif [[ -f "$backup_path/volumes/${service_name}.tar.lz4" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar.lz4" - elif [[ -f "$backup_path/volumes/${service_name}.tar" ]]; then - backup_file="$backup_path/volumes/${service_name}.tar" - elif [[ -f "$backup_path/${service_name}.tar.gz" ]]; then - # Legacy format - backup_file="$backup_path/${service_name}.tar.gz" - elif [[ "$service_name" == "elasticsearch" && -f "$backup_path/es.tar.gz" ]]; then - # Legacy format uses short names for the search engines - backup_file="$backup_path/es.tar.gz" - elif [[ "$service_name" == "opensearch" && -f "$backup_path/os.tar.gz" ]]; then - backup_file="$backup_path/os.tar.gz" - else - logMessage WARNING "Backup file not found for service: $service_name" - logVerbose "Searched in: $backup_path/volumes/ and $backup_path/" + if [[ -z "$src_file" ]]; then + logMessage INFO "No source code archive found in backup" return 0 fi - # A search-index volume is only mounted when the matching engine is enabled - # in this environment's .env.roll — warn instead of silently restoring into - # a volume that no service will ever use - if [[ "$service_name" == "elasticsearch" && "${ROLL_ELASTICSEARCH:-0}" != "1" ]]; then - logMessage WARNING "Backup contains an Elasticsearch index but ROLL_ELASTICSEARCH is not enabled - the restored volume will not be mounted by 'roll env up'" - elif [[ "$service_name" == "opensearch" && "${ROLL_OPENSEARCH:-0}" != "1" ]]; then - logMessage WARNING "Backup contains an OpenSearch index but ROLL_OPENSEARCH is not enabled - the restored volume will not be mounted by 'roll env up'" - fi - - logVerbose "Found backup file: $backup_file" - logVerbose "Encrypted: $is_encrypted" - if [[ $RESTORE_DRY_RUN -eq 1 ]]; then - if [[ $is_encrypted == true ]]; then - logMessage INFO "[DRY RUN] Would decrypt and restore $service_name from $backup_file to volume $volume_name" - else - logMessage INFO "[DRY RUN] Would restore $service_name from $backup_file to volume $volume_name" - fi + logMessage INFO "[DRY RUN] Would extract source code to $target_dir" return 0 fi - - # Validate decryption password if file is encrypted + + mkdir -p "$target_dir" + + # Use direct file reading instead of piping for cross-platform compatibility + # BSD tar (macOS) doesn't reliably handle piped stdin with -C flag + # Both BSD and GNU tar auto-detect compression format with -xf if [[ $is_encrypted == true ]]; then if [[ -z "$RESTORE_DECRYPT" ]]; then - logMessage ERROR "Encrypted backup file found but no decryption password provided" + logMessage ERROR "Encrypted source archive found but no decryption password provided" return 1 fi - fi - - # Get Docker Compose version for proper labeling - local docker_compose_version=$(docker compose version 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+' | head -1) - local volume_base_name=$(echo "$volume_name" | sed "s/${ROLL_ENV_NAME}_//") - - # Remove existing volume if it exists - if docker volume inspect "$volume_name" >/dev/null 2>&1; then - if [[ $RESTORE_FORCE -eq 1 ]]; then - logMessage INFO "Removing existing volume: $volume_name" - logVerbose "Executing: docker volume rm $volume_name" - docker volume rm "$volume_name" >/dev/null 2>&1 + # Decrypt to temp file first, then extract directly + local temp_file="$backup_path/source_decrypted.tar" + case "$src_file" in + *.tar.gz.gpg) temp_file="$backup_path/source_decrypted.tar.gz" ;; + *.tar.xz.gpg) temp_file="$backup_path/source_decrypted.tar.xz" ;; + *.tar.lz4.gpg) temp_file="$backup_path/source_decrypted.tar.lz4" ;; + esac + + if echo "$RESTORE_DECRYPT" | gpg --batch --yes --quiet --passphrase-fd 0 --decrypt "$src_file" > "$temp_file"; then + if tar -xf "$temp_file" -C "$target_dir"; then + rm -f "$temp_file" + logMessage SUCCESS "Source code restored" + return 0 + else + rm -f "$temp_file" + logMessage ERROR "Failed to extract source code" + return 1 + fi else - logMessage ERROR "Volume $volume_name already exists. Use --force to overwrite." + rm -f "$temp_file" 2>/dev/null + logMessage ERROR "Failed to decrypt source archive" return 1 fi else - logVerbose "Volume $volume_name does not exist yet" - fi - - # Create new volume with proper labels - logVerbose "Creating volume: $volume_name with labels" - logVerbose " - com.docker.compose.project=$ROLL_ENV_NAME" - logVerbose " - com.docker.compose.version=$docker_compose_version" - logVerbose " - com.docker.compose.volume=$volume_base_name" - docker volume create "$volume_name" \ - --label com.docker.compose.project="$ROLL_ENV_NAME" \ - --label com.docker.compose.version="$docker_compose_version" \ - --label com.docker.compose.volume="$volume_base_name" >/dev/null 2>&1 - - # Restore the volume data with decryption if needed - local temp_container="${ROLL_ENV_NAME}_restore_${service_name}_$$" - - if [[ $is_encrypted == true ]]; then - # Decrypt and decompress pipeline - use ubuntu and original tar approach with strip components - # Use passphrase-fd to avoid shell escaping issues with passwords - # Determine the correct tar command based on the backup file format - local tar_cmd="tar -xf -" - case "$backup_file" in - *.tar.gz.gpg) tar_cmd="tar -xzf -" ;; - *.tar.xz.gpg) tar_cmd="tar -xJf -" ;; - *.tar.lz4.gpg) tar_cmd="lz4 -d - | tar -xf -" ;; - esac - - if echo "$RESTORE_DECRYPT" | gpg --batch --yes --quiet --passphrase-fd 0 --decrypt "$backup_file" | docker run --rm --name "$temp_container" --mount source="$volume_name",target=/data -i ubuntu bash -c "cd /data && $tar_cmd --strip-components=1" 2>/dev/null; then - logMessage SUCCESS "Successfully restored and decrypted $service_name volume" + # Direct extraction - works on both BSD tar (macOS) and GNU tar (Linux) + if tar -xf "$src_file" -C "$target_dir"; then + logMessage SUCCESS "Source code restored" return 0 else - logMessage ERROR "Failed to decrypt and restore $service_name volume" + logMessage ERROR "Failed to restore source code" return 1 fi - else - # Regular restore without decryption - use ubuntu and original tar approach with strip components - # For compressed files, we need to handle decompression properly - case "$backup_file" in - *.tar.gz) - if docker run --rm --name "$temp_container" \ - --mount source="$volume_name",target=/data \ - -v "$(dirname "$backup_file")":/backup \ - ubuntu bash \ - -c "cd /data && tar -xzf /backup/$(basename "$backup_file") --strip-components=1" 2>/dev/null; then - - logMessage SUCCESS "Successfully restored $service_name volume" - return 0 - else - logMessage ERROR "Failed to restore $service_name volume" - return 1 - fi - ;; - *.tar.xz) - if docker run --rm --name "$temp_container" \ - --mount source="$volume_name",target=/data \ - -v "$(dirname "$backup_file")":/backup \ - ubuntu bash \ - -c "cd /data && tar -xJf /backup/$(basename "$backup_file") --strip-components=1" 2>/dev/null; then - - logMessage SUCCESS "Successfully restored $service_name volume" - return 0 - else - logMessage ERROR "Failed to restore $service_name volume" - return 1 - fi - ;; - *.tar.lz4) - if docker run --rm --name "$temp_container" \ - --mount source="$volume_name",target=/data \ - -v "$(dirname "$backup_file")":/backup \ - ubuntu bash \ - -c "cd /data && lz4 -d /backup/$(basename "$backup_file") - | tar -xf - --strip-components=1" 2>/dev/null; then - - logMessage SUCCESS "Successfully restored $service_name volume" - return 0 - else - logMessage ERROR "Failed to restore $service_name volume" - return 1 - fi - ;; - *.tar) - if docker run --rm --name "$temp_container" \ - --mount source="$volume_name",target=/data \ - -v "$(dirname "$backup_file")":/backup \ - ubuntu bash \ - -c "cd /data && tar -xf /backup/$(basename "$backup_file") --strip-components=1" 2>/dev/null; then - - logMessage SUCCESS "Successfully restored $service_name volume" - return 0 - else - logMessage ERROR "Failed to restore $service_name volume" - return 1 - fi - ;; - *) - logMessage ERROR "Unsupported backup file format: $backup_file" - return 1 - ;; - esac fi } -function restoreConfigurations() { - local backup_path="$1" - local step="$2" - local total="$3" - - if [[ $RESTORE_CONFIG -eq 0 ]]; then - return 0 - fi - - showProgress $step $total "Restoring configuration files" - - local config_source_dir="$backup_path/config" - local current_dir="$(pwd)" - - # Legacy format support - if [[ ! -d "$config_source_dir" ]]; then - # Check for legacy files in backup root (both encrypted and unencrypted) - local legacy_files=("env.php" "auth.json") - for file in "${legacy_files[@]}"; do - local source_file="" - local is_encrypted=false - - # Check for encrypted version first - if [[ -f "$backup_path/${file}.gpg" ]]; then - source_file="$backup_path/${file}.gpg" - is_encrypted=true - elif [[ -f "$backup_path/$file" ]]; then - source_file="$backup_path/$file" - fi - - if [[ -n "$source_file" ]]; then - local target_path="" - case "$file" in - env.php) target_path="$current_dir/app/etc/env.php" ;; - auth.json) target_path="$current_dir/auth.json" ;; - esac - - if [[ -n "$target_path" ]]; then - if [[ $RESTORE_DRY_RUN -eq 1 ]]; then - if [[ $is_encrypted == true ]]; then - logMessage INFO "[DRY RUN] Would decrypt and restore $file to $target_path" - else - logMessage INFO "[DRY RUN] Would restore $file to $target_path" - fi - else - mkdir -p "$(dirname "$target_path")" - - if [[ $is_encrypted == true ]]; then - # Decrypt the file directly to target location - if [[ -n "$RESTORE_DECRYPT" ]]; then - if echo "$RESTORE_DECRYPT" | gpg --batch --yes --quiet --passphrase-fd 0 --decrypt "$source_file" > "$target_path"; then - logMessage INFO "Decrypted and restored $file" - else - logMessage ERROR "Failed to decrypt $file" - return 1 - fi - else - logMessage ERROR "Encrypted config file found but no decryption password provided" - return 1 - fi - else - cp "$source_file" "$target_path" - logMessage INFO "Restored $file" - fi - fi - fi - fi - done - return 0 - fi - - # New format with structured config directory - if [[ $RESTORE_DRY_RUN -eq 1 ]]; then - logMessage INFO "[DRY RUN] Would restore configuration files from $config_source_dir" - return 0 - fi - - # Restore configuration files (both encrypted and unencrypted) - if [[ -d "$config_source_dir" ]]; then - # Process all files including .gpg files - find "$config_source_dir" -type f | while read -r config_file; do - local relative_path="${config_file#$config_source_dir/}" - local is_encrypted=false - - # Check if file is encrypted - if [[ "$config_file" == *.gpg ]]; then - is_encrypted=true - # Remove .gpg extension for target path - relative_path="${relative_path%.gpg}" - fi - - local target_path="$current_dir/$relative_path" - - # Create target directory if needed - mkdir -p "$(dirname "$target_path")" - - # Backup existing file if it exists - if [[ -f "$target_path" ]]; then - if [[ $is_encrypted == true ]]; then - # For encrypted files, we can't easily compare so always backup - cp "$target_path" "$target_path.backup.$(date +%s)" - logMessage INFO "Backed up existing $relative_path" - elif ! cmp -s "$config_file" "$target_path"; then - cp "$target_path" "$target_path.backup.$(date +%s)" - logMessage INFO "Backed up existing $relative_path" - fi - fi - - if [[ $is_encrypted == true ]]; then - # Decrypt the file - if [[ -n "$RESTORE_DECRYPT" ]]; then - if echo "$RESTORE_DECRYPT" | gpg --batch --yes --quiet --passphrase-fd 0 --decrypt "$config_file" > "$target_path"; then - logMessage INFO "Decrypted and restored $relative_path" - else - logMessage ERROR "Failed to decrypt $relative_path" - return 1 - fi - else - logMessage ERROR "Encrypted config file found but no decryption password provided" - return 1 - fi - else - # Copy unencrypted file - cp "$config_file" "$target_path" - logMessage INFO "Restored $relative_path" - fi - done - fi - - logMessage SUCCESS "Configuration restore completed" -} +function performFullRestore() { + logMessage INFO "Starting full restore from $(basename "$RESTORE_BACKUP_FILE")..." + + logVerbose "Backup file: $RESTORE_BACKUP_FILE" + logVerbose "Output directory: $RESTORE_OUTPUT_DIR" + logVerbose "Environment path: $ROLL_ENV_PATH" -function performRestore() { - local backup_id="$1" - # Perform legacy migration if needed performLegacyMigration - + # Validate database environment if [[ ${ROLL_DB:-1} -eq 0 ]]; then logMessage ERROR "Database environment is not enabled (ROLL_DB=0)" exit 1 fi - - # Find backup if not specified - if [[ -z "$backup_id" ]]; then - backup_id=$(findLatestBackup) - if [[ -z "$backup_id" ]]; then - logMessage ERROR "No backups found and no backup ID specified" - exit 1 - fi - logMessage INFO "Using latest backup: $backup_id" - fi - - # Determine backup path - local backup_path="$(pwd)/.roll/backups/$backup_id" - - # Check if backup exists as directory - if [[ ! -d "$backup_path" ]]; then - # Try to extract from archive - backup_path=$(extractBackupArchive "$backup_id") - if [[ $? -ne 0 ]]; then - logMessage ERROR "Backup not found: $backup_id" - exit 1 - fi + + # Determine backup path from archive argument + local backup_path="" + + logVerbose "Checking backup file type..." + if [[ -f "$RESTORE_BACKUP_FILE" ]]; then + logMessage INFO "Extracting backup archive..." + logVerbose "Backup is a file, extracting..." + backup_path=$(extractBackupArchiveFile "$RESTORE_BACKUP_FILE") + logMessage SUCCESS "Archive extracted" + elif [[ -d "$RESTORE_BACKUP_FILE" ]]; then + logVerbose "Backup is a directory" + backup_path="$RESTORE_BACKUP_FILE" + else + logMessage ERROR "Backup file not found: $RESTORE_BACKUP_FILE" + exit 1 fi + + logVerbose "Backup path resolved to: $backup_path" - # Detect if backup is encrypted and handle password prompting + # Detect if backup is encrypted and handle password prompting. Quiet mode is an explicit + # request for no interaction, so it stays a hard error even when stdin is a terminal. if detectEncryptedBackup "$backup_path"; then - if [[ -z "$RESTORE_DECRYPT" ]]; then - # No password provided, prompt for it - RESTORE_DECRYPT=$(promptPassword "Encrypted backup detected. Enter decryption password") - elif [[ "$RESTORE_DECRYPT" == "PROMPT" ]]; then - # Explicit prompt requested - RESTORE_DECRYPT=$(promptPassword "Enter decryption password") + if [[ -z "$RESTORE_DECRYPT" ]] || [[ "$RESTORE_DECRYPT" == "PROMPT" ]]; then + if [[ $RESTORE_QUIET -eq 1 ]]; then + fatal "Password required but running in quiet mode. Use --decrypt= instead." + fi + RESTORE_DECRYPT="" + promptPassword RESTORE_DECRYPT "--decrypt=" "Encrypted backup detected. Enter decryption password" fi - + if [[ -z "$RESTORE_DECRYPT" ]]; then logMessage ERROR "Encrypted backup requires a password. Use --decrypt=password or --decrypt to prompt." exit 1 @@ -828,7 +485,23 @@ function performRestore() { # Get backup metadata local metadata=$(getBackupMetadata "$backup_path") - logMessage INFO "Restoring backup: $backup_id" + logMessage INFO "Restoring backup from: $(basename \"$RESTORE_BACKUP_FILE\")" + logVerbose "Backup metadata: $metadata" + + local source_exists=0 + for ext in ".tar.gz" ".tar.xz" ".tar.lz4" ".tar"; do + if [[ -f "$backup_path/source${ext}" ]] || [[ -f "$backup_path/source${ext}.gpg" ]]; then + logVerbose "Found source archive: source${ext}" + source_exists=1 + break + fi + done + logVerbose "Source code exists in backup: $source_exists" + + if [[ $ROLL_ENV_LOADED -eq 0 ]]; then + ROLL_ENV_NAME=$(echo "$metadata" | grep -o '"environment"[^"]*"' | head -1 | sed 's/.*"environment"[ ]*:[ ]*"\([^"]*\)".*/\1/') + logVerbose "Extracted ROLL_ENV_NAME from metadata: ${ROLL_ENV_NAME:-}" + fi # Detect available services in backup local available_services=($(detectBackupServices "$backup_path")) @@ -838,10 +511,11 @@ function performRestore() { fi logMessage INFO "Available services in backup: ${available_services[*]}" - + # Determine which services to restore local services_to_restore=() if [[ ${#RESTORE_SERVICES[@]} -gt 0 ]]; then + logVerbose "User specified services to restore: ${RESTORE_SERVICES[*]}" # Use specified services for service in "${RESTORE_SERVICES[@]}"; do if containsElement "$service" "${available_services[@]}"; then @@ -851,6 +525,7 @@ function performRestore() { fi done else + logVerbose "No specific services requested, restoring all available" # Restore all available services services_to_restore=("${available_services[@]}") fi @@ -870,22 +545,36 @@ function performRestore() { if [[ $RESTORE_CONFIG -eq 1 ]]; then total_steps=$((total_steps + 1)) fi + if [[ $source_exists -eq 1 ]]; then + total_steps=$((total_steps + 1)) + fi local current_step=0 - # Restore volumes - for service in "${services_to_restore[@]}"; do + # Restore source code if available + if [[ $source_exists -eq 1 ]]; then current_step=$((current_step + 1)) - logMessage INFO "Restoring ${service} volume..." - restoreVolume "$service" "$backup_path" $current_step $total_steps - done + logMessage INFO "Restoring source code..." + restoreSourceCode "$backup_path" "$ROLL_ENV_PATH" $current_step $total_steps + fi # Restore configurations if [[ $RESTORE_CONFIG -eq 1 ]]; then current_step=$((current_step + 1)) logMessage INFO "Restoring configuration files..." restoreConfigurations "$backup_path" $current_step $total_steps + if [[ $ROLL_ENV_LOADED -eq 0 ]]; then + loadEnvConfig "$ROLL_ENV_PATH" || exit 1 + ROLL_ENV_LOADED=1 + fi fi + + # Restore volumes + for service in "${services_to_restore[@]}"; do + current_step=$((current_step + 1)) + logMessage INFO "Restoring ${service} volume..." + restoreVolume "$service" "$backup_path" $current_step $total_steps + done # Clean up extracted backup if it was temporary if [[ "$backup_path" =~ _extracted$ ]]; then @@ -904,72 +593,18 @@ function performRestore() { fi } -function signEnvironmentCertificate() { - # Determine the domain to sign certificate for - local domain="${TRAEFIK_DOMAIN:-}" - - # If TRAEFIK_DOMAIN not set, try to construct from env name - if [[ -z "$domain" ]]; then - domain="${ROLL_ENV_NAME}.test" - fi - - if [[ -z "$domain" ]]; then - logVerbose "No domain found for certificate signing, skipping" - return 0 - fi - - # Check if root CA exists - if [[ ! -f "${ROLL_SSL_DIR}/rootca/certs/ca.cert.pem" ]]; then - logMessage WARNING "Root CA not found. Run 'roll install' first to enable SSL certificates." - return 0 - fi - - logMessage INFO "Signing SSL certificates for ${domain}..." - - # Sign certificate for the main domain (includes domain and *.domain) - logVerbose "Signing certificate for: ${domain}" - if [[ $RESTORE_VERBOSE -eq 1 ]]; then - "${ROLL_DIR}/bin/roll" sign-certificate "$domain" || { - logMessage WARNING "Failed to sign certificate for ${domain}" - } - else - "${ROLL_DIR}/bin/roll" sign-certificate "$domain" >/dev/null 2>&1 || { - logMessage WARNING "Failed to sign certificate for ${domain}" - } - fi - - # Sign separate certificate for wildcard domain - local wildcard_domain="*.${domain}" - logVerbose "Signing certificate for: ${wildcard_domain}" - if [[ $RESTORE_VERBOSE -eq 1 ]]; then - "${ROLL_DIR}/bin/roll" sign-certificate "$wildcard_domain" || { - logMessage WARNING "Failed to sign certificate for ${wildcard_domain}" - } - else - "${ROLL_DIR}/bin/roll" sign-certificate "$wildcard_domain" >/dev/null 2>&1 || { - logMessage WARNING "Failed to sign certificate for ${wildcard_domain}" - } - fi - - # Regenerate traefik dynamic config and restart to pick up new certificates - logVerbose "Regenerating traefik configuration..." - if [[ $RESTORE_VERBOSE -eq 1 ]]; then - "${ROLL_DIR}/bin/roll" svc up traefik || { - logMessage WARNING "Failed to restart traefik" - } - else - "${ROLL_DIR}/bin/roll" svc up traefik >/dev/null 2>&1 || { - logMessage WARNING "Failed to restart traefik" - } - fi - - logMessage SUCCESS "SSL certificates signed for ${domain}" -} - # Main execution +if [[ ${RESTORE_INCLUDE_SOURCE} -eq 1 ]]; then + performFullRestore + exit $? +fi + if [[ -z "$RESTORE_BACKUP_ID" ]]; then # If no backup ID provided, use the latest - RESTORE_BACKUP_ID=$(findLatestBackup) + ## findLatestBackup returns 1 when the directory holds no backup, and an unguarded command + ## substitution in an assignment is not exempt from set -e - without || true the CLI died on + ## the ERR trap instead of printing the message just below + RESTORE_BACKUP_ID=$(findLatestBackup) || true if [[ -z "$RESTORE_BACKUP_ID" ]]; then error "No backups found. Please create a backup first with: roll backup" exit 1 diff --git a/commands/restore.help b/commands/restore.help index 6cb9214..a406722 100755 --- a/commands/restore.help +++ b/commands/restore.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Restore project data and configuration from a previous backup. +## @category: backup ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Open a root shell in the project php-fpm container. +## @category: utility ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run a one-off command as root in the project php-fpm container without a TTY. +## @category: utility ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 -if [[ -f "${ROLL_HOME_DIR}/.env" ]]; then - eval "$(cat "${ROLL_HOME_DIR}/.env" | sed 's/\r$//g' | grep "^ROLL_ENV_SHELL_COMMAND")" -fi - +## ROLL_ENV_SHELL_COMMAND from the global ~/.roll/.env is already loaded (schema-validated) by +## loadEnvConfig below, which reads that same file before applying project-level overrides. ROLL_ENV_PATH="$(locateEnvPath)" || exit $? loadEnvConfig "${ROLL_ENV_PATH}" || exit $? diff --git a/commands/rootshell.help b/commands/rootshell.help index 2ecd22b..7600889 100644 --- a/commands/rootshell.help +++ b/commands/rootshell.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Open a root shell in the project container using its configured shell. +## @category: utility ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 -if [[ -f "${ROLL_HOME_DIR}/.env" ]]; then - eval "$(cat "${ROLL_HOME_DIR}/.env" | sed 's/\r$//g' | grep "^ROLL_ENV_SHELL_COMMAND")" -fi - +## ROLL_ENV_SHELL_COMMAND from the global ~/.roll/.env is already loaded (schema-validated) by +## loadEnvConfig below, which reads that same file before applying project-level overrides. ROLL_ENV_PATH="$(locateEnvPath)" || exit $? loadEnvConfig "${ROLL_ENV_PATH}" || exit $? diff --git a/commands/shell.help b/commands/shell.help index 5b02158..97b8243 100755 --- a/commands/shell.help +++ b/commands/shell.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Open a shell in the project container using its configured shell. +## @category: utility ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Generate and sign a local SSL certificate for one or more hostnames. +## @category: services ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Start the RollDev core services. +## @category: environment ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## `status` is on roll's ROLL_CMD_ANYARGS list (needed so --format reaches this script), so roll's +## own parser stops at the first dash-prefixed argument and passes it straight through here with +## ROLL_PARAMS empty. Parse flags from "$@", not ROLL_PARAMS. +STATUS_FORMAT="human" +while (( "$#" )); do + case "$1" in + -h|--help) + ## Do NOT re-invoke `roll status --help` here - status is on ROLL_CMD_ANYARGS, so + ## re-invoking roll lands right back on this branch and forks until killed. + source "${ROLL_DIR}/commands/usage.cmd" + ;; + --format=*) + STATUS_FORMAT="${1#*=}" + shift + ;; + --format) + ## `shift 2` would abort the whole CLI under bin/roll's `set -e` if --format is the + ## last argument ($# == 1): shift returns non-zero when the count exceeds $#. + STATUS_FORMAT="${2:-}" + shift + (( $# )) && shift + ;; + *) + fatal "Unsupported argument $1" + ;; + esac +done + +if [[ "${STATUS_FORMAT}" != "human" && "${STATUS_FORMAT}" != "json" ]]; then + fatal "Unsupported --format value '${STATUS_FORMAT}' (expected: human, json)" +fi + assertDockerRunning -rollNetworkName=$(grep -A3 'networks:' "${ROLL_DIR}/docker/docker-compose.yml" | tail -n1 | sed -e 's/[[:blank:]]*name:[[:blank:]]*//g') +## The roll core network is always named "roll" (docker/docker-compose.yml networks.default.name); +## reading it from a fixed constant instead of scraping the compose YAML. +rollNetworkName="roll" rollNetworkId=$(docker network ls -q --filter name="${rollNetworkName}") -if [[ -z "${rollNetworkId}" ]]; then - echo -e "[\033[33;1m!!\033[0m] \033[31mRollDev is not currently running.\033[0m Run \033[36mroll svc up\033[0m to start RollDev core services." -fi - OLDIFS="$IFS" IFS=$'\n' if command -v mapfile >/dev/null 2>&1; then @@ -22,6 +52,15 @@ else fi IFS="$OLDIFS" +## Raw per-project fields, parallel arrays, collected alongside messageList so json rendering +## below needs no extra docker calls. +jsonProjectNames=() +jsonProjectTypes=() +jsonProjectDirs=() +jsonProjectUrls=() +jsonProjectNetworks=() +jsonProjectContainerCounts=() + messageList=() if (( ${#projectNetworkList[@]} > 0 )); then lastIdx=$(( ${#projectNetworkList[@]} - 1 )) @@ -46,56 +85,101 @@ for projectNetwork in "${projectNetworkList[@]}"; do traefikDomain=$(grep -m1 '^TRAEFIK_DOMAIN=' "${projectDir}/.env.roll" | cut -d '=' -f2- | tr -d '\r') traefikSubDomain=$(grep -m1 '^TRAEFIK_SUBDOMAIN=' "${projectDir}/.env.roll" | cut -d '=' -f2- | tr -d '\r') containerCount=$(echo "$projectContainers" | wc -l | tr -d ' ') + projectUrl="https://${traefikSubDomain}.${traefikDomain}" + + jsonProjectNames+=("${projectName}") + jsonProjectTypes+=("${projectType}") + jsonProjectDirs+=("${projectDir}") + jsonProjectUrls+=("${projectUrl}") + jsonProjectNetworks+=("${projectNetwork}") + jsonProjectContainerCounts+=("${containerCount}") messageList+=(" \033[1;35m${projectName}\033[0m a \033[36m${projectType}\033[0m project") messageList+=(" Project Directory: \033[33m${projectDir}\033[0m") - messageList+=(" Project URL: \033[94mhttps://${traefikSubDomain}.${traefikDomain}\033[0m") + messageList+=(" Project URL: \033[94m${projectUrl}\033[0m") messageList+=(" Docker Network: \033[33m${projectNetwork}\033[0m") messageList+=(" Containers Running: \033[33m${containerCount}\033[0m") [[ "$projectNetwork" != "$lastNetwork" ]] && messageList+=("") done -if (( ${#messageList[@]} > 0 )); then - if [[ -z "${rollNetworkId}" ]]; then - echo -e "Found the following \033[32mrunning\033[0m projects; however, \033[31mRollDev core services are currently not running\033[0m:" +## Enabled core services (name, running/stopped) collected alongside the human table below. +jsonServiceNames=() +jsonServiceStates=() + +if [[ "${STATUS_FORMAT}" != "json" ]]; then + if (( ${#messageList[@]} > 0 )); then + if [[ -z "${rollNetworkId}" ]]; then + echo -e "Found the following \033[32mrunning\033[0m projects; however, \033[31mRollDev core services are currently not running\033[0m:" + else + echo -e "Found the following \033[32mrunning\033[0m environments:" + fi + for line in "${messageList[@]}"; do + echo -e "$line" + done else - echo -e "Found the following \033[32mrunning\033[0m environments:" + echo "No running environments found." fi - for line in "${messageList[@]}"; do - echo -e "$line" - done -else - echo "No running environments found." fi if [[ -n "${rollNetworkId}" ]]; then - echo - echo -e "RollDev Services (enabled -> running):" + if [[ "${STATUS_FORMAT}" != "json" ]]; then + echo + echo -e "RollDev Services (enabled -> running):" + fi - portainerEnabled=0 - startpageEnabled=1 if [[ -f "${ROLL_HOME_DIR}/.env" ]]; then - portainerEnabled=$(grep -m1 '^ROLL_SERVICE_PORTAINER=' "${ROLL_HOME_DIR}/.env" | cut -d '=' -f2- | tr -d '\r') - startpageEnabled=$(grep -m1 '^ROLL_SERVICE_STARTPAGE=' "${ROLL_HOME_DIR}/.env" | cut -d '=' -f2- | tr -d '\r') + initConfigSchema + loadConfigFromFile "${ROLL_HOME_DIR}/.env" fi - portainerEnabled=${portainerEnabled:-0} - startpageEnabled=${startpageEnabled:-1} + # Read through the schema so the default (1) is authoritative, matching svc.cmd + portainerEnabled="$(getConfig ROLL_SERVICE_PORTAINER 1)" + startpageEnabled="$(getConfig ROLL_SERVICE_STARTPAGE 1)" services=(traefik dnsmasq mailhog tunnel) [[ "${portainerEnabled}" == 1 ]] && services+=(portainer) [[ "${startpageEnabled}" == 1 ]] && services+=(startpage) - printf ' %-12s %-10s %-20s %s\n' "NAME" "STATE" "STATUS" "PORTS" + [[ "${STATUS_FORMAT}" != "json" ]] && printf ' %-12s %-10s %-20s %s\n' "NAME" "STATE" "STATUS" "PORTS" for svc in "${services[@]}"; do name=$(docker ps --filter "name=^${svc}$" --format '{{.Names}}') state=$(docker ps --filter "name=^${svc}$" --format '{{.State}}') status=$(docker ps --filter "name=^${svc}$" --format '{{.Status}}') ports=$(docker ps --filter "name=^${svc}$" --format '{{.Ports}}') + + jsonServiceNames+=("${svc}") if [[ -z "${name}" ]]; then - printf ' %-12s %-10s %-20s -\n' "${svc}" "stopped" "Exited" + jsonServiceStates+=("stopped") + [[ "${STATUS_FORMAT}" != "json" ]] && printf ' %-12s %-10s %-20s -\n' "${svc}" "stopped" "Exited" else - printf ' %-12s %-10s %-20s %s\n' "${name}" "${state}" "${status}" "${ports}" + [[ "${state}" == "running" ]] && jsonServiceStates+=("running") || jsonServiceStates+=("stopped") + [[ "${STATUS_FORMAT}" != "json" ]] && printf ' %-12s %-10s %-20s %s\n' "${name}" "${state}" "${status}" "${ports}" fi done fi + +if [[ "${STATUS_FORMAT}" == "json" ]]; then + out="{\"projects\":[" + i=0 + while [[ $i -lt ${#jsonProjectNames[@]} ]]; do + (( i > 0 )) && out+="," + out+="{" + out+="\"name\":\"$(jsonEscape "${jsonProjectNames[$i]}")\"," + out+="\"type\":\"$(jsonEscape "${jsonProjectTypes[$i]}")\"," + out+="\"dir\":\"$(jsonEscape "${jsonProjectDirs[$i]}")\"," + out+="\"url\":\"$(jsonEscape "${jsonProjectUrls[$i]}")\"," + out+="\"network\":\"$(jsonEscape "${jsonProjectNetworks[$i]}")\"," + out+="\"containers\":${jsonProjectContainerCounts[$i]}" + out+="}" + i=$((i + 1)) + done + out+="],\"services\":[" + i=0 + while [[ $i -lt ${#jsonServiceNames[@]} ]]; do + (( i > 0 )) && out+="," + out+="{\"name\":\"$(jsonEscape "${jsonServiceNames[$i]}")\",\"status\":\"${jsonServiceStates[$i]}\"}" + i=$((i + 1)) + done + out+="]}" + printf '%s\n' "${out}" +fi diff --git a/commands/status.help b/commands/status.help index 398a904..7913ce7 100644 --- a/commands/status.help +++ b/commands/status.help @@ -1,14 +1,21 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: List running RollDev project environments and their services. +## @category: environment ROLL_USAGE=$(cat <<'USAGE' \033[33mUsage:\033[0m status Display list of all running RollDev project environments + status --format json Same information, as machine-readable JSON on stdout -The command shows each project's name, directory, primary URL and other +For each project the command shows its name, directory, primary URL and other details such as the Docker network and the number of running containers. When RollDev core services are running, a summary table of enabled services is also displayed similar to the output of \`docker ps\`. + +With --format json, the same data is printed as a single JSON object with no +ANSI escapes and no credentials: a projects array (name, type, dir, url, +network, containers) and a services array (name, status). USAGE ) diff --git a/commands/stop.help b/commands/stop.help index ff5af54..d7c234e 100644 --- a/commands/stop.help +++ b/commands/stop.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Stop the RollDev core services. +## @category: environment ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run docker compose subcommands scoped to the RollDev core services. +## @category: services COMPOSE_USAGE=$(docker compose "${ROLL_PARAMS[@]}" "$@" \ | grep -vE '(--file|--project-name|--project-directory|default: docker-compose.yml|default: directory name)' \ diff --git a/commands/sync.help b/commands/sync.help index 728ac1b..f290ead 100755 --- a/commands/sync.help +++ b/commands/sync.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Manage the Mutagen file sync session for the current project. +## @category: services ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## Open the current project database in TablePlus. +## +## TablePlus is macOS-only, so this command is too. It is kept rather than hidden on Linux because a +## clear message beats "command not found" when someone follows documentation written on a Mac. + +if [[ "${OSTYPE}" != darwin* ]]; then + error "roll tableplus needs TablePlus, which is macOS-only." + fatal "On Linux use \`roll db connect\`, or point your own client at the tunnel on localhost:2222." +fi ROLL_ENV_PATH="$(locateEnvPath)" || exit $? loadEnvConfig "${ROLL_ENV_PATH}" || exit $? assertDockerRunning if [[ ${ROLL_DB:-1} -eq 0 ]]; then - fatal "Database environment is not used (ROLL_DB=0)." + fatal "Database environment is not used (ROLL_DB=0)." fi -TABLEPLUS_CMD="/Applications/TablePlus.app/Contents/MacOS/TablePlus" - -if [ ! -d "/Applications/TablePlus.app" ]; then - if [ -d "/Applications/Setapp/TablePlus.app" ]; then - TABLEPLUS_CMD="/Applications/Setapp/TablePlus.app/Contents/MacOS/TablePlus" - else - fatal "Tableplus app not exists on machine" - fi +## Locate the app. Setapp installs it somewhere else entirely, which is why this looks in two places. +TABLEPLUS_APP="" +for candidate in "/Applications/TablePlus.app" "/Applications/Setapp/TablePlus.app" "${HOME}/Applications/TablePlus.app"; do + if [[ -d "${candidate}" ]]; then + TABLEPLUS_APP="${candidate}" + break + fi +done + +if [[ -z "${TABLEPLUS_APP}" ]]; then + fatal "TablePlus was not found in /Applications, /Applications/Setapp or ~/Applications." fi -### load connection information for the mysql service -DB_CONTAINER=$(roll env ps -q db) +## load connection information from the running db container +DB_CONTAINER="$(roll env ps -q db)" || true if [[ ! ${DB_CONTAINER} ]]; then - fatal "No container found for db service." + fatal "No container found for db service. Is the environment running?" fi eval "$( - docker container inspect ${DB_CONTAINER} --format ' + docker container inspect "${DB_CONTAINER}" --format ' {{- range .Config.Env }}{{with split . "=" -}} {{- index . 0 }}='\''{{ range $i, $v := . }}{{ if $i }}{{ $v }}{{ end }}{{ end }}'\''{{println}} {{- end }}{{ end -}} ' | grep "^MYSQL_" )" -eval "MYSQL_HOST=$(docker container inspect ${DB_CONTAINER} --format='{{.Name}}' | cut -c2-)" +MYSQL_HOST="$(docker container inspect "${DB_CONTAINER}" --format='{{.Name}}' | cut -c2-)" + +TABLEPLUS_URI="mariadb+ssh://user@tunnel.roll.test:2222/${MYSQL_USER}:${MYSQL_PASSWORD}@${MYSQL_HOST}/${MYSQL_DATABASE}?statusColor=686B6F&enviroment=local&name=${ROLL_ENV_NAME}%20DOCKER&tLSMode=0&usePrivateKey=true&safeModeLevel=0&advancedSafeModeLevel=0" -query="mariadb+ssh://user@tunnel.roll.test:2222/${MYSQL_USER}:${MYSQL_PASSWORD}@${MYSQL_HOST}/${MYSQL_DATABASE}?statusColor=686B6F&enviroment=local&name=${ROLL_ENV_NAME}%20DOCKER&tLSMode=0&usePrivateKey=true&safeModeLevel=0&advancedSafeModeLevel=0" -open "$query" -a $TABLEPLUS_CMD +openInTablePlus "${TABLEPLUS_URI}" diff --git a/commands/tableplus.help b/commands/tableplus.help index 80503fa..337b9d9 100755 --- a/commands/tableplus.help +++ b/commands/tableplus.help @@ -1,10 +1,11 @@ - #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Launch TablePlus connected to the project database. +## @category: database ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Display the RollDev command usage banner. +## @category: utility ROLL_HEADER=' \n @@ -49,9 +51,12 @@ RollDev version $(cat ${ROLL_DIR}/version) svc Orchestrates global services such as traefik, portainer and dnsmasq via docker-compose env-init Configure environment by adding \033[31m'.env.roll'\033[0m file to the current working directory magento2-init Scaffold a complete Magento 2 project from scratch - env Controls an environment from any point within the root project directory - config Manage and validate Roll configuration (see \033[31m'roll config -h'\033[0m for details) + env Controls an environment (see \033[31m'roll env -h'\033[0m for details, including \033[31m'roll env sh'\033[0m) + config Manage and validate Roll configuration (see \033[31m'roll config -h'\033[0m for check-pins, fix-pins) registry Manage and inspect command registry (see \033[31m'roll registry -h'\033[0m for details) + has-command Check whether a command is available; communicates via exit code only + copyfromcontainer Copy a file, folder, or debug artifact from the container to the host + copytocontainer Copy a file, folder, or the entire project source from the host into the container db Interacts with the db service on an environment (see \033[31m'roll db -h'\033[0m for details) redis Interacts with the redis service on an environment (see \033[31m'roll redis -h'\033[0m for details) backup Create archive backups of an environment (see \033[31m'roll backup -h'\033[0m for details) @@ -60,8 +65,14 @@ RollDev version $(cat ${ROLL_DIR}/version) install Initializes or updates roll configuration on host machine shell Launches into a shell within the current project environment status Display list of all running RollDev project environments + describe Display detailed information about the current environment + doctor Diagnose whether the current project environment is fit to run debug Launches debug enabled shell within current project environment sign-certificate Signs a wildcard certificate including all passed hostnames on the SAN list + convert Convert a Warden project in the current directory to RollDev + duplicate Duplicate the current environment to a new environment with a different name + multistore Manage Magento multi-store configuration + vnc Connect to Selenium VNC server version Show version information \033[33mEnvironment commands:\033[0m @@ -71,10 +82,14 @@ RollDev version $(cat ${ROLL_DIR}/version) bash Allocate TTY with shell (simulair to \033[31m'roll shell'\033[0m) cli \033[32m%command%\033[0m Run command inside container with TTY allocation clinotty \033[32m%command%\033[0m Run command inside container without TTY allocation + cliq \033[32m%command%\033[0m Run command inside container without TTY allocation and suppress output root \033[32m%command%\033[0m Run command inside container with TTY allocation as root user rootnotty \033[32m%command%\033[0m Run command inside container without TTY allocation as root user rootshell Allocate TTY with shell as root user composer \033[32m%command%\033[0m run composer inside php-fpm container with params + node \033[32m%command%\033[0m Run node commands inside the container + npm \033[32m%command%\033[0m Run npm commands inside the container + magerun \033[32m%command%\033[0m Run magerun commands inside the container add-php-ext \033[32m%ext%\033[0m add custom php extensions to the container browsersync See browsersync domain and port values ${ENV_TYPE_USAGE:1} diff --git a/commands/version.help b/commands/version.help index 58a065d..c208127 100644 --- a/commands/version.help +++ b/commands/version.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Display the installed RollDev version. +## @category: utility ROLL_USAGE=$(cat </dev/null | sed -n "${ROLL_SELENIUM_INDEX}p") || true +if [[ -z "${ROLL_SELENIUM_CONTAINER}" ]]; then + fatal "No running container found for the \"${ROLL_SELENIUM_SERVICE}\" service (index ${ROLL_SELENIUM_INDEX})." +fi + +ROLL_SELENIUM_VNC=$(docker container inspect --format '{{ .Name }}' "${ROLL_SELENIUM_CONTAINER}" 2>/dev/null) || true +ROLL_SELENIUM_VNC="${ROLL_SELENIUM_VNC#/}" +if [[ -z "${ROLL_SELENIUM_VNC}" ]]; then + fatal "Could not resolve the container name for the \"${ROLL_SELENIUM_SERVICE}\" service." +fi if ! which remmina >/dev/null; then EXPOSE_PORT=$((5900 + ROLL_SELENIUM_INDEX)) diff --git a/commands/vnc.help b/commands/vnc.help index 02e7020..89fd64a 100644 --- a/commands/vnc.help +++ b/commands/vnc.help @@ -1,5 +1,7 @@ #!/usr/bin/env bash [[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Open a VNC session to a project container. +## @category: services ROLL_USAGE=$(cat <&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 +## @description: Run WP-CLI inside the project php-fpm container. +## @category: wordpress WORDPRESS_HELP=$("${ROLL_DIR}/bin/roll" cli "wp") diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a12e8e1..ce1e605 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -9,7 +9,7 @@ services: - ${ROLL_HOME_DIR}/etc/traefik/traefik.yml:/etc/traefik/traefik.yml - ${ROLL_HOME_DIR}/etc/traefik/dynamic.yml:/etc/traefik/dynamic.yml - ${ROLL_HOME_DIR}/ssl/certs:/etc/ssl/certs - - /var/run/docker.sock:/var/run/docker.sock + - /var/run/docker.sock:/var/run/docker.sock:ro labels: - traefik.enable=true - traefik.http.routers.traefik.tls=true diff --git a/docs/changelog.md b/docs/changelog.md index 3a6943d..16a4f03 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,3 +1,7 @@ # Change Log -See the full change log in GitHub: [https://github.com/epartment/rolldev/releases](https://github.com/epartment/rolldev/releases) \ No newline at end of file +```{include} ../CHANGELOG.md +--- +start-line: 2 +--- +``` diff --git a/docs/conf.py b/docs/conf.py index 68d63ed..0c6c8eb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -40,9 +40,7 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] html_show_sourcelink = False -html_static_path = ['_static'] templates_path = ['_templates'] -html_extra_path = ['_redirects'] myst_enable_extensions = [ "html_admonition", diff --git a/docs/configuration/copy-container.md b/docs/configuration/copy-container.md new file mode 100644 index 0000000..49309e8 --- /dev/null +++ b/docs/configuration/copy-container.md @@ -0,0 +1,52 @@ +# Copying files between host and container + +`roll copyfromcontainer` and `roll copytocontainer` copy files between the project directory on the +host and `/var/www/html` inside the `php-fpm` container, without going through a bind mount. + +## copyfromcontainer + +``` +roll copyfromcontainer vendor/autoload.php +``` + +Copies the given file or folder from the container to the same relative path on the host. + +``` +roll copyfromcontainer --all +``` + +Copies the entire project source from the container to the host. + +``` +roll copyfromcontainer --cachegrind [file] +roll copyfromcontainer --traces [file] +``` + +Copies a cachegrind profile or xdebug trace from `/tmp` inside the `php-debug` container into +`tmp/profiles` or `tmp/traces` in the project directory. Without a file argument this prompts +interactively for which file to copy (requires a terminal); non-interactively, pass the filename +explicitly. + +``` +roll copyfromcontainer --realpath +``` + +Copies an absolute container path into the project `tmp` folder, instead of the default +`/var/www/html`-relative path. + +## copytocontainer + +``` +roll copytocontainer vendor +``` + +Copies the given file or folder from the host into the container, then runs `roll fixowns` and +`roll fixperms` on that path. + +``` +roll copytocontainer --all +``` + +Copies the entire project source from the host into the container, then runs `roll fixowns` and +`roll fixperms` for the whole project. Useful after a container rebuild to restore files that only +existed inside the previous container. diff --git a/docs/configuration/doctor.md b/docs/configuration/doctor.md new file mode 100644 index 0000000..2842ea1 --- /dev/null +++ b/docs/configuration/doctor.md @@ -0,0 +1,58 @@ +# Diagnosing an environment with `roll env doctor` + +`roll env doctor` (also available as the top-level `roll doctor`) answers a single question: is +the current project environment actually fit to run? Instead of letting a problem surface later as +an opaque Docker or application error deep into a build, it runs a fixed set of checks and reports +one line per check. + +``` +roll env doctor +``` + +``` +roll env doctor --format json +``` + +## What it checks + +- **`env-config`** — `.env.roll` for this project loads. +- **`docker`** — the Docker daemon is reachable. +- **`container:`** — every container this project has is running, and if it carries a + Docker healthcheck, that healthcheck reports healthy. A container with no healthcheck configured + at all — for example one started before its compose fragment gained a healthcheck — is reported + as running but unverified, not as a failure. +- **`port-80` / `port-443` / `port-53`** — the ports the shared roll core services need (Traefik on + 80/443, dnsmasq on 53) are bound by those services, or free for them to bind. Project containers + route through Traefik rather than publishing ports of their own, so these are the ports every + project actually depends on. +- **`port-browsersync`** — when BrowserSync is enabled with host port publication on, whether its + ports are bound. +- **`search-engine:`** — the configured search engine (OpenSearch or Elasticsearch, reached + through Traefik on its own subdomain) answers `/_cluster/health` with a status other than red. +- **`search-engine-write:`** — the search engine also accepts a throwaway index write, + immediately deleted again. A green cluster that refuses writes — for example after hitting a disk + watermark — is exactly the failure a health-only probe would miss. +- **`disk`** — the Docker data root has at least 5GB free. Probed from inside a running container + belonging to the project so it works the same way whether the Docker daemon runs natively on + Linux or inside a VM (Docker Desktop, OrbStack). + +## Exit status and output + +Exit status is `0` when every check passes and `1` when any check fails, so `roll env doctor` is +safe to use as a build pipeline gate. + +`--format json` prints a single JSON object instead of the human-readable box, with no ANSI escapes +and no credentials: + +```json +{ + "checks": [ + {"check": "docker", "ok": true, "detail": "Docker daemon is reachable."} + ], + "ok": true +} +``` + +Each entry in `checks` has `check` (the check name), `ok` (boolean) and `detail` (a human-readable +explanation) — the same shape the failing check would have in the human-readable report, so +automation and interactive use see the same information. diff --git a/docs/configuration/unattended-operation.md b/docs/configuration/unattended-operation.md new file mode 100644 index 0000000..4417f73 --- /dev/null +++ b/docs/configuration/unattended-operation.md @@ -0,0 +1,41 @@ +# Unattended operation + +These toggles matter most on a build server that provisions many environments in parallel, +unattended. Interactively running a single project usually needs neither. + +## Suppressing published host ports + +When [BrowserSync](livereload.md) is enabled with `ROLL_BROWSERSYNC=1`, RollDev publishes its web +and UI ports on the host so a browser can reach them directly. Those ports are fixed, so two +environments on the same host with BrowserSync enabled collide on the same host port — Compose +reports `Bind for 0.0.0.0: failed: port is already allocated` and the `php-fpm` container +never starts. + +Set `ROLL_PUBLISH_PORTS=0` in `.env.roll` to suppress host port publication for the whole +environment: + +``` +ROLL_PUBLISH_PORTS=0 +``` + +The default is `1` (publish ports, current behaviour). With it set to `0`, `BROWSERSYNC_PORT_WEB` +and `BROWSERSYNC_PORT_UI` are still passed into the `php-fpm` container as environment variables — +only the host port mapping is skipped, so BrowserSync itself is unaffected for anything reaching it +through the container network (e.g. Traefik). + +## Search-engine heap size + +`ROLL_ELASTICSEARCH` and `ROLL_OPENSEARCH` both default their JVM heap to `-Xms64m -Xmx512m`. That +is enough for a small catalog but too little for a large one — the container gets OOM-killed part +way through indexing, and the visible symptom is a connection error from the application rather +than anything that points at memory. + +Override it per project in `.env.roll`: + +``` +ELASTICSEARCH_JAVA_OPTS=-Xms256m -Xmx2g +OPENSEARCH_JAVA_OPTS=-Xms256m -Xmx2g +``` + +Each applies only to its own service — set the one matching whichever engine +(`ROLL_ELASTICSEARCH`/`ROLL_OPENSEARCH`) the project has enabled. diff --git a/docs/configuration/version-pins.md b/docs/configuration/version-pins.md new file mode 100644 index 0000000..799f802 --- /dev/null +++ b/docs/configuration/version-pins.md @@ -0,0 +1,101 @@ +# Service version pins + +Every service RollDev starts runs a specific image tag — `nginx:1.27`, `mariadb:10.4`, +`elasticsearch:7.17`. Those versions come from your project's `.env.roll`. + +## What changed in 0.8.0 + +Before 0.8.0, a version you did not set fell back to a default built into RollDev itself. That made +the version a property of *your installed RollDev* rather than of your project: upgrading RollDev +could change which PHP or database image a project ran, without the project changing at all. Two +people on the same branch could be running different images. + +From 0.8.0, RollDev warns when an enabled service has no version pinned: + +``` +WARNING: This magento2 environment enables services whose versions are not pinned. +WARNING: Roll is falling back to a built-in version, which means upgrading roll can silently +WARNING: change which image this project runs. From 0.9.0 this will be an error. +WARNING: These are the versions the project is running right now - run `roll config fix-pins` +WARNING: to write them into .env.roll, or add them by hand: + + NGINX_VERSION=1.27 +``` + +**Nothing breaks in 0.8.0.** The fallback still applies, and it is deliberately the version your +project was already running — so the warning never comes with a change in behaviour. In 0.9.0 the +fallback is removed and a missing pin becomes an error. + +## Fixing it + +```bash +roll config fix-pins +``` + +This appends the missing pins to `.env.roll`, using the versions the project runs today, after +taking a timestamped backup. Because the values are the ones already in effect, running it never +changes which images come up — verify with `roll env config` before and after if you want to see +that for yourself. + +To see what is missing without writing anything: + +```bash +roll config check-pins +``` + +It lists the missing pins and exits `1` when there are any, so a script or CI job can gate on it. + +## Which versions are required + +`PHP_VERSION` and `NODE_VERSION` are required for every environment type except `local`, which +never starts a PHP container. Beyond that, a version is required only when its service is switched +on: + +| Service toggle | Version variable | +|---|---| +| `ROLL_NGINX` | `NGINX_VERSION` | +| `ROLL_DB` | `DB_DISTRIBUTION_VERSION` | +| `ROLL_REDIS` | `REDIS_VERSION` | +| `ROLL_DRAGONFLY` | `DRAGONFLY_VERSION` | +| `ROLL_VARNISH` | `VARNISH_VERSION` | +| `ROLL_ELASTICSEARCH` | `ELASTICSEARCH_VERSION` | +| `ROLL_OPENSEARCH` | `OPENSEARCH_VERSION` | +| `ROLL_RABBITMQ` | `RABBITMQ_VERSION` | +| `ROLL_MONGODB` | `MONGO_VERSION` | +| `ROLL_MAGEPACK` | `MAGEPACK_VERSION` | +| `ROLL_SELENIUM` | `ROLL_SELENIUM_VERSION` | + +Turning a service off removes its pin requirement. + +## Global config does not count as a pin + +A version set in `~/.roll/.env` or `~/.roll/.env.roll` is loaded before your project's `.env.roll`, +so it resolves — but it pins nothing. It lives on your machine, not in the repository, so a +colleague who does not have that line gets a different image from the same branch, which is exactly +what pinning exists to prevent. `check-pins` therefore still reports such a version as missing. + +`fix-pins` writes the value you are inheriting, not RollDev's built-in default, so pinning it does +not change the image your environment comes up with: + +```bash +# ~/.roll/.env carries PHP_VERSION=8.3, the project pins nothing +roll config fix-pins # writes PHP_VERSION=8.3 into .env.roll +``` + +## New projects + +`roll env-init` seeds `.env.roll` from the environment type's `init.env`, which pins everything that +type enables. Projects created with 0.8.0 or later start out fully pinned and never see the warning. + +## Upgrading a version deliberately + +Edit the value in `.env.roll` and bring the environment back up: + +```bash +roll env down +roll env up +``` + +Changing a database or search-engine version usually means the existing data volume was written by +the old version. Check that engine's upgrade path first — RollDev only selects the image, it does +not migrate data. diff --git a/docs/environments/types.md b/docs/environments/types.md index c54d749..5f5e414 100644 --- a/docs/environments/types.md +++ b/docs/environments/types.md @@ -2,13 +2,24 @@ RollDev currently supports three environment types. These types are passed to `env-init` when configuring a project for local development for the first time. This list of environment types can also be seen by running `roll env-init --help` on your command line. The `docker-compose` configuration used to assemble each environment type can be found in the [environments directory](https://github.com/epartment/rolldev/tree/master/environments) on Github. +`roll env-init` follows RollDev's flag/env-first prompt convention: if you pass a project name and type as arguments (`roll env-init myproject magento2`), it runs non-interactively and never prompts. Leave either out on a terminal and it prompts for the missing value; leave either out with no terminal attached (CI, a script) and it errors naming the argument you need to supply instead of hanging. Re-running `env-init` where a `.env.roll` already exists normally asks whether to overwrite it; set `ROLL_ENV_INIT_FORCE=1` to overwrite without prompting. + ## Local -The `local` environment type does nothing more than declare the `docker-compose` version and label the project network so RollDev will recognize it as belonging to an environment orchestrated by RollDev. +The `local` environment type is a network-only container — it provides no services of its own. Use it when you need RollDev's networking layer (Traefik, dnsmasq, the shared tunnel) but want to define your own services in a `.roll/roll-env.yml` file. + +**Required:** Every usable `local` project must provide its own `.roll/roll-env.yml` defining which containers to run. If you use the default service toggles (which default to enabled), `roll env config` will fail because it expects those services to be defined in your custom compose fragment. + +**To use the `local` type:** + +1. Run `roll env-init myproject local` +2. Create `.roll/roll-env.yml` in your project root with your desired services (nginx, php-fpm, db, etc.) +3. Set service toggles to 0 in `.env.roll` for any services you are NOT providing in your custom fragment: `ROLL_NGINX=0`, `ROLL_DB=0`, etc. +4. Run `roll env up` -When this type is used, a `.roll/roll-env.yml` may be placed in the root directory of the project workspace to define the desired containers, volumes, etc needed for the project. An example of a `local` environment type being used can be found in the [m2demo project](https://github.com/davidalger/m2demo). +**Platform-specific overrides:** RollDev supports `roll-env.darwin.yml` and `roll-env.linux.yml` for macOS/Linux differences, following the same pattern as other environment types. -Similar to the other environment type's base definitions, RollDev supports a `roll-env.darwin.yml` and `roll-env.linux.yml` +For an example, see the [m2demo project](https://github.com/davidalger/m2demo). ## Magento 2 diff --git a/docs/index.md b/docs/index.md index 73b2802..7c8d2c6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -37,6 +37,7 @@ backup-restore-quick-reference registry environments configuration +machine-interface ``` ```{toctree} diff --git a/docs/installing.md b/docs/installing.md index c50c5a4..1cc8332 100644 --- a/docs/installing.md +++ b/docs/installing.md @@ -6,6 +6,20 @@ * [Docker Desktop for Mac](https://hub.docker.com/editions/community/docker-ce-desktop-mac) 2.2.0.0 or later or [Docker for Linux](https://docs.docker.com/install/) or [Docker for Windows](https://docs.docker.com/desktop/windows/install/) * `docker-compose` version 2.0 or later is required (this can be installed via `brew`, `apt`, `dnf`, or `pip3` as needed) * [Mutagen](https://mutagen.io/) 0.11.4 or later is required for environments leveraging sync sessions on Mac OS. RollDev will attempt to install this via `brew` if not present. +* [gum](https://github.com/charmbracelet/gum) 0.14.0 or later backs every interactive prompt (`roll env-init`, and the password prompts in `backup`, `restore` and `duplicate`). The Homebrew formula installs it for you. Installing it by hand: + + | Platform | Command | + |---|---| + | macOS | `brew install gum` | + | Debian/Ubuntu | Charm's apt repository — see [gum's install instructions](https://github.com/charmbracelet/gum#installation) | + | Fedora/RHEL | `sudo dnf install gum` | + | Arch | `sudo pacman -S gum` | + | Anywhere | download a binary from [the releases page](https://github.com/charmbracelet/gum/releases) — no root required | + + gum is only needed for prompts. Every prompt can also be answered by a flag, an environment + variable or a positional argument, so scripted and CI use works without it — see + [Service version pins](configuration/version-pins.md) and each command's `--help` for the + non-interactive form. `roll install` warns if gum is missing rather than failing. :::{warning} **By default Docker Desktop for Mac allocates 2GB memory.** diff --git a/docs/machine-interface.md b/docs/machine-interface.md new file mode 100644 index 0000000..9a80c6f --- /dev/null +++ b/docs/machine-interface.md @@ -0,0 +1,174 @@ +# Driving RollDev from a script + +RollDev is normally used interactively, but everything it does is also reachable without a terminal: +from a CI job, a deployment script, a build server rebuilding environments overnight, or an AI +assistant. This page collects the parts of the interface that exist for that. + +The rule underneath all of it: **never parse RollDev's human output.** It is formatted for people, +carries ANSI colour, and its layout is not a contract. The JSON surfaces below are. + +## Machine-readable output + +Four commands accept `--format json`: + +```bash +roll status --format json # every project on the host +roll env describe --format json # one project, its services and their state +roll registry list --format json # every available command, with descriptions +roll env doctor --format json # per-check diagnostics +``` + +The output carries no ANSI escapes and no credentials. Values are escaped, so a project path +containing a quote or a non-ASCII character still parses. + +The two project commands share a vocabulary — `name`, `type`, `dir`, `url`, `network`, `containers` +— but nest it differently, because one covers every project and the other covers one: + +- `status` returns `{"projects": [ … ], "services": [ … ]}`, where `projects` holds one object per + environment on the host and `services` holds the **shared** RollDev services (traefik, dnsmasq and + friends). +- `describe` returns a single project object directly, with its own `services` array for that + project's containers. + +Service entries carry at least `name` and `status`. + +```bash +# which of this project's services are not running +roll env describe --format json | jq -r '.services[] | select(.status != "running") | .name' + +# every environment on the host, by name +roll status --format json | jq -r '.projects[].name' +``` + +`doctor` returns an overall `ok` alongside a `checks` array, one row per check: + +```json +{ + "ok": true, + "checks": [ + {"check": "search-engine-write:opensearch", "ok": true, "detail": "..."} + ] +} +``` + +`registry list` returns `command`, `category`, `description`, `priority` and `source`. Priority is +the registry search tier — project `.roll/commands` (1), `~/.roll/commands` (2), `~/.roll/reclu` (3), +built-in (4) — and **lower wins**, so a project-local command shadows a built-in of the same name. + +## Feature detection + +Do not assume a command exists. Command availability varies: RollDev version, whether an internal +command pack is installed, and whether the project defines its own commands. + +```bash +if roll has-command pull; then + roll pull +fi +``` + +`has-command` prints nothing and exits `0` or `1`. It resolves through the registry, so it sees +project-local and add-on commands, not just built-ins. + +## Waiting for services + +`roll env up` returns as soon as Compose has started the containers, which is not the same as the +services being ready. A script that continues immediately will race the database. + +```bash +roll env up --wait +roll db connect -e 'SELECT 1' # safe: --wait returned only once db reported healthy +``` + +`--wait` relies on the healthchecks RollDev defines for `db`, `redis`, `elasticsearch`, +`opensearch`, `rabbitmq`, `varnish` and `nginx`. The search engines are checked through +`/_cluster/health` rather than a port probe, because those containers can hold a port open while the +cluster inside them is dead. + +One caveat: containers created **before** the healthchecks existed report no health status, so +`--wait` has nothing to wait on for them. Recreate the environment once (`roll env down && roll env +up`) after upgrading. + +## Prompts, flags and no terminal + +Every prompt resolves in this order: **a value supplied by flag, positional argument or environment +variable, then gum on a terminal, then a hard error naming the flag that would have supplied it.** + +A prompt never blocks waiting for input that cannot arrive. Without a terminal, an unanswerable +prompt fails immediately and tells you what to pass: + +``` +ERROR: Cannot prompt for a password: no terminal attached. +ERROR: Supply it non-interactively with --encrypt=. +``` + +So a script never hangs, and the fix is always in the error. The non-interactive forms: + +| Interactive prompt | Non-interactive form | +|---|---| +| `roll env-init` name and type | `roll env-init ` | +| `roll env-init` overwrite confirmation | `ROLL_ENV_INIT_FORCE=1` | +| `roll backup` encryption password | `--encrypt=` | +| `roll restore` decryption password | `--decrypt=` | +| `roll duplicate` encryption password | `--encrypt=` | + +`gum` is only needed for the interactive path. A fully flag-driven run works without it installed. + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Success. For `doctor`, every check passed. For `has-command`, the command exists. | +| `1` | Failure. For `doctor`, at least one check failed. For `has-command`, no such command. | +| `130` | Cancelled at an interactive prompt (Ctrl-C). Only reachable on a terminal. | + +`--help` also exits `1`, which is long-standing behaviour rather than an error signal — do not treat +a non-zero exit from `--help` as a failed command. + +## Pinning versions + +From 0.8.0 an enabled service with no version pinned produces a warning, and from 0.9.0 it will be +an error. For unattended use, pin explicitly so that upgrading RollDev cannot change which images a +project runs: + +```bash +roll config check-pins # exits 1 if any enabled service lacks a pin +roll config fix-pins # writes the versions the project runs today +``` + +`check-pins` is the one to put in CI. See [Service version pins](configuration/version-pins.md). + +## Running many environments on one host + +Two settings exist for build servers running several environments at once: + +- `ROLL_PUBLISH_PORTS=0` stops the browsersync partial publishing host ports, which otherwise + collide across environments and cannot be overridden from `.roll/roll-env.yml`. +- `ELASTICSEARCH_JAVA_OPTS` / `OPENSEARCH_JAVA_OPTS` replace the fixed search-engine heap, so + several search containers can coexist. + +See [Unattended operation](configuration/unattended-operation.md). + +## A worked example + +```bash +#!/usr/bin/env bash +set -euo pipefail + +roll config check-pins # fail early on unpinned versions +roll env up --wait # block until services are actually ready + +if ! roll env doctor --format json > doctor.json; then + jq -r '.checks[] | select(.ok == false) | "\(.check): \(.detail)"' doctor.json >&2 + exit 1 +fi + +if roll has-command pull; then # optional add-on command + roll pull +fi + +roll env sh php-fpm 'bin/magento setup:upgrade' # quoted: redirects apply in the container +``` + +Note the last line. `roll env exec php-fpm bin/magento ... > out.log` writes `out.log` on the +**host**; `roll env sh` wraps the command in `sh -c` inside the container so redirects and pipes +behave as written. diff --git a/docs/usage.md b/docs/usage.md index 84edd6b..b420839 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -43,6 +43,17 @@ Flush varnish: roll env exec -T varnish varnishadm 'ban req.url ~ .' +Run a shell command inside a container (with proper redirect handling): + + roll env sh php-fpm 'cat app/etc/env.php | grep MODE' + +Copy a file from the container to the host, or the whole project back in after a rebuild (see the +[Copying files between host and container](configuration/copy-container.md) page for the full flag +list, including cachegrind/trace profile pickup): + + roll copyfromcontainer vendor/autoload.php + roll copytocontainer --all + Connect to redis: roll redis @@ -59,6 +70,28 @@ Remove volumes completely: roll env down -v +Build every Magento 2 frontend theme found under `app/design/frontend`: + + roll theme all + +Build or watch one theme directly, without the interactive picker: + + roll theme Vendor/theme build + roll theme Vendor/theme watch + +## Converting a Warden Project + +Run from the root of a Warden project to convert it to RollDev: renames `WARDEN_*` variables to +`ROLL_*`, moves `.env` to `.env.roll` and `.warden` to `.roll`, validates the result, then signs a +certificate and starts the environment. + + roll convert + +Existing service version pins (including `ELASTICSEARCH_VERSION`) are left as the project had +them. If a pin is missing afterwards, `roll config validate` reports a warning; run +`roll config fix-pins` to write one in. `roll convert` does nothing if `.env.roll` already exists +or `.env` does not reference `WARDEN`. + ## Environment Duplication Duplicate the current environment to create a new environment with a different name: diff --git a/environments/akeneo/init.env b/environments/akeneo/init.env index 3422488..65bb417 100644 --- a/environments/akeneo/init.env +++ b/environments/akeneo/init.env @@ -15,3 +15,8 @@ PHP_XDEBUG_3=1 # New Relic monitoring (set to 1 to enable) ROLL_NEWRELIC=0 + +# Web server +NGINX_VERSION=1.27 +ELASTICSEARCH_VERSION=7.17 +REDIS_VERSION=7.0 diff --git a/environments/includes/browsersync.noports.base.yml b/environments/includes/browsersync.noports.base.yml new file mode 100644 index 0000000..b4cec8f --- /dev/null +++ b/environments/includes/browsersync.noports.base.yml @@ -0,0 +1,5 @@ +services: + php-fpm: + environment: + - BROWSERSYNC_PORT_WEB=${BROWSERSYNC_PORT_WEB:-3000} + - BROWSERSYNC_PORT_UI=${BROWSERSYNC_PORT_UI:-3001} \ No newline at end of file diff --git a/environments/includes/db.base.yml b/environments/includes/db.base.yml index b74a732..225bba9 100644 --- a/environments/includes/db.base.yml +++ b/environments/includes/db.base.yml @@ -5,7 +5,7 @@ services: db: hostname: "${ROLL_ENV_NAME}-db" - image: ${ROLL_IMAGE_REPOSITORY}/${DB_DISTRIBUTION:-mariadb}:${DB_DISTRIBUTION_VERSION:-10.4} + image: ${ROLL_IMAGE_REPOSITORY}/${DB_DISTRIBUTION:-mariadb}:${DB_DISTRIBUTION_VERSION} environment: - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-app} - MYSQL_DATABASE=${MYSQL_DATABASE:-app} @@ -15,6 +15,20 @@ services: volumes: - dbdata:/var/lib/mysql - sqlhistory:/sql_history + healthcheck: + ## MariaDB 11 images dropped the mysqladmin compatibility symlink and only ship + ## mariadb-admin, while the mysql images ship only mysqladmin; try both so either + ## distribution works (mirrors the resolveDbBinary probe in commands/db.cmd). + ## + ## Deliberately unauthenticated: ping reports whether the server answers, and verified on + ## both engines it exits 0 with no credentials and with wrong ones alike, so passing the + ## root password would add nothing except a credential in argv on every interval - which + ## mysqladmin itself warns about ("Using a password on the command line can be insecure"). + test: ["CMD-SHELL", "mariadb-admin ping --silent 2>/dev/null || mysqladmin ping --silent 2>/dev/null"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 60s volumes: dbdata: diff --git a/environments/includes/dragonfly.base.yml b/environments/includes/dragonfly.base.yml index 661bcbf..0a19cbb 100644 --- a/environments/includes/dragonfly.base.yml +++ b/environments/includes/dragonfly.base.yml @@ -1,7 +1,7 @@ services: redis: hostname: "${ROLL_ENV_NAME}-dragonfly" - image: ${ROLL_IMAGE_REPOSITORY}/dragonfly:${DRAGONFLY_VERSION:-1.3} + image: ${ROLL_IMAGE_REPOSITORY}/dragonfly:${DRAGONFLY_VERSION} volumes: - dragonfly:/data diff --git a/environments/includes/elasticsearch.base.yml b/environments/includes/elasticsearch.base.yml index 852590a..00c4f8c 100644 --- a/environments/includes/elasticsearch.base.yml +++ b/environments/includes/elasticsearch.base.yml @@ -1,7 +1,7 @@ services: elasticsearch: hostname: "${ROLL_ENV_NAME}-elasticsearch" - image: ${ROLL_IMAGE_REPOSITORY}/elasticsearch:${ELASTICSEARCH_VERSION:-8.11} + image: ${ROLL_IMAGE_REPOSITORY}/elasticsearch:${ELASTICSEARCH_VERSION} labels: - traefik.enable=true - traefik.http.routers.${ROLL_ENV_NAME}-elasticsearch.tls=true @@ -12,11 +12,19 @@ services: - indices.query.bool.max_clause_count=16384 - discovery.type=single-node - xpack.security.enabled=false - - "ES_JAVA_OPTS=-Xms64m -Xmx512m" + - "ES_JAVA_OPTS=${ELASTICSEARCH_JAVA_OPTS}" - http.cors.enabled=true - http.cors.allow-origin=/.*/ volumes: - esdata:/usr/share/elasticsearch/data + healthcheck: + ## a bare port probe is not enough here: the container's fixed heap has historically let the + ## process look healthy while it is actually dead, so this hits the cluster-health endpoint + test: ["CMD", "curl", "-fs", "http://localhost:9200/_cluster/health"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 60s volumes: esdata: diff --git a/environments/includes/mongodb.base.yml b/environments/includes/mongodb.base.yml index f6342f0..8c79287 100644 --- a/environments/includes/mongodb.base.yml +++ b/environments/includes/mongodb.base.yml @@ -5,7 +5,7 @@ services: mongodb: hostname: "${ROLL_ENV_NAME}-mongodb" - image: ${ROLL_IMAGE_REPOSITORY}/mongo:${MONGODB_VERSION:-7} + image: ${ROLL_IMAGE_REPOSITORY}/mongo:${MONGO_VERSION} environment: - MONGO_INITDB_ROOT_USERNAME=${MONGODB_ROOT_USER:-app} - MONGO_INITDB_ROOT_PASSWORD=${MONGODB_ROOT_PASSWORD:-app} diff --git a/environments/includes/nginx.base.yml b/environments/includes/nginx.base.yml index c03fec5..ebdb775 100644 --- a/environments/includes/nginx.base.yml +++ b/environments/includes/nginx.base.yml @@ -1,7 +1,7 @@ services: nginx: hostname: "${ROLL_ENV_NAME}-nginx" - image: ${ROLL_IMAGE_REPOSITORY}/nginx:${NGINX_VERSION:-1.26} + image: ${ROLL_IMAGE_REPOSITORY}/nginx:${NGINX_VERSION} labels: - traefik.enable=true - traefik.http.routers.${ROLL_ENV_NAME}-nginx.tls=true @@ -16,3 +16,12 @@ services: - XDEBUG_CONNECT_BACK_HOST=${XDEBUG_CONNECT_BACK_HOST:-''} - NGINX_PUBLIC=${NGINX_PUBLIC:-/} - NGINX_TEMPLATE=${NGINX_TEMPLATE:-application.conf} + healthcheck: + ## checks nginx itself answers, not that the application returns 200 - a fresh project + ## with no code deployed yet legitimately serves 403/404, which must not read as unhealthy; + ## only a connection failure or a 5xx (nginx/upstream broken) fails the probe + test: ["CMD-SHELL", "curl -s -o /dev/null -w '%{http_code}' http://localhost/ | grep -qE '^[234]'"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s diff --git a/environments/includes/opensearch.base.yml b/environments/includes/opensearch.base.yml index 847a15c..a1d22f4 100644 --- a/environments/includes/opensearch.base.yml +++ b/environments/includes/opensearch.base.yml @@ -1,7 +1,7 @@ services: opensearch: hostname: "${ROLL_ENV_NAME}-opensearch" - image: ${ROLL_IMAGE_REPOSITORY}/opensearch:${OPENSEARCH_VERSION:-2.19} + image: ${ROLL_IMAGE_REPOSITORY}/opensearch:${OPENSEARCH_VERSION} labels: - traefik.enable=true - traefik.http.routers.${ROLL_ENV_NAME}-opensearch.tls=true @@ -12,11 +12,19 @@ services: - discovery.type=single-node - indices.query.bool.max_clause_count=16384 - indices.id_field_data.enabled=true - - "OPENSEARCH_JAVA_OPTS=-Xms64m -Xmx512m" + - "OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS}" - DISABLE_INSTALL_DEMO_CONFIG=true - DISABLE_SECURITY_PLUGIN=true volumes: - osdata:/usr/share/opensearch/data + healthcheck: + ## a bare port probe is not enough here: the container's fixed heap has historically let the + ## process look healthy while it is actually dead, so this hits the cluster-health endpoint + test: ["CMD", "curl", "-fs", "http://localhost:9200/_cluster/health"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 60s volumes: osdata: diff --git a/environments/includes/php-fpm.base.yml b/environments/includes/php-fpm.base.yml index 3d95fdb..a64d65d 100644 --- a/environments/includes/php-fpm.base.yml +++ b/environments/includes/php-fpm.base.yml @@ -19,12 +19,12 @@ services: php-fpm: hostname: "${ROLL_ENV_NAME}-php-fpm" - image: ${ROLL_IMAGE_REPOSITORY}/php-fpm${ROLL_SVC_PHP_VARIANT:-}:${PHP_VERSION:-8.3}${ROLL_SVC_PHP_NODE:-} + image: ${ROLL_IMAGE_REPOSITORY}/php-fpm${ROLL_SVC_PHP_VARIANT:-}:${PHP_VERSION}${ROLL_SVC_PHP_NODE:-} environment: - TRAEFIK_DOMAIN - TRAEFIK_SUBDOMAIN - SSH_AUTH_SOCK=${SSH_AUTH_SOCK_PATH_ENV:-/tmp/ssh-auth.sock} - - NODE_VERSION=${NODE_VERSION:-22} + - NODE_VERSION=${NODE_VERSION} - COMPOSER_VERSION=${COMPOSER_VERSION:-2} - COMPOSER_MEMORY_LIMIT=-1 - HISTFILE=/bash_history/.bash_history @@ -42,12 +42,12 @@ services: php-debug: hostname: "${ROLL_ENV_NAME}-php-debug" - image: ${ROLL_IMAGE_REPOSITORY}/php-fpm${ROLL_SVC_PHP_VARIANT:-}-debug:${PHP_VERSION:-8.3}${ROLL_SVC_PHP_NODE:-} + image: ${ROLL_IMAGE_REPOSITORY}/php-fpm${ROLL_SVC_PHP_VARIANT:-}-debug:${PHP_VERSION}${ROLL_SVC_PHP_NODE:-} environment: - TRAEFIK_DOMAIN - TRAEFIK_SUBDOMAIN - SSH_AUTH_SOCK=${SSH_AUTH_SOCK_PATH_ENV:-/tmp/ssh-auth.sock} - - NODE_VERSION=${NODE_VERSION:-22} + - NODE_VERSION=${NODE_VERSION} - COMPOSER_VERSION=${COMPOSER_VERSION:-2} - COMPOSER_MEMORY_LIMIT=-1 - PHP_IDE_CONFIG=serverName=${ROLL_ENV_NAME}-docker diff --git a/environments/includes/rabbitmq.base.yml b/environments/includes/rabbitmq.base.yml index 472e8a2..ae216d2 100644 --- a/environments/includes/rabbitmq.base.yml +++ b/environments/includes/rabbitmq.base.yml @@ -1,7 +1,7 @@ services: rabbitmq: hostname: "${ROLL_ENV_NAME}-rabbitmq" - image: ${ROLL_IMAGE_REPOSITORY}/rabbitmq:${RABBITMQ_VERSION:-3.8} + image: ${ROLL_IMAGE_REPOSITORY}/rabbitmq:${RABBITMQ_VERSION} labels: - traefik.enable=true - traefik.http.routers.${ROLL_ENV_NAME}-rabbitmq.tls=true @@ -10,6 +10,12 @@ services: - traefik.docker.network=${ROLL_ENV_NAME}_default volumes: - rabbitmq:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s volumes: rabbitmq: diff --git a/environments/includes/redis.base.yml b/environments/includes/redis.base.yml index 03b2daa..509a997 100644 --- a/environments/includes/redis.base.yml +++ b/environments/includes/redis.base.yml @@ -1,9 +1,15 @@ services: redis: hostname: "${ROLL_ENV_NAME}-redis" - image: ${ROLL_IMAGE_REPOSITORY}/redis:${REDIS_VERSION:-5.0} + image: ${ROLL_IMAGE_REPOSITORY}/redis:${REDIS_VERSION} volumes: - redis:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s volumes: redis: diff --git a/environments/includes/selenium.base.yml b/environments/includes/selenium.base.yml index f0adeaa..69667db 100644 --- a/environments/includes/selenium.base.yml +++ b/environments/includes/selenium.base.yml @@ -1,7 +1,7 @@ services: selenium: hostname: ${ROLL_ENV_NAME}_selenium - image: selenium/standalone-chrome${ROLL_SELENIUM_DEBUG:-}:${ROLL_SELENIUM_VERSION:-3.141.59} + image: selenium/standalone-chrome${ROLL_SELENIUM_DEBUG:-}:${ROLL_SELENIUM_VERSION} extra_hosts: - ${TRAEFIK_DOMAIN}:${TRAEFIK_ADDRESS:-0.0.0.0} - ${TRAEFIK_SUBDOMAIN:-app}.${TRAEFIK_DOMAIN}:${TRAEFIK_ADDRESS:-0.0.0.0} diff --git a/environments/includes/varnish.base.yml b/environments/includes/varnish.base.yml index ad5c37f..4457528 100644 --- a/environments/includes/varnish.base.yml +++ b/environments/includes/varnish.base.yml @@ -7,7 +7,7 @@ services: varnish: hostname: "${ROLL_ENV_NAME}-varnish" - image: ${ROLL_IMAGE_REPOSITORY}/varnish:${VARNISH_VERSION:-6.0} + image: ${ROLL_IMAGE_REPOSITORY}/varnish:${VARNISH_VERSION} depends_on: - nginx labels: @@ -18,3 +18,11 @@ services: HostRegexp(`^.+\.${TRAEFIK_DOMAIN}$$`) || Host(`${TRAEFIK_DOMAIN}`) - traefik.http.services.${ROLL_ENV_NAME}-varnish.loadbalancer.server.port=80 - traefik.docker.network=${ROLL_ENV_NAME}_default + healthcheck: + ## the varnish image ships neither curl nor bash, so probe the running varnishd directly + ## instead of the backend it fronts + test: ["CMD", "varnishadm", "ping"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s diff --git a/environments/laravel/init.env b/environments/laravel/init.env index 50e6690..dbdaf68 100644 --- a/environments/laravel/init.env +++ b/environments/laravel/init.env @@ -6,7 +6,7 @@ COMPOSER_VERSION=2 PHP_VERSION=8.3 PHP_XDEBUG_3=1 REDIS_VERSION=5.0 -MONGODB_VERSION=7 +MONGO_VERSION=7 ROLL_DB=1 ROLL_REDIS=1 @@ -40,3 +40,6 @@ MAIL_DRIVER=sendmail # New Relic monitoring (set to 1 to enable) ROLL_NEWRELIC=0 + +# Web server +NGINX_VERSION=1.27 diff --git a/environments/local/init.env b/environments/local/init.env new file mode 100644 index 0000000..3ca3405 --- /dev/null +++ b/environments/local/init.env @@ -0,0 +1,11 @@ +# The local environment type ships no compose fragment of its own - bring your own +# .roll/roll-env.yml. These pins reproduce the versions this type resolved to before service +# versions had to be declared explicitly; drop a service's toggle to 0 if you do not want it. +ROLL_NGINX=1 +ROLL_DB=1 +ROLL_REDIS=1 + +NGINX_VERSION=1.27 +DB_DISTRIBUTION=mariadb +DB_DISTRIBUTION_VERSION=10.4 +REDIS_VERSION=7.0 diff --git a/environments/local/local.base.yml b/environments/local/local.base.yml index e69de29..864db81 100644 --- a/environments/local/local.base.yml +++ b/environments/local/local.base.yml @@ -0,0 +1,3 @@ +# The local environment type is network-only. Bring your own .roll/roll-env.yml to define services. +# Service toggles default to 1, so set ROLL_NGINX, ROLL_DB, ROLL_REDIS to 0 unless you provide those +# services in your custom roll-env.yml. diff --git a/environments/magento1/init.env b/environments/magento1/init.env index 46c4ada..75f8ec0 100644 --- a/environments/magento1/init.env +++ b/environments/magento1/init.env @@ -20,3 +20,6 @@ ROLL_MAGENTO_STATIC_CACHING=1 # New Relic monitoring (set to 1 to enable) ROLL_NEWRELIC=0 + +# Web server +NGINX_VERSION=1.27 diff --git a/environments/magento2/init.env b/environments/magento2/init.env index 06b504a..d898851 100644 --- a/environments/magento2/init.env +++ b/environments/magento2/init.env @@ -38,4 +38,6 @@ ROLL_ADMIN_AUTOLOGIN=0 # NEWRELIC_LICENSE_KEY=your_license_key_here # New Relic monitoring (set to 1 to enable) -ROLL_NEWRELIC=0 \ No newline at end of file +ROLL_NEWRELIC=0 +# Web server +NGINX_VERSION=1.27 diff --git a/environments/magento2/magento2.magepack.base.yml b/environments/magento2/magento2.magepack.base.yml index ba5c026..927c52a 100644 --- a/environments/magento2/magento2.magepack.base.yml +++ b/environments/magento2/magento2.magepack.base.yml @@ -1,7 +1,7 @@ services: magepack: hostname: "${ROLL_ENV_NAME}-magepack" - image: ${ROLL_IMAGE_REPOSITORY}/magepack:${MAGEPACK_VERSION:-2.3} + image: ${ROLL_IMAGE_REPOSITORY}/magepack:${MAGEPACK_VERSION} environment: - TRAEFIK_DOMAIN - TRAEFIK_SUBDOMAIN diff --git a/environments/php/init.env b/environments/php/init.env index 75d2687..2b07f67 100644 --- a/environments/php/init.env +++ b/environments/php/init.env @@ -16,3 +16,6 @@ DRAGONFLY_VERSION=1.3 # New Relic monitoring (set to 1 to enable) ROLL_NEWRELIC=0 + +# Web server +NGINX_VERSION=1.27 diff --git a/environments/shopware/init.env b/environments/shopware/init.env index 5e04d7c..a9ac59c 100644 --- a/environments/shopware/init.env +++ b/environments/shopware/init.env @@ -24,3 +24,6 @@ VARNISH_VERSION=6.0 # New Relic monitoring (set to 1 to enable) ROLL_NEWRELIC=0 + +# Web server +NGINX_VERSION=1.27 diff --git a/environments/symfony/init.env b/environments/symfony/init.env index 5e04d7c..a9ac59c 100644 --- a/environments/symfony/init.env +++ b/environments/symfony/init.env @@ -24,3 +24,6 @@ VARNISH_VERSION=6.0 # New Relic monitoring (set to 1 to enable) ROLL_NEWRELIC=0 + +# Web server +NGINX_VERSION=1.27 diff --git a/environments/typo3/init.env b/environments/typo3/init.env index 97cac8a..9459653 100644 --- a/environments/typo3/init.env +++ b/environments/typo3/init.env @@ -17,3 +17,6 @@ DRAGONFLY_VERSION=1.3 # New Relic monitoring (set to 1 to enable) ROLL_NEWRELIC=0 + +# Web server +NGINX_VERSION=1.27 diff --git a/environments/vuejs/init.env b/environments/vuejs/init.env new file mode 100644 index 0000000..ea34a10 --- /dev/null +++ b/environments/vuejs/init.env @@ -0,0 +1,13 @@ +# Vue.js environment. These pins reproduce the versions this type resolved to before service +# versions had to be declared explicitly, so existing projects keep the images they already ran. +ROLL_DB=1 +ROLL_REDIS=1 + +PHP_VERSION=8.1 +NODE_VERSION=18 +COMPOSER_VERSION=2 + +NGINX_VERSION=1.27 +DB_DISTRIBUTION=mariadb +DB_DISTRIBUTION_VERSION=10.4 +REDIS_VERSION=7.0 diff --git a/environments/wordpress/init.env b/environments/wordpress/init.env index 56ebdc1..e64c003 100644 --- a/environments/wordpress/init.env +++ b/environments/wordpress/init.env @@ -25,3 +25,6 @@ DB_PASSWORD=wordpress # New Relic monitoring (set to 1 to enable) ROLL_NEWRELIC=0 + +# Web server +NGINX_VERSION=1.27 diff --git a/utils/backup.sh b/utils/backup.sh new file mode 100644 index 0000000..c776d13 --- /dev/null +++ b/utils/backup.sh @@ -0,0 +1,777 @@ +#!/usr/bin/env bash +[[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 + +## Primitives shared by backup, restore, restore-full and duplicate. +## +## These four commands grew as independent copies of one another: eleven of the sixteen functions +## restore and restore-full defined were byte-identical, and three of the four defined their own +## logMessage. This file is the single definition; the commands keep only what is genuinely their +## own. + +## The directory backups are staged in and looked for. +## +## Deliberately keyed on the working directory rather than ROLL_ENV_PATH. That means `roll backup` +## run from a project SUB-directory stages into /.roll/backups, and restore run from the same +## place looks there too, so the pair agrees. It is pre-existing behaviour and all four commands +## share it; changing it here would silently relocate where every existing backup is found, so it +## is preserved exactly and lives in one place so it cannot drift again. +## +## restore-full is the exception that proves the point: it has no configured environment at all +## (the config comes out of the backup), so it cds into its target directory first and this +## resolver then answers relative to that. +function resolveBackupBaseDir() { + echo "$(pwd)/.roll/backups" +} + +function logMessage() { + ## Each command in this family carries its own quiet flag; only one of them is ever set in a + ## given invocation, so honouring whichever is present keeps every existing call site working + ## without threading a parameter through hundreds of them. + if [[ ${BACKUP_QUIET:-0} -eq 1 || ${RESTORE_QUIET:-0} -eq 1 || ${DUPLICATE_QUIET:-0} -eq 1 ]]; then + return 0 + fi + + local level="$1" + shift + case "$level" in + INFO) info "$@" ;; + SUCCESS) success "$@" ;; + WARNING) warning "$@" ;; + ERROR) error "$@" ;; + VERBOSE) [[ ${BACKUP_VERBOSE:-0} -eq 1 || ${RESTORE_VERBOSE:-0} -eq 1 || ${DUPLICATE_VERBOSE:-0} -eq 1 ]] && info "[VERBOSE] $*" ;; + esac + + ## a case branch whose test is false would otherwise become this function's return value and + ## kill the CLI under set -e + return 0 +} + +function logVerbose() { + logMessage VERBOSE "$@" + return 0 +} + +function showProgress() { + [[ $PROGRESS -eq 0 ]] && return + local current=$1 + local total=$2 + local description="$3" + local percent=$((current * 100 / total)) + local bar_length=30 + local filled_length=$((percent * bar_length / 100)) + + printf "\r[" + printf "%*s" $filled_length | tr ' ' '=' + printf "%*s" $((bar_length - filled_length)) | tr ' ' '-' + printf "] %d%% %s" $percent "$description" + + # Always end with a newline for clean output + echo "" +} + +function detectEncryptedBackup() { + local backup_path="$1" + + # Check if backup contains .gpg files + if [[ -d "$backup_path" ]]; then + # Directory format - check for .gpg files + if find "$backup_path" -name "*.gpg" -type f | head -1 | grep -q .; then + return 0 # Encrypted + fi + else + # Archive format - check if archive contains .gpg files + local archive_file="$backup_path" + if [[ -f "$archive_file" ]]; then + # Determine decompression command + local decompress_cmd="cat" + case "$archive_file" in + *.tar.gz) decompress_cmd="gzip -dc" ;; + *.tar.xz) decompress_cmd="xz -dc" ;; + *.tar.lz4) decompress_cmd="lz4 -dc" ;; + esac + + # Check if archive contains .gpg files + if $decompress_cmd "$archive_file" | tar -tf - 2>/dev/null | grep -q "\.gpg$"; then + return 0 # Encrypted + fi + fi + fi + + return 1 # Not encrypted +} + +function findLatestBackup() { + local backup_dir="$(pwd)/.roll/backups" + + if [[ ! -d "$backup_dir" ]]; then + return 1 + fi + + # Look for timestamped directories first (new format) + local latest_dir=$(ls "$backup_dir" 2>/dev/null | grep '^[0-9]\{10\}$' | sort -n | tail -1) + if [[ -n "$latest_dir" ]]; then + echo "$latest_dir" + return 0 + fi + + # Look for compressed archives + local latest_archive=$(ls "$backup_dir"/backup_*_*.tar* 2>/dev/null | sort | tail -1) + if [[ -n "$latest_archive" ]]; then + # Extract timestamp from filename + local timestamp=$(basename "$latest_archive" | grep -o '[0-9]\{10\}') + echo "$timestamp" + return 0 + fi + + return 1 +} + +function extractBackupArchive() { + local backup_id="$1" + local backup_dir="$(pwd)/.roll/backups" + local extract_dir="$backup_dir/${backup_id}_extracted" + + logVerbose "Looking for backup archive with ID: $backup_id" + logVerbose "Backup directory: $backup_dir" + + # Check if already extracted + if [[ -d "$extract_dir" ]]; then + logVerbose "Found already extracted backup at: $extract_dir" + echo "$extract_dir" + return 0 + fi + + # Find the archive file + local archive_file="" + for ext in ".tar.gz" ".tar.xz" ".tar.lz4" ".tar"; do + local potential_file="$backup_dir/backup_${ROLL_ENV_NAME}_${backup_id}${ext}" + logVerbose "Checking for archive: $potential_file" + if [[ -f "$potential_file" ]]; then + archive_file="$potential_file" + break + fi + done + + # Also check for generic archive names + if [[ -z "$archive_file" ]]; then + logVerbose "Trying generic archive name pattern: *${backup_id}*.tar*" + archive_file=$(ls "$backup_dir"/*"$backup_id"*.tar* 2>/dev/null | head -1) + fi + + if [[ -z "$archive_file" ]]; then + logMessage ERROR "Backup archive not found for ID: $backup_id" + return 1 + fi + + logMessage INFO "Extracting backup archive: $(basename "$archive_file")" + logVerbose "Full archive path: $archive_file" + logVerbose "Extract destination: $extract_dir" + + mkdir -p "$extract_dir" + + # Use direct file reading for cross-platform compatibility + # BSD tar (macOS) doesn't reliably handle piped stdin with -C flag + # Both BSD and GNU tar auto-detect compression format with -xf + logVerbose "Extracting archive using direct file reading" + + if tar -xf "$archive_file" -C "$extract_dir" --strip-components=1; then + logVerbose "Successfully extracted archive to: $extract_dir" + echo "$extract_dir" + return 0 + else + logMessage ERROR "Failed to extract backup archive" + rm -rf "$extract_dir" + return 1 + fi +} + +function validateBackup() { + local backup_path="$1" + + if [[ $RESTORE_VERIFY -eq 0 ]]; then + return 0 + fi + + logMessage INFO "Validating backup integrity..." + + # Check if backup metadata exists + if [[ ! -f "$backup_path/metadata/backup.json" ]]; then + logMessage WARNING "Backup metadata not found, proceeding with legacy format" + return 0 + fi + + # Verify checksums if available + if [[ -f "$backup_path/metadata/checksums.sha256" ]]; then + if (cd "$backup_path" && sha256sum -c metadata/checksums.sha256 >/dev/null 2>&1); then + logMessage SUCCESS "Backup integrity verified" + return 0 + else + logMessage ERROR "Backup integrity check failed" + return 1 + fi + fi + + logMessage SUCCESS "Backup validation completed" + return 0 +} + +function getBackupMetadata() { + local backup_path="$1" + local metadata_file="$backup_path/metadata/backup.json" + + if [[ -f "$metadata_file" ]]; then + cat "$metadata_file" + else + # Return empty JSON for legacy backups + echo "{}" + fi +} + +function detectBackupServices() { + local backup_path="$1" + local services=() + + # Check for volume backups + if [[ -d "$backup_path/volumes" ]]; then + for volume_file in "$backup_path/volumes"/*; do + if [[ -f "$volume_file" ]]; then + local service_name=$(basename "$volume_file" | sed 's/\.tar.*//') + services+=("$service_name") + fi + done + else + # Legacy format detection + if [[ -f "$backup_path/db.tar.gz" ]]; then + services+=("db") + fi + if [[ -f "$backup_path/redis.tar.gz" ]]; then + services+=("redis") + fi + if [[ -f "$backup_path/es.tar.gz" ]]; then + services+=("elasticsearch") + fi + if [[ -f "$backup_path/os.tar.gz" ]]; then + services+=("opensearch") + fi + fi + + echo "${services[@]}" +} + +function getVolumeMapping() { + local service_name="$1" + + case "$service_name" in + db) + case "${DB_DISTRIBUTION:-mariadb}" in + mysql|mariadb) echo "${ROLL_ENV_NAME}_dbdata:mysql" ;; + postgres) echo "${ROLL_ENV_NAME}_dbdata:postgres" ;; + *) echo "${ROLL_ENV_NAME}_dbdata:mysql" ;; + esac + ;; + redis) echo "${ROLL_ENV_NAME}_redis:redis" ;; + dragonfly) echo "${ROLL_ENV_NAME}_dragonfly:dragonfly" ;; + elasticsearch) echo "${ROLL_ENV_NAME}_esdata:elasticsearch" ;; + opensearch) echo "${ROLL_ENV_NAME}_osdata:opensearch" ;; + mongodb) echo "${ROLL_ENV_NAME}_mongodb:mongodb" ;; + rabbitmq) echo "${ROLL_ENV_NAME}_rabbitmq:rabbitmq" ;; + varnish) echo "${ROLL_ENV_NAME}_varnish:varnish" ;; + *) echo "${ROLL_ENV_NAME}_${service_name}:generic" ;; + esac +} + +function restoreVolume() { + local service_name="$1" + local backup_path="$2" + local step="$3" + local total="$4" + + showProgress $step $total "Restoring $service_name volume" + + local volume_mapping=$(getVolumeMapping "$service_name") + IFS=':' read -r volume_name service_type <<< "$volume_mapping" + + logVerbose "Restoring service: $service_name" + logVerbose "Volume mapping: $volume_mapping" + logVerbose "Volume name: $volume_name, Service type: $service_type" + + # Determine backup file location (check for both encrypted and unencrypted) + local backup_file="" + local is_encrypted=false + + # Check for encrypted files first (.gpg extension) + if [[ -f "$backup_path/volumes/${service_name}.tar.gz.gpg" ]]; then + backup_file="$backup_path/volumes/${service_name}.tar.gz.gpg" + is_encrypted=true + elif [[ -f "$backup_path/volumes/${service_name}.tar.xz.gpg" ]]; then + backup_file="$backup_path/volumes/${service_name}.tar.xz.gpg" + is_encrypted=true + elif [[ -f "$backup_path/volumes/${service_name}.tar.lz4.gpg" ]]; then + backup_file="$backup_path/volumes/${service_name}.tar.lz4.gpg" + is_encrypted=true + elif [[ -f "$backup_path/volumes/${service_name}.tar.gpg" ]]; then + backup_file="$backup_path/volumes/${service_name}.tar.gpg" + is_encrypted=true + # Check for unencrypted files + elif [[ -f "$backup_path/volumes/${service_name}.tar.gz" ]]; then + backup_file="$backup_path/volumes/${service_name}.tar.gz" + elif [[ -f "$backup_path/volumes/${service_name}.tar.xz" ]]; then + backup_file="$backup_path/volumes/${service_name}.tar.xz" + elif [[ -f "$backup_path/volumes/${service_name}.tar.lz4" ]]; then + backup_file="$backup_path/volumes/${service_name}.tar.lz4" + elif [[ -f "$backup_path/volumes/${service_name}.tar" ]]; then + backup_file="$backup_path/volumes/${service_name}.tar" + elif [[ -f "$backup_path/${service_name}.tar.gz" ]]; then + # Legacy format + backup_file="$backup_path/${service_name}.tar.gz" + elif [[ "$service_name" == "elasticsearch" && -f "$backup_path/es.tar.gz" ]]; then + # Legacy format uses short names for the search engines + backup_file="$backup_path/es.tar.gz" + elif [[ "$service_name" == "opensearch" && -f "$backup_path/os.tar.gz" ]]; then + backup_file="$backup_path/os.tar.gz" + else + logMessage WARNING "Backup file not found for service: $service_name" + logVerbose "Searched in: $backup_path/volumes/ and $backup_path/" + return 0 + fi + + # A search-index volume is only mounted when the matching engine is enabled + # in this environment's .env.roll — warn instead of silently restoring into + # a volume that no service will ever use. + # + # Skipped entirely when there is no .env.roll yet: a full restore takes its configuration out + # of the backup, so at this point every toggle reads as unset and the warning would fire for + # every search volume regardless of what the restored environment will actually enable. + if [[ ! -f "${ROLL_ENV_PATH}/.env.roll" ]]; then + : + elif [[ "$service_name" == "elasticsearch" && "${ROLL_ELASTICSEARCH:-0}" != "1" ]]; then + logMessage WARNING "Backup contains an Elasticsearch index but ROLL_ELASTICSEARCH is not enabled - the restored volume will not be mounted by 'roll env up'" + elif [[ "$service_name" == "opensearch" && "${ROLL_OPENSEARCH:-0}" != "1" ]]; then + logMessage WARNING "Backup contains an OpenSearch index but ROLL_OPENSEARCH is not enabled - the restored volume will not be mounted by 'roll env up'" + fi + + logVerbose "Found backup file: $backup_file" + logVerbose "Encrypted: $is_encrypted" + + if [[ $RESTORE_DRY_RUN -eq 1 ]]; then + if [[ $is_encrypted == true ]]; then + logMessage INFO "[DRY RUN] Would decrypt and restore $service_name from $backup_file to volume $volume_name" + else + logMessage INFO "[DRY RUN] Would restore $service_name from $backup_file to volume $volume_name" + fi + return 0 + fi + + # Validate decryption password if file is encrypted + if [[ $is_encrypted == true ]]; then + if [[ -z "$RESTORE_DECRYPT" ]]; then + logMessage ERROR "Encrypted backup file found but no decryption password provided" + return 1 + fi + fi + + # Get Docker Compose version for proper labeling + local docker_compose_version=$(docker compose version 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+' | head -1) + local volume_base_name=$(echo "$volume_name" | sed "s/${ROLL_ENV_NAME}_//") + + # Remove existing volume if it exists + if docker volume inspect "$volume_name" >/dev/null 2>&1; then + if [[ $RESTORE_FORCE -eq 1 ]]; then + logMessage INFO "Removing existing volume: $volume_name" + logVerbose "Executing: docker volume rm $volume_name" + docker volume rm "$volume_name" >/dev/null 2>&1 + else + logMessage ERROR "Volume $volume_name already exists. Use --force to overwrite." + return 1 + fi + else + logVerbose "Volume $volume_name does not exist yet" + fi + + # Create new volume with proper labels + logVerbose "Creating volume: $volume_name with labels" + logVerbose " - com.docker.compose.project=$ROLL_ENV_NAME" + logVerbose " - com.docker.compose.version=$docker_compose_version" + logVerbose " - com.docker.compose.volume=$volume_base_name" + docker volume create "$volume_name" \ + --label com.docker.compose.project="$ROLL_ENV_NAME" \ + --label com.docker.compose.version="$docker_compose_version" \ + --label com.docker.compose.volume="$volume_base_name" >/dev/null 2>&1 + + # Restore the volume data with decryption if needed + local temp_container="${ROLL_ENV_NAME}_restore_${service_name}_$$" + + # `docker volume create` above leaves the volume root owned root:root, and the extraction runs + # as root with `--strip-components=1`, discarding the archive's top-level entry - the only one + # carrying the data directory's own ownership. Copy-up does not compensate: it fires only for an + # EMPTY volume, and this one is populated before any service mounts it. Containers that start as + # root chown their data directory themselves, but Elasticsearch and OpenSearch run as uid 1000 + # and then die at boot with `AccessDeniedException: .../data/.es_temp_file`. + # + # Owner comes from the restored content, not a per-service table: it is what that service's own + # volume held, so it survives an image changing uid and covers `getVolumeMapping`'s generic + # `*)` branch. No chmod - 0755 already grants the new owner rwx, and the archive's real mode is + # gone with its top-level entry anyway. Empty archive: no repair needed, copy-up still applies. + # The braces matter - a bare `a; b` list would let a failed extraction report success. + local fix_volume_root='{ root_ref=$(find /data -mindepth 1 -maxdepth 1 -print -quit); [ -z "$root_ref" ] || chown "$(stat -c %u:%g "$root_ref")" /data; }' + + if [[ $is_encrypted == true ]]; then + # Decrypt and decompress pipeline - use ubuntu and original tar approach with strip components + # Use passphrase-fd to avoid shell escaping issues with passwords + # Determine the correct tar command based on the backup file format + local tar_cmd="tar -xf -" + case "$backup_file" in + *.tar.gz.gpg) tar_cmd="tar -xzf -" ;; + *.tar.xz.gpg) tar_cmd="tar -xJf -" ;; + *.tar.lz4.gpg) tar_cmd="lz4 -d - | tar -xf -" ;; + esac + + if echo "$RESTORE_DECRYPT" | gpg --batch --yes --quiet --passphrase-fd 0 --decrypt "$backup_file" | docker run --rm --name "$temp_container" --mount source="$volume_name",target=/data -i ubuntu bash -c "cd /data && $tar_cmd --strip-components=1 && $fix_volume_root" 2>/dev/null; then + logMessage SUCCESS "Successfully restored and decrypted $service_name volume" + return 0 + else + logMessage ERROR "Failed to decrypt and restore $service_name volume" + return 1 + fi + else + # Regular restore without decryption - use ubuntu and original tar approach with strip components + # For compressed files, we need to handle decompression properly + case "$backup_file" in + *.tar.gz) + if docker run --rm --name "$temp_container" \ + --mount source="$volume_name",target=/data \ + -v "$(dirname "$backup_file")":/backup \ + ubuntu bash \ + -c "cd /data && tar -xzf /backup/$(basename "$backup_file") --strip-components=1 && $fix_volume_root" 2>/dev/null; then + + logMessage SUCCESS "Successfully restored $service_name volume" + return 0 + else + logMessage ERROR "Failed to restore $service_name volume" + return 1 + fi + ;; + *.tar.xz) + if docker run --rm --name "$temp_container" \ + --mount source="$volume_name",target=/data \ + -v "$(dirname "$backup_file")":/backup \ + ubuntu bash \ + -c "cd /data && tar -xJf /backup/$(basename "$backup_file") --strip-components=1 && $fix_volume_root" 2>/dev/null; then + + logMessage SUCCESS "Successfully restored $service_name volume" + return 0 + else + logMessage ERROR "Failed to restore $service_name volume" + return 1 + fi + ;; + *.tar.lz4) + if docker run --rm --name "$temp_container" \ + --mount source="$volume_name",target=/data \ + -v "$(dirname "$backup_file")":/backup \ + ubuntu bash \ + -c "cd /data && lz4 -d /backup/$(basename "$backup_file") - | tar -xf - --strip-components=1 && $fix_volume_root" 2>/dev/null; then + + logMessage SUCCESS "Successfully restored $service_name volume" + return 0 + else + logMessage ERROR "Failed to restore $service_name volume" + return 1 + fi + ;; + *.tar) + if docker run --rm --name "$temp_container" \ + --mount source="$volume_name",target=/data \ + -v "$(dirname "$backup_file")":/backup \ + ubuntu bash \ + -c "cd /data && tar -xf /backup/$(basename "$backup_file") --strip-components=1 && $fix_volume_root" 2>/dev/null; then + + logMessage SUCCESS "Successfully restored $service_name volume" + return 0 + else + logMessage ERROR "Failed to restore $service_name volume" + return 1 + fi + ;; + *) + logMessage ERROR "Unsupported backup file format: $backup_file" + return 1 + ;; + esac + fi +} + +function performLegacyMigration() { + if [[ $RESTORE_LEGACY_MIGRATION -eq 0 ]]; then + return 0 + fi + + local current_dir="$(pwd)" + + # Handle Warden to Roll migration + if [[ ! -f "$current_dir/.env.roll" ]]; then + if [[ -f "$current_dir/.env" ]]; then + logMessage INFO "Performing legacy Warden to Roll migration..." + + # Create backup of original .env + cp "$current_dir/.env" "$current_dir/.env.backup.$(date +%s)" + + # Convert WARDEN to ROLL + sed -i.warden 's/WARDEN/ROLL/g' "$current_dir/.env" + + # Migrate .warden directory to .roll + if [[ -d "$current_dir/.warden" ]]; then + mv "$current_dir/.warden" "$current_dir/.roll" + + if [[ -f "$current_dir/.roll/warden-env.yml" ]]; then + mv "$current_dir/.roll/warden-env.yml" "$current_dir/.roll/roll-env.yml" + sed -i.warden 's/WARDEN/ROLL/g;s/warden/roll/g' "$current_dir/.roll/roll-env.yml" + fi + fi + + # Translate the legacy negative-form key to the supported positive-form one, inverting + # the value: ROLL_NO_STATIC_CACHING is in no schema, while ROLL_MAGENTO_STATIC_CACHING=1 + # selects the production nginx template. Absent means schema default 0, so write nothing. + if grep -q '^[[:space:]]*ROLL_NO_STATIC_CACHING=' "$current_dir/.env"; then + local legacy_no_static_caching="" + legacy_no_static_caching="$(sed -n 's/^[[:space:]]*ROLL_NO_STATIC_CACHING=//p' "$current_dir/.env" | tail -n 1)" + legacy_no_static_caching="${legacy_no_static_caching//[[:space:]]/}" + local magento_static_caching=1 + if [[ "$legacy_no_static_caching" == "1" ]]; then + magento_static_caching=0 + fi + sed -i.legacy '/^[[:space:]]*ROLL_NO_STATIC_CACHING=/d' "$current_dir/.env" + echo "ROLL_MAGENTO_STATIC_CACHING=${magento_static_caching}" >> "$current_dir/.env" + fi + + # Move to .env.roll if it contains ROLL_ variables + if [[ -n "$(grep -r 'ROLL_' "$current_dir/.env")" ]]; then + mv "$current_dir/.env" "$current_dir/.env.roll" + fi + + logMessage SUCCESS "Legacy migration completed" + fi + fi +} + +function restoreConfigurations() { + local backup_path="$1" + local step="$2" + local total="$3" + + if [[ $RESTORE_CONFIG -eq 0 ]]; then + return 0 + fi + + showProgress $step $total "Restoring configuration files" + + local config_source_dir="$backup_path/config" + local current_dir="$(pwd)" + + # Legacy format support + if [[ ! -d "$config_source_dir" ]]; then + # Check for legacy files in backup root (both encrypted and unencrypted) + local legacy_files=("env.php" "auth.json") + for file in "${legacy_files[@]}"; do + local source_file="" + local is_encrypted=false + + # Check for encrypted version first + if [[ -f "$backup_path/${file}.gpg" ]]; then + source_file="$backup_path/${file}.gpg" + is_encrypted=true + elif [[ -f "$backup_path/$file" ]]; then + source_file="$backup_path/$file" + fi + + if [[ -n "$source_file" ]]; then + local target_path="" + case "$file" in + env.php) target_path="$current_dir/app/etc/env.php" ;; + auth.json) target_path="$current_dir/auth.json" ;; + esac + + if [[ -n "$target_path" ]]; then + if [[ $RESTORE_DRY_RUN -eq 1 ]]; then + if [[ $is_encrypted == true ]]; then + logMessage INFO "[DRY RUN] Would decrypt and restore $file to $target_path" + else + logMessage INFO "[DRY RUN] Would restore $file to $target_path" + fi + else + mkdir -p "$(dirname "$target_path")" + + if [[ $is_encrypted == true ]]; then + # Decrypt the file directly to target location + if [[ -n "$RESTORE_DECRYPT" ]]; then + if echo "$RESTORE_DECRYPT" | gpg --batch --yes --quiet --passphrase-fd 0 --decrypt "$source_file" > "$target_path"; then + logMessage INFO "Decrypted and restored $file" + else + logMessage ERROR "Failed to decrypt $file" + return 1 + fi + else + logMessage ERROR "Encrypted config file found but no decryption password provided" + return 1 + fi + else + cp "$source_file" "$target_path" + logMessage INFO "Restored $file" + fi + fi + fi + fi + done + return 0 + fi + + # New format with structured config directory + if [[ $RESTORE_DRY_RUN -eq 1 ]]; then + logMessage INFO "[DRY RUN] Would restore configuration files from $config_source_dir" + return 0 + fi + + # Restore configuration files (both encrypted and unencrypted) + if [[ -d "$config_source_dir" ]]; then + # Process all files including .gpg files + find "$config_source_dir" -type f | while read -r config_file; do + local relative_path="${config_file#$config_source_dir/}" + local is_encrypted=false + + # Check if file is encrypted + if [[ "$config_file" == *.gpg ]]; then + is_encrypted=true + # Remove .gpg extension for target path + relative_path="${relative_path%.gpg}" + fi + + local target_path="$current_dir/$relative_path" + + # Create target directory if needed + mkdir -p "$(dirname "$target_path")" + + # Backup existing file if it exists + if [[ -f "$target_path" ]]; then + if [[ $is_encrypted == true ]]; then + # For encrypted files, we can't easily compare so always backup + cp "$target_path" "$target_path.backup.$(date +%s)" + logMessage INFO "Backed up existing $relative_path" + elif ! cmp -s "$config_file" "$target_path"; then + cp "$target_path" "$target_path.backup.$(date +%s)" + logMessage INFO "Backed up existing $relative_path" + fi + fi + + if [[ $is_encrypted == true ]]; then + # Decrypt the file + if [[ -n "$RESTORE_DECRYPT" ]]; then + if echo "$RESTORE_DECRYPT" | gpg --batch --yes --quiet --passphrase-fd 0 --decrypt "$config_file" > "$target_path"; then + logMessage INFO "Decrypted and restored $relative_path" + else + logMessage ERROR "Failed to decrypt $relative_path" + return 1 + fi + else + logMessage ERROR "Encrypted config file found but no decryption password provided" + return 1 + fi + else + # Copy unencrypted file + cp "$config_file" "$target_path" + logMessage INFO "Restored $relative_path" + fi + done + fi + + logMessage SUCCESS "Configuration restore completed" +} + +function signEnvironmentCertificate() { + # Determine the domain to sign certificate for + local domain="${TRAEFIK_DOMAIN:-}" + + # If TRAEFIK_DOMAIN not set, try to construct from env name + if [[ -z "$domain" ]]; then + domain="${ROLL_ENV_NAME}.test" + fi + + if [[ -z "$domain" ]]; then + logVerbose "No domain found for certificate signing, skipping" + return 0 + fi + + # Check if root CA exists + if [[ ! -f "${ROLL_SSL_DIR}/rootca/certs/ca.cert.pem" ]]; then + logMessage WARNING "Root CA not found. Run 'roll install' first to enable SSL certificates." + return 0 + fi + + logMessage INFO "Signing SSL certificates for ${domain}..." + + # Sign certificate for the main domain (includes domain and *.domain) + logVerbose "Signing certificate for: ${domain}" + if [[ $RESTORE_VERBOSE -eq 1 ]]; then + "${ROLL_DIR}/bin/roll" sign-certificate "$domain" || { + logMessage WARNING "Failed to sign certificate for ${domain}" + } + else + "${ROLL_DIR}/bin/roll" sign-certificate "$domain" >/dev/null 2>&1 || { + logMessage WARNING "Failed to sign certificate for ${domain}" + } + fi + + # Sign separate certificate for wildcard domain + local wildcard_domain="*.${domain}" + logVerbose "Signing certificate for: ${wildcard_domain}" + if [[ $RESTORE_VERBOSE -eq 1 ]]; then + "${ROLL_DIR}/bin/roll" sign-certificate "$wildcard_domain" || { + logMessage WARNING "Failed to sign certificate for ${wildcard_domain}" + } + else + "${ROLL_DIR}/bin/roll" sign-certificate "$wildcard_domain" >/dev/null 2>&1 || { + logMessage WARNING "Failed to sign certificate for ${wildcard_domain}" + } + fi + + # Regenerate traefik dynamic config and restart to pick up new certificates + logVerbose "Regenerating traefik configuration..." + if [[ $RESTORE_VERBOSE -eq 1 ]]; then + "${ROLL_DIR}/bin/roll" svc up traefik || { + logMessage WARNING "Failed to restart traefik" + } + else + "${ROLL_DIR}/bin/roll" svc up traefik >/dev/null 2>&1 || { + logMessage WARNING "Failed to restart traefik" + } + fi + + logMessage SUCCESS "SSL certificates signed for ${domain}" +} + + +## Bring the environment down before restoring into it. +## +## Restore and restore-full disagreed here: a full restore has no .env.roll yet, because the +## configuration is inside the backup, and the env commands require one. The guard covers both, +## so the two callers no longer need separate copies. +function stopEnvironment() { + logVerbose "Environment path: ${ROLL_ENV_PATH}" + + if [[ ! -f "${ROLL_ENV_PATH}/.env.roll" ]]; then + logMessage INFO "No environment configuration found yet, skipping stop" + return 0 + fi + + logVerbose "Checking for running containers with project: ${ROLL_ENV_NAME}" + local running_containers + running_containers="$(docker ps --filter "label=com.docker.compose.project=${ROLL_ENV_NAME}" -q)" || true + + if [[ -n "${running_containers}" ]]; then + logMessage INFO "Stopping environment..." + "${ROLL_DIR}/bin/roll" env down || true + else + logVerbose "No running containers for this environment" + fi + + return 0 +} diff --git a/utils/config.sh b/utils/config.sh index 3b9f4a3..9a067b4 100644 --- a/utils/config.sh +++ b/utils/config.sh @@ -14,6 +14,13 @@ ROLL_CONFIG_LOADED_FILES=() ROLL_CONFIG_SCHEMA_KEYS=() ROLL_CONFIG_SCHEMA_VALUES=() +# Which config file each key came from. Global config (~/.roll/.env{,.roll}) is loaded into the same +# cache as the project's .env.roll, so the resolved value alone cannot answer "did THIS project pin +# this?" - the version-pin checks below need the origin, not the value. +ROLL_GLOBAL_CONFIG_KEYS=() +ROLL_GLOBAL_CONFIG_VALUES=() +ROLL_PROJECT_CONFIG_KEYS=() + ## Helper function to find index of key in array function findConfigIndex() { local key="$1" @@ -79,6 +86,7 @@ function initConfigSchema() { ROLL_CONFIG_SCHEMA_KEYS+=(ROLL_RABBITMQ); ROLL_CONFIG_SCHEMA_VALUES+=("boolean:0") ROLL_CONFIG_SCHEMA_KEYS+=(ROLL_MONGODB); ROLL_CONFIG_SCHEMA_VALUES+=("boolean:0") ROLL_CONFIG_SCHEMA_KEYS+=(ROLL_BROWSERSYNC); ROLL_CONFIG_SCHEMA_VALUES+=("boolean:0") + ROLL_CONFIG_SCHEMA_KEYS+=(ROLL_PUBLISH_PORTS); ROLL_CONFIG_SCHEMA_VALUES+=("boolean:1") ROLL_CONFIG_SCHEMA_KEYS+=(ROLL_SELENIUM); ROLL_CONFIG_SCHEMA_VALUES+=("boolean:0") ROLL_CONFIG_SCHEMA_KEYS+=(ROLL_SELENIUM_DEBUG); ROLL_CONFIG_SCHEMA_VALUES+=("boolean:0") ROLL_CONFIG_SCHEMA_KEYS+=(ROLL_TEST_DB); ROLL_CONFIG_SCHEMA_VALUES+=("boolean:0") @@ -92,7 +100,7 @@ function initConfigSchema() { ROLL_CONFIG_SCHEMA_KEYS+=(TRAEFIK_LISTEN); ROLL_CONFIG_SCHEMA_VALUES+=("string:127.0.0.1") # PHP configuration - ROLL_CONFIG_SCHEMA_KEYS+=(PHP_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:8.1") + ROLL_CONFIG_SCHEMA_KEYS+=(PHP_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") ROLL_CONFIG_SCHEMA_KEYS+=(PHP_XDEBUG_3); ROLL_CONFIG_SCHEMA_VALUES+=("boolean:1") ROLL_CONFIG_SCHEMA_KEYS+=(PHP_MEMORY_LIMIT); ROLL_CONFIG_SCHEMA_VALUES+=("string:2G") @@ -101,24 +109,30 @@ function initConfigSchema() { # Database configuration ROLL_CONFIG_SCHEMA_KEYS+=(DB_DISTRIBUTION); ROLL_CONFIG_SCHEMA_VALUES+=("string:mariadb") - ROLL_CONFIG_SCHEMA_KEYS+=(DB_DISTRIBUTION_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:10.4") - ROLL_CONFIG_SCHEMA_KEYS+=(MYSQL_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:8.0") - ROLL_CONFIG_SCHEMA_KEYS+=(MARIADB_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:10.4") + ROLL_CONFIG_SCHEMA_KEYS+=(DB_DISTRIBUTION_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") + ROLL_CONFIG_SCHEMA_KEYS+=(MYSQL_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") + ROLL_CONFIG_SCHEMA_KEYS+=(MARIADB_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") # Service version configurations - ROLL_CONFIG_SCHEMA_KEYS+=(ELASTICSEARCH_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:7.17") - ROLL_CONFIG_SCHEMA_KEYS+=(RABBITMQ_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:3.11") - ROLL_CONFIG_SCHEMA_KEYS+=(REDIS_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:7.0") - ROLL_CONFIG_SCHEMA_KEYS+=(DRAGONFLY_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:latest") - ROLL_CONFIG_SCHEMA_KEYS+=(VARNISH_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:7.0") - ROLL_CONFIG_SCHEMA_KEYS+=(OPENSEARCH_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:2.5") - ROLL_CONFIG_SCHEMA_KEYS+=(MONGO_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:6.0") - ROLL_CONFIG_SCHEMA_KEYS+=(NGINX_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:1.27") - ROLL_CONFIG_SCHEMA_KEYS+=(MAGEPACK_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:2.3") - ROLL_CONFIG_SCHEMA_KEYS+=(ROLL_SELENIUM_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:3.141.59") + ROLL_CONFIG_SCHEMA_KEYS+=(ELASTICSEARCH_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") + ROLL_CONFIG_SCHEMA_KEYS+=(ELASTICSEARCH_JAVA_OPTS); ROLL_CONFIG_SCHEMA_VALUES+=("string:-Xms64m -Xmx512m") + ROLL_CONFIG_SCHEMA_KEYS+=(RABBITMQ_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") + ROLL_CONFIG_SCHEMA_KEYS+=(REDIS_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") + ROLL_CONFIG_SCHEMA_KEYS+=(DRAGONFLY_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") + ROLL_CONFIG_SCHEMA_KEYS+=(VARNISH_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") + ROLL_CONFIG_SCHEMA_KEYS+=(OPENSEARCH_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") + ROLL_CONFIG_SCHEMA_KEYS+=(OPENSEARCH_JAVA_OPTS); ROLL_CONFIG_SCHEMA_VALUES+=("string:-Xms64m -Xmx512m") + ROLL_CONFIG_SCHEMA_KEYS+=(MONGO_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") + ROLL_CONFIG_SCHEMA_KEYS+=(NGINX_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") + ROLL_CONFIG_SCHEMA_KEYS+=(MAGEPACK_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") + ROLL_CONFIG_SCHEMA_KEYS+=(ROLL_SELENIUM_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") # Node configuration - ROLL_CONFIG_SCHEMA_KEYS+=(NODE_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:18") + ROLL_CONFIG_SCHEMA_KEYS+=(NODE_VERSION); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") + # Both are read by `roll theme`. Optional rather than defaulted on purpose: an unset + # ROLL_YARN_INSTEAD_OF_GULP means "decide per theme", which a 0/1 default would destroy. + ROLL_CONFIG_SCHEMA_KEYS+=(ROLL_NODE_PACKAGE_MANAGER); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") + ROLL_CONFIG_SCHEMA_KEYS+=(ROLL_YARN_INSTEAD_OF_GULP); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") # Nginx configuration ROLL_CONFIG_SCHEMA_KEYS+=(NGINX_TEMPLATE); ROLL_CONFIG_SCHEMA_VALUES+=("string:optional") @@ -196,6 +210,9 @@ function validateConfigValue() { error "Configuration $key is required but empty" return 1 fi + if [[ "$key" == "ROLL_ENV_NAME" && -n "$value" && ! "$value" =~ ^[a-z0-9][a-z0-9_-]*$ ]]; then + fatal "ROLL_ENV_NAME=\"$value\" is invalid: it becomes the Docker Compose project name, which must match ^[a-z0-9][a-z0-9_-]*\$ (lowercase alphanumeric characters, hyphens, and underscores, starting with a letter or number). Note that the env name also prefixes every named volume, so renaming an existing environment orphans its old volumes and it comes up on an empty database." + fi ;; integer) if ! [[ "$value" =~ ^[0-9]+$ ]]; then @@ -235,6 +252,7 @@ function setConfigDefault() { function loadConfigFromFile() { local config_file="$1" local validate_only="${2:-false}" + local source_label="${3:-}" if [[ ! -f "$config_file" ]]; then error "Configuration file not found: $config_file" @@ -287,6 +305,16 @@ function loadConfigFromFile() { ROLL_CONFIG_CACHE_VALUES+=("$value") fi export "$key"="$value" + + case "$source_label" in + global) + ROLL_GLOBAL_CONFIG_KEYS+=("$key") + ROLL_GLOBAL_CONFIG_VALUES+=("$value") + ;; + project) + ROLL_PROJECT_CONFIG_KEYS+=("$key") + ;; + esac fi elif [[ "$line" =~ ^[[:space:]]*[^=]+$ ]]; then @@ -323,12 +351,16 @@ function loadRollConfig() { # Initialize schema if not done initConfigSchema + ROLL_GLOBAL_CONFIG_KEYS=() + ROLL_GLOBAL_CONFIG_VALUES=() + ROLL_PROJECT_CONFIG_KEYS=() + # Load global configuration first from ROLL_HOME_DIR local global_config_loaded=0 # Check for new-style global config file if [[ -f "${ROLL_HOME_DIR}/.env.roll" ]]; then - if loadConfigFromFile "${ROLL_HOME_DIR}/.env.roll"; then + if loadConfigFromFile "${ROLL_HOME_DIR}/.env.roll" "false" "global"; then global_config_loaded=1 else warning "Failed to load global configuration from ${ROLL_HOME_DIR}/.env.roll" @@ -337,7 +369,7 @@ function loadRollConfig() { # Check for legacy global config file if [[ -f "${ROLL_HOME_DIR}/.env" ]]; then - if loadConfigFromFile "${ROLL_HOME_DIR}/.env"; then + if loadConfigFromFile "${ROLL_HOME_DIR}/.env" "false" "global"; then global_config_loaded=1 else warning "Failed to load global configuration from ${ROLL_HOME_DIR}/.env" @@ -345,7 +377,7 @@ function loadRollConfig() { fi # Load project-specific configuration (this will override global settings) - if ! loadConfigFromFile "$config_file"; then + if ! loadConfigFromFile "$config_file" "false" "project"; then return 1 fi @@ -373,19 +405,26 @@ function loadRollConfig() { export GROUP_ID="$(id -g)" export OSTYPE="${OSTYPE}" - # Set defaults for unset values + # Validate environment type before anything derives from it + if ! assertValidEnvType; then + return 1 + fi + + # Environment-type defaults first: they must see the variables still unset, so they can fill in + # what this type needs without overriding what the project explicitly configured. + applyEnvTypeDefaults + + # Schema literals fill whatever nothing else has set by now local i=0 while [[ $i -lt ${#ROLL_CONFIG_SCHEMA_KEYS[@]} ]]; do setConfigDefault "${ROLL_CONFIG_SCHEMA_KEYS[$i]}" i=$((i + 1)) done - # Validate environment type - if ! assertValidEnvType; then - return 1 - fi + # Warn about unpinned versions of enabled services and fall back to the recommended value + applyVersionPinFallbacks - # Post-processing for specific configurations + # Derived values last, once every input has its final value postProcessConfig return 0 @@ -408,6 +447,230 @@ function setConfigValue() { export "$key"="$value" } +## Apply an environment-type default. Unlike setConfigDefault, which fills in the schema literal, +## this expresses "what this environment type needs unless the project said otherwise", so it must +## run BEFORE the schema loop and must never override a value the user actually wrote in +## .env.roll. A key already in the cache came from a config file, so it is left alone. +function setConfigDerived() { + local key="$1" + local value="$2" + + if [[ $(findConfigIndex "$key") -ge 0 ]]; then + return 0 + fi + + setConfigValue "$key" "$value" +} + +## Environment-type service defaults. These used to live at the bottom of postProcessConfig, where +## they could never take effect: the schema loop had already set every toggle to its literal, so +## the ${VAR:-1} fallbacks only ever saw a value that was present but zero. Running them here, on +## the still-unset variables, is what makes a magento2 project bring up Varnish, a search engine +## and RabbitMQ without having to spell out the toggles. +function applyEnvTypeDefaults() { + if [[ "${ROLL_ENV_TYPE}" != "local" ]]; then + setConfigDerived ROLL_NGINX 1 + setConfigDerived ROLL_DB 1 + setConfigDerived ROLL_REDIS 1 + fi + + if [[ "${ROLL_ENV_TYPE}" == "magento2" ]]; then + setConfigDerived ROLL_VARNISH 1 + setConfigDerived ROLL_ELASTICSEARCH 1 + setConfigDerived ROLL_RABBITMQ 1 + fi + + ## DB_DISTRIBUTION_VERSION is the value the image tag is built from; MYSQL_VERSION and + ## MARIADB_VERSION are the older per-distribution spellings. Derive from whichever matches the + ## selected distribution, and only when that one was actually provided. + if [[ $(findConfigIndex DB_DISTRIBUTION_VERSION) -lt 0 ]]; then + if [[ "${DB_DISTRIBUTION}" == "mysql" ]]; then + [[ -n "${MYSQL_VERSION}" ]] && setConfigDerived DB_DISTRIBUTION_VERSION "${MYSQL_VERSION}" + else + [[ -n "${MARIADB_VERSION}" ]] && setConfigDerived DB_DISTRIBUTION_VERSION "${MARIADB_VERSION}" + fi + fi + + return 0 +} + +## The version a project is currently running for an unpinned key: the literal that used to be +## this key's schema default. It is deliberately NOT the environment type's init.env value - a +## project that never pinned the key has been running the schema default all along, so that is what +## must be written into .env.roll. Recommending init.env here would jump, for example, an existing +## magento2 project from Elasticsearch 7.17 to 8.11 on upgrade, which is precisely the silent +## version change this whole mechanism exists to prevent. New projects get init.env copied by +## env-init, so they never reach this path. +function getLegacyDefaultVersion() { + local key="$1" + + case "$key" in + PHP_VERSION) echo "8.1" ;; + DB_DISTRIBUTION_VERSION) echo "10.4" ;; + MYSQL_VERSION) echo "8.0" ;; + MARIADB_VERSION) echo "10.4" ;; + ELASTICSEARCH_VERSION) echo "7.17" ;; + RABBITMQ_VERSION) echo "3.11" ;; + REDIS_VERSION) echo "7.0" ;; + DRAGONFLY_VERSION) echo "latest" ;; + VARNISH_VERSION) echo "7.0" ;; + OPENSEARCH_VERSION) echo "2.5" ;; + # the mongodb fragment interpolated MONGODB_VERSION, a key the schema never defined, so + # its own ":-7" literal is what actually ran - not the schema's unused 6.0 + MONGO_VERSION) echo "7" ;; + NGINX_VERSION) echo "1.27" ;; + MAGEPACK_VERSION) echo "2.3" ;; + ROLL_SELENIUM_VERSION) echo "3.141.59" ;; + NODE_VERSION) echo "18" ;; + *) echo "" ;; + esac +} + +## The older per-distribution spelling that applyEnvTypeDefaults turns into DB_DISTRIBUTION_VERSION. +function dbDistributionVersionKey() { + if [[ "${DB_DISTRIBUTION}" == "mysql" ]]; then + echo "MYSQL_VERSION" + else + echo "MARIADB_VERSION" + fi +} + +## Look up a key's value in the global config files, if it was set there at all. +function getGlobalConfigValue() { + local key="$1" + local i=0 + + while [[ $i -lt ${#ROLL_GLOBAL_CONFIG_KEYS[@]} ]]; do + if [[ "${ROLL_GLOBAL_CONFIG_KEYS[$i]}" == "${key}" ]]; then + echo "${ROLL_GLOBAL_CONFIG_VALUES[$i]}" + return 0 + fi + i=$((i + 1)) + done + + echo "" + return 0 +} + +## The version an unpinned key is running RIGHT NOW, which is what has to be written into +## .env.roll. A value inherited from global config wins over the legacy literal: pinning the +## literal instead would change the image a project running on a global override already uses. +## DB_DISTRIBUTION_VERSION additionally accepts the older per-distribution spellings, because +## applyEnvTypeDefaults derives it from those and that has not happened yet when pins are collected. +function getRunningVersion() { + local key="$1" + local inherited="" + + inherited="$(getGlobalConfigValue "${key}")" + + if [[ -z "${inherited}" && "${key}" == "DB_DISTRIBUTION_VERSION" ]]; then + inherited="$(getGlobalConfigValue "$(dbDistributionVersionKey)")" + fi + + if [[ -n "${inherited}" ]]; then + echo "${inherited}" + return 0 + fi + + getLegacyDefaultVersion "${key}" +} + +## Whether the project's own config file pins this key. DB_DISTRIBUTION_VERSION also counts as +## pinned when the project uses one of the older per-distribution spellings, which +## applyEnvTypeDefaults turns into it. +function isVersionPinned() { + local key="$1" + + if containsElement "${key}" "${ROLL_PROJECT_CONFIG_KEYS[@]}"; then + return 0 + fi + + if [[ "${key}" == "DB_DISTRIBUTION_VERSION" ]] && containsElement "$(dbDistributionVersionKey)" "${ROLL_PROJECT_CONFIG_KEYS[@]}"; then + return 0 + fi + + return 1 +} + +## Every enabled service should have its version pinned in .env.roll. Previously an omitted pin +## silently resolved to a schema literal, so a project could change PHP or database version just by +## upgrading roll. +## +## In 0.8.0 this warns and falls back to the recommended value, so existing projects keep running +## the images they already ran. In 0.9.0 the fallback goes away and a missing pin becomes fatal. +## `roll config fix-pins` writes the recommended lines into .env.roll. +function collectMissingVersionPins() { + ROLL_MISSING_PINS=() + + ## A pin is judged on the project's own .env.roll, never on the resolved value: global config is + ## loaded into the same cache first, so a version in ~/.roll/.env would report as pinned while + ## the project file stays empty and a teammate without that global resolves a different image. + ## env.cmd never appends the php-fpm partial for the local type, so it needs no PHP or Node + if [[ "${ROLL_ENV_TYPE}" != "local" ]]; then + isVersionPinned PHP_VERSION || ROLL_MISSING_PINS+=(PHP_VERSION) + isVersionPinned NODE_VERSION || ROLL_MISSING_PINS+=(NODE_VERSION) + fi + + ## toggle:version pairs; the version is required only when its service is switched on + local requirements=( + "ROLL_NGINX:NGINX_VERSION" + "ROLL_DB:DB_DISTRIBUTION_VERSION" + "ROLL_REDIS:REDIS_VERSION" + "ROLL_DRAGONFLY:DRAGONFLY_VERSION" + "ROLL_VARNISH:VARNISH_VERSION" + "ROLL_ELASTICSEARCH:ELASTICSEARCH_VERSION" + "ROLL_OPENSEARCH:OPENSEARCH_VERSION" + "ROLL_RABBITMQ:RABBITMQ_VERSION" + "ROLL_MONGODB:MONGO_VERSION" + "ROLL_MAGEPACK:MAGEPACK_VERSION" + "ROLL_SELENIUM:ROLL_SELENIUM_VERSION" + ) + + local i=0 + local toggle="" version_key="" toggle_value="" + while [[ $i -lt ${#requirements[@]} ]]; do + toggle="${requirements[$i]%%:*}" + version_key="${requirements[$i]##*:}" + eval "toggle_value=\${${toggle}:-0}" + + if [[ "${toggle_value}" == "1" ]] && ! isVersionPinned "${version_key}"; then + ROLL_MISSING_PINS+=("${version_key}") + fi + i=$((i + 1)) + done + + return 0 +} + +## Warn about unpinned versions and fall back to the recommended value so the environment still +## comes up exactly as it did before. Becomes a hard error in 0.9.0. +function applyVersionPinFallbacks() { + collectMissingVersionPins + + if (( ${#ROLL_MISSING_PINS[@]} == 0 )); then + return 0 + fi + + local i=0 + local key="" value="" + warning "This ${ROLL_ENV_TYPE} environment enables services whose versions are not pinned." + warning "Roll is falling back to a built-in version, which means upgrading roll can silently" + warning "change which image this project runs. From 0.9.0 this will be an error." + warning "These are the versions the project is running right now - run \`roll config fix-pins\`" + warning "to write them into .env.roll, or add them by hand:" + >&2 echo "" + while [[ $i -lt ${#ROLL_MISSING_PINS[@]} ]]; do + key="${ROLL_MISSING_PINS[$i]}" + value="$(getRunningVersion "${key}")" + >&2 echo " ${key}=${value}" + setConfigValue "${key}" "${value}" + i=$((i + 1)) + done + >&2 echo "" + + return 0 +} + ## Post-process configuration after loading function postProcessConfig() { # Set PHP variant based on environment type @@ -420,15 +683,6 @@ function postProcessConfig() { export ROLL_SVC_PHP_NODE="-node${NODE_VERSION}" fi - # Database distribution defaults - if [[ -z "${DB_DISTRIBUTION_VERSION}" ]]; then - if [[ "${DB_DISTRIBUTION}" == "mysql" ]]; then - export DB_DISTRIBUTION_VERSION="${MYSQL_VERSION:-8.0}" - else - export DB_DISTRIBUTION_VERSION="${MARIADB_VERSION:-10.4}" - fi - fi - # XDebug version configuration if [[ "${PHP_XDEBUG_3}" == "1" ]]; then export XDEBUG_VERSION="xdebug3" @@ -446,14 +700,12 @@ function postProcessConfig() { export SSH_AUTH_SOCK_PATH_ENV="/run/host-services/ssh-auth.sock" fi - # Environment-specific defaults + # Bash history and SSH directories. Always exported, even when empty: the compose fragments + # interpolate it unconditionally, and an unset variable makes docker compose warn. if [[ "${ROLL_ENV_TYPE}" != "local" ]]; then - export ROLL_NGINX="${ROLL_NGINX:-1}" - export ROLL_DB="${ROLL_DB:-1}" - export ROLL_REDIS="${ROLL_REDIS:-1}" - - # Bash history and SSH directories export CHOWN_DIR_LIST="/bash_history /home/www-data/.ssh ${ROLL_CHOWN_DIR_LIST:-}" + else + export CHOWN_DIR_LIST="" fi # Magento 1 specific configuration @@ -471,10 +723,6 @@ function postProcessConfig() { # Magento 2 specific configuration if [[ "${ROLL_ENV_TYPE}" == "magento2" ]]; then - export ROLL_VARNISH="${ROLL_VARNISH:-1}" - export ROLL_ELASTICSEARCH="${ROLL_ELASTICSEARCH:-1}" - export ROLL_RABBITMQ="${ROLL_RABBITMQ:-1}" - if [[ "${ROLL_MAGENTO_STATIC_CACHING}" == "1" ]]; then if [[ "${ROLL_ADMIN_AUTOLOGIN}" == "1" ]]; then export NGINX_TEMPLATE="${NGINX_TEMPLATE:-magento2-autologin.conf}" @@ -489,6 +737,10 @@ function postProcessConfig() { fi fi fi + # env.cmd used to export these unconditionally; the compose fragments still interpolate them + # for every environment type, so keep them defined even when nothing derived a value. + export NGINX_TEMPLATE="${NGINX_TEMPLATE:-}" + export NGINX_PUBLIC="${NGINX_PUBLIC:-}" } ## Validate configuration file without loading diff --git a/utils/core.sh b/utils/core.sh index 51136c2..234b478 100644 --- a/utils/core.sh +++ b/utils/core.sh @@ -23,42 +23,16 @@ function error { function fatal { error "$@" - exit -1 + exit 1 } -function boxinfo() { - local s=("$@") b w use_tput=0 - - # only enable colors if we have a tty and TERM is set (and not "dumb") - if [[ -t 1 && -n "${TERM:-}" && "${TERM:-}" != "dumb" ]] && command -v tput >/dev/null 2>&1; then - use_tput=1 - fi - - for l in "${s[@]}"; do - ((w < ${#l})) && { b="$l"; w="${#l}"; } - done - - ((use_tput)) && tput setaf 3 - echo " -${b//?/-}- - | ${b//?/ } |" - - for l in "${s[@]}"; do - if ((use_tput)); then - printf '| %s%*s%s |\n' "$(tput setaf 7)" "-$w" "$l" "$(tput setaf 3)" - else - printf '| %*s |\n' "-$w" "$l" - fi - done - - echo "| ${b//?/ } | - -${b//?/-}-" - ((use_tput)) && tput sgr 0 - - return 0 - } - -function boxsuccess() { - local s=("$@") b w use_tput=0 +## draw a box around the given lines, with the text rendered in the given tput setaf color +function box() { + local color="$1"; shift + ## b and w must be initialised, not merely declared: bash 4.4 and later apply `set -u` inside + ## arithmetic contexts, so the `((w < ${#l}))` below aborts on an unset w. bash 3.2 treats it as + ## 0, which is why this only ever failed on Linux. + local s=("$@") b="" w=0 use_tput=0 # only enable colors if we have a tty and TERM is set (and not "dumb") if [[ -t 1 && -n "${TERM:-}" && "${TERM:-}" != "dumb" ]] && command -v tput >/dev/null 2>&1; then @@ -75,7 +49,7 @@ function boxsuccess() { for l in "${s[@]}"; do if ((use_tput)); then - printf '| %s%*s%s |\n' "$(tput setaf 2)" "-$w" "$l" "$(tput setaf 3)" + printf '| %s%*s%s |\n' "$(tput setaf "$color")" "-$w" "$l" "$(tput setaf 3)" else printf '| %*s |\n' "-$w" "$l" fi @@ -88,41 +62,28 @@ function boxsuccess() { return 0 } -function boxerror() { - local s=("$@") b w use_tput=0 - - # only enable colors if we have a tty and TERM is set (and not "dumb") - if [[ -t 1 && -n "${TERM:-}" && "${TERM:-}" != "dumb" ]] && command -v tput >/dev/null 2>&1; then - use_tput=1 - fi - - for l in "${s[@]}"; do - ((w < ${#l})) && { b="$l"; w="${#l}"; } - done - - ((use_tput)) && tput setaf 3 - echo " -${b//?/-}- -| ${b//?/ } |" - - for l in "${s[@]}"; do - if ((use_tput)); then - printf '| %s%*s%s |\n' "$(tput setaf 1)" "-$w" "$l" "$(tput setaf 3)" - else - printf '| %*s |\n' "-$w" "$l" - fi - done +function boxinfo() { + box 7 "$@" +} - echo "| ${b//?/ } | - -${b//?/-}-" - ((use_tput)) && tput sgr 0 +function boxsuccess() { + box 2 "$@" +} - return 0 +function boxerror() { + box 1 "$@" } function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } +## capitalize the first letter of a string; bash 3.2 compatible +function capitalize() { + local s="$1" + printf '%s%s' "$(printf '%s' "${s:0:1}" | tr '[:lower:]' '[:upper:]')" "${s:1}" +} + ## determines if value is present in an array; returns 0 if element is present ## in array, otherwise returns 1 ## @@ -144,22 +105,41 @@ function assertDockerRunning { ## methods to peer global services requiring network connectivity with project networks function connectPeeredServices { - for svc in ${DOCKER_PEERED_SERVICES[@]}; do + for svc in "${DOCKER_PEERED_SERVICES[@]}"; do echo "Connecting ${svc} to $1 network" (docker network connect "$1" ${svc} 2>&1| grep -v 'already exists in network') || true done } function disconnectPeeredServices { - for svc in ${DOCKER_PEERED_SERVICES[@]}; do + for svc in "${DOCKER_PEERED_SERVICES[@]}"; do echo "Disconnecting ${svc} from $1 network" (docker network disconnect "$1" ${svc} 2>&1| grep -v 'is not connected') || true done } -# Main logic with the timeout function +# Probe the registry that `svc pull` actually needs, with a 3-second timeout. +# Accept any HTTP response as proof of reachability (even 401 proves we reached the server). +# Do not use -f so 401 is treated as successful connection. function isOnline() { - (ping -q -c1 -t 2 8.8.8.8 &>/dev/null && echo "true") || (ping -q -c1 -t 2 1.1.1.1 &>/dev/null && echo "true") || echo "false" + if curl -s -m 3 -o /dev/null "https://ghcr.io/v2/" 2>/dev/null; then + echo "true" + else + echo "false" + fi +} + +## escape a raw string for embedding as a JSON string value; the caller supplies the +## surrounding quotes. bash 3.2 safe (pattern substitution only, no external tools). +## usage: printf '"%s"' "$(jsonEscape "$rawValue")" +function jsonEscape() { + local raw="$1" + raw="${raw//\\/\\\\}" + raw="${raw//\"/\\\"}" + raw="${raw//$'\n'/\\n}" + raw="${raw//$'\r'/\\r}" + raw="${raw//$'\t'/\\t}" + printf '%s' "$raw" } ## cross-platform sed in-place editing function @@ -181,4 +161,29 @@ function sed_inplace() { if [[ "$backup_ext" == ".bak" && -f "${file}${backup_ext}" ]]; then rm -f "${file}${backup_ext}" fi -} \ No newline at end of file +} +## Hand a database URI to TablePlus without putting it in the process list. +## +## The URI carries the database password, and `open "$uri" -a TablePlus` makes it an argument, so +## anything able to read `ps` can see it for as long as the call lasts - that is finding H2 against +## the internal command pack, where the URI holds real staging and production credentials. +## +## osascript reads its script from stdin, so only "osascript -" appears in the process list. +## `open location` routes by URL scheme, which is what TablePlus registers itself for. +function openInTablePlus() { + local uri="$1" + local escaped + + ## AppleScript string literals take the same escapes as C: backslash first, then quote + escaped="${uri//\\/\\\\}" + escaped="${escaped//\"/\\\"}" + + if printf 'open location "%s"\n' "${escaped}" | osascript - >/dev/null 2>&1; then + return 0 + fi + + ## Falling back to `open` puts the password in argv, so say so rather than exposing it silently. + warning "Could not hand the connection to TablePlus via osascript; falling back to \`open\`," + warning "which briefly exposes the database password in the process list." + open "${uri}" -a "${TABLEPLUS_APP:-TablePlus}" +} diff --git a/utils/env.sh b/utils/env.sh index 0d54070..9d1028b 100644 --- a/utils/env.sh +++ b/utils/env.sh @@ -90,19 +90,29 @@ function assertValidEnvType () { function appendEnvPartialIfExists () { local PARTIAL_NAME="${1}" local PARTIAL_PATH="" + local PARTIAL_DIR="" + local SUFFIX="" - for PARTIAL_PATH in \ - "${ROLL_DIR}/environments/includes/${PARTIAL_NAME}.base.yml" \ - "${ROLL_DIR}/environments/includes/${PARTIAL_NAME}.${ROLL_ENV_SUBT}.yml" \ - "${ROLL_DIR}/environments/${ROLL_ENV_TYPE}/${PARTIAL_NAME}.base.yml" \ - "${ROLL_DIR}/environments/${ROLL_ENV_TYPE}/${PARTIAL_NAME}.${ROLL_ENV_SUBT}.yml" \ - "${ROLL_HOME_DIR}/environments/includes/${PARTIAL_NAME}.base.yml" \ - "${ROLL_HOME_DIR}/environments/includes/${PARTIAL_NAME}.${ROLL_ENV_SUBT}.yml" \ - "${ROLL_HOME_DIR}/environments/${ROLL_ENV_TYPE}/${PARTIAL_NAME}.base.yml" \ - "${ROLL_HOME_DIR}/environments/${ROLL_ENV_TYPE}/${PARTIAL_NAME}.${ROLL_ENV_SUBT}.yml" - do - if [[ -f "${PARTIAL_PATH}" ]]; then - DOCKER_COMPOSE_ARGS+=("-f" "${PARTIAL_PATH}") - fi + local PARTIAL_DIRS=( + "${ROLL_DIR}/environments/includes" + "${ROLL_DIR}/environments/${ROLL_ENV_TYPE}" + "${ROLL_HOME_DIR}/environments/includes" + "${ROLL_HOME_DIR}/environments/${ROLL_ENV_TYPE}" + ) + + # WSL2 is Linux, so it also needs the .linux.yml fragments; a future + # .wsl.yml fragment is layered last so it can override the linux one. + local SUFFIXES=("base" "${ROLL_ENV_SUBT}") + if [[ "${ROLL_ENV_SUBT}" == "wsl" ]]; then + SUFFIXES=("base" "linux" "wsl") + fi + + for PARTIAL_DIR in "${PARTIAL_DIRS[@]}"; do + for SUFFIX in "${SUFFIXES[@]}"; do + PARTIAL_PATH="${PARTIAL_DIR}/${PARTIAL_NAME}.${SUFFIX}.yml" + if [[ -f "${PARTIAL_PATH}" ]]; then + DOCKER_COMPOSE_ARGS+=("-f" "${PARTIAL_PATH}") + fi + done done } diff --git a/utils/interact.sh b/utils/interact.sh new file mode 100644 index 0000000..6a31ff4 --- /dev/null +++ b/utils/interact.sh @@ -0,0 +1,269 @@ +#!/usr/bin/env bash +[[ ! ${ROLL_DIR} ]] && >&2 echo -e "\033[31mThis script is not intended to be run directly!\033[0m" && exit 1 + +## Interactive prompts and styled output, all of it gum-backed. +## +## Every prompt in this file follows one rule: a value supplied by a flag or an environment variable +## wins, gum runs only when there is a terminal to run it in, and anything else is a hard error that +## names the flag which would have supplied the value. gum has no non-interactive mode - `gum choose` +## and `gum input` open /dev/tty directly and exit 1 when there is no terminal, and piping into them +## supplies options rather than an answer - so a prompt must never be reached to obtain a value that +## was not already resolvable without one. That is what makes roll drivable from a script, a CI job +## or an AI assistant. +## +## The plain messaging helpers in core.sh (fatal/error/warning/info/success) stay gum-free on +## purpose: they have to work in pipes and CI, where gum is neither present nor wanted. + +## Minimum gum with the uniform exit-code contract: 0 chosen, 1 cancelled with ESC, 130 Ctrl-C. +ROLL_GUM_REQUIRE="0.14.0" + +## Is there a terminal on both ends? gum needs one; nothing here may assume it. +function isInteractive() { + [[ -t 0 && -t 1 ]] +} + +## Per-platform install instructions, shared by the lazy assertGum below and by `roll install`. +function gumInstallHint() { + >&2 echo "" + >&2 echo " macOS: brew install gum" + >&2 echo " Debian/Ubuntu: sudo mkdir -p /etc/apt/keyrings \\" + >&2 echo " && curl -fsSL https://repo.charm.sh/apt/gpg.key \\" + >&2 echo " | sudo gpg --dearmor -o /etc/apt/keyrings/charm.gpg \\" + >&2 echo " && echo 'deb [signed-by=/etc/apt/keyrings/charm.gpg] https://repo.charm.sh/apt/ * *' \\" + >&2 echo " | sudo tee /etc/apt/sources.list.d/charm.list \\" + >&2 echo " && sudo apt update && sudo apt install gum" + >&2 echo " Fedora/RHEL: sudo dnf install gum" + >&2 echo " Arch: sudo pacman -S gum" + >&2 echo " Anywhere: download a binary from https://github.com/charmbracelet/gum/releases" + >&2 echo "" +} + +## The installed gum version, empty when gum is absent or does not report one. +function gumVersion() { + local installed="" + command -v gum >/dev/null 2>&1 || return 0 + installed="$(gum --version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -n1)" || true + echo "${installed}" +} + +## Non-fatal check for `roll install`: report gum's state and move on. Installing roll on a machine +## that never runs an interactive prompt must not be blocked by a missing gum. +function checkGum() { + local installed="" + installed="$(gumVersion)" + + if [[ -z "${installed}" ]]; then + warning "gum is not installed. Interactive prompts (env-init, backup, restore) will not work" + warning "until it is; non-interactive use with flags is unaffected." + gumInstallHint + return 0 + fi + + if ! test "$(version "${installed}")" -ge "$(version "${ROLL_GUM_REQUIRE}")"; then + warning "gum ${installed} is older than the required ${ROLL_GUM_REQUIRE}; please upgrade it." + gumInstallHint + return 0 + fi + + success "gum ${installed} found." + return 0 +} + +## Called lazily by the wrappers below, never at startup: a non-interactive invocation of a command +## that happens not to prompt must keep working on a machine with no gum installed. +function assertGum() { + if ! command -v gum >/dev/null 2>&1; then + error "This prompt needs \`gum\`, which is not installed." + gumInstallHint + fatal "Install gum and try again, or supply the value non-interactively." + fi + + local installed="" + installed="$(gumVersion)" + + if [[ -z "${installed}" ]]; then + warning "Could not determine the installed gum version; continuing." + return 0 + fi + + if ! test "$(version "${installed}")" -ge "$(version "${ROLL_GUM_REQUIRE}")"; then + fatal "gum ${ROLL_GUM_REQUIRE} or newer is required (${installed} installed); please upgrade it." + fi + + return 0 +} + +## Turn gum's exit status into a message and an exit. ESC is 1 and Ctrl-C is 130; both mean the user +## chose not to answer, and 130 is propagated so a caller in a pipeline sees a real interrupt. +function handleGumCancel() { + local status="$1" + local what="$2" + + if [[ "${status}" == "130" ]]; then + >&2 echo "" + error "Cancelled." + exit 130 + fi + + error "No ${what} selected." + exit 1 +} + +## Explain how to supply a value when there is no terminal to ask on. +function fatalNoTty() { + local what="$1" + local how="$2" + + error "Cannot prompt for ${what}: no terminal attached." + fatal "Supply it non-interactively with ${how}." +} + +## promptInput [placeholder] +## Leaves the variable untouched when it already holds a value. +function promptInput() { + local var="$1" how="$2" prompt="$3" placeholder="${4:-}" + local current="" value="" status=0 + + eval "current=\${${var}:-}" + if [[ -n "${current}" ]]; then + return 0 + fi + + if ! isInteractive; then + fatalNoTty "${prompt}" "${how}" + fi + + assertGum + value="$(gum input --prompt "${prompt} " --placeholder "${placeholder}")" || status=$? + if [[ ${status} -ne 0 ]]; then + handleGumCancel "${status}" "value" + fi + + if [[ -z "${value}" ]]; then + error "No value entered." + exit 1 + fi + + printf -v "${var}" '%s' "${value}" + return 0 +} + +## promptChoose