diff --git a/.github/workflows/test-templates.yml b/.github/workflows/test-templates.yml new file mode 100644 index 0000000..bb582c6 --- /dev/null +++ b/.github/workflows/test-templates.yml @@ -0,0 +1,16 @@ +name: Test templates + +on: + pull_request: + push: + branches: [main] + +jobs: + test: + name: Render, build, deploy and verify templates + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run template tests + run: make test diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..7d76a57 --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +# Template testing pipeline — see tests/README.md. +# Tests always run inside the container, against the icp CLI pinned in the +# image; the host only needs Docker. + +# Official ICP dev environment; the tag pins the icp CLI version under test. +IMAGE ?= ghcr.io/dfinity/icp-dev-env-all:0.3.2 +# Optional permutation filter, e.g. `make test FILTER=hello-world:rust:react` +FILTER ?= + +# Named volumes keep dependency caches warm across local runs. +CACHE_VOLUMES = \ + -v icp-template-tests-cargo:/usr/local/cargo/registry \ + -v icp-template-tests-npm:/root/.npm \ + -v icp-template-tests-cache:/root/.cache + +.PHONY: test + +test: + docker run --rm -v "$(CURDIR)":/repo:ro $(CACHE_VOLUMES) $(IMAGE) /repo/tests/run.sh $(FILTER) diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..1f0e678 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,69 @@ +# Template tests + +End-to-end tests that verify every template in this repo renders, builds, +deploys, and serves traffic. For each permutation in +[`matrix.yaml`](./matrix.yaml) the harness: + +1. renders the template from the local checkout with `icp new` (the same code + path end users take), +2. asserts the rendered layout (directory renames, agent files), +3. runs `icp build`, +4. starts a local network (`icp network start -d`), +5. runs `icp deploy`, +6. runs the checks configured per canister in `matrix.yaml`: canister calls + with an expected reply, and/or a curl through the gateway + (`http://.local.localhost:8000/`, pinned to `127.0.0.1` + via `curl --resolve`, expecting HTTP 200), +7. stops the network and cleans up. + +Tests always run inside a container, against the `icp` CLI pinned in the +image — the host only needs Docker. They run directly on +[`ghcr.io/dfinity/icp-dev-env-all`](https://github.com/dfinity/icp-dev-env) +(tag pinned in the [`Makefile`](../Makefile)), which provides the icp CLI, +Rust + wasm32 target, mops, and Node.js. The harness installs `yq` at +startup if the image doesn't have it yet. + +## Running + +```bash +# everything +make test + +# a subset, by permutation-id prefix +make test FILTER=hello-world +make test FILTER=hello-world:rust:react + +# debugging: shell into the image and drive the harness manually +docker run --rm -it -v "$PWD":/repo:ro ghcr.io/dfinity/icp-dev-env-all:0.3.2 bash +$ /repo/tests/run.sh --fail-fast hello-world +``` + +Permutations run sequentially in one container; the full suite takes a few +minutes. Local runs mount named volumes for the cargo/npm/mops caches, so +repeated runs skip re-downloading dependencies. + +The repo is mounted read-only and the harness copies only files git would +track (tracked + untracked-but-not-ignored), so uncommitted template changes +are tested while local build artifacts (`target/`, `node_modules/`) are not. + +## Adding a template or permutation + +Everything is configured in [`matrix.yaml`](./matrix.yaml). Each template +entry declares: + +- `permutations` — variable sets passed to `icp new` as `--define` (an empty + entry `{}` renders with the template defaults). Each one gets a test run; + the permutation id used for filtering and reporting is the template name + plus the values, e.g. `hello-world:motoko:react`. +- `canisters` — what to verify after deploy, per canister: `curl: true` for + an HTTP 200 check through the gateway, and/or `call` with `method`, `args`, + and the `expect`-ed reply substring. A `name` of `{{project-name}}` is + replaced with the rendered project's name (for templates whose `icp.yaml` + uses that placeholder). + +## CI + +[`test-templates.yml`](../.github/workflows/test-templates.yml) runs the full +suite as a single sequential job on every pull request and on pushes to +`main`. If the matrix grows enough that this is too slow, split it into a job +matrix using the `FILTER` argument — the harness needs no changes. diff --git a/tests/lib.sh b/tests/lib.sh new file mode 100644 index 0000000..90ae2f1 --- /dev/null +++ b/tests/lib.sh @@ -0,0 +1,180 @@ +# Shared test steps for the template testing pipeline. +# Sourced by run.sh; runs inside the test container (see tests/Dockerfile). + +# Repo root as mounted into the container (read-only). +REPO="${REPO:-/repo}" +GATEWAY_PORT=8000 +CONFIG="${CONFIG:-$(dirname "${BASH_SOURCE[0]}")/matrix.yaml}" + +# Never send icp telemetry from test runs. +export DO_NOT_TRACK=1 + +# The container runs as root but doesn't set $USER, which cargo-generate +# (inside `icp new`) requires. +export USER="${USER:-root}" + +# Stopgap until ghcr.io/dfinity/icp-dev-env-all ships yq: install it at +# startup. Becomes a no-op once the base image includes it. +YQ_VERSION=v4.53.3 +ensure_yq() { + command -v yq >/dev/null && return 0 + step "setup: installing yq $YQ_VERSION" + curl -fsSL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_$(dpkg --print-architecture)" \ + -o /usr/local/bin/yq \ + && chmod +x /usr/local/bin/yq \ + || fail "could not install yq" +} + +step() { printf '\n==> %s\n' "$*"; } +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } + +# prepare_template_source +# Copies the repo into , restricted to files git would track +# (tracked + untracked-but-not-ignored). Local build artifacts in template +# dirs (target/, node_modules/, ...) would otherwise be copied and +# template-substituted by cargo-generate, which is slow and fails on +# binaries. Sets TEMPLATE_SRC. +prepare_template_source() { + TEMPLATE_SRC="$1" + mkdir -p "$TEMPLATE_SRC" + # The repo bind-mount is owned by the host user; git refuses to read it + # from a different uid without this. + git config --global --add safe.directory "$REPO" + (cd "$REPO" && git ls-files --cached --others --exclude-standard -z \ + | tar --null --ignore-failed-read -cf - -T -) \ + | tar -xf - -C "$TEMPLATE_SRC" \ + || fail "could not copy template source from $REPO" +} + +# render