From 9bb4b57824b6098d2695876befa3fe88ccee2210 Mon Sep 17 00:00:00 2001
From: Raymond Khalife <raymond.khalife@dfinity.org>
Date: Thu, 11 Jun 2026 16:12:53 -0400
Subject: [PATCH 1/5] feat(tests): containerized template test harness (M1)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds the first milestone of the template testing pipeline: a thin test
image over ghcr.io/dfinity/icp-dev-env-all and a bash harness that
renders a template from the local checkout via `icp new`, builds,
deploys to a local network, and verifies a backend call — all inside
the container, against the icp CLI pinned in the image.

Covers a single hardcoded permutation (rust template, defaults);
the full matrix follows in M2.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 Makefile         |  13 ++++++
 tests/Dockerfile |  19 +++++++++
 tests/lib.sh     | 107 +++++++++++++++++++++++++++++++++++++++++++++++
 tests/run.sh     |  32 ++++++++++++++
 4 files changed, 171 insertions(+)
 create mode 100644 Makefile
 create mode 100644 tests/Dockerfile
 create mode 100644 tests/lib.sh
 create mode 100755 tests/run.sh

diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..c200ac4
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,13 @@
+# Template testing pipeline — see docs/template-testing-prd.md.
+# Tests always run inside the container, against the icp CLI pinned in the
+# image; the host only needs Docker.
+
+IMAGE ?= icp-template-tests
+
+.PHONY: test test-image
+
+test-image:
+	docker build -t $(IMAGE) tests
+
+test: test-image
+	docker run --rm -v "$(CURDIR)":/repo:ro $(IMAGE) /repo/tests/run.sh
diff --git a/tests/Dockerfile b/tests/Dockerfile
new file mode 100644
index 0000000..b4f507a
--- /dev/null
+++ b/tests/Dockerfile
@@ -0,0 +1,19 @@
+# Test image for the template testing pipeline (docs/template-testing-prd.md).
+#
+# Thin layer over the official ICP dev environment, which already ships the
+# icp CLI (with embedded cargo-generate and PocketIC), Rust + wasm32 target,
+# mops, and Node.js. We only add yq for parsing the rendered icp.yaml files.
+ARG BASE_TAG=0.3.2
+FROM ghcr.io/dfinity/icp-dev-env-all:${BASE_TAG}
+
+# cargo-generate (inside `icp new`) requires $USER; the base image runs as
+# root without it. Also opt out of icp telemetry for test runs.
+ENV USER=root \
+    DO_NOT_TRACK=1
+
+ARG YQ_VERSION=v4.53.3
+RUN ARCH="$(dpkg --print-architecture)" \
+    && curl -fsSL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH}" \
+       -o /usr/local/bin/yq \
+    && chmod +x /usr/local/bin/yq \
+    && yq --version
diff --git a/tests/lib.sh b/tests/lib.sh
new file mode 100644
index 0000000..f15e556
--- /dev/null
+++ b/tests/lib.sh
@@ -0,0 +1,107 @@
+# Shared test steps for the template testing pipeline.
+# Sourced by run.sh; runs inside the test container (see tests/Dockerfile).
+
+# Repo root as mounted into the container (read-only).
+REPO="${REPO:-/repo}"
+GATEWAY_PORT=8000
+
+# Never send icp telemetry from test runs (also set in tests/Dockerfile, but
+# exported here so it holds however the harness is invoked).
+export DO_NOT_TRACK=1
+
+step() { printf '\n==> %s\n' "$*"; }
+fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
+
+# prepare_template_source <dest-dir>
+# Copies the repo into <dest-dir>, restricted to files git would track
+# (tracked + untracked-but-not-ignored). Local build artifacts in template
+# dirs (target/, node_modules/, ...) would otherwise be copied and
+# template-substituted by cargo-generate, which is slow and fails on
+# binaries. Sets TEMPLATE_SRC.
+prepare_template_source() {
+  TEMPLATE_SRC="$1"
+  mkdir -p "$TEMPLATE_SRC"
+  # The repo bind-mount is owned by the host user; git refuses to read it
+  # from a different uid without this.
+  git config --global --add safe.directory "$REPO"
+  (cd "$REPO" && git ls-files --cached --others --exclude-standard -z \
+    | tar --null --ignore-failed-read -cf - -T -) \
+    | tar -xf - -C "$TEMPLATE_SRC" \
+    || fail "could not copy template source from $REPO"
+}
+
+# render <template> <project-name> [-d key=value ...]
+# Renders the template from the clean copy of the local checkout via
+# `icp new` — the same code path end users take — into the current directory.
+render() {
+  local template="$1" project="$2"
+  shift 2
+  step "render: icp new $project --subfolder $template $*"
+  icp new "$project" --path "$TEMPLATE_SRC" --subfolder "$template" --silent "$@" \
+    || fail "icp new failed for template '$template'"
+
+  [ -f "$project/icp.yaml" ] || fail "rendered project is missing icp.yaml"
+  [ -f "$project/AGENTS.md" ] || fail "rendered project is missing AGENTS.md (post-hook)"
+  [ -f "$project/CLAUDE.md" ] || fail "rendered project is missing CLAUDE.md (post-hook)"
+}
+
+# All remaining steps run from inside the rendered project directory.
+
+build() {
+  step "build: icp build"
+  icp build || fail "icp build failed"
+}
+
+network_start() {
+  step "network: icp network start -d"
+  icp network start -d || fail "icp network start failed"
+}
+
+network_stop() {
+  icp network stop >/dev/null 2>&1 || true
+}
+
+deploy() {
+  step "deploy: icp deploy"
+  icp deploy || fail "icp deploy failed"
+}
+
+# call_greet <canister-name>
+call_greet() {
+  local canister="$1" reply
+  step "call: icp canister call $canister greet '(\"ICP\")'"
+  reply="$(icp canister call "$canister" greet '("ICP")')" \
+    || fail "icp canister call failed"
+  printf '%s\n' "$reply"
+  printf '%s' "$reply" | grep -qF 'Hello, ICP!' \
+    || fail "unexpected greet reply: $reply"
+}
+
+# http_check <canister-name>
+# Curls the asset canister root through the gateway using the name-based
+# domain. --resolve is always used because *.localhost DNS resolution is not
+# guaranteed inside containers.
+http_check() {
+  local canister="$1" host status
+  host="$canister.local.localhost"
+  step "http: curl http://$host:$GATEWAY_PORT/"
+  status="$(curl -s -o /tmp/frontend-body -w '%{http_code}' \
+    --resolve "$host:$GATEWAY_PORT:127.0.0.1" \
+    "http://$host:$GATEWAY_PORT/")" \
+    || fail "curl to $host failed"
+  [ "$status" = "200" ] || fail "expected HTTP 200 from $host, got $status"
+  [ -s /tmp/frontend-body ] || fail "frontend returned an empty body"
+}
+
+# first_canister_name <project-dir>
+# Reads the first canister's name from the rendered icp.yaml. Entries are
+# either inline objects (`name: backend`) or paths to canister.yaml files.
+first_canister_name() {
+  local dir="$1" entry
+  entry="$(yq -r '.canisters[0]' "$dir/icp.yaml")"
+  if [ "$(yq -r '.canisters[0] | type' "$dir/icp.yaml")" = "!!map" ]; then
+    yq -r '.canisters[0].name' "$dir/icp.yaml"
+  else
+    yq -r '.name' "$dir/$entry/canister.yaml"
+  fi
+}
diff --git a/tests/run.sh b/tests/run.sh
new file mode 100755
index 0000000..1cdb3a0
--- /dev/null
+++ b/tests/run.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# In-container entrypoint for the template testing pipeline.
+# See docs/template-testing-prd.md. M1: a single hardcoded permutation
+# (rust template, default arguments). The full matrix arrives in M2.
+set -euo pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+
+TEMPLATE=rust
+PROJECT=e2e-rust
+
+WORKDIR="$(mktemp -d)"
+cleanup() {
+  network_stop
+  rm -rf "$WORKDIR"
+}
+trap cleanup EXIT
+
+prepare_template_source "$WORKDIR/template-src"
+
+cd "$WORKDIR"
+render "$TEMPLATE" "$PROJECT" -d network_type=Default
+
+cd "$PROJECT"
+backend="$(first_canister_name .)"
+
+build
+network_start
+deploy
+call_greet "$backend"
+
+printf '\nPASS: %s (%s)\n' "$TEMPLATE" "$PROJECT"

From b53f2636c47a361779d759bb82d1ca55b1ce57bc Mon Sep 17 00:00:00 2001
From: Raymond Khalife <raymond.khalife@dfinity.org>
Date: Thu, 11 Jun 2026 16:24:20 -0400
Subject: [PATCH 2/5] feat(tests): full template test matrix (M2)

Replaces the hardcoded M1 permutation with a declarative matrix
(tests/matrix.txt) covering rust, motoko, and all four hello-world
backend/frontend combinations. The driver gains filtering by
permutation-id prefix, a --fail-fast flag, and a summary table.

Each permutation now verifies every canister in the rendered project:
asset canisters get an HTTP 200 check through the name-based gateway
domain (curl --resolve <name>.local.localhost:8000), everything else
gets the greet call. hello-world renders additionally assert the
variant directory renames.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 Makefile         |  4 ++-
 tests/lib.sh     | 81 +++++++++++++++++++++++++++++++++++++++++-------
 tests/matrix.txt | 11 +++++++
 tests/run.sh     | 72 +++++++++++++++++++++++++++++++-----------
 4 files changed, 138 insertions(+), 30 deletions(-)
 create mode 100644 tests/matrix.txt

diff --git a/Makefile b/Makefile
index c200ac4..91e78aa 100644
--- a/Makefile
+++ b/Makefile
@@ -3,6 +3,8 @@
 # image; the host only needs Docker.
 
 IMAGE ?= icp-template-tests
+# Optional permutation filter, e.g. `make test FILTER=hello-world:rust:react`
+FILTER ?=
 
 .PHONY: test test-image
 
@@ -10,4 +12,4 @@ test-image:
 	docker build -t $(IMAGE) tests
 
 test: test-image
-	docker run --rm -v "$(CURDIR)":/repo:ro $(IMAGE) /repo/tests/run.sh
+	docker run --rm -v "$(CURDIR)":/repo:ro $(IMAGE) /repo/tests/run.sh $(FILTER)
diff --git a/tests/lib.sh b/tests/lib.sh
index f15e556..03d48ab 100644
--- a/tests/lib.sh
+++ b/tests/lib.sh
@@ -93,15 +93,74 @@ http_check() {
   [ -s /tmp/frontend-body ] || fail "frontend returned an empty body"
 }
 
-# first_canister_name <project-dir>
-# Reads the first canister's name from the rendered icp.yaml. Entries are
-# either inline objects (`name: backend`) or paths to canister.yaml files.
-first_canister_name() {
-  local dir="$1" entry
-  entry="$(yq -r '.canisters[0]' "$dir/icp.yaml")"
-  if [ "$(yq -r '.canisters[0] | type' "$dir/icp.yaml")" = "!!map" ]; then
-    yq -r '.canisters[0].name' "$dir/icp.yaml"
-  else
-    yq -r '.name' "$dir/$entry/canister.yaml"
-  fi
+# canister_specs <project-dir>
+# Prints one "name<TAB>recipe-type" line per canister in the rendered
+# project. icp.yaml entries are either inline objects (`name: ...`) or paths
+# to directories containing a canister.yaml.
+canister_specs() {
+  local dir="$1" count i entry
+  count="$(yq -r '.canisters | length' "$dir/icp.yaml")"
+  for ((i = 0; i < count; i++)); do
+    if [ "$(yq -r ".canisters[$i] | type" "$dir/icp.yaml")" = "!!map" ]; then
+      yq -r ".canisters[$i] | .name + \"\t\" + .recipe.type" "$dir/icp.yaml"
+    else
+      entry="$(yq -r ".canisters[$i]" "$dir/icp.yaml")"
+      [ -f "$dir/$entry/canister.yaml" ] \
+        || fail "icp.yaml references '$entry' but $entry/canister.yaml is missing"
+      yq -r '.name + "\t" + .recipe.type' "$dir/$entry/canister.yaml"
+    fi
+  done
+}
+
+# verify_canisters <project-dir>
+# Exercises every deployed canister: asset canisters get the HTTP check,
+# everything else gets the greet call.
+verify_canisters() {
+  local dir="$1" name type
+  while IFS=$'\t' read -r name type; do
+    case "$type" in
+      *asset-canister*) http_check "$name" ;;
+      *) call_greet "$name" ;;
+    esac
+  done < <(canister_specs "$dir")
+}
+
+# assert_hello_world_layout <project-dir>
+# The hello-world pre-hook renames the chosen variant dirs to backend/ and
+# frontend/; the cargo-generate conditionals drop the unchosen ones.
+assert_hello_world_layout() {
+  local dir="$1" leftover
+  [ -d "$dir/backend" ] || fail "hello-world: backend/ directory missing after render"
+  [ -d "$dir/frontend" ] || fail "hello-world: frontend/ directory missing after render"
+  for leftover in rust-backend motoko-backend react-frontend vue-frontend; do
+    [ ! -e "$dir/$leftover" ] || fail "hello-world: unexpected leftover directory $leftover/"
+  done
+}
+
+# run_permutation <id> <template> [key=value ...]
+# Runs the full flow for one matrix entry in its own project dir under
+# $WORKDIR. Always attempts to stop the permutation's network afterwards so
+# sequential runs don't fight over the gateway port.
+run_permutation() {
+  local id="$1" template="$2"
+  shift 2
+  local project="e2e-$(printf '%s' "$id" | tr ':' '-')"
+  local dir="$WORKDIR/$project" rc=0
+  mkdir -p "$dir"
+
+  (
+    cd "$dir"
+    local dargs=() kv
+    for kv in "$@"; do dargs+=(-d "$kv"); done
+    render "$template" "$project" "${dargs[@]}"
+    [ "$template" = hello-world ] && assert_hello_world_layout "$project"
+    cd "$project"
+    build
+    network_start
+    deploy
+    verify_canisters .
+  ) || rc=$?
+
+  icp network stop --project-root-override "$dir/$project" >/dev/null 2>&1 || true
+  return $rc
 }
diff --git a/tests/matrix.txt b/tests/matrix.txt
new file mode 100644
index 0000000..4bbe2d3
--- /dev/null
+++ b/tests/matrix.txt
@@ -0,0 +1,11 @@
+# Template test matrix — one permutation per line:
+#   <template> [key=value ...]
+# Values are passed to `icp new` as -d defines; omitted variables use the
+# template defaults. The permutation id is the template name plus the values,
+# e.g. `hello-world:motoko:react`.
+rust
+motoko
+hello-world backend_type=motoko frontend_type=react
+hello-world backend_type=motoko frontend_type=vue
+hello-world backend_type=rust frontend_type=react
+hello-world backend_type=rust frontend_type=vue
diff --git a/tests/run.sh b/tests/run.sh
index 1cdb3a0..cf445b3 100755
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -1,32 +1,68 @@
 #!/usr/bin/env bash
 # In-container entrypoint for the template testing pipeline.
-# See docs/template-testing-prd.md. M1: a single hardcoded permutation
-# (rust template, default arguments). The full matrix arrives in M2.
+# See docs/template-testing-prd.md.
+#
+# Usage: run.sh [--fail-fast] [FILTER ...]
+#   FILTER matches permutation ids by prefix, e.g. `hello-world` or
+#   `hello-world:rust:react`. With no filter, the whole matrix runs.
 set -euo pipefail
 
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+MATRIX_FILE="$(dirname "${BASH_SOURCE[0]}")/matrix.txt"
 
-TEMPLATE=rust
-PROJECT=e2e-rust
+FAIL_FAST=0
+FILTERS=()
+for arg in "$@"; do
+  case "$arg" in
+    --fail-fast) FAIL_FAST=1 ;;
+    -*) fail "unknown option: $arg" ;;
+    *) FILTERS+=("$arg") ;;
+  esac
+done
 
-WORKDIR="$(mktemp -d)"
-cleanup() {
-  network_stop
-  rm -rf "$WORKDIR"
+matches_filter() {
+  local id="$1" f
+  [ ${#FILTERS[@]} -eq 0 ] && return 0
+  for f in "${FILTERS[@]}"; do
+    [[ "$id" == "$f" || "$id" == "$f":* ]] && return 0
+  done
+  return 1
 }
-trap cleanup EXIT
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
 
 prepare_template_source "$WORKDIR/template-src"
 
-cd "$WORKDIR"
-render "$TEMPLATE" "$PROJECT" -d network_type=Default
+RESULTS=()
+overall=0
+ran=0
+
+while read -r template defines; do
+  [[ -z "$template" || "$template" == \#* ]] && continue
+
+  read -ra define_args <<<"${defines:-}"
+  id="$template"
+  for kv in ${define_args[@]+"${define_args[@]}"}; do
+    id+=":${kv#*=}"
+  done
+
+  matches_filter "$id" || continue
+  ran=$((ran + 1))
 
-cd "$PROJECT"
-backend="$(first_canister_name .)"
+  printf '\n========== %s ==========\n' "$id"
+  start="$(date +%s)"
+  if run_permutation "$id" "$template" ${define_args[@]+"${define_args[@]}"}; then
+    RESULTS+=("PASS  $id  ($(($(date +%s) - start))s)")
+  else
+    RESULTS+=("FAIL  $id  ($(($(date +%s) - start))s)")
+    overall=1
+    [ "$FAIL_FAST" -eq 1 ] && break
+  fi
+done <"$MATRIX_FILE"
 
-build
-network_start
-deploy
-call_greet "$backend"
+[ "$ran" -gt 0 ] || fail "no permutations matched filter: ${FILTERS[*]:-<none>}"
 
-printf '\nPASS: %s (%s)\n' "$TEMPLATE" "$PROJECT"
+printf '\n========== summary ==========\n'
+printf '%s\n' "${RESULTS[@]}"
+exit "$overall"

From 51b734fb2cb5533bc23eb1d1dcd73e9e7a2790b7 Mon Sep 17 00:00:00 2001
From: Raymond Khalife <raymond.khalife@dfinity.org>
Date: Thu, 11 Jun 2026 17:06:57 -0400
Subject: [PATCH 3/5] feat(tests): CI workflow, cache volumes, and docs (M3)

Adds a GitHub Actions workflow that runs the full template test suite
as a single sequential job on pull requests and pushes to main. The
suite stays sequential in one container: dependency caches warm up on
the first permutation, making the full matrix faster than parallel
cold containers would be. If the matrix grows, CI can split into a
job matrix via the harness's permutation filter without changes.

Local `make test` runs now mount named volumes for the cargo, npm and
mops caches so repeated runs skip re-downloading dependencies.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/test-templates.yml | 16 +++++++
 Makefile                             |  8 +++-
 tests/README.md                      | 65 ++++++++++++++++++++++++++++
 3 files changed, 88 insertions(+), 1 deletion(-)
 create mode 100644 .github/workflows/test-templates.yml
 create mode 100644 tests/README.md

diff --git a/.github/workflows/test-templates.yml b/.github/workflows/test-templates.yml
new file mode 100644
index 0000000..bb582c6
--- /dev/null
+++ b/.github/workflows/test-templates.yml
@@ -0,0 +1,16 @@
+name: Test templates
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  test:
+    name: Render, build, deploy and verify templates
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Run template tests
+        run: make test
diff --git a/Makefile b/Makefile
index 91e78aa..92b5983 100644
--- a/Makefile
+++ b/Makefile
@@ -11,5 +11,11 @@ FILTER ?=
 test-image:
 	docker build -t $(IMAGE) tests
 
+# Named volumes keep dependency caches warm across local runs.
+CACHE_VOLUMES = \
+	-v $(IMAGE)-cargo:/usr/local/cargo/registry \
+	-v $(IMAGE)-npm:/root/.npm \
+	-v $(IMAGE)-cache:/root/.cache
+
 test: test-image
-	docker run --rm -v "$(CURDIR)":/repo:ro $(IMAGE) /repo/tests/run.sh $(FILTER)
+	docker run --rm -v "$(CURDIR)":/repo:ro $(CACHE_VOLUMES) $(IMAGE) /repo/tests/run.sh $(FILTER)
diff --git a/tests/README.md b/tests/README.md
new file mode 100644
index 0000000..66d6900
--- /dev/null
+++ b/tests/README.md
@@ -0,0 +1,65 @@
+# Template tests
+
+End-to-end tests that verify every template in this repo renders, builds,
+deploys, and serves traffic. For each permutation in [`matrix.txt`](./matrix.txt)
+the harness:
+
+1. renders the template from the local checkout with `icp new` (the same code
+   path end users take),
+2. asserts the rendered layout (directory renames, agent files),
+3. runs `icp build`,
+4. starts a local network (`icp network start -d`),
+5. runs `icp deploy`,
+6. calls `greet` on every backend canister and curls every asset canister
+   through the gateway (`http://<canister-name>.local.localhost:8000/`,
+   pinned to `127.0.0.1` via `curl --resolve`),
+7. stops the network and cleans up.
+
+Tests always run inside a container, against the `icp` CLI pinned in the
+image — the host only needs Docker. The image
+([`Dockerfile`](./Dockerfile)) is a thin layer over
+[`ghcr.io/dfinity/icp-dev-env-all`](https://github.com/dfinity/icp-dev-env),
+which provides the icp CLI, Rust + wasm32 target, mops, and Node.js.
+
+## Running
+
+```bash
+# everything
+make test
+
+# a subset, by permutation-id prefix
+make test FILTER=hello-world
+make test FILTER=hello-world:rust:react
+
+# debugging: shell into the image and drive the harness manually
+docker run --rm -it -v "$PWD":/repo:ro icp-template-tests bash
+$ /repo/tests/run.sh --fail-fast hello-world
+```
+
+Permutations run sequentially in one container; the full suite takes a few
+minutes. Local runs mount named volumes for the cargo/npm/mops caches, so
+repeated runs skip re-downloading dependencies.
+
+The repo is mounted read-only and the harness copies only files git would
+track (tracked + untracked-but-not-ignored), so uncommitted template changes
+are tested while local build artifacts (`target/`, `node_modules/`) are not.
+
+## Adding a permutation
+
+Add a line to [`matrix.txt`](./matrix.txt):
+
+```
+<template> [key=value ...]
+```
+
+`key=value` pairs are passed to `icp new` as `--define`; omitted template
+variables fall back to their defaults. The permutation id used for filtering
+and reporting is the template name plus the values, e.g.
+`hello-world:motoko:react`.
+
+## CI
+
+[`test-templates.yml`](../.github/workflows/test-templates.yml) runs the full
+suite as a single sequential job on every pull request and on pushes to
+`main`. If the matrix grows enough that this is too slow, split it into a job
+matrix using the `FILTER` argument — the harness needs no changes.

From a9a079030257726709ed336ea553d510b3bef4d8 Mon Sep 17 00:00:00 2001
From: Raymond Khalife <raymond.khalife@dfinity.org>
Date: Thu, 11 Jun 2026 17:34:00 -0400
Subject: [PATCH 4/5] refactor(tests): drop custom Dockerfile, run on
 icp-dev-env-all directly

The only thing the custom image added was yq, which the base image
will eventually ship. The harness now installs yq at startup if it's
missing (a no-op once the image includes it) and exports the env vars
the image doesn't set (USER for cargo-generate, DO_NOT_TRACK).

The icp CLI pin moves to the Makefile's IMAGE variable; an alternate
CLI can be tested with `make test IMAGE=...`.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 Makefile         | 20 +++++++++-----------
 tests/Dockerfile | 19 -------------------
 tests/README.md  | 11 ++++++-----
 tests/lib.sh     | 19 +++++++++++++++++--
 tests/run.sh     |  1 +
 5 files changed, 33 insertions(+), 37 deletions(-)
 delete mode 100644 tests/Dockerfile

diff --git a/Makefile b/Makefile
index 92b5983..7d76a57 100644
--- a/Makefile
+++ b/Makefile
@@ -1,21 +1,19 @@
-# Template testing pipeline — see docs/template-testing-prd.md.
+# Template testing pipeline — see tests/README.md.
 # Tests always run inside the container, against the icp CLI pinned in the
 # image; the host only needs Docker.
 
-IMAGE ?= icp-template-tests
+# Official ICP dev environment; the tag pins the icp CLI version under test.
+IMAGE ?= ghcr.io/dfinity/icp-dev-env-all:0.3.2
 # Optional permutation filter, e.g. `make test FILTER=hello-world:rust:react`
 FILTER ?=
 
-.PHONY: test test-image
-
-test-image:
-	docker build -t $(IMAGE) tests
-
 # Named volumes keep dependency caches warm across local runs.
 CACHE_VOLUMES = \
-	-v $(IMAGE)-cargo:/usr/local/cargo/registry \
-	-v $(IMAGE)-npm:/root/.npm \
-	-v $(IMAGE)-cache:/root/.cache
+	-v icp-template-tests-cargo:/usr/local/cargo/registry \
+	-v icp-template-tests-npm:/root/.npm \
+	-v icp-template-tests-cache:/root/.cache
+
+.PHONY: test
 
-test: test-image
+test:
 	docker run --rm -v "$(CURDIR)":/repo:ro $(CACHE_VOLUMES) $(IMAGE) /repo/tests/run.sh $(FILTER)
diff --git a/tests/Dockerfile b/tests/Dockerfile
deleted file mode 100644
index b4f507a..0000000
--- a/tests/Dockerfile
+++ /dev/null
@@ -1,19 +0,0 @@
-# Test image for the template testing pipeline (docs/template-testing-prd.md).
-#
-# Thin layer over the official ICP dev environment, which already ships the
-# icp CLI (with embedded cargo-generate and PocketIC), Rust + wasm32 target,
-# mops, and Node.js. We only add yq for parsing the rendered icp.yaml files.
-ARG BASE_TAG=0.3.2
-FROM ghcr.io/dfinity/icp-dev-env-all:${BASE_TAG}
-
-# cargo-generate (inside `icp new`) requires $USER; the base image runs as
-# root without it. Also opt out of icp telemetry for test runs.
-ENV USER=root \
-    DO_NOT_TRACK=1
-
-ARG YQ_VERSION=v4.53.3
-RUN ARCH="$(dpkg --print-architecture)" \
-    && curl -fsSL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${ARCH}" \
-       -o /usr/local/bin/yq \
-    && chmod +x /usr/local/bin/yq \
-    && yq --version
diff --git a/tests/README.md b/tests/README.md
index 66d6900..05065bc 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -16,10 +16,11 @@ the harness:
 7. stops the network and cleans up.
 
 Tests always run inside a container, against the `icp` CLI pinned in the
-image — the host only needs Docker. The image
-([`Dockerfile`](./Dockerfile)) is a thin layer over
-[`ghcr.io/dfinity/icp-dev-env-all`](https://github.com/dfinity/icp-dev-env),
-which provides the icp CLI, Rust + wasm32 target, mops, and Node.js.
+image — the host only needs Docker. They run directly on
+[`ghcr.io/dfinity/icp-dev-env-all`](https://github.com/dfinity/icp-dev-env)
+(tag pinned in the [`Makefile`](../Makefile)), which provides the icp CLI,
+Rust + wasm32 target, mops, and Node.js. The harness installs `yq` at
+startup if the image doesn't have it yet.
 
 ## Running
 
@@ -32,7 +33,7 @@ make test FILTER=hello-world
 make test FILTER=hello-world:rust:react
 
 # debugging: shell into the image and drive the harness manually
-docker run --rm -it -v "$PWD":/repo:ro icp-template-tests bash
+docker run --rm -it -v "$PWD":/repo:ro ghcr.io/dfinity/icp-dev-env-all:0.3.2 bash
 $ /repo/tests/run.sh --fail-fast hello-world
 ```
 
diff --git a/tests/lib.sh b/tests/lib.sh
index 03d48ab..9309673 100644
--- a/tests/lib.sh
+++ b/tests/lib.sh
@@ -5,10 +5,25 @@
 REPO="${REPO:-/repo}"
 GATEWAY_PORT=8000
 
-# Never send icp telemetry from test runs (also set in tests/Dockerfile, but
-# exported here so it holds however the harness is invoked).
+# Never send icp telemetry from test runs.
 export DO_NOT_TRACK=1
 
+# The container runs as root but doesn't set $USER, which cargo-generate
+# (inside `icp new`) requires.
+export USER="${USER:-root}"
+
+# Stopgap until ghcr.io/dfinity/icp-dev-env-all ships yq: install it at
+# startup. Becomes a no-op once the base image includes it.
+YQ_VERSION=v4.53.3
+ensure_yq() {
+  command -v yq >/dev/null && return 0
+  step "setup: installing yq $YQ_VERSION"
+  curl -fsSL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_$(dpkg --print-architecture)" \
+    -o /usr/local/bin/yq \
+    && chmod +x /usr/local/bin/yq \
+    || fail "could not install yq"
+}
+
 step() { printf '\n==> %s\n' "$*"; }
 fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; }
 
diff --git a/tests/run.sh b/tests/run.sh
index cf445b3..108458c 100755
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -32,6 +32,7 @@ matches_filter() {
 WORKDIR="$(mktemp -d)"
 trap 'rm -rf "$WORKDIR"' EXIT
 
+ensure_yq
 prepare_template_source "$WORKDIR/template-src"
 
 RESULTS=()

From bcf373d2d3d680a7c29bb33d99a4e647c5f895bc Mon Sep 17 00:00:00 2001
From: Raymond Khalife <raymond.khalife@dfinity.org>
Date: Thu, 11 Jun 2026 19:20:11 -0400
Subject: [PATCH 5/5] refactor(tests): configure canister checks per template
 in matrix.yaml

Replaces recipe-type discovery with explicit per-template test
configuration. matrix.txt becomes matrix.yaml, where each template
declares its permutations and, per canister, what to verify after
deploy: a gateway curl (curl: true) and/or a canister call with
method, args, and an expected reply substring.

A configured canister name of {{project-name}} is substituted with
the rendered project's name, for templates whose icp.yaml names the
canister after the project.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 tests/README.md   | 33 ++++++++++++-----------
 tests/lib.sh      | 69 +++++++++++++++++++++++------------------------
 tests/matrix.txt  | 11 --------
 tests/matrix.yaml | 51 +++++++++++++++++++++++++++++++++++
 tests/run.sh      | 49 +++++++++++++++++++--------------
 5 files changed, 132 insertions(+), 81 deletions(-)
 delete mode 100644 tests/matrix.txt
 create mode 100644 tests/matrix.yaml

diff --git a/tests/README.md b/tests/README.md
index 05065bc..1f0e678 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -1,8 +1,8 @@
 # Template tests
 
 End-to-end tests that verify every template in this repo renders, builds,
-deploys, and serves traffic. For each permutation in [`matrix.txt`](./matrix.txt)
-the harness:
+deploys, and serves traffic. For each permutation in
+[`matrix.yaml`](./matrix.yaml) the harness:
 
 1. renders the template from the local checkout with `icp new` (the same code
    path end users take),
@@ -10,9 +10,10 @@ the harness:
 3. runs `icp build`,
 4. starts a local network (`icp network start -d`),
 5. runs `icp deploy`,
-6. calls `greet` on every backend canister and curls every asset canister
-   through the gateway (`http://<canister-name>.local.localhost:8000/`,
-   pinned to `127.0.0.1` via `curl --resolve`),
+6. runs the checks configured per canister in `matrix.yaml`: canister calls
+   with an expected reply, and/or a curl through the gateway
+   (`http://<canister-name>.local.localhost:8000/`, pinned to `127.0.0.1`
+   via `curl --resolve`, expecting HTTP 200),
 7. stops the network and cleans up.
 
 Tests always run inside a container, against the `icp` CLI pinned in the
@@ -45,18 +46,20 @@ The repo is mounted read-only and the harness copies only files git would
 track (tracked + untracked-but-not-ignored), so uncommitted template changes
 are tested while local build artifacts (`target/`, `node_modules/`) are not.
 
-## Adding a permutation
+## Adding a template or permutation
 
-Add a line to [`matrix.txt`](./matrix.txt):
+Everything is configured in [`matrix.yaml`](./matrix.yaml). Each template
+entry declares:
 
-```
-<template> [key=value ...]
-```
-
-`key=value` pairs are passed to `icp new` as `--define`; omitted template
-variables fall back to their defaults. The permutation id used for filtering
-and reporting is the template name plus the values, e.g.
-`hello-world:motoko:react`.
+- `permutations` — variable sets passed to `icp new` as `--define` (an empty
+  entry `{}` renders with the template defaults). Each one gets a test run;
+  the permutation id used for filtering and reporting is the template name
+  plus the values, e.g. `hello-world:motoko:react`.
+- `canisters` — what to verify after deploy, per canister: `curl: true` for
+  an HTTP 200 check through the gateway, and/or `call` with `method`, `args`,
+  and the `expect`-ed reply substring. A `name` of `{{project-name}}` is
+  replaced with the rendered project's name (for templates whose `icp.yaml`
+  uses that placeholder).
 
 ## CI
 
diff --git a/tests/lib.sh b/tests/lib.sh
index 9309673..90ae2f1 100644
--- a/tests/lib.sh
+++ b/tests/lib.sh
@@ -4,6 +4,7 @@
 # Repo root as mounted into the container (read-only).
 REPO="${REPO:-/repo}"
 GATEWAY_PORT=8000
+CONFIG="${CONFIG:-$(dirname "${BASH_SOURCE[0]}")/matrix.yaml}"
 
 # Never send icp telemetry from test runs.
 export DO_NOT_TRACK=1
@@ -81,15 +82,15 @@ deploy() {
   icp deploy || fail "icp deploy failed"
 }
 
-# call_greet <canister-name>
-call_greet() {
-  local canister="$1" reply
-  step "call: icp canister call $canister greet '(\"ICP\")'"
-  reply="$(icp canister call "$canister" greet '("ICP")')" \
+# call_check <canister-name> <method> <args> <expected-substring>
+call_check() {
+  local canister="$1" method="$2" args="$3" expect="$4" reply
+  step "call: icp canister call $canister $method '$args'"
+  reply="$(icp canister call "$canister" "$method" "$args")" \
     || fail "icp canister call failed"
   printf '%s\n' "$reply"
-  printf '%s' "$reply" | grep -qF 'Hello, ICP!' \
-    || fail "unexpected greet reply: $reply"
+  printf '%s' "$reply" | grep -qF "$expect" \
+    || fail "expected reply to contain '$expect', got: $reply"
 }
 
 # http_check <canister-name>
@@ -108,38 +109,36 @@ http_check() {
   [ -s /tmp/frontend-body ] || fail "frontend returned an empty body"
 }
 
-# canister_specs <project-dir>
-# Prints one "name<TAB>recipe-type" line per canister in the rendered
-# project. icp.yaml entries are either inline objects (`name: ...`) or paths
-# to directories containing a canister.yaml.
-canister_specs() {
-  local dir="$1" count i entry
-  count="$(yq -r '.canisters | length' "$dir/icp.yaml")"
+# verify_canisters <template> <project-name>
+# Runs the checks configured in matrix.yaml for each of the template's
+# canisters: a gateway curl if `curl: true`, a canister call if `call` is
+# configured (both, if both are present).
+verify_canisters() {
+  local template="$1" project="$2"
+  local base count i name
+  count="$(yq -r ".templates.\"$template\".canisters | length" "$CONFIG")"
+  [ "$count" -gt 0 ] 2>/dev/null \
+    || fail "no canister checks configured for template '$template' in $CONFIG"
+
   for ((i = 0; i < count; i++)); do
-    if [ "$(yq -r ".canisters[$i] | type" "$dir/icp.yaml")" = "!!map" ]; then
-      yq -r ".canisters[$i] | .name + \"\t\" + .recipe.type" "$dir/icp.yaml"
-    else
-      entry="$(yq -r ".canisters[$i]" "$dir/icp.yaml")"
-      [ -f "$dir/$entry/canister.yaml" ] \
-        || fail "icp.yaml references '$entry' but $entry/canister.yaml is missing"
-      yq -r '.name + "\t" + .recipe.type' "$dir/$entry/canister.yaml"
+    base=".templates.\"$template\".canisters[$i]"
+    name="$(yq -r "$base.name" "$CONFIG")"
+    [ -n "$name" ] && [ "$name" != "null" ] \
+      || fail "canister #$i of template '$template' has no name in $CONFIG"
+    name="${name//"{{project-name}}"/$project}"
+
+    if [ "$(yq -r "$base.curl // false" "$CONFIG")" = "true" ]; then
+      http_check "$name"
+    fi
+    if [ "$(yq -r "$base.call" "$CONFIG")" != "null" ]; then
+      call_check "$name" \
+        "$(yq -r "$base.call.method" "$CONFIG")" \
+        "$(yq -r "$base.call.args" "$CONFIG")" \
+        "$(yq -r "$base.call.expect" "$CONFIG")"
     fi
   done
 }
 
-# verify_canisters <project-dir>
-# Exercises every deployed canister: asset canisters get the HTTP check,
-# everything else gets the greet call.
-verify_canisters() {
-  local dir="$1" name type
-  while IFS=$'\t' read -r name type; do
-    case "$type" in
-      *asset-canister*) http_check "$name" ;;
-      *) call_greet "$name" ;;
-    esac
-  done < <(canister_specs "$dir")
-}
-
 # assert_hello_world_layout <project-dir>
 # The hello-world pre-hook renames the chosen variant dirs to backend/ and
 # frontend/; the cargo-generate conditionals drop the unchosen ones.
@@ -173,7 +172,7 @@ run_permutation() {
     build
     network_start
     deploy
-    verify_canisters .
+    verify_canisters "$template" "$project"
   ) || rc=$?
 
   icp network stop --project-root-override "$dir/$project" >/dev/null 2>&1 || true
diff --git a/tests/matrix.txt b/tests/matrix.txt
deleted file mode 100644
index 4bbe2d3..0000000
--- a/tests/matrix.txt
+++ /dev/null
@@ -1,11 +0,0 @@
-# Template test matrix — one permutation per line:
-#   <template> [key=value ...]
-# Values are passed to `icp new` as -d defines; omitted variables use the
-# template defaults. The permutation id is the template name plus the values,
-# e.g. `hello-world:motoko:react`.
-rust
-motoko
-hello-world backend_type=motoko frontend_type=react
-hello-world backend_type=motoko frontend_type=vue
-hello-world backend_type=rust frontend_type=react
-hello-world backend_type=rust frontend_type=vue
diff --git a/tests/matrix.yaml b/tests/matrix.yaml
new file mode 100644
index 0000000..fff5f4c
--- /dev/null
+++ b/tests/matrix.yaml
@@ -0,0 +1,51 @@
+# Template test configuration.
+#
+# templates.<name>.permutations — list of variable sets passed to `icp new`
+#   as --define; an empty entry ({}) renders with the template defaults.
+#   Each permutation gets a test run. The permutation id used for filtering
+#   and reporting is the template name plus the values, e.g.
+#   `hello-world:motoko:react`.
+#
+# templates.<name>.canisters — what to verify after deploy, per canister:
+#   name — canister name as deployed; the literal {{project-name}} is
+#          replaced with the rendered project's name.
+#   curl — if true, curl the canister through the gateway
+#          (http://<name>.local.localhost:8000/) and expect HTTP 200.
+#   call — call `icp canister call <name> <method> '<args>'` and expect the
+#          reply to contain `expect`.
+
+templates:
+  rust:
+    permutations:
+      - {}
+    canisters:
+      - name: backend
+        call:
+          method: greet
+          args: '("ICP")'
+          expect: 'Hello, ICP!'
+
+  motoko:
+    permutations:
+      - {}
+    canisters:
+      - name: '{{project-name}}'
+        call:
+          method: greet
+          args: '("ICP")'
+          expect: 'Hello, ICP!'
+
+  hello-world:
+    permutations:
+      - { backend_type: motoko, frontend_type: react }
+      - { backend_type: motoko, frontend_type: vue }
+      - { backend_type: rust, frontend_type: react }
+      - { backend_type: rust, frontend_type: vue }
+    canisters:
+      - name: backend
+        call:
+          method: greet
+          args: '("ICP")'
+          expect: 'Hello, ICP!'
+      - name: frontend
+        curl: true
diff --git a/tests/run.sh b/tests/run.sh
index 108458c..7b6702a 100755
--- a/tests/run.sh
+++ b/tests/run.sh
@@ -8,7 +8,6 @@
 set -euo pipefail
 
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
-MATRIX_FILE="$(dirname "${BASH_SOURCE[0]}")/matrix.txt"
 
 FAIL_FAST=0
 FILTERS=()
@@ -38,29 +37,39 @@ prepare_template_source "$WORKDIR/template-src"
 RESULTS=()
 overall=0
 ran=0
+done_loop=0
 
-while read -r template defines; do
-  [[ -z "$template" || "$template" == \#* ]] && continue
+while IFS= read -r template; do
+  [ "$done_loop" -eq 1 ] && break
+  pcount="$(yq -r ".templates.\"$template\".permutations | length" "$CONFIG")"
+  [ "$pcount" -gt 0 ] 2>/dev/null \
+    || fail "template '$template' has no permutations in $CONFIG"
 
-  read -ra define_args <<<"${defines:-}"
-  id="$template"
-  for kv in ${define_args[@]+"${define_args[@]}"}; do
-    id+=":${kv#*=}"
-  done
+  for ((p = 0; p < pcount; p++)); do
+    define_args=()
+    while IFS= read -r kv; do
+      [ -n "$kv" ] && define_args+=("$kv")
+    done < <(yq -r ".templates.\"$template\".permutations[$p] | to_entries | .[] | .key + \"=\" + .value" "$CONFIG")
+
+    id="$template"
+    for kv in ${define_args[@]+"${define_args[@]}"}; do
+      id+=":${kv#*=}"
+    done
 
-  matches_filter "$id" || continue
-  ran=$((ran + 1))
+    matches_filter "$id" || continue
+    ran=$((ran + 1))
 
-  printf '\n========== %s ==========\n' "$id"
-  start="$(date +%s)"
-  if run_permutation "$id" "$template" ${define_args[@]+"${define_args[@]}"}; then
-    RESULTS+=("PASS  $id  ($(($(date +%s) - start))s)")
-  else
-    RESULTS+=("FAIL  $id  ($(($(date +%s) - start))s)")
-    overall=1
-    [ "$FAIL_FAST" -eq 1 ] && break
-  fi
-done <"$MATRIX_FILE"
+    printf '\n========== %s ==========\n' "$id"
+    start="$(date +%s)"
+    if run_permutation "$id" "$template" ${define_args[@]+"${define_args[@]}"}; then
+      RESULTS+=("PASS  $id  ($(($(date +%s) - start))s)")
+    else
+      RESULTS+=("FAIL  $id  ($(($(date +%s) - start))s)")
+      overall=1
+      [ "$FAIL_FAST" -eq 1 ] && { done_loop=1; break; }
+    fi
+  done
+done < <(yq -r '.templates | keys | .[]' "$CONFIG")
 
 [ "$ran" -gt 0 ] || fail "no permutations matched filter: ${FILTERS[*]:-<none>}"