From 87542fbd6cedc9e47c8317429bb584cc8e6e7a8b Mon Sep 17 00:00:00 2001 From: Adrian Moldovan <3854374+adimoldovan@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:02:12 +0300 Subject: [PATCH 01/11] Build/Test Tools: Check the upgraded site over HTTP. Upgrade Tests end at `wp core version`, which reads a string off the filesystem. No step in the job makes an HTTP request. A site that upgrades to the right version and then fatals on every page still reports green. Serve the upgraded site with `wp server` and check what a working install must produce: the version in the generator tag, the default post, a 404 for a missing one, the REST index, the login form, an authentication cookie from a real login, and the admin bar on the dashboard. A check for a missing error string breaks as soon as someone rewords the string. This skips multisite. The network domain lives in `wp-config.php` and in the `site` and `blogs` tables. WordPress strips only `:80` and `:443` from the host before it matches a network, so a move to another port needs more than an option update. --- .../workflows/reusable-upgrade-testing.yml | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/.github/workflows/reusable-upgrade-testing.yml b/.github/workflows/reusable-upgrade-testing.yml index 169abe20006a4..ed248c101c85c 100644 --- a/.github/workflows/reusable-upgrade-testing.yml +++ b/.github/workflows/reusable-upgrade-testing.yml @@ -56,6 +56,7 @@ jobs: # - Updates to the version of WordPress being tested. # - Updates the database. # - Checks the version of WordPress after the upgrade. + # - Serves the upgraded site and confirms it renders. upgrade-tests: name: PHP ${{ inputs.php }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }}${{ inputs.multisite && ' multisite' || '' }} permissions: @@ -135,3 +136,66 @@ jobs: - name: Post-upgrade version check run: wp core version + + # `wp core version` reads a string off the filesystem, and nothing else in this job makes an + # HTTP request, so a site that upgrades and then fatals on every page still reports green. + # Every assertion below is for something a working install must produce, not for the absence + # of an error string, which stops matching the moment the error is reworded. + # + # Multisite is excluded for now: the network domain is stored in `wp-config.php` and the + # `site` and `blogs` tables, and only `:80` and `:443` are stripped from the host when + # matching a network, so moving the site to another port takes more than an option update. + - name: Post-upgrade smoke check + if: ${{ ! inputs.multisite }} + run: | + wp option update home "${SITE_URL}" + wp option update siteurl "${SITE_URL}" + + wp server --host=127.0.0.1 --port=8889 > "${RUNNER_TEMP}/wp-server.log" 2>&1 & + trap '[ $? -eq 0 ] || cat "${RUNNER_TEMP}/wp-server.log"' EXIT + timeout 30 bash -c 'until curl --silent --output /dev/null "${SITE_URL}/"; do sleep 1; done' + + jar="${RUNNER_TEMP}/cookies.txt" + + # Fetches a path and fails the step unless the response body contains the given string. + check() { + local body + body="$(curl --silent --show-error --fail --cookie "${jar}" --cookie-jar "${jar}" "${SITE_URL}${1}")" + if ! grep -qF "${2}" <<< "${body}"; then + echo "::error::${1} did not contain: ${2}" + exit 1 + fi + echo "ok ${1}" + } + + version="$(wp core version)" + + check '/' "" + check '/?p=1' 'Hello world!' + check '/?rest_route=/' '"name":"Upgrade Test"' + check '/wp-admin/upgrade.php' 'No Update Required' + check '/wp-login.php' 'id="loginform"' + + status="$(curl --silent --output /dev/null --write-out '%{http_code}' "${SITE_URL}/?p=99999999")" + if [ "${status}" != '404' ]; then + echo "::error::a missing post returned ${status}, expected 404" + exit 1 + fi + echo "ok 404 handling" + + curl --silent --show-error --fail --output /dev/null \ + --cookie "${jar}" --cookie-jar "${jar}" \ + --data-urlencode 'log=admin' --data-urlencode 'pwd=password' \ + "${SITE_URL}/wp-login.php" + if ! grep -q 'wordpress_logged_in_' "${jar}"; then + echo '::error::logging in did not set an authentication cookie' + exit 1 + fi + echo 'ok login' + + # The admin bar only renders for a logged-in user, and /wp-admin/ redirects when logged out. + check '/wp-admin/' 'id="wpadminbar"' + env: + SITE_URL: http://127.0.0.1:8889 + # PHP's built-in server is single threaded, and WordPress makes loopback requests. + PHP_CLI_SERVER_WORKERS: 4 From 7abc187e081bbbc535165389fae5e728344592df Mon Sep 17 00:00:00 2001 From: Adrian Moldovan <3854374+adimoldovan@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:13:42 +0300 Subject: [PATCH 02/11] Build/Test Tools: Trim the smoke check comments. Drop the block above the step. Keep one line on the `if`, which says why the step skips multisite. --- .github/workflows/reusable-upgrade-testing.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/reusable-upgrade-testing.yml b/.github/workflows/reusable-upgrade-testing.yml index ed248c101c85c..1d91f2962b93f 100644 --- a/.github/workflows/reusable-upgrade-testing.yml +++ b/.github/workflows/reusable-upgrade-testing.yml @@ -137,15 +137,8 @@ jobs: - name: Post-upgrade version check run: wp core version - # `wp core version` reads a string off the filesystem, and nothing else in this job makes an - # HTTP request, so a site that upgrades and then fatals on every page still reports green. - # Every assertion below is for something a working install must produce, not for the absence - # of an error string, which stops matching the moment the error is reworded. - # - # Multisite is excluded for now: the network domain is stored in `wp-config.php` and the - # `site` and `blogs` tables, and only `:80` and `:443` are stripped from the host when - # matching a network, so moving the site to another port takes more than an option update. - name: Post-upgrade smoke check + # A network keeps its domain in wp-config.php and the database, not just an option. if: ${{ ! inputs.multisite }} run: | wp option update home "${SITE_URL}" From 17e47e293bad54bad5e6216b695acb6c534c9e88 Mon Sep 17 00:00:00 2001 From: Adrian Moldovan <3854374+adimoldovan@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:20:10 +0300 Subject: [PATCH 03/11] Build/Test Tools: Stop the smoke check deadlocking on cron. Six of 264 upgrade jobs failed. A request hung for 30 seconds and died: Fatal error: Maximum execution time of 30+2 seconds exceeded (terminated) in wp-includes/rest-api.php on line 2820 WordPress cron spawns a request back to the same server, and PHP's built-in server has nothing left to serve it. Four worker processes did not prevent this, and that mode is experimental: two jobs also died with a segmentation fault. Turn cron off instead, and drop the workers. Give every request a 30 second limit, so a hang that slips through fails the step instead of spending the 20 minute job timeout. Let the outer shell expand the URL in the wait loop. actionlint reads the single-quoted form as shellcheck SC2016. --- .github/workflows/reusable-upgrade-testing.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/reusable-upgrade-testing.yml b/.github/workflows/reusable-upgrade-testing.yml index 1d91f2962b93f..639c0b2d4de17 100644 --- a/.github/workflows/reusable-upgrade-testing.yml +++ b/.github/workflows/reusable-upgrade-testing.yml @@ -143,17 +143,19 @@ jobs: run: | wp option update home "${SITE_URL}" wp option update siteurl "${SITE_URL}" + # Cron spawns a request back to this server, which then has nothing left to serve it. + wp config set DISABLE_WP_CRON true --raw wp server --host=127.0.0.1 --port=8889 > "${RUNNER_TEMP}/wp-server.log" 2>&1 & trap '[ $? -eq 0 ] || cat "${RUNNER_TEMP}/wp-server.log"' EXIT - timeout 30 bash -c 'until curl --silent --output /dev/null "${SITE_URL}/"; do sleep 1; done' + timeout 30 bash -c "until curl --silent --output /dev/null \"${SITE_URL}/\"; do sleep 1; done" jar="${RUNNER_TEMP}/cookies.txt" # Fetches a path and fails the step unless the response body contains the given string. check() { local body - body="$(curl --silent --show-error --fail --cookie "${jar}" --cookie-jar "${jar}" "${SITE_URL}${1}")" + body="$(curl --silent --show-error --fail --max-time 30 --cookie "${jar}" --cookie-jar "${jar}" "${SITE_URL}${1}")" if ! grep -qF "${2}" <<< "${body}"; then echo "::error::${1} did not contain: ${2}" exit 1 @@ -169,14 +171,14 @@ jobs: check '/wp-admin/upgrade.php' 'No Update Required' check '/wp-login.php' 'id="loginform"' - status="$(curl --silent --output /dev/null --write-out '%{http_code}' "${SITE_URL}/?p=99999999")" + status="$(curl --silent --output /dev/null --max-time 30 --write-out '%{http_code}' "${SITE_URL}/?p=99999999")" if [ "${status}" != '404' ]; then echo "::error::a missing post returned ${status}, expected 404" exit 1 fi echo "ok 404 handling" - curl --silent --show-error --fail --output /dev/null \ + curl --silent --show-error --fail --max-time 30 --output /dev/null \ --cookie "${jar}" --cookie-jar "${jar}" \ --data-urlencode 'log=admin' --data-urlencode 'pwd=password' \ "${SITE_URL}/wp-login.php" @@ -190,5 +192,3 @@ jobs: check '/wp-admin/' 'id="wpadminbar"' env: SITE_URL: http://127.0.0.1:8889 - # PHP's built-in server is single threaded, and WordPress makes loopback requests. - PHP_CLI_SERVER_WORKERS: 4 From e4e6d777bd93c84bca734371f815b1d4111fe609 Mon Sep 17 00:00:00 2001 From: Adrian Moldovan <3854374+adimoldovan@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:02:38 +0300 Subject: [PATCH 04/11] Build/Test Tools: Harden the post-upgrade smoke check. Stop the background server when the step ends, and keep the step's exit code. Print the response body when a check finds the wrong content. Read the port from the site URL. Share the admin credentials with the install step. Match the generator tag without its closing slash. Stop the wait loop when the server dies. --- .../workflows/reusable-upgrade-testing.yml | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/.github/workflows/reusable-upgrade-testing.yml b/.github/workflows/reusable-upgrade-testing.yml index 639c0b2d4de17..ce16ad692c206 100644 --- a/.github/workflows/reusable-upgrade-testing.yml +++ b/.github/workflows/reusable-upgrade-testing.yml @@ -64,6 +64,10 @@ jobs: runs-on: ${{ inputs.os }} timeout-minutes: 20 + env: + WP_ADMIN_USER: admin + WP_ADMIN_PASSWORD: password + services: database: image: ${{ inputs.db-type }}:${{ inputs.db-version }} @@ -98,8 +102,8 @@ jobs: - name: Install WordPress run: | wp core ${{ inputs.multisite && 'multisite-install' || 'install' }} \ - --url=http://localhost/ --title="Upgrade Test" --admin_user=admin \ - --admin_password=password --admin_email=me@example.org --skip-email + --url=http://localhost/ --title="Upgrade Test" --admin_user="${WP_ADMIN_USER}" \ + --admin_password="${WP_ADMIN_PASSWORD}" --admin_email=me@example.org --skip-email - name: Pre-upgrade version check run: wp core version @@ -146,9 +150,19 @@ jobs: # Cron spawns a request back to this server, which then has nothing left to serve it. wp config set DISABLE_WP_CRON true --raw - wp server --host=127.0.0.1 --port=8889 > "${RUNNER_TEMP}/wp-server.log" 2>&1 & - trap '[ $? -eq 0 ] || cat "${RUNNER_TEMP}/wp-server.log"' EXIT - timeout 30 bash -c "until curl --silent --output /dev/null \"${SITE_URL}/\"; do sleep 1; done" + wp server --host=127.0.0.1 --port="${SITE_URL##*:}" > "${RUNNER_TEMP}/wp-server.log" 2>&1 & + server_pid=$! + + stop_server() { + local code=$? + pkill -P "${server_pid}" || true + kill "${server_pid}" 2> /dev/null || true + [ "${code}" -eq 0 ] || cat "${RUNNER_TEMP}/wp-server.log" + exit "${code}" + } + trap stop_server EXIT + + timeout 30 bash -c "until curl --silent --output /dev/null \"${SITE_URL}/\"; do kill -0 ${server_pid} 2> /dev/null || exit 1; sleep 1; done" jar="${RUNNER_TEMP}/cookies.txt" @@ -158,6 +172,7 @@ jobs: body="$(curl --silent --show-error --fail --max-time 30 --cookie "${jar}" --cookie-jar "${jar}" "${SITE_URL}${1}")" if ! grep -qF "${2}" <<< "${body}"; then echo "::error::${1} did not contain: ${2}" + head -n 20 <<< "${body}" exit 1 fi echo "ok ${1}" @@ -165,7 +180,7 @@ jobs: version="$(wp core version)" - check '/' "" + check '/' "content=\"WordPress ${version}\"" check '/?p=1' 'Hello world!' check '/?rest_route=/' '"name":"Upgrade Test"' check '/wp-admin/upgrade.php' 'No Update Required' @@ -180,7 +195,7 @@ jobs: curl --silent --show-error --fail --max-time 30 --output /dev/null \ --cookie "${jar}" --cookie-jar "${jar}" \ - --data-urlencode 'log=admin' --data-urlencode 'pwd=password' \ + --data-urlencode "log=${WP_ADMIN_USER}" --data-urlencode "pwd=${WP_ADMIN_PASSWORD}" \ "${SITE_URL}/wp-login.php" if ! grep -q 'wordpress_logged_in_' "${jar}"; then echo '::error::logging in did not set an authentication cookie' From 66ff167ce76ac15a15eaf49ada05c0460ad54793 Mon Sep 17 00:00:00 2001 From: Adrian Moldovan <3854374+adimoldovan@users.noreply.github.com> Date: Tue, 1 Sep 2026 18:54:17 +0300 Subject: [PATCH 05/11] Build/Test Tools: Make the smoke check report what broke. Every marker sat near the top of the response. A fatal in the second half of a page leaves an HTTP 200 with half a document, and the check passed. Each HTML check now also looks for the closing tag, and the REST index must parse as whole JSON. Neither marker depends on the wording of an error message. A failed request printed almost nothing. All requests now go through one helper that keeps the body and the status code. A response that fails prints what the site returned. The login POST fails the same way as the other checks. An empty reply came back as a bare curl error. The step now checks whether the server still runs. A page that kills the PHP process says so and prints the exit status. The step reads the server log at the end. A fatal during shutdown leaves every response whole and the site broken. --- .../workflows/reusable-upgrade-testing.yml | 117 ++++++++++++------ 1 file changed, 79 insertions(+), 38 deletions(-) diff --git a/.github/workflows/reusable-upgrade-testing.yml b/.github/workflows/reusable-upgrade-testing.yml index ce16ad692c206..01b9816048876 100644 --- a/.github/workflows/reusable-upgrade-testing.yml +++ b/.github/workflows/reusable-upgrade-testing.yml @@ -145,65 +145,106 @@ jobs: # A network keeps its domain in wp-config.php and the database, not just an option. if: ${{ ! inputs.multisite }} run: | - wp option update home "${SITE_URL}" - wp option update siteurl "${SITE_URL}" + site_url="http://127.0.0.1:${SITE_PORT}" + server_log="${RUNNER_TEMP}/wp-server.log" + jar="${RUNNER_TEMP}/cookies.txt" + response="${RUNNER_TEMP}/response.txt" + code="${RUNNER_TEMP}/http-code.txt" + + # Moving the site and turning off cron are not undone, so this step has to stay last. + wp option update home "${site_url}" + wp option update siteurl "${site_url}" # Cron spawns a request back to this server, which then has nothing left to serve it. wp config set DISABLE_WP_CRON true --raw - wp server --host=127.0.0.1 --port="${SITE_URL##*:}" > "${RUNNER_TEMP}/wp-server.log" 2>&1 & + wp server --host=127.0.0.1 --port="${SITE_PORT}" > "${server_log}" 2>&1 & server_pid=$! stop_server() { - local code=$? + local status=$? pkill -P "${server_pid}" || true kill "${server_pid}" 2> /dev/null || true - [ "${code}" -eq 0 ] || cat "${RUNNER_TEMP}/wp-server.log" - exit "${code}" + [ "${status}" -eq 0 ] || cat "${server_log}" + exit "${status}" } trap stop_server EXIT - timeout 30 bash -c "until curl --silent --output /dev/null \"${SITE_URL}/\"; do kill -0 ${server_pid} 2> /dev/null || exit 1; sleep 1; done" + timeout 30 bash -c "until curl --silent --output /dev/null \"${site_url}/\"; do kill -0 ${server_pid} 2> /dev/null || exit 1; sleep 1; done" - jar="${RUNNER_TEMP}/cookies.txt" + # Prints what came back and fails the step. + fail() { + echo "::error::${1}" + head -n 20 "${response}" + exit 1 + } - # Fetches a path and fails the step unless the response body contains the given string. - check() { - local body - body="$(curl --silent --show-error --fail --max-time 30 --cookie "${jar}" --cookie-jar "${jar}" "${SITE_URL}${1}")" - if ! grep -qF "${2}" <<< "${body}"; then - echo "::error::${1} did not contain: ${2}" - head -n 20 <<< "${body}" - exit 1 + # Requests a path, leaving the body in ${response} and the status in ${http_code}. An + # HTTP error is a result and comes back to the caller. No reply at all is not: the + # step says whether the server is still up, because that tells the two causes apart. + fetch() { + local path="${1}" status=0 + shift + : > "${response}" + curl --silent --show-error --max-time 30 --output "${response}" \ + --write-out '%{http_code}' --cookie "${jar}" --cookie-jar "${jar}" \ + "$@" "${site_url}${path}" > "${code}" || status=$? + http_code="$(cat "${code}")" + if [ "${status}" -ne 0 ] && [ "${status}" -ne 22 ]; then + if ! kill -0 "${server_pid}" 2> /dev/null; then + # 139 here means a signal 11, so the page brought the interpreter down with it. + server_status=0 + wait "${server_pid}" || server_status=$? + fail "${path} took the server down, which left ${server_status}" + fi + fail "${path} got no reply, curl exited ${status}" fi - echo "ok ${1}" + return "${status}" + } + + # Fails the step unless the response arrives whole and contains every given string. + check() { + local path="${1}" marker status=0 + shift + fetch "${path}" --fail-with-body || status=$? + [ "${status}" -eq 0 ] || fail "${path} returned HTTP ${http_code}" + for marker in "$@"; do + grep -qF "${marker}" "${response}" || fail "${path} did not contain: ${marker}" + done + echo "ok ${path}" } version="$(wp core version)" - check '/' "content=\"WordPress ${version}\"" - check '/?p=1' 'Hello world!' + # A fatal after the first bytes leaves a 200 holding half a page, so every check also + # asks for the end of the document. Unlike an error string, that cannot be reworded. + check '/' "content=\"WordPress ${version}\"" '' + check '/?p=1' 'Hello world!' '' + check '/?rest_route=/' '"name":"Upgrade Test"' - check '/wp-admin/upgrade.php' 'No Update Required' - check '/wp-login.php' 'id="loginform"' + jq -e . "${response}" > /dev/null || fail 'the REST index was not whole JSON' - status="$(curl --silent --output /dev/null --max-time 30 --write-out '%{http_code}' "${SITE_URL}/?p=99999999")" - if [ "${status}" != '404' ]; then - echo "::error::a missing post returned ${status}, expected 404" - exit 1 - fi - echo "ok 404 handling" - - curl --silent --show-error --fail --max-time 30 --output /dev/null \ - --cookie "${jar}" --cookie-jar "${jar}" \ - --data-urlencode "log=${WP_ADMIN_USER}" --data-urlencode "pwd=${WP_ADMIN_PASSWORD}" \ - "${SITE_URL}/wp-login.php" - if ! grep -q 'wordpress_logged_in_' "${jar}"; then - echo '::error::logging in did not set an authentication cookie' - exit 1 - fi + check '/wp-admin/upgrade.php' 'No Update Required' '' + check '/wp-login.php' 'id="loginform"' '' + + fetch '/?p=99999999' + [ "${http_code}" = '404' ] || fail "a missing post returned ${http_code}, expected 404" + echo 'ok 404 handling' + + login_status=0 + fetch '/wp-login.php' --fail-with-body \ + --data-urlencode "log=${WP_ADMIN_USER}" --data-urlencode "pwd=${WP_ADMIN_PASSWORD}" || login_status=$? + [ "${login_status}" -eq 0 ] || fail "logging in returned HTTP ${http_code}" + grep -q 'wordpress_logged_in_' "${jar}" || fail 'logging in did not set an authentication cookie' echo 'ok login' # The admin bar only renders for a logged-in user, and /wp-admin/ redirects when logged out. - check '/wp-admin/' 'id="wpadminbar"' + check '/wp-admin/' 'id="wpadminbar"' '' + + # A fatal that lands after the markers, or during shutdown, still leaves a broken site. + if grep -qE 'Fatal error|Uncaught|Segmentation fault' "${server_log}"; then + echo '::error::the server logged a fatal error' + exit 1 + fi + echo 'ok server log' env: - SITE_URL: http://127.0.0.1:8889 + SITE_PORT: 8889 From 4adce2469e0c7333775ac8f1d8648dd2b4b7097e Mon Sep 17 00:00:00 2001 From: Adrian Moldovan <3854374+adimoldovan@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:04:48 +0300 Subject: [PATCH 06/11] Build/Test Tools: Report why a smoke check request got no reply. The step reported an empty reply as a bare curl error. It now reports two facts: whether the server process is still up, and whether anything still listens on the port. The process that holds the port is a grandchild of the step, so one fact alone cannot tell the two causes apart. The wait for the server to start had no message. A server that never binds now says so, instead of leaving a bare exit 124 in the log. The REST index check read a scratch file that the previous request filled. It now makes its own request and asserts the parsed value, so line order no longer decides what it reads. The 404 page must arrive whole, as every other page must. A missing server log can no longer replace the exit status the step meant to report. --- .../workflows/reusable-upgrade-testing.yml | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/workflows/reusable-upgrade-testing.yml b/.github/workflows/reusable-upgrade-testing.yml index 01b9816048876..102cd5d4e274f 100644 --- a/.github/workflows/reusable-upgrade-testing.yml +++ b/.github/workflows/reusable-upgrade-testing.yml @@ -164,12 +164,13 @@ jobs: local status=$? pkill -P "${server_pid}" || true kill "${server_pid}" 2> /dev/null || true - [ "${status}" -eq 0 ] || cat "${server_log}" + [ "${status}" -eq 0 ] || cat "${server_log}" || true exit "${status}" } trap stop_server EXIT - timeout 30 bash -c "until curl --silent --output /dev/null \"${site_url}/\"; do kill -0 ${server_pid} 2> /dev/null || exit 1; sleep 1; done" + timeout 30 bash -c "until curl --silent --output /dev/null \"${site_url}/\"; do kill -0 ${server_pid} 2> /dev/null || exit 1; sleep 1; done" \ + || { echo '::error::the server never answered'; exit 1; } # Prints what came back and fails the step. fail() { @@ -190,13 +191,18 @@ jobs: "$@" "${site_url}${path}" > "${code}" || status=$? http_code="$(cat "${code}")" if [ "${status}" -ne 0 ] && [ "${status}" -ne 22 ]; then - if ! kill -0 "${server_pid}" 2> /dev/null; then - # 139 here means a signal 11, so the page brought the interpreter down with it. + # Two facts, because the process holding the port is a grandchild of this one: + # a status of 128 and up names a signal, and the socket says who is left listening. + if kill -0 "${server_pid}" 2> /dev/null; then + server_state='is up' + else server_status=0 wait "${server_pid}" || server_status=$? - fail "${path} took the server down, which left ${server_status}" + server_state="left ${server_status}" fi - fail "${path} got no reply, curl exited ${status}" + port_state='nothing listens on the port' + ss -ltnH "sport = :${SITE_PORT}" | grep -q . && port_state='the port still listens' + fail "${path} got no reply, curl exited ${status}. The server ${server_state}, ${port_state}." fi return "${status}" } @@ -220,21 +226,24 @@ jobs: check '/' "content=\"WordPress ${version}\"" '' check '/?p=1' 'Hello world!' '' - check '/?rest_route=/' '"name":"Upgrade Test"' - jq -e . "${response}" > /dev/null || fail 'the REST index was not whole JSON' + fetch '/?rest_route=/' --fail-with-body || fail "/?rest_route=/ returned HTTP ${http_code}" + jq -e '.name == "Upgrade Test"' "${response}" > /dev/null \ + || fail 'the REST index was not the expected JSON' + echo 'ok /?rest_route=/' check '/wp-admin/upgrade.php' 'No Update Required' '' check '/wp-login.php' 'id="loginform"' '' fetch '/?p=99999999' [ "${http_code}" = '404' ] || fail "a missing post returned ${http_code}, expected 404" + grep -qF '' "${response}" || fail 'the 404 page was cut short' echo 'ok 404 handling' login_status=0 fetch '/wp-login.php' --fail-with-body \ --data-urlencode "log=${WP_ADMIN_USER}" --data-urlencode "pwd=${WP_ADMIN_PASSWORD}" || login_status=$? [ "${login_status}" -eq 0 ] || fail "logging in returned HTTP ${http_code}" - grep -q 'wordpress_logged_in_' "${jar}" || fail 'logging in did not set an authentication cookie' + grep -qF 'wordpress_logged_in_' "${jar}" || fail 'logging in did not set an authentication cookie' echo 'ok login' # The admin bar only renders for a logged-in user, and /wp-admin/ redirects when logged out. From ca82687bf2c1c7f405624640db404bf46cc26ea1 Mon Sep 17 00:00:00 2001 From: Adrian Moldovan <3854374+adimoldovan@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:25:29 +0300 Subject: [PATCH 07/11] Build/Test Tools: Serve the smoke check with php -S. `wp server` wraps `php -S` in a router and puts the server two processes below the step. Every path the step asks for resolves without that router: `/` and `/wp-admin/` reach their `index.php` as directory indexes, and the rest are files on disk. Dropping it removes a layer. The server is now the step's own child, so when a request gets no reply the step reports the status the server exited with, rather than the status of the process that started it. --- .../workflows/reusable-upgrade-testing.yml | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/reusable-upgrade-testing.yml b/.github/workflows/reusable-upgrade-testing.yml index 102cd5d4e274f..2d730a07d0061 100644 --- a/.github/workflows/reusable-upgrade-testing.yml +++ b/.github/workflows/reusable-upgrade-testing.yml @@ -157,12 +157,11 @@ jobs: # Cron spawns a request back to this server, which then has nothing left to serve it. wp config set DISABLE_WP_CRON true --raw - wp server --host=127.0.0.1 --port="${SITE_PORT}" > "${server_log}" 2>&1 & + php -S "127.0.0.1:${SITE_PORT}" -t . > "${server_log}" 2>&1 & server_pid=$! stop_server() { local status=$? - pkill -P "${server_pid}" || true kill "${server_pid}" 2> /dev/null || true [ "${status}" -eq 0 ] || cat "${server_log}" || true exit "${status}" @@ -191,18 +190,14 @@ jobs: "$@" "${site_url}${path}" > "${code}" || status=$? http_code="$(cat "${code}")" if [ "${status}" -ne 0 ] && [ "${status}" -ne 22 ]; then - # Two facts, because the process holding the port is a grandchild of this one: - # a status of 128 and up names a signal, and the socket says who is left listening. if kill -0 "${server_pid}" 2> /dev/null; then - server_state='is up' - else - server_status=0 - wait "${server_pid}" || server_status=$? - server_state="left ${server_status}" + fail "${path} got no reply, curl exited ${status}, and the server is still up" fi - port_state='nothing listens on the port' - ss -ltnH "sport = :${SITE_PORT}" | grep -q . && port_state='the port still listens' - fail "${path} got no reply, curl exited ${status}. The server ${server_state}, ${port_state}." + # The server is this step's own child, so its status is the one that killed it: + # 128 and up is 128 plus the signal number. + server_status=0 + wait "${server_pid}" || server_status=$? + fail "${path} got no reply and the server exited ${server_status}" fi return "${status}" } From b4cbacccef33666c17079babfc671352cf4b10ec Mon Sep 17 00:00:00 2001 From: Adrian Moldovan <3854374+adimoldovan@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:16:17 +0300 Subject: [PATCH 08/11] Build/Test Tools: Serve the smoke check with nginx and PHP-FPM. PHP's built-in server answered the connection and then closed it with no reply, on about four jobs in every run. One process accepts the connection and runs WordPress, so when that process stalls, nothing answers. nginx now owns the listening socket, and PHP-FPM runs the site behind it. A worker that dies mid-request becomes an HTTP 502 the step reports, and the pool starts another. Both programs are on the runner image, and the step runs them as the runner user with its own config in RUNNER_TEMP. The step also blocks external HTTP. The update checks on /wp-admin/ left the runner for api.wordpress.org, which no check asked for. --- .../workflows/reusable-upgrade-testing.yml | 110 ++++++++++++++---- 1 file changed, 86 insertions(+), 24 deletions(-) diff --git a/.github/workflows/reusable-upgrade-testing.yml b/.github/workflows/reusable-upgrade-testing.yml index 2d730a07d0061..700fc95cf725d 100644 --- a/.github/workflows/reusable-upgrade-testing.yml +++ b/.github/workflows/reusable-upgrade-testing.yml @@ -146,41 +146,106 @@ jobs: if: ${{ ! inputs.multisite }} run: | site_url="http://127.0.0.1:${SITE_PORT}" - server_log="${RUNNER_TEMP}/wp-server.log" + fpm_conf="${RUNNER_TEMP}/php-fpm.conf" + fpm_log="${RUNNER_TEMP}/php-fpm.log" + fpm_sock="${RUNNER_TEMP}/php-fpm.sock" + nginx_conf="${RUNNER_TEMP}/nginx.conf" + nginx_log="${RUNNER_TEMP}/nginx.log" jar="${RUNNER_TEMP}/cookies.txt" response="${RUNNER_TEMP}/response.txt" code="${RUNNER_TEMP}/http-code.txt" + http_code='' # Moving the site and turning off cron are not undone, so this step has to stay last. wp option update home "${site_url}" wp option update siteurl "${site_url}" - # Cron spawns a request back to this server, which then has nothing left to serve it. + # A cron spawn and the update checks both make requests the checks did not ask for, + # and the update checks leave the runner for api.wordpress.org. wp config set DISABLE_WP_CRON true --raw + wp config set WP_HTTP_BLOCK_EXTERNAL true --raw + + # The pool is the step's own. The packaged one runs as www-data and needs root. + cat > "${fpm_conf}" < "${nginx_conf}" < "${server_log}" 2>&1 & - server_pid=$! + fpm_pid='' + nginx_pid='' stop_server() { local status=$? - kill "${server_pid}" 2> /dev/null || true - [ "${status}" -eq 0 ] || cat "${server_log}" || true + kill "${nginx_pid}" "${fpm_pid}" 2> /dev/null || true + wait "${nginx_pid}" "${fpm_pid}" 2> /dev/null || true + [ "${status}" -eq 0 ] || tail -n +1 "${nginx_log}" "${fpm_log}" || true exit "${status}" } trap stop_server EXIT - timeout 30 bash -c "until curl --silent --output /dev/null \"${site_url}/\"; do kill -0 ${server_pid} 2> /dev/null || exit 1; sleep 1; done" \ - || { echo '::error::the server never answered'; exit 1; } + # setup-php registers the unversioned name only when it installs PHP itself, and it + # installs nothing when the runner image already carries the version being tested. + command -v "php-fpm${PHP_VERSION}" > /dev/null \ + || { echo "::error::php-fpm${PHP_VERSION} is not on PATH"; exit 1; } + "php-fpm${PHP_VERSION}" --nodaemonize --fpm-config "${fpm_conf}" & + fpm_pid=$! + nginx -c "${nginx_conf}" -e "${nginx_log}" & + nginx_pid=$! + + # nginx answers a request before the pool is listening, and that answer is a 502. + # Neither that nor an unanswered request is a result a check can read. + deadline=$(( SECONDS + 30 )) + while :; do + # A poll that cannot connect prints 000 and exits 7, which would end the step here. + ready_code="$(curl --silent --max-time 5 --output /dev/null --write-out '%{http_code}' "${site_url}/" || true)" + [ "${ready_code}" = '000' ] || [ "${ready_code}" = '502' ] || break + [ "${SECONDS}" -lt "${deadline}" ] \ + || { echo "::error::the site did not come up, and the last status was ${ready_code}"; exit 1; } + sleep 1 + done # Prints what came back and fails the step. fail() { echo "::error::${1}" - head -n 20 "${response}" + tail -n 20 "${response}" exit 1 } # Requests a path, leaving the body in ${response} and the status in ${http_code}. An - # HTTP error is a result and comes back to the caller. No reply at all is not: the - # step says whether the server is still up, because that tells the two causes apart. + # HTTP error is a result and comes back to the caller. No reply at all is not. nginx + # answers a request even when PHP cannot, so no reply means nginx itself stopped. fetch() { local path="${1}" status=0 shift @@ -189,16 +254,8 @@ jobs: --write-out '%{http_code}' --cookie "${jar}" --cookie-jar "${jar}" \ "$@" "${site_url}${path}" > "${code}" || status=$? http_code="$(cat "${code}")" - if [ "${status}" -ne 0 ] && [ "${status}" -ne 22 ]; then - if kill -0 "${server_pid}" 2> /dev/null; then - fail "${path} got no reply, curl exited ${status}, and the server is still up" - fi - # The server is this step's own child, so its status is the one that killed it: - # 128 and up is 128 plus the signal number. - server_status=0 - wait "${server_pid}" || server_status=$? - fail "${path} got no reply and the server exited ${server_status}" - fi + [ "${status}" -eq 0 ] || [ "${status}" -eq 22 ] \ + || fail "${path} got no reply and curl exited ${status}" return "${status}" } @@ -209,7 +266,7 @@ jobs: fetch "${path}" --fail-with-body || status=$? [ "${status}" -eq 0 ] || fail "${path} returned HTTP ${http_code}" for marker in "$@"; do - grep -qF "${marker}" "${response}" || fail "${path} did not contain: ${marker}" + grep -qF -e "${marker}" "${response}" || fail "${path} did not contain: ${marker}" done echo "ok ${path}" } @@ -226,10 +283,12 @@ jobs: || fail 'the REST index was not the expected JSON' echo 'ok /?rest_route=/' + # The page prints this for a database that needs no upgrade and for one that holds no + # install at all. The checks above have already ruled the second one out. check '/wp-admin/upgrade.php' 'No Update Required' '' check '/wp-login.php' 'id="loginform"' '' - fetch '/?p=99999999' + fetch '/?p=99999999' || true [ "${http_code}" = '404' ] || fail "a missing post returned ${http_code}, expected 404" grep -qF '' "${response}" || fail 'the 404 page was cut short' echo 'ok 404 handling' @@ -245,10 +304,13 @@ jobs: check '/wp-admin/' 'id="wpadminbar"' '' # A fatal that lands after the markers, or during shutdown, still leaves a broken site. - if grep -qE 'Fatal error|Uncaught|Segmentation fault' "${server_log}"; then + # The pool always writes a start line, so an empty log means the check reads nothing. + [ -s "${fpm_log}" ] || { echo '::error::the server wrote no log'; exit 1; } + if grep -qE 'Fatal error|Uncaught|Segmentation fault|exited on signal' "${fpm_log}"; then echo '::error::the server logged a fatal error' exit 1 fi echo 'ok server log' env: + PHP_VERSION: ${{ inputs.php }} SITE_PORT: 8889 From 3d386982d6b63fe477492fc6415f90ca045f69af Mon Sep 17 00:00:00 2001 From: Adrian Moldovan <3854374+adimoldovan@users.noreply.github.com> Date: Wed, 2 Sep 2026 16:54:06 +0300 Subject: [PATCH 09/11] Build/Test Tools: Turn the JIT off for the smoke check pool. PHP workers crashed with SIGSEGV on about five percent of cold pools, on PHP 8.4 and 8.5, while they served the first request. A run of 1200 pools on the runner image found the cause: the runner sets opcache.jit to tracing with a 256M buffer. Stock PHP ships the JIT off, so the crash needs a setup no site runs. With the JIT off and opcache still on, none of 400 pools crashed. With the JIT on, 19 of 400 crashed. --- .github/workflows/reusable-upgrade-testing.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/reusable-upgrade-testing.yml b/.github/workflows/reusable-upgrade-testing.yml index 700fc95cf725d..0efa5dfb51c54 100644 --- a/.github/workflows/reusable-upgrade-testing.yml +++ b/.github/workflows/reusable-upgrade-testing.yml @@ -175,6 +175,9 @@ jobs: catch_workers_output = yes php_admin_flag[log_errors] = On php_admin_value[error_log] = ${fpm_log} + ; The runner turns the JIT on, which stock PHP does not. It crashes the worker on + ; about five percent of cold pools, so the check runs what a site runs instead. + php_admin_value[opcache.jit] = disable CONF cat > "${nginx_conf}" < Date: Wed, 2 Sep 2026 17:53:58 +0300 Subject: [PATCH 10/11] Build/Test Tools: Drop nginx and PHP-FPM from the smoke check. A core dump named the crash: every frame sat in opcache.so, and the server that produced it was php -S. opcache counts the built-in server as a web SAPI, not as the CLI, so opcache.enable_cli never kept it out and it ran the JIT the runner turns on. One bug hit both servers. With the JIT off, php -S crashed on none of 1600 cold starts, against 38 with it on. It serves every path the check asks for, so the pool, the server block, the socket and the second process all go. --- .../workflows/reusable-upgrade-testing.yml | 98 ++++--------------- 1 file changed, 21 insertions(+), 77 deletions(-) diff --git a/.github/workflows/reusable-upgrade-testing.yml b/.github/workflows/reusable-upgrade-testing.yml index 0efa5dfb51c54..73dd291b56fd9 100644 --- a/.github/workflows/reusable-upgrade-testing.yml +++ b/.github/workflows/reusable-upgrade-testing.yml @@ -146,96 +146,41 @@ jobs: if: ${{ ! inputs.multisite }} run: | site_url="http://127.0.0.1:${SITE_PORT}" - fpm_conf="${RUNNER_TEMP}/php-fpm.conf" - fpm_log="${RUNNER_TEMP}/php-fpm.log" - fpm_sock="${RUNNER_TEMP}/php-fpm.sock" - nginx_conf="${RUNNER_TEMP}/nginx.conf" - nginx_log="${RUNNER_TEMP}/nginx.log" + server_log="${RUNNER_TEMP}/server.log" jar="${RUNNER_TEMP}/cookies.txt" response="${RUNNER_TEMP}/response.txt" code="${RUNNER_TEMP}/http-code.txt" http_code='' - # Moving the site and turning off cron are not undone, so this step has to stay last. + # None of these three are undone, so this step has to stay last. wp option update home "${site_url}" wp option update siteurl "${site_url}" - # A cron spawn and the update checks both make requests the checks did not ask for, - # and the update checks leave the runner for api.wordpress.org. + # A cron spawn and the update checks both make requests no check asked for, and the + # update checks leave the runner for api.wordpress.org. wp config set DISABLE_WP_CRON true --raw wp config set WP_HTTP_BLOCK_EXTERNAL true --raw - # The pool is the step's own. The packaged one runs as www-data and needs root. - cat > "${fpm_conf}" < "${nginx_conf}" < "${server_log}" 2>&1 & + server_pid=$! stop_server() { local status=$? - kill "${nginx_pid}" "${fpm_pid}" 2> /dev/null || true - wait "${nginx_pid}" "${fpm_pid}" 2> /dev/null || true - [ "${status}" -eq 0 ] || tail -n +1 "${nginx_log}" "${fpm_log}" || true + kill "${server_pid}" 2> /dev/null || true + wait "${server_pid}" 2> /dev/null || true + [ "${status}" -eq 0 ] || cat "${server_log}" || true exit "${status}" } trap stop_server EXIT - # setup-php registers the unversioned name only when it installs PHP itself, and it - # installs nothing when the runner image already carries the version being tested. - command -v "php-fpm${PHP_VERSION}" > /dev/null \ - || { echo "::error::php-fpm${PHP_VERSION} is not on PATH"; exit 1; } - "php-fpm${PHP_VERSION}" --nodaemonize --fpm-config "${fpm_conf}" & - fpm_pid=$! - nginx -c "${nginx_conf}" -e "${nginx_log}" & - nginx_pid=$! - - # nginx answers a request before the pool is listening, and that answer is a 502. - # Neither that nor an unanswered request is a result a check can read. + # A poll that cannot connect prints 000 and exits 7, which would end the step here. deadline=$(( SECONDS + 30 )) while :; do - # A poll that cannot connect prints 000 and exits 7, which would end the step here. ready_code="$(curl --silent --max-time 5 --output /dev/null --write-out '%{http_code}' "${site_url}/" || true)" - [ "${ready_code}" = '000' ] || [ "${ready_code}" = '502' ] || break - [ "${SECONDS}" -lt "${deadline}" ] \ - || { echo "::error::the site did not come up, and the last status was ${ready_code}"; exit 1; } + [ "${ready_code}" = '000' ] || break + [ "${SECONDS}" -lt "${deadline}" ] || { echo '::error::the server never answered'; exit 1; } sleep 1 done @@ -246,9 +191,9 @@ jobs: exit 1 } - # Requests a path, leaving the body in ${response} and the status in ${http_code}. An - # HTTP error is a result and comes back to the caller. No reply at all is not. nginx - # answers a request even when PHP cannot, so no reply means nginx itself stopped. + # Requests a path, leaving the body in ${response} and the status in ${http_code}. + # An HTTP error is a result and comes back to the caller. No reply at all is not: + # the server is gone, and the log the trap prints says why. fetch() { local path="${1}" status=0 shift @@ -307,13 +252,12 @@ jobs: check '/wp-admin/' 'id="wpadminbar"' '' # A fatal that lands after the markers, or during shutdown, still leaves a broken site. - # The pool always writes a start line, so an empty log means the check reads nothing. - [ -s "${fpm_log}" ] || { echo '::error::the server wrote no log'; exit 1; } - if grep -qE 'Fatal error|Uncaught|Segmentation fault|exited on signal' "${fpm_log}"; then + # The server always writes a start line, so an empty log means the check reads nothing. + [ -s "${server_log}" ] || { echo '::error::the server wrote no log'; exit 1; } + if grep -qE 'Fatal error|Uncaught|Segmentation fault' "${server_log}"; then echo '::error::the server logged a fatal error' exit 1 fi echo 'ok server log' env: - PHP_VERSION: ${{ inputs.php }} SITE_PORT: 8889 From 9f0baf9522bf822de533a39d3d23b8ca7a4ba2c3 Mon Sep 17 00:00:00 2001 From: Adrian Moldovan <3854374+adimoldovan@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:40:21 +0300 Subject: [PATCH 11/11] Build/Test Tools: Drop two redundant smoke check requests. The login page GET repeated the login POST. The POST asks for the same URL and already fails when that page is broken. The single post view repeated the front page, which prints post titles. The front page check now asserts the title, so a site that renders but lost its content still fails. --- .github/workflows/reusable-upgrade-testing.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/reusable-upgrade-testing.yml b/.github/workflows/reusable-upgrade-testing.yml index 73dd291b56fd9..4da42ff39921d 100644 --- a/.github/workflows/reusable-upgrade-testing.yml +++ b/.github/workflows/reusable-upgrade-testing.yml @@ -223,8 +223,7 @@ jobs: # A fatal after the first bytes leaves a 200 holding half a page, so every check also # asks for the end of the document. Unlike an error string, that cannot be reworded. - check '/' "content=\"WordPress ${version}\"" '' - check '/?p=1' 'Hello world!' '' + check '/' "content=\"WordPress ${version}\"" 'Hello world!' '' fetch '/?rest_route=/' --fail-with-body || fail "/?rest_route=/ returned HTTP ${http_code}" jq -e '.name == "Upgrade Test"' "${response}" > /dev/null \ @@ -234,7 +233,6 @@ jobs: # The page prints this for a database that needs no upgrade and for one that holds no # install at all. The checks above have already ruled the second one out. check '/wp-admin/upgrade.php' 'No Update Required' '' - check '/wp-login.php' 'id="loginform"' '' fetch '/?p=99999999' || true [ "${http_code}" = '404' ] || fail "a missing post returned ${http_code}, expected 404"