test(appstore): cover the release Nextcloud setup and validate the unsigned package

.github/workflows/playwright.yml

@@ -51,6 +51,8 @@ jobs:
               - 'playwright.config.ts'
               - 'package.json'
               - 'package-lock.json'
+              - 'Makefile'
+              - '.nextcloud-release.yml'
 
   playwright:
     runs-on: ubuntu-latest
@@ -87,7 +89,8 @@ jobs:
           DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
             poppler-utils \
             libgd3 \
-            libc-client2007e
+            libc-client2007e \
+            make
 
       - name: Checkout server
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -162,6 +165,10 @@ jobs:
           npm ci --prefer-offline
           TESTING=true npm run build --if-present
 
+      - name: Package the app without signing
+        working-directory: apps/${{ env.APP_NAME }}
+        run: make appstore verify-appstore-package
+
       - name: Cache LibreSign binary payloads
         id: libresign-binaries-cache
         uses: actions/cache@v6

.github/workflows/release-metadata.yml

@@ -11,6 +11,8 @@ on:
       - 'package-lock.json'
       - 'docs/changelogs/**'
       - 'Makefile'
+      - 'tests/ci/**'
+      - '.github/workflows/release-metadata.yml'
   push:
     branches:
       - main
@@ -22,6 +24,8 @@ on:
       - 'package-lock.json'
       - 'docs/changelogs/**'
       - 'Makefile'
+      - 'tests/ci/**'
+      - '.github/workflows/release-metadata.yml'
 
 permissions:
   contents: read
@@ -47,3 +51,14 @@ jobs:
         env:
           EXPECTED_CHANGELOG: ${{ steps.release.outputs.changelog-path }}
         run: test "$(make -s print-release-changelog)" = "${EXPECTED_CHANGELOG}"
+
+      - name: Setup Bats-core
+        uses: bats-core/bats-action@77d6fb60505b4d0d1d73e48bd035b55074bbfb43 # 4.0.0
+        with:
+          support-install: false
+          assert-install: false
+          detik-install: false
+          file-install: false
+
+      - name: Test release packaging behavior
+        run: bats --print-output-on-failure tests/ci

Makefile

@@ -14,13 +14,14 @@ appstore_build_directory=$(CURDIR)/build/artifacts
 appstore_package_name=$(appstore_build_directory)/$(app_name)
 appstore_sign_dir=$(appstore_build_directory)/sign
 cert_dir=$(build_tools_directory)/certificates
+nextcloud_directory=$(CURDIR)/../nextcloud
 release_version=$(shell sed -n 's:.*<version>\([^<]*\)</version>.*:\1:p' appinfo/info.xml)
 release_major=$(word 1,$(subst ., ,$(release_version)))
 release_changelog=$(CURDIR)/docs/changelogs/changelog-$(release_major).md
 npm=$(shell which npm 2> /dev/null)
 composer=$(shell which composer 2> /dev/null)
-ifneq (,$(wildcard $(CURDIR)/../nextcloud/occ))
-	occ=php $(CURDIR)/../nextcloud/occ
+ifneq (,$(wildcard $(nextcloud_directory)/occ))
+	occ=php $(nextcloud_directory)/occ
 else ifneq (,$(wildcard $(CURDIR)/../../occ))
 	occ=php $(CURDIR)/../../occ
 endif
@@ -205,24 +206,7 @@ appstore: verify-release-metadata
 	cp tests/php/fixtures/pdfs/small_valid.pdf $(appstore_sign_dir)/$(app_name)/tests/php/fixtures
 
 	mkdir -p $(cert_dir)
-	if [ -f $(cert_dir)/$(app_name).key ] && [ "$(GITHUB_ACTIONS)" = "true" ]; then \
-		set -e; \
-		echo "⌛️ Starting Nextcloud setup..."; \
-		mkdir $(CURDIR)/../nextcloud/data; \
-		ln -s $(CURDIR) $(CURDIR)/../nextcloud/apps/libresign; \
-		$(occ) maintenance:install \
-			--verbose \
-			--database=sqlite \
-			--database-name=nextcloud \
-			--database-host=127.0.0.1 \
-			--database-user=root \
-			--database-pass=rootpassword \
-			--admin-user admin \
-			--admin-pass admin; \
-		$(occ) --version; \
-		$(occ) app:enable --force libresign; \
-		echo "🏁 Setup finished"; \
-	fi
+	$(MAKE) --no-print-directory _appstore-nextcloud-setup
 
 	if [ -f $(cert_dir)/$(app_name).key ]; then \
 		set -e; \
@@ -251,6 +235,35 @@ appstore: verify-release-metadata
 		openssl dgst -sha512 -sign $(cert_dir)/$(app_name).key $(appstore_package_name).tar.gz | openssl base64; \
 	fi
 
+# Internal step of appstore, kept as a target so it can be tested on its own
+# (tests/ci/). Prepares the Nextcloud instance that signs the release; it only
+# runs on GitHub Actions when the app private key is present.
+.PHONY: _appstore-nextcloud-setup
+_appstore-nextcloud-setup:
+	if [ -f $(cert_dir)/$(app_name).key ] && [ "$(GITHUB_ACTIONS)" = "true" ]; then \
+		set -e; \
+		echo "⌛️ Starting Nextcloud setup..."; \
+		mkdir $(nextcloud_directory)/data; \
+		ln -s $(CURDIR) $(nextcloud_directory)/apps/libresign; \
+		$(occ) maintenance:install \
+			--verbose \
+			--database=sqlite \
+			--database-name=nextcloud \
+			--database-host=127.0.0.1 \
+			--database-user=root \
+			--database-pass=rootpassword \
+			--admin-user admin \
+			--admin-pass admin; \
+		$(occ) --version; \
+		$(occ) app:enable --force libresign; \
+		echo "🏁 Setup finished"; \
+	fi
+
+# Package contract, mirroring package.required_paths and package.forbidden_paths
+# in .nextcloud-release.yml.
+appstore_required_paths=appinfo/info.xml composer css img js l10n lib templates vendor 3rdparty openapi.json openapi-administration.json openapi-full.json CHANGELOG.md
+appstore_forbidden_paths=.git .github .devcontainer .patches build docs node_modules playwright src vendor-bin img/screenshot 3rdparty/.git 3rdparty/.github 3rdparty/vendor 3rdparty/vendor-bin 3rdparty/scoper.inc.php
+
 .PHONY: verify-appstore-package
 verify-appstore-package: verify-release-metadata
 	test -f $(appstore_sign_dir)/$(app_name)/CHANGELOG.md
@@ -268,3 +281,13 @@ verify-appstore-package: verify-release-metadata
 		setup_signature_count=$$(tar -tzf $(appstore_package_name).tar.gz | grep -E -c '^$(app_name)/appinfo/install-.*\.json$$' || true); \
 		test "$$setup_signature_count" -eq 10 || (echo "Expected 10 setup integrity metadata files in app store package, found $$setup_signature_count" >&2; exit 1); \
 	fi
+	@package_paths=$$(tar -tzf $(appstore_package_name).tar.gz) || { echo "Unable to read $(appstore_package_name).tar.gz" >&2; exit 1; }; \
+	has_path() { printf '%s\n' "$$package_paths" | awk -v path="$(app_name)/$$1" '$$0 == path || index($$0, path "/") == 1 { found = 1 } END { exit !found }'; }; \
+	outside=$$(printf '%s\n' "$$package_paths" | grep -v '^$(app_name)/' || true); \
+	test -z "$$outside" || { echo "App store package has entries outside $(app_name)/: $$outside" >&2; exit 1; }; \
+	for path in $(appstore_required_paths); do \
+		has_path "$$path" || { echo "App store package is missing $$path" >&2; exit 1; }; \
+	done; \
+	for path in $(appstore_forbidden_paths); do \
+		! has_path "$$path" || { echo "App store package must not contain $$path" >&2; exit 1; }; \
+	done

tests/README.md

@@ -15,6 +15,7 @@ Use this file only as a repository map.
 - `tests/integration/`: Behat scenarios and support code.
 - `src/tests/`: frontend unit tests executed with Vitest.
 - `playwright/`: browser/end-to-end tests.
+- `tests/ci/`: [Bats](https://bats-core.readthedocs.io/) tests for the release packaging in the `Makefile`.
 
 Prefer the narrowest useful validation while implementing a change, then broaden before opening or updating a pull request.
 
@@ -37,4 +38,19 @@ vendor/bin/behat -dl
 vendor/bin/behat features/path/to/feature.feature:LINE -v
 ```
 
+The release packaging tests run the real `make appstore` and
+`make verify-appstore-package` against a temporary minimal app, with `occ`,
+`curl` and `openssl` stubbed. They need only Bats, GNU Make and tar, use no
+network and leave the repository untouched:
+
+```bash
+bats tests/ci
+```
+
+They cover when `appstore` initializes the Nextcloud instance that signs a
+release (only on GitHub Actions with the app private key present), that a
+failing initialization stops the build, and that the package keeps its
+required paths and excludes development ones. Pull requests to `main` also
+build the real unsigned package in the Playwright workflow.
+
 See the Developer Manual for linting, static analysis, full validation, and environment requirements.

tests/ci/appstore-nextcloud-setup.bats

@@ -0,0 +1,76 @@
+#!/usr/bin/env bats
+# SPDX-FileCopyrightText: 2026 LibreCode coop and contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# Behavior of the Nextcloud initialization that `make appstore` runs before
+# signing a release.
+
+load helpers
+
+setup() {
+	setup_appstore_test
+}
+
+@test "initializes Nextcloud on GitHub Actions when the signing key is present" {
+	with_signing_key
+
+	run_make true "$REPO_ROOT" _appstore-nextcloud-setup
+
+	[ "$status" -eq 0 ]
+	[ -d "$NEXTCLOUD_DIR/data" ]
+	[ "$(readlink "$NEXTCLOUD_DIR/apps/libresign")" = "$REPO_ROOT" ]
+	ran 'occ maintenance:install'
+	grep -q -- '^occ maintenance:install .*--database=sqlite' "$COMMAND_LOG"
+	ran 'occ --version'
+	ran 'occ app:enable --force libresign'
+	[ "$(call_line 'occ maintenance:install')" -lt "$(call_line 'occ --version')" ]
+	[ "$(call_line 'occ --version')" -lt "$(call_line 'occ app:enable')" ]
+}
+
+@test "skips the initialization outside GitHub Actions" {
+	with_signing_key
+
+	run_make '' "$REPO_ROOT" _appstore-nextcloud-setup
+
+	[ "$status" -eq 0 ]
+	[ ! -s "$COMMAND_LOG" ]
+	[ ! -e "$NEXTCLOUD_DIR/data" ]
+	[ ! -e "$NEXTCLOUD_DIR/apps/libresign" ]
+}
+
+@test "skips the initialization when the signing key is absent" {
+	run_make true "$REPO_ROOT" _appstore-nextcloud-setup
+
+	[ "$status" -eq 0 ]
+	[ ! -s "$COMMAND_LOG" ]
+	[ ! -e "$NEXTCLOUD_DIR/data" ]
+	[ ! -e "$NEXTCLOUD_DIR/apps/libresign" ]
+}
+
+@test "stops with an error when a required initialization command fails" {
+	with_signing_key
+	export OCC_FAIL_ON=maintenance:install
+
+	run_make true "$REPO_ROOT" _appstore-nextcloud-setup
+
+	[ "$status" -ne 0 ]
+	ran 'occ maintenance:install'
+	[ -z "$(call_line 'occ app:enable')" ]
+}
+
+@test "appstore initializes Nextcloud before signing the release" {
+	local app
+	app="$(make_app_fixture)"
+	with_signing_key
+
+	run_make true "$app" appstore
+
+	[ "$status" -eq 0 ]
+	[ -f "$app/build/artifacts/$APP_NAME.tar.gz" ]
+	[ "$(readlink "$NEXTCLOUD_DIR/apps/libresign")" = "$app" ]
+	ran 'occ maintenance:install'
+	ran 'occ app:enable --force libresign'
+	ran 'occ integrity:sign-app'
+	[ "$(call_line 'occ app:enable')" -lt "$(call_line 'occ libresign:install')" ]
+	[ "$(call_line 'occ app:enable')" -lt "$(call_line 'occ integrity:sign-app')" ]
+}

tests/ci/appstore-package.bats

@@ -0,0 +1,54 @@
+#!/usr/bin/env bats
+# SPDX-FileCopyrightText: 2026 LibreCode coop and contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# Behavior of `make verify-appstore-package` against the archive built by
+# `make appstore` without a signing key, as on pull requests.
+
+load helpers
+
+setup() {
+	setup_appstore_test
+	APP="$(make_app_fixture)"
+	SIGN_DIR="$APP/build/artifacts/sign"
+	PACKAGE="$APP/build/artifacts/$APP_NAME.tar.gz"
+}
+
+# Rebuilds the archive from the staging directory after the test altered it.
+repack() {
+	tar -czf "$PACKAGE" -C "$SIGN_DIR" "$APP_NAME"
+}
+
+@test "builds and accepts the unsigned package" {
+	run_make true "$APP" appstore verify-appstore-package
+
+	[ "$status" -eq 0 ]
+	[ -f "$PACKAGE" ]
+	[ ! -s "$COMMAND_LOG" ]
+	tar -tzf "$PACKAGE" | grep -qx "$APP_NAME/appinfo/info.xml"
+}
+
+@test "rejects a package that ships a development path" {
+	run_make true "$APP" appstore
+	[ "$status" -eq 0 ]
+	mkdir -p "$SIGN_DIR/$APP_NAME/src"
+	touch "$SIGN_DIR/$APP_NAME/src/main.ts"
+	repack
+
+	run_make true "$APP" verify-appstore-package
+
+	[ "$status" -ne 0 ]
+	[[ "$output" == *"must not contain src"* ]]
+}
+
+@test "rejects a package that is missing a required path" {
+	run_make true "$APP" appstore
+	[ "$status" -eq 0 ]
+	rm -rf "$SIGN_DIR/$APP_NAME/templates"
+	repack
+
+	run_make true "$APP" verify-appstore-package
+
+	[ "$status" -ne 0 ]
+	[[ "$output" == *"missing templates"* ]]
+}

tests/ci/helpers.bash

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: 2026 LibreCode coop and contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+# Shared setup for the release packaging tests. Nextcloud, occ and the network
+# are replaced by stubs that record what was executed; nothing outside
+# BATS_TEST_TMPDIR is written.
+
+setup_appstore_test() {
+	REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd -P)"
+	APP_NAME="$(basename "$REPO_ROOT")"
+
+	NEXTCLOUD_DIR="$BATS_TEST_TMPDIR/nextcloud"
+	CERT_DIR="$BATS_TEST_TMPDIR/certificates"
+	STUB_BIN="$BATS_TEST_TMPDIR/bin"
+	COMMAND_LOG="$BATS_TEST_TMPDIR/commands.log"
+	mkdir -p "$NEXTCLOUD_DIR/apps" "$CERT_DIR" "$STUB_BIN"
+	: > "$COMMAND_LOG"
+
+	# occ records its arguments and fails for the subcommand named in OCC_FAIL_ON.
+	cat > "$STUB_BIN/occ" <<-'EOF'
+		#!/usr/bin/env bash
+		printf 'occ %s\n' "$*" >> "$COMMAND_LOG"
+		[ "$1" != "${OCC_FAIL_ON:-}" ]
+	EOF
+	# The signing steps of `appstore` download the certificate and sign the
+	# archive; neither may reach the network or need a real key.
+	cat > "$STUB_BIN/curl" <<-'EOF'
+		#!/usr/bin/env bash
+		printf 'curl %s\n' "$*" >> "$COMMAND_LOG"
+	EOF
+	cat > "$STUB_BIN/openssl" <<-'EOF'
+		#!/usr/bin/env bash
+		printf 'openssl %s\n' "$1" >> "$COMMAND_LOG"
+	EOF
+	chmod +x "$STUB_BIN"/*
+	export COMMAND_LOG
+}
+
+with_signing_key() {
+	touch "$CERT_DIR/$APP_NAME.key"
+}
+
+# Runs make like CI does: GITHUB_ACTIONS comes from the environment, never from
+# the command line. Any value inherited from the runner is dropped first.
+run_make() {
+	local github_actions="$1" directory="$2"
+	shift 2
+	run env -u GITHUB_ACTIONS -u MAKEFLAGS -u MAKELEVEL -u MFLAGS \
+		${github_actions:+GITHUB_ACTIONS="$github_actions"} \
+		PATH="$STUB_BIN:$PATH" \
+		make --no-print-directory -C "$directory" "$@" \
+		cert_dir="$CERT_DIR" \
+		nextcloud_directory="$NEXTCLOUD_DIR" \
+		occ="$STUB_BIN/occ"
+}
+
+# Line of the first recorded call whose arguments start with "$1"; empty when
+# the command never ran.
+call_line() {
+	grep -n -- "^$1\( \|\$\)" "$COMMAND_LOG" | head -n 1 | cut -d: -f1
+}
+
+ran() {
+	[ -n "$(call_line "$1")" ]
+}
+
+# Creates a minimal app with the inputs `appstore` copies and links the real
+# Makefile into it, so packaging runs without touching the repository. Prints
+# the app directory.
+make_app_fixture() {
+	local app
+	app="$(cd "$BATS_TEST_TMPDIR" && pwd -P)/work/$APP_NAME"
+	mkdir -p "$app"/{appinfo,composer,css,img,js,l10n,lib,templates,vendor,3rdparty} \
+		"$app/docs/changelogs" "$app/tests/php/fixtures/pdfs"
+	printf '<info><id>libresign</id><version>1.0.0</version></info>\n' > "$app/appinfo/info.xml"
+	printf '# Changelog\n' > "$app/docs/changelogs/changelog-1.md"
+	for spec in openapi openapi-administration openapi-full; do
+		printf '{}\n' > "$app/$spec.json"
+	done
+	touch "$app/js/pdf.worker.min-fixture.mjs" "$app/tests/php/fixtures/pdfs/small_valid.pdf"
+	ln -s "$REPO_ROOT/Makefile" "$app/Makefile"
+	printf '%s\n' "$app"
+}