diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ff6d299..9c6c87a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,12 +14,13 @@ # "stay current" layer, not the "react to a 0-day" layer. # # Dependabot only edits requirements.txt — it has no concept of the -# pip-compile lockfile the image installs from. Every pip PR therefore turns -# ci.yml's `lockfile-drift` job red until requirements.lock is recompiled: -# either run the `deps-lock` workflow on the PR branch, or take the lockfile -# that the failing job uploads as an artifact. That red gate is deliberate — -# when keeping the lockfile current was invisible manual work, it fell four -# packages behind. +# pip-compile lockfile the image installs from. A pip PR that raises a floor +# past the locked version therefore turns ci.yml's `lockfile-drift` job red +# until requirements.lock is recompiled — and `lint-and-test` too, whose +# install is constrained to the lockfile. Either run the `deps-lock` workflow +# on the PR branch, or take the lockfile that `lockfile-drift` uploads as an +# artifact. Those red gates are deliberate — when keeping the lockfile current +# was invisible manual work, it fell four packages behind. # # A base-image bump also moves the Python version the app ships on. The # `python-version consistency` gate in ci.yml fails until the workflows and diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b265036..b74694d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,8 +56,39 @@ jobs: sudo apt-get update sudo apt-get install -y ffmpeg ghostscript libheif-dev libcairo2 libpangocairo-1.0-0 libgdk-pixbuf2.0-0 + # Tests run against the versions the image ships. requirements-dev.txt + # pulls in requirements.txt, whose `>=` ranges resolve to the newest + # releases: SQLAlchemy 2.1.0 (2026-09-24) reached CI that way and failed a + # test on every branch, while the image stayed on 2.0.52. The lockfile — + # what the Dockerfile installs — therefore constrains every runtime + # package. Dev-only tools (pytest, ruff, uv, aiosqlite, …) are not in it + # and resolve as before; a dependency they share with the app (packaging, + # requests, …) stays at its locked version. The unpinned install runs + # weekly in deps-latest.yml as an early warning. + # + # The hashes are stripped because pip switches the whole install to + # --require-hashes as soon as one constraint carries a hash, and the dev + # tools are unhashed. The step fails if a lockfile entry did not become a + # constraint (or none did), instead of letting it install unpinned. + # + # A resolver conflict here usually means requirements.txt raised a floor + # past the locked version, as Dependabot pip PRs do: lockfile-drift is red + # as well, and recompiling the lockfile fixes both. If lockfile-drift is + # green, a dev tool wants a different version of a locked package. If it + # caps one below the pin, hold that dev-tool bump. If it needs a newer + # one, which a plain recompile does not move, move it explicitly — that + # changes what ships, so review it like a runtime bump (uv leaves the flag + # out of the lockfile header, so lockfile-drift stays green): + # uv pip compile --generate-hashes --python-version 3.14 --python-platform x86_64-unknown-linux-gnu --upgrade-package --output-file requirements.lock requirements.txt - name: Install Python dependencies - run: pip install -r requirements-dev.txt + run: | + constraints="$RUNNER_TEMP/constraints.txt" + grep -E '^[A-Za-z0-9._-]+==' requirements.lock | cut -d' ' -f1 > "$constraints" + if [ ! -s "$constraints" ] || grep -E '^[^[:space:]#]' requirements.lock | grep -qvE '^[A-Za-z0-9._-]+=='; then + echo "::error::Not every requirements.lock entry could be turned into a constraint (see the comment above this step in ci.yml)." + exit 1 + fi + pip install -r requirements-dev.txt -c "$constraints" - name: Lint (ruff) run: ruff check . diff --git a/.github/workflows/deps-latest.yml b/.github/workflows/deps-latest.yml new file mode 100644 index 0000000..4b85c29 --- /dev/null +++ b/.github/workflows/deps-latest.yml @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: AGPL-3.0-or-later +# +# deps-latest — early warning: run the test suite against the newest releases +# that requirements.txt allows. +# +# ci.yml's lint-and-test constrains every runtime package to requirements.lock, +# so a PR is tested against what the image ships. That keeps an upstream +# release from turning every branch red at once, as SQLAlchemy 2.1.0 did in +# September 2026 while production stayed on 2.0.52. But it also means PR CI no +# longer notices such a release at all, and that notice was worth having: it +# arrived before any lockfile bump would have pulled 2.1 in. +# +# This workflow keeps it. It installs requirements-dev.txt unpinned, the way +# lint-and-test used to, and runs the tests. It gates nothing — a red run means +# the next lockfile bump needs work first, and the failing test is where to +# start. GitHub reports a failed scheduled run through its usual +# failed-workflow notification. To run it by hand, use the Actions tab or: +# +# gh workflow run deps-latest.yml + +name: deps-latest + +on: + schedule: + # Mondays — the day Dependabot opens its weekly pip PRs (dependabot.yml), + # so the result is at hand when reviewing them. + - cron: "23 4 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + test-latest: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + # The shipped version, like every workflow (scripts/check_python_version.py). + python-version: "3.14" + + # Same packages, and the same Google-Chrome apt-source workaround, as + # ci.yml's lint-and-test — see the comment there. + - name: Install system dependencies + run: | + sudo find /etc/apt/sources.list.d -iname '*google*' -delete + sudo apt-get update + sudo apt-get install -y ffmpeg ghostscript libheif-dev libcairo2 libpangocairo-1.0-0 libgdk-pixbuf2.0-0 + + # Deliberately without the lockfile constraints: the newest releases the + # ranges allow are what this job is for. + - name: Install Python dependencies (unpinned) + run: pip install -r requirements-dev.txt + + - name: Run tests + run: pytest tests/ -v --tb=short diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b1facab..01f23fd 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,117 +1,117 @@ -name: Docker - -on: - push: - branches: ["main"] - tags: ["v*"] - # Manual rebuild. Without this there is no way to produce an image, or to - # reach `notify-ops` and the deploy that follows it, other than pushing to - # main — and main is protected, so that means opening a PR. - # - # That gap cost three weeks in September 2026: the credential `notify-ops` - # uses had expired, so it failed with HTTP 401 on every run and nothing - # reached production. Once it was replaced there was no way to prove the fix, - # or to redeploy, without inventing a code change. A dispatch here rebuilds - # from the current main and runs the whole chain, which is also what you want - # during an incident. - workflow_dispatch: - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - build-and-push: - runs-on: ubuntu-latest - strategy: - # Build slim + office in parallel; fail-fast off because the two - # variants are independent — a Pango regression in slim shouldn't - # mask an upstream LibreOffice break in office (or vice versa). - fail-fast: false - matrix: - include: - - target: base - suffix: "" - description: "Slim image (mammoth+WeasyPrint docx→pdf path)" - - target: office - suffix: "-office" - description: "Office image (adds LibreOffice for high-fidelity docx→pdf)" - permissions: - contents: read - packages: write - # NEU-B.4: cosign keyless signing uses the GitHub Actions OIDC - # token to prove identity to Sigstore's Fulcio CA — no long-lived - # signing key to manage or rotate. Verifiable later with - # cosign verify ghcr.io//@sha256: \ - # --certificate-identity-regexp "^https://github\\.com///" \ - # --certificate-oidc-issuer https://token.actions.githubusercontent.com - id-token: write - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - # The default ``docker`` driver on GHA runners doesn't support - # ``cache-to: type=gha`` (GitHub Actions cache backend). Switching - # to buildx's ``docker-container`` driver enables the cache export - # so the matrix matrix-base / matrix-office leg can reuse the - # base-stage layers it already pulled. - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata - id: meta - uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - # The office variant gets the same tag set as slim but with a - # ``-office`` suffix so a consumer can pick either by tag — - # filemorph:1.1.0 vs filemorph:1.1.0-office. The slim image - # keeps ``:latest`` (default for naive pulls); the office image - # gets ``:office`` for the matching shorthand. - tags: | - type=semver,pattern={{version}},suffix=${{ matrix.suffix }} - type=semver,pattern={{major}}.{{minor}},suffix=${{ matrix.suffix }} - type=raw,value=${{ matrix.target == 'base' && 'latest' || 'office' }} - type=sha,format=short,prefix=sha-,suffix=${{ matrix.suffix }} - - - name: Build and push Docker image (${{ matrix.target }}) - id: build - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 - with: - context: . - target: ${{ matrix.target }} - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - # GHA-cache lets the office stage reuse every layer the base - # stage already produced — without it the office build redoes - # ffmpeg, ghostscript, pip install. Scope is per-target so the - # two matrix legs don't trample each other's cache. - cache-from: type=gha,scope=${{ matrix.target }} - cache-to: type=gha,scope=${{ matrix.target }},mode=max - - - name: Install cosign - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 - with: - cosign-release: "v2.4.1" - - - name: Sign published image (keyless / Sigstore) - env: - COSIGN_EXPERIMENTAL: "true" - DIGEST: ${{ steps.build.outputs.digest }} - TAGS: ${{ steps.meta.outputs.tags }} - # Sign every tag that got pushed at the SAME digest so a downstream - # `cosign verify ghcr.io/.../filemorph:` works regardless of - # which alias the consumer pulls. - run: | - set -euo pipefail - for tag in $TAGS; do - cosign sign --yes "${tag}@${DIGEST}" - done +name: Docker + +on: + push: + branches: ["main"] + tags: ["v*"] + # Manual rebuild. Without this there is no way to produce an image, or to + # reach `notify-ops` and the deploy that follows it, other than pushing to + # main — and main is protected, so that means opening a PR. + # + # That gap cost three weeks in September 2026: the credential `notify-ops` + # uses had expired, so it failed with HTTP 401 on every run and nothing + # reached production. Once it was replaced there was no way to prove the fix, + # or to redeploy, without inventing a code change. A dispatch here rebuilds + # from the current main and runs the whole chain, which is also what you want + # during an incident. + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push: + runs-on: ubuntu-latest + strategy: + # Build slim + office in parallel; fail-fast off because the two + # variants are independent — a Pango regression in slim shouldn't + # mask an upstream LibreOffice break in office (or vice versa). + fail-fast: false + matrix: + include: + - target: base + suffix: "" + description: "Slim image (mammoth+WeasyPrint docx→pdf path)" + - target: office + suffix: "-office" + description: "Office image (adds LibreOffice for high-fidelity docx→pdf)" + permissions: + contents: read + packages: write + # NEU-B.4: cosign keyless signing uses the GitHub Actions OIDC + # token to prove identity to Sigstore's Fulcio CA — no long-lived + # signing key to manage or rotate. Verifiable later with + # cosign verify ghcr.io//@sha256: \ + # --certificate-identity-regexp "^https://github\\.com///" \ + # --certificate-oidc-issuer https://token.actions.githubusercontent.com + id-token: write + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # The default ``docker`` driver on GHA runners doesn't support + # ``cache-to: type=gha`` (GitHub Actions cache backend). Switching + # to buildx's ``docker-container`` driver enables the cache export + # so the matrix matrix-base / matrix-office leg can reuse the + # base-stage layers it already pulled. + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + # The office variant gets the same tag set as slim but with a + # ``-office`` suffix so a consumer can pick either by tag — + # filemorph:1.1.0 vs filemorph:1.1.0-office. The slim image + # keeps ``:latest`` (default for naive pulls); the office image + # gets ``:office`` for the matching shorthand. + tags: | + type=semver,pattern={{version}},suffix=${{ matrix.suffix }} + type=semver,pattern={{major}}.{{minor}},suffix=${{ matrix.suffix }} + type=raw,value=${{ matrix.target == 'base' && 'latest' || 'office' }} + type=sha,format=short,prefix=sha-,suffix=${{ matrix.suffix }} + + - name: Build and push Docker image (${{ matrix.target }}) + id: build + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + target: ${{ matrix.target }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + # GHA-cache lets the office stage reuse every layer the base + # stage already produced — without it the office build redoes + # ffmpeg, ghostscript, pip install. Scope is per-target so the + # two matrix legs don't trample each other's cache. + cache-from: type=gha,scope=${{ matrix.target }} + cache-to: type=gha,scope=${{ matrix.target }},mode=max + + - name: Install cosign + uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + with: + cosign-release: "v2.4.1" + + - name: Sign published image (keyless / Sigstore) + env: + COSIGN_EXPERIMENTAL: "true" + DIGEST: ${{ steps.build.outputs.digest }} + TAGS: ${{ steps.meta.outputs.tags }} + # Sign every tag that got pushed at the SAME digest so a downstream + # `cosign verify ghcr.io/.../filemorph:` works regardless of + # which alias the consumer pulls. + run: | + set -euo pipefail + for tag in $TAGS; do + cosign sign --yes "${tag}@${DIGEST}" + done diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 259df0e..01d3e0e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -76,8 +76,11 @@ jobs: # release-triggered SBOM workflow would silently never run — which is # exactly what happened for v1.1.0. Bundling it here guarantees every # release tag ships its CycloneDX SBOM (docs/patch-policy.md promise + - # EVB-IT-March-2026 SBOM mandate). `environment` captures the resolved - # transitive dependency graph, matching the SBOM workflow's main-push path. + # EVB-IT-March-2026 SBOM mandate). The method is the SBOM workflow's + # main-push one — the image's install (Dockerfile: `pip install + # --require-hashes -r requirements.lock`) into a venv of its own, the + # generator outside it; sbom.yml's header explains why. Keep the two in + # step: this path only runs on a release, sbom.yml on every push to main. - name: Set up Python (for SBOM) uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -90,9 +93,10 @@ jobs: - name: Generate CycloneDX SBOM run: | TAG="${GITHUB_REF#refs/tags/}" - pip install -r requirements.txt + python -m venv "$RUNNER_TEMP/image-env" + "$RUNNER_TEMP/image-env/bin/pip" install --require-hashes -r requirements.lock pip install "cyclonedx-bom>=5,<6" - cyclonedx-py environment \ + cyclonedx-py environment "$RUNNER_TEMP/image-env/bin/python" \ --output-file "filemorph-${TAG}.cdx.json" \ --output-format JSON diff --git a/.github/workflows/sbom.yml b/.github/workflows/sbom.yml index a36ab66..a979879 100644 --- a/.github/workflows/sbom.yml +++ b/.github/workflows/sbom.yml @@ -13,11 +13,17 @@ name: SBOM # `workflow_dispatch` stays so a maintainer can regenerate the artifact on # demand (e.g. to back-fill a release cut before this split landed). # -# Why "environment" and not "requirements": `cyclonedx-py environment` -# captures the actual installed graph including transitive dependencies -# with their resolved versions, which is what a vulnerability scanner -# wants to see. `cyclonedx-py requirements` would only emit the direct -# pins from requirements.txt and miss everything pulled in transitively. +# What it describes: a fresh venv installed exactly as the Dockerfile installs +# the image (`pip install --require-hashes -r requirements.lock`), with the +# generator installed outside it. It used to describe the runner's own Python +# after `pip install -r requirements.txt` plus the generator — so the `>=` +# ranges put versions into the SBOM that the image does not contain, and the +# generator added its own dependencies (and moved one the image ships). +# +# Why "environment" and not "requirements": `cyclonedx-py environment` reads +# the installed distributions, so the SBOM carries their licences and the +# dependency graph. `cyclonedx-py requirements requirements.lock` parses the +# lockfile, hashes and all, but emits neither. on: push: @@ -41,12 +47,17 @@ jobs: python-version: "3.14" cache: "pip" - - name: Install runtime dependencies - run: pip install -r requirements.txt + # The Dockerfile's builder-stage install, into a venv of its own so nothing + # else on the runner ends up in the SBOM. + - name: Install the image's dependency set + run: | + python -m venv "$RUNNER_TEMP/image-env" + "$RUNNER_TEMP/image-env/bin/pip" install --require-hashes -r requirements.lock - name: Install CycloneDX generator # Pinned to the 5.x line for reproducibility; bump deliberately when - # CycloneDX schema spec advances. + # CycloneDX schema spec advances. Goes into the runner's Python, not + # the venv above, so the generator is not part of what it describes. run: pip install "cyclonedx-bom>=5,<6" - name: Determine version label @@ -57,7 +68,7 @@ jobs: - name: Generate CycloneDX SBOM (JSON) run: | - cyclonedx-py environment \ + cyclonedx-py environment "$RUNNER_TEMP/image-env/bin/python" \ --output-file "filemorph-${{ steps.version.outputs.version }}.cdx.json" \ --output-format JSON diff --git a/.github/workflows/verapdf.yml b/.github/workflows/verapdf.yml index 9a622ec..0d6abce 100644 --- a/.github/workflows/verapdf.yml +++ b/.github/workflows/verapdf.yml @@ -79,8 +79,11 @@ jobs: sudo apt-get update sudo apt-get install -y ghostscript libcairo2 libpangocairo-1.0-0 libgdk-pixbuf2.0-0 + # The image's install (Dockerfile), so the fixture is built with the + # package versions that actually ship — pikepdf above all. + # requirements.txt would resolve its `>=` ranges to the newest releases. - name: Install Python dependencies - run: pip install -r requirements.txt + run: pip install --require-hashes -r requirements.lock - name: Generate PDF/A fixture via PdfToPdfaConverter run: python scripts/verapdf_check.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b0a370..07acd7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,51 @@ Versions follow [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Fixed — CI tests, and the SBOM lists, the versions the image ships + +The image installs `requirements.lock`. Four workflows still installed +`requirements.txt`, whose `>=` ranges resolve to the newest releases, so what +they tested or described was not what ships. + +- **Tests.** `lint-and-test` ran against whatever was newest on PyPI — that is + how SQLAlchemy 2.1.0 failed a test on every branch while production stayed + on 2.0.52. It now installs with the lockfile as a constraints file, so every + runtime package is at its shipped version. The dev-only tools (pytest, ruff, + uv, aiosqlite, …) are not in the lockfile and resolve as before; a dependency + they share with the app stays at the locked version. The constraints are the + lockfile's pins without their hashes, because pip turns on `--require-hashes` + for the whole install as soon as one constraint carries a hash. A + `requirements.txt` floor raised past the locked version (the usual + Dependabot pip PR) now fails this job's install as well as `lockfile-drift`; + recompiling the lockfile fixes both. +- **The early warning stays.** The unpinned run caught SQLAlchemy 2.1 before + any lockfile bump would have pulled it in. It now runs in the new + `deps-latest` workflow — weekly on Mondays and on demand — and gates nothing. +- **SBOM.** `sbom.yml`, and `release.yml` for the copy attached to releases, + built the SBOM from the runner's Python after `pip install -r + requirements.txt` plus the CycloneDX generator. The one produced for main + at `1d7bad6` listed 106 packages against 77 in the lockfile: 19 locked + packages at a version the image does not contain (SQLAlchemy 2.1.1 instead + of 2.0.52), plus 28 that belong to the generator (`cyclonedx-bom` and its + dependencies) — whose install had also downgraded `packaging` to 25.0, + while the image ships 26.3. Both workflows now install the lockfile into a + fresh venv exactly as the Dockerfile does, install the generator outside it, + and point `cyclonedx-py environment` at that venv. (`cyclonedx-py requirements + requirements.lock` reads the hashed lockfile fine, but its SBOM has no + licence data and no dependency graph.) An SBOM from before this change can + list extra packages and versions ahead of the image. +- **veraPDF.** The PDF/A-2b gate built its fixture with the newest pikepdf; it + now installs the lockfile the way the image does. + +`build-desktop.yml` still installs `requirements.txt`: it builds on Windows, +and the lockfile is resolved for Linux. + +`tests/test_supply_chain_hygiene.py` pins all four, so none of them can +quietly go back to the manifest. Also: `.github/workflows/docker.yml` had been +committed with CRLF line endings (an API commit in PR #129) despite +`*.yml text eol=lf`, so every checkout showed it as modified. It is +renormalized to LF; no content change. + ### Changed — homepage shows seven quick actions; "More tools" box removed Before a file was chosen, the homepage's tool card offered no concrete diff --git a/docs/development.md b/docs/development.md index 2170719..effa57e 100644 --- a/docs/development.md +++ b/docs/development.md @@ -324,7 +324,7 @@ and only keys listed in [`release-signing.md`](release-signing.md) verify. ``` The tag push triggers `release.yml` (verifies the signature, publishes the -GitHub Release with a source tarball + `IMAGE_DIGEST.txt`), `docker.yml` -(builds + cosign-signs the slim and office images to GHCR) and `sbom.yml` -(attaches the CycloneDX SBOM). See [`release-signing.md`](release-signing.md) +GitHub Release with a source tarball, the CycloneDX SBOM and +`IMAGE_DIGEST.txt`) and `docker.yml` (builds + cosign-signs the slim and +office images to GHCR). See [`release-signing.md`](release-signing.md) for key setup/rotation and `docs/patch-policy.md` for the versioning rules. diff --git a/docs/patch-policy.md b/docs/patch-policy.md index 09e09f0..92832c4 100644 --- a/docs/patch-policy.md +++ b/docs/patch-policy.md @@ -70,10 +70,15 @@ dependencies, and a re-uploaded or tampered wheel fails the hash check. CI fails if the two files drift apart or if the lockfile was compiled on a different Python version than the image ships. -The full dependency manifest is available as a +The full Python dependency manifest is available as a [CycloneDX SBOM](https://cyclonedx.org/) attached to each GitHub -release as `filemorph-{version}.cdx.json`. Use it for vulnerability -scanning against your existing CVE pipeline. +release as `filemorph-{version}.cdx.json`. It is generated from a +clean environment installed from `requirements.lock` exactly the way +the image installs it, so it lists the application's Python +dependencies at the versions the image ships. System packages from +the base image and `apt` (FFmpeg, Ghostscript, LibreOffice) are not +in it. Use it for vulnerability scanning against your existing CVE +pipeline. ## Release announcements diff --git a/docs/third-party-licenses.md b/docs/third-party-licenses.md index 0eb2eaf..7db4c1a 100644 --- a/docs/third-party-licenses.md +++ b/docs/third-party-licenses.md @@ -117,8 +117,8 @@ converters need: - **Machine-readable, full transitive list:** the CycloneDX-JSON SBOM `filemorph-{version}.cdx.json` attached to every GitHub release (generated by - the `sbom` workflow with `cyclonedx-py environment`). Run it through your - existing licence/CVE pipeline. + the `release` workflow with `cyclonedx-py environment`, over a clean install + of `requirements.lock`). Run it through your existing licence/CVE pipeline. - **Human-readable regeneration:** ```bash pip install pip-licenses diff --git a/tests/test_supply_chain_hygiene.py b/tests/test_supply_chain_hygiene.py index 9e6a2e9..04ff031 100644 --- a/tests/test_supply_chain_hygiene.py +++ b/tests/test_supply_chain_hygiene.py @@ -15,7 +15,10 @@ repo-wide default (OpenSSF Scorecard "Token-Permissions"); * ``.github/dependabot.yml`` exists and covers all three ecosystems we pin manually (``pip`` / ``github-actions`` / ``docker``) so the pins - above don't rot. + above don't rot; + * what CI tests, validates and publishes is the lockfile's dependency set — + the test job installs with the lockfile as constraints, and the SBOM and + veraPDF workflows install it the way the image does. This is a tripwire, not a substitute for the server-side Scorecard run / review: the per-job permissions check here is a heuristic (it asserts a @@ -56,6 +59,12 @@ def _workflow_files() -> list[Path]: return files +def _workflow_code(name: str) -> str: + """A workflow's text without its comment lines, which quote old commands.""" + lines = (_WORKFLOW_DIR / name).read_text(encoding="utf-8").splitlines() + return "\n".join(line for line in lines if not line.lstrip().startswith("#")) + + def test_workflow_dir_exists() -> None: assert _WORKFLOW_DIR.is_dir(), f"{_WORKFLOW_DIR} missing" @@ -190,6 +199,83 @@ def test_lockfile_is_hash_pinned_and_matches_the_image_python() -> None: ) +def test_ci_tests_run_against_the_locked_versions() -> None: + """lint-and-test installs with the lockfile as constraints. + + requirements-dev.txt pulls in requirements.txt, whose ``>=`` ranges + resolve to the newest releases — that is how CI came to test SQLAlchemy + 2.1.0 while the image shipped 2.0.52. The unpinned install is the early + warning in ``deps-latest.yml``, not the gate. + """ + text = _workflow_code("ci.yml") + installs = [ + line.strip() + for line in text.splitlines() + if re.search(r"pip install .*-r requirements(-dev)?\.txt", line) + ] + assert installs, "ci.yml no longer installs requirements-dev.txt — update this guard" + for line in installs: + assert re.search(r"\s-c\s", line), ( + f"ci.yml installs a manifest without constraints: {line!r}. " + f"Tests would run against the newest releases, not what the image ships." + ) + assert re.search(r"requirements\.lock.*>.*constraints", text), ( + "ci.yml's constraints are no longer derived from requirements.lock" + ) + + +def test_deps_latest_mirrors_lint_and_test() -> None: + """deps-latest differs from lint-and-test only in the pinning. + + Otherwise a system package or pytest option added to ci.yml alone turns the + weekly run red, and its header tells the reader to blame an upstream + release. + """ + ci, latest = _workflow_code("ci.yml"), _workflow_code("deps-latest.yml") + for pattern in (r"apt-get install -y .*", r"pytest tests/.*"): + assert re.findall(pattern, latest) == re.findall(pattern, ci), ( + f"deps-latest.yml and ci.yml disagree on `{pattern}` — keep them in step" + ) + + +@pytest.mark.parametrize("workflow", ["sbom.yml", "release.yml", "verapdf.yml"]) +def test_workflow_installs_what_the_image_ships(workflow: str) -> None: + """The SBOM lists, and veraPDF validates, the image's dependency set. + + Installing requirements.txt resolves its ``>=`` ranges to the newest + releases: the SBOM published from main listed 19 of the 77 shipped + packages at versions the image does not contain. + """ + text = _workflow_code(workflow) + assert "install --require-hashes -r requirements.lock" in text, ( + f"{workflow} does not install requirements.lock the way the Dockerfile does" + ) + assert not re.search(r"-r requirements(-dev)?\.txt", text), ( + f"{workflow} installs the manifest — its `>=` ranges resolve to versions " + f"the image does not ship. Install requirements.lock with --require-hashes." + ) + + +@pytest.mark.parametrize("workflow", ["sbom.yml", "release.yml"]) +def test_sbom_describes_the_lockfile_venv_only(workflow: str) -> None: + """``cyclonedx-py environment`` reads the lockfile venv, not its own. + + Without an interpreter argument it describes the Python it runs in, which + holds the generator too: 28 packages of its own in the SBOM, and a + ``packaging`` its install had downgraded below the shipped version. + """ + text = _workflow_code(workflow) + target = re.search(r'cyclonedx-py environment\s+"?([^\s"\\]+)/bin/python', text) + assert target, f"{workflow}: cyclonedx-py environment is not given a venv to describe" + venv = re.escape(target.group(1)) + assert re.search(venv + r'/bin/pip"? install --require-hashes -r requirements\.lock', text), ( + f"{workflow}: the venv the SBOM describes is not the one requirements.lock is installed into" + ) + assert len(re.findall(venv + r'/bin/pip"? install', text)) == 1, ( + f"{workflow}: something besides requirements.lock is installed into the SBOM's venv" + ) + + def test_dependabot_config_covers_all_pinned_ecosystems() -> None: assert _DEPENDABOT.is_file(), ( ".github/dependabot.yml missing — the manual SHA/digest pins will rot "