diff --git a/.specify/feature.json b/.specify/feature.json index c2d99ccce4..dd58e2bf69 100644 --- a/.specify/feature.json +++ b/.specify/feature.json @@ -1,3 +1,3 @@ { - "feature_directory": "specs/017-pm-index-version-provenance" + "feature_directory": "specs/018-packages-stage-invalidation" } diff --git a/docs/pages_en/usage/build/stapel/instructions.md b/docs/pages_en/usage/build/stapel/instructions.md index b6e0d7749d..c03054ed28 100644 --- a/docs/pages_en/usage/build/stapel/instructions.md +++ b/docs/pages_en/usage/build/stapel/instructions.md @@ -276,6 +276,23 @@ packages: - libssl-dev ``` +For file-based types, declare the spec and lock files in `git.stageDependencies.packages` — otherwise changes to their contents will not rebuild the packages stage, leaving installed dependencies stale while the SBOM reports the updated files: + +```yaml +git: + - add: / + to: /app + stageDependencies: + packages: + - go.mod + - go.sum +packages: + - type: go-mod + workdir: /app +``` + +The `os-pm` type does not need `stageDependencies`: its package list lives in `werf.yaml` itself, so any change to it rebuilds the stage automatically. + ## Syntax There are two mutually exclusive top-level ***builder directives*** for assembly instructions: `shell` and `ansible`. You can build an image either via ***shell instructions*** or via their ***ansible counterparts***. diff --git a/docs/pages_ru/usage/build/stapel/instructions.md b/docs/pages_ru/usage/build/stapel/instructions.md index 4032e5c865..0a0f0b2828 100644 --- a/docs/pages_ru/usage/build/stapel/instructions.md +++ b/docs/pages_ru/usage/build/stapel/instructions.md @@ -289,6 +289,23 @@ packages: - libssl-dev ``` +Для файловых типов укажите spec- и lock-файлы в `git.stageDependencies.packages` — иначе изменение их содержимого не приведет к пересборке стадии packages: установленные зависимости устареют, а SBOM при этом будет отражать обновленные файлы: + +```yaml +git: + - add: / + to: /app + stageDependencies: + packages: + - go.mod + - go.sum +packages: + - type: go-mod + workdir: /app +``` + +Типу `os-pm` директива `stageDependencies` не требуется: список пакетов находится в самом `werf.yaml`, поэтому любое его изменение автоматически пересобирает стадию. + ## Синтаксис Пользовательские стадии и инструкции сборки определяются внутри двух взаимоисключающих директив — `shell` и `ansible`. Каждый образ может собираться либо используя сборочные инструкции ***shell***, либо задачи ***ansible***, описанные в соответствующих директивах. diff --git a/pkg/build/image/stapel.go b/pkg/build/image/stapel.go index 542f829f80..d37695af26 100644 --- a/pkg/build/image/stapel.go +++ b/pkg/build/image/stapel.go @@ -7,6 +7,8 @@ import ( "path/filepath" "sync" + "github.com/samber/lo" + "github.com/werf/logboek" "github.com/werf/werf/v2/pkg/build/stage" "github.com/werf/werf/v2/pkg/config" @@ -106,6 +108,7 @@ func initStages(ctx context.Context, image *Image, metaConfig *config.Meta, stap // TODO(v3): make this a hard error instead of a warning. warnStageDependenciesWithoutInstructions(ctx, imageBaseConfig, gitMappings) + warnFileBasedPackagesWithoutStageDependencies(ctx, imageBaseConfig, gitMappings) imageCacheVersion := option.ValueOrDefault(stapelImageConfig.CacheVersion(), metaConfig.Build.CacheVersion) @@ -208,6 +211,45 @@ func warnStageDependenciesWithoutInstructions(ctx context.Context, imageBaseConf } } +// warnFileBasedPackagesWithoutStageDependencies warns when a file-based packages directive +// (any type except os-pm) is used, but no git mapping tracks its spec/lock files via +// stageDependencies.packages: without it, changes to those files do not rebuild the packages +// stage, so installed dependencies go stale while the SBOM keeps reporting the new file contents. +func warnFileBasedPackagesWithoutStageDependencies(ctx context.Context, imageBaseConfig *config.StapelImageBase, gitMappings []*stage.GitMapping) { + if !shouldWarnFileBasedPackagesWithoutStageDependencies(imageBaseConfig, gitMappings) { + return + } + + global_warnings.GlobalWarningLn(ctx, fmt.Sprintf( + "Image %q uses a file-based packages directive, but no git mapping declares git.stageDependencies.packages. "+ + "Changes to the spec/lock files (e.g. go.mod, requirements.txt) will not rebuild the packages stage, "+ + "leaving installed dependencies stale while the SBOM reports the updated files. "+ + "Declare git.stageDependencies.packages with the spec/lock file paths.", + imageBaseConfig.Name, + )) +} + +func shouldWarnFileBasedPackagesWithoutStageDependencies(imageBaseConfig *config.StapelImageBase, gitMappings []*stage.GitMapping) bool { + if len(gitMappings) == 0 { + return false + } + + hasFileBasedPackages := lo.SomeBy(imageBaseConfig.Packages, func(pkg *config.PackagesDirective) bool { + return pkg.Type != config.PackagesDirectiveTypeOSPM + }) + if !hasFileBasedPackages { + return false + } + + for _, gitMapping := range gitMappings { + if len(gitMapping.StagesDependencies[stage.Packages]) > 0 { + return false + } + } + + return true +} + func hasStageInstructions(imageBaseConfig *config.StapelImageBase, stageName stage.StageName) bool { if imageBaseConfig.Shell != nil { switch stageName { diff --git a/pkg/build/image/stapel_test.go b/pkg/build/image/stapel_test.go new file mode 100644 index 0000000000..483cba959d --- /dev/null +++ b/pkg/build/image/stapel_test.go @@ -0,0 +1,68 @@ +package image + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/v2/pkg/build/stage" + "github.com/werf/werf/v2/pkg/config" +) + +var _ = Describe("shouldWarnFileBasedPackagesWithoutStageDependencies", func() { + newImageConfig := func(pkgTypes ...config.PackagesDirectiveType) *config.StapelImageBase { + imageBaseConfig := &config.StapelImageBase{Name: "app"} + for _, pkgType := range pkgTypes { + imageBaseConfig.Packages = append(imageBaseConfig.Packages, &config.PackagesDirective{Type: pkgType}) + } + return imageBaseConfig + } + + newGitMapping := func(packagesDeps ...string) *stage.GitMapping { + gitMapping := stage.NewGitMapping() + gitMapping.StagesDependencies = map[stage.StageName][]string{ + stage.Packages: packagesDeps, + } + return gitMapping + } + + DescribeTable("warning decision", + func(imageBaseConfig *config.StapelImageBase, gitMappings []*stage.GitMapping, expected bool) { + Expect(shouldWarnFileBasedPackagesWithoutStageDependencies(imageBaseConfig, gitMappings)).To(Equal(expected)) + }, + Entry("file-based packages without stageDependencies.packages", + newImageConfig(config.PackagesDirectiveTypeGoMod), + []*stage.GitMapping{newGitMapping()}, + true, + ), + Entry("file-based packages mixed with os-pm, no stageDependencies.packages", + newImageConfig(config.PackagesDirectiveTypeOSPM, config.PackagesDirectiveTypePythonPip), + []*stage.GitMapping{newGitMapping()}, + true, + ), + Entry("file-based packages with stageDependencies.packages declared", + newImageConfig(config.PackagesDirectiveTypeGoMod), + []*stage.GitMapping{newGitMapping("go.mod", "go.sum")}, + false, + ), + Entry("file-based packages with stageDependencies.packages on one of several mappings", + newImageConfig(config.PackagesDirectiveTypeGoMod), + []*stage.GitMapping{newGitMapping(), newGitMapping("go.mod")}, + false, + ), + Entry("only os-pm packages", + newImageConfig(config.PackagesDirectiveTypeOSPM), + []*stage.GitMapping{newGitMapping()}, + false, + ), + Entry("no packages at all", + newImageConfig(), + []*stage.GitMapping{newGitMapping()}, + false, + ), + Entry("file-based packages without git mappings", + newImageConfig(config.PackagesDirectiveTypeGoMod), + nil, + false, + ), + ) +}) diff --git a/pkg/build/image/suite_test.go b/pkg/build/image/suite_test.go new file mode 100644 index 0000000000..254f2b8e3a --- /dev/null +++ b/pkg/build/image/suite_test.go @@ -0,0 +1,13 @@ +package image + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestImageSuite(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Build Image Suite") +} diff --git a/specs/018-packages-stage-invalidation/checklists/requirements.md b/specs/018-packages-stage-invalidation/checklists/requirements.md new file mode 100644 index 0000000000..53320db69d --- /dev/null +++ b/specs/018-packages-stage-invalidation/checklists/requirements.md @@ -0,0 +1,35 @@ +# Specification Quality Checklist: Reliable packages stage rebuilds for file-based package ecosystems + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-08-21 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +- The spec describes an already-implemented change (branch `fix/sbom/warn-missing-stage-deps`); Status is set to Implemented rather than Draft. +- Domain terms (`git.stageDependencies.packages`, `werf.yaml`, SBOM, stage names) are user-facing configuration surface in this product, not implementation details. diff --git a/specs/018-packages-stage-invalidation/spec.md b/specs/018-packages-stage-invalidation/spec.md new file mode 100644 index 0000000000..a01ec28e3b --- /dev/null +++ b/specs/018-packages-stage-invalidation/spec.md @@ -0,0 +1,116 @@ +# Feature Specification: Reliable packages stage rebuilds for file-based package ecosystems + +**Feature Branch**: `fix/sbom/warn-missing-stage-deps` + +**Created**: 2026-08-21 + +**Status**: Implemented + +**Input**: User description: "With a file-based packages directive (go-mod, python-pip, rust-cargo, javascript-*, lua-rock), changing the spec/lock file contents never rebuilds the packages stage unless the files are tracked via git.stageDependencies.packages: installed dependencies silently go stale while the SBOM keeps reporting the updated files. Protect users from this trap." + +## Project Context + +**Delivery Kit** is a Go CLI tool for full-cycle CI/CD to Kubernetes. It is built on top of werf with Deckhouse Platform extensions. Key subsystems: + +- **Build** (`pkg/build/`) — Container image building via Buildah +- **Deploy** (`pkg/deploy/`) — Kubernetes deployment via werf/nelm (Helm-based) +- **SBOM** (`pkg/sbom/`) — Software Bill of Materials generation and validation +- **Cleanup** (`pkg/cleaning/`) — Container registry cleanup policies +- **Signature** (`pkg/signature/`) — Container image signing and verification +- **Docker Registry** (`pkg/docker_registry/`) — OCI registry operations +- **Config** (`pkg/config/`) — werf.yaml configuration parsing + +## Problem + +The packages stage digest is derived from the generated install command plus the checksum of files listed in `git.stageDependencies.packages`. For file-based ecosystems the install command contains only paths (e.g. `cd "/app" && go mod download`), so: + +- Without `stageDependencies.packages`, a change to spec/lock file contents (e.g. a dependency bump in `go.mod`) leaves the stage digest intact and the install command is never re-run. +- The updated spec/lock files still reach the final image through the git patch, and the SBOM cataloger reads them from the image — so the SBOM reports packages that were never actually installed. +- The result is a silently stale image with a lying SBOM: a supply-chain artifact defect the user has no way to notice. + +The inline `os-pm` type is not affected: its package list lives in `werf.yaml` itself and feeds the stage digest through the generated command. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Misconfiguration is surfaced, not silent (Priority: P1) + +A user declares `packages: [{type: go-mod, workdir: /app}]` with a git mapping but forgets `stageDependencies.packages`. Today nothing tells them their dependency bumps will be ignored. After this change, every build emits a warning naming the image and the fix, and the warning is repeated in the end-of-run WARNINGS summary. + +**Why this priority**: This is the only layer that reaches users with already-broken configurations; docs and fixtures only help those who read them. + +**Independent Test**: Build a stapel image with a file-based packages directive, a git mapping, and no `stageDependencies.packages` — the warning appears. Add `stageDependencies.packages` — the warning disappears. + +**Acceptance Scenarios**: + +1. **Given** an image with a file-based packages directive and git mappings none of which declare `stageDependencies.packages`, **When** the user runs a build, **Then** a global warning names the image, explains that spec/lock changes will not rebuild the packages stage, and tells the user to declare `git.stageDependencies.packages`. +2. **Given** the same image with `stageDependencies.packages` declared on any git mapping, **When** the user runs a build, **Then** no such warning is emitted. +3. **Given** an image whose only packages directive is `os-pm`, **When** the user runs a build, **Then** no such warning is emitted. +4. **Given** an image with a file-based packages directive but no git mappings at all, **When** the user runs a build, **Then** no such warning is emitted (the files cannot come from git, so `stageDependencies` cannot help). + +--- + +### User Story 2 - Correct configuration provably works (Priority: P2) + +A user who declares `stageDependencies.packages: [go.mod, go.sum]` must get the promised behavior: bumping a dependency in `go.mod` rebuilds the packages stage and the SBOM reflects the new dependency; rebuilding without changes reuses the cached stage. + +**Why this priority**: The warning tells users to adopt this pattern; the pattern itself must be covered by an automated end-to-end check, otherwise a regression would invalidate the advice. + +**Independent Test**: The e2e suite drives a two-state go-mod fixture where only `go.mod` changes between states (werf.yaml identical) and asserts the packages stage rebuild directly via build output, not only via SBOM regeneration (which fires on any image change and would mask the regression). + +**Acceptance Scenarios**: + +1. **Given** a go-mod image with `stageDependencies.packages: [go.mod, go.sum]`, **When** it is built twice without changes, **Then** the second build reuses the cached packages stage and the cached SBOM. +2. **Given** the same image, **When** a new module is added to `go.mod` (werf.yaml unchanged), **Then** the packages stage is rebuilt and the regenerated SBOM contains the new module. + +--- + +### User Story 3 - Shipped examples model the correct pattern (Priority: P3) + +Fixtures and documentation are the templates users copy from. Every file-based e2e fixture declares `stageDependencies.packages`, and the stapel documentation (EN and RU) states the requirement with an example and explains why `os-pm` does not need it. + +**Why this priority**: Prevents the trap from propagating through copy-paste and keeps e2e runs free of the new warning. + +**Independent Test**: Grep the e2e fixtures: every werf.yaml with a file-based packages type also declares `stageDependencies.packages`; docs pages contain the pattern in both languages. + +**Acceptance Scenarios**: + +1. **Given** the e2e fixture set, **When** any fixture uses a file-based packages type, **Then** its git mapping declares `stageDependencies.packages` with the fixture's actual spec/lock files. +2. **Given** the stapel instructions documentation, **When** a reader reaches the file-based packages section, **Then** both the EN and RU versions document the `stageDependencies.packages` requirement with an example and note that `os-pm` is exempt. + +### Edge Cases + +- Multiple git mappings where only one declares `stageDependencies.packages`: no warning — any tracked mapping is sufficient. +- Mixed directives (`os-pm` + file-based) without `stageDependencies.packages`: warning fires — the file-based part is still unprotected. +- A `stageDependencies.packages` entry pointing to a file that does not exist (negative fixtures): the unmatched path contributes nothing to the checksum; builds behave as before. +- Multi-image projects: the warning is emitted per image (it names the image), unlike the once-per-run network warning, because the fix is per-image. + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: The build MUST emit a global warning for every stapel image that has at least one file-based packages directive (any type except `os-pm`), at least one git mapping, and no git mapping declaring a non-empty `stageDependencies.packages`. +- **FR-002**: The warning MUST name the image, state the consequence (spec/lock changes do not rebuild the packages stage; installed dependencies go stale while the SBOM reports the updated files), and name the fix (`git.stageDependencies.packages`). +- **FR-003**: The warning MUST be suppressed when the image has no git mappings, has no file-based packages directives, or any git mapping declares `stageDependencies.packages`. +- **FR-004**: Existing digest behavior MUST NOT change: no automatic injection of spec/lock paths into stage dependencies (no cache invalidation for existing users). +- **FR-005**: E2e coverage MUST assert the packages stage rebuild itself (via build output) when a tracked spec file changes with werf.yaml unchanged, and assert stage cache reuse on an unchanged rebuild. +- **FR-006**: All file-based e2e fixtures MUST declare `stageDependencies.packages` listing their actual spec/lock files, keeping e2e runs free of the new warning. +- **FR-007**: The stapel instructions documentation MUST describe the requirement identically in EN and RU, including an example and the `os-pm` exemption. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: A user with an untracked file-based packages directive sees the warning on every build, including in the end-of-run WARNINGS summary, and can resolve it by following the message alone (no docs lookup required). +- **SC-002**: With `stageDependencies.packages` declared, a spec-file-only change triggers a packages stage rebuild in 100% of builds, and an unchanged rebuild never triggers one. +- **SC-003**: Zero occurrences of the new warning across the whole e2e SBOM suite output. +- **SC-004**: Upgrading werf with this change causes zero stage cache invalidations for existing projects. + +## Rejected Alternative + +Automatically injecting spec/lock paths into the packages stage dependencies was rejected: `workdir` is a container path that cannot be reliably mapped back to git-repository paths (multiple git mappings, files produced by earlier stages or present in the base image), it would deviate from the explicit `stageDependencies` model shared by the install/beforeSetup/setup stages, and it would invalidate build caches for every existing user of file-based packages. + +## Assumptions + +- The trusted mechanism for file-change-driven stage invalidation is `git.stageDependencies` (shared with install/beforeSetup/setup stages); this feature documents and enforces awareness of it rather than replacing it. +- A warning (not a hard error) is the right severity: existing configurations keep building, matching the precedent of the stage-dependencies-without-instructions warning, with a potential future upgrade to an error. +- Users of file-based ecosystems keep their spec/lock files in git; files produced during the build or baked into the base image are out of scope for the warning (covered by the no-git-mappings suppression). diff --git a/test/e2e/sbom/_fixtures/inject/cargo_simple/werf.yaml b/test/e2e/sbom/_fixtures/inject/cargo_simple/werf.yaml index 7a32d02361..4a4751b9af 100644 --- a/test/e2e/sbom/_fixtures/inject/cargo_simple/werf.yaml +++ b/test/e2e/sbom/_fixtures/inject/cargo_simple/werf.yaml @@ -10,6 +10,10 @@ from: {{ env "BUILDER_BASE_IMAGE" }} git: - add: / to: /app + stageDependencies: + packages: + - Cargo.toml + - Cargo.lock shell: install: - cd /app && cargo build --release diff --git a/test/e2e/sbom/_fixtures/inject/gomod_replace/werf.yaml b/test/e2e/sbom/_fixtures/inject/gomod_replace/werf.yaml index 821ca16a65..8485add5b6 100644 --- a/test/e2e/sbom/_fixtures/inject/gomod_replace/werf.yaml +++ b/test/e2e/sbom/_fixtures/inject/gomod_replace/werf.yaml @@ -10,6 +10,10 @@ from: {{ env "BUILDER_BASE_IMAGE" }} git: - add: / to: /app + stageDependencies: + packages: + - go.mod + - go.sum shell: install: - cd /app && CGO_ENABLED=0 go build -o /bin/app . diff --git a/test/e2e/sbom/_fixtures/inject/lua_simple/werf.yaml b/test/e2e/sbom/_fixtures/inject/lua_simple/werf.yaml index 7d4a026e63..021c4b988f 100644 --- a/test/e2e/sbom/_fixtures/inject/lua_simple/werf.yaml +++ b/test/e2e/sbom/_fixtures/inject/lua_simple/werf.yaml @@ -10,6 +10,9 @@ from: {{ env "BUILDER_BASE_IMAGE" }} git: - add: / to: /app + stageDependencies: + packages: + - werf-sbom-lua-app-0.1-1.rockspec packages: - type: lua-rock workdir: /app diff --git a/test/e2e/sbom/_fixtures/inject/npm_simple/werf.yaml b/test/e2e/sbom/_fixtures/inject/npm_simple/werf.yaml index 118f2d04b4..afdb538bef 100644 --- a/test/e2e/sbom/_fixtures/inject/npm_simple/werf.yaml +++ b/test/e2e/sbom/_fixtures/inject/npm_simple/werf.yaml @@ -10,6 +10,10 @@ from: {{ env "BUILDER_BASE_IMAGE" }} git: - add: / to: /app + stageDependencies: + packages: + - package.json + - package-lock.json packages: - type: javascript-npm workdir: /app \ No newline at end of file diff --git a/test/e2e/sbom/_fixtures/inject/pip_simple/werf.yaml b/test/e2e/sbom/_fixtures/inject/pip_simple/werf.yaml index d1534065a0..0403f83262 100644 --- a/test/e2e/sbom/_fixtures/inject/pip_simple/werf.yaml +++ b/test/e2e/sbom/_fixtures/inject/pip_simple/werf.yaml @@ -10,6 +10,9 @@ from: {{ env "BUILDER_BASE_IMAGE" }} git: - add: / to: /app + stageDependencies: + packages: + - requirements.txt packages: - type: python-pip workdir: /app diff --git a/test/e2e/sbom/_fixtures/inject/pnpm_simple/werf.yaml b/test/e2e/sbom/_fixtures/inject/pnpm_simple/werf.yaml index 02abb59da0..f091acc20d 100644 --- a/test/e2e/sbom/_fixtures/inject/pnpm_simple/werf.yaml +++ b/test/e2e/sbom/_fixtures/inject/pnpm_simple/werf.yaml @@ -10,6 +10,10 @@ from: {{ env "BUILDER_BASE_IMAGE" }} git: - add: / to: /app + stageDependencies: + packages: + - package.json + - pnpm-lock.yaml packages: - type: javascript-pnpm workdir: /app \ No newline at end of file diff --git a/test/e2e/sbom/_fixtures/inject/poetry_simple/werf.yaml b/test/e2e/sbom/_fixtures/inject/poetry_simple/werf.yaml index d587a0a29b..b1d3c0a66c 100644 --- a/test/e2e/sbom/_fixtures/inject/poetry_simple/werf.yaml +++ b/test/e2e/sbom/_fixtures/inject/poetry_simple/werf.yaml @@ -10,6 +10,10 @@ from: {{ env "BUILDER_BASE_IMAGE" }} git: - add: / to: /app + stageDependencies: + packages: + - pyproject.toml + - poetry.lock packages: - type: python-poetry workdir: /app diff --git a/test/e2e/sbom/_fixtures/inject/uv_simple/werf.yaml b/test/e2e/sbom/_fixtures/inject/uv_simple/werf.yaml index 3d27c7e49c..ab7779e9f3 100644 --- a/test/e2e/sbom/_fixtures/inject/uv_simple/werf.yaml +++ b/test/e2e/sbom/_fixtures/inject/uv_simple/werf.yaml @@ -10,6 +10,10 @@ from: {{ env "BUILDER_BASE_IMAGE" }} git: - add: / to: /app + stageDependencies: + packages: + - pyproject.toml + - uv.lock packages: - type: python-uv workdir: /app diff --git a/test/e2e/sbom/_fixtures/inject/yarn_simple/werf.yaml b/test/e2e/sbom/_fixtures/inject/yarn_simple/werf.yaml index 3e16bdff0c..71f6dd46bb 100644 --- a/test/e2e/sbom/_fixtures/inject/yarn_simple/werf.yaml +++ b/test/e2e/sbom/_fixtures/inject/yarn_simple/werf.yaml @@ -10,6 +10,10 @@ from: {{ env "BUILDER_BASE_IMAGE" }} git: - add: / to: /app + stageDependencies: + packages: + - package.json + - yarn.lock packages: - type: javascript-yarn workdir: /app \ No newline at end of file diff --git a/test/e2e/sbom/_fixtures/negative/lua_missing_rockspec/werf.yaml b/test/e2e/sbom/_fixtures/negative/lua_missing_rockspec/werf.yaml index 3c69443e5b..764d42f48a 100644 --- a/test/e2e/sbom/_fixtures/negative/lua_missing_rockspec/werf.yaml +++ b/test/e2e/sbom/_fixtures/negative/lua_missing_rockspec/werf.yaml @@ -10,6 +10,9 @@ from: {{ env "BUILDER_BASE_IMAGE" }} git: - add: / to: /app + stageDependencies: + packages: + - app-0.1-1.rockspec packages: - type: lua-rock workdir: /app diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/Dockerfile.builder-base b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/Dockerfile.builder-base new file mode 100644 index 0000000000..30ad6d272e --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/Dockerfile.builder-base @@ -0,0 +1,2 @@ +FROM registry.werf.io/base/golang:1.12-alpine3.9 +LABEL io.deckhouse.internal.builder=true diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/go.mod b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/go.mod new file mode 100644 index 0000000000..8a49e6ffae --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/go.mod @@ -0,0 +1,7 @@ +module example.com/app + +go 1.12 + +require example.com/mylib v0.0.0 + +replace example.com/mylib => ./mylib diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/go.sum b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/go.sum new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/main.go b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/main.go new file mode 100644 index 0000000000..cb32316f85 --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/main.go @@ -0,0 +1,11 @@ +package main + +import ( + "fmt" + + "example.com/mylib" +) + +func main() { + fmt.Println(mylib.Hello()) +} diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/mylib/go.mod b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/mylib/go.mod new file mode 100644 index 0000000000..773ea0f75a --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/mylib/go.mod @@ -0,0 +1,3 @@ +module example.com/mylib + +go 1.12 diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/mylib/lib.go b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/mylib/lib.go new file mode 100644 index 0000000000..2c120e9107 --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/mylib/lib.go @@ -0,0 +1,5 @@ +package mylib + +func Hello() string { + return "hello from mylib" +} diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/werf-giterminism.yaml b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/werf-giterminism.yaml new file mode 100644 index 0000000000..9483c3670c --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/werf-giterminism.yaml @@ -0,0 +1,5 @@ +giterminismConfigVersion: 1 +config: + goTemplateRendering: + allowEnvVariables: + - BUILDER_BASE_IMAGE diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/werf.yaml b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/werf.yaml new file mode 100644 index 0000000000..6b665daded --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state0/werf.yaml @@ -0,0 +1,22 @@ +project: werf-test-e2e-sbom-stage-deps-gomod +configVersion: 1 +build: + sbom: + enable: true + standard: cyclonedx@1.6 +--- +image: app +from: {{ env "BUILDER_BASE_IMAGE" }} +git: + - add: / + to: /app + stageDependencies: + packages: + - go.mod + - go.sum +shell: + install: + - cd /app && CGO_ENABLED=0 go build -o /bin/app . +packages: + - type: go-mod + workdir: /app diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/Dockerfile.builder-base b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/Dockerfile.builder-base new file mode 100644 index 0000000000..30ad6d272e --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/Dockerfile.builder-base @@ -0,0 +1,2 @@ +FROM registry.werf.io/base/golang:1.12-alpine3.9 +LABEL io.deckhouse.internal.builder=true diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/go.mod b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/go.mod new file mode 100644 index 0000000000..d0866bd9e8 --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/go.mod @@ -0,0 +1,12 @@ +module example.com/app + +go 1.12 + +require ( + example.com/mylib v0.0.0 + example.com/otherlib v0.0.0 +) + +replace example.com/mylib => ./mylib + +replace example.com/otherlib => ./otherlib diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/go.sum b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/go.sum new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/main.go b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/main.go new file mode 100644 index 0000000000..cb32316f85 --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/main.go @@ -0,0 +1,11 @@ +package main + +import ( + "fmt" + + "example.com/mylib" +) + +func main() { + fmt.Println(mylib.Hello()) +} diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/mylib/go.mod b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/mylib/go.mod new file mode 100644 index 0000000000..773ea0f75a --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/mylib/go.mod @@ -0,0 +1,3 @@ +module example.com/mylib + +go 1.12 diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/mylib/lib.go b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/mylib/lib.go new file mode 100644 index 0000000000..2c120e9107 --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/mylib/lib.go @@ -0,0 +1,5 @@ +package mylib + +func Hello() string { + return "hello from mylib" +} diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/otherlib/go.mod b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/otherlib/go.mod new file mode 100644 index 0000000000..d93ce988ef --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/otherlib/go.mod @@ -0,0 +1,3 @@ +module example.com/otherlib + +go 1.12 diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/otherlib/lib.go b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/otherlib/lib.go new file mode 100644 index 0000000000..fdd128e47b --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/otherlib/lib.go @@ -0,0 +1,5 @@ +package otherlib + +func Hello() string { + return "hello from otherlib" +} diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/werf-giterminism.yaml b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/werf-giterminism.yaml new file mode 100644 index 0000000000..9483c3670c --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/werf-giterminism.yaml @@ -0,0 +1,5 @@ +giterminismConfigVersion: 1 +config: + goTemplateRendering: + allowEnvVariables: + - BUILDER_BASE_IMAGE diff --git a/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/werf.yaml b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/werf.yaml new file mode 100644 index 0000000000..6b665daded --- /dev/null +++ b/test/e2e/sbom/_fixtures/stage_deps_gomod/state1/werf.yaml @@ -0,0 +1,22 @@ +project: werf-test-e2e-sbom-stage-deps-gomod +configVersion: 1 +build: + sbom: + enable: true + standard: cyclonedx@1.6 +--- +image: app +from: {{ env "BUILDER_BASE_IMAGE" }} +git: + - add: / + to: /app + stageDependencies: + packages: + - go.mod + - go.sum +shell: + install: + - cd /app && CGO_ENABLED=0 go build -o /bin/app . +packages: + - type: go-mod + workdir: /app diff --git a/test/e2e/sbom/_fixtures/type_change/state1/werf.yaml b/test/e2e/sbom/_fixtures/type_change/state1/werf.yaml index c73a976d8b..23edc2cfeb 100644 --- a/test/e2e/sbom/_fixtures/type_change/state1/werf.yaml +++ b/test/e2e/sbom/_fixtures/type_change/state1/werf.yaml @@ -10,6 +10,9 @@ from: {{ env "BUILDER_BASE_IMAGE" }} git: - add: / to: /app + stageDependencies: + packages: + - go.mod shell: install: - cd /app && CGO_ENABLED=0 go build -o /bin/app . diff --git a/test/e2e/sbom/stage_dependencies_test.go b/test/e2e/sbom/stage_dependencies_test.go index 8a09a92794..5f746c39f0 100644 --- a/test/e2e/sbom/stage_dependencies_test.go +++ b/test/e2e/sbom/stage_dependencies_test.go @@ -116,6 +116,62 @@ var _ = Describe("SBOM stageDependencies cache invalidation", Label("e2e", "sbom XEntry("with local repo using Native Buildah with rootless isolation", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "native-rootless"}}), ) + DescribeTable("go.mod change tracked by git.stageDependencies.packages rebuilds the packages stage", + func(ctx SpecContext, testOpts sbomTestOptions) { + setupSbomBuildEnv(testOpts.setupEnvOptions) + + const packagesStageBuildingMarker = "Building stage app/packages" + + repoDirname := "repo_sbom_stage_deps_gomod" + SuiteData.InitTestRepo(ctx, repoDirname, "stage_deps_gomod/state0") + testRepoPath := SuiteData.GetTestRepoPath(repoDirname) + + utils.RunSucceedCommand(ctx, testRepoPath, "git", "tag", "v1.0.0") + + builderEnv := buildTrustedBuilderBase(ctx, testRepoPath, "sbom-stage-deps-gomod-builder") + + werfProject := werf.NewProject(SuiteData.WerfBinPath, testRepoPath) + + By("state0: initial build → packages stage built, SBOM generated") + out0 := werfProject.Build(ctx, &werf.BuildOptions{CommonOptions: werf.CommonOptions{Envs: builderEnv}}) + Expect(out0).To(ContainSubstring(packagesStageBuildingMarker), "expected initial packages stage build") + Expect(out0).To(ContainSubstring(sbomRegenMarker), "expected initial SBOM generation") + + bom0 := sbomtest.MustParseSBOMOutput(werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ExtraArgs: []string{"app"}, Envs: builderEnv}, + })) + Expect(sbomtest.FindComponent(bom0, "example.com/mylib", "v1.0.0")).NotTo(BeNil(), + "expected example.com/mylib@v1.0.0 in state0 BOM") + + By("rebuild without changes → packages stage from cache, SBOM cache hit") + outCached := werfProject.Build(ctx, &werf.BuildOptions{CommonOptions: werf.CommonOptions{Envs: builderEnv}}) + Expect(outCached).NotTo(ContainSubstring(packagesStageBuildingMarker), + "expected packages stage cache hit on unchanged build; output:\n%s", outCached) + Expect(outCached).To(ContainSubstring(sbomCachedMarker), + "expected cached SBOM marker; output:\n%s", outCached) + + By("state1: add module to go.mod (werf.yaml unchanged) → packages stage must rebuild") + SuiteData.UpdateTestRepo(ctx, repoDirname, "stage_deps_gomod/state1") + utils.RunSucceedCommand(ctx, testRepoPath, "git", "tag", "v1.1.0") + + out1 := werfProject.Build(ctx, &werf.BuildOptions{CommonOptions: werf.CommonOptions{Envs: builderEnv}}) + Expect(out1).To(ContainSubstring(packagesStageBuildingMarker), + "expected packages stage rebuild after go.mod tracked by stageDependencies.packages changed; output:\n%s", out1) + Expect(out1).To(ContainSubstring(sbomRegenMarker), + "expected SBOM regen after packages stage rebuild; output:\n%s", out1) + + bom1 := sbomtest.MustParseSBOMOutput(werfProject.SbomGet(ctx, &werf.SbomGetOptions{ + CommonOptions: werf.CommonOptions{ExtraArgs: []string{"app"}, Envs: builderEnv}, + })) + Expect(sbomtest.FindComponent(bom1, "example.com/otherlib", "v1.1.0")).NotTo(BeNil(), + "expected example.com/otherlib@v1.1.0 in state1 BOM") + }, + Entry("with local repo using Vanilla Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "vanilla-docker"}}), + Entry("with local repo using BuildKit Docker", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "buildkit-docker"}}), + XEntry("with local repo using Native Buildah with chroot isolation", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "native-chroot"}}), + XEntry("with local repo using Native Buildah with rootless isolation", sbomTestOptions{setupEnvOptions{ContainerBackendMode: "native-rootless"}}), + ) + DescribeTable("switching packages type from os-pm to go-mod regenerates SBOM with new scanner output", func(ctx SpecContext, testOpts sbomTestOptions) { setupSbomBuildEnv(testOpts.setupEnvOptions)