diff --git a/hydra-gates/scripts/lib/check_visual_coverage.py b/hydra-gates/scripts/lib/check_visual_coverage.py index fa4566b46..dee90bf21 100644 --- a/hydra-gates/scripts/lib/check_visual_coverage.py +++ b/hydra-gates/scripts/lib/check_visual_coverage.py @@ -83,7 +83,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from exclusion_reason import exclude_pattern, is_reason_bearing # noqa: E402 -from source_scope import js_comment_mask # noqa: E402 +from source_scope import js_code_mask, js_comment_mask, js_comments_only_mask # noqa: E402 GATE_NUM = 26 @@ -415,11 +415,14 @@ def _visual_exclude_status(vue_text: str) -> tuple[bool, str | None]: return (True, reason if is_reason_bearing(reason) else None) -def _e2e_corpus(app_dir: Path, visual_only: bool) -> str: +def _e2e_corpus(app_dir: Path, visual_only: bool, include_visual: bool = True) -> str: """Concatenated EXECUTABLE text of e2e test files. visual_only=True → only tests/e2e/visual/**. visual_only=False → all of tests/e2e/**. + include_visual=False → tests/e2e/visual/** contributes NOTHING, in either + mode. The caller passes it when CI never runs that directory; see + ``visual_suite_runs_in_ci``. A COMMENT IS NOT A BASELINE (.github#358) ----------------------------------------- @@ -454,12 +457,17 @@ def _e2e_corpus(app_dir: Path, visual_only: bool) -> str: alone. """ root = app_dir / "tests" / "e2e" + visual_root = root / "visual" if visual_only: - root = root / "visual" + if not include_visual: + return "" + root = visual_root if not root.is_dir(): return "" buf: list[str] = [] for p in root.rglob("*"): + if not include_visual and (p == visual_root or visual_root in p.parents): + continue if p.is_file() and p.suffix in (".ts", ".js", ".png", ".txt", ".json"): # Binary PNG baselines: we only need their FILENAME to match, so # record the name rather than the bytes. @@ -472,6 +480,167 @@ def _e2e_corpus(app_dir: Path, visual_only: bool) -> str: return "\n".join(buf) +# --------------------------------------------------------------------------- +# A BASELINE CI NEVER RUNS IS NOT A PROOF (keepiq#198) +# --------------------------------------------------------------------------- +# `tests/e2e/visual/**` used to be credited unconditionally, and FIRST. But the +# fleet's CI Playwright configs (keepiq, pipelinq, ...) put `'**/visual/**'` in +# their only project's `testIgnore`, because the PNGs are generated on a +# developer machine and would diff on the runner. So a new page whose only +# proof was a baseline passed this gate while CI never opened it. +# +# The credit now follows what CI EXECUTES. The CI job runs +# `npx playwright test --config=/playwright.config.ts` +# (falling back to ./playwright.config.ts) with no `--project`, so every +# declared project runs. The visual directory is credited only when +# +# * some entry of `projects: [...]` would collect it: its effective +# `testIgnore` (its own, which REPLACES the top-level one in Playwright, +# else the top-level one) does not name `visual`, its effective `testDir` +# (its own, else the top-level one, else the config's directory) contains +# or is inside tests/e2e/visual, and its `testMatch`, when it has one, +# names `visual` or matches any directory (`**/...`). +# +# Anything else, including no config at all, withholds the credit. Executed +# e2e tests outside tests/e2e/visual and `@visual exclude ` still count. +# --------------------------------------------------------------------------- +_TEST_PATH_RE = re.compile(r"^\s*playwright-test-path\s*:\s*['\"]?([^'\"#\s]+)", re.MULTILINE) +_DEFAULT_TEST_PATH = "tests/e2e" + + +def _playwright_test_path(app_dir: Path) -> str: + """The `playwright-test-path` the app's CI passes, else the default. + + `PLAYWRIGHT_TEST_PATH` in the environment wins (the quality workflow sets + it from the input); otherwise the app's own caller workflow is read. + """ + env = os.environ.get("PLAYWRIGHT_TEST_PATH", "").strip() + if env: + return env.rstrip("/") + wf_dir = app_dir / ".github" / "workflows" + if wf_dir.is_dir(): + for wf in sorted(wf_dir.glob("*.y*ml")): + m = _TEST_PATH_RE.search(_read(wf)) + if m and "${{" not in m.group(1): + return m.group(1).rstrip("/") + return _DEFAULT_TEST_PATH + + +def _ci_playwright_config(app_dir: Path) -> Path | None: + cand = app_dir / _playwright_test_path(app_dir) / "playwright.config.ts" + if cand.is_file(): + return cand + root = app_dir / "playwright.config.ts" + return root if root.is_file() else None + + +def _match_close(code: str, i: int) -> int: + """Index just past the bracket that closes the one at *i* (on a code mask).""" + pairs = {"[": "]", "{": "}", "(": ")"} + stack = [pairs[code[i]]] + j = i + 1 + while j < len(code) and stack: + c = code[j] + if c in pairs: + stack.append(pairs[c]) + elif stack and c == stack[-1]: + stack.pop() + j += 1 + return j + + +def _top_level_keys(code: str, a: int, b: int) -> dict[str, tuple[int, int]]: + """{key: (value_start, value_end)} for the keys at depth 0 of code[a:b].""" + out: dict[str, tuple[int, int]] = {} + key_re = re.compile(r"([A-Za-z_$][\w$]*)\s*:") + i = a + while i < b: + c = code[i] + if c in "[{(": + i = _match_close(code, i) + continue + m = key_re.match(code, i) + if m and (i == a or not (code[i - 1].isalnum() or code[i - 1] in "_$")): + vs = m.end() + j = vs + while j < b and code[j] != ",": + if code[j] in "[{(": + j = _match_close(code, j) + continue + j += 1 + out.setdefault(m.group(1), (vs, j)) + i = j + 1 + continue + i += 1 + return out + + +def _resolve_test_dir(raw: str, config_dir: Path) -> Path: + """Best-effort value of a `testDir:` expression. Unknown shapes mean the + config's own directory, which is Playwright's default.""" + # `__dirname`, `'./x'`, `path.join(__dirname, 'x')` and + # `path.resolve(__dirname, '..', 'x')` all reduce to the config directory + # joined with the string literals in order. + parts = re.findall(r"""['"]([^'"]*)['"]""", raw) + path = config_dir + for part in parts: + path = path / part + return path.resolve() + + +def _names_visual(raw: str) -> bool: + return "visual" in raw + + +def visual_suite_runs_in_ci(app_dir: Path) -> tuple[bool, str]: + """(runs, why). True only when the CI Playwright config collects + tests/e2e/visual/**. See the block comment above.""" + visual_dir = (app_dir / "tests" / "e2e" / "visual").resolve() + config = _ci_playwright_config(app_dir) + if config is None: + return (False, f"no CI Playwright config at {_playwright_test_path(app_dir)}/playwright.config.ts") + text = js_comments_only_mask(_read(config)) + code = js_code_mask(_read(config)) + rel_cfg = str(config.relative_to(app_dir)) + m = re.search(r"defineConfig\s*\(\s*\{", code) or re.search(r"export\s+default\s*\{", code) + if not m: + return (False, f"{rel_cfg}: no defineConfig({{...}}) / export default {{...}} object found") + obj_a = m.end() - 1 + obj_b = _match_close(code, obj_a) + top = _top_level_keys(code, obj_a + 1, obj_b - 1) + top_ignore = text[slice(*top["testIgnore"])] if "testIgnore" in top else None + if "projects" not in top: + return (False, f"{rel_cfg}: declares no projects, so none runs visual/") + top_dir = _resolve_test_dir(text[slice(*top["testDir"])], config.parent) if "testDir" in top else config.parent.resolve() + top_match = text[slice(*top["testMatch"])] if "testMatch" in top else None + pa, pb = top["projects"] + arr = code.find("[", pa, pb) + if arr < 0: + return (False, f"{rel_cfg}: projects is not an array literal") + arr_end = _match_close(code, arr) + i = arr + 1 + while i < arr_end - 1: + if code[i] == "{": + end = _match_close(code, i) + keys = _top_level_keys(code, i + 1, end - 1) + name_raw = text[slice(*keys["name"])].strip() if "name" in keys else "?" + i = end + # A project's own testIgnore REPLACES the top-level one in + # Playwright; it does not add to it. + ignore = text[slice(*keys["testIgnore"])] if "testIgnore" in keys else top_ignore + if ignore is not None and _names_visual(ignore): + continue + tdir = _resolve_test_dir(text[slice(*keys["testDir"])], config.parent) if "testDir" in keys else top_dir + if not (tdir == visual_dir or tdir in visual_dir.parents or visual_dir in tdir.parents): + continue + match = text[slice(*keys["testMatch"])] if "testMatch" in keys else top_match + if match is not None and not (_names_visual(match) or "**/" in match): + continue + return (True, f"{rel_cfg}: project {name_raw} runs tests/e2e/visual") + i += 1 + return (False, f"{rel_cfg}: no project collects tests/e2e/visual (ignored or outside its testDir)") + + def is_covered(page: dict, visual_corpus: str, e2e_corpus: str) -> bool: """True if a visual baseline/spec or an e2e test references the page.""" stem = Path(page["path"]).stem @@ -540,8 +709,14 @@ def run_gate(app_dir: Path) -> int: f"manifest \"type\":\"page\" entry resolving to a .vue)" ) return EXIT_EMPTY_SCOPE - visual_corpus = _e2e_corpus(app_dir, visual_only=True) - e2e_corpus = _e2e_corpus(app_dir, visual_only=False) + visual_runs, visual_why = visual_suite_runs_in_ci(app_dir) + if not visual_runs and (app_dir / "tests" / "e2e" / "visual").is_dir(): + print( + f"[gate-{GATE_NUM}] tests/e2e/visual/** is NOT credited: {visual_why}. " + f"Only executed e2e tests and @visual exclude count." + ) + visual_corpus = _e2e_corpus(app_dir, visual_only=True, include_visual=visual_runs) + e2e_corpus = _e2e_corpus(app_dir, visual_only=False, include_visual=visual_runs) findings: list[str] = [] for page in pages: excluded, reason = _visual_exclude_status(_read(app_dir / page["path"])) @@ -553,7 +728,7 @@ def run_gate(app_dir: Path) -> int: if not is_covered(page, visual_corpus, e2e_corpus): findings.append( f"{page['path']} — new page component missing visual-regression " - f"baseline (tests/e2e/visual/**) / e2e test / @visual exclude" + f"baseline that CI runs (tests/e2e/visual/**) / e2e test / @visual exclude" ) for line in sorted(set(findings)): print(line) @@ -577,8 +752,9 @@ def run_gate(app_dir: Path) -> int: def run_report(app_dir: Path) -> int: base_ref = os.environ.get("HYDRA_GATE_BASE_REF") or None pages = _collect(app_dir, base_ref) - visual_corpus = _e2e_corpus(app_dir, visual_only=True) - e2e_corpus = _e2e_corpus(app_dir, visual_only=False) + visual_runs, visual_why = visual_suite_runs_in_ci(app_dir) + visual_corpus = _e2e_corpus(app_dir, visual_only=True, include_visual=visual_runs) + e2e_corpus = _e2e_corpus(app_dir, visual_only=False, include_visual=visual_runs) covered = uncovered = excluded = 0 rows = [] for page in pages: @@ -604,6 +780,8 @@ def run_report(app_dir: Path) -> int: "uncovered": uncovered, }, "pages": rows, + "visual_suite_credited": visual_runs, + "visual_suite_reason": visual_why, } print(json.dumps(out, indent=2)) return 0 diff --git a/hydra-gates/scripts/lib/source_scope.py b/hydra-gates/scripts/lib/source_scope.py index 5c2e130f7..d55453da8 100644 --- a/hydra-gates/scripts/lib/source_scope.py +++ b/hydra-gates/scripts/lib/source_scope.py @@ -212,7 +212,7 @@ def _regex_can_start(prev_char: str, prev_word: str) -> bool: return True -def js_mask(text: str, *, blank_strings: bool = True) -> str: +def js_mask(text: str, *, blank_strings: bool = True, blank_regex: bool = True) -> str: """A same-length copy of *text* with the non-code regions blanked. Comments and regex literals always go. String / template CONTENTS go only @@ -280,7 +280,8 @@ def blank(a: int, b: int) -> None: if c == "/" and _regex_can_start(prev_char, prev_word): j = _skip_regex(text, i) if j > 0: - blank(i, j) + if blank_regex: + blank(i, j) prev_char, prev_word = ")", "" # a regex literal is a value i = j continue @@ -312,6 +313,16 @@ def js_comment_mask(text: str) -> str: return js_mask(text, blank_strings=False) +def js_comments_only_mask(text: str) -> str: + r"""Comments blanked; string AND regex literals left intact. + + For reading configuration whose values are patterns: gate-26 reads a + Playwright `testMatch: /visual\/.*\.spec\.ts/` or `testIgnore: + ['**/visual/**']`, and a regex is exactly the evidence there. + """ + return js_mask(text, blank_strings=False, blank_regex=False) + + # --------------------------------------------------------------------------- # PHP # --------------------------------------------------------------------------- diff --git a/hydra-gates/scripts/lib/test_check_visual_coverage.py b/hydra-gates/scripts/lib/test_check_visual_coverage.py index d50e9d4a5..6539b8627 100644 --- a/hydra-gates/scripts/lib/test_check_visual_coverage.py +++ b/hydra-gates/scripts/lib/test_check_visual_coverage.py @@ -67,6 +67,19 @@ def check(name: str, cond: bool, detail: str = "") -> None: """ +# keepiq#198: tests/e2e/visual/** is credited only when the app's CI Playwright +# config runs it. Every fixture below that proves coverage THROUGH the visual +# directory therefore ships this config, which declares a project collecting it. +_CI_CONFIG_RUNS_VISUAL = """import { defineConfig, devices } from '@playwright/test' +export default defineConfig({ + testDir: __dirname, + projects: [ + { name: 'chromium', testIgnore: ['**/visual/**'], use: { ...devices['Desktop Chrome'] } }, + { name: 'visual', testMatch: /visual\\/.*\\.spec\\.(ts|js)/, use: { ...devices['Desktop Chrome'] } }, + ], +}) +""" + def _git(args: list[str], cwd: Path) -> None: subprocess.run( ["git", "-c", "user.email=t@t", "-c", "user.name=t", *args], @@ -91,6 +104,7 @@ def _repo(n_pages: int, *, covered: bool = False) -> tempfile.TemporaryDirectory if covered: vis = root / "tests" / "e2e" / "visual" vis.mkdir(parents=True) + (root / "tests" / "e2e" / "playwright.config.ts").write_text(_CI_CONFIG_RUNS_VISUAL) for i in range(1, n_pages + 1): (vis / f"p{i}.spec.js").write_text( f"test('P{i}', async () => {{ await expect(page)" @@ -410,7 +424,8 @@ def _repo_mixed(*, manifest: str | None = None, }) """ -with _repo_mixed(extra={"tests/e2e/visual/x.spec.ts": _VISUAL_COMMENT_ONLY}) as t: +with _repo_mixed(extra={"tests/e2e/visual/x.spec.ts": _VISUAL_COMMENT_ONLY, + "tests/e2e/playwright.config.ts": _CI_CONFIG_RUNS_VISUAL}) as t: rc, out = _run(Path(t), None) check( "a comment saying a page still OWES a baseline is not coverage", @@ -418,7 +433,8 @@ def _repo_mixed(*, manifest: str | None = None, f"rc={rc} out={out.strip()[-220:]}", ) -with _repo_mixed(extra={"tests/e2e/visual/x.spec.ts": _VISUAL_REAL}) as t: +with _repo_mixed(extra={"tests/e2e/visual/x.spec.ts": _VISUAL_REAL, + "tests/e2e/playwright.config.ts": _CI_CONFIG_RUNS_VISUAL}) as t: rc, out = _run(Path(t), None) check( "a spec that navigates to the page and screenshots it still counts", @@ -434,7 +450,8 @@ def _repo_mixed(*, manifest: str | None = None, \tawait expect(page).toHaveScreenshot('FlowDetailPage.png') }) """ -with _repo_mixed(extra={"tests/e2e/visual/x.spec.ts": _VISUAL_STRING_ONLY}) as t: +with _repo_mixed(extra={"tests/e2e/visual/x.spec.ts": _VISUAL_STRING_ONLY, + "tests/e2e/playwright.config.ts": _CI_CONFIG_RUNS_VISUAL}) as t: rc, out = _run(Path(t), None) check( "a reference that exists only as a STRING literal still counts", @@ -444,7 +461,7 @@ def _repo_mixed(*, manifest: str | None = None, # A .png baseline named after the component is the canonical proof and must be # untouched by the corpus change — its filename is the whole reference. -with _repo_mixed() as t: +with _repo_mixed(extra={"tests/e2e/playwright.config.ts": _CI_CONFIG_RUNS_VISUAL}) as t: _b = Path(t) / "tests" / "e2e" / "visual" _b.mkdir(parents=True) (_b / "FlowDetailPage.png").write_bytes(b"\x89PNG\r\n") @@ -455,6 +472,167 @@ def _repo_mixed(*, manifest: str | None = None, f"rc={rc} out={out.strip()[-220:]}", ) +# --------------------------------------------------------------------------- +# keepiq#198: A BASELINE CI NEVER RUNS IS NOT A PROOF +# --------------------------------------------------------------------------- +# The fleet's CI Playwright configs (keepiq, pipelinq) give their only project +# `testIgnore: ['**/visual/**']`, because the PNG baselines are made on a +# developer machine. tests/e2e/visual/** was still credited, so a page whose +# only proof was a baseline passed while CI never opened it. Every arm that +# withholds credit is paired with one that keeps it, so "never credit visual" +# would not pass this block either. +print() +print("== gate-26: tests/e2e/visual counts only when CI runs it (keepiq#198) ==") + +# The keepiq shape, verbatim in the part that matters: one project, with +# '**/visual/**' in its own testIgnore, and a COMMENT naming visual that must +# not be read as configuration. +_CI_CONFIG_IGNORES_VISUAL = """import { defineConfig, devices } from '@playwright/test' +export default defineConfig({ +\ttestDir: __dirname, +\t// the visual project runs locally only: `npx playwright test --project visual` +\tprojects: [ +\t\t{ +\t\t\tname: 'chromium', +\t\t\ttestIgnore: [ +\t\t\t\t'**/node_modules/**', +\t\t\t\t'**/visual/**', +\t\t\t], +\t\t\tuse: { ...devices['Desktop Chrome'] }, +\t\t}, +\t], +}) +""" + +_BASELINE_ONLY = {"tests/e2e/visual/x.spec.ts": _VISUAL_REAL} + +with _repo_mixed(extra={**_BASELINE_ONLY, + "tests/e2e/playwright.config.ts": _CI_CONFIG_IGNORES_VISUAL}) as t: + rc, out = _run(Path(t), None) + check( + "#198 RED-BEFORE: a baseline the CI config ignores does NOT cover the page", + rc == EXIT_FAIL and "FlowDetailPage.vue" in out, + f"rc={rc} out={out.strip()[-260:]}", + ) + check( + "#198: the run says WHY the visual directory earned no credit", + "is NOT credited" in out and "no project collects" in out, + out.strip()[-260:], + ) + +with _repo_mixed(extra=dict(_BASELINE_ONLY)) as t: + rc, out = _run(Path(t), None) + check( + "#198: no CI Playwright config at all -> a baseline earns no credit", + rc == EXIT_FAIL and "no CI Playwright config" in out, + f"rc={rc} out={out.strip()[-260:]}", + ) + +with _repo_mixed(extra={**_BASELINE_ONLY, + "tests/e2e/playwright.config.ts": _CI_CONFIG_RUNS_VISUAL}) as t: + rc, out = _run(Path(t), None) + check( + "#198 control: a config with a project that runs visual/ keeps the credit", + rc == EXIT_PASS, + f"rc={rc} out={out.strip()[-260:]}", + ) + +# Executed e2e tests outside visual/ still count when visual/ is ignored. +_EXECUTED = """import { test, expect } from '@playwright/test' +test('flow detail opens', async ({ page }) => { +\tawait page.goto('/index.php/apps/fx/#/flow-detail') +\tawait expect(page.getByTestId('FlowDetailPage')).toBeVisible() +}) +""" +with _repo_mixed(extra={**_BASELINE_ONLY, + "tests/e2e/workflows/flow.spec.ts": _EXECUTED, + "tests/e2e/playwright.config.ts": _CI_CONFIG_IGNORES_VISUAL}) as t: + rc, out = _run(Path(t), None) + check( + "#198: an EXECUTED e2e test still covers the page when visual/ is ignored", + rc == EXIT_PASS, + f"rc={rc} out={out.strip()[-260:]}", + ) + +with _repo_mixed(extra={**_BASELINE_ONLY, + "tests/e2e/playwright.config.ts": _CI_CONFIG_IGNORES_VISUAL}) as t: + pg = Path(t) / "src" / "views" / "FlowDetailPage.vue" + pg.write_text("\n" + + pg.read_text()) + rc, out = _run(Path(t), None) + check( + "#198: @visual exclude still counts when visual/ is ignored", + rc == EXIT_PASS, + f"rc={rc} out={out.strip()[-260:]}", + ) + +# Playwright semantics: a project's own testIgnore REPLACES the top-level one. +_CI_CONFIG_TOP_IGNORE_OVERRIDDEN = """import { defineConfig } from '@playwright/test' +export default defineConfig({ +\ttestIgnore: ['**/visual/**'], +\tprojects: [ +\t\t{ name: 'chromium' }, +\t\t{ name: 'visual', testIgnore: ['**/node_modules/**'] }, +\t], +}) +""" +with _repo_mixed(extra={**_BASELINE_ONLY, + "tests/e2e/playwright.config.ts": _CI_CONFIG_TOP_IGNORE_OVERRIDDEN}) as t: + rc, out = _run(Path(t), None) + check( + "#198: a project whose own testIgnore replaces a top-level visual ignore runs it", + rc == EXIT_PASS, + f"rc={rc} out={out.strip()[-260:]}", + ) + +_CI_CONFIG_TOP_IGNORE_ONLY = _CI_CONFIG_TOP_IGNORE_OVERRIDDEN.replace( + "{ name: 'visual', testIgnore: ['**/node_modules/**'] }", "{ name: 'firefox' }") +with _repo_mixed(extra={**_BASELINE_ONLY, + "tests/e2e/playwright.config.ts": _CI_CONFIG_TOP_IGNORE_ONLY}) as t: + rc, out = _run(Path(t), None) + check( + "#198: a top-level visual ignore that every project inherits withholds credit", + rc == EXIT_FAIL, + f"rc={rc} out={out.strip()[-260:]}", + ) + +# The CI config is found through the app's `playwright-test-path`, read from +# its caller workflow. openregister uses tests/e2e/ci, whose testDir does not +# reach tests/e2e/visual unless it says so. +_CALLER = """jobs: + quality: + uses: ConductionNL/.github/.github/workflows/quality.yml@main + with: + playwright-test-path: tests/e2e/ci +""" +_CI_SUBDIR_CONFIG = """import { defineConfig } from '@playwright/test' +export default defineConfig({ +\ttestDir: __dirname, +\tprojects: [{ name: 'chromium' }], +}) +""" +with _repo_mixed(extra={**_BASELINE_ONLY, + ".github/workflows/code-quality.yml": _CALLER, + "tests/e2e/ci/playwright.config.ts": _CI_SUBDIR_CONFIG}) as t: + rc, out = _run(Path(t), None) + check( + "#198: a CI config whose testDir does not reach tests/e2e/visual withholds credit", + rc == EXIT_FAIL, + f"rc={rc} out={out.strip()[-260:]}", + ) + +with _repo_mixed(extra={**_BASELINE_ONLY, + ".github/workflows/code-quality.yml": _CALLER, + "tests/e2e/ci/playwright.config.ts": + _CI_SUBDIR_CONFIG.replace("testDir: __dirname", + "testDir: path.join(__dirname, '..')")}) as t: + rc, out = _run(Path(t), None) + check( + "#198: the same config with testDir reaching tests/e2e keeps the credit", + rc == EXIT_PASS, + f"rc={rc} out={out.strip()[-260:]}", + ) + print() print(f"{_passed}/{_passed + len(_failed)} passed") if _failed: diff --git a/hydra-gates/scripts/lib/test_gates_23_33_never_green_over_nothing.sh b/hydra-gates/scripts/lib/test_gates_23_33_never_green_over_nothing.sh index c7208420e..622ebb1b7 100644 --- a/hydra-gates/scripts/lib/test_gates_23_33_never_green_over_nothing.sh +++ b/hydra-gates/scripts/lib/test_gates_23_33_never_green_over_nothing.sh @@ -202,7 +202,7 @@ echo "== clean/ — the SAME gates must not fire (no widening) ==" # =========================================================================== if _run "${FIXTURES}/clean"; then _expect 23 "PASS" "gate-23 clean on an app that consumes the OR abstraction" - _expect 26 "PASS" "gate-26 clean when the page has a visual baseline" + _expect 26 "PASS" "gate-26 clean when the page has a visual baseline that CI runs" _expect 27 "PASS" "gate-27 clean on the ADR-041 event recipe" _expect 30 "PASS" "gate-30 clean when both scrape targets declare #[PublicPage]" _expect 31 "PASS" "gate-31 clean when every carries alt" diff --git a/hydra-gates/scripts/test-fixtures/gate-acceptance/prose-not-proof/clean/tests/e2e/playwright.config.ts b/hydra-gates/scripts/test-fixtures/gate-acceptance/prose-not-proof/clean/tests/e2e/playwright.config.ts new file mode 100644 index 000000000..6f77c02e0 --- /dev/null +++ b/hydra-gates/scripts/test-fixtures/gate-acceptance/prose-not-proof/clean/tests/e2e/playwright.config.ts @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: EUPL-1.2 +// gate-26 credits tests/e2e/visual/** only when the CI config runs it +// (keepiq#198), so the clean arm declares a project that does. Without it the +// visual spec beside this file would be a baseline CI never opens. +import { defineConfig } from '@playwright/test' + +export default defineConfig({ + testDir: __dirname, + projects: [ + { name: 'chromium', testIgnore: ['**/visual/**'] }, + { name: 'visual', testMatch: /visual\/.*\.spec\.ts/ }, + ], +}) diff --git a/hydra-gates/scripts/test-fixtures/gates-23-33/clean/tests/e2e/playwright.config.ts b/hydra-gates/scripts/test-fixtures/gates-23-33/clean/tests/e2e/playwright.config.ts new file mode 100644 index 000000000..c0a5990d4 --- /dev/null +++ b/hydra-gates/scripts/test-fixtures/gates-23-33/clean/tests/e2e/playwright.config.ts @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: EUPL-1.2 +// gate-26 credits tests/e2e/visual/** only when the CI config runs it +// (keepiq#198), so this clean fixture declares a project that does. +import { defineConfig } from '@playwright/test' + +export default defineConfig({ + testDir: __dirname, + projects: [ + { name: 'chromium', testIgnore: ['**/visual/**'] }, + { name: 'visual', testMatch: /visual\/.*\.spec\.(ts|js)/ }, + ], +}) diff --git a/hydra-gates/tests/test-hydra-gates-bin.sh b/hydra-gates/tests/test-hydra-gates-bin.sh index 34f675f9d..ca07a690e 100755 --- a/hydra-gates/tests/test-hydra-gates-bin.sh +++ b/hydra-gates/tests/test-hydra-gates-bin.sh @@ -418,6 +418,14 @@ test('FixtureView', async ({ page }) => { await expect(page).toHaveScreenshot('FixtureView.png') }) JS +# gate-26 credits tests/e2e/visual only when the CI config runs it (keepiq#198). +cat > "${FIX}/tests/e2e/playwright.config.ts" <<'TS' +import { defineConfig } from '@playwright/test' +export default defineConfig({ + testDir: __dirname, + projects: [{ name: 'visual', testMatch: /visual\/.*\.spec\.js/ }], +}) +TS git -C "${FIX}" add -A >/dev/null 2>&1 git -C "${FIX}" commit -qm "fixture: register an integration leaf" >/dev/null 2>&1 OUT_STRUCT="$("${BIN}" --app-dir "${FIX}" --base "${BASE_SHA}" --require-full-coverage 2>&1)"; RC_STRUCT=$?