diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9febbaaf67..d8cbb74102 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3491,15 +3491,9 @@ jobs: # Proper fix: hoist well-known-binding lookup out of # build_optimized_libs into link.rs so it runs even when # auto-optimize is skipped. Tracked separately. - # #8475: the three fastify-importing snippets cannot run under the - # harness's PERRY_NO_AUTO_OPTIMIZE=1 speed flag — `perry compile` - # hard-errors on `import 'fastify'` there, because the prebuilt - # stdlib is not built with `external-fastify-pump` and the request - # loop would hang. Excluded on the same terms as the HTTP aggregate - # above until the harness can run this class in a second, - # auto-optimizing pass. COVERAGE GAP, tracked in #8475 — not a - # statement that these snippets work. - cmd_exclude_gallery: "./scripts/run_doc_tests.sh --verbose --skip-xcompile --filter-exclude ui/gallery.ts --filter-exclude stdlib/http/snippets.ts --filter-exclude getting-started/npm_packages.ts --filter-exclude stdlib/http/fastify_json.ts --filter-exclude stdlib/overview/snippets.ts" + # Fastify examples declare `requires: auto-optimize`; the harness + # builds their specialized libraries and counts them in this run. + cmd_exclude_gallery: "./scripts/run_doc_tests.sh --verbose --skip-xcompile --filter-exclude ui/gallery.ts --filter-exclude stdlib/http/snippets.ts" cmd_gallery: "./scripts/run_doc_tests.sh --verbose --skip-xcompile --filter ui/gallery.ts" # Repeat `--xcompile-only-target=…` per target rather than a # single comma-delimited value because PowerShell splits even @@ -3542,9 +3536,7 @@ jobs: shell: pwsh # The well-known HTTP aggregate remains excluded on every host # while its no-auto ext-archive routing issue is tracked. - # #8475: same fastify / PERRY_NO_AUTO_OPTIMIZE exclusion as the - # macOS entry above. COVERAGE GAP, tracked there. - cmd_exclude_gallery: "./scripts/run_doc_tests.ps1 --verbose --skip-xcompile --filter-exclude ui/gallery.ts --filter-exclude stdlib/http/snippets.ts --filter-exclude getting-started/npm_packages.ts --filter-exclude stdlib/http/fastify_json.ts --filter-exclude stdlib/overview/snippets.ts" + cmd_exclude_gallery: "./scripts/run_doc_tests.ps1 --verbose --skip-xcompile --filter-exclude ui/gallery.ts --filter-exclude stdlib/http/snippets.ts" cmd_gallery: "./scripts/run_doc_tests.ps1 --verbose --skip-xcompile --filter ui/gallery.ts" cmd_xcompile_blocking: "./scripts/run_doc_tests.ps1 --verbose --xcompile-only --xcompile-only-target=web --xcompile-only-target=wasm" cmd_xcompile_advisory: "./scripts/run_doc_tests.ps1 --verbose --xcompile-only" diff --git a/changelog.d/9820-fastify-doc-test-coverage.md b/changelog.d/9820-fastify-doc-test-coverage.md new file mode 100644 index 0000000000..4ba08e5f82 --- /dev/null +++ b/changelog.d/9820-fastify-doc-test-coverage.md @@ -0,0 +1,6 @@ +Restore the three Fastify documentation examples to the host doc-test CI run. +The examples declare `requires: auto-optimize` in their banners, which lets the +harness rebuild their specialized runtime libraries while ordinary examples +continue using prebuilt archives. Required examples participate in the normal +pass/fail report; compiler failures remain gate failures. Their existing +compile-only setting avoids starting servers or connecting to external services. diff --git a/crates/perry-doc-tests/src/main.rs b/crates/perry-doc-tests/src/main.rs index adc3837802..94f1ac009b 100644 --- a/crates/perry-doc-tests/src/main.rs +++ b/crates/perry-doc-tests/src/main.rs @@ -15,6 +15,8 @@ use serde::Serialize; mod image_diff; mod lint; +#[cfg(test)] +mod tests; #[derive(Parser, Debug)] #[command(name = "doc-tests", about = "Perry documentation-example test harness")] @@ -357,6 +359,7 @@ struct Example { platforms: BTreeSet, targets: BTreeSet, compile_only: bool, + requires_auto_optimize: bool, widget_bundle_id: Option, } @@ -402,6 +405,7 @@ fn discover_examples(root: &Path) -> Result> { platforms: banner.platforms, targets: banner.targets, compile_only: banner.compile_only, + requires_auto_optimize: banner.requires_auto_optimize, widget_bundle_id: banner.widget_bundle_id, }); } @@ -419,6 +423,7 @@ struct Banner { /// single-program timeout. Catches API/TS drift without the /// integration-test overhead. compile_only: bool, + requires_auto_optimize: bool, /// Required for any `*-widget` / `wearos-tile` target — passed as /// `--app-bundle-id` on the perry compile invocation. widget_bundle_id: Option, @@ -455,6 +460,18 @@ fn read_banner(path: &Path) -> Result { if v.eq_ignore_ascii_case("false") || v == "0" || v.eq_ignore_ascii_case("no") { b.compile_only = true; } + } else if let Some(rest) = body.strip_prefix("requires:") { + for requirement in rest.split(',').map(str::trim) { + match requirement { + "auto-optimize" => b.requires_auto_optimize = true, + _ => { + return Err(anyhow!( + "{}: unknown doc-example requirement `{requirement}`", + path.display() + )) + } + } + } } else if let Some(rest) = body.strip_prefix("widget-bundle-id:") { let v = rest.trim(); if !v.is_empty() { @@ -494,7 +511,7 @@ fn run_one( }); if !no_compile { - if let Err(e) = compile(perry_bin, &ex.path, &bin_path) { + if let Err(e) = compile(perry_bin, ex, &bin_path) { return ExampleReport { file: rel.to_string(), kind: ex.kind, @@ -733,6 +750,7 @@ fn cross_compile_one( }; let mut cmd = Command::new(perry_bin); + configure_compile_environment(&mut cmd, ex); cmd.arg("compile") .arg(&ex.path) .arg("--target") @@ -807,13 +825,25 @@ fn cross_compile_one( } } -fn compile(perry_bin: &Path, src: &Path, out: &Path) -> Result<()> { - let out_status = Command::new(perry_bin) - .arg(src) +fn configure_compile_environment(cmd: &mut Command, example: &Example) { + // The host wrappers select prebuilt libraries for the ordinary examples. + // Fastify requires a specialized stdlib with its request pump, so remove + // the override only from this compiler child. Never mutate the harness's + // environment: subsequent examples still benefit from the prebuilt libs. + if example.requires_auto_optimize { + cmd.env_remove("PERRY_NO_AUTO_OPTIMIZE"); + } +} + +fn compile(perry_bin: &Path, example: &Example, out: &Path) -> Result<()> { + let mut cmd = Command::new(perry_bin); + configure_compile_environment(&mut cmd, example); + let out_status = cmd + .arg(&example.path) .arg("-o") .arg(out) .output() - .with_context(|| format!("launching perry for {}", src.display()))?; + .with_context(|| format!("launching perry for {}", example.path.display()))?; if !out_status.status.success() { return Err(anyhow!( "perry {}: {}", diff --git a/crates/perry-doc-tests/src/tests.rs b/crates/perry-doc-tests/src/tests.rs new file mode 100644 index 0000000000..ddb8edd022 --- /dev/null +++ b/crates/perry-doc-tests/src/tests.rs @@ -0,0 +1,41 @@ +use super::*; + +#[test] +fn fastify_examples_request_specialized_compilation_without_being_skipped() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../docs/examples"); + for file in [ + "getting-started/npm_packages.ts", + "stdlib/http/fastify_json.ts", + "stdlib/overview/snippets.ts", + ] { + let banner = read_banner(&root.join(file)).unwrap(); + assert!( + banner.requires_auto_optimize, + "{file}: must rebuild the Fastify pump" + ); + assert!( + banner.compile_only, + "{file}: requires external services to run" + ); + for host in ["macos", "linux", "windows"] { + assert!( + banner.platforms.contains(host), + "{file}: {host} must compile it" + ); + } + } +} + +#[test] +fn unknown_requirement_is_an_error_instead_of_silently_disabling_coverage() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("typo.ts"); + std::fs::write( + &path, + "// requires: auto-optmize\nconsole.log('example');\n", + ) + .unwrap(); + let error = read_banner(&path).unwrap_err().to_string(); + assert!(error.contains("typo.ts")); + assert!(error.contains("unknown doc-example requirement `auto-optmize`")); +} diff --git a/crates/perry-doc-tests/tests/compiler_environment.rs b/crates/perry-doc-tests/tests/compiler_environment.rs new file mode 100644 index 0000000000..2d9884a143 --- /dev/null +++ b/crates/perry-doc-tests/tests/compiler_environment.rs @@ -0,0 +1,109 @@ +#![cfg(unix)] + +use std::os::unix::fs::PermissionsExt; +use std::process::Command; + +fn run_examples(reject_required: bool) -> (std::process::Output, serde_json::Value, String) { + let dir = tempfile::tempdir().unwrap(); + let examples = dir.path().join("examples"); + std::fs::create_dir(&examples).unwrap(); + std::fs::write( + examples.join("required.ts"), + "// requires: auto-optimize\n// run: false\nconsole.log('required');\n", + ) + .unwrap(); + std::fs::write( + examples.join("ordinary.ts"), + "// run: false\nconsole.log('ordinary');\n", + ) + .unwrap(); + std::fs::write( + examples.join("z_after.ts"), + "// run: false\nconsole.log('after');\n", + ) + .unwrap(); + let compiler = dir.path().join("compiler.sh"); + std::fs::write( + &compiler, + r#"#!/bin/sh +case "$1" in + */required.ts) + if test "${PERRY_NO_AUTO_OPTIMIZE+x}" = x; then + echo 'required example inherited PERRY_NO_AUTO_OPTIMIZE' >&2 + exit 7 + fi + echo required:auto >> "$PERRY_DOC_TEST_COMPILER_LOG" + if test "$PERRY_DOC_TEST_REJECT_REQUIRED" = 1; then + echo 'required example compilation failed' >&2 + exit 17 + fi + ;; + */ordinary.ts|*/z_after.ts) + if test "$PERRY_NO_AUTO_OPTIMIZE" != 1; then + echo 'ordinary example lost its prebuilt-archive setting' >&2 + exit 8 + fi + echo "$(basename "$1" .ts):prebuilt" >> "$PERRY_DOC_TEST_COMPILER_LOG" + ;; + *) exit 9 ;; +esac +"#, + ) + .unwrap(); + std::fs::set_permissions(&compiler, std::fs::Permissions::from_mode(0o755)).unwrap(); + let report = dir.path().join("report.json"); + let log = dir.path().join("compiler.log"); + let output = Command::new(env!("CARGO_BIN_EXE_doc-tests")) + .current_dir(env!("CARGO_MANIFEST_DIR")) + .args(["--skip-xcompile", "--examples-dir"]) + .arg(&examples) + .arg("--perry") + .arg(&compiler) + .arg("--json") + .arg(&report) + .env("PERRY_NO_AUTO_OPTIMIZE", "1") + .env("PERRY_DOC_TEST_COMPILER_LOG", &log) + .env( + "PERRY_DOC_TEST_REJECT_REQUIRED", + if reject_required { "1" } else { "0" }, + ) + .output() + .unwrap(); + let report = serde_json::from_slice(&std::fs::read(report).unwrap()).unwrap(); + (output, report, std::fs::read_to_string(log).unwrap()) +} + +#[test] +fn required_compilation_removes_only_its_own_no_auto_override() { + let (output, report, calls) = run_examples(false); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stdout) + ); + assert_eq!(report["passed"], 3); + assert_eq!(report["failed"], 0); + assert_eq!(report["skipped"], 0); + assert_eq!( + calls, + "ordinary:prebuilt\nrequired:auto\nz_after:prebuilt\n" + ); +} + +#[test] +fn required_compilation_failures_are_counted_and_fail_the_harness() { + let (output, report, calls) = run_examples(true); + assert_eq!(output.status.code(), Some(1)); + assert_eq!(report["passed"], 2); + assert_eq!(report["failed"], 1); + assert_eq!(report["skipped"], 0); + assert_eq!(report["results"][1]["status"], "compile_fail"); + assert!(report["results"][1]["detail"] + .as_str() + .unwrap() + .contains("exit=17")); + assert_eq!( + calls, + "ordinary:prebuilt\nrequired:auto\nz_after:prebuilt\n" + ); +} diff --git a/docs/examples/README.md b/docs/examples/README.md index 15029951be..daba8a2e72 100644 --- a/docs/examples/README.md +++ b/docs/examples/README.md @@ -1,8 +1,9 @@ # Perry Doc Examples Every `.ts` file under this directory is a real, compilable program that is -verified by `cargo run -p perry-doc-tests` on every PR. Documentation pages in -`docs/src/` pull these files in via mdBook's `{{#include}}` directive, so the +verified by `cargo run -p perry-doc-tests` in the full doc-tests CI job. +Documentation pages in `docs/src/` pull these files in via mdBook's +`{{#include}}` directive, so the code you see on the rendered docs site is the same code CI is checking. ## Adding an example @@ -20,6 +21,18 @@ Runtime examples (non-UI) should list all three platforms. UI examples list whichever platforms their widgets support. The harness skips an example whose banner doesn't include the current host platform. +Examples that need specialized runtime libraries, such as Fastify's request +pump, add `// requires: auto-optimize` to the opening banner (within its first +15 lines). The harness removes `PERRY_NO_AUTO_OPTIMIZE` from that example's +compiler process, so it builds the required libraries and participates in the +normal pass/fail report. Other examples keep using the host runner's prebuilt +libraries. Unknown requirements fail discovery rather than silently being +ignored. + +Use `// run: false` for examples that need external services or run indefinitely. +They still compile and link; combining it with `requires: auto-optimize` tests +the specialized build without starting a server or connecting to a database. + 3. Reference it from markdown: ```markdown diff --git a/docs/examples/getting-started/npm_packages.ts b/docs/examples/getting-started/npm_packages.ts index 71eb348999..ea0852ed33 100644 --- a/docs/examples/getting-started/npm_packages.ts +++ b/docs/examples/getting-started/npm_packages.ts @@ -1,6 +1,7 @@ // demonstrates: importing built-in stdlib npm packages (project-config.md) // docs: docs/src/getting-started/project-config.md // platforms: macos, linux, windows +// requires: auto-optimize // run: false // These four imports are Perry's most-used built-in stdlib shims: diff --git a/docs/examples/stdlib/http/fastify_json.ts b/docs/examples/stdlib/http/fastify_json.ts index 7342617ba0..f9c6fc0378 100644 --- a/docs/examples/stdlib/http/fastify_json.ts +++ b/docs/examples/stdlib/http/fastify_json.ts @@ -11,6 +11,7 @@ // // docs: docs/src/stdlib/http.md // platforms: macos, linux, windows +// requires: auto-optimize // run: false import Fastify from "fastify" diff --git a/docs/examples/stdlib/overview/snippets.ts b/docs/examples/stdlib/overview/snippets.ts index 26632fb2c5..e0268e2fc1 100644 --- a/docs/examples/stdlib/overview/snippets.ts +++ b/docs/examples/stdlib/overview/snippets.ts @@ -2,6 +2,7 @@ // in docs/src/stdlib/overview.md // docs: docs/src/stdlib/overview.md // platforms: macos, linux, windows +// requires: auto-optimize // run: false // The overview page exists to show "these imports compile". So we just