diff --git a/.changeset/tidy-turnstile-forms.md b/.changeset/tidy-turnstile-forms.md new file mode 100644 index 00000000..8cf95e7e --- /dev/null +++ b/.changeset/tidy-turnstile-forms.md @@ -0,0 +1,5 @@ +--- +"@forkpoint/agent-lighthouse-core": minor +--- + +Stop reporting a Cloudflare bot wall when a readable page loads Turnstile for form protection. Keep managed challenge-page detection and allow content audits to assess the fetched page. diff --git a/docs/evidence/audits/access-crawl-control/no-bot-detection.md b/docs/evidence/audits/access-crawl-control/no-bot-detection.md index 1e1c2123..b234412f 100644 --- a/docs/evidence/audits/access-crawl-control/no-bot-detection.md +++ b/docs/evidence/audits/access-crawl-control/no-bot-detection.md @@ -82,6 +82,16 @@ a pause is the only way to get a verdict from a rate-limited origin. ## Implementation deviations +- 2026-10-06 — the shared WAF detector no longer treats the + `challenges.cloudflare.com` hostname as proof of an access wall. A live fetch + of a storefront returned HTTP 200 with the storefront HTML and a + Turnstile loader configured for login, registration and newsletter forms. + The detector still recognises challenge headers, refusal responses and + managed challenge-page loaders. The page-script warning remains separate; + this correction removes the false wall and restores content audit evidence. + Regression tests cover Turnstile script/preconnect markup, JavaScript + Detections, a managed challenge loader and the scan evidence path. + - 2026-08-28 — the audit declines when the scan holds no response it can attribute to this site. It read the scripts on the scanned pages, and `ctx.pages`/`ctx.rootFiles` carry whatever answered 200 — on a parked domain diff --git a/packages/core/src/scan-evidence.test.ts b/packages/core/src/scan-evidence.test.ts index 66fe0b6c..fb8bb1ce 100644 --- a/packages/core/src/scan-evidence.test.ts +++ b/packages/core/src/scan-evidence.test.ts @@ -9,6 +9,7 @@ import { import { mockPageContext, mockFetchResult } from "./__tests__/test-utils"; import type { FetchResult } from "./fetcher"; import type { PageContext } from "./check-context"; +import { detectWafProtection } from "./waf-detector"; /** A homepage fetch result, overridable field by field. */ function homepage(overrides: Partial = {}): FetchResult { @@ -411,6 +412,31 @@ describe("allEvidenceMet", () => { }); describe("scanReadTheSite", () => { + it("keeps a readable storefront with form Turnstile available to audits", () => { + const page = mockPageContext( + "https://www.shop.test/", + '' + + `
${"Browse clothing and accessories in our shop. ".repeat(20)}
` + + '
', + ); + const response = homepage({ + ...page.fetchResult, + url: "https://shop.test/", + finalUrl: page.url, + headers: { server: "cloudflare", "cf-ray": "abc" }, + }); + const evidence = build({ + requestedUrl: response.url, + homepageResult: response, + pages: [page], + wafProtection: detectWafProtection(response.url, response, {}, 1), + }); + + expect(evidence.met["unblocked-fetches"]).toBe(true); + expect(evidence.met["rendered-body"]).toBe(true); + expect(scanReadTheSite(evidence)).toBe(true); + }); + // The guard 36 audits consult. It used to read `origin-reachable` alone, // which a bot wall satisfies whenever the wall answers 200. it("is false when the origin answered but the scan was refused", () => { diff --git a/packages/core/src/waf-detector.test.ts b/packages/core/src/waf-detector.test.ts index 481f5295..535f8bcd 100644 --- a/packages/core/src/waf-detector.test.ts +++ b/packages/core/src/waf-detector.test.ts @@ -80,6 +80,35 @@ describe("detectWafProtection", () => { ).toBeNull(); }); + it.each([ + '', + '', + '', + ])( + "does not flag a readable storefront with Turnstile markup: %s", + (markup) => { + expect( + onHomepage({ + headers: { server: "cloudflare", "cf-ray": "abc" }, + body: `Example shop${markup}

Clothing

Browse our collection.

`, + }), + ).toBeNull(); + }, + ); + + it("detects a managed challenge script on a 200 without a challenge header", () => { + expect( + onHomepage({ + headers: { server: "cloudflare" }, + body: '', + }), + ).toMatchObject({ + isBlocked: true, + provider: "cloudflare", + statusCode: 200, + }); + }); + it("names the status code in the reason for a 403", () => { expect( onHomepage({ headers: { "cf-ray": "abc" }, status: 403 })?.reason, diff --git a/packages/core/src/waf-detector.ts b/packages/core/src/waf-detector.ts index 54571e46..5e5fb8bc 100644 --- a/packages/core/src/waf-detector.ts +++ b/packages/core/src/waf-detector.ts @@ -81,7 +81,9 @@ export function detectWafProtection( (body.includes("just a moment...") || body.includes("checking your browser") || body.includes("cf-challenge") || - body.includes("challenges.cloudflare.com"))) + // Turnstile's shared host also serves form widgets on readable pages. + // Only the managed challenge-page loader is evidence of a page wall. + body.includes("/orchestrate/chl_page/"))) ) { return { isBlocked: true,