diff --git a/docs/CLI_REFERENCE.md b/docs/CLI_REFERENCE.md index fa4d03e..3125419 100644 --- a/docs/CLI_REFERENCE.md +++ b/docs/CLI_REFERENCE.md @@ -200,7 +200,7 @@ field came back empty — one field matching is enough to count it as extracted. This is the closest pobear gets to reporting a soft block it doesn't recognize. A `200` challenge page from a known anti-bot vendor (Amazon's CAPTCHA, Cloudflare, -DataDome, PerimeterX) is classified as `[blocked]` like a `403`, so `--ladder` +DataDome, PerimeterX, Google Search) is classified as `[blocked]` like a `403`, so `--ladder` climbs past it; any other interstitial answers with a normal `200` and real HTML. What pobear can say is that the selectors you asked for found nothing, which is the same signal a challenge page, a layout change, or a wrong selector all produce. `empty 200` out diff --git a/docs/JOB_SPEC_REFERENCE.md b/docs/JOB_SPEC_REFERENCE.md index 72ecc9c..8a94651 100644 --- a/docs/JOB_SPEC_REFERENCE.md +++ b/docs/JOB_SPEC_REFERENCE.md @@ -170,7 +170,7 @@ reports which tier actually served each URL, whether or not `ladder` ever climbe On `local` and `proxy`, `blocked` covers `403`, `503`, and a `2xx` whose body is a recognized anti-bot page (Amazon's CAPTCHA, a Cloudflare challenge, DataDome, -PerimeterX). Markers are vendor-specific, so an unrecognized challenge page still +PerimeterX, Google Search's block page). Markers are vendor-specific, so an unrecognized challenge page still comes back as a successful fetch. **`profile` only spoofs the TLS ClientHello, nothing else:** it doesn't touch diff --git a/services/core/internal/fetch/tiers/http.go b/services/core/internal/fetch/tiers/http.go index 53cd3db..fd0d92a 100644 --- a/services/core/internal/fetch/tiers/http.go +++ b/services/core/internal/fetch/tiers/http.go @@ -95,6 +95,7 @@ var blockPageMarkers = [][]byte{ []byte("_cf_chl_opt"), // Cloudflare challenge []byte("captcha-delivery.com"), // DataDome []byte("px-captcha"), // PerimeterX + []byte("emsg=SG_REL"), // Google Search } func isBlockPage(body []byte) bool { diff --git a/services/core/internal/fetch/tiers/local_test.go b/services/core/internal/fetch/tiers/local_test.go index 077ad6c..3fc5e59 100644 --- a/services/core/internal/fetch/tiers/local_test.go +++ b/services/core/internal/fetch/tiers/local_test.go @@ -77,6 +77,15 @@ func TestLocalFetch(t *testing.T) { wantFault: fault.Blocked, wantErr: true, }, + { + name: "200-google-search-block-is-blocked", + handler: func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`click here`)) + }, + wantStatus: 200, + wantFault: fault.Blocked, + wantErr: true, + }, { name: "200-mentioning-captcha-is-ok", handler: func(w http.ResponseWriter, r *http.Request) {