From 7a0ad6c3a088f4d56d144bafae5c153590757705 Mon Sep 17 00:00:00 2001 From: Petr Date: Wed, 30 Sep 2026 19:26:43 +0000 Subject: [PATCH] http_scraper.py: the three modes with no browser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The data endpoints are ungated, so the browser in the three engines is insurance for the day AWS WAF moves in front of them, not a requirement today. This engine drives the same page_flow loop over requests: same flags minus the ones that describe a browser, same output, same exit codes, requests' own User-Agent (§24). Measured 2026-09-30, one 3-page run per mode: identical rows to Playwright in the same order (one copy-trading AUM moved between the runs), P2P 3.0 s vs 5.2 s, copy-trading 3.4 s vs 8.8 s, announcements 2.7 s vs 9.1 s. The shared loop no longer waits for a WAF challenge script when the engine cannot run one (`runs_scripts`): a control showed the wait re-read the empty 202 body without its status and turned a plain block (exit 3) into "never arrived" (exit 5). The suite adds the engine to the operations, binding, masking, import and banned-flag checks, pins its flags as the browser set minus a named list, and drives it through the loop with requests stubbed. The canary runs P2P through it daily with no browser installed. Co-Authored-By: Claude Opus 5.5 --- .github/ci_checks.py | 2 +- .github/workflows/canary.yml | 23 ++- CHANGELOG.md | 14 ++ README.md | 24 ++- http_scraper.py | 339 +++++++++++++++++++++++++++++++++++ page_flow.py | 6 +- pyproject.toml | 1 + smoke_test.py | 153 +++++++++++++++- 8 files changed, 546 insertions(+), 16 deletions(-) create mode 100644 http_scraper.py diff --git a/.github/ci_checks.py b/.github/ci_checks.py index 6c36406..4a74574 100644 --- a/.github/ci_checks.py +++ b/.github/ci_checks.py @@ -34,7 +34,7 @@ ENGINE_LIBS = ("playwright", "pyppeteer", "selenium", "webdriver_manager") CLIS = ["playwright_scraper.py", "puppeteer_scraper.py", "selenium_scraper.py", - "scraper_api_client.py", "fingerprint_client.py", "env_config.py", + "http_scraper.py", "scraper_api_client.py", "fingerprint_client.py", "env_config.py", "diff_runs.py"] # One real run per mode: three row classes, three samples. diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 4265692..a9e8cfa 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -33,7 +33,7 @@ permissions: jobs: run: - name: ${{ matrix.mode }} + name: ${{ matrix.mode }} (${{ matrix.engine }}) runs-on: ubuntu-latest timeout-minutes: 20 strategy: @@ -43,14 +43,24 @@ jobs: # USDT/EUR had ~190-200 adverts on 2026-09-24, so three pages of 20 # are full; the floor leaves room for a thin morning. - mode: p2p + engine: playwright args: "--asset USDT --fiat EUR --side buy" floor: 40 - mode: copytrading + engine: playwright args: "--mode copytrading --time-range 30D --sort-by roi" floor: 60 - mode: announcements + engine: playwright args: "--mode announcements --category new-listings" floor: 100 + # The browserless engine, so the README's "no browser needed" is + # under test every morning too. One mode: the loop is shared, and + # P2P is the POST that needs the most of it. + - mode: p2p + engine: http + args: "--asset USDT --fiat EUR --side buy" + floor: 40 env: BINANCE_PROXY: ${{ secrets.BINANCE_PROXY }} steps: @@ -62,8 +72,11 @@ jobs: - name: Install run: | python -m pip install --upgrade pip - pip install -r requirements.txt -r requirements-playwright.txt - playwright install --with-deps chromium + pip install -r requirements.txt + if [ "${{ matrix.engine }}" = "playwright" ]; then + pip install -r requirements-playwright.txt + playwright install --with-deps chromium + fi # >= 3 pages, because with one page pagination is never exercised (§11). - name: Run @@ -71,7 +84,7 @@ jobs: run: | set +e # `run:` is `bash -e`; without this a non-zero exit aborts # the step before the echo below records it. - python playwright_scraper.py ${{ matrix.args }} --pages 3 \ + python ${{ matrix.engine }}_scraper.py ${{ matrix.args }} --pages 3 \ --out canary --format both --dump-html canary_dump.json echo "exit_code=$?" >> "$GITHUB_OUTPUT" @@ -137,7 +150,7 @@ jobs: - uses: actions/upload-artifact@v4 if: always() with: - name: canary-${{ matrix.mode }} + name: canary-${{ matrix.mode }}-${{ matrix.engine }} path: | canary.json canary.csv diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bde354..895a664 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ toolkit can: a patch release means **fixes**, not that every flag and default is frozen. A default that changes behaviour for an existing user is said so at the top of its release notes. +## [Unreleased] + +### Added + +- `http_scraper.py`: the three modes with no browser, through the same + fetch loop, output and exit codes. Same flags as the browser engines + minus the ones that describe a browser. It sends requests' own + User-Agent rather than a Chrome one over a non-Chrome TLS handshake. + Measured 2026-09-30: identical rows to Playwright in all three modes, in + about a third to a half of the time. If AWS WAF answers instead of an + endpoint it reports exit 3 straight away, without waiting for a + challenge script that nothing here can run. +- The canary runs P2P through it daily, with no browser installed. + ## [0.1.1] — 2026-09-30 > **A moved payload shape is no longer reported as an empty listing.** A diff --git a/README.md b/README.md index 30a55d8..4bf9d7e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![canary](https://github.com/2scraper/binance-scraper/actions/workflows/canary.yml/badge.svg)](https://github.com/2scraper/binance-scraper/actions/workflows/canary.yml) ![python](https://img.shields.io/badge/python-3.9%20%E2%80%93%203.14-blue) [![licence](https://img.shields.io/badge/licence-MIT-green)](LICENSE) -![engines](https://img.shields.io/badge/engines-playwright%20%7C%20selenium%20%7C%20pyppeteer-lightgrey) +![engines](https://img.shields.io/badge/engines-playwright%20%7C%20selenium%20%7C%20pyppeteer%20%7C%20http-lightgrey) ![runs without an account](https://img.shields.io/badge/all%20three%20modes-no%20account%20needed-brightgreen) Scrapes three things from [binance.com](https://www.binance.com) that its @@ -43,6 +43,15 @@ one of those endpoints and issues every page as a same-origin `fetch()`, so no page is ever rendered. A full USDT/EUR buy-side order book, 10 pages, came back as **196 of 196 adverts** in 13 seconds with nothing configured. +Which also means the data needs **no browser at all**. `http_scraper.py` +sends the same requests with `requests` and nothing else, through the same +loop, into the same files. Measured 2026-09-30, one 3-page run per mode +from the same VPS: P2P 3.0 s against Playwright's 5.2 s, copy-trading 3.4 s +against 8.8 s, announcements 2.7 s against 9.1 s, with the same rows in the +same order. What the browser buys is the day the WAF moves in front of the +endpoints: then only a browser engine can run its challenge or have its +CAPTCHA solved, and the HTTP engine says so and exits 3. + What the paid products buy here is insurance and scale, and the section below says exactly which one does what. @@ -59,6 +68,14 @@ python3 -m venv venv Install **one** engine per virtualenv: the three libraries pin versions of their dependencies that cannot all be satisfied at once. +Or no browser at all, for the HTTP engine: + +```bash +python3 -m venv venv +./venv/bin/pip install -r requirements.txt +./venv/bin/python http_scraper.py --asset USDT --fiat EUR --pages 3 +``` + ## Run ```bash @@ -172,11 +189,12 @@ retried, and the exit code is 5 ("the data never arrived"), not 3. | `playwright_scraper.py` | **Primary.** Authenticates a proxy and a remote CDP endpoint. | | `puppeteer_scraper.py` | pyppeteer is effectively unmaintained; here for parity. Authenticates a proxy and a CDP endpoint. `--chromium-path` points it at another browser if its own will not start. | | `selenium_scraper.py` | Drives the Chrome you already have. **Cannot authenticate a proxy or a remote CDP endpoint** (`debuggerAddress` is a bare `host:port`), so it refuses a credentialled `--cdp-endpoint` with exit 2. None of the modes needs either. | +| `http_scraper.py` | **No browser**: `requests` only, the same flags minus the ones that describe a browser (`--headless`, `--cdp-endpoint`, `--fingerprint*`, `--locale`, the solver's). Proxies and rotation work. Cannot pass AWS WAF: if it answers, the run is exit 3 at once, naming the engine that can. | | `scraper_api_client.py` | No local browser: the 2Captcha Scraper API fetches the page. Reads `--mode announcements` only, the one endpoint with a URL. The P2P and copy-trading endpoints answer POST, and **this repo does not implement a POST through the Scraper API**. Measured: 2 pages, 100 rows, $0.0005 a page. | The fetch loop itself (landing, the WAF, retries, throttling, rotation, -parsing) is one implementation in `page_flow.py` that all three browser -engines drive, so they cannot disagree about a page. Each was run live on +parsing) is one implementation in `page_flow.py` that all four engines +drive, so they cannot disagree about a page. Each was run live on 2026-09-24 through the same seven scenarios (P2P buy and sell, copy-trading, announcements, `--concurrency`, a refused `--pay-type`, `--url`) with the same results. diff --git a/http_scraper.py b/http_scraper.py new file mode 100644 index 0000000..e7f2531 --- /dev/null +++ b/http_scraper.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +""" +binance-scraper — HTTP edition (no browser) +=========================================== + +The same three modes, the same query flags, the same output, sidecar and +exit codes as playwright_scraper.py, with no browser at all: each page is +one `requests` call to the endpoint the site's own front end calls. + +Why it exists +------------- +The browser engines never render a page either. They land on an ungated +JSON endpoint and issue every request as a same-origin fetch(), and the +browser is there for one reason: if AWS WAF ever moves in front of the +endpoints, its challenge can run and its CAPTCHA can be solved. Until then +it costs a Chromium download, a launch per run and per rotation, and the +memory of a browser for what is a few JSON requests. + +Measured 2026-09-30 from a datacentre VPS, this engine's own client: +all three endpoints answered HTTP 200 with complete JSON, on every page +their totals implied (P2P 12 of 12, copy-trading 304, announcements 46). + +What it gives up, and says so rather than failing quietly +--------------------------------------------------------- +* **No WAF.** If a response is AWS WAF's challenge or CAPTCHA, nothing here + can run the one or solve the other. The page reports blocked (exit 3) + with that reason, straight away rather than after the browser engines' + 15-second wait for a script to run, and names the engine that can. +* **No --cdp-endpoint, --fingerprint, --locale or --headless**: they + describe a browser. `--proxy`, `--proxy-file` and rotation work: a + rotation is a fresh session, so no cookie crosses exits (§8). +* **Its own identity.** It sends requests' own User-Agent rather than + claiming to be Chrome over a TLS handshake that is not Chrome's, which is + the contradiction a sibling site refused (§24). Measured here: P2P + answered both identities 6 of 6. + +Usage +----- + python http_scraper.py --asset USDT --fiat EUR --pages 3 + python http_scraper.py --mode copytrading --pages 5 --concurrency 3 + python http_scraper.py --mode announcements --category delisting + +Requires: pip install -r requirements.txt (nothing else) +""" + +import argparse +import logging +import queue +import re +import sys +import threading +import time + +import requests + +from product_parser import (ANN_CATALOGS, COPY_SORTS, COPY_TIME_RANGES, + DEFAULT_ANN_CATALOG, DEFAULT_COPY_SORT, P2P_SIDES) +import page_flow +from proxy_pool import (from_args as proxy_pool_from_args, mask, ROTATE_MODES, + ProxyError) +import env_config + +logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s") +logger = logging.getLogger("http_scraper") + +CONNECT_TIMEOUT_S = 15.0 + +# The words requests and urllib3 use for a proxy that could not be used. +# Distinguished from a timeout because the two want opposite responses: a +# timeout deserves another try at the same exit, a dead proxy a different +# one (§8). +_PROXY_ERROR_MARKERS = ( + "ProxyError", + "Unable to connect to proxy", + "Tunnel connection failed", + "407 Proxy Authentication Required", +) + +# Every `scheme://user:pass@` in a string, however many times it occurs. +_CREDENTIALS_IN_URL_RE = re.compile(r"([a-z][a-z0-9+.\-]*://)[^\s/@]+:[^\s/@]+@", + re.IGNORECASE) + + +def _mask_credentials(text: str) -> str: + """`text` with any username:password in an embedded URL replaced.""" + return _CREDENTIALS_IN_URL_RE.sub(r"\1***:***@", text or "") + + +def _proxy_failure(text: str) -> str: + """The proxy-error marker in `text`, or "" if it is not one.""" + for marker in _PROXY_ERROR_MARKERS: + if marker in (text or ""): + return marker + return "" + + +class _Ops: + """One requests.Session, exposed as page_flow's named operations. + + `runs_scripts` is False, which tells the shared loop not to wait for an + AWS WAF challenge to clear itself: that is a script, and there is no + browser here to run it. + """ + + runs_scripts = False + + def __init__(self, args, pool): + self.args, self.pool = args, pool + self.session = None + self.landed = False + self._last_text = "" + self._warned_waf = False + + def open(self): + self.session = requests.Session() + self.session.headers["Accept"] = "application/json, text/plain, */*" + if self.pool: + exit_url = self.pool.current + self.session.proxies = {"http": exit_url, "https": exit_url} + logger.info("Using proxy exit %s", mask(exit_url)) + self.landed = False + return self + + def _request(self, method: str, url: str, body=None): + headers = {"Content-Type": "application/json"} if body is not None else {} + return self.session.request( + method, url, data=body, headers=headers, + timeout=(CONNECT_TIMEOUT_S, page_flow.FETCH_TIMEOUT_MS / 1000.0)) + + # ---- page_flow's operations ----------------------------------------- + + def goto(self, url: str): + try: + resp = self._request("GET", url) + except requests.RequestException as e: + raise page_flow.TransportError(_mask_credentials(str(e))) from None + self._last_text = resp.text + return resp.status_code, resp.headers.get("x-amzn-waf-action") + + def document_text(self) -> str: + return self._last_text + + def wait_ms(self, ms: int) -> None: + time.sleep(ms / 1000.0) + + def solve_captcha(self) -> bool: + if not self._warned_waf: + logger.error("AWS WAF answered instead of the endpoint. This engine " + "has no browser, so its challenge cannot run and its " + "CAPTCHA cannot be solved here. playwright_scraper.py " + "can do both, with TWOCAPTCHA_KEY for the CAPTCHA.") + self._warned_waf = True + return False + + def fetch(self, req, timeout_ms: int = page_flow.FETCH_TIMEOUT_MS): + try: + resp = self._request(req.method, req.url, req.body_json) + except requests.RequestException as e: + return None, "", None, _mask_credentials("%s: %s" % (type(e).__name__, e)) + return resp.status_code, resp.text, resp.headers.get("x-amzn-waf-action"), None + + def proxy_failure(self, text: str) -> str: + return _proxy_failure(text) + + def relaunch(self): + """A fresh session on the pool's current exit, so no cookie issued + against one exit is replayed from another (§8).""" + self.close() + self.open() + + def close(self): + if self.session is not None: + self.session.close() + self.session = None + + +def _fetch_pages_concurrently(args, pool, query, page_nums, concurrency: int): + """Fetch `page_nums` across `concurrency` workers, each with its own + session and its own exit. The page loop is page_flow.worker_loop, the + same one the browser engines use.""" + work = queue.Queue() + for n in page_nums: + work.put(n) + results, results_lock = [], threading.Lock() + exhausted = threading.Event() + + def worker(index: int): + name = f"worker-{index + 1}" + ops = None + try: + ops = _Ops(args, page_flow.worker_pool(pool, index)).open() + page_flow.worker_loop(ops, args, query, work, results, results_lock, + exhausted, name, _mask_credentials) + except Exception: # noqa: BLE001 — a dead worker must not hang the run + logger.exception("[%s] died; its pages will be reported as failed.", name) + finally: + if ops is not None: + ops.close() + + threads = [threading.Thread(target=worker, args=(i,), name=f"page-worker-{i + 1}") + for i in range(concurrency)] + for t in threads: + t.start() + for t in threads: + t.join() + + unattempted = [] + while True: + try: + unattempted.append(work.get_nowait()) + except queue.Empty: + break + return results, sorted(unattempted), exhausted.is_set() + + +def scrape(args) -> int: + pool = proxy_pool_from_args(args) + concurrency = page_flow.concurrency_for(args, pool) + return page_flow.run_pages( + lambda: _Ops(args, pool).open(), + lambda ops: ops.close(), + lambda pages: _fetch_pages_concurrently(args, pool, args.query, pages, + concurrency), + args, pool, args.query, concurrency, _mask_credentials) + + +def parse_args(argv=None): + p = argparse.ArgumentParser( + description="binance.com scraper — P2P adverts, copy-trading lead " + "portfolios and announcements (HTTP edition: no browser)") + p.add_argument("--mode", choices=["p2p", "copytrading", "announcements"], + default=None, + help="p2p (default): the P2P order book for --asset/--fiat. " + "copytrading: Futures copy-trading lead portfolios. " + "announcements: one announcement catalogue, newest " + "first. Inferred from --url when that is given.") + p.add_argument("--url", default=None, + help="A binance.com page to read the query from instead of " + "the flags: a P2P trade page " + "(p2p.binance.com/en/trade/all-payments/USDT?fiat=EUR, " + "…/trade/sell/BTC?fiat=TRY), /en/copy-trading, or " + "/en/support/announcement/list/{id}. The page itself " + "is never fetched — it is behind AWS WAF — only the " + "query in it is read. Also read from BINANCE_URL.") + g = p.add_argument_group("p2p") + g.add_argument("--asset", default=None, + help="The crypto asset (default USDT).") + g.add_argument("--fiat", default=None, + help="The fiat currency, ISO 4217 (default USD).") + g.add_argument("--side", choices=P2P_SIDES, default=None, + help="buy (default): adverts you could BUY the asset from. " + "Those adverts are marked SELL by the site, because " + "an advert carries the maker's side; the row keeps " + "both as `side` and `advertiser_side`.") + g.add_argument("--pay-type", action="append", default=None, metavar="ID", + help="Only adverts taking this payment method, by the " + "site's identifier (SEPAinstant, Wise, BANK, …). " + "Repeatable. Checked against the site's own list for " + "the fiat before the search runs: the search answers " + "an unknown one with an EMPTY result, not an error.") + g.add_argument("--amount", type=float, default=None, + help="Only adverts whose limits admit an order of this much " + "fiat.") + g = p.add_argument_group("copytrading") + g.add_argument("--time-range", choices=COPY_TIME_RANGES, default=None, + help="The period ROI, PnL and drawdown cover (default 30D).") + g.add_argument("--sort-by", choices=sorted(COPY_SORTS), default=None, + help="Ordering (default %s). Not cosmetic: a capped run " + "holds the first N portfolios by this key, so it " + "decides WHICH portfolios are in the file. `sharpe` " + "also filters: portfolios without a Sharpe ratio are " + "left out. Win rate is not offered: the API gave a " + "nonsense key the same answer." % DEFAULT_COPY_SORT) + g.add_argument("--order", choices=["desc", "asc"], default=None, + help="desc (default) or asc.") + g.add_argument("--hide-full", action="store_true", + help="Leave out portfolios with no copier seat free.") + g = p.add_argument_group("announcements") + g.add_argument("--category", default=None, + help="The announcement catalogue: %s, or a numeric " + "catalogue id (default %s)." + % (", ".join(ANN_CATALOGS), DEFAULT_ANN_CATALOG)) + p.add_argument("--pages", type=int, default=1, + help="Pages to fetch (20 adverts, 30 portfolios or 50 " + "announcements each). Planned against the total the " + "site states on page 1, so asking for more than exist " + "fetches all of them.") + p.add_argument("--delay", type=float, default=1.0, + help="Delay between pages, seconds (default %(default)s)") + p.add_argument("--concurrency", type=int, default=1, metavar="N", + help="Fetch pages through N parallel workers (default 1). " + "Each worker holds its own session and proxy exit.") + p.add_argument("--retries", type=int, default=3, + help="Attempts per page on a transport failure (default 3). " + "The pause doubles each time. A request the endpoint " + "REFUSED is not retried: its parameters would be " + "refused again.") + p.add_argument("--retry-delay", type=float, default=2.0, + help="Seconds before the first retry, doubling thereafter.") + p.add_argument("--format", choices=["json", "csv", "both"], default="both") + p.add_argument("--out", default="binance_rows", help="Output file prefix") + p.add_argument("--proxy", default=None, + help="Proxy URL, e.g. http://ACCOUNT:PASSWORD@HOST:9999 " + "(2captcha.com/proxy)") + p.add_argument("--proxy-file", default=None, + help="File with one proxy URL per line to rotate across. " + "Wins over --proxy.") + p.add_argument("--proxy-rotate", choices=list(ROTATE_MODES), default="per-run", + help="per-run (default): one exit for the whole run. " + "per-page: a new exit, and a fresh session, per page.") + p.add_argument("--proxy-shuffle", action="store_true", + help="Shuffle the pool at startup.") + p.add_argument("--proxy-block-retries", type=int, default=2, + help="When a page is refused (403, 451, AWS WAF), retry it " + "from this many OTHER exits (default 2). Needs a pool " + "of more than one.") + p.add_argument("--allow-empty", action="store_true", + help="Write output files even when 0 rows were found.") + p.add_argument("--dump-html", default=None, metavar="PATH", + help="Save the exact response the parser is given, on " + "success as well as failure. It is JSON; the flag " + "keeps the family's name.") + args = p.parse_args(argv) + env_config.apply(args) + # Read by the shared loop, which serves the browser engines too. There is + # no remote browser here. + args.cdp_endpoint = None + args.query = page_flow.build_query(args, p.error) + args.mode = args.query.mode + return args + + +if __name__ == "__main__": + args = parse_args() + try: + sys.exit(scrape(args)) + except ProxyError as e: + logger.error("%s", e) + sys.exit(2) diff --git a/page_flow.py b/page_flow.py index e1ebdef..3d2f474 100644 --- a/page_flow.py +++ b/page_flow.py @@ -457,8 +457,10 @@ def land(ops, args) -> Tuple[str, Optional[str]]: state = classify(ops.document_text(), status, ORIGIN_URL, waf) if state == "challenge": # The WAF's challenge action computes a token and reloads the page - # by itself, so let it run before paying for anything. - waited = 0 + # by itself, so let it run before paying for anything. Only a + # browser runs it: an engine without one (`runs_scripts` False) + # would spend the wait re-reading a document that cannot change. + waited = 0 if getattr(ops, "runs_scripts", True) else CHALLENGE_SETTLE_MS while waited < CHALLENGE_SETTLE_MS and state == "challenge": ops.wait_ms(CHALLENGE_POLL_MS) waited += CHALLENGE_POLL_MS diff --git a/pyproject.toml b/pyproject.toml index 1c41b45..549e71e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ py-modules = [ "diff_runs", "env_config", "fingerprint_client", + "http_scraper", "output_writer", "page_flow", "playwright_scraper", diff --git a/smoke_test.py b/smoke_test.py index 8df9a8c..24aff6d 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -77,9 +77,15 @@ def fx(name) -> str: ENGINES = ("playwright_scraper", "selenium_scraper", "puppeteer_scraper") +# The browserless engine. Kept out of ENGINES, whose checks are about a +# browser (its flags, its fetch() JavaScript), and added by name to every +# check that is about the shared loop, the log or the imports. +HTTP_ENGINE = "http_scraper" +ALL_ENGINES = ENGINES + (HTTP_ENGINE,) DRIVER_IMPORTS = {"playwright_scraper": "playwright", "selenium_scraper": "selenium", - "puppeteer_scraper": "pyppeteer"} + "puppeteer_scraper": "pyppeteer", + "http_scraper": "requests"} def _import_engine(name): @@ -644,6 +650,119 @@ def check_a_multi_page_run_merges_in_page_order_and_ends_on_data(): equal("...and the run is still complete", (rc, meta["status"]), (0, "complete")) +class _FakeResponse: + def __init__(self, status, text, waf=None): + self.status_code, self.text = status, text + self.headers = {"x-amzn-waf-action": waf} if waf else {} + + +def _http_run(answer, pages=1, query=None, **extra): + """Drive http_scraper through the shared loop with requests stubbed: + `answer(method, url, body)` returns a _FakeResponse or raises.""" + import http_scraper + import page_flow + import requests + from product_parser import Query + calls, waits = [], [] + real_request, real_sleep = requests.Session.request, http_scraper.time.sleep + + def fake_request(self, method, url, data=None, headers=None, timeout=None, **kw): + calls.append((method, url)) + check("every request is bounded by a timeout", timeout is not None) + return answer(method, url, data) + + requests.Session.request = fake_request + http_scraper.time.sleep = lambda s: waits.append(s) + try: + with tempfile.TemporaryDirectory() as tmp: + args = types.SimpleNamespace( + pages=pages, retries=2, retry_delay=0, delay=0, + proxy_block_retries=2, out=os.path.join(tmp, "out"), format="json", + allow_empty=False, dump_html=None, url=None, cdp_endpoint=None, + concurrency=1) + for k, v in extra.items(): + setattr(args, k, v) + q = query or Query("p2p", fiat="EUR") + rc = page_flow.run_pages( + lambda: http_scraper._Ops(args, None).open(), lambda o: o.close(), + lambda nums: http_scraper._fetch_pages_concurrently(args, None, q, nums, 2), + args, None, q, 1, http_scraper._mask_credentials) + meta_path = args.out + ".meta.json" + meta = json.load(open(meta_path)) if os.path.exists(meta_path) else None + finally: + requests.Session.request = real_request + http_scraper.time.sleep = real_sleep + return rc, meta, calls, waits + + +def check_the_http_engine_drives_the_shared_loop(): + """The HTTP engine is page_flow's loop over requests, so the same answers + must give the same exit codes the browser engines give. requests is the + one library the offline suite always has, so this never skips.""" + if _import_engine(HTTP_ENGINE) is None: + return + from product_parser import ORIGIN_URL, Query + + def served(method, url, body): + if url == ORIGIN_URL: + return _FakeResponse(200, fx("ann_new_listings")) + if "copy-trade" in url: + return _FakeResponse(200, fx("copy_30d_roi")) + return _FakeResponse(200, _p2p_page(1)) + + rc, meta, calls, waits = _http_run(served) + equal("http: a served listing is exit 0, complete", (rc, meta["status"]), + (0, "complete")) + equal("...landing on the GET endpoint, then POSTing the search", + [m for m, _ in calls], ["GET", "POST"]) + + def challenged(method, url, body): + return _FakeResponse(202, "", "challenge") + + rc, meta, calls, waits = _http_run(challenged) + equal("http: an AWS WAF challenge is exit 3, blocked", rc, 3) + # Any pause at all is the settle poll: --delay and --retry-delay are 0 + # here. Measured by a control: with the wait, the first poll re-read the + # empty 202 body without its status, called it `unknown`, and the run + # became exit 5 "never arrived" for a page that was plainly blocked. + check("...without the browser engines' wait for a script to run " + "(nothing here runs it)", not any(w > 0 for w in waits), + "waited %r" % waits) + + def dead_proxy(method, url, body): + import requests + raise requests.exceptions.ProxyError( + "Unable to connect to proxy http://u:supersecret@exit.example:2334") + + rc, meta, calls, waits = _http_run(dead_proxy) + equal("http: a dead proxy is exit 5, the data never arrived", rc, 5) + import http_scraper + check("...recognised as a PROXY failure, not a timeout", + http_scraper._proxy_failure("ProxyError: Unable to connect to proxy") != "") + ops = http_scraper._Ops(types.SimpleNamespace(), None).open() + try: + import requests + real = requests.Session.request + requests.Session.request = lambda *a, **k: dead_proxy(None, None, None) + status, text, waf, error = ops.fetch( + __import__("product_parser").request_for(Query("p2p", fiat="EUR"), 1)) + finally: + requests.Session.request = real + ops.close() + check("...and its error text carries no password", "supersecret" not in (error or ""), + error) + check("...but keeps the exit", "exit.example:2334" in (error or ""), error) + + def ann_pages(method, url, body): + if url == ORIGIN_URL: + return _FakeResponse(200, fx("ann_new_listings")) + return _FakeResponse(200, fx("ann_new_listings")) + + rc, meta, calls, waits = _http_run(ann_pages, pages=1, + query=Query("announcements")) + equal("http: announcements, a GET listing, exit 0", rc, 0) + + def check_a_moved_payload_shape_is_not_an_empty_listing(): """The endpoints are undocumented, so the shape WILL move one day. Both ways it can move were silent before: a renamed container parsed to zero @@ -708,7 +827,7 @@ def check_every_engine_implements_the_operations_page_flow_uses(): check("...and the fake driver this suite uses implements every one", all(hasattr(_FakeOps({}), name) for name in used), repr(sorted(n for n in used if not hasattr(_FakeOps({}), n)))) - for module in ENGINES: + for module in ALL_ENGINES: path = os.path.join(HERE, module + ".py") tree = ast.parse(open(path, encoding="utf-8").read()) ops_cls = next((n for n in tree.body @@ -876,7 +995,7 @@ def check_shared_calls_bind_against_the_real_signature(): "output_writer": output_writer, "captcha_solver": captcha_solver, "proxy_pool": proxy_pool} bound = 0 - for module in ENGINES + ("scraper_api_client", "diff_runs", "page_flow"): + for module in ALL_ENGINES + ("scraper_api_client", "diff_runs", "page_flow"): tree = ast.parse(open(os.path.join(HERE, module + ".py"), encoding="utf-8").read()) local_names = {n.arg for n in ast.walk(tree) if isinstance(n, ast.arg)} direct = {} @@ -974,10 +1093,34 @@ def check_engine_flag_sets(): "--chromium-path" in sets["puppeteer_scraper"]) +# The flags that describe a browser, and so are absent from the HTTP engine. +# The list IS the documentation (§17): the check fails if the HTTP engine +# gains one, loses a shared one, or if one of these leaves the browser +# engines while still listed here. +BROWSER_ONLY_FLAGS = {"--headless", "--headful", "--cdp-endpoint", "--locale", + "--fingerprint", "--fp-tags", "--fp-country", + "--twocaptcha-key", "--captcha-api", "--solve-captcha", + "--min-score"} + + +def check_http_engine_flags_are_the_browser_ones_minus_the_browser(): + pw = _argparse_flags("playwright_scraper") + http = _argparse_flags(HTTP_ENGINE) + check("every browser-only flag is still a browser engine's flag", + BROWSER_ONLY_FLAGS <= pw, "gone from playwright: %s" + % sorted(BROWSER_ONLY_FLAGS - pw)) + check("the HTTP engine defines exactly the rest", + http == pw - BROWSER_ONLY_FLAGS, + "only in http: %s; missing from http: %s" + % (sorted(http - (pw - BROWSER_ONLY_FLAGS)), + sorted((pw - BROWSER_ONLY_FLAGS) - http))) + check("...which includes every query flag", BINANCE_FLAGS <= http) + + def check_banned_and_removed_flags(): """Scoped to the engines. `--country` is banned: it could disagree with the --url, and a P2P query's country is its --fiat.""" - for module in ENGINES: + for module in ALL_ENGINES: source = open(os.path.join(HERE, module + ".py"), encoding="utf-8").read() for flag in ("--antidetect", "--country", "--country-code"): check("%s does not define %s" % (module, flag), '"%s"' % flag not in source) @@ -1309,7 +1452,7 @@ def check_fetch_js_is_one_request_in_three_dialects(): def check_credentials_never_reach_a_log(): - for module in ENGINES: + for module in ALL_ENGINES: engine = _import_engine(module) if engine is None: continue