From f2996219c623e9a204a2983c82f7152ee89bdef6 Mon Sep 17 00:00:00 2001 From: Petr Date: Wed, 30 Sep 2026 19:07:50 +0000 Subject: [PATCH] A moved payload shape is not an empty listing; CI tests 3.14 The three endpoints are the front end's, not a published API, so a field will be renamed one day. Both ways that can happen were silent: - a renamed row container parsed to zero rows and ended the run as exit 4, "the listing has nothing in it", on a response still counting 187 adverts. A page whose OWN total puts rows on it and from which none were read is now `unread`: exit 5 on page 1, exit 6 later, stop_reason parser_found_nothing. Measured 2026-09-30 that every page up to the one the total implies carries rows on all three endpoints, and the next carries none, so the end of a listing is still end_of_listing. - a renamed column wrote a complete file with the column null on every row and one warning. The sidecar now carries core_field_shortfall per page, and the canary fails on it. One test simulated "the listing shrank" with a past-the-end capture whose total still covered the page, which the live endpoints never answer; it now states the shrunk total. CI's newest Python is 3.14 (was 3.12); 3.9 stays as the floor. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/canary.yml | 8 +++- .github/workflows/tests.yml | 5 ++- CHANGELOG.md | 23 ++++++++++++ README.md | 28 ++++++++++++-- page_flow.py | 62 ++++++++++++++++++++++++++++--- smoke_test.py | 72 +++++++++++++++++++++++++++++++++++- 6 files changed, 185 insertions(+), 13 deletions(-) diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index 21f0c84..4265692 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -83,7 +83,8 @@ jobs: 0) ;; 3) echo "::error::Blocked. If the log names aws-waf, the WAF has moved in front of the data endpoints, which is the README's central claim failing. If it names http-403 or geo-451, the runner's address was refused: set the BINANCE_PROXY secret (see the header)."; exit 1 ;; 4) echo "::error::Zero rows from a listing that is never empty. The endpoint's shape has moved; see canary_dump.json."; exit 1 ;; - 5) echo "::error::The data never arrived, or the endpoint REFUSED the parameters (the log names the site's own complaint)."; exit 1 ;; + 5) echo "::error::The data never arrived, the endpoint REFUSED the parameters (the log names the site's own complaint), or page 1 was served and none of its rows could be read (the log says 'none were read': the endpoint's shape has moved)."; exit 1 ;; + 6) echo "::error::Partial. If the sidecar's stop_reason is parser_found_nothing, the endpoint's shape moved part-way through the run; see canary_dump.json."; exit 1 ;; *) echo "::error::Unexpected exit $code."; exit 1 ;; esac @@ -103,6 +104,11 @@ jobs: problems.append("%d rows, floor %d" % (len(rows), floor)) if not meta.get("total_results"): problems.append("no total_results in the sidecar") + # A renamed field writes its rows with the column null and exits 0, + # so the exit code cannot catch it; the sidecar says it. + if meta.get("core_field_shortfall"): + problems.append("the payload's shape has moved: core columns " + "missing %r" % meta["core_field_shortfall"]) if {r["mode"] for r in rows} != {mode}: problems.append("rows name modes %r" % {r["mode"] for r in rows}) pairs = [(r["page"], r["position"]) for r in rows] diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 24894d6..ed2fb5d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,10 @@ jobs: matrix: # 3.9 is the floor the README claims. Claiming it without testing it is # how a walrus operator or a `X | None` annotation ships and breaks it. - python-version: ["3.9", "3.12"] + # It stays although upstream support ended in October 2025: it is the + # Python macOS still ships, which is the one a reader of the README + # meets first. The other end is the newest release, not a fixed one. + python-version: ["3.9", "3.14"] steps: - uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 77be5ed..5f6d606 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,29 @@ toolkit can: a patch release means **fixes**, not that every flag and default is frozen. A default that changes behaviour for an existing user is said so at the top of its release notes. +## [Unreleased] + +> **A moved payload shape is no longer reported as an empty listing.** A +> page the endpoint served, whose own total says it holds rows, and from +> which no row could be read, used to end the run as exit 4 ("the listing +> has nothing in it"). It is now exit 5 on page 1 and exit 6 on a later +> page, with `stop_reason: parser_found_nothing`. Measured by renaming the +> row container in a real P2P capture that still counted 187 adverts. + +### Changed + +- The sidecar carries `core_field_shortfall`: for each page, the core + columns filled on fewer than 99% of its rows. `{}` on a healthy run. A + renamed field used to write a complete-looking file with the column null + on every row and one warning in the log; the daily canary now fails on it. +- The canary names a moved shape on exit 5 and 6. +- CI tests Python 3.9 and 3.14 (was 3.9 and 3.12). 3.9 stays as the floor. +- The AWS WAF token cookie is set on the domain the challenge page lists in + `awsWafCookieDomainList`, or on the page's own host when it lists none, + rather than always on the registrable domain. +- `engine-smoke` checks that the engine imports, rather than grepping the + suite's output for a phrase it never printed. + ## [0.1.0] — 2026-09-24 First release. Three modes over binance.com's own JSON endpoints, three diff --git a/README.md b/README.md index 71518da..30a55d8 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![release](https://img.shields.io/github/v/release/2scraper/binance-scraper)](https://github.com/2scraper/binance-scraper/releases) [![tests](https://github.com/2scraper/binance-scraper/actions/workflows/tests.yml/badge.svg)](https://github.com/2scraper/binance-scraper/actions/workflows/tests.yml) [![canary](https://github.com/2scraper/binance-scraper/actions/workflows/canary.yml/badge.svg)](https://github.com/2scraper/binance-scraper/actions/workflows/canary.yml) -![python](https://img.shields.io/badge/python-3.9%20%7C%203.12-blue) +![python](https://img.shields.io/badge/python-3.9%20%E2%80%93%203.14-blue) [![licence](https://img.shields.io/badge/licence-MIT-green)](LICENSE) ![engines](https://img.shields.io/badge/engines-playwright%20%7C%20selenium%20%7C%20pyppeteer-lightgrey) ![runs without an account](https://img.shields.io/badge/all%20three%20modes-no%20account%20needed-brightgreen) @@ -135,7 +135,27 @@ multi-page run of a live listing can see one row twice (the dedupe drops it and the log says so) or miss one that moved up across a page boundary after its page was fetched. No scraper can see the second. -### 5. A refused parameter is not a block +### 5. These endpoints are the front end's, not a published API + +Binance documents its official API and says undocumented interfaces are not +something to build on. These three are exactly that, so one day a field will +be renamed. The run is built to say so rather than to look healthy: + +| the shape moves | what you get | +|---|---| +| the row container is renamed, on page 1 | exit 5, nothing written, and the log says the page was served but **none were read**. Not exit 4: the same response still counts its rows | +| the same, on a later page | exit 6, `stop_reason: parser_found_nothing`, that page in `pages_failed` | +| a column is renamed | the rows are written (exit 0), and the sidecar's `core_field_shortfall` names the column and the page, e.g. `{"1": {"price": 0.0}}` | + +The first two rest on a measurement: on 2026-09-30 every page up to the one +the endpoint's own total implies carried rows, on all three (P2P 228 → 12 +pages, copy-trading 9,093 → 304, announcements 2,275 → 46, the last page +partial each time), and the page after it carried none. A page INSIDE that +range with no rows is the parser missing them. If the total itself is +renamed, nothing tells the page from an empty listing: that is what the +canary's row floor is for, and it runs every day. + +### 6. A refused parameter is not a block The announcements endpoint takes a page size from a fixed set, and answers anything else (12, 25, 30, 100) with **HTTP 400 and an empty body**. Classify @@ -214,8 +234,8 @@ Nothing here integrates a competitor. | 2 | bad usage, including a `--pay-type` the site does not offer | | 3 | blocked: AWS WAF, a 403 or a 451, distinct from an empty listing | | 4 | zero rows: the listing has nothing in it | -| 5 | the data never arrived: a timeout, a dead proxy, a refused parameter, a remote API error | -| 6 | partial: some pages came back and some did not | +| 5 | the data never arrived: a timeout, a dead proxy, a refused parameter, a remote API error, or page 1 served in a shape the parser cannot read (see 5 above) | +| 6 | partial: some pages came back and some did not, or a later page's shape moved (`stop_reason: parser_found_nothing`) | **A run that finds nothing writes nothing**, so a failure never replaces last night's good output with `[]`. `--allow-empty` is the opt-out. diff --git a/page_flow.py b/page_flow.py index 1d381a8..e1ebdef 100644 --- a/page_flow.py +++ b/page_flow.py @@ -209,6 +209,8 @@ def refusal_advice(state: str) -> str: def stop_reason_for(outcome) -> str: """The run's stop_reason when `outcome` is the page that ended it.""" + if getattr(outcome, "unread", False): + return "parser_found_nothing" if getattr(outcome, "rejected", None): return "api_rejected" if getattr(outcome, "blocked_by", None): @@ -319,7 +321,13 @@ def finish(args, query: Query, outcomes: List, stop_reason: str, start_url=args.url or request_for(query, 1).url, final_url=request_for(query, last_ok).url, extra={"total_results": total, "pages_available": available, - "query": query_summary(query)}) + "query": query_summary(query), + # Empty on a healthy run. Keyed by page, because a shape can + # move mid-run and which pages it moved on is the point. + "core_field_shortfall": { + str(o.page_num): o.field_shortfall + for o in sorted(outcomes, key=lambda o: o.page_num) + if o.field_shortfall}}) _WAF_COOKIE_DOMAINS_RE = re.compile(r"awsWafCookieDomainList\s*=\s*\[([^\]]*)\]") @@ -408,11 +416,17 @@ class PageOutcome: # The site's own count of what matched, from this page's response. total_available: Optional[int] = None pages_available: Optional[int] = None + # The page was served and its own total says it holds rows, and the + # parser read none: the payload's shape has moved (see _unread). + unread: bool = False + # Core columns filled on fewer than CORE_FIELD_FLOOR% of this page's + # rows, as {column: share}. Empty when the page looks as captured. + field_shortfall: dict = field(default_factory=dict) @property def ok(self) -> bool: return (not self.load_failed and self.blocked_by is None - and self.rejected is None) + and self.rejected is None and not self.unread) # The columns the site filled on EVERY record of every capture, per mode. @@ -460,17 +474,47 @@ def land(ops, args) -> Tuple[str, Optional[str]]: return state, None -def _core_field_warnings(rows: List, mode: str, page_num: int) -> None: +def _core_field_shortfall(rows: List, mode: str, page_num: int) -> dict: + """{column: share} for each core column below the floor, logged. + + Returned rather than only logged, so the sidecar can carry it: a warning + in a log is read by nobody once the run exits 0, and a renamed field + otherwise yields a complete-looking file with the column null on every + row (measured by renaming `adv.price` in a real P2P capture). + """ + short = {} for name in CORE_FIELDS.get(mode, ()): if not rows: - return + return short filled = sum(1 for r in rows if getattr(r, name, None) not in (None, "", [])) share = 100.0 * filled / len(rows) if share < CORE_FIELD_FLOOR: + short[name] = round(share, 1) log.warning("Only %.0f%% of page %d carries `%s`, against a " "measured floor of %d%%. Every record of every capture " "had one, so the payload shape has moved — re-run with " "--dump-html.", share, page_num, name, CORE_FIELD_FLOOR) + return short + + +def _unread(rows: List, total: Optional[int], pages: Optional[int], + page_num: int) -> bool: + """True when the site says this page holds rows and none were read. + + A page past the end of a listing holds none and says so by arithmetic: + its own total puts it beyond the last page. A page INSIDE that range + with zero rows is the parser failing to find them, and reported as an + empty page it was exit 4, "nobody trades here", on a market the same + response counts 187 adverts in (measured by renaming `data` in a real + P2P capture). The total is read from THIS page, so a listing that + shrank during the run still ends as end_of_listing. + + It cannot fire when the total itself has moved (None): that page is + indistinguishable from an empty listing, and the canary's row floor is + what catches it. + """ + return (not rows and bool(total) and pages is not None + and page_num <= pages) def _dump(args, page_num: int, text: str) -> None: @@ -613,7 +657,15 @@ def fetch_one_page(ops, args, pool, query: Query, page_num: int, if page_num == 1 and outcome.total_available is not None: log.info("The site reports %d match(es) — %s page(s) at this page " "size.", outcome.total_available, outcome.pages_available) - _core_field_warnings(rows, query.mode, page_num) + outcome.field_shortfall = _core_field_shortfall(rows, query.mode, page_num) + if _unread(rows, outcome.total_available, outcome.pages_available, page_num): + outcome.unread = True + debug = _save_debug(args, page_num, text) + log.error("Page %d was served, and the site counts %d match(es) that " + "put rows on it, but none were read — the payload's shape " + "has moved. Saved to %s. This is NOT an empty listing; " + "open an issue with that file.", page_num, + outcome.total_available, debug) return outcome diff --git a/smoke_test.py b/smoke_test.py index e31b99f..8df9a8c 100644 --- a/smoke_test.py +++ b/smoke_test.py @@ -604,15 +604,32 @@ def check_the_shared_loop_end_to_end(): equal("...no search page was fetched", [p for p in ops.fetches if p != 0], []) +def _copy_shrunk_to(total): + """The captured past-the-end copy-trading page, with the total a listing + that SHRANK to `total` states. Measured 2026-09-30 on all three + endpoints: every page up to the one the total implies carries rows + (P2P 228 -> 12 pages, copy-trading 9,093 -> 304, announcements 2,275 -> + 46, the last page partial each time) and the next one carries none. So + an empty page whose OWN total still covers it does not happen on a + healthy response — the listing ending is a page past its own total.""" + payload = copy.deepcopy(FIXTURES["copy_past_the_end"]) + payload["data"]["total"] = total + return json.dumps(payload) + + def check_a_multi_page_run_merges_in_page_order_and_ends_on_data(): import product_parser as P q = P.Query("copytrading") page2 = copy.deepcopy(FIXTURES["copy_30d_roi"]) for i, rec in enumerate(page2["data"]["list"]): rec["leadPortfolioId"] = "90000000000000000%02d" % i + # Page 3 of a listing that had shrunk to two pages of 30 by the time + # page 3 was asked for. This used the capture of page 999 with its total + # of 8,921 untouched, which says page 3 SHOULD hold rows: since the + # shape-moved check, that reads as a parser failure, which is right. answers = {1: [(200, fx("copy_30d_roi"), None)], 2: [(200, json.dumps(page2), None)], - 3: [(200, fx("copy_past_the_end"), None)]} + 3: [(200, _copy_shrunk_to(60), None)]} rc, meta, rows, ops = _run(answers, pages=5, query=q) equal("an empty page 3 of a planned 5 ends the run: exit 0", rc, 0) equal("...as a complete run", meta["status"], "complete") @@ -620,10 +637,61 @@ def check_a_multi_page_run_merges_in_page_order_and_ends_on_data(): equal("...pages 1-3 fetched, not 4-5", ops.fetches, [1, 2, 3]) equal("rows are in page order", [r["page"] for r in rows], [1, 1, 1, 2, 2, 2]) dup = {1: [(200, fx("copy_30d_roi"), None)], 2: [(200, fx("copy_30d_roi"), None)], - 3: [(200, fx("copy_past_the_end"), None)]} + 3: [(200, _copy_shrunk_to(60), None)]} rc, meta, rows, ops = _run(dup, pages=3, query=q) equal("a row seen twice across pages (a live listing moving) is kept once", len(rows), 3) + equal("...and the run is still complete", (rc, meta["status"]), (0, "complete")) + + +def check_a_moved_payload_shape_is_not_an_empty_listing(): + """The endpoints are undocumented, so the shape WILL move one day. Both + ways it can move were silent before: a renamed container parsed to zero + rows and reported exit 4, "nobody trades here", on a response counting + the adverts; a renamed field wrote a complete file with the column null + on every row and one warning in a log. Measured by renaming each in a + real capture.""" + import product_parser as P + renamed = copy.deepcopy(FIXTURES["p2p_usdt_eur_buy"]) + renamed["items"] = renamed.pop("data") + rc, meta, rows, ops = _run({1: [(200, json.dumps(renamed), None)]}, pages=1) + equal("a renamed row container on page 1 is NOT an empty listing (exit 4): " + "exit 5, nothing concluded", rc, 5) + equal("...and the previous good output is not replaced", rows, None) + equal("...and the page is not re-fetched (the same shape comes back)", + ops.fetches, [1]) + + q = P.Query("copytrading") + page2 = copy.deepcopy(FIXTURES["copy_30d_roi"]) + page2["data"]["items"] = page2["data"].pop("list") + rc, meta, rows, ops = _run({1: [(200, fx("copy_30d_roi"), None)], + 2: [(200, json.dumps(page2), None)]}, + pages=2, query=q) + equal("a shape that moves on page 2: exit 6, partial", (rc, meta["status"]), + (6, "partial")) + equal("...named by its own stop_reason", meta["stop_reason"], + "parser_found_nothing") + equal("...with the page that moved listed", meta["pages_failed"], [2]) + + unpriced = copy.deepcopy(FIXTURES["p2p_usdt_eur_buy"]) + for rec in unpriced["data"]: + rec["adv"]["unitPrice"] = rec["adv"].pop("price") + rc, meta, rows, ops = _run({1: [(200, json.dumps(unpriced), None)]}, pages=1) + equal("a renamed FIELD still writes its rows (exit 0: the listing was read)", + rc, 0) + equal("...and the sidecar says which column went missing, on which page", + meta["core_field_shortfall"], {"1": {"price": 0.0}}) + + rc, meta, rows, ops = _run({1: [(200, _p2p_page(1), None)]}, pages=1) + equal("a healthy run's sidecar carries an EMPTY shortfall (a key the " + "canary can assert on)", meta["core_field_shortfall"], {}) + + import page_flow + equal("a page past its OWN total is the end of the listing, not a moved shape", + page_flow._unread([], 228, 12, 13), False) + equal("...a page inside it with no rows is", page_flow._unread([], 228, 12, 12), True) + equal("...and with no total at all nothing can be told apart", + page_flow._unread([], None, None, 1), False) def check_every_engine_implements_the_operations_page_flow_uses():