From b9f913fe41f1dff5c4e7c01b186a21968937cc11 Mon Sep 17 00:00:00 2001 From: anil-bd <210667061+anil-bd@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:12:08 +0200 Subject: [PATCH 1/2] Move README CLI commands to the CLI 0.4.0 syntax and catch CLI drift daily CLI 0.4.0 (12 Oct) removed 'bdata pipelines ', so every CLI line in the README failed for readers. The Monday run caught only block 10: blocks 11-14 ran a pinned 0.3.4 and stayed green. - README: pipelines run collect_by_url, with the four Instagram IDs - live.yml: one parser in .github/scripts/readme_blocks.py for plan and block jobs - live.yml: new secretless cli-syntax job runs every README CLI line on the newest CLI with a rejected key, on every push and schedule - live.yml: one BDATA_CLI pin, applied to npx lines too; unused CLI install dropped - live.yml: failing blocks show their last six lines; bot commit names the real state - live.yml: a failed scheduled run opens or comments on one issue - ci.yml: concurrency, timeouts, pip cache --- .github/scripts/readme_blocks.py | 138 +++++++++++++++++++++++++ .github/workflows/ci.yml | 8 ++ .github/workflows/live.yml | 166 ++++++++++++++----------------- README.md | 34 +++---- 4 files changed, 234 insertions(+), 112 deletions(-) create mode 100644 .github/scripts/readme_blocks.py diff --git a/.github/scripts/readme_blocks.py b/.github/scripts/readme_blocks.py new file mode 100644 index 0000000..1db42ab --- /dev/null +++ b/.github/scripts/readme_blocks.py @@ -0,0 +1,138 @@ +"""The README's code blocks, as the live check runs them. + + python readme_blocks.py list # JSON matrix for the block jobs (ONLY filters) + python readme_blocks.py run N # run block N with the real token, one retry + python readme_blocks.py cli-syntax # every README CLI line with a dummy key + +One parser for all three, so the list and the runner can never disagree on +what block N is. +""" + +import json +import os +import pathlib +import re +import subprocess +import sys +import time + +README = pathlib.Path(__file__).resolve().parents[2] / "README.md" +SKIP = ("pip install", "export ", "npm i", "npm install", + "npx -p @brightdata/cli bdata login", "bdata login") +CLI = os.environ.get("BDATA_CLI", "") # the pinned CLI version; empty means whatever is installed + + +def blocks(): + readme = README.read_text(encoding="utf-8") + # never execute anything inside the generated field table: its text comes from + # the API, and a fence in a description must not become a command + readme = re.sub(r".*?", "", readme, flags=re.S) + found, heading, seen = [], "README", {} + for m in re.finditer(r"^#{2,3} ([^\n]+)$|```(python|bash)\n(.*?)```", readme, re.M | re.S): + if m.group(1): + heading = m.group(1).strip() # a block is named after its section, not its first line + continue + lang, code = m.group(2), m.group(3) + lines = [line for line in code.splitlines() + if line.strip() and not line.strip().startswith("#")] + if lang == "bash": + lines = [line for line in lines if not line.startswith(SKIP)] + if not lines: + continue + code = "\n".join(lines) + seen[heading] = seen.get(heading, 0) + 1 + title = heading if seen[heading] == 1 else f"{heading} ({seen[heading]})" + found.append({"lang": lang, "title": title[:50], "code": code}) + return found + + +def cli_lines(code, key): + """The CLI cannot open a browser here, so the key goes in by flag, and an + unpinned npx is held to the version the job installed.""" + out = [] + for line in code.splitlines(): + if CLI: + line = line.replace("npx -p @brightdata/cli ", f"npx -p @brightdata/cli@{CLI} ") + if "bdata pipelines" in line: + line += f" -k {key}" + out.append(line) + return "\n".join(out) + + +def list_blocks(): + wanted = {int(n) for n in os.environ.get("ONLY", "").split(",") if n.strip()} + out = [{"n": i + 1, "lang": b["lang"], "title": b["title"]} + for i, b in enumerate(blocks()) if not wanted or i + 1 in wanted] + print("blocks=" + json.dumps(out)) + + +def run_block(n): + b = blocks()[n - 1] + if b["lang"] == "bash": + # only the shell ever sees the key's value + argv = ["bash", "-e", "-c", cli_lines(b["code"], '"$BRIGHTDATA_API_TOKEN"')] + else: + argv = [sys.executable, "-c", b["code"]] + secret = os.environ.get("BRIGHTDATA_API_TOKEN", "") + for _attempt in (1, 2): + started = time.time() + try: + run = subprocess.run(argv, capture_output=True, text=True, timeout=480) + except subprocess.TimeoutExpired: + run = subprocess.CompletedProcess( + argv, 1, "", "block timeout: no result within 8 minutes") + secs = round(time.time() - started) + out, err = run.stdout.strip(), run.stderr.strip() + if secret: # summaries are not secret-masked the way logs are + out, err = out.replace(secret, "***"), err.replace(secret, "***") + ok = run.returncode == 0 and bool(out if b["lang"] == "python" else out or err) + if ok or ((out or err) and "timeout" not in (out + err).lower()): + break + time.sleep(30) # the API timed out or answered with nothing at all; one retry + if ok: + shown = (out.splitlines() or [""])[:1] + else: + # the last few lines: a CLI error often puts the cause above its hint + shown = (err.splitlines() or out.splitlines() or ["no output"])[-6:] + body = "\n".join(" " + line[:160] for line in shown) + state = "ok" if ok else "FAILED" + print(f"\n`{b['title']}`: {state} in {secs}s (exit {run.returncode})\n\n{body}") + sys.exit(0 if ok else 1) + + +def cli_syntax(): + """Run every README CLI line with a key the API rejects. A line the CLI + still understands reaches the API and gets a 401; a line it no longer + understands fails before that. Free, and needs no secret.""" + failed = 0 + print("| block | CLI line | result |\n| --- | --- | --- |") + for i, b in enumerate(blocks(), 1): + if b["lang"] != "bash": + continue + for line in b["code"].splitlines(): + if "bdata pipelines" not in line: + continue + run = subprocess.run(["bash", "-c", cli_lines(line, "invalid-key-for-syntax-check")], + capture_output=True, text=True, timeout=120) + text = run.stdout + run.stderr + ok = "401" in text or "Invalid credentials" in text + last = (run.stderr.strip().splitlines() or run.stdout.strip().splitlines() + or ["no output"]) + why = "reached the API" if ok else " / ".join(x.strip() for x in last[-3:])[:200] + print(f"| {i} | `{line[:90]}` | {'ok' if ok else 'FAILED'}: {why.replace('|', '/')} |") + if not ok: + failed = 1 + print(f"::error::README block {i}: the CLI rejects `{line}`", file=sys.stderr) + sys.exit(failed) + + +if __name__ == "__main__": + mode = sys.argv[1] if len(sys.argv) > 1 else "" + if mode == "list": + list_blocks() + elif mode == "run": + run_block(int(sys.argv[2])) + elif mode == "cli-syntax": + cli_syntax() + else: + sys.exit(__doc__) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2eb6175..df6a178 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,15 +10,22 @@ on: permissions: contents: read +concurrency: # a newer push to the same branch or PR replaces the run still going + group: ci-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: true + jobs: tests: name: Run the tests and the code style check runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml - name: Install the package and its test tools run: pip install -e ".[dev]" - name: Check code style with ruff @@ -40,6 +47,7 @@ jobs: coldstart: name: Follow the README from scratch on an empty machine runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 diff --git a/.github/workflows/live.yml b/.github/workflows/live.yml index 8ffe47e..627c9a8 100644 --- a/.github/workflows/live.yml +++ b/.github/workflows/live.yml @@ -11,6 +11,13 @@ # every Monday at 06:00 UTC: the same, plus every code block in the README, # about 35 credits and 30 minutes # +# Every run also checks, for free and with no secret, that the newest Bright +# Data CLI still accepts every CLI line in the README. Readers run `npx -p +# @brightdata/cli`, which is always the newest release, while the jobs that hold +# the secret run the version pinned in BDATA_CLI below. CLI 0.4.0 removed the +# README's commands and only the Monday run noticed; this check sees it the +# same day. When it warns about a new release, bump BDATA_CLI. +# # A slow day at the API shows up as a red run. Rerun it before reading more # into it. # @@ -47,6 +54,9 @@ on: permissions: contents: write # the daily check commits the regenerated field table +env: + BDATA_CLI: "0.4.0" # pinned for the jobs that hold the API secret; bump after cli-syntax passes + concurrency: group: live-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: ${{ github.event_name == 'push' }} # only the newest commit matters @@ -68,10 +78,10 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: python-version: "3.12" - - name: Install the package and the Bright Data CLI - run: | - pip install -e . - npm i -g @brightdata/cli@0.3.4 # pinned: this job holds the API secret + cache: pip + cache-dependency-path: pyproject.toml + - name: Install the package + run: pip install -e . - name: Do the README's links still resolve? run: | @@ -251,40 +261,7 @@ jobs: - id: list env: ONLY: ${{ inputs.only }} - run: | - python - <<'PY' >> "$GITHUB_OUTPUT" - import os - import json - import pathlib - import re - - readme = pathlib.Path("README.md").read_text(encoding="utf-8") - # never execute anything inside the generated field table: its text comes from - # the API, and a fence in a description must not become a command - readme = re.sub(r".*?", "", readme, flags=re.S) - SKIP = ("pip install", "export ", "npm i", "npm install", - "npx -p @brightdata/cli bdata login", "bdata login") - blocks, heading, seen = [], "README", {} - for m in re.finditer(r"^#{2,3} ([^\n]+)$|```(python|bash)\n(.*?)```", readme, re.M | re.S): - if m.group(1): - heading = m.group(1).strip() # a block is named after its section, not its first line - continue - lang, code = m.group(2), m.group(3) - lines = [l for l in code.splitlines() if l.strip() and not l.strip().startswith("#")] - if lang == "bash": - lines = [l for l in lines if not l.startswith(SKIP)] - if not lines: - continue - # the CLI cannot open a browser here; the secret goes in by flag, and - # only the shell ever sees its value - code = "\n".join(l + ' -k "$BRIGHTDATA_API_TOKEN"' if "bdata pipelines" in l else l for l in lines) - seen[heading] = seen.get(heading, 0) + 1 - title = heading if seen[heading] == 1 else f"{heading} ({seen[heading]})" - blocks.append({"lang": lang, "title": title[:50], "code": code}) - wanted = {int(n) for n in os.environ.get("ONLY", "").split(",") if n.strip()} - out = [{"n": i + 1, "lang": b["lang"], "title": b["title"]} for i, b in enumerate(blocks) if not wanted or i + 1 in wanted] - print("blocks=" + json.dumps(out)) - PY + run: python .github/scripts/readme_blocks.py list >> "$GITHUB_OUTPUT" block: name: "${{ matrix.n }}. ${{ matrix.title }}" @@ -302,72 +279,46 @@ jobs: - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml - name: Install the package and, for CLI blocks, the Bright Data CLI + env: + LANG_: ${{ matrix.lang }} run: | pip install -e . - if [ "${{ matrix.lang }}" = "bash" ]; then npm i -g @brightdata/cli@0.3.4; fi + if [ "$LANG_" = "bash" ]; then npm i -g "@brightdata/cli@$BDATA_CLI"; fi - name: Run block ${{ matrix.n }} exactly as the README shows it env: N: ${{ matrix.n }} run: | set -o pipefail # without this, tee's exit code hides a failing block - python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY" - import os - import subprocess - import sys - import time - import json - import pathlib - import re + python .github/scripts/readme_blocks.py run "$N" | tee -a "$GITHUB_STEP_SUMMARY" - readme = pathlib.Path("README.md").read_text(encoding="utf-8") - # never execute anything inside the generated field table: its text comes from - # the API, and a fence in a description must not become a command - readme = re.sub(r".*?", "", readme, flags=re.S) - SKIP = ("pip install", "export ", "npm i", "npm install", - "npx -p @brightdata/cli bdata login", "bdata login") - blocks, heading, seen = [], "README", {} - for m in re.finditer(r"^#{2,3} ([^\n]+)$|```(python|bash)\n(.*?)```", readme, re.M | re.S): - if m.group(1): - heading = m.group(1).strip() # a block is named after its section, not its first line - continue - lang, code = m.group(2), m.group(3) - lines = [l for l in code.splitlines() if l.strip() and not l.strip().startswith("#")] - if lang == "bash": - lines = [l for l in lines if not l.startswith(SKIP)] - if not lines: - continue - # the CLI cannot open a browser here; the secret goes in by flag, and - # only the shell ever sees its value - code = "\n".join(l + ' -k "$BRIGHTDATA_API_TOKEN"' if "bdata pipelines" in l else l for l in lines) - seen[heading] = seen.get(heading, 0) + 1 - title = heading if seen[heading] == 1 else f"{heading} ({seen[heading]})" - blocks.append({"lang": lang, "title": title[:50], "code": code}) - b = blocks[int(os.environ["N"]) - 1] - argv = ["bash", "-e", "-c", b["code"]] if b["lang"] == "bash" else [sys.executable, "-c", b["code"]] - secret = os.environ.get("BRIGHTDATA_API_TOKEN", "") - for attempt in (1, 2): - started = time.time() - try: - run = subprocess.run(argv, capture_output=True, text=True, timeout=480) - except subprocess.TimeoutExpired: - run = subprocess.CompletedProcess(argv, 1, "", "block timeout: no result within 8 minutes") - secs = round(time.time() - started) - out, err = run.stdout.strip(), run.stderr.strip() - if secret: # summaries are not secret-masked the way logs are - out, err = out.replace(secret, "***"), err.replace(secret, "***") - ok = run.returncode == 0 and bool(out if b["lang"] == "python" else out or err) - if ok or ((out or err) and "timeout" not in (out + err).lower()): - break - time.sleep(30) # the API timed out or answered with nothing at all; one retry - shown = (out.splitlines() or [""])[0] if ok else (err.splitlines() or out.splitlines() or ["no output"])[-1] - print(f"\n`{b['title']}`: {'ok' if ok else 'FAILED'} in {secs}s\n\n {shown[:120]}") - sys.exit(0 if ok else 1) - PY + cli-syntax: + name: Does the newest CLI still accept the README's commands? + runs-on: ubuntu-latest + timeout-minutes: 10 + # no secret here on purpose: this job runs whatever npm calls latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - name: Install the newest CLI, as a reader's npx would + run: | + npm i -g @brightdata/cli@latest + latest=$(bdata --version) + echo "Newest CLI: $latest. Pinned for the live jobs: $BDATA_CLI." + if [ "$latest" != "$BDATA_CLI" ]; then + echo "::warning::@brightdata/cli $latest is out; the live jobs still pin $BDATA_CLI. Bump BDATA_CLI in live.yml once this job passes." + fi + - name: Run each README CLI line with a key the API rejects + env: + BDATA_CLI: "" # use the newest, not the pin + run: | + set -o pipefail + python3 .github/scripts/readme_blocks.py cli-syntax | tee -a "$GITHUB_STEP_SUMMARY" badge: name: Put today's date on the README's "last verified" badge - needs: [scrape, plan, block] + needs: [scrape, plan, block, cli-syntax] # scheduled and manual runs only, and only in the main repository: a push is # never followed by a bot commit, and a copy never diverges from the original if: always() && !cancelled() && github.event_name != 'push' && github.repository == 'brightdata/instagram-scraper-python' @@ -379,8 +330,9 @@ jobs: env: SCRAPE: ${{ needs.scrape.result }} BLOCKS: ${{ needs.block.result }} # skipped on the daily run, and that is fine + SYNTAX: ${{ needs.cli-syntax.result }} run: | - if [ "$SCRAPE" = "success" ] && { [ "$BLOCKS" = "success" ] || [ "$BLOCKS" = "skipped" ]; }; then + if [ "$SCRAPE" = "success" ] && [ "$SYNTAX" = "success" ] && { [ "$BLOCKS" = "success" ] || [ "$BLOCKS" = "skipped" ]; }; then label="last verified"; color="brightgreen" else label="last check failed"; color="red" @@ -398,7 +350,33 @@ jobs: if git diff --quiet; then echo "badge already shows $today"; exit 0; fi git config user.name "anil-bd" git config user.email "210667061+anil-bd@users.noreply.github.com" - git commit -qam "Last verified $today (automated daily check)" + git commit -qam "${label^} $today (automated daily check)" git pull -q --rebase origin main git push -q origin main echo "badge now shows: $label, $today" + + # A red scheduled run used to show only as a red badge. Open one issue and keep + # adding to it, so a failure reaches the people watching the repository. + alert: + name: Open or update the live-check issue + needs: [scrape, block, cli-syntax] + if: failure() && github.event_name == 'schedule' && github.repository == 'brightdata/instagram-scraper-python' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + issues: write + steps: + - env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RUN: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + RESULTS: "scrape ${{ needs.scrape.result }}, README blocks ${{ needs.block.result }}, CLI syntax ${{ needs.cli-syntax.result }}" + run: | + title="Live check is failing" + body="$(date -u +'%-d %b %Y'): $RESULTS. $RUN" + n=$(gh issue list --state open --search "\"$title\" in:title" --json number --jq '.[0].number') + if [ -n "$n" ]; then + gh issue comment "$n" --body "$body" + else + gh issue create --title "$title" --body "$body" + fi diff --git a/README.md b/README.md index e4f5f26..93d7736 100644 --- a/README.md +++ b/README.md @@ -406,12 +406,12 @@ once, or use `bdata login --device` over SSH and in CI: ```bash npx -p @brightdata/cli bdata login -npx -p @brightdata/cli bdata pipelines instagram_posts "https://www.instagram.com/p/Db_SePSltfz/" +npx -p @brightdata/cli bdata pipelines run gd_lk5ns7kz21pck8jpis collect_by_url "https://www.instagram.com/p/Db_SePSltfz/" ``` -The CLI's four Instagram pipelines each take a post, reel or profile URL and -return that one record. Recent posts from a profile is the SDK call in the -Quickstart; the CLI has no route for it. +The CLI runs a scraper by its dataset ID. The four Instagram scrapers each +take a post, reel or profile URL and return that one record; +`bdata pipelines search instagram` lists their IDs and methods. `npx skills add brightdata/skills` teaches Claude Code, Cursor and Codex these commands and the docs, so plain language works afterwards. Full guide: @@ -432,43 +432,41 @@ Data connects to, from LangChain to Zapier and n8n:
Every Instagram command the CLI has, each run today -Four types: `instagram_profiles`, `instagram_posts`, `instagram_reels`, -`instagram_comments`. Each takes URLs, prints JSON, and costs one credit per -record. Prefix each command with `npx -p @brightdata/cli`, or install once with +Four scrapers, each run with `bdata pipelines run collect_by_url`: +profiles `gd_l1vikfch901nx3by4`, posts `gd_lk5ns7kz21pck8jpis`, reels +`gd_lyclm20il4r5helnj` and comments `gd_ltppn085pokosxh13`. Each takes URLs, +prints JSON, and costs one credit per record. Prefix each command with `npx -p @brightdata/cli`, or install once with `npm i -g @brightdata/cli`. ```bash -bdata pipelines instagram_profiles "https://www.instagram.com/nasa/" --pretty +bdata pipelines run gd_l1vikfch901nx3by4 collect_by_url "https://www.instagram.com/nasa/" --pretty ``` 1 record, 25 fields, 50 seconds. `"account": "nasa"`, `"followers": 104396028`. ```bash -bdata pipelines instagram_posts "https://www.instagram.com/p/Dc1W1uFj-CW/" --format csv -o posts.csv -``` - -``` -Triggered collection with snapshot ID:sd_mtrcu6qpyebqgtcjg -Output written to posts.csv +bdata pipelines run gd_lk5ns7kz21pck8jpis collect_by_url "https://www.instagram.com/p/Dc1W1uFj-CW/" --format csv -o posts.csv ``` 37 seconds. The CSV header carries the same field names as the table below. ```bash -bdata pipelines instagram_reels "https://www.instagram.com/reel/DcMXl1IPNtB/" --pretty +bdata pipelines run gd_lyclm20il4r5helnj collect_by_url "https://www.instagram.com/reel/DcMXl1IPNtB/" --pretty ``` 1 record, 25 fields, 23 seconds. ```bash -bdata pipelines instagram_comments "https://www.instagram.com/p/Dc1W1uFj-CW/" --pretty +bdata pipelines run gd_ltppn085pokosxh13 collect_by_url "https://www.instagram.com/p/Dc1W1uFj-CW/" --pretty ``` 5 records, 11 fields, 9 seconds. One credit per comment, so check the post's `num_comments` first. -`bdata pipelines list` prints every type. `bdata pipelines --help` shows -`--format json|csv|ndjson|jsonl`, `-o FILE` and `--timeout`. +`bdata pipelines info ` shows a scraper's input fields. +`bdata pipelines run --help` shows `--format json|csv|ndjson|jsonl`, `-o FILE`, +`--timeout` and `--async`. CLI 0.4.0 replaced the old +`bdata pipelines instagram_posts ` form with these commands.
From f73d9bf6c8547113dd4bae0336e3d45a25201676 Mon Sep 17 00:00:00 2001 From: anil-bd <210667061+anil-bd@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:12:51 +0200 Subject: [PATCH 2/2] Live check: commit the badge and field table only from main --- .github/workflows/live.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/live.yml b/.github/workflows/live.yml index 627c9a8..2fce5a4 100644 --- a/.github/workflows/live.yml +++ b/.github/workflows/live.yml @@ -182,7 +182,7 @@ jobs: PY - name: Regenerate the README field table from the live schema - if: always() && env.CHECK == 'true' && github.repository == 'brightdata/instagram-scraper-python' + if: always() && env.CHECK == 'true' && github.repository == 'brightdata/instagram-scraper-python' && github.ref == 'refs/heads/main' run: | python - <<'PY' | tee -a "$GITHUB_STEP_SUMMARY" import json @@ -319,9 +319,10 @@ jobs: badge: name: Put today's date on the README's "last verified" badge needs: [scrape, plan, block, cli-syntax] - # scheduled and manual runs only, and only in the main repository: a push is - # never followed by a bot commit, and a copy never diverges from the original - if: always() && !cancelled() && github.event_name != 'push' && github.repository == 'brightdata/instagram-scraper-python' + # scheduled and manual runs on main only, and only in the main repository: a + # push is never followed by a bot commit, a copy never diverges from the + # original, and a manual run on a branch never tries to push main + if: always() && !cancelled() && github.event_name != 'push' && github.repository == 'brightdata/instagram-scraper-python' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest timeout-minutes: 5 steps: