From f5dc10a5406787304e81e8697b48785f1153812e Mon Sep 17 00:00:00 2001 From: galargh Date: Sun, 12 Apr 2026 15:17:58 +0200 Subject: [PATCH 01/21] test: filecoin-pin in CI as a testing scenario for multi-copy upload --- scenarios/test_multi_copy_upload.py | 113 ++++++++++++++++++++++++++++ scripts/verify_cid.mjs | 21 ++++++ 2 files changed, 134 insertions(+) create mode 100644 scenarios/test_multi_copy_upload.py create mode 100644 scripts/verify_cid.mjs diff --git a/scenarios/test_multi_copy_upload.py b/scenarios/test_multi_copy_upload.py new file mode 100644 index 0000000..b300932 --- /dev/null +++ b/scenarios/test_multi_copy_upload.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Multi-copy upload test: upload a random file via filecoin-pin against the devnet.""" + +import os +import subprocess +import sys +import tempfile +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.request import urlopen + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from scenarios.helpers import assert_eq, assert_ok, fail, info, run_cmd, write_random_file + +RAND_FILE_NAME = "random_file" +RAND_FILE_SIZE = 20 * 1024 * 1024 +RAND_FILE_SEED = 42 + + +def run(): + assert_ok("command -v node", "node is installed") + assert_ok("command -v pnpm", "pnpm is installed") + + scripts_dir = Path("scripts").resolve() + + with tempfile.TemporaryDirectory(prefix="filecoin-pin-") as tmp: + if not run_cmd(["pnpm", "install", "-g", "filecoin-pin"], label="pnpm install -g filecoin-pin"): + return + if not run_cmd(["pnpm", "install", "-g", "multiformats"], label="pnpm install -g multiformats"): + return + + random_file = Path(tmp) / RAND_FILE_NAME + info(f"Creating random file ({RAND_FILE_SIZE} bytes)") + write_random_file(random_file, RAND_FILE_SIZE, RAND_FILE_SEED) + assert_eq( + random_file.stat().st_size, + RAND_FILE_SIZE, + f"{RAND_FILE_NAME} created with exact size {RAND_FILE_SIZE} bytes", + ) + + info("Running filecoin-pin multi-copy upload script against devnet") + + add_result = subprocess.run( + ["filecoin-pin", "add", "--network", "devnet", tmp], + text=True, + capture_output=True, + ) + add_details = (add_result.stderr or add_result.stdout or "").strip() + if add_result.returncode != 0: + fail(f"filecoin-pin add --network devnet {tmp} (exit={add_result.returncode}) {add_details}") + return + + root_cid = None + piece_cid = None + piece_retrieval_urls = [] + + for line in add_details.splitlines(): + stripped = line.strip() + + if root_cid is None and stripped.startswith("Root CID:"): + root_cid = stripped.split(":", 1)[1].strip() + + if piece_cid is None and stripped.startswith("Piece CID:"): + piece_cid = stripped.split(":", 1)[1].strip() + + if stripped.startswith("Retrieval URL:"): + piece_retrieval_urls.append(stripped.split(":", 1)[1].strip()) + + if not root_cid: + fail(f"Could not parse Root CID from output: {add_details}") + return + + if not piece_cid: + fail(f"Could not parse Piece CID from output: {add_details}") + return + + if not piece_retrieval_urls: + fail(f"Could not parse Piece Retrieval URLs from output: {add_details}") + return + + root_retrieval_urls = [ + url.replace("/piece/", "/ipfs/").replace(piece_cid, root_cid) + for url in piece_retrieval_urls + ] + + ipfs_dir = Path("ipfs") + ipfs_dir.mkdir(parents=True, exist_ok=True) + + for i, url in enumerate(root_retrieval_urls, start=1): + file = ipfs_dir / f"{root_cid}_{i}.bin" + download_url = f"{url}?format=raw" + + try: + with urlopen(download_url, timeout=60) as resp, open(file, "wb") as f: + while True: + chunk = resp.read(1024 * 1024) + if not chunk: + break + f.write(chunk) + except (URLError, OSError) as e: + fail(f"Failed to download {download_url}: {e}") + return + + if not run_cmd( + ["node", str(scripts_dir / "verify_cid.mjs"), root_cid, str(file)], + label=f"node {scripts_dir / 'verify_cid.mjs'} {root_cid} {file} ({download_url})", + ): + return + + +if __name__ == "__main__": + run() diff --git a/scripts/verify_cid.mjs b/scripts/verify_cid.mjs new file mode 100644 index 0000000..cbb35cb --- /dev/null +++ b/scripts/verify_cid.mjs @@ -0,0 +1,21 @@ +import { readFileSync } from "node:fs"; +import { CID } from "multiformats/cid"; +import { sha256 } from "multiformats/hashes/sha2"; + +const [cidString, filePath] = process.argv.slice(2); + +if (!cidString || !filePath) { + console.error("usage: node verify.mjs "); + process.exit(1); +} + +const cid = CID.parse(cidString); +const bytes = readFileSync(filePath); +const digest = await sha256.digest(bytes); + +if (Buffer.from(digest.digest).equals(Buffer.from(cid.multihash.digest))) { + console.log("OK"); +} else { + console.error("CID mismatch"); + process.exit(1); +} From 94847933751f1d7832f57bb9a4c60f05293da791 Mon Sep 17 00:00:00 2001 From: galargh Date: Sun, 12 Apr 2026 15:22:28 +0200 Subject: [PATCH 02/21] fmt --- scenarios/test_multi_copy_upload.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/scenarios/test_multi_copy_upload.py b/scenarios/test_multi_copy_upload.py index b300932..25cee9f 100644 --- a/scenarios/test_multi_copy_upload.py +++ b/scenarios/test_multi_copy_upload.py @@ -11,7 +11,14 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from scenarios.helpers import assert_eq, assert_ok, fail, info, run_cmd, write_random_file +from scenarios.helpers import ( + assert_eq, + assert_ok, + fail, + info, + run_cmd, + write_random_file, +) RAND_FILE_NAME = "random_file" RAND_FILE_SIZE = 20 * 1024 * 1024 @@ -25,9 +32,15 @@ def run(): scripts_dir = Path("scripts").resolve() with tempfile.TemporaryDirectory(prefix="filecoin-pin-") as tmp: - if not run_cmd(["pnpm", "install", "-g", "filecoin-pin"], label="pnpm install -g filecoin-pin"): + if not run_cmd( + ["pnpm", "install", "-g", "filecoin-pin"], + label="pnpm install -g filecoin-pin", + ): return - if not run_cmd(["pnpm", "install", "-g", "multiformats"], label="pnpm install -g multiformats"): + if not run_cmd( + ["pnpm", "install", "-g", "multiformats"], + label="pnpm install -g multiformats", + ): return random_file = Path(tmp) / RAND_FILE_NAME @@ -48,7 +61,9 @@ def run(): ) add_details = (add_result.stderr or add_result.stdout or "").strip() if add_result.returncode != 0: - fail(f"filecoin-pin add --network devnet {tmp} (exit={add_result.returncode}) {add_details}") + fail( + f"filecoin-pin add --network devnet {tmp} (exit={add_result.returncode}) {add_details}" + ) return root_cid = None From 9914af25593ae00581eb22a5607f238f599389af Mon Sep 17 00:00:00 2001 From: galargh Date: Sun, 12 Apr 2026 18:14:51 +0200 Subject: [PATCH 03/21] test: include new test in the suite --- scenarios/run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scenarios/run.py b/scenarios/run.py index 8eac053..51be410 100755 --- a/scenarios/run.py +++ b/scenarios/run.py @@ -27,6 +27,7 @@ ("test_containers", 5), ("test_basic_balances", 10), ("test_storage_e2e", 100), + ("test_multi_copy_upload", 200), ("test_caching_subsystem", 200), ] From 74699ee63a1a5886ba29aa78b6a0af87cf1ad223 Mon Sep 17 00:00:00 2001 From: galargh Date: Sun, 12 Apr 2026 18:48:45 +0200 Subject: [PATCH 04/21] pnpm install -g -> npm install -g --- scenarios/test_multi_copy_upload.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scenarios/test_multi_copy_upload.py b/scenarios/test_multi_copy_upload.py index 25cee9f..4085814 100644 --- a/scenarios/test_multi_copy_upload.py +++ b/scenarios/test_multi_copy_upload.py @@ -33,13 +33,13 @@ def run(): with tempfile.TemporaryDirectory(prefix="filecoin-pin-") as tmp: if not run_cmd( - ["pnpm", "install", "-g", "filecoin-pin"], - label="pnpm install -g filecoin-pin", + ["npm", "install", "-g", "filecoin-pin"], + label="npm install -g filecoin-pin", ): return if not run_cmd( - ["pnpm", "install", "-g", "multiformats"], - label="pnpm install -g multiformats", + ["npm", "install", "-g", "multiformats"], + label="npm install -g multiformats", ): return From ed79bb3e9e7c24c8fe79919225cdb3ba8bc2eb0b Mon Sep 17 00:00:00 2001 From: galargh Date: Sun, 12 Apr 2026 19:19:12 +0200 Subject: [PATCH 05/21] debug: show stdout and stderr on failure --- scenarios/test_multi_copy_upload.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scenarios/test_multi_copy_upload.py b/scenarios/test_multi_copy_upload.py index 4085814..e670fdc 100644 --- a/scenarios/test_multi_copy_upload.py +++ b/scenarios/test_multi_copy_upload.py @@ -59,10 +59,15 @@ def run(): text=True, capture_output=True, ) - add_details = (add_result.stderr or add_result.stdout or "").strip() + add_stdout = (add_result.stdout or "").strip() + add_stderr = (add_result.stderr or "").strip() if add_result.returncode != 0: fail( - f"filecoin-pin add --network devnet {tmp} (exit={add_result.returncode}) {add_details}" + f""" + filecoin-pin add --network devnet {tmp} (exit={add_result.returncode}) + {add_stdout} + {add_stderr} + """.strip() ) return @@ -70,7 +75,7 @@ def run(): piece_cid = None piece_retrieval_urls = [] - for line in add_details.splitlines(): + for line in add_stdout.splitlines(): stripped = line.strip() if root_cid is None and stripped.startswith("Root CID:"): From f44000d901b721ea100bd247a656618de5eb6eba Mon Sep 17 00:00:00 2001 From: galargh Date: Sun, 12 Apr 2026 20:10:52 +0200 Subject: [PATCH 06/21] fmt --- scenarios/test_multi_copy_upload.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scenarios/test_multi_copy_upload.py b/scenarios/test_multi_copy_upload.py index e670fdc..003cf34 100644 --- a/scenarios/test_multi_copy_upload.py +++ b/scenarios/test_multi_copy_upload.py @@ -62,13 +62,11 @@ def run(): add_stdout = (add_result.stdout or "").strip() add_stderr = (add_result.stderr or "").strip() if add_result.returncode != 0: - fail( - f""" + fail(f""" filecoin-pin add --network devnet {tmp} (exit={add_result.returncode}) {add_stdout} {add_stderr} - """.strip() - ) + """.strip()) return root_cid = None From 0671bd4733b485790eaff2852aa501d30570304a Mon Sep 17 00:00:00 2001 From: galargh Date: Sun, 3 May 2026 16:57:31 +0200 Subject: [PATCH 07/21] ci: pin synapse-core --- scenarios/test_multi_copy_upload.py | 57 +++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/scenarios/test_multi_copy_upload.py b/scenarios/test_multi_copy_upload.py index 003cf34..a39dafb 100644 --- a/scenarios/test_multi_copy_upload.py +++ b/scenarios/test_multi_copy_upload.py @@ -6,7 +6,7 @@ import sys import tempfile from pathlib import Path -from urllib.error import HTTPError, URLError +from urllib.error import URLError from urllib.request import urlopen sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -31,19 +31,49 @@ def run(): scripts_dir = Path("scripts").resolve() - with tempfile.TemporaryDirectory(prefix="filecoin-pin-") as tmp: + with tempfile.TemporaryDirectory(prefix="filecoin-pin-upload-") as upload_tmp, tempfile.TemporaryDirectory( + prefix="filecoin-pin-npm-" + ) as npm_tmp: + upload_dir = Path(upload_tmp) + npm_dir = Path(npm_tmp) + + if not run_cmd( + ["npm", "init", "-y"], + label="npm init", + cwd=npm_dir, + ): + return + + if not run_cmd( + [ + "npm", + "pkg", + "set", + "type=module", + "dependencies.filecoin-pin=0.20.0", + "overrides.@filoz/synapse-core=0.4.1", + ], + label="pin filecoin-pin dependencies", + cwd=npm_dir, + ): + return + if not run_cmd( - ["npm", "install", "-g", "filecoin-pin"], - label="npm install -g filecoin-pin", + ["npm", "install"], + label="npm install", + cwd=npm_dir, ): return + + # Keep this global for now; verify_cid.mjs likely imports multiformats + # outside the temporary npm project. if not run_cmd( ["npm", "install", "-g", "multiformats"], label="npm install -g multiformats", ): return - random_file = Path(tmp) / RAND_FILE_NAME + random_file = upload_dir / RAND_FILE_NAME info(f"Creating random file ({RAND_FILE_SIZE} bytes)") write_random_file(random_file, RAND_FILE_SIZE, RAND_FILE_SEED) assert_eq( @@ -54,19 +84,24 @@ def run(): info("Running filecoin-pin multi-copy upload script against devnet") + filecoin_pin_bin = npm_dir / "node_modules" / ".bin" / "filecoin-pin" + add_result = subprocess.run( - ["filecoin-pin", "add", "--network", "devnet", tmp], + [str(filecoin_pin_bin), "add", "--network", "devnet", str(upload_dir)], text=True, capture_output=True, ) add_stdout = (add_result.stdout or "").strip() add_stderr = (add_result.stderr or "").strip() + add_details = f"{add_stdout}\n{add_stderr}".strip() + if add_result.returncode != 0: - fail(f""" - filecoin-pin add --network devnet {tmp} (exit={add_result.returncode}) - {add_stdout} - {add_stderr} - """.strip()) + fail( + f""" + filecoin-pin add --network devnet {upload_dir} (exit={add_result.returncode}) + {add_details} + """.strip() + ) return root_cid = None From 9012362cddac5c26975f2f23f78d7accc2932263 Mon Sep 17 00:00:00 2001 From: galargh Date: Sun, 3 May 2026 17:01:33 +0200 Subject: [PATCH 08/21] ci: fix pr trigger --- .github/workflows/ci_pull_request.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci_pull_request.yml b/.github/workflows/ci_pull_request.yml index 5e218f2..d911cde 100644 --- a/.github/workflows/ci_pull_request.yml +++ b/.github/workflows/ci_pull_request.yml @@ -9,7 +9,6 @@ name: CI (Pull Request) on: pull_request: - branches: ['main'] push: branches: ['main'] From 38c839fbe722f8f94f110e1c99cb84b4cff26804 Mon Sep 17 00:00:00 2001 From: galargh Date: Sun, 3 May 2026 17:03:09 +0200 Subject: [PATCH 09/21] fmt --- scenarios/test_multi_copy_upload.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scenarios/test_multi_copy_upload.py b/scenarios/test_multi_copy_upload.py index a39dafb..7c882c6 100644 --- a/scenarios/test_multi_copy_upload.py +++ b/scenarios/test_multi_copy_upload.py @@ -31,9 +31,9 @@ def run(): scripts_dir = Path("scripts").resolve() - with tempfile.TemporaryDirectory(prefix="filecoin-pin-upload-") as upload_tmp, tempfile.TemporaryDirectory( - prefix="filecoin-pin-npm-" - ) as npm_tmp: + with tempfile.TemporaryDirectory( + prefix="filecoin-pin-upload-" + ) as upload_tmp, tempfile.TemporaryDirectory(prefix="filecoin-pin-npm-") as npm_tmp: upload_dir = Path(upload_tmp) npm_dir = Path(npm_tmp) @@ -96,12 +96,10 @@ def run(): add_details = f"{add_stdout}\n{add_stderr}".strip() if add_result.returncode != 0: - fail( - f""" + fail(f""" filecoin-pin add --network devnet {upload_dir} (exit={add_result.returncode}) {add_details} - """.strip() - ) + """.strip()) return root_cid = None From 7c7e3ae9f21e05f4cc607cc2b97e0876b14f7264 Mon Sep 17 00:00:00 2001 From: galargh Date: Sun, 3 May 2026 17:44:49 +0200 Subject: [PATCH 10/21] ci: older version --- scenarios/test_multi_copy_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/test_multi_copy_upload.py b/scenarios/test_multi_copy_upload.py index 7c882c6..5d0ce39 100644 --- a/scenarios/test_multi_copy_upload.py +++ b/scenarios/test_multi_copy_upload.py @@ -51,7 +51,7 @@ def run(): "set", "type=module", "dependencies.filecoin-pin=0.20.0", - "overrides.@filoz/synapse-core=0.4.1", + "overrides.@filoz/synapse-core=0.3.3", ], label="pin filecoin-pin dependencies", cwd=npm_dir, From 30a2f93112fba79d916f5cfa27e2ae8e9f2e7686 Mon Sep 17 00:00:00 2001 From: galargh Date: Sun, 3 May 2026 18:42:56 +0200 Subject: [PATCH 11/21] fix: pin filecoin-pin test dependencies --- scenarios/run.py | 2 +- scenarios/test_multi_copy_upload.py | 141 +++++++++++++++++++++------- scripts/verify_cid.mjs | 21 ----- 3 files changed, 107 insertions(+), 57 deletions(-) delete mode 100644 scripts/verify_cid.mjs diff --git a/scenarios/run.py b/scenarios/run.py index 51be410..38ebde8 100755 --- a/scenarios/run.py +++ b/scenarios/run.py @@ -27,7 +27,7 @@ ("test_containers", 5), ("test_basic_balances", 10), ("test_storage_e2e", 100), - ("test_multi_copy_upload", 200), + ("test_multi_copy_upload", 300), ("test_caching_subsystem", 200), ] diff --git a/scenarios/test_multi_copy_upload.py b/scenarios/test_multi_copy_upload.py index 5d0ce39..0878ed1 100644 --- a/scenarios/test_multi_copy_upload.py +++ b/scenarios/test_multi_copy_upload.py @@ -2,9 +2,11 @@ """Multi-copy upload test: upload a random file via filecoin-pin against the devnet.""" import os +import re import subprocess import sys import tempfile +import time from pathlib import Path from urllib.error import URLError from urllib.request import urlopen @@ -20,17 +22,98 @@ write_random_file, ) +ANSI_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") RAND_FILE_NAME = "random_file" RAND_FILE_SIZE = 20 * 1024 * 1024 RAND_FILE_SEED = 42 +RETRIEVAL_DEADLINE_SECS = 90 +RETRIEVAL_INTERVAL_SECS = 5 +RETRIEVAL_REQUEST_TIMEOUT_SECS = 10 +VERIFY_CID_SCRIPT = """ +import { readFileSync } from "node:fs"; +import { CID } from "multiformats"; +import { sha256 } from "multiformats/hashes/sha2"; + +const [cidString, filePath] = process.argv.slice(1); + +if (!cidString || !filePath) { + console.error("usage: node --input-type=module --eval