From 2cf92f6950eb598b2116c82aa0af65ec3fb23bb8 Mon Sep 17 00:00:00 2001 From: AxelNoun <150222552+AxelNoun@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:23:58 +0200 Subject: [PATCH 01/13] feat(datasets): add PTB-XL ECG loader and YAML config Introduce PTBXLDataset (BaseDataset + YAML) with optional wfdb extra, root-keyed metadata cache, and a resolved cache YAML so file_path is correct before BaseDataset init. Co-authored-by: Cursor --- pyhealth/datasets/__init__.py | 1 + pyhealth/datasets/configs/ptbxl.yaml | 22 ++ pyhealth/datasets/ptbxl.py | 535 +++++++++++++++++++++++++++ pyproject.toml | 5 + 4 files changed, 563 insertions(+) create mode 100644 pyhealth/datasets/configs/ptbxl.yaml create mode 100644 pyhealth/datasets/ptbxl.py diff --git a/pyhealth/datasets/__init__.py b/pyhealth/datasets/__init__.py index 57a9956c2..5831bebba 100644 --- a/pyhealth/datasets/__init__.py +++ b/pyhealth/datasets/__init__.py @@ -84,6 +84,7 @@ def __init__(self, *args, **kwargs): split_by_visit_conformal, ) from .eegbci import EEGBCIDataset as EEGBCIDataset # noqa: E402 +from .ptbxl import PTBXLDataset as PTBXLDataset from .tuab import TUABDataset from .tuev import TUEVDataset from .utils import ( diff --git a/pyhealth/datasets/configs/ptbxl.yaml b/pyhealth/datasets/configs/ptbxl.yaml new file mode 100644 index 000000000..6b5b969b9 --- /dev/null +++ b/pyhealth/datasets/configs/ptbxl.yaml @@ -0,0 +1,22 @@ +version: "1.0.3" +tables: + records: + # Template only: PTBXLDataset writes a resolved config whose file_path + # is the derived cache CSV (rate + data-root hash) before BaseDataset init. + file_path: "ptbxl-pyhealth-100hz.csv" + patient_id: "patient_id" + # recording_date from ptbxl_database.csv (e.g. 1984-11-09 09:17:34) + timestamp: "recording_date" + timestamp_format: "%Y-%m-%d %H:%M:%S" + attributes: + - "record_id" + - "signal_file" + - "sampling_rate" + - "strat_fold" + - "age" + - "age_is_censored" + - "age_is_missing" + - "sex" # 0 = female, 1 = male + - "site" + - "device" + - "scp_codes" diff --git a/pyhealth/datasets/ptbxl.py b/pyhealth/datasets/ptbxl.py new file mode 100644 index 000000000..54fee0c0d --- /dev/null +++ b/pyhealth/datasets/ptbxl.py @@ -0,0 +1,535 @@ +""" +PyHealth dataset for PTB-XL (12-lead ECG, PhysioNet). + +Dataset link: + https://physionet.org/content/ptb-xl/1.0.3/ + +Dataset paper: (please cite if you use this dataset) + Patrick Wagner, Nils Strodthoff, Ralf-Dieter Bousseljot, Dieter Kreiseler, + Fatima I. Lunze, Wojciech Samek, and Tobias Schaeffter. "PTB-XL, a large + publicly available electrocardiography dataset." Scientific Data 7, 154 + (2020). + +Dataset paper link: + https://www.nature.com/articles/s41597-020-0495-6 + +Author: + AxelNoun (GitHub: @AxelNoun) — external contributor, no NetID + +Description: + Implements ``PTBXLDataset`` (BaseDataset + YAML) for PTB-XL v1.0.3, + including metadata preparation, HIPAA age-censor handling, and lazy + ``wfdb`` signal loading. Task and split helpers live in separate modules. +""" + +from __future__ import annotations + +import ast +import hashlib +import logging +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import numpy as np +import pandas as pd +import yaml + +from .base_dataset import BaseDataset +from .configs.config import load_yaml_config +from .utils import MODULE_CACHE_PATH + +if TYPE_CHECKING: + from pyhealth.tasks.ptbxl import PTBXLSuperclassClassification + +logger = logging.getLogger(__name__) + +# HIPAA: ages ≥ 90 are encoded as this sentinel in ptbxl_database.csv. +AGE_CENSOR_SENTINEL = 300 + +# Official diagnostic superclasses (diagnostic_class in scp_statements.csv). +PTBXL_DIAGNOSTIC_SUPERCLASSES = ("NORM", "MI", "STTC", "CD", "HYP") + +# Sex encoding in ptbxl_database.csv (PhysioNet / Scientific Data). +PTBXL_SEX_FEMALE = 0 +PTBXL_SEX_MALE = 1 + +PTBXL_DATABASE_CSV = "ptbxl_database.csv" +PTBXL_SCP_STATEMENTS_CSV = "scp_statements.csv" + +_DEFAULT_METADATA_CACHE = Path(MODULE_CACHE_PATH) / "ptbxl" + + +def format_patient_id(value: Any, *, ecg_id: Any | None = None) -> str: + """Cast PTB-XL ``patient_id`` (stored as float, e.g. ``15709.0``) to ``str``. + + Without an explicit int cast, stringification yields ``\"15709.0\"`` and + silently breaks patient-level splits. Called from + :meth:`PTBXLDataset.prepare_metadata`. + + Args: + value (Any): Raw ``patient_id`` cell (float, int, or numeric string). + ecg_id (Any | None): Optional ``ecg_id`` included in error messages. + + Returns: + str: Integer patient id as a string (e.g. ``\"15709\"``). + + Examples: + >>> format_patient_id(15709.0) + '15709' + """ + if value is None or (isinstance(value, float) and np.isnan(value)): + suffix = f" (ecg_id={ecg_id})" if ecg_id is not None else "" + raise ValueError(f"patient_id is missing{suffix}") + try: + return str(int(float(value))) + except (TypeError, ValueError) as exc: + suffix = f" (ecg_id={ecg_id})" if ecg_id is not None else "" + raise ValueError( + f"patient_id is not numeric: {value!r}{suffix}" + ) from exc + + +def parse_scp_codes(scp_codes: Any) -> dict[str, float]: + """Parse the stringified ``scp_codes`` dict from ``ptbxl_database.csv``. + + PhysioNet stores entries as ``statement: likelihood`` where likelihood is + in ``[0, 100]`` and **0 means unknown confidence, not absence**. All keys + present in the dict are therefore treated as positive statements; only an + empty dict means no statements. Used by task aggregation helpers. + + Args: + scp_codes (Any): Raw cell value (dict or stringified dict). + + Returns: + dict[str, float]: Mapping of SCP statement acronym → likelihood. + + Examples: + >>> parse_scp_codes("{'IMI': 80.0, 'SR': 0.0}")["SR"] + 0.0 + """ + if scp_codes is None or (isinstance(scp_codes, float) and np.isnan(scp_codes)): + return {} + if isinstance(scp_codes, dict): + return {str(k): float(v) for k, v in scp_codes.items()} + text = str(scp_codes).strip() + if not text or text.lower() in {"nan", "none", "{}"}: + return {} + parsed = ast.literal_eval(text) + if not isinstance(parsed, dict): + raise TypeError( + f"Expected scp_codes dict, got {type(parsed)!r}: {scp_codes!r}" + ) + return {str(k): float(v) for k, v in parsed.items()} + + +def is_age_censored(age: Any, sentinel: int = AGE_CENSOR_SENTINEL) -> bool: + """Return True when age is the HIPAA ≥90 sentinel (default 300). + + Distinct from missing age (NaN), which is not censored. Used when writing + ``age_is_censored`` in metadata. + + Args: + age (Any): Raw age cell from ``ptbxl_database.csv``. + sentinel (int): Censor value (default ``300``). + + Returns: + bool: True if ``age`` equals the HIPAA sentinel. + + Examples: + >>> is_age_censored(300), is_age_censored(float("nan")) + (True, False) + """ + if age is None or (isinstance(age, float) and np.isnan(age)): + return False + if isinstance(age, str) and not age.strip(): + return False + try: + return int(float(age)) == int(sentinel) + except (TypeError, ValueError): + return False + + +def is_age_missing(age: Any) -> bool: + """Return True when age is genuinely missing (NaN / empty), not censored. + + Args: + age (Any): Raw age cell from ``ptbxl_database.csv``. + + Returns: + bool: True if age is missing (not the 300 sentinel). + + Examples: + >>> is_age_missing(float("nan")), is_age_missing(300) + (True, False) + """ + if age is None: + return True + if isinstance(age, float) and np.isnan(age): + return True + return isinstance(age, str) and not str(age).strip() + + +def root_cache_key(data_root: str | Path) -> str: + """Return a short stable hash of the resolved data root path. + + Args: + data_root (str | Path): PTB-XL version root. + + Returns: + str: 10-character hex digest used in metadata filenames. + + Examples: + >>> len(root_cache_key("/data/ptb-xl/1.0.3")) + 10 + """ + resolved = str(Path(data_root).resolve()) + return hashlib.sha1(resolved.encode("utf-8")).hexdigest()[:10] + + +def metadata_filename(sampling_rate: int, data_root: str | Path) -> str: + """Return the rate- and root-specific derived metadata CSV name. + + Args: + sampling_rate (int): ``100`` or ``500``. + data_root (str | Path): Absolute/relative PTB-XL data root. + + Returns: + str: Filename such as ``ptbxl-pyhealth-100hz-.csv``. + + Examples: + >>> name = metadata_filename(100, "/data/ptb-xl/1.0.3") + >>> name.startswith("ptbxl-pyhealth-100hz-") + True + """ + return ( + f"ptbxl-pyhealth-{int(sampling_rate)}hz-" + f"{root_cache_key(data_root)}.csv" + ) + + +def load_ptbxl_record(record_path: str | Path) -> np.ndarray: + """Load a PTB-XL WFDB record as ``(n_leads, n_samples)``. + + ``wfdb.rdsamp`` returns ``(n_samples, n_channels)`` (e.g. ``(1000, 12)`` at + 100 Hz). This helper **transposes** to ``(n_leads, n_samples)`` so the layout + matches PyHealth signal tasks such as EEGBCI / SleepEDF, which use + ``mne.io.BaseRaw.get_data()`` → ``(n_channels, n_times)``. The + ``\"tensor\"`` processor preserves that shape. + + The path must be a WFDB *record base* without ``.hea`` / ``.dat`` (as in + ``filename_lr`` / ``filename_hr``). Extensions are stripped only if the + caller accidentally includes them; they are never appended. + + Args: + record_path (str | Path): WFDB record base path (no extension). + + Returns: + np.ndarray: Signal of shape ``(n_leads, n_samples)``, float32, + physical units. + + Raises: + ImportError: If the optional ``wfdb`` extra is not installed. + FileNotFoundError: If the ``.hea`` header cannot be found. + + Examples: + >>> # doctest: +SKIP + >>> signal = load_ptbxl_record("/data/ptb-xl/1.0.3/records100/00000/00001_lr") + >>> signal.shape[0] + 12 + """ + try: + import wfdb + except ImportError as exc: + raise ImportError( + "Reading PTB-XL waveforms requires the optional 'wfdb' dependency. " + "Install it with: pip install 'pyhealth[ptbxl]'" + ) from exc + + path = Path(record_path) + # Strip accidental extensions; never add .hea/.dat for rdsamp. + if path.suffix.lower() in {".hea", ".dat"}: + path = path.with_suffix("") + header = Path(str(path) + ".hea") + if not header.is_file(): + raise FileNotFoundError( + f"PTB-XL WFDB header not found for record base: {record_path}" + ) + + signals, _ = wfdb.rdsamp(str(path)) + # (n_samples, n_leads) → (n_leads, n_samples) to match mne.get_data(). + return np.asarray(signals, dtype=np.float32).T + + +class PTBXLDataset(BaseDataset): + """PhysioNet PTB-XL ECG dataset (v1.0.3). + + Dataset: https://physionet.org/content/ptb-xl/1.0.3/ + + Expects ``root`` (``data_root``) to point at the extracted version directory + containing ``ptbxl_database.csv``, ``scp_statements.csv``, ``records100/``, + and ``records500/``. Raw data must live outside the git repo. + + Derived metadata CSVs are written under PyHealth's dataset cache + (``~/.cache/pyhealth/datasets/ptbxl/`` by default), **not** into ``root``, + so read-only / shared data mounts stay untouched. Override with + ``metadata_cache_dir``. Filenames include the sampling rate and a short + hash of the resolved data root so different roots never share a cache. + + Args: + root (str): Version root of the PTB-XL download (signal + official CSVs). + dataset_name (str | None): Optional name; defaults to + ``ptbxl_{sampling_rate}hz``. + config_path (str | Path | None): Optional YAML config; defaults to + ``configs/ptbxl.yaml``. + sampling_rate (int): ``100`` (default) or ``500``. + metadata_cache_dir (str | Path | None): Directory for derived + ``ptbxl-pyhealth-*.csv``. Defaults to + ``MODULE_CACHE_PATH / \"ptbxl\"``. + **kwargs: Forwarded to :class:`BaseDataset` (``cache_dir``, ``dev``, …). + + Attributes: + data_root (Path): User-provided PTB-XL version root (waveforms + CSVs). + sampling_rate (int): Selected waveform rate (100 or 500). + metadata_cache_dir (Path): Directory holding derived metadata CSVs. + + Note: + Age missing (NaN) vs HIPAA-censored (``age == 300`` for ≥90) are + exposed as ``age_is_missing`` / ``age_is_censored``. Sex is encoded + as ``0`` = female, ``1`` = male. ``scp_codes`` keeps the original + stringified dict; likelihood ``0`` means unknown confidence. Label + aggregation lives in task modules. ``recording_date`` is used as the + event timestamp. + + Examples: + >>> dataset = PTBXLDataset(root="/data/ptb-xl/1.0.3") + >>> dataset.stats() + >>> patient = dataset.get_patient(dataset.unique_patient_ids[0]) + >>> event = patient.get_events(event_type="records")[0] + >>> signal = load_ptbxl_record(event.signal_file) + """ + + def __init__( + self, + root: str, + dataset_name: str | None = None, + config_path: str | Path | None = None, + sampling_rate: int = 100, + metadata_cache_dir: str | Path | None = None, + **kwargs, + ) -> None: + if sampling_rate not in {100, 500}: + raise ValueError( + f"sampling_rate must be 100 or 500, got {sampling_rate}" + ) + package_config = ( + Path(config_path) + if config_path is not None + else Path(__file__).parent / "configs" / "ptbxl.yaml" + ) + + self.data_root = Path(root) + self.sampling_rate = int(sampling_rate) + self.metadata_cache_dir = Path( + metadata_cache_dir + if metadata_cache_dir is not None + else _DEFAULT_METADATA_CACHE + ) + self.metadata_file_name = metadata_filename( + self.sampling_rate, self.data_root + ) + self.prepare_metadata() + + # BaseDataset only accepts config_path (not an in-memory config). Write + # a resolved YAML whose file_path matches the derived CSV *before* + # super().__init__ — loading is lazy via global_event_df, but the + # config must already be correct when that first happens. + resolved_config_path = self._write_resolved_config(package_config) + + # BaseDataset.root is the metadata cache (CSV location); waveforms stay + # under data_root via absolute signal_file paths in the CSV. + super().__init__( + root=str(self.metadata_cache_dir), + tables=["records"], + dataset_name=dataset_name or f"ptbxl_{self.sampling_rate}hz", + config_path=str(resolved_config_path), + **kwargs, + ) + + def _write_resolved_config(self, package_config: Path) -> Path: + """Write a cache-local YAML with the derived metadata CSV filename. + + Args: + package_config (Path): Packaged ``configs/ptbxl.yaml`` template. + + Returns: + Path: Written config path passed to :class:`BaseDataset`. + """ + config = load_yaml_config(str(package_config)) + config.tables["records"].file_path = self.metadata_file_name + out_path = self.metadata_cache_dir / ( + f"ptbxl-config-{self.sampling_rate}hz-" + f"{root_cache_key(self.data_root)}.yaml" + ) + self.metadata_cache_dir.mkdir(parents=True, exist_ok=True) + with open(out_path, "w", encoding="utf-8") as handle: + yaml.safe_dump(config.model_dump(), handle, sort_keys=False) + return out_path + + @property + def scp_statements_path(self) -> Path: + """Path to official ``scp_statements.csv`` under the data root. + + Returns: + Path: Absolute path to ``scp_statements.csv``. + """ + return self.data_root / PTBXL_SCP_STATEMENTS_CSV + + def prepare_metadata(self) -> None: + """Build rate-/root-specific metadata CSV under ``metadata_cache_dir``. + + Returns: + None + """ + csv_path = self.metadata_cache_dir / self.metadata_file_name + if csv_path.exists() and self._metadata_matches_request(csv_path): + return + + db_path = self.data_root / PTBXL_DATABASE_CSV + if not db_path.is_file(): + raise FileNotFoundError( + f"Expected {PTBXL_DATABASE_CSV} under root={self.data_root}. " + "Download PTB-XL from https://physionet.org/content/ptb-xl/1.0.3/" + ) + + db = pd.read_csv(db_path) + required = { + "ecg_id", + "patient_id", + "age", + "sex", + "site", + "device", + "scp_codes", + "strat_fold", + "filename_lr", + "filename_hr", + "recording_date", + } + missing = required - set(db.columns) + if missing: + raise ValueError( + f"ptbxl_database.csv missing columns: {sorted(missing)}" + ) + + filename_col = "filename_lr" if self.sampling_rate == 100 else "filename_hr" + rows: list[dict[str, Any]] = [] + for row in db.to_dict(orient="records"): + ecg_id = row["ecg_id"] + rel = str(row[filename_col]).strip() + # Record base path only — no .hea/.dat (matches wfdb.rdsamp). + if rel.endswith((".hea", ".dat")): + rel = rel.rsplit(".", 1)[0] + signal_file = str((self.data_root / rel).resolve()) + + age = row["age"] + missing_age = is_age_missing(age) + censored = is_age_censored(age) + if missing_age: + age_out: Any = pd.NA + else: + # Keep numeric age (including sentinel 300). + age_out = int(float(age)) + + rows.append( + { + "patient_id": format_patient_id( + row["patient_id"], ecg_id=ecg_id + ), + "record_id": str(int(float(ecg_id))), + "signal_file": signal_file, + "sampling_rate": self.sampling_rate, + "strat_fold": int(row["strat_fold"]), + "age": age_out, + "age_is_censored": int(censored), + "age_is_missing": int(missing_age), + "sex": int(row["sex"]), + "site": row["site"], + "device": row["device"], + "scp_codes": row["scp_codes"], + "recording_date": row["recording_date"], + } + ) + + out = pd.DataFrame(rows) + out["patient_id"] = out["patient_id"].astype(str) + out["record_id"] = out["record_id"].astype(str) + out.sort_values( + ["patient_id", "record_id"], + key=lambda col: col.astype(int), + inplace=True, + ) + out.reset_index(drop=True, inplace=True) + self.metadata_cache_dir.mkdir(parents=True, exist_ok=True) + out.to_csv(csv_path, index=False) + logger.info( + "Wrote PTB-XL metadata (%d records, %d Hz) to %s", + len(out), + self.sampling_rate, + csv_path, + ) + + def _metadata_matches_request(self, csv_path: Path) -> bool: + """Reuse cached metadata when schema and signal roots match. + + Args: + csv_path (Path): Candidate derived metadata CSV. + + Returns: + bool: True if the cache is safe to reuse for this ``data_root``. + """ + try: + df = pd.read_csv(csv_path, nrows=5) + except (OSError, ValueError, pd.errors.ParserError): + return False + needed = { + "patient_id", + "record_id", + "signal_file", + "sampling_rate", + "strat_fold", + "age", + "age_is_censored", + "age_is_missing", + "sex", + "site", + "device", + "scp_codes", + "recording_date", + } + if not needed.issubset(df.columns) or df.empty: + return False + # Absolute signal paths must still sit under the current data_root. + data_root = self.data_root.resolve() + for path in df["signal_file"].astype(str): + try: + Path(path).resolve().relative_to(data_root) + except ValueError: + return False + return True + + @property + def default_task(self) -> PTBXLSuperclassClassification: + """Return the 5-superclass multi-label task wired to this data root. + + Matches other signal datasets (EEGBCI / TUAB / SleepEDF): BaseDataset + exposes ``default_task`` as a read-only property and never assigns to + it in ``__init__``. + + Returns: + PTBXLSuperclassClassification: Default task instance. + """ + from pyhealth.tasks.ptbxl import PTBXLSuperclassClassification + + return PTBXLSuperclassClassification( + scp_statements_path=str(self.scp_statements_path), + ) diff --git a/pyproject.toml b/pyproject.toml index b4626e649..494469d9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,9 @@ nlp = [ "rouge_score~=0.1.2", "nltk~=3.9.1", ] +ptbxl = [ + "wfdb>=4.1.0", +] lint = [ "ruff~=0.15", ] @@ -135,6 +138,8 @@ pyhealth = { path = ".", editable = true } [tool.pixi.feature.test.pypi-dependencies] pyhealth = { path = ".", editable = true } +# Optional PTB-XL waveform dependency — keep CI covering load_ptbxl_record. +wfdb = ">=4.1.0" [tool.pixi.feature.nlp.pypi-dependencies] pyhealth = { path = ".", editable = true } From f4cf33e76bc68bf7ceaf6303c37ba91dc10fab2c Mon Sep 17 00:00:00 2001 From: AxelNoun <150222552+AxelNoun@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:23:58 +0200 Subject: [PATCH 02/13] feat: add PTB-XL 5-superclass task and strat_fold split Add PTBXLSuperclassClassification and split_by_strat_fold (folds 1-8 / 9 / 10) in separate modules so they can move to benchmarks later. Co-authored-by: Cursor --- pyhealth/datasets/__init__.py | 1 + pyhealth/datasets/splitter.py | 71 ++++++++++ pyhealth/tasks/__init__.py | 4 + pyhealth/tasks/ptbxl.py | 239 ++++++++++++++++++++++++++++++++++ 4 files changed, 315 insertions(+) create mode 100644 pyhealth/tasks/ptbxl.py diff --git a/pyhealth/datasets/__init__.py b/pyhealth/datasets/__init__.py index 5831bebba..d489a8beb 100644 --- a/pyhealth/datasets/__init__.py +++ b/pyhealth/datasets/__init__.py @@ -80,6 +80,7 @@ def __init__(self, *args, **kwargs): split_by_sample_conformal, split_by_sample_conformal_tuh, split_by_sample_tuh, + split_by_strat_fold as split_by_strat_fold, split_by_visit, split_by_visit_conformal, ) diff --git a/pyhealth/datasets/splitter.py b/pyhealth/datasets/splitter.py index 2dbc94186..bebdabc06 100644 --- a/pyhealth/datasets/splitter.py +++ b/pyhealth/datasets/splitter.py @@ -180,6 +180,77 @@ def split_by_patient( return train_dataset, val_dataset, test_dataset +def split_by_strat_fold( + dataset: SampleDataset, + train_folds: tuple[int, ...] | list[int] = tuple(range(1, 9)), + val_folds: tuple[int, ...] | list[int] = (9,), + test_folds: tuple[int, ...] | list[int] = (10,), +): + """Split a sample dataset using PTB-XL official ``strat_fold`` assignments. + + The recommended PTB-XL protocol uses folds 1–8 for training, fold 9 for + validation, and fold 10 for testing (patient-disjoint by construction). + + Samples must contain an integer ``strat_fold`` field (as emitted by + :class:`~pyhealth.tasks.ptbxl.PTBXLSuperclassClassification`). + + Args: + dataset (SampleDataset): A :class:`~pyhealth.datasets.SampleDataset` + (or compatible object exposing ``subset`` and ``__getitem__``). + train_folds (Tuple[int, ...] | List[int]): Folds assigned to train + (default ``1..8``). + val_folds (Tuple[int, ...] | List[int]): Folds assigned to validation + (default ``9``). + test_folds (Tuple[int, ...] | List[int]): Folds assigned to test + (default ``10``). + + Returns: + tuple: ``(train_dataset, val_dataset, test_dataset)`` subsets. + + Raises: + KeyError: If a sample is missing ``strat_fold``. + ValueError: If fold sets overlap or a sample fold is unassigned. + + Examples: + >>> # doctest: +SKIP + >>> from pyhealth.datasets import split_by_strat_fold + >>> train, val, test = split_by_strat_fold(sample_dataset) + """ + train_set = {int(f) for f in train_folds} + val_set = {int(f) for f in val_folds} + test_set = {int(f) for f in test_folds} + if train_set & val_set or train_set & test_set or val_set & test_set: + raise ValueError("train_folds, val_folds, and test_folds must be disjoint") + + train_index: list[int] = [] + val_index: list[int] = [] + test_index: list[int] = [] + for i in range(len(dataset)): + sample = dataset[i] + if "strat_fold" not in sample: + raise KeyError( + "sample is missing 'strat_fold'; run a PTB-XL task that " + "copies strat_fold onto each sample" + ) + fold = int(sample["strat_fold"]) + if fold in train_set: + train_index.append(i) + elif fold in val_set: + val_index.append(i) + elif fold in test_set: + test_index.append(i) + else: + raise ValueError( + f"strat_fold={fold} is not in train/val/test fold sets" + ) + + return ( + dataset.subset(train_index), # type: ignore + dataset.subset(val_index), # type: ignore + dataset.subset(test_index), # type: ignore + ) + + def split_by_sample( dataset: SampleDataset, ratios: Union[Tuple[float, float, float], List[float]], diff --git a/pyhealth/tasks/__init__.py b/pyhealth/tasks/__init__.py index df8411db0..8f30912cb 100644 --- a/pyhealth/tasks/__init__.py +++ b/pyhealth/tasks/__init__.py @@ -88,4 +88,8 @@ def __getattr__(name: str): from .mpf_clinical_prediction import MPFClinicalPredictionTask return MPFClinicalPredictionTask + if name == "PTBXLSuperclassClassification": + from .ptbxl import PTBXLSuperclassClassification + + return PTBXLSuperclassClassification raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/pyhealth/tasks/ptbxl.py b/pyhealth/tasks/ptbxl.py new file mode 100644 index 000000000..4ea5d7bc9 --- /dev/null +++ b/pyhealth/tasks/ptbxl.py @@ -0,0 +1,239 @@ +""" +PyHealth tasks for PTB-XL multi-label ECG diagnosis. + +Dataset link: + https://physionet.org/content/ptb-xl/1.0.3/ + +Dataset paper: (please cite if you use this dataset) + Patrick Wagner, Nils Strodthoff, Ralf-Dieter Bousseljot, Dieter Kreiseler, + Fatima I. Lunze, Wojciech Samek, and Tobias Schaeffter. "PTB-XL, a large + publicly available electrocardiography dataset." Scientific Data 7, 154 + (2020). + +Dataset paper link: + https://www.nature.com/articles/s41597-020-0495-6 + +Author: + AxelNoun (GitHub: @AxelNoun) — external contributor, no NetID + +Description: + Implements the official 5-diagnostic-superclass multi-label task and + aggregation helpers. Separable for a future ``pyhealth.benchmarks`` + package. Official fold splitting lives in + ``pyhealth.datasets.split_by_strat_fold``. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any, ClassVar + +import pandas as pd + +from pyhealth.data import Patient +from pyhealth.tasks.base_task import BaseTask + +# Official diagnostic superclasses (duplicated here to avoid importing the +# dataset package at module import time — that would circular-import via +# ``datasets.__init__`` → ``BaseDataset`` → ``tasks``). +PTBXL_DIAGNOSTIC_SUPERCLASSES = ("NORM", "MI", "STTC", "CD", "HYP") + +# Scientific Data (Wagner et al., Table 9): after aggregating diagnostic +# statements to the 5 superclasses, 407 of 21,799 records have an empty +# label set (mainly pacemaker ECGs with form/rhythm-only annotations). +# Default drop_empty_labels=True matches the common PTB-XL benchmarking +# practice and avoids all-zero multi-hot targets that break most +# conformal / nonconformity scores used in CPBench. +PTBXL_EMPTY_SUPERCLASS_COUNT = 407 +PTBXL_TOTAL_RECORDS = 21799 + + +def load_diagnostic_class_map( + scp_statements_path: str | Path, +) -> dict[str, str]: + """Map SCP acronym → ``diagnostic_class`` for diagnostic statements only. + + Called when constructing :class:`PTBXLSuperclassClassification`. + + Args: + scp_statements_path (str | Path): Path to official ``scp_statements.csv``. + + Returns: + dict[str, str]: Mapping used for 5-superclass aggregation. + + Examples: + >>> # doctest: +SKIP + >>> mapping = load_diagnostic_class_map("/data/ptb-xl/1.0.3/scp_statements.csv") + >>> mapping["NORM"] + 'NORM' + """ + path = Path(scp_statements_path) + if not path.is_file(): + raise FileNotFoundError(f"scp_statements.csv not found: {path}") + df = pd.read_csv(path, index_col=0) + if "diagnostic" not in df.columns or "diagnostic_class" not in df.columns: + raise ValueError( + f"{path} must contain 'diagnostic' and 'diagnostic_class' columns" + ) + diag = df[df["diagnostic"] == 1] + mapping: dict[str, str] = {} + for code, row in diag.iterrows(): + cls = row["diagnostic_class"] + if pd.isna(cls): + continue + mapping[str(code)] = str(cls) + return mapping + + +def aggregate_diagnostic_superclasses( + scp_codes: Any, + diagnostic_class_map: Mapping[str, str], +) -> list[str]: + """Aggregate stringified ``scp_codes`` to unique diagnostic superclasses. + + Includes every SCP key present in the dict (likelihood ``0`` = unknown + confidence still counts). Only codes with ``diagnostic == 1`` in + ``scp_statements.csv`` contribute a superclass. + + Args: + scp_codes (Any): Raw / stringified SCP dict from PTB-XL metadata. + diagnostic_class_map (Mapping[str, str]): From + :func:`load_diagnostic_class_map`. + + Returns: + list[str]: Sorted unique superclass labels (subset of + NORM/MI/STTC/CD/HYP). + + Examples: + >>> aggregate_diagnostic_superclasses( + ... "{'IMI': 80.0, 'SR': 0.0}", {"IMI": "MI"} + ... ) + ['MI'] + """ + from pyhealth.datasets.ptbxl import parse_scp_codes + + codes = parse_scp_codes(scp_codes) + labels = { + diagnostic_class_map[code] + for code in codes + if code in diagnostic_class_map + } + return sorted(labels) + + +class PTBXLSuperclassClassification(BaseTask): + """5-superclass multi-label classification on PTB-XL. + + Labels are the official diagnostic superclasses + ``NORM``, ``MI``, ``STTC``, ``CD``, ``HYP`` obtained by mapping diagnostic + SCP statements via ``scp_statements.csv``. + + Empty label sets + ---------------- + After aggregation, **407 / 21,799** records have no diagnostic superclass + (Scientific Data Table 9; mainly pacemaker ECGs). By default these are + **dropped** (``drop_empty_labels=True``) because an all-zero multi-hot + target breaks typical multi-label nonconformity scores used in CPBench, + and matches common PTB-XL literature practice. + + Each sample also carries ``strat_fold``, ``site``, ``device``, ``age``, + ``sex``, ``age_is_censored``, and ``age_is_missing`` for official splits + and downstream shift evaluations. + + Note: + 71-SCP multi-label classification and age regression are intentionally + not implemented here; add them in this module in a follow-up PR. + + Args: + scp_statements_path (str | Path | None): Path to ``scp_statements.csv``. + Required unless set later via the dataset's ``default_task``. + drop_empty_labels (bool): Drop records with no superclass after + aggregation. Defaults to ``True``. + diagnostic_superclasses (Sequence[str]): Label vocabulary order + (defaults to the official five). + + Examples: + >>> # doctest: +SKIP + >>> from pyhealth.datasets import PTBXLDataset + >>> from pyhealth.tasks import PTBXLSuperclassClassification + >>> ds = PTBXLDataset(root="/data/ptb-xl/1.0.3") + >>> samples = ds.set_task(PTBXLSuperclassClassification( + ... scp_statements_path="/data/ptb-xl/1.0.3/scp_statements.csv" + ... )) + """ + + task_name: str = "PTBXLSuperclassClassification" + input_schema: ClassVar[dict[str, str]] = {"signal": "tensor"} + output_schema: ClassVar[dict[str, str]] = {"labels": "multilabel"} + + def __init__( + self, + scp_statements_path: str | Path | None = None, + drop_empty_labels: bool = True, + diagnostic_superclasses: Sequence[str] = PTBXL_DIAGNOSTIC_SUPERCLASSES, + ) -> None: + self.scp_statements_path = ( + Path(scp_statements_path) if scp_statements_path is not None else None + ) + self.drop_empty_labels = drop_empty_labels + self.diagnostic_superclasses = tuple(diagnostic_superclasses) + self._diagnostic_class_map: dict[str, str] | None = None + super().__init__() + + def _class_map(self) -> dict[str, str]: + if self._diagnostic_class_map is None: + if self.scp_statements_path is None: + raise ValueError( + "scp_statements_path is required. Pass it to " + "PTBXLSuperclassClassification(...) or use " + "PTBXLDataset.default_task." + ) + self._diagnostic_class_map = load_diagnostic_class_map( + self.scp_statements_path + ) + return self._diagnostic_class_map + + def __call__(self, patient: Patient) -> list[dict[str, Any]]: + """Build one sample per ECG record for the patient.""" + from pyhealth.datasets.ptbxl import load_ptbxl_record + + class_map = self._class_map() + samples: list[dict[str, Any]] = [] + for event in patient.get_events(event_type="records"): + labels = aggregate_diagnostic_superclasses(event.scp_codes, class_map) + if not labels and self.drop_empty_labels: + continue + + signal = load_ptbxl_record(event.signal_file) + age_raw = event.age + age_missing = str(getattr(event, "age_is_missing", "0")) in { + "1", + "True", + "true", + } + if age_missing or age_raw is None or str(age_raw).strip() == "": + age: Any = None + else: + age = int(float(age_raw)) + + samples.append( + { + "patient_id": patient.patient_id, + "record_id": str(event.record_id), + "signal": signal, + "labels": labels, + "strat_fold": int(float(event.strat_fold)), + "site": event.site, + "device": event.device, + "age": age, + "age_is_censored": str( + getattr(event, "age_is_censored", "0") + ) + in {"1", "True", "true"}, + "age_is_missing": age_missing, + # sex: 0 = female, 1 = male + "sex": int(float(event.sex)), + } + ) + return samples From 95ec5c47e6180b48d53075e06db5a9acd8aa2618 Mon Sep 17 00:00:00 2001 From: AxelNoun <150222552+AxelNoun@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:23:59 +0200 Subject: [PATCH 03/13] test: add PTB-XL synthetic fixtures and end-to-end coverage Cover helpers, metadata cache keyed by root, event reads via PTBXLDataset, waveform IO (wfdb), and the 5-superclass task path. Co-authored-by: Cursor --- test-resources/ptbxl/README.md | 14 + test-resources/ptbxl/ptbxl_database.csv | 6 + .../ptbxl/records100/00000/00001_lr.dat | Bin 0 -> 1200 bytes .../ptbxl/records100/00000/00001_lr.hea | 13 + .../ptbxl/records100/00000/00002_lr.dat | Bin 0 -> 1200 bytes .../ptbxl/records100/00000/00002_lr.hea | 13 + .../ptbxl/records100/00000/00003_lr.dat | Bin 0 -> 1200 bytes .../ptbxl/records100/00000/00003_lr.hea | 13 + .../ptbxl/records100/00000/00004_lr.dat | Bin 0 -> 1200 bytes .../ptbxl/records100/00000/00004_lr.hea | 13 + .../ptbxl/records100/00000/00005_lr.dat | Bin 0 -> 1200 bytes .../ptbxl/records100/00000/00005_lr.hea | 13 + .../ptbxl/records500/00000/00001_hr.dat | Bin 0 -> 6000 bytes .../ptbxl/records500/00000/00001_hr.hea | 13 + .../ptbxl/records500/00000/00002_hr.dat | Bin 0 -> 6000 bytes .../ptbxl/records500/00000/00002_hr.hea | 13 + .../ptbxl/records500/00000/00003_hr.dat | Bin 0 -> 6000 bytes .../ptbxl/records500/00000/00003_hr.hea | 13 + .../ptbxl/records500/00000/00004_hr.dat | Bin 0 -> 6000 bytes .../ptbxl/records500/00000/00004_hr.hea | 13 + .../ptbxl/records500/00000/00005_hr.dat | Bin 0 -> 6000 bytes .../ptbxl/records500/00000/00005_hr.hea | 13 + test-resources/ptbxl/scp_statements.csv | 9 + tests/core/test_ptbxl.py | 376 ++++++++++++++++++ 24 files changed, 535 insertions(+) create mode 100644 test-resources/ptbxl/README.md create mode 100644 test-resources/ptbxl/ptbxl_database.csv create mode 100644 test-resources/ptbxl/records100/00000/00001_lr.dat create mode 100644 test-resources/ptbxl/records100/00000/00001_lr.hea create mode 100644 test-resources/ptbxl/records100/00000/00002_lr.dat create mode 100644 test-resources/ptbxl/records100/00000/00002_lr.hea create mode 100644 test-resources/ptbxl/records100/00000/00003_lr.dat create mode 100644 test-resources/ptbxl/records100/00000/00003_lr.hea create mode 100644 test-resources/ptbxl/records100/00000/00004_lr.dat create mode 100644 test-resources/ptbxl/records100/00000/00004_lr.hea create mode 100644 test-resources/ptbxl/records100/00000/00005_lr.dat create mode 100644 test-resources/ptbxl/records100/00000/00005_lr.hea create mode 100644 test-resources/ptbxl/records500/00000/00001_hr.dat create mode 100644 test-resources/ptbxl/records500/00000/00001_hr.hea create mode 100644 test-resources/ptbxl/records500/00000/00002_hr.dat create mode 100644 test-resources/ptbxl/records500/00000/00002_hr.hea create mode 100644 test-resources/ptbxl/records500/00000/00003_hr.dat create mode 100644 test-resources/ptbxl/records500/00000/00003_hr.hea create mode 100644 test-resources/ptbxl/records500/00000/00004_hr.dat create mode 100644 test-resources/ptbxl/records500/00000/00004_hr.hea create mode 100644 test-resources/ptbxl/records500/00000/00005_hr.dat create mode 100644 test-resources/ptbxl/records500/00000/00005_hr.hea create mode 100644 test-resources/ptbxl/scp_statements.csv create mode 100644 tests/core/test_ptbxl.py diff --git a/test-resources/ptbxl/README.md b/test-resources/ptbxl/README.md new file mode 100644 index 000000000..e5ab58a1e --- /dev/null +++ b/test-resources/ptbxl/README.md @@ -0,0 +1,14 @@ +# Synthetic PTB-XL-shaped fixture for unit tests (NOT real PhysioNet data). +# +# Contents are programmatically generated: +# - ptbxl_database.csv / scp_statements.csv: tiny CSV stubs +# - records100/*.hea+.dat and records500/*.hea+.dat: synthetic WFDB +# (12 leads x 50 samples at 100 Hz; 12 x 250 at 500 Hz — unequal dims +# so a missing transpose fails the shape assertion) +# +# Edge-case coverage: +# - ecg_id=2: age=300 (HIPAA censored ≥90); likelihood 0 (SR) +# - ecg_id=3: missing age + non-empty non-diagnostic dict (PACE-only) +# - ecg_id=4: true multi-label (IMI+LVH → MI and HYP) +# - ecg_id=5: empty scp_codes dict {} +# - ecg_id=1 and 2 share patient_id (patient-level fold leakage checks) diff --git a/test-resources/ptbxl/ptbxl_database.csv b/test-resources/ptbxl/ptbxl_database.csv new file mode 100644 index 000000000..b72999108 --- /dev/null +++ b/test-resources/ptbxl/ptbxl_database.csv @@ -0,0 +1,6 @@ +ecg_id,patient_id,age,sex,site,device,recording_date,scp_codes,strat_fold,filename_lr,filename_hr +1,15709.0,65.0,1,0,CS-12,1990-01-01 10:00:00,"{'NORM': 100.0}",1,records100/00000/00001_lr,records500/00000/00001_hr +2,15709.0,300.0,0,1,CS-12,1991-06-15 11:30:00,"{'IMI': 80.0, 'SR': 0.0}",9,records100/00000/00002_lr,records500/00000/00002_hr +3,99.0,,1,2,CS-12,1992-03-20 09:15:00,"{'PACE': 100.0}",10,records100/00000/00003_lr,records500/00000/00003_hr +4,42.0,55.0,0,0,CS-12,1993-01-01 08:00:00,"{'IMI': 100.0, 'LVH': 80.0}",2,records100/00000/00004_lr,records500/00000/00004_hr +5,7.0,40.0,1,1,CS-12,1994-01-01 08:00:00,"{}",3,records100/00000/00005_lr,records500/00000/00005_hr diff --git a/test-resources/ptbxl/records100/00000/00001_lr.dat b/test-resources/ptbxl/records100/00000/00001_lr.dat new file mode 100644 index 0000000000000000000000000000000000000000..5f5a9a203530337c5b546b833f7bbae83de6b4fe GIT binary patch literal 1200 zcmZ|K*H%I?6h+a6qN1X75TsY>AlUu??=t4Ll#|yvGEVk_92Z=Y=Zb4?C~(Ui_W>=6 zZ;Lz#7yk9pyBlfA`7P11wIS1rvCPzJy4TiPT;Xb6xB1#o#gaAy+R|+$%2cRQqfUb+ zEuK`dr0sxqgbQstJbQN|?K;0D+Osxf+BcS&I!O20I*cn^9qBe-$EsM;NkFH%txJzS z1BQ$kGhxb16-zn~=t8(KXTghiH`1l^TcRs#L#AtEnW>v}udUm-!quH_^L4L^B|QZ6 zsN0sTShHcvjy(sCysBbJPXRp(7fzgc^X^9a?);YMhqWQoPh**>U+G?3zvBv5e{`F# Gzx)9sjYd%b literal 0 HcmV?d00001 diff --git a/test-resources/ptbxl/records100/00000/00001_lr.hea b/test-resources/ptbxl/records100/00000/00001_lr.hea new file mode 100644 index 000000000..203e612da --- /dev/null +++ b/test-resources/ptbxl/records100/00000/00001_lr.hea @@ -0,0 +1,13 @@ +00001_lr 12 100 50 +00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 +00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 +00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 +00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 +00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 +00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 +00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 +00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 +00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 +00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 +00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 +00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records100/00000/00002_lr.dat b/test-resources/ptbxl/records100/00000/00002_lr.dat new file mode 100644 index 0000000000000000000000000000000000000000..5f5a9a203530337c5b546b833f7bbae83de6b4fe GIT binary patch literal 1200 zcmZ|K*H%I?6h+a6qN1X75TsY>AlUu??=t4Ll#|yvGEVk_92Z=Y=Zb4?C~(Ui_W>=6 zZ;Lz#7yk9pyBlfA`7P11wIS1rvCPzJy4TiPT;Xb6xB1#o#gaAy+R|+$%2cRQqfUb+ zEuK`dr0sxqgbQstJbQN|?K;0D+Osxf+BcS&I!O20I*cn^9qBe-$EsM;NkFH%txJzS z1BQ$kGhxb16-zn~=t8(KXTghiH`1l^TcRs#L#AtEnW>v}udUm-!quH_^L4L^B|QZ6 zsN0sTShHcvjy(sCysBbJPXRp(7fzgc^X^9a?);YMhqWQoPh**>U+G?3zvBv5e{`F# Gzx)9sjYd%b literal 0 HcmV?d00001 diff --git a/test-resources/ptbxl/records100/00000/00002_lr.hea b/test-resources/ptbxl/records100/00000/00002_lr.hea new file mode 100644 index 000000000..f998f2254 --- /dev/null +++ b/test-resources/ptbxl/records100/00000/00002_lr.hea @@ -0,0 +1,13 @@ +00002_lr 12 100 50 +00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 +00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 +00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 +00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 +00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 +00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 +00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 +00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 +00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 +00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 +00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 +00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records100/00000/00003_lr.dat b/test-resources/ptbxl/records100/00000/00003_lr.dat new file mode 100644 index 0000000000000000000000000000000000000000..5f5a9a203530337c5b546b833f7bbae83de6b4fe GIT binary patch literal 1200 zcmZ|K*H%I?6h+a6qN1X75TsY>AlUu??=t4Ll#|yvGEVk_92Z=Y=Zb4?C~(Ui_W>=6 zZ;Lz#7yk9pyBlfA`7P11wIS1rvCPzJy4TiPT;Xb6xB1#o#gaAy+R|+$%2cRQqfUb+ zEuK`dr0sxqgbQstJbQN|?K;0D+Osxf+BcS&I!O20I*cn^9qBe-$EsM;NkFH%txJzS z1BQ$kGhxb16-zn~=t8(KXTghiH`1l^TcRs#L#AtEnW>v}udUm-!quH_^L4L^B|QZ6 zsN0sTShHcvjy(sCysBbJPXRp(7fzgc^X^9a?);YMhqWQoPh**>U+G?3zvBv5e{`F# Gzx)9sjYd%b literal 0 HcmV?d00001 diff --git a/test-resources/ptbxl/records100/00000/00003_lr.hea b/test-resources/ptbxl/records100/00000/00003_lr.hea new file mode 100644 index 000000000..717cb6d98 --- /dev/null +++ b/test-resources/ptbxl/records100/00000/00003_lr.hea @@ -0,0 +1,13 @@ +00003_lr 12 100 50 +00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 +00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 +00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 +00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 +00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 +00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 +00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 +00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 +00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 +00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 +00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 +00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records100/00000/00004_lr.dat b/test-resources/ptbxl/records100/00000/00004_lr.dat new file mode 100644 index 0000000000000000000000000000000000000000..5f5a9a203530337c5b546b833f7bbae83de6b4fe GIT binary patch literal 1200 zcmZ|K*H%I?6h+a6qN1X75TsY>AlUu??=t4Ll#|yvGEVk_92Z=Y=Zb4?C~(Ui_W>=6 zZ;Lz#7yk9pyBlfA`7P11wIS1rvCPzJy4TiPT;Xb6xB1#o#gaAy+R|+$%2cRQqfUb+ zEuK`dr0sxqgbQstJbQN|?K;0D+Osxf+BcS&I!O20I*cn^9qBe-$EsM;NkFH%txJzS z1BQ$kGhxb16-zn~=t8(KXTghiH`1l^TcRs#L#AtEnW>v}udUm-!quH_^L4L^B|QZ6 zsN0sTShHcvjy(sCysBbJPXRp(7fzgc^X^9a?);YMhqWQoPh**>U+G?3zvBv5e{`F# Gzx)9sjYd%b literal 0 HcmV?d00001 diff --git a/test-resources/ptbxl/records100/00000/00004_lr.hea b/test-resources/ptbxl/records100/00000/00004_lr.hea new file mode 100644 index 000000000..3c0edc5b4 --- /dev/null +++ b/test-resources/ptbxl/records100/00000/00004_lr.hea @@ -0,0 +1,13 @@ +00004_lr 12 100 50 +00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 +00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 +00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 +00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 +00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 +00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 +00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 +00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 +00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 +00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 +00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 +00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records100/00000/00005_lr.dat b/test-resources/ptbxl/records100/00000/00005_lr.dat new file mode 100644 index 0000000000000000000000000000000000000000..5f5a9a203530337c5b546b833f7bbae83de6b4fe GIT binary patch literal 1200 zcmZ|K*H%I?6h+a6qN1X75TsY>AlUu??=t4Ll#|yvGEVk_92Z=Y=Zb4?C~(Ui_W>=6 zZ;Lz#7yk9pyBlfA`7P11wIS1rvCPzJy4TiPT;Xb6xB1#o#gaAy+R|+$%2cRQqfUb+ zEuK`dr0sxqgbQstJbQN|?K;0D+Osxf+BcS&I!O20I*cn^9qBe-$EsM;NkFH%txJzS z1BQ$kGhxb16-zn~=t8(KXTghiH`1l^TcRs#L#AtEnW>v}udUm-!quH_^L4L^B|QZ6 zsN0sTShHcvjy(sCysBbJPXRp(7fzgc^X^9a?);YMhqWQoPh**>U+G?3zvBv5e{`F# Gzx)9sjYd%b literal 0 HcmV?d00001 diff --git a/test-resources/ptbxl/records100/00000/00005_lr.hea b/test-resources/ptbxl/records100/00000/00005_lr.hea new file mode 100644 index 000000000..2f42af92d --- /dev/null +++ b/test-resources/ptbxl/records100/00000/00005_lr.hea @@ -0,0 +1,13 @@ +00005_lr 12 100 50 +00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 +00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 +00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 +00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 +00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 +00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 +00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 +00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 +00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 +00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 +00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 +00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records500/00000/00001_hr.dat b/test-resources/ptbxl/records500/00000/00001_hr.dat new file mode 100644 index 0000000000000000000000000000000000000000..69632dc434819f9da0e901ad91cd25621420fc9f GIT binary patch literal 6000 zcmZ|KcNouA7>D8KhqgjeQAjD-L^e^#h|CfZMP!q*iU>tRHj$!8Lqa68$jFFnBFc&i zNo04N?>h6}`#RTqJ`bcwLt4^t1y^zvS91;5a-B!hJKv_~ddG#!dV_U0&VrH_+RS-y+SfZAdhSvP{*S@m^i;2rEp@CEIMhQxv=OE|2DxZMT!19OUE(KB77Qy)EhO7)Ei8&%TEwG8 zWm{hIk)Hw-q!5KELQ(D$#V#%8(c+E^#VF4G*4;o$7{5hYQrnPdDP@_erQ^N2mI*6N zEi2n>EhmayTHd1-WLpVJQi{@)p)BPnPX!(j#V)Pr(MpaB6{*C7*4;oKGJcD+vbG`7 zD#|idtHyhEeK@QzwVG_RwYn&FX$_Colx+`DnJQG}VX9G`8r0+wQS8!M9b3; zYTXUAj`3Thb+rwN)>D?LT0h>aYlE=D)P}On*2hG#OCR@WBiUAmy40gS4QR+?JWeB? z5XCNS?9nEU3yo>Qlh)lpn;O4G+DzM!=u^rvRh!3qb!`z=nEJGAv-KHK?9!GVZ6({9 z(u}8QP79vq8CuecXGO6~TYI#P<3elN@SJrw(6+{Jk+#z|B-&nCrfP?HudW@#3R64D zHd{N3VwZODXjj?RmUgtK10Cr^XS&dp=S8tgyLt2l$AxaZz>C)1K)V~iMcPB#kZ4b3 znX0|wy}I@eD@=V!w%Pi!D0XQdkM@;q-RVJ3deNJgc$q%*z%;n6o`+W-bKh}U?X!3^OI-sCM& z?9#VAI@EFDZHDrWbvMvq#&3}h*ES?NLRqHj$at@=qrwVPN6R)_$B1H=j`ip`**1*f zj9?_A7|j^QGLCmeu}jB$bb{l;cqZ_kbvMw7#&40nuWd;517(@2lj6O)P7W(f{ZO{q z`jIGh=@gGnm2DGwpAVSCWIp60rZAO{MX^h#dGr&oGnvI~z7oYQo#WBDjtg^`%h%T3 zK<631Mf#1lA<_BDGF2DEdv#qHR+zd-w%NK^6uWeZN57SA^Z17OEMOsvSj-Z>^mH_(m7Z;@`&HYB=PS*Gfic(1No!wOTk$u?WJ zi(;4V@aRt2wvkP2W(!-{#&&kFlb=PgOLuv6x8uStcJqsMH_$!CZ;|fRHYED1vP{+A z;=Q`=3oA_BFWYQAAc|dj(4&WB+aC7vE5EUi{T$#RhxlC-yY#R}k2o$I<_LdScLV*? z_$|`Ev<-Y9S$6|HWBeBBS#3k2=aglto{#tHdLgVZ^`dOE^^z!dDJi$6QkUD#aF%nN Y=K>eG1gTUiP0FolQ?9pZQ|VIw18Vc+%K!iX literal 0 HcmV?d00001 diff --git a/test-resources/ptbxl/records500/00000/00001_hr.hea b/test-resources/ptbxl/records500/00000/00001_hr.hea new file mode 100644 index 000000000..e7e69164f --- /dev/null +++ b/test-resources/ptbxl/records500/00000/00001_hr.hea @@ -0,0 +1,13 @@ +00001_hr 12 500 250 +00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 +00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 +00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 +00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 +00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 +00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 +00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 +00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 +00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 +00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 +00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 +00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records500/00000/00002_hr.dat b/test-resources/ptbxl/records500/00000/00002_hr.dat new file mode 100644 index 0000000000000000000000000000000000000000..69632dc434819f9da0e901ad91cd25621420fc9f GIT binary patch literal 6000 zcmZ|KcNouA7>D8KhqgjeQAjD-L^e^#h|CfZMP!q*iU>tRHj$!8Lqa68$jFFnBFc&i zNo04N?>h6}`#RTqJ`bcwLt4^t1y^zvS91;5a-B!hJKv_~ddG#!dV_U0&VrH_+RS-y+SfZAdhSvP{*S@m^i;2rEp@CEIMhQxv=OE|2DxZMT!19OUE(KB77Qy)EhO7)Ei8&%TEwG8 zWm{hIk)Hw-q!5KELQ(D$#V#%8(c+E^#VF4G*4;o$7{5hYQrnPdDP@_erQ^N2mI*6N zEi2n>EhmayTHd1-WLpVJQi{@)p)BPnPX!(j#V)Pr(MpaB6{*C7*4;oKGJcD+vbG`7 zD#|idtHyhEeK@QzwVG_RwYn&FX$_Colx+`DnJQG}VX9G`8r0+wQS8!M9b3; zYTXUAj`3Thb+rwN)>D?LT0h>aYlE=D)P}On*2hG#OCR@WBiUAmy40gS4QR+?JWeB? z5XCNS?9nEU3yo>Qlh)lpn;O4G+DzM!=u^rvRh!3qb!`z=nEJGAv-KHK?9!GVZ6({9 z(u}8QP79vq8CuecXGO6~TYI#P<3elN@SJrw(6+{Jk+#z|B-&nCrfP?HudW@#3R64D zHd{N3VwZODXjj?RmUgtK10Cr^XS&dp=S8tgyLt2l$AxaZz>C)1K)V~iMcPB#kZ4b3 znX0|wy}I@eD@=V!w%Pi!D0XQdkM@;q-RVJ3deNJgc$q%*z%;n6o`+W-bKh}U?X!3^OI-sCM& z?9#VAI@EFDZHDrWbvMvq#&3}h*ES?NLRqHj$at@=qrwVPN6R)_$B1H=j`ip`**1*f zj9?_A7|j^QGLCmeu}jB$bb{l;cqZ_kbvMw7#&40nuWd;517(@2lj6O)P7W(f{ZO{q z`jIGh=@gGnm2DGwpAVSCWIp60rZAO{MX^h#dGr&oGnvI~z7oYQo#WBDjtg^`%h%T3 zK<631Mf#1lA<_BDGF2DEdv#qHR+zd-w%NK^6uWeZN57SA^Z17OEMOsvSj-Z>^mH_(m7Z;@`&HYB=PS*Gfic(1No!wOTk$u?WJ zi(;4V@aRt2wvkP2W(!-{#&&kFlb=PgOLuv6x8uStcJqsMH_$!CZ;|fRHYED1vP{+A z;=Q`=3oA_BFWYQAAc|dj(4&WB+aC7vE5EUi{T$#RhxlC-yY#R}k2o$I<_LdScLV*? z_$|`Ev<-Y9S$6|HWBeBBS#3k2=aglto{#tHdLgVZ^`dOE^^z!dDJi$6QkUD#aF%nN Y=K>eG1gTUiP0FolQ?9pZQ|VIw18Vc+%K!iX literal 0 HcmV?d00001 diff --git a/test-resources/ptbxl/records500/00000/00002_hr.hea b/test-resources/ptbxl/records500/00000/00002_hr.hea new file mode 100644 index 000000000..23e343c08 --- /dev/null +++ b/test-resources/ptbxl/records500/00000/00002_hr.hea @@ -0,0 +1,13 @@ +00002_hr 12 500 250 +00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 +00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 +00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 +00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 +00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 +00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 +00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 +00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 +00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 +00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 +00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 +00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records500/00000/00003_hr.dat b/test-resources/ptbxl/records500/00000/00003_hr.dat new file mode 100644 index 0000000000000000000000000000000000000000..69632dc434819f9da0e901ad91cd25621420fc9f GIT binary patch literal 6000 zcmZ|KcNouA7>D8KhqgjeQAjD-L^e^#h|CfZMP!q*iU>tRHj$!8Lqa68$jFFnBFc&i zNo04N?>h6}`#RTqJ`bcwLt4^t1y^zvS91;5a-B!hJKv_~ddG#!dV_U0&VrH_+RS-y+SfZAdhSvP{*S@m^i;2rEp@CEIMhQxv=OE|2DxZMT!19OUE(KB77Qy)EhO7)Ei8&%TEwG8 zWm{hIk)Hw-q!5KELQ(D$#V#%8(c+E^#VF4G*4;o$7{5hYQrnPdDP@_erQ^N2mI*6N zEi2n>EhmayTHd1-WLpVJQi{@)p)BPnPX!(j#V)Pr(MpaB6{*C7*4;oKGJcD+vbG`7 zD#|idtHyhEeK@QzwVG_RwYn&FX$_Colx+`DnJQG}VX9G`8r0+wQS8!M9b3; zYTXUAj`3Thb+rwN)>D?LT0h>aYlE=D)P}On*2hG#OCR@WBiUAmy40gS4QR+?JWeB? z5XCNS?9nEU3yo>Qlh)lpn;O4G+DzM!=u^rvRh!3qb!`z=nEJGAv-KHK?9!GVZ6({9 z(u}8QP79vq8CuecXGO6~TYI#P<3elN@SJrw(6+{Jk+#z|B-&nCrfP?HudW@#3R64D zHd{N3VwZODXjj?RmUgtK10Cr^XS&dp=S8tgyLt2l$AxaZz>C)1K)V~iMcPB#kZ4b3 znX0|wy}I@eD@=V!w%Pi!D0XQdkM@;q-RVJ3deNJgc$q%*z%;n6o`+W-bKh}U?X!3^OI-sCM& z?9#VAI@EFDZHDrWbvMvq#&3}h*ES?NLRqHj$at@=qrwVPN6R)_$B1H=j`ip`**1*f zj9?_A7|j^QGLCmeu}jB$bb{l;cqZ_kbvMw7#&40nuWd;517(@2lj6O)P7W(f{ZO{q z`jIGh=@gGnm2DGwpAVSCWIp60rZAO{MX^h#dGr&oGnvI~z7oYQo#WBDjtg^`%h%T3 zK<631Mf#1lA<_BDGF2DEdv#qHR+zd-w%NK^6uWeZN57SA^Z17OEMOsvSj-Z>^mH_(m7Z;@`&HYB=PS*Gfic(1No!wOTk$u?WJ zi(;4V@aRt2wvkP2W(!-{#&&kFlb=PgOLuv6x8uStcJqsMH_$!CZ;|fRHYED1vP{+A z;=Q`=3oA_BFWYQAAc|dj(4&WB+aC7vE5EUi{T$#RhxlC-yY#R}k2o$I<_LdScLV*? z_$|`Ev<-Y9S$6|HWBeBBS#3k2=aglto{#tHdLgVZ^`dOE^^z!dDJi$6QkUD#aF%nN Y=K>eG1gTUiP0FolQ?9pZQ|VIw18Vc+%K!iX literal 0 HcmV?d00001 diff --git a/test-resources/ptbxl/records500/00000/00003_hr.hea b/test-resources/ptbxl/records500/00000/00003_hr.hea new file mode 100644 index 000000000..51105c487 --- /dev/null +++ b/test-resources/ptbxl/records500/00000/00003_hr.hea @@ -0,0 +1,13 @@ +00003_hr 12 500 250 +00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 +00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 +00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 +00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 +00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 +00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 +00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 +00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 +00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 +00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 +00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 +00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records500/00000/00004_hr.dat b/test-resources/ptbxl/records500/00000/00004_hr.dat new file mode 100644 index 0000000000000000000000000000000000000000..69632dc434819f9da0e901ad91cd25621420fc9f GIT binary patch literal 6000 zcmZ|KcNouA7>D8KhqgjeQAjD-L^e^#h|CfZMP!q*iU>tRHj$!8Lqa68$jFFnBFc&i zNo04N?>h6}`#RTqJ`bcwLt4^t1y^zvS91;5a-B!hJKv_~ddG#!dV_U0&VrH_+RS-y+SfZAdhSvP{*S@m^i;2rEp@CEIMhQxv=OE|2DxZMT!19OUE(KB77Qy)EhO7)Ei8&%TEwG8 zWm{hIk)Hw-q!5KELQ(D$#V#%8(c+E^#VF4G*4;o$7{5hYQrnPdDP@_erQ^N2mI*6N zEi2n>EhmayTHd1-WLpVJQi{@)p)BPnPX!(j#V)Pr(MpaB6{*C7*4;oKGJcD+vbG`7 zD#|idtHyhEeK@QzwVG_RwYn&FX$_Colx+`DnJQG}VX9G`8r0+wQS8!M9b3; zYTXUAj`3Thb+rwN)>D?LT0h>aYlE=D)P}On*2hG#OCR@WBiUAmy40gS4QR+?JWeB? z5XCNS?9nEU3yo>Qlh)lpn;O4G+DzM!=u^rvRh!3qb!`z=nEJGAv-KHK?9!GVZ6({9 z(u}8QP79vq8CuecXGO6~TYI#P<3elN@SJrw(6+{Jk+#z|B-&nCrfP?HudW@#3R64D zHd{N3VwZODXjj?RmUgtK10Cr^XS&dp=S8tgyLt2l$AxaZz>C)1K)V~iMcPB#kZ4b3 znX0|wy}I@eD@=V!w%Pi!D0XQdkM@;q-RVJ3deNJgc$q%*z%;n6o`+W-bKh}U?X!3^OI-sCM& z?9#VAI@EFDZHDrWbvMvq#&3}h*ES?NLRqHj$at@=qrwVPN6R)_$B1H=j`ip`**1*f zj9?_A7|j^QGLCmeu}jB$bb{l;cqZ_kbvMw7#&40nuWd;517(@2lj6O)P7W(f{ZO{q z`jIGh=@gGnm2DGwpAVSCWIp60rZAO{MX^h#dGr&oGnvI~z7oYQo#WBDjtg^`%h%T3 zK<631Mf#1lA<_BDGF2DEdv#qHR+zd-w%NK^6uWeZN57SA^Z17OEMOsvSj-Z>^mH_(m7Z;@`&HYB=PS*Gfic(1No!wOTk$u?WJ zi(;4V@aRt2wvkP2W(!-{#&&kFlb=PgOLuv6x8uStcJqsMH_$!CZ;|fRHYED1vP{+A z;=Q`=3oA_BFWYQAAc|dj(4&WB+aC7vE5EUi{T$#RhxlC-yY#R}k2o$I<_LdScLV*? z_$|`Ev<-Y9S$6|HWBeBBS#3k2=aglto{#tHdLgVZ^`dOE^^z!dDJi$6QkUD#aF%nN Y=K>eG1gTUiP0FolQ?9pZQ|VIw18Vc+%K!iX literal 0 HcmV?d00001 diff --git a/test-resources/ptbxl/records500/00000/00004_hr.hea b/test-resources/ptbxl/records500/00000/00004_hr.hea new file mode 100644 index 000000000..768d38e2e --- /dev/null +++ b/test-resources/ptbxl/records500/00000/00004_hr.hea @@ -0,0 +1,13 @@ +00004_hr 12 500 250 +00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 +00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 +00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 +00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 +00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 +00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 +00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 +00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 +00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 +00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 +00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 +00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records500/00000/00005_hr.dat b/test-resources/ptbxl/records500/00000/00005_hr.dat new file mode 100644 index 0000000000000000000000000000000000000000..69632dc434819f9da0e901ad91cd25621420fc9f GIT binary patch literal 6000 zcmZ|KcNouA7>D8KhqgjeQAjD-L^e^#h|CfZMP!q*iU>tRHj$!8Lqa68$jFFnBFc&i zNo04N?>h6}`#RTqJ`bcwLt4^t1y^zvS91;5a-B!hJKv_~ddG#!dV_U0&VrH_+RS-y+SfZAdhSvP{*S@m^i;2rEp@CEIMhQxv=OE|2DxZMT!19OUE(KB77Qy)EhO7)Ei8&%TEwG8 zWm{hIk)Hw-q!5KELQ(D$#V#%8(c+E^#VF4G*4;o$7{5hYQrnPdDP@_erQ^N2mI*6N zEi2n>EhmayTHd1-WLpVJQi{@)p)BPnPX!(j#V)Pr(MpaB6{*C7*4;oKGJcD+vbG`7 zD#|idtHyhEeK@QzwVG_RwYn&FX$_Colx+`DnJQG}VX9G`8r0+wQS8!M9b3; zYTXUAj`3Thb+rwN)>D?LT0h>aYlE=D)P}On*2hG#OCR@WBiUAmy40gS4QR+?JWeB? z5XCNS?9nEU3yo>Qlh)lpn;O4G+DzM!=u^rvRh!3qb!`z=nEJGAv-KHK?9!GVZ6({9 z(u}8QP79vq8CuecXGO6~TYI#P<3elN@SJrw(6+{Jk+#z|B-&nCrfP?HudW@#3R64D zHd{N3VwZODXjj?RmUgtK10Cr^XS&dp=S8tgyLt2l$AxaZz>C)1K)V~iMcPB#kZ4b3 znX0|wy}I@eD@=V!w%Pi!D0XQdkM@;q-RVJ3deNJgc$q%*z%;n6o`+W-bKh}U?X!3^OI-sCM& z?9#VAI@EFDZHDrWbvMvq#&3}h*ES?NLRqHj$at@=qrwVPN6R)_$B1H=j`ip`**1*f zj9?_A7|j^QGLCmeu}jB$bb{l;cqZ_kbvMw7#&40nuWd;517(@2lj6O)P7W(f{ZO{q z`jIGh=@gGnm2DGwpAVSCWIp60rZAO{MX^h#dGr&oGnvI~z7oYQo#WBDjtg^`%h%T3 zK<631Mf#1lA<_BDGF2DEdv#qHR+zd-w%NK^6uWeZN57SA^Z17OEMOsvSj-Z>^mH_(m7Z;@`&HYB=PS*Gfic(1No!wOTk$u?WJ zi(;4V@aRt2wvkP2W(!-{#&&kFlb=PgOLuv6x8uStcJqsMH_$!CZ;|fRHYED1vP{+A z;=Q`=3oA_BFWYQAAc|dj(4&WB+aC7vE5EUi{T$#RhxlC-yY#R}k2o$I<_LdScLV*? z_$|`Ev<-Y9S$6|HWBeBBS#3k2=aglto{#tHdLgVZ^`dOE^^z!dDJi$6QkUD#aF%nN Y=K>eG1gTUiP0FolQ?9pZQ|VIw18Vc+%K!iX literal 0 HcmV?d00001 diff --git a/test-resources/ptbxl/records500/00000/00005_hr.hea b/test-resources/ptbxl/records500/00000/00005_hr.hea new file mode 100644 index 000000000..e7ab1b3f2 --- /dev/null +++ b/test-resources/ptbxl/records500/00000/00005_hr.hea @@ -0,0 +1,13 @@ +00005_hr 12 500 250 +00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 +00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 +00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 +00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 +00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 +00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 +00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 +00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 +00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 +00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 +00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 +00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/scp_statements.csv b/test-resources/ptbxl/scp_statements.csv new file mode 100644 index 000000000..2e08b8d55 --- /dev/null +++ b/test-resources/ptbxl/scp_statements.csv @@ -0,0 +1,9 @@ +,description,diagnostic,form,rhythm,diagnostic_class,diagnostic_subclass,Statement Category,SCP-ECG Statement Description,AHA code,aECG REFID,CDISC Code,DICOM Code +NORM,normal ECG,1.0,,,NORM,NORM,Normal/abnormal,normal ECG,1,,,F-000B7 +IMI,inferior myocardial infarction,1.0,,,MI,IMI,Myocardial Infarction,inferior myocardial infarction,161,,, +LVH,left ventricular hypertrophy,1.0,,,HYP,LVH,Ventricular Hypertrophy,left ventricular hypertrophy,142,,C71076, +SR,sinus rhythm,,,1.0,,,Statements related to impulse formation (abnormalities),sinus rhythm,20,MDC_ECG_RHY_SINUS_RHY,, +PACE,normal functioning artificial pacemaker,,,1.0,,,Pacemaker types and pacemaker function,normal functioning artificial pacemaker,,,, +STTC,non-specific ST-T changes,1.0,,,STTC,STTC,other ST-T descriptive statements,non-specific ST-T changes,,,, +CD,conduction disturbance,1.0,,,CD,CD,Intraventricular and intra-atrial Conduction disturbances,conduction disturbance,,,, +HYP,hypertrophy,1.0,,,HYP,HYP,Ventricular Hypertrophy,hypertrophy,,,, diff --git a/tests/core/test_ptbxl.py b/tests/core/test_ptbxl.py new file mode 100644 index 000000000..5d29a9c9f --- /dev/null +++ b/tests/core/test_ptbxl.py @@ -0,0 +1,376 @@ +""" +Unit tests for PTBXLDataset, PTBXLSuperclassClassification, and +split_by_strat_fold. + +Uses small synthetic WFDB fixtures under test-resources/ptbxl/ (not real +PhysioNet records). Covers censored age (300), missing age, and empty +diagnostic-superclass labels. + +Author: + AxelNoun (GitHub: @AxelNoun) — external contributor, no NetID +""" + +from __future__ import annotations + +import shutil +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import numpy as np +import pandas as pd + +from pyhealth.datasets.ptbxl import ( + AGE_CENSOR_SENTINEL, + PTBXLDataset, + format_patient_id, + is_age_censored, + is_age_missing, + load_ptbxl_record, + metadata_filename, + parse_scp_codes, +) +from pyhealth.datasets.splitter import split_by_strat_fold +from pyhealth.tasks.ptbxl import ( + PTBXL_EMPTY_SUPERCLASS_COUNT, + PTBXLSuperclassClassification, + aggregate_diagnostic_superclasses, + load_diagnostic_class_map, +) + +FIXTURE_ROOT = Path(__file__).resolve().parents[1] / ".." / "test-resources" / "ptbxl" +FIXTURE_ROOT = FIXTURE_ROOT.resolve() + + +def _write_dummy_wfdb( + record_base: Path, + n_leads: int = 12, + n_samples: int = 50, + fs: int = 100, +) -> None: + """Write a minimal WFDB record (header + int16 dat) for tests.""" + record_base.parent.mkdir(parents=True, exist_ok=True) + data = np.zeros((n_samples, n_leads), dtype=np.int16) + for lead in range(n_leads): + data[:, lead] = np.arange(n_samples, dtype=np.int16) + lead + Path(str(record_base) + ".dat").write_bytes(data.tobytes()) + name = record_base.name + lines = [f"{name} {n_leads} {fs} {n_samples}"] + for i in range(n_leads): + lines.append(f"{name}.dat 16 1000.0(0)/uV 16 0 0 0 0 {i}") + Path(str(record_base) + ".hea").write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def _materialize_fixture(dest: Path) -> Path: + """Copy committed CSVs and WFDB records under ``dest``.""" + dest.mkdir(parents=True, exist_ok=True) + shutil.copy(FIXTURE_ROOT / "ptbxl_database.csv", dest / "ptbxl_database.csv") + shutil.copy(FIXTURE_ROOT / "scp_statements.csv", dest / "scp_statements.csv") + for records_dir in ("records100", "records500"): + src = FIXTURE_ROOT / records_dir + if src.is_dir(): + shutil.copytree(src, dest / records_dir, dirs_exist_ok=True) + # Fallback: synthesize WFDB if committed waveforms are missing. + db = pd.read_csv(dest / "ptbxl_database.csv") + for col, fs, n_samples in ( + ("filename_lr", 100, 50), + ("filename_hr", 500, 250), + ): + for rel in db[col]: + base = dest / str(rel) + if not Path(str(base) + ".hea").is_file(): + _write_dummy_wfdb(base, n_samples=n_samples, fs=fs) + return dest + + +class TestPTBXLHelpers(unittest.TestCase): + def test_format_patient_id_strips_float(self): + self.assertEqual(format_patient_id(15709.0), "15709") + self.assertEqual(format_patient_id("15709.0"), "15709") + + def test_format_patient_id_includes_ecg_id_in_errors(self): + with self.assertRaisesRegex(ValueError, r"ecg_id=99"): + format_patient_id(None, ecg_id=99) + with self.assertRaisesRegex(ValueError, r"ecg_id=7"): + format_patient_id("not-a-number", ecg_id=7) + + def test_parse_scp_codes_keeps_likelihood_zero(self): + codes = parse_scp_codes("{'IMI': 80.0, 'SR': 0.0}") + self.assertEqual(codes["SR"], 0.0) + self.assertIn("IMI", codes) + + def test_parse_scp_codes_empty(self): + self.assertEqual(parse_scp_codes("{}"), {}) + self.assertEqual(parse_scp_codes(None), {}) + + def test_age_missing_vs_censored(self): + self.assertTrue(is_age_censored(AGE_CENSOR_SENTINEL)) + self.assertFalse(is_age_missing(AGE_CENSOR_SENTINEL)) + self.assertTrue(is_age_missing(float("nan"))) + self.assertTrue(is_age_missing("")) + self.assertFalse(is_age_censored(float("nan"))) + self.assertFalse(is_age_censored(65)) + + def test_metadata_filename_includes_rate_and_root(self): + name_a = metadata_filename(100, "/data/ptb-xl/a") + name_b = metadata_filename(100, "/data/ptb-xl/b") + self.assertTrue(name_a.startswith("ptbxl-pyhealth-100hz-")) + self.assertTrue(name_a.endswith(".csv")) + self.assertNotEqual(name_a, name_b) + self.assertNotEqual( + metadata_filename(100, "/tmp/x"), metadata_filename(500, "/tmp/x") + ) + + def test_empty_superclass_count_documented(self): + self.assertEqual(PTBXL_EMPTY_SUPERCLASS_COUNT, 407) + + +class TestPTBXLAggregation(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.class_map = load_diagnostic_class_map(FIXTURE_ROOT / "scp_statements.csv") + + def test_aggregate_superclasses(self): + labels = aggregate_diagnostic_superclasses( + "{'NORM': 100.0}", self.class_map + ) + self.assertEqual(labels, ["NORM"]) + + labels = aggregate_diagnostic_superclasses( + "{'IMI': 80.0, 'SR': 0.0}", self.class_map + ) + self.assertEqual(labels, ["MI"]) + + # Non-empty dict with no diagnostic statements (PACE is rhythm-only). + labels = aggregate_diagnostic_superclasses( + "{'PACE': 100.0}", self.class_map + ) + self.assertEqual(labels, []) + + # Empty dict — distinct path from PACE-only. + labels = aggregate_diagnostic_superclasses("{}", self.class_map) + self.assertEqual(labels, []) + + # True multi-label: two distinct diagnostic superclasses. + labels = aggregate_diagnostic_superclasses( + "{'IMI': 100.0, 'LVH': 80.0}", self.class_map + ) + self.assertEqual(labels, ["HYP", "MI"]) + + +class TestPTBXLDatasetMetadata(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()) + self.data_root = _materialize_fixture(self.tmp / "data") + self.cache_dir = self.tmp / "meta_cache" + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_prepare_metadata_writes_cache_not_root(self): + ds = PTBXLDataset( + root=str(self.data_root), + metadata_cache_dir=self.cache_dir, + sampling_rate=100, + cache_dir=self.tmp / "pyhealth_cache", + ) + csv_path = self.cache_dir / ds.metadata_file_name + self.assertTrue(csv_path.is_file()) + self.assertFalse((self.data_root / ds.metadata_file_name).exists()) + meta = pd.read_csv(csv_path) + self.assertEqual( + sorted(meta["patient_id"].astype(str).unique()), + ["15709", "42", "7", "99"], + ) + self.assertNotIn("15709.0", set(meta["patient_id"].astype(str))) + self.assertEqual(len(meta), 5) + # Age flags: record 2 censored, record 3 missing + row2 = meta[meta["record_id"].astype(str) == "2"].iloc[0] + row3 = meta[meta["record_id"].astype(str) == "3"].iloc[0] + self.assertEqual(int(row2["age_is_censored"]), 1) + self.assertEqual(int(row2["age_is_missing"]), 0) + self.assertEqual(int(row2["age"]), 300) + self.assertEqual(int(row3["age_is_missing"]), 1) + self.assertEqual(int(row3["age_is_censored"]), 0) + # Signal paths are extension-free record bases + for path in meta["signal_file"]: + self.assertFalse(str(path).endswith(".hea")) + self.assertFalse(str(path).endswith(".dat")) + self.assertEqual(ds.sampling_rate, 100) + + def test_100_and_500_hz_coexist(self): + ds = PTBXLDataset( + root=str(self.data_root), + metadata_cache_dir=self.cache_dir, + sampling_rate=100, + cache_dir=self.tmp / "c100", + ) + self.assertTrue((self.cache_dir / ds.metadata_file_name).is_file()) + # Second rate gets a distinct filename (rate + root hash). + ds500 = PTBXLDataset( + root=str(self.data_root), + metadata_cache_dir=self.cache_dir, + sampling_rate=500, + cache_dir=self.tmp / "c500", + ) + self.assertTrue((self.cache_dir / ds500.metadata_file_name).is_file()) + self.assertNotEqual(ds.metadata_file_name, ds500.metadata_file_name) + + def test_end_to_end_reads_event_from_fixture(self): + """Instantiate PTBXLDataset and read a real event (not just helpers).""" + ds = PTBXLDataset( + root=str(self.data_root), + metadata_cache_dir=self.cache_dir, + sampling_rate=100, + cache_dir=self.tmp / "pyhealth_cache_e2e", + ) + # Config must already point at the derived CSV before any load. + self.assertEqual( + ds.config.tables["records"].file_path, ds.metadata_file_name + ) + self.assertTrue( + (Path(ds.root) / ds.config.tables["records"].file_path).is_file() + ) + + patient_ids = ds.unique_patient_ids + self.assertGreaterEqual(len(patient_ids), 1) + patient = ds.get_patient(patient_ids[0]) + events = patient.get_events(event_type="records") + self.assertGreaterEqual(len(events), 1) + event = events[0] + self.assertTrue(hasattr(event, "signal_file")) + self.assertTrue(hasattr(event, "strat_fold")) + self.assertTrue(hasattr(event, "scp_codes")) + self.assertTrue(str(event.signal_file)) + # Absolute waveform path must live under the fixture data root. + Path(str(event.signal_file)).resolve().relative_to(self.data_root.resolve()) + + +@unittest.skipUnless( + __import__("importlib").util.find_spec("wfdb") is not None, + "wfdb optional extra not installed", +) +class TestPTBXLSignalIO(unittest.TestCase): + def test_load_committed_fixture_shape_channels_time(self): + record = FIXTURE_ROOT / "records100" / "00000" / "00001_lr" + self.assertTrue(Path(str(record) + ".hea").is_file()) + self.assertTrue(Path(str(record) + ".dat").is_file()) + signal = load_ptbxl_record(record) + # 12 leads != 50 samples: catches a missing transpose. + self.assertEqual(signal.shape, (12, 50)) + self.assertNotEqual(signal.shape[0], signal.shape[1]) + + def test_load_strips_extension_never_appends(self): + record = FIXTURE_ROOT / "records100" / "00000" / "00001_lr" + signal = load_ptbxl_record(Path(str(record) + ".hea")) + self.assertEqual(signal.shape, (12, 50)) + self.assertNotEqual(signal.shape[0], signal.shape[1]) + + +class TestPTBXLTaskAndSplit(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()) + self.data_root = _materialize_fixture(self.tmp / "data") + self.cache_dir = self.tmp / "meta_cache" + self.dataset = PTBXLDataset( + root=str(self.data_root), + metadata_cache_dir=self.cache_dir, + sampling_rate=100, + cache_dir=self.tmp / "pyhealth_cache", + ) + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_task_drops_empty_labels_by_default(self): + fake_signal = np.zeros((12, 50), dtype=np.float32) + + def _fake_load(_path): + return fake_signal + + task = PTBXLSuperclassClassification( + scp_statements_path=self.data_root / "scp_statements.csv", + drop_empty_labels=True, + ) + with patch( + "pyhealth.datasets.ptbxl.load_ptbxl_record", side_effect=_fake_load + ): + samples = [] + for patient in self.dataset.iter_patients(): + samples.extend(task(patient)) + # Keep: 1 NORM, 2 MI, 4 HYP+MI. Drop: 3 PACE-only and 5 empty {}. + self.assertEqual(len(samples), 3) + labels = {tuple(s["labels"]) for s in samples} + self.assertEqual(labels, {("NORM",), ("MI",), ("HYP", "MI")}) + multi = next(s for s in samples if len(s["labels"]) > 1) + self.assertEqual(multi["labels"], ["HYP", "MI"]) + for s in samples: + self.assertIn("strat_fold", s) + self.assertIn("site", s) + self.assertIn("device", s) + self.assertIn("sex", s) + + def test_task_keeps_empty_when_disabled(self): + fake_signal = np.zeros((12, 50), dtype=np.float32) + task = PTBXLSuperclassClassification( + scp_statements_path=self.data_root / "scp_statements.csv", + drop_empty_labels=False, + ) + with patch( + "pyhealth.datasets.ptbxl.load_ptbxl_record", return_value=fake_signal + ): + samples = [] + for patient in self.dataset.iter_patients(): + samples.extend(task(patient)) + self.assertEqual(len(samples), 5) + empty = [s for s in samples if s["labels"] == []] + # Both PACE-only and empty-dict {} yield empty superclass sets. + self.assertEqual(len(empty), 2) + + def test_split_by_strat_fold(self): + # Lightweight fake SampleDataset-like object + class _FakeDS: + def __init__(self, samples): + self._samples = samples + + def __len__(self): + return len(self._samples) + + def __getitem__(self, i): + return self._samples[i] + + def subset(self, indices): + return _FakeDS([self._samples[i] for i in indices]) + + samples = [ + {"strat_fold": 1, "id": "a"}, + {"strat_fold": 9, "id": "b"}, + {"strat_fold": 10, "id": "c"}, + {"strat_fold": 3, "id": "d"}, + ] + train, val, test = split_by_strat_fold(_FakeDS(samples)) + self.assertEqual([s["id"] for s in train._samples], ["a", "d"]) + self.assertEqual([s["id"] for s in val._samples], ["b"]) + self.assertEqual([s["id"] for s in test._samples], ["c"]) + + +class TestWFDBImportError(unittest.TestCase): + def test_import_error_points_at_extra(self): + import builtins + + real_import = builtins.__import__ + + def _guard(name, *args, **kwargs): + if name == "wfdb" or name.startswith("wfdb."): + raise ImportError("blocked") + return real_import(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=_guard): + with self.assertRaises(ImportError) as ctx: + load_ptbxl_record("/tmp/does-not-matter") + self.assertIn("pyhealth[ptbxl]", str(ctx.exception)) + + +if __name__ == "__main__": + unittest.main() From 2d10c696d1723e9f069c7d5c0dbd41145ab389c4 Mon Sep 17 00:00:00 2001 From: AxelNoun <150222552+AxelNoun@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:23:59 +0200 Subject: [PATCH 04/13] docs: add PTB-XL API pages and quickstart example Document the dataset/task Overview RST pages and a minimal ECG superclass example for CPBench users. Co-authored-by: Cursor --- docs/api/datasets.rst | 1 + docs/api/datasets/pyhealth.datasets.ptbxl.rst | 25 ++++++ docs/api/tasks.rst | 1 + docs/api/tasks/pyhealth.tasks.ptbxl.rst | 22 ++++++ .../ecg/ptbxl/ptbxl_superclass_quickstart.py | 77 +++++++++++++++++++ 5 files changed, 126 insertions(+) create mode 100644 docs/api/datasets/pyhealth.datasets.ptbxl.rst create mode 100644 docs/api/tasks/pyhealth.tasks.ptbxl.rst create mode 100644 examples/ecg/ptbxl/ptbxl_superclass_quickstart.py diff --git a/docs/api/datasets.rst b/docs/api/datasets.rst index c9a88b7ff..02dd68a57 100644 --- a/docs/api/datasets.rst +++ b/docs/api/datasets.rst @@ -243,6 +243,7 @@ Available Datasets datasets/pyhealth.datasets.ChestXray14Dataset datasets/pyhealth.datasets.PhysioNetDeIDDataset datasets/pyhealth.datasets.EEGBCIDataset + datasets/pyhealth.datasets.ptbxl datasets/pyhealth.datasets.TUABDataset datasets/pyhealth.datasets.TUEVDataset datasets/pyhealth.datasets.ClinVarDataset diff --git a/docs/api/datasets/pyhealth.datasets.ptbxl.rst b/docs/api/datasets/pyhealth.datasets.ptbxl.rst new file mode 100644 index 000000000..a7a8c927c --- /dev/null +++ b/docs/api/datasets/pyhealth.datasets.ptbxl.rst @@ -0,0 +1,25 @@ +pyhealth.datasets.ptbxl +======================= + +Overview +-------- + +PTB-XL is a large publicly available 12-lead ECG dataset from PhysioNet +(version 1.0.3). It contains 21,799 clinical ECG records of 10 seconds from +18,885 patients, with multi-label SCP-ECG statements, official stratified +folds (``strat_fold``), and demographic / site / device metadata suited to +shift-aware evaluation. + +For more information see `PhysioNet PTB-XL v1.0.3 +`_ and Wagner et al., +`Scientific Data 2020 `_. + +Optional dependency: install waveform I/O with ``pip install 'pyhealth[ptbxl]'``. + +API Reference +------------- + +.. autoclass:: pyhealth.datasets.PTBXLDataset + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/api/tasks.rst b/docs/api/tasks.rst index bdaa9599a..bd7a4a627 100644 --- a/docs/api/tasks.rst +++ b/docs/api/tasks.rst @@ -225,6 +225,7 @@ Available Tasks Sleep Staging (SleepEDF) Temple University EEG Tasks EEGBCI Tasks + PTB-XL Tasks Sleep Staging v2 Benchmark EHRShot ChestX-ray14 Binary Classification diff --git a/docs/api/tasks/pyhealth.tasks.ptbxl.rst b/docs/api/tasks/pyhealth.tasks.ptbxl.rst new file mode 100644 index 000000000..19b48c404 --- /dev/null +++ b/docs/api/tasks/pyhealth.tasks.ptbxl.rst @@ -0,0 +1,22 @@ +pyhealth.tasks.ptbxl +==================== + +Overview +-------- + +Task helpers for PTB-XL multi-label ECG diagnosis. This module currently +implements the official **5-diagnostic-superclass** classification task +(``NORM``, ``MI``, ``STTC``, ``CD``, ``HYP``) by aggregating diagnostic SCP +statements via ``scp_statements.csv``. + +Empty superclass label sets (407 / 21,799 records after aggregation; mainly +pacemaker ECGs) are dropped by default. Official fold splitting is provided +by :func:`pyhealth.datasets.split_by_strat_fold`. + +API Reference +------------- + +.. automodule:: pyhealth.tasks.ptbxl + :members: + :undoc-members: + :show-inheritance: diff --git a/examples/ecg/ptbxl/ptbxl_superclass_quickstart.py b/examples/ecg/ptbxl/ptbxl_superclass_quickstart.py new file mode 100644 index 000000000..21922c238 --- /dev/null +++ b/examples/ecg/ptbxl/ptbxl_superclass_quickstart.py @@ -0,0 +1,77 @@ +"""PTB-XL 5-superclass multi-label quickstart. + +Requires:: + + pip install 'pyhealth[ptbxl]' + +Download PTB-XL v1.0.3 from https://physionet.org/content/ptb-xl/1.0.3/ +and point ``--root`` at the extracted version directory (contains +``ptbxl_database.csv``, ``scp_statements.csv``, ``records100/``). + +Example:: + + python examples/ecg/ptbxl/ptbxl_superclass_quickstart.py \\ + --root /data/ptb-xl/1.0.3 --dev +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from pyhealth.datasets import PTBXLDataset, split_by_strat_fold +from pyhealth.tasks import PTBXLSuperclassClassification + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--root", + type=Path, + required=True, + help="PTB-XL version root (ptbxl_database.csv + records*/)", + ) + parser.add_argument( + "--sampling-rate", + type=int, + default=100, + choices=(100, 500), + help="Waveform sampling rate (default: 100 Hz / filename_lr)", + ) + parser.add_argument( + "--dev", + action="store_true", + help="Limit patients via BaseDataset.dev mode", + ) + args = parser.parse_args() + + dataset = PTBXLDataset( + root=str(args.root), + sampling_rate=args.sampling_rate, + dev=args.dev, + ) + task = PTBXLSuperclassClassification( + scp_statements_path=str(args.root / "scp_statements.csv"), + ) + samples = dataset.set_task(task) + train, val, test = split_by_strat_fold(samples) + print( + f"samples={len(samples)} " + f"train={len(train)} val={len(val)} test={len(test)}" + ) + if len(samples): + first = samples[0] + print( + "first sample:", + { + "patient_id": first["patient_id"], + "record_id": first["record_id"], + "labels": first["labels"], + "strat_fold": first["strat_fold"], + "signal_shape": tuple(first["signal"].shape), + }, + ) + + +if __name__ == "__main__": + main() From 6e9ef584bb310ca54db9230a994798c0aa7ede17 Mon Sep 17 00:00:00 2001 From: AxelNoun <150222552+AxelNoun@users.noreply.github.com> Date: Wed, 19 Aug 2026 00:29:55 +0200 Subject: [PATCH 05/13] fix(datasets): isolate PTB-XL cache identity and write metadata atomically BaseDataset keys global_event_df on root+dataset_name; hash data-root path and CSV bytes into dataset_name (EEGBCI pattern) and write CSV/YAML via tmp+os.replace under FileLock so DDP cannot read a truncated file. Co-authored-by: Cursor --- pyhealth/datasets/configs/ptbxl.yaml | 2 +- pyhealth/datasets/ptbxl.py | 172 +++++++++++++----- test-resources/{ => core}/ptbxl/README.md | 11 +- .../{ => core}/ptbxl/ptbxl_database.csv | 0 .../{ => core}/ptbxl/scp_statements.csv | 0 .../ptbxl/records100/00000/00001_lr.dat | Bin 1200 -> 0 bytes .../ptbxl/records100/00000/00001_lr.hea | 13 -- .../ptbxl/records100/00000/00002_lr.dat | Bin 1200 -> 0 bytes .../ptbxl/records100/00000/00002_lr.hea | 13 -- .../ptbxl/records100/00000/00003_lr.dat | Bin 1200 -> 0 bytes .../ptbxl/records100/00000/00003_lr.hea | 13 -- .../ptbxl/records100/00000/00004_lr.dat | Bin 1200 -> 0 bytes .../ptbxl/records100/00000/00004_lr.hea | 13 -- .../ptbxl/records100/00000/00005_lr.dat | Bin 1200 -> 0 bytes .../ptbxl/records100/00000/00005_lr.hea | 13 -- .../ptbxl/records500/00000/00001_hr.dat | Bin 6000 -> 0 bytes .../ptbxl/records500/00000/00001_hr.hea | 13 -- .../ptbxl/records500/00000/00002_hr.dat | Bin 6000 -> 0 bytes .../ptbxl/records500/00000/00002_hr.hea | 13 -- .../ptbxl/records500/00000/00003_hr.dat | Bin 6000 -> 0 bytes .../ptbxl/records500/00000/00003_hr.hea | 13 -- .../ptbxl/records500/00000/00004_hr.dat | Bin 6000 -> 0 bytes .../ptbxl/records500/00000/00004_hr.hea | 13 -- .../ptbxl/records500/00000/00005_hr.dat | Bin 6000 -> 0 bytes .../ptbxl/records500/00000/00005_hr.hea | 13 -- 25 files changed, 134 insertions(+), 181 deletions(-) rename test-resources/{ => core}/ptbxl/README.md (52%) rename test-resources/{ => core}/ptbxl/ptbxl_database.csv (100%) rename test-resources/{ => core}/ptbxl/scp_statements.csv (100%) delete mode 100644 test-resources/ptbxl/records100/00000/00001_lr.dat delete mode 100644 test-resources/ptbxl/records100/00000/00001_lr.hea delete mode 100644 test-resources/ptbxl/records100/00000/00002_lr.dat delete mode 100644 test-resources/ptbxl/records100/00000/00002_lr.hea delete mode 100644 test-resources/ptbxl/records100/00000/00003_lr.dat delete mode 100644 test-resources/ptbxl/records100/00000/00003_lr.hea delete mode 100644 test-resources/ptbxl/records100/00000/00004_lr.dat delete mode 100644 test-resources/ptbxl/records100/00000/00004_lr.hea delete mode 100644 test-resources/ptbxl/records100/00000/00005_lr.dat delete mode 100644 test-resources/ptbxl/records100/00000/00005_lr.hea delete mode 100644 test-resources/ptbxl/records500/00000/00001_hr.dat delete mode 100644 test-resources/ptbxl/records500/00000/00001_hr.hea delete mode 100644 test-resources/ptbxl/records500/00000/00002_hr.dat delete mode 100644 test-resources/ptbxl/records500/00000/00002_hr.hea delete mode 100644 test-resources/ptbxl/records500/00000/00003_hr.dat delete mode 100644 test-resources/ptbxl/records500/00000/00003_hr.hea delete mode 100644 test-resources/ptbxl/records500/00000/00004_hr.dat delete mode 100644 test-resources/ptbxl/records500/00000/00004_hr.hea delete mode 100644 test-resources/ptbxl/records500/00000/00005_hr.dat delete mode 100644 test-resources/ptbxl/records500/00000/00005_hr.hea diff --git a/pyhealth/datasets/configs/ptbxl.yaml b/pyhealth/datasets/configs/ptbxl.yaml index 6b5b969b9..322114542 100644 --- a/pyhealth/datasets/configs/ptbxl.yaml +++ b/pyhealth/datasets/configs/ptbxl.yaml @@ -1,4 +1,4 @@ -version: "1.0.3" +version: "1.0.3" # PhysioNet PTB-XL v1.0.3 only; earlier releases are not supported. tables: records: # Template only: PTBXLDataset writes a resolved config whose file_path diff --git a/pyhealth/datasets/ptbxl.py b/pyhealth/datasets/ptbxl.py index 54fee0c0d..cdc2d36c9 100644 --- a/pyhealth/datasets/ptbxl.py +++ b/pyhealth/datasets/ptbxl.py @@ -27,12 +27,14 @@ import ast import hashlib import logging +import os from pathlib import Path from typing import TYPE_CHECKING, Any import numpy as np import pandas as pd import yaml +from filelock import FileLock from .base_dataset import BaseDataset from .configs.config import load_yaml_config @@ -46,18 +48,38 @@ # HIPAA: ages ≥ 90 are encoded as this sentinel in ptbxl_database.csv. AGE_CENSOR_SENTINEL = 300 -# Official diagnostic superclasses (diagnostic_class in scp_statements.csv). -PTBXL_DIAGNOSTIC_SUPERCLASSES = ("NORM", "MI", "STTC", "CD", "HYP") - -# Sex encoding in ptbxl_database.csv (PhysioNet / Scientific Data). -PTBXL_SEX_FEMALE = 0 -PTBXL_SEX_MALE = 1 - PTBXL_DATABASE_CSV = "ptbxl_database.csv" PTBXL_SCP_STATEMENTS_CSV = "scp_statements.csv" _DEFAULT_METADATA_CACHE = Path(MODULE_CACHE_PATH) / "ptbxl" +def _atomic_replace(tmp_path: Path, dest: Path) -> None: + """Replace ``dest`` with ``tmp_path`` (same-filesystem atomic on POSIX/NT).""" + os.replace(tmp_path, dest) + + +def _write_csv_atomic(df: pd.DataFrame, dest: Path) -> None: + tmp_path = dest.with_name(dest.name + ".tmp") + try: + df.to_csv(tmp_path, index=False) + _atomic_replace(tmp_path, dest) + except Exception: + if tmp_path.exists(): + tmp_path.unlink(missing_ok=True) + raise + + +def _write_yaml_atomic(payload: dict[str, Any], dest: Path) -> None: + tmp_path = dest.with_name(dest.name + ".tmp") + try: + with open(tmp_path, "w", encoding="utf-8") as handle: + yaml.safe_dump(payload, handle, sort_keys=False) + _atomic_replace(tmp_path, dest) + except Exception: + if tmp_path.exists(): + tmp_path.unlink(missing_ok=True) + raise + def format_patient_id(value: Any, *, ecg_id: Any | None = None) -> str: """Cast PTB-XL ``patient_id`` (stored as float, e.g. ``15709.0``) to ``str``. @@ -186,25 +208,41 @@ def root_cache_key(data_root: str | Path) -> str: return hashlib.sha1(resolved.encode("utf-8")).hexdigest()[:10] -def metadata_filename(sampling_rate: int, data_root: str | Path) -> str: - """Return the rate- and root-specific derived metadata CSV name. +def metadata_filename( + sampling_rate: int, + data_root: str | Path, + source_key: str | None = None, +) -> str: + """Return the rate-, root-, and source-specific derived metadata CSV name. + + ``source_key`` is a content hash of ``ptbxl_database.csv``. Including it + in the filename forces ``prepare_metadata`` to regenerate when the + official CSV is replaced in-place (e.g. extracting v1.0.3 over v1.0.1). Args: sampling_rate (int): ``100`` or ``500``. data_root (str | Path): Absolute/relative PTB-XL data root. + source_key (str | None): Optional SHA1[:10] of the source + ``ptbxl_database.csv`` bytes. Returns: - str: Filename such as ``ptbxl-pyhealth-100hz-.csv``. + str: Filename such as ``ptbxl-pyhealth-100hz-[-].csv``. Examples: >>> name = metadata_filename(100, "/data/ptb-xl/1.0.3") >>> name.startswith("ptbxl-pyhealth-100hz-") True + >>> keyed = metadata_filename(100, "/data/ptb-xl/1.0.3", "abc123def0") + >>> "abc123def0" in keyed + True """ - return ( + name = ( f"ptbxl-pyhealth-{int(sampling_rate)}hz-" - f"{root_cache_key(data_root)}.csv" + f"{root_cache_key(data_root)}" ) + if source_key: + name = f"{name}-{source_key}" + return f"{name}.csv" def load_ptbxl_record(record_path: str | Path) -> np.ndarray: @@ -265,20 +303,26 @@ class PTBXLDataset(BaseDataset): Dataset: https://physionet.org/content/ptb-xl/1.0.3/ - Expects ``root`` (``data_root``) to point at the extracted version directory - containing ``ptbxl_database.csv``, ``scp_statements.csv``, ``records100/``, - and ``records500/``. Raw data must live outside the git repo. + Expects ``root`` (``data_root``) to point at the extracted **v1.0.3** + directory containing ``ptbxl_database.csv``, ``scp_statements.csv``, + ``records100/``, and ``records500/``. Raw data must live outside the git + repo. Earlier PhysioNet releases (v1.0.1 / v1.0.2) are not supported: + required columns and record counts differ. Derived metadata CSVs are written under PyHealth's dataset cache (``~/.cache/pyhealth/datasets/ptbxl/`` by default), **not** into ``root``, so read-only / shared data mounts stay untouched. Override with - ``metadata_cache_dir``. Filenames include the sampling rate and a short - hash of the resolved data root so different roots never share a cache. + ``metadata_cache_dir``. Filenames include the sampling rate, a short + hash of the resolved data root, and a content hash of + ``ptbxl_database.csv`` so different roots and source versions never share + a derived CSV. ``BaseDataset`` cache identity (``global_event_df``) further + includes those hashes via ``dataset_name`` (same pattern as EEGBCI). Args: root (str): Version root of the PTB-XL download (signal + official CSVs). - dataset_name (str | None): Optional name; defaults to - ``ptbxl_{sampling_rate}hz``. + dataset_name (str | None): Optional name prefix; defaults to + ``ptbxl_{sampling_rate}hz``. Root and metadata content hashes are + appended so two data roots cannot share a ``BaseDataset`` cache. config_path (str | Path | None): Optional YAML config; defaults to ``configs/ptbxl.yaml``. sampling_rate (int): ``100`` (default) or ``500``. @@ -301,11 +345,11 @@ class PTBXLDataset(BaseDataset): event timestamp. Examples: - >>> dataset = PTBXLDataset(root="/data/ptb-xl/1.0.3") - >>> dataset.stats() - >>> patient = dataset.get_patient(dataset.unique_patient_ids[0]) - >>> event = patient.get_events(event_type="records")[0] - >>> signal = load_ptbxl_record(event.signal_file) + >>> dataset = PTBXLDataset(root="/data/ptb-xl/1.0.3") # doctest: +SKIP + >>> dataset.stats() # doctest: +SKIP + >>> patient = dataset.get_patient(dataset.unique_patient_ids[0]) # doctest: +SKIP + >>> event = patient.get_events(event_type="records")[0] # doctest: +SKIP + >>> signal = load_ptbxl_record(event.signal_file) # doctest: +SKIP """ def __init__( @@ -334,8 +378,15 @@ def __init__( if metadata_cache_dir is not None else _DEFAULT_METADATA_CACHE ) + db_path = self.data_root / PTBXL_DATABASE_CSV + if not db_path.is_file(): + raise FileNotFoundError( + f"Expected {PTBXL_DATABASE_CSV} under root={self.data_root}. " + "Download PTB-XL from https://physionet.org/content/ptb-xl/1.0.3/" + ) + self._source_key = hashlib.sha1(db_path.read_bytes()).hexdigest()[:10] self.metadata_file_name = metadata_filename( - self.sampling_rate, self.data_root + self.sampling_rate, self.data_root, self._source_key ) self.prepare_metadata() @@ -345,12 +396,22 @@ def __init__( # config must already be correct when that first happens. resolved_config_path = self._write_resolved_config(package_config) + # BaseDataset._init_cache_dir keys on {root, tables, dataset_name, dev}. + # root is the shared metadata cache (see comment on super().__init__), + # so uniqueness of global_event_df must come from dataset_name — same + # pattern as EEGBCIDataset._metadata_cache_key. + base_name = dataset_name or f"ptbxl_{self.sampling_rate}hz" + dataset_name = ( + f"{base_name}_{root_cache_key(self.data_root)}_" + f"{self._metadata_cache_key()}" + ) + # BaseDataset.root is the metadata cache (CSV location); waveforms stay # under data_root via absolute signal_file paths in the CSV. super().__init__( root=str(self.metadata_cache_dir), tables=["records"], - dataset_name=dataset_name or f"ptbxl_{self.sampling_rate}hz", + dataset_name=dataset_name, config_path=str(resolved_config_path), **kwargs, ) @@ -368,11 +429,12 @@ def _write_resolved_config(self, package_config: Path) -> Path: config.tables["records"].file_path = self.metadata_file_name out_path = self.metadata_cache_dir / ( f"ptbxl-config-{self.sampling_rate}hz-" - f"{root_cache_key(self.data_root)}.yaml" + f"{root_cache_key(self.data_root)}-{self._source_key}.yaml" ) self.metadata_cache_dir.mkdir(parents=True, exist_ok=True) - with open(out_path, "w", encoding="utf-8") as handle: - yaml.safe_dump(config.model_dump(), handle, sort_keys=False) + lock_path = out_path.with_name(out_path.name + ".lock") + with FileLock(str(lock_path)): + _write_yaml_atomic(config.model_dump(), out_path) return out_path @property @@ -384,16 +446,41 @@ def scp_statements_path(self) -> Path: """ return self.data_root / PTBXL_SCP_STATEMENTS_CSV + def _metadata_cache_key(self) -> str: + """SHA1[:10] of the derived metadata CSV bytes (EEGBCI pattern). + + Must be called after :meth:`prepare_metadata` so the file exists. + Injected into ``dataset_name`` so ``BaseDataset._init_cache_dir`` + does not collapse two data roots onto one ``global_event_df``. + """ + csv_path = self.metadata_cache_dir / self.metadata_file_name + return hashlib.sha1(csv_path.read_bytes()).hexdigest()[:10] + def prepare_metadata(self) -> None: """Build rate-/root-specific metadata CSV under ``metadata_cache_dir``. Returns: None + + Examples: + >>> # doctest: +SKIP + >>> ds = PTBXLDataset(root="/data/ptb-xl/1.0.3") + >>> ds.prepare_metadata() """ csv_path = self.metadata_cache_dir / self.metadata_file_name if csv_path.exists() and self._metadata_matches_request(csv_path): return + self.metadata_cache_dir.mkdir(parents=True, exist_ok=True) + lock_path = csv_path.with_name(csv_path.name + ".lock") + with FileLock(str(lock_path)): + # Another process may have finished while we waited. + if csv_path.exists() and self._metadata_matches_request(csv_path): + return + self._write_derived_metadata(csv_path) + + def _write_derived_metadata(self, csv_path: Path) -> None: + """Generate the derived metadata CSV (caller holds the file lock).""" db_path = self.data_root / PTBXL_DATABASE_CSV if not db_path.is_file(): raise FileNotFoundError( @@ -418,7 +505,9 @@ def prepare_metadata(self) -> None: missing = required - set(db.columns) if missing: raise ValueError( - f"ptbxl_database.csv missing columns: {sorted(missing)}" + f"ptbxl_database.csv missing columns: {sorted(missing)}. " + "PTBXLDataset supports PhysioNet PTB-XL v1.0.3 only " + "(https://physionet.org/content/ptb-xl/1.0.3/)." ) filename_col = "filename_lr" if self.sampling_rate == 100 else "filename_hr" @@ -469,8 +558,7 @@ def prepare_metadata(self) -> None: inplace=True, ) out.reset_index(drop=True, inplace=True) - self.metadata_cache_dir.mkdir(parents=True, exist_ok=True) - out.to_csv(csv_path, index=False) + _write_csv_atomic(out, csv_path) logger.info( "Wrote PTB-XL metadata (%d records, %d Hz) to %s", len(out), @@ -479,16 +567,19 @@ def prepare_metadata(self) -> None: ) def _metadata_matches_request(self, csv_path: Path) -> bool: - """Reuse cached metadata when schema and signal roots match. + """Reuse cached metadata when the derived CSV has the expected schema. + + Filename already encodes sampling rate, data-root path, and source CSV + content, so this only guards against a truncated or partial write. Args: csv_path (Path): Candidate derived metadata CSV. Returns: - bool: True if the cache is safe to reuse for this ``data_root``. + bool: True if the cache is safe to reuse. """ try: - df = pd.read_csv(csv_path, nrows=5) + df = pd.read_csv(csv_path, nrows=0) except (OSError, ValueError, pd.errors.ParserError): return False needed = { @@ -506,16 +597,7 @@ def _metadata_matches_request(self, csv_path: Path) -> bool: "scp_codes", "recording_date", } - if not needed.issubset(df.columns) or df.empty: - return False - # Absolute signal paths must still sit under the current data_root. - data_root = self.data_root.resolve() - for path in df["signal_file"].astype(str): - try: - Path(path).resolve().relative_to(data_root) - except ValueError: - return False - return True + return needed.issubset(df.columns) @property def default_task(self) -> PTBXLSuperclassClassification: diff --git a/test-resources/ptbxl/README.md b/test-resources/core/ptbxl/README.md similarity index 52% rename from test-resources/ptbxl/README.md rename to test-resources/core/ptbxl/README.md index e5ab58a1e..d34c35cd1 100644 --- a/test-resources/ptbxl/README.md +++ b/test-resources/core/ptbxl/README.md @@ -1,10 +1,11 @@ # Synthetic PTB-XL-shaped fixture for unit tests (NOT real PhysioNet data). # -# Contents are programmatically generated: -# - ptbxl_database.csv / scp_statements.csv: tiny CSV stubs -# - records100/*.hea+.dat and records500/*.hea+.dat: synthetic WFDB -# (12 leads x 50 samples at 100 Hz; 12 x 250 at 500 Hz — unequal dims -# so a missing transpose fails the shape assertion) +# Contents: +# - ptbxl_database.csv / scp_statements.csv: tiny CSV stubs (committed) +# - records100 / records500: synthesized at test time by +# tests/core/test_ptbxl.py::_materialize_fixture (12 leads x 50 samples +# at 100 Hz; 12 x 250 at 500 Hz — unequal dims so a missing transpose +# fails the shape assertion). Waveform binaries are not committed. # # Edge-case coverage: # - ecg_id=2: age=300 (HIPAA censored ≥90); likelihood 0 (SR) diff --git a/test-resources/ptbxl/ptbxl_database.csv b/test-resources/core/ptbxl/ptbxl_database.csv similarity index 100% rename from test-resources/ptbxl/ptbxl_database.csv rename to test-resources/core/ptbxl/ptbxl_database.csv diff --git a/test-resources/ptbxl/scp_statements.csv b/test-resources/core/ptbxl/scp_statements.csv similarity index 100% rename from test-resources/ptbxl/scp_statements.csv rename to test-resources/core/ptbxl/scp_statements.csv diff --git a/test-resources/ptbxl/records100/00000/00001_lr.dat b/test-resources/ptbxl/records100/00000/00001_lr.dat deleted file mode 100644 index 5f5a9a203530337c5b546b833f7bbae83de6b4fe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1200 zcmZ|K*H%I?6h+a6qN1X75TsY>AlUu??=t4Ll#|yvGEVk_92Z=Y=Zb4?C~(Ui_W>=6 zZ;Lz#7yk9pyBlfA`7P11wIS1rvCPzJy4TiPT;Xb6xB1#o#gaAy+R|+$%2cRQqfUb+ zEuK`dr0sxqgbQstJbQN|?K;0D+Osxf+BcS&I!O20I*cn^9qBe-$EsM;NkFH%txJzS z1BQ$kGhxb16-zn~=t8(KXTghiH`1l^TcRs#L#AtEnW>v}udUm-!quH_^L4L^B|QZ6 zsN0sTShHcvjy(sCysBbJPXRp(7fzgc^X^9a?);YMhqWQoPh**>U+G?3zvBv5e{`F# Gzx)9sjYd%b diff --git a/test-resources/ptbxl/records100/00000/00001_lr.hea b/test-resources/ptbxl/records100/00000/00001_lr.hea deleted file mode 100644 index 203e612da..000000000 --- a/test-resources/ptbxl/records100/00000/00001_lr.hea +++ /dev/null @@ -1,13 +0,0 @@ -00001_lr 12 100 50 -00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 -00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 -00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 -00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 -00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 -00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 -00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 -00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 -00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 -00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 -00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 -00001_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records100/00000/00002_lr.dat b/test-resources/ptbxl/records100/00000/00002_lr.dat deleted file mode 100644 index 5f5a9a203530337c5b546b833f7bbae83de6b4fe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1200 zcmZ|K*H%I?6h+a6qN1X75TsY>AlUu??=t4Ll#|yvGEVk_92Z=Y=Zb4?C~(Ui_W>=6 zZ;Lz#7yk9pyBlfA`7P11wIS1rvCPzJy4TiPT;Xb6xB1#o#gaAy+R|+$%2cRQqfUb+ zEuK`dr0sxqgbQstJbQN|?K;0D+Osxf+BcS&I!O20I*cn^9qBe-$EsM;NkFH%txJzS z1BQ$kGhxb16-zn~=t8(KXTghiH`1l^TcRs#L#AtEnW>v}udUm-!quH_^L4L^B|QZ6 zsN0sTShHcvjy(sCysBbJPXRp(7fzgc^X^9a?);YMhqWQoPh**>U+G?3zvBv5e{`F# Gzx)9sjYd%b diff --git a/test-resources/ptbxl/records100/00000/00002_lr.hea b/test-resources/ptbxl/records100/00000/00002_lr.hea deleted file mode 100644 index f998f2254..000000000 --- a/test-resources/ptbxl/records100/00000/00002_lr.hea +++ /dev/null @@ -1,13 +0,0 @@ -00002_lr 12 100 50 -00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 -00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 -00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 -00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 -00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 -00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 -00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 -00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 -00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 -00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 -00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 -00002_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records100/00000/00003_lr.dat b/test-resources/ptbxl/records100/00000/00003_lr.dat deleted file mode 100644 index 5f5a9a203530337c5b546b833f7bbae83de6b4fe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1200 zcmZ|K*H%I?6h+a6qN1X75TsY>AlUu??=t4Ll#|yvGEVk_92Z=Y=Zb4?C~(Ui_W>=6 zZ;Lz#7yk9pyBlfA`7P11wIS1rvCPzJy4TiPT;Xb6xB1#o#gaAy+R|+$%2cRQqfUb+ zEuK`dr0sxqgbQstJbQN|?K;0D+Osxf+BcS&I!O20I*cn^9qBe-$EsM;NkFH%txJzS z1BQ$kGhxb16-zn~=t8(KXTghiH`1l^TcRs#L#AtEnW>v}udUm-!quH_^L4L^B|QZ6 zsN0sTShHcvjy(sCysBbJPXRp(7fzgc^X^9a?);YMhqWQoPh**>U+G?3zvBv5e{`F# Gzx)9sjYd%b diff --git a/test-resources/ptbxl/records100/00000/00003_lr.hea b/test-resources/ptbxl/records100/00000/00003_lr.hea deleted file mode 100644 index 717cb6d98..000000000 --- a/test-resources/ptbxl/records100/00000/00003_lr.hea +++ /dev/null @@ -1,13 +0,0 @@ -00003_lr 12 100 50 -00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 -00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 -00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 -00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 -00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 -00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 -00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 -00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 -00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 -00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 -00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 -00003_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records100/00000/00004_lr.dat b/test-resources/ptbxl/records100/00000/00004_lr.dat deleted file mode 100644 index 5f5a9a203530337c5b546b833f7bbae83de6b4fe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1200 zcmZ|K*H%I?6h+a6qN1X75TsY>AlUu??=t4Ll#|yvGEVk_92Z=Y=Zb4?C~(Ui_W>=6 zZ;Lz#7yk9pyBlfA`7P11wIS1rvCPzJy4TiPT;Xb6xB1#o#gaAy+R|+$%2cRQqfUb+ zEuK`dr0sxqgbQstJbQN|?K;0D+Osxf+BcS&I!O20I*cn^9qBe-$EsM;NkFH%txJzS z1BQ$kGhxb16-zn~=t8(KXTghiH`1l^TcRs#L#AtEnW>v}udUm-!quH_^L4L^B|QZ6 zsN0sTShHcvjy(sCysBbJPXRp(7fzgc^X^9a?);YMhqWQoPh**>U+G?3zvBv5e{`F# Gzx)9sjYd%b diff --git a/test-resources/ptbxl/records100/00000/00004_lr.hea b/test-resources/ptbxl/records100/00000/00004_lr.hea deleted file mode 100644 index 3c0edc5b4..000000000 --- a/test-resources/ptbxl/records100/00000/00004_lr.hea +++ /dev/null @@ -1,13 +0,0 @@ -00004_lr 12 100 50 -00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 -00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 -00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 -00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 -00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 -00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 -00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 -00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 -00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 -00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 -00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 -00004_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records100/00000/00005_lr.dat b/test-resources/ptbxl/records100/00000/00005_lr.dat deleted file mode 100644 index 5f5a9a203530337c5b546b833f7bbae83de6b4fe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1200 zcmZ|K*H%I?6h+a6qN1X75TsY>AlUu??=t4Ll#|yvGEVk_92Z=Y=Zb4?C~(Ui_W>=6 zZ;Lz#7yk9pyBlfA`7P11wIS1rvCPzJy4TiPT;Xb6xB1#o#gaAy+R|+$%2cRQqfUb+ zEuK`dr0sxqgbQstJbQN|?K;0D+Osxf+BcS&I!O20I*cn^9qBe-$EsM;NkFH%txJzS z1BQ$kGhxb16-zn~=t8(KXTghiH`1l^TcRs#L#AtEnW>v}udUm-!quH_^L4L^B|QZ6 zsN0sTShHcvjy(sCysBbJPXRp(7fzgc^X^9a?);YMhqWQoPh**>U+G?3zvBv5e{`F# Gzx)9sjYd%b diff --git a/test-resources/ptbxl/records100/00000/00005_lr.hea b/test-resources/ptbxl/records100/00000/00005_lr.hea deleted file mode 100644 index 2f42af92d..000000000 --- a/test-resources/ptbxl/records100/00000/00005_lr.hea +++ /dev/null @@ -1,13 +0,0 @@ -00005_lr 12 100 50 -00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 -00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 -00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 -00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 -00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 -00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 -00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 -00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 -00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 -00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 -00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 -00005_lr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records500/00000/00001_hr.dat b/test-resources/ptbxl/records500/00000/00001_hr.dat deleted file mode 100644 index 69632dc434819f9da0e901ad91cd25621420fc9f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6000 zcmZ|KcNouA7>D8KhqgjeQAjD-L^e^#h|CfZMP!q*iU>tRHj$!8Lqa68$jFFnBFc&i zNo04N?>h6}`#RTqJ`bcwLt4^t1y^zvS91;5a-B!hJKv_~ddG#!dV_U0&VrH_+RS-y+SfZAdhSvP{*S@m^i;2rEp@CEIMhQxv=OE|2DxZMT!19OUE(KB77Qy)EhO7)Ei8&%TEwG8 zWm{hIk)Hw-q!5KELQ(D$#V#%8(c+E^#VF4G*4;o$7{5hYQrnPdDP@_erQ^N2mI*6N zEi2n>EhmayTHd1-WLpVJQi{@)p)BPnPX!(j#V)Pr(MpaB6{*C7*4;oKGJcD+vbG`7 zD#|idtHyhEeK@QzwVG_RwYn&FX$_Colx+`DnJQG}VX9G`8r0+wQS8!M9b3; zYTXUAj`3Thb+rwN)>D?LT0h>aYlE=D)P}On*2hG#OCR@WBiUAmy40gS4QR+?JWeB? z5XCNS?9nEU3yo>Qlh)lpn;O4G+DzM!=u^rvRh!3qb!`z=nEJGAv-KHK?9!GVZ6({9 z(u}8QP79vq8CuecXGO6~TYI#P<3elN@SJrw(6+{Jk+#z|B-&nCrfP?HudW@#3R64D zHd{N3VwZODXjj?RmUgtK10Cr^XS&dp=S8tgyLt2l$AxaZz>C)1K)V~iMcPB#kZ4b3 znX0|wy}I@eD@=V!w%Pi!D0XQdkM@;q-RVJ3deNJgc$q%*z%;n6o`+W-bKh}U?X!3^OI-sCM& z?9#VAI@EFDZHDrWbvMvq#&3}h*ES?NLRqHj$at@=qrwVPN6R)_$B1H=j`ip`**1*f zj9?_A7|j^QGLCmeu}jB$bb{l;cqZ_kbvMw7#&40nuWd;517(@2lj6O)P7W(f{ZO{q z`jIGh=@gGnm2DGwpAVSCWIp60rZAO{MX^h#dGr&oGnvI~z7oYQo#WBDjtg^`%h%T3 zK<631Mf#1lA<_BDGF2DEdv#qHR+zd-w%NK^6uWeZN57SA^Z17OEMOsvSj-Z>^mH_(m7Z;@`&HYB=PS*Gfic(1No!wOTk$u?WJ zi(;4V@aRt2wvkP2W(!-{#&&kFlb=PgOLuv6x8uStcJqsMH_$!CZ;|fRHYED1vP{+A z;=Q`=3oA_BFWYQAAc|dj(4&WB+aC7vE5EUi{T$#RhxlC-yY#R}k2o$I<_LdScLV*? z_$|`Ev<-Y9S$6|HWBeBBS#3k2=aglto{#tHdLgVZ^`dOE^^z!dDJi$6QkUD#aF%nN Y=K>eG1gTUiP0FolQ?9pZQ|VIw18Vc+%K!iX diff --git a/test-resources/ptbxl/records500/00000/00001_hr.hea b/test-resources/ptbxl/records500/00000/00001_hr.hea deleted file mode 100644 index e7e69164f..000000000 --- a/test-resources/ptbxl/records500/00000/00001_hr.hea +++ /dev/null @@ -1,13 +0,0 @@ -00001_hr 12 500 250 -00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 -00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 -00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 -00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 -00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 -00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 -00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 -00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 -00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 -00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 -00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 -00001_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records500/00000/00002_hr.dat b/test-resources/ptbxl/records500/00000/00002_hr.dat deleted file mode 100644 index 69632dc434819f9da0e901ad91cd25621420fc9f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6000 zcmZ|KcNouA7>D8KhqgjeQAjD-L^e^#h|CfZMP!q*iU>tRHj$!8Lqa68$jFFnBFc&i zNo04N?>h6}`#RTqJ`bcwLt4^t1y^zvS91;5a-B!hJKv_~ddG#!dV_U0&VrH_+RS-y+SfZAdhSvP{*S@m^i;2rEp@CEIMhQxv=OE|2DxZMT!19OUE(KB77Qy)EhO7)Ei8&%TEwG8 zWm{hIk)Hw-q!5KELQ(D$#V#%8(c+E^#VF4G*4;o$7{5hYQrnPdDP@_erQ^N2mI*6N zEi2n>EhmayTHd1-WLpVJQi{@)p)BPnPX!(j#V)Pr(MpaB6{*C7*4;oKGJcD+vbG`7 zD#|idtHyhEeK@QzwVG_RwYn&FX$_Colx+`DnJQG}VX9G`8r0+wQS8!M9b3; zYTXUAj`3Thb+rwN)>D?LT0h>aYlE=D)P}On*2hG#OCR@WBiUAmy40gS4QR+?JWeB? z5XCNS?9nEU3yo>Qlh)lpn;O4G+DzM!=u^rvRh!3qb!`z=nEJGAv-KHK?9!GVZ6({9 z(u}8QP79vq8CuecXGO6~TYI#P<3elN@SJrw(6+{Jk+#z|B-&nCrfP?HudW@#3R64D zHd{N3VwZODXjj?RmUgtK10Cr^XS&dp=S8tgyLt2l$AxaZz>C)1K)V~iMcPB#kZ4b3 znX0|wy}I@eD@=V!w%Pi!D0XQdkM@;q-RVJ3deNJgc$q%*z%;n6o`+W-bKh}U?X!3^OI-sCM& z?9#VAI@EFDZHDrWbvMvq#&3}h*ES?NLRqHj$at@=qrwVPN6R)_$B1H=j`ip`**1*f zj9?_A7|j^QGLCmeu}jB$bb{l;cqZ_kbvMw7#&40nuWd;517(@2lj6O)P7W(f{ZO{q z`jIGh=@gGnm2DGwpAVSCWIp60rZAO{MX^h#dGr&oGnvI~z7oYQo#WBDjtg^`%h%T3 zK<631Mf#1lA<_BDGF2DEdv#qHR+zd-w%NK^6uWeZN57SA^Z17OEMOsvSj-Z>^mH_(m7Z;@`&HYB=PS*Gfic(1No!wOTk$u?WJ zi(;4V@aRt2wvkP2W(!-{#&&kFlb=PgOLuv6x8uStcJqsMH_$!CZ;|fRHYED1vP{+A z;=Q`=3oA_BFWYQAAc|dj(4&WB+aC7vE5EUi{T$#RhxlC-yY#R}k2o$I<_LdScLV*? z_$|`Ev<-Y9S$6|HWBeBBS#3k2=aglto{#tHdLgVZ^`dOE^^z!dDJi$6QkUD#aF%nN Y=K>eG1gTUiP0FolQ?9pZQ|VIw18Vc+%K!iX diff --git a/test-resources/ptbxl/records500/00000/00002_hr.hea b/test-resources/ptbxl/records500/00000/00002_hr.hea deleted file mode 100644 index 23e343c08..000000000 --- a/test-resources/ptbxl/records500/00000/00002_hr.hea +++ /dev/null @@ -1,13 +0,0 @@ -00002_hr 12 500 250 -00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 -00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 -00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 -00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 -00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 -00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 -00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 -00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 -00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 -00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 -00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 -00002_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records500/00000/00003_hr.dat b/test-resources/ptbxl/records500/00000/00003_hr.dat deleted file mode 100644 index 69632dc434819f9da0e901ad91cd25621420fc9f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6000 zcmZ|KcNouA7>D8KhqgjeQAjD-L^e^#h|CfZMP!q*iU>tRHj$!8Lqa68$jFFnBFc&i zNo04N?>h6}`#RTqJ`bcwLt4^t1y^zvS91;5a-B!hJKv_~ddG#!dV_U0&VrH_+RS-y+SfZAdhSvP{*S@m^i;2rEp@CEIMhQxv=OE|2DxZMT!19OUE(KB77Qy)EhO7)Ei8&%TEwG8 zWm{hIk)Hw-q!5KELQ(D$#V#%8(c+E^#VF4G*4;o$7{5hYQrnPdDP@_erQ^N2mI*6N zEi2n>EhmayTHd1-WLpVJQi{@)p)BPnPX!(j#V)Pr(MpaB6{*C7*4;oKGJcD+vbG`7 zD#|idtHyhEeK@QzwVG_RwYn&FX$_Colx+`DnJQG}VX9G`8r0+wQS8!M9b3; zYTXUAj`3Thb+rwN)>D?LT0h>aYlE=D)P}On*2hG#OCR@WBiUAmy40gS4QR+?JWeB? z5XCNS?9nEU3yo>Qlh)lpn;O4G+DzM!=u^rvRh!3qb!`z=nEJGAv-KHK?9!GVZ6({9 z(u}8QP79vq8CuecXGO6~TYI#P<3elN@SJrw(6+{Jk+#z|B-&nCrfP?HudW@#3R64D zHd{N3VwZODXjj?RmUgtK10Cr^XS&dp=S8tgyLt2l$AxaZz>C)1K)V~iMcPB#kZ4b3 znX0|wy}I@eD@=V!w%Pi!D0XQdkM@;q-RVJ3deNJgc$q%*z%;n6o`+W-bKh}U?X!3^OI-sCM& z?9#VAI@EFDZHDrWbvMvq#&3}h*ES?NLRqHj$at@=qrwVPN6R)_$B1H=j`ip`**1*f zj9?_A7|j^QGLCmeu}jB$bb{l;cqZ_kbvMw7#&40nuWd;517(@2lj6O)P7W(f{ZO{q z`jIGh=@gGnm2DGwpAVSCWIp60rZAO{MX^h#dGr&oGnvI~z7oYQo#WBDjtg^`%h%T3 zK<631Mf#1lA<_BDGF2DEdv#qHR+zd-w%NK^6uWeZN57SA^Z17OEMOsvSj-Z>^mH_(m7Z;@`&HYB=PS*Gfic(1No!wOTk$u?WJ zi(;4V@aRt2wvkP2W(!-{#&&kFlb=PgOLuv6x8uStcJqsMH_$!CZ;|fRHYED1vP{+A z;=Q`=3oA_BFWYQAAc|dj(4&WB+aC7vE5EUi{T$#RhxlC-yY#R}k2o$I<_LdScLV*? z_$|`Ev<-Y9S$6|HWBeBBS#3k2=aglto{#tHdLgVZ^`dOE^^z!dDJi$6QkUD#aF%nN Y=K>eG1gTUiP0FolQ?9pZQ|VIw18Vc+%K!iX diff --git a/test-resources/ptbxl/records500/00000/00003_hr.hea b/test-resources/ptbxl/records500/00000/00003_hr.hea deleted file mode 100644 index 51105c487..000000000 --- a/test-resources/ptbxl/records500/00000/00003_hr.hea +++ /dev/null @@ -1,13 +0,0 @@ -00003_hr 12 500 250 -00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 -00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 -00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 -00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 -00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 -00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 -00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 -00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 -00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 -00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 -00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 -00003_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records500/00000/00004_hr.dat b/test-resources/ptbxl/records500/00000/00004_hr.dat deleted file mode 100644 index 69632dc434819f9da0e901ad91cd25621420fc9f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6000 zcmZ|KcNouA7>D8KhqgjeQAjD-L^e^#h|CfZMP!q*iU>tRHj$!8Lqa68$jFFnBFc&i zNo04N?>h6}`#RTqJ`bcwLt4^t1y^zvS91;5a-B!hJKv_~ddG#!dV_U0&VrH_+RS-y+SfZAdhSvP{*S@m^i;2rEp@CEIMhQxv=OE|2DxZMT!19OUE(KB77Qy)EhO7)Ei8&%TEwG8 zWm{hIk)Hw-q!5KELQ(D$#V#%8(c+E^#VF4G*4;o$7{5hYQrnPdDP@_erQ^N2mI*6N zEi2n>EhmayTHd1-WLpVJQi{@)p)BPnPX!(j#V)Pr(MpaB6{*C7*4;oKGJcD+vbG`7 zD#|idtHyhEeK@QzwVG_RwYn&FX$_Colx+`DnJQG}VX9G`8r0+wQS8!M9b3; zYTXUAj`3Thb+rwN)>D?LT0h>aYlE=D)P}On*2hG#OCR@WBiUAmy40gS4QR+?JWeB? z5XCNS?9nEU3yo>Qlh)lpn;O4G+DzM!=u^rvRh!3qb!`z=nEJGAv-KHK?9!GVZ6({9 z(u}8QP79vq8CuecXGO6~TYI#P<3elN@SJrw(6+{Jk+#z|B-&nCrfP?HudW@#3R64D zHd{N3VwZODXjj?RmUgtK10Cr^XS&dp=S8tgyLt2l$AxaZz>C)1K)V~iMcPB#kZ4b3 znX0|wy}I@eD@=V!w%Pi!D0XQdkM@;q-RVJ3deNJgc$q%*z%;n6o`+W-bKh}U?X!3^OI-sCM& z?9#VAI@EFDZHDrWbvMvq#&3}h*ES?NLRqHj$at@=qrwVPN6R)_$B1H=j`ip`**1*f zj9?_A7|j^QGLCmeu}jB$bb{l;cqZ_kbvMw7#&40nuWd;517(@2lj6O)P7W(f{ZO{q z`jIGh=@gGnm2DGwpAVSCWIp60rZAO{MX^h#dGr&oGnvI~z7oYQo#WBDjtg^`%h%T3 zK<631Mf#1lA<_BDGF2DEdv#qHR+zd-w%NK^6uWeZN57SA^Z17OEMOsvSj-Z>^mH_(m7Z;@`&HYB=PS*Gfic(1No!wOTk$u?WJ zi(;4V@aRt2wvkP2W(!-{#&&kFlb=PgOLuv6x8uStcJqsMH_$!CZ;|fRHYED1vP{+A z;=Q`=3oA_BFWYQAAc|dj(4&WB+aC7vE5EUi{T$#RhxlC-yY#R}k2o$I<_LdScLV*? z_$|`Ev<-Y9S$6|HWBeBBS#3k2=aglto{#tHdLgVZ^`dOE^^z!dDJi$6QkUD#aF%nN Y=K>eG1gTUiP0FolQ?9pZQ|VIw18Vc+%K!iX diff --git a/test-resources/ptbxl/records500/00000/00004_hr.hea b/test-resources/ptbxl/records500/00000/00004_hr.hea deleted file mode 100644 index 768d38e2e..000000000 --- a/test-resources/ptbxl/records500/00000/00004_hr.hea +++ /dev/null @@ -1,13 +0,0 @@ -00004_hr 12 500 250 -00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 -00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 -00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 -00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 -00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 -00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 -00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 -00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 -00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 -00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 -00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 -00004_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 diff --git a/test-resources/ptbxl/records500/00000/00005_hr.dat b/test-resources/ptbxl/records500/00000/00005_hr.dat deleted file mode 100644 index 69632dc434819f9da0e901ad91cd25621420fc9f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6000 zcmZ|KcNouA7>D8KhqgjeQAjD-L^e^#h|CfZMP!q*iU>tRHj$!8Lqa68$jFFnBFc&i zNo04N?>h6}`#RTqJ`bcwLt4^t1y^zvS91;5a-B!hJKv_~ddG#!dV_U0&VrH_+RS-y+SfZAdhSvP{*S@m^i;2rEp@CEIMhQxv=OE|2DxZMT!19OUE(KB77Qy)EhO7)Ei8&%TEwG8 zWm{hIk)Hw-q!5KELQ(D$#V#%8(c+E^#VF4G*4;o$7{5hYQrnPdDP@_erQ^N2mI*6N zEi2n>EhmayTHd1-WLpVJQi{@)p)BPnPX!(j#V)Pr(MpaB6{*C7*4;oKGJcD+vbG`7 zD#|idtHyhEeK@QzwVG_RwYn&FX$_Colx+`DnJQG}VX9G`8r0+wQS8!M9b3; zYTXUAj`3Thb+rwN)>D?LT0h>aYlE=D)P}On*2hG#OCR@WBiUAmy40gS4QR+?JWeB? z5XCNS?9nEU3yo>Qlh)lpn;O4G+DzM!=u^rvRh!3qb!`z=nEJGAv-KHK?9!GVZ6({9 z(u}8QP79vq8CuecXGO6~TYI#P<3elN@SJrw(6+{Jk+#z|B-&nCrfP?HudW@#3R64D zHd{N3VwZODXjj?RmUgtK10Cr^XS&dp=S8tgyLt2l$AxaZz>C)1K)V~iMcPB#kZ4b3 znX0|wy}I@eD@=V!w%Pi!D0XQdkM@;q-RVJ3deNJgc$q%*z%;n6o`+W-bKh}U?X!3^OI-sCM& z?9#VAI@EFDZHDrWbvMvq#&3}h*ES?NLRqHj$at@=qrwVPN6R)_$B1H=j`ip`**1*f zj9?_A7|j^QGLCmeu}jB$bb{l;cqZ_kbvMw7#&40nuWd;517(@2lj6O)P7W(f{ZO{q z`jIGh=@gGnm2DGwpAVSCWIp60rZAO{MX^h#dGr&oGnvI~z7oYQo#WBDjtg^`%h%T3 zK<631Mf#1lA<_BDGF2DEdv#qHR+zd-w%NK^6uWeZN57SA^Z17OEMOsvSj-Z>^mH_(m7Z;@`&HYB=PS*Gfic(1No!wOTk$u?WJ zi(;4V@aRt2wvkP2W(!-{#&&kFlb=PgOLuv6x8uStcJqsMH_$!CZ;|fRHYED1vP{+A z;=Q`=3oA_BFWYQAAc|dj(4&WB+aC7vE5EUi{T$#RhxlC-yY#R}k2o$I<_LdScLV*? z_$|`Ev<-Y9S$6|HWBeBBS#3k2=aglto{#tHdLgVZ^`dOE^^z!dDJi$6QkUD#aF%nN Y=K>eG1gTUiP0FolQ?9pZQ|VIw18Vc+%K!iX diff --git a/test-resources/ptbxl/records500/00000/00005_hr.hea b/test-resources/ptbxl/records500/00000/00005_hr.hea deleted file mode 100644 index e7ab1b3f2..000000000 --- a/test-resources/ptbxl/records500/00000/00005_hr.hea +++ /dev/null @@ -1,13 +0,0 @@ -00005_hr 12 500 250 -00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 0 -00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 1 -00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 2 -00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 3 -00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 4 -00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 5 -00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 6 -00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 7 -00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 8 -00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 9 -00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 10 -00005_hr.dat 16 1000.0(0)/uV 16 0 0 0 0 11 From 00f9591b185b2233ec4f64bfd6f83ea7f82babeb Mon Sep 17 00:00:00 2001 From: AxelNoun <150222552+AxelNoun@users.noreply.github.com> Date: Wed, 19 Aug 2026 00:30:17 +0200 Subject: [PATCH 06/13] fix(tasks): keep PTB-XL task cache keys stable and emit integer ages Move the SCP class map to functools.cache so vars(task) is unchanged after __call__. Emit age=-1 when missing and clip HIPAA-censored 300 to 90 so litdata can serialize the field. Co-authored-by: Cursor --- pyhealth/tasks/ptbxl.py | 144 +++++++++++++++++++++++++--------------- 1 file changed, 89 insertions(+), 55 deletions(-) diff --git a/pyhealth/tasks/ptbxl.py b/pyhealth/tasks/ptbxl.py index 4ea5d7bc9..daa3041ed 100644 --- a/pyhealth/tasks/ptbxl.py +++ b/pyhealth/tasks/ptbxl.py @@ -25,7 +25,8 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence +import functools +from collections.abc import Mapping from pathlib import Path from typing import Any, ClassVar @@ -39,14 +40,34 @@ # ``datasets.__init__`` → ``BaseDataset`` → ``tasks``). PTBXL_DIAGNOSTIC_SUPERCLASSES = ("NORM", "MI", "STTC", "CD", "HYP") -# Scientific Data (Wagner et al., Table 9): after aggregating diagnostic -# statements to the 5 superclasses, 407 of 21,799 records have an empty -# label set (mainly pacemaker ECGs with form/rhythm-only annotations). -# Default drop_empty_labels=True matches the common PTB-XL benchmarking -# practice and avoids all-zero multi-hot targets that break most -# conformal / nonconformity scores used in CPBench. -PTBXL_EMPTY_SUPERCLASS_COUNT = 407 -PTBXL_TOTAL_RECORDS = 21799 +# Missing age is emitted as this integer so litdata / default_collate can +# batch the field. ``age_is_missing`` disambiguates it from a real age. +AGE_MISSING_SENTINEL = -1 +# HIPAA-censored ages (raw 300 in ptbxl_database.csv) are clipped to 90 in +# the sample. ``age_is_censored`` remains True; the raw 300 stays in the +# derived metadata CSV. Matches the common PTB-XL literature convention. +HIPAA_AGE_CLIP = 90 + + +@functools.cache +def _load_diagnostic_class_map_cached(path: str) -> tuple[tuple[str, str], ...]: + """Load diagnostic_class map as a hashable tuple; cached on resolved path.""" + csv_path = Path(path) + if not csv_path.is_file(): + raise FileNotFoundError(f"scp_statements.csv not found: {csv_path}") + df = pd.read_csv(csv_path, index_col=0) + if "diagnostic" not in df.columns or "diagnostic_class" not in df.columns: + raise ValueError( + f"{csv_path} must contain 'diagnostic' and 'diagnostic_class' columns" + ) + diag = df[df["diagnostic"] == 1] + mapping: list[tuple[str, str]] = [] + for code, row in diag.iterrows(): + cls = row["diagnostic_class"] + if pd.isna(cls): + continue + mapping.append((str(code), str(cls))) + return tuple(mapping) def load_diagnostic_class_map( @@ -68,22 +89,8 @@ def load_diagnostic_class_map( >>> mapping["NORM"] 'NORM' """ - path = Path(scp_statements_path) - if not path.is_file(): - raise FileNotFoundError(f"scp_statements.csv not found: {path}") - df = pd.read_csv(path, index_col=0) - if "diagnostic" not in df.columns or "diagnostic_class" not in df.columns: - raise ValueError( - f"{path} must contain 'diagnostic' and 'diagnostic_class' columns" - ) - diag = df[df["diagnostic"] == 1] - mapping: dict[str, str] = {} - for code, row in diag.iterrows(): - cls = row["diagnostic_class"] - if pd.isna(cls): - continue - mapping[str(code)] = str(cls) - return mapping + resolved = str(Path(scp_statements_path).resolve()) + return dict(_load_diagnostic_class_map_cached(resolved)) def aggregate_diagnostic_superclasses( @@ -122,6 +129,29 @@ def aggregate_diagnostic_superclasses( return sorted(labels) +def _to_bool(value: Any) -> bool: + """Coerce PTB-XL flag cells (0/1, bool, ``\"1\"``/``\"True\"``) to ``bool``. + + Args: + value (Any): Raw event attribute. + + Returns: + bool: Parsed flag. + + Examples: + >>> _to_bool(1), _to_bool("true"), _to_bool(0) + (True, True, False) + """ + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + if pd.isna(value): + return False + return int(value) != 0 + text = str(value).strip() + return text in {"1", "True", "true"} + + class PTBXLSuperclassClassification(BaseTask): """5-superclass multi-label classification on PTB-XL. @@ -129,18 +159,33 @@ class PTBXLSuperclassClassification(BaseTask): ``NORM``, ``MI``, ``STTC``, ``CD``, ``HYP`` obtained by mapping diagnostic SCP statements via ``scp_statements.csv``. + Output label order + ------------------ + ``MultiLabelProcessor.fit`` sorts **observed** labels alphabetically, so + on the full v1.0.3 corpus the multi-hot vector is + ``CD, HYP, MI, NORM, STTC``. The constructor does not pin this order. + Empty label sets ---------------- - After aggregation, **407 / 21,799** records have no diagnostic superclass - (Scientific Data Table 9; mainly pacemaker ECGs). By default these are - **dropped** (``drop_empty_labels=True``) because an all-zero multi-hot - target breaks typical multi-label nonconformity scores used in CPBench, - and matches common PTB-XL literature practice. + After aggregation, about 400 records have no diagnostic superclass + (Wagner et al., Scientific Data Table 9, counted on v1.0.1's 21,837 + records; mainly pacemaker ECGs). By default these are **dropped** + (``drop_empty_labels=True``) because an all-zero multi-hot target breaks + typical multi-label nonconformity scores used in CPBench, and matches + common PTB-XL literature practice. Each sample also carries ``strat_fold``, ``site``, ``device``, ``age``, ``sex``, ``age_is_censored``, and ``age_is_missing`` for official splits and downstream shift evaluations. + Age convention + -------------- + ``age`` is always an integer so litdata and ``default_collate`` can batch + it. Missing age is ``-1`` (see ``age_is_missing``). HIPAA-censored ages + (≥90, stored as 300 in ``ptbxl_database.csv``) are clipped to ``90`` in + the sample while ``age_is_censored`` stays ``True``. The raw 300 remains + in the derived metadata CSV. + Note: 71-SCP multi-label classification and age regression are intentionally not implemented here; add them in this module in a follow-up PR. @@ -150,8 +195,6 @@ class PTBXLSuperclassClassification(BaseTask): Required unless set later via the dataset's ``default_task``. drop_empty_labels (bool): Drop records with no superclass after aggregation. Defaults to ``True``. - diagnostic_superclasses (Sequence[str]): Label vocabulary order - (defaults to the official five). Examples: >>> # doctest: +SKIP @@ -166,33 +209,28 @@ class PTBXLSuperclassClassification(BaseTask): task_name: str = "PTBXLSuperclassClassification" input_schema: ClassVar[dict[str, str]] = {"signal": "tensor"} output_schema: ClassVar[dict[str, str]] = {"labels": "multilabel"} + # Official five; MultiLabelProcessor still sorts observed labels. + superclass_names: ClassVar[tuple[str, ...]] = PTBXL_DIAGNOSTIC_SUPERCLASSES def __init__( self, scp_statements_path: str | Path | None = None, drop_empty_labels: bool = True, - diagnostic_superclasses: Sequence[str] = PTBXL_DIAGNOSTIC_SUPERCLASSES, ) -> None: self.scp_statements_path = ( Path(scp_statements_path) if scp_statements_path is not None else None ) self.drop_empty_labels = drop_empty_labels - self.diagnostic_superclasses = tuple(diagnostic_superclasses) - self._diagnostic_class_map: dict[str, str] | None = None super().__init__() def _class_map(self) -> dict[str, str]: - if self._diagnostic_class_map is None: - if self.scp_statements_path is None: - raise ValueError( - "scp_statements_path is required. Pass it to " - "PTBXLSuperclassClassification(...) or use " - "PTBXLDataset.default_task." - ) - self._diagnostic_class_map = load_diagnostic_class_map( - self.scp_statements_path + if self.scp_statements_path is None: + raise ValueError( + "scp_statements_path is required. Pass it to " + "PTBXLSuperclassClassification(...) or use " + "PTBXLDataset.default_task." ) - return self._diagnostic_class_map + return load_diagnostic_class_map(self.scp_statements_path) def __call__(self, patient: Patient) -> list[dict[str, Any]]: """Build one sample per ECG record for the patient.""" @@ -206,14 +244,13 @@ def __call__(self, patient: Patient) -> list[dict[str, Any]]: continue signal = load_ptbxl_record(event.signal_file) + age_missing = _to_bool(event.age_is_missing) + age_censored = _to_bool(event.age_is_censored) age_raw = event.age - age_missing = str(getattr(event, "age_is_missing", "0")) in { - "1", - "True", - "true", - } if age_missing or age_raw is None or str(age_raw).strip() == "": - age: Any = None + age: int = AGE_MISSING_SENTINEL + elif age_censored: + age = HIPAA_AGE_CLIP else: age = int(float(age_raw)) @@ -227,10 +264,7 @@ def __call__(self, patient: Patient) -> list[dict[str, Any]]: "site": event.site, "device": event.device, "age": age, - "age_is_censored": str( - getattr(event, "age_is_censored", "0") - ) - in {"1", "True", "true"}, + "age_is_censored": age_censored, "age_is_missing": age_missing, # sex: 0 = female, 1 = male "sex": int(float(event.sex)), From 5a2ba11a82ac4207d83d432f347d11f9bc3015ab Mon Sep 17 00:00:00 2001 From: AxelNoun <150222552+AxelNoun@users.noreply.github.com> Date: Wed, 19 Aug 2026 00:30:26 +0200 Subject: [PATCH 07/13] refactor(datasets): avoid decoding samples in split_by_strat_fold Accept a precomputed folds sequence and a generic fold_field so callers need not materialize 12-lead signals just to read an integer. Error messages no longer mention PTB-XL. Co-authored-by: Cursor --- pyhealth/datasets/splitter.py | 49 ++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/pyhealth/datasets/splitter.py b/pyhealth/datasets/splitter.py index bebdabc06..6138d6f7e 100644 --- a/pyhealth/datasets/splitter.py +++ b/pyhealth/datasets/splitter.py @@ -185,14 +185,17 @@ def split_by_strat_fold( train_folds: tuple[int, ...] | list[int] = tuple(range(1, 9)), val_folds: tuple[int, ...] | list[int] = (9,), test_folds: tuple[int, ...] | list[int] = (10,), + fold_field: str = "strat_fold", + folds: tuple[int, ...] | list[int] | None = None, ): - """Split a sample dataset using PTB-XL official ``strat_fold`` assignments. + """Split a sample dataset using integer fold assignments. The recommended PTB-XL protocol uses folds 1–8 for training, fold 9 for - validation, and fold 10 for testing (patient-disjoint by construction). + validation, and fold 10 for testing. - Samples must contain an integer ``strat_fold`` field (as emitted by - :class:`~pyhealth.tasks.ptbxl.PTBXLSuperclassClassification`). + Pass ``folds`` (one integer per sample, aligned with dataset indices) to + avoid decoding every sample via ``dataset[i]``. When omitted, fold values + are read from ``sample[fold_field]``. Args: dataset (SampleDataset): A :class:`~pyhealth.datasets.SampleDataset` @@ -203,13 +206,18 @@ def split_by_strat_fold( (default ``9``). test_folds (Tuple[int, ...] | List[int]): Folds assigned to test (default ``10``). + fold_field (str): Sample key holding the fold integer when ``folds`` + is not provided. Defaults to ``\"strat_fold\"``. + folds (Tuple[int, ...] | List[int] | None): Optional precomputed fold + for each index. Length must equal ``len(dataset)``. Returns: tuple: ``(train_dataset, val_dataset, test_dataset)`` subsets. Raises: - KeyError: If a sample is missing ``strat_fold``. - ValueError: If fold sets overlap or a sample fold is unassigned. + KeyError: If ``folds`` is omitted and a sample is missing ``fold_field``. + ValueError: If fold sets overlap, ``folds`` has the wrong length, or + a sample fold is unassigned. Examples: >>> # doctest: +SKIP @@ -222,17 +230,28 @@ def split_by_strat_fold( if train_set & val_set or train_set & test_set or val_set & test_set: raise ValueError("train_folds, val_folds, and test_folds must be disjoint") + n_samples = len(dataset) + if folds is not None: + if len(folds) != n_samples: + raise ValueError( + f"folds has length {len(folds)} but dataset has {n_samples} samples" + ) + fold_values = [int(f) for f in folds] + else: + fold_values = [] + for i in range(n_samples): + sample = dataset[i] + if fold_field not in sample: + raise KeyError( + f"sample is missing {fold_field!r}; pass folds= to avoid " + "decoding samples, or emit this field from the task" + ) + fold_values.append(int(sample[fold_field])) + train_index: list[int] = [] val_index: list[int] = [] test_index: list[int] = [] - for i in range(len(dataset)): - sample = dataset[i] - if "strat_fold" not in sample: - raise KeyError( - "sample is missing 'strat_fold'; run a PTB-XL task that " - "copies strat_fold onto each sample" - ) - fold = int(sample["strat_fold"]) + for i, fold in enumerate(fold_values): if fold in train_set: train_index.append(i) elif fold in val_set: @@ -241,7 +260,7 @@ def split_by_strat_fold( test_index.append(i) else: raise ValueError( - f"strat_fold={fold} is not in train/val/test fold sets" + f"{fold_field}={fold} is not in train/val/test fold sets" ) return ( From ce76939e52cf04a0b8bf2713a14c3a4fc0a694f5 Mon Sep 17 00:00:00 2001 From: AxelNoun <150222552+AxelNoun@users.noreply.github.com> Date: Wed, 19 Aug 2026 00:30:35 +0200 Subject: [PATCH 08/13] test: cover PTB-XL set_task, cache isolation, and age sentinels Exercise the real litdata path without mocking wfdb, assert MultiLabelProcessor vocab order, and pin cache-dir uniqueness plus vars(task) stability. Co-authored-by: Cursor --- tests/core/test_ptbxl.py | 291 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 270 insertions(+), 21 deletions(-) diff --git a/tests/core/test_ptbxl.py b/tests/core/test_ptbxl.py index 5d29a9c9f..e4f4b9cf7 100644 --- a/tests/core/test_ptbxl.py +++ b/tests/core/test_ptbxl.py @@ -2,9 +2,9 @@ Unit tests for PTBXLDataset, PTBXLSuperclassClassification, and split_by_strat_fold. -Uses small synthetic WFDB fixtures under test-resources/ptbxl/ (not real -PhysioNet records). Covers censored age (300), missing age, and empty -diagnostic-superclass labels. +Uses small synthetic WFDB fixtures under test-resources/core/ptbxl/ (not real +PhysioNet records). Waveforms are generated at test time. Covers censored +age (300), missing age, and empty diagnostic-superclass labels. Author: AxelNoun (GitHub: @AxelNoun) — external contributor, no NetID @@ -20,6 +20,7 @@ import numpy as np import pandas as pd +import torch from pyhealth.datasets.ptbxl import ( AGE_CENSOR_SENTINEL, @@ -33,14 +34,15 @@ ) from pyhealth.datasets.splitter import split_by_strat_fold from pyhealth.tasks.ptbxl import ( - PTBXL_EMPTY_SUPERCLASS_COUNT, PTBXLSuperclassClassification, + _to_bool, aggregate_diagnostic_superclasses, load_diagnostic_class_map, ) -FIXTURE_ROOT = Path(__file__).resolve().parents[1] / ".." / "test-resources" / "ptbxl" -FIXTURE_ROOT = FIXTURE_ROOT.resolve() +FIXTURE_ROOT = ( + Path(__file__).resolve().parents[2] / "test-resources" / "core" / "ptbxl" +) def _write_dummy_wfdb( @@ -63,24 +65,17 @@ def _write_dummy_wfdb( def _materialize_fixture(dest: Path) -> Path: - """Copy committed CSVs and WFDB records under ``dest``.""" + """Copy committed CSVs and synthesize WFDB records under ``dest``.""" dest.mkdir(parents=True, exist_ok=True) shutil.copy(FIXTURE_ROOT / "ptbxl_database.csv", dest / "ptbxl_database.csv") shutil.copy(FIXTURE_ROOT / "scp_statements.csv", dest / "scp_statements.csv") - for records_dir in ("records100", "records500"): - src = FIXTURE_ROOT / records_dir - if src.is_dir(): - shutil.copytree(src, dest / records_dir, dirs_exist_ok=True) - # Fallback: synthesize WFDB if committed waveforms are missing. db = pd.read_csv(dest / "ptbxl_database.csv") for col, fs, n_samples in ( ("filename_lr", 100, 50), ("filename_hr", 500, 250), ): for rel in db[col]: - base = dest / str(rel) - if not Path(str(base) + ".hea").is_file(): - _write_dummy_wfdb(base, n_samples=n_samples, fs=fs) + _write_dummy_wfdb(dest / str(rel), n_samples=n_samples, fs=fs) return dest @@ -112,6 +107,15 @@ def test_age_missing_vs_censored(self): self.assertFalse(is_age_censored(float("nan"))) self.assertFalse(is_age_censored(65)) + def test_to_bool_parses_flag_cells(self): + self.assertTrue(_to_bool(1)) + self.assertTrue(_to_bool("True")) + self.assertTrue(_to_bool("true")) + self.assertTrue(_to_bool(True)) + self.assertFalse(_to_bool(0)) + self.assertFalse(_to_bool("0")) + self.assertFalse(_to_bool(False)) + def test_metadata_filename_includes_rate_and_root(self): name_a = metadata_filename(100, "/data/ptb-xl/a") name_b = metadata_filename(100, "/data/ptb-xl/b") @@ -121,9 +125,10 @@ def test_metadata_filename_includes_rate_and_root(self): self.assertNotEqual( metadata_filename(100, "/tmp/x"), metadata_filename(500, "/tmp/x") ) - - def test_empty_superclass_count_documented(self): - self.assertEqual(PTBXL_EMPTY_SUPERCLASS_COUNT, 407) + keyed_a = metadata_filename(100, "/tmp/x", "aaaaaaaaaa") + keyed_b = metadata_filename(100, "/tmp/x", "bbbbbbbbbb") + self.assertIn("aaaaaaaaaa", keyed_a) + self.assertNotEqual(keyed_a, keyed_b) class TestPTBXLAggregation(unittest.TestCase): @@ -252,8 +257,15 @@ def test_end_to_end_reads_event_from_fixture(self): "wfdb optional extra not installed", ) class TestPTBXLSignalIO(unittest.TestCase): - def test_load_committed_fixture_shape_channels_time(self): - record = FIXTURE_ROOT / "records100" / "00000" / "00001_lr" + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()) + self.data_root = _materialize_fixture(self.tmp / "data") + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_load_fixture_shape_channels_time(self): + record = self.data_root / "records100" / "00000" / "00001_lr" self.assertTrue(Path(str(record) + ".hea").is_file()) self.assertTrue(Path(str(record) + ".dat").is_file()) signal = load_ptbxl_record(record) @@ -262,7 +274,7 @@ def test_load_committed_fixture_shape_channels_time(self): self.assertNotEqual(signal.shape[0], signal.shape[1]) def test_load_strips_extension_never_appends(self): - record = FIXTURE_ROOT / "records100" / "00000" / "00001_lr" + record = self.data_root / "records100" / "00000" / "00001_lr" signal = load_ptbxl_record(Path(str(record) + ".hea")) self.assertEqual(signal.shape, (12, 50)) self.assertNotEqual(signal.shape[0], signal.shape[1]) @@ -354,6 +366,243 @@ def subset(self, indices): self.assertEqual([s["id"] for s in val._samples], ["b"]) self.assertEqual([s["id"] for s in test._samples], ["c"]) + def test_split_by_strat_fold_uses_precomputed_folds(self): + class _BoomDS: + def __init__(self, n): + self._n = n + + def __len__(self): + return self._n + + def __getitem__(self, i): + raise AssertionError( + "dataset[i] must not be called when folds= is provided" + ) + + def subset(self, indices): + return list(indices) + + train, val, test = split_by_strat_fold( + _BoomDS(4), + train_folds=(1, 3), + val_folds=(9,), + test_folds=(10,), + folds=[1, 9, 10, 3], + ) + self.assertEqual(train, [0, 3]) + self.assertEqual(val, [1]) + self.assertEqual(test, [2]) + + +@unittest.skipUnless( + __import__("importlib").util.find_spec("wfdb") is not None, + "wfdb optional extra not installed", +) +class TestPTBXLSetTaskE2E(unittest.TestCase): + """Exercise the real set_task / litdata / SampleDataset path (no mocks).""" + + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()) + self.data_root = _materialize_fixture(self.tmp / "data") + self.cache_dir = self.tmp / "meta_cache" + self.dataset = PTBXLDataset( + root=str(self.data_root), + metadata_cache_dir=self.cache_dir, + sampling_rate=100, + cache_dir=self.tmp / "pyhealth_cache", + ) + self.task = PTBXLSuperclassClassification( + scp_statements_path=self.data_root / "scp_statements.csv", + ) + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_set_task_builds_samples_without_mocks(self): + samples = self.dataset.set_task(self.task, num_workers=1) + # Keep: ecg 1 NORM, 2 MI, 4 HYP+MI. Drop: 3 PACE-only and 5 empty {}. + self.assertEqual(len(samples), 3) + + sample = samples[0] + self.assertEqual(tuple(sample["signal"].shape), (12, 50)) + self.assertNotEqual(sample["signal"].shape[0], sample["signal"].shape[1]) + self.assertIn("strat_fold", sample) + self.assertIn("labels", sample) + # Multi-hot from MultiLabelProcessor, not a list of strings. + self.assertEqual(tuple(sample["labels"].shape), (3,)) + self.assertTrue(torch.is_tensor(sample["labels"])) + + def test_multilabel_vocab_order_is_alphabetical_observed(self): + samples = self.dataset.set_task(self.task, num_workers=1) + vocab = samples.output_processors["labels"].label_vocab + # Fixture only observes HYP, MI, NORM — not CD/STTC. Processor sorts. + self.assertEqual(list(vocab.keys()), ["HYP", "MI", "NORM"]) + self.assertEqual(vocab, {"HYP": 0, "MI": 1, "NORM": 2}) + + by_record = {str(s["record_id"]): s for s in samples} + # ecg 1 = NORM → [0, 0, 1]; ecg 2 = MI → [0, 1, 0]; ecg 4 = HYP+MI → [1, 1, 0] + self.assertEqual(by_record["1"]["labels"].tolist(), [0.0, 0.0, 1.0]) + self.assertEqual(by_record["2"]["labels"].tolist(), [0.0, 1.0, 0.0]) + self.assertEqual(by_record["4"]["labels"].tolist(), [1.0, 1.0, 0.0]) + + def test_split_by_strat_fold_on_sample_dataset(self): + samples = self.dataset.set_task(self.task, num_workers=1) + # Kept folds are 1 (ecg 1), 2 (ecg 4), 9 (ecg 2). Fold 10 was dropped. + train, val, test = split_by_strat_fold( + samples, + train_folds=(1,), + val_folds=(2,), + test_folds=(9,), + ) + self.assertEqual(len(train), 1) + self.assertEqual(len(val), 1) + self.assertEqual(len(test), 1) + self.assertEqual(int(train[0]["strat_fold"]), 1) + self.assertEqual(int(val[0]["strat_fold"]), 2) + self.assertEqual(int(test[0]["strat_fold"]), 9) + self.assertEqual(tuple(train[0]["signal"].shape), (12, 50)) + + def test_age_sentinel_through_set_task(self): + """Missing age must be an int sentinel; censored age is clipped to 90.""" + task = PTBXLSuperclassClassification( + scp_statements_path=self.data_root / "scp_statements.csv", + drop_empty_labels=False, + ) + samples = self.dataset.set_task(task, num_workers=1) + by_record = {str(s["record_id"]): s for s in samples} + + missing = by_record["3"] + self.assertTrue(missing["age_is_missing"]) + self.assertIsInstance(missing["age"], (int, np.integer)) + self.assertEqual(int(missing["age"]), -1) + self.assertIsNotNone(missing["age"]) + + censored = by_record["2"] + self.assertTrue(censored["age_is_censored"]) + self.assertFalse(censored["age_is_missing"]) + self.assertEqual(int(censored["age"]), 90) + + +class TestPTBXLCacheIsolation(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()) + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_distinct_data_roots_get_distinct_cache_dirs(self): + root_a = _materialize_fixture(self.tmp / "a") + root_b = _materialize_fixture(self.tmp / "b") + db_b = pd.read_csv(root_b / "ptbxl_database.csv") + db_b.loc[0, "scp_codes"] = "{'LVH': 100.0}" + db_b.to_csv(root_b / "ptbxl_database.csv", index=False) + + shared_meta = self.tmp / "meta" + shared_cache = self.tmp / "pyhealth_cache" + ds_a = PTBXLDataset( + root=str(root_a), + metadata_cache_dir=shared_meta, + sampling_rate=100, + cache_dir=shared_cache, + ) + ds_b = PTBXLDataset( + root=str(root_b), + metadata_cache_dir=shared_meta, + sampling_rate=100, + cache_dir=shared_cache, + ) + self.assertNotEqual(ds_a.cache_dir, ds_b.cache_dir) + self.assertNotEqual(ds_a.dataset_name, ds_b.dataset_name) + + def test_source_csv_change_invalidates_derived_metadata(self): + data_root = _materialize_fixture(self.tmp / "data") + meta = self.tmp / "meta" + ds1 = PTBXLDataset( + root=str(data_root), + metadata_cache_dir=meta, + sampling_rate=100, + cache_dir=self.tmp / "c1", + ) + name1 = ds1.metadata_file_name + derived1 = pd.read_csv(meta / name1) + + db = pd.read_csv(data_root / "ptbxl_database.csv") + db.loc[0, "scp_codes"] = "{'LVH': 100.0}" + db.to_csv(data_root / "ptbxl_database.csv", index=False) + + ds2 = PTBXLDataset( + root=str(data_root), + metadata_cache_dir=meta, + sampling_rate=100, + cache_dir=self.tmp / "c2", + ) + self.assertNotEqual(ds2.metadata_file_name, name1) + derived2 = pd.read_csv(meta / ds2.metadata_file_name) + codes1 = str( + derived1.loc[derived1["record_id"].astype(str) == "1", "scp_codes"].iloc[0] + ) + codes2 = str( + derived2.loc[derived2["record_id"].astype(str) == "1", "scp_codes"].iloc[0] + ) + self.assertNotEqual(codes1, codes2) + + +class TestPTBXLTaskStability(unittest.TestCase): + def setUp(self): + self.tmp = Path(tempfile.mkdtemp()) + self.data_root = _materialize_fixture(self.tmp / "data") + self.dataset = PTBXLDataset( + root=str(self.data_root), + metadata_cache_dir=self.tmp / "meta_cache", + sampling_rate=100, + cache_dir=self.tmp / "pyhealth_cache", + ) + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def test_vars_task_stable_after_call(self): + task = PTBXLSuperclassClassification( + scp_statements_path=self.data_root / "scp_statements.csv", + ) + before = dict(vars(task)) + fake_signal = np.zeros((12, 50), dtype=np.float32) + with patch( + "pyhealth.datasets.ptbxl.load_ptbxl_record", return_value=fake_signal + ): + patient = self.dataset.get_patient(self.dataset.unique_patient_ids[0]) + task(patient) + after = dict(vars(task)) + self.assertEqual(before, after) + + def test_age_sentinel_from_task_call(self): + task = PTBXLSuperclassClassification( + scp_statements_path=self.data_root / "scp_statements.csv", + drop_empty_labels=False, + ) + fake_signal = np.zeros((12, 50), dtype=np.float32) + with patch( + "pyhealth.datasets.ptbxl.load_ptbxl_record", return_value=fake_signal + ): + samples = [] + for patient in self.dataset.iter_patients(): + samples.extend(task(patient)) + by_record = {str(s["record_id"]): s for s in samples} + + missing = by_record["3"] + self.assertTrue(missing["age_is_missing"]) + self.assertIsInstance(missing["age"], int) + self.assertEqual(missing["age"], -1) + + censored = by_record["2"] + self.assertTrue(censored["age_is_censored"]) + self.assertEqual(censored["age"], 90) + + present = by_record["1"] + self.assertFalse(present["age_is_missing"]) + self.assertFalse(present["age_is_censored"]) + self.assertEqual(present["age"], 65) + class TestWFDBImportError(unittest.TestCase): def test_import_error_points_at_extra(self): From be6fb1c1e221cdee6f46bb34acb5a8c671363706 Mon Sep 17 00:00:00 2001 From: AxelNoun <150222552+AxelNoun@users.noreply.github.com> Date: Wed, 19 Aug 2026 00:30:44 +0200 Subject: [PATCH 09/13] docs: correct PTB-XL v1.0.3 counts, label order, and API toctree v1.0.3 is 21,799 records / 18,869 patients. Document that MultiLabelProcessor emits CD, HYP, MI, NORM, STTC. Harmonize the datasets toctree and reuse the ptbxl extra in the pixi test env. Co-authored-by: Cursor --- docs/api/datasets.rst | 2 +- ...tbxl.rst => pyhealth.datasets.PTBXLDataset.rst} | 14 ++++++++------ docs/api/tasks/pyhealth.tasks.ptbxl.rst | 6 ++++-- pyhealth/datasets/__init__.py | 2 +- pyproject.toml | 4 +--- 5 files changed, 15 insertions(+), 13 deletions(-) rename docs/api/datasets/{pyhealth.datasets.ptbxl.rst => pyhealth.datasets.PTBXLDataset.rst} (50%) diff --git a/docs/api/datasets.rst b/docs/api/datasets.rst index 02dd68a57..933fa34a5 100644 --- a/docs/api/datasets.rst +++ b/docs/api/datasets.rst @@ -243,7 +243,7 @@ Available Datasets datasets/pyhealth.datasets.ChestXray14Dataset datasets/pyhealth.datasets.PhysioNetDeIDDataset datasets/pyhealth.datasets.EEGBCIDataset - datasets/pyhealth.datasets.ptbxl + PTB-XL Dataset datasets/pyhealth.datasets.TUABDataset datasets/pyhealth.datasets.TUEVDataset datasets/pyhealth.datasets.ClinVarDataset diff --git a/docs/api/datasets/pyhealth.datasets.ptbxl.rst b/docs/api/datasets/pyhealth.datasets.PTBXLDataset.rst similarity index 50% rename from docs/api/datasets/pyhealth.datasets.ptbxl.rst rename to docs/api/datasets/pyhealth.datasets.PTBXLDataset.rst index a7a8c927c..27a77785d 100644 --- a/docs/api/datasets/pyhealth.datasets.ptbxl.rst +++ b/docs/api/datasets/pyhealth.datasets.PTBXLDataset.rst @@ -1,14 +1,16 @@ -pyhealth.datasets.ptbxl -======================= +pyhealth.datasets.PTBXLDataset +============================== Overview -------- PTB-XL is a large publicly available 12-lead ECG dataset from PhysioNet -(version 1.0.3). It contains 21,799 clinical ECG records of 10 seconds from -18,885 patients, with multi-label SCP-ECG statements, official stratified -folds (``strat_fold``), and demographic / site / device metadata suited to -shift-aware evaluation. +(version 1.0.3). It contains **21,799** clinical ECG records of 10 seconds +from **18,869** patients, with multi-label SCP-ECG statements, official +stratified folds (``strat_fold``), and demographic / site / device metadata +suited to shift-aware evaluation. Only v1.0.3 is supported (v1.0.1 figures +of 21,837 records / 18,885 patients come from Wagner et al., Scientific +Data 2020, and must not be mixed with the PhysioNet v1.0.3 counts). For more information see `PhysioNet PTB-XL v1.0.3 `_ and Wagner et al., diff --git a/docs/api/tasks/pyhealth.tasks.ptbxl.rst b/docs/api/tasks/pyhealth.tasks.ptbxl.rst index 19b48c404..8e8fba74b 100644 --- a/docs/api/tasks/pyhealth.tasks.ptbxl.rst +++ b/docs/api/tasks/pyhealth.tasks.ptbxl.rst @@ -9,8 +9,10 @@ implements the official **5-diagnostic-superclass** classification task (``NORM``, ``MI``, ``STTC``, ``CD``, ``HYP``) by aggregating diagnostic SCP statements via ``scp_statements.csv``. -Empty superclass label sets (407 / 21,799 records after aggregation; mainly -pacemaker ECGs) are dropped by default. Official fold splitting is provided +Empty superclass label sets (≈400 records on v1.0.1, Wagner et al. +Table 9; mainly pacemaker ECGs) are dropped by default. On the full +v1.0.3 corpus, ``MultiLabelProcessor`` emits labels in alphabetical +order: ``CD, HYP, MI, NORM, STTC``. Official fold splitting is provided by :func:`pyhealth.datasets.split_by_strat_fold`. API Reference diff --git a/pyhealth/datasets/__init__.py b/pyhealth/datasets/__init__.py index d489a8beb..2a92c6ab1 100644 --- a/pyhealth/datasets/__init__.py +++ b/pyhealth/datasets/__init__.py @@ -80,7 +80,7 @@ def __init__(self, *args, **kwargs): split_by_sample_conformal, split_by_sample_conformal_tuh, split_by_sample_tuh, - split_by_strat_fold as split_by_strat_fold, + split_by_strat_fold, split_by_visit, split_by_visit_conformal, ) diff --git a/pyproject.toml b/pyproject.toml index 494469d9a..1976c7a66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,9 +137,7 @@ python = "~=3.13" pyhealth = { path = ".", editable = true } [tool.pixi.feature.test.pypi-dependencies] -pyhealth = { path = ".", editable = true } -# Optional PTB-XL waveform dependency — keep CI covering load_ptbxl_record. -wfdb = ">=4.1.0" +pyhealth = { path = ".", editable = true, extras = ["ptbxl"] } [tool.pixi.feature.nlp.pypi-dependencies] pyhealth = { path = ".", editable = true } From b1e1891aed14345653cab735e556c4e1fa8395cc Mon Sep 17 00:00:00 2001 From: AxelNoun <150222552+AxelNoun@users.noreply.github.com> Date: Wed, 19 Aug 2026 00:31:37 +0200 Subject: [PATCH 10/13] fix(datasets): keep split_by_strat_fold re-export alias for ruff F401 The redundant-looking 'as' is required by ruff's unused-import rule for module re-exports, same as EEGBCIDataset / PTBXLDataset in this file. Co-authored-by: Cursor --- pyhealth/datasets/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyhealth/datasets/__init__.py b/pyhealth/datasets/__init__.py index 2a92c6ab1..d489a8beb 100644 --- a/pyhealth/datasets/__init__.py +++ b/pyhealth/datasets/__init__.py @@ -80,7 +80,7 @@ def __init__(self, *args, **kwargs): split_by_sample_conformal, split_by_sample_conformal_tuh, split_by_sample_tuh, - split_by_strat_fold, + split_by_strat_fold as split_by_strat_fold, split_by_visit, split_by_visit_conformal, ) From a8fb19410a383b66a8d68fbe7a39770bedd6d92c Mon Sep 17 00:00:00 2001 From: AxelNoun <150222552+AxelNoun@users.noreply.github.com> Date: Wed, 19 Aug 2026 00:43:07 +0200 Subject: [PATCH 11/13] fix(datasets): keep PTBXLDataset.root as the user data path BaseDataset.load_table concatenates root/file_path and rejects absolute paths, so override load_table to read the derived CSV from metadata_cache_dir while leaving self.root equal to the constructor path. Co-authored-by: Cursor --- .../pyhealth.datasets.PTBXLDataset.rst | 7 ++ pyhealth/datasets/ptbxl.py | 36 ++++++++-- tests/core/test_ptbxl.py | 68 ++++++++++++++++++- 3 files changed, 104 insertions(+), 7 deletions(-) diff --git a/docs/api/datasets/pyhealth.datasets.PTBXLDataset.rst b/docs/api/datasets/pyhealth.datasets.PTBXLDataset.rst index 27a77785d..fca949e18 100644 --- a/docs/api/datasets/pyhealth.datasets.PTBXLDataset.rst +++ b/docs/api/datasets/pyhealth.datasets.PTBXLDataset.rst @@ -18,6 +18,13 @@ For more information see `PhysioNet PTB-XL v1.0.3 Optional dependency: install waveform I/O with ``pip install 'pyhealth[ptbxl]'``. +``PTBXLDataset.root`` is the version directory passed to the constructor. +Derived metadata CSVs are written under ``metadata_cache_dir`` (default +``~/.cache/pyhealth/datasets/ptbxl/``), not into ``root``, so read-only +data mounts stay untouched. ``BaseDataset.load_table`` concatenates +``root / file_path`` and cannot take an absolute path; PTB-XL overrides +``load_table`` to read the derived CSV from the metadata cache. + API Reference ------------- diff --git a/pyhealth/datasets/ptbxl.py b/pyhealth/datasets/ptbxl.py index cdc2d36c9..1f49f6375 100644 --- a/pyhealth/datasets/ptbxl.py +++ b/pyhealth/datasets/ptbxl.py @@ -332,6 +332,8 @@ class PTBXLDataset(BaseDataset): **kwargs: Forwarded to :class:`BaseDataset` (``cache_dir``, ``dev``, …). Attributes: + root (str): Same as the constructor ``root`` (the PTB-XL version + directory). Derived metadata is *not* stored here. data_root (Path): User-provided PTB-XL version root (waveforms + CSVs). sampling_rate (int): Selected waveform rate (100 or 500). metadata_cache_dir (Path): Directory holding derived metadata CSVs. @@ -397,25 +399,47 @@ def __init__( resolved_config_path = self._write_resolved_config(package_config) # BaseDataset._init_cache_dir keys on {root, tables, dataset_name, dev}. - # root is the shared metadata cache (see comment on super().__init__), - # so uniqueness of global_event_df must come from dataset_name — same - # pattern as EEGBCIDataset._metadata_cache_key. + # root is the user data path (see load_table). dataset_name still + # includes a content hash of the derived CSV so replacing + # ptbxl_database.csv in-place does not reuse a stale global_event_df. base_name = dataset_name or f"ptbxl_{self.sampling_rate}hz" dataset_name = ( f"{base_name}_{root_cache_key(self.data_root)}_" f"{self._metadata_cache_key()}" ) - # BaseDataset.root is the metadata cache (CSV location); waveforms stay - # under data_root via absolute signal_file paths in the CSV. + # Keep self.root as the user-provided PTB-XL path so stats()/logs + # match what the caller passed. Derived CSVs live in + # metadata_cache_dir (read-only data mounts). BaseDataset.load_table + # builds csv_path as f"{self.root}/{file_path}" and does not accept + # an absolute file_path, so load_table (below) temporarily points + # root at the metadata cache for the scan only. super().__init__( - root=str(self.metadata_cache_dir), + root=str(self.data_root), tables=["records"], dataset_name=dataset_name, config_path=str(resolved_config_path), **kwargs, ) + def load_table(self, table_name: str): + """Load tables from ``metadata_cache_dir``, not the user data root. + + See the comment on ``super().__init__`` in :meth:`__init__`. + + Args: + table_name (str): Table name from the YAML config. + + Returns: + The Dask dataframe produced by :meth:`BaseDataset.load_table`. + """ + original_root = self.root + self.root = str(self.metadata_cache_dir) + try: + return super().load_table(table_name) + finally: + self.root = original_root + def _write_resolved_config(self, package_config: Path) -> Path: """Write a cache-local YAML with the derived metadata CSV filename. diff --git a/tests/core/test_ptbxl.py b/tests/core/test_ptbxl.py index e4f4b9cf7..238806628 100644 --- a/tests/core/test_ptbxl.py +++ b/tests/core/test_ptbxl.py @@ -234,8 +234,9 @@ def test_end_to_end_reads_event_from_fixture(self): self.assertEqual( ds.config.tables["records"].file_path, ds.metadata_file_name ) + self.assertEqual(Path(ds.root).resolve(), self.data_root.resolve()) self.assertTrue( - (Path(ds.root) / ds.config.tables["records"].file_path).is_file() + (ds.metadata_cache_dir / ds.config.tables["records"].file_path).is_file() ) patient_ids = ds.unique_patient_ids @@ -250,6 +251,8 @@ def test_end_to_end_reads_event_from_fixture(self): self.assertTrue(str(event.signal_file)) # Absolute waveform path must live under the fixture data root. Path(str(event.signal_file)).resolve().relative_to(self.data_root.resolve()) + # load_table must restore self.root after scanning the cache CSV. + self.assertEqual(Path(ds.root).resolve(), self.data_root.resolve()) @unittest.skipUnless( @@ -366,6 +369,57 @@ def subset(self, indices): self.assertEqual([s["id"] for s in val._samples], ["b"]) self.assertEqual([s["id"] for s in test._samples], ["c"]) + def test_split_by_strat_fold_check_patient_disjoint(self): + class _FakeDS: + def __init__(self, samples): + self._samples = samples + self.patient_to_index = {} + for i, sample in enumerate(samples): + self.patient_to_index.setdefault(sample["patient_id"], []).append( + i + ) + + def __len__(self): + return len(self._samples) + + def __getitem__(self, i): + return self._samples[i] + + def subset(self, indices): + return _FakeDS([self._samples[i] for i in indices]) + + leaked = [ + {"strat_fold": 1, "patient_id": "15709", "id": "a"}, + {"strat_fold": 9, "patient_id": "15709", "id": "b"}, + {"strat_fold": 10, "patient_id": "99", "id": "c"}, + {"strat_fold": 3, "patient_id": "42", "id": "d"}, + ] + train, val, test = split_by_strat_fold(_FakeDS(leaked)) + self.assertEqual([s["id"] for s in train._samples], ["a", "d"]) + self.assertEqual([s["id"] for s in val._samples], ["b"]) + self.assertEqual([s["id"] for s in test._samples], ["c"]) + with self.assertRaisesRegex(ValueError, r"15709"): + split_by_strat_fold(_FakeDS(leaked), check_patient_disjoint=True) + + class _NoMap: + def __len__(self): + return 1 + + def __getitem__(self, i): + return {"strat_fold": 1} + + def subset(self, indices): + return indices + + with self.assertRaises(TypeError): + split_by_strat_fold( + _NoMap(), + train_folds=(1,), + val_folds=(9,), + test_folds=(10,), + check_patient_disjoint=True, + ) + def test_split_by_strat_fold_uses_precomputed_folds(self): class _BoomDS: def __init__(self, n): @@ -462,6 +516,18 @@ def test_split_by_strat_fold_on_sample_dataset(self): self.assertEqual(int(test[0]["strat_fold"]), 9) self.assertEqual(tuple(train[0]["signal"].shape), (12, 50)) + def test_split_by_strat_fold_detects_fixture_patient_leak(self): + samples = self.dataset.set_task(self.task, num_workers=1) + # Fixture: ecg 1 (fold 1) and ecg 2 (fold 9) share patient_id 15709. + with self.assertRaisesRegex(ValueError, r"15709"): + split_by_strat_fold( + samples, + train_folds=(1,), + val_folds=(2,), + test_folds=(9,), + check_patient_disjoint=True, + ) + def test_age_sentinel_through_set_task(self): """Missing age must be an int sentinel; censored age is clipped to 90.""" task = PTBXLSuperclassClassification( From fe1007f346d88da5703a4bdfacc093b7263d5073 Mon Sep 17 00:00:00 2001 From: AxelNoun <150222552+AxelNoun@users.noreply.github.com> Date: Wed, 19 Aug 2026 00:43:27 +0200 Subject: [PATCH 12/13] feat(datasets): add check_patient_disjoint to split_by_strat_fold Official PTB-XL folds are patient-disjoint; this flag verifies that property via patient_to_index. Default False keeps current callers working. The synthetic fixture leaks patient 15709 across folds 1 and 9 so the check is actually exercised. Co-authored-by: Cursor --- pyhealth/datasets/splitter.py | 42 ++++++++++++++++++++++++++--- test-resources/core/ptbxl/README.md | 2 +- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/pyhealth/datasets/splitter.py b/pyhealth/datasets/splitter.py index 6138d6f7e..3e3d33235 100644 --- a/pyhealth/datasets/splitter.py +++ b/pyhealth/datasets/splitter.py @@ -187,11 +187,15 @@ def split_by_strat_fold( test_folds: tuple[int, ...] | list[int] = (10,), fold_field: str = "strat_fold", folds: tuple[int, ...] | list[int] | None = None, + check_patient_disjoint: bool = False, ): """Split a sample dataset using integer fold assignments. The recommended PTB-XL protocol uses folds 1–8 for training, fold 9 for - validation, and fold 10 for testing. + validation, and fold 10 for testing. Official PTB-XL folds are + patient-disjoint; this helper does not assume that of arbitrary data. + Pass ``check_patient_disjoint=True`` to verify no ``patient_id`` appears + in more than one of train/val/test (uses ``dataset.patient_to_index``). Pass ``folds`` (one integer per sample, aligned with dataset indices) to avoid decoding every sample via ``dataset[i]``. When omitted, fold values @@ -210,14 +214,19 @@ def split_by_strat_fold( is not provided. Defaults to ``\"strat_fold\"``. folds (Tuple[int, ...] | List[int] | None): Optional precomputed fold for each index. Length must equal ``len(dataset)``. + check_patient_disjoint (bool): If True, raise ``ValueError`` when a + ``patient_id`` has samples in more than one split. Requires + ``dataset.patient_to_index``. Defaults to False. Returns: tuple: ``(train_dataset, val_dataset, test_dataset)`` subsets. Raises: KeyError: If ``folds`` is omitted and a sample is missing ``fold_field``. - ValueError: If fold sets overlap, ``folds`` has the wrong length, or - a sample fold is unassigned. + TypeError: If ``check_patient_disjoint`` is True but the dataset has + no ``patient_to_index``. + ValueError: If fold sets overlap, ``folds`` has the wrong length, a + sample fold is unassigned, or a patient leaks across splits. Examples: >>> # doctest: +SKIP @@ -263,6 +272,33 @@ def split_by_strat_fold( f"{fold_field}={fold} is not in train/val/test fold sets" ) + if check_patient_disjoint: + patient_to_index = getattr(dataset, "patient_to_index", None) + if patient_to_index is None: + raise TypeError( + "check_patient_disjoint=True requires dataset.patient_to_index" + ) + split_of_index = {} + for i in train_index: + split_of_index[i] = "train" + for i in val_index: + split_of_index[i] = "val" + for i in test_index: + split_of_index[i] = "test" + leaks = [] + for pid, indices in patient_to_index.items(): + splits = {split_of_index[i] for i in indices if i in split_of_index} + if len(splits) > 1: + leaks.append((str(pid), splits)) + if leaks: + details = "; ".join( + f"{pid} in {sorted(splits)}" for pid, splits in leaks + ) + raise ValueError( + "check_patient_disjoint: " + f"{len(leaks)} patient(s) appear in multiple splits: {details}" + ) + return ( dataset.subset(train_index), # type: ignore dataset.subset(val_index), # type: ignore diff --git a/test-resources/core/ptbxl/README.md b/test-resources/core/ptbxl/README.md index d34c35cd1..7b4762849 100644 --- a/test-resources/core/ptbxl/README.md +++ b/test-resources/core/ptbxl/README.md @@ -12,4 +12,4 @@ # - ecg_id=3: missing age + non-empty non-diagnostic dict (PACE-only) # - ecg_id=4: true multi-label (IMI+LVH → MI and HYP) # - ecg_id=5: empty scp_codes dict {} -# - ecg_id=1 and 2 share patient_id (patient-level fold leakage checks) +# - ecg_id=1 and 2 share patient_id (used by check_patient_disjoint tests) From 0fccca0d472e2a49e16953ac1563b1e9eafbf5df Mon Sep 17 00:00:00 2001 From: AxelNoun <150222552+AxelNoun@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:53:25 +0200 Subject: [PATCH 13/13] docs(datasets): describe the post-#1265 cache key in PTBXLDataset BaseDataset._init_cache_dir now also hashes the resolved config and fingerprints the table sources. The source fingerprint resolves the derived CSV against the user data root, where it does not live, so it is empty for PTBXLDataset; say so, and say what the cache identity rests on instead. --- pyhealth/datasets/ptbxl.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pyhealth/datasets/ptbxl.py b/pyhealth/datasets/ptbxl.py index 1f49f6375..e527527ce 100644 --- a/pyhealth/datasets/ptbxl.py +++ b/pyhealth/datasets/ptbxl.py @@ -398,9 +398,14 @@ def __init__( # config must already be correct when that first happens. resolved_config_path = self._write_resolved_config(package_config) - # BaseDataset._init_cache_dir keys on {root, tables, dataset_name, dev}. - # root is the user data path (see load_table). dataset_name still - # includes a content hash of the derived CSV so replacing + # BaseDataset._init_cache_dir keys on root, tables, dataset_name, dev, + # a hash of the resolved config and a size/mtime fingerprint of each + # table source. The fingerprint resolves file_path against root (the + # user data path, see load_table), where the derived CSV does not + # live, so it is empty here (key["sources"] == {"records": [[]]}). + # Cache identity therefore rests on the config hash (file_path + # encodes the root and ptbxl_database.csv hashes) and on dataset_name, + # which adds a content hash of the derived CSV so replacing # ptbxl_database.csv in-place does not reuse a stale global_event_df. base_name = dataset_name or f"ptbxl_{self.sampling_rate}hz" dataset_name = (