From 04797a543d67b67e64808e944797ac6313178277 Mon Sep 17 00:00:00 2001 From: Sam Minot Date: Wed, 19 Aug 2026 11:08:23 -0700 Subject: [PATCH 1/8] Add package orientation and document SDK authentication The `cirro` and `cirro.sdk` packages had no module docstrings, so the pdoc landing page and `help(cirro)` -- the first place a reader lands -- said nothing about how to authenticate or how the classes relate. `DataPortal()` falls through to a device-code login that blocks on a browser flow, which hangs in any non-interactive context. Neither the constructor nor `DataPortalLogin` mentioned this or pointed at `ClientCredentialsAuth`. Co-Authored-By: Claude Opus 5 --- cirro/__init__.py | 109 ++++++++++++++++++++++++++++++++++++++++++ cirro/sdk/__init__.py | 20 ++++++++ cirro/sdk/login.py | 28 ++++++++++- cirro/sdk/portal.py | 54 +++++++++++++++++++-- 4 files changed, 204 insertions(+), 7 deletions(-) diff --git a/cirro/__init__.py b/cirro/__init__.py index 119f6b4c..93f20fcc 100644 --- a/cirro/__init__.py +++ b/cirro/__init__.py @@ -1,3 +1,112 @@ +""" +Python SDK and command-line interface for the [Cirro](https://cirro.bio) platform. + +Install with `pip install cirro`. + +## Authentication + +`DataPortal` needs to know which Cirro instance to talk to and how to +authenticate. The instance comes from the `base_url` argument, falling back to +the `CIRRO_BASE_URL` environment variable, then to the saved configuration in +`~/.cirro/config.ini` (written by `cirro configure`). + +There are three ways to authenticate: + +1. **Interactive (the default).** `DataPortal()` uses the saved configuration. + If none exists it starts a device-code login, prints a URL, and **blocks + until the user completes the login in a browser**. Avoid this in scripts and + automated sessions -- there is nobody to click the link, so it will hang + until the device code expires. + +2. **Headless.** OAuth client credentials never prompt, so this is the option + to use for automation: + + ```python + import os + from cirro import CirroApi, DataPortal + from cirro.auth.client_creds import ClientCredentialsAuth + from cirro.config import AppConfig + + config = AppConfig(base_url="app.cirro.bio") + auth = ClientCredentialsAuth( + os.environ["CIRRO_CLIENT_ID"], + os.environ["CIRRO_CLIENT_SECRET"], + auth_endpoint=config.auth_endpoint + ) + portal = DataPortal(client=CirroApi(auth_info=auth)) + ``` + + See [OAuth Apps](https://docs.cirro.bio/cli-sdk/oauth-apps/) for how to + create the client ID and secret. + +3. **Non-blocking browser login.** `cirro.sdk.login.DataPortalLogin` returns + the authorization message so you can display it yourself, and blocks only + when you call `await_completion()`. + +## Quickstart + +```python +from cirro import DataPortal + +portal = DataPortal(base_url="app.cirro.bio") + +# Browse +for project in portal.list_projects(): + print(project.name) + +# Read a file straight into a DataFrame, without downloading it +df = portal.read_file("Name of Project", "Name of Dataset", glob="*.csv") + +# Launch an analysis on an existing dataset +dataset = portal.get_dataset(project="Name of Project", dataset="Name of Dataset") +new_dataset_id = dataset.run_analysis( + name="Name of the output dataset", + process="Name or ID of the process to run", + params={} +) +``` + +## Object model + +- `cirro.sdk.portal.DataPortal` -- entry point; lists projects, processes, + and reference types. +- `cirro.sdk.project.DataPortalProject` -- a permissions boundary holding + datasets and reference data; uploads new datasets. +- `cirro.sdk.dataset.DataPortalDataset` -- a collection of files, either + uploaded or produced by an analysis; reads files and launches analyses. +- `cirro.sdk.file.DataPortalFile` -- one file; read it into memory + (`read_csv`, `read_json`, ...) or download it. +- `cirro.sdk.process.DataPortalProcess` -- a pipeline that can be run, or a + data type that datasets can be uploaded as. +- `cirro.sdk.task.DataPortalTask` -- one task from a Nextflow execution, used + for debugging failed analyses. +- `cirro.sdk.reference.DataPortalReference` -- reference data (genomes, + annotations) available to a project. +- `cirro.cirro_client.CirroApi` -- the lower-level typed API client; use it + when the classes above do not cover what you need. + +Projects, datasets, and processes can be looked up by either name or ID -- +`get_project`, `get_dataset`, and `run_analysis` all accept either. + +Every `list_*` method returns a `list` subclass +(`cirro.sdk.asset.DataPortalAssets`) with extra lookup helpers: +`get_by_name`, `get_by_id`, and `filter_by_pattern`. + +## Freshness + +These objects hold a snapshot of what the API returned when they were built. +Properties such as `cirro.sdk.dataset.DataPortalDataset.status` and +`cirro.sdk.dataset.DataPortalDataset.logs` will not change on an object you +already have. To watch a running analysis, call `portal.get_dataset(...)` again +each time round the loop. + +## Worked examples + +The [samples directory](https://github.com/CirroBio/Cirro-SDK-Python/tree/main/samples) +holds runnable notebooks for uploading, downloading, reading files, running and +debugging analyses, managing reference data, and integrating pipelines. +""" + import cirro.file_utils # noqa from cirro.cirro_client import CirroApi from cirro.sdk.dataset import DataPortalDataset diff --git a/cirro/sdk/__init__.py b/cirro/sdk/__init__.py index e69de29b..7987bda8 100644 --- a/cirro/sdk/__init__.py +++ b/cirro/sdk/__init__.py @@ -0,0 +1,20 @@ +""" +The high-level, object-oriented interface to Cirro. + +Start from `cirro.sdk.portal.DataPortal`, which is re-exported as +`cirro.DataPortal`. Every other class in this package is reached from it rather +than constructed directly: + +``` +DataPortal +├── list_projects() -> DataPortalProject +│ ├── list_datasets() -> DataPortalDataset +│ │ ├── list_files() -> DataPortalFile +│ │ └── tasks -> DataPortalTask +│ └── list_references() -> DataPortalReference +├── list_processes() -> DataPortalProcess +└── list_reference_types() -> DataPortalReferenceType +``` + +For the lower-level typed API client, see `cirro.cirro_client.CirroApi`. +""" diff --git a/cirro/sdk/login.py b/cirro/sdk/login.py index 1d86320e..bf06be5a 100644 --- a/cirro/sdk/login.py +++ b/cirro/sdk/login.py @@ -9,7 +9,13 @@ class DataPortalLogin: Start the login process, obtaining the authorization message from Cirro needed to confirm the user identity. - Useful when you need to authenticate a user in a non-blocking way. + Use this when a person is available to complete the login but you need to + control when your code blocks -- for example to render the authorization + message in a web page or notebook before waiting. Constructing the object + does not block; only `await_completion` does. + + For automation with no person in the loop, use OAuth client credentials + instead -- see `cirro.sdk.portal.DataPortal`. Usage: @@ -29,6 +35,16 @@ class DataPortalLogin: auth_info: DeviceCodeAuth def __init__(self, base_url: str = None, enable_cache=False): + """ + Begin a device-code login without waiting for it to complete. + + Args: + base_url (str): Base URL of the Cirro instance, e.g. `app.cirro.bio`. + If omitted, falls back to the `CIRRO_BASE_URL` environment variable, + then to the saved configuration. + enable_cache (bool): If True, save the resulting token to the system + keychain so later sessions can reuse it. + """ app_config = AppConfig(base_url=base_url) self.base_url = base_url @@ -52,7 +68,15 @@ def auth_message_markdown(self) -> str: return self.auth_info.auth_message_markdown def await_completion(self) -> DataPortal: - """Complete the login process and return an authenticated client""" + """ + Block until the user completes the login in their browser. + + Returns: + `cirro.sdk.portal.DataPortal`: an authenticated portal object. + + Raises: + RuntimeError: if the device code expires before the login completes. + """ # Block until the user completes the login flow self.auth_info.await_completion() diff --git a/cirro/sdk/portal.py b/cirro/sdk/portal.py index 7f4727c4..999355c3 100644 --- a/cirro/sdk/portal.py +++ b/cirro/sdk/portal.py @@ -11,18 +11,62 @@ class DataPortal: """ - Helper functions for exploring the Projects, Datasets, Samples, and Files - available in the Data Portal. + The entry point for the SDK: explore the Projects, Datasets, Samples, and + Files available in Cirro, and launch analyses on them. + + Projects and datasets can be looked up by either name or ID. + + ```python + from cirro import DataPortal + + portal = DataPortal(base_url="app.cirro.bio") + dataset = portal.get_dataset(project="Name of Project", dataset="Name of Dataset") + ``` """ def __init__(self, base_url: str = None, client: CirroApi = None): """ Set up the DataPortal object, establishing an authenticated connection. + **This may block on an interactive login.** With no `client` and no + saved configuration, the constructor starts a device-code login: it + prints a URL and waits until someone completes the login in a browser. + In a script or an automated session there is nobody to click the link, + so it will hang until the device code expires. + + To authenticate without prompting, build a `cirro.cirro_client.CirroApi` + with OAuth client credentials and pass it as `client`: + + ```python + import os + from cirro import CirroApi, DataPortal + from cirro.auth.client_creds import ClientCredentialsAuth + from cirro.config import AppConfig + + config = AppConfig(base_url="app.cirro.bio") + auth = ClientCredentialsAuth( + os.environ["CIRRO_CLIENT_ID"], + os.environ["CIRRO_CLIENT_SECRET"], + auth_endpoint=config.auth_endpoint + ) + portal = DataPortal(client=CirroApi(auth_info=auth)) + ``` + + To drive the browser login yourself without blocking in the + constructor, use `cirro.sdk.login.DataPortalLogin` instead. + Args: - base_url (str): Optional base URL of the Cirro instance - (if not provided, it uses the `CIRRO_BASE_URL` environment variable, or the config file) - client (`cirro.cirro_client.CirroApi`): Optional pre-configured client + base_url (str): Base URL of the Cirro instance, e.g. `app.cirro.bio`. + If omitted, falls back to the `CIRRO_BASE_URL` environment variable, + then to the `base_url` saved in `~/.cirro/config.ini` by + `cirro configure`. Raises `RuntimeError` if none of these is set. + client (`cirro.cirro_client.CirroApi`): Pre-configured client. Supply + this to control how authentication happens; when given, `base_url` + is ignored. + + Raises: + RuntimeError: if no base URL can be determined, or the instance + cannot be reached. Example: ```python From 93acd7d93e64decf43260ae5c58fdd6ba8ccd8f9 Mon Sep 17 00:00:00 2001 From: Sam Minot Date: Wed, 19 Aug 2026 11:09:32 -0700 Subject: [PATCH 2/8] Fix unrunnable examples and undiscoverable run_analysis params Four docstring examples opened with `from cirro import DataPortal()`, which is a SyntaxError. Two `Returns:` blocks named an import statement rather than the type returned. `run_analysis` documented `params` only as "Analysis parameters", with no pointer to `get_parameter_spec`, which is the only way to learn what keys a process accepts. It also did not say that the call is asynchronous or how to follow the resulting dataset. Co-Authored-By: Claude Opus 5 --- cirro/sdk/dataset.py | 37 ++++++++++++++++++++++++++++++++++++- cirro/sdk/portal.py | 10 ++++++++-- cirro/sdk/process.py | 16 +++++++++++++++- cirro/sdk/project.py | 2 +- cirro/sdk/reference.py | 2 +- 5 files changed, 61 insertions(+), 6 deletions(-) diff --git a/cirro/sdk/dataset.py b/cirro/sdk/dataset.py index 21495f64..e7e2dac8 100644 --- a/cirro/sdk/dataset.py +++ b/cirro/sdk/dataset.py @@ -340,11 +340,17 @@ def get_file(self, relative_path: str) -> DataPortalFile: """ Get a file from the dataset using its relative path. + The leading `data/` prefix is optional -- it is tried automatically if + the path is not found as given. + Args: relative_path (str): Relative path of file within the dataset Returns: - `from cirro.sdk.file import DataPortalFile` + `cirro.sdk.file.DataPortalFile` + + Raises: + DataPortalAssetNotFound: if no file in the dataset has this path. """ # Get the list of files in this dataset @@ -546,6 +552,35 @@ def run_analysis( The process can be provided as either a DataPortalProcess object, or a string which corresponds to the name or ID of the process. + The analysis runs asynchronously. The output dataset is registered + immediately in a `PENDING` state and this method returns as soon as the + job is submitted -- it does not wait for the analysis to finish. + + To find out which `params` a process accepts, ask the process itself: + + ```python + process = portal.get_process_by_name("Name of process") + spec = process.get_parameter_spec() + spec.print() # human-readable listing of every parameter + spec.validate_params(params) # raises if params do not fit the schema + ``` + + To follow the analysis, re-fetch the dataset each time round the loop. + `status` on a dataset object you already hold reflects the moment that + object was built and will never change: + + ```python + from time import sleep + from cirro_api_client.v1.models import Status + + dataset_id = dataset.run_analysis(name="Output", process="Name of process") + while True: + result = portal.get_dataset(project=dataset.project_id, dataset=dataset_id) + if result.status in (Status.COMPLETED, Status.FAILED): + break + sleep(30) + ``` + Args: name (str): Name of newly created dataset description (str): Description of newly created dataset diff --git a/cirro/sdk/portal.py b/cirro/sdk/portal.py index 999355c3..b970bac2 100644 --- a/cirro/sdk/portal.py +++ b/cirro/sdk/portal.py @@ -108,11 +108,17 @@ def get_project(self, project: str = None) -> DataPortalProject: """ Return a project identified by ID or name. + Tries to match by ID first, then falls back to matching by name. + Args: project (str): ID or name of project Returns: - `from cirro.sdk.project import DataPortalProject` + `cirro.sdk.project.DataPortalProject` + + Raises: + DataPortalAssetNotFound: if no project matches by either ID or name. + DataPortalInputError: if more than one project has this name. """ try: return self.get_project_by_id(project) @@ -131,7 +137,7 @@ def get_dataset(self, project: str = None, dataset: str = None) -> DataPortalDat `cirro.sdk.dataset.DataPortalDataset` ```python - from cirro import DataPortal() + from cirro import DataPortal portal = DataPortal() dataset = portal.get_dataset( project="id-or-name-of-project", diff --git a/cirro/sdk/process.py b/cirro/sdk/process.py index 83bcf8a3..c99d4640 100644 --- a/cirro/sdk/process.py +++ b/cirro/sdk/process.py @@ -17,7 +17,7 @@ def __init__(self, process: Union[Process, ProcessDetail], client: CirroApi): Instantiate with helper method ```python - from cirro import DataPortal() + from cirro import DataPortal portal = DataPortal() process = portal.get_process_by_name("Process Name") ``` @@ -94,6 +94,14 @@ def __str__(self): def get_parameter_spec(self) -> ParameterSpecification: """ Gets a specification used to describe the parameters used in the process. + + This is the authoritative source for the keys accepted by the `params` + argument of `run_analysis`, along with their types and default values. + Call `print()` on the result for a readable listing, or + `validate_params()` to check a `params` dict before submitting it. + + Returns: + `cirro.models.form_specification.ParameterSpecification` """ return self._client.processes.get_parameter_spec(self.id) @@ -112,6 +120,12 @@ def run_analysis( """ Runs this process on one or more input datasets, returns the ID of the newly created dataset. + The analysis runs asynchronously; this returns as soon as the job is + submitted. Call `get_parameter_spec` to discover which `params` this + process accepts, and see + `cirro.sdk.dataset.DataPortalDataset.run_analysis` for how to follow the + analysis to completion. + Args: name (str): Name of newly created dataset project_id (str): ID of the project to run the analysis in diff --git a/cirro/sdk/project.py b/cirro/sdk/project.py index 89f58c91..d5a4a865 100644 --- a/cirro/sdk/project.py +++ b/cirro/sdk/project.py @@ -27,7 +27,7 @@ def __init__(self, proj: Project, client: CirroApi): Instantiate with helper method ```python - from cirro import DataPortal() + from cirro import DataPortal portal = DataPortal() project = portal.get_project_by_name("Project Name") ``` diff --git a/cirro/sdk/reference.py b/cirro/sdk/reference.py index cfb2a542..0711aa3b 100644 --- a/cirro/sdk/reference.py +++ b/cirro/sdk/reference.py @@ -16,7 +16,7 @@ def __init__(self, ref: Reference, project_id: str, client: CirroApi): """ Instantiate by listing the references which have been added to a particular project ```python - from cirro import DataPortal() + from cirro import DataPortal portal = DataPortal() project = portal.get_project_by_name("Project Name") references = project.list_references() From 90a8f9be90c78680cdd0746bce2d980f6eacaa3b Mon Sep 17 00:00:00 2001 From: Sam Minot Date: Wed, 19 Aug 2026 11:12:29 -0700 Subject: [PATCH 3/8] Document the SDK's caching, staleness and truncation behaviour Several properties return a snapshot that never updates, which reads as a hang when polling a running analysis: `dataset.status` is fixed at construction, and `dataset.logs` / `dataset.tasks` are `cached_property`, so an empty log read before the job starts stays empty for the object's lifetime. Each now says to re-fetch the dataset. `dataset.logs`, `dataset.get_logs()` and `task.logs` are three different logs with near-identical names; each now says which one it is and points at the others. `list_files` and `samples` now state that they truncate silently, and `force_refresh` says what it discards. Also documents the members that had no docstring at all -- `file_count`, `total_size_bytes`, `total_size`, `developer_helper`, `absolute_path`, the two `get_by_id` overrides that only raise -- and the lookup helpers on `DataPortalAssets`, which are what make every `list_*` result more than a list. Co-Authored-By: Claude Opus 5 --- cirro/sdk/asset.py | 75 +++++++++++++++++++++-- cirro/sdk/dataset.py | 77 ++++++++++++++++++++++-- cirro/sdk/developer.py | 14 +++++ cirro/sdk/exceptions.py | 2 +- cirro/sdk/portal.py | 74 +++++++++++++++++++++-- cirro/sdk/project.py | 115 +++++++++++++++++++++++++++++++++--- cirro/sdk/reference.py | 12 ++++ cirro/sdk/reference_type.py | 17 ++++++ 8 files changed, 361 insertions(+), 25 deletions(-) diff --git a/cirro/sdk/asset.py b/cirro/sdk/asset.py index 082200fe..68c7a920 100644 --- a/cirro/sdk/asset.py +++ b/cirro/sdk/asset.py @@ -6,7 +6,12 @@ class DataPortalAsset: - """Base class used for all Data Portal Assets""" + """ + Base class used for all Data Portal Assets. + + Assets are not constructed directly -- each one is obtained from a method on + `cirro.sdk.portal.DataPortal` or on another asset. + """ @property @abstractmethod @@ -23,7 +28,22 @@ def __repr__(self): class DataPortalAssets(List[T]): """ - Generic class with helper functions for any group of assets (projects, datasets, etc.) + A `list` of assets (projects, datasets, files, ...) with lookup helpers. + + Every `list_*` method in the SDK returns one of these rather than a plain + list, so anything you can do with a list works, plus lookup by name or ID + and filtering by wildcard: + + ```python + projects = portal.list_projects() + + for project in projects: # ordinary list iteration + print(project.name) + + project = projects.get_by_name("My Project") + subset = projects.filter_by_pattern("RNA-seq*") + print(projects.description()) # printable summary of them all + ``` """ # Overridden by child classes @@ -36,7 +56,12 @@ def __str__(self): return "\n".join([str(i) for i in self]) def description(self): - """Render a text summary of the assets.""" + """ + Render a text summary of the assets, one block per asset. + + Returns: + str + """ return '\n\n---\n\n'.join([ str(i) @@ -44,7 +69,21 @@ def description(self): ]) def get_by_name(self, name: str) -> T: - """Return the item which matches with name attribute.""" + """ + Return the single item whose `name` attribute matches exactly. + + Args: + name (str): Name to match. Matching is exact and case-sensitive; + use `filter_by_pattern` for wildcards. + + Returns: + The matching item. + + Raises: + DataPortalInputError: if `name` is None, or if several items share + the name -- in which case use `get_by_id`. + DataPortalAssetNotFound: if nothing matches. + """ if name is None: raise DataPortalInputError(f"Must provide name to identify {self.asset_name}") @@ -65,7 +104,21 @@ def get_by_name(self, name: str) -> T: return matching_queries[0] def get_by_id(self, _id: str) -> T: - """Return the item which matches by id attribute.""" + """ + Return the single item whose `id` attribute matches exactly. + + For files, the `id` is the relative path within the dataset. + + Args: + _id (str): ID to match. + + Returns: + The matching item. + + Raises: + DataPortalInputError: if `_id` is None. + DataPortalAssetNotFound: if nothing matches. + """ if _id is None: raise DataPortalInputError(f"Must provide id to identify {self.asset_name}") @@ -81,7 +134,17 @@ def get_by_id(self, _id: str) -> T: return matching_queries[0] def filter_by_pattern(self, pattern: str) -> 'DataPortalAssets[T]': - """Filter the items to just those whose name attribute matches the pattern.""" + """ + Return the items whose `name` matches a shell-style wildcard pattern. + + Args: + pattern (str): Wildcard pattern, matched with `fnmatch` -- `*` for + any run of characters, `?` for one, `[seq]` for a character set. + + Returns: + A new collection of the same type holding the matching items, empty + if none match. + """ # Get a list of the names to search against all_names = [i.name for i in self] diff --git a/cirro/sdk/dataset.py b/cirro/sdk/dataset.py index e7e2dac8..b8605fc9 100644 --- a/cirro/sdk/dataset.py +++ b/cirro/sdk/dataset.py @@ -172,7 +172,13 @@ def project_id(self) -> str: @property def status(self) -> Status: """ - Status of the dataset + Status of the dataset: one of `PENDING`, `STARTING`, `RUNNING`, + `COMPLETED`, `FAILED`, `STOPPING`, `SUSPENDED`, `ARCHIVED`, `DELETING`, + `DELETED`, or `UNKNOWN`. + + This is a snapshot taken when the object was built, and it does not + update. To watch a running analysis, call `portal.get_dataset(...)` + again each time round the loop rather than re-reading this property. """ return self._data.status @@ -224,14 +230,27 @@ def share(self) -> Optional[NamedItem]: @property def file_count(self) -> int: + """ + Number of files in the dataset. + + Fetches the full dataset detail from the API on first access if this + object was built from a listing. + """ return self._get_detail().file_count @property def total_size_bytes(self) -> int: + """ + Combined size of every file in the dataset, in bytes. + + Fetches the full dataset detail from the API on first access if this + object was built from a listing. + """ return self._get_detail().total_size_bytes @property def total_size(self) -> str: + """Combined size of every file in the dataset, human-readable (e.g. 4.50 GB).""" return bytes_to_human_readable(self.total_size_bytes) @property @@ -249,8 +268,17 @@ def logs(self) -> str: """ Return the top-level execution log for this dataset. + This is the log from the head node driving the workflow -- Nextflow's + own output, including which tasks it submitted and why the run stopped. + For the stdout/stderr of one task, use `cirro.sdk.task.DataPortalTask.logs`; + for the log file archived once the run finishes, use `get_logs`. + Returns an empty string if no log events are available (e.g. the job has not started yet). + Cached after the first access. Reading this while the analysis is still + starting up returns `''` and will keep returning `''` for the lifetime + of this object -- re-fetch the dataset to try again. + Returns: str: Execution log text, or an empty string if unavailable. """ @@ -267,8 +295,12 @@ def tasks(self) -> List[DataPortalTask]: """ List of tasks from the workflow execution, fetched via the execution API. + Cached after the first access, so a list read while the analysis is + still running will not pick up tasks that start later -- re-fetch the + dataset for an up-to-date list. + Returns: - `List[DataPortalTask]` + `List[cirro.sdk.task.DataPortalTask]` """ return self._load_tasks_from_api() @@ -374,8 +406,17 @@ def list_files(self, file_limit: int = 100000) -> DataPortalFiles: """ Return the list of files which make up the dataset. + The result is a `cirro.sdk.asset.DataPortalAssets` list, so it also + offers `get_by_name`, `get_by_id`, and `filter_by_pattern`. Files are + listed by their relative path within the dataset, most of which sit + under a `data/` prefix. + Args: - file_limit (int): Maximum number of files to return (default 100,000) + file_limit (int): Maximum number of files to return (default 100,000). + A dataset with more files than this is truncated silently. + + Returns: + `cirro.sdk.file.DataPortalFiles` """ assets = self._client.datasets.get_assets_listing( project_id=self.project_id, @@ -479,23 +520,49 @@ def get_trace(self) -> Any: """ Read the Nextflow workflow trace file for this dataset as a DataFrame. + One row per task, with timing, resource usage, and exit status. Written + when the run finishes. + Returns: `pandas.DataFrame` + + Raises: + DataPortalAssetNotFound: if the dataset has no workflow trace + artifact, which is the case for datasets that were uploaded + rather than produced by a Nextflow analysis, and for runs that + have not finished. """ return self.get_artifact(ArtifactType.WORKFLOW_TRACE).read_csv(sep='\t') def get_logs(self) -> str: """ - Read the Nextflow workflow logs for this dataset as a string. + Read the archived Nextflow workflow log for this dataset as a string. + + This reads the `WORKFLOW_LOGS` artifact, written when the run finishes. + For the live head-node log of a run in progress, use `logs` instead. Returns: str + + Raises: + DataPortalAssetNotFound: if the dataset has no workflow log + artifact yet. """ return self.get_artifact(ArtifactType.WORKFLOW_LOGS).read() def get_artifact(self, artifact_type: ArtifactType) -> DataPortalFile: """ - Get the artifact of a particular type from the dataset + Get the artifact of a particular type from the dataset. + + Args: + artifact_type (`cirro_api_client.v1.models.ArtifactType`): Type of + artifact to return, e.g. `ArtifactType.WORKFLOW_TRACE`. + + Returns: + `cirro.sdk.file.DataPortalFile` + + Raises: + DataPortalAssetNotFound: if the dataset has no artifact of this type. """ artifacts = self._get_assets().artifacts artifact = next((a for a in artifacts if a.artifact_type == artifact_type), None) diff --git a/cirro/sdk/developer.py b/cirro/sdk/developer.py index b178d622..847a11de 100644 --- a/cirro/sdk/developer.py +++ b/cirro/sdk/developer.py @@ -9,6 +9,11 @@ class Matches(list[FileNameMatch]): + """ + The file name matches produced by + `DeveloperHelper.test_file_name_validation`. + """ + def print(self): """ Prints the file name validation matches in a readable format. @@ -30,6 +35,15 @@ class DeveloperHelper: """ def __init__(self, client: CirroApi): + """ + Obtained from `cirro.sdk.portal.DataPortal.developer_helper`. + + ```python + from cirro import DataPortal + portal = DataPortal() + helper = portal.developer_helper + ``` + """ self.client = client def generate_preprocess_for_input_datasets(self, diff --git a/cirro/sdk/exceptions.py b/cirro/sdk/exceptions.py index 68f6e563..622a04ac 100644 --- a/cirro/sdk/exceptions.py +++ b/cirro/sdk/exceptions.py @@ -4,5 +4,5 @@ class DataPortalAssetNotFound(Exception): class DataPortalInputError(Exception): - """Exception raised invalid inputs are provided to the Data Portal.""" + """Exception raised when invalid inputs are provided to the Data Portal.""" pass diff --git a/cirro/sdk/portal.py b/cirro/sdk/portal.py index b970bac2..cf3049a3 100644 --- a/cirro/sdk/portal.py +++ b/cirro/sdk/portal.py @@ -85,7 +85,13 @@ def __init__(self, base_url: str = None, client: CirroApi = None): self._client = CirroApi(base_url=base_url) def list_projects(self) -> DataPortalProjects: - """List all the projects available in the Data Portal.""" + """ + List all the projects available in the Data Portal. + + Returns: + `cirro.sdk.project.DataPortalProjects`, a list which also offers + `get_by_name`, `get_by_id`, and `filter_by_pattern`. + """ return DataPortalProjects( [ @@ -95,12 +101,35 @@ def list_projects(self) -> DataPortalProjects: ) def get_project_by_name(self, name: str = None) -> DataPortalProject: - """Return the project with the specified name.""" + """ + Return the project with the specified name. + + Args: + name (str): Name of the project. + + Returns: + `cirro.sdk.project.DataPortalProject` + + Raises: + DataPortalAssetNotFound: if no project has this name. + DataPortalInputError: if more than one project has this name. + """ return self.list_projects().get_by_name(name) def get_project_by_id(self, _id: str = None) -> DataPortalProject: - """Return the project with the specified id.""" + """ + Return the project with the specified id. + + Args: + _id (str): ID of the project. + + Returns: + `cirro.sdk.project.DataPortalProject` + + Raises: + DataPortalAssetNotFound: if no project has this ID. + """ return self.list_projects().get_by_id(_id) @@ -289,6 +318,9 @@ def list_processes(self, ingest=False) -> DataPortalProcesses: Args: ingest (bool): If True, only list those processes which can be used to ingest datasets directly + + Returns: + `cirro.sdk.process.DataPortalProcesses` """ return DataPortalProcesses( @@ -305,23 +337,48 @@ def get_process_by_name(self, name: str, ingest=False) -> DataPortalProcess: Args: name (str): Name of process + ingest (bool): If True, search only the processes which can be used + to ingest datasets directly. A data type used for uploading will + not be found unless this is set. + + Returns: + `cirro.sdk.process.DataPortalProcess` + + Raises: + DataPortalAssetNotFound: if no process has this name. """ return self.list_processes(ingest=ingest).get_by_name(name) def get_process_by_id(self, id: str, ingest=False) -> DataPortalProcess: """ - Return the process with the specified id + Return the process with the specified id. Args: id (str): ID of process + ingest (bool): If True, search only the processes which can be used + to ingest datasets directly. + + Returns: + `cirro.sdk.process.DataPortalProcess` + + Raises: + DataPortalAssetNotFound: if no process has this ID. """ return self.list_processes(ingest=ingest).get_by_id(id) def list_reference_types(self) -> DataPortalReferenceTypes: """ - Return the list of all available reference types + Return the list of all available reference types. + + These are the categories that reference data is organized into, such as + `genome_fasta`. Pass a type name to + `cirro.sdk.project.DataPortalProject.list_references` to see the + references of that type held by a project. + + Returns: + `cirro.sdk.reference_type.DataPortalReferenceTypes` """ return DataPortalReferenceTypes( @@ -333,4 +390,11 @@ def list_reference_types(self) -> DataPortalReferenceTypes: @property def developer_helper(self) -> DeveloperHelper: + """ + Helpers for developing Cirro pipelines and data types, rather than for + analysing data. + + Returns: + `cirro.sdk.developer.DeveloperHelper` + """ return DeveloperHelper(self._client) diff --git a/cirro/sdk/project.py b/cirro/sdk/project.py index d5a4a865..ea5e610e 100644 --- a/cirro/sdk/project.py +++ b/cirro/sdk/project.py @@ -78,7 +78,20 @@ def _get_datasets(self) -> List[Dataset]: client=self._client) def list_datasets(self, force_refresh=False) -> DataPortalDatasets: - """List all the datasets available in the project.""" + """ + List all the datasets available in the project. + + The listing is fetched once and cached on this object, since a project + may hold many thousands of datasets. + + Args: + force_refresh (bool): Discard the cached listing and fetch it again. + Needed to see datasets created since this object was built. + + Returns: + `cirro.sdk.dataset.DataPortalDatasets`, a list which also offers + `get_by_name`, `get_by_id`, and `filter_by_pattern`. + """ if force_refresh: self._get_datasets.cache_clear() @@ -90,10 +103,24 @@ def list_datasets(self, force_refresh=False) -> DataPortalDatasets: ) def get_dataset(self, name_or_id: str, force_refresh=False) -> DataPortalDataset: - """Return the dataset matching the given ID or name. + """ + Return the dataset matching the given ID or name. Tries to match by ID first, then by name. - Raises an error if the name matches multiple datasets. + + Args: + name_or_id (str): ID or name of the dataset. + force_refresh (bool): Discard the cached dataset listing before + matching by name. Needed to find a dataset created since this + object was built. + + Returns: + `cirro.sdk.dataset.DataPortalDataset` + + Raises: + DataPortalAssetNotFound: if nothing matches by either ID or name. + DataPortalInputError: if more than one dataset has this name, in + which case use `get_dataset_by_id`. """ if force_refresh: self._get_datasets.cache_clear() @@ -115,7 +142,22 @@ def get_dataset(self, name_or_id: str, force_refresh=False) -> DataPortalDataset return self.get_dataset_by_id(matches[0].id) def get_dataset_by_name(self, name: str, force_refresh=False) -> DataPortalDataset: - """Return the dataset with the specified name.""" + """ + Return the dataset with the specified name. + + If several datasets share the name, the first one in the listing is + returned. Use `get_dataset` if you would rather that were an error. + + Args: + name (str): Name of the dataset. + force_refresh (bool): Discard the cached dataset listing first. + + Returns: + `cirro.sdk.dataset.DataPortalDataset` + + Raises: + DataPortalAssetNotFound: if no dataset in the project has this name. + """ if force_refresh: self._get_datasets.cache_clear() @@ -125,7 +167,20 @@ def get_dataset_by_name(self, name: str, force_refresh=False) -> DataPortalDatas return self.get_dataset_by_id(dataset.id) def get_dataset_by_id(self, _id: str = None) -> DataPortalDataset: - """Return the dataset with the specified id.""" + """ + Return the dataset with the specified id. + + Fetches the dataset directly, bypassing the cached project listing. + + Args: + _id (str): ID of the dataset. + + Returns: + `cirro.sdk.dataset.DataPortalDataset` + + Raises: + DataPortalAssetNotFound: if the project has no dataset with this ID. + """ dataset = self._client.datasets.get(project_id=self.id, dataset_id=_id) if dataset is None: @@ -135,7 +190,18 @@ def get_dataset_by_id(self, _id: str = None) -> DataPortalDataset: def list_references(self, reference_type: str = None) -> DataPortalReferences: """ List the references available in a project. - Optionally filter to references of a particular type (identified by name) + + Args: + reference_type (str): Optionally restrict the results to references + of one type, identified by name. Call + `cirro.sdk.portal.DataPortal.list_reference_types` for the + available type names. + + Returns: + `cirro.sdk.reference.DataPortalReferences` + + Raises: + DataPortalAssetNotFound: if `reference_type` matches no known type. """ # Get the complete list of references which are available @@ -164,7 +230,20 @@ def list_references(self, reference_type: str = None) -> DataPortalReferences: ) def get_reference_by_name(self, name: str = None, ref_type: str = None) -> DataPortalReference: - """Return the reference of a particular type with the specified name.""" + """ + Return the reference of a particular type with the specified name. + + Args: + name (str): Name of the reference. + ref_type (str): Optionally restrict the search to one reference type. + + Returns: + `cirro.sdk.reference.DataPortalReference` + + Raises: + DataPortalInputError: if `name` is not provided. + DataPortalAssetNotFound: if no matching reference exists. + """ if name is None: raise DataPortalInputError("Must specify the reference name") @@ -185,6 +264,13 @@ def upload_dataset( If the files parameter is not provided, it will upload all files in the upload folder + The `process` here is a data type rather than a pipeline: it declares + what kind of data is being uploaded and which files the dataset must + contain. List the valid options with + `portal.list_processes(ingest=True)`. Cirro validates the file names + against the data type's requirements before any upload starts, so a + mismatch fails fast. + Args: name (str): Name of newly created dataset description (str): Description of newly created dataset @@ -192,6 +278,14 @@ def upload_dataset( upload_folder (str): Folder containing files to upload files (List[str]): Optional subset of files to upload from the folder tags (List[str]): Optional list of tags to apply to the dataset + + Returns: + `cirro.sdk.dataset.DataPortalDataset`: the newly created dataset. + + Raises: + DataPortalInputError: if `name`, `process`, or `upload_folder` is + missing, or if the files do not meet the data type's requirements. + RuntimeWarning: if there are no files to upload. """ if name is None: @@ -256,7 +350,12 @@ def samples(self, max_items: int = 10000) -> List[Sample]: Retrieves a list of samples associated with a project along with their metadata Args: - max_items (int): Maximum number of records to get (default 10,000) + max_items (int): Maximum number of records to get (default 10,000). + A project with more samples than this is truncated silently. + + Returns: + `List[cirro_api_client.v1.models.Sample]` -- each carrying the + sample's `id`, `name`, and its `metadata` dict. """ return self._client.metadata.get_project_samples(self.id, max_items) diff --git a/cirro/sdk/reference.py b/cirro/sdk/reference.py index 0711aa3b..1c9a43b7 100644 --- a/cirro/sdk/reference.py +++ b/cirro/sdk/reference.py @@ -44,6 +44,12 @@ def type(self) -> str: @property def absolute_path(self): + """ + S3 URI of the reference's first file, or `None` if it has no files. + + A reference may contain several files (e.g. a FASTA alongside its index). + Use `files` to reach the others. + """ if len(self._files) == 0: return None return self._files[0].absolute_path @@ -57,4 +63,10 @@ class DataPortalReferences(DataPortalAssets[DataPortalReference]): asset_name = "reference" def get_by_id(self, _id: str) -> DataPortalReference: + """ + Not supported for references, which are identified by name only. + + Raises: + NotImplementedError: always. Use `get_by_name` instead. + """ raise NotImplementedError("Filtering by ID is not supported, use get_by_name") diff --git a/cirro/sdk/reference_type.py b/cirro/sdk/reference_type.py index f12af797..81dbc026 100644 --- a/cirro/sdk/reference_type.py +++ b/cirro/sdk/reference_type.py @@ -9,6 +9,15 @@ class DataPortalReferenceType(DataPortalAsset): """ def __init__(self, ref_type: ReferenceType): + """ + Obtained from `cirro.sdk.portal.DataPortal.list_reference_types`. + + ```python + from cirro import DataPortal + portal = DataPortal() + reference_types = portal.list_reference_types() + ``` + """ self._data = ref_type @property @@ -23,10 +32,12 @@ def description(self): @property def directory(self): + """Folder that references of this type are stored under.""" return self._data.directory @property def validation(self): + """Rules describing the files a reference of this type must contain.""" return self._data.validation def __str__(self): @@ -41,4 +52,10 @@ class DataPortalReferenceTypes(DataPortalAssets[DataPortalReferenceType]): asset_name = "reference type" def get_by_id(self, _id: str) -> DataPortalReferenceType: + """ + Not supported for reference types, which are identified by name only. + + Raises: + NotImplementedError: always. Use `get_by_name` instead. + """ raise NotImplementedError("Filtering by ID is not supported, use get_by_name") From 1846959c9be61da0329e420db6594fd5540e1708 Mon Sep 17 00:00:00 2001 From: Sam Minot Date: Wed, 19 Aug 2026 11:15:35 -0700 Subject: [PATCH 4/8] Normalize docstring markup and fill in remaining Args sections The corpus mixed three dialects -- Google sections, reStructuredText roles and literals, and Markdown fences -- sometimes within one docstring. `make_docs.py` sets `docformat="google"`, and raw `help()` output shows the RST verbatim, so RST roles and ``literals`` are converted to Markdown backticks throughout. `DataPortalDataset.read_files` / `read_file` deferred to their `DataPortal` equivalents for the glob and pattern rules, leaving the docstring on the object you actually hold nearly empty; both are now self-contained. Fills in the Args, Returns and Raises sections still missing on the file-reading mixin, file downloads, the developer helpers, and `CirroApi.__init__`. Co-Authored-By: Claude Opus 5 --- cirro/cirro_client.py | 8 +++- cirro/sdk/dataset.py | 95 ++++++++++++++++++++++++++++++---------- cirro/sdk/developer.py | 62 ++++++++++++++++++++++++-- cirro/sdk/file.py | 20 ++++++++- cirro/sdk/file_mixins.py | 66 ++++++++++++++++++++-------- cirro/sdk/helpers.py | 14 +++++- cirro/sdk/portal.py | 82 +++++++++++++++++----------------- cirro/sdk/task.py | 24 +++++----- 8 files changed, 268 insertions(+), 103 deletions(-) diff --git a/cirro/cirro_client.py b/cirro/cirro_client.py index 041882b5..41b66bed 100644 --- a/cirro/cirro_client.py +++ b/cirro/cirro_client.py @@ -17,9 +17,15 @@ def __init__(self, auth_info: AuthInfo = None, base_url: str = None, user_agent: Instantiates the Cirro API object Args: - auth_info (cirro.auth.base.AuthInfo): + auth_info (`cirro.auth.base.AuthInfo`): How to authenticate. If + omitted, this is read from the saved configuration, which falls back + to an interactive device-code login that blocks on a browser flow. + Pass `cirro.auth.client_creds.ClientCredentialsAuth` to authenticate + without prompting. base_url (str): Optional base URL of the Cirro instance (if not provided, it uses the `CIRRO_BASE_URL` environment variable, or the config file) + user_agent (str): Name reported to the API for this client, which + shows up in Cirro's audit logs. Returns: Authenticated Cirro API object, which can be used to call endpoint functions. diff --git a/cirro/sdk/dataset.py b/cirro/sdk/dataset.py index b8605fc9..dc0ded4e 100644 --- a/cirro/sdk/dataset.py +++ b/cirro/sdk/dataset.py @@ -24,17 +24,17 @@ def _pattern_to_captures_regex(pattern: str): """ - Convert a glob pattern that may contain ``{name}`` capture placeholders into - a compiled regex and return ``(compiled_regex, capture_names)``. + Convert a glob pattern that may contain `{name}` capture placeholders into + a compiled regex and return `(compiled_regex, capture_names)`. Conversion rules: - - ``{name}`` → named group matching a single path segment (no ``/``) - - ``*`` → matches any characters within a single path segment - - ``**`` → matches any characters including ``/`` (multiple segments) + - `{name}` → named group matching a single path segment (no `/`) + - `*` → matches any characters within a single path segment + - `**` → matches any characters including `/` (multiple segments) - All other characters are regex-escaped. - The resulting regex is suffix-anchored (like ``pathlib.PurePath.match``): - a pattern without a leading ``/`` will match at any depth in the path. + The resulting regex is suffix-anchored (like `pathlib.PurePath.match`): + a pattern without a leading `/` will match at any depth in the path. """ capture_names = re.findall(r'\{(\w+)\}', pattern) tokens = re.split(r'(\*\*|\*|\{\w+\})', pattern) @@ -333,11 +333,11 @@ def primary_failed_task(self) -> Optional[DataPortalTask]: """ Find the root-cause failed task in this workflow execution. - Returns ``None`` gracefully when no tasks are available or none have - a ``FAILED`` status. + Returns `None` gracefully when no tasks are available or none have + a `FAILED` status. Returns: - `cirro.sdk.task.DataPortalTask`, or ``None`` if no failed task is found. + `cirro.sdk.task.DataPortalTask`, or `None` if no failed task is found. """ from cirro.helpers.nextflow_utils import find_primary_failed_task @@ -440,24 +440,58 @@ def read_files( **kwargs ): """ - Read the contents of files in the dataset. + Read the contents of files in the dataset, without downloading them. - See :meth:`~cirro.sdk.portal.DataPortal.read_files` for full details - on ``glob``/``pattern`` matching and filetype options. + Exactly one of `glob` or `pattern` must be provided. + + **glob** -- standard wildcard matching; yields the file content for each + matching file: + + - `*` matches any characters within a single path segment + - `**` matches zero or more path segments + - Matching is suffix-anchored (`*.csv` matches at any depth) + + **pattern** -- like `glob` but `{name}` placeholders capture portions of + the path automatically; yields `(content, meta)` pairs where *meta* is a + `dict` of extracted values: + + - `{name}` captures one path segment (no `/`) + - `*` and `**` wildcards work as in `glob` + + See `cirro.sdk.portal.DataPortal.read_files()` for the full list of + `filetype` values and the extensions each one is inferred from. + + ```python + # Every CSV in the dataset, as DataFrames + for df in dataset.read_files(glob='*.csv'): + print(df.shape) + + # Capture the sample name from each filename + for df, meta in dataset.read_files(pattern='{sample}.csv'): + print(meta['sample'], df.shape) + + # Gzipped TSVs at any depth + for df in dataset.read_files(glob='**/*.tsv.gz', filetype='csv', sep='\\t'): + print(df.shape) + ``` Args: glob (str): Wildcard expression to match files. Yields one item per matching file: the parsed content. - pattern (str): Wildcard expression with ``{name}`` capture - placeholders. Yields ``(content, meta)`` per matching file. + pattern (str): Wildcard expression with `{name}` capture + placeholders. Yields `(content, meta)` per matching file. filetype (str): File format used to parse each file - (or ``None`` to infer from extension). + (or `None` to infer from extension). **kwargs: Additional keyword arguments forwarded to the - file-parsing function. + file-parsing function (e.g. `sep='\\t'` for TSV files). Yields: - - When using ``glob``: *content* for each matching file - - When using ``pattern``: ``(content, meta)`` for each matching file + - When using `glob`: *content* for each matching file + - When using `pattern`: `(content, meta)` for each matching file + + Raises: + DataPortalInputError: if both `glob` and `pattern` are provided, or + if neither is. """ if glob is not None and pattern is not None: raise DataPortalInputError("Cannot specify both 'glob' and 'pattern' — use one or the other") @@ -482,20 +516,33 @@ def read_file( **kwargs ) -> Any: """ - Read the contents of a single file from the dataset. + Read the contents of a single file from the dataset, without + downloading it. + + Provide either `path` (the exact relative path) or `glob` (a wildcard + expression, which must match exactly one file). - See :meth:`~cirro.sdk.portal.DataPortal.read_file` for full details. + ```python + df = dataset.read_file(path='data/counts.csv') + df = dataset.read_file(glob='**/counts.csv') + ``` Args: path (str): Exact relative path of the file within the dataset. glob (str): Wildcard expression matching exactly one file. filetype (str): File format used to parse the file. Supported values - are the same as :meth:`~cirro.sdk.portal.DataPortal.read_files`. + are the same as `cirro.sdk.portal.DataPortal.read_files()`. **kwargs: Additional keyword arguments forwarded to the file-parsing function. Returns: - Parsed file content. + Parsed file content -- a `pandas.DataFrame` for tabular formats, a + `str` for text, and so on depending on `filetype`. + + Raises: + DataPortalInputError: if both or neither of `path`/`glob` are given, + or if `glob` matches more than one file. + DataPortalAssetNotFound: if nothing matches. """ if path is not None and glob is not None: raise DataPortalInputError("Cannot specify both 'path' and 'glob' — use one or the other") @@ -593,7 +640,7 @@ def download_files(self, download_location: str = None, glob: str = None) -> Non Args: download_location (str): Path to local directory glob (str): Optional wildcard expression to filter which files are downloaded - (e.g., ``'*.csv'``, ``'data/**/*.tsv.gz'``). + (e.g., `'*.csv'`, `'data/**/*.tsv.gz'`). If omitted, all files are downloaded. """ diff --git a/cirro/sdk/developer.py b/cirro/sdk/developer.py index 847a11de..a5620905 100644 --- a/cirro/sdk/developer.py +++ b/cirro/sdk/developer.py @@ -32,6 +32,10 @@ class DeveloperHelper: Helper class for developer-related tasks, such as adding samplesheet preprocessing for a pipeline or testing file name validation and sample autopopulation. + + These are for building and debugging Cirro pipelines and data types, not for + analysing data. Obtained from + `cirro.sdk.portal.DataPortal.developer_helper`. """ def __init__(self, client: CirroApi): @@ -51,10 +55,22 @@ def generate_preprocess_for_input_datasets(self, input_dataset_ids: list[str], params=None) -> PreprocessDataset: """ - Generates a PreprocessDataset object for the given datasets + Generates a PreprocessDataset object for the given datasets. + + Use this to develop and test a pipeline's `preprocess.py` locally against + real datasets, without running the pipeline. - With optional parameters to pass into the preprocess script. - Certain properties of `metadata` are available in this context. + Args: + project_id (str): ID of the project holding the input datasets. + input_dataset_ids (list[str]): IDs of the datasets to use as input. + params (dict): Parameters to expose to the preprocess script, as if + they had been entered in the analysis form. + + Returns: + `cirro.helpers.preprocess_dataset.PreprocessDataset` -- with real + samplesheet and file listings, and partially mocked `metadata`: only + `project` and `inputs` are populated, while `dataset` and `process` + are empty. """ samplesheets = self._generate_samplesheets_for_datasets(project_id, input_dataset_ids) project = self.client.projects.get(project_id) @@ -86,6 +102,16 @@ def test_file_name_validation_for_dataset(self, Used when configuring Cirro's sample autopopulation feature. More info: https://docs.cirro.bio/features/samples/#using-auto-population + + Args: + project_id (str): ID of the project holding the dataset. + dataset_id (str): ID of the dataset whose file names to test against. + file_name_patterns (list[str]): Regex patterns to test, as they would + appear in a process definition. + + Returns: + `Matches` -- call `print()` on it for a readable report of which + files matched which pattern, and what sample name each produced. """ dataset_files = self.client.datasets.get_assets_listing(project_id=project_id, dataset_id=dataset_id).files file_names = [file.relative_path for file in dataset_files] @@ -96,6 +122,16 @@ def test_file_name_validation(self, file_name_patterns: list[str]) -> Matches: """ Tests the file name validation for a list of file names against specified regex patterns. + + The same as `test_file_name_validation_for_dataset`, but against file + names you supply rather than a dataset's. + + Args: + file_names (list[str]): File names to test. + file_name_patterns (list[str]): Regex patterns to test them against. + + Returns: + `Matches` """ request_body = ValidateFileNamePatternsRequest( file_names=file_names, @@ -111,7 +147,18 @@ def test_file_name_validation(self, def generate_samplesheets_for_dataset(self, project_id: str, dataset_id: str) -> SampleSheets: """ - Generates Cirro samplesheets for a given dataset + Generates Cirro samplesheets for a given dataset. + + These are the `samplesheet.csv` and `files.csv` that Cirro stages for a + pipeline run, useful for checking what a pipeline will actually receive. + + Args: + project_id (str): ID of the project holding the dataset. + dataset_id (str): ID of the dataset. + + Returns: + `cirro_api_client.v1.models.SampleSheets` -- with `samples` and + `files` each holding CSV text. """ return get_sample_sheets.sync( project_id=project_id, @@ -123,6 +170,13 @@ def rerun_sample_ingest_for_dataset(self, project_id: str, dataset_id: str): """ Reruns the sample ingest process for a given dataset. You'll want to do this if you have updated the file name patterns in your pipeline (or data type) + + This re-derives the dataset's samples and their metadata from its file + names, replacing what is currently recorded. + + Args: + project_id (str): ID of the project holding the dataset. + dataset_id (str): ID of the dataset to re-ingest samples for. """ ingest_samples.sync_detailed( project_id=project_id, diff --git a/cirro/sdk/file.py b/cirro/sdk/file.py index 3cdbb4d8..104fea1c 100644 --- a/cirro/sdk/file.py +++ b/cirro/sdk/file.py @@ -86,8 +86,15 @@ def download(self, download_location: str = None) -> Path: """ Download the file to a local directory. + Args: + download_location (str): Local directory to write the file into. The + file keeps its relative path within that directory. + Returns: - Path to download file + `pathlib.Path`: path to the downloaded file. + + Raises: + DataPortalInputError: if `download_location` is not provided. """ if download_location is None: @@ -137,8 +144,17 @@ def download(self, download_location: str = None) -> List[Path]: """ Download the collection of files to a local directory. + Files are downloaded one at a time, each keeping its relative path + within the dataset. + + Args: + download_location (str): Local directory to write the files into. + Returns: - List of paths to downloaded files. + `List[pathlib.Path]`: paths to the downloaded files. + + Raises: + DataPortalInputError: if `download_location` is not provided. """ local_paths = [] diff --git a/cirro/sdk/file_mixins.py b/cirro/sdk/file_mixins.py index e9ba08ea..ad8ac139 100644 --- a/cirro/sdk/file_mixins.py +++ b/cirro/sdk/file_mixins.py @@ -15,7 +15,7 @@ class FileReadMixin(ABC): """ Mixin that adds file-reading methods to any class that provides - ``_get() -> bytes`` and a ``name`` property. + `_get() -> bytes` and a `name` property. """ @property @@ -28,7 +28,19 @@ def _get(self) -> bytes: """Return the raw file bytes.""" def read(self, encoding='utf-8', compression=None) -> str: - """Read the file contents as text.""" + """ + Read the file contents as text. + + Args: + encoding (str): Text encoding to decode with. + compression (str): Pass `'gzip'` to decompress first, or `None` to + read the bytes as-is. Unlike `read_csv`, this is not inferred + from the file extension. + + Raises: + DataPortalInputError: if `compression` is anything other than + `'gzip'` or `None`. + """ cont = self._get() if compression is None: return cont.decode(encoding) @@ -38,24 +50,42 @@ def read(self, encoding='utf-8', compression=None) -> str: return handle.read() def readlines(self, encoding='utf-8', compression=None) -> List[str]: - """Read the file contents as a list of lines.""" + """ + Read the file contents as a list of lines, without trailing newlines. + + Args: + encoding (str): Text encoding to decode with. + compression (str): Pass `'gzip'` to decompress first, or `None` to + read the bytes as-is. + """ return self.read(encoding=encoding, compression=compression).splitlines() def read_bytes(self) -> BytesIO: - """Get a BytesIO object for the file contents, to pass into arbitrary readers.""" + """ + Get a BytesIO object for the file contents, to pass into arbitrary readers. + + Use this for formats the mixin does not cover -- the whole file is read + into memory and handed over as a file-like object. + """ return BytesIO(self._get()) def read_csv(self, compression='infer', encoding='utf-8', **kwargs) -> 'DataFrame': """ Parse the file as a Pandas DataFrame. - The default field separator is a comma (for CSV), use sep='\\t' for TSV. - - File compression is inferred from the extension, but can be set - explicitly with the compression= flag. - - All other keyword arguments are passed to pandas.read_csv - https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html + Args: + compression (str | dict): How the file is compressed. The default, + `'infer'`, picks gzip/bz2/xz/zstd from a `.gz`, `.bz2`, `.xz`, + or `.zst` extension, and no compression otherwise. Pass `None` + to force reading as plain text. + encoding (str): Text encoding to decode with. + **kwargs: Passed through to + [pandas.read_csv](https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html). + The field separator defaults to a comma, so pass `sep='\\t'` + for TSV files. + + Returns: + `pandas.DataFrame` """ import pandas @@ -102,8 +132,8 @@ def read_parquet(self, **kwargs) -> 'DataFrame': """ Read a Parquet file as a Pandas DataFrame. - Requires ``pyarrow`` or ``fastparquet`` to be installed. - All keyword arguments are passed to :func:`pandas.read_parquet`. + Requires `pyarrow` or `fastparquet` to be installed. + All keyword arguments are passed to `pandas.read_parquet()`. """ import pandas return pandas.read_parquet(BytesIO(self._get()), **kwargs) @@ -112,8 +142,8 @@ def read_feather(self, **kwargs) -> 'DataFrame': """ Read a Feather file as a Pandas DataFrame. - Requires ``pyarrow`` to be installed. - All keyword arguments are passed to :func:`pandas.read_feather`. + Requires `pyarrow` to be installed. + All keyword arguments are passed to `pandas.read_feather()`. """ import pandas return pandas.read_feather(BytesIO(self._get()), **kwargs) @@ -124,10 +154,10 @@ def read_pickle(self, **kwargs): def read_excel(self, **kwargs) -> 'DataFrame': """ - Read an Excel file (``.xlsx`` / ``.xls``) as a Pandas DataFrame. + Read an Excel file (`.xlsx` / `.xls`) as a Pandas DataFrame. - Requires ``openpyxl`` (for ``.xlsx``) or ``xlrd`` (for ``.xls``). - All keyword arguments are passed to :func:`pandas.read_excel`. + Requires `openpyxl` (for `.xlsx`) or `xlrd` (for `.xls`). + All keyword arguments are passed to `pandas.read_excel()`. """ import pandas return pandas.read_excel(BytesIO(self._get()), **kwargs) diff --git a/cirro/sdk/helpers.py b/cirro/sdk/helpers.py index fef80b7c..ff70605a 100644 --- a/cirro/sdk/helpers.py +++ b/cirro/sdk/helpers.py @@ -10,7 +10,19 @@ def parse_process_name_or_id(process: Union[DataPortalProcess, str], client: CirroApi): """ - If the process is a string, try to parse it as a process name or ID. + Resolve a process given as a name, an ID, or an already-built object. + + Args: + process (str | `cirro.sdk.process.DataPortalProcess`): Name or ID of the + process, or the process object itself (returned unchanged). + client (`cirro.cirro_client.CirroApi`): Client to look the process up with. + + Returns: + `cirro.sdk.process.DataPortalProcess` + + Raises: + DataPortalInputError: if `process` is not a string, or if no process + matches it by either ID or name. """ # If the process object is already a DataPortalProcess object diff --git a/cirro/sdk/portal.py b/cirro/sdk/portal.py index cf3049a3..c7e8b5d0 100644 --- a/cirro/sdk/portal.py +++ b/cirro/sdk/portal.py @@ -194,62 +194,62 @@ def read_files( Read the contents of files from a dataset. The project and dataset can each be identified by name or ID. - Exactly one of ``glob`` or ``pattern`` must be provided. + Exactly one of `glob` or `pattern` must be provided. **glob** — standard wildcard matching; yields the file content for each matching file: - - ``*`` matches any characters within a single path segment - - ``**`` matches zero or more path segments - - Matching is suffix-anchored (``*.csv`` matches at any depth) + - `*` matches any characters within a single path segment + - `**` matches zero or more path segments + - Matching is suffix-anchored (`*.csv` matches at any depth) - **pattern** — like ``glob`` but ``{name}`` placeholders capture portions - of the path automatically; yields ``(content, meta)`` pairs where - *meta* is a ``dict`` of extracted values: + **pattern** — like `glob` but `{name}` placeholders capture portions + of the path automatically; yields `(content, meta)` pairs where + *meta* is a `dict` of extracted values: - - ``{name}`` captures one path segment (no ``/``) - - ``*`` and ``**`` wildcards work as in ``glob`` + - `{name}` captures one path segment (no `/`) + - `*` and `**` wildcards work as in `glob` Args: project (str): ID or name of the project. dataset (str): ID or name of the dataset. glob (str): Wildcard expression to match files - (e.g., ``'*.csv'``, ``'data/**/*.tsv.gz'``). + (e.g., `'*.csv'`, `'data/**/*.tsv.gz'`). Yields one item per matching file: the parsed content. - pattern (str): Wildcard expression with ``{name}`` capture - placeholders (e.g., ``'{sample}.csv'``, - ``'{condition}/{sample}.csv'``). - Yields ``(content, meta)`` per matching file. + pattern (str): Wildcard expression with `{name}` capture + placeholders (e.g., `'{sample}.csv'`, + `'{condition}/{sample}.csv'`). + Yields `(content, meta)` per matching file. filetype (str): File format used to parse each file. Supported values: - - ``'csv'``: parse with :func:`pandas.read_csv`, returns a ``DataFrame`` - - ``'h5ad'``: parse as AnnData (requires ``anndata`` package) - - ``'json'``: parse with :func:`json.loads`, returns a Python object - - ``'parquet'``: parse with :func:`pandas.read_parquet`, returns a ``DataFrame`` - (requires ``pyarrow`` or ``fastparquet``) - - ``'feather'``: parse with :func:`pandas.read_feather`, returns a ``DataFrame`` - (requires ``pyarrow``) - - ``'pickle'``: deserialize with :mod:`pickle`, returns a Python object - - ``'excel'``: parse with :func:`pandas.read_excel`, returns a ``DataFrame`` - (requires ``openpyxl`` for ``.xlsx`` or ``xlrd`` for ``.xls``) - - ``'text'``: read as plain text, returns a ``str`` - - ``'bytes'``: read as raw bytes, returns ``bytes`` - - ``None`` (default): infer from file extension - (``.csv``/``.tsv`` → ``'csv'``, ``.h5ad`` → ``'h5ad'``, - ``.json`` → ``'json'``, ``.parquet`` → ``'parquet'``, - ``.feather`` → ``'feather'``, ``.pkl``/``.pickle`` → ``'pickle'``, - ``.xlsx``/``.xls`` → ``'excel'``, otherwise ``'text'``) + - `'csv'`: parse with `pandas.read_csv()`, returns a `DataFrame` + - `'h5ad'`: parse as AnnData (requires `anndata` package) + - `'json'`: parse with `json.loads()`, returns a Python object + - `'parquet'`: parse with `pandas.read_parquet()`, returns a `DataFrame` + (requires `pyarrow` or `fastparquet`) + - `'feather'`: parse with `pandas.read_feather()`, returns a `DataFrame` + (requires `pyarrow`) + - `'pickle'`: deserialize with `pickle`, returns a Python object + - `'excel'`: parse with `pandas.read_excel()`, returns a `DataFrame` + (requires `openpyxl` for `.xlsx` or `xlrd` for `.xls`) + - `'text'`: read as plain text, returns a `str` + - `'bytes'`: read as raw bytes, returns `bytes` + - `None` (default): infer from file extension + (`.csv`/`.tsv` → `'csv'`, `.h5ad` → `'h5ad'`, + `.json` → `'json'`, `.parquet` → `'parquet'`, + `.feather` → `'feather'`, `.pkl`/`.pickle` → `'pickle'`, + `.xlsx`/`.xls` → `'excel'`, otherwise `'text'`) **kwargs: Additional keyword arguments forwarded to the file-parsing - function (e.g., ``sep='\\t'`` for CSV/TSV files). + function (e.g., `sep='\\t'` for CSV/TSV files). Yields: - - When using ``glob``: *content* for each matching file - - When using ``pattern``: ``(content, meta)`` for each matching file, - where *meta* is a ``dict`` of values extracted from ``{name}`` + - When using `glob`: *content* for each matching file + - When using `pattern`: `(content, meta)` for each matching file, + where *meta* is a `dict` of values extracted from `{name}` placeholders Raises: - DataPortalInputError: if both ``glob`` and ``pattern`` are provided, + DataPortalInputError: if both `glob` and `pattern` are provided, or if neither is provided. Example: @@ -287,8 +287,8 @@ def read_file( Read the contents of a single file from a dataset. The project and dataset can each be identified by name or ID. - Provide either ``path`` (exact relative path) or ``glob`` (wildcard - expression). If ``glob`` is used it must match exactly one file. + Provide either `path` (exact relative path) or `glob` (wildcard + expression). If `glob` is used it must match exactly one file. Args: project (str): ID or name of the project. @@ -296,7 +296,7 @@ def read_file( path (str): Exact relative path of the file within the dataset. glob (str): Wildcard expression matching exactly one file. filetype (str): File format used to parse the file. Supported values - are the same as :meth:`read_files`. + are the same as `read_files()`. **kwargs: Additional keyword arguments forwarded to the file-parsing function. @@ -304,8 +304,8 @@ def read_file( Parsed file content. Raises: - DataPortalInputError: if both or neither of ``path``/``glob`` are - provided, or if ``glob`` matches zero or more than one file. + DataPortalInputError: if both or neither of `path`/`glob` are + provided, or if `glob` matches zero or more than one file. """ ds = self.get_dataset(project=project, dataset=dataset) return ds.read_file(path=path, glob=glob, filetype=filetype, **kwargs) diff --git a/cirro/sdk/task.py b/cirro/sdk/task.py index 5e9e65b6..d87b4a62 100644 --- a/cirro/sdk/task.py +++ b/cirro/sdk/task.py @@ -28,8 +28,8 @@ class WorkDirFile(FileReadMixin): A file that lives in a Nextflow work directory or a dataset staging area. Each WorkDirFile either originated from another task's work directory - (``source_task`` is set) or was a primary/staged input to the workflow - (``source_task`` is ``None``). + (`source_task` is set) or was a primary/staged input to the workflow + (`source_task` is `None`). """ def __init__( @@ -42,7 +42,7 @@ def __init__( dataset_id: str = '' ): """ - Obtained from a task's ``inputs`` or ``outputs`` property. + Obtained from a task's `inputs` or `outputs` property. ```python for task in dataset.tasks: @@ -60,7 +60,7 @@ def __init__( @property def source_task(self) -> Optional['DataPortalTask']: - """The task that produced this file, or ``None`` for staged/primary inputs.""" + """The task that produced this file, or `None` for staged/primary inputs.""" return self._source_task @property @@ -143,7 +143,7 @@ def __init__( task_id: int = 0 ): """ - Obtained from a dataset's ``tasks`` property. + Obtained from a dataset's `tasks` property. ```python for task in dataset.tasks: @@ -157,7 +157,7 @@ def __init__( project_id (str): ID of the project that owns this dataset. dataset_id (str): ID of the dataset (execution) that owns this task. all_tasks_ref (list): A shared list that will contain all tasks once they - are all built. Used by ``inputs`` to resolve ``source_task``. + are all built. Used by `inputs` to resolve `source_task`. task_id (int): Numeric index of this task in the execution's task list. """ self._task = task @@ -178,12 +178,12 @@ def task_id(self) -> int: @property def name(self) -> str: - """Full task name, e.g. ``NFCORE_RNASEQ:RNASEQ:TRIMGALORE (sample1)``.""" + """Full task name, e.g. `NFCORE_RNASEQ:RNASEQ:TRIMGALORE (sample1)`.""" return self._task.name @property def status(self) -> str: - """Task status string, e.g. ``COMPLETED``, ``FAILED``, ``ABORTED``.""" + """Task status string, e.g. `COMPLETED`, `FAILED`, `ABORTED`.""" return self._task.status @property @@ -279,7 +279,7 @@ def logs(self) -> str: Fetches via the Cirro execution API when a native job ID is available, which works even when the S3 scratch bucket is not directly accessible. - Falls back to reading ``.command.log`` from the S3 work directory. + Falls back to reading `.command.log` from the S3 work directory. Returns an empty string if neither source can be read. """ if self._dataset_id and self.native_id: @@ -296,11 +296,11 @@ def logs(self) -> str: @cached_property def script(self) -> str: """ - Return the contents of ``.command.sh`` from the task's work directory. + Return the contents of `.command.sh` from the task's work directory. This is the actual shell script that Nextflow executed — the user's pipeline code for this task. Falls back to parsing the script from the - ``WORKFLOW_LOGS`` artifact when the work directory is not accessible + `WORKFLOW_LOGS` artifact when the work directory is not accessible (scratch bucket requires elevated permissions). Returns an empty string if the script cannot be obtained. """ @@ -379,7 +379,7 @@ def inputs(self) -> List[WorkDirFile]: """ List of input files for this task, fetched from the execution API. - Each file is annotated with ``source_task`` if its URI falls within + Each file is annotated with `source_task` if its URI falls within another task's work directory. """ return self._build_inputs() From f82a2e98baab1b057e454d14d9d833fa3c9b36ee Mon Sep 17 00:00:00 2001 From: Sam Minot Date: Wed, 19 Aug 2026 11:29:09 -0700 Subject: [PATCH 5/8] Correct executor assumptions and the status value list The head node may be running Nextflow or Cromwell depending on the process executor, so `dataset.logs` no longer calls it Nextflow's output. `get_logs` reads the `WORKFLOW_LOGS` artifact for either. `get_trace` stays Nextflow-only, since a Cromwell analysis produces no trace artifact -- now stated, along with Cromwell as a reason the artifact can be missing. `status` no longer enumerates the `Status` values. The list had omitted `DELETE`, and hardcoding a generated enum into a docstring only invites drift; it now points at `cirro_api_client.v1.models.Status` and names the values an analysis actually moves through. Restore the RST literals in `_pattern_to_captures_regex`. It is private, so pdoc renders no member section for it and `help()` is never called on it -- the only way anyone reads that docstring is as source, where the doubled delimiters mark where a punctuation literal like `*` or `**` starts and ends. Co-Authored-By: Claude Opus 5 --- cirro/sdk/dataset.py | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/cirro/sdk/dataset.py b/cirro/sdk/dataset.py index dc0ded4e..35b60119 100644 --- a/cirro/sdk/dataset.py +++ b/cirro/sdk/dataset.py @@ -24,17 +24,17 @@ def _pattern_to_captures_regex(pattern: str): """ - Convert a glob pattern that may contain `{name}` capture placeholders into - a compiled regex and return `(compiled_regex, capture_names)`. + Convert a glob pattern that may contain ``{name}`` capture placeholders into + a compiled regex and return ``(compiled_regex, capture_names)``. Conversion rules: - - `{name}` → named group matching a single path segment (no `/`) - - `*` → matches any characters within a single path segment - - `**` → matches any characters including `/` (multiple segments) + - ``{name}`` → named group matching a single path segment (no ``/``) + - ``*`` → matches any characters within a single path segment + - ``**`` → matches any characters including ``/`` (multiple segments) - All other characters are regex-escaped. - The resulting regex is suffix-anchored (like `pathlib.PurePath.match`): - a pattern without a leading `/` will match at any depth in the path. + The resulting regex is suffix-anchored (like ``pathlib.PurePath.match``): + a pattern without a leading ``/`` will match at any depth in the path. """ capture_names = re.findall(r'\{(\w+)\}', pattern) tokens = re.split(r'(\*\*|\*|\{\w+\})', pattern) @@ -172,9 +172,10 @@ def project_id(self) -> str: @property def status(self) -> Status: """ - Status of the dataset: one of `PENDING`, `STARTING`, `RUNNING`, - `COMPLETED`, `FAILED`, `STOPPING`, `SUSPENDED`, `ARCHIVED`, `DELETING`, - `DELETED`, or `UNKNOWN`. + Status of the dataset, as a `cirro_api_client.v1.models.Status` -- see + that enum for the full set of values. An analysis moves through + `PENDING`, `STARTING` and `RUNNING` before reaching `COMPLETED` or + `FAILED`. This is a snapshot taken when the object was built, and it does not update. To watch a running analysis, call `portal.get_dataset(...)` @@ -268,10 +269,11 @@ def logs(self) -> str: """ Return the top-level execution log for this dataset. - This is the log from the head node driving the workflow -- Nextflow's - own output, including which tasks it submitted and why the run stopped. - For the stdout/stderr of one task, use `cirro.sdk.task.DataPortalTask.logs`; - for the log file archived once the run finishes, use `get_logs`. + This is the log from the head node driving the workflow -- the output of + Nextflow or Cromwell itself, depending on the process executor, + including which tasks it submitted and why the run stopped. For the + stdout/stderr of one task, use `cirro.sdk.task.DataPortalTask.logs`; for + the log file archived once the run finishes, use `get_logs`. Returns an empty string if no log events are available (e.g. the job has not started yet). @@ -568,22 +570,22 @@ def get_trace(self) -> Any: Read the Nextflow workflow trace file for this dataset as a DataFrame. One row per task, with timing, resource usage, and exit status. Written - when the run finishes. + when the run finishes. This artifact is specific to Nextflow -- a + Cromwell (WDL) analysis does not produce one. Returns: `pandas.DataFrame` Raises: DataPortalAssetNotFound: if the dataset has no workflow trace - artifact, which is the case for datasets that were uploaded - rather than produced by a Nextflow analysis, and for runs that - have not finished. + artifact -- true for uploaded datasets, for Cromwell analyses, + and for runs that have not finished. """ return self.get_artifact(ArtifactType.WORKFLOW_TRACE).read_csv(sep='\t') def get_logs(self) -> str: """ - Read the archived Nextflow workflow log for this dataset as a string. + Read the archived workflow log for this dataset as a string. This reads the `WORKFLOW_LOGS` artifact, written when the run finishes. For the live head-node log of a run in progress, use `logs` instead. From 1594d558ce2af1c98d1de3af2f83bc2aacd82876 Mon Sep 17 00:00:00 2001 From: Sam Minot Date: Wed, 19 Aug 2026 11:37:11 -0700 Subject: [PATCH 6/8] Revert the docstring markup conversion Rendering the pages before and after the conversion produces byte-identical HTML: pdoc already turns :func:`pandas.read_csv` into `pandas.read_csv()` and ``DataFrame`` into `DataFrame` under docformat="google", so the conversion was reproducing by hand what pdoc does anyway. It also discarded semantic markup that a Sphinx build would resolve into real links, and left `cirro/services` inconsistent with `cirro/sdk`. The docstrings rewritten for content keep the local style: RST for inline literals, Markdown fences for code blocks, matching DataPortal.read_files. task.py is no longer touched -- its whole diff was this churn. Two cross-references drop the `~` prefix, which pdoc does not honour: it renders ":meth:`~cirro.sdk.portal.DataPortal.read_files`" with the tilde still in the text, on main as well. Co-Authored-By: Claude Opus 5 --- cirro/sdk/dataset.py | 56 +++++++++++++-------------- cirro/sdk/file_mixins.py | 16 ++++---- cirro/sdk/portal.py | 82 ++++++++++++++++++++-------------------- cirro/sdk/task.py | 24 ++++++------ 4 files changed, 89 insertions(+), 89 deletions(-) diff --git a/cirro/sdk/dataset.py b/cirro/sdk/dataset.py index 35b60119..60a1cf28 100644 --- a/cirro/sdk/dataset.py +++ b/cirro/sdk/dataset.py @@ -335,11 +335,11 @@ def primary_failed_task(self) -> Optional[DataPortalTask]: """ Find the root-cause failed task in this workflow execution. - Returns `None` gracefully when no tasks are available or none have - a `FAILED` status. + Returns ``None`` gracefully when no tasks are available or none have + a ``FAILED`` status. Returns: - `cirro.sdk.task.DataPortalTask`, or `None` if no failed task is found. + `cirro.sdk.task.DataPortalTask`, or ``None`` if no failed task is found. """ from cirro.helpers.nextflow_utils import find_primary_failed_task @@ -444,24 +444,24 @@ def read_files( """ Read the contents of files in the dataset, without downloading them. - Exactly one of `glob` or `pattern` must be provided. + Exactly one of ``glob`` or ``pattern`` must be provided. **glob** -- standard wildcard matching; yields the file content for each matching file: - - `*` matches any characters within a single path segment - - `**` matches zero or more path segments - - Matching is suffix-anchored (`*.csv` matches at any depth) + - ``*`` matches any characters within a single path segment + - ``**`` matches zero or more path segments + - Matching is suffix-anchored (``*.csv`` matches at any depth) - **pattern** -- like `glob` but `{name}` placeholders capture portions of - the path automatically; yields `(content, meta)` pairs where *meta* is a - `dict` of extracted values: + **pattern** -- like ``glob`` but ``{name}`` placeholders capture portions of + the path automatically; yields ``(content, meta)`` pairs where *meta* is a + ``dict`` of extracted values: - - `{name}` captures one path segment (no `/`) - - `*` and `**` wildcards work as in `glob` + - ``{name}`` captures one path segment (no ``/``) + - ``*`` and ``**`` wildcards work as in ``glob`` - See `cirro.sdk.portal.DataPortal.read_files()` for the full list of - `filetype` values and the extensions each one is inferred from. + See :meth:`cirro.sdk.portal.DataPortal.read_files` for the full list of + ``filetype`` values and the extensions each one is inferred from. ```python # Every CSV in the dataset, as DataFrames @@ -480,19 +480,19 @@ def read_files( Args: glob (str): Wildcard expression to match files. Yields one item per matching file: the parsed content. - pattern (str): Wildcard expression with `{name}` capture - placeholders. Yields `(content, meta)` per matching file. + pattern (str): Wildcard expression with ``{name}`` capture + placeholders. Yields ``(content, meta)`` per matching file. filetype (str): File format used to parse each file - (or `None` to infer from extension). + (or ``None`` to infer from extension). **kwargs: Additional keyword arguments forwarded to the - file-parsing function (e.g. `sep='\\t'` for TSV files). + file-parsing function (e.g. ``sep='\\t'`` for TSV files). Yields: - - When using `glob`: *content* for each matching file - - When using `pattern`: `(content, meta)` for each matching file + - When using ``glob``: *content* for each matching file + - When using ``pattern``: ``(content, meta)`` for each matching file Raises: - DataPortalInputError: if both `glob` and `pattern` are provided, or + DataPortalInputError: if both ``glob`` and ``pattern`` are provided, or if neither is. """ if glob is not None and pattern is not None: @@ -521,7 +521,7 @@ def read_file( Read the contents of a single file from the dataset, without downloading it. - Provide either `path` (the exact relative path) or `glob` (a wildcard + Provide either ``path`` (the exact relative path) or ``glob`` (a wildcard expression, which must match exactly one file). ```python @@ -533,17 +533,17 @@ def read_file( path (str): Exact relative path of the file within the dataset. glob (str): Wildcard expression matching exactly one file. filetype (str): File format used to parse the file. Supported values - are the same as `cirro.sdk.portal.DataPortal.read_files()`. + are the same as :meth:`cirro.sdk.portal.DataPortal.read_files`. **kwargs: Additional keyword arguments forwarded to the file-parsing function. Returns: - Parsed file content -- a `pandas.DataFrame` for tabular formats, a - `str` for text, and so on depending on `filetype`. + Parsed file content -- a ``pandas.DataFrame`` for tabular formats, a + ``str`` for text, and so on depending on ``filetype``. Raises: - DataPortalInputError: if both or neither of `path`/`glob` are given, - or if `glob` matches more than one file. + DataPortalInputError: if both or neither of ``path``/``glob`` are given, + or if ``glob`` matches more than one file. DataPortalAssetNotFound: if nothing matches. """ if path is not None and glob is not None: @@ -642,7 +642,7 @@ def download_files(self, download_location: str = None, glob: str = None) -> Non Args: download_location (str): Path to local directory glob (str): Optional wildcard expression to filter which files are downloaded - (e.g., `'*.csv'`, `'data/**/*.tsv.gz'`). + (e.g., ``'*.csv'``, ``'data/**/*.tsv.gz'``). If omitted, all files are downloaded. """ diff --git a/cirro/sdk/file_mixins.py b/cirro/sdk/file_mixins.py index ad8ac139..066533dc 100644 --- a/cirro/sdk/file_mixins.py +++ b/cirro/sdk/file_mixins.py @@ -15,7 +15,7 @@ class FileReadMixin(ABC): """ Mixin that adds file-reading methods to any class that provides - `_get() -> bytes` and a `name` property. + ``_get() -> bytes`` and a ``name`` property. """ @property @@ -132,8 +132,8 @@ def read_parquet(self, **kwargs) -> 'DataFrame': """ Read a Parquet file as a Pandas DataFrame. - Requires `pyarrow` or `fastparquet` to be installed. - All keyword arguments are passed to `pandas.read_parquet()`. + Requires ``pyarrow`` or ``fastparquet`` to be installed. + All keyword arguments are passed to :func:`pandas.read_parquet`. """ import pandas return pandas.read_parquet(BytesIO(self._get()), **kwargs) @@ -142,8 +142,8 @@ def read_feather(self, **kwargs) -> 'DataFrame': """ Read a Feather file as a Pandas DataFrame. - Requires `pyarrow` to be installed. - All keyword arguments are passed to `pandas.read_feather()`. + Requires ``pyarrow`` to be installed. + All keyword arguments are passed to :func:`pandas.read_feather`. """ import pandas return pandas.read_feather(BytesIO(self._get()), **kwargs) @@ -154,10 +154,10 @@ def read_pickle(self, **kwargs): def read_excel(self, **kwargs) -> 'DataFrame': """ - Read an Excel file (`.xlsx` / `.xls`) as a Pandas DataFrame. + Read an Excel file (``.xlsx`` / ``.xls``) as a Pandas DataFrame. - Requires `openpyxl` (for `.xlsx`) or `xlrd` (for `.xls`). - All keyword arguments are passed to `pandas.read_excel()`. + Requires ``openpyxl`` (for ``.xlsx``) or ``xlrd`` (for ``.xls``). + All keyword arguments are passed to :func:`pandas.read_excel`. """ import pandas return pandas.read_excel(BytesIO(self._get()), **kwargs) diff --git a/cirro/sdk/portal.py b/cirro/sdk/portal.py index c7e8b5d0..cf3049a3 100644 --- a/cirro/sdk/portal.py +++ b/cirro/sdk/portal.py @@ -194,62 +194,62 @@ def read_files( Read the contents of files from a dataset. The project and dataset can each be identified by name or ID. - Exactly one of `glob` or `pattern` must be provided. + Exactly one of ``glob`` or ``pattern`` must be provided. **glob** — standard wildcard matching; yields the file content for each matching file: - - `*` matches any characters within a single path segment - - `**` matches zero or more path segments - - Matching is suffix-anchored (`*.csv` matches at any depth) + - ``*`` matches any characters within a single path segment + - ``**`` matches zero or more path segments + - Matching is suffix-anchored (``*.csv`` matches at any depth) - **pattern** — like `glob` but `{name}` placeholders capture portions - of the path automatically; yields `(content, meta)` pairs where - *meta* is a `dict` of extracted values: + **pattern** — like ``glob`` but ``{name}`` placeholders capture portions + of the path automatically; yields ``(content, meta)`` pairs where + *meta* is a ``dict`` of extracted values: - - `{name}` captures one path segment (no `/`) - - `*` and `**` wildcards work as in `glob` + - ``{name}`` captures one path segment (no ``/``) + - ``*`` and ``**`` wildcards work as in ``glob`` Args: project (str): ID or name of the project. dataset (str): ID or name of the dataset. glob (str): Wildcard expression to match files - (e.g., `'*.csv'`, `'data/**/*.tsv.gz'`). + (e.g., ``'*.csv'``, ``'data/**/*.tsv.gz'``). Yields one item per matching file: the parsed content. - pattern (str): Wildcard expression with `{name}` capture - placeholders (e.g., `'{sample}.csv'`, - `'{condition}/{sample}.csv'`). - Yields `(content, meta)` per matching file. + pattern (str): Wildcard expression with ``{name}`` capture + placeholders (e.g., ``'{sample}.csv'``, + ``'{condition}/{sample}.csv'``). + Yields ``(content, meta)`` per matching file. filetype (str): File format used to parse each file. Supported values: - - `'csv'`: parse with `pandas.read_csv()`, returns a `DataFrame` - - `'h5ad'`: parse as AnnData (requires `anndata` package) - - `'json'`: parse with `json.loads()`, returns a Python object - - `'parquet'`: parse with `pandas.read_parquet()`, returns a `DataFrame` - (requires `pyarrow` or `fastparquet`) - - `'feather'`: parse with `pandas.read_feather()`, returns a `DataFrame` - (requires `pyarrow`) - - `'pickle'`: deserialize with `pickle`, returns a Python object - - `'excel'`: parse with `pandas.read_excel()`, returns a `DataFrame` - (requires `openpyxl` for `.xlsx` or `xlrd` for `.xls`) - - `'text'`: read as plain text, returns a `str` - - `'bytes'`: read as raw bytes, returns `bytes` - - `None` (default): infer from file extension - (`.csv`/`.tsv` → `'csv'`, `.h5ad` → `'h5ad'`, - `.json` → `'json'`, `.parquet` → `'parquet'`, - `.feather` → `'feather'`, `.pkl`/`.pickle` → `'pickle'`, - `.xlsx`/`.xls` → `'excel'`, otherwise `'text'`) + - ``'csv'``: parse with :func:`pandas.read_csv`, returns a ``DataFrame`` + - ``'h5ad'``: parse as AnnData (requires ``anndata`` package) + - ``'json'``: parse with :func:`json.loads`, returns a Python object + - ``'parquet'``: parse with :func:`pandas.read_parquet`, returns a ``DataFrame`` + (requires ``pyarrow`` or ``fastparquet``) + - ``'feather'``: parse with :func:`pandas.read_feather`, returns a ``DataFrame`` + (requires ``pyarrow``) + - ``'pickle'``: deserialize with :mod:`pickle`, returns a Python object + - ``'excel'``: parse with :func:`pandas.read_excel`, returns a ``DataFrame`` + (requires ``openpyxl`` for ``.xlsx`` or ``xlrd`` for ``.xls``) + - ``'text'``: read as plain text, returns a ``str`` + - ``'bytes'``: read as raw bytes, returns ``bytes`` + - ``None`` (default): infer from file extension + (``.csv``/``.tsv`` → ``'csv'``, ``.h5ad`` → ``'h5ad'``, + ``.json`` → ``'json'``, ``.parquet`` → ``'parquet'``, + ``.feather`` → ``'feather'``, ``.pkl``/``.pickle`` → ``'pickle'``, + ``.xlsx``/``.xls`` → ``'excel'``, otherwise ``'text'``) **kwargs: Additional keyword arguments forwarded to the file-parsing - function (e.g., `sep='\\t'` for CSV/TSV files). + function (e.g., ``sep='\\t'`` for CSV/TSV files). Yields: - - When using `glob`: *content* for each matching file - - When using `pattern`: `(content, meta)` for each matching file, - where *meta* is a `dict` of values extracted from `{name}` + - When using ``glob``: *content* for each matching file + - When using ``pattern``: ``(content, meta)`` for each matching file, + where *meta* is a ``dict`` of values extracted from ``{name}`` placeholders Raises: - DataPortalInputError: if both `glob` and `pattern` are provided, + DataPortalInputError: if both ``glob`` and ``pattern`` are provided, or if neither is provided. Example: @@ -287,8 +287,8 @@ def read_file( Read the contents of a single file from a dataset. The project and dataset can each be identified by name or ID. - Provide either `path` (exact relative path) or `glob` (wildcard - expression). If `glob` is used it must match exactly one file. + Provide either ``path`` (exact relative path) or ``glob`` (wildcard + expression). If ``glob`` is used it must match exactly one file. Args: project (str): ID or name of the project. @@ -296,7 +296,7 @@ def read_file( path (str): Exact relative path of the file within the dataset. glob (str): Wildcard expression matching exactly one file. filetype (str): File format used to parse the file. Supported values - are the same as `read_files()`. + are the same as :meth:`read_files`. **kwargs: Additional keyword arguments forwarded to the file-parsing function. @@ -304,8 +304,8 @@ def read_file( Parsed file content. Raises: - DataPortalInputError: if both or neither of `path`/`glob` are - provided, or if `glob` matches zero or more than one file. + DataPortalInputError: if both or neither of ``path``/``glob`` are + provided, or if ``glob`` matches zero or more than one file. """ ds = self.get_dataset(project=project, dataset=dataset) return ds.read_file(path=path, glob=glob, filetype=filetype, **kwargs) diff --git a/cirro/sdk/task.py b/cirro/sdk/task.py index d87b4a62..5e9e65b6 100644 --- a/cirro/sdk/task.py +++ b/cirro/sdk/task.py @@ -28,8 +28,8 @@ class WorkDirFile(FileReadMixin): A file that lives in a Nextflow work directory or a dataset staging area. Each WorkDirFile either originated from another task's work directory - (`source_task` is set) or was a primary/staged input to the workflow - (`source_task` is `None`). + (``source_task`` is set) or was a primary/staged input to the workflow + (``source_task`` is ``None``). """ def __init__( @@ -42,7 +42,7 @@ def __init__( dataset_id: str = '' ): """ - Obtained from a task's `inputs` or `outputs` property. + Obtained from a task's ``inputs`` or ``outputs`` property. ```python for task in dataset.tasks: @@ -60,7 +60,7 @@ def __init__( @property def source_task(self) -> Optional['DataPortalTask']: - """The task that produced this file, or `None` for staged/primary inputs.""" + """The task that produced this file, or ``None`` for staged/primary inputs.""" return self._source_task @property @@ -143,7 +143,7 @@ def __init__( task_id: int = 0 ): """ - Obtained from a dataset's `tasks` property. + Obtained from a dataset's ``tasks`` property. ```python for task in dataset.tasks: @@ -157,7 +157,7 @@ def __init__( project_id (str): ID of the project that owns this dataset. dataset_id (str): ID of the dataset (execution) that owns this task. all_tasks_ref (list): A shared list that will contain all tasks once they - are all built. Used by `inputs` to resolve `source_task`. + are all built. Used by ``inputs`` to resolve ``source_task``. task_id (int): Numeric index of this task in the execution's task list. """ self._task = task @@ -178,12 +178,12 @@ def task_id(self) -> int: @property def name(self) -> str: - """Full task name, e.g. `NFCORE_RNASEQ:RNASEQ:TRIMGALORE (sample1)`.""" + """Full task name, e.g. ``NFCORE_RNASEQ:RNASEQ:TRIMGALORE (sample1)``.""" return self._task.name @property def status(self) -> str: - """Task status string, e.g. `COMPLETED`, `FAILED`, `ABORTED`.""" + """Task status string, e.g. ``COMPLETED``, ``FAILED``, ``ABORTED``.""" return self._task.status @property @@ -279,7 +279,7 @@ def logs(self) -> str: Fetches via the Cirro execution API when a native job ID is available, which works even when the S3 scratch bucket is not directly accessible. - Falls back to reading `.command.log` from the S3 work directory. + Falls back to reading ``.command.log`` from the S3 work directory. Returns an empty string if neither source can be read. """ if self._dataset_id and self.native_id: @@ -296,11 +296,11 @@ def logs(self) -> str: @cached_property def script(self) -> str: """ - Return the contents of `.command.sh` from the task's work directory. + Return the contents of ``.command.sh`` from the task's work directory. This is the actual shell script that Nextflow executed — the user's pipeline code for this task. Falls back to parsing the script from the - `WORKFLOW_LOGS` artifact when the work directory is not accessible + ``WORKFLOW_LOGS`` artifact when the work directory is not accessible (scratch bucket requires elevated permissions). Returns an empty string if the script cannot be obtained. """ @@ -379,7 +379,7 @@ def inputs(self) -> List[WorkDirFile]: """ List of input files for this task, fetched from the execution API. - Each file is annotated with `source_task` if its URI falls within + Each file is annotated with ``source_task`` if its URI falls within another task's work directory. """ return self._build_inputs() From 726ec7658350c1adba28e240461f8e34b8e5dd2d Mon Sep 17 00:00:00 2001 From: Sam Minot Date: Thu, 27 Aug 2026 10:07:11 -0700 Subject: [PATCH 7/8] version = "1.13.0" --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 845d74d2..658a1147 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cirro" -version = "1.13.1" +version = "1.13.0" description = "CLI tool and SDK for interacting with the Cirro platform" authors = ["Cirro Bio "] license = "MIT" From 14a511b5c6d47e13160e4e77f68483f6ac282115 Mon Sep 17 00:00:00 2001 From: Sam Minot Date: Thu, 27 Aug 2026 10:08:15 -0700 Subject: [PATCH 8/8] asset description type hint --- cirro/sdk/asset.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cirro/sdk/asset.py b/cirro/sdk/asset.py index 68c7a920..05b3db5b 100644 --- a/cirro/sdk/asset.py +++ b/cirro/sdk/asset.py @@ -55,12 +55,9 @@ def __init__(self, input_list: List[T]): def __str__(self): return "\n".join([str(i) for i in self]) - def description(self): + def description(self) -> str: """ Render a text summary of the assets, one block per asset. - - Returns: - str """ return '\n\n---\n\n'.join([