diff --git a/cirro/__init__.py b/cirro/__init__.py index 119f6b4..93f20fc 100644 --- a/cirro/__init__.py +++ b/cirro/__init__.py @@ -1,3 +1,112 @@ +""" +Python SDK and command-line interface for the [Cirro](https://cirro.bio) platform. + +Install with `pip install cirro`. + +## Authentication + +`DataPortal` needs to know which Cirro instance to talk to and how to +authenticate. The instance comes from the `base_url` argument, falling back to +the `CIRRO_BASE_URL` environment variable, then to the saved configuration in +`~/.cirro/config.ini` (written by `cirro configure`). + +There are three ways to authenticate: + +1. **Interactive (the default).** `DataPortal()` uses the saved configuration. + If none exists it starts a device-code login, prints a URL, and **blocks + until the user completes the login in a browser**. Avoid this in scripts and + automated sessions -- there is nobody to click the link, so it will hang + until the device code expires. + +2. **Headless.** OAuth client credentials never prompt, so this is the option + to use for automation: + + ```python + import os + from cirro import CirroApi, DataPortal + from cirro.auth.client_creds import ClientCredentialsAuth + from cirro.config import AppConfig + + config = AppConfig(base_url="app.cirro.bio") + auth = ClientCredentialsAuth( + os.environ["CIRRO_CLIENT_ID"], + os.environ["CIRRO_CLIENT_SECRET"], + auth_endpoint=config.auth_endpoint + ) + portal = DataPortal(client=CirroApi(auth_info=auth)) + ``` + + See [OAuth Apps](https://docs.cirro.bio/cli-sdk/oauth-apps/) for how to + create the client ID and secret. + +3. **Non-blocking browser login.** `cirro.sdk.login.DataPortalLogin` returns + the authorization message so you can display it yourself, and blocks only + when you call `await_completion()`. + +## Quickstart + +```python +from cirro import DataPortal + +portal = DataPortal(base_url="app.cirro.bio") + +# Browse +for project in portal.list_projects(): + print(project.name) + +# Read a file straight into a DataFrame, without downloading it +df = portal.read_file("Name of Project", "Name of Dataset", glob="*.csv") + +# Launch an analysis on an existing dataset +dataset = portal.get_dataset(project="Name of Project", dataset="Name of Dataset") +new_dataset_id = dataset.run_analysis( + name="Name of the output dataset", + process="Name or ID of the process to run", + params={} +) +``` + +## Object model + +- `cirro.sdk.portal.DataPortal` -- entry point; lists projects, processes, + and reference types. +- `cirro.sdk.project.DataPortalProject` -- a permissions boundary holding + datasets and reference data; uploads new datasets. +- `cirro.sdk.dataset.DataPortalDataset` -- a collection of files, either + uploaded or produced by an analysis; reads files and launches analyses. +- `cirro.sdk.file.DataPortalFile` -- one file; read it into memory + (`read_csv`, `read_json`, ...) or download it. +- `cirro.sdk.process.DataPortalProcess` -- a pipeline that can be run, or a + data type that datasets can be uploaded as. +- `cirro.sdk.task.DataPortalTask` -- one task from a Nextflow execution, used + for debugging failed analyses. +- `cirro.sdk.reference.DataPortalReference` -- reference data (genomes, + annotations) available to a project. +- `cirro.cirro_client.CirroApi` -- the lower-level typed API client; use it + when the classes above do not cover what you need. + +Projects, datasets, and processes can be looked up by either name or ID -- +`get_project`, `get_dataset`, and `run_analysis` all accept either. + +Every `list_*` method returns a `list` subclass +(`cirro.sdk.asset.DataPortalAssets`) with extra lookup helpers: +`get_by_name`, `get_by_id`, and `filter_by_pattern`. + +## Freshness + +These objects hold a snapshot of what the API returned when they were built. +Properties such as `cirro.sdk.dataset.DataPortalDataset.status` and +`cirro.sdk.dataset.DataPortalDataset.logs` will not change on an object you +already have. To watch a running analysis, call `portal.get_dataset(...)` again +each time round the loop. + +## Worked examples + +The [samples directory](https://github.com/CirroBio/Cirro-SDK-Python/tree/main/samples) +holds runnable notebooks for uploading, downloading, reading files, running and +debugging analyses, managing reference data, and integrating pipelines. +""" + import cirro.file_utils # noqa from cirro.cirro_client import CirroApi from cirro.sdk.dataset import DataPortalDataset diff --git a/cirro/cirro_client.py b/cirro/cirro_client.py index 041882b..41b66be 100644 --- a/cirro/cirro_client.py +++ b/cirro/cirro_client.py @@ -17,9 +17,15 @@ def __init__(self, auth_info: AuthInfo = None, base_url: str = None, user_agent: Instantiates the Cirro API object Args: - auth_info (cirro.auth.base.AuthInfo): + auth_info (`cirro.auth.base.AuthInfo`): How to authenticate. If + omitted, this is read from the saved configuration, which falls back + to an interactive device-code login that blocks on a browser flow. + Pass `cirro.auth.client_creds.ClientCredentialsAuth` to authenticate + without prompting. base_url (str): Optional base URL of the Cirro instance (if not provided, it uses the `CIRRO_BASE_URL` environment variable, or the config file) + user_agent (str): Name reported to the API for this client, which + shows up in Cirro's audit logs. Returns: Authenticated Cirro API object, which can be used to call endpoint functions. diff --git a/cirro/sdk/__init__.py b/cirro/sdk/__init__.py index e69de29..7987bda 100644 --- a/cirro/sdk/__init__.py +++ b/cirro/sdk/__init__.py @@ -0,0 +1,20 @@ +""" +The high-level, object-oriented interface to Cirro. + +Start from `cirro.sdk.portal.DataPortal`, which is re-exported as +`cirro.DataPortal`. Every other class in this package is reached from it rather +than constructed directly: + +``` +DataPortal +├── list_projects() -> DataPortalProject +│ ├── list_datasets() -> DataPortalDataset +│ │ ├── list_files() -> DataPortalFile +│ │ └── tasks -> DataPortalTask +│ └── list_references() -> DataPortalReference +├── list_processes() -> DataPortalProcess +└── list_reference_types() -> DataPortalReferenceType +``` + +For the lower-level typed API client, see `cirro.cirro_client.CirroApi`. +""" diff --git a/cirro/sdk/asset.py b/cirro/sdk/asset.py index 082200f..05b3db5 100644 --- a/cirro/sdk/asset.py +++ b/cirro/sdk/asset.py @@ -6,7 +6,12 @@ class DataPortalAsset: - """Base class used for all Data Portal Assets""" + """ + Base class used for all Data Portal Assets. + + Assets are not constructed directly -- each one is obtained from a method on + `cirro.sdk.portal.DataPortal` or on another asset. + """ @property @abstractmethod @@ -23,7 +28,22 @@ def __repr__(self): class DataPortalAssets(List[T]): """ - Generic class with helper functions for any group of assets (projects, datasets, etc.) + A `list` of assets (projects, datasets, files, ...) with lookup helpers. + + Every `list_*` method in the SDK returns one of these rather than a plain + list, so anything you can do with a list works, plus lookup by name or ID + and filtering by wildcard: + + ```python + projects = portal.list_projects() + + for project in projects: # ordinary list iteration + print(project.name) + + project = projects.get_by_name("My Project") + subset = projects.filter_by_pattern("RNA-seq*") + print(projects.description()) # printable summary of them all + ``` """ # Overridden by child classes @@ -35,8 +55,10 @@ def __init__(self, input_list: List[T]): def __str__(self): return "\n".join([str(i) for i in self]) - def description(self): - """Render a text summary of the assets.""" + def description(self) -> str: + """ + Render a text summary of the assets, one block per asset. + """ return '\n\n---\n\n'.join([ str(i) @@ -44,7 +66,21 @@ def description(self): ]) def get_by_name(self, name: str) -> T: - """Return the item which matches with name attribute.""" + """ + Return the single item whose `name` attribute matches exactly. + + Args: + name (str): Name to match. Matching is exact and case-sensitive; + use `filter_by_pattern` for wildcards. + + Returns: + The matching item. + + Raises: + DataPortalInputError: if `name` is None, or if several items share + the name -- in which case use `get_by_id`. + DataPortalAssetNotFound: if nothing matches. + """ if name is None: raise DataPortalInputError(f"Must provide name to identify {self.asset_name}") @@ -65,7 +101,21 @@ def get_by_name(self, name: str) -> T: return matching_queries[0] def get_by_id(self, _id: str) -> T: - """Return the item which matches by id attribute.""" + """ + Return the single item whose `id` attribute matches exactly. + + For files, the `id` is the relative path within the dataset. + + Args: + _id (str): ID to match. + + Returns: + The matching item. + + Raises: + DataPortalInputError: if `_id` is None. + DataPortalAssetNotFound: if nothing matches. + """ if _id is None: raise DataPortalInputError(f"Must provide id to identify {self.asset_name}") @@ -81,7 +131,17 @@ def get_by_id(self, _id: str) -> T: return matching_queries[0] def filter_by_pattern(self, pattern: str) -> 'DataPortalAssets[T]': - """Filter the items to just those whose name attribute matches the pattern.""" + """ + Return the items whose `name` matches a shell-style wildcard pattern. + + Args: + pattern (str): Wildcard pattern, matched with `fnmatch` -- `*` for + any run of characters, `?` for one, `[seq]` for a character set. + + Returns: + A new collection of the same type holding the matching items, empty + if none match. + """ # Get a list of the names to search against all_names = [i.name for i in self] diff --git a/cirro/sdk/dataset.py b/cirro/sdk/dataset.py index a2f0d87..cdb95ae 100644 --- a/cirro/sdk/dataset.py +++ b/cirro/sdk/dataset.py @@ -173,7 +173,14 @@ def project_id(self) -> str: @property def status(self) -> Status: """ - Status of the dataset + Status of the dataset, as a `cirro_api_client.v1.models.Status` -- see + that enum for the full set of values. An analysis moves through + `PENDING`, `STARTING` and `RUNNING` before reaching `COMPLETED` or + `FAILED`. + + This is a snapshot taken when the object was built, and it does not + update. To watch a running analysis, call `portal.get_dataset(...)` + again each time round the loop rather than re-reading this property. """ return self._data.status @@ -225,14 +232,27 @@ def share(self) -> Optional[NamedItem]: @property def file_count(self) -> int: + """ + Number of files in the dataset. + + Fetches the full dataset detail from the API on first access if this + object was built from a listing. + """ return self._get_detail().file_count @property def total_size_bytes(self) -> int: + """ + Combined size of every file in the dataset, in bytes. + + Fetches the full dataset detail from the API on first access if this + object was built from a listing. + """ return self._get_detail().total_size_bytes @property def total_size(self) -> str: + """Combined size of every file in the dataset, human-readable (e.g. 4.50 GB).""" return bytes_to_human_readable(self.total_size_bytes) @property @@ -250,8 +270,18 @@ def logs(self) -> str: """ Return the top-level execution log for this dataset. + This is the log from the head node driving the workflow -- the output of + Nextflow or Cromwell itself, depending on the process executor, + including which tasks it submitted and why the run stopped. For the + stdout/stderr of one task, use `cirro.sdk.task.DataPortalTask.logs`; for + the log file archived once the run finishes, use `get_logs`. + Returns an empty string if no log events are available (e.g. the job has not started yet). + Cached after the first access. Reading this while the analysis is still + starting up returns `''` and will keep returning `''` for the lifetime + of this object -- re-fetch the dataset to try again. + Returns: str: Execution log text, or an empty string if unavailable. """ @@ -268,8 +298,12 @@ def tasks(self) -> List[DataPortalTask]: """ List of tasks from the workflow execution, fetched via the execution API. + Cached after the first access, so a list read while the analysis is + still running will not pick up tasks that start later -- re-fetch the + dataset for an up-to-date list. + Returns: - `List[DataPortalTask]` + `List[cirro.sdk.task.DataPortalTask]` """ return self._load_tasks_from_api() @@ -341,11 +375,17 @@ def get_file(self, relative_path: str) -> DataPortalFile: """ Get a file from the dataset using its relative path. + The leading `data/` prefix is optional -- it is tried automatically if + the path is not found as given. + Args: relative_path (str): Relative path of file within the dataset Returns: - `from cirro.sdk.file import DataPortalFile` + `cirro.sdk.file.DataPortalFile` + + Raises: + DataPortalAssetNotFound: if no file in the dataset has this path. """ # Get the list of files in this dataset @@ -369,8 +409,17 @@ def list_files(self, file_limit: int = 100000) -> DataPortalFiles: """ Return the list of files which make up the dataset. + The result is a `cirro.sdk.asset.DataPortalAssets` list, so it also + offers `get_by_name`, `get_by_id`, and `filter_by_pattern`. Files are + listed by their relative path within the dataset, most of which sit + under a `data/` prefix. + Args: - file_limit (int): Maximum number of files to return (default 100,000) + file_limit (int): Maximum number of files to return (default 100,000). + A dataset with more files than this is truncated silently. + + Returns: + `cirro.sdk.file.DataPortalFiles` """ assets = self._client.datasets.get_assets_listing( project_id=self.project_id, @@ -394,10 +443,40 @@ def read_files( **kwargs ): """ - Read the contents of files in the dataset. + Read the contents of files in the dataset, without downloading them. + + Exactly one of ``glob`` or ``pattern`` must be provided. + + **glob** -- standard wildcard matching; yields the file content for each + matching file: + + - ``*`` matches any characters within a single path segment + - ``**`` matches zero or more path segments + - Matching is suffix-anchored (``*.csv`` matches at any depth) + + **pattern** -- like ``glob`` but ``{name}`` placeholders capture portions of + the path automatically; yields ``(content, meta)`` pairs where *meta* is a + ``dict`` of extracted values: + + - ``{name}`` captures one path segment (no ``/``) + - ``*`` and ``**`` wildcards work as in ``glob`` + + See :meth:`cirro.sdk.portal.DataPortal.read_files` for the full list of + ``filetype`` values and the extensions each one is inferred from. + + ```python + # Every CSV in the dataset, as DataFrames + for df in dataset.read_files(glob='*.csv'): + print(df.shape) + + # Capture the sample name from each filename + for df, meta in dataset.read_files(pattern='{sample}.csv'): + print(meta['sample'], df.shape) - See :meth:`~cirro.sdk.portal.DataPortal.read_files` for full details - on ``glob``/``pattern`` matching and filetype options. + # Gzipped TSVs at any depth + for df in dataset.read_files(glob='**/*.tsv.gz', filetype='csv', sep='\\t'): + print(df.shape) + ``` Args: glob (str): Wildcard expression to match files. @@ -407,11 +486,15 @@ def read_files( filetype (str): File format used to parse each file (or ``None`` to infer from extension). **kwargs: Additional keyword arguments forwarded to the - file-parsing function. + file-parsing function (e.g. ``sep='\\t'`` for TSV files). Yields: - When using ``glob``: *content* for each matching file - When using ``pattern``: ``(content, meta)`` for each matching file + + Raises: + DataPortalInputError: if both ``glob`` and ``pattern`` are provided, or + if neither is. """ if glob is not None and pattern is not None: raise DataPortalInputError("Cannot specify both 'glob' and 'pattern' — use one or the other") @@ -436,20 +519,33 @@ def read_file( **kwargs ) -> Any: """ - Read the contents of a single file from the dataset. + Read the contents of a single file from the dataset, without + downloading it. + + Provide either ``path`` (the exact relative path) or ``glob`` (a wildcard + expression, which must match exactly one file). - See :meth:`~cirro.sdk.portal.DataPortal.read_file` for full details. + ```python + df = dataset.read_file(path='data/counts.csv') + df = dataset.read_file(glob='**/counts.csv') + ``` Args: path (str): Exact relative path of the file within the dataset. glob (str): Wildcard expression matching exactly one file. filetype (str): File format used to parse the file. Supported values - are the same as :meth:`~cirro.sdk.portal.DataPortal.read_files`. + are the same as :meth:`cirro.sdk.portal.DataPortal.read_files`. **kwargs: Additional keyword arguments forwarded to the file-parsing function. Returns: - Parsed file content. + Parsed file content -- a ``pandas.DataFrame`` for tabular formats, a + ``str`` for text, and so on depending on ``filetype``. + + Raises: + DataPortalInputError: if both or neither of ``path``/``glob`` are given, + or if ``glob`` matches more than one file. + DataPortalAssetNotFound: if nothing matches. """ if path is not None and glob is not None: raise DataPortalInputError("Cannot specify both 'path' and 'glob' — use one or the other") @@ -474,23 +570,49 @@ def get_trace(self) -> Any: """ Read the Nextflow workflow trace file for this dataset as a DataFrame. + One row per task, with timing, resource usage, and exit status. Written + when the run finishes. This artifact is specific to Nextflow -- a + Cromwell (WDL) analysis does not produce one. + Returns: `pandas.DataFrame` + + Raises: + DataPortalAssetNotFound: if the dataset has no workflow trace + artifact -- true for uploaded datasets, for Cromwell analyses, + and for runs that have not finished. """ return self.get_artifact(ArtifactType.WORKFLOW_TRACE).read_csv(sep='\t') def get_logs(self) -> str: """ - Read the Nextflow workflow logs for this dataset as a string. + Read the archived workflow log for this dataset as a string. + + This reads the `WORKFLOW_LOGS` artifact, written when the run finishes. + For the live head-node log of a run in progress, use `logs` instead. Returns: str + + Raises: + DataPortalAssetNotFound: if the dataset has no workflow log + artifact yet. """ return self.get_artifact(ArtifactType.WORKFLOW_LOGS).read() def get_artifact(self, artifact_type: ArtifactType) -> DataPortalFile: """ - Get the artifact of a particular type from the dataset + Get the artifact of a particular type from the dataset. + + Args: + artifact_type (`cirro_api_client.v1.models.ArtifactType`): Type of + artifact to return, e.g. `ArtifactType.WORKFLOW_TRACE`. + + Returns: + `cirro.sdk.file.DataPortalFile` + + Raises: + DataPortalAssetNotFound: if the dataset has no artifact of this type. """ artifacts = self._get_assets().artifacts artifact = next((a for a in artifacts if a.artifact_type == artifact_type), None) @@ -549,6 +671,35 @@ def run_analysis( The process can be provided as either a DataPortalProcess object, or a string which corresponds to the name or ID of the process. + The analysis runs asynchronously. The output dataset is registered + immediately in a `PENDING` state and this method returns as soon as the + job is submitted -- it does not wait for the analysis to finish. + + To find out which `params` a process accepts, ask the process itself: + + ```python + process = portal.get_process_by_name("Name of process") + spec = process.get_parameter_spec() + spec.print() # human-readable listing of every parameter + spec.validate_params(params) # raises if params do not fit the schema + ``` + + To follow the analysis, re-fetch the dataset each time round the loop. + `status` on a dataset object you already hold reflects the moment that + object was built and will never change: + + ```python + from time import sleep + from cirro_api_client.v1.models import Status + + dataset_id = dataset.run_analysis(name="Output", process="Name of process") + while True: + result = portal.get_dataset(project=dataset.project_id, dataset=dataset_id) + if result.status in (Status.COMPLETED, Status.FAILED): + break + sleep(30) + ``` + Args: name (str): Name of newly created dataset description (str): Description of newly created dataset diff --git a/cirro/sdk/developer.py b/cirro/sdk/developer.py index b178d62..a562090 100644 --- a/cirro/sdk/developer.py +++ b/cirro/sdk/developer.py @@ -9,6 +9,11 @@ class Matches(list[FileNameMatch]): + """ + The file name matches produced by + `DeveloperHelper.test_file_name_validation`. + """ + def print(self): """ Prints the file name validation matches in a readable format. @@ -27,9 +32,22 @@ class DeveloperHelper: Helper class for developer-related tasks, such as adding samplesheet preprocessing for a pipeline or testing file name validation and sample autopopulation. + + These are for building and debugging Cirro pipelines and data types, not for + analysing data. Obtained from + `cirro.sdk.portal.DataPortal.developer_helper`. """ def __init__(self, client: CirroApi): + """ + Obtained from `cirro.sdk.portal.DataPortal.developer_helper`. + + ```python + from cirro import DataPortal + portal = DataPortal() + helper = portal.developer_helper + ``` + """ self.client = client def generate_preprocess_for_input_datasets(self, @@ -37,10 +55,22 @@ def generate_preprocess_for_input_datasets(self, input_dataset_ids: list[str], params=None) -> PreprocessDataset: """ - Generates a PreprocessDataset object for the given datasets - - With optional parameters to pass into the preprocess script. - Certain properties of `metadata` are available in this context. + Generates a PreprocessDataset object for the given datasets. + + Use this to develop and test a pipeline's `preprocess.py` locally against + real datasets, without running the pipeline. + + Args: + project_id (str): ID of the project holding the input datasets. + input_dataset_ids (list[str]): IDs of the datasets to use as input. + params (dict): Parameters to expose to the preprocess script, as if + they had been entered in the analysis form. + + Returns: + `cirro.helpers.preprocess_dataset.PreprocessDataset` -- with real + samplesheet and file listings, and partially mocked `metadata`: only + `project` and `inputs` are populated, while `dataset` and `process` + are empty. """ samplesheets = self._generate_samplesheets_for_datasets(project_id, input_dataset_ids) project = self.client.projects.get(project_id) @@ -72,6 +102,16 @@ def test_file_name_validation_for_dataset(self, Used when configuring Cirro's sample autopopulation feature. More info: https://docs.cirro.bio/features/samples/#using-auto-population + + Args: + project_id (str): ID of the project holding the dataset. + dataset_id (str): ID of the dataset whose file names to test against. + file_name_patterns (list[str]): Regex patterns to test, as they would + appear in a process definition. + + Returns: + `Matches` -- call `print()` on it for a readable report of which + files matched which pattern, and what sample name each produced. """ dataset_files = self.client.datasets.get_assets_listing(project_id=project_id, dataset_id=dataset_id).files file_names = [file.relative_path for file in dataset_files] @@ -82,6 +122,16 @@ def test_file_name_validation(self, file_name_patterns: list[str]) -> Matches: """ Tests the file name validation for a list of file names against specified regex patterns. + + The same as `test_file_name_validation_for_dataset`, but against file + names you supply rather than a dataset's. + + Args: + file_names (list[str]): File names to test. + file_name_patterns (list[str]): Regex patterns to test them against. + + Returns: + `Matches` """ request_body = ValidateFileNamePatternsRequest( file_names=file_names, @@ -97,7 +147,18 @@ def test_file_name_validation(self, def generate_samplesheets_for_dataset(self, project_id: str, dataset_id: str) -> SampleSheets: """ - Generates Cirro samplesheets for a given dataset + Generates Cirro samplesheets for a given dataset. + + These are the `samplesheet.csv` and `files.csv` that Cirro stages for a + pipeline run, useful for checking what a pipeline will actually receive. + + Args: + project_id (str): ID of the project holding the dataset. + dataset_id (str): ID of the dataset. + + Returns: + `cirro_api_client.v1.models.SampleSheets` -- with `samples` and + `files` each holding CSV text. """ return get_sample_sheets.sync( project_id=project_id, @@ -109,6 +170,13 @@ def rerun_sample_ingest_for_dataset(self, project_id: str, dataset_id: str): """ Reruns the sample ingest process for a given dataset. You'll want to do this if you have updated the file name patterns in your pipeline (or data type) + + This re-derives the dataset's samples and their metadata from its file + names, replacing what is currently recorded. + + Args: + project_id (str): ID of the project holding the dataset. + dataset_id (str): ID of the dataset to re-ingest samples for. """ ingest_samples.sync_detailed( project_id=project_id, diff --git a/cirro/sdk/exceptions.py b/cirro/sdk/exceptions.py index 68f6e56..622a04a 100644 --- a/cirro/sdk/exceptions.py +++ b/cirro/sdk/exceptions.py @@ -4,5 +4,5 @@ class DataPortalAssetNotFound(Exception): class DataPortalInputError(Exception): - """Exception raised invalid inputs are provided to the Data Portal.""" + """Exception raised when invalid inputs are provided to the Data Portal.""" pass diff --git a/cirro/sdk/file.py b/cirro/sdk/file.py index f73eae7..36608ee 100644 --- a/cirro/sdk/file.py +++ b/cirro/sdk/file.py @@ -87,8 +87,15 @@ def download(self, download_location: str = None) -> Path: """ Download the file to a local directory. + Args: + download_location (str): Local directory to write the file into. The + file keeps its relative path within that directory. + Returns: - Path to download file + `pathlib.Path`: path to the downloaded file. + + Raises: + DataPortalInputError: if `download_location` is not provided. """ if download_location is None: @@ -139,12 +146,19 @@ def download(self, download_location: str = None, """ Download the collection of files to a local directory. + Each file keeps its relative path within the dataset, and the files + transfer concurrently. + Args: - download_location (str): Path to local directory - threads (int): Number of files to download at once. 1 disables threading. + download_location (str): Local directory to write the files into. + threads (int): Number of files to download at once (default 8). + 1 disables threading. Returns: - List of paths to downloaded files. + `List[pathlib.Path]`: paths to the downloaded files. + + Raises: + DataPortalInputError: if `download_location` is not provided. """ if len(self) == 0: diff --git a/cirro/sdk/file_mixins.py b/cirro/sdk/file_mixins.py index e9ba08e..066533d 100644 --- a/cirro/sdk/file_mixins.py +++ b/cirro/sdk/file_mixins.py @@ -28,7 +28,19 @@ def _get(self) -> bytes: """Return the raw file bytes.""" def read(self, encoding='utf-8', compression=None) -> str: - """Read the file contents as text.""" + """ + Read the file contents as text. + + Args: + encoding (str): Text encoding to decode with. + compression (str): Pass `'gzip'` to decompress first, or `None` to + read the bytes as-is. Unlike `read_csv`, this is not inferred + from the file extension. + + Raises: + DataPortalInputError: if `compression` is anything other than + `'gzip'` or `None`. + """ cont = self._get() if compression is None: return cont.decode(encoding) @@ -38,24 +50,42 @@ def read(self, encoding='utf-8', compression=None) -> str: return handle.read() def readlines(self, encoding='utf-8', compression=None) -> List[str]: - """Read the file contents as a list of lines.""" + """ + Read the file contents as a list of lines, without trailing newlines. + + Args: + encoding (str): Text encoding to decode with. + compression (str): Pass `'gzip'` to decompress first, or `None` to + read the bytes as-is. + """ return self.read(encoding=encoding, compression=compression).splitlines() def read_bytes(self) -> BytesIO: - """Get a BytesIO object for the file contents, to pass into arbitrary readers.""" + """ + Get a BytesIO object for the file contents, to pass into arbitrary readers. + + Use this for formats the mixin does not cover -- the whole file is read + into memory and handed over as a file-like object. + """ return BytesIO(self._get()) def read_csv(self, compression='infer', encoding='utf-8', **kwargs) -> 'DataFrame': """ Parse the file as a Pandas DataFrame. - The default field separator is a comma (for CSV), use sep='\\t' for TSV. - - File compression is inferred from the extension, but can be set - explicitly with the compression= flag. - - All other keyword arguments are passed to pandas.read_csv - https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html + Args: + compression (str | dict): How the file is compressed. The default, + `'infer'`, picks gzip/bz2/xz/zstd from a `.gz`, `.bz2`, `.xz`, + or `.zst` extension, and no compression otherwise. Pass `None` + to force reading as plain text. + encoding (str): Text encoding to decode with. + **kwargs: Passed through to + [pandas.read_csv](https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html). + The field separator defaults to a comma, so pass `sep='\\t'` + for TSV files. + + Returns: + `pandas.DataFrame` """ import pandas diff --git a/cirro/sdk/helpers.py b/cirro/sdk/helpers.py index fef80b7..ff70605 100644 --- a/cirro/sdk/helpers.py +++ b/cirro/sdk/helpers.py @@ -10,7 +10,19 @@ def parse_process_name_or_id(process: Union[DataPortalProcess, str], client: CirroApi): """ - If the process is a string, try to parse it as a process name or ID. + Resolve a process given as a name, an ID, or an already-built object. + + Args: + process (str | `cirro.sdk.process.DataPortalProcess`): Name or ID of the + process, or the process object itself (returned unchanged). + client (`cirro.cirro_client.CirroApi`): Client to look the process up with. + + Returns: + `cirro.sdk.process.DataPortalProcess` + + Raises: + DataPortalInputError: if `process` is not a string, or if no process + matches it by either ID or name. """ # If the process object is already a DataPortalProcess object diff --git a/cirro/sdk/login.py b/cirro/sdk/login.py index 1d86320..bf06be5 100644 --- a/cirro/sdk/login.py +++ b/cirro/sdk/login.py @@ -9,7 +9,13 @@ class DataPortalLogin: Start the login process, obtaining the authorization message from Cirro needed to confirm the user identity. - Useful when you need to authenticate a user in a non-blocking way. + Use this when a person is available to complete the login but you need to + control when your code blocks -- for example to render the authorization + message in a web page or notebook before waiting. Constructing the object + does not block; only `await_completion` does. + + For automation with no person in the loop, use OAuth client credentials + instead -- see `cirro.sdk.portal.DataPortal`. Usage: @@ -29,6 +35,16 @@ class DataPortalLogin: auth_info: DeviceCodeAuth def __init__(self, base_url: str = None, enable_cache=False): + """ + Begin a device-code login without waiting for it to complete. + + Args: + base_url (str): Base URL of the Cirro instance, e.g. `app.cirro.bio`. + If omitted, falls back to the `CIRRO_BASE_URL` environment variable, + then to the saved configuration. + enable_cache (bool): If True, save the resulting token to the system + keychain so later sessions can reuse it. + """ app_config = AppConfig(base_url=base_url) self.base_url = base_url @@ -52,7 +68,15 @@ def auth_message_markdown(self) -> str: return self.auth_info.auth_message_markdown def await_completion(self) -> DataPortal: - """Complete the login process and return an authenticated client""" + """ + Block until the user completes the login in their browser. + + Returns: + `cirro.sdk.portal.DataPortal`: an authenticated portal object. + + Raises: + RuntimeError: if the device code expires before the login completes. + """ # Block until the user completes the login flow self.auth_info.await_completion() diff --git a/cirro/sdk/portal.py b/cirro/sdk/portal.py index 9f6b2ef..9602377 100644 --- a/cirro/sdk/portal.py +++ b/cirro/sdk/portal.py @@ -13,18 +13,62 @@ class DataPortal: """ - Helper functions for exploring the Projects, Datasets, Samples, and Files - available in the Data Portal. + The entry point for the SDK: explore the Projects, Datasets, Samples, and + Files available in Cirro, and launch analyses on them. + + Projects and datasets can be looked up by either name or ID. + + ```python + from cirro import DataPortal + + portal = DataPortal(base_url="app.cirro.bio") + dataset = portal.get_dataset(project="Name of Project", dataset="Name of Dataset") + ``` """ def __init__(self, base_url: str = None, client: CirroApi = None): """ Set up the DataPortal object, establishing an authenticated connection. + **This may block on an interactive login.** With no `client` and no + saved configuration, the constructor starts a device-code login: it + prints a URL and waits until someone completes the login in a browser. + In a script or an automated session there is nobody to click the link, + so it will hang until the device code expires. + + To authenticate without prompting, build a `cirro.cirro_client.CirroApi` + with OAuth client credentials and pass it as `client`: + + ```python + import os + from cirro import CirroApi, DataPortal + from cirro.auth.client_creds import ClientCredentialsAuth + from cirro.config import AppConfig + + config = AppConfig(base_url="app.cirro.bio") + auth = ClientCredentialsAuth( + os.environ["CIRRO_CLIENT_ID"], + os.environ["CIRRO_CLIENT_SECRET"], + auth_endpoint=config.auth_endpoint + ) + portal = DataPortal(client=CirroApi(auth_info=auth)) + ``` + + To drive the browser login yourself without blocking in the + constructor, use `cirro.sdk.login.DataPortalLogin` instead. + Args: - base_url (str): Optional base URL of the Cirro instance - (if not provided, it uses the `CIRRO_BASE_URL` environment variable, or the config file) - client (`cirro.cirro_client.CirroApi`): Optional pre-configured client + base_url (str): Base URL of the Cirro instance, e.g. `app.cirro.bio`. + If omitted, falls back to the `CIRRO_BASE_URL` environment variable, + then to the `base_url` saved in `~/.cirro/config.ini` by + `cirro configure`. Raises `RuntimeError` if none of these is set. + client (`cirro.cirro_client.CirroApi`): Pre-configured client. Supply + this to control how authentication happens; when given, `base_url` + is ignored. + + Raises: + RuntimeError: if no base URL can be determined, or the instance + cannot be reached. Example: ```python @@ -77,7 +121,13 @@ def from_client_credentials(cls, client_id: str, client_secret: str, base_url: s return cls(client=client) def list_projects(self) -> DataPortalProjects: - """List all the projects available in the Data Portal.""" + """ + List all the projects available in the Data Portal. + + Returns: + `cirro.sdk.project.DataPortalProjects`, a list which also offers + `get_by_name`, `get_by_id`, and `filter_by_pattern`. + """ return DataPortalProjects( [ @@ -87,12 +137,35 @@ def list_projects(self) -> DataPortalProjects: ) def get_project_by_name(self, name: str = None) -> DataPortalProject: - """Return the project with the specified name.""" + """ + Return the project with the specified name. + + Args: + name (str): Name of the project. + + Returns: + `cirro.sdk.project.DataPortalProject` + + Raises: + DataPortalAssetNotFound: if no project has this name. + DataPortalInputError: if more than one project has this name. + """ return self.list_projects().get_by_name(name) def get_project_by_id(self, _id: str = None) -> DataPortalProject: - """Return the project with the specified id.""" + """ + Return the project with the specified id. + + Args: + _id (str): ID of the project. + + Returns: + `cirro.sdk.project.DataPortalProject` + + Raises: + DataPortalAssetNotFound: if no project has this ID. + """ return self.list_projects().get_by_id(_id) @@ -100,11 +173,17 @@ def get_project(self, project: str = None) -> DataPortalProject: """ Return a project identified by ID or name. + Tries to match by ID first, then falls back to matching by name. + Args: project (str): ID or name of project Returns: - `from cirro.sdk.project import DataPortalProject` + `cirro.sdk.project.DataPortalProject` + + Raises: + DataPortalAssetNotFound: if no project matches by either ID or name. + DataPortalInputError: if more than one project has this name. """ try: return self.get_project_by_id(project) @@ -123,7 +202,7 @@ def get_dataset(self, project: str = None, dataset: str = None) -> DataPortalDat `cirro.sdk.dataset.DataPortalDataset` ```python - from cirro import DataPortal() + from cirro import DataPortal portal = DataPortal() dataset = portal.get_dataset( project="id-or-name-of-project", @@ -275,6 +354,9 @@ def list_processes(self, ingest=False) -> DataPortalProcesses: Args: ingest (bool): If True, only list those processes which can be used to ingest datasets directly + + Returns: + `cirro.sdk.process.DataPortalProcesses` """ return DataPortalProcesses( @@ -291,23 +373,48 @@ def get_process_by_name(self, name: str, ingest=False) -> DataPortalProcess: Args: name (str): Name of process + ingest (bool): If True, search only the processes which can be used + to ingest datasets directly. A data type used for uploading will + not be found unless this is set. + + Returns: + `cirro.sdk.process.DataPortalProcess` + + Raises: + DataPortalAssetNotFound: if no process has this name. """ return self.list_processes(ingest=ingest).get_by_name(name) def get_process_by_id(self, id: str, ingest=False) -> DataPortalProcess: """ - Return the process with the specified id + Return the process with the specified id. Args: id (str): ID of process + ingest (bool): If True, search only the processes which can be used + to ingest datasets directly. + + Returns: + `cirro.sdk.process.DataPortalProcess` + + Raises: + DataPortalAssetNotFound: if no process has this ID. """ return self.list_processes(ingest=ingest).get_by_id(id) def list_reference_types(self) -> DataPortalReferenceTypes: """ - Return the list of all available reference types + Return the list of all available reference types. + + These are the categories that reference data is organized into, such as + `genome_fasta`. Pass a type name to + `cirro.sdk.project.DataPortalProject.list_references` to see the + references of that type held by a project. + + Returns: + `cirro.sdk.reference_type.DataPortalReferenceTypes` """ return DataPortalReferenceTypes( @@ -319,4 +426,11 @@ def list_reference_types(self) -> DataPortalReferenceTypes: @property def developer_helper(self) -> DeveloperHelper: + """ + Helpers for developing Cirro pipelines and data types, rather than for + analysing data. + + Returns: + `cirro.sdk.developer.DeveloperHelper` + """ return DeveloperHelper(self._client) diff --git a/cirro/sdk/process.py b/cirro/sdk/process.py index 83bcf8a..c99d464 100644 --- a/cirro/sdk/process.py +++ b/cirro/sdk/process.py @@ -17,7 +17,7 @@ def __init__(self, process: Union[Process, ProcessDetail], client: CirroApi): Instantiate with helper method ```python - from cirro import DataPortal() + from cirro import DataPortal portal = DataPortal() process = portal.get_process_by_name("Process Name") ``` @@ -94,6 +94,14 @@ def __str__(self): def get_parameter_spec(self) -> ParameterSpecification: """ Gets a specification used to describe the parameters used in the process. + + This is the authoritative source for the keys accepted by the `params` + argument of `run_analysis`, along with their types and default values. + Call `print()` on the result for a readable listing, or + `validate_params()` to check a `params` dict before submitting it. + + Returns: + `cirro.models.form_specification.ParameterSpecification` """ return self._client.processes.get_parameter_spec(self.id) @@ -112,6 +120,12 @@ def run_analysis( """ Runs this process on one or more input datasets, returns the ID of the newly created dataset. + The analysis runs asynchronously; this returns as soon as the job is + submitted. Call `get_parameter_spec` to discover which `params` this + process accepts, and see + `cirro.sdk.dataset.DataPortalDataset.run_analysis` for how to follow the + analysis to completion. + Args: name (str): Name of newly created dataset project_id (str): ID of the project to run the analysis in diff --git a/cirro/sdk/project.py b/cirro/sdk/project.py index 89f58c9..ea5e610 100644 --- a/cirro/sdk/project.py +++ b/cirro/sdk/project.py @@ -27,7 +27,7 @@ def __init__(self, proj: Project, client: CirroApi): Instantiate with helper method ```python - from cirro import DataPortal() + from cirro import DataPortal portal = DataPortal() project = portal.get_project_by_name("Project Name") ``` @@ -78,7 +78,20 @@ def _get_datasets(self) -> List[Dataset]: client=self._client) def list_datasets(self, force_refresh=False) -> DataPortalDatasets: - """List all the datasets available in the project.""" + """ + List all the datasets available in the project. + + The listing is fetched once and cached on this object, since a project + may hold many thousands of datasets. + + Args: + force_refresh (bool): Discard the cached listing and fetch it again. + Needed to see datasets created since this object was built. + + Returns: + `cirro.sdk.dataset.DataPortalDatasets`, a list which also offers + `get_by_name`, `get_by_id`, and `filter_by_pattern`. + """ if force_refresh: self._get_datasets.cache_clear() @@ -90,10 +103,24 @@ def list_datasets(self, force_refresh=False) -> DataPortalDatasets: ) def get_dataset(self, name_or_id: str, force_refresh=False) -> DataPortalDataset: - """Return the dataset matching the given ID or name. + """ + Return the dataset matching the given ID or name. Tries to match by ID first, then by name. - Raises an error if the name matches multiple datasets. + + Args: + name_or_id (str): ID or name of the dataset. + force_refresh (bool): Discard the cached dataset listing before + matching by name. Needed to find a dataset created since this + object was built. + + Returns: + `cirro.sdk.dataset.DataPortalDataset` + + Raises: + DataPortalAssetNotFound: if nothing matches by either ID or name. + DataPortalInputError: if more than one dataset has this name, in + which case use `get_dataset_by_id`. """ if force_refresh: self._get_datasets.cache_clear() @@ -115,7 +142,22 @@ def get_dataset(self, name_or_id: str, force_refresh=False) -> DataPortalDataset return self.get_dataset_by_id(matches[0].id) def get_dataset_by_name(self, name: str, force_refresh=False) -> DataPortalDataset: - """Return the dataset with the specified name.""" + """ + Return the dataset with the specified name. + + If several datasets share the name, the first one in the listing is + returned. Use `get_dataset` if you would rather that were an error. + + Args: + name (str): Name of the dataset. + force_refresh (bool): Discard the cached dataset listing first. + + Returns: + `cirro.sdk.dataset.DataPortalDataset` + + Raises: + DataPortalAssetNotFound: if no dataset in the project has this name. + """ if force_refresh: self._get_datasets.cache_clear() @@ -125,7 +167,20 @@ def get_dataset_by_name(self, name: str, force_refresh=False) -> DataPortalDatas return self.get_dataset_by_id(dataset.id) def get_dataset_by_id(self, _id: str = None) -> DataPortalDataset: - """Return the dataset with the specified id.""" + """ + Return the dataset with the specified id. + + Fetches the dataset directly, bypassing the cached project listing. + + Args: + _id (str): ID of the dataset. + + Returns: + `cirro.sdk.dataset.DataPortalDataset` + + Raises: + DataPortalAssetNotFound: if the project has no dataset with this ID. + """ dataset = self._client.datasets.get(project_id=self.id, dataset_id=_id) if dataset is None: @@ -135,7 +190,18 @@ def get_dataset_by_id(self, _id: str = None) -> DataPortalDataset: def list_references(self, reference_type: str = None) -> DataPortalReferences: """ List the references available in a project. - Optionally filter to references of a particular type (identified by name) + + Args: + reference_type (str): Optionally restrict the results to references + of one type, identified by name. Call + `cirro.sdk.portal.DataPortal.list_reference_types` for the + available type names. + + Returns: + `cirro.sdk.reference.DataPortalReferences` + + Raises: + DataPortalAssetNotFound: if `reference_type` matches no known type. """ # Get the complete list of references which are available @@ -164,7 +230,20 @@ def list_references(self, reference_type: str = None) -> DataPortalReferences: ) def get_reference_by_name(self, name: str = None, ref_type: str = None) -> DataPortalReference: - """Return the reference of a particular type with the specified name.""" + """ + Return the reference of a particular type with the specified name. + + Args: + name (str): Name of the reference. + ref_type (str): Optionally restrict the search to one reference type. + + Returns: + `cirro.sdk.reference.DataPortalReference` + + Raises: + DataPortalInputError: if `name` is not provided. + DataPortalAssetNotFound: if no matching reference exists. + """ if name is None: raise DataPortalInputError("Must specify the reference name") @@ -185,6 +264,13 @@ def upload_dataset( If the files parameter is not provided, it will upload all files in the upload folder + The `process` here is a data type rather than a pipeline: it declares + what kind of data is being uploaded and which files the dataset must + contain. List the valid options with + `portal.list_processes(ingest=True)`. Cirro validates the file names + against the data type's requirements before any upload starts, so a + mismatch fails fast. + Args: name (str): Name of newly created dataset description (str): Description of newly created dataset @@ -192,6 +278,14 @@ def upload_dataset( upload_folder (str): Folder containing files to upload files (List[str]): Optional subset of files to upload from the folder tags (List[str]): Optional list of tags to apply to the dataset + + Returns: + `cirro.sdk.dataset.DataPortalDataset`: the newly created dataset. + + Raises: + DataPortalInputError: if `name`, `process`, or `upload_folder` is + missing, or if the files do not meet the data type's requirements. + RuntimeWarning: if there are no files to upload. """ if name is None: @@ -256,7 +350,12 @@ def samples(self, max_items: int = 10000) -> List[Sample]: Retrieves a list of samples associated with a project along with their metadata Args: - max_items (int): Maximum number of records to get (default 10,000) + max_items (int): Maximum number of records to get (default 10,000). + A project with more samples than this is truncated silently. + + Returns: + `List[cirro_api_client.v1.models.Sample]` -- each carrying the + sample's `id`, `name`, and its `metadata` dict. """ return self._client.metadata.get_project_samples(self.id, max_items) diff --git a/cirro/sdk/reference.py b/cirro/sdk/reference.py index cfb2a54..1c9a43b 100644 --- a/cirro/sdk/reference.py +++ b/cirro/sdk/reference.py @@ -16,7 +16,7 @@ def __init__(self, ref: Reference, project_id: str, client: CirroApi): """ Instantiate by listing the references which have been added to a particular project ```python - from cirro import DataPortal() + from cirro import DataPortal portal = DataPortal() project = portal.get_project_by_name("Project Name") references = project.list_references() @@ -44,6 +44,12 @@ def type(self) -> str: @property def absolute_path(self): + """ + S3 URI of the reference's first file, or `None` if it has no files. + + A reference may contain several files (e.g. a FASTA alongside its index). + Use `files` to reach the others. + """ if len(self._files) == 0: return None return self._files[0].absolute_path @@ -57,4 +63,10 @@ class DataPortalReferences(DataPortalAssets[DataPortalReference]): asset_name = "reference" def get_by_id(self, _id: str) -> DataPortalReference: + """ + Not supported for references, which are identified by name only. + + Raises: + NotImplementedError: always. Use `get_by_name` instead. + """ raise NotImplementedError("Filtering by ID is not supported, use get_by_name") diff --git a/cirro/sdk/reference_type.py b/cirro/sdk/reference_type.py index f12af79..81dbc02 100644 --- a/cirro/sdk/reference_type.py +++ b/cirro/sdk/reference_type.py @@ -9,6 +9,15 @@ class DataPortalReferenceType(DataPortalAsset): """ def __init__(self, ref_type: ReferenceType): + """ + Obtained from `cirro.sdk.portal.DataPortal.list_reference_types`. + + ```python + from cirro import DataPortal + portal = DataPortal() + reference_types = portal.list_reference_types() + ``` + """ self._data = ref_type @property @@ -23,10 +32,12 @@ def description(self): @property def directory(self): + """Folder that references of this type are stored under.""" return self._data.directory @property def validation(self): + """Rules describing the files a reference of this type must contain.""" return self._data.validation def __str__(self): @@ -41,4 +52,10 @@ class DataPortalReferenceTypes(DataPortalAssets[DataPortalReferenceType]): asset_name = "reference type" def get_by_id(self, _id: str) -> DataPortalReferenceType: + """ + Not supported for reference types, which are identified by name only. + + Raises: + NotImplementedError: always. Use `get_by_name` instead. + """ raise NotImplementedError("Filtering by ID is not supported, use get_by_name") diff --git a/pyproject.toml b/pyproject.toml index 845d74d..658a114 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "cirro" -version = "1.13.1" +version = "1.13.0" description = "CLI tool and SDK for interacting with the Cirro platform" authors = ["Cirro Bio "] license = "MIT"