diff --git a/dataconnect/client.py b/dataconnect/client.py index 577c6b4..6acbf22 100644 --- a/dataconnect/client.py +++ b/dataconnect/client.py @@ -15,6 +15,7 @@ from dataconnect.models import ( Dataset, DatasetVersion, + DatetimeFormatsResult, DryPublishResult, PaginatedResponse, PublishResult, @@ -169,6 +170,32 @@ def publish( datetime_formats=datetime_formats, ) + def get_datetime_formats( + self, + project_token: str, + format_type: str = "all", + ) -> DatetimeFormatsResult: + """Return the supported datetime formats filtered by ``format_type``. + + Delegates directly to :meth:`DataConnectService.get_datetime_formats`. + + Args: + project_token: Base64-encoded project token identifying the target + study, study environment, and project. + format_type: Server-side filter to apply. One of ``"all"`` + (default), ``"date"``, or ``"datetime"``. + + Returns: + A :class:`DatetimeFormatsResult` exposing the classified list via + :meth:`~DatetimeFormatsResult.all` and the type-filtered views via + :meth:`~DatetimeFormatsResult.dates` and + :meth:`~DatetimeFormatsResult.datetimes`. + """ + return self._service.get_datetime_formats( + project_token=project_token, + format_type=format_type, + ) + # Lifecycle def close(self) -> None: diff --git a/dataconnect/models.py b/dataconnect/models.py index e6bbe60..4e38bd0 100644 --- a/dataconnect/models.py +++ b/dataconnect/models.py @@ -97,3 +97,45 @@ class PublishResult: duplicate_record_count: int | None = None invalid_record_count: int | None = None invalid_records: pd.DataFrame | None = None + + +@dataclass(frozen=True) +class DatetimeFormat: + """A single supported datetime format string with its classification.""" + + format: str + """The format string as returned by the server (e.g. ``"yyyy-MM-dd"``).""" + + type: str + """Either ``"date"`` (date-only) or ``"datetime"`` (date with time component).""" + + +@dataclass +class DatetimeFormatsResult: + """Result of a :meth:`DataConnectClient.get_datetime_formats` call. + + Holds the full list of supported datetime formats returned by the server + and provides convenience accessors for the common views. + + Examples: + >>> result = client.get_datetime_formats(project_token="...") + >>> for fmt in result.all(): + ... print(fmt.format, fmt.type) + >>> only_dates = result.dates() # list[str] + >>> only_datetimes = result.datetimes() # list[str] + """ + + formats: list[DatetimeFormat] = field(default_factory=list) + """The full list of supported formats, in the order returned by the server.""" + + def all(self) -> list[DatetimeFormat]: + """Return every supported format with its type classification.""" + return list(self.formats) + + def dates(self) -> list[str]: + """Return only the date-style format strings (no time component).""" + return [f.format for f in self.formats if f.type == "date"] + + def datetimes(self) -> list[str]: + """Return only the datetime-style format strings (with a time component).""" + return [f.format for f in self.formats if f.type == "datetime"] diff --git a/dataconnect/service/base.py b/dataconnect/service/base.py index 261eb48..4ac7011 100644 --- a/dataconnect/service/base.py +++ b/dataconnect/service/base.py @@ -10,6 +10,7 @@ from dataconnect.models import ( Dataset, DatasetVersion, + DatetimeFormatsResult, DryPublishResult, PaginatedResponse, PublishResult, @@ -68,5 +69,14 @@ def publish( """Publish a dataset to the server and return the result.""" ... + @abstractmethod + def get_datetime_formats( + self, + project_token: str, + format_type: str = "all", + ) -> DatetimeFormatsResult: + """Return the supported datetime formats filtered by ``format_type``.""" + ... + @abstractmethod def close(self) -> None: ... diff --git a/dataconnect/service/default.py b/dataconnect/service/default.py index 38b1fda..fc5ffa3 100644 --- a/dataconnect/service/default.py +++ b/dataconnect/service/default.py @@ -3,13 +3,17 @@ from __future__ import annotations import json +from datetime import UTC, datetime from uuid import UUID import pandas as pd +from dataconnect.exceptions import ValidationError from dataconnect.models import ( Dataset, DatasetVersion, + DatetimeFormat, + DatetimeFormatsResult, DryPublishResult, PaginatedResponse, Pagination, @@ -28,13 +32,22 @@ ) from dataconnect.transport.base import Transport from dataconnect.transport.errors import TransportError -from dataconnect.transport.models import DatasetTicket, PublishRequest, ResourceQuery +from dataconnect.transport.models import ( + DatasetTicket, + DatetimeFormatsRequest, + PublishRequest, + ResourceQuery, +) # Server action identifiers _ACTION_LIST_STUDIES = "studies.list" _ACTION_LIST_DATASETS = "datasets.list" _ACTION_LIST_DATASET_VERSIONS = "dataset_versions.list" +# Accepted values for the ``format_type`` filter passed to +# :meth:`DefaultDataConnectService.get_datetime_formats`. +_VALID_DATETIME_FORMAT_TYPES: frozenset[str] = frozenset({"all", "date", "datetime"}) + class DefaultDataConnectService(DataConnectService): """Concrete service injected with an abstract ``Transport``.""" @@ -276,6 +289,69 @@ def publish( except TransportError as ex: raise translate_error(ex) from ex + def get_datetime_formats( + self, + project_token: str, + format_type: str = "all", + ) -> DatetimeFormatsResult: + """Return the supported datetime formats filtered by ``format_type``. + + The ``format_type`` argument is normalised (stripped + lower-cased) and + validated client-side against the accepted set ``{"all", "date", + "datetime"}``. Invalid values raise :class:`ValidationError` *before* + any transport call is made. + + The server filters the list according to ``format_type``; the service + then classifies each returned format as ``"date"`` or ``"datetime"`` + (based on whether the format contains a time component ``"HH:mm"``) and + wraps everything in a :class:`DatetimeFormatsResult` that exposes the + :meth:`~DatetimeFormatsResult.all`, + :meth:`~DatetimeFormatsResult.dates` and + :meth:`~DatetimeFormatsResult.datetimes` helpers. + + Args: + project_token: Base64-encoded project token identifying the target + study, study environment, and project. Required by the server + for authorization. + format_type: Filter to apply server-side. One of ``"all"`` + (default), ``"date"``, or ``"datetime"``. + + Returns: + A :class:`DatetimeFormatsResult` containing the classified formats. + + Raises: + ValidationError: When ``format_type`` is not one of the accepted + values. + DataConnectError: Any :class:`TransportError` from the transport + layer is translated by :func:`translate_error` into the public + API's :class:`DataConnectError` hierarchy. + """ + normalized = (format_type or "all").strip().lower() + + if normalized not in _VALID_DATETIME_FORMAT_TYPES: + raise ValidationError( + error_code="VAL_001", + message=(f"Invalid format_type: {format_type!r}. Accepted values: all, date, datetime."), + timestamp=datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + ) + + request = DatetimeFormatsRequest(project_token=project_token, format_type=normalized) + + try: + raw_formats = self._transport.get_datetime_formats(request) + except TransportError as ex: + raise translate_error(ex) from ex + + formats = [ + DatetimeFormat( + format=fmt, + type="datetime" if "HH:mm" in fmt else "date", + ) + for fmt in raw_formats + ] + + return DatetimeFormatsResult(formats=formats) + def close(self) -> None: """Close the underlying transport connection.""" diff --git a/dataconnect/transport/arrow_flight/transport.py b/dataconnect/transport/arrow_flight/transport.py index 48d8232..9e9f224 100644 --- a/dataconnect/transport/arrow_flight/transport.py +++ b/dataconnect/transport/arrow_flight/transport.py @@ -26,6 +26,7 @@ DataRef, DatasetTicket, DataTable, + DatetimeFormatsRequest, DryPublishResponse, PublishRequest, PublishResponse, @@ -382,5 +383,39 @@ def publish_dataset(self, publish_request: PublishRequest) -> PublishResponse: except Exception as ex: raise parse_dataconnect_error(ex) from ex + def get_datetime_formats(self, request: DatetimeFormatsRequest) -> list[str]: + """Invoke the Arrow Flight ``get_datetime_formats`` action and return the format list. + + The server expects a JSON body of the form + ``{"project_token": "...", "type": "all|date|datetime"}`` and responds + with a single :class:`flight.Result` whose body is a JSON-encoded list + of format strings already filtered server-side. + + Args: + request: A :class:`DatetimeFormatsRequest` carrying the project + token and the (already-validated) ``format_type`` filter. + + Returns: + A list of supported format strings. Returns an empty list if the + server produced no results. + + Raises: + TransportError: Any Arrow Flight or gRPC error is translated by + :func:`parse_dataconnect_error` before propagating. + """ + payload = {"project_token": request.project_token, "type": request.format_type} + action = flight.Action("get_datetime_formats", json.dumps(payload).encode("utf-8")) + + try: + results = list(self._client.do_action(action, self._options())) + + if not results: + return [] + + return json.loads(results[0].body.to_pybytes().decode("utf-8")) + + except Exception as ex: + raise parse_dataconnect_error(ex) from ex + def close(self) -> None: self._client.close() diff --git a/dataconnect/transport/base.py b/dataconnect/transport/base.py index 7898d3e..91ce61f 100644 --- a/dataconnect/transport/base.py +++ b/dataconnect/transport/base.py @@ -12,6 +12,7 @@ from dataconnect.transport.models import ( DatasetTicket, DataTable, + DatetimeFormatsRequest, DryPublishResponse, PublishRequest, PublishResponse, @@ -55,6 +56,15 @@ def publish_dataset(self, publish_request: PublishRequest) -> PublishResponse: ``PublishResponse`` containing the complete result set. """ + @abstractmethod + def get_datetime_formats(self, request: DatetimeFormatsRequest) -> list[str]: + """Return the supported datetime format strings for the project. + + The transport does not interpret ``format_type`` — that is the service + layer's responsibility. The server applies the filter and returns the + already-filtered list of format strings. + """ + @abstractmethod def close(self) -> None: """Close the transport connection.""" diff --git a/dataconnect/transport/models.py b/dataconnect/transport/models.py index a6f87d9..7634171 100644 --- a/dataconnect/transport/models.py +++ b/dataconnect/transport/models.py @@ -65,6 +65,21 @@ class DataTable: ipc_bytes: bytes +@dataclass(frozen=True) +class DatetimeFormatsRequest: + """An outbound request to fetch the supported datetime formats. + + Attributes: + project_token: Base64-encoded project token used by the server to + authorize the request. + format_type: Filter applied server-side. One of ``"all"``, ``"date"``, + or ``"datetime"``. + """ + + project_token: str + format_type: str = "all" + + @dataclass(frozen=True) class PublishRequest: """A publish request containing the input configuration and the dataset to be published. diff --git a/guides/dataconnect_usage.ipynb b/guides/dataconnect_usage.ipynb index 646cfe6..3af2607 100644 --- a/guides/dataconnect_usage.ipynb +++ b/guides/dataconnect_usage.ipynb @@ -271,6 +271,47 @@ "You can retrieve this by navigating to `iMedidata` > `Data Connect` > `Transformations` and by clicking on the kebab menu on a project row and selecting `View/Modify`." ] }, + { + "cell_type": "markdown", + "id": "24dc262e", + "metadata": {}, + "source": [ + "### Discover supported datetime formats\n", + "\n", + "Use `get_datetime_formats` to list the format patterns the server accepts for the `datetime_formats` argument of `dry_publish` / `publish`. Pass `format_type=\"date\"` or `format_type=\"datetime\"` to restrict the result; the default `\"all\"` returns both kinds." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "addc2370", + "metadata": {}, + "outputs": [], + "source": [ + "from dataconnect import DataConnectClient\n", + "\n", + "user_token = \"usertoken\" # From iMedidata > Data Connect > Developer Center\n", + "project_token = \"project_token\" # From iMedidata > Data Connect > Transformations\n", + "\n", + "try:\n", + " with DataConnectClient.connect(token=user_token) as dataconnect_client:\n", + " # Default: return both date and datetime formats\n", + " formats_result = dataconnect_client.get_datetime_formats(\n", + " project_token=project_token,\n", + " format_type=\"all\",\n", + " )\n", + "\n", + " print(\"All supported formats:\")\n", + " for fmt in formats_result.all():\n", + " print(f\" {fmt.format} [{fmt.type}]\")\n", + "\n", + " print(\"\\nDate-only formats: \", formats_result.dates())\n", + " print(\"Datetime formats: \", formats_result.datetimes())\n", + "\n", + "except Exception as e:\n", + " print(e)" + ] + }, { "cell_type": "markdown", "id": "69e733cb", diff --git a/readme/README-v1.1.0.md b/readme/README-v1.1.0.md new file mode 100644 index 0000000..ddd52c3 --- /dev/null +++ b/readme/README-v1.1.0.md @@ -0,0 +1,390 @@ +# DataConnect Python Library v1.0.0 + +The DataConnect Python library provides a Python client for connecting to Medidata DataConnect and retrieving relevant data programmatically. +To use this library, you must have a valid iMedidata account and access to required building blocks in the Medidata Platform. For details, see the Medidata [Knowledge Hub](https://learn.medidata.com/en-US/bundle/data-connect/page/developer_center.html). + +## Table of Contents + +- [DataConnect Python Library v1.0.0](#dataconnect-python-library-v100) + - [Table of Contents](#table-of-contents) + - [Setup and Usage](#setup-and-usage) + - [Authentication and Connectivity](#authentication-and-connectivity) + - [Functions](#functions) + - [connect()](#connect) + - [Description](#description) + - [Usage](#usage) + - [Arguments](#arguments) + - [Output](#output) + - [get\_studies()](#get_studies) + - [Description](#description-1) + - [Usage](#usage-1) + - [Arguments](#arguments-1) + - [Output](#output-1) + - [get\_datasets()](#get_datasets) + - [Description](#description-2) + - [Usage](#usage-2) + - [Arguments](#arguments-2) + - [Output](#output-2) + - [get\_dataset\_versions()](#get_dataset_versions) + - [Description](#description-3) + - [Usage](#usage-3) + - [Arguments](#arguments-3) + - [Output](#output-3) + - [fetch\_data()](#fetch_data) + - [Description](#description-4) + - [Usage](#usage-4) + - [Arguments](#arguments-4) + - [Output](#output-4) + - [dry\_publish()](#dry_publish) + - [Description](#description-5) + - [Usage](#usage-5) + - [Arguments](#arguments-5) + - [Output](#output-5) + - [Data Validations](#data-validations) + - [publish()](#publish) + - [Description](#description-6) + - [Usage](#usage-6) + - [Arguments](#arguments-6) + - [Output](#output-6) + - [Data Validations](#data-validations-1) + - [Data Validation Failures](#data-validation-failures) + - [get\_datetime\_formats()](#get_datetime_formats) + - [Description](#description-7) + - [Usage](#usage-7) + - [Arguments](#arguments-7) + - [Output](#output-7) + - [close()](#close) + - [Description](#description-8) + - [Usage](#usage-8) + - [Arguments](#arguments-8) + - [Output](#output-8) + - [Errors](#errors) +- [Reporting known issues](#reporting-known-issues) +- [Backend](#backend) +- [Licensing](#licensing) + +## Setup and Usage + +* For instructions on how to install and use this library, follow the [Usage Notebook](../guides/dataconnect_usage.ipynb). +* For an example of how to transform data, go [here](../guides/transform_example.md). + +### Authentication and Connectivity + +* **Retrieving data:** You must have a user token to establish a connection between your Python environment and Medidata Data Connect. You can generate this token through Data Connect’s Developer Center. For details, visit the [knowledge hub](https://learn.medidata.com/en-US/bundle/data-connect/page/developer_center.html). Medidata recommends that you save the token in a local file and input it into the below initiation function. + +* **Publish data:** You must have a project token to publish a dataset from your Python environment to Medidata Data Connect. You can generate this token through Data Connect > Transformations, by creating a Custom Code project. For details, visit the [knowledge hub](https://learn.medidata.com/en-US/bundle/data-connect/page/generate_custom_code_projects.html). + +## Functions + +The main public entry point is `DataconnectClient`. + +### connect() + +#### Description +Creates a connected client. + +#### Usage +`connect(token="")` + +#### Arguments +| Argument | Type | Description | +|---|---|---| +| host | str | Server host. Default host="enodia-gateway.platform.imedidata.com" | +| port | int | Server port. Default port="443" | +| use_tls | bool | Denotes whether to use TLS. Default use_tls = True | +| token | str | Authentication token, this is the user authentication token generated from the Developer Center in Medidata Data Connect | + +#### Output +DataconnectClient object. This enables you to interact with Medidata Data Connect data in Python environment. + +--- + +### get_studies() + +#### Description +Retrieves a list of studies where the user has permission to manage custom code projects. Use the optional study name search parameter to filter results. + +#### Usage +`get_studies(search_study_name=None)` + +#### Arguments +| Argument | Type | Description | +|---|---|---| +| search_study_name | str or None | Optional. The approximate name of the study | + +#### Output +Returns a list containing `total_records` (total studies available) and a `studies` array. Each study includes `name`, `uuid`, and an environments array. Each environment includes `name` and `uuid`. + +--- + +### get_datasets() + +#### Description +Retrieves datasets for a specific study environment and returns paginated results. + +#### Usage +`get_datasets(study_environment_uuid, search_dataset_name="", page=1, page_size=50)` + +#### Arguments +| Argument | Type | Description | +|---|---|---| +| study_environment_uuid | UUID | Unique iMedidata study environment identifier. You can find this in iMedidata’s Developer Info details | +| search_dataset_name | str | Optional. The approximate name of the dataset | +| page | int | Optional. Page number for paginated results. Default: 1 | +| page_size | int | Optional. Number of results per page. Default: 50 | + +#### Output +Returns a list containing `total_records` (total datasets available across all pages), `pagination` and `datasets` array. + +--- + +### get_dataset_versions() + +#### Description +Retrieves all available versions for a dataset. + +#### Usage +`get_dataset_versions(dataset_uuid)` + +#### Arguments +| Argument | Type | Description | +|---|---|---| +| dataset_uuid | UUID | Unique iMedidata dataset identifier. This is available in the output of datasets() function | + +#### Output +Returns all available versions of the dataset. + +--- + +### fetch_data() + +#### Description +Fetches dataset rows into a pandas DataFrame. + +#### Usage +`fetch_data(dataset_uuid, first_n_rows=None)` + +#### Arguments +| Argument | Type | Description | +|---|---|---| +| dataset_uuid | UUID | Unique iMedidata dataset identifier. This is available in the output of datasets() and dataset_versions() functions | +| first_n_rows | int or None | Optional positive row limit | + +#### Output +Returns data from a specific dataset. + +--- + +### dry_publish() + +#### Description +Check if the publish results meet validation requirements. + +#### Usage + +```python +dry_publish(project_token, dataset_name, key_columns, source_datasets, data, datetime_formats = None) +``` + +#### Arguments + +| Argument | Description | +|:---------------------| :---------- | +| **project_token** | You can generate this from the Data Connect > Transformations > Custom Code project type. | +| **dataset_name** | Data Connect expects the dataset name to be unique within the study | +| **key_columns** | List of columns that form the composite key that identifies each unique record in the data to be validated. Key columns must not contain null/missing values (for example, `None`) in any row. | +| **source_datasets** | List of source dataset unique identifiers (UUIDs) to be used to create the data being validated | +| **data** | Data frame that needs to be validated | +| **datetime_formats** | Optional. The expected format for date or datetime fields in the data frame. This is used to validate that the date or datetime fields in the data frame are in the correct format before publishing to Data Connect. This should be `None` when none of the fields in the data frame are expected to be in date or datetime type.| + +#### Output + +Returns the result of publishing validations as a list containing clean, server-side data-quality metrics: +* **`valid_record_count`**: Number of clean records matching platform requirements (always ≥ 0). +* **`duplicate_record_count`**: Gross duplicate records identified across the payload composite keys. +* **`invalid_record_count`**: Number of records containing validation errors or missing required keys. +* **`invalid_records`**: A data frame containing the rows that failed validation. + +#### Data Validations + +| Validations | Description | +|:---------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Invalid Input** | Required argument is missing | +| **project_token** | 1. Project Token is valid and generated from the Data Connect > Transformations > Custom Code project type.
2. More than one dataset cannot be published into a project
3. Only the project owner can publish datasets into a project. | +| **dataset_name** | 1. Maximum length of 15 characters and must only contain alphanumeric characters and underscores
2. This is the new name of the resulting dataset created by the user | +| **key_columns** | 1. Key columns are valid column names from the data frame being published
2. Key columns must not contain null/missing values (for example, `None`) in any row
3. Maps directly to the server-side metrics payload: `valid_record_count`, `duplicate_record_count`, and `invalid_record_count` without double-penalizing overlapping row states. | +| **source_datasets** | 1. Source Dataset is a valid dataset UUID
2. Source Dataset is from the same study environment. | +| **data** | Invalid column name '{column.name}', it must only contain alphanumeric characters and underscores, with a maximum length of 20 characters. | +| **datetime_formats** | 1. Date or Date time format is not from the acceptable list of formats
2. Date/Datetime format cannot be provided for a field that is not parsed as a Date/DateTime field in data frame. | + + +### publish() + +#### Description + +Publish dataset to Data Connect. + + +#### Usage + +```python +publish(project_token, dataset_name, key_columns, source_datasets, data, datetime_formats = None) +``` + +#### Arguments + +| Argument | Description | +|:---------------------| :---------- | +| **project_token** | You can generate this from the Data Connect > Transformations > Custom Code project type | +| **dataset_name** | This is the new name of the resulting dataset being created by the user. Data Connect expects the dataset name to be unique within the study | +| **key_columns** | List of columns that form the composite key that identifies each unique record. Rows with null/missing values (for example, `None`) are flagged as invalid. Key fields are mandatory, they cannot be omitted.| +| **source_datasets** | List of source dataset UUIDs within the study environment where the dataset is published and used to create the data that is being published | +| **data** | Data frame which needs to be published | +| **datetime_formats** | Optional. The expected format for datetime fields in the data frame. This is used to validate that datetime fields in the data frame are in the correct format before publishing to Data Connect. This should be `None` when none of the fields in the data frame are expected to be in date or datetime type.| + + +#### Output + +Returns the status of publish as a list containing the final backend execution results: +* **`valid_record_count`**: Total structural records written successfully to the destination table. +* **`duplicate_record_count`**: Gross row duplication counters. +* **`invalid_record_count`**: Total failure rows excluded during the network stream. +* **`invalid_records`**: A data frame containing the rows that failed validation. + + +#### Data Validations + +| Validations | Description | +|:---------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Invalid Input** | Required argument is missing | +| **project_token** | 1. Project Token is valid and generated from the Data Connect > Transformations > Custom Code project type.
2. More than one dataset cannot be published into a project
3. Only the project owner can publish datasets into a project. | +| **dataset_name** | 1. Maximum length of 15 characters and must only contain alphanumeric characters and underscores
2. This is the new name of the resulting dataset created by the user | +| **key_columns** | 1. Key columns are valid column names from the data frame being published
2. Key columns must not contain null/missing values (for example, `None`) in any row
3. Maps directly to the server-side metrics payload: `valid_record_count`, `duplicate_record_count`, and `invalid_record_count` without double-penalizing overlapping row states. | +| **source_datasets** | 1. Source Dataset is a valid dataset UUID
2. Source Dataset is from the same study environment. | +| **data** | Invalid column name '{column.name}', it must only contain alphanumeric characters and underscores, with a maximum length of 20 characters. | +| **datetime_formats** | 1. Date or Date time format is not from the acceptable list of formats
2. Date/Datetime format cannot be provided for a field that is not parsed as a Date/DateTime field in data frame. | + +### Data Validation Failures +- When validation fails, the SDK returns the original data frame with an appended `error` column. +- Each invalid record appears once per error type (a row with multiple errors produces multiple result rows). +- Supported error names: `NULL_KEY` (null/empty value in key column), `INVALID_VALUE` (invalid value in key column). +- A summary is printed to the console for immediate visibility. +- The full invalid records table is accessible programmatically from the error object. + + + +### get_datetime_formats() + +#### Description +Returns the list of date/datetime format patterns accepted by Data Connect when publishing data. Use this to discover valid values for the `datetime_formats` argument of [`dry_publish()`](#dry_publish) and [`publish()`](#publish). + +#### Usage + +```python +get_datetime_formats(project_token, format_type="all") +``` + +#### Arguments + +| Argument | Type | Description | +|:------------------|:-----|:------------| +| **project_token** | str | You can generate this from the Data Connect > Transformations > Custom Code project type. | +| **format_type** | str | Optional. Filters the returned formats. One of `"all"` (default), `"date"`, or `"datetime"`. | + +#### Output + +Returns a `DatetimeFormatsResult` object exposing the following methods: + +| Method | Returns | Description | +|:----------------|:----------------------|:------------| +| `all()` | `list[DatetimeFormat]`| Every supported format, each tagged with its `type` (`"date"` or `"datetime"`). | +| `dates()` | `list[str]` | Format strings classified as date-only. | +| `datetimes()` | `list[str]` | Format strings classified as datetime. | + +Each `DatetimeFormat` item has two fields: + +| Field | Type | Description | +|:---------|:-----|:------------| +| `format` | str | The format pattern (for example, `"yyyy-MM-dd"`). | +| `type` | str | Either `"date"` or `"datetime"`. | + +Example: + +```python +with DataConnectClient.connect(token=user_token) as dc: + result = dc.get_datetime_formats(project_token=project_token) + for fmt in result.all(): + print(f"{fmt.format} [{fmt.type}]") + + date_only = result.dates() + datetime_only = result.datetimes() +``` + +When `format_type="date"` or `format_type="datetime"` is supplied, the server returns only formats of that kind; `all()` will contain just those entries and the matching accessor (`dates()` / `datetimes()`) will mirror them. + + +### close() + +#### Description +Closes the underlying transport connection. + +#### Usage +`close()` + +#### Arguments +None + +#### Output +None + +## Errors +The library raises exceptions for many reasons, such as invalid parameters, authentication errors, and validation failures. We have introduced error codes for each category of errors to be handled programmatically. + +| Error Code | Type | Scenario| +| :--- | :--- |:---| +| AUTHZ_001 | Authorization | Authorization service check failed | +| VAL_002 | Validation - Page Number | Page number is not a positive integer +| VAL_003 | Validation - Page Size | Page size is out of range [1, 100] +| VAL_004 | Validation - Study Parameter | Invalid study uuid +| VAL_005 | Validation - Study Environment Parameter | Missing or invalid study environment uuid +| VAL_006 | Validation - Dataset Parameter | Invalid dataset uuid +| VAL_007 | Validation - Configuration Error | Required input parameters are missing or invalid in configuration +| VAL_008 | Validation - Project Token | Invalid project token +| VAL_009 | Validation - Unsupported Data Type | Unsupported data types. +| VAL_010 | Validation - Unsupported Data Type | Unsupported datetime formats. +| VAL_011 | Validation - Pagination | Pagination is out of range +| VAL_012 | Validation - Concurrency | Project actively being published +| VAL_013 | Validation - Formatting Error | Data validation failed. One or more records contain formatting errors. +| RES_002 | Resource Exceptions - Study Environment | No authorized Study Environments found for the authenticated user +| RES_003 | Resource Exceptions - Invalid parameter | Incorrect UUID combination. +| RES_004 | Resource Exceptions - Invalid parameter | Incorrect UUID combination. +| RES_005 | Resource Exceptions - Study Group | Study Group not found for the Dataset's Study Environment. +| RES_006 | Resource Exceptions - Study | Study Group not found for the Dataset's Study Environment. +| RES_007 | Resource Exceptions - Client Division | Client Division not found for the Dataset's Study Environment. +| RES_008 | Resource Exceptions - Custom Code Project | Transformation Project is not found. +| INT_001 | Internal Application Exception | Something went wrong on our end. + +# Reporting known issues + +If you believe you have found an issue, please contact Medidata Support by submitting a ticket to Medidata Support. All issue reports should include a minimal reproducible example to ensure our team can diagnose the issue. + +Additionally, all known issues are available [here](https://learn.medidata.com/en-US/bundle/current-issues/page/current_known_issues_for_data_connect.html). + +# Backend + +This library uses the Arrow open source library and the Iceberg open table format to enable data interoperability across platforms. + +* [Apache arrow](https://arrow.apache.org/docs/r/): This library uses Arrow’s highly efficient format [pyarrow](https://arrow.apache.org/cookbook/py/flight.html) to transfer massive datasets over the network, allowing users to access & interact with remote datasets. + +* [Apache Iceberg](https://iceberg.apache.org/): This is the open table format underlying Medidata Data Connect's structured data management to support high-performance and reliable data analytics and storage. + +# Licensing + +BY DOWNLOADING THIS FILE (“DOWNLOAD”) YOU AGREE TO THE FOLLOWING TERMS: +MEDIDATA SOLUTIONS, INC. AND ITS AFFILIATES (COLLECTIVELY “MEDIDATA”) GRANT A FREE OF CHARGE, NON-EXCLUSIVE AND NON-TRANSFERABLE RIGHT TO USE THE DOWNLOAD. USE OF THIS DOWNLOAD IS PERMITTED FOR INTERNAL BUSINESS PURPOSES ONLY. + +THIS DOWNLOAD IS MADE AVAILABLE ON AN "AS IS" BASIS WITHOUT WARRANTY OF ANY KIND, WHETHER EXPRESS OR IMPLIED, ORAL OR WRITTEN, INCLUDING, WITHOUT LIMITATION, ANY IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON-INFRINGEMENT. + +MEDIDATA SHALL HAVE NO LIABILITY FOR DIRECT, INDIRECT, INCIDENTAL, CONSEQUENTIAL OR PUNITIVE DAMAGES, INCLUDING, WITHOUT LIMITATION, CLAIMS FOR LOST PROFITS, BUSINESS INTERRUPTION AND LOSS OF DATA THAT IN ANY WAY RELATE TO THIS DOWNLOAD, WHETHER OR NOT MEDIDATA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES AND NOTWITHSTANDING THE FAILURE OF THE ESSENTIAL PURPOSE OF ANY REMEDY. + +YOUR USE OF THIS DOWNLOAD SHALL BE AT YOUR SOLE RISK. NO SUPPORT OF ANY KIND OF THE DOWNLOAD IS PROVIDED BY MEDIDATA. diff --git a/tests/test_dry_publish.py b/tests/test_dry_publish.py index 4f2ec8f..3d5c0a1 100644 --- a/tests/test_dry_publish.py +++ b/tests/test_dry_publish.py @@ -18,7 +18,7 @@ import pytest from dataconnect.exceptions import ValidationError -from dataconnect.models import DryPublishResult +from dataconnect.models import DatetimeFormatsResult, DryPublishResult from dataconnect.service.default import DefaultDataConnectService from dataconnect.service.mappers import dry_publish_response_to_domain from dataconnect.transport.arrow_flight.transport import ( @@ -233,6 +233,9 @@ def dry_publish_dataset(self, publish_request: PublishRequest) -> DryPublishResp def publish_dataset(self, publish_request: PublishRequest) -> PublishResponse: raise NotImplementedError + def get_datetime_formats(self, request: DatetimeFormatsResult) -> list[str]: # type: ignore[override] + raise NotImplementedError + def close(self) -> None: pass diff --git a/tests/test_get_datetime_formats.py b/tests/test_get_datetime_formats.py new file mode 100644 index 0000000..eb0c318 --- /dev/null +++ b/tests/test_get_datetime_formats.py @@ -0,0 +1,422 @@ +"""Unit tests for the ``get_datetime_formats`` feature. + +Covers: +- ``DatetimeFormatsResult`` (domain model helpers) +- ``DefaultDataConnectService.get_datetime_formats`` (service layer) +- ``ArrowFlightTransport.get_datetime_formats`` (transport layer) +- ``DataConnectClient.get_datetime_formats`` (client façade) +""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from dataconnect.client import DataConnectClient +from dataconnect.exceptions import ValidationError +from dataconnect.models import DatetimeFormat, DatetimeFormatsResult +from dataconnect.service.default import DefaultDataConnectService +from dataconnect.transport.arrow_flight.transport import ArrowFlightTransport +from dataconnect.transport.base import Transport +from dataconnect.transport.errors import TransportValidationError +from dataconnect.transport.models import ( + DatasetTicket, + DataTable, + DatetimeFormatsRequest, + DryPublishResponse, + PublishRequest, + PublishResponse, + ResourceInfo, + ResourceQuery, +) + +# --------------------------------------------------------------------------- +# Shared fixtures / stubs +# --------------------------------------------------------------------------- + + +_SAMPLE_FORMATS: list[str] = [ + "yyyy-MM-dd", + "MM/dd/yyyy", + "yyyy-MM-dd HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ssXXX", +] + + +class _StubTransport(Transport): + """Minimal stub that satisfies the Transport ABC for get_datetime_formats tests.""" + + def __init__( + self, + formats: list[str] | None = None, + raise_error: Exception | None = None, + ) -> None: + self._formats = formats if formats is not None else list(_SAMPLE_FORMATS) + self._raise = raise_error + self.last_request: DatetimeFormatsRequest | None = None + + def list_resources(self, request: ResourceQuery) -> list[ResourceInfo]: + return [] + + def get_ticket(self, ticket: DatasetTicket) -> DataTable: + raise NotImplementedError + + def dry_publish_dataset(self, publish_request: PublishRequest) -> DryPublishResponse: + raise NotImplementedError + + def publish_dataset(self, publish_request: PublishRequest) -> PublishResponse: + raise NotImplementedError + + def get_datetime_formats(self, request: DatetimeFormatsRequest) -> list[str]: + self.last_request = request + if self._raise is not None: + raise self._raise + return self._formats + + def close(self) -> None: + pass + + +def _make_service( + formats: list[str] | None = None, + raise_error: Exception | None = None, +) -> tuple[DefaultDataConnectService, _StubTransport]: + transport = _StubTransport(formats=formats, raise_error=raise_error) + return DefaultDataConnectService(transport), transport + + +# --------------------------------------------------------------------------- +# DatetimeFormatsResult — domain model helpers +# --------------------------------------------------------------------------- + + +class TestDatetimeFormatsResult: + """``DatetimeFormatsResult.all/dates/datetimes`` must return the correct + views without mutating the underlying list. + """ + + def _make(self) -> DatetimeFormatsResult: + return DatetimeFormatsResult( + formats=[ + DatetimeFormat(format="yyyy-MM-dd", type="date"), + DatetimeFormat(format="MM/dd/yyyy", type="date"), + DatetimeFormat(format="yyyy-MM-dd HH:mm:ss", type="datetime"), + DatetimeFormat(format="yyyy-MM-dd'T'HH:mm:ssXXX", type="datetime"), + ] + ) + + def test_all_returns_every_format_object(self) -> None: + result = self._make() + items = result.all() + assert len(items) == 4 + assert all(isinstance(item, DatetimeFormat) for item in items) + assert [item.format for item in items] == [ + "yyyy-MM-dd", + "MM/dd/yyyy", + "yyyy-MM-dd HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ssXXX", + ] + + def test_all_returns_a_copy_not_the_internal_list(self) -> None: + result = self._make() + items = result.all() + items.pop() + assert len(result.all()) == 4 + + def test_dates_returns_only_date_format_strings(self) -> None: + result = self._make() + assert result.dates() == ["yyyy-MM-dd", "MM/dd/yyyy"] + + def test_datetimes_returns_only_datetime_format_strings(self) -> None: + result = self._make() + assert result.datetimes() == [ + "yyyy-MM-dd HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ssXXX", + ] + + def test_dates_returns_list_of_strings(self) -> None: + result = self._make() + assert all(isinstance(fmt, str) for fmt in result.dates()) + + def test_datetimes_returns_list_of_strings(self) -> None: + result = self._make() + assert all(isinstance(fmt, str) for fmt in result.datetimes()) + + def test_empty_result_methods_return_empty_lists(self) -> None: + result = DatetimeFormatsResult(formats=[]) + assert result.all() == [] + assert result.dates() == [] + assert result.datetimes() == [] + + def test_default_formats_is_empty_list_and_not_shared(self) -> None: + a = DatetimeFormatsResult() + b = DatetimeFormatsResult() + a.formats.append(DatetimeFormat(format="yyyy", type="date")) + assert b.formats == [] + + +# --------------------------------------------------------------------------- +# DefaultDataConnectService.get_datetime_formats +# --------------------------------------------------------------------------- + + +class TestServiceGetDatetimeFormats: + """``DefaultDataConnectService.get_datetime_formats`` must validate the + ``format_type`` filter, forward the request to the transport, and classify + each returned format. + """ + + def test_returns_datetime_formats_result_instance(self) -> None: + service, _ = _make_service() + result = service.get_datetime_formats(project_token="tok") + assert isinstance(result, DatetimeFormatsResult) + + def test_default_format_type_is_all(self) -> None: + service, transport = _make_service() + service.get_datetime_formats(project_token="tok") + assert transport.last_request is not None + assert transport.last_request.format_type == "all" + + def test_project_token_forwarded_to_transport(self) -> None: + service, transport = _make_service() + service.get_datetime_formats(project_token="abc123", format_type="all") + assert transport.last_request is not None + assert transport.last_request.project_token == "abc123" + + def test_format_type_date_forwarded_to_transport(self) -> None: + service, transport = _make_service() + service.get_datetime_formats(project_token="tok", format_type="date") + assert transport.last_request is not None + assert transport.last_request.format_type == "date" + + def test_format_type_datetime_forwarded_to_transport(self) -> None: + service, transport = _make_service() + service.get_datetime_formats(project_token="tok", format_type="datetime") + assert transport.last_request is not None + assert transport.last_request.format_type == "datetime" + + def test_format_type_is_normalised_case_insensitively(self) -> None: + service, transport = _make_service() + service.get_datetime_formats(project_token="tok", format_type="DATE") + assert transport.last_request is not None + assert transport.last_request.format_type == "date" + + def test_format_type_is_normalised_with_whitespace(self) -> None: + service, transport = _make_service() + service.get_datetime_formats(project_token="tok", format_type=" Datetime ") + assert transport.last_request is not None + assert transport.last_request.format_type == "datetime" + + def test_empty_string_format_type_is_treated_as_all(self) -> None: + service, transport = _make_service() + service.get_datetime_formats(project_token="tok", format_type="") + assert transport.last_request is not None + assert transport.last_request.format_type == "all" + + # --- response classification --- + + def test_result_preserves_server_order(self) -> None: + service, _ = _make_service() + result = service.get_datetime_formats(project_token="tok") + assert [f.format for f in result.formats] == _SAMPLE_FORMATS + + def test_date_only_formats_classified_as_date(self) -> None: + service, _ = _make_service(formats=["yyyy-MM-dd", "MM/dd/yyyy"]) + result = service.get_datetime_formats(project_token="tok", format_type="date") + assert [f.type for f in result.formats] == ["date", "date"] + + def test_formats_with_time_component_classified_as_datetime(self) -> None: + service, _ = _make_service( + formats=["yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd'T'HH:mm:ssXXX"], + ) + result = service.get_datetime_formats(project_token="tok", format_type="datetime") + assert [f.type for f in result.formats] == ["datetime", "datetime"] + + def test_mixed_classification_when_format_type_is_all(self) -> None: + service, _ = _make_service() # uses _SAMPLE_FORMATS + result = service.get_datetime_formats(project_token="tok", format_type="all") + assert [f.type for f in result.formats] == [ + "date", + "date", + "datetime", + "datetime", + ] + + def test_filter_helpers_work_on_service_result(self) -> None: + service, _ = _make_service() + result = service.get_datetime_formats(project_token="tok") + assert result.dates() == ["yyyy-MM-dd", "MM/dd/yyyy"] + assert result.datetimes() == [ + "yyyy-MM-dd HH:mm:ss", + "yyyy-MM-dd'T'HH:mm:ssXXX", + ] + + def test_empty_server_response_yields_empty_result(self) -> None: + service, _ = _make_service(formats=[]) + result = service.get_datetime_formats(project_token="tok") + assert result.all() == [] + assert result.dates() == [] + assert result.datetimes() == [] + + # --- invalid input --- + + @pytest.mark.parametrize("bad_type", ["NA", "invalid", "datetimes", "DATETIMEZ", "1"]) + def test_invalid_format_type_raises_validation_error(self, bad_type: str) -> None: + service, transport = _make_service() + with pytest.raises(ValidationError): + service.get_datetime_formats(project_token="tok", format_type=bad_type) + # Transport must not be invoked when validation fails up-front. + assert transport.last_request is None + + # --- error translation --- + + def test_transport_validation_error_is_translated_to_service_error(self) -> None: + err = TransportValidationError( + error_code="VAL_008", + message="invalid project token", + timestamp="2024-01-01T00:00:00Z", + ) + service, _ = _make_service(raise_error=err) + with pytest.raises(ValidationError): + service.get_datetime_formats(project_token="bad", format_type="all") + + +# --------------------------------------------------------------------------- +# ArrowFlightTransport.get_datetime_formats +# --------------------------------------------------------------------------- + + +def _make_flight_transport() -> ArrowFlightTransport: + """Create an ``ArrowFlightTransport`` with a mocked ``FlightClient``.""" + with patch.object(ArrowFlightTransport, "_get_client", return_value=MagicMock()): + return ArrowFlightTransport(host="localhost", port=5005, use_tls=False) + + +def _make_action_result(payload: list[str]) -> MagicMock: + body = MagicMock() + body.to_pybytes.return_value = json.dumps(payload).encode("utf-8") + result = MagicMock() + result.body = body + return result + + +class TestTransportGetDatetimeFormats: + """``ArrowFlightTransport.get_datetime_formats`` must encode the request as a + Flight ``Action`` and decode the JSON response. + """ + + def test_action_type_is_get_datetime_formats(self) -> None: + transport = _make_flight_transport() + transport._client.do_action.return_value = iter([_make_action_result(_SAMPLE_FORMATS)]) + + transport.get_datetime_formats(DatetimeFormatsRequest(project_token="tok", format_type="all")) + + action_arg = transport._client.do_action.call_args[0][0] + assert action_arg.type == "get_datetime_formats" + + def test_action_body_contains_project_token_and_type(self) -> None: + transport = _make_flight_transport() + transport._client.do_action.return_value = iter([_make_action_result(_SAMPLE_FORMATS)]) + + transport.get_datetime_formats(DatetimeFormatsRequest(project_token="my_token", format_type="datetime")) + + action_arg = transport._client.do_action.call_args[0][0] + body = json.loads(action_arg.body.to_pybytes().decode("utf-8")) + assert body == {"project_token": "my_token", "type": "datetime"} + + def test_returns_decoded_format_list(self) -> None: + transport = _make_flight_transport() + transport._client.do_action.return_value = iter([_make_action_result(_SAMPLE_FORMATS)]) + + result = transport.get_datetime_formats(DatetimeFormatsRequest(project_token="tok", format_type="all")) + + assert result == _SAMPLE_FORMATS + + def test_empty_result_iterator_returns_empty_list(self) -> None: + transport = _make_flight_transport() + transport._client.do_action.return_value = iter([]) + + result = transport.get_datetime_formats(DatetimeFormatsRequest(project_token="tok", format_type="all")) + + assert result == [] + + def test_underlying_exception_is_translated_to_transport_error(self) -> None: + from dataconnect.transport.errors import TransportError + + transport = _make_flight_transport() + transport._client.do_action.side_effect = RuntimeError("boom") + + with pytest.raises(TransportError): + transport.get_datetime_formats(DatetimeFormatsRequest(project_token="tok", format_type="all")) + + +# --------------------------------------------------------------------------- +# DataConnectClient.get_datetime_formats (façade) +# --------------------------------------------------------------------------- + + +class _FakeService: + """Minimal fake service to verify the client delegates correctly.""" + + def __init__(self, return_value: DatetimeFormatsResult | None = None) -> None: + self._return = return_value or DatetimeFormatsResult(formats=[DatetimeFormat(format="yyyy-MM-dd", type="date")]) + self.calls: list[dict] = [] + + def get_datetime_formats(self, **kwargs: object) -> DatetimeFormatsResult: + self.calls.append(kwargs) + return self._return + + # Unused service methods — must exist to satisfy the implicit protocol used + # by DataConnectClient. + + def get_studies(self, **kwargs: object) -> None: # type: ignore[return] + pass + + def get_datasets(self, **kwargs: object) -> None: # type: ignore[return] + pass + + def get_dataset_versions(self, **kwargs: object) -> None: # type: ignore[return] + pass + + def fetch_data(self, **kwargs: object) -> None: # type: ignore[return] + pass + + def dry_publish(self, **kwargs: object) -> None: # type: ignore[return] + pass + + def publish(self, **kwargs: object) -> None: # type: ignore[return] + pass + + def close(self) -> None: + pass + + +class TestClientGetDatetimeFormats: + """``DataConnectClient.get_datetime_formats`` must delegate to the service.""" + + def test_returns_datetime_formats_result(self) -> None: + service = _FakeService() + client = DataConnectClient(service) # type: ignore[arg-type] + result = client.get_datetime_formats(project_token="tok") + assert isinstance(result, DatetimeFormatsResult) + + def test_default_format_type_is_all(self) -> None: + service = _FakeService() + client = DataConnectClient(service) # type: ignore[arg-type] + client.get_datetime_formats(project_token="tok") + assert service.calls == [{"project_token": "tok", "format_type": "all"}] + + def test_format_type_is_forwarded(self) -> None: + service = _FakeService() + client = DataConnectClient(service) # type: ignore[arg-type] + client.get_datetime_formats(project_token="tok", format_type="datetime") + assert service.calls == [{"project_token": "tok", "format_type": "datetime"}] + + def test_result_from_service_is_returned_unchanged(self) -> None: + expected = DatetimeFormatsResult(formats=[DatetimeFormat(format="yyyy-MM-dd HH:mm", type="datetime")]) + service = _FakeService(return_value=expected) + client = DataConnectClient(service) # type: ignore[arg-type] + result = client.get_datetime_formats(project_token="tok", format_type="datetime") + assert result is expected diff --git a/tests/test_publish.py b/tests/test_publish.py index 04384bc..74ed3a2 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -29,6 +29,7 @@ from dataconnect.transport.models import ( DatasetTicket, DataTable, + DatetimeFormatsRequest, DryPublishResponse, PublishRequest, PublishResponse, @@ -165,6 +166,9 @@ def publish_dataset(self, publish_request: PublishRequest) -> PublishResponse: raise self._raise return self._return # type: ignore[return-value] + def get_datetime_formats(self, request: DatetimeFormatsRequest) -> list[str]: # type: ignore[override] + raise NotImplementedError + def close(self) -> None: pass