diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 60371125ade90..691ec69550b82 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -2884,7 +2884,7 @@ def test_upgrade_to_newer_dependencies( ("providers/common/sql/src/airflow/providers/common/sql/common_sql_python.py",), { "docs-list-as-string": "amazon apache.drill apache.druid apache.hive apache.iceberg " - "apache.impala apache.pinot clickhousedb common.ai common.compat common.sql databricks elasticsearch " + "apache.impala apache.pinot clickhousedb common.ai common.compat common.dataquality common.sql databricks elasticsearch " "exasol google informatica jdbc microsoft.mssql mysql odbc openlineage " "oracle pgvector postgres presto slack snowflake sqlite teradata trino vertica ydb", }, diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 3382600169700..d4327fad46968 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -294,6 +294,7 @@ conda conf Config config +ConfigDict configfile configMap configmap diff --git a/providers/common/dataquality/.pre-commit-config.yaml b/providers/common/dataquality/.pre-commit-config.yaml new file mode 100644 index 0000000000000..173d5a5527520 --- /dev/null +++ b/providers/common/dataquality/.pre-commit-config.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +default_stages: [pre-commit, pre-push] +minimum_prek_version: '0.3.4' +default_language_version: + python: python3 + node: 22.19.0 + golang: 1.24.0 +repos: + - repo: local + hooks: + - id: generate-common-dataquality-ruleset-schema + name: Generate Data Quality RuleSet schema + language: python + entry: ../../../scripts/ci/prek/generate_common_dataquality_ruleset_schema.py + pass_filenames: false + always_run: true + files: > + (?x) + ^src/airflow/providers/common/dataquality/rules/.*\.py$| + ^src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset\.schema\.json$ diff --git a/providers/common/dataquality/README.rst b/providers/common/dataquality/README.rst index baddbee922875..b9e552d12d263 100644 --- a/providers/common/dataquality/README.rst +++ b/providers/common/dataquality/README.rst @@ -19,7 +19,11 @@ Package ``apache-airflow-providers-common-dataquality`` Release: ``0.1.0`` -Common Data Quality Provider +``Data Quality Provider`` + +Declarative data quality rules with durable, per-rule execution history. Checks run through +``common.sql`` DB-API hooks; results are persisted to a configurable results store (object +storage or local files) so task, run, and rule-level quality can be inspected over time. Provider package ---------------- @@ -37,8 +41,12 @@ see ``Requirements`` below. Requirements ------------ -================== ================== -PIP package Version required -================== ================== -``apache-airflow`` ``>=3.0.0`` -================== ================== +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== diff --git a/providers/common/dataquality/docs/agents.rst b/providers/common/dataquality/docs/agents.rst new file mode 100644 index 0000000000000..8a0d160a4e2f2 --- /dev/null +++ b/providers/common/dataquality/docs/agents.rst @@ -0,0 +1,55 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _dq:agents: + +Generating rules with an LLM +============================== + +Writing a :class:`~airflow.providers.common.dataquality.rules.RuleSet` by hand for every table doesn't scale. +An LLM can propose one from a table's column definitions instead, given the check catalog as +context. + +The ``dataquality-rule-authoring`` skill +---------------------------------------- + +This provider ships an `Agent Skill `__ at +``airflow/providers/common/dataquality/skills/dataquality-rule-authoring/``: a ``SKILL.md`` documenting the +``RuleSet``/``DQRule`` fields and check catalog, plus a generated JSON Schema +(``references/ruleset.schema.json``) for validation. + +Point ``common.ai``'s :doc:`AgentSkillsToolset ` at +it, and give the model ``output_type=RuleSet`` so pydantic-ai validates -- and self-corrects -- +its output before the task completes: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py + :language: python + :start-after: [START howto_task_dq_generate_ruleset_with_llm] + :end-before: [END howto_task_dq_generate_ruleset_with_llm] + +Requires ``apache-airflow-providers-common-ai[skills]`` and a configured ``llm_conn_id``. + +Wiring the result into a check +--------------------------------- + +``@task.dq_check`` can leave ``ruleset=`` unset and return the LLM task's result at execution +time instead: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py + :language: python + :start-after: [START howto_decorator_dq_check_llm_runtime_ruleset] + :end-before: [END howto_decorator_dq_check_llm_runtime_ruleset] diff --git a/providers/common/dataquality/docs/assets.rst b/providers/common/dataquality/docs/assets.rst new file mode 100644 index 0000000000000..00cb0e7c2444d --- /dev/null +++ b/providers/common/dataquality/docs/assets.rst @@ -0,0 +1,70 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _dq:assets: + +Assets and quality gating +============================ + +Quality rules can travel with the :class:`~airflow.sdk.Asset` they describe instead of being +scattered across every Dag that checks it, and a downstream consumer Dag can refuse to run when +the data it was triggered by did not meet a minimum quality bar. + +Attaching a ruleset to an asset +---------------------------------- + +:func:`~airflow.providers.common.dataquality.assets.asset_quality` stores ruleset, connection, and table +configuration inside ``Asset.extra`` under the ``airflow.dataquality`` key, so it is serialized with the +Dag and needs no Airflow core changes: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py + :language: python + :start-after: [START howto_asset_quality] + :end-before: [END howto_asset_quality] + +Pass the resulting asset to :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` +via ``asset=`` (see :doc:`operators`) instead of ``table``/``ruleset``/``conn_id``: the operator +resolves all three from the asset's config, adds the asset to its own outlets, and attaches its +summary -- including the quality ``score`` used below -- to the asset event. + +Only one Dag should call ``asset_quality()`` for a given asset ``name``/``uri``. Airflow keeps one +shared record per asset across all Dags, so if more than one Dag attaches (or omits) config for the +same asset, whichever Dag parsed most recently determines what's stored. + +Gating a consumer Dag on quality +------------------------------------ + +:func:`~airflow.providers.common.dataquality.assets.require_quality` builds a ``@task.short_circuit`` task that +reads the ``score`` off the asset event that triggered the current run, and skips every +downstream task when that event has no quality summary at all, or its score is below +``min_score``: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py + :language: python + :start-after: [START howto_require_quality] + :end-before: [END howto_require_quality] + +Put the gate first in a Dag scheduled by the asset, and chain everything else after it: + +.. code-block:: python + + with DAG("orders_consumer", schedule=orders_asset) as consumer: + gate = require_quality(orders_asset, min_score=0.95) + gate >> process_orders() + +``min_score`` must be between ``0`` and ``1``; the check considers only the *most recent* +triggering event for the asset when a run was triggered by several. diff --git a/providers/common/dataquality/docs/configurations-ref.rst b/providers/common/dataquality/docs/configurations-ref.rst new file mode 100644 index 0000000000000..ea8e668d75793 --- /dev/null +++ b/providers/common/dataquality/docs/configurations-ref.rst @@ -0,0 +1,19 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/providers-configurations-ref.rst +.. include:: /../../../../devel-common/src/sphinx_exts/includes/sections-and-options.rst diff --git a/providers/common/dataquality/docs/decorators.rst b/providers/common/dataquality/docs/decorators.rst new file mode 100644 index 0000000000000..1ec79dd5bd0b9 --- /dev/null +++ b/providers/common/dataquality/docs/decorators.rst @@ -0,0 +1,52 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/decorator:dq_check: + +``@task.dq_check`` +===================== + +``@task.dq_check`` wraps :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` in +the TaskFlow API. ``ruleset`` may be declared as a decorator argument when it exists at +Dag-parse time, or returned by the decorated function as a runtime ruleset. ``table`` or +``asset`` are declared as decorator arguments exactly like the plain operator. The decorated +function is optional plumbing on top: return ``None`` to run the check exactly as declared. + +.. exampleinclude:: /../tests/system/common/dataquality/example_dq_check.py + :language: python + :dedent: 4 + :start-after: [START howto_decorator_dq_check] + :end-before: [END howto_decorator_dq_check] + +Runtime rule sets +-------------------- + +Return a :class:`~airflow.providers.common.dataquality.rules.RuleSet`, its dict form, or a YAML path to use a +ruleset that is only known at task-execution time -- for example one produced by an upstream +task, loaded from a Variable, or generated by an LLM. Return ``None`` to use the ruleset declared +on the decorator. + +Swapping in a different ruleset at execution time: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py + :language: python + :start-after: [START howto_decorator_dq_check_runtime_ruleset] + :end-before: [END howto_decorator_dq_check_runtime_ruleset] + +Everything documented for the plain operator in :doc:`operators` -- ``fail_on``, persistence, +``asset``, ``partition_clause``, ``custom_sql`` -- applies unchanged; the decorator only adds the +optional runtime ruleset step before the check runs. diff --git a/providers/common/dataquality/docs/index.rst b/providers/common/dataquality/docs/index.rst index 445360c2d4933..0e26c1be95b17 100644 --- a/providers/common/dataquality/docs/index.rst +++ b/providers/common/dataquality/docs/index.rst @@ -31,16 +31,21 @@ .. toctree:: :hidden: :maxdepth: 1 - :caption: References + :caption: Guides - Python API <_api/airflow/providers/common/dataquality/index> + Rules, rule sets, and checks + Operators + Decorators + Assets and quality gating + Generating rules with an LLM .. toctree:: :hidden: :maxdepth: 1 - :caption: System tests + :caption: References - System Tests <_api/tests/system/common/dataquality/index> + Configuration + Python API <_api/airflow/providers/common/dataquality/index> .. toctree:: :hidden: @@ -50,6 +55,64 @@ PyPI Repository Installing from sources +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/common/dataquality/index> + + +apache-airflow-providers-common-dataquality package +--------------------------------------------------- + +``Data Quality Provider`` + +Declarative data quality rules with durable, per-rule execution history. Checks run through +``common.sql`` DB-API hooks; results are persisted to a configurable results store (object +storage or local files) so task, run, and rule-level quality can be inspected over time. + +See :doc:`rules` for the built-in check catalog (and its cross-database caveats), +:doc:`operators`/:doc:`decorators` for running checks, and :doc:`assets` for attaching rules to +an asset and gating a downstream Dag on its quality score. + +Release: 0.1.0 + +Provider package +---------------- + +This package is for the ``common.dataquality`` provider. +All classes for this package are included in the ``airflow.providers.common.dataquality`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-common-dataquality``. For the minimum Airflow version supported, +see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``. + +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== + .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! @@ -64,7 +127,12 @@ apache-airflow-providers-common-dataquality package ------------------------------------------------------ -Common Data Quality Provider +``Data Quality Provider`` + +Declarative data quality rules with durable, per-rule execution history. +Checks run through ``common.sql`` DB-API hooks; results are persisted to a +configurable results store (object storage or local files) so task, run, +and rule-level quality can be inspected over time. Release: 0.1.0 @@ -87,11 +155,15 @@ Requirements The minimum Apache Airflow version supported by this provider distribution is ``3.0.0``. -================== ================== -PIP package Version required -================== ================== -``apache-airflow`` ``>=3.0.0`` -================== ================== +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.15.0`` +``apache-airflow-providers-common-sql`` ``>=2.0.0`` +``pydantic`` ``>=2.11.0`` +``pyyaml`` ``>=6.0.2`` +========================================== ================== Downloading official packages ----------------------------- diff --git a/providers/common/dataquality/docs/operators.rst b/providers/common/dataquality/docs/operators.rst new file mode 100644 index 0000000000000..d12b0dd79d524 --- /dev/null +++ b/providers/common/dataquality/docs/operators.rst @@ -0,0 +1,132 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _howto/operator:DQCheckOperator: + +``DQCheckOperator`` +===================== + +Use :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` to run a +:doc:`ruleset ` against a table and persist per-rule results to the configured results +store. Every rule is evaluated and recorded regardless of the task outcome -- a rule failing +doesn't stop the others from running, and the full set of results is always written before the +task decides whether to fail. + +Basic usage +------------ + +Pass a connection, table, and ruleset directly: + +.. exampleinclude:: /../tests/system/common/dataquality/example_dq_check.py + :language: python + :dedent: 4 + :start-after: [START howto_operator_dq_check] + :end-before: [END howto_operator_dq_check] + +Parameters +----------- + +- ``ruleset`` -- a :class:`~airflow.providers.common.dataquality.rules.RuleSet`, its dict form, or a path to a + YAML ruleset file (see :doc:`rules`). Optional when ``asset`` carries one. +- ``table`` -- the table to check. Optional when ``asset`` carries one (falling back to the + asset's name). +- ``asset`` -- an :class:`~airflow.sdk.Asset` decorated with + :func:`~airflow.providers.common.dataquality.assets.asset_quality`. Supplies defaults for ``ruleset``, + ``table``, and ``conn_id`` (explicit arguments win) and is automatically added to the task's + outlets so its asset events carry the check summary -- see :doc:`assets`. +- ``partition_clause`` -- predicate ANDed into every built-in check's ``WHERE`` clause, e.g. + ``"ds = '{{ ds }}'"`` (templated). +- ``fail_on`` -- severity that fails the task: + + - ``error`` (default) -- only ``error``-severity rule failures fail the task; ``warn`` + failures are recorded but don't fail it. + - ``warn`` -- any rule failure (``error`` or ``warn`` severity) fails the task. + - ``never`` -- rule failures never fail the task; only an execution error (the check query + itself failing) does. + +- ``conn_id`` -- connection to the database to check, any ``common.sql`` ``DbApiHook``- + compatible type. +- ``database`` -- optional database/schema, overriding the connection's default. + +Regardless of ``fail_on``, an execution error -- the check query itself failing, as opposed to +a rule failing its condition -- always fails the task; there's no way to check data whose query +can't even run. + +Persisting results +-------------------- + +Results are persisted to the backend configured under ``[common.dataquality] results_path`` (see +:doc:`configurations-ref`). When that's unset, checks still run and the task still passes or +fails normally -- only persisted data quality history is unavailable. There is no +per-operator override: every check in a deployment shares one results store, so history stays +available across tasks and Dags without stitching together several stores. + +Run quality checks inside your own tasks +------------------------------------------ + +If quality checks are one step inside a larger task, use +:func:`~airflow.providers.common.dataquality.execution.run_quality_checks` directly, then call +:func:`~airflow.providers.common.dataquality.execution.persist_quality_results` to write the same +history records that ``DQCheckOperator`` writes: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py + :language: python + :start-after: [START howto_run_dq_inside_task] + :end-before: [END howto_run_dq_inside_task] + +Checking an asset +-------------------- + +Attach a ruleset to an :class:`~airflow.sdk.Asset` with +:func:`~airflow.providers.common.dataquality.assets.asset_quality`, then pass the asset instead of ``table``/ +``ruleset``/``conn_id``: + +.. exampleinclude:: /../tests/system/common/dataquality/example_dq_check.py + :language: python + :start-after: [START howto_operator_dq_check_asset] + :end-before: [END howto_operator_dq_check_asset] + +The operator adds the asset to its own outlets automatically, and attaches the check's summary +(including its quality ``score``) to the asset event -- which is what makes +:func:`~airflow.providers.common.dataquality.assets.require_quality` (see :doc:`assets`) able to gate a +downstream consumer Dag on it. + +``custom_sql`` checks +------------------------ + +Built-in checks are all single-column. For cross-column comparisons, joins, or anything the +catalog doesn't cover, use ``custom_sql``: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py + :language: python + :start-after: [START howto_operator_dq_check_custom_sql] + :end-before: [END howto_operator_dq_check_custom_sql] + +See :doc:`rules` for the full built-in check catalog and the ``custom_sql`` grammar. + +Loading a ruleset from YAML +------------------------------ + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py + :language: python + :start-after: [START howto_operator_dq_check_ruleset_from_yaml] + :end-before: [END howto_operator_dq_check_ruleset_from_yaml] + +TaskFlow decorator +-------------------- + +See :doc:`decorators` for the ``@task.dq_check`` equivalent, including runtime rule sets. diff --git a/providers/common/dataquality/docs/rules.rst b/providers/common/dataquality/docs/rules.rst new file mode 100644 index 0000000000000..efcd8c0e0f136 --- /dev/null +++ b/providers/common/dataquality/docs/rules.rst @@ -0,0 +1,223 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. _dq:rules: + +Rules, rule sets, and checks +============================ + +Rules are data, not code. A :class:`~airflow.providers.common.dataquality.rules.RuleSet` is a named tuple of +:class:`~airflow.providers.common.dataquality.rules.DQRule` -- plain, serializable objects with no behavior of +their own. They describe *what* to check; :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` +(see :doc:`operators`) is what actually runs them. Rules are usually written by hand, but they +can also be proposed by an LLM -- see :doc:`agents`. + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py + :language: python + :start-after: [START howto_rules_dq_ruleset] + :end-before: [END howto_rules_dq_ruleset] + +``DQRule`` fields +------------------ + +- ``name`` -- unique within its ruleset. Shows up in data quality results and logs. +- ``check`` -- one of the built-in checks below, or ``custom_sql``. +- ``condition`` -- the pass/fail condition for the observed value; see `Conditions`_. +- ``column`` -- target column. Required for column-level built-in checks, unused for + ``row_count`` and ``custom_sql``. +- ``sql`` -- a SQL statement returning a single scalar. Required for, and only valid with, + ``check="custom_sql"``. May reference the table being checked as ``{table}``. +- ``severity`` -- ``error`` (default) or ``warn``. Controls whether a failing rule fails the + task, subject to the operator's ``fail_on`` (see :doc:`operators`). +- ``partition_clause`` -- extra SQL predicate ANDed into this rule's ``WHERE`` clause, e.g. + ``"region = 'EU'"``. Combines with the operator-level ``partition_clause``, if any. +- ``previous_name`` -- set when renaming a rule so its execution history stays continuous + (see `Identity and history`_). +- ``id`` -- explicit, stable identity for this rule's history, used directly as ``rule_uid`` + instead of the derived hash (see `Identity and history`_). +- ``description`` -- optional human-readable text shown in data quality results. + When omitted, the provider generates a short default description from the rule and condition. +- ``dimension`` -- one of ``completeness``, ``uniqueness``, ``validity``, ``freshness``, + ``volume``, ``consistency``. Defaults to the check's catalog dimension (see the table below; + ``validity`` for ``custom_sql``) when left unset. Only set this explicitly for a ``custom_sql`` + rule that measures something the default dimension doesn't capture. + +Built-in checks +---------------- + +Each built-in check is backed by a plain SQL expression, rendered with ``{column}`` for +column-level checks: + +.. list-table:: + :header-rows: 1 + + * - Check + - SQL expression + - Column required + - Default dimension + * - ``null_count`` + - ``SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END)`` + - yes + - ``completeness`` + * - ``null_ratio`` + - ``SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*)`` + - yes + - ``completeness`` + * - ``distinct_count`` + - ``COUNT(DISTINCT {column})`` + - yes + - ``uniqueness`` + * - ``unique_violations`` + - ``COUNT({column}) - COUNT(DISTINCT {column})`` + - yes + - ``uniqueness`` + * - ``min`` + - ``MIN({column})`` + - yes + - ``validity`` + * - ``max`` + - ``MAX({column})`` + - yes + - ``validity`` + * - ``mean`` + - ``AVG({column})`` + - yes + - ``validity`` + * - ``row_count`` + - ``COUNT(*)`` + - no + - ``volume`` + +.. note:: + + ``null_ratio`` divides by ``COUNT(*)`` with no guard against an empty table (or an empty + partition, when combined with ``partition_clause``); running it against zero rows is a + division-by-zero at the database level, not a clean "not applicable" result. Guard with a + ``row_count`` rule upstream, or a ``partition_clause`` that guarantees a non-empty set. A + portable guard would wrap the denominator in ``NULLIF(COUNT(*), 0)``, but ``NULLIF`` isn't + supported by every ``DbApiHook`` (see `Supported checks and databases`_) -- this is one + instance of the general rule below: if a built-in check's SQL doesn't work against your + database, express it as ``custom_sql`` instead. + +Conditions +----------- + +A :class:`~airflow.providers.common.dataquality.rules.Condition` is the pass/fail rule applied to the observed +value, using the same grammar as the ``common.sql`` check operators: + +- ``equal_to`` -- exact match. Cannot be combined with any other comparison. +- ``greater_than`` / ``geq_to`` -- lower bound, exclusive/inclusive. +- ``less_than`` / ``leq_to`` -- upper bound, exclusive/inclusive. +- ``tolerance`` -- a percentage that widens the comparison bounds. + +``greater_than``/``less_than``/``geq_to``/``leq_to`` may be combined to express a range +(e.g. ``{"geq_to": 0, "leq_to": 10}``). + +``custom_sql``: the escape hatch +---------------------------------- + +Built-in checks are all single-column. The moment a rule needs to compare two columns, join +another table, or use a function the catalog doesn't cover, use ``custom_sql`` -- any SQL +statement that resolves to a single scalar, evaluated exactly like a built-in check: + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py + :language: python + :start-after: [START howto_operator_dq_check_custom_sql] + :end-before: [END howto_operator_dq_check_custom_sql] + +Supported checks and databases +-------------------------------- + +Built-in checks are plain SQL expressions executed through whichever ``common.sql`` +:class:`~airflow.providers.common.sql.hooks.sql.DbApiHook` your connection resolves to. +Airflow does not validate those expressions per database dialect. + +The catalog intentionally uses simple expressions that work on common relational databases +and many distributed SQL engines, but support is not guaranteed for every ``DbApiHook``. +Some hooks expose non-standard SQL layers and may reject or evaluate an expression differently. + +For example, ``null_ratio`` is currently rendered as: + +.. code-block:: text + + SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*) + +This is simple and portable, but it is not guarded against an empty table or an empty +partition. A database-specific version could use a different expression, for example: + +.. code-block:: text + + CASE WHEN COUNT(*) = 0 THEN NULL + ELSE SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*) END + +If a built-in check does not work for your database or you need different empty-table +semantics, use ``custom_sql`` and write the expression for your dialect. + +Loading rules from YAML +------------------------- + +Anywhere a ``RuleSet`` is accepted -- ``DQCheckOperator(ruleset=...)``, +``@task.dq_check(ruleset=...)``, :func:`~airflow.providers.common.dataquality.assets.asset_quality` -- a path +string is accepted too, and resolved via :meth:`~airflow.providers.common.dataquality.rules.RuleSet.from_file` +at Dag-parse time. This keeps rules editable by people who don't write Python. + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/orders_ruleset.yaml + :language: yaml + +.. exampleinclude:: /../src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py + :language: python + :start-after: [START howto_operator_dq_check_ruleset_from_yaml] + :end-before: [END howto_operator_dq_check_ruleset_from_yaml] + +Identity and history +---------------------- + +Each rule has a stable ``rule_uid``, a hash of the fields that define *what is being measured*: +``name`` (or ``previous_name``, if set), ``check``, ``column``, ``sql``, and ``condition``. +Everything else -- including ``severity``, ``partition_clause``, ``description``, and +``dimension`` -- can change between Dag runs without breaking the rule's execution history, +because it isn't part of the identity hash. + +Renaming a rule outright would normally start a new history under the new name; set +``previous_name`` to the old name for one deploy to carry the old identity forward instead: + +.. code-block:: python + + DQRule( + name="order_id_is_unique", # renamed from order_id_unique + previous_name="order_id_unique", + check="unique_violations", + column="order_id", + condition={"equal_to": 0}, + ) + +The derived hash isn't guaranteed unique across every possible ruleset: two rules can end up +with the same ``rule_uid`` if their identity fields happen to line up (for example, a renamed +rule's ``previous_name`` matching another rule's current ``name``, ``check``, ``column``, and +``condition``). ``RuleSet`` validates against this and raises if it happens. If you'd rather +not depend on the hash at all, set ``id`` and it's used as the ``rule_uid`` directly: + +.. code-block:: python + + DQRule( + name="order_id_is_unique", + previous_name="order_id_unique", + check="unique_violations", + column="order_id", + condition={"equal_to": 0}, + id="order_id_uniqueness", # stable regardless of future renames or condition tweaks + ) diff --git a/providers/common/dataquality/provider.yaml b/providers/common/dataquality/provider.yaml index ae44126145dd5..ffe1ad33ef588 100644 --- a/providers/common/dataquality/provider.yaml +++ b/providers/common/dataquality/provider.yaml @@ -19,7 +19,12 @@ package-name: apache-airflow-providers-common-dataquality name: Common Data Quality description: | - Common Data Quality Provider + ``Data Quality Provider`` + + Declarative data quality rules with durable, per-rule execution history. + Checks run through ``common.sql`` DB-API hooks; results are persisted to a + configurable results store (object storage or local files) so task, run, + and rule-level quality can be inspected over time. state: ready lifecycle: incubation @@ -32,3 +37,41 @@ build-system: flit_core # to be done in the same PR versions: - 0.1.0 + +integrations: + - integration-name: Common Data Quality + external-doc-url: https://airflow.apache.org/docs/apache-airflow-providers-common-dataquality/ + how-to-guide: + - /docs/apache-airflow-providers-common-dataquality/operators.rst + tags: [software] + +operators: + - integration-name: Common Data Quality + python-modules: + - airflow.providers.common.dataquality.operators.dq_check + +task-decorators: + - class-name: airflow.providers.common.dataquality.decorators.dq_check.dq_check_task + name: dq_check + +config: + common.dataquality: + description: | + Configuration for the Data Quality provider results store. + options: + results_path: + description: | + Any fsspec-compatible URL understood by ``ObjectStoragePath`` where data quality + results are persisted, e.g. ``s3://bucket/airflow-dq`` or ``file:///opt/airflow/dq``. + When unset, checks still run but no history is persisted. + version_added: 0.1.0 + type: string + example: s3://data-platform/airflow-dq + default: ~ + results_conn_id: + description: | + Optional Airflow connection used to access ``results_path``. + version_added: 0.1.0 + type: string + example: aws_default + default: ~ diff --git a/providers/common/dataquality/pyproject.toml b/providers/common/dataquality/pyproject.toml index fd65bbbbd02ef..bbf3f9764da68 100644 --- a/providers/common/dataquality/pyproject.toml +++ b/providers/common/dataquality/pyproject.toml @@ -60,6 +60,10 @@ requires-python = ">=3.10" # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` dependencies = [ "apache-airflow>=3.0.0", + "apache-airflow-providers-common-compat>=1.15.0", + "apache-airflow-providers-common-sql>=2.0.0", + "pydantic>=2.11.0", + "pyyaml>=6.0.2", ] [dependency-groups] @@ -67,6 +71,8 @@ dev = [ "apache-airflow", "apache-airflow-task-sdk", "apache-airflow-devel-common", + "apache-airflow-providers-common-compat", + "apache-airflow-providers-common-sql", # Additional devel dependencies (do not remove this line and add extra development dependencies) ] diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/assets.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/assets.py new file mode 100644 index 0000000000000..263cd53924503 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/assets.py @@ -0,0 +1,190 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Asset-level data quality declarations. + +Quality configuration lives inside ``Asset.extra`` under the ``airflow.dataquality`` key, so it is +serialized with the Dag and needs no Airflow core changes. The rules travel with the asset +definition instead of being scattered across the Dags that check it. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +from airflow.providers.common.dataquality.exceptions import DQRuleValidationError +from airflow.providers.common.dataquality.rules import RuleSet +from airflow.sdk import task + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from airflow.sdk import Asset + from airflow.sdk.execution_time.comms import AssetEventDagRunReferenceResult + +log = logging.getLogger(__name__) + +DQ_EXTRA_KEY = "airflow.dataquality" +DQ_RESULT_EXTRA_KEY = "airflow.dataquality.result" + + +def asset_quality( + asset: Asset, + *, + ruleset: RuleSet | dict[str, Any] | str, + conn_id: str | None = None, + table: str | None = None, +) -> Asset: + """ + Attach data quality configuration to an asset, returning the same asset. + + :param asset: The asset the rules describe. + :param ruleset: A :class:`~airflow.providers.common.dataquality.rules.RuleSet`, its dict form, or a path + to a YAML ruleset file (resolved eagerly, at Dag-parse time). + :param conn_id: Default connection a check operator should use for this asset. + :param table: Default table to check; falls back to the asset name when unset. + + Usage:: + + orders = asset_quality( + Asset("orders", uri="postgres://warehouse/analytics/orders"), + ruleset=rules, + conn_id="warehouse", + table="analytics.orders", + ) + check = DQCheckOperator(task_id="dq", asset=orders) + """ + if isinstance(ruleset, str): + ruleset = RuleSet.from_file(ruleset) + elif isinstance(ruleset, dict): + ruleset = RuleSet.from_dict(ruleset) + config: dict[str, Any] = {"ruleset": ruleset.to_dict()} + if conn_id: + config["conn_id"] = conn_id + if table: + config["table"] = table + asset.extra[DQ_EXTRA_KEY] = config + return asset + + +def get_asset_quality_config(asset: Asset) -> dict[str, Any] | None: + """Return the raw ``airflow.dataquality`` config attached to an asset, if any.""" + config = asset.extra.get(DQ_EXTRA_KEY) + if not isinstance(config, dict): + return None + return config + + +def get_asset_ruleset(asset: Asset) -> RuleSet: + """Return the ruleset attached to an asset, raising when none is attached.""" + config = get_asset_quality_config(asset) + if not config or "ruleset" not in config: + raise DQRuleValidationError( + f"Asset {asset.name!r} has no data quality config; attach one with asset_quality()" + ) + return RuleSet.from_dict(config["ruleset"]) + + +def _quality_score_passes( + asset: Asset, + min_score: float, + triggering_asset_events: Mapping[Asset, Sequence[AssetEventDagRunReferenceResult]], + *, + require_all: bool = True, +) -> bool: + """ + Pure decision logic behind :func:`require_quality`, kept separate so it is testable without a Dag. + + ``consumed_asset_events`` carries no ordering guarantee, so the triggering event is never + picked positionally. By default every triggering event must pass, since several producer + runs can coalesce into one consumer run; pass ``require_all=False`` to gate on the single + most recent event (by ``timestamp``) instead. + """ + events = triggering_asset_events.get(asset, []) + if not events: + log.warning( + "require_quality(%s): no triggering event for this run; skipping downstream tasks", + asset.name, + ) + return False + + events_to_check = events if require_all else [max(events, key=lambda event: event.timestamp)] + + for event in events_to_check: + summary = event.extra.get(DQ_RESULT_EXTRA_KEY) + score = summary.get("score") if isinstance(summary, dict) else None + if not isinstance(score, (int, float)) or isinstance(score, bool): + log.warning( + "require_quality(%s): triggering event has no data quality summary; skipping downstream tasks", + asset.name, + ) + return False + + if score < min_score: + log.warning( + "require_quality(%s): score %s below required minimum %s; skipping downstream tasks", + asset.name, + score, + min_score, + ) + return False + + return True + + +def require_quality( + asset: Asset, + *, + min_score: float, + task_id: str | None = None, + require_all: bool = True, +) -> Any: + """ + Gate a Dag run on the data quality score attached to its triggering asset event(s). + + Reads the summary a :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` + attaches to ``asset``'s outlet event, under ``asset_event.extra["airflow.dataquality.result"]`` + (see :func:`asset_quality`), and short-circuits the run — skipping every downstream task + — when that summary is missing or its score is below ``min_score``. Call it inside a Dag + scheduled by ``asset``:: + + with DAG("orders_consumer", schedule=orders) as consumer: + start = require_quality(orders, min_score=0.95) + start >> process_orders() + + :param asset: The asset whose triggering event(s) carry the quality summary. + :param min_score: Minimum required score in ``[0, 1]``; the run proceeds only when the + checked event(s)' score is at least this value. + :param task_id: Task id for the generated gate task. Defaults to + ``f"require_quality_{asset.name}"`` so gating on several assets in one Dag doesn't + collide on task id. + :param require_all: When ``True`` (default), every asset event that triggered this run must + pass ``min_score`` — the safe default when several producer runs coalesce into one + consumer run. Set to ``False`` to gate on only the most recent triggering event instead. + """ + if not 0 <= min_score <= 1: + raise ValueError(f"min_score must be between 0 and 1, got {min_score!r}") + gate_task_id = task_id or f"require_quality_{asset.name}" + + @task.short_circuit(task_id=gate_task_id) + def _require_quality(**context: Any) -> bool: + return _quality_score_passes( + asset, min_score, context["triggering_asset_events"], require_all=require_all + ) + + return _require_quality() diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/__init__.py new file mode 100644 index 0000000000000..6008fc360bd41 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/__init__.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend + +__all__ = ["get_backend_from_config"] + + +def get_backend_from_config() -> ObjectStorageResultsBackend | None: + """Build the results backend configured under ``[common.dataquality]``, or ``None`` when not configured.""" + from airflow.providers.common.compat.sdk import conf + + results_path = conf.get("common.dataquality", "results_path", fallback=None) + if not results_path: + return None + + # Imported lazily: ObjectStoragePath pulls in fsspec/upath, which shouldn't be paid at + # Dag-parse time (this module is imported eagerly by the operator) when results_path unset. + from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend + + conn_id = conf.get("common.dataquality", "results_conn_id", fallback=None) + return ObjectStorageResultsBackend(results_path=results_path, conn_id=conn_id or None) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py new file mode 100644 index 0000000000000..34003bdda8a06 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/backends/object_storage.py @@ -0,0 +1,265 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Object-storage results backend. + +Each DQ check writes a keyed JSON document plus read indexes optimized for the UI: + + runs/by_task/dag_id=/task_id=/date=<2026-07-04>/__.json + Canonical run record: ``{"run": ..., "results": [...], "summary": ...}``. + + runs/by_task_instance/dag_id=/task_id=/__.json + Latest result for a task-instance page. Last write wins across retries. + + rules/by_task_rule/dag_id=/task_id=/rule_uid=/__.json + One rule result plus run context: ``{"run": ..., "result": ...}``. + +The duplicate files are intentional read indexes: DQ tasks write once, while the UI reads +many times. Keeping these indexes avoids scanning all task runs for common UI views. +""" + +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone +from typing import Any, TypedDict + +from airflow.providers.common.dataquality.results import DQRun, RuleResult, build_summary +from airflow.sdk import ObjectStoragePath + +log = logging.getLogger(__name__) + + +class DQPageResult(TypedDict): + """Paginated DQ read result.""" + + items: list[dict[str, Any]] + next_cursor: str | None + + +class ObjectStorageResultsBackend: + """Persist DQ results as JSON files via ``ObjectStoragePath``.""" + + def __init__(self, *, results_path: str, conn_id: str | None = None) -> None: + self.root = ObjectStoragePath(results_path, conn_id=conn_id) + + def write_run(self, *, run: DQRun, results: list[RuleResult]) -> None: + timestamp = run.started_at or datetime.now(tz=timezone.utc).isoformat() + compact_ts = self._get_safe_key(timestamp) + payload = self._build_run_payload(run=run, results=results) + + self._write_run_file(run=run, date_part=timestamp[:10], compact_ts=compact_ts, payload=payload) + self._write_task_instance_index(run=run, payload=payload) + self._write_rule_indexes(run=run, results=results, timestamp=timestamp) + + def read_task_rule_history( + self, *, dag_id: str, task_id: str, rule_uid: str, limit: int = 100, before: str | None = None + ) -> DQPageResult: + """Return recent results for one rule produced by one task, newest first.""" + rule_dir = ( + self.root + / "rules" + / "by_task_rule" + / f"dag_id={dag_id}" + / f"task_id={task_id}" + / f"rule_uid={rule_uid}" + ) + return self._read_rule_history_dir(rule_dir=rule_dir, limit=limit, before=before) + + def read_task_runs( + self, *, dag_id: str, task_id: str, limit: int = 50, before: str | None = None + ) -> DQPageResult: + """ + Return recent data quality runs for one task, newest first. + + Returns ``{"items": [...], "next_cursor": ...}``. ``before`` is the opaque + ``next_cursor`` from the previous page, so "load more" only reads runs it hasn't + shown yet. ``next_cursor`` is set by reading one extra entry past ``limit``: cheap + on every page except the last one, where there's no way to confirm history is + exhausted without walking to the end. + + ``date=`` partition names sort correctly as plain strings, so directories are walked + newest-first. Filenames are ``{started_at}__{run_uid}.json``, so sorting each partition's + filenames before scanning also walks newest-first within the partition -- required for + the early exit below to stop on the actual newest runs rather than an arbitrary subset. + Scanning stops as soon as ``limit + 1`` matching runs have been collected — a task with + years of history doesn't pay for a full scan on every page. + """ + task_dir = self.root / "runs" / "by_task" / f"dag_id={dag_id}" / f"task_id={task_id}" + if not task_dir.exists(): + return {"items": [], "next_cursor": None} + + date_dirs = sorted((path for path in task_dir.iterdir() if path.is_dir()), reverse=True) + runs = [] + for date_dir in date_dirs: + for path in sorted(date_dir.iterdir(), key=lambda p: p.name, reverse=True): + if not path.name.endswith(".json"): + continue + if (payload := self._read_json(path)) is None: + continue + if before is not None and self._get_run_payload_cursor(payload) >= before: + continue + runs.append(payload) + if len(runs) > limit: + break + if len(runs) > limit: + break + + ordered = sorted(runs, key=self._get_run_payload_cursor, reverse=True) + page = ordered[:limit] + next_cursor = self._get_run_payload_cursor(page[-1]) if len(ordered) > limit and page else None + return {"items": page, "next_cursor": next_cursor} + + def read_by_task_instance( + self, *, dag_id: str, task_id: str, run_id: str, map_index: int = -1 + ) -> dict[str, Any]: + """Read the latest run for one task instance as ``{"run": ..., "results": ..., "summary": ...}``.""" + path = ( + self.root + / "runs" + / "by_task_instance" + / f"dag_id={dag_id}" + / f"task_id={task_id}" + / f"{self._get_safe_key(run_id)}__{map_index}.json" + ) + return self._read_json_or_raise(path) + + def _write_run_file( + self, *, run: DQRun, date_part: str, compact_ts: str, payload: dict[str, Any] + ) -> None: + run_dir = ( + self.root + / "runs" + / "by_task" + / f"dag_id={run.dag_id}" + / f"task_id={run.task_id}" + / f"date={date_part}" + ) + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / f"{compact_ts}__{run.run_uid}.json").write_text(json.dumps(payload, default=str)) + + def _write_task_instance_index(self, *, run: DQRun, payload: dict[str, Any]) -> None: + ti_dir = self.root / "runs" / "by_task_instance" / f"dag_id={run.dag_id}" / f"task_id={run.task_id}" + ti_dir.mkdir(parents=True, exist_ok=True) + (ti_dir / f"{self._get_safe_key(run.run_id)}__{run.map_index}.json").write_text( + json.dumps(payload, default=str) + ) + + def _write_rule_indexes(self, *, run: DQRun, results: list[RuleResult], timestamp: str) -> None: + run_context = self._build_run_context(run) + compact_ts = self._get_safe_key(timestamp) + for result in results: + payload = {"run": run_context, "result": result.to_dict()} + self._write_rule_index( + rule_dir=( + self.root + / "rules" + / "by_task_rule" + / f"dag_id={run.dag_id}" + / f"task_id={run.task_id}" + / f"rule_uid={result.rule_uid}" + ), + compact_ts=compact_ts, + run_uid=run.run_uid, + payload=payload, + ) + + def _write_rule_index( + self, *, rule_dir: ObjectStoragePath, compact_ts: str, run_uid: str, payload: dict[str, Any] + ) -> None: + rule_dir.mkdir(parents=True, exist_ok=True) + (rule_dir / f"{compact_ts}__{run_uid}.json").write_text(json.dumps(payload, default=str)) + + def _read_rule_history_dir( + self, *, rule_dir: ObjectStoragePath, limit: int, before: str | None = None + ) -> DQPageResult: + """ + Read rule-result records newest-first, as ``{"items": [...], "next_cursor": ...}``. + + Reads one entry past ``limit`` to determine ``next_cursor``: cheap on every page + except the last one, where confirming there's nothing older means reading to the end. + """ + if not rule_dir.exists(): + return {"items": [], "next_cursor": None} + + history: list[dict[str, Any]] = [] + for path in sorted(rule_dir.iterdir(), key=lambda p: p.name, reverse=True): + payload = self._read_json(path) + if payload is None: + continue + record = {**payload["result"], "run": payload["run"]} + cursor = self._get_rule_history_cursor(record) + if before is not None and cursor >= before: + continue + history.append(record) + if len(history) > limit: + break + + page = history[:limit] + next_cursor = self._get_rule_history_cursor(page[-1]) if len(history) > limit and page else None + return {"items": page, "next_cursor": next_cursor} + + @staticmethod + def _get_safe_key(value: str) -> str: + """Sanitize a value (for example an Airflow ``run_id``) for an object key segment.""" + return value.replace("/", "_").replace(":", "_").replace("+", "_") + + @staticmethod + def _build_run_payload(*, run: DQRun, results: list[RuleResult]) -> dict[str, Any]: + result_records = [result.to_dict() for result in results] + return { + "run": run.to_dict(), + "results": result_records, + "summary": build_summary(run=run, results=results), + } + + @staticmethod + def _build_run_context(run: DQRun) -> dict[str, Any]: + return { + "run_uid": run.run_uid, + "dag_id": run.dag_id, + "task_id": run.task_id, + "run_id": run.run_id, + "map_index": run.map_index, + "started_at": run.started_at, + "table_ref": run.table_ref, + } + + @staticmethod + def _get_run_payload_cursor(payload: dict[str, Any]) -> str: + run = payload["run"] + return ObjectStorageResultsBackend._build_cursor(run.get("started_at"), run.get("run_uid")) + + @staticmethod + def _get_rule_history_cursor(record: dict[str, Any]) -> str: + run = record["run"] + return ObjectStorageResultsBackend._build_cursor(run.get("started_at"), run.get("run_uid")) + + @staticmethod + def _build_cursor(started_at: str | None, run_uid: str | None) -> str: + return f"{started_at or ''}|{run_uid or ''}" + + def _read_json_or_raise(self, path: ObjectStoragePath) -> dict[str, Any]: + return json.loads(path.read_text()) + + def _read_json(self, path: ObjectStoragePath) -> dict[str, Any] | None: + try: + return self._read_json_or_raise(path) + except (OSError, json.JSONDecodeError): + log.warning("Skipping unreadable DQ result file %s", path) + return None diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/__init__.py new file mode 100644 index 0000000000000..21d298ede6ed3 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/dq_check.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/dq_check.py new file mode 100644 index 0000000000000..ba4285c063fd1 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/decorators/dq_check.py @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""TaskFlow decorator for :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator`.""" + +from __future__ import annotations + +from collections.abc import Callable, Collection, Mapping, Sequence +from typing import TYPE_CHECKING, Any, ClassVar + +from airflow.providers.common.compat.sdk import ( + DecoratedOperator, + context_merge, + determine_kwargs, + task_decorator_factory, +) +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import RuleSet +from airflow.sdk.definitions._internal.types import SET_DURING_EXECUTION + +if TYPE_CHECKING: + from airflow.sdk import Context + from airflow.sdk.bases.decorator import TaskDecorator + + +class _DQCheckDecoratedOperator(DecoratedOperator, DQCheckOperator): + """ + Wraps a callable that optionally returns a runtime ruleset for a data quality check. + + :param python_callable: A reference to a callable returning ``None`` or a ruleset for this run. + :param op_args: Positional arguments for the callable. + :param op_kwargs: Keyword arguments for the callable. + """ + + template_fields: Sequence[str] = ( + *DecoratedOperator.template_fields, + *DQCheckOperator.template_fields, + ) + template_fields_renderers: ClassVar[dict[str, str]] = { + **DecoratedOperator.template_fields_renderers, + } + + custom_operator_name = "@task.dq_check" + + def __init__( + self, + *, + python_callable: Callable, + op_args: Collection[Any] | None = None, + op_kwargs: Mapping[str, Any] | None = None, + **kwargs, + ) -> None: + kwargs.setdefault("ruleset", SET_DURING_EXECUTION) + super().__init__(python_callable=python_callable, op_args=op_args, op_kwargs=op_kwargs, **kwargs) + + def execute(self, context: Context) -> Any: + context_merge(context, self.op_kwargs) + kwargs = determine_kwargs(self.python_callable, self.op_args, context) + ruleset = self.python_callable(*self.op_args, **kwargs) + if ruleset is not None: + if not isinstance(ruleset, (RuleSet, dict, str)): + raise TypeError( + f"{self.custom_operator_name} function must return a RuleSet, dict, path, or None, " + f"got {type(ruleset).__name__}" + ) + self.ruleset = ruleset + elif self.ruleset is SET_DURING_EXECUTION: + raise ValueError("ruleset is required, either directly or returned by @task.dq_check") + return DQCheckOperator.execute(self, context) + + +def dq_check_task( + python_callable: Callable | None = None, + **kwargs, +) -> TaskDecorator: + """ + Turn a function into a data quality check task. + + ``ruleset`` may be passed as a decorator argument when known at Dag-parse time, or returned + by the decorated function at runtime. ``table`` or ``asset`` are passed through ``kwargs`` + exactly as on :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator`. The + function itself is optional plumbing: return ``None`` to run the check as declared, or a + ruleset to use for this run. + + Usage:: + + @task.dq_check(conn_id="warehouse", ruleset=rules) + def orders_quality(ds=None): + return None + + :param python_callable: Function to decorate. + """ + return task_decorator_factory( + python_callable=python_callable, + decorated_operator_class=_DQCheckDecoratedOperator, + **kwargs, + ) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/__init__.py new file mode 100644 index 0000000000000..cf6a86506a954 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.common.dataquality.engines.sql import Observation, SQLDQEngine + +__all__ = ["Observation", "SQLDQEngine"] diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py new file mode 100644 index 0000000000000..abfb468c2ff17 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/engines/sql.py @@ -0,0 +1,159 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""SQL execution engine: compiles a ruleset into check queries run through a DB-API hook.""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from airflow.providers.common.dataquality.rules import CUSTOM_SQL_CHECK +from airflow.providers.common.dataquality.rules.checks import CHECK_SPECS + +if TYPE_CHECKING: + from airflow.providers.common.dataquality.rules import DQRule, RuleSet + from airflow.providers.common.sql.hooks.sql import DbApiHook + +log = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class Observation: + """Raw value a rule observed, before its condition is evaluated.""" + + rule: DQRule + observed_value: Any = None + duration_ms: float | None = None + error_message: str | None = None + sql: str | None = None + + +class SQLDQEngine: + """ + Runs built-in checks as a single UNION ALL query and custom SQL rules individually. + + Table, column, and partition-clause values come from the Dag author and are interpolated + into SQL the same way the ``common.sql`` check operators do — they are trusted input. + """ + + check_sql_template = "SELECT '{rule_uid}' AS rule_uid, {expression} AS observed FROM {table}{where}" + + def __init__(self, hook: DbApiHook) -> None: + self.hook = hook + + def measure( + self, *, ruleset: RuleSet, table: str, partition_clause: str | None = None + ) -> list[Observation]: + builtin_rules = [rule for rule in ruleset.rules if rule.check != CUSTOM_SQL_CHECK] + custom_rules = [rule for rule in ruleset.rules if rule.check == CUSTOM_SQL_CHECK] + + observations = [] + if builtin_rules: + observations.extend( + self._measure_builtin(rules=builtin_rules, table=table, partition_clause=partition_clause) + ) + for rule in custom_rules: + observations.append(self._measure_custom(rule=rule, table=table)) + return observations + + def build_batch_sql(self, *, rules: list[DQRule], table: str, partition_clause: str | None) -> str: + return " UNION ALL ".join( + self.build_rule_sql(rule=rule, table=table, partition_clause=partition_clause) for rule in rules + ) + + def build_rule_sql(self, *, rule: DQRule, table: str, partition_clause: str | None = None) -> str: + """Build the SQL used to measure one built-in rule.""" + expression = CHECK_SPECS[rule.check].expression.format(column=rule.column) + predicates = [p for p in (partition_clause, rule.partition_clause) if p] + where = f" WHERE {' AND '.join(predicates)}" if predicates else "" + return self.check_sql_template.format( + rule_uid=rule.rule_uid, expression=expression, table=table, where=where + ) + + def _measure_builtin( + self, *, rules: list[DQRule], table: str, partition_clause: str | None + ) -> list[Observation]: + # Built once per rule and reused below, instead of re-deriving each rule's SQL for the + # batch, for the returned Observation, and again in the per-rule fallback. + rule_sql = { + rule.rule_uid: self.build_rule_sql(rule=rule, table=table, partition_clause=partition_clause) + for rule in rules + } + sql = " UNION ALL ".join(rule_sql.values()) + log.info("Running %d built-in checks against %s", len(rules), table) + started = time.monotonic() + try: + records = self.hook.get_records(sql) + except Exception: + log.exception( + "Batched check query failed for table %s; falling back to one query execution per rule", + table, + ) + + return [self._measure_builtin_single(rule=rule, sql=rule_sql[rule.rule_uid]) for rule in rules] + elapsed_ms = (time.monotonic() - started) * 1000 + observed_by_uid = {str(row[0]): row[1] for row in records or []} + return [ + Observation( + rule=rule, + observed_value=observed_by_uid.get(rule.rule_uid), + duration_ms=elapsed_ms, + error_message=None if rule.rule_uid in observed_by_uid else "No result returned for rule", + sql=rule_sql[rule.rule_uid], + ) + for rule in rules + ] + + def _measure_builtin_single(self, *, rule: DQRule, sql: str) -> Observation: + started = time.monotonic() + try: + row = self.hook.get_first(sql) + except Exception as e: + elapsed_ms = (time.monotonic() - started) * 1000 + log.exception("Check query failed for rule %s", rule.name) + return Observation(rule=rule, duration_ms=elapsed_ms, error_message=str(e), sql=sql) + elapsed_ms = (time.monotonic() - started) * 1000 + if row is None: + return Observation( + rule=rule, duration_ms=elapsed_ms, error_message="No result returned for rule", sql=sql + ) + return Observation(rule=rule, observed_value=row[1], duration_ms=elapsed_ms, sql=sql) + + def _measure_custom(self, *, rule: DQRule, table: str) -> Observation: + if rule.sql is None: + raise ValueError(f"Rule {rule.name!r} has no SQL to execute") + sql = rule.sql.replace("{table}", table) + log.info("Running custom SQL check %s", rule.name) + started = time.monotonic() + try: + row = self.hook.get_first(sql) + except Exception as e: + log.exception("Custom SQL check %s failed", rule.name) + return Observation( + rule=rule, + duration_ms=(time.monotonic() - started) * 1000, + error_message=str(e), + sql=sql, + ) + elapsed_ms = (time.monotonic() - started) * 1000 + if row is None: + return Observation( + rule=rule, duration_ms=elapsed_ms, error_message="Query returned no rows", sql=sql + ) + return Observation(rule=rule, observed_value=row[0], duration_ms=elapsed_ms, sql=sql) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py new file mode 100644 index 0000000000000..e0b0086e5e938 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_custom_sql.py @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: ``custom_sql`` for checks the built-in catalog can't express. + +Built-in checks (``null_count``, ``min``, ``max``, ...) are all single-column. The moment a +rule needs to compare two columns, join against another table, or use a SQL function the +built-in catalog doesn't cover (or that your database's ``DbApiHook`` doesn't support -- see +"Supported checks and databases" in the provider docs), ``custom_sql`` is the escape hatch: +any statement that resolves to a single scalar, evaluated the same way as a built-in check. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet, Severity +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.sdk import DAG + +DAG_ID = "example_dq_check_custom_sql" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_custom_sql_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") + +# [START howto_operator_dq_check_custom_sql] +custom_sql_ruleset = RuleSet( + name="orders_custom_sql", + rules=( + DQRule( + name="shipped_after_ordered", + check="custom_sql", + # {table} is substituted by the SQL engine at check time, not an f-string + # placeholder -- a cross-column comparison no single-column built-in can express. + sql="SELECT COUNT(*) FROM {table} WHERE shipped_at < ordered_at", + condition=Condition(equal_to=0), + ), + DQRule( + name="high_value_order_ratio", + check="custom_sql", + sql=( + "SELECT CAST(SUM(CASE WHEN amount > 100 THEN 1 ELSE 0 END) AS REAL) / COUNT(*) FROM {table}" + ), + condition=Condition(leq_to=0.5), + severity=Severity.WARN, + ), + ), +) +# [END howto_operator_dq_check_custom_sql] + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f""" + CREATE TABLE {TABLE_NAME} ( + order_id INTEGER, + amount REAL, + ordered_at TEXT, + shipped_at TEXT + ); + """, + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount, ordered_at, shipped_at) VALUES + (1, 42.0, '2026-07-01', '2026-07-02'), + (2, 150.0, '2026-07-01', '2026-07-03'), + (3, 15.5, '2026-07-02', '2026-07-02'); + """, + ) + + check_orders = DQCheckOperator( + task_id="check_orders", + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=custom_sql_ruleset, + fail_on="never", + ) + + create_table >> insert_orders >> check_orders diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py new file mode 100644 index 0000000000000..bc92664b3aa1a --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_check_decorator_dynamic.py @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: ``@task.dq_check`` with a runtime ruleset. + +``table`` (or ``asset``) is declared as a decorator argument, exactly like the plain operator. +``ruleset`` may be declared at Dag-parse time, or returned by the decorated function at +execution time. This is useful when an upstream task, variable, or generated value decides which +ruleset to run. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.sdk import DAG, task + +DAG_ID = "example_dq_check_decorator_dynamic" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_dynamic_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") + +orders_ruleset = RuleSet( + name="orders_dynamic", + rules=( + DQRule( + name="order_id_not_null", check="null_count", column="order_id", condition=Condition(equal_to=0) + ), + DQRule(name="row_count_present", check="row_count", condition=Condition(greater_than=0)), + ), +) + +strict_ruleset = RuleSet( + name="orders_dynamic_strict", + rules=( + *orders_ruleset.rules, + DQRule(name="amount_min_ge_zero", check="min", column="amount", condition=Condition(geq_to=0)), + ), +) + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f"CREATE TABLE {TABLE_NAME} (order_id INTEGER, amount REAL, ds TEXT);", + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount, ds) VALUES + (1, 10.0, '{{{{ ds }}}}'), + (2, 25.5, '{{{{ ds }}}}'), + (3, 7.25, '2020-01-01'); + """, + ) + + # [START howto_decorator_dq_check_runtime_ruleset] + @task.dq_check(conn_id=CONN_ID, table=TABLE_NAME, ruleset=orders_ruleset, fail_on="never") + def check_with_ruleset_from_upstream(strict: bool = True): + """Swap in a stricter ruleset based on a signal only known at task-execution time.""" + return strict_ruleset if strict else None + + # [END howto_decorator_dq_check_runtime_ruleset] + + create_table >> insert_orders >> check_with_ruleset_from_upstream() diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py new file mode 100644 index 0000000000000..dcab9cfebedf8 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_in_taskflow.py @@ -0,0 +1,121 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example Dag: run data quality checks as part of a TaskFlow task. + +The task owns the surrounding Python flow -- it can persist DQ results, inspect the summary, +and decide what to return or raise -- while the rule execution still uses the same +``DbApiHook`` path as ``DQCheckOperator``. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.dataquality.execution import persist_quality_results, run_quality_checks +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.sdk import DAG, get_current_context, task + +DAG_ID = "example_dq_in_taskflow" +CONN_ID = "sqlite_default" +RAW_TABLE = "dq_taskflow_raw_orders" +READY_TABLE = "dq_taskflow_ready_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +# [START howto_rules_dq_ruleset] +orders_ruleset = RuleSet( + name="orders_taskflow_quality", + rules=( + DQRule( + name="order_id_not_null", + check="null_count", + column="order_id", + condition=Condition(equal_to=0), + ), + DQRule( + name="amount_min_ge_zero", + check="min", + column="amount", + condition=Condition(geq_to=0), + ), + DQRule( + name="row_count_present", + check="row_count", + condition=Condition(greater_than=0), + ), + ), +) +# [END howto_rules_dq_ruleset] + +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_tables = SQLExecuteQueryOperator( + task_id="create_tables", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {RAW_TABLE};", + f"DROP TABLE IF EXISTS {READY_TABLE};", + f"CREATE TABLE {RAW_TABLE} (order_id INTEGER, customer_id INTEGER, amount REAL);", + f"CREATE TABLE {READY_TABLE} (order_id INTEGER, customer_id INTEGER, amount REAL);", + ], + ) + + load_orders = SQLExecuteQueryOperator( + task_id="load_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {RAW_TABLE} (order_id, customer_id, amount) VALUES + (1, 101, 10.0), + (2, 102, 25.5), + (3, 103, 7.25); + """, + ) + + # [START howto_run_dq_inside_task] + @task + def validate_and_choose_source() -> str: + result = run_quality_checks( + conn_id=CONN_ID, + table=RAW_TABLE, + ruleset=orders_ruleset, + ) + summary = persist_quality_results(result, context=get_current_context()) + + if summary["failed"] or summary["errored"]: + raise ValueError(f"Order quality checks failed: {summary}") + + return RAW_TABLE + + # [END howto_run_dq_inside_task] + + publish_orders = SQLExecuteQueryOperator( + task_id="publish_orders", + conn_id=CONN_ID, + sql=f"INSERT INTO {READY_TABLE} SELECT * FROM {{{{ ti.xcom_pull(task_ids='validate_and_choose_source') }}}};", + ) + + create_tables >> load_orders >> validate_and_choose_source() >> publish_orders diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py new file mode 100644 index 0000000000000..5f4ee75d361bd --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_llm_generated_ruleset.py @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: generate a ``RuleSet`` with an LLM, guided by the ``dataquality-rule-authoring`` skill. + +Requires the optional ``apache-airflow-providers-common-ai[skills]`` package and a configured +LLM connection (``llm_conn_id``); this Dag is not registered when common.ai isn't installed. + +An LLM is asked to propose data quality rules for a table's columns. Its ``system_prompt`` +points it at the ``dataquality-rule-authoring`` skill shipped with this provider (via +``AgentSkillsToolset``), so it knows the exact ``RuleSet``/``DQRule`` schema, the built-in check +catalog, and the ``Condition`` grammar, instead of guessing at field or check names. +``output_type=RuleSet`` makes pydantic-ai validate -- and self-correct -- the model's output +against the real ``RuleSet`` model before the task completes, so a malformed rule never reaches +``DQCheckOperator`` in the first place. + +The generated ``RuleSet`` is then returned by ``@task.dq_check`` at runtime, since the real +ruleset doesn't exist until the LLM task runs. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.dataquality.rules import RuleSet +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.sdk import DAG, task + +try: + from airflow.providers.common.ai.toolsets.skills import AgentSkillsToolset +except ImportError: + AgentSkillsToolset = None # type: ignore[assignment,misc] + +DAG_ID = "example_dq_llm_generated_ruleset" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_llm_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") +# The skill ships next to this Dag's package, not next to this file -- resolve relative to +# __file__ so the path holds regardless of the Dag processor's working directory. +SKILLS_DIR = Path(__file__).parent.parent / "skills" / "dataquality-rule-authoring" + +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") + +if AgentSkillsToolset is not None: + with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq", "ai"], + ) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f""" + CREATE TABLE {TABLE_NAME} ( + order_id INTEGER, + customer_id INTEGER, + amount REAL, + region TEXT, + created_at TEXT + ); + """, + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, region, created_at) VALUES + (1, 101, 10.0, 'US', '2026-07-01'), + (2, 102, 25.5, 'EU', '2026-07-02'), + (3, NULL, 7.25, 'US', '2026-07-03'); + """, + ) + + # [START howto_task_dq_generate_ruleset_with_llm] + @task.llm( + llm_conn_id="pydanticai_default", + system_prompt=( + "You are a data quality engineer. Before answering, consult the " + "dataquality-rule-authoring skill for the exact RuleSet/DQRule schema, the built-in " + "check catalog, and the Condition grammar -- do not invent field or check names." + ), + output_type=RuleSet, + agent_params={"toolsets": [AgentSkillsToolset(sources=[str(SKILLS_DIR)])]}, + ) + def generate_ruleset(): + return ( + "Generate a RuleSet named 'orders_llm_quality' for a table with these columns: " + "order_id (integer, primary key), customer_id (integer, nullable foreign key), " + "amount (float, must be non-negative), region (short string, low-cardinality), " + "created_at (date). Include at least a not-null check on order_id, a uniqueness " + "check on order_id, and a row-count check." + ) + + # [END howto_task_dq_generate_ruleset_with_llm] + + # [START howto_decorator_dq_check_llm_runtime_ruleset] + @task.dq_check(conn_id=CONN_ID, table=TABLE_NAME, fail_on="never") + def check_with_llm_ruleset(ruleset: RuleSet): + return ruleset + + # [END howto_decorator_dq_check_llm_runtime_ruleset] + + generated_ruleset = generate_ruleset() + checked = check_with_llm_ruleset(generated_ruleset) + + create_table >> insert_orders >> generated_ruleset >> checked diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py new file mode 100644 index 0000000000000..2b9c93c661489 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_require_quality.py @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAGs: gate a consumer Dag on the data quality score of the asset that triggers it. + +Two Dags, wired together only through the ``dq_gated_orders`` asset: + +- ``example_dq_require_quality_producer`` checks a table and produces the asset. + :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` attaches its summary + (including the quality ``score``) to the asset event automatically because the asset carries + quality config attached with :func:`~airflow.providers.common.dataquality.assets.asset_quality`. +- ``example_dq_require_quality_consumer`` is scheduled by that asset. Its first task, built by + :func:`~airflow.providers.common.dataquality.assets.require_quality`, reads the score off the triggering + asset event and short-circuits -- skipping every downstream task -- when it's missing or + below ``min_score``. Downstream tasks never see a run triggered by a bad batch of data. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.dataquality.assets import asset_quality, require_quality +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.sdk import DAG, Asset, task + +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_gated_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") + +# [START howto_asset_quality] +gated_orders_ruleset = RuleSet( + name="gated_orders_quality", + rules=( + DQRule( + name="order_id_not_null", + check="null_count", + column="order_id", + condition=Condition(equal_to=0), + id="order_id_not_null", # explicit rule_uid, stable across future renames + ), + DQRule(name="amount_min_ge_zero", check="min", column="amount", condition=Condition(geq_to=0)), + DQRule(name="row_count_present", check="row_count", condition=Condition(greater_than=0)), + ), +) + +gated_orders_asset = asset_quality( + Asset("dq_gated_orders", uri="file:///tmp/airflow_dq_example/gated_orders"), + ruleset=gated_orders_ruleset, + conn_id=CONN_ID, + table=TABLE_NAME, +) +# [END howto_asset_quality] + +with DAG( + dag_id="example_dq_require_quality_producer", + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as producer_dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f"CREATE TABLE {TABLE_NAME} (order_id INTEGER, amount REAL);", + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount) VALUES + (1, 10.0), + (2, 25.5), + (3, 7.25); + """, + ) + + # Producing check: no explicit outlets needed, DQCheckOperator adds the asset itself + # because it was passed via asset=. + check_orders = DQCheckOperator( + task_id="check_orders", + asset=gated_orders_asset, + fail_on="never", + ) + + create_table >> insert_orders >> check_orders + +# [START howto_require_quality] +with DAG( + dag_id="example_dq_require_quality_consumer", + schedule=gated_orders_asset, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as consumer_dag: + gate = require_quality(gated_orders_asset, min_score=0.95) + + @task + def process_orders(): + print("Processing orders -- quality gate passed.") + + gate >> process_orders() +# [END howto_require_quality] diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py new file mode 100644 index 0000000000000..a1bee5a2431b9 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/example_dq_ruleset_from_yaml.py @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG: load a ruleset from a YAML file instead of declaring it in Python. + +A path string is accepted anywhere a :class:`~airflow.providers.common.dataquality.rules.RuleSet` is -- +``DQCheckOperator(ruleset=...)``, ``@task.dq_check(ruleset=...)``, and +:func:`~airflow.providers.common.dataquality.assets.asset_quality` all resolve it via +:meth:`~airflow.providers.common.dataquality.rules.RuleSet.from_file` at Dag-parse time. This keeps rules +editable by people who don't write Python -- a data steward can change ``orders_ruleset.yaml`` +without touching the Dag file. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.sdk import DAG + +DAG_ID = "example_dq_ruleset_from_yaml" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_yaml_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") +RULESET_FILE = Path(__file__).parent / "orders_ruleset.yaml" + +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f"CREATE TABLE {TABLE_NAME} (order_id INTEGER, amount REAL);", + ], + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + INSERT INTO {TABLE_NAME} (order_id, amount) VALUES + (1, 10.0), + (2, 25.5), + (3, 7.25); + """, + ) + + # [START howto_operator_dq_check_ruleset_from_yaml] + check_orders = DQCheckOperator( + task_id="check_orders", + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=str(RULESET_FILE), + fail_on="never", + ) + # [END howto_operator_dq_check_ruleset_from_yaml] + + create_table >> insert_orders >> check_orders diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/orders_ruleset.yaml b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/orders_ruleset.yaml new file mode 100644 index 0000000000000..049e69543bc32 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/example_dags/orders_ruleset.yaml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- + +name: orders_from_yaml +rules: + - name: order_id_not_null + check: null_count + column: order_id + condition: + equal_to: 0 + dimension: completeness + - name: amount_min_ge_zero + check: min + column: amount + condition: + geq_to: 0 + - name: row_count_present + check: row_count + condition: + greater_than: 0 + severity: warn diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/exceptions.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/exceptions.py new file mode 100644 index 0000000000000..9350308d46501 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/exceptions.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + + +class DQRuleValidationError(ValueError): + """Raised when a rule or ruleset definition is invalid.""" + + +class DQCheckFailedError(RuntimeError): + """Raised when a data quality check fails at or above the operator's ``fail_on`` severity.""" diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/execution.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/execution.py new file mode 100644 index 0000000000000..4b4b01d7613b2 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/execution.py @@ -0,0 +1,240 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Helpers for running data quality rules from operators or custom Python tasks.""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, cast + +from airflow.providers.common.compat.sdk import BaseHook +from airflow.providers.common.dataquality.assets import DQ_RESULT_EXTRA_KEY +from airflow.providers.common.dataquality.backends import get_backend_from_config +from airflow.providers.common.dataquality.engines.sql import SQLDQEngine +from airflow.providers.common.dataquality.results import ( + ERROR, + FAIL, + PASS, + WARN, + DQRun, + RuleResult, + build_summary, +) +from airflow.providers.common.dataquality.rules import RuleSet, describe_rule + +if TYPE_CHECKING: + from collections.abc import Sequence + + from airflow.providers.common.dataquality.engines.sql import Observation + from airflow.providers.common.dataquality.rules.rule import Condition, Dimension + from airflow.providers.common.sql.hooks.sql import DbApiHook + from airflow.sdk import Asset, Context + +log = logging.getLogger(__name__) + +RulesetArg = RuleSet | dict[str, Any] | str + + +@dataclass(frozen=True) +class DataQualityResult: + """Evaluated data quality results before Airflow task metadata is attached.""" + + ruleset: RuleSet + table: str + results: tuple[RuleResult, ...] + started_at: str + finished_at: str + + def to_dict(self) -> dict[str, Any]: + return { + "ruleset": self.ruleset.name, + "table": self.table, + "started_at": self.started_at, + "finished_at": self.finished_at, + "results": [result.to_dict() for result in self.results], + } + + +def run_quality_checks( + *, + ruleset: RulesetArg, + table: str, + conn_id: str | None = None, + hook: DbApiHook | None = None, + hook_params: dict[str, Any] | None = None, + partition_clause: str | None = None, +) -> DataQualityResult: + """ + Run data quality rules through a ``DbApiHook``-compatible connection. + + Use this from custom Python tasks when the standard + :class:`~airflow.providers.common.dataquality.operators.dq_check.DQCheckOperator` + does not fit the task shape. + """ + resolved_ruleset = _resolve_ruleset(ruleset) + resolved_hook = hook if hook is not None else _get_hook(conn_id, hook_params) + started_at = datetime.now(tz=timezone.utc).isoformat() + observations = SQLDQEngine(resolved_hook).measure( + ruleset=resolved_ruleset, + table=table, + partition_clause=partition_clause, + ) + finished_at = datetime.now(tz=timezone.utc).isoformat() + return DataQualityResult( + ruleset=resolved_ruleset, + table=table, + results=tuple(_evaluate_observation(observation) for observation in observations), + started_at=started_at, + finished_at=finished_at, + ) + + +def persist_quality_results( + result: DataQualityResult, + *, + context: Context, + outlets: Sequence[Asset] | None = None, +) -> dict[str, Any]: + """ + Persist data quality results for the current task and return the run summary. + + When ``[common.dataquality] results_path`` is not configured, this still returns + the same summary and skips writing history. When ``outlets`` are supplied, the + summary is also attached to outlet asset events for ``require_quality()``. + """ + run = _build_run_from_context( + context=context, + ruleset=result.ruleset, + table=result.table, + outlets=outlets or (), + started_at=result.started_at, + finished_at=result.finished_at, + ) + results = list(result.results) + summary = build_summary(run=run, results=results) + + backend = get_backend_from_config() + if backend is None: + log.info("No [common.dataquality] results_path configured; skipping results persistence") + else: + # Persistence is best-effort: an unreachable results store leaves a gap in + # history but must not change the outcome of the check itself. + try: + backend.write_run(run=run, results=results) + except Exception: + log.exception("Failed to persist data quality results; continuing") + + if outlets: + _attach_to_outlet_events(context, outlets, summary) + + return summary + + +def _get_hook(conn_id: str | None, hook_params: dict[str, Any] | None) -> DbApiHook: + if conn_id is None: + raise ValueError("Either conn_id or hook is required") + connection = BaseHook.get_connection(conn_id) + return connection.get_hook(hook_params=hook_params) + + +def _resolve_ruleset(ruleset: RulesetArg) -> RuleSet: + if isinstance(ruleset, RuleSet): + return ruleset + if isinstance(ruleset, dict): + return RuleSet.from_dict(ruleset) + return RuleSet.from_file(ruleset) + + +def _evaluate_observation(observation: Observation) -> RuleResult: + rule = observation.rule + condition = cast("Condition", rule.condition) + dimension = cast("Dimension", rule.dimension) + error_message = observation.error_message + if error_message is None: + try: + passed = condition.evaluate(observation.observed_value) + except (TypeError, ValueError) as e: + passed = False + error_message = f"Could not evaluate observed value {observation.observed_value!r}: {e}" + else: + passed = False + + if error_message is not None: + status = ERROR + elif passed: + status = PASS + else: + status = WARN if rule.severity == "warn" else FAIL + observed = observation.observed_value + if observed is not None and not isinstance(observed, int | float | str): + observed = str(observed) + return RuleResult( + rule_uid=rule.rule_uid, + rule_name=rule.name, + status=status, + observed_value=observed, + condition=condition.to_dict(), + dimension=dimension.value, + severity=rule.severity.value, + duration_ms=observation.duration_ms, + error_message=error_message, + description=rule.description or describe_rule(rule), + sql=observation.sql, + ) + + +def _build_run_from_context( + *, + context: Context, + ruleset: RuleSet, + table: str, + outlets: Sequence[Asset], + started_at: str, + finished_at: str, +) -> DQRun: + ti = context["ti"] + return DQRun( + dag_id=ti.dag_id, + task_id=ti.task_id, + run_id=ti.run_id, + try_number=ti.try_number, + map_index=ti.map_index if ti.map_index is not None else -1, + ruleset_name=ruleset.name, + table_ref=table, + asset_names=tuple(_get_outlet_asset_names(outlets)), + started_at=started_at, + finished_at=finished_at, + ) + + +def _get_outlet_asset_names(outlets: Sequence[Asset]) -> list[str]: + return [asset.name for asset in outlets if getattr(asset, "name", None)] + + +def _attach_to_outlet_events(context: Context, outlets: Sequence[Asset], summary: dict[str, Any]) -> None: + try: + outlet_events = context["outlet_events"] + except KeyError: + return + for outlet in outlets: + if getattr(outlet, "name", None): + try: + outlet_events[outlet].extra[DQ_RESULT_EXTRA_KEY] = summary + except Exception: + log.warning("Could not attach data quality summary to outlet event for %s", outlet) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py index 961705e59b515..756efc23b8549 100644 --- a/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/get_provider_info.py @@ -25,5 +25,46 @@ def get_provider_info(): return { "package-name": "apache-airflow-providers-common-dataquality", "name": "Common Data Quality", - "description": "Common Data Quality Provider\n", + "description": "``Data Quality Provider``\n\nDeclarative data quality rules with durable, per-rule execution history.\nChecks run through ``common.sql`` DB-API hooks; results are persisted to a\nconfigurable results store (object storage or local files) so task, run,\nand rule-level quality can be inspected over time.\n", + "integrations": [ + { + "integration-name": "Common Data Quality", + "external-doc-url": "https://airflow.apache.org/docs/apache-airflow-providers-common-dataquality/", + "how-to-guide": ["/docs/apache-airflow-providers-common-dataquality/operators.rst"], + "tags": ["software"], + } + ], + "operators": [ + { + "integration-name": "Common Data Quality", + "python-modules": ["airflow.providers.common.dataquality.operators.dq_check"], + } + ], + "task-decorators": [ + { + "class-name": "airflow.providers.common.dataquality.decorators.dq_check.dq_check_task", + "name": "dq_check", + } + ], + "config": { + "common.dataquality": { + "description": "Configuration for the Data Quality provider results store.\n", + "options": { + "results_path": { + "description": "Any fsspec-compatible URL understood by ``ObjectStoragePath`` where data quality\nresults are persisted, e.g. ``s3://bucket/airflow-dq`` or ``file:///opt/airflow/dq``.\nWhen unset, checks still run but no history is persisted.\n", + "version_added": "0.1.0", + "type": "string", + "example": "s3://data-platform/airflow-dq", + "default": None, + }, + "results_conn_id": { + "description": "Optional Airflow connection used to access ``results_path``.\n", + "version_added": "0.1.0", + "type": "string", + "example": "aws_default", + "default": None, + }, + }, + } + }, } diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/__init__.py new file mode 100644 index 0000000000000..048c2ff752049 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator + +__all__ = ["DQCheckOperator"] diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py new file mode 100644 index 0000000000000..fa2a7b5f28c1f --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/operators/dq_check.py @@ -0,0 +1,158 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any + +from airflow.providers.common.dataquality.assets import get_asset_quality_config +from airflow.providers.common.dataquality.exceptions import DQCheckFailedError +from airflow.providers.common.dataquality.execution import ( + DataQualityResult, + _resolve_ruleset, + persist_quality_results, + run_quality_checks, +) +from airflow.providers.common.dataquality.results import ( + ERROR, + FAIL, + WARN, + RuleResult, +) +from airflow.providers.common.sql.operators.sql import BaseSQLOperator + +if TYPE_CHECKING: + from airflow.providers.common.dataquality.rules import RuleSet + from airflow.sdk import Asset, Context + +log = logging.getLogger(__name__) + +FAIL_ON_CHOICES = ("error", "warn", "never") + + +class DQCheckOperator(BaseSQLOperator): + """ + Run a ruleset against a table and persist per-rule results to the configured results store. + + Every rule is evaluated and recorded regardless of the task outcome. Execution errors + (a check query failing) always fail the task; rule failures fail the task according to + ``fail_on``. + + :param ruleset: A :class:`~airflow.providers.common.dataquality.rules.RuleSet`, its dict form, or a path + to a YAML ruleset file. Optional when ``asset`` carries one. + :param table: The table to run checks against. Optional when ``asset`` carries one + (falling back to the asset name). + :param asset: An asset decorated with :func:`~airflow.providers.common.dataquality.assets.asset_quality`. + Supplies defaults for ``ruleset``, ``table``, and ``conn_id`` (explicit arguments + win) and is automatically added to the task's outlets so its asset events carry + the check summary. + :param partition_clause: Predicate ANDed into every built-in check's WHERE clause, + e.g. ``"ds = '{{ ds }}'"``. + :param fail_on: Severity that fails the task: ``error`` (default — warn-severity rule + failures are recorded but don't fail), ``warn`` (any rule failure fails), or + ``never`` (only execution errors fail). + :param conn_id: Connection to the database to check (any ``DbApiHook``-compatible type). + :param database: Optional database/schema overriding the connection's default. + + Results are persisted to the backend configured under ``[common.dataquality] results_path``; when that's + unset, checks still run but no history is persisted. There is no per-operator override -- + all checks in a deployment share one results store. + """ + + template_fields: Sequence[str] = ("table", "partition_clause", *BaseSQLOperator.template_fields) + ui_color: str = "#87ceeb" + + def __init__( + self, + *, + ruleset: RuleSet | dict[str, Any] | str | None = None, + table: str | None = None, + asset: Asset | None = None, + partition_clause: str | None = None, + fail_on: str = "error", + **kwargs, + ) -> None: + if asset is not None: + config = get_asset_quality_config(asset) or {} + ruleset = ruleset if ruleset is not None else config.get("ruleset") + table = table or config.get("table") or asset.name + asset_conn_id = config.get("conn_id") + if asset_conn_id is not None: + kwargs.setdefault("conn_id", asset_conn_id) + outlets = list(kwargs.get("outlets") or []) + if asset not in outlets: + outlets.append(asset) + kwargs["outlets"] = outlets + super().__init__(**kwargs) + if fail_on not in FAIL_ON_CHOICES: + raise ValueError(f"fail_on must be one of {FAIL_ON_CHOICES}, got {fail_on!r}") + if ruleset is None: + raise ValueError("ruleset is required, either directly or via an asset_quality() asset") + if not table: + raise ValueError("table is required, either directly or via an asset_quality() asset") + if not self.conn_id: + raise ValueError("conn_id is required, either directly or via an asset_quality() asset") + self.ruleset = ruleset + self.table = table + self.asset = asset + self.partition_clause = partition_clause + self.fail_on = fail_on + + def execute(self, context: Context) -> dict[str, Any]: + result = run_quality_checks( + hook=self.get_db_hook(), + ruleset=_resolve_ruleset(self.ruleset), + table=self.table, + partition_clause=self.partition_clause, + ) + results = list(result.results) + summary = self._persist_result(result, context) + self._log_results(results, summary) + self._raise_for_failures(results, summary) + return summary + + def _persist_result(self, result: DataQualityResult, context: Context) -> dict[str, Any]: + return persist_quality_results(result, context=context, outlets=list(self.outlets or ())) + + def _log_results(self, results: list[RuleResult], summary: dict[str, Any]) -> None: + for result in results: + self.log.info( + "Rule %s [%s/%s]: %s (observed=%s, condition=%s)", + result.rule_name, + result.dimension, + result.severity, + result.status.upper(), + result.observed_value, + result.condition, + ) + self.log.info("Data quality score: %s", summary["score"]) + + def _raise_for_failures(self, results: list[RuleResult], summary: dict[str, Any]) -> None: + errored = [r.rule_name for r in results if r.status == ERROR] + if errored: + raise DQCheckFailedError(f"Check execution errored for rules: {errored}") + failed = [r.rule_name for r in results if r.status == FAIL] + warned = [r.rule_name for r in results if r.status == WARN] + if self.fail_on == "error" and failed: + raise DQCheckFailedError(f"Data quality rules failed: {failed} (score={summary['score']})") + if self.fail_on == "warn" and (failed or warned): + raise DQCheckFailedError( + f"Data quality rules failed: {failed + warned} (score={summary['score']})" + ) + if failed or warned: + self.log.warning("Rule failures below fail_on=%s threshold: %s", self.fail_on, failed + warned) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/results.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/results.py new file mode 100644 index 0000000000000..256b147e04fac --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/results.py @@ -0,0 +1,116 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Result records produced by a data quality check run — immutable facts, JSON-serializable.""" + +from __future__ import annotations + +import uuid +from dataclasses import asdict, dataclass, field +from typing import Any + +PASS = "pass" +WARN = "warn" +FAIL = "fail" +ERROR = "error" + +# Weight of a warn-severity failure in the run score, relative to an error-severity failure. +WARN_SCORE_WEIGHT = 0.25 + + +@dataclass(frozen=True) +class RuleResult: + """Outcome of evaluating one rule in one run.""" + + rule_uid: str + rule_name: str + status: str # pass | warn | fail | error + observed_value: float | str | None = None + condition: dict[str, Any] = field(default_factory=dict) + dimension: str = "validity" + severity: str = "error" + duration_ms: float | None = None + error_message: str | None = None + description: str | None = None + sql: str | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> RuleResult: + return cls(**data) + + +@dataclass(frozen=True) +class DQRun: + """One execution of a ruleset by one task instance.""" + + dag_id: str + task_id: str + run_id: str + try_number: int = 1 + map_index: int = -1 + run_uid: str = field(default_factory=lambda: uuid.uuid4().hex) + ruleset_name: str | None = None + table_ref: str | None = None + asset_names: tuple[str, ...] = () + started_at: str | None = None + finished_at: str | None = None + + def to_dict(self) -> dict[str, Any]: + data = asdict(self) + data["asset_names"] = list(self.asset_names) + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> DQRun: + data = {**data, "asset_names": tuple(data.get("asset_names", ()))} + return cls(**data) + + +def compute_score(results: list[RuleResult]) -> float | None: + """ + Weighted pass rate for a run in [0, 1]. + + Error-severity failures (and execution errors) count fully against the score; + warn-severity failures count at ``WARN_SCORE_WEIGHT``. + """ + if not results: + return None + penalty = 0.0 + for result in results: + if result.status in (FAIL, ERROR): + penalty += 1.0 + elif result.status == WARN: + penalty += WARN_SCORE_WEIGHT + return round(1.0 - penalty / len(results), 4) + + +def build_summary(*, run: DQRun, results: list[RuleResult]) -> dict[str, Any]: + """Compact run summary attached to XCom and outlet asset events.""" + return { + "run_uid": run.run_uid, + "ruleset": run.ruleset_name, + "table": run.table_ref, + "score": compute_score(results), + "passed": sum(1 for r in results if r.status == PASS), + "warned": sum(1 for r in results if r.status == WARN), + "failed": sum(1 for r in results if r.status == FAIL), + "errored": sum(1 for r in results if r.status == ERROR), + "failed_rules": sorted(r.rule_name for r in results if r.status in (FAIL, ERROR)), + "warned_rules": sorted(r.rule_name for r in results if r.status == WARN), + } diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/__init__.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/__init__.py new file mode 100644 index 0000000000000..e50c9a99f43b9 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/__init__.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.common.dataquality.rules.checks import CHECK_SPECS, CheckSpec, Dimension +from airflow.providers.common.dataquality.rules.rule import ( + CUSTOM_SQL_CHECK, + Condition, + DQRule, + RuleSet, + Severity, + describe_rule, +) + +__all__ = [ + "CHECK_SPECS", + "CUSTOM_SQL_CHECK", + "CheckSpec", + "Condition", + "DQRule", + "Dimension", + "RuleSet", + "Severity", + "describe_rule", +] diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/checks.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/checks.py new file mode 100644 index 0000000000000..11e73a5208f30 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/checks.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Built-in check catalog: one place for each check's SQL expression and metadata.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any + + +class Dimension(str, Enum): + """Data quality dimension assigned to a rule result.""" + + COMPLETENESS = "completeness" + UNIQUENESS = "uniqueness" + VALIDITY = "validity" + FRESHNESS = "freshness" + VOLUME = "volume" + CONSISTENCY = "consistency" + + +@dataclass(frozen=True) +class CheckSpec: + """ + Metadata for one built-in check. + + :param expression: SQL expression template. Column-level checks are rendered with + ``{column}``; table-level checks (``requires_column=False``) ignore it. + :param dimension: Default value for ``DQRule.dimension`` when a rule doesn't set it + explicitly. ``DQRule.dimension`` is a settable field; this is only the fallback. + :param requires_column: Whether a rule using this check must set ``column``. + :param default_condition: Default ``DQRule.condition`` (as a dict) for a rule that doesn't + set one explicitly; ``None`` means the rule must always specify a condition. + """ + + expression: str + dimension: Dimension = Dimension.VALIDITY + requires_column: bool = True + default_condition: dict[str, Any] | None = None + + +CHECK_SPECS: dict[str, CheckSpec] = { + "null_count": CheckSpec( + expression="SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END)", + dimension=Dimension.COMPLETENESS, + ), + "null_ratio": CheckSpec( + expression="SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*)", + dimension=Dimension.COMPLETENESS, + ), + "distinct_count": CheckSpec( + expression="COUNT(DISTINCT {column})", + dimension=Dimension.UNIQUENESS, + ), + "unique_violations": CheckSpec( + expression="COUNT({column}) - COUNT(DISTINCT {column})", + dimension=Dimension.UNIQUENESS, + ), + "min": CheckSpec(expression="MIN({column})"), + "max": CheckSpec(expression="MAX({column})"), + "mean": CheckSpec(expression="AVG({column})"), + "row_count": CheckSpec( + expression="COUNT(*)", + dimension=Dimension.VOLUME, + requires_column=False, + ), +} diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py new file mode 100644 index 0000000000000..6efa97faf423c --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/rules/rule.py @@ -0,0 +1,314 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Declarative data quality rule model: rules are data, not code.""" + +from __future__ import annotations + +import hashlib +import json +from enum import Enum +from typing import Any, cast + +import yaml +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from airflow.providers.common.dataquality.exceptions import DQRuleValidationError +from airflow.providers.common.dataquality.rules.checks import CHECK_SPECS, Dimension + +CUSTOM_SQL_CHECK = "custom_sql" + + +class Severity(str, Enum): + """Severity used to decide whether a failing rule fails the task.""" + + WARN = "warn" + ERROR = "error" + + +class Condition(BaseModel): + """ + Pass/fail condition evaluated against a rule's observed value. + + Uses the same grammar as the ``common.sql`` check operators: ``equal_to``, + ``greater_than``, ``less_than``, ``geq_to``, ``leq_to``, plus a percentage + ``tolerance`` that widens comparisons. + + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + equal_to: float | None = None + greater_than: float | None = None + less_than: float | None = None + geq_to: float | None = None + leq_to: float | None = None + tolerance: float | None = None + + @model_validator(mode="after") + def _validate_comparisons(self) -> Condition: + comparisons = { + "equal_to": self.equal_to, + "greater_than": self.greater_than, + "less_than": self.less_than, + "geq_to": self.geq_to, + "leq_to": self.leq_to, + } + set_comparisons = {name for name, value in comparisons.items() if value is not None} + if not set_comparisons: + raise ValueError(f"Condition needs at least one comparison out of: {', '.join(comparisons)}") + if self.equal_to is not None and len(set_comparisons) > 1: + raise ValueError("equal_to cannot be combined with other comparisons") + return self + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> Condition: + return cls(**data) + + def to_dict(self) -> dict[str, float]: + data = { + "equal_to": self.equal_to, + "greater_than": self.greater_than, + "less_than": self.less_than, + "geq_to": self.geq_to, + "leq_to": self.leq_to, + "tolerance": self.tolerance, + } + return {k: v for k, v in data.items() if v is not None} + + def evaluate(self, observed: Any) -> bool: + if observed is None: + return False + value = float(observed) + if self.equal_to is not None: + if self.tolerance is not None: + delta = abs(self.equal_to) * self.tolerance + return self.equal_to - delta <= value <= self.equal_to + delta + return value == self.equal_to + geq_to = self.geq_to + greater_than = self.greater_than + leq_to = self.leq_to + less_than = self.less_than + if self.tolerance is not None: + if geq_to is not None: + geq_to -= abs(geq_to) * self.tolerance + if greater_than is not None: + greater_than -= abs(greater_than) * self.tolerance + if leq_to is not None: + leq_to += abs(leq_to) * self.tolerance + if less_than is not None: + less_than += abs(less_than) * self.tolerance + return ( + (greater_than is None or value > greater_than) + and (less_than is None or value < less_than) + and (geq_to is None or value >= geq_to) + and (leq_to is None or value <= leq_to) + ) + + +class DQRule(BaseModel): + """ + A single, named data quality rule. + + :param name: Rule name, unique within its ruleset. + :param check: One of the built-in checks (see ``CHECK_SPECS``) or ``custom_sql``. Built-in + checks are plain ANSI SQL and are not guaranteed to work against every ``DbApiHook``; + see "Supported checks and databases" in the provider docs. Use ``custom_sql`` if a + built-in check doesn't fit your database's dialect. + :param condition: Pass condition for the observed value (``Condition`` or its dict form). + Optional only for checks whose ``CheckSpec.default_condition`` is set; every current + built-in check requires one explicitly. + :param column: Target column; required for column-level built-in checks. + :param sql: SQL statement returning a single scalar; required for ``custom_sql``. + May reference the target table as ``{table}``. + :param severity: ``error`` (fails the task by default) or ``warn`` (recorded only). + :param partition_clause: Extra predicate ANDed into the check's WHERE clause. + :param previous_name: Set when renaming a rule, to keep its history continuous. A rule whose + ``previous_name`` happens to match another rule's identity (same check/column/condition) + collides on ``rule_uid``; set an explicit ``id`` on one of them to avoid this. + :param description: Human-readable description shown in results and the UI. When omitted, + the provider generates a short default description from the rule and condition. + :param id: Explicit, stable identity for this rule's history. When set, it is used directly + as the ``rule_uid`` instead of the derived hash, so it survives ``previous_name`` chains + and sidesteps any collision between rules that would otherwise hash the same. + + Invalid input raises pydantic's own :class:`~pydantic.ValidationError`. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str + check: str = Field(description="One of the built-in checks, or custom_sql.") + condition: Condition | None = None + column: str | None = None + sql: str | None = None + severity: Severity = Severity.ERROR + partition_clause: str | None = None + previous_name: str | None = None + id: str | None = Field( + default=None, + description=( + "Explicit, stable identity for this rule's history, used directly as rule_uid " + "instead of a derived hash." + ), + ) + description: str | None = Field( + default=None, + description=( + "Human-readable description shown in results and the UI. When omitted, the provider " + "generates a short default description from the rule and condition." + ), + ) + dimension: Dimension | None = Field( + default=None, + description=( + "Data quality dimension recorded for this rule's results. Defaults to the check's " + "catalog dimension (validity for custom_sql) when not set explicitly." + ), + ) + + @model_validator(mode="after") + def _validate(self) -> DQRule: + if not self.name: + raise ValueError("Rule name cannot be empty") + catalog_spec = None + if self.check == CUSTOM_SQL_CHECK: + if not self.sql: + raise ValueError(f"Rule {self.name!r}: custom_sql check requires 'sql'") + else: + catalog_spec = CHECK_SPECS.get(self.check) + if catalog_spec is None: + supported = sorted([*CHECK_SPECS, CUSTOM_SQL_CHECK]) + raise ValueError(f"Rule {self.name!r}: unknown check {self.check!r}; supported: {supported}") + if self.sql: + raise ValueError(f"Rule {self.name!r}: 'sql' is only valid with custom_sql") + if catalog_spec.requires_column and not self.column: + raise ValueError(f"Rule {self.name!r}: check {self.check!r} requires 'column'") + if self.condition is None and catalog_spec.default_condition is not None: + object.__setattr__(self, "condition", Condition.from_dict(catalog_spec.default_condition)) + if self.condition is None: + raise ValueError(f"Rule {self.name!r}: condition is required for check {self.check!r}") + if self.dimension is None: + object.__setattr__( + self, "dimension", catalog_spec.dimension if catalog_spec else Dimension.VALIDITY + ) + return self + + @property + def rule_uid(self) -> str: + """ + Stable identity across runs: survives severity/dimension tweaks and Dag refactors. + + Uses ``id`` directly when set. Otherwise derives a hash from the rule's identity -- + set ``id`` explicitly to sidestep a collision between two rules that would otherwise + hash the same (see ``previous_name``). + """ + if self.id: + return self.id + condition = cast("Condition", self.condition) + identity = { + "name": self.previous_name or self.name, + "check": self.check, + "column": self.column, + "sql": self.sql, + "condition": condition.to_dict(), + } + digest = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest() + return digest[:16] + + def to_dict(self) -> dict[str, Any]: + condition = cast("Condition", self.condition) + data: dict[str, Any] = { + "name": self.name, + "check": self.check, + "condition": condition.to_dict(), + "severity": self.severity.value, + } + for optional in ("column", "sql", "partition_clause", "previous_name", "id", "description"): + value = getattr(self, optional) + if value is not None: + data[optional] = value + # Only emit dimension when it overrides the check's catalog default, so a rule that + # never set one explicitly round-trips through to_dict()/from_dict() unchanged. + catalog_spec = CHECK_SPECS.get(self.check) + default_dimension = catalog_spec.dimension if catalog_spec else Dimension.VALIDITY + if self.dimension != default_dimension: + data["dimension"] = cast("Dimension", self.dimension).value + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> DQRule: + return cls(**data) + + +def describe_rule(rule: DQRule) -> str: + """Return a default human-readable description for a rule.""" + condition = cast("Condition", rule.condition).to_dict() + subject = rule.column or rule.name.replace("_", " ") + if "equal_to" in condition: + return f"{subject} should equal {condition['equal_to']}" + if "geq_to" in condition: + return f"{subject} should be greater than or equal to {condition['geq_to']}" + if "greater_than" in condition: + return f"{subject} should be greater than {condition['greater_than']}" + if "leq_to" in condition: + return f"{subject} should be less than or equal to {condition['leq_to']}" + if "less_than" in condition: + return f"{subject} should be less than {condition['less_than']}" + return subject + + +class RuleSet(BaseModel): + """A named collection of rules, typically attached to one table or asset.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + name: str + rules: tuple[DQRule, ...] = () + + @model_validator(mode="after") + def _validate(self) -> RuleSet: + if not self.name: + raise ValueError("RuleSet name cannot be empty") + names = [rule.name for rule in self.rules] + duplicates = {name for name in names if names.count(name) > 1} + if duplicates: + raise ValueError(f"Duplicate rule names in ruleset {self.name!r}: {sorted(duplicates)}") + uids = [rule.rule_uid for rule in self.rules] + colliding_uids = {uid for uid in uids if uids.count(uid) > 1} + if colliding_uids: + colliding_names = sorted(rule.name for rule in self.rules if rule.rule_uid in colliding_uids) + raise ValueError(f"Rules {colliding_names} in ruleset {self.name!r} collide on rule_uid") + return self + + def to_dict(self) -> dict[str, Any]: + return {"name": self.name, "rules": [rule.to_dict() for rule in self.rules]} + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> RuleSet: + rules = [ + rule if isinstance(rule, DQRule) else DQRule.from_dict(rule) for rule in data.get("rules", []) + ] + return cls(name=data.get("name", ""), rules=tuple(rules)) + + @classmethod + def from_file(cls, path: str) -> RuleSet: + """Load a ruleset from a YAML (or JSON, being a YAML subset) file.""" + with open(path) as f: + data = yaml.safe_load(f) + if not isinstance(data, dict): + raise DQRuleValidationError(f"Ruleset file {path!r} must contain a mapping at top level") + return cls.from_dict(data) diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md new file mode 100644 index 0000000000000..202d2b650f3a1 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/SKILL.md @@ -0,0 +1,179 @@ +--- +name: dataquality-rule-authoring +description: "Generate a RuleSet/DQRule JSON payload for Apache Airflow's dataquality provider from a table's column definitions. Use this whenever asked to write, generate, or suggest data quality rules, checks, or a ruleset for a table or dataset -- especially when the output is structured JSON handed to DQCheckOperator, @task.dq_check, asset_quality(), or RuleSet.from_file()." +--- + + + +# dataquality rule authoring + +You are producing a **RuleSet**: a JSON object naming a table's data quality checks. This +document is the schema and the full list of valid values -- do not guess field names or check +names, and do not invent checks that aren't in the catalog below. + +## Output shape + +```json +{ + "name": "orders_quality", + "rules": [ + { + "name": "order_id_not_null", + "check": "null_count", + "column": "order_id", + "condition": {"equal_to": 0} + } + ] +} +``` + +`name` (ruleset name) and `rules` (array) are the only top-level keys. Every rule name must be +unique within the ruleset. + +## `DQRule` fields + +| Field | Required | Notes | +|---|---|---| +| `name` | yes | Unique within the ruleset. Shows up in data quality results and logs. | +| `check` | yes | One of the catalog names below, or `"custom_sql"`. Exact string match -- there is no fuzzy matching. | +| `condition` | yes* | See `Condition` grammar below. *Every catalog check currently requires one explicitly (no defaults) -- always include it. | +| `column` | check-dependent | Required for every catalog check except `row_count`. Not used with `custom_sql`. | +| `sql` | `custom_sql` only | A SQL statement returning a single scalar. May reference the checked table as `{table}`. Invalid together with any catalog check. | +| `severity` | no | `"error"` (default) or `"warn"`. `"error"` fails the task on a failing rule (subject to the operator's `fail_on`); `"warn"` only records it. | +| `partition_clause` | no | Extra SQL predicate ANDed into this rule's `WHERE` clause, e.g. `"region = 'EU'"`. | +| `previous_name` | no | Only set when told a rule is being renamed, to keep its history continuous. | +| `id` | no | Explicit stable identity for this rule's history, used directly instead of the derived hash. Only set this when told to, e.g. to avoid a collision between a renamed rule and an unrelated rule that reuses its old name. | +| `description` | no | Human-readable text shown in data quality results. Use it when a clear business meaning is known; otherwise omit it and let Airflow generate a default description. | +| `dimension` | no | One of `completeness`, `uniqueness`, `validity`, `freshness`, `volume`, `consistency`. Defaults to the check's catalog dimension (`validity` for `custom_sql`) -- only set this explicitly when a `custom_sql` rule measures something the default doesn't capture (e.g. a freshness check written as `custom_sql` should set `"dimension": "freshness"`). Leave it unset for every built-in check. | + +**Do not add any key not listed above.** The schema rejects unrecognized fields. + +## Built-in check catalog + +| `check` | SQL expression | needs `column` | typical use | +|---|---|---|---| +| `null_count` | `SUM(CASE WHEN {column} IS NULL THEN 1 ELSE 0 END)` | yes | count of nulls in a column | +| `null_ratio` | `SUM(CASE WHEN {column} IS NULL THEN 1.0 ELSE 0.0 END) / COUNT(*)` | yes | fraction of nulls, 0.0-1.0 | +| `distinct_count` | `COUNT(DISTINCT {column})` | yes | number of distinct values | +| `unique_violations` | `COUNT({column}) - COUNT(DISTINCT {column})` | yes | 0 means the column is fully unique | +| `min` | `MIN({column})` | yes | minimum value | +| `max` | `MAX({column})` | yes | maximum value | +| `mean` | `AVG({column})` | yes | average value | +| `row_count` | `COUNT(*)` | no | total row count -- omit `column` entirely | + +These are the *only* built-in check names. Anything else -- comparing two columns, joining +another table, checking a format/regex, freshness against `now()`, referential integrity -- must +be written as `custom_sql`. + +## `Condition` grammar + +`condition` is an object with these optional numeric keys; at least one is required: + +- `equal_to` -- exact match. Cannot be combined with any other key below. +- `greater_than` / `geq_to` -- lower bound, exclusive / inclusive. +- `less_than` / `leq_to` -- upper bound, exclusive / inclusive. +- `tolerance` -- a fraction (e.g. `0.1` for 10%) that widens the comparison bounds. + +Combine `greater_than`/`less_than`/`geq_to`/`leq_to` freely to express a range, e.g. +`{"geq_to": 0, "leq_to": 100}`. Never combine `equal_to` with another comparison key (other than +`tolerance`). + +## `custom_sql`: when a catalog check doesn't fit + +> **Security note:** `custom_sql` executes the given statement verbatim against the configured +> connection. Prefer a built-in check whenever one fits. Treat any model-generated `custom_sql` +> rule as requiring human review before it runs against a production connection, and use +> read-only credentials for data quality connections. + +Use `check: "custom_sql"` with a `sql` statement resolving to a single scalar whenever: + +- The check needs more than one column (e.g. `end_date >= start_date`), a join, or a subquery. +- A catalog expression doesn't run correctly against the target database's SQL dialect (for + example, some engines don't support `NULLIF`, or evaluate `CASE`/`COUNT DISTINCT` + differently) -- write the equivalent expression for that dialect instead. +- The check is conceptually one of the checks above but needs different null/empty-table + semantics than the catalog expression provides. + +Reference the table being checked as `{table}` in the SQL: + +```json +{ + "name": "no_future_order_dates", + "check": "custom_sql", + "sql": "SELECT COUNT(*) FROM {table} WHERE order_date > CURRENT_DATE", + "condition": {"equal_to": 0} +} +``` + +## Worked example + +Given these column definitions for an `orders` table -- +`order_id` (integer, primary key), `customer_id` (integer, nullable foreign key), +`amount` (decimal, must be non-negative), `region` (string, low-cardinality), +`created_at` (timestamp) -- a reasonable ruleset: + +```json +{ + "name": "orders_quality", + "rules": [ + { + "name": "order_id_not_null", + "check": "null_count", + "column": "order_id", + "condition": {"equal_to": 0} + }, + { + "name": "order_id_unique", + "check": "unique_violations", + "column": "order_id", + "condition": {"equal_to": 0} + }, + { + "name": "customer_id_null_ratio_low", + "check": "null_ratio", + "column": "customer_id", + "condition": {"leq_to": 0.05}, + "severity": "warn" + }, + { + "name": "amount_non_negative", + "check": "min", + "column": "amount", + "condition": {"geq_to": 0} + }, + { + "name": "region_cardinality_reasonable", + "check": "distinct_count", + "column": "region", + "condition": {"leq_to": 20} + }, + { + "name": "table_not_empty", + "check": "row_count", + "condition": {"greater_than": 0} + }, + { + "name": "no_future_order_dates", + "check": "custom_sql", + "sql": "SELECT COUNT(*) FROM {table} WHERE created_at > CURRENT_TIMESTAMP", + "condition": {"equal_to": 0} + } + ] +} +``` + +## Reference schema + +`references/ruleset.schema.json` in this skill's directory is the pydantic-generated JSON +Schema for this exact shape (`RuleSet.model_json_schema()`), useful for validating output +structurally. It does not enumerate valid `check` values (that field is a plain string) -- +this document is the source of truth for which check names exist. + +## How this is consumed + +The generated JSON is typically produced by an LLM task (e.g. `@task.llm(output_type=RuleSet, ...)` +from `common.ai`) and passed straight to `DQCheckOperator(ruleset=...)`, `@task.dq_check(ruleset=...)`, +or `asset_quality(ruleset=...)` -- all three accept a `RuleSet`, its dict form, or a path to a +YAML file written in this same shape. Invalid output raises a `pydantic.ValidationError` +describing exactly which field or value was wrong. diff --git a/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json new file mode 100644 index 0000000000000..2f56d1867fe99 --- /dev/null +++ b/providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json @@ -0,0 +1,249 @@ +{ + "$defs": { + "Condition": { + "additionalProperties": false, + "description": "Pass/fail condition evaluated against a rule's observed value.\n\nUses the same grammar as the ``common.sql`` check operators: ``equal_to``,\n``greater_than``, ``less_than``, ``geq_to``, ``leq_to``, plus a percentage\n``tolerance`` that widens comparisons.", + "properties": { + "equal_to": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Equal To" + }, + "greater_than": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Greater Than" + }, + "less_than": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Less Than" + }, + "geq_to": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Geq To" + }, + "leq_to": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Leq To" + }, + "tolerance": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tolerance" + } + }, + "title": "Condition", + "type": "object" + }, + "DQRule": { + "additionalProperties": false, + "description": "A single, named data quality rule.\n\n:param name: Rule name, unique within its ruleset.\n:param check: One of the built-in checks (see ``CHECK_SPECS``) or ``custom_sql``. Built-in\n checks are plain ANSI SQL and are not guaranteed to work against every ``DbApiHook``;\n see \"Supported checks and databases\" in the provider docs. Use ``custom_sql`` if a\n built-in check doesn't fit your database's dialect.\n:param condition: Pass condition for the observed value (``Condition`` or its dict form).\n Optional only for checks whose ``CheckSpec.default_condition`` is set; every current\n built-in check requires one explicitly.\n:param column: Target column; required for column-level built-in checks.\n:param sql: SQL statement returning a single scalar; required for ``custom_sql``.\n May reference the target table as ``{table}``.\n:param severity: ``error`` (fails the task by default) or ``warn`` (recorded only).\n:param partition_clause: Extra predicate ANDed into the check's WHERE clause.\n:param previous_name: Set when renaming a rule, to keep its history continuous. A rule whose\n ``previous_name`` happens to match another rule's identity (same check/column/condition)\n collides on ``rule_uid``; set an explicit ``id`` on one of them to avoid this.\n:param description: Human-readable description shown in results and the UI. When omitted,\n the provider generates a short default description from the rule and condition.\n:param id: Explicit, stable identity for this rule's history. When set, it is used directly\n as the ``rule_uid`` instead of the derived hash, so it survives ``previous_name`` chains\n and sidesteps any collision between rules that would otherwise hash the same.\n\nInvalid input raises pydantic's own :class:`~pydantic.ValidationError`.", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "check": { + "description": "One of the built-in checks, or custom_sql.", + "title": "Check", + "type": "string" + }, + "condition": { + "anyOf": [ + { + "$ref": "#/$defs/Condition" + }, + { + "type": "null" + } + ], + "default": null + }, + "column": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Column" + }, + "sql": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sql" + }, + "severity": { + "$ref": "#/$defs/Severity", + "default": "error" + }, + "partition_clause": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Partition Clause" + }, + "previous_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Previous Name" + }, + "id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Explicit, stable identity for this rule's history, used directly as rule_uid instead of a derived hash.", + "title": "Id" + }, + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Human-readable description shown in results and the UI. When omitted, the provider generates a short default description from the rule and condition.", + "title": "Description" + }, + "dimension": { + "anyOf": [ + { + "$ref": "#/$defs/Dimension" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Data quality dimension recorded for this rule's results. Defaults to the check's catalog dimension (validity for custom_sql) when not set explicitly." + } + }, + "required": [ + "name", + "check" + ], + "title": "DQRule", + "type": "object" + }, + "Dimension": { + "description": "Data quality dimension assigned to a rule result.", + "enum": [ + "completeness", + "uniqueness", + "validity", + "freshness", + "volume", + "consistency" + ], + "title": "Dimension", + "type": "string" + }, + "Severity": { + "description": "Severity used to decide whether a failing rule fails the task.", + "enum": [ + "warn", + "error" + ], + "title": "Severity", + "type": "string" + } + }, + "additionalProperties": false, + "description": "A named collection of rules, typically attached to one table or asset.", + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "rules": { + "default": [], + "items": { + "$ref": "#/$defs/DQRule" + }, + "title": "Rules", + "type": "array" + } + }, + "required": [ + "name" + ], + "title": "RuleSet", + "type": "object" +} diff --git a/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py b/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py new file mode 100644 index 0000000000000..a2adde9d64395 --- /dev/null +++ b/providers/common/dataquality/tests/system/common/dataquality/example_dq_check.py @@ -0,0 +1,354 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example DAG for the Data Quality provider's ``DQCheckOperator``. + +Runs against the ``sqlite_default`` connection (Airflow's default local backend, no extra +infrastructure required). Results are persisted through the ``[common.dataquality] results_path`` config +option, seeded here via ``AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH`` so the history can be inspected +afterwards under ``/tmp/airflow_dq_example/results`` -- a real deployment would instead set +``results_path`` in ``airflow.cfg`` (or its env var) once, for every check to share. + +Trigger the Dag repeatedly with different ``dq_scenario`` values to create useful +persisted data quality history: + +- ``pass`` inserts clean data. +- ``negative`` inserts a negative amount, failing ``amount_non_negative``. +- ``duplicate_order`` inserts a duplicate ``order_id``, failing uniqueness. +- ``nulls`` inserts null ``order_id``/``customer_id`` values and a high discount null ratio. +- ``outlier`` inserts amounts above the allowed maximum. +- ``small_table`` inserts too few rows for volume checks. +- ``zero_quantity`` inserts an invalid quantity minimum. +- ``empty`` leaves the table empty. +- ``mixed`` combines several bad values to show multiple failures in one run. + +Demonstrates +three equivalent ways to run the same ruleset: + +- Plain ``DQCheckOperator`` with an explicit ``table``/``conn_id``. +- ``asset_quality()`` attaching the ruleset to an ``Asset``, then ``DQCheckOperator(asset=...)`` + picking up ``table``/``conn_id``/``ruleset`` from it and adding the asset to the task's + outlets automatically. +- A second asset-producing ``DQCheckOperator`` with stricter rules, so the Browse > Data Quality + page shows multiple producers for the same asset with both pass and fail outcomes. +- The ``@task.dq_check`` TaskFlow decorator. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path + +from airflow.providers.common.dataquality.assets import asset_quality +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet, Severity +from airflow.providers.common.sql.operators.sql import SQLExecuteQueryOperator +from airflow.sdk import DAG, Asset, task + +DAG_ID = "example_dq_check" +CONN_ID = "sqlite_default" +TABLE_NAME = "dq_example_orders" +RESULTS_PATH = Path("/tmp/airflow_dq_example/results") + +# DQCheckOperator has no per-operator results-store override; every check in a deployment +# shares the one store configured under [common.dataquality] results_path. setdefault() so a real deployment's +# own config is never overridden. +os.environ.setdefault("AIRFLOW__COMMON_DATAQUALITY__RESULTS_PATH", f"file://{RESULTS_PATH}") + +# [START howto_operator_dq_check_ruleset] +orders_ruleset = RuleSet( + name="orders_quality", + rules=( + DQRule( + name="order_id_not_null", + check="null_count", + column="order_id", + condition=Condition(equal_to=0), + ), + DQRule( + name="order_id_unique", + check="unique_violations", + column="order_id", + condition=Condition(equal_to=0), + ), + DQRule( + name="customer_id_not_null", + check="null_count", + column="customer_id", + condition=Condition(equal_to=0), + severity=Severity.WARN, + ), + DQRule( + name="discount_null_ratio", + check="null_ratio", + column="discount", + condition=Condition(leq_to=0.25), + severity=Severity.WARN, + ), + DQRule( + name="region_distinct_count", + check="distinct_count", + column="region", + condition=Condition(geq_to=2), + severity=Severity.WARN, + ), + DQRule( + name="amount_min_ge_zero", + check="min", + column="amount", + condition=Condition(geq_to=0), + ), + DQRule( + name="amount_max_le_100", + check="max", + column="amount", + condition=Condition(leq_to=100), + ), + DQRule( + name="amount_mean_between_5_and_60", + check="mean", + column="amount", + condition=Condition(geq_to=5, leq_to=60), + ), + DQRule( + name="quantity_min_ge_one", + check="min", + column="quantity", + condition=Condition(geq_to=1), + ), + DQRule( + name="quantity_max_le_ten", + check="max", + column="quantity", + condition=Condition(leq_to=10), + ), + DQRule( + name="amount_non_negative", + check="custom_sql", + # {table} is substituted by the SQL engine at check time, not an f-string placeholder. + sql="SELECT COUNT(*) FROM {table} WHERE amount < 0", + condition=Condition(equal_to=0), + ), + DQRule( + name="high_value_order_count", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE amount > 100", + condition=Condition(equal_to=0), + severity=Severity.WARN, + ), + DQRule( + name="row_count_present", + check="row_count", + condition=Condition(greater_than=0), + severity=Severity.WARN, + ), + DQRule( + name="row_count_at_least_three", + check="row_count", + condition=Condition(geq_to=3), + ), + ), +) +# [END howto_operator_dq_check_ruleset] + +# [START howto_operator_dq_check_asset] +orders_asset = asset_quality( + Asset("dq_example_orders", uri="file:///tmp/airflow_dq_example/orders"), + ruleset=orders_ruleset, + conn_id=CONN_ID, + table=TABLE_NAME, +) +# [END howto_operator_dq_check_asset] + +strict_orders_ruleset = RuleSet( + name="orders_strict_quality", + rules=( + DQRule( + name="strict_order_id_not_null", + check="null_count", + column="order_id", + condition=Condition(equal_to=0), + ), + DQRule( + name="strict_amount_max_le_20", + check="max", + column="amount", + condition=Condition(leq_to=20), + ), + DQRule( + name="strict_row_count_equal_two", + check="row_count", + condition=Condition(equal_to=2), + ), + DQRule( + name="strict_region_distinct_count", + check="distinct_count", + column="region", + condition=Condition(geq_to=4), + severity=Severity.WARN, + ), + ), +) + +strict_orders_asset = asset_quality( + Asset("dq_example_orders", uri="file:///tmp/airflow_dq_example/orders"), + ruleset=strict_orders_ruleset, + conn_id=CONN_ID, + table=TABLE_NAME, +) + + +with DAG( + dag_id=DAG_ID, + schedule=None, + start_date=datetime(2021, 1, 1), + catchup=False, + tags=["example", "dq"], +) as dag: + create_table = SQLExecuteQueryOperator( + task_id="create_table", + conn_id=CONN_ID, + sql=[ + f"DROP TABLE IF EXISTS {TABLE_NAME};", + f""" + CREATE TABLE {TABLE_NAME} ( + order_id INTEGER, + customer_id INTEGER, + amount REAL, + discount REAL, + quantity INTEGER, + region TEXT + ); + """, + ], + ) + + clear_table = SQLExecuteQueryOperator( + task_id="clear_table", + conn_id=CONN_ID, + sql=f"DELETE FROM {TABLE_NAME};", + ) + + insert_orders = SQLExecuteQueryOperator( + task_id="insert_orders", + conn_id=CONN_ID, + sql=f""" + {{% set scenario = dag_run.conf.get("dq_scenario", "pass") if dag_run else "pass" %}} + {{% if scenario == "negative" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (2, 102, -25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "duplicate_order" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (1, 102, 25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "nulls" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, NULL, 1, 'US'), + (NULL, NULL, 25.5, NULL, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "outlier" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (2, 102, 120.0, 0.10, 2, 'EU'), + (3, 103, 150.0, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "small_table" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'); + {{% elif scenario == "zero_quantity" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 0, 'US'), + (2, 102, 25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% elif scenario == "mixed" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, NULL, 0, 'US'), + (1, NULL, -25.5, NULL, 2, 'US'), + (NULL, 103, 150.0, NULL, 11, 'US'), + (4, 104, 55.0, 0.20, 4, 'US'); + {{% elif scenario == "empty" %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) + SELECT NULL, NULL, NULL, NULL, NULL, NULL WHERE 0; + {{% else %}} + INSERT INTO {TABLE_NAME} (order_id, customer_id, amount, discount, quantity, region) VALUES + (1, 101, 10.0, 0.00, 1, 'US'), + (2, 102, 25.5, 0.10, 2, 'EU'), + (3, 103, 7.25, NULL, 3, 'US'), + (4, 104, 55.0, 0.20, 4, 'APAC'); + {{% endif %}} + """, + ) + + # [START howto_operator_dq_check] + check_orders = DQCheckOperator( + task_id="check_orders", + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=orders_ruleset, + fail_on="never", + ) + # [END howto_operator_dq_check] + + check_orders_via_asset = DQCheckOperator( + task_id="check_orders_via_asset", + asset=orders_asset, + fail_on="never", + ) + + check_orders_strict_via_asset = DQCheckOperator( + task_id="check_orders_strict_via_asset", + asset=strict_orders_asset, + fail_on="never", + ) + + # [START howto_decorator_dq_check] + @task.dq_check( + conn_id=CONN_ID, + table=TABLE_NAME, + ruleset=orders_ruleset, + fail_on="never", + ) + def check_orders_decorated(): + return None + + # [END howto_decorator_dq_check] + + ( + create_table + >> clear_table + >> insert_orders + >> [check_orders, check_orders_via_asset, check_orders_strict_via_asset, check_orders_decorated()] + ) + + from tests_common.test_utils.watcher import watcher + + # This test needs watcher in order to properly mark success/failure + # when "tearDown" task with trigger rule is part of the DAG + list(dag.tasks) >> watcher() + +from tests_common.test_utils.system_tests import get_test_run # noqa: E402 + +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) +test_run = get_test_run(dag) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/backends/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/backends/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/backends/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/tests/unit/common/dataquality/backends/test_backends_init.py b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_backends_init.py new file mode 100644 index 0000000000000..3b027c1b73950 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_backends_init.py @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import subprocess +import sys + + +def test_get_backend_from_config_does_not_eagerly_import_object_storage(): + """ + ``ObjectStorageResultsBackend`` pulls in ``ObjectStoragePath`` (fsspec/upath), which + ``get_backend_from_config()`` shouldn't pay for at import time when no ``results_path`` is + configured -- it's only needed once a backend is actually built. Runs in a subprocess for a + clean ``sys.modules`` unaffected by other tests importing the submodule directly. + """ + script = ( + "import sys\n" + "from airflow.providers.common.dataquality.backends import get_backend_from_config\n" + "assert 'airflow.providers.common.dataquality.backends.object_storage' not in sys.modules\n" + "assert get_backend_from_config() is None\n" + "assert 'airflow.providers.common.dataquality.backends.object_storage' not in sys.modules\n" + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, check=False) + assert result.returncode == 0, result.stderr diff --git a/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py new file mode 100644 index 0000000000000..4e67f45691893 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/backends/test_object_storage.py @@ -0,0 +1,438 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +from typing import Any + +import pytest + +from airflow.providers.common.dataquality.backends import get_backend_from_config +from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.common.dataquality.results import DQRun, RuleResult +from airflow.providers.common.dataquality.rules import Severity + +from tests_common.test_utils.config import conf_vars + + +def make_run(run_uid: str = "abc123", started_at: str = "2026-07-04T06:00:00+00:00") -> DQRun: + return DQRun( + dag_id="orders_pipeline", + task_id="dq", + run_id="scheduled__2026-07-04", + run_uid=run_uid, + ruleset_name="orders", + table_ref="analytics.orders", + asset_names=("dq_example_orders",), + started_at=started_at, + finished_at="2026-07-04T06:00:03+00:00", + ) + + +def make_result(rule_uid: str = "rule-1", status: str = "pass") -> RuleResult: + return RuleResult( + rule_uid=rule_uid, + rule_name="ids_not_null", + status=status, + observed_value=0, + condition={"equal_to": 0}, + dimension="completeness", + severity=Severity.ERROR, + duration_ms=12.5, + ) + + +class TestObjectStorageResultsBackend: + @pytest.fixture + def backend(self, tmp_path): + return ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + + def test_write_run_creates_run_file_with_run_and_results(self, backend): + backend.write_run(run=make_run(), results=[make_result()]) + + record = backend.read_by_task_instance( + dag_id="orders_pipeline", task_id="dq", run_id="scheduled__2026-07-04" + ) + assert record["run"]["run_uid"] == "abc123" + assert record["run"]["dag_id"] == "orders_pipeline" + assert record["results"][0]["rule_uid"] == "rule-1" + assert record["results"][0]["status"] == "pass" + + def test_write_run_stores_keyed_json_payload(self, backend): + backend.write_run(run=make_run(), results=[make_result()]) + + path = ( + backend.root + / "runs" + / "by_task" + / "dag_id=orders_pipeline" + / "task_id=dq" + / "date=2026-07-04" + / "2026-07-04T06_00_00_00_00__abc123.json" + ) + payload = json.loads(path.read_text()) + + assert payload["run"]["run_uid"] == "abc123" + assert payload["results"][0]["rule_uid"] == "rule-1" + assert payload["summary"]["passed"] == 1 + + def test_write_run_stores_task_rule_index(self, backend): + backend.write_run(run=make_run(), results=[make_result()]) + + path = ( + backend.root + / "rules" + / "by_task_rule" + / "dag_id=orders_pipeline" + / "task_id=dq" + / "rule_uid=rule-1" + / "2026-07-04T06_00_00_00_00__abc123.json" + ) + payload = json.loads(path.read_text()) + + assert payload["run"]["dag_id"] == "orders_pipeline" + assert payload["run"]["task_id"] == "dq" + assert payload["result"]["rule_uid"] == "rule-1" + + def test_write_run_does_not_store_global_rule_index(self, backend): + backend.write_run(run=make_run(), results=[make_result()]) + + path = backend.root / "rules" / "by_rule" / "rule_uid=rule-1" + + assert not path.exists() + + def test_rule_history_is_newest_first(self, backend): + backend.write_run( + run=make_run(run_uid="run1", started_at="2026-07-01T06:00:00+00:00"), + results=[make_result(status="pass")], + ) + backend.write_run( + run=make_run(run_uid="run2", started_at="2026-07-02T06:00:00+00:00"), + results=[make_result(status="fail")], + ) + + result = backend.read_task_rule_history(dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1") + + assert [record["status"] for record in result["items"]] == ["fail", "pass"] + assert result["items"][0]["run"]["run_uid"] == "run2" + assert result["next_cursor"] is None + + def test_rule_history_includes_task_instance_route_context(self, backend): + backend.write_run(run=make_run(), results=[make_result()]) + + history = backend.read_task_rule_history(dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1")[ + "items" + ] + + assert history[0]["run"]["dag_id"] == "orders_pipeline" + assert history[0]["run"]["task_id"] == "dq" + assert history[0]["run"]["run_id"] == "scheduled__2026-07-04" + assert history[0]["run"]["map_index"] == -1 + + def test_rule_history_respects_limit(self, backend): + for day in range(1, 4): + backend.write_run( + run=make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + results=[make_result()], + ) + + result = backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1", limit=2 + ) + + assert len(result["items"]) == 2 + assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" + + def test_rule_history_before_cursor_pages_further_back(self, backend): + for day in range(1, 4): + backend.write_run( + run=make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + results=[make_result()], + ) + + first_page = backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1", limit=2 + ) + second_page = backend.read_task_rule_history( + dag_id="orders_pipeline", + task_id="dq", + rule_uid="rule-1", + limit=2, + before=first_page["next_cursor"], + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_concurrent_style_writes_do_not_collide(self, backend): + backend.write_run(run=make_run(run_uid="run1"), results=[make_result()]) + backend.write_run(run=make_run(run_uid="run2"), results=[make_result()]) + + runs = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq")["items"] + assert {run["run"]["run_uid"] for run in runs} == {"run1", "run2"} + + def test_read_by_task_instance_matches_run(self, backend): + backend.write_run(run=make_run(), results=[make_result()]) + + record = backend.read_by_task_instance( + dag_id="orders_pipeline", task_id="dq", run_id="scheduled__2026-07-04" + ) + + assert record["run"]["run_uid"] == "abc123" + assert record["results"][0]["rule_uid"] == "rule-1" + assert record["summary"]["passed"] == 1 + + def test_read_by_task_instance_unknown_raises(self, backend): + with pytest.raises(FileNotFoundError): + backend.read_by_task_instance(dag_id="orders_pipeline", task_id="dq", run_id="no_such_run") + + def test_read_by_task_instance_last_write_wins_across_retries(self, backend): + first = make_run(run_uid="try1") + backend.write_run(run=first, results=[make_result(status="fail")]) + second = make_run(run_uid="try2") + backend.write_run(run=second, results=[make_result(status="pass")]) + + record = backend.read_by_task_instance( + dag_id="orders_pipeline", task_id="dq", run_id="scheduled__2026-07-04" + ) + + assert record["run"]["run_uid"] == "try2" + assert record["results"][0]["status"] == "pass" + assert record["summary"]["passed"] == 1 + + def test_read_by_task_instance_sanitizes_run_id(self, backend): + run_id = "manual__2026-07-04T06:00:00+00:00" + run = DQRun(dag_id="orders_pipeline", task_id="dq", run_id=run_id, run_uid="abc123") + backend.write_run(run=run, results=[make_result()]) + + record = backend.read_by_task_instance(dag_id="orders_pipeline", task_id="dq", run_id=run_id) + + assert record["run"]["run_id"] == run_id + + def test_read_task_runs_returns_newest_first_with_summaries(self, backend): + backend.write_run( + run=make_run(run_uid="run1", started_at="2026-07-01T06:00:00+00:00"), + results=[make_result(status="pass")], + ) + backend.write_run( + run=make_run(run_uid="run2", started_at="2026-07-02T06:00:00+00:00"), + results=[make_result(status="fail")], + ) + + result = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq") + runs = result["items"] + + assert [record["run"]["run_uid"] for record in runs] == ["run2", "run1"] + assert runs[0]["summary"]["failed"] == 1 + assert runs[1]["summary"]["passed"] == 1 + assert result["next_cursor"] is None + + def test_read_task_runs_respects_limit(self, backend): + for day in range(1, 4): + backend.write_run( + run=make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + results=[make_result()], + ) + + result = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq", limit=2) + + assert len(result["items"]) == 2 + assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" + + def test_read_task_runs_before_cursor_pages_further_back(self, backend): + for day in range(1, 4): + backend.write_run( + run=make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + results=[make_result()], + ) + + first_page = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq", limit=2) + second_page = backend.read_task_runs( + dag_id="orders_pipeline", + task_id="dq", + limit=2, + before=first_page["next_cursor"], + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_read_task_runs_cursor_keeps_same_timestamp_records(self, backend): + for run_uid in ("run1", "run2", "run3"): + backend.write_run( + run=make_run(run_uid=run_uid, started_at="2026-07-04T06:00:00+00:00"), + results=[make_result()], + ) + + first_page = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq", limit=2) + second_page = backend.read_task_runs( + dag_id="orders_pipeline", + task_id="dq", + limit=2, + before=first_page["next_cursor"], + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_read_task_runs_stops_scanning_once_limit_is_reached(self, backend, monkeypatch): + for day in range(1, 5): + backend.write_run( + run=make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + results=[make_result()], + ) + + read_paths: list[Any] = [] + original_read_json = backend._read_json + monkeypatch.setattr( + backend, "_read_json", lambda path: (read_paths.append(path), original_read_json(path))[1] + ) + + result = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq", limit=1) + + assert [record["run"]["run_uid"] for record in result["items"]] == ["run4"] + assert result["next_cursor"] == "2026-07-04T06:00:00+00:00|run4" + # Determining next_cursor reads one entry past the limit (date=2026-07-03's run3), but + # no further — date=2026-07-02/01 must not be read. + assert len(read_paths) == 2 + + def test_read_task_runs_stops_inside_date_partition_once_limit_is_reached(self, backend, monkeypatch): + for index in range(1, 5): + backend.write_run( + run=make_run(run_uid=f"run{index}", started_at=f"2026-07-04T06:00:0{index}+00:00"), + results=[make_result()], + ) + + read_paths: list[Any] = [] + original_read_json = backend._read_json + monkeypatch.setattr( + backend, "_read_json", lambda path: (read_paths.append(path), original_read_json(path))[1] + ) + + result = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq", limit=1) + + assert [r["run"]["run_uid"] for r in result["items"]] == ["run4"] + assert result["next_cursor"] == "2026-07-04T06:00:04+00:00|run4" + assert len(read_paths) == 2 + + def test_read_task_runs_within_partition_returns_newest_first_regardless_of_write_order(self, backend): + """More than limit+1 runs in one date partition must still surface the true newest ones. + + Filenames are random run_uids with no ordering guarantee from iterdir(), so this only + passes when the partition is sorted by its started_at-prefixed filename before scanning. + """ + for index in (3, 1, 4, 2): + backend.write_run( + run=make_run(run_uid=f"run{index}", started_at=f"2026-07-04T06:00:0{index}+00:00"), + results=[make_result()], + ) + + result = backend.read_task_runs(dag_id="orders_pipeline", task_id="dq", limit=2) + + assert [r["run"]["run_uid"] for r in result["items"]] == ["run4", "run3"] + assert result["next_cursor"] == "2026-07-04T06:00:03+00:00|run3" + + def test_read_task_rule_history_filters_to_task(self, backend): + backend.write_run(run=make_run(run_uid="run1"), results=[make_result()]) + backend.write_run( + run=DQRun( + dag_id="other_pipeline", + task_id="dq", + run_id="scheduled__2026-07-04", + run_uid="run2", + started_at="2026-07-04T07:00:00+00:00", + ), + results=[make_result()], + ) + + history = backend.read_task_rule_history(dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1")[ + "items" + ] + + assert len(history) == 1 + assert history[0]["run"]["dag_id"] == "orders_pipeline" + + def test_read_task_rule_history_respects_limit(self, backend): + for day in range(1, 4): + backend.write_run( + run=make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + results=[make_result()], + ) + + result = backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1", limit=2 + ) + + assert [record["run"]["run_uid"] for record in result["items"]] == ["run3", "run2"] + assert result["next_cursor"] == "2026-07-02T06:00:00+00:00|run2" + + def test_read_task_rule_history_before_cursor_pages_further_back(self, backend): + for day in range(1, 4): + backend.write_run( + run=make_run(run_uid=f"run{day}", started_at=f"2026-07-0{day}T06:00:00+00:00"), + results=[make_result()], + ) + + first_page = backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1", limit=2 + ) + second_page = backend.read_task_rule_history( + dag_id="orders_pipeline", + task_id="dq", + rule_uid="rule-1", + limit=2, + before=first_page["next_cursor"], + ) + + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + def test_read_task_rule_history_cursor_keeps_same_timestamp_records(self, backend): + for run_uid in ("run1", "run2", "run3"): + backend.write_run( + run=make_run(run_uid=run_uid, started_at="2026-07-04T06:00:00+00:00"), + results=[make_result()], + ) + + first_page = backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid="rule-1", limit=2 + ) + second_page = backend.read_task_rule_history( + dag_id="orders_pipeline", + task_id="dq", + rule_uid="rule-1", + limit=2, + before=first_page["next_cursor"], + ) + + assert [r["run"]["run_uid"] for r in first_page["items"]] == ["run3", "run2"] + assert [r["run"]["run_uid"] for r in second_page["items"]] == ["run1"] + assert second_page["next_cursor"] is None + + +class TestGetBackendFromConfig: + @conf_vars({("common.dataquality", "results_path"): None}) + def test_no_results_path_returns_none(self): + assert get_backend_from_config() is None + + def test_results_path_builds_object_storage_backend(self, tmp_path): + with conf_vars({("common.dataquality", "results_path"): f"file://{tmp_path}"}): + backend = get_backend_from_config() + assert isinstance(backend, ObjectStorageResultsBackend) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/decorators/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/decorators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/decorators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py b/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py new file mode 100644 index 0000000000000..d532b0e09a18d --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/decorators/test_dq_check.py @@ -0,0 +1,126 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.common.dataquality.decorators.dq_check import _DQCheckDecoratedOperator +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet +from airflow.providers.common.sql.hooks.sql import DbApiHook +from airflow.sdk.execution_time.context import OutletEventAccessors + +NULLS = DQRule(name="nulls", check="null_count", column="id", condition=Condition(equal_to=0)) +RULESET = RuleSet(name="orders", rules=(NULLS,)) + + +def make_context(): + # spec'd to the attributes DQCheckOperator actually reads off `ti`, since the real + # RuntimeTaskInstance is a heavyweight pydantic model with many unrelated required fields. + ti = mock.Mock(spec=["dag_id", "task_id", "run_id", "try_number", "map_index"]) + ti.dag_id = "orders_pipeline" + ti.task_id = "dq" + ti.run_id = "manual__2026-07-04" + ti.try_number = 1 + ti.map_index = -1 + return {"ti": ti, "outlet_events": mock.create_autospec(OutletEventAccessors, instance=True)} + + +@pytest.fixture(autouse=True) +def results_backend(tmp_path): + """Patch the module-level config lookup so persisted results land in an isolated tmp_path.""" + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + with mock.patch( + "airflow.providers.common.dataquality.execution.get_backend_from_config", + return_value=backend, + ): + yield backend + + +def make_decorated_operator(records, python_callable, **kwargs): + kwargs.setdefault("task_id", "dq") + if not kwargs.pop("omit_ruleset", False): + kwargs.setdefault("ruleset", RULESET) + kwargs.setdefault("table", "orders") + operator = _DQCheckDecoratedOperator(conn_id="warehouse", python_callable=python_callable, **kwargs) + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = records + return operator, hook + + +class TestDQCheckDecoratedOperator: + def test_custom_operator_name(self): + assert _DQCheckDecoratedOperator.custom_operator_name == "@task.dq_check" + + @mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook") + def test_none_return_runs_check_as_declared(self, mock_get_db_hook): + operator, hook = make_decorated_operator(records=[(NULLS.rule_uid, 0)], python_callable=lambda: None) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert operator.table == "orders" + assert summary["passed"] == 1 + + @mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook") + def test_ruleset_can_be_provided_only_at_runtime(self, mock_get_db_hook): + other_rule = DQRule(name="row_count_ok", check="row_count", condition=Condition(greater_than=0)) + other_ruleset = RuleSet(name="dynamic", rules=(other_rule,)) + + operator, hook = make_decorated_operator( + records=[(other_rule.rule_uid, 10)], + python_callable=lambda: other_ruleset, + omit_ruleset=True, + ) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["passed"] == 1 + + def test_missing_runtime_ruleset_raises_value_error(self): + operator, hook = make_decorated_operator(records=[], python_callable=lambda: None, omit_ruleset=True) + with mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook", return_value=hook): + with pytest.raises(ValueError, match="ruleset is required"): + operator.execute(make_context()) + + @mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook") + def test_op_kwargs_are_passed_to_callable(self, mock_get_db_hook): + seen = {} + + def resolve_target(suffix): + seen["suffix"] = suffix + return None + + operator, hook = make_decorated_operator( + records=[(NULLS.rule_uid, 0)], + python_callable=resolve_target, + op_kwargs={"suffix": "eu"}, + ) + mock_get_db_hook.return_value = hook + + operator.execute(make_context()) + + assert seen["suffix"] == "eu" + + def test_invalid_return_raises_type_error(self): + operator, hook = make_decorated_operator(records=[], python_callable=lambda: 42) + with mock.patch.object(_DQCheckDecoratedOperator, "get_db_hook", return_value=hook): + with pytest.raises(TypeError, match="must return a RuleSet"): + operator.execute(make_context()) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/engines/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/engines/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/engines/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py b/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py new file mode 100644 index 0000000000000..995444c95ba19 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/engines/test_sql.py @@ -0,0 +1,170 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.common.dataquality.engines.sql import SQLDQEngine +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet +from airflow.providers.common.sql.hooks.sql import DbApiHook + + +@pytest.fixture +def hook(): + return mock.create_autospec(DbApiHook, instance=True) + + +NULLS = DQRule(name="nulls", check="null_count", column="id", condition=Condition(equal_to=0)) +VOLUME = DQRule(name="volume", check="row_count", condition=Condition(greater_than=0)) +CUSTOM = DQRule( + name="negatives", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE amount < 0", + condition=Condition(equal_to=0), +) + + +class TestBuildBatchSql: + def test_batch_sql_unions_one_select_per_rule(self, hook): + engine = SQLDQEngine(hook) + sql = engine.build_batch_sql(rules=[NULLS, VOLUME], table="orders", partition_clause=None) + assert sql.count("UNION ALL") == 1 + assert f"'{NULLS.rule_uid}' AS rule_uid" in sql + assert "SUM(CASE WHEN id IS NULL THEN 1 ELSE 0 END)" in sql + assert "COUNT(*)" in sql + assert "WHERE" not in sql + + def test_partition_clauses_are_anded(self, hook): + rule = DQRule( + name="nulls", + check="null_count", + column="id", + condition=Condition(equal_to=0), + partition_clause="region = 'EU'", + ) + sql = SQLDQEngine(hook).build_batch_sql( + rules=[rule], table="orders", partition_clause="ds = '2026-07-04'" + ) + assert "WHERE ds = '2026-07-04' AND region = 'EU'" in sql + + +class TestMeasure: + def test_builtin_rules_measured_from_single_query(self, hook): + hook.get_records.return_value = [(NULLS.rule_uid, 0), (VOLUME.rule_uid, 42)] + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + + observations = SQLDQEngine(hook).measure(ruleset=ruleset, table="orders") + + hook.get_records.assert_called_once() + by_name = {obs.rule.name: obs for obs in observations} + assert by_name["nulls"].observed_value == 0 + assert by_name["volume"].observed_value == 42 + assert by_name["nulls"].sql == SQLDQEngine(hook).build_rule_sql(rule=NULLS, table="orders") + assert by_name["volume"].sql == SQLDQEngine(hook).build_rule_sql(rule=VOLUME, table="orders") + assert all(obs.error_message is None for obs in observations) + + def test_builds_each_rules_sql_exactly_once_on_success(self, hook): + hook.get_records.return_value = [(NULLS.rule_uid, 0), (VOLUME.rule_uid, 42)] + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + engine = SQLDQEngine(hook) + + with mock.patch.object(engine, "build_rule_sql", wraps=engine.build_rule_sql) as build_rule_sql: + engine.measure(ruleset=ruleset, table="orders") + + assert build_rule_sql.call_count == 2 + + def test_builds_each_rules_sql_exactly_once_on_batch_failure(self, hook): + hook.get_records.side_effect = RuntimeError("connection refused") + hook.get_first.side_effect = [(NULLS.rule_uid, 0), (VOLUME.rule_uid, 0)] + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + engine = SQLDQEngine(hook) + + with mock.patch.object(engine, "build_rule_sql", wraps=engine.build_rule_sql) as build_rule_sql: + engine.measure(ruleset=ruleset, table="orders") + + assert build_rule_sql.call_count == 2 + + def test_batch_failure_falls_back_to_per_rule_queries(self, hook): + hook.get_records.side_effect = RuntimeError("connection refused") + hook.get_first.side_effect = [(NULLS.rule_uid, 0), (VOLUME.rule_uid, 0)] + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + + observations = SQLDQEngine(hook).measure(ruleset=ruleset, table="orders") + + assert hook.get_first.call_count == 2 + assert len(observations) == 2 + assert all(obs.error_message is None for obs in observations) + assert all(obs.observed_value == 0 for obs in observations) + assert all(obs.sql for obs in observations) + + def test_batch_failure_fallback_isolates_a_single_bad_rule(self, hook): + """One rule's query failing in the per-rule fallback must not fail the other rules.""" + hook.get_records.side_effect = RuntimeError("connection refused") + hook.get_first.side_effect = [ + (NULLS.rule_uid, 0), + RuntimeError("no such column: no_such_column"), + ] + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + + observations = SQLDQEngine(hook).measure(ruleset=ruleset, table="orders") + + by_name = {obs.rule.name: obs for obs in observations} + assert by_name["nulls"].observed_value == 0 + assert by_name["nulls"].error_message is None + assert by_name["volume"].observed_value is None + assert by_name["volume"].error_message == "no such column: no_such_column" + + def test_per_rule_fallback_query_with_no_rows_is_an_error(self, hook): + hook.get_records.side_effect = RuntimeError("connection refused") + hook.get_first.return_value = None + ruleset = RuleSet(name="s", rules=(NULLS,)) + + observations = SQLDQEngine(hook).measure(ruleset=ruleset, table="orders") + + assert observations[0].error_message == "No result returned for rule" + assert observations[0].observed_value is None + + def test_missing_rule_in_result_is_an_error(self, hook): + hook.get_records.return_value = [(NULLS.rule_uid, 0)] + ruleset = RuleSet(name="s", rules=(NULLS, VOLUME)) + + observations = SQLDQEngine(hook).measure(ruleset=ruleset, table="orders") + + by_name = {obs.rule.name: obs for obs in observations} + assert by_name["nulls"].error_message is None + assert by_name["volume"].error_message == "No result returned for rule" + + def test_custom_sql_rule_runs_individually_with_table_substituted(self, hook): + hook.get_first.return_value = (3,) + ruleset = RuleSet(name="s", rules=(CUSTOM,)) + + observations = SQLDQEngine(hook).measure(ruleset=ruleset, table="orders") + + hook.get_first.assert_called_once_with("SELECT COUNT(*) FROM orders WHERE amount < 0") + hook.get_records.assert_not_called() + assert observations[0].observed_value == 3 + assert observations[0].sql == "SELECT COUNT(*) FROM orders WHERE amount < 0" + + def test_custom_sql_no_rows_is_an_error(self, hook): + hook.get_first.return_value = None + ruleset = RuleSet(name="s", rules=(CUSTOM,)) + + observations = SQLDQEngine(hook).measure(ruleset=ruleset, table="orders") + + assert observations[0].error_message == "Query returned no rows" diff --git a/providers/common/dataquality/tests/unit/common/dataquality/operators/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/operators/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/operators/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py new file mode 100644 index 0000000000000..1eef94d7319cd --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/operators/test_dq_check.py @@ -0,0 +1,271 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.common.dataquality.assets import asset_quality +from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.common.dataquality.exceptions import DQCheckFailedError +from airflow.providers.common.dataquality.operators.dq_check import DQCheckOperator +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet, Severity +from airflow.providers.common.sql.hooks.sql import DbApiHook +from airflow.sdk import Asset +from airflow.sdk.execution_time.context import OutletEventAccessors + +NULLS = DQRule(name="nulls", check="null_count", column="id", condition=Condition(equal_to=0)) +VOLUME_WARN = DQRule( + name="volume", check="row_count", condition=Condition(greater_than=100), severity=Severity.WARN +) +RULESET = RuleSet(name="orders", rules=(NULLS, VOLUME_WARN)) + + +def make_context(): + # spec'd to the attributes DQCheckOperator actually reads off `ti`, since the real + # RuntimeTaskInstance is a heavyweight pydantic model with many unrelated required fields. + ti = mock.Mock(spec=["dag_id", "task_id", "run_id", "try_number", "map_index"]) + ti.dag_id = "orders_pipeline" + ti.task_id = "dq" + ti.run_id = "manual__2026-07-04" + ti.try_number = 1 + ti.map_index = -1 + return {"ti": ti, "outlet_events": mock.create_autospec(OutletEventAccessors, instance=True)} + + +def make_operator(records, **kwargs): + kwargs.setdefault("task_id", "dq") + kwargs.setdefault("ruleset", RULESET) + kwargs.setdefault("table", "orders") + operator = DQCheckOperator(conn_id="warehouse", **kwargs) + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = records + return operator, hook + + +@pytest.fixture(autouse=True) +def results_backend(tmp_path): + """Patch the module-level config lookup so persisted results land in an isolated tmp_path.""" + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + with mock.patch( + "airflow.providers.common.dataquality.execution.get_backend_from_config", + return_value=backend, + ): + yield backend + + +class TestDQCheckOperatorConstruction: + def test_requires_ruleset(self): + with pytest.raises(ValueError, match="ruleset is required"): + DQCheckOperator(task_id="dq", table="orders", conn_id="c") + + def test_requires_table(self): + with pytest.raises(ValueError, match="table is required"): + DQCheckOperator(task_id="dq", ruleset=RULESET, conn_id="c") + + def test_rejects_bad_fail_on(self): + with pytest.raises(ValueError, match="fail_on"): + DQCheckOperator(task_id="dq", ruleset=RULESET, table="orders", conn_id="c", fail_on="maybe") + + def test_requires_conn_id(self): + with pytest.raises(ValueError, match="conn_id is required"): + DQCheckOperator(task_id="dq", ruleset=RULESET, table="orders") + + def test_asset_without_conn_id_fails_fast(self): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + table="analytics.orders", + ) + + with pytest.raises(ValueError, match="conn_id is required"): + DQCheckOperator(task_id="dq", asset=asset) + + def test_asset_supplies_defaults_and_outlet(self): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="analytics.orders", + ) + operator = DQCheckOperator(task_id="dq", asset=asset) + assert operator.table == "analytics.orders" + assert operator.conn_id == "warehouse" + assert asset in operator.outlets + + def test_explicit_arguments_beat_asset_config(self): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="analytics.orders", + ) + operator = DQCheckOperator(task_id="dq", asset=asset, table="other_table", conn_id="other_conn") + assert operator.table == "other_table" + assert operator.conn_id == "other_conn" + + +class TestDQCheckOperatorExecute: + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_all_pass_returns_summary_and_persists(self, mock_get_db_hook, results_backend): + operator, hook = make_operator(records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 500)]) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["passed"] == 2 + assert summary["failed"] == 0 + assert summary["score"] == 1.0 + history = results_backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid=NULLS.rule_uid + )["items"] + assert len(history) == 1 + assert history[0]["status"] == "pass" + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_error_severity_failure_fails_task(self, mock_get_db_hook, results_backend): + operator, hook = make_operator(records=[(NULLS.rule_uid, 3), (VOLUME_WARN.rule_uid, 500)]) + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="nulls"): + operator.execute(make_context()) + + # The failing result is persisted even though the task fails. + assert ( + results_backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid=NULLS.rule_uid + )["items"][0]["status"] + == "fail" + ) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_warn_severity_failure_does_not_fail_task_by_default(self, mock_get_db_hook): + operator, hook = make_operator(records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 5)]) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["warned"] == 1 + assert summary["warned_rules"] == ["volume"] + assert summary["score"] == pytest.approx(0.875) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_rule_description_is_persisted(self, mock_get_db_hook, results_backend): + described_rule = DQRule( + name="nulls", + check="null_count", + column="id", + condition=Condition(equal_to=0), + description="Order IDs must always be present.", + ) + operator, hook = make_operator( + records=[(described_rule.rule_uid, 0)], + ruleset=RuleSet(name="orders", rules=(described_rule,)), + ) + mock_get_db_hook.return_value = hook + + operator.execute(make_context()) + + history = results_backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid=described_rule.rule_uid + )["items"] + assert history[0]["description"] == "Order IDs must always be present." + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_fail_on_warn_fails_task_on_warn_failure(self, mock_get_db_hook): + operator, hook = make_operator( + records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 5)], fail_on="warn" + ) + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="volume"): + operator.execute(make_context()) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_fail_on_never_records_failures_without_raising(self, mock_get_db_hook): + operator, hook = make_operator( + records=[(NULLS.rule_uid, 3), (VOLUME_WARN.rule_uid, 5)], fail_on="never" + ) + mock_get_db_hook.return_value = hook + + summary = operator.execute(make_context()) + + assert summary["failed"] == 1 + assert summary["warned"] == 1 + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_execution_error_always_fails_task(self, mock_get_db_hook): + """A connection that's actually down fails the batch query and every per-rule fallback query.""" + operator, hook = make_operator(records=None, fail_on="never") + hook.get_records.side_effect = RuntimeError("boom") + hook.get_first.side_effect = RuntimeError("boom") + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="errored"): + operator.execute(make_context()) + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_non_numeric_observed_value_is_persisted_as_error(self, mock_get_db_hook, results_backend): + operator, hook = make_operator(records=[(NULLS.rule_uid, "not-a-number")], fail_on="never") + mock_get_db_hook.return_value = hook + + with pytest.raises(DQCheckFailedError, match="errored"): + operator.execute(make_context()) + + history = results_backend.read_task_rule_history( + dag_id="orders_pipeline", task_id="dq", rule_uid=NULLS.rule_uid + )["items"] + assert history[0]["status"] == "error" + assert "Could not evaluate observed value" in history[0]["error_message"] + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_backend_failure_does_not_fail_the_check(self, mock_get_db_hook): + backend = mock.create_autospec(ObjectStorageResultsBackend, instance=True) + backend.write_run.side_effect = OSError("bucket unreachable") + operator, hook = make_operator(records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 500)]) + mock_get_db_hook.return_value = hook + + with mock.patch( + "airflow.providers.common.dataquality.execution.get_backend_from_config", + return_value=backend, + ): + summary = operator.execute(make_context()) + + assert summary["passed"] == 2 + backend.write_run.assert_called_once() + + @mock.patch.object(DQCheckOperator, "get_db_hook") + def test_summary_attached_to_outlet_events(self, mock_get_db_hook): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="orders", + ) + operator, hook = make_operator( + records=[(NULLS.rule_uid, 0), (VOLUME_WARN.rule_uid, 500)], asset=asset + ) + mock_get_db_hook.return_value = hook + context = make_context() + + summary = operator.execute(context) + + outlet_events = context["outlet_events"] + outlet_events.__getitem__.assert_called_with(asset) + extra = outlet_events.__getitem__.return_value.extra + extra.__setitem__.assert_called_once_with("airflow.dataquality.result", summary) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/rules/__init__.py b/providers/common/dataquality/tests/unit/common/dataquality/rules/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/rules/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/common/dataquality/tests/unit/common/dataquality/rules/test_checks.py b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_checks.py new file mode 100644 index 0000000000000..a06ae0ef38e02 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_checks.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.common.dataquality.rules import CHECK_SPECS + + +class TestCheckSpecs: + def test_every_column_check_expression_renders(self): + for name, spec in CHECK_SPECS.items(): + if spec.requires_column: + assert "{column}" in spec.expression, name + + def test_row_count_is_the_only_table_level_check(self): + table_level = {name for name, spec in CHECK_SPECS.items() if not spec.requires_column} + assert table_level == {"row_count"} diff --git a/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py new file mode 100644 index 0000000000000..50c4ddd13e8e1 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/rules/test_rule.py @@ -0,0 +1,372 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +import airflow.providers.common.dataquality +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet, Severity, describe_rule + + +def test_skill_reference_schema_matches_live_model(): + """ + The dataquality-rule-authoring skill ships a generated RuleSet.model_json_schema() snapshot for + agents to consult. Guard against it silently drifting out of sync with the real model -- + regenerate with ``json.dumps(RuleSet.model_json_schema(), indent=2)`` (plus a trailing + newline) if this fails after an intentional schema change. + """ + schema_path = ( + Path(airflow.providers.common.dataquality.__file__).parent + / "skills" + / "dataquality-rule-authoring" + / "references" + / "ruleset.schema.json" + ) + checked_in = json.loads(schema_path.read_text()) + assert checked_in == RuleSet.model_json_schema() + + +class TestCondition: + @pytest.mark.parametrize( + ("condition", "observed", "expected"), + [ + ({"equal_to": 0}, None, False), + ({"equal_to": 0}, 0, True), + ({"equal_to": 0}, 1, False), + ({"equal_to": 100, "tolerance": 0.1}, 105, True), + ({"equal_to": 100, "tolerance": 0.1}, 111, False), + ({"equal_to": -100, "tolerance": 0.1}, -95, True), + ({"equal_to": -100, "tolerance": 0.1}, -105, True), + ({"equal_to": -100, "tolerance": 0.1}, -89, False), + ({"equal_to": -100, "tolerance": 0.1}, -111, False), + ({"geq_to": 10, "tolerance": 0.1}, 9, True), + ({"geq_to": 10, "tolerance": 0.1}, 8.9, False), + ({"geq_to": -1000, "tolerance": 0.1}, -1000, True), + ({"geq_to": -1000, "tolerance": 0.1}, -1100, True), + ({"geq_to": -1000, "tolerance": 0.1}, -1101, False), + ({"greater_than": 5}, 6, True), + ({"greater_than": 5}, 5, False), + ({"greater_than": 10, "tolerance": 0.1}, 9.1, True), + ({"greater_than": 10, "tolerance": 0.1}, 9, False), + ({"greater_than": -100, "tolerance": 0.1}, -109.9, True), + ({"greater_than": -100, "tolerance": 0.1}, -110, False), + ({"leq_to": 5}, 6, False), + ({"leq_to": 10, "tolerance": 0.1}, 11, True), + ({"leq_to": 10, "tolerance": 0.1}, 11.1, False), + ({"leq_to": -10, "tolerance": 0.1}, -10, True), + ({"leq_to": -10, "tolerance": 0.1}, -9, True), + ({"leq_to": -10, "tolerance": 0.1}, -8.9, False), + ({"less_than": 5}, 4, True), + ({"less_than": 10, "tolerance": 0.1}, 10.9, True), + ({"less_than": 10, "tolerance": 0.1}, 11, False), + ({"less_than": -100, "tolerance": 0.1}, -90.1, True), + ({"less_than": -100, "tolerance": 0.1}, -90, False), + ({"geq_to": 0, "leq_to": 10}, 5, True), + ({"geq_to": 0, "leq_to": 10}, 11, False), + ({"geq_to": 10, "leq_to": 20, "tolerance": 0.1}, 9, True), + ({"geq_to": 10, "leq_to": 20, "tolerance": 0.1}, 22, True), + ({"geq_to": 10, "leq_to": 20, "tolerance": 0.1}, 8.9, False), + ({"geq_to": 10, "leq_to": 20, "tolerance": 0.1}, 22.1, False), + ({"geq_to": -20, "leq_to": -10, "tolerance": 0.1}, -22, True), + ({"geq_to": -20, "leq_to": -10, "tolerance": 0.1}, -9, True), + ({"geq_to": -20, "leq_to": -10, "tolerance": 0.1}, -22.1, False), + ({"geq_to": -20, "leq_to": -10, "tolerance": 0.1}, -8.9, False), + ], + ) + def test_evaluate(self, condition, observed, expected): + assert Condition.from_dict(condition).evaluate(observed) is expected + + @pytest.mark.parametrize( + "condition", + [ + {}, + {"tolerance": 0.1}, + {"equal_to": 1, "greater_than": 0}, + {"nonsense": 1}, + ], + ) + def test_invalid_conditions_rejected(self, condition): + with pytest.raises(ValidationError): + Condition.from_dict(condition) + + def test_to_dict_round_trip(self): + condition = Condition.from_dict({"geq_to": 1, "leq_to": 2}) + assert Condition.from_dict(condition.to_dict()) == condition + + +class TestDQRule: + def test_condition_dict_is_coerced(self): + rule = DQRule(name="r", check="null_count", column="c", condition={"equal_to": 0}) # type: ignore[arg-type] + assert isinstance(rule.condition, Condition) + + @pytest.mark.parametrize( + "condition", + [ + {"nonsense": 1}, + {"tolerance": 0.1}, + {"equal_to": 1, "greater_than": 0}, + ], + ) + def test_malformed_condition_dict_rejected_at_construction(self, condition): + with pytest.raises(ValidationError): + DQRule(name="r", check="null_count", column="c", condition=condition) + + def test_rule_uid_is_stable_across_severity(self): + base = DQRule(name="r", check="null_count", column="c", condition=Condition(equal_to=0)) + tweaked = DQRule( + name="r", + check="null_count", + column="c", + condition=Condition(equal_to=0), + severity=Severity.WARN, + ) + assert base.rule_uid == tweaked.rule_uid + + def test_rule_uid_is_stable_across_description(self): + base = DQRule(name="r", check="null_count", column="c", condition=Condition(equal_to=0)) + described = DQRule( + name="r", + check="null_count", + column="c", + condition=Condition(equal_to=0), + description="Column c must be populated.", + ) + assert base.rule_uid == described.rule_uid + + def test_rule_uid_changes_with_condition(self): + one = DQRule(name="r", check="null_count", column="c", condition=Condition(equal_to=0)) + two = DQRule(name="r", check="null_count", column="c", condition=Condition(equal_to=1)) + assert one.rule_uid != two.rule_uid + + def test_previous_name_keeps_uid(self): + old = DQRule(name="old_name", check="null_count", column="c", condition=Condition(equal_to=0)) + renamed = DQRule( + name="new_name", + check="null_count", + column="c", + condition=Condition(equal_to=0), + previous_name="old_name", + ) + assert renamed.rule_uid == old.rule_uid + + def test_previous_name_can_collide_with_another_rules_identity(self): + """ + Documents the known collision this ``rule_uid`` scheme can produce. + + A rule renamed away from ``old_name`` derives the same uid as another, unrelated rule + that is still actually named ``old_name`` with the same check/column/condition. Set an + explicit ``id`` on one of them to avoid this (see ``test_explicit_id_avoids_collision``). + """ + renamed = DQRule( + name="new_name", + check="null_count", + column="c", + condition=Condition(equal_to=0), + previous_name="old_name", + ) + unrelated = DQRule(name="old_name", check="null_count", column="c", condition=Condition(equal_to=0)) + assert renamed.rule_uid == unrelated.rule_uid + + def test_explicit_id_is_used_directly_as_rule_uid(self): + rule = DQRule( + name="r", check="null_count", column="c", condition=Condition(equal_to=0), id="my-stable-id" + ) + assert rule.rule_uid == "my-stable-id" + + def test_explicit_id_avoids_collision(self): + renamed = DQRule( + name="new_name", + check="null_count", + column="c", + condition=Condition(equal_to=0), + previous_name="old_name", + id="renamed-rule", + ) + unrelated = DQRule(name="old_name", check="null_count", column="c", condition=Condition(equal_to=0)) + assert renamed.rule_uid != unrelated.rule_uid + + def test_explicit_id_survives_condition_and_previous_name_changes(self): + first = DQRule(name="r", check="null_count", column="c", condition=Condition(equal_to=0), id="stable") + tightened = DQRule( + name="r", check="null_count", column="c", condition=Condition(equal_to=5), id="stable" + ) + assert first.rule_uid == tightened.rule_uid + + def test_id_is_serialized_and_round_trips(self): + rule = DQRule( + name="r", check="null_count", column="c", condition=Condition(equal_to=0), id="my-stable-id" + ) + assert rule.to_dict()["id"] == "my-stable-id" + assert DQRule.from_dict(rule.to_dict()) == rule + + @pytest.mark.parametrize( + "kwargs", + [ + {"name": "", "check": "row_count", "condition": {"equal_to": 0}}, + {"name": "r", "check": "no_such_check", "condition": {"equal_to": 0}}, + {"name": "r", "check": "null_count", "condition": {"equal_to": 0}}, # missing column + {"name": "r", "check": "custom_sql", "condition": {"equal_to": 0}}, # missing sql + {"name": "r", "check": "row_count", "condition": {"equal_to": 0}, "sql": "SELECT 1"}, + {"name": "r", "check": "row_count", "condition": {"equal_to": 0}, "severity": "fatal"}, + ], + ) + def test_invalid_rules_rejected(self, kwargs): + with pytest.raises(ValidationError): + DQRule(**kwargs) + + def test_row_count_needs_no_column(self): + rule = DQRule(name="volume", check="row_count", condition=Condition(greater_than=0)) + assert rule.column is None + + @pytest.mark.parametrize( + ("check", "column", "expected_dimension"), + [ + ("null_count", "c", "completeness"), + ("null_ratio", "c", "completeness"), + ("distinct_count", "c", "uniqueness"), + ("unique_violations", "c", "uniqueness"), + ("min", "c", "validity"), + ("max", "c", "validity"), + ("mean", "c", "validity"), + ("row_count", None, "volume"), + ], + ) + def test_default_dimension_comes_from_check_catalog(self, check, column, expected_dimension): + rule = DQRule(name="r", check=check, column=column, condition=Condition(equal_to=0)) + assert rule.dimension.value == expected_dimension + + def test_custom_sql_dimension_can_be_overridden(self): + rule = DQRule( + name="freshness_check", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE updated_at < NOW() - INTERVAL '1 day'", + condition=Condition(equal_to=0), + dimension="freshness", + ) + assert rule.dimension.value == "freshness" + + def test_invalid_dimension_rejected(self): + with pytest.raises(ValidationError): + DQRule( + name="r", check="row_count", condition=Condition(greater_than=0), dimension="not_a_dimension" + ) + + def test_explicit_dimension_matching_default_is_omitted_from_to_dict(self): + rule = DQRule( + name="r", + check="null_count", + column="c", + condition=Condition(equal_to=0), + dimension="completeness", + ) + assert "dimension" not in rule.to_dict() + + def test_dimension_override_round_trips(self): + rule = DQRule( + name="freshness_check", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE updated_at < NOW() - INTERVAL '1 day'", + condition=Condition(equal_to=0), + dimension="freshness", + ) + assert rule.to_dict()["dimension"] == "freshness" + assert DQRule.from_dict(rule.to_dict()) == rule + + def test_missing_condition_without_catalog_default_raises(self): + with pytest.raises(ValidationError, match="condition is required"): + DQRule(name="r", check="null_count", column="c") + + def test_to_dict_round_trip(self): + rule = DQRule( + name="r", + check="custom_sql", + sql="SELECT COUNT(*) FROM {table} WHERE x < 0", + condition=Condition(equal_to=0), + severity=Severity.WARN, + partition_clause="ds = '2026-07-04'", + description="No rows should have negative x.", + ) + assert DQRule.from_dict(rule.to_dict()) == rule + + def test_description_is_serialized(self): + rule = DQRule( + name="r", + check="row_count", + condition=Condition(greater_than=0), + description="Orders table should not be empty.", + ) + assert rule.to_dict()["description"] == "Orders table should not be empty." + + def test_default_description_uses_column_when_available(self): + rule = DQRule(name="amount_min", check="min", column="amount", condition=Condition(geq_to=0)) + assert describe_rule(rule) == "amount should be greater than or equal to 0.0" + + +class TestRuleSet: + def test_duplicate_rule_names_rejected(self): + rule = DQRule(name="r", check="row_count", condition=Condition(greater_than=0)) + with pytest.raises(ValidationError, match="Duplicate rule names"): + RuleSet(name="s", rules=(rule, rule)) + + def test_colliding_rule_uids_rejected(self): + renamed = DQRule( + name="new_name", + check="null_count", + column="c", + condition=Condition(equal_to=0), + previous_name="old_name", + ) + unrelated = DQRule(name="old_name", check="null_count", column="c", condition=Condition(equal_to=0)) + with pytest.raises(ValidationError, match="collide on rule_uid"): + RuleSet(name="s", rules=(renamed, unrelated)) + + def test_from_dict_round_trip(self): + ruleset = RuleSet( + name="orders", + rules=( + DQRule(name="volume", check="row_count", condition=Condition(greater_than=0)), + DQRule(name="ids", check="null_count", column="id", condition=Condition(equal_to=0)), + ), + ) + assert RuleSet.from_dict(ruleset.to_dict()) == ruleset + + def test_from_file(self, tmp_path): + ruleset_file = tmp_path / "orders.yaml" + ruleset_file.write_text( + """ + name: orders + rules: + - name: ids_not_null + check: null_count + column: id + condition: + equal_to: 0 + - name: volume + check: row_count + condition: + greater_than: 100 + severity: warn + """ + ) + ruleset = RuleSet.from_file(str(ruleset_file)) + assert ruleset.name == "orders" + assert [rule.name for rule in ruleset.rules] == ["ids_not_null", "volume"] + assert ruleset.rules[1].severity.value == "warn" diff --git a/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py b/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py new file mode 100644 index 0000000000000..95a3310e50f34 --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_assets.py @@ -0,0 +1,195 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.common.dataquality.assets import ( + DQ_EXTRA_KEY, + DQ_RESULT_EXTRA_KEY, + _quality_score_passes, + asset_quality, + get_asset_quality_config, + get_asset_ruleset, + require_quality, +) +from airflow.providers.common.dataquality.exceptions import DQRuleValidationError +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet +from airflow.sdk import Asset +from airflow.sdk.execution_time.comms import AssetEventDagRunReferenceResult +from airflow.sdk.execution_time.context import TriggeringAssetEventsAccessor + +RULESET = RuleSet( + name="orders", + rules=(DQRule(name="volume", check="row_count", condition=Condition(greater_than=0)),), +) + +ORDERS = Asset("orders") + + +def make_event( + *, + extra: dict, + asset: Asset = ORDERS, + run_id: str = "manual__2026-07-04", + timestamp: str = "2026-07-04T06:00:00Z", +) -> AssetEventDagRunReferenceResult: + event = { + "asset": {"name": asset.name, "uri": asset.uri, "extra": {}}, + "extra": extra, + "source_task_id": "dq", + "source_dag_id": "orders_pipeline", + "source_run_id": run_id, + "source_map_index": -1, + "source_aliases": [], + "timestamp": timestamp, + } + return AssetEventDagRunReferenceResult.model_validate(event) + + +def make_triggering_events(*, extra: dict, asset: Asset = ORDERS) -> TriggeringAssetEventsAccessor: + return TriggeringAssetEventsAccessor.build([make_event(extra=extra, asset=asset)]) + + +class TestAssetQuality: + def test_config_stored_under_dq_extra_key(self): + asset = asset_quality( + Asset("orders", uri="postgres://wh/warehouse/analytics/orders"), + ruleset=RULESET, + conn_id="warehouse", + table="analytics.orders", + ) + config = asset.extra[DQ_EXTRA_KEY] + assert config["conn_id"] == "warehouse" + assert config["table"] == "analytics.orders" + assert config["ruleset"] == RULESET.to_dict() + + def test_config_is_json_serializable(self): + import json + + asset = asset_quality(Asset("orders"), ruleset=RULESET) + json.dumps(asset.extra) + + def test_ruleset_round_trips_through_asset(self): + asset = asset_quality(Asset("orders"), ruleset=RULESET) + assert get_asset_ruleset(asset) == RULESET + + def test_ruleset_from_yaml_file(self, tmp_path): + ruleset_file = tmp_path / "orders.yaml" + ruleset_file.write_text( + """ + name: orders + rules: + - name: volume + check: row_count + condition: + greater_than: 0 + """ + ) + asset = asset_quality(Asset("orders"), ruleset=str(ruleset_file)) + assert get_asset_ruleset(asset).rules[0].name == "volume" + + def test_get_config_returns_none_without_quality(self): + assert get_asset_quality_config(Asset("plain")) is None + + def test_get_ruleset_raises_without_quality(self): + with pytest.raises(DQRuleValidationError, match="no data quality config"): + get_asset_ruleset(Asset("plain")) + + +class TestQualityScorePasses: + """ + Covers the pure decision logic behind ``require_quality()``. + + Kept independent of ``@task.short_circuit`` / Dag wiring, which is exercised by the + standard provider's own tests — this only needs to prove "given this triggering event, + does the gate pass or fail". + """ + + def test_passes_when_score_at_min_score(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"score": 0.95}}) + assert _quality_score_passes(ORDERS, 0.95, events) is True + + def test_fails_when_score_below_min_score(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"score": 0.8}}) + assert _quality_score_passes(ORDERS, 0.95, events) is False + + def test_fails_when_no_triggering_event_for_asset(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"score": 1.0}}, asset=Asset("other")) + assert _quality_score_passes(ORDERS, 0.5, events) is False + + def test_fails_when_event_has_no_dq_summary(self): + events = make_triggering_events(extra={}) + assert _quality_score_passes(ORDERS, 0.5, events) is False + + def test_fails_when_summary_has_no_score(self): + events = make_triggering_events(extra={DQ_RESULT_EXTRA_KEY: {"passed": 3}}) + assert _quality_score_passes(ORDERS, 0.5, events) is False + + def test_require_all_fails_when_any_event_is_below_min_score(self): + low = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.4}}, run_id="run1", timestamp="2026-07-03T06:00:00Z" + ) + high = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.99}}, run_id="run2", timestamp="2026-07-04T06:00:00Z" + ) + events = TriggeringAssetEventsAccessor.build([low, high]) + + assert _quality_score_passes(ORDERS, 0.9, events) is False + + def test_require_all_passes_when_every_event_meets_min_score(self): + first = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.95}}, run_id="run1", timestamp="2026-07-03T06:00:00Z" + ) + second = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.99}}, run_id="run2", timestamp="2026-07-04T06:00:00Z" + ) + events = TriggeringAssetEventsAccessor.build([first, second]) + + assert _quality_score_passes(ORDERS, 0.9, events) is True + + def test_latest_only_uses_most_recent_event_when_require_all_false(self): + first = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.4}}, run_id="run1", timestamp="2026-07-03T06:00:00Z" + ) + second = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.99}}, run_id="run2", timestamp="2026-07-04T06:00:00Z" + ) + events = TriggeringAssetEventsAccessor.build([first, second]) + + assert _quality_score_passes(ORDERS, 0.9, events, require_all=False) is True + + def test_latest_only_fails_when_most_recent_event_is_below_min_score(self): + first = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.99}}, run_id="run1", timestamp="2026-07-03T06:00:00Z" + ) + second = make_event( + extra={DQ_RESULT_EXTRA_KEY: {"score": 0.4}}, run_id="run2", timestamp="2026-07-04T06:00:00Z" + ) + events = TriggeringAssetEventsAccessor.build([first, second]) + + assert _quality_score_passes(ORDERS, 0.9, events, require_all=False) is False + + +class TestRequireQuality: + def test_rejects_min_score_below_zero(self): + with pytest.raises(ValueError, match="min_score must be between 0 and 1"): + require_quality(ORDERS, min_score=-0.1) + + def test_rejects_min_score_above_one(self): + with pytest.raises(ValueError, match="min_score must be between 0 and 1"): + require_quality(ORDERS, min_score=1.1) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/test_exceptions.py b/providers/common/dataquality/tests/unit/common/dataquality/test_exceptions.py new file mode 100644 index 0000000000000..712a8775a0a0c --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_exceptions.py @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from airflow.providers.common.dataquality.exceptions import DQCheckFailedError, DQRuleValidationError + + +def test_dq_rule_validation_error_is_value_error(): + assert isinstance(DQRuleValidationError("invalid rule"), ValueError) + + +def test_dq_check_failed_error_is_runtime_error(): + assert isinstance(DQCheckFailedError("failed check"), RuntimeError) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/test_execution.py b/providers/common/dataquality/tests/unit/common/dataquality/test_execution.py new file mode 100644 index 0000000000000..10bc6921f928b --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_execution.py @@ -0,0 +1,285 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow.providers.common.dataquality.backends.object_storage import ObjectStorageResultsBackend +from airflow.providers.common.dataquality.engines.sql import Observation +from airflow.providers.common.dataquality.execution import ( + _attach_to_outlet_events, + _build_run_from_context, + _evaluate_observation, + _get_hook, + _get_outlet_asset_names, + _resolve_ruleset, + persist_quality_results, + run_quality_checks, +) +from airflow.providers.common.dataquality.rules import Condition, DQRule, RuleSet +from airflow.providers.common.sql.hooks.sql import DbApiHook +from airflow.sdk import Asset +from airflow.sdk.execution_time.context import OutletEventAccessors + +ORDER_ID_NOT_NULL = DQRule( + name="order_id_not_null", + check="null_count", + column="order_id", + condition=Condition(equal_to=0), +) +ORDER_AMOUNT_VALID = DQRule( + name="order_amount_valid", + check="min", + column="amount", + condition=Condition(geq_to=0), +) +ORDER_RULESET = RuleSet(name="orders_quality", rules=(ORDER_ID_NOT_NULL, ORDER_AMOUNT_VALID)) + + +def make_context(): + ti = mock.Mock(spec=["dag_id", "task_id", "run_id", "try_number", "map_index"]) + ti.dag_id = "orders_pipeline" + ti.task_id = "custom_quality_task" + ti.run_id = "manual__2026-07-04" + ti.try_number = 1 + ti.map_index = -1 + return {"ti": ti, "outlet_events": mock.create_autospec(OutletEventAccessors, instance=True)} + + +class TestRunQualityChecks: + def test_runs_rules_with_supplied_hook(self): + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = [ + (ORDER_ID_NOT_NULL.rule_uid, 0), + (ORDER_AMOUNT_VALID.rule_uid, 5), + ] + + result = run_quality_checks(hook=hook, table="orders", ruleset=ORDER_RULESET) + + assert result.ruleset == ORDER_RULESET + assert result.table == "orders" + assert [rule_result.status for rule_result in result.results] == ["pass", "pass"] + hook.get_records.assert_called_once() + + @mock.patch("airflow.providers.common.dataquality.execution.BaseHook.get_connection", autospec=True) + def test_runs_rules_with_conn_id(self, mock_get_connection): + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = [(ORDER_ID_NOT_NULL.rule_uid, 1)] + connection = mock.Mock(spec=["get_hook"]) + connection.get_hook.return_value = hook + mock_get_connection.return_value = connection + + result = run_quality_checks( + conn_id="warehouse", + hook_params={"schema": "analytics"}, + table="orders", + ruleset=RuleSet(name="orders_quality", rules=(ORDER_ID_NOT_NULL,)), + ) + + assert result.results[0].status == "fail" + mock_get_connection.assert_called_once_with("warehouse") + connection.get_hook.assert_called_once_with(hook_params={"schema": "analytics"}) + + def test_returns_error_result_when_value_cannot_be_evaluated(self): + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = [(ORDER_ID_NOT_NULL.rule_uid, "not-a-number")] + + result = run_quality_checks( + hook=hook, + table="orders", + ruleset=RuleSet(name="orders_quality", rules=(ORDER_ID_NOT_NULL,)), + ) + + assert result.results[0].status == "error" + assert "Could not evaluate observed value" in result.results[0].error_message + + +class TestPersistQualityResults: + def test_persists_results_for_custom_task(self, tmp_path): + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = [ + (ORDER_ID_NOT_NULL.rule_uid, 0), + (ORDER_AMOUNT_VALID.rule_uid, 5), + ] + result = run_quality_checks(hook=hook, table="orders", ruleset=ORDER_RULESET) + backend = ObjectStorageResultsBackend(results_path=f"file://{tmp_path}") + + with mock.patch( + "airflow.providers.common.dataquality.execution.get_backend_from_config", + return_value=backend, + ): + summary = persist_quality_results(result, context=make_context()) + + assert summary["passed"] == 2 + history = backend.read_task_rule_history( + dag_id="orders_pipeline", + task_id="custom_quality_task", + rule_uid=ORDER_ID_NOT_NULL.rule_uid, + )["items"] + assert len(history) == 1 + assert history[0]["status"] == "pass" + + def test_attaches_summary_to_outlet_asset_event(self): + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = [(ORDER_ID_NOT_NULL.rule_uid, 0)] + result = run_quality_checks( + hook=hook, + table="orders", + ruleset=RuleSet(name="orders_quality", rules=(ORDER_ID_NOT_NULL,)), + ) + asset = Asset("orders") + context = make_context() + + with mock.patch( + "airflow.providers.common.dataquality.execution.get_backend_from_config", return_value=None + ): + summary = persist_quality_results(result, context=context, outlets=[asset]) + + outlet_events = context["outlet_events"] + outlet_events.__getitem__.assert_called_with(asset) + outlet_events.__getitem__.return_value.extra.__setitem__.assert_called_once_with( + "airflow.dataquality.result", summary + ) + + def test_backend_failure_does_not_fail_custom_task(self): + hook = mock.create_autospec(DbApiHook, instance=True) + hook.get_records.return_value = [(ORDER_ID_NOT_NULL.rule_uid, 0)] + result = run_quality_checks( + hook=hook, + table="orders", + ruleset=RuleSet(name="orders_quality", rules=(ORDER_ID_NOT_NULL,)), + ) + backend = mock.create_autospec(ObjectStorageResultsBackend, instance=True) + backend.write_run.side_effect = OSError("bucket unreachable") + + with mock.patch( + "airflow.providers.common.dataquality.execution.get_backend_from_config", + return_value=backend, + ): + summary = persist_quality_results(result, context=make_context()) + + assert summary["passed"] == 1 + backend.write_run.assert_called_once() + + +class TestPrivateHelpers: + @pytest.mark.parametrize( + "ruleset_arg", + [ + ORDER_RULESET, + ORDER_RULESET.to_dict(), + ], + ) + def test_resolve_ruleset_from_model_or_dict(self, ruleset_arg): + assert _resolve_ruleset(ruleset_arg) == ORDER_RULESET + + def test_resolve_ruleset_from_file(self, tmp_path): + ruleset_file = tmp_path / "orders_ruleset.yaml" + ruleset_file.write_text( + """ +name: orders_quality +rules: + - name: order_id_not_null + check: null_count + column: order_id + condition: + equal_to: 0 +""", + ) + + ruleset = _resolve_ruleset(str(ruleset_file)) + + assert ruleset.name == "orders_quality" + assert ruleset.rules[0].name == "order_id_not_null" + + def test_get_hook_requires_conn_id_when_hook_is_not_supplied(self): + with pytest.raises(ValueError, match="Either conn_id or hook is required"): + _get_hook(None, None) + + @mock.patch("airflow.providers.common.dataquality.execution.BaseHook.get_connection", autospec=True) + def test_get_hook_uses_conn_id_and_hook_params(self, mock_get_connection): + hook = mock.create_autospec(DbApiHook, instance=True) + connection = mock.Mock(spec=["get_hook"]) + connection.get_hook.return_value = hook + mock_get_connection.return_value = connection + + assert _get_hook("warehouse", {"schema": "analytics"}) == hook + mock_get_connection.assert_called_once_with("warehouse") + connection.get_hook.assert_called_once_with(hook_params={"schema": "analytics"}) + + def test_evaluate_observation_records_warn_for_warn_severity_failure(self): + warn_rule = DQRule( + name="volume", + check="row_count", + condition=Condition(greater_than=100), + severity="warn", + ) + + result = _evaluate_observation(Observation(rule=warn_rule, observed_value=5, duration_ms=12.3)) + + assert result.status == "warn" + assert result.observed_value == 5 + assert result.duration_ms == 12.3 + + def test_evaluate_observation_records_error_for_query_error(self): + result = _evaluate_observation(Observation(rule=ORDER_ID_NOT_NULL, error_message="query failed")) + + assert result.status == "error" + assert result.error_message == "query failed" + + def test_build_run_from_context_uses_task_metadata_and_outlets(self): + context = make_context() + context["ti"].map_index = None + asset = Asset("orders") + + run = _build_run_from_context( + context=context, + ruleset=ORDER_RULESET, + table="orders", + outlets=[asset], + started_at="2026-07-04T12:00:00+00:00", + finished_at="2026-07-04T12:00:01+00:00", + ) + + assert run.dag_id == "orders_pipeline" + assert run.task_id == "custom_quality_task" + assert run.map_index == -1 + assert run.ruleset_name == "orders_quality" + assert run.table_ref == "orders" + assert run.asset_names == ("orders",) + assert run.started_at == "2026-07-04T12:00:00+00:00" + assert run.finished_at == "2026-07-04T12:00:01+00:00" + + def test_get_outlet_asset_names_ignores_outlets_without_names(self): + named_asset = Asset("orders") + unnamed_outlet = mock.Mock(spec=[]) + + assert _get_outlet_asset_names([named_asset, unnamed_outlet]) == ["orders"] + + def test_attach_to_outlet_events_noops_without_context_key(self): + _attach_to_outlet_events({}, [Asset("orders")], {"score": 1.0}) + + def test_attach_to_outlet_events_ignores_single_outlet_failure(self): + asset = Asset("orders") + outlet_events = mock.create_autospec(OutletEventAccessors, instance=True) + outlet_events.__getitem__.side_effect = RuntimeError("not available") + + _attach_to_outlet_events({"outlet_events": outlet_events}, [asset], {"score": 1.0}) + + outlet_events.__getitem__.assert_called_once_with(asset) diff --git a/providers/common/dataquality/tests/unit/common/dataquality/test_results.py b/providers/common/dataquality/tests/unit/common/dataquality/test_results.py new file mode 100644 index 0000000000000..3490a55af087d --- /dev/null +++ b/providers/common/dataquality/tests/unit/common/dataquality/test_results.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.common.dataquality.results import DQRun, RuleResult, build_summary, compute_score + + +def make_result(status: str, name: str = "r") -> RuleResult: + return RuleResult(rule_uid=f"uid-{name}", rule_name=name, status=status) + + +class TestComputeScore: + @pytest.mark.parametrize( + ("statuses", "expected"), + [ + ([], None), + (["pass", "pass"], 1.0), + (["pass", "fail"], 0.5), + (["pass", "error"], 0.5), + (["pass", "pass", "pass", "warn"], 0.9375), + (["fail", "fail"], 0.0), + ], + ) + def test_score(self, statuses, expected): + results = [make_result(status, name=f"r{i}") for i, status in enumerate(statuses)] + assert compute_score(results) == expected + + +class TestBuildSummary: + def test_summary_counts_and_rule_names(self): + run = DQRun(dag_id="d", task_id="t", run_id="r", ruleset_name="orders", table_ref="orders") + results = [ + make_result("pass", "a"), + make_result("warn", "b"), + make_result("fail", "c"), + make_result("error", "d"), + ] + + summary = build_summary(run=run, results=results) + + assert summary["passed"] == 1 + assert summary["warned"] == 1 + assert summary["failed"] == 1 + assert summary["errored"] == 1 + assert summary["failed_rules"] == ["c", "d"] + assert summary["warned_rules"] == ["b"] + assert summary["run_uid"] == run.run_uid + + +class TestSerialization: + def test_rule_result_round_trip(self): + result = RuleResult( + rule_uid="u", + rule_name="r", + status="fail", + observed_value=3, + condition={"equal_to": 0}, + description="r should equal 0", + duration_ms=1.5, + sql="SELECT 3", + ) + assert RuleResult.from_dict(result.to_dict()) == result + + def test_run_round_trip(self): + run = DQRun(dag_id="d", task_id="t", run_id="r", asset_names=("orders",)) + assert DQRun.from_dict(run.to_dict()) == run diff --git a/scripts/ci/prek/generate_common_dataquality_ruleset_schema.py b/scripts/ci/prek/generate_common_dataquality_ruleset_schema.py new file mode 100755 index 0000000000000..dab079ae734d9 --- /dev/null +++ b/scripts/ci/prek/generate_common_dataquality_ruleset_schema.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Regenerate the Data Quality RuleSet JSON schema snapshot at +``providers/common/dataquality/src/airflow/providers/common/dataquality/skills/dataquality-rule-authoring/references/ruleset.schema.json``. +""" + +from __future__ import annotations + +import subprocess +import sys + +from common_prek_utils import AIRFLOW_ROOT_PATH, console + +DQ_PROVIDER_PATH = AIRFLOW_ROOT_PATH / "providers" / "common" / "dataquality" +SCHEMA_PATH = DQ_PROVIDER_PATH.joinpath( + "src", + "airflow", + "providers", + "common", + "dataquality", + "skills", + "dataquality-rule-authoring", + "references", + "ruleset.schema.json", +) + +DUMP_SCHEMA = r""" +import json +import sys + +from airflow.providers.common.dataquality.rules import RuleSet + +sys.stdout.write(json.dumps(RuleSet.model_json_schema(), indent=2)) +sys.stdout.write("\n") +""" + + +def dump_schema() -> str: + """Run the schema-dump snippet in the dataquality provider project and return its stdout.""" + result = subprocess.run( + [ + "uv", + "run", + "--frozen", + "--no-progress", + "--project", + str(DQ_PROVIDER_PATH), + "python", + "-c", + DUMP_SCHEMA, + ], + cwd=DQ_PROVIDER_PATH, + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError(f"Schema generation failed: {result.stderr}") + return result.stdout + + +def main() -> int: + try: + new_content = dump_schema() + except Exception as e: + console.print(f"[bold red]ERROR:[/] {e}") + return 1 + + if SCHEMA_PATH.exists(): + old_content = SCHEMA_PATH.read_text() + if old_content == new_content: + return 0 + else: + SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True) + + SCHEMA_PATH.write_text(new_content) + rel = SCHEMA_PATH.relative_to(AIRFLOW_ROOT_PATH) + console.print(f"[yellow]Regenerated[/] [cyan]{rel}[/]. Please review the diff and re-stage the file.") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/uv.lock b/uv.lock index dc07848533e42..0983a47d28960 100644 --- a/uv.lock +++ b/uv.lock @@ -64,9 +64,9 @@ apache-airflow-providers-apache-cassandra = false apache-airflow-providers-asana = false apache-airflow-providers-oracle = false apache-airflow-providers-mysql = false +apache-airflow-providers-teradata = false apache-airflow-providers-alibaba = false apache-airflow-providers-microsoft-mssql = false -apache-airflow-providers-teradata = false apache-airflow-providers-jdbc = false apache-airflow-helm-chart = false apache-airflow-providers-anthropic = false @@ -330,7 +330,7 @@ name = "adbc-driver-manager" version = "1.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e9/5e/50aab18cb501e42d3aca3cd2cc26c6637094fcaf5b6576e350c444188f1f/adbc_driver_manager-1.11.0.tar.gz", hash = "sha256:c64aaabeb5810109ab3d2961008f1b014e9f2d87b3df4416c2a080a40237af50", size = 233059, upload-time = "2026-04-07T00:17:28.263Z" } wheels = [ @@ -375,8 +375,8 @@ name = "adbc-driver-postgresql" version = "1.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "adbc-driver-manager", marker = "python_full_version < '3.13'" }, - { name = "importlib-resources", marker = "python_full_version < '3.13'" }, + { name = "adbc-driver-manager" }, + { name = "importlib-resources" }, ] sdist = { url = "https://files.pythonhosted.org/packages/68/10/2962c25035887cd03af3b348eac3302493936f45048c220021a802d07f12/adbc_driver_postgresql-1.11.0.tar.gz", hash = "sha256:f5688b8648ac7a86d8b89340231bb3686ac5df56ee95d1ca0b875dad5d52b48a", size = 32328, upload-time = "2026-04-07T00:17:29.232Z" } wheels = [ @@ -392,8 +392,8 @@ name = "adbc-driver-sqlite" version = "1.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "adbc-driver-manager", marker = "python_full_version < '3.13'" }, - { name = "importlib-resources", marker = "python_full_version < '3.13'" }, + { name = "adbc-driver-manager" }, + { name = "importlib-resources" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3a/dd/8a5f4908aa4bdec64dcd672734fa314d692517458ce169591639d0123fe1/adbc_driver_sqlite-1.11.0.tar.gz", hash = "sha256:a4c6b4962610f7cd67cd754c42dd74e18a2c11fabeec9488c5501d73ae62dc62", size = 28885, upload-time = "2026-04-07T00:17:31.325Z" } wheels = [ @@ -456,7 +456,7 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "caio", marker = "python_full_version < '3.11'" }, + { name = "caio" }, ] sdist = { url = "https://files.pythonhosted.org/packages/67/e2/d7cb819de8df6b5c1968a2756c3cb4122d4fa2b8fc768b53b7c9e5edb646/aiofile-3.9.0.tar.gz", hash = "sha256:e5ad718bb148b265b6df1b3752c4d1d83024b93da9bd599df74b9d9ffcf7919b", size = 17943, upload-time = "2024-10-08T10:39:35.846Z" } wheels = [ @@ -480,7 +480,7 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "caio", marker = "python_full_version >= '3.11'" }, + { name = "caio" }, ] sdist = { url = "https://files.pythonhosted.org/packages/48/41/2fea7e193e061ce54eacc3b7bc0e6a99e4fcff43c78cf0a76dd781ed8334/aiofile-3.11.1.tar.gz", hash = "sha256:1f91912c6643d2a4e49ca4ae3514f0bf3867ce948a36d99a6411b8f4755f4cf9", size = 19342, upload-time = "2026-05-16T08:18:33.538Z" } wheels = [ @@ -964,16 +964,16 @@ vertex = [ [[package]] name = "anyio" -version = "4.14.1" +version = "4.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, { name = "idna" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3b/72/5562aabb8dd7181e8e860622a38bea08d17842b99ecd4c91f84ac95251b0/anyio-4.14.1.tar.gz", hash = "sha256:8d648a3544c1a700e3ff78615cd679e4c5c3f149904287e73687b2596963629e", size = 254831, upload-time = "2026-06-24T20:56:06.017Z" } +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/7b/90df4a0a816d98d6ea26f559d87836d494a2cf1fcf063be67df50a7bcc30/anyio-4.14.1-py3-none-any.whl", hash = "sha256:4e5533c5b8ff0a24f5d7a176cbe6877129cd183893f66b537f8f227d10527d72", size = 124875, upload-time = "2026-06-24T20:56:04.413Z" }, + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] [[package]] @@ -4582,12 +4582,18 @@ version = "0.1.0" source = { editable = "providers/common/dataquality" } dependencies = [ { name = "apache-airflow" }, + { name = "apache-airflow-providers-common-compat" }, + { name = "apache-airflow-providers-common-sql" }, + { name = "pydantic" }, + { name = "pyyaml" }, ] [package.dev-dependencies] dev = [ { name = "apache-airflow" }, { name = "apache-airflow-devel-common" }, + { name = "apache-airflow-providers-common-compat" }, + { name = "apache-airflow-providers-common-sql" }, { name = "apache-airflow-task-sdk" }, ] docs = [ @@ -4595,12 +4601,20 @@ docs = [ ] [package.metadata] -requires-dist = [{ name = "apache-airflow", editable = "." }] +requires-dist = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, + { name = "pydantic", specifier = ">=2.11.0" }, + { name = "pyyaml", specifier = ">=6.0.2" }, +] [package.metadata.requires-dev] dev = [ { name = "apache-airflow", editable = "." }, { name = "apache-airflow-devel-common", editable = "devel-common" }, + { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, + { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, { name = "apache-airflow-task-sdk", editable = "task-sdk" }, ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] @@ -4813,6 +4827,9 @@ dependencies = [ ] [package.optional-dependencies] +amazon = [ + { name = "apache-airflow-providers-amazon" }, +] avro = [ { name = "fastavro" }, ] @@ -4842,6 +4859,7 @@ standard = [ dev = [ { name = "apache-airflow" }, { name = "apache-airflow-devel-common" }, + { name = "apache-airflow-providers-amazon" }, { name = "apache-airflow-providers-common-compat" }, { name = "apache-airflow-providers-common-sql", extra = ["pandas", "polars"] }, { name = "apache-airflow-providers-databricks", extra = ["sqlalchemy"] }, @@ -4860,6 +4878,7 @@ docs = [ requires-dist = [ { name = "aiohttp", specifier = ">=3.14.0,<4" }, { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-providers-amazon", marker = "extra == 'amazon'", editable = "providers/amazon" }, { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, { name = "apache-airflow-providers-fab", marker = "extra == 'fab'", editable = "providers/fab" }, @@ -4882,12 +4901,13 @@ requires-dist = [ { name = "pyarrow", marker = "python_full_version >= '3.14'", specifier = ">=22.0.0" }, { name = "requests", specifier = ">=2.32.0,<3" }, ] -provides-extras = ["avro", "azure-identity", "fab", "google", "sdk", "standard", "openlineage", "sqlalchemy"] +provides-extras = ["avro", "amazon", "azure-identity", "fab", "google", "sdk", "standard", "openlineage", "sqlalchemy"] [package.metadata.requires-dev] dev = [ { name = "apache-airflow", editable = "." }, { name = "apache-airflow-devel-common", editable = "devel-common" }, + { name = "apache-airflow-providers-amazon", editable = "providers/amazon" }, { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, { name = "apache-airflow-providers-common-sql", extras = ["pandas", "polars"], editable = "providers/common/sql" }, @@ -10425,8 +10445,8 @@ name = "cassandra-driver" version = "3.30.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "deprecated", marker = "python_full_version < '3.14'" }, - { name = "geomet", marker = "python_full_version < '3.14'" }, + { name = "deprecated" }, + { name = "geomet" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bb/ed/4e16210e194660f107929ee494f1cf18557252655067d9b39029c241be2d/cassandra_driver-3.30.1.tar.gz", hash = "sha256:0c6a3e1428f7c6a9aa6c944b9c47a37cd2cdbeb5b5a82d42c33afdd56f14e398", size = 289233, upload-time = "2026-07-06T20:14:31.37Z" } wheels = [ @@ -11048,100 +11068,100 @@ wheels = [ [[package]] name = "coverage" -version = "7.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/8b/adeb62ea8951f13c4c7fef2e7a85e1a06b499c8d8237ea589d496029e53f/coverage-7.15.0.tar.gz", hash = "sha256:9ac3fe7a1435986463eaa8ee253ae2f2a268709ba4ae5c7dd1f52a05391ad78f", size = 925362, upload-time = "2026-07-02T13:10:50.535Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/97/c52dc440c390b6cfa87be9432b141a956e2d56d9b9f5fc8bd71c5f471722/coverage-7.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:50913d4bf5ddafa6ca3693da5e4dd833dd1b772e0283c99ca7f7d287db67331a", size = 220539, upload-time = "2026-07-02T13:08:19.252Z" }, - { url = "https://files.pythonhosted.org/packages/3f/26/602de8c2aec7e2e3e99ebfb8e04ba65598f746275396eea5f6794ff4673f/coverage-7.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:359e141ccd33893ce3f1ad5525f8b96083003677c82182e5907d62d4ea5799fc", size = 221058, upload-time = "2026-07-02T13:08:21.013Z" }, - { url = "https://files.pythonhosted.org/packages/fc/13/ebab0743138891c1d646d61e247ec29639afcbb6c4e1905e6a0f0c75291a/coverage-7.15.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3200b6204935f928c64b2ca1f923ab8c1acb7c9de45ec61569711b34d25cccaf", size = 247797, upload-time = "2026-07-02T13:08:22.474Z" }, - { url = "https://files.pythonhosted.org/packages/d3/b7/b6ffb9e042aa48dc4144a8a65529affaec8dca0685309353614a2a7386ad/coverage-7.15.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:be616bf61346883b2cfdc5178669647e03531d81ab761a7e378558b7e8bcb628", size = 249626, upload-time = "2026-07-02T13:08:23.803Z" }, - { url = "https://files.pythonhosted.org/packages/9c/06/243ff05b652333d8e3d060c11223efc2723b19cacf6605e433fa686ab5d4/coverage-7.15.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cc7bafc3fe1059463a8fdd97ca79972d6e2bf819d775c7d54991b5b1971201d6", size = 251493, upload-time = "2026-07-02T13:08:25.397Z" }, - { url = "https://files.pythonhosted.org/packages/d3/2b/867faa17030a806114dae388b32a3fa929d8cd4bf39226fbc11f6e6bb705/coverage-7.15.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b713aa7fcf325a01d4184d848acb46fd84f78fdb0978470c636b23a06a753d91", size = 253406, upload-time = "2026-07-02T13:08:26.842Z" }, - { url = "https://files.pythonhosted.org/packages/94/c0/d789ce18f6605afc4895db75723424be2ef494282f77f61d8e5832923183/coverage-7.15.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e38e6fba2d56652fdfaf0231f8f78aeb805234a912de25dc291ee5cce5b8faa4", size = 248512, upload-time = "2026-07-02T13:08:28.398Z" }, - { url = "https://files.pythonhosted.org/packages/c9/b6/b2673c30739f4a2e06649a0a38ad8b093c4d865462dc7bab0e9524a2c3b1/coverage-7.15.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:884499f42e382675be80770391983b90e0c0c774d87dbeeebf5f991cf6612b20", size = 249532, upload-time = "2026-07-02T13:08:29.731Z" }, - { url = "https://files.pythonhosted.org/packages/3c/2e/acd79e9a41beabee92b623afe4f30b549916f48566271475f2907e752828/coverage-7.15.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:840481b12e083dbcbafab14794a8781a958edf327c8d3d70b4eee42f9b8253aa", size = 247537, upload-time = "2026-07-02T13:08:31.173Z" }, - { url = "https://files.pythonhosted.org/packages/12/d4/2d301c4d1b3238d7c88b70ab9d13fd53ed9505662a7ff1b46ba1e2e4e3c3/coverage-7.15.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:276646e9481703d09f854f3b2f018f24e19fd7049ae670a92570043eb97203b1", size = 251348, upload-time = "2026-07-02T13:08:32.63Z" }, - { url = "https://files.pythonhosted.org/packages/35/bb/c67708b2bc00f32e12805ec23d5fa677a0a51652f449341a89f9d6b1b715/coverage-7.15.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4de4b4d3f5545aa6c60dc4efd9c63b5b5dcc3bf00fe83146b2bdfffb8f6613bd", size = 247806, upload-time = "2026-07-02T13:08:33.931Z" }, - { url = "https://files.pythonhosted.org/packages/eb/6c/57c4f653c47a6e917748f8938e389e72fbcae44e3643cd906664f0477a13/coverage-7.15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5c504097b2a89b1e85bc6070d920df77daec701337e3aeef2c17775a5dd0ca90", size = 248410, upload-time = "2026-07-02T13:08:35.189Z" }, - { url = "https://files.pythonhosted.org/packages/6c/94/bb083041aef828903668f134273f319f2bd49224962875359c52faa5497f/coverage-7.15.0-cp310-cp310-win32.whl", hash = "sha256:f6e80ed91f98316e86b9c137206b04b2bcfbffccbdff49bd2eb09dddb1cf14e0", size = 222588, upload-time = "2026-07-02T13:08:36.486Z" }, - { url = "https://files.pythonhosted.org/packages/ef/94/a09d8ee618956f626741b0734854bac4425a00e10c0565f5abca64e7e751/coverage-7.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:b3b3e22030f3f6f5e01a5ce69936552a5c0f6992b7698777377b99041961031f", size = 223214, upload-time = "2026-07-02T13:08:37.885Z" }, - { url = "https://files.pythonhosted.org/packages/ae/23/82e910835ef4b8391047025e1d53aa48d66029f444eb8b25373c849bf503/coverage-7.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:003fff99412ea848c0aaebcc78ed2b6ce7d8a1227ed17e68470672770b78a02a", size = 220662, upload-time = "2026-07-02T13:08:39.205Z" }, - { url = "https://files.pythonhosted.org/packages/6d/0d/c7b213dde2f1579de5231062b386d8413f79c11667eb58c39319b25991da/coverage-7.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5cbd804bf2784ce7b45114516050f346ecd50f960c4bb630a7ee9e1d78fa2118", size = 221168, upload-time = "2026-07-02T13:08:40.471Z" }, - { url = "https://files.pythonhosted.org/packages/33/77/d000aeedfac085088337b3c7becdad328474b1f8a9e4c9368a0c99605d68/coverage-7.15.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8773e15c23305b58882a4611fb9b2755977eae0dc2e515366a1b6c98866cc4c2", size = 251587, upload-time = "2026-07-02T13:08:42.033Z" }, - { url = "https://files.pythonhosted.org/packages/cc/e0/86787c56b9df17afd370d5e293515dd4d9a107a561d13054873eefad8ecc/coverage-7.15.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f50e40081494c1dc4239ebb202014cbcc3306ea96fb6302a34c8cc0967fc5ae8", size = 253497, upload-time = "2026-07-02T13:08:43.387Z" }, - { url = "https://files.pythonhosted.org/packages/3f/02/181bc917359299c07dead6270f94e411151c8b60cec905c33499da69afe6/coverage-7.15.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:daf96f37f5fc3a7b6c6da862eb4aee61c426bd63da236ed4a73ef0e503b4bca5", size = 255607, upload-time = "2026-07-02T13:08:44.897Z" }, - { url = "https://files.pythonhosted.org/packages/b9/35/ca5e7427699913da6788c4f910e73ab16c5f4b59ec5d3a999dce2a45112f/coverage-7.15.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:51aa20f6ae2788fd197747766edf4cd8234fd9423309b934257fa6b21a592723", size = 257563, upload-time = "2026-07-02T13:08:46.334Z" }, - { url = "https://files.pythonhosted.org/packages/0b/4d/b8220bacc2fc3c4e9078e27c32e99fb411479a4718a72bdd00036a9891c8/coverage-7.15.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:03d1f922757662eb7af586e77834792274cff776bc7b1d1a0b66a49ea9d84735", size = 251726, upload-time = "2026-07-02T13:08:47.941Z" }, - { url = "https://files.pythonhosted.org/packages/c4/e4/2e145da1991d72189b9c3cf7eca05c716ee7080d099aaea6757cfc7df008/coverage-7.15.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a6d6acc9a7666245e6133dd15144ca038a85a9cd5026bb06d6bbae9e77440dc9", size = 253301, upload-time = "2026-07-02T13:08:49.5Z" }, - { url = "https://files.pythonhosted.org/packages/72/28/d2c841d698bf762e481f08bd4839d370246b6d9b61dab085a7b20b201a08/coverage-7.15.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1ac2c4c27c7df851dc9a017c2d7de00b69147e84ba3d96f37a530b0b6fb51035", size = 251361, upload-time = "2026-07-02T13:08:51.304Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ed/55d9ffde994fba3897c0c783f77a7d053b0c18787f6892ed5b0aed73f469/coverage-7.15.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b761a1d504fd4bd1f20f418753964dca9f5862a511fc854dac58296b3b223671", size = 255129, upload-time = "2026-07-02T13:08:52.661Z" }, - { url = "https://files.pythonhosted.org/packages/1d/c0/ecbf33b8c460ea2718aeb813e2df8140d0370e5f67261c31524ceb0a2a8d/coverage-7.15.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:e43b045e11c16e897895758ae90e4a90cf99e93d58549e2f90c0e2272e155695", size = 251081, upload-time = "2026-07-02T13:08:54.188Z" }, - { url = "https://files.pythonhosted.org/packages/a9/de/fb87b4261f54448dd2b9504ef19a58be42cef0d9520595fbfe1219b15234/coverage-7.15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:589b54513e901739f4b4582c705ce96b80c96f57641b1464607e2367a270e540", size = 251988, upload-time = "2026-07-02T13:08:55.726Z" }, - { url = "https://files.pythonhosted.org/packages/df/27/3494d5f291b9a4cb868f73c11221a8bd2d5bd761a8f9acea61ff57128dd1/coverage-7.15.0-cp311-cp311-win32.whl", hash = "sha256:106781b8482749162d0b47056937ba0933508e5d9447f65a5e7d5c422f0d6bb4", size = 222754, upload-time = "2026-07-02T13:08:57.091Z" }, - { url = "https://files.pythonhosted.org/packages/2a/ee/cd4847ebc9be6a9c0123d763645a6f1f3be6b8c58c962706368b79cbac07/coverage-7.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:821e92b3631d762a339695824cadbbc73020354eba2a23a551a99ad34938fbe6", size = 223225, upload-time = "2026-07-02T13:08:58.594Z" }, - { url = "https://files.pythonhosted.org/packages/57/37/5011581aa7f2be498b97dcc7c9902192442a42f4f9a748aeadb3d6506b42/coverage-7.15.0-cp311-cp311-win_arm64.whl", hash = "sha256:309990eb5fb8014b9f67cb211f7fd41876ec8a88a88d3ae76de0ed1d611e3640", size = 222774, upload-time = "2026-07-02T13:09:00.074Z" }, - { url = "https://files.pythonhosted.org/packages/2a/74/fd4c0901137c4f8d81a76ada99e43c65163b4c94a02ece107a4ec0c6b615/coverage-7.15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b75ee5e8cb7575636ac598719b4307ac529ec8fcd79608a35c3cd4d4dada812d", size = 220838, upload-time = "2026-07-02T13:09:02.084Z" }, - { url = "https://files.pythonhosted.org/packages/0f/2e/2347583467bd7f0402635101a916961915cc68fce652cd0db5f173ea04fc/coverage-7.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffb31267816b93b075302248cc1737506081b4f163df4401e9df1a6424aafabe", size = 221197, upload-time = "2026-07-02T13:09:03.617Z" }, - { url = "https://files.pythonhosted.org/packages/f0/17/99fa688541ae1d6e84543a0e544f83de0c944815b63e9e7b1ed411d15036/coverage-7.15.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e4d0bb73455bf97ab243a8f12c37c686ccf1c13bb614b7b85f1d062f06f42b2c", size = 252705, upload-time = "2026-07-02T13:09:05.059Z" }, - { url = "https://files.pythonhosted.org/packages/fb/02/6a95a5cd83b74839017ef9cf48d2d8c9ae60af919e17a3f336e6f9f1b7bd/coverage-7.15.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:20d9ccc4ebd0edc434d86dfd2a1dd2a8efa6b6b3073d0485a394fee86459ebb4", size = 255441, upload-time = "2026-07-02T13:09:06.559Z" }, - { url = "https://files.pythonhosted.org/packages/67/f2/406f6c57d600f68185942422c4c00f1a3255d60aee6e5fd961425cd9987e/coverage-7.15.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20c8a976c365c8cb12f0cbd099508772ea41fb5fa80657a8506df0e11bd278c5", size = 256556, upload-time = "2026-07-02T13:09:08.197Z" }, - { url = "https://files.pythonhosted.org/packages/74/8e/d3fa48489c15ecdec1ba48fd61f68798555dddd2f6716f9ad42adeb1a2a9/coverage-7.15.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f948fd5ba1b9cbca91f0ae08b4c1ce2b139509149a435e2585d056d57d70bf01", size = 258815, upload-time = "2026-07-02T13:09:09.691Z" }, - { url = "https://files.pythonhosted.org/packages/47/2e/2d40ddd110462c6a2769677cf7f1c119a52b45f568978fc6c98e4cc0dd0f/coverage-7.15.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f58185f06edf6ad68ec9fb155d63ef650c82f3fbd7e1770e2867751fb13158f4", size = 253117, upload-time = "2026-07-02T13:09:11.212Z" }, - { url = "https://files.pythonhosted.org/packages/51/c0/310782f0d7c3cb2b5ac05ba8d205fe91f24a36f6bf3256098f1782181c38/coverage-7.15.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:02adc79a920c73c647c5d117f55747df7f2de94571884758ce8bc58e04f0a796", size = 254475, upload-time = "2026-07-02T13:09:13.029Z" }, - { url = "https://files.pythonhosted.org/packages/86/f7/702da6c275f8ae6ade423d2877243122932c9b27f5403003b9ef8c927d12/coverage-7.15.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:6eb7c300fbed667fd6e3588eba71c1904cdb06110ca6fdf908c26bdd88b8e382", size = 252619, upload-time = "2026-07-02T13:09:14.699Z" }, - { url = "https://files.pythonhosted.org/packages/fb/84/c5b15a7e5ecba4e56218d772d99fe80a63e63f8d11f12783723a6005ab45/coverage-7.15.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:b5fb23fa2de9dce1f5c36c09066d8fcda16cd96e8e26686caa2d7cb9b567d65c", size = 256689, upload-time = "2026-07-02T13:09:16.103Z" }, - { url = "https://files.pythonhosted.org/packages/95/2f/c8b07559b57701230c61b23a953858c052890c12ef568d81780c6c46e92e/coverage-7.15.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:cec79341dbe6281484024979976d0c7f22beae08b4a254655decd25d42cbe766", size = 252189, upload-time = "2026-07-02T13:09:17.828Z" }, - { url = "https://files.pythonhosted.org/packages/6b/80/6d2f049dd3fd3dbfd60b62ba6b2162a04009e2c002ce70b24cf3878dec7a/coverage-7.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6c664c5444b1d970b1b2a450e21fb19ee5c9cfdf151ded2dda37260031cca0da", size = 254059, upload-time = "2026-07-02T13:09:19.304Z" }, - { url = "https://files.pythonhosted.org/packages/ce/92/b0287a2c42031d25c628f815f89a3cd9f8268ee78bb1252c9356cda1c689/coverage-7.15.0-cp312-cp312-win32.whl", hash = "sha256:5f764a3fa339bde6b3aa97657f5a6a3a9451e4a5b4ea98a2892c773a43525f77", size = 222893, upload-time = "2026-07-02T13:09:20.812Z" }, - { url = "https://files.pythonhosted.org/packages/a9/69/e34c481915fecb499b3146975061dac528752e37706edc1804f32c822469/coverage-7.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:52f9a4d2c4c56c8848bc2f524916698354b0211488b38c49ad9ae54f6cafbff6", size = 223429, upload-time = "2026-07-02T13:09:22.315Z" }, - { url = "https://files.pythonhosted.org/packages/fe/98/6e878f0b571d32684ef3f38d7c03db241ca5b82a5da8a5391596a8f209c4/coverage-7.15.0-cp312-cp312-win_arm64.whl", hash = "sha256:31e5c3e70c85307ea35a12964e2e40f56ca2ee4b1c8c721ccf4609d17071080b", size = 222810, upload-time = "2026-07-02T13:09:23.812Z" }, - { url = "https://files.pythonhosted.org/packages/76/04/145a3748098bcc86b631a85408d2c3dc5c104e0bd86d605468239b25b6c4/coverage-7.15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5be4caf3b28836f078abe700f8944dac4a65d78f16d6c600c89cb624e5535782", size = 220863, upload-time = "2026-07-02T13:09:25.371Z" }, - { url = "https://files.pythonhosted.org/packages/a4/5c/4ed55708fed2c64b63c9bc5715daef670872202101938869b7fe5d5fbb8f/coverage-7.15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dd58ad1404704303ca8d4f4b8a1095e7cbc7040ef17a66df1e6619aa10176430", size = 221230, upload-time = "2026-07-02T13:09:26.897Z" }, - { url = "https://files.pythonhosted.org/packages/7b/19/3a80b97d3b2a5c77a01ae359c6bed20c13738fe3d9380f08616d4fec0281/coverage-7.15.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bbcbb317c2e5ded5b21104af81c29f391be2af98d065693ffbe8d23949b948e5", size = 252227, upload-time = "2026-07-02T13:09:28.543Z" }, - { url = "https://files.pythonhosted.org/packages/a1/fa/b70062750686bd7da454da27927622f48bbac6990ac7a4c4a4653e7b0036/coverage-7.15.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:27f31ecb458da3f859aab3f15ada871eb7a7768807d88df4a9f186bb17737970", size = 254823, upload-time = "2026-07-02T13:09:30.177Z" }, - { url = "https://files.pythonhosted.org/packages/a9/09/dad6a75a2e561b9dc5086a8c5257a7591d584246f67e23e70d2995b89ab6/coverage-7.15.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fb759be317fdc62e0f56bffdf61cfcb45c7761ad6b71e3e583e71a67ae753c", size = 256059, upload-time = "2026-07-02T13:09:31.979Z" }, - { url = "https://files.pythonhosted.org/packages/e6/e7/b5d2941fa9564573d44b693a871ff3156f0c42cbefe977a09fa7fdc59971/coverage-7.15.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d5cf007add5ab4bb8fa9f4c77e3732127c9e6cad501d7db43355fbfafca0be84", size = 258190, upload-time = "2026-07-02T13:09:34.035Z" }, - { url = "https://files.pythonhosted.org/packages/7c/1d/8e895bcde3c57ccd46d896dda5f2b3d5df761a1b0c6c9d450d175dedc632/coverage-7.15.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cc78d9843bd576fbe2118248258d485e968dc535f95ed504a7b0867ba9b51389", size = 252456, upload-time = "2026-07-02T13:09:35.765Z" }, - { url = "https://files.pythonhosted.org/packages/14/4c/f6997da343ddeb959be82c3b05322793f92c071ad45f7cb8a96336e2dd5f/coverage-7.15.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a263060f1de0b4b74b4e089c2a70b8003b3781c733329a9c8fd54995328f9950", size = 254192, upload-time = "2026-07-02T13:09:37.445Z" }, - { url = "https://files.pythonhosted.org/packages/17/27/a0bc09d032267b9da89d95a2d874cfbef2a5aebbf0e87cf7aba221d79a99/coverage-7.15.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c48decf16e0dfd5b049c7d5e82200c23c08126719142998d4f172444e3d0529e", size = 252153, upload-time = "2026-07-02T13:09:39.422Z" }, - { url = "https://files.pythonhosted.org/packages/54/c0/77fc233d9fba07b244c40948c53fe27308b8f21732fb3417f87fbd6fd992/coverage-7.15.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:08fb028000ed0aaa0a4cbdfbb98be7cb42f370db973fbbb469733505ab20e13e", size = 256310, upload-time = "2026-07-02T13:09:41.006Z" }, - { url = "https://files.pythonhosted.org/packages/d5/24/601cecfb5825becacb8d45219a018a3b55b9dbaec624efdb0ea249d08be2/coverage-7.15.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb7dc0c3b7d8a1077abea0b8546ebc5e26d6ef6ecefc2f0f5ad2b8a53bdad837", size = 251974, upload-time = "2026-07-02T13:09:42.733Z" }, - { url = "https://files.pythonhosted.org/packages/47/1e/6f45e5a5b3d5484318d368702af6716b5ab8913b0428bec981a562fcf296/coverage-7.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6cb3602054ccbe9f0d8c2dc04bbeba90d5719236e2cd06e042ddd6d3fc7b6e37", size = 253745, upload-time = "2026-07-02T13:09:44.376Z" }, - { url = "https://files.pythonhosted.org/packages/8e/db/4df027a77bd11d0e527f44c53557c76e54ad027413d0304252ea3a78d67e/coverage-7.15.0-cp313-cp313-win32.whl", hash = "sha256:0bf781da64326b677be344df505171435b6f58716108606621d5d27d964fff8b", size = 222902, upload-time = "2026-07-02T13:09:46.122Z" }, - { url = "https://files.pythonhosted.org/packages/a0/10/0355894d34e231f2c5449e71287e81a50793a325df2e2b027b7bcd9dfd19/coverage-7.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:2c57a275078ee3fa185f83e400f765bc764a549de66d99b47881645cbd4ea629", size = 223444, upload-time = "2026-07-02T13:09:47.687Z" }, - { url = "https://files.pythonhosted.org/packages/06/ef/bb725f263befaaff851203ab338e68af15e195d7f7b5f323162532d9b6a8/coverage-7.15.0-cp313-cp313-win_arm64.whl", hash = "sha256:3812c61afc6685c7999b39320779ab8f43b7a3081fdb0def39976e56fbdb9a21", size = 222839, upload-time = "2026-07-02T13:09:49.717Z" }, - { url = "https://files.pythonhosted.org/packages/4f/9c/1e3ca54f72a3185ece06c58d871099898c48f0ed6430d17b6ab75f0d180a/coverage-7.15.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:41cb79af843222e11da87127ad0ecbfa878abadd0f770a4a99391a27d3887324", size = 220906, upload-time = "2026-07-02T13:09:51.339Z" }, - { url = "https://files.pythonhosted.org/packages/09/37/f718613d83b274880382f6b67e78f3802549ae39b0b3e65ae5b5974df56e/coverage-7.15.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7d2008989ef8fe54188d3f3bfa2e3099b025af11e90a6a1b9e7dc433d04263d8", size = 221239, upload-time = "2026-07-02T13:09:53.138Z" }, - { url = "https://files.pythonhosted.org/packages/a7/ce/22bae91e0b75445f68d365c7643ed0aa4880bbf77450ee74ca65bdae53a7/coverage-7.15.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:769e8ece11a596315ebf5aa7ec383aeeed016c091d2bf6363ffb996d41529092", size = 252286, upload-time = "2026-07-02T13:09:54.996Z" }, - { url = "https://files.pythonhosted.org/packages/dd/1e/bec5e32aa508615d9d7a2790effb25fb4dc28606e995816afe400b25ece3/coverage-7.15.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:65a6b6164ee5c39e2f3803f314292d6c61a607ba7fee253d1e03c42dc3903502", size = 254789, upload-time = "2026-07-02T13:09:56.678Z" }, - { url = "https://files.pythonhosted.org/packages/17/29/0e865435b4354e4a7c03b1b7920046d31d0a273d55decefea27e011cb9bf/coverage-7.15.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:75128817f95a5c45bb01d65fd2d8b9cb54bbe03d81608fb70e3e14b437ad56c2", size = 256135, upload-time = "2026-07-02T13:09:58.343Z" }, - { url = "https://files.pythonhosted.org/packages/84/ff/33a870b58a13325d62fc0a6c8f01fa0ff667cef60c7498e2382a147dfa18/coverage-7.15.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9887bb428fe2d4cd4bee89bac1a6c9932f484afd5b36fbd4ff6ea5f825bb1f5e", size = 258449, upload-time = "2026-07-02T13:10:00.057Z" }, - { url = "https://files.pythonhosted.org/packages/18/7b/6fffe596bf3ddba8462758d02c5dad730fd91055a6634aa2e4226229181a/coverage-7.15.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0bfc0be1f702042207a93a00523b1065ee1fe951e96edf311581c0bbc2e34888", size = 252313, upload-time = "2026-07-02T13:10:01.946Z" }, - { url = "https://files.pythonhosted.org/packages/58/1b/11468dd6c1676ab831a70cb9a8d4e198e8607fa0b7220ab918b73fe9bfbd/coverage-7.15.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f64627d55def5a43282d70e08396672692f77e4da610a5bb8bb4060b432b6859", size = 254142, upload-time = "2026-07-02T13:10:04.065Z" }, - { url = "https://files.pythonhosted.org/packages/79/41/29328e21d16b1b95092c30dd700e08cf915bd3734f836df8f3bdb0e8fa9f/coverage-7.15.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:2c6f0fa473003905c6d5bac328ee4eba9fbea654f15bc24b8a3274b23363fa99", size = 252108, upload-time = "2026-07-02T13:10:06.11Z" }, - { url = "https://files.pythonhosted.org/packages/9b/de/05ccfb990439655b35afbfd8e0d13fe66677565a7d4eb38c3f5ef2635e1c/coverage-7.15.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2bcf9afaf064172c6ec3c58a325a9957ad1178c05dd934e25f253321776e0676", size = 256385, upload-time = "2026-07-02T13:10:08.141Z" }, - { url = "https://files.pythonhosted.org/packages/51/0e/486828a3d2695ea7a2609f17ff572f6b01905e608379440a11da4b8dffbe/coverage-7.15.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:baf06bc987115d6fb938d403f7eab684a057766c490367999a2b71a6883110c6", size = 251923, upload-time = "2026-07-02T13:10:10.179Z" }, - { url = "https://files.pythonhosted.org/packages/18/c7/03582b6715f078e5e558354c87616d945b9894cda2dace8e4009b17035e4/coverage-7.15.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f0405f2ff97b1c4c0e782cb32e02f32369bcf2e6b618b591d67e1ea754575dfe", size = 253580, upload-time = "2026-07-02T13:10:12.052Z" }, - { url = "https://files.pythonhosted.org/packages/db/dc/9e578bbaf2ecb4959a81b7e7601ad8cca772cba2892e8d144cb749b4a71a/coverage-7.15.0-cp314-cp314-win32.whl", hash = "sha256:ab282853ed5fbd64bbb162f19cb8fcb7087187508a6374b4f9c34ec1577c4e8f", size = 223107, upload-time = "2026-07-02T13:10:13.994Z" }, - { url = "https://files.pythonhosted.org/packages/ae/3e/c8c3b75d8dbe0e35f7b0cc3ff5e949fc59500f70b21d0398813f66740664/coverage-7.15.0-cp314-cp314-win_amd64.whl", hash = "sha256:3bb3040e9f4bbe26fcb0cd7cc85ac63e630d3f3a9c74f027abf4caa27e706663", size = 223597, upload-time = "2026-07-02T13:10:15.906Z" }, - { url = "https://files.pythonhosted.org/packages/cd/bc/3cbc9fb036eb388519bccd521f783499c39b64256013fbc362782f196fe1/coverage-7.15.0-cp314-cp314-win_arm64.whl", hash = "sha256:346771144d34f7fa84ec28386f78e0f31653f33cf35e19d253d5b35f9e8201da", size = 223020, upload-time = "2026-07-02T13:10:17.844Z" }, - { url = "https://files.pythonhosted.org/packages/28/00/199c4a8d656dff63102577a056c0fce2ff6a79e40adac092fc986c49cbf1/coverage-7.15.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d34a010905fb6401324ba016b5da03d574967f7b21ce48ea41e66f0f1f95f641", size = 221638, upload-time = "2026-07-02T13:10:19.703Z" }, - { url = "https://files.pythonhosted.org/packages/ba/8e/9d0092c96a3d3a26951ecc7020826aa57bcb1b119ca81acbba996884ab13/coverage-7.15.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bb25d825d885ca8036795dacfc3924d33091fc76d71ebc99420c6b79e77d96fa", size = 221903, upload-time = "2026-07-02T13:10:21.514Z" }, - { url = "https://files.pythonhosted.org/packages/6d/b4/c0ca3028f42c9a08e51feb4561ef1192e5de99797cd1db5b04590c215bda/coverage-7.15.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:94c9686bfe8a9a6810297aecbd99beaa3445f9e8dc2f80b1382cca0d86b64461", size = 263267, upload-time = "2026-07-02T13:10:23.261Z" }, - { url = "https://files.pythonhosted.org/packages/5f/aa/a375e3846e5d3c013dc600b2a3231089055c73d77f5393dd2192a8d64da6/coverage-7.15.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9bd671c25f9d85f09d7ec481d0e43d5139f486c06a37139847a7ce569788af72", size = 265390, upload-time = "2026-07-02T13:10:25.152Z" }, - { url = "https://files.pythonhosted.org/packages/92/e1/5783cdabb797305e1c9e4809fea496d31834c51fa772514f73dc148bcfc9/coverage-7.15.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:110cbdf8d2e216577312cf06ccf85539c0e5a5420ef747e4a4719b5e483c88cd", size = 267811, upload-time = "2026-07-02T13:10:27.249Z" }, - { url = "https://files.pythonhosted.org/packages/85/31/96d8bbf58b8e9193bc8389574a91a0db48355ee98feb66aa6bf8d1b32eea/coverage-7.15.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2c5d4619214f1d9993e7b00a8600d14614b7e9d84e89507460b126aa5e6559e5", size = 268928, upload-time = "2026-07-02T13:10:29.242Z" }, - { url = "https://files.pythonhosted.org/packages/5e/7a/5294567e811a1cb7eda93140c628fa050d66189da28da320f93d1d815c73/coverage-7.15.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:781a704516e2d8346fbbd5be6c6f3412dd824785146528b3a01816f26c081007", size = 262378, upload-time = "2026-07-02T13:10:31.107Z" }, - { url = "https://files.pythonhosted.org/packages/69/3f/3f48538421f899f28946f90a3d272136a4686e1abf461cc9249a783ee0f3/coverage-7.15.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd4a1b44bcb65ee29e947ac92bbee04956df3a6bfc6143641bb6cae7ede00fc9", size = 265263, upload-time = "2026-07-02T13:10:32.942Z" }, - { url = "https://files.pythonhosted.org/packages/ce/d3/092df15efcab8a9c1467ee960eb8019bbad3f9300d115d89ea6195f369ff/coverage-7.15.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0e4950c9d6d3e39c64c991814ff315e2d0b9cb8152363594212c9e55208c0a8f", size = 262866, upload-time = "2026-07-02T13:10:35.104Z" }, - { url = "https://files.pythonhosted.org/packages/e5/ab/0254d2b88665efb2c57ad368cc77ab5de3435bd8d5add4729c1b0e79431e/coverage-7.15.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:fe9c87ff42e5472d80d21704972e1f96e104a0a599d77c5e35db5a3c562e2571", size = 266599, upload-time = "2026-07-02T13:10:37.05Z" }, - { url = "https://files.pythonhosted.org/packages/a8/79/1cfa4023e489ce6fbc7be4a5d442dbc375edb4f4fda39a352cedb53263c2/coverage-7.15.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f00d5ae1dd2fe13fb8186e3e7d37bcbd8b25c0d764ff7d1b32cef9be058510a8", size = 261714, upload-time = "2026-07-02T13:10:38.966Z" }, - { url = "https://files.pythonhosted.org/packages/b7/eb/fee5c8665656be63f497418d410484637c438172568688e8ac92e06574e7/coverage-7.15.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:363ab38cc78b615f11c9cac3cf1d7eef950c18b9fdedfb9066f59461dcf84d68", size = 264025, upload-time = "2026-07-02T13:10:40.789Z" }, - { url = "https://files.pythonhosted.org/packages/ab/99/63005db722f91edc81abc16302f9cc2f6228c1679e46e15be9ae144b14d0/coverage-7.15.0-cp314-cp314t-win32.whl", hash = "sha256:54fd9c53a5fafff509195f1b6a3f9be615d8e8362a3629ff1de23d270c03c86b", size = 223413, upload-time = "2026-07-02T13:10:42.597Z" }, - { url = "https://files.pythonhosted.org/packages/c1/e8/2bc6181c4fb06f1a6b981eb85330cc57bfad7e3f710fc9c9d350013ba228/coverage-7.15.0-cp314-cp314t-win_amd64.whl", hash = "sha256:87b47553097ba185ed964866078e7e63adea9f5f51b5f39691c34f30afd21080", size = 224245, upload-time = "2026-07-02T13:10:44.47Z" }, - { url = "https://files.pythonhosted.org/packages/79/b8/4d959bf9cc45d0cfed2f4d35cafcab978cdb6ea02eb5100009cd740632a3/coverage-7.15.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aeefb2dd178fe7eee79f0ad25d75855cb35ee9ed472db2c5ea06f5b4fd00cec5", size = 223558, upload-time = "2026-07-02T13:10:46.368Z" }, - { url = "https://files.pythonhosted.org/packages/52/30/21b2ad45959cd50e909e02ebac1e30b4ceb7162e91c11d4c570223a458b7/coverage-7.15.0-py3-none-any.whl", hash = "sha256:56da6a4cbe8f7e9e80bd072ca9cefe67d7106a440a7ec06519ec6507ac94ad19", size = 212632, upload-time = "2026-07-02T13:10:48.641Z" }, +version = "7.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/83/6051c2a2feab48ae5bd27c84ef047191d2d4a3172f689e38eaa48ed17db1/coverage-7.15.1.tar.gz", hash = "sha256:165e9949eaf222ef1f018635d0d7f368a23bfe0212af558534c40d8c04686d67", size = 927640, upload-time = "2026-07-12T20:58:19.908Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/76/eef242d6bb95fb737fefd9bbf3a318897e4e82cf543b53a61c2a6e8588eb/coverage-7.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:05d87c2a43373ad6b976d0a99ad58c48633633bcdeb896dc645a006472cc4a71", size = 221195, upload-time = "2026-07-12T20:55:55.82Z" }, + { url = "https://files.pythonhosted.org/packages/89/73/41cd50da59d513a30fd3a34b5516b962ac17514defdb6705948446a5c0b1/coverage-7.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2afce82f2cf8f4c9002746a42755e1dc61baff33d9f7ab5569b4c9101f8f4d1d", size = 221714, upload-time = "2026-07-12T20:55:58.104Z" }, + { url = "https://files.pythonhosted.org/packages/22/9d/6df6ac6677719a087c839208186525b7b1f3653c50196c43246482ada65e/coverage-7.15.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0a545ef5384d787d0fcac6c349afc2c5f99dcc39e13ed3c191b2c06305f64c04", size = 248454, upload-time = "2026-07-12T20:55:59.336Z" }, + { url = "https://files.pythonhosted.org/packages/9f/f5/86f92c429fbf942986d01c3bb7b0b6c3ad06b09f7fa745023877413dde83/coverage-7.15.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:afaa144b8f5b3bc69fe0ce50d401c46b01ab264782553bfd05a3f98804524ecb", size = 250282, upload-time = "2026-07-12T20:56:00.589Z" }, + { url = "https://files.pythonhosted.org/packages/47/c6/99d0f1c1dd1583aaf8c1deca447e44426be54a76027a8c55e29970f22b15/coverage-7.15.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e1b6099490e5f88569c46b18605f556c3b30acc9a0a219cf7ef8fab8f7161ec4", size = 252149, upload-time = "2026-07-12T20:56:01.853Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c3/5d30740ec96f5fd8a659e46b6ee9224197f87aa66ce4a20e93dcf46221ac/coverage-7.15.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bb5f6c2cf1ffd0bf2bd925c7cdcae9b4f208e9696d453ed51eb1f5fa0cc5b45b", size = 254061, upload-time = "2026-07-12T20:56:03.277Z" }, + { url = "https://files.pythonhosted.org/packages/71/e5/3afb2950673ca6cd22bc9ad6a9d6058c853bef8fe27a6d1df3adc274fd88/coverage-7.15.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:81572011fc1fc271317da35da593944daef7bfd507085e35751abbe702b74f69", size = 249152, upload-time = "2026-07-12T20:56:04.618Z" }, + { url = "https://files.pythonhosted.org/packages/ca/53/25ab3c0a7cf517ed4d30cd4dd82331e005d4b57fec8c19c0ffd94fb50baa/coverage-7.15.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8f765d13c08497687d0780cca66115c6aa4ba6703ad43b61e94fab9db689e3a3", size = 250187, upload-time = "2026-07-12T20:56:06Z" }, + { url = "https://files.pythonhosted.org/packages/ad/2a/8afb5f994e26e919e5dca5164f1c7b6b7ce6090aab1aac32b1ae1782f047/coverage-7.15.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:771caba880fee96493d18dfc465c318e08ab74e3bc2a3e4089e52514be6a6e54", size = 248193, upload-time = "2026-07-12T20:56:07.376Z" }, + { url = "https://files.pythonhosted.org/packages/d4/62/b6616e15288c9a7b628f7745e832fd8af2c03ec1f7dc11da25947f5ab16f/coverage-7.15.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:433a73200848e80f27712fc113b6ff5311f29b479a7d3bd4b1106138a77f9674", size = 252004, upload-time = "2026-07-12T20:56:08.787Z" }, + { url = "https://files.pythonhosted.org/packages/9e/21/5d80d65e4df3018dedb23a44f276f20ce26e429ffd76df03de03bcf9a767/coverage-7.15.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:5d6fa45079db9fbeba0a69e3d91189f05301d6ac918162a53179d32fc9ed4910", size = 248463, upload-time = "2026-07-12T20:56:10.245Z" }, + { url = "https://files.pythonhosted.org/packages/53/4b/04a22708691561c44d77e1d5a60857b351310fdfbf253533780bb31c8b6f/coverage-7.15.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:00b6703c6640075cdce5124e9335dfdf9167272475301828acfdd09c0e5ee731", size = 249064, upload-time = "2026-07-12T20:56:11.614Z" }, + { url = "https://files.pythonhosted.org/packages/82/c9/a5c1e8954e657559de478acc1ef73e9393d3fc9b4a2ab4427501d765aca2/coverage-7.15.1-cp310-cp310-win32.whl", hash = "sha256:3ad9a0eac4728327fd870d52f74d2e631d176c5f178eaea2d9983ab5b9755a55", size = 223248, upload-time = "2026-07-12T20:56:13.068Z" }, + { url = "https://files.pythonhosted.org/packages/54/49/5f6566e14611a58e93a926f3a8a7b22e4e0702ebb4c467ac38f452c7f4cf/coverage-7.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:eb5fa75dc3d30e3a1b75da97973479b20ffa9b0641ff56d6e94b5f3e210daa54", size = 223874, upload-time = "2026-07-12T20:56:14.396Z" }, + { url = "https://files.pythonhosted.org/packages/67/ee/5095e07a0986930174fc153fd0bb115f7f2724f5344cc672e016d4f23a4e/coverage-7.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6506330b4a8dcf53b95bd84d8d0e817107cdb3fc1438e835029cdf0bc6612eb0", size = 221320, upload-time = "2026-07-12T20:56:16.036Z" }, + { url = "https://files.pythonhosted.org/packages/43/7a/7e2598ce98433a214020a61c0755d5961f9f26df0c630dc73e06b86d3416/coverage-7.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8a51a8ec382c39d939cba0ab07ae949077ae4e842343bd4eed22d432358cea9", size = 221823, upload-time = "2026-07-12T20:56:17.54Z" }, + { url = "https://files.pythonhosted.org/packages/8b/4f/67dac6a69139b490a239d91b6d5ef127982ffb89159769241656ab879ee5/coverage-7.15.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:18ea20e3922d7f8ca9e0ef1084408d08c4ad62d5e531cb9c1f6896a99297ebea", size = 252242, upload-time = "2026-07-12T20:56:18.868Z" }, + { url = "https://files.pythonhosted.org/packages/05/37/5e439725a96de592309725550c8df639c359c209dcb4b152ed46032d4c0d/coverage-7.15.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:aab9902a64b8390e3b56e539fddae1d79a267807fe5cb0c18d7d2f544ce867e2", size = 254153, upload-time = "2026-07-12T20:56:20.118Z" }, + { url = "https://files.pythonhosted.org/packages/0f/1c/671ba9e15f9651a99962fb588a1fe4fb267cae4bb85f9bb4ac4413c6f7ef/coverage-7.15.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cc316264317b07a9e90d7f2b4188a15e36e9b54e651081b791b0515fa612a29", size = 256261, upload-time = "2026-07-12T20:56:21.682Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3c/88305322b4d015b4536b7174b8f9b87f850f01af9d89008cdbf984b65ff2/coverage-7.15.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d4e47e7eea81a8ccf060a07627654151d929da62c7b715738387c200905cec89", size = 258222, upload-time = "2026-07-12T20:56:22.962Z" }, + { url = "https://files.pythonhosted.org/packages/8d/20/d02e42333a57f266ded61ed4fb3821da1f2dc143734aaa3dcc9c117204c5/coverage-7.15.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c6829d9a3b55ad2b73ef5fda8302e5be03683789e88b1a079dcf4a773229c21d", size = 252368, upload-time = "2026-07-12T20:56:24.475Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d0/94ebc010926a191d83e83b1f1236f8bd0d14d0eb98b5c324aa4be2589a8e/coverage-7.15.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:764e045811f9c8cda436641f3f088283351d331a519b5807f19041cd0a68da1c", size = 253953, upload-time = "2026-07-12T20:56:26.036Z" }, + { url = "https://files.pythonhosted.org/packages/2e/a6/2c8553191da0c2da2c43ff294fb9bab57a5b0fae03560304a82a8b5addc3/coverage-7.15.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:09b3b088aa24489c4082bcc35fcc8224281ab94a653dfb6d3f0c8165b0d628ab", size = 252016, upload-time = "2026-07-12T20:56:27.374Z" }, + { url = "https://files.pythonhosted.org/packages/17/99/20f47986d8360fa1a31d9858dd2ba0be009d44ecfa8d02120b1eb93c028b/coverage-7.15.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7911b02f57053adf8164ae63edb1c26574d24dfccabadc5268cf69310a69a358", size = 255783, upload-time = "2026-07-12T20:56:28.921Z" }, + { url = "https://files.pythonhosted.org/packages/17/36/fb61983b5259f78fa5e62ee74e91fe4de4c556fcbc9a3b9c9021c2f8b57b/coverage-7.15.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:664e279ed40599b8ed16f4db18d92a7e212c73129672bec8f5d96d4da48d2404", size = 251735, upload-time = "2026-07-12T20:56:30.465Z" }, + { url = "https://files.pythonhosted.org/packages/6c/2e/d109d8d2d6c726ba2e78125297073918a2a91464f0ea777e5398fa3cf75c/coverage-7.15.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:34fe7cf79d5f1f87f2e8ce7dc1c32950841f50e10d0120f263856acfad66de34", size = 252645, upload-time = "2026-07-12T20:56:32.076Z" }, + { url = "https://files.pythonhosted.org/packages/bc/a0/5b3219cbb46135e14673427f2bf5890c4da4e6f655243692f3448aa3f42c/coverage-7.15.1-cp311-cp311-win32.whl", hash = "sha256:5e2d2536d2f57a354aa382ed303ac0e2e5c9522a508c05b998d26181b94163a7", size = 223414, upload-time = "2026-07-12T20:56:33.432Z" }, + { url = "https://files.pythonhosted.org/packages/5b/80/24e13ade0b6df8090e2492f9c55044bf1423f7ef28c76fc1af8c827a61cb/coverage-7.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:c337da8fca7ea93ab43f3868cfcde6cf6dad32c3906b273cfbad5d7390bc423b", size = 223888, upload-time = "2026-07-12T20:56:34.784Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9e/34659930ae380491af48e2f5f5f9a2e99cd9fb7de3d30f27be5ac5425137/coverage-7.15.1-cp311-cp311-win_arm64.whl", hash = "sha256:db3403fdb7a94d5eb73e099befad8104d2a7d110a0f0d99df0de61c5d1fa756c", size = 223435, upload-time = "2026-07-12T20:56:36.302Z" }, + { url = "https://files.pythonhosted.org/packages/d9/76/32c1826309beaf4604c54accef108fdd611e5e5e93f2f5192f050cd5f6bd/coverage-7.15.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d9476292594309db922cc841dd13b303b3c388f4c25d279884f7e2341c681f80", size = 221497, upload-time = "2026-07-12T20:56:37.628Z" }, + { url = "https://files.pythonhosted.org/packages/db/5c/b88ce0d68fa550c7f3b58617fbf363bce64df5bf8295a01b627e4696e022/coverage-7.15.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0c579056b0de461b3a62318b63d0b6ce90aed7f8158d3f00da094df82f29d189", size = 221854, upload-time = "2026-07-12T20:56:39.033Z" }, + { url = "https://files.pythonhosted.org/packages/0e/fe/8509fd2a66fc4e0a829f76a0f0b1dc3cc163368352435b5f243168658077/coverage-7.15.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:23214bdbe226f2b0e9c66a7d6a1d59d4a88045dcf86e702cf0fe0d0935e3d615", size = 253359, upload-time = "2026-07-12T20:56:40.423Z" }, + { url = "https://files.pythonhosted.org/packages/e5/81/c7009ed7ea9765adb2b9d095054d748266fae5f07ac6c5f925f33715fcde/coverage-7.15.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:df164be93b46b4825cc39339440a05edc54c4d1d865ba4a60fd43d151a2a1cd3", size = 256096, upload-time = "2026-07-12T20:56:42.115Z" }, + { url = "https://files.pythonhosted.org/packages/21/52/dc8ee03968a5ba86e2da5aa48ddc9e3747bd65d63825fdce2d96acb9c5ff/coverage-7.15.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a524fca1a6f08927d9dc2d4c873cfb7bd7202c247f08b14bdc02424071b8b304", size = 257211, upload-time = "2026-07-12T20:56:43.513Z" }, + { url = "https://files.pythonhosted.org/packages/b8/27/95d7623908da8937deb53d48efcdbf423907a47540e63c62fa21372c652b/coverage-7.15.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d70f3542cd38de85a9e257dcb1ac4c1ab4b6d7d2c2a645809207556628755d1c", size = 259473, upload-time = "2026-07-12T20:56:44.974Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3b/730d761928de97d585465680b568ae69622fb40716babadeabffe75cb51b/coverage-7.15.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d78aa537237212c4313aabe5e964b66acc86350ed19ebc56a3e202df33b6077b", size = 253759, upload-time = "2026-07-12T20:56:46.615Z" }, + { url = "https://files.pythonhosted.org/packages/f1/fc/6b9277acff1f9484b6c12857af5774689d1a6a95e13265f7405329d2f5da/coverage-7.15.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a318112bb4f79d9d04766196d5a3388caa825908a6a9b052aa87de3d9aea7c61", size = 255131, upload-time = "2026-07-12T20:56:48.073Z" }, + { url = "https://files.pythonhosted.org/packages/3d/f2/c704f86129594ba34e25a64695d2068c71d51c2b98907184d716c94f4aec/coverage-7.15.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e55d24cada901963eed5bc89fa562aa033f0d84b9d3de4ecf363737c13aed11e", size = 253275, upload-time = "2026-07-12T20:56:49.538Z" }, + { url = "https://files.pythonhosted.org/packages/f6/29/80fee8af47de4a6dce71ccf2938491f444687a756af258a56d8469b8f1b0/coverage-7.15.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3c78f0cea7275342cf2adc2ad5fdd0aafa106ad91e66d573568f2fcf62c41df5", size = 257345, upload-time = "2026-07-12T20:56:51.038Z" }, + { url = "https://files.pythonhosted.org/packages/20/21/a1e7d7ed1b48a8adf8fd5154d9e83fcc5ad8e6ff20ae00e44865057dce8d/coverage-7.15.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:86bd37eabe39977216f630a7fc1b698e7f5e81a191c7186013245c6c3d313f9d", size = 252844, upload-time = "2026-07-12T20:56:52.535Z" }, + { url = "https://files.pythonhosted.org/packages/a7/8c/a4bc26e6ee207d412f3678f04d74be1550e83140563ca0e4997510579712/coverage-7.15.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c6db15c217693bdc3ca0b84de1ba9afafe1c14c26a8a29d77f4ed0de2b6132e2", size = 254716, upload-time = "2026-07-12T20:56:53.968Z" }, + { url = "https://files.pythonhosted.org/packages/11/9d/8ad0266ecfada6353cf6627a1a02294cf55a907521b6ee0bd7b770cfd659/coverage-7.15.1-cp312-cp312-win32.whl", hash = "sha256:359f3fbe09a51500c51966596ee4ee4070b356552c70b3b2420eb200d68e0f76", size = 223554, upload-time = "2026-07-12T20:56:55.583Z" }, + { url = "https://files.pythonhosted.org/packages/81/6d/24224929e06c6e05a93f738bc5f9e8e6ab658f8f1d9b823e7b85430e28b8/coverage-7.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:fa75dc099c126e941a9c0baa8ebd2cbc78bd778687534fe410baf754f6d9e374", size = 224087, upload-time = "2026-07-12T20:56:57.041Z" }, + { url = "https://files.pythonhosted.org/packages/35/23/f81441dd01de88e53c97842e706907b307d9078918c3f4998b11e9ac7250/coverage-7.15.1-cp312-cp312-win_arm64.whl", hash = "sha256:26f89cf6d0634375f454fa71057945ad18edb0f1607a90fecf22c57dc3dc289a", size = 223472, upload-time = "2026-07-12T20:56:58.594Z" }, + { url = "https://files.pythonhosted.org/packages/ca/1e/6fa289d7993a2a39f1b283ddb58c4bfec80f7800be654b8ba8a9f6a07c63/coverage-7.15.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:71ac4ca1658ca99160fd58cc6967110e989c34b04627f24ed6ec9f70fb24571a", size = 221519, upload-time = "2026-07-12T20:57:00.081Z" }, + { url = "https://files.pythonhosted.org/packages/6f/e1/0db4902a0588234a70ab0218073c0b20fbc5c740aa35f91d360160a2ebc9/coverage-7.15.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:26a40cbf2b13bd94af53ee02a424cb3bb96a9edfac0d00834bd068512a62714b", size = 221895, upload-time = "2026-07-12T20:57:01.867Z" }, + { url = "https://files.pythonhosted.org/packages/b4/cb/3719783865092dac5e08df842730305ee9ab1973ae7ddb6fbdf27d401f30/coverage-7.15.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f4c5a5eff4ad4f9f7088fd3fc7a66d98d06566ee294b3b053309fb0a3b45be1e", size = 252882, upload-time = "2026-07-12T20:57:03.459Z" }, + { url = "https://files.pythonhosted.org/packages/1c/5e/caf3abbdbb22629626160ffc9c017eb995b7cb11c0be46b974834cef1792/coverage-7.15.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:962aa56c1c9b016d681265880eb6acc9966029d2c4c559319cc43a1abbb9b59a", size = 255479, upload-time = "2026-07-12T20:57:04.984Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f1/d60f375bfe095fef944f0f19427aefdbf9bdd5a9571c41a4bf6e2f5fdb81/coverage-7.15.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1678eb2dc57a8ce67601b029582ef6d41e9e6ca22692aaeccd4107e40f27386c", size = 256715, upload-time = "2026-07-12T20:57:06.446Z" }, + { url = "https://files.pythonhosted.org/packages/d7/17/8b0cbc90d02dc5adad4d9034c1824ec3fa567771b4c39d9c1e3f9b1431b8/coverage-7.15.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1174900a43f6f8c425fee10d7dbddc308adefcdc78aaced32357f5ab750a0e90", size = 258845, upload-time = "2026-07-12T20:57:08.092Z" }, + { url = "https://files.pythonhosted.org/packages/92/29/c5e69f5fb75c322e9a3e4ef64d02eebfc3d66efceccc8514ff80a3c13a56/coverage-7.15.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:98847557a6859cadf693792ce89f440cb89692993f60dc6d3a7e35f3d340216f", size = 253098, upload-time = "2026-07-12T20:57:09.636Z" }, + { url = "https://files.pythonhosted.org/packages/64/57/21144252fdd0c01d707d48fbcea13a80b0b7c42ced3f299f885ab8978c3a/coverage-7.15.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8697b2edb57143546a24389efc11e1b000cd5800fc20d84f04edb601e4a7cfb8", size = 254844, upload-time = "2026-07-12T20:57:11.141Z" }, + { url = "https://files.pythonhosted.org/packages/59/2a/499a28a322b0ce6768328e6c5bb2e2ad00ac068a7c7adb2ecd8533c8c5d9/coverage-7.15.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6827ac0519be3fe91bf96b4060eb00d1d24f82649b29862cd75a3cfca248b02a", size = 252807, upload-time = "2026-07-12T20:57:12.678Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6f/928a95da5da8b60f2b00e1482c7787b3316188e6d2d227fb8e124ada43a1/coverage-7.15.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2de8ecbbc77c7e4d22572779920ed8979c69168675e96be3a548c996568c6c31", size = 256965, upload-time = "2026-07-12T20:57:14.326Z" }, + { url = "https://files.pythonhosted.org/packages/16/10/889adbc1b8c9f866ed51e18a98bcafc0259fb9d29b81f50a719407c64ea8/coverage-7.15.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:2b25f0f0fa5260df9d7bb55d47c8bdc23fa3382c1a18f7c9cae122e6c320b1ad", size = 252628, upload-time = "2026-07-12T20:57:15.892Z" }, + { url = "https://files.pythonhosted.org/packages/1a/30/a5e1871e5d93416511f8e359d1ccebfe0cbb050a1bbf7dd20228533ec0cf/coverage-7.15.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a2effcbd93ae340a58db718fe4181d967f84d352c4cefeaab4ff82ce813901a", size = 254399, upload-time = "2026-07-12T20:57:17.703Z" }, + { url = "https://files.pythonhosted.org/packages/2d/26/c36fbffd549dadbdd1a75827528fb00a4c46aa3187b007b750b1e2cebbf2/coverage-7.15.1-cp313-cp313-win32.whl", hash = "sha256:895e65c96aef0cecea250f6e35e9a32f11375514e1a0cb5210e0fda128c04e8e", size = 223564, upload-time = "2026-07-12T20:57:19.253Z" }, + { url = "https://files.pythonhosted.org/packages/16/fc/becbb9d2c4206d242b9b1e1e8e24a42f7926c0200dd3c788b9fab4bb96d5/coverage-7.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6d0a28b63a0d75f9ed5118105d1154fc3aa40a8605a30d5d87e3d043ad90fe7", size = 224106, upload-time = "2026-07-12T20:57:21.108Z" }, + { url = "https://files.pythonhosted.org/packages/d3/30/1cfc641461369b6858799fca61c0a8b5edc490c519bf7c636ffa6bbf556f/coverage-7.15.1-cp313-cp313-win_arm64.whl", hash = "sha256:b4ee9818e8bae3544379ad2c09b851c4fb886aaa8860d57a1c1316ddcb16db49", size = 223497, upload-time = "2026-07-12T20:57:22.734Z" }, + { url = "https://files.pythonhosted.org/packages/f0/46/81961952e7aebfb38ad0ae4264e8954cc607a7af9e7ac111f9fa986595cc/coverage-7.15.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a886af95f59edf67d5770fd3564d53f4a8af93f25f8c1d60d27e00d7f5674ee8", size = 221560, upload-time = "2026-07-12T20:57:24.282Z" }, + { url = "https://files.pythonhosted.org/packages/13/d2/ee14d715889f216baf47301d9f469e08fff6995552aaf67e897b282865f6/coverage-7.15.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:985657ebd707941de90d488d1cbb5efac20bdf81f7b91eba771624ccda4d36f4", size = 221894, upload-time = "2026-07-12T20:57:25.87Z" }, + { url = "https://files.pythonhosted.org/packages/f3/38/f830bc6e6c2c5f23f43847125e6c650d378872f7eeba8d49f1d42193e8a9/coverage-7.15.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5bbe2a06e0a5e1404d9ffbdb49b819bbd6a3bb198ebea4c8dfe7ad9f1e1c2e81", size = 252938, upload-time = "2026-07-12T20:57:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/e1/53/0d3dd963631259d794c898735d5436e68d6a8d40749c419a07ff7c171469/coverage-7.15.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bde0fe24083d0b7b3dbafa7a09f0796410af1afa2523f28f5f208d8340a4aaca", size = 255445, upload-time = "2026-07-12T20:57:29.234Z" }, + { url = "https://files.pythonhosted.org/packages/b1/fd/aabed228557565c958259251b89bab8c5669b31291fa63b3e2154ebb017a/coverage-7.15.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f89f7453d6d46db14cf233e2cd8edcd78de2b9c49d4f1dc109590b4e5dbfbb74", size = 256790, upload-time = "2026-07-12T20:57:30.826Z" }, + { url = "https://files.pythonhosted.org/packages/bc/aa/1cc888e5d3623e603c4e5399653cb25728bb2b40d7519188a3e293d24620/coverage-7.15.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cc3656c9ecc27b36bd0907455b77f83c0069ca9ad4a66dec892b76c696eb6047", size = 259104, upload-time = "2026-07-12T20:57:32.63Z" }, + { url = "https://files.pythonhosted.org/packages/5f/61/fc16d5f5e53098dae41efa21e8ccc611a9b4fe922750dd03dc56db552182/coverage-7.15.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:24d8e85a2a45e44883b488c2659f51fa761dad5353fdb319b672a93facbd2ca9", size = 252956, upload-time = "2026-07-12T20:57:34.316Z" }, + { url = "https://files.pythonhosted.org/packages/a1/f3/52384668c3de4519ca770bf1975a89e4d6eb5aa2faf0da0577a14008cba4/coverage-7.15.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:68931b5fe746ed4fdaa8892989cab9e6c35781eeb3b0ab2ded893d561e1b3652", size = 254797, upload-time = "2026-07-12T20:57:35.947Z" }, + { url = "https://files.pythonhosted.org/packages/ce/68/54b807e7c1868178e902fd8360b5d4e559394462f97285c50edf1c4608db/coverage-7.15.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:1ce6947e2a95534ecaa5a15e73c21e550514c980d80eda204d064d789a95f6a4", size = 252762, upload-time = "2026-07-12T20:57:37.856Z" }, + { url = "https://files.pythonhosted.org/packages/7b/48/dde8adf0338e3ace738757dccf1ce817e5fdcadfae77e1b48a77e5a3b265/coverage-7.15.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:841befdbc89b9c82435fc25b0f4f41858b6238693e45af758bec4cfc1968171c", size = 257037, upload-time = "2026-07-12T20:57:39.488Z" }, + { url = "https://files.pythonhosted.org/packages/07/f2/179dd88cf60a0aeeee16a970ffe250dccea8b80ed4beab4c5d3f6c41ad4b/coverage-7.15.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5d3de58b837375e7f4c0e1a088ccab5f655efb2fd7427b729df02c862a559633", size = 252577, upload-time = "2026-07-12T20:57:41.363Z" }, + { url = "https://files.pythonhosted.org/packages/2c/37/8a593d69ab521beb6a105a2017cac4ba94425ee0a8349e29c3c0b522d24f/coverage-7.15.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b1801963f9f44ae0c0f6d737bc7aeb2bbcde7d1fe7e3b43cddc1961af42d3b41", size = 254235, upload-time = "2026-07-12T20:57:43.025Z" }, + { url = "https://files.pythonhosted.org/packages/6d/34/bc9b3bced66f2cdad4bf5e57ae51c54ea226e8aaaebfc9370a9a11877bf3/coverage-7.15.1-cp314-cp314-win32.whl", hash = "sha256:8c7953c4128ef53b6ffb5f90d87c87d4ce26731df294760bb2314eb0e069e44b", size = 223771, upload-time = "2026-07-12T20:57:44.662Z" }, + { url = "https://files.pythonhosted.org/packages/4c/f3/4d20337bed61915d14349e62b88d5e4144d5a9872b64adbe90e9906db6db/coverage-7.15.1-cp314-cp314-win_amd64.whl", hash = "sha256:6f0bab60a582d415f0fb535ccff13ba334a47a1538f98913330a525d23bd535a", size = 224257, upload-time = "2026-07-12T20:57:46.412Z" }, + { url = "https://files.pythonhosted.org/packages/7b/df/bbfeae4948f3ded516f92b32f2d57952427fc5ecfc0924487bb6ee6a5f38/coverage-7.15.1-cp314-cp314-win_arm64.whl", hash = "sha256:0f410ee8f0ac4ec7db71bc0b7632a8b9994e1cad2755bd1566c17e6a162caa74", size = 223683, upload-time = "2026-07-12T20:57:48.106Z" }, + { url = "https://files.pythonhosted.org/packages/35/65/0b431856064e387d1f5cf474625e4a0465e907024d42f35de6af19ced0be/coverage-7.15.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:fc868bab88e049d41fcd41766810d790a8b960053be2a45e060f5ce0d31d258b", size = 222298, upload-time = "2026-07-12T20:57:49.882Z" }, + { url = "https://files.pythonhosted.org/packages/a6/96/50eac9bd49df8a3df5f3d38746d1bf332299dffb554486c94ebd55c9dc49/coverage-7.15.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:206d4ec6028f2773b40932d09f074539d6bcdd8f6b318d40cb04bdbd68ed0b49", size = 222561, upload-time = "2026-07-12T20:57:51.688Z" }, + { url = "https://files.pythonhosted.org/packages/e0/5b/6ba1c4a27e10b8816fd2622b98162c83d3bdf1185097360373611bf96364/coverage-7.15.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:620482ef1c9f4e61f962e159325fe77dea59d16e39d9c9470d069053b244d864", size = 263923, upload-time = "2026-07-12T20:57:53.392Z" }, + { url = "https://files.pythonhosted.org/packages/e4/59/fe03ade97a3ca2d890e98c572cf48a99fda9adba85757c34b823f41efe1e/coverage-7.15.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d385fc9b054e309ad3cecdc77b586d2af0c98aeec2fdb3773544586f366e817c", size = 266043, upload-time = "2026-07-12T20:57:55.095Z" }, + { url = "https://files.pythonhosted.org/packages/16/e0/55c4b1217a572a43e13b39e1eb78d0da29fb23679003bd0cdf22c50b1978/coverage-7.15.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1198bca9c0dd7c188aae1f185b0c0b5fc4f0a2b6909000858c29550320bdb07", size = 268465, upload-time = "2026-07-12T20:57:57.017Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e2/ee47944f76afc03909119b036fe9e0da8cbd274a5141287de79791a0fb6d/coverage-7.15.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5d0297e6a070eadb49df7cddd0ab6f420b8b689dd8904c7dd815a323168fa57e", size = 269584, upload-time = "2026-07-12T20:57:58.958Z" }, + { url = "https://files.pythonhosted.org/packages/cb/8a/6b4d9779c7b2e21c3d12c3425e3261aa7411399319e27aa402dfec4db5d0/coverage-7.15.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:916fcf2214f56960e409561b37fc32a160a42b6e85483d0652d7b70fa55d707e", size = 263019, upload-time = "2026-07-12T20:58:00.979Z" }, + { url = "https://files.pythonhosted.org/packages/c4/1e/db5c7fa0c8ba5ece390a1e1a3f30db71d440240a80589df28e66a7503c40/coverage-7.15.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f837bae572c7869ffaa502e604c87e182543012831cf87aae4586ad090ac6dcf", size = 265916, upload-time = "2026-07-12T20:58:03.005Z" }, + { url = "https://files.pythonhosted.org/packages/83/53/fe5176682b00709b13fab36addd26883139d0dea430816fea412e69255e2/coverage-7.15.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3ea65e3ee6c7c32349fd00559927a9e577bdd72386087eeed1c42b62dfce9b82", size = 263520, upload-time = "2026-07-12T20:58:04.994Z" }, + { url = "https://files.pythonhosted.org/packages/f9/e7/16f15127be93fbc70c667df5ec5dce934fc76c9b0888d84969a5d5341e2c/coverage-7.15.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:345034976f46a1c54bd17f4e43eb30bb92cb7082fcddff03250cff136cc4eb82", size = 267254, upload-time = "2026-07-12T20:58:06.824Z" }, + { url = "https://files.pythonhosted.org/packages/cd/73/e5119111f6f065376395a525f7ce6e9174d83f3db6d217ea0211a61cca4d/coverage-7.15.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4f051a64eb8f8addb4661c2b41d6eea5b7ebc68ad4b2baea8d9bc54e1956e5f7", size = 262366, upload-time = "2026-07-12T20:58:08.555Z" }, + { url = "https://files.pythonhosted.org/packages/d7/9c/6d0a81182df18a73b081e7a8630f0e2a52b12dfd7898c6ab839551a454d2/coverage-7.15.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a7625770f7720b49bb30d194ad2f8d50fab3c5177874af3d2399676f95f9c594", size = 264680, upload-time = "2026-07-12T20:58:10.359Z" }, + { url = "https://files.pythonhosted.org/packages/d9/a2/bac0cbd4450638f1be2041a464b1766c8cc94abf705a2df6f1c8d4be870d/coverage-7.15.1-cp314-cp314t-win32.whl", hash = "sha256:81e503d130a472ad1bd38199ecd35116b40d92bcd31e27a2cacde035381f2070", size = 224077, upload-time = "2026-07-12T20:58:12.065Z" }, + { url = "https://files.pythonhosted.org/packages/f0/b2/d83c5403155172a43ba47c08641bad3f89822d8405102423a41339d2c857/coverage-7.15.1-cp314-cp314t-win_amd64.whl", hash = "sha256:724e878b213b302ad46e9f2fc872d386613f20ebfc492a211482d917ea76c14f", size = 224908, upload-time = "2026-07-12T20:58:13.956Z" }, + { url = "https://files.pythonhosted.org/packages/cc/41/442b74cad832cc77712080585455482e7cc4f4a9a13192f65731dcd18231/coverage-7.15.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ce2f05c14d077f406fefc4fa5e4f093ad0e0787549f6582535d6e28766f0361b", size = 224219, upload-time = "2026-07-12T20:58:16.315Z" }, + { url = "https://files.pythonhosted.org/packages/34/98/07a67cf1a26e795d617ed5c540c042b0ac87b72f810c30c07f076cf334f3/coverage-7.15.1-py3-none-any.whl", hash = "sha256:717d01e6e00bed56ad13306f19e0dd2f4f645ee8159d2c72c72301d6cfc7090c", size = 213284, upload-time = "2026-07-12T20:58:18.079Z" }, ] [package.optional-dependencies] @@ -11939,7 +11959,7 @@ wheels = [ [[package]] name = "fastcore" -version = "2.0.4" +version = "2.0.5" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.15' and sys_platform == 'win32'", @@ -11953,9 +11973,9 @@ resolution-markers = [ "python_full_version == '3.11.*' and platform_machine == 'arm64' and sys_platform == 'darwin'", "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] -sdist = { url = "https://files.pythonhosted.org/packages/3c/3a/6ab189da89201ea24cd294bc56a20b6de621b7899485b6497c9240963a5b/fastcore-2.0.4.tar.gz", hash = "sha256:5b65f7fc39ab716c62e9117746c6ea897ae111f9d71f4d7d6d0c08daac8c9480", size = 103215, upload-time = "2026-07-11T04:46:52.234Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/83/a4452a9e0c078d844e0882745d0a44d3f57c968e1ae0d5dbc06892a403d0/fastcore-2.0.5.tar.gz", hash = "sha256:7be70ec5517723e4caeac0725a75942f9f081d152470d1401699a1ba3f536c01", size = 103271, upload-time = "2026-07-13T07:19:35.999Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/40/25/4094eede4ca19d712c646402a9db5c9ab67cba4e6ff192754805655ec8a9/fastcore-2.0.4-py3-none-any.whl", hash = "sha256:15a4044114dc54d1c6bfdfec1a6c0e5b64c9b115d5881e9ef879de2a0f9b9a97", size = 108099, upload-time = "2026-07-11T04:46:50.641Z" }, + { url = "https://files.pythonhosted.org/packages/5e/cc/4bc2c4514444fe27514700a4fb2d8385f32cb65f3f0e0cf752eceed5854a/fastcore-2.0.5-py3-none-any.whl", hash = "sha256:88a7149aea4457717a02bdc74a4e3e4669d1cc9b611759da984333f83a280d85", size = 108123, upload-time = "2026-07-13T07:19:34.53Z" }, ] [[package]] @@ -12493,7 +12513,7 @@ name = "geomet" version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click", marker = "python_full_version < '3.14'" }, + { name = "click" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2a/8c/dde022aa6747b114f6b14a7392871275dea8867e2bd26cddb80cc6d66620/geomet-1.1.0.tar.gz", hash = "sha256:51e92231a0ef6aaa63ac20c443377ba78a303fd2ecd179dc3567de79f3c11605", size = 28732, upload-time = "2023-11-14T15:43:36.764Z" } wheels = [ @@ -12574,14 +12594,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.50" +version = "3.1.51" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/33/f6/354ae6491228b5eb40e10d89c4d13c651fe1cf7556e35ebdded50cff57ce/gitpython-3.1.50.tar.gz", hash = "sha256:80da2d12504d52e1f998772dc5baf6e553f8d2fcfe1fcc226c9d9a2ee3372dcc", size = 219798, upload-time = "2026-05-06T04:01:26.571Z" } +sdist = { url = "https://files.pythonhosted.org/packages/59/30/a8a0c15f9480dc91b5b7f11ebd26105e5f80898d7ff02da197fef35d8395/gitpython-3.1.51.tar.gz", hash = "sha256:22c9c94bb6b0b9f3c7157c684fece45a414cea204586b600beae6cd4570dcd6d", size = 223519, upload-time = "2026-07-12T13:40:07.922Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, + { url = "https://files.pythonhosted.org/packages/f8/11/1232bb1ba52a230f20ff08e1b3df7cd9d43a71b61cc0a1c37941064fe4c7/gitpython-3.1.51-py3-none-any.whl", hash = "sha256:d5b29685708f3c80a56db040b665bfd69492c8e150b5848532af8013b686c5a4", size = 215246, upload-time = "2026-07-12T13:40:06.43Z" }, ] [[package]] @@ -13926,7 +13946,7 @@ name = "gssapi" version = "1.11.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "decorator", marker = "python_full_version < '3.15' or sys_platform != 'win32'" }, + { name = "decorator" }, ] sdist = { url = "https://files.pythonhosted.org/packages/23/52/c1e90623c259a42ab0587078bb04f959867b970add46ff66750ead8fc7c5/gssapi-1.11.1.tar.gz", hash = "sha256:2049ee4b1d0c363163a1344b7282a363f9f4094e51d2c36de0cf01d4735e0ae2", size = 95233, upload-time = "2026-01-26T21:01:39.463Z" } wheels = [ @@ -14711,17 +14731,17 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version < '3.11'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "jedi", marker = "python_full_version < '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, - { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "stack-data", marker = "python_full_version < '3.11'" }, - { name = "traitlets", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "exceptiongroup" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } wheels = [ @@ -14745,18 +14765,18 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version >= '3.11'" }, - { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, - { name = "jedi", marker = "python_full_version >= '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, - { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, - { name = "psutil", marker = "python_full_version >= '3.11' and sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "stack-data", marker = "python_full_version >= '3.11'" }, - { name = "traitlets", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "psutil", marker = "sys_platform != 'cygwin' and sys_platform != 'emscripten'" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/59/165d3b4d75cc34add3122c4417ecb229085140ac573103c223cd01dde96f/ipython-9.15.0.tar.gz", hash = "sha256:da2819ce2aa83135257df830660b1176d986c3d2876db24df01974fa955b2756", size = 4442580, upload-time = "2026-06-26T11:03:35.913Z" } wheels = [ @@ -14768,7 +14788,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -18096,9 +18116,9 @@ wheels = [ [package.optional-dependencies] sql-other = [ - { name = "adbc-driver-postgresql", marker = "python_full_version < '3.13'" }, - { name = "adbc-driver-sqlite", marker = "python_full_version < '3.13'" }, - { name = "sqlalchemy", marker = "python_full_version < '3.13'" }, + { name = "adbc-driver-postgresql" }, + { name = "adbc-driver-sqlite" }, + { name = "sqlalchemy" }, ] [[package]] @@ -20506,9 +20526,9 @@ name = "python3-saml" version = "1.16.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "isodate", marker = "python_full_version < '3.13'" }, - { name = "lxml", marker = "python_full_version < '3.13'" }, - { name = "xmlsec", marker = "python_full_version < '3.13'" }, + { name = "isodate" }, + { name = "lxml" }, + { name = "xmlsec" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5d/98/6e0268c3a9893af3d4c5cf670183e0314cd6b5cb034a612d6a7cc5060df8/python3-saml-1.16.0.tar.gz", hash = "sha256:97c9669aecabc283c6e5fb4eb264f446b6e006f5267d01c9734f9d8bffdac133", size = 83468, upload-time = "2023-10-09T10:37:43.128Z" } wheels = [ @@ -20572,7 +20592,7 @@ dependencies = [ { name = "cryptography" }, { name = "docker" }, { name = "fastcore", version = "1.14.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "fastcore", version = "2.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "fastcore", version = "2.0.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "httpr" }, { name = "httpx", extra = ["http2"] }, { name = "jinja2" }, @@ -21641,10 +21661,10 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "joblib", marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "threadpoolctl", marker = "python_full_version < '3.11'" }, + { name = "joblib" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, + { name = "threadpoolctl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/37/59/44985a2bdc95c74e34fef3d10cb5d93ce13b0e2a7baefffe1b53853b502d/scikit_learn-1.5.2.tar.gz", hash = "sha256:b4237ed7b3fdd0a4882792e68ef2545d5baa50aca3bb45aa7df468138ad8f94d", size = 7001680, upload-time = "2024-09-11T15:50:10.957Z" } wheels = [ @@ -21687,13 +21707,13 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "joblib", marker = "python_full_version >= '3.11'" }, - { name = "narwhals", marker = "python_full_version >= '3.11'" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "joblib" }, + { name = "narwhals" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "scipy", version = "1.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "threadpoolctl", marker = "python_full_version >= '3.11'" }, + { name = "threadpoolctl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } wheels = [ @@ -21738,7 +21758,7 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -21798,7 +21818,7 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -21879,7 +21899,7 @@ resolution-markers = [ "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [ @@ -21964,8 +21984,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "python_full_version >= '3.14' or platform_machine != 'arm64' or sys_platform != 'darwin'" }, - { name = "jeepney", marker = "python_full_version >= '3.14' or platform_machine != 'arm64' or sys_platform != 'darwin'" }, + { name = "cryptography" }, + { name = "jeepney" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -22268,15 +22288,15 @@ name = "snowflake-snowpark-python" version = "1.53.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cloudpickle", version = "3.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" }, - { name = "protobuf", marker = "python_full_version < '3.14'" }, - { name = "python-dateutil", marker = "python_full_version < '3.14'" }, - { name = "pyyaml", marker = "python_full_version < '3.14'" }, - { name = "setuptools", marker = "python_full_version < '3.14'" }, - { name = "snowflake-connector-python", marker = "python_full_version < '3.14'" }, - { name = "typing-extensions", marker = "python_full_version < '3.14'" }, - { name = "tzlocal", marker = "python_full_version < '3.14'" }, - { name = "wheel", marker = "python_full_version < '3.14'" }, + { name = "cloudpickle", version = "3.1.1", source = { registry = "https://pypi.org/simple" } }, + { name = "protobuf" }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "setuptools" }, + { name = "snowflake-connector-python" }, + { name = "typing-extensions" }, + { name = "tzlocal" }, + { name = "wheel" }, ] sdist = { url = "https://files.pythonhosted.org/packages/67/98/38189e919c54fc6f09a43e778d850ebf2116ced491595971ae5f9ca01b0c/snowflake_snowpark_python-1.53.0.tar.gz", hash = "sha256:6eab04d5703fac72982f3666140c006d6fb9964d46a04fd953766410af8fc62e", size = 1783158, upload-time = "2026-07-09T16:15:19.599Z" } wheels = [ @@ -22323,23 +22343,23 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "alabaster", marker = "python_full_version < '3.11'" }, - { name = "babel", marker = "python_full_version < '3.11'" }, - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "imagesize", marker = "python_full_version < '3.11'" }, - { name = "jinja2", marker = "python_full_version < '3.11'" }, - { name = "packaging", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "requests", marker = "python_full_version < '3.11'" }, - { name = "snowballstemmer", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version < '3.11'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version < '3.11'" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" } }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, + { name = "tomli" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/be0b61178fe2cdcb67e2a92fc9ebb488e3c51c4f74a36a7824c0adf23425/sphinx-8.1.3.tar.gz", hash = "sha256:43c1911eecb0d3e161ad78611bc905d1ad0e523e4ddc202a58a821773dc4c927", size = 8184611, upload-time = "2024-10-13T20:27:13.93Z" } wheels = [ @@ -22355,23 +22375,23 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "alabaster", marker = "python_full_version == '3.11.*'" }, - { name = "babel", marker = "python_full_version == '3.11.*'" }, - { name = "colorama", marker = "python_full_version == '3.11.*' and sys_platform == 'win32'" }, - { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, - { name = "imagesize", marker = "python_full_version == '3.11.*'" }, - { name = "jinja2", marker = "python_full_version == '3.11.*'" }, - { name = "packaging", marker = "python_full_version == '3.11.*'" }, - { name = "pygments", marker = "python_full_version == '3.11.*'" }, - { name = "requests", marker = "python_full_version == '3.11.*'" }, - { name = "roman-numerals", marker = "python_full_version == '3.11.*'" }, - { name = "snowballstemmer", marker = "python_full_version == '3.11.*'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version == '3.11.*'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version == '3.11.*'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version == '3.11.*'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version == '3.11.*'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version == '3.11.*'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version == '3.11.*'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" } }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/50/a8c6ccc36d5eacdfd7913ddccd15a9cee03ecafc5ee2bc40e1f168d85022/sphinx-9.0.4.tar.gz", hash = "sha256:594ef59d042972abbc581d8baa577404abe4e6c3b04ef61bd7fc2acbd51f3fa3", size = 8710502, upload-time = "2025-12-04T07:45:27.343Z" } wheels = [ @@ -22393,23 +22413,23 @@ resolution-markers = [ "(python_full_version == '3.12.*' and platform_machine != 'arm64') or (python_full_version == '3.12.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "alabaster", marker = "python_full_version >= '3.12'" }, - { name = "babel", marker = "python_full_version >= '3.12'" }, - { name = "colorama", marker = "python_full_version >= '3.12' and sys_platform == 'win32'" }, - { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "imagesize", marker = "python_full_version >= '3.12'" }, - { name = "jinja2", marker = "python_full_version >= '3.12'" }, - { name = "packaging", marker = "python_full_version >= '3.12'" }, - { name = "pygments", marker = "python_full_version >= '3.12'" }, - { name = "requests", marker = "python_full_version >= '3.12'" }, - { name = "roman-numerals", marker = "python_full_version >= '3.12'" }, - { name = "snowballstemmer", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-applehelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-devhelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-htmlhelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-qthelp", marker = "python_full_version >= '3.12'" }, - { name = "sphinxcontrib-serializinghtml", marker = "python_full_version >= '3.12'" }, + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils", version = "0.22.4", source = { registry = "https://pypi.org/simple" } }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } wheels = [ @@ -22475,12 +22495,12 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11'" }, - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "starlette", marker = "python_full_version < '3.11'" }, - { name = "uvicorn", marker = "python_full_version < '3.11'" }, - { name = "watchfiles", marker = "python_full_version < '3.11'" }, - { name = "websockets", marker = "python_full_version < '3.11'" }, + { name = "colorama" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" } }, + { name = "starlette" }, + { name = "uvicorn" }, + { name = "watchfiles" }, + { name = "websockets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a5/2c/155e1de2c1ba96a72e5dba152c509a8b41e047ee5c2def9e9f0d812f8be7/sphinx_autobuild-2024.10.3.tar.gz", hash = "sha256:248150f8f333e825107b6d4b86113ab28fa51750e5f9ae63b59dc339be951fb1", size = 14023, upload-time = "2024-10-02T23:15:30.172Z" } wheels = [ @@ -22504,13 +22524,13 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11'" }, - { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "colorama" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "starlette", marker = "python_full_version >= '3.11'" }, - { name = "uvicorn", marker = "python_full_version >= '3.11'" }, - { name = "watchfiles", marker = "python_full_version >= '3.11'" }, - { name = "websockets", marker = "python_full_version >= '3.11'" }, + { name = "starlette" }, + { name = "uvicorn" }, + { name = "watchfiles" }, + { name = "websockets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e0/3c/a59a3a453d4133777f7ed2e83c80b7dc817d43c74b74298ca0af869662ad/sphinx_autobuild-2025.8.25.tar.gz", hash = "sha256:9cf5aab32853c8c31af572e4fecdc09c997e2b8be5a07daf2a389e270e85b213", size = 15200, upload-time = "2025-08-25T18:44:55.436Z" } wheels = [ @@ -22540,7 +22560,7 @@ resolution-markers = [ "(python_full_version < '3.11' and platform_machine != 'arm64') or (python_full_version < '3.11' and sys_platform != 'darwin')", ] dependencies = [ - { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/2b/69/b34e0cb5336f09c6866d53b4a19d76c227cdec1bbc7ac4de63ca7d58c9c7/sphinx_design-0.6.1.tar.gz", hash = "sha256:b44eea3719386d04d765c1a8257caca2b3e6f8421d7b3a5e742c0fd45f84e632", size = 2193689, upload-time = "2024-08-02T13:48:44.277Z" } wheels = [ @@ -22564,7 +22584,7 @@ resolution-markers = [ "(python_full_version == '3.11.*' and platform_machine != 'arm64') or (python_full_version == '3.11.*' and sys_platform != 'darwin')", ] dependencies = [ - { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "sphinx", version = "9.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "sphinx", version = "9.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/13/7b/804f311da4663a4aecc6cf7abd83443f3d4ded970826d0c958edc77d4527/sphinx_design-0.7.0.tar.gz", hash = "sha256:d2a3f5b19c24b916adb52f97c5f00efab4009ca337812001109084a740ec9b7a", size = 2203582, upload-time = "2026-01-19T13:12:53.297Z" } @@ -22977,14 +22997,15 @@ wheels = [ [[package]] name = "svcs" -version = "25.1.0" +version = "26.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "attrs" }, + { name = "typing-extensions", marker = "python_full_version < '3.15'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/08/e1/2d56ec21820cae24a6215dc1d6a224167b0e24368bf53166551064742e0d/svcs-25.1.0.tar.gz", hash = "sha256:64dd74d0c4e8fee79a9ac7550a9d824670b228df390ba1911614e29b59b3f2c2", size = 714086, upload-time = "2025-01-25T13:15:21.505Z" } +sdist = { url = "https://files.pythonhosted.org/packages/50/ea/10957367bf49b401a8100ff434f453a340cc30c9c2fc3b2367e6c282f03d/svcs-26.1.0.tar.gz", hash = "sha256:71550e3b228d529448136d78013b2d65528eec4632b8779f8f89c1d8585eed99", size = 906116, upload-time = "2026-07-13T10:09:38.322Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/1c/a6b5d90a9ca479805798276728ccbbdff0c7228e2ea93b1f731779d635e3/svcs-25.1.0-py3-none-any.whl", hash = "sha256:df49cb7d1a05dfd2dd60af1a2cb84b9c3bb0a74728833cb8c54a7ceeecce6c97", size = 19456, upload-time = "2025-01-25T13:15:19.677Z" }, + { url = "https://files.pythonhosted.org/packages/5e/8c/5f9441100851bea8f4735cf54a2980d2ad9300a5933c1d5e416448fd789a/svcs-26.1.0-py3-none-any.whl", hash = "sha256:87428c6371af6ae5d0936193613ba4e463118f36b43e15a708f417a8e0b85520", size = 23609, upload-time = "2026-07-13T10:09:36.945Z" }, ] [[package]] @@ -23575,11 +23596,11 @@ wheels = [ [[package]] name = "types-cachetools" -version = "7.0.0.20260518" +version = "7.0.0.20260713" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c8/14/1e4fca2b250dbc75be9f0beab083acb3cd1151711e1031eb4a854dfd71be/types_cachetools-7.0.0.20260518.tar.gz", hash = "sha256:7730014e4fef0c6f01e2cd0f980f8ce6d1b1d2472c8459c1f382348ec1a6b435", size = 10072, upload-time = "2026-05-18T06:02:20.396Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/64/66d7efdb36ecf6826aca5415e59fe2df96e97d24157147e53acfbe8dda11/types_cachetools-7.0.0.20260713.tar.gz", hash = "sha256:f1acf079e9c66a81e096a897ef0b261a82117cf856834e37b4bd0c9a116a076a", size = 10199, upload-time = "2026-07-13T05:22:21.845Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/45/e0/767be6b60859fd2edc4512fabdedbce703fc8d4ec5007b31abaf37a51c6c/types_cachetools-7.0.0.20260518-py3-none-any.whl", hash = "sha256:997b356870915f8bbc9b2cdb4e7271c01d487996fdac2a9c8e91cc5b1261b3d1", size = 9500, upload-time = "2026-05-18T06:02:19.042Z" }, + { url = "https://files.pythonhosted.org/packages/0e/c7/d3525c9dbdc1be7786bad46655ef051b6e7993f656d304719ec40079c91c/types_cachetools-7.0.0.20260713-py3-none-any.whl", hash = "sha256:6db9bcc7a3840d39e91c04117d85a9d0937eacc9d14d12a873e2b01a2d24a71d", size = 9615, upload-time = "2026-07-13T05:22:20.76Z" }, ] [[package]] @@ -24520,7 +24541,7 @@ name = "xmlsec" version = "1.3.17" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "lxml", marker = "python_full_version < '3.13'" }, + { name = "lxml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/49/14/538b75379e6ab8f688f14d8663e2ab138d9c778bac4999d155b5f33c71c1/xmlsec-1.3.17.tar.gz", hash = "sha256:f3fac9ae679f66585925cc00c5f6839ae36c1d03157619571dee18acc05b9c01", size = 115637, upload-time = "2025-11-11T16:20:46.019Z" } wheels = [