diff --git a/.github/workflows/CD_dagster_branch.yml b/.github/workflows/CD_dagster_branch.yml new file mode 100644 index 000000000..69394860a --- /dev/null +++ b/.github/workflows/CD_dagster_branch.yml @@ -0,0 +1,90 @@ +# Creates a Dagster+ branch deployment for a pull request, so ingestion changes +# can be materialized against an isolated deployment before they reach prod. +# +# Path-filtered: most PRs in this repository touch only the API and should not +# create a Dagster+ deployment at all. +name: CD (Dagster+ branch deployment) + +on: + pull_request: + types: [opened, synchronize, reopened, closed] + paths: + - "automated_ingestion/**" + # The code location imports db/ models and domain/ rules in-process, + # so a change to either alters what this image runs even when no + # ingestion file moves. Without these, a domain fix merged to + # production would leave the pipeline running the old rule against + # the live database. The cost is that ordinary API changes to these + # directories also trigger a build; a stale code location is worse. + - "db/**" + - "domain/**" + - "dagster_cloud.yaml" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/CD_dagster_branch.yml" + +permissions: + contents: read + pull-requests: write + +# One deployment per PR; a force-push supersedes the run it interrupts. +concurrency: + group: dagster-branch-deploy-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + dagster-branch-deploy: + runs-on: ubuntu-latest + # Forks cannot read the Dagster+ secrets, and a branch deployment from an + # untrusted fork would run our code against our infrastructure regardless. + if: github.event.pull_request.head.repo.full_name == github.repository + + # The action's notify steps post build status as a PR comment and read the + # token from the workflow environment -- `env.GITHUB_TOKEN`, not the + # `secrets` context. Without this the run dies on an empty-token assertion + # before it ever reaches Dagster+, which reads as an auth failure but is + # not one. + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + steps: + - name: Check out source repository + uses: actions/checkout@v7.0.1 + + # parse_workspace performs its own `actions/checkout`, which cleans the + # working tree. It has to run *before* requirements.txt is generated, or + # the generated file is deleted before the deploy step can use it. + - name: Parse dagster_cloud.yaml + id: parse + uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v1.13.18 + with: + dagster_cloud_file: dagster_cloud.yaml + + - name: Install uv in container + uses: astral-sh/setup-uv@v10.0.1 + with: + version: "latest" + + - name: Generate requirements.txt + run: | + uv export \ + --format requirements-txt \ + --no-emit-project \ + --no-dev \ + --group ingestion \ + --output-file requirements.txt + + # Runs on `closed` too: the action tears the branch deployment down when + # the PR is merged or abandoned, so stale deployments do not accumulate. + - name: Deploy to Dagster+ branch deployment + uses: dagster-io/dagster-cloud-action/actions/serverless_branch_deploy@v1.13.18 + with: + organization_id: ${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }} + dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + location: ${{ toJson(fromJson(steps.parse.outputs.build_info)[0]) }} + checkout_repo: false + # The action defaults to python:3.8-slim, which cannot install a + # lockfile resolved for requires-python >= 3.13 -- pip reports the + # pins as having no matching distribution rather than as a version + # conflict, which reads like a broken requirements file. + base_image: python:3.13-slim diff --git a/.github/workflows/CD_dagster_prod.yml b/.github/workflows/CD_dagster_prod.yml new file mode 100644 index 000000000..1ff7d5de4 --- /dev/null +++ b/.github/workflows/CD_dagster_prod.yml @@ -0,0 +1,89 @@ +# Deploys the `ocotillo-automated-ingestion` code location to the Dagster+ prod +# deployment. +# +# Triggered on `production`, not `main`: `main` was abandoned in July 2025 and +# the release flow runs feature -> staging -> production (docs/release-flow.md). +# The plan document's reference to `main` predates that being checked. +# +# Path-filtered so an ordinary API change does not spend a Dagster+ build. The +# filter includes pyproject.toml and uv.lock because the location's dependency +# set is exported from them, so a lockfile bump changes the built image even +# when no ingestion source file does. +name: CD (Dagster+ prod) + +on: + push: + branches: [production] + paths: + - "automated_ingestion/**" + # The code location imports db/ models and domain/ rules in-process, + # so a change to either alters what this image runs even when no + # ingestion file moves. Without these, a domain fix merged to + # production would leave the pipeline running the old rule against + # the live database. The cost is that ordinary API changes to these + # directories also trigger a build; a stale code location is worse. + - "db/**" + - "domain/**" + - "dagster_cloud.yaml" + - "pyproject.toml" + - "uv.lock" + - ".github/workflows/CD_dagster_prod.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: dagster-prod-deploy + cancel-in-progress: false + +jobs: + dagster-prod-deploy: + runs-on: ubuntu-latest + environment: production + + steps: + - name: Check out source repository + uses: actions/checkout@v7.0.1 + + # parse_workspace performs its own `actions/checkout`, which cleans the + # working tree. It has to run *before* requirements.txt is generated, or + # the generated file is deleted before the deploy step can use it. + - name: Parse dagster_cloud.yaml + id: parse + uses: dagster-io/dagster-cloud-action/actions/utils/parse_workspace@v1.13.18 + with: + dagster_cloud_file: dagster_cloud.yaml + + - name: Install uv in container + uses: astral-sh/setup-uv@v10.0.1 + with: + version: "latest" + + # Dagster+ builds from a requirements.txt, which the repo does not keep + # under version control. `--group ingestion` adds dagster and dlt on top + # of the runtime dependencies; the runtime ones are needed too, because + # the loader imports `db/` and `domain/`. + - name: Generate requirements.txt + run: | + uv export \ + --format requirements-txt \ + --no-emit-project \ + --no-dev \ + --group ingestion \ + --output-file requirements.txt + + # checkout_repo is false because requirements.txt is generated above and + # a second checkout would discard it. + - name: Deploy to Dagster+ prod + uses: dagster-io/dagster-cloud-action/actions/serverless_prod_deploy@v1.13.18 + with: + organization_id: ${{ vars.DAGSTER_CLOUD_ORGANIZATION_ID }} + dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }} + location: ${{ toJson(fromJson(steps.parse.outputs.build_info)[0]) }} + checkout_repo: false + # The action defaults to python:3.8-slim, which cannot install a + # lockfile resolved for requires-python >= 3.13 -- pip reports the + # pins as having no matching distribution rather than as a version + # conflict, which reads like a broken requirements file. + base_image: python:3.13-slim diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 650a748b3..fb68c00e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,7 +86,7 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('uv.lock') }} - name: Install the project - run: uv sync --locked --all-extras --dev --group cli + run: uv sync --locked --all-extras --dev --group cli --group ingestion - name: Show Alembic heads run: uv run alembic heads @@ -214,7 +214,7 @@ jobs: key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('uv.lock') }} - name: Install the project - run: uv sync --locked --all-extras --dev --group cli + run: uv sync --locked --all-extras --dev --group cli --group ingestion - name: Show Alembic heads run: uv run alembic heads diff --git a/automated_ingestion/__init__.py b/automated_ingestion/__init__.py new file mode 100644 index 000000000..1fb584ba5 --- /dev/null +++ b/automated_ingestion/__init__.py @@ -0,0 +1,33 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Automated ingestion: scheduled pipelines that land external monitoring data in +Ocotillo without anyone hand-carrying a file. + +This package is deployed as its own Dagster+ code location, separate from the +API process, but it lives in this repository so the loader can import ``db/`` +models and ``domain/`` rules directly instead of maintaining a second copy of +the Ocotillo schema elsewhere. + +Shape of a source: a dlt pipeline extracts the vendor API into a GCS raw zone, +an adapter maps raw records onto Ocotillo structures, and a loader writes them +to Postgres over a direct connection. San Acacia Reach (Van Essen divers) is +the first source; ``shared/`` holds what later sources reuse. + +See ``docs/automated-ingestion-pipeline-plan.md``. +""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/__init__.py b/automated_ingestion/defs/__init__.py new file mode 100644 index 000000000..4bfea2869 --- /dev/null +++ b/automated_ingestion/defs/__init__.py @@ -0,0 +1,18 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Dagster definitions: the code location's assets, jobs, and schedules.""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/assets/__init__.py b/automated_ingestion/defs/assets/__init__.py new file mode 100644 index 000000000..3671abfa5 --- /dev/null +++ b/automated_ingestion/defs/assets/__init__.py @@ -0,0 +1,43 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Asset collection for the code location. + +Per-source assets are declared in their own modules and gathered here so +``definitions.py`` never has to know which sources exist. +""" + +from dagster import AssetsDefinition + +from automated_ingestion.defs.assets.connectivity import database_connectivity +from automated_ingestion.defs.assets.heartbeat import ingestion_heartbeat +from automated_ingestion.sources.san_acacia.ingest import ( + raw_san_acacia_locations, + raw_san_acacia_readings, +) + + +def all_assets() -> list[AssetsDefinition]: + """Every asset the code location exposes.""" + return [ + ingestion_heartbeat, + database_connectivity, + raw_san_acacia_locations, + raw_san_acacia_readings, + ] + + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/assets/connectivity.py b/automated_ingestion/defs/assets/connectivity.py new file mode 100644 index 000000000..c8c83fcbc --- /dev/null +++ b/automated_ingestion/defs/assets/connectivity.py @@ -0,0 +1,62 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Proves the Dagster+ runtime can reach Ocotillo Postgres. + +Dagster+ Serverless runs outside the VPC, so Cloud SQL's private IP is +unreachable from it -- the connection has to go through the Cloud SQL connector +instead. That is the single riskiest assumption in the foundations task, and it +fails at run time rather than at deploy time. This asset makes it fail loudly, +on its own, in an asset whose only job is to fail there. + +It reads and never writes: connectivity and permission are separable problems, +and a write here would leave test rows in a real table. +""" + +from dagster import AssetExecutionContext, MetadataValue, Output, asset + +from automated_ingestion.defs.resources import OcotilloDatabase + + +@asset( + group_name="operations", + description="Reads from Ocotillo Postgres to prove the runtime can connect.", +) +def database_connectivity( + context: AssetExecutionContext, database: OcotilloDatabase +) -> Output[int]: + """Count transducer observations, returning the count as metadata.""" + from sqlalchemy import func, select + + from db.transducer import TransducerObservation + + with database.session() as session: + count = session.scalar(select(func.count()).select_from(TransducerObservation)) + + count = int(count or 0) + context.log.info("connected to Ocotillo; transducer_observation rows: %s", count) + return Output( + count, + metadata={ + "transducer_observation_rows": MetadataValue.int(count), + "note": MetadataValue.text( + "Read-only. A failure here is connectivity or grants, not data." + ), + }, + ) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/assets/heartbeat.py b/automated_ingestion/defs/assets/heartbeat.py new file mode 100644 index 000000000..2fe21ee28 --- /dev/null +++ b/automated_ingestion/defs/assets/heartbeat.py @@ -0,0 +1,41 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +A trivial asset that proves the code location deploys and materializes. + +It touches nothing -- no database, no network, no GCS -- so a failure here is +unambiguously a packaging or deployment problem rather than a credential or +connectivity one. The Postgres connectivity check that BDMS task 1.4 calls for +is a separate asset, added when the least-privilege role exists. +""" + +from datetime import datetime, timezone + +from dagster import AssetExecutionContext, asset + + +@asset( + group_name="operations", + description="Static heartbeat proving the code location loaded and can run.", +) +def ingestion_heartbeat(context: AssetExecutionContext) -> str: + """Return the materialization timestamp.""" + stamp = datetime.now(timezone.utc).isoformat() + context.log.info("automated_ingestion code location alive at %s", stamp) + return stamp + + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/definitions.py b/automated_ingestion/defs/definitions.py new file mode 100644 index 000000000..aabacc42a --- /dev/null +++ b/automated_ingestion/defs/definitions.py @@ -0,0 +1,34 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Entry point for the ``ocotillo-automated-ingestion`` Dagster+ code location. + +``[tool.dagster] module_name`` in ``pyproject.toml`` points here, so this is +what ``dagster dev`` and the Dagster+ agent import. Keep it thin: it collects +definitions declared elsewhere in the package rather than declaring them here. +""" + +from dagster import Definitions + +from automated_ingestion.defs.assets import all_assets +from automated_ingestion.defs.resources import OcotilloDatabase + +defs = Definitions( + assets=all_assets(), + resources={"database": OcotilloDatabase()}, +) + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/jobs/__init__.py b/automated_ingestion/defs/jobs/__init__.py new file mode 100644 index 000000000..a33f53655 --- /dev/null +++ b/automated_ingestion/defs/jobs/__init__.py @@ -0,0 +1,18 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Jobs: backfill and any other non-schedule-driven runs.""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/jobs/backfill.py b/automated_ingestion/defs/jobs/backfill.py new file mode 100644 index 000000000..18236a0b2 --- /dev/null +++ b/automated_ingestion/defs/jobs/backfill.py @@ -0,0 +1,30 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Backfill job factory. + +Two modes are planned, both filled in under BDMS task 4: + +- **Mode A (refetch)** re-pulls a window from the vendor API when a gap is real + data we never collected. +- **Mode B (replay)** reprocesses parquet already in the GCS raw zone through + the current adapter, with no API calls, when the bug was in our mapping. + +Both chunk the window, checkpoint per chunk so an interrupted run resumes, and +default to ``dry_run=True``. +""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/defs/resources.py b/automated_ingestion/defs/resources.py new file mode 100644 index 000000000..6e11316d8 --- /dev/null +++ b/automated_ingestion/defs/resources.py @@ -0,0 +1,51 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Dagster resources: the pipeline's handles on the outside world. + +The database resource deliberately delegates to ``db/engine.py`` rather than +building its own engine. Connection setup for Cloud SQL -- the connector, IAM +auth, the IP-type choice -- is intricate and already solved there; a second +implementation would be a second thing to get wrong, and would drift. +""" + +from collections.abc import Iterator +from contextlib import contextmanager + +from dagster import ConfigurableResource + + +class OcotilloDatabase(ConfigurableResource): + """A session against the Ocotillo database. + + Configured entirely through the environment that ``db/engine.py`` reads + (``DB_DRIVER``, ``CLOUD_SQL_*``), so the Dagster+ code location is + configured the same way the API is, with different credentials. + """ + + @contextmanager + def session(self) -> Iterator[object]: + """Yield a SQLAlchemy session, rolled back and closed on the way out.""" + # Imported lazily: importing db.engine builds an engine from the + # environment at import time, which should happen when a run asks for a + # session, not when Dagster loads the code location to list assets. + from db.engine import session_ctx + + with session_ctx() as session: + yield session + + +# ============= EOF ============================================= diff --git a/automated_ingestion/iac/.gitignore b/automated_ingestion/iac/.gitignore new file mode 100644 index 000000000..72869f3b0 --- /dev/null +++ b/automated_ingestion/iac/.gitignore @@ -0,0 +1,6 @@ +.terraform/ +.terraform.lock.hcl +terraform.tfstate +terraform.tfstate.* +terraform.tfvars +*.tfplan diff --git a/automated_ingestion/iac/main.tf b/automated_ingestion/iac/main.tf new file mode 100644 index 000000000..9df05f48a --- /dev/null +++ b/automated_ingestion/iac/main.tf @@ -0,0 +1,103 @@ +# Raw-zone storage for the automated ingestion pipeline. +# +# Two buckets, one per environment, plus the service account the Dagster+ code +# location uses to write to them. Deliberately narrow: this configuration owns +# ingestion storage and nothing else, so a mistake here cannot affect the API's +# uploads bucket or any other project resource. +# +# Not applied by CI. Run it by hand, review the plan, and record the applied +# state -- see README.md. + +terraform { + required_version = ">= 1.5" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 6.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} + +locals { + environments = toset(["production", "staging"]) +} + +resource "google_storage_bucket" "ingestion_raw" { + for_each = local.environments + + name = "ocotillo-ingestion-${each.key}" + project = var.project_id + location = var.bucket_location + + # The raw zone is the replay source for Mode B backfill: reprocessing a + # mapping bug must not depend on the vendor still serving that window. + # Deleting an object here is therefore a data-loss event, not a cleanup. + force_destroy = false + uniform_bucket_level_access = true + public_access_prevention = "enforced" + + versioning { + enabled = true + } + + # Raw payloads are read constantly for the first month (recent-window + # replays), then almost never. Age-out to colder classes rather than + # deleting: an old window is exactly what a historical replay needs. + lifecycle_rule { + condition { + age = 30 + } + action { + type = "SetStorageClass" + storage_class = "NEARLINE" + } + } + + lifecycle_rule { + condition { + age = 365 + } + action { + type = "SetStorageClass" + storage_class = "COLDLINE" + } + } + + # Bucket versioning would otherwise retain every superseded object forever. + lifecycle_rule { + condition { + num_newer_versions = 3 + with_state = "ARCHIVED" + } + action { + type = "Delete" + } + } + + labels = { + component = "automated-ingestion" + env = each.key + } +} + +resource "google_service_account" "ingestion" { + account_id = "ocotillo-ingestion" + display_name = "Ocotillo automated ingestion" + description = "Writes raw vendor payloads to the ingestion buckets from the Dagster+ code location." + project = var.project_id +} + +# Scoped to the two buckets, not granted at project level. objectAdmin rather +# than objectCreator because a replay overwrite rewrites an existing object. +resource "google_storage_bucket_iam_member" "ingestion_object_admin" { + for_each = google_storage_bucket.ingestion_raw + + bucket = each.value.name + role = "roles/storage.objectAdmin" + member = "serviceAccount:${google_service_account.ingestion.email}" +} diff --git a/automated_ingestion/iac/outputs.tf b/automated_ingestion/iac/outputs.tf new file mode 100644 index 000000000..77b57e6fc --- /dev/null +++ b/automated_ingestion/iac/outputs.tf @@ -0,0 +1,9 @@ +output "bucket_names" { + description = "Raw-zone bucket per environment. The matching value goes into INGESTION_GCS_BUCKET on the Dagster+ code location." + value = { for k, b in google_storage_bucket.ingestion_raw : k => b.name } +} + +output "service_account_email" { + description = "Ingestion service account. Grant nothing else to it without revisiting the least-privilege rationale in README.md." + value = google_service_account.ingestion.email +} diff --git a/automated_ingestion/iac/terraform.tfvars.example b/automated_ingestion/iac/terraform.tfvars.example new file mode 100644 index 000000000..6e1830113 --- /dev/null +++ b/automated_ingestion/iac/terraform.tfvars.example @@ -0,0 +1 @@ +project_id = "waterdatainitiative-271000" diff --git a/automated_ingestion/iac/variables.tf b/automated_ingestion/iac/variables.tf new file mode 100644 index 000000000..c06187f4e --- /dev/null +++ b/automated_ingestion/iac/variables.tf @@ -0,0 +1,16 @@ +variable "project_id" { + type = string + description = "GCP project that owns the ingestion buckets and service account." +} + +variable "region" { + type = string + description = "Default provider region." + default = "us-central1" +} + +variable "bucket_location" { + type = string + description = "Bucket location. US-CENTRAL1 keeps the raw zone in the same region as Cloud SQL, so replay reads do not cross regions." + default = "US-CENTRAL1" +} diff --git a/automated_ingestion/ocotillo/__init__.py b/automated_ingestion/ocotillo/__init__.py new file mode 100644 index 000000000..1b346fc5d --- /dev/null +++ b/automated_ingestion/ocotillo/__init__.py @@ -0,0 +1,24 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +The Ocotillo-facing half of ingestion: adapters and the structures they emit. + +Source packages know their vendor's payload shape; this package knows +Ocotillo's. An adapter is the seam between them, so adding a source means +writing an adapter rather than touching the loader. +""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/ocotillo/adapter.py b/automated_ingestion/ocotillo/adapter.py new file mode 100644 index 000000000..7ba38f2f6 --- /dev/null +++ b/automated_ingestion/ocotillo/adapter.py @@ -0,0 +1,46 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Base class for source adapters. + +An adapter converts one source's raw records into Ocotillo structures. It is +the only place a vendor's vocabulary appears alongside Ocotillo's, which keeps +vendor quirks out of ``domain/`` and out of the loader. +""" + +from abc import ABC, abstractmethod +from collections.abc import Iterable, Iterator +from typing import Any + +from automated_ingestion.ocotillo.structs import ObservationRecord + + +class SourceAdapter(ABC): + """Maps one source's raw records onto Ocotillo structures.""" + + @property + @abstractmethod + def source_key(self) -> str: + """Registry key of the source this adapter serves.""" + + @abstractmethod + def to_observations( + self, records: Iterable[dict[str, Any]] + ) -> Iterator[ObservationRecord]: + """Convert raw vendor records into observation records.""" + + +# ============= EOF ============================================= diff --git a/automated_ingestion/ocotillo/structs.py b/automated_ingestion/ocotillo/structs.py new file mode 100644 index 000000000..90cc29868 --- /dev/null +++ b/automated_ingestion/ocotillo/structs.py @@ -0,0 +1,44 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Plain structures passed from an adapter to the loader. + +These are deliberately not SQLAlchemy models. An adapter is pure and testable +without a database session; turning these into rows is the loader's job. +""" + +from dataclasses import dataclass +from datetime import datetime + + +@dataclass(frozen=True) +class ObservationRecord: + """One timestamped reading, already in Ocotillo's units and datum.""" + + external_point_id: str + """The vendor's identifier for the monitoring point.""" + + observation_datetime: datetime + """Timezone-aware instant of the reading.""" + + value: float + """Measurement in ``units``, on the datum the source's mapping fixes.""" + + units: str + """Unit symbol as it appears in the Ocotillo lexicon.""" + + +# ============= EOF ============================================= diff --git a/automated_ingestion/scripts/__init__.py b/automated_ingestion/scripts/__init__.py new file mode 100644 index 000000000..94919dc13 --- /dev/null +++ b/automated_ingestion/scripts/__init__.py @@ -0,0 +1,18 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""One-off instruments. Nothing here is imported by the pipeline.""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/scripts/probe_diverhub.py b/automated_ingestion/scripts/probe_diverhub.py new file mode 100644 index 000000000..b71cca638 --- /dev/null +++ b/automated_ingestion/scripts/probe_diverhub.py @@ -0,0 +1,302 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Answer the open questions in BDMS task 2.1 against the live Diver-HUB API. + +Run once, with credentials, and fold the output into +``docs/sources/san_acacia.md``. It reads and never writes. + + export DIVERHUB_USERNAME=... DIVERHUB_PASSWORD=... + uv run --group ingestion python -m automated_ingestion.scripts.probe_diverhub + +What it settles: + +* Which project holds San Acacia Reach, and whether it really has 33 points. +* **Which ``reference`` value is ground surface.** The swagger declares the enum + as ``[0, 1, 2, 3]`` and says nothing else, so this prints a sample from each + side by side. The ground-surface series is recognisable by magnitude and sign + against a well whose depth to water is roughly known -- a judgement a person + has to make, which is why this script prints rather than decides. +* The window ceiling. Three months is known good; this widens until the API + answers 500, so the production span is measured rather than guessed. + +Nothing here is imported by the pipeline. It is a one-off instrument. +""" + +import sys +from datetime import datetime, timezone + +from automated_ingestion.shared.windows import DAY +from automated_ingestion.sources.san_acacia.client import ( + DiverHubClient, + DiverHubError, +) + +REFERENCE_VALUES = (0, 1, 2, 3) + + +def _session(): + import requests + + return requests.Session() + + +def _iso(unix: int) -> str: + return datetime.fromtimestamp(unix, tz=timezone.utc).isoformat() + + +def probe_projects(client: DiverHubClient) -> list[dict]: + print("== Projects visible to these credentials ==") + projects = client.projects() + for project in projects: + print(f" {project['id']:>6} {project['name']}") + return projects + + +def probe_points(client: DiverHubClient, project_id: int) -> list[dict]: + print(f"\n== Monitoring points in project {project_id} ==") + points = client.monitoring_points(project_id) + print(f" {len(points)} points (the plan expects 33)") + if len(points) != 33: + print( + " ^ count differs from the plan; listing all so the extras\n can be identified before anything is reconciled." + ) + for point in points: + print(f" {point['id']:>6} {point['name']}") + return points + for point in points[:5]: + print(f" {point['id']:>6} {point['name']}") + if len(points) > 5: + print(f" ... and {len(points) - 5} more") + return points + + +def probe_reference_values( + client: DiverHubClient, points: list[dict], end: int +) -> None: + """Sample each reference value so a human can tell which is ground surface. + + Searches over a year, and moves on to another point if the first has gone + quiet -- a diver that stopped reporting months ago tells us nothing about + what the enum means. + """ + print("\n== WaterLevelReference values ==") + print(" Ground surface reads as depth below ground: positive, and") + print(" plausible as feet below surface. An elevation is a much larger") + print(" number. A vrd/TOC series looks like ground surface but is offset") + print(" by the stickup, so compare against a well you know.\n") + + start = end - 365 * DAY + for point in points[:6]: + point_id, name = point["id"], point["name"] + found = False + for reference in REFERENCE_VALUES: + try: + rows = list( + client.water_levels(point_id, start, end, reference=reference) + ) + except DiverHubError as exc: + print(f" {name} reference={reference}: error -- {exc}") + continue + if not rows: + print(f" {name} reference={reference}: no rows in 365d") + continue + found = True + levels = [r["level"] for r in rows if r.get("level") is not None] + print( + f" {name} reference={reference}: {len(rows):>5} rows, " + f"min={min(levels):>10.3f} max={max(levels):>10.3f} " + f"first={rows[0].get('dateAndTime')} last={rows[-1].get('dateAndTime')}" + ) + if found: + print(f"\n ^ compare these four for {name} and pick the datum.") + return + print(" No point returned water levels in the last year.") + + +def probe_window_ceiling(client: DiverHubClient, point_id: int, end: int) -> None: + """Find what actually triggers a 500. + + Widening from the present tests span. Sliding a fixed narrow window back + through time tests whether the failure is instead about *when* -- a range + that predates the point's data. The two look identical from the status + code, so both are worth separating here. + """ + print(f"\n== Window behaviour for point {point_id} ==") + print(" Widening back from now (tests span):") + for days in (90, 180, 365, 545, 730): + start = end - days * DAY + try: + rows = list( + client.water_levels( + point_id, + start, + end, + reference=REFERENCE_VALUES[0], + span=days * DAY, + ) + ) + print(f" {days:>5}d: ok, {len(rows)} rows") + except DiverHubError: + print(f" {days:>5}d: 500 even at the one-day floor") + + print(" Fixed 30-day window slid backwards (tests age, not span):") + for years_back in (0, 1, 2, 3): + window_end = end - years_back * 365 * DAY + window_start = window_end - 30 * DAY + label = f"{years_back}y ago" + try: + rows = list( + client.water_levels( + point_id, + window_start, + window_end, + reference=REFERENCE_VALUES[0], + span=30 * DAY, + ) + ) + print(f" {label:>8}: ok, {len(rows)} rows") + except DiverHubError: + print(f" {label:>8}: 500 at the floor") + + +def probe_datum_relationships( + client: DiverHubClient, point_id: int, name: str, start: int, end: int +) -> None: + """Settle what the four reference values mean, using the API against itself. + + Two questions the min/max summary cannot answer: + + 1. **Is any of them an elevation rather than a depth?** An elevation moves + opposite to a depth, so ``elevation + depth`` is constant while + ``depth - depth`` is constant. Comparing aligned rows distinguishes them; + comparing ranges does not, because both look like the same spread. + 2. **Which is ground surface?** ``ManualMeasurements`` reports + ``waterLevelToc`` -- explicitly top of casing. Whichever reference tracks + it *is* the TOC series, and ground surface is then the one shallower than + it by the casing stickup. + """ + print(f"\n== Datum relationships for {name} ==") + series: dict[int, dict[str, float]] = {} + for reference in REFERENCE_VALUES: + rows = list(client.water_levels(point_id, start, end, reference=reference)) + series[reference] = { + r["dateAndTime"]: r["level"] for r in rows if r.get("level") is not None + } + + shared = set.intersection(*(set(v) for v in series.values())) if series else set() + stamps = sorted(shared)[:3] + if not stamps: + print(" No overlapping timestamps across references.") + return + + print(" Aligned samples:") + print(f" {'timestamp':<22}" + "".join(f"ref{r:<14}" for r in REFERENCE_VALUES)) + for stamp in stamps: + cells = "".join(f"{series[r][stamp]:<17.3f}" for r in REFERENCE_VALUES) + print(f" {stamp:<22}{cells}") + + base = REFERENCE_VALUES[0] + print(f"\n Relationship to ref={base} across those samples:") + for reference in REFERENCE_VALUES[1:]: + diffs = {round(series[reference][t] - series[base][t], 3) for t in stamps} + sums = {round(series[reference][t] + series[base][t], 3) for t in stamps} + if len(diffs) == 1: + print( + f" ref={reference}: constant OFFSET {diffs.pop():+.3f} " + "-- same direction, so also a depth" + ) + elif len(sums) == 1: + print( + f" ref={reference}: constant SUM {sums.pop():.3f} " + "-- INVERTED, so this one is an elevation" + ) + else: + print(f" ref={reference}: neither constant; not a simple datum shift") + + print("\n Manual measurements (waterLevelToc = top of casing):") + try: + # Sparse by nature -- a few per year at best -- so search the whole + # record rather than the window used for the logged series. + manual = client.manual_measurements(point_id, end - 3650 * DAY, end) + except DiverHubError as exc: + print(f" unavailable -- {exc}") + return + if not manual: + print(" none in this window; try a wider one.") + return + for record in manual[:3]: + stamp = record.get("dateAndTime") + toc = record.get("waterLevelToc") + print(f" {stamp} toc={toc}") + nearest = min(stamps, key=lambda t: abs(_epoch(t) - _epoch(stamp))) + print(f" nearest logged sample {nearest}:") + for reference in REFERENCE_VALUES: + delta = series[reference][nearest] - toc if toc is not None else None + if delta is not None: + print( + f" ref={reference}: {series[reference][nearest]:.3f} " + f"(toc{delta:+.3f})" + ) + print("\n The reference nearest zero against toc IS the TOC series.") + print(" Ground surface is shallower than TOC by the casing stickup.") + + +def _epoch(stamp: str) -> float: + from datetime import datetime, timezone + + parsed = datetime.fromisoformat(stamp.replace("Z", "+00:00")) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.timestamp() + + +def main() -> int: + try: + client = DiverHubClient(_session()) + except DiverHubError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + + end = int(datetime.now(tz=timezone.utc).timestamp()) + + projects = probe_projects(client) + if not projects: + print("No projects visible; nothing further to probe.", file=sys.stderr) + return 1 + + project_id = projects[0]["id"] + if len(projects) > 1: + print(f"\n(using project {project_id}; pass another by editing this script)") + + points = probe_points(client, project_id) + if not points: + return 1 + + point_id = points[0]["id"] + probe_reference_values(client, points, end) + probe_window_ceiling(client, point_id, end) + probe_datum_relationships(client, point_id, points[0]["name"], end - 730 * DAY, end) + + print("\nRecord the findings in docs/sources/san_acacia.md and set") + print("GROUND_SURFACE_REFERENCE in sources/san_acacia/client.py.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) + +# ============= EOF ============================================= diff --git a/automated_ingestion/shared/__init__.py b/automated_ingestion/shared/__init__.py new file mode 100644 index 000000000..809f48e4e --- /dev/null +++ b/automated_ingestion/shared/__init__.py @@ -0,0 +1,18 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Source-agnostic machinery reused by every ingestion source.""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/shared/backfill.py b/automated_ingestion/shared/backfill.py new file mode 100644 index 000000000..e884f36f2 --- /dev/null +++ b/automated_ingestion/shared/backfill.py @@ -0,0 +1,24 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Backfill primitives shared by every source. + +Ported from Aqueduct under BDMS task 4.1 -- ``month_chunks``, ``ChunkResult``, +and ``BackfillCheckpointStore``. Ported rather than imported: the two +repositories deploy separately and are allowed to diverge. +""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/shared/gcs.py b/automated_ingestion/shared/gcs.py new file mode 100644 index 000000000..b98b8836f --- /dev/null +++ b/automated_ingestion/shared/gcs.py @@ -0,0 +1,60 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +GCS raw-zone conventions. + +Every source writes date-partitioned parquet under one bucket per environment, +so a replay backfill can select an exact window by prefix without reading the +files. + +``services/gcs_helper.py`` serves user uploads from ``GCS_BUCKET_NAME``. +Ingestion deliberately reads a different variable: sharing it would let a +misconfigured deployment write raw vendor payloads into the uploads bucket. +""" + +BUCKET_ENV_VAR = "INGESTION_GCS_BUCKET" +"""Environment variable naming the raw-zone bucket. Never ``GCS_BUCKET_NAME``.""" + +RAW_LAYOUT = "{table_name}/year={YYYY}/month={MM}/day={DD}/{load_id}.{file_id}.{ext}" +"""dlt filesystem layout for the raw zone.""" + + +def raw_zone_bucket() -> str: + """Name of the raw-zone bucket for this environment. + + Raises rather than defaulting. A wrong bucket name is not a condition worth + guessing through: the failure would be a run that reports success while + writing nowhere useful, or worse, into a bucket that belongs to something + else. + """ + import os + + bucket = os.environ.get(BUCKET_ENV_VAR, "").strip() + if not bucket: + raise RuntimeError( + f"{BUCKET_ENV_VAR} is not set. The ingestion raw zone has no default; " + "set it on the Dagster+ code location to the bucket Terraform " + "created (see automated_ingestion/iac)." + ) + if bucket == os.environ.get("GCS_BUCKET_NAME", "").strip(): + raise RuntimeError( + f"{BUCKET_ENV_VAR} points at GCS_BUCKET_NAME, the API's user-upload " + "bucket. Raw vendor payloads must not be written there." + ) + return bucket + + +# ============= EOF ============================================= diff --git a/automated_ingestion/shared/http.py b/automated_ingestion/shared/http.py new file mode 100644 index 000000000..32adeb87d --- /dev/null +++ b/automated_ingestion/shared/http.py @@ -0,0 +1,24 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +HTTP session construction for vendor APIs. + +Centralized so every source inherits the same timeout, retry, and backoff +posture, and so one source's flaky endpoint cannot hang a run indefinitely. +Filled in alongside the first live extraction under BDMS task 2. +""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/shared/source_registry.py b/automated_ingestion/shared/source_registry.py new file mode 100644 index 000000000..a5ef76ab6 --- /dev/null +++ b/automated_ingestion/shared/source_registry.py @@ -0,0 +1,64 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Registry of ingestion sources. + +Each source declares itself once here so jobs, schedules, and the backfill +factory can enumerate sources without importing each one by name. +""" + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SourceDefinition: + """Static description of one ingestion source.""" + + key: str + """Stable identifier, used in asset keys and GCS prefixes.""" + + display_name: str + """Human-readable name for logs and the Dagster UI.""" + + dataset_name: str + """dlt dataset name; becomes the top-level GCS prefix.""" + + +_SOURCES: dict[str, SourceDefinition] = {} + + +def register(source: SourceDefinition) -> SourceDefinition: + """Add a source to the registry, rejecting duplicate keys.""" + if source.key in _SOURCES: + raise ValueError(f"Source {source.key!r} is already registered.") + _SOURCES[source.key] = source + return source + + +def get_source(key: str) -> SourceDefinition: + """Look up a registered source by key.""" + try: + return _SOURCES[key] + except KeyError: + raise KeyError(f"No ingestion source registered under {key!r}.") from None + + +def all_sources() -> tuple[SourceDefinition, ...]: + """Every registered source, ordered by key.""" + return tuple(_SOURCES[k] for k in sorted(_SOURCES)) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/shared/windows.py b/automated_ingestion/shared/windows.py new file mode 100644 index 000000000..9fc8765d6 --- /dev/null +++ b/automated_ingestion/shared/windows.py @@ -0,0 +1,83 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Time-window arithmetic for sources that cannot be asked for an open range. + +Diver-HUB answers `DiverData` and `WaterLevels` for an explicit +``startTime``/``endTime`` in Unix seconds, and returns HTTP 500 -- not a +pagination cursor, not a 413 -- when the span is too wide. So a "fetch this +series" operation is always a sequence of bounded windows, and the useful +response to a 500 is to ask for less rather than to give up. + +Pure arithmetic, no HTTP: the retry policy that uses it is in the client, and +the point of separating them is that the tricky part is testable without a +network. +""" + +from collections.abc import Iterator +from dataclasses import dataclass + +DAY = 86_400 + +DEFAULT_SPAN = 90 * DAY +"""Starting window width. Three months is confirmed to work; the ceiling is +not yet measured, so this is the largest span known to be safe rather than the +largest span that is.""" + +MINIMUM_SPAN = DAY +"""Floor for bisection. A 500 on a single day is a real failure -- something +other than volume -- and must surface rather than shrink forever.""" + + +@dataclass(frozen=True) +class Window: + """A half-open interval in Unix seconds, ``start`` inclusive.""" + + start: int + end: int + + def __post_init__(self) -> None: + if self.end < self.start: + raise ValueError(f"Window end {self.end} precedes start {self.start}.") + + @property + def span(self) -> int: + return self.end - self.start + + def bisect(self) -> tuple["Window", "Window"]: + """Split in two. Raises at the floor rather than shrinking forever.""" + if self.span <= MINIMUM_SPAN: + raise ValueError( + f"Refusing to split a {self.span}s window below the {MINIMUM_SPAN}s " + "floor. A failure this narrow is not a volume problem." + ) + midpoint = self.start + self.span // 2 + return Window(self.start, midpoint), Window(midpoint, self.end) + + +def iter_windows(start: int, end: int, span: int = DEFAULT_SPAN) -> Iterator[Window]: + """Walk ``[start, end]`` in windows of at most ``span`` seconds.""" + if span <= 0: + raise ValueError(f"Window span must be positive, got {span}.") + if end < start: + raise ValueError(f"End {end} precedes start {start}.") + cursor = start + while cursor < end: + yield Window(cursor, min(cursor + span, end)) + cursor += span + + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/__init__.py b/automated_ingestion/sources/__init__.py new file mode 100644 index 000000000..4fbe9cf77 --- /dev/null +++ b/automated_ingestion/sources/__init__.py @@ -0,0 +1,18 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""One subpackage per ingestion source.""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/__init__.py b/automated_ingestion/sources/san_acacia/__init__.py new file mode 100644 index 000000000..28d879b9d --- /dev/null +++ b/automated_ingestion/sources/san_acacia/__init__.py @@ -0,0 +1,35 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +San Acacia Reach -- 33 Van Essen divers, one depth-to-groundwater series each. + +The pilot source: small and already mapped, so it exercises the whole path end +to end without a large or unfamiliar dataset complicating the first build. + +Readings come from the private Diver-HUB API, which shapes the extraction in +two ways. Requests carry a JWT good for one hour, so anything long-running +refreshes mid-run rather than authenticating once at the start. And +``DiverData/ByMonitoringPoint/{id}`` returns HTTP 500 when asked for too wide a +span instead of paginating, so reads are always bounded windows in Unix +seconds -- roughly three months is known to work. + +Readings land on the **ground-surface** datum (Van Essen's ``gs`` +arrays, never ``vrd``), public but provisional, and always ``not reviewed`` -- +the vendor's own approval flag records what the vendor approved, not a Bureau +review. +""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/adapter.py b/automated_ingestion/sources/san_acacia/adapter.py new file mode 100644 index 000000000..0d5d89731 --- /dev/null +++ b/automated_ingestion/sources/san_acacia/adapter.py @@ -0,0 +1,18 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Van Essen records to Ocotillo structures. Implemented under BDMS task 3.1.""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/backfill.py b/automated_ingestion/sources/san_acacia/backfill.py new file mode 100644 index 000000000..eb3cd127e --- /dev/null +++ b/automated_ingestion/sources/san_acacia/backfill.py @@ -0,0 +1,18 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""San Acacia backfill wiring. Built under BDMS task 4.""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/client.py b/automated_ingestion/sources/san_acacia/client.py new file mode 100644 index 000000000..8e3b269e5 --- /dev/null +++ b/automated_ingestion/sources/san_acacia/client.py @@ -0,0 +1,291 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Client for the private Diver-HUB API. + +Three things about this API shape the code, and all three differ from what the +retired FROST pipeline suggested: + +1. **Bearer JWT with a real expiry.** ``POST /Accounts/Login`` returns a token + and a ``validTo`` timestamp. The token is refreshed against that timestamp + rather than against an assumed lifetime, and once more on a 401 -- a clock + difference between us and the server should not end a backfill. +2. **Bounded windows.** Readings endpoints take ``startTime``/``endTime`` in + Unix seconds and answer HTTP 500 when the span is too wide, so a fetch walks + windows and narrows on failure. +3. **Datum is a request parameter, not a response field.** ``WaterLevels`` + returns ``{dateAndTime, level}``; which datum that level is on depends on the + ``reference`` value sent. See ``GROUND_SURFACE_REFERENCE``. + +Credentials come from the environment and are never logged. The token is not +logged either: it is a bearer credential for the whole account. +""" + +import os +import time +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Any, Protocol + +from automated_ingestion.shared.windows import DEFAULT_SPAN, Window, iter_windows + +BASE_URL = "https://diver-hub.com/private/api/v1" + +USERNAME_ENV_VAR = "DIVERHUB_USERNAME" +PASSWORD_ENV_VAR = "DIVERHUB_PASSWORD" + +EXPIRY_SKEW_SECONDS = 60 +"""Refresh this long before ``validTo``, so a request in flight at the boundary +does not arrive expired.""" + +GROUND_SURFACE_REFERENCE = 3 +"""Which ``WaterLevelReference`` value means depth below ground surface. + +The swagger declares the enum as ``[0, 1, 2, 3]`` with no names, so this was +determined by measurement rather than read off the specification. Probing +SO-0125 showed all four values return the same rows at the same timestamps, +related by constants that held identically across two windows eighteen months +apart: + + ref1 + ref0 = 518.160 ref3 + ref0 = 472.704 ref2 - ref0 = 139001.296 + +``ref0`` and ``ref2`` rise with the water; ``ref1`` and ``ref3`` fall, so the +latter pair are depths. ``ref1`` is deeper than ``ref3`` by a fixed 45.456 cm +(1.49 ft) -- a casing stickup -- which makes ``ref1`` top of casing and ``ref3`` +ground surface. The reading checks out physically: ground surface lands at +1394.74 m (4576 ft), right for San Acacia, and depth to water runs 2.2-4.7 m, +right for a riparian piezometer. + +See ``docs/sources/san_acacia.md``. Do not change this without re-running +``scripts/probe_diverhub.py``: the wrong value returns plausible numbers on the +wrong datum rather than an error. +""" + +TOP_OF_CASING_REFERENCE = 1 +"""Depth below top of casing. Not ingested -- recorded so the value is not +mistaken for ground surface, which it resembles to within a stickup.""" + +ELEVATION_REFERENCE = 2 +"""Water-surface elevation above sea level. Not ingested.""" + +SOURCE_UNIT = "cm" +"""Diver-HUB reports centimeters; Ocotillo stores feet. Convert with +``domain.units.convert_cm_to_ft`` -- never store a raw value.""" + + +class Response(Protocol): + """The subset of a `requests` response this module uses.""" + + status_code: int + + def json(self) -> Any: ... + + +class Transport(Protocol): + """The subset of a `requests` session this module uses.""" + + def post(self, url: str, **kwargs: Any) -> Response: ... + + def get(self, url: str, **kwargs: Any) -> Response: ... + + +class DiverHubError(RuntimeError): + """The API refused a request in a way retrying will not fix.""" + + +@dataclass +class _Token: + value: str + valid_to: float + + def expired(self, now: float) -> bool: + return now >= self.valid_to - EXPIRY_SKEW_SECONDS + + +class DiverHubClient: + """Authenticated, window-aware access to Diver-HUB.""" + + def __init__( + self, + transport: Transport, + username: str | None = None, + password: str | None = None, + base_url: str = BASE_URL, + timeout: int = 60, + ) -> None: + self._transport = transport + self._base_url = base_url.rstrip("/") + self._timeout = timeout + self._username = username or os.environ.get(USERNAME_ENV_VAR, "") + self._password = password or os.environ.get(PASSWORD_ENV_VAR, "") + self._token: _Token | None = None + if not self._username or not self._password: + raise DiverHubError( + f"Diver-HUB credentials are not set. Provide {USERNAME_ENV_VAR} and " + f"{PASSWORD_ENV_VAR} in the environment." + ) + + # -- authentication ---------------------------------------------------- + + def _login(self) -> _Token: + response = self._transport.post( + f"{self._base_url}/Accounts/Login", + json={"username": self._username, "password": self._password}, + timeout=self._timeout, + ) + if response.status_code == 401: + raise DiverHubError("Diver-HUB rejected the credentials.") + if response.status_code != 200: + raise DiverHubError(f"Login failed with HTTP {response.status_code}.") + payload = response.json() + return _Token( + value=payload["token"], + valid_to=_parse_timestamp(payload["validTo"]), + ) + + def _authorization(self) -> dict[str, str]: + if self._token is None or self._token.expired(time.time()): + self._token = self._login() + return {"Authorization": f"Bearer {self._token.value}"} + + def _get(self, path: str, params: dict[str, Any] | None = None) -> Response: + """GET with one forced re-login if the token is rejected.""" + response = self._transport.get( + f"{self._base_url}/{path.lstrip('/')}", + headers=self._authorization(), + params=params, + timeout=self._timeout, + ) + if response.status_code == 401: + self._token = None + response = self._transport.get( + f"{self._base_url}/{path.lstrip('/')}", + headers=self._authorization(), + params=params, + timeout=self._timeout, + ) + return response + + # -- reference data ---------------------------------------------------- + + def projects(self) -> list[dict[str, Any]]: + """Projects visible to these credentials.""" + return _expect_ok(self._get("Projects"), "Projects").json() + + def monitoring_points(self, project_id: int) -> list[dict[str, Any]]: + """Monitoring points in a project. Returns ``{id, name}`` only -- + no coordinates and no construction detail, so geometry and depth have + to be resolved from Ocotillo rather than from here.""" + path = f"MonitoringPoints/ByProject/{project_id}" + return _expect_ok(self._get(path), path).json() + + def manual_measurements( + self, monitoring_point_id: int, start: int, end: int + ) -> list[dict[str, Any]]: + """Manual readings, reported against top of casing. + + Not ingested -- Ocotillo's manual-measurement path owns these. Fetched + only to identify which ``reference`` value is the TOC series, since the + swagger names the enum members not at all. + """ + path = f"ManualMeasurements/ByMonitoringPoint/{monitoring_point_id}" + response = self._get(path, {"startTime": start, "endTime": end}) + return _expect_ok(response, path).json() + + # -- series ------------------------------------------------------------ + + def water_levels( + self, + monitoring_point_id: int, + start: int, + end: int, + reference: int, + approved: bool | None = None, + span: int = DEFAULT_SPAN, + ) -> Iterator[dict[str, Any]]: + """Yield ``{dateAndTime, level}`` records across bounded windows. + + ``reference`` selects the datum and is required: there is no safe + default, because the wrong value produces plausible numbers rather than + an error. + """ + params: dict[str, Any] = {"reference": reference} + if approved is not None: + params["approved"] = approved + path = f"WaterLevels/ByMonitoringPoint/{monitoring_point_id}" + for window in iter_windows(start, end, span): + yield from self._fetch_window(path, window, params) + + def diver_data( + self, + monitoring_point_id: int, + start: int, + end: int, + span: int = DEFAULT_SPAN, + ) -> Iterator[dict[str, Any]]: + """Yield raw ``DataPoint`` records -- pressure, temperature, and the + rest. Not water level; see ``water_levels`` for that.""" + path = f"DiverData/ByMonitoringPoint/{monitoring_point_id}" + for window in iter_windows(start, end, span): + yield from self._fetch_window(path, window, {}) + + def _fetch_window( + self, path: str, window: Window, params: dict[str, Any] + ) -> Iterator[dict[str, Any]]: + """Fetch one window, halving it on a 500 until it succeeds or hits the + floor. A 500 here means "too much data", which is the API's way of + asking to be given a narrower range.""" + response = self._get( + path, {**params, "startTime": window.start, "endTime": window.end} + ) + if response.status_code == 500: + try: + left, right = window.bisect() + except ValueError as exc: + raise DiverHubError( + f"{path} returned HTTP 500 for {window.span}s starting " + f"{window.start}, which is already at the minimum window. " + "This is not a volume problem." + ) from exc + yield from self._fetch_window(path, left, params) + yield from self._fetch_window(path, right, params) + return + yield from _expect_ok(response, path).json() + + +def _expect_ok(response: Response, what: str) -> Response: + if response.status_code != 200: + raise DiverHubError(f"{what} returned HTTP {response.status_code}.") + return response + + +def _parse_timestamp(value: str) -> float: + """Parse an ISO-8601 instant into a Unix timestamp. + + The API reports UTC but does not always mark it, so a naive value is read + as UTC rather than as local time -- reading it as local would shift token + expiry by the machine's offset and, worse, shift every reading. + """ + from datetime import datetime, timezone + + text = value.replace("Z", "+00:00") + parsed = datetime.fromisoformat(text) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.timestamp() + + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/dlt_pipeline.py b/automated_ingestion/sources/san_acacia/dlt_pipeline.py new file mode 100644 index 000000000..c7b9e7319 --- /dev/null +++ b/automated_ingestion/sources/san_acacia/dlt_pipeline.py @@ -0,0 +1,186 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +dlt resources landing San Acacia in the GCS raw zone. + +Two resources, with deliberately different dispositions: + +* ``vanessen_locations`` -- the monitoring point roster, ``replace``. It is a + snapshot of what the vendor currently lists, and a point disappearing is + information we want to see rather than accumulate. +* ``vanessen_readings`` -- the water level series, ``append``, incremental on + the reading timestamp. Appending is what makes Mode B replay possible: the + raw zone keeps what the vendor said at the time, not just what it says now. + +Nothing is transformed here. The raw zone stores the vendor's payload as it +arrived, in the vendor's units and on the vendor's datum, so a mapping bug is a +reprocess rather than a re-fetch. Conversion to Ocotillo's model happens in the +adapter, downstream. +""" + +from collections.abc import Iterator +from typing import Any + +import dlt + +from automated_ingestion.shared.gcs import RAW_LAYOUT, raw_zone_bucket +from automated_ingestion.shared.windows import DAY +from automated_ingestion.shared.source_registry import SourceDefinition, register +from automated_ingestion.sources.san_acacia.client import ( + GROUND_SURFACE_REFERENCE, + SOURCE_UNIT, + DiverHubClient, + DiverHubError, +) + +PROJECT_ID = 4317 +"""Diver-HUB project ``SanAcaciaReach``. Confirmed by probing, not assumed.""" + +READING_SPAN = 365 * DAY +"""Window width for this source, measured rather than assumed. + +``WaterLevels`` served 730 days and 18111 rows in a single request when probed, +so the generic 90-day default in ``shared/windows.py`` would quadruple the +request count for no benefit -- a first run for one point covers a decade. This +sits at half the largest span observed to work, leaving room for a denser point +than SO-0125. +""" + +INITIAL_START = "2015-01-01T00:00:00+00:00" +"""Floor for a point that has never been ingested. + +A floor, never a backfill lever: moving it forward does not delete anything +already landed, and moving it backward does not fetch history for a point whose +cursor has advanced past it. Use a backfill job for that +(``BACKFILL_STRATEGY.md`` section 2). +""" + +SOURCE = register( + SourceDefinition( + key="san_acacia", + display_name="San Acacia Reach", + dataset_name="raw_sanacaciareach", + ) +) + + +@dlt.resource(name="vanessen_locations", write_disposition="replace") +def vanessen_locations(client: DiverHubClient) -> Iterator[dict[str, Any]]: + """The monitoring point roster. + + One request, no pagination. The payload is ``{id, name}`` and nothing more + -- no coordinates, no construction detail -- so this cannot be the source + of a well's geometry. It exists to enumerate the points a reading fetch + walks, and to record what the vendor listed on a given day. + """ + for point in client.monitoring_points(PROJECT_ID): + yield { + "monitoring_point_id": point["id"], + "name": point["name"], + "project_id": PROJECT_ID, + } + + +@dlt.resource(name="vanessen_readings", write_disposition="append") +def vanessen_readings( + client: DiverHubClient, + monitoring_points: list[dict[str, Any]], + end: int, + failures: list[dict[str, Any]], + cursor: dlt.sources.incremental[str] = dlt.sources.incremental( + "dateAndTime", initial_value=INITIAL_START + ), +) -> Iterator[dict[str, Any]]: + """Water levels for every point, from each point's watermark to ``end``. + + Failure is isolated per point. One diver returning a 500 for its whole + history should cost that diver's data for this run, not the other + thirty-seven -- so exceptions are caught here and appended to ``failures`` + rather than raised. + + ``failures`` is supplied by the caller rather than stashed on the resource: + a dlt resource is a module-level object shared by every run, so recording + per-run state on it would have one run overwriting another's. + """ + from automated_ingestion.sources.san_acacia.client import _parse_timestamp + + start = int(_parse_timestamp(cursor.last_value)) + + for point in monitoring_points: + point_id = point["monitoring_point_id"] + try: + approved_at = _approved_timestamps(client, point_id, start, end) + for row in client.water_levels( + point_id, + start, + end, + reference=GROUND_SURFACE_REFERENCE, + span=READING_SPAN, + ): + yield { + "monitoring_point_id": point_id, + "name": point["name"], + "dateAndTime": row["dateAndTime"], + "level": row["level"], + "unit": SOURCE_UNIT, + "reference": GROUND_SURFACE_REFERENCE, + "vendor_approved": row["dateAndTime"] in approved_at, + } + except DiverHubError as exc: + failures.append({"monitoring_point_id": point_id, "error": str(exc)}) + + +def _approved_timestamps( + client: DiverHubClient, point_id: int, start: int, end: int +) -> set[str]: + """Timestamps the vendor has marked approved. + + ``approved`` is a request parameter rather than a response field, so the + flag has to be recovered by asking twice. We take the unfiltered series as + the authoritative row set and use this only to tag it -- fetching + ``approved=true`` and ``approved=false`` separately and concatenating would + duplicate every row if the two sets overlap, which is not yet known. + + A failure here is not fatal: an untagged reading is worth more than no + reading, and the vendor flag is not Ocotillo's review status anyway. + """ + try: + rows = client.water_levels( + point_id, + start, + end, + reference=GROUND_SURFACE_REFERENCE, + approved=True, + span=READING_SPAN, + ) + return {row["dateAndTime"] for row in rows} + except DiverHubError: + return set() + + +def build_pipeline(environment: str) -> Any: + """A dlt pipeline writing parquet to the raw zone for one environment.""" + return dlt.pipeline( + pipeline_name=f"san_acacia_{environment}", + destination=dlt.destinations.filesystem( + bucket_url=f"gs://{raw_zone_bucket()}", + layout=RAW_LAYOUT, + ), + dataset_name=SOURCE.dataset_name, + ) + + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/ingest.py b/automated_ingestion/sources/san_acacia/ingest.py new file mode 100644 index 000000000..6486f2a2d --- /dev/null +++ b/automated_ingestion/sources/san_acacia/ingest.py @@ -0,0 +1,123 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Dagster assets for San Acacia. + +Both assets land raw payloads in GCS and report what happened as metadata -- +row counts, and for readings the number of points that failed. A run that +silently ingests nothing looks identical to a run with nothing to ingest, and +the metadata is what separates them. +""" + +from datetime import datetime, timezone +from typing import Any + +from dagster import AssetExecutionContext, MetadataValue, Output, asset + +from automated_ingestion.sources.san_acacia.client import DiverHubClient + + +def _client() -> DiverHubClient: + import requests + + return DiverHubClient(requests.Session()) + + +@asset( + group_name="san_acacia", + description="Monitoring point roster for the San Acacia project, landed raw.", +) +def raw_san_acacia_locations(context: AssetExecutionContext) -> Output[int]: + """Land the point roster in the raw zone.""" + from automated_ingestion.sources.san_acacia.dlt_pipeline import ( + PROJECT_ID, + build_pipeline, + vanessen_locations, + ) + + client = _client() + points = list(client.monitoring_points(PROJECT_ID)) + pipeline = build_pipeline(context.run.tags.get("environment", "staging")) + pipeline.run(vanessen_locations(client)) + + context.log.info("landed %s monitoring points", len(points)) + return Output( + len(points), + metadata={ + "monitoring_points": MetadataValue.int(len(points)), + "project_id": MetadataValue.int(PROJECT_ID), + "names": MetadataValue.text(", ".join(p["name"] for p in points[:10])), + }, + ) + + +@asset( + group_name="san_acacia", + deps=[raw_san_acacia_locations], + description="Water level readings for every San Acacia point, landed raw.", +) +def raw_san_acacia_readings(context: AssetExecutionContext) -> Output[int]: + """Land water levels for every point, isolating per-point failure.""" + from automated_ingestion.sources.san_acacia.dlt_pipeline import ( + PROJECT_ID, + build_pipeline, + vanessen_readings, + ) + + client = _client() + points = [ + {"monitoring_point_id": p["id"], "name": p["name"]} + for p in client.monitoring_points(PROJECT_ID) + ] + end = int(datetime.now(tz=timezone.utc).timestamp()) + + pipeline = build_pipeline(context.run.tags.get("environment", "staging")) + failures: list[dict[str, Any]] = [] + info = pipeline.run(vanessen_readings(client, points, end, failures)) + rows = _row_count(info) + + if failures: + context.log.warning( + "%s of %s points failed: %s", + len(failures), + len(points), + ", ".join(str(f["monitoring_point_id"]) for f in failures), + ) + + return Output( + rows, + metadata={ + "rows_ingested": MetadataValue.int(rows), + "points_attempted": MetadataValue.int(len(points)), + "points_failed": MetadataValue.int(len(failures)), + "failures": MetadataValue.json(failures), + }, + ) + + +def _row_count(load_info: Any) -> int: + """Rows dlt reports as loaded, or 0 when it reports nothing.""" + try: + return sum( + metrics.get("rows_count", 0) + for job in load_info.load_packages + for metrics in getattr(job, "jobs", {}).values() + ) + except Exception: # noqa: BLE001 - metadata must never fail a good load + return 0 + + +# ============= EOF ============================================= diff --git a/automated_ingestion/sources/san_acacia/transform.py b/automated_ingestion/sources/san_acacia/transform.py new file mode 100644 index 000000000..4630ae8a2 --- /dev/null +++ b/automated_ingestion/sources/san_acacia/transform.py @@ -0,0 +1,30 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Reshaping that precedes adaptation. + +Less is needed here than the plan first assumed. The retired FROST pipeline +suggested Van Essen returned parallel arrays that had to be zipped into +records; the live API returns ``[{dateAndTime, level}]`` already, and selects +datum and approval through query parameters rather than through which array a +value came from. + +What remains for this module is timestamp normalisation and whatever +per-record tidying the live responses turn out to need. Filled in under BDMS +task 3.1, once the probe has run. +""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/sql/ingestion_role.sql b/automated_ingestion/sql/ingestion_role.sql new file mode 100644 index 000000000..65afb657b --- /dev/null +++ b/automated_ingestion/sql/ingestion_role.sql @@ -0,0 +1,68 @@ +-- Least-privilege Postgres role for the automated ingestion pipeline. +-- +-- Run by hand against each environment as a superuser. Not an Alembic +-- migration: roles and grants are per-environment infrastructure, not schema, +-- and migrations run under this database's application role rather than a +-- superuser. +-- +-- The point of the role is blast radius. The pipeline writes observations and +-- the reference rows they hang from, and reads everything it must resolve +-- against. It cannot touch chemistry, contacts, assets, or the legacy NMA_* +-- and NMW_* tables, so a bug in an adapter cannot corrupt data no ingestion +-- path should ever reach. + +-- Set the password out of band; do not commit it. It belongs in Secret +-- Manager alongside internal-ogc-api-keys. +-- CREATE ROLE ocotillo_ingestion LOGIN PASSWORD '...'; +-- +-- Or, preferred, use IAM database authentication and create the role for the +-- service account instead, so there is no password to rotate: +-- CREATE ROLE "ocotillo-ingestion@PROJECT.iam" WITH LOGIN; +-- GRANT cloudsqliamuser TO "ocotillo-ingestion@PROJECT.iam"; + +\set role_name ocotillo_ingestion + +GRANT CONNECT ON DATABASE :"db_name" TO :"role_name"; +GRANT USAGE ON SCHEMA public TO :"role_name"; + +-- Written: the observations themselves and the rows a new series needs. +GRANT SELECT, INSERT, UPDATE ON + transducer_observation, + transducer_observation_block, + deployment, + sensor, + parameter +TO :"role_name"; + +-- `parameter` is versioned by sqlalchemy-continuum, so an insert there also +-- writes a version row and a transaction row. Without these two grants the +-- write fails at runtime with a permission error on a table the code never +-- names directly -- an unpleasant thing to debug. +GRANT SELECT, INSERT ON parameter_version, transaction TO :"role_name"; + +-- Read-only: resolved against, never written. `thing` and `location` are +-- deliberately not writable. Reconciling the 33 San Acacia wells means +-- matching them to rows that already exist; if reconciliation finds a well +-- missing, that is a decision for a human, not a row the pipeline invents. +GRANT SELECT ON + thing, + thing_id_link, + location, + lexicon_term, + lexicon_category, + lexicon_term_category_association +TO :"role_name"; + +-- Inserts need the sequences behind the autoincrement primary keys. +GRANT USAGE, SELECT ON SEQUENCE + transducer_observation_id_seq, + transducer_observation_block_id_seq, + deployment_id_seq, + sensor_id_seq, + parameter_id_seq, + transaction_id_seq +TO :"role_name"; + +-- No default privileges are granted. A table added later is invisible to this +-- role until someone grants it deliberately, which is the intended failure +-- mode: a new table reaching the pipeline should be a decision. diff --git a/automated_ingestion/tests/__init__.py b/automated_ingestion/tests/__init__.py new file mode 100644 index 000000000..6df237838 --- /dev/null +++ b/automated_ingestion/tests/__init__.py @@ -0,0 +1,18 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Tests for the automated ingestion code location.""" + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/conftest.py b/automated_ingestion/tests/conftest.py new file mode 100644 index 000000000..801f5d827 --- /dev/null +++ b/automated_ingestion/tests/conftest.py @@ -0,0 +1,29 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Skip this directory when the ingestion dependency group is not installed. + +dagster lives in the optional ``ingestion`` group, so a developer who ran a +plain ``uv sync`` has no dagster in the environment. Without this guard, +collecting these modules raises ``ImportError`` and takes the whole suite down +with it -- the API tests would fail for a package the API never imports. +""" + +from importlib.util import find_spec + +collect_ignore_glob = [] if find_spec("dagster") else ["*.py"] + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_connectivity.py b/automated_ingestion/tests/test_connectivity.py new file mode 100644 index 000000000..da5453139 --- /dev/null +++ b/automated_ingestion/tests/test_connectivity.py @@ -0,0 +1,60 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +The connectivity asset is wired up, and does not reach the database until run. + +Loading the code location must not open a connection: Dagster lists assets far +more often than it runs them, and a code location that needs a database to load +is a code location that breaks whenever the database is briefly unreachable. +""" + +from dagster import AssetKey + +from automated_ingestion.defs.definitions import defs + + +def test_connectivity_asset_is_registered(): + assert AssetKey(["database_connectivity"]) in defs.resolve_all_asset_keys() + + +def test_database_resource_is_provided(): + assert "database" in defs.resources + + +def test_loading_definitions_does_not_import_db_engine(): + # db.engine builds its engine at import time, so listing assets must not + # reach it. Checking sys.modules in-process would only observe whichever + # test imported it first, so ask a clean interpreter instead. + import subprocess + import sys + + result = subprocess.run( + [ + sys.executable, + "-c", + "import automated_ingestion.defs.definitions as d; " + "import sys; " + "assert d.defs is not None; " + "print('db.engine' in sys.modules)", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "False", result.stdout + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_definitions.py b/automated_ingestion/tests/test_definitions.py new file mode 100644 index 000000000..b0745dcb0 --- /dev/null +++ b/automated_ingestion/tests/test_definitions.py @@ -0,0 +1,45 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +The code location loads. + +Cheap, but it is the check that catches the failure this package is most prone +to: a Dagster+ deploy that builds fine and then cannot import. +""" + +from dagster import AssetKey, Definitions + +from automated_ingestion.defs.definitions import defs + + +def test_definitions_object_is_loadable(): + assert isinstance(defs, Definitions) + + +def test_heartbeat_asset_is_registered(): + assert AssetKey(["ingestion_heartbeat"]) in defs.resolve_all_asset_keys() + + +def test_heartbeat_materializes_without_external_dependencies(): + from dagster import materialize + + from automated_ingestion.defs.assets.heartbeat import ingestion_heartbeat + + result = materialize([ingestion_heartbeat]) + assert result.success + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_diverhub_client.py b/automated_ingestion/tests/test_diverhub_client.py new file mode 100644 index 000000000..78ccd37b8 --- /dev/null +++ b/automated_ingestion/tests/test_diverhub_client.py @@ -0,0 +1,200 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Client behaviour that is easy to get wrong and expensive to get wrong: +token refresh, the 401 retry, and narrowing on a 500. + +No network. The transport is a stub that records what it was asked for. +""" + +import pytest + +from automated_ingestion.shared.windows import DAY +from automated_ingestion.sources.san_acacia.client import ( + DiverHubClient, + DiverHubError, +) + + +class FakeResponse: + def __init__(self, status_code=200, payload=None): + self.status_code = status_code + self._payload = payload if payload is not None else [] + + def json(self): + return self._payload + + +class FakeTransport: + """Records calls and replays queued responses.""" + + def __init__(self, get_responses=None, token_valid_for=3600): + self.posts = [] + self.gets = [] + self._get_responses = list(get_responses or []) + self._token_valid_for = token_valid_for + self.login_count = 0 + + def post(self, url, **kwargs): + self.posts.append((url, kwargs)) + self.login_count += 1 + from datetime import datetime, timedelta, timezone + + valid_to = datetime.now(tz=timezone.utc) + timedelta( + seconds=self._token_valid_for + ) + return FakeResponse( + 200, + {"token": f"token-{self.login_count}", "validTo": valid_to.isoformat()}, + ) + + def get(self, url, **kwargs): + self.gets.append((url, kwargs)) + if self._get_responses: + return self._get_responses.pop(0) + return FakeResponse(200, []) + + +def _client(transport): + return DiverHubClient(transport, username="u", password="p") + + +def test_missing_credentials_fail_fast(monkeypatch): + monkeypatch.delenv("DIVERHUB_USERNAME", raising=False) + monkeypatch.delenv("DIVERHUB_PASSWORD", raising=False) + with pytest.raises(DiverHubError, match="credentials"): + DiverHubClient(FakeTransport()) + + +def test_token_is_reused_across_calls(): + transport = FakeTransport() + client = _client(transport) + client.projects() + client.projects() + assert transport.login_count == 1 + + +def test_token_is_refreshed_once_expired(): + # validTo in the past means every call re-authenticates. + transport = FakeTransport(token_valid_for=-10) + client = _client(transport) + client.projects() + client.projects() + assert transport.login_count == 2 + + +def test_expiry_skew_refreshes_before_the_deadline(): + # A token valid for 30s is already inside the skew window, so it must not + # be used: a request in flight at the boundary would arrive expired. + transport = FakeTransport(token_valid_for=30) + client = _client(transport) + client.projects() + client.projects() + assert transport.login_count == 2 + + +def test_401_forces_one_reauthentication_and_retry(): + transport = FakeTransport( + get_responses=[FakeResponse(401), FakeResponse(200, [{"id": 1}])] + ) + client = _client(transport) + assert client.projects() == [{"id": 1}] + assert transport.login_count == 2 + assert len(transport.gets) == 2 + + +def test_500_narrows_the_window_and_stitches_the_halves(): + # First window 500s; each half then succeeds and both are returned. + transport = FakeTransport( + get_responses=[ + FakeResponse(500), + FakeResponse(200, [{"level": 1.0}]), + FakeResponse(200, [{"level": 2.0}]), + ] + ) + client = _client(transport) + rows = list(client.water_levels(40, 0, 100 * DAY, reference=0, span=100 * DAY)) + assert [r["level"] for r in rows] == [1.0, 2.0] + + +def test_persistent_500_at_the_floor_is_an_error_not_a_loop(): + transport = FakeTransport(get_responses=[FakeResponse(500)] * 50) + client = _client(transport) + with pytest.raises(DiverHubError, match="not a volume problem"): + list(client.water_levels(40, 0, DAY, reference=0, span=DAY)) + + +def test_water_levels_sends_reference_and_unix_seconds(): + transport = FakeTransport() + client = _client(transport) + list(client.water_levels(40, 0, DAY, reference=2, span=DAY)) + _, kwargs = transport.gets[0] + params = kwargs["params"] + assert params["reference"] == 2 + assert params["startTime"] == 0 + assert params["endTime"] == DAY + assert isinstance(params["startTime"], int) + + +def test_approved_is_omitted_unless_asked_for(): + transport = FakeTransport() + client = _client(transport) + list(client.water_levels(40, 0, DAY, reference=0, span=DAY)) + assert "approved" not in transport.gets[0][1]["params"] + + +def test_naive_valid_to_is_read_as_utc(): + # The API documents UTC but does not always mark it. Reading a naive + # timestamp as local time would shift expiry by the machine's offset. + from automated_ingestion.sources.san_acacia.client import _parse_timestamp + + naive = _parse_timestamp("2026-08-18T20:00:00") + aware = _parse_timestamp("2026-08-18T20:00:00Z") + assert naive == aware + + +def test_datum_constants_match_the_measured_relationships(): + # Determined by probing, not read from the spec: ref0/ref2 rise with the + # water and ref1/ref3 fall, so the depths are 1 and 3, and ref1 is deeper + # than ref3 by a fixed casing stickup. Getting this wrong does not raise -- + # it silently records every reading on the wrong datum. + from automated_ingestion.sources.san_acacia import client as module + + assert module.GROUND_SURFACE_REFERENCE == 3 + assert module.TOP_OF_CASING_REFERENCE == 1 + assert module.ELEVATION_REFERENCE == 2 + assert module.GROUND_SURFACE_REFERENCE != module.TOP_OF_CASING_REFERENCE + + +def test_source_unit_is_centimeters_not_feet(): + # The vendor reports cm and Ocotillo stores ft. A value passed through + # unconverted is wrong by a factor of 30.48 and still looks like a plausible + # depth, which is exactly the kind of error that survives review. + from domain.units import convert_cm_to_ft + from automated_ingestion.sources.san_acacia import client as module + + assert module.SOURCE_UNIT == "cm" + # SO-0125 on 2024-10-30: 471.518 cm below ground surface. + assert convert_cm_to_ft(471.518) == 15.469751 + + +def test_cm_conversion_passes_none_through(): + from domain.units import convert_cm_to_ft + + assert convert_cm_to_ft(None) is None + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_gcs.py b/automated_ingestion/tests/test_gcs.py new file mode 100644 index 000000000..b9875a1d5 --- /dev/null +++ b/automated_ingestion/tests/test_gcs.py @@ -0,0 +1,62 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Bucket resolution refuses to guess. + +The uploads-bucket check is the one worth testing: `services/gcs_helper.py` +already uses GCS_BUCKET_NAME, and the two variables being confused is a +configuration mistake that would otherwise succeed quietly. +""" + +import pytest + +from automated_ingestion.shared.gcs import BUCKET_ENV_VAR, RAW_LAYOUT, raw_zone_bucket + + +def test_returns_the_configured_bucket(monkeypatch): + monkeypatch.setenv(BUCKET_ENV_VAR, "ocotillo-ingestion-staging") + monkeypatch.delenv("GCS_BUCKET_NAME", raising=False) + assert raw_zone_bucket() == "ocotillo-ingestion-staging" + + +def test_unset_bucket_raises(monkeypatch): + monkeypatch.delenv(BUCKET_ENV_VAR, raising=False) + with pytest.raises(RuntimeError, match=BUCKET_ENV_VAR): + raw_zone_bucket() + + +def test_blank_bucket_raises(monkeypatch): + monkeypatch.setenv(BUCKET_ENV_VAR, " ") + with pytest.raises(RuntimeError, match=BUCKET_ENV_VAR): + raw_zone_bucket() + + +def test_uploads_bucket_is_rejected(monkeypatch): + monkeypatch.setenv(BUCKET_ENV_VAR, "ocotillo-uploads") + monkeypatch.setenv("GCS_BUCKET_NAME", "ocotillo-uploads") + with pytest.raises(RuntimeError, match="user-upload"): + raw_zone_bucket() + + +def test_layout_partitions_by_date(): + # Mode B replay selects a window by prefix, which only works if the date + # is in the path rather than inside the file. + assert "year={YYYY}" in RAW_LAYOUT + assert "month={MM}" in RAW_LAYOUT + assert "day={DD}" in RAW_LAYOUT + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_san_acacia_resources.py b/automated_ingestion/tests/test_san_acacia_resources.py new file mode 100644 index 000000000..64fe21fbb --- /dev/null +++ b/automated_ingestion/tests/test_san_acacia_resources.py @@ -0,0 +1,164 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Resource behaviour: failure isolation, approval tagging, and the raw-zone +contract that nothing is converted on the way in. +""" + +from automated_ingestion.sources.san_acacia.client import ( + GROUND_SURFACE_REFERENCE, + DiverHubClient, +) +from automated_ingestion.sources.san_acacia.dlt_pipeline import ( + PROJECT_ID, + vanessen_locations, + vanessen_readings, +) +from automated_ingestion.tests.test_diverhub_client import FakeResponse, FakeTransport + + +class ScriptedTransport(FakeTransport): + """Answers per-path so a single point can be made to fail.""" + + def __init__(self, handler): + super().__init__() + self._handler = handler + + def get(self, url, **kwargs): + self.gets.append((url, kwargs)) + return self._handler(url, kwargs) + + +def _points_payload(): + return [{"id": 39, "name": "SO-0125"}, {"id": 40, "name": "SO-0131"}] + + +def test_locations_flatten_to_the_raw_shape(): + transport = ScriptedTransport(lambda url, kw: FakeResponse(200, _points_payload())) + client = DiverHubClient(transport, username="u", password="p") + rows = list(vanessen_locations(client)) + assert rows == [ + {"monitoring_point_id": 39, "name": "SO-0125", "project_id": PROJECT_ID}, + {"monitoring_point_id": 40, "name": "SO-0131", "project_id": PROJECT_ID}, + ] + + +READINGS = [ + {"dateAndTime": "2026-04-15T22:45:00", "level": 199.356}, + {"dateAndTime": "2026-04-15T23:00:00", "level": 200.0}, +] + + +def _within(rows, params): + """Return only rows inside the requested window, as the API does. + + A stub that ignores startTime/endTime returns its whole payload for every + window, which turns a decade-long fetch into fifty copies of the same rows + and hides whether the caller is windowing correctly at all. + """ + from automated_ingestion.sources.san_acacia.client import _parse_timestamp + + start, end = params["startTime"], params["endTime"] + return [r for r in rows if start <= _parse_timestamp(r["dateAndTime"]) <= end] + + +def _reading_handler(failing_point=None, approved_stamps=()): + def handler(url, kwargs): + if "WaterLevels" in url: + point_id = int(url.rstrip("/").split("/")[-1]) + if point_id == failing_point: + return FakeResponse(500) + params = kwargs.get("params", {}) + if params.get("approved"): + approved = [{"dateAndTime": s, "level": 1.0} for s in approved_stamps] + return FakeResponse(200, _within(approved, params)) + return FakeResponse(200, _within(READINGS, params)) + return FakeResponse(200, []) + + return handler + + +def _run_readings(handler, points=None, failures=None): + transport = ScriptedTransport(handler) + client = DiverHubClient(transport, username="u", password="p") + points = ( + points + if points is not None + else [ + {"monitoring_point_id": 39, "name": "SO-0125"}, + {"monitoring_point_id": 40, "name": "SO-0131"}, + ] + ) + collected = failures if failures is not None else [] + resource = vanessen_readings(client, points, 1_800_000_000, collected) + return list(resource), collected + + +def test_readings_carry_unit_and_reference_untransformed(): + # The raw zone stores what the vendor said, on the vendor's datum in the + # vendor's units. Converting here would make a mapping bug a re-fetch + # instead of a reprocess. + rows, _ = _run_readings(_reading_handler()) + assert rows[0]["level"] == 199.356 + assert rows[0]["unit"] == "cm" + assert rows[0]["reference"] == GROUND_SURFACE_REFERENCE + + +def test_one_failing_point_does_not_lose_the_others(): + rows, failures = _run_readings(_reading_handler(failing_point=39)) + assert [r["monitoring_point_id"] for r in rows] == [40, 40] + assert len(failures) == 1 + assert failures[0]["monitoring_point_id"] == 39 + + +def test_failures_are_recorded_for_the_caller_not_the_resource(): + # Per-run state on a module-level resource would have concurrent runs + # overwriting one another. + own = [] + _run_readings(_reading_handler(failing_point=39), failures=own) + assert len(own) == 1 + assert not hasattr(vanessen_readings, "failures") + + +def test_vendor_approval_tags_rows_without_duplicating_them(): + rows, _ = _run_readings( + _reading_handler(approved_stamps=["2026-04-15T22:45:00"]), + points=[{"monitoring_point_id": 39, "name": "SO-0125"}], + ) + # Two readings in, two readings out -- the approved fetch tags, never adds. + assert len(rows) == 2 + assert rows[0]["vendor_approved"] is True + assert rows[1]["vendor_approved"] is False + + +def test_unavailable_approval_flag_does_not_lose_readings(): + def handler(url, kwargs): + if "WaterLevels" in url: + params = kwargs.get("params", {}) + if params.get("approved"): + return FakeResponse(500) + return FakeResponse(200, _within(READINGS[:1], params)) + return FakeResponse(200, []) + + rows, failures = _run_readings( + handler, points=[{"monitoring_point_id": 39, "name": "SO-0125"}] + ) + assert len(rows) == 1 + assert rows[0]["vendor_approved"] is False + assert failures == [] + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_source_registry.py b/automated_ingestion/tests/test_source_registry.py new file mode 100644 index 000000000..18b4e6efd --- /dev/null +++ b/automated_ingestion/tests/test_source_registry.py @@ -0,0 +1,59 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Registry behavior: duplicate keys are a bug, not a silent overwrite.""" + +import pytest + +from automated_ingestion.shared import source_registry +from automated_ingestion.shared.source_registry import SourceDefinition + + +@pytest.fixture(autouse=True) +def _isolated_registry(monkeypatch): + monkeypatch.setattr(source_registry, "_SOURCES", {}) + + +def _definition(key="san_acacia"): + return SourceDefinition( + key=key, + display_name="San Acacia Reach", + dataset_name="raw_sanacaciareach", + ) + + +def test_registered_source_is_retrievable(): + source_registry.register(_definition()) + assert source_registry.get_source("san_acacia").display_name == "San Acacia Reach" + + +def test_duplicate_key_is_rejected(): + source_registry.register(_definition()) + with pytest.raises(ValueError, match="already registered"): + source_registry.register(_definition()) + + +def test_unknown_key_raises(): + with pytest.raises(KeyError, match="san_acacia"): + source_registry.get_source("san_acacia") + + +def test_all_sources_is_sorted_by_key(): + source_registry.register(_definition("van_essen")) + source_registry.register(_definition("bernco")) + assert [s.key for s in source_registry.all_sources()] == ["bernco", "van_essen"] + + +# ============= EOF ============================================= diff --git a/automated_ingestion/tests/test_windows.py b/automated_ingestion/tests/test_windows.py new file mode 100644 index 000000000..34d68602d --- /dev/null +++ b/automated_ingestion/tests/test_windows.py @@ -0,0 +1,71 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Window arithmetic, including the refusal to shrink past the floor.""" + +import pytest + +from automated_ingestion.shared.windows import ( + DAY, + MINIMUM_SPAN, + Window, + iter_windows, +) + + +def test_windows_cover_the_range_without_gaps_or_overlap(): + windows = list(iter_windows(0, 10 * DAY, span=3 * DAY)) + assert windows[0].start == 0 + assert windows[-1].end == 10 * DAY + for earlier, later in zip(windows, windows[1:]): + assert earlier.end == later.start + + +def test_final_window_is_truncated_not_overshot(): + # Overshooting would ask the API for a future range, which is at best waste + # and at worst a 400. + windows = list(iter_windows(0, 10 * DAY, span=3 * DAY)) + assert windows[-1].end == 10 * DAY + assert windows[-1].span == DAY + + +def test_range_shorter_than_span_is_a_single_window(): + assert list(iter_windows(0, DAY, span=90 * DAY)) == [Window(0, DAY)] + + +def test_empty_range_yields_nothing(): + assert list(iter_windows(500, 500)) == [] + + +def test_reversed_range_is_rejected(): + with pytest.raises(ValueError, match="precedes"): + list(iter_windows(10, 5)) + + +def test_bisect_splits_in_half(): + left, right = Window(0, 100 * DAY).bisect() + assert left.start == 0 + assert left.end == right.start + assert right.end == 100 * DAY + + +def test_bisect_refuses_below_the_floor(): + # A 500 on one day is not a volume problem, and silently halving forever + # would turn one real failure into an unbounded pile of requests. + with pytest.raises(ValueError, match="floor"): + Window(0, MINIMUM_SPAN).bisect() + + +# ============= EOF ============================================= diff --git a/dagster_cloud.yaml b/dagster_cloud.yaml new file mode 100644 index 000000000..342a9a4d3 --- /dev/null +++ b/dagster_cloud.yaml @@ -0,0 +1,16 @@ +# Dagster+ code locations for this repository. +# +# The API and the ingestion pipeline share a repo but not a runtime: this file +# describes only what Dagster+ builds and runs. `module_name` mirrors +# `[tool.dagster]` in pyproject.toml, so `dagster dev` locally and the Dagster+ +# agent load the same entry point. +# +# `directory` is the repository root rather than `automated_ingestion/` because +# the loader imports `db/` models and `domain/` rules -- the package is not +# self-contained by design (see docs/automated-ingestion-pipeline-plan.md). +locations: + - location_name: ocotillo-automated-ingestion + code_source: + module_name: automated_ingestion.defs.definitions + build: + directory: ./ diff --git a/docs/automated-ingestion-pipeline-plan.md b/docs/automated-ingestion-pipeline-plan.md new file mode 100644 index 000000000..7a3773081 --- /dev/null +++ b/docs/automated-ingestion-pipeline-plan.md @@ -0,0 +1,357 @@ +# Draft: Automated Ingestion Pipeline Epic (BDMS) + +1 new Epic → 4 Tasks → 17 Sub-tasks. **Nothing written to Jira yet.** + +## TL;DR + +Build the Bureau's first automated data ingestion pipeline, in the OcotilloAPI repo, so continuous depth-to-groundwater readings reach Ocotillo on a schedule instead of by hand. San Acacia Reach (33 Van Essen divers) is the pilot source; the structure it establishes is what every later source inherits. + +Stack: **Dagster+** code location → **dlt** extraction → **GCS** raw parquet → **`domain/`** mapping → direct **Postgres** load. Watermark and backfill mechanics are ported from Aqueduct, with two deliberate improvements a relational destination allows: the watermark is read from Postgres rather than a GCS sidecar, and an upsert replaces Aqueduct's delete-then-repost (removing its known window where data goes temporarily missing). + +**Decided** — four calls already made, so reviewers don't reopen them: + +- **Owned by OcotilloAPI, not Aqueduct.** The loader writes over a direct database connection, which wants the `db/` SQLAlchemy models and `domain/` rules in the same process. Running it as a third Aqueduct source would mean maintaining a copy of Ocotillo's schema in another repo. Aqueduct stays the FROST/SensorThings pipeline; shared code is **ported, not imported**, so the two can diverge without breaking each other. +- **Ground-surface datum.** `TransducerObservation.value` stores depth to water below ground surface, in feet. That picks Van Essen's `gs` arrays and drops `vrd` entirely. No measuring-point correction on ingest — `domain/water_levels.py`'s MP reconciliation belongs to the manual-measurement path, where a field crew measured the height on the day. Datum shifts are the Hydrograph Corrector's job, downstream. +- **Public + provisional.** Visible from the first run, and marked provisional so no consumer mistakes an uncorrected diver series for a reviewed one. This matches what the retired FROST pipeline asserted for this source (`is_provisional: true`) — adopted deliberately here rather than inherited silently, which was the open question left in Aqueduct's mapping doc. It needs a schema change: `release_status` is one column, and its lexicon lists `public` and `provisional` as siblings, so visibility and maturity — two orthogonal axes — currently collide. +- **Vendor approval flag ≠ Ocotillo review status.** Van Essen's `approvedWaterLevels*` records what *the vendor* approved. Ocotillo's `review_status` is `approved` / `not reviewed`, and `TransducerObservationBlock.reviewer_id` FKs a Bureau `Contact` — so `approved` asserts a Bureau human reviewed it. Mapping one onto the other would manufacture provenance that doesn't exist. All San Acacia blocks land `not reviewed`; the vendor flag is preserved as a separate per-row attribute. + +**Watch:** two schema changes — a unique constraint on `transducer_observation`, and a new field because `release_status` cannot hold "public" and "provisional" at once. The vendor blocker cleared on 2026-08-18: the readings endpoint works, but only through the private Diver-HUB API, only with a 1-hour JWT, and only in bounded time windows. + +**Sequencing:** Task 1 gates everything. Tasks 2 and 3 run largely in parallel after it. Nothing is vendor-blocked any more. + +## All tasks + +| # | Item | In one line | Blocked by | +|---|---|---|---| +| **T1** | **Foundations** | Package, Dagster+ code location, GCS, DB connectivity | — | +| 1.1 | Scaffold package + Dagster skeleton | `automated_ingestion/` layout, deps, loads in `dagster dev` | — | +| 1.2 | Register Dagster+ code location | `dagster_cloud.yaml` + prod/branch deploy workflows | 1.1 | +| 1.3 | GCS buckets + service account | `ocotillo-ingestion-{production,staging}`, date-partitioned layout | — | +| 1.4 | DB connectivity + least-privilege role | Cloud SQL connector from serverless; scoped Postgres role | 1.2 | +| **T2** | **Source extraction** | Van Essen API → GCS raw zone | T1 | +| 2.1 | Confirm endpoint + finalize mapping | **Unblocked.** Diver-HUB swagger, JWT login, measure the window ceiling | — | +| 2.2 | dlt resource: locations | 33 wells, `replace`, one call, no pagination | 1.3 | +| 2.3 | dlt resource: readings, incremental | Windowed per-point fetch, dlt cursor, `append`, token refresh, failure isolation | 2.1 | +| **T3** | **Domain mapping + load** | Van Essen records → Ocotillo Postgres | T1 | +| 3.1 | Domain layer | Pure functions: units, datum, timestamps, geometry, external keys | — | +| 3.2 | Bootstrap reference data | Reconcile 33 wells; seed parameter, sensor, deployments | 3.1 | +| 3.3 | Represent "public but provisional" | **Schema change.** `release_status` can't hold both axes | — | +| 3.4 | Unique constraint + upsert loader | **Schema change.** `ON CONFLICT DO UPDATE`; makes backfill idempotent | 3.2, 3.3 | +| 3.5 | Watermark from Postgres | `MAX(observation_datetime)` per series; no GCS sidecar | 3.4 | +| **T4** | **Backfill + operations** | Recover from gaps, bugs, and vendor corrections | T3 | +| 4.1 | Port shared backfill primitives | `month_chunks`, `BackfillCheckpointStore`, `ChunkResult` from Aqueduct | — | +| 4.2 | Mode A — refetch | Re-fetch from API for a window; `dry_run: true` default; chunked, resumable | 4.1, 2.3 | +| 4.3 | Mode B — replay | Reprocess GCS parquet through the current adapter; no API calls | 4.1, 3.4 | +| 4.4 | Schedule, observability, alerting | Daily schedule, log bridge, failure notification, run metadata | 4.2 | +| 4.5 | Documentation | Source mapping, storage conventions, backfill runbook, new-source checklist | 4.3 | + +--- + +# EPIC — Automated Ingestion Pipeline + +**Goal:** continuous depth-to-groundwater data lands in Ocotillo automatically, on a schedule, with no one hand-carrying files — starting with San Acacia Reach. + +The Hydrograph Corrector UI exists and works (BDMS-1137 done), but has no automatic supply of raw data. San Acacia Reach's 33 Van Essen divers historically flowed through the retired FROST/`st2` stack and now flow nowhere. This epic builds the supply. Correction, review, and publication workflows are **out of scope** and belong to their own epic. + +New top-level `automated_ingestion/` package in OcotilloAPI, deployed as its own Dagster+ code location in the existing `nmbgmr-data-services` org. dlt extracts the Van Essen API to a GCS raw zone; a `domain/` layer maps to the Ocotillo model; a loader writes to Ocotillo Postgres over a direct DB connection. Watermark and backfill mechanics come from Aqueduct. + +San Acacia first: 33 wells, one DTW series each, and already mapped in `Aqueduct/docs/sources/san_acacia.md`. It authenticates with a short-lived JWT and must be read in bounded time windows — both cheap enough here to establish the pattern before a harder source needs it. What it establishes — source registry, per-source dlt pipeline, adapter, backfill job factory — every later source inherits. + +**Ownership: OcotilloAPI.** Not a third Aqueduct source writing into Ocotillo. The loader writes over a direct database connection, which wants the `db/` SQLAlchemy models and `domain/` rules in-process rather than a duplicated schema in another repo. Aqueduct stays the FROST/SensorThings pipeline; this is Ocotillo's own. The two share code by porting (see below), not by importing. + +### Adopted from Aqueduct + +| Artifact | Adoption | +|---|---| +| `docs/BACKFILL_STRATEGY.md` | Wholesale: Mode A refetch / Mode B replay, per-source generated jobs, calendar-month chunking, `dry_run: true` default, `initial_start_date` as a floor only | +| `shared/backfill.py`, `shared/gcs.py` | Port near-verbatim — already destination-agnostic | +| `shared/source_registry.py` | Port the pattern; registry drives job + schedule generation | +| `canonical/base_adapter.py` | Adapt: same `extract`/`to_*`/`run` shape and per-record failure isolation, emitting Ocotillo structs | +| `loader/watermark_store.py` | **Adapt, not port** — see deviation 1 | +| `docs/STORAGE_CONVENTIONS.md` | Adopt, renamed for `ocotillo-ingestion-` | + +### Deviations from Aqueduct + +1. **Watermark in Postgres, not a GCS sidecar.** Aqueduct needs `_frost_watermarks.json` because FROST has no transactional read. Ocotillo's destination does: `MAX(observation_datetime)` per `(thing_id, parameter_id)`, read in the write transaction. No sidecar drift, no recovery path. +2. **Upsert replaces delete-then-repost.** `BACKFILL_STRATEGY.md` §4.4 accepts a temporary hole in FROST because observations have no dedup key there. Postgres does — unique constraint plus `ON CONFLICT DO UPDATE` makes load and backfill idempotent with no destructive delete. Resolves that doc's §6 open question. +3. **Target is `Thing → Deployment → TransducerObservation`**, not `FieldEvent → … → Observation`. 5-minute diver series are continuous, not field visits. + +### Data classification — decided + +- **Datum: ground surface.** `TransducerObservation.value` = depth to water below ground surface, feet. Ingest Van Essen's `gs` arrays, not `vrd`. No measuring-point correction on ingest — `domain/water_levels.py`'s MP reconciliation is the manual-measurement path. Datum shifts are the corrector's business. +- **Visibility public, maturity provisional.** Public from the first run, marked provisional so nobody mistakes an uncorrected diver series for a reviewed one. Matches what the old FROST pipeline asserted (`is_provisional: true`) — adopted deliberately, not inherited silently. +- **Schema cannot express this today.** `release_status` is one scalar column (`ReleaseMixin` → `lexicon_term.term`), and its lexicon category holds `public` *and* `provisional` as siblings. Visibility and maturity are orthogonal; the lexicon conflates them. Sub-task 3.3 resolves it. +- **Vendor approval ≠ Ocotillo review status.** `approvedWaterLevels*` records what the *vendor* approved. Ocotillo `review_status` is `approved` / `not reviewed`, and `TransducerObservationBlock.reviewer_id` FKs a Bureau `Contact` — `approved` means a Bureau human reviewed it. All San Acacia blocks land `not reviewed`; the vendor flag is kept as a separate per-row attribute. + +### Epic acceptance criteria + +- `automated_ingestion/` deploys as a Dagster+ code location on merge; jobs visible in the Dagster UI. +- Scheduled job runs end to end: Van Essen API → GCS parquet → domain mapping → Ocotillo Postgres. +- Re-running over an already-loaded window: zero duplicates, zero errors. +- Both backfill jobs exist, default `dry_run: true`, chunk by month, resume from last completed chunk. +- 33 wells resolve to `Thing` records — matched or created, no duplicates. +- Readings are public, marked provisional, stored as DTW below ground surface in feet. +- Series render in the Hydrograph Corrector. +- Domain mapping unit-tested with no database, per `ADR4.md`. + +### Blocker — resolved 2026-08-18 + +The 500s were never a vendor outage. Two things were wrong on our side, both reported by Chase Martin: + +1. **Wrong API.** Readings come from the private Diver-HUB API — `GET https://diver-hub.com/private/api/v1/DiverData/ByMonitoringPoint/{id}` — not the doubled-segment `/api/api/monitoringPoint/{project}/{id}` path the earlier draft assumed. Swagger: `https://diver-hub.com/private/swagger/index.html`, which is now the authority over anything inferred from retired FROST data. +2. **Window too large.** The endpoint 500s rather than paginating or erroring cleanly when asked for too much. A confirmed-good request is a ~3-month window in **Unix seconds**: + + ``` + https://diver-hub.com/private/api/v1/DiverData/ByMonitoringPoint/40?startTime=1767225600&endTime=1775001600 + ``` + +**Auth:** POST to the login endpoint with the credentials Ethan circulated; it returns a **JWT valid for one hour**. This overturns the "unauthenticated" assumption in the earlier draft and has two consequences: the token is a secret needing the same handling as the DB credentials, and any run outliving an hour — every backfill — must refresh mid-run rather than acquire once at start. + +**Still open:** the actual window ceiling. Three months works; the limit is unmeasured. Until it is, chunk conservatively and treat a 500 as "too much data" rather than a hard failure. + +### Related + +BDMS-1137 (corrector zoom/selection, Done — the consumer of this data, not part of this epic) · BDMS-1090 (Wellpy Revival Discovery) · BDMS-362 (WellPy Ocotillo) · `DataIntegrationGroup/Aqueduct` · OcotilloAPI `ADR4.md`, `db/transducer.py`, `db/engine.py` + +--- + +# TASK 1 — Foundations: code location, GCS, DB connectivity + +Nothing in this repo runs on a schedule today. This task creates the package, gets it deploying to Dagster+, provisions GCS, and proves the Dagster runtime can reach Ocotillo Postgres. Carries the workstream's two infrastructure risks: build size and serverless→Cloud SQL connectivity. + +**Done when:** package loads in `dagster dev`; merge deploys to prod and PRs produce branch deployments; buckets exist with a least-privilege SA; a trivial asset reads Ocotillo Postgres from both deployments; pytest/ruff/mypy pass. + +### 1.1 — Scaffold `automated_ingestion/` and the Dagster skeleton + +``` +automated_ingestion/ +├── defs/ definitions.py (entry point), assets/, jobs/backfill.py +├── shared/ source_registry.py, backfill.py, gcs.py, http.py +├── ocotillo/ adapter base + Ocotillo structs +├── sources/san_acacia/ ingest / dlt_pipeline / adapter / transform / backfill +└── tests/ +``` + +- Layout above created; `automated_ingestion` added to `[tool.setuptools] packages` (same fix as `f33cd063` for `domain`). +- Deps added: dagster, dagster-cloud, `dlt[filesystem,gs]`, gcsfs, pyarrow. `[tool.dagster] module_name = "automated_ingestion.defs.definitions"`. +- `dagster dev` loads the location with no import errors; ruff/mypy cover it; pytest still green. + +Lives in this repo so the loader can import `db/` models and `domain/` rules rather than duplicate the schema. If the Dagster+ build proves too large, fall back to a `[project.optional-dependencies]` split. + +### 1.2 — Register as a Dagster+ code location with CI deploy + +Files written; nothing deployed yet — the secrets do not exist, so neither workflow has run. + +- ✅ `dagster_cloud.yaml` declaring `ocotillo-automated-ingestion` → `automated_ingestion.defs.definitions`, build directory `./`. The build directory is the repository root, not `automated_ingestion/`, because the loader imports `db/` and `domain/`. +- ✅ `CD_dagster_prod.yml` and `CD_dagster_branch.yml`, both on `dagster-io/dagster-cloud-action@v1.13.18` — pinned to the same version as the installed dagster. +- ✅ Path-filtered to `automated_ingestion/**`, `dagster_cloud.yaml`, `pyproject.toml`, and `uv.lock`. The last two matter: the location's dependency set is exported from them, so a lockfile bump changes the built image even when no ingestion file moves. +- ⬜ `DAGSTER_CLOUD_API_TOKEN` as a repository **secret**, `DAGSTER_CLOUD_ORGANIZATION_ID` as a repository **variable**. The token is a CI credential the action uses to reach Dagster+, so it belongs with `CLOUD_DEPLOY_SERVICE_ACCOUNT_KEY` rather than in Secret Manager -- reading it from Secret Manager would still require a GitHub secret to authenticate to GCP first, adding a hop without removing a trust root. The organization ID is not sensitive; it appears in the Dagster+ console URL. +- ⬜ Runtime secrets are a different question and are **not** GitHub's. The Diver-HUB login (2.1), the ingestion service account (1.3), and the Postgres role (1.4) are read by the pipeline while it runs, not by the deploy, so they belong in Secret Manager on the `internal-ogc-api-keys` precedent, reached from Dagster+ at runtime. +- ⬜ Test PR yields a working branch deployment; merge to `production` yields a working prod location. + +**Prod deploys from `production`, not `main`.** `main` was abandoned in July 2025 — it is 3,839 commits behind and is not part of the release flow (`docs/release-flow.md`). The `main` reference in the original draft was inherited from Aqueduct's layout without checking this repository's. + +**PEX vs Docker — answered: Docker.** `serverless_prod_deploy` and `serverless_branch_deploy` build with `docker/build-push-action` and a copied Dockerfile template; there is no PEX fast-deploy path in these actions. So build time is a full image build, and the dependency set matters: the image installs all 197 exported packages (the 135 runtime ones plus dagster, dlt, gcsfs, pyarrow). `pymssql` and `psycopg2-binary` are in that set and compile from source on some base images — the first real build is where that surfaces. + +**Ordering constraint, easy to break.** `utils/parse_workspace` runs its own `actions/checkout`, which cleans the working tree. It must run *before* `requirements.txt` is generated; putting the generation first silently deletes it, and the deploy fails on a missing file rather than on the real cause. + +Both workflows generate `requirements.txt` with `uv export --group ingestion`, since Dagster+ builds from a requirements file and the repository does not keep one under version control. + +### 1.3 — Provision GCS buckets and ingestion service account + +Terraform written in `automated_ingestion/iac/`; **not applied**. `terraform validate` and `fmt` pass, but no GCP credentials were available, so no resource exists yet. + +- ✅ `ocotillo-ingestion-production` and `-staging`, uniform bucket-level access, public access prevention enforced, `force_destroy = false`. +- ✅ Service account `ocotillo-ingestion` with `roles/storage.objectAdmin` bound **on the two buckets**, not at project level. `objectAdmin` rather than `objectCreator` because a Mode B replay overwrites an existing object. +- ✅ Lifecycle: NEARLINE at 30 days, COLDLINE at 365, and archived-version pruning past 3. Aged out rather than deleted — an old window is exactly what a historical replay reads. +- ✅ `INGESTION_GCS_BUCKET` resolved by `shared/gcs.raw_zone_bucket()`, which raises rather than defaulting and explicitly rejects a value equal to `GCS_BUCKET_NAME`. `services/gcs_helper.py` uses that variable for user uploads; the two being confused would write raw vendor payloads into the uploads bucket, and would otherwise do so silently. +- ⬜ `terraform apply`, then set `INGESTION_GCS_BUCKET` on the Dagster+ code location. + +The dlt layout `{table_name}/year={YYYY}/month={MM}/day={DD}/{load_id}.{file_id}.{ext}` is asserted by a test, because Mode B replay selects a window by prefix — the date has to be in the path, not inside the file. + +### 1.4 — DB connectivity from Dagster+ with a least-privilege role + +Dagster+ Serverless is outside the VPC, so Cloud SQL's private IP is unreachable from it. Code written; **nothing run against a database**. + +- ✅ `OcotilloDatabase` resource delegating to `db/engine.py`'s `DB_DRIVER=cloudsql` path rather than building a second engine. The import is lazy: `db.engine` builds its engine at import time, and a code location that needs a reachable database merely to *list* its assets breaks every time the database blips. A test asserts loading the definitions leaves `db.engine` unimported. +- ✅ `database_connectivity` asset, read-only. Connectivity and grants are separable problems, and a write here would leave test rows in a real table. +- ✅ Role DDL in `automated_ingestion/sql/ingestion_role.sql`, kept out of Alembic: roles and grants are per-environment infrastructure, not schema, and migrations do not run as a superuser. +- ⬜ Run the DDL per environment; set `DB_DRIVER`, `CLOUD_SQL_*` on the code location; materialize the asset from both a branch and prod deployment. + +**The grant list is narrower than the draft assumed, and one part of it is non-obvious.** Writable: `transducer_observation`, `transducer_observation_block`, `deployment`, `sensor`, `parameter`. Read-only: `thing`, `thing_id_link`, `location`, and the three `lexicon_*` tables — `thing` and `location` deliberately *not* writable, because reconciling the 33 wells means matching rows that already exist. A well found missing is a decision for a human, not a row the pipeline invents. + +`parameter` is versioned by sqlalchemy-continuum, so inserting one also writes to `parameter_version` and `transaction`. Without those two grants the write fails on a table the code never names — the kind of error that costs an afternoon. (`transducer_observation` itself is not versioned; only `aquifer_system`, `geologic_formation`, `location`, `observation`, `parameter`, `regulatory_limit`, and `thing` are.) Sequence `USAGE` is granted explicitly, and no default privileges are set: a table added later stays invisible until someone grants it deliberately. + +Fallback if the connector path fails: Hybrid agent in GCP. + +--- + +# TASK 2 — Source extraction: Van Essen → GCS raw zone + +Land locations and readings untransformed in GCS as date-partitioned parquet. Raw storage is what makes Mode B replay possible — a mapping bug becomes a reprocess, not a re-fetch. Carries the external blocker. + +**Done when:** both land at the documented paths; readings extraction is incremental; a per-entity failure doesn't abort the run; fixtures exist so downstream work needs no live API. + +### 2.1 — Confirm the readings endpoint; finalize the source mapping + +**The swagger is public** (`https://diver-hub.com/private/swagger/v1/swagger.json`) and reading it settled most of this without credentials. Full mapping in `docs/sources/san_acacia.md`; four corrections that invalidate parts of the original draft: + +- ✅ **No `/api/api/` segment, no `locations/sanacaciareach`.** Seven endpoints under `/api/v1/`. Reference data is `Projects` → `MonitoringPoints/ByProject/{id}`. +- ✅ **No `gs`/`vrd` arrays.** `WaterLevels/ByMonitoringPoint` returns a flat `[{dateAndTime, level}]`. Datum and approval are *query parameters* (`reference`, `approved`), not fields to pick out of parallel arrays. The reshaping `transform.py` was scaffolded for does not exist. +- ✅ **`DiverData` is not the series we want.** It returns `DataPoint` — pressure, temperature, conductivity, salinity — with no water level. It is what the known-good example URL fetches, which is why it looked like the readings endpoint. +- ✅ **`MonitoringPoint` is `{id, name}` only.** No coordinates, no `drillingDepth`. The planned centimetre conversion and geometry mapping have no source here; both must come from the Ocotillo rows the points reconcile against. + +Built, and testable without the network: + +- ✅ `sources/san_acacia/client.py` — JWT auth refreshed against `validTo` with a skew, one forced re-login on a 401, and windowed fetches that halve on a 500 and refuse to shrink past a one-day floor. +- ✅ `shared/windows.py` — the window arithmetic, kept pure so the tricky part is testable. +- ✅ `scripts/probe_diverhub.py` — a one-off instrument that answers the remaining questions against the live API. + +⬜ **Run the probe.** It needs the credentials Ethan circulated. Until then: + +**`WaterLevelReference` is `enum [0,1,2,3]` with no names in the spec, and this is the highest-risk unknown in the epic.** Which value means ground surface is not derivable, and choosing wrong does not fail — it returns plausible numbers on the wrong datum and silently poisons every reading. `GROUND_SURFACE_REFERENCE` is `None` in code and the client will not guess. The probe samples all four side by side so a person can identify it against a well whose depth to water is known. + +Also still open: the window ceiling (three months works, the limit is unmeasured), whether `approved=true`/`false` partition or overlap, whether `dateAndTime` is marked UTC, and whether `level` is feet. That last one gates correctness rather than completeness, same as the datum. + +### 2.2 — dlt resource: locations → GCS + +- ✅ `@dlt.resource(name="vanessen_locations")`, `write_disposition="replace"`, on `MonitoringPoints/ByProject/4317` — **not** the `locations/sanacaciareach` path in the original draft, which does not exist. One request, no pagination. +- ✅ Asset `raw_san_acacia_locations` emits the point count, project id, and a sample of names. Tested against a stub, no network. +- ✅ `replace` rather than `append`: this is a snapshot of what the vendor currently lists, and a point disappearing is information rather than something to accumulate. + +The payload is `{id, name}` only, so this cannot be a source of geometry or construction detail — it enumerates the points a reading fetch walks. **38 points, not the 33 the plan assumes**, still unexplained. + +### 2.3 — dlt resource: readings → GCS, incremental + +- ✅ `@dlt.resource(name="vanessen_readings")`, `write_disposition="append"`, dlt incremental cursor on `dateAndTime`, walking each point from its watermark. +- ✅ `INITIAL_START` (2015-01-01) documented as a floor for a point with no cursor, never a backfill lever. +- ✅ Per-point failure isolation: one diver failing costs that diver's data for the run, not the other thirty-seven. Failures are collected into a list **the caller owns** — a dlt resource is a module-level object shared by every run, so per-run state stashed on it would have concurrent runs overwriting each other. +- ✅ Asset `raw_san_acacia_readings` emits rows ingested, points attempted, points failed, and the failures themselves. +- ✅ Nothing is converted on the way in. The raw zone stores the vendor's `level` in the vendor's centimetres on the vendor's datum, with `unit` and `reference` recorded alongside, so a mapping bug is a reprocess rather than a re-fetch. + +**Vendor approval needs two requests.** `approved` is a query parameter, not a response field, so the flag cannot be read off a row. Fetching `approved=true` and `approved=false` separately and concatenating would duplicate every reading if the two sets overlap — which is still unknown (open question 4). Instead the unfiltered series is authoritative and a second `approved=true` fetch supplies a set of timestamps used only to tag it. A failure of that second fetch leaves rows tagged `false` rather than losing them: an untagged reading is worth more than no reading, and the vendor flag is not Ocotillo's review status regardless. + +**Window span is measured, not inherited.** `READING_SPAN` is 365 days for this source rather than the cautious 90-day default in `shared/windows.py`, because probing showed `WaterLevels` serving 730 days and 18111 rows in one request. At 90 days a first run for a single point would issue four times the requests for no benefit. It sits at half the largest span observed to work, leaving headroom for a denser point than SO-0125. + +--- + +# TASK 3 — Domain mapping and load into Ocotillo + +Where this stops resembling Aqueduct: the destination is a relational database with constraints and transactions, and mapping rules belong in `domain/` per `ADR4.md`. Three risks — matching 33 wells without duplicating them, representing "public but provisional" when the schema can't, and making the write idempotent so backfill is safe. + +**Done when:** mapping rules are pure functions tested without a database; 33 wells resolve with no duplicates; data is public and separately marked provisional; `transducer_observation` has a unique constraint and the loader upserts against it; loading the same window twice leaves the row count unchanged; the watermark comes from Postgres. + +### 3.1 — Domain layer: Van Essen record → Ocotillo model + +Per `ADR4.md`, `domain/` imports nothing from `api/`, `db/`, `schemas/`, `services/`, and no fastapi/sqlalchemy/pydantic/httpx. + +`domain/van_essen.py`, pure functions: +- `drillingDepth` cm → ft (÷ 30.48), reusing `domain/units.py` where it fits +- reading timestamp → tz-aware UTC `datetime` +- `gs` reading → DTW below ground surface, feet (datum fixed — see Epic) +- `lat`/`lng` → WGS84 point (SRID 4326) +- deterministic external key per well and per series, so repeat runs resolve to the same records + +Plus an adapter in Aqueduct's `BaseAdapter` shape, with the same per-record failure isolation: a bad record is logged and counted, never fatal. Domain errors subclass `ValueError`, matching the CSV importers' per-row contract. Tests need no database and no network. Every value the mapping *invents* rather than reads is listed in the module docstring with its justification. + +### 3.2 — Bootstrap reference data: reconcile wells, seed parameter, sensor, deployments + +Some of the 33 may already exist in Ocotillo under Bureau point IDs. Duplicates are the main risk — the `group_type` collision elsewhere in this database is the reminder that "looks new" isn't proof. + +- Reconciliation report **first**: per well, whether a matching `Thing` exists — on name, on `monitoringPoints[].name` (e.g. `SO-0125`), and on coordinate proximity. Ambiguous matches escalate to a human, never auto-merge. +- Data migration (existing `data_migrations/` runner, already supports dry-run) creates missing `Location`/`Thing`, links existing ones. Idempotent, dry-run-clean before running for real. +- Lexicon terms, a DTW `Parameter`, and a `VanEssenDiver` `Sensor` created if absent. +- One `Deployment` per well (thing → sensor), `recording_interval` ~5 min where known. +- Van Essen `uid` (e.g. `sanacaciareach-40`) persisted as external identifier. +- `DataProvenance` recorded for Van Essen-sourced well attributes: depth, coordinates, installation date. + +### 3.3 — Represent "public but provisional" + +`release_status` is one scalar column and its lexicon category holds `public` and `provisional` as siblings, so both cannot be set. Visibility and maturity are orthogonal axes. + +- Decide the representation. Recommended: keep `release_status = "public"` for visibility, add an explicit maturity field (`is_provisional` boolean, or a `data_maturity` lexicon term) on `TransducerObservation` / `TransducerObservationBlock`. Rejected alternative: overloading `review_status`, which means Bureau review and carries a `reviewer_id` FK. +- Follow the Model Change Workflow in `CLAUDE.md`: db model → schemas → alembic migration → tests → transfer scripts. +- Provisional state is visible wherever the data surfaces — API responses and the Hydrograph Corrector. +- Check the blast radius of `release_status = "public"` before shipping: `services/ngwmn_helper.py` filters `Thing.release_status == "public"` for NGWMN publication. Confirm San Acacia data becoming public is intended there too. +- Existing rows keep their current behavior; the migration has a defined default. + +### 3.4 — Unique constraint on `transducer_observation` + idempotent upsert loader + +`db/transducer.py` defines only an index — no unique constraint, so nothing prevents inserting the same reading twice. That absence is what forces Aqueduct's delete-then-repost in FROST. + +- Alembic migration adds `UniqueConstraint(thing_id, parameter_id, observation_datetime)`. Existing duplicates found and resolved first — the migration must not fail on production data. +- Loader batches and issues `INSERT … ON CONFLICT … DO UPDATE`, through the `db/` SQLAlchemy models, not raw SQL. Batch size tuned and documented; a full backfill month fits in memory; each batch commits in its own transaction. +- `TransducerObservationBlock` rows created/extended for the loaded window, `review_status = "not reviewed"`. +- Loader reports rows inserted, rows updated, adapter failures as Dagster metadata. +- Test: loading the same window twice leaves the row count unchanged. + +### 3.5 — Watermark from Postgres + +- Keep Aqueduct's `WatermarkStore` interface; Postgres implementation returns `MAX(observation_datetime)` for a `(thing_id, parameter_id)`, read in the same session as the write. No GCS sidecar for normal runs. +- Backfill never advances the normal watermark implicitly — inherent with upsert, but asserted in a test. +- In-memory implementation kept for tests. First-ever run for a series falls back to the `initial_start_date` floor. +- Divergence from Aqueduct recorded in the module docstring, so it reads as a decision not an oversight. + +--- + +# TASK 4 — Backfill and operations + +A forward-only pipeline isn't enough. `BACKFILL_STRATEGY.md` §3 lists twelve situations demanding backfill; most come from ongoing operation, not onboarding — outage gaps, vendor corrections, adapter bugs found later, newly mapped properties. + +**Done when:** both backfill jobs are registry-generated, unscheduled, default `dry_run: true`, chunk by month sequentially in one run, and resume from the last completed chunk; the daily pipeline is scheduled and alerts a human on failure; docs carry the runbook. + +### 4.1 — Port shared backfill primitives from Aqueduct + +- `month_chunks()`, `ChunkResult`, `sum_chunk_results()`, `parse_backfill_date()`, `validate_date_order()`, `attach_run_timestamp()`, `sanitize_run_key()`, `resolve_location_ids()`, `chunk_key()`, `BackfillCheckpointStore` → `automated_ingestion/shared/backfill.py`. `atomic_write_json_with_retry()` → `shared/gcs.py`. +- `ChunkResult` adjusted for Postgres: `rows_upserted` replaces `observations_posted`/`observations_deleted`. +- Aqueduct's tests ported alongside and passing. +- Each docstring notes provenance and what changed, so the two can be diffed later. + +### 4.2 — Backfill Mode A (refetch) + +Covers data never ingested: onboarding, a late-added well, an outage gap beyond the retry budget, a vendor correction, extending history past the original floor (§3A). + +- `san_acacia_backfill_refetch`, generated from the registry via a factory so a second source needs a registry entry, not new wiring. No schedule; launched from the Launchpad. +- Run config: `location_ids` (empty = every location the API returns), `start_date`, `end_date`, `run_key`, `dry_run`. +- **`dry_run: true` default.** Logs the full plan — entities, range, chunk list, expected counts — making exactly one read-only API call to resolve and validate the entity list, writing nothing. +- An unknown `location_id` fails the run naming the bad IDs, rather than silently backfilling nothing. +- Calendar-month chunks, sequential within one Dagster run — one billed run regardless of chunk count. +- Ingest writes to `vanessen_backfill_readings` under isolated dlt pipeline state, so backfill can't roll back or race the scheduled cursor. +- A chunk checkpoints only after ingest + transform + load all succeed; same `run_key` resumes from the last completed chunk. +- Same idempotent upsert as normal load — no delete step, no window where data is missing. +- Metadata reports per-chunk and total rows ingested, rows upserted, adapter failures. + +### 4.3 — Backfill Mode B (replay) + +Covers raw already in GCS with only the mapping wrong: adapter or unit bug, newly mapped property, storage migration, upstream rename, Ocotillo-side loss with parquet intact (§3B). Aqueduct notes this is almost entirely generic — build it that way. + +- `san_acacia_backfill_replay` from the same factory. Never contacts the Van Essen API. +- Reads raw parquet for an explicit range, filtered on event time, re-running the source's *current* adapter — so fixing a domain bug and replaying picks it up automatically. +- Same chunking, checkpointing, `dry_run: true` default, and upsert load path as Mode A. +- Source-agnostic: a second source gets replay free once it has an adapter and a registry entry. Anything that can't be generic is called out in the docstring. +- Test: a deliberately wrong mapping, once corrected, is fully repaired by a replay over the affected window. + +### 4.4 — Schedule, observability, alerting + +- `san_acacia_schedule` runs the daily pipeline; cron avoids contention with existing Dagster+ jobs in the org, recorded in the source registry. +- Dagster logs bridge into the repo's existing logging setup, so ingestion failures surface where the team already looks. Confirm which error-tracking destination is current before wiring this — do not assume the repo's existing integrations are live. +- A failed run notifies someone — not discovered via a stale hydrograph. +- Every run emits rows ingested, rows upserted, entities processed, entities failed, adapter failures, resulting watermark per series. +- A zero-new-rows run succeeds and is distinguishable in the logs from a failure. + +### 4.5 — Documentation + +- `docs/sources/san_acacia.md` — confirmed mapping. +- `docs/ingestion-storage-conventions.md` — bucket/dataset/table naming, date partitioning, control-file convention, checklist for adding a source or agency. +- `docs/ingestion-backfill.md` — Modes A and B, chunking, checkpoints, `dry_run` policy, and why Ocotillo upserts where Aqueduct deletes-then-reposts. +- `automated_ingestion/README.md` — architecture, local dev, deploy path. Runbook: launching each mode, reading a dry-run plan, recovering a failed run. +- `CLAUDE.md` section pointing at the above, in the style of the existing "Domain Rules" section. +- "Adding a new source" checklist usable without reading the San Acacia implementation. + +--- + +## Open questions + +1. **Provisional representation** (3.3) — new boolean, new lexicon category, or something else? Recommendation is in the sub-task. +2. **NGWMN** — `release_status = "public"` makes San Acacia wells eligible for NGWMN publication via `services/ngwmn_helper.py`. Intended? +3. **Epic name** — keep "Automated Ingestion Pipeline", or use "Hydrograph Corrector" as originally asked? diff --git a/docs/sources/san_acacia.md b/docs/sources/san_acacia.md new file mode 100644 index 000000000..55725aa40 --- /dev/null +++ b/docs/sources/san_acacia.md @@ -0,0 +1,196 @@ +# Source: San Acacia Reach (Van Essen divers, Diver-HUB) + +The pilot source for automated ingestion. Project **4317 `SanAcaciaReach`**, +containing **38 monitoring points** named `SO-####` — the plan and the Aqueduct +mapping both say 33, so five are unaccounted for and must be identified before +3.2 reconciles anything. Ingestion never creates wells, so an unexpected point +is a decision, not a row. Historically flowed through the retired +FROST/`st2` stack; now flows nowhere. + +This document supersedes the mapping in `Aqueduct/docs/sources/san_acacia.md`, +which described the FROST-era payload rather than the live API. + +## API + +Base URL `https://diver-hub.com/private/api/v1`. +Specification: `https://diver-hub.com/private/swagger/v1/swagger.json` — public, +no authentication needed to read it. **Treat the swagger as authoritative over +anything inherited from the FROST pipeline.** + +There is no `/api/api/` doubled path segment and no `locations/sanacaciareach` +endpoint. Both appeared in earlier drafts and neither exists. + +| Endpoint | Returns | Used for | +|---|---|---| +| `POST /Accounts/Login` | `{token, validTo}` | Authentication | +| `GET /Projects` | `[{id, name}]` | Finding the San Acacia project id | +| `GET /MonitoringPoints/ByProject/{projectId}` | `[{id, name}]` | The 33 points | +| `GET /WaterLevels/ByMonitoringPoint/{id}` | `[{dateAndTime, level}]` | **The series we ingest** | +| `GET /DiverData/ByMonitoringPoint/{id}` | `[DataPoint]` | Raw sensor output; not ingested | +| `GET /ManualMeasurements/ByMonitoringPoint/{id}` | `[{dateAndTime, waterLevelToc}]` | Not ingested — see below | +| `GET /WeatherStationData/AirPressure/ByMonitoringPoint/{id}` | `[DataPoint]` | Not ingested | + +### Authentication + +`POST /Accounts/Login` with `{username, password}` returns a bearer JWT and a +`validTo` timestamp. Every other endpoint requires +`Authorization: Bearer {token}` and answers `401` without it. + +The token is short-lived — about an hour. Refresh against `validTo` rather than +against an assumed lifetime, with a skew so a request in flight at the boundary +does not arrive expired, and re-authenticate once on a `401` so a clock +difference cannot end a backfill. Implemented in +`automated_ingestion/sources/san_acacia/client.py`. + +Credentials live in Secret Manager, never in GitHub secrets and never in the +repository. They are read from `DIVERHUB_USERNAME` / `DIVERHUB_PASSWORD`. + +### Windowing — measured 2026-08-18 + +All series endpoints take `startTime` and `endTime` as **Unix seconds, UTC**, +inclusive of both ends. + +**The 500 is endpoint-specific, and `WaterLevels` — the endpoint we ingest — +did not exhibit it.** Measured against point 39 (SO-0125): + +| Span back from now | `WaterLevels` | +|---|---| +| 90 d | ok, 0 rows | +| 180 d | ok, 1054 rows | +| 365 d | ok, 1054 rows | +| 545 d | ok, 9302 rows | +| 730 d | ok, **18111 rows** | + +A fixed 30-day window slid back 0/1/2/3 years also succeeded every time, so +there is no age-based cutoff on this endpoint either. + +`DiverData` is a different story: a 730-day request failed, and bisecting it +ten times down to a **17-hour** window still returned 500. That is not a volume +ceiling — a 17-hour window of raw diver data is trivial. The failing slice was +the oldest part of the range, starting 2024-08-18. Whatever the cause, it is +specific to `DiverData`, which we do not ingest. + +Practical consequence: the windowing machinery in +`automated_ingestion/shared/windows.py` stays, because 18111 rows in one +response is already large and the ceiling is untested above 730 days, but the +halve-on-500 recovery is **not** a routine path for `WaterLevels`. Do not +assume a 500 there means "too much data" without re-measuring; on `DiverData` +that assumption is provably wrong. + +## Field mapping + +### Water levels — the ingested series + +`WaterLevel` is `{dateAndTime: date-time, level: double}`. That is the whole +schema. Two consequences worth stating plainly, because earlier drafts assumed +otherwise: + +- **There are no `gs` / `vrd` arrays**, and no `approvedWaterLevelsGs` / + `unApprovedWaterLevelsGs`. Nothing in the response says which datum `level` + is on or whether the vendor approved it. +- **Datum and approval are request parameters.** `reference` selects the datum; + `approved` (boolean) selects the vendor's approval state. The same point and + time range returns different numbers depending on what was asked for. + +### WaterLevelReference — resolved 2026-08-18 + +The swagger declares `"WaterLevelReference": { "enum": [0, 1, 2, 3] }` with no +names, so this was determined by measurement. + +All four values return **the same rows at the same timestamps**, related by +constants that held identically across two windows eighteen months apart: + +``` +ref1 + ref0 = 518.160 ref3 + ref0 = 472.704 ref2 - ref0 = 139001.296 +``` + +A constant *sum* means the two move in opposite directions; a constant +*difference* means they move together. So `ref0` and `ref2` rise with the water +and `ref1`/`ref3` fall — the latter pair are depths. `ref1` is deeper than +`ref3` by a fixed **45.456 cm (1.49 ft)**, which is a casing stickup. + +| Value | Meaning | Ingested | +|---|---|---| +| 0 | Water height above the diver | No | +| **3** | **Depth below ground surface** | **Yes** | +| 1 | Depth below top of casing | No | +| 2 | Water-surface elevation above sea level | No | + +`GROUND_SURFACE_REFERENCE = 3`. + +Sample values for SO-0125, 2024-10-30T20:00:00Z: + +| ref0 | ref1 | ref2 | ref3 | +|---|---|---|---| +| 1.186 | 516.974 | 139002.482 | 471.518 | + +The reading checks out physically. The sensor sits at 1390.01 m; ground surface +is 4.727 m above it at **1394.74 m (4576 ft)**, right for San Acacia. Depth to +water runs 4.72 m in October 2024 to 2.2–2.7 m in April 2026, right for a +riparian piezometer. + +**Not independently corroborated.** `ManualMeasurements`, which reports +`waterLevelToc` explicitly, returned nothing in the sampled window, so `ref1` +being TOC is inferred from the stickup rather than confirmed against a measured +one. The probe now searches ten years for a manual reading; a single one would +close this. + +### Units — centimetres, not feet + +`ref2` is only an elevation if the unit is centimetres: 139002 cm is 1390 m, +which matches San Acacia, whereas any other unit puts the ground somewhere +impossible. That fixes the unit for every value the API returns. + +**Ocotillo stores feet.** Convert with `domain.units.convert_cm_to_ft` +(`/100 * 3.28084`). An unconverted value is wrong by a factor of 30.48 and +still reads as a plausible depth, so it would survive review. + +### Monitoring points — thinner than expected + +`MonitoringPoint` is `{id: int, name: string}`. **No coordinates, no +`drillingDepth`, no construction detail.** + +So the planned `drillingDepth` centimetre conversion (÷ 30.48) has no source in +this API, and neither does geometry. Both have to come from the Ocotillo +`Thing` and `Location` records the points reconcile against. That is consistent +with the decision that ingestion never creates wells: matching to an existing +row is the only way it learns where a point is. + +### Not ingested + +- **`DiverData`** returns `DataPoint` — `pressure`, `temperature`, + `conductivity`, `salinity`, `airPressure`, `precipitation`. Useful for + diagnostics, and it is what the known-good example URL fetches, but it + contains no water level. +- **`ManualMeasurements`** returns `waterLevelToc` — top of casing. Ocotillo's + manual-measurement path already owns this, and mixing a TOC-referenced series + into a ground-surface one is the datum error above by another route. +- **`AirPressure`** matters only for barometric compensation, which is the + Hydrograph Corrector's job downstream. + +## Decisions inherited from the epic + +Settled, not to be relitigated per source: + +- **Ground-surface datum.** Never `vrd`, never TOC. No measuring-point + correction on ingest. +- **Public but provisional.** Visible from the first run, marked so no consumer + mistakes an uncorrected diver series for a reviewed one. +- **The vendor `approved` flag is not Ocotillo `review_status`.** Ocotillo's + `approved` asserts a *Bureau* human reviewed it and carries a `reviewer_id` + FK. All San Acacia blocks land `not reviewed`; the vendor flag is preserved + as a separate per-row attribute. + +## Open questions + +| # | Question | How to settle | +|---|---|---| +| 1 | ~~Which `reference` value is ground surface?~~ | **Answered: 3.** Corroboration via `ManualMeasurements` still outstanding | +| 2 | ~~What is the window ceiling?~~ | **`WaterLevels` took 730 d / 18111 rows. The 500 is a `DiverData` problem** | +| 3 | ~~Which project id, how many points?~~ | **Answered: 4317, 38 points (not 33)** | +| 4 | Do `approved=true` and `approved=false` partition the series, or overlap? | Fetch both for one window and compare timestamps | +| 5 | Is `dateAndTime` UTC in the response, and is it marked as such? | Inspect a live payload | +| 6 | ~~Is `level` in feet?~~ | **No — centimetres.** Convert with `convert_cm_to_ft` | + +Questions 1 and 6 both gate correctness rather than completeness: wrong answers +produce data that looks fine. diff --git a/domain/units.py b/domain/units.py index 66fd5d917..18231e1a8 100644 --- a/domain/units.py +++ b/domain/units.py @@ -42,4 +42,18 @@ def convert_m_to_ft(meters: float | None, ndigits: int = 6) -> float | None: return round(meters * METERS_TO_FEET, ndigits) +CENTIMETERS_PER_METER = 100.0 + + +def convert_cm_to_ft(centimeters: float | None, ndigits: int = 6) -> float | None: + """Convert a length from centimeters to feet. + + Diver-HUB reports water levels in centimeters while Ocotillo stores feet, + so every ingested reading passes through here. + """ + if centimeters is None: + return None + return round(centimeters / CENTIMETERS_PER_METER * METERS_TO_FEET, ndigits) + + # ============= EOF ============================================= diff --git a/pyproject.toml b/pyproject.toml index 7b746073b..0cdc5c86a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -106,7 +106,30 @@ dependencies = [ package = true [tool.setuptools] -packages = ["alembic", "cli", "core", "data_migrations", "db", "domain", "schemas", "services", "transfers"] +packages = [ + "alembic", + "automated_ingestion", + "automated_ingestion.defs", + "automated_ingestion.defs.assets", + "automated_ingestion.defs.jobs", + "automated_ingestion.ocotillo", + "automated_ingestion.shared", + "automated_ingestion.sources", + "automated_ingestion.sources.san_acacia", + "cli", + "core", + "data_migrations", + "db", + "domain", + "schemas", + "services", + "transfers", +] + +# Entry point for the `ocotillo-automated-ingestion` Dagster+ code location. +# `dagster dev` and the Dagster+ agent both read this. +[tool.dagster] +module_name = "automated_ingestion.defs.definitions" # Bare `--cov` measures every imported module, which pulls the whole virtualenv # into the report. Scope it to first-party code instead. Keep this a single @@ -119,6 +142,7 @@ relative_files = true omit = [ ".venv/*", "alembic/*", + "automated_ingestion/tests/*", "docker/*", "features/*", "geoserver_iac/*", @@ -173,6 +197,17 @@ cli = [ "openpyxl==3.1.5", "google-api-python-client==2.198.0", ] +# Dagster+ code location dependencies. The API runtime never imports +# `automated_ingestion`, so keeping these out of `dependencies` stops dagster +# and its transitive tree from shipping in the API image. CI and the Dagster+ +# build install them explicitly with `uv sync --group ingestion`. +ingestion = [ + "dagster>=1.13.18", + "dagster-cloud>=1.13.18", + "dlt[filesystem,gs]>=1.30.0", + "gcsfs>=2026.8.0", + "pyarrow>=25.0.1", +] # timezone to use when rendering the date within the migration file # as well as the filename. diff --git a/uv.lock b/uv.lock index e3dbf53cd..526f81af9 100644 --- a/uv.lock +++ b/uv.lock @@ -2,7 +2,8 @@ version = 1 revision = 3 requires-python = ">=3.13" resolution-markers = [ - "python_full_version >= '3.14'", + "python_full_version >= '3.14' and platform_python_implementation != 'PyPy' and sys_platform != 'emscripten'", + "(python_full_version >= '3.14' and platform_python_implementation == 'PyPy') or (python_full_version >= '3.14' and sys_platform == 'emscripten')", "python_full_version < '3.14'", ] @@ -15,6 +16,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/f7/85273299ab57117850cc0a936c64151171fac4da49bc6fba0dad984a7c5f/affine-2.4.0-py3-none-any.whl", hash = "sha256:8a3df80e2b2378aef598a83c1392efd47967afec4242021a0b06b4c7cbc61a92", size = 15662, upload-time = "2023-01-19T23:44:28.833Z" }, ] +[[package]] +name = "aiobotocore" +version = "3.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "aioitertools" }, + { name = "botocore" }, + { name = "jmespath" }, + { name = "multidict" }, + { name = "python-dateutil" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/c0/18abcb7e4e504a68714c280853fd180afe376a4a55e5511fb04ba76702e4/aiobotocore-3.9.0.tar.gz", hash = "sha256:5d344e97c518b010bea167c7f7ba4f9e785f9d2b8ac7af4fd00846c62f2c0a10", size = 514972, upload-time = "2026-08-01T11:54:07.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/c5/6290519dec32f3cdf6827e3bbcbf7a9f4fb29a55a9204199901fed69957b/aiobotocore-3.9.0-py3-none-any.whl", hash = "sha256:7354659eac9ba6034675b3ea178330b7de97c45989d6fda1bf01d3da167b6135", size = 100764, upload-time = "2026-08-01T11:54:06.128Z" }, +] + [[package]] name = "aiofiles" version = "24.1.0" @@ -114,6 +133,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a4/37/cfd1ed540a4d318da025590d96b728e63713c09e9377950fc655dadeb856/aiohttp-3.14.3-cp314-cp314t-win_arm64.whl", hash = "sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7", size = 469280, upload-time = "2026-07-23T01:57:24.241Z" }, ] +[[package]] +name = "aioitertools" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/3c/53c4a17a05fb9ea2313ee1777ff53f5e001aefd5cc85aa2f4c2d982e1e38/aioitertools-0.13.0.tar.gz", hash = "sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c", size = 19322, upload-time = "2025-11-06T22:17:07.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/a1/510b0a7fadc6f43a6ce50152e69dbd86415240835868bb0bd9b5b88b1e06/aioitertools-0.13.0-py3-none-any.whl", hash = "sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be", size = 24182, upload-time = "2025-11-06T22:17:06.502Z" }, +] + [[package]] name = "aiosignal" version = "1.4.0" @@ -167,6 +195,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, ] +[[package]] +name = "antlr4-python3-runtime" +version = "4.13.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/5f/2cdf6f7aca3b20d3f316e9f505292e1f256a32089bd702034c29ebde6242/antlr4_python3_runtime-4.13.2.tar.gz", hash = "sha256:909b647e1d2fc2b70180ac586df3933e38919c85f98ccc656a96cd3f25ef3916", size = 117467, upload-time = "2024-08-03T19:00:12.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/03/a851e84fcbb85214dc637b6378121ef9a0dd61b4c65264675d8a5c9b1ae7/antlr4_python3_runtime-4.13.2-py3-none-any.whl", hash = "sha256:fe3835eb8d33daece0e799090eda89719dbccee7aa39ef94eed3818cafa5a7e8", size = 144462, upload-time = "2024-08-03T19:00:11.134Z" }, +] + [[package]] name = "anyio" version = "4.14.2" @@ -393,6 +430,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, ] +[[package]] +name = "botocore" +version = "1.43.56" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/cc/7f84a5d3071fe878380e9f610ab36ca87b8cbbc4aa81ba2727f90e1f3ea3/botocore-1.43.56.tar.gz", hash = "sha256:6c01f85f0ff9863076f4c761e74ee3aa96c5ccc1ad09fc1efd62ef8f2d22bf57", size = 15733117, upload-time = "2026-07-24T19:31:38.125Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/cd/86fe9e659e9699f62f8dd5ecd8c6725474334b23cab8aa71d82b5f56f1a4/botocore-1.43.56-py3-none-any.whl", hash = "sha256:aafc741f1b10f6fd63253eaf6ea029680c1ff436d87e1b8969d62aefa0c76976", size = 15418773, upload-time = "2026-07-24T19:31:34.758Z" }, +] + [[package]] name = "cachetools" version = "7.1.7" @@ -660,6 +711,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "coloredlogs" +version = "14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "humanfriendly" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/1b/1ecdd371fa68839cfbda15cc671d0f6c92d2c42688df995a9bf6e36f3511/coloredlogs-14.0.tar.gz", hash = "sha256:a1fab193d2053aa6c0a97608c4342d031f1f93a3d1218432c59322441d31a505", size = 275863, upload-time = "2020-02-16T20:51:12.172Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/2f/12747be360d6dea432e7b5dfae3419132cb008535cfe614af73b9ce2643b/coloredlogs-14.0-py2.py3-none-any.whl", hash = "sha256:346f58aad6afd48444c2468618623638dadab76e4e70d5e10822676f2d32226a", size = 43888, upload-time = "2020-02-16T20:51:09.712Z" }, +] + [[package]] name = "coverage" version = "7.10.2" @@ -781,6 +844,111 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/51/51ae3ab3b8553ec61f6558e9a0a9e8c500a9db844f9cf00a732b19c9a6ea/cucumber_tag_expressions-8.0.0-py3-none-any.whl", hash = "sha256:bfe552226f62a4462ee91c9643582f524af84ac84952643fb09057580cbb110a", size = 9726, upload-time = "2025-10-14T17:01:26.098Z" }, ] +[[package]] +name = "dagster" +version = "1.13.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "alembic" }, + { name = "antlr4-python3-runtime" }, + { name = "click" }, + { name = "coloredlogs" }, + { name = "dagster-pipes" }, + { name = "dagster-shared" }, + { name = "docstring-parser" }, + { name = "filelock" }, + { name = "grpcio" }, + { name = "grpcio-health-checking" }, + { name = "jinja2" }, + { name = "protobuf" }, + { name = "psutil", marker = "sys_platform == 'win32'" }, + { name = "python-dotenv" }, + { name = "pytz" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "requests" }, + { name = "rich" }, + { name = "six" }, + { name = "sqlalchemy" }, + { name = "structlog" }, + { name = "tabulate" }, + { name = "tomli" }, + { name = "toposort" }, + { name = "tqdm" }, + { name = "tzdata" }, + { name = "universal-pathlib" }, + { name = "watchdog" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/6b/bac47b75b9ddb3301c345255e87e66d66516755a66be55374f739c3b4da4/dagster-1.13.18.tar.gz", hash = "sha256:b443164a1fad04e4da45fbb729b9ed4ffd0cbf0faf8aa7edc9f8a11cc4a29024", size = 3629353, upload-time = "2026-08-14T19:15:09.753Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/b9/0a37f7460d7391bfb20391a8944d9cc3334cffd0dbd81149efc01ca286ba/dagster-1.13.18-py3-none-any.whl", hash = "sha256:fd9cd4041245e1ae2e71660c45ad6bbc9999489b59dd07d49c09c54d798800c0", size = 2026007, upload-time = "2026-08-14T19:15:07.192Z" }, +] + +[[package]] +name = "dagster-cloud" +version = "1.13.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dagster" }, + { name = "dagster-cloud-cli" }, + { name = "dagster-shared" }, + { name = "pex" }, + { name = "questionary" }, + { name = "requests" }, + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/41/1554b9b5a0ccc10488c66d41a1f03569dfe219f53a57ff8401498d7238f5/dagster_cloud-1.13.18.tar.gz", hash = "sha256:a705e6ce04d438187c46fe72a74c649fc6e7bbb18a116a2297559cab3b389331", size = 737131, upload-time = "2026-08-14T19:15:31.319Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/1e/b9778c03289847be65dffa8fe1ef898e518a3c37395fae4e8d2d2ad9648c/dagster_cloud-1.13.18-py3-none-any.whl", hash = "sha256:d854af1985e54600b6e8bfb34530133289b8956ee0aca5fa7312aef62eda4781", size = 204300, upload-time = "2026-08-14T19:15:29.879Z" }, +] + +[[package]] +name = "dagster-cloud-cli" +version = "1.13.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "dagster-shared" }, + { name = "github3-py" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "questionary" }, + { name = "requests" }, + { name = "setuptools" }, + { name = "typer" }, + { name = "validators" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/27/cc/4eb6b2b63489533a0d8182051a7a54acca917dcd13485ad2fbb268bd2d91/dagster_cloud_cli-1.13.18.tar.gz", hash = "sha256:33a939a0320145beab61d5db8bacde0d40ed72c77328174b130667b9ebe140af", size = 176645, upload-time = "2026-08-14T19:29:52.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/65/67d9a1f36999578b6d13517da5f64e835177312e9b6be2d11c9cd3761924/dagster_cloud_cli-1.13.18-py3-none-any.whl", hash = "sha256:7fc5900404049d1d1e8399947e74b80aa41c3ef3a42962ff1ce3e9a083c3cb25", size = 122353, upload-time = "2026-08-14T19:29:51.67Z" }, +] + +[[package]] +name = "dagster-pipes" +version = "1.13.18" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/90/6e38aac87786e71cabc003978a9a43aae4e5eae7755c45b7078196ae009f/dagster_pipes-1.13.18.tar.gz", hash = "sha256:29b27cdc386664e8c0842b1cc65f970dcc7c568d9e53a67732452b42cb265cd8", size = 149679, upload-time = "2026-08-14T19:15:41.361Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/41/a57cb37bded94a2f77712ed4af7d0a5306f0014313ee8aacd79b34e8778c/dagster_pipes-1.13.18-py3-none-any.whl", hash = "sha256:76eccd1d3d784223a3954a9064b787f21c96f6cb9c470f8d2e755e7ce768b529", size = 20245, upload-time = "2026-08-14T19:15:40.258Z" }, +] + +[[package]] +name = "dagster-shared" +version = "1.13.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, + { name = "platformdirs" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "tomlkit" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/c8/aa3a0b803501437906e9605af83e62bb86dd8860d8797c9d67f72ecfbbde/dagster_shared-1.13.18.tar.gz", hash = "sha256:c081b6cdb1fa79399328e2adaa22fcdf1f336b83fedc5783678b8e40b26eda33", size = 124087, upload-time = "2026-08-14T19:26:53.281Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/2d/b08a5071ab05243a283ca8159069904c37ec26199c9c33d9f0d32627ae7c/dagster_shared-1.13.18-py3-none-any.whl", hash = "sha256:a549a941494fc6b0a860fffcb7018025294fd63d2d804603848e9711e02c4c39", size = 96420, upload-time = "2026-08-14T19:26:51.923Z" }, +] + [[package]] name = "dateparser" version = "1.3.0" @@ -796,6 +964,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/c7/95349670e193b2891176e1b8e5f43e12b31bff6d9994f70e74ab385047f6/dateparser-1.3.0-py3-none-any.whl", hash = "sha256:8dc678b0a526e103379f02ae44337d424bd366aac727d3c6cf52ce1b01efbb5a", size = 318688, upload-time = "2026-02-04T16:00:04.652Z" }, ] +[[package]] +name = "decorator" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/60/8b/32f9823da46cde7df2087faa08cd98d01b908f8dcab982cdba9c84e85355/decorator-5.3.1.tar.gz", hash = "sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82", size = 58084, upload-time = "2026-05-18T06:03:28.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/7f/798705f5296a58ca505d600456748d1be48078eac8a7050d8a98bc9edb89/decorator-5.3.1-py3-none-any.whl", hash = "sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c", size = 10365, upload-time = "2026-05-18T06:03:26.517Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -805,6 +982,51 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, ] +[[package]] +name = "dlt" +version = "1.30.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "fsspec" }, + { name = "gitpython" }, + { name = "giturlparse" }, + { name = "humanize" }, + { name = "jsonpath-ng" }, + { name = "orjson", marker = "(python_full_version >= '3.14' and platform_python_implementation == 'PyPy') or (python_full_version >= '3.14' and sys_platform == 'emscripten') or (platform_python_implementation != 'PyPy' and sys_platform != 'emscripten')" }, + { name = "packaging" }, + { name = "pathvalidate" }, + { name = "pendulum" }, + { name = "pluggy" }, + { name = "pytz" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "requirements-parser" }, + { name = "rich-argparse" }, + { name = "semver" }, + { name = "setuptools" }, + { name = "simplejson" }, + { name = "sqlglot" }, + { name = "tenacity" }, + { name = "tomlkit" }, + { name = "typing-extensions" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/a8/fab4e86b8c9a6f7c04c5ecdb9a7d18297b6ecf6c92e13115eed033714ee7/dlt-1.30.0.tar.gz", hash = "sha256:46157b4c75aabde40c8b12af005e27d51ddde693ebbc2d338682ee0b19527d5b", size = 1155778, upload-time = "2026-08-11T13:21:57.707Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/09/7111a1dfda0b1a92648854182507df1a0c53b17cd258ea5dd206bc65d11f/dlt-1.30.0-py3-none-any.whl", hash = "sha256:7e3c66fc9f8874438539e15123c7ff4f587b5779939e5fca3a43bb3e865cbdab", size = 1432587, upload-time = "2026-08-11T13:21:59.734Z" }, +] + +[package.optional-dependencies] +filesystem = [ + { name = "botocore" }, + { name = "s3fs" }, +] +gs = [ + { name = "gcsfs" }, +] + [[package]] name = "dnspython" version = "2.8.0" @@ -814,6 +1036,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, ] +[[package]] +name = "docstring-parser" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, +] + [[package]] name = "dotenv" version = "0.9.9" @@ -1014,6 +1245,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] +[[package]] +name = "fsspec" +version = "2026.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/78/f34251dadb8f3921264a1d9b8946f5e542014ee2614b285261b4e40e6775/fsspec-2026.7.0.tar.gz", hash = "sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88", size = 317040, upload-time = "2026-07-28T16:34:51.052Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/3c/6a2bf344106328fd04963664a60b9bb6496fc25df8e962fcdc1367285fb9/fsspec-2026.7.0-py3-none-any.whl", hash = "sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279", size = 206583, upload-time = "2026-07-28T16:34:49.538Z" }, +] + +[[package]] +name = "gcsfs" +version = "2026.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "decorator" }, + { name = "fsspec" }, + { name = "google-auth" }, + { name = "google-auth-oauthlib" }, + { name = "google-cloud-storage" }, + { name = "google-cloud-storage-control" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/3a/194b5e67b78586a45fb36958800a30783c24c07b7f1c0e82d347f01346c6/gcsfs-2026.8.0.tar.gz", hash = "sha256:c2a7c0ffee2d0837243b4f838efaa185c152a4eb4cc529e41b3569bf00b9781e", size = 1090432, upload-time = "2026-08-13T11:46:49.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/a0/ad756cfa675322303eba197fdd87ff6dd5b6fc0425ade6e6ee2348c94418/gcsfs-2026.8.0-py3-none-any.whl", hash = "sha256:adf616a543ac38557ae87dcf6a282020fad54cdd6778cc9e30ab01a85ef91fdc", size = 91402, upload-time = "2026-08-13T11:46:47.359Z" }, +] + [[package]] name = "geoalchemy2" version = "0.20.0" @@ -1027,6 +1286,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4e/08/b66ad4239f592e05202e25925c08cdd04cc14c3994000ec70ec61fea202c/geoalchemy2-0.20.0-py3-none-any.whl", hash = "sha256:1489a1d106519542a79c97cd0b4c537d80462c353610ebc2429cf2c43daac717", size = 96467, upload-time = "2026-05-12T14:50:24.998Z" }, ] +[[package]] +name = "gitdb" +version = "4.0.12" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "smmap" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/94/63b0fc47eb32792c7ba1fe1b694daec9a63620db1e313033d18140c2320a/gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571", size = 394684, upload-time = "2025-01-02T07:20:46.413Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/61/5c78b91c3143ed5c14207f463aecfc8f9dbb5092fb2869baf37c273b2705/gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf", size = 62794, upload-time = "2025-01-02T07:20:43.624Z" }, +] + +[[package]] +name = "github3-py" +version = "4.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyjwt", extra = ["crypto"] }, + { name = "python-dateutil" }, + { name = "requests" }, + { name = "uritemplate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/89/91/603bcaf8cd1b3927de64bf56c3a8915f6653ea7281919140c5bcff2bfe7b/github3.py-4.0.1.tar.gz", hash = "sha256:30d571076753efc389edc7f9aaef338a4fcb24b54d8968d5f39b1342f45ddd36", size = 36214038, upload-time = "2023-04-26T17:56:37.677Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/ad/2394d4fb542574678b0ba342daf734d4d811768da3c2ee0c84d509dcb26c/github3.py-4.0.1-py3-none-any.whl", hash = "sha256:a89af7de25650612d1da2f0609622bcdeb07ee8a45a1c06b2d16a05e4234e753", size = 151800, upload-time = "2023-04-26T17:56:25.015Z" }, +] + +[[package]] +name = "gitpython" +version = "3.1.59" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gitdb" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ca/dc/126b28e76b24a9268ba931ad3e012f71ebdadf62fd9f17758f7074bb0b20/gitpython-3.1.59.tar.gz", hash = "sha256:0a1475cfdc38a5bfba1a3e9a4a9da52a39749ecec322b772915c019f94e5b7e4", size = 230445, upload-time = "2026-08-10T12:03:20.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/ed/ae57eb7d344f43f87b74b3a281ead6ec7d6394eef72a7b1dcb28dd089550/gitpython-3.1.59-py3-none-any.whl", hash = "sha256:67a82f537384578643624c8b2c531938a9b82be431663e575dcf638526631d4c", size = 220996, upload-time = "2026-08-10T12:03:18.804Z" }, +] + +[[package]] +name = "giturlparse" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/8a/b70b84cc78f9059d627f09560c81a11e8f046570d220a24e169489cad6c9/giturlparse-0.15.0.tar.gz", hash = "sha256:9af3f1fd5c4a0cac94ddb283593635005646393ee0debbe330d1bdff8866bf2c", size = 16138, upload-time = "2026-06-16T07:28:56.021Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/96/147a2771ab655b9353781fb2f95c94eaf1d8576dccc991c1a61d0d355067/giturlparse-0.15.0-py2.py3-none-any.whl", hash = "sha256:76d2e6983b037356ab99b30683e533ac3db96409b68e2163a20fc3aff6446f10", size = 16683, upload-time = "2026-06-16T07:28:55.184Z" }, +] + [[package]] name = "google-api-core" version = "2.34.0" @@ -1043,6 +1350,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/c1/a8a92ae1bc4b1a8f804c776d7d3f0c771b78a62c3ad4df1be41b3fd8c767/google_api_core-2.34.0-py3-none-any.whl", hash = "sha256:cdf9c67e7ca2402d86ccbfde5f2503fc83e3cc3f58cc78456ae96cad24a6d2de", size = 180545, upload-time = "2026-08-06T06:22:47.502Z" }, ] +[package.optional-dependencies] +grpc = [ + { name = "grpcio" }, + { name = "grpcio-status" }, +] + [[package]] name = "google-api-python-client" version = "2.198.0" @@ -1085,6 +1398,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/97/be/954c35a62b9e31de66b0a43c225c9b6bb9e0f98d6b1dc110a2308e3644f5/google_auth_httplib2-0.4.0-py3-none-any.whl", hash = "sha256:8e55cfafa3358cba85f6cad4a886138e88e158d71e7e5c9ee5936a5c1507fb91", size = 9529, upload-time = "2026-05-07T08:02:12.375Z" }, ] +[[package]] +name = "google-auth-oauthlib" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "requests-oauthlib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/70/18/90c7fac516e63cf2058166fce0c88c353647c677b51cc036c09c49bb5cbb/google_auth_oauthlib-1.4.0.tar.gz", hash = "sha256:18b5e28880eb8eba9065c436becdc0ee8e4b59117a73a510679c82f70cd363d2", size = 21675, upload-time = "2026-05-07T08:03:47.816Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/d3/d7dff0d58a9e9244b48044bfb6a898bfcc8ecc42e0031d1bebc695344725/google_auth_oauthlib-1.4.0-py3-none-any.whl", hash = "sha256:251314f213a9ee46a5ae73988e84fd7cca8bb68e7ecf4bfd45940f9e7f51d070", size = 19261, upload-time = "2026-05-07T08:02:13.798Z" }, +] + [[package]] name = "google-cloud-core" version = "2.6.1" @@ -1115,6 +1441,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/06/6f/d69f0e185e08ddb58c323a0a935af2b492907b5de362bc08933b0a3b5644/google_cloud_storage-3.13.1-py3-none-any.whl", hash = "sha256:98208de6c21e85cecd3eb44551894efff33d98365500e178867d4305854a770a", size = 341486, upload-time = "2026-08-06T06:23:36.548Z" }, ] +[[package]] +name = "google-cloud-storage-control" +version = "1.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"] }, + { name = "google-auth" }, + { name = "grpc-google-iam-v1" }, + { name = "grpcio" }, + { name = "proto-plus" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3e/65/b90fe3397596f7066336cf3979fd147aea68e1fcea10d71dd1f8dab974bc/google_cloud_storage_control-1.13.0.tar.gz", hash = "sha256:48351122e3375d2f00a393d6fbfed929c614246a2bd27c9410382b615a4641c6", size = 153274, upload-time = "2026-08-06T06:24:40.386Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/00/937a64affecabc07bdf71041f463d044e3064be7133aa0176a6f1612793f/google_cloud_storage_control-1.13.0-py3-none-any.whl", hash = "sha256:6c8b0b922c38eb1614b5b6bf5d388b581c50a59705c00ec53d9ddd8c36470066", size = 110217, upload-time = "2026-08-06T06:23:35.2Z" }, +] + [[package]] name = "google-crc32c" version = "1.8.0" @@ -1157,6 +1500,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/51/186c02b8549b69ccda44429cf6ff5081e4b61a602ddfe6a8020d1be31d1b/googleapis_common_protos-1.75.1-py3-none-any.whl", hash = "sha256:28a1934bcd33b9c9da66ac301a0a4227e3367f095a17d0375cb98f0a09d93b79", size = 300626, upload-time = "2026-08-06T06:23:46.696Z" }, ] +[package.optional-dependencies] +grpc = [ + { name = "grpcio" }, +] + [[package]] name = "greenlet" version = "3.5.5" @@ -1214,6 +1562,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/df/af/419a4e383bd600858a9b67e9b280a60fdc383ee3f2fe5b6c0c1ef04e74d1/greenlet-3.5.5-cp315-cp315t-win_arm64.whl", hash = "sha256:7f049911ee81a16a03c33d5450d8d5867d27f596ca5fb201b86f4524e874468b", size = 315093, upload-time = "2026-08-10T13:29:34.949Z" }, ] +[[package]] +name = "grpc-google-iam-v1" +version = "0.14.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos", extra = ["grpc"] }, + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d2/d0/fa5bdd5f3f421bb68dc6dc162e9caaf942897ca41ce7255b524723c80f0b/grpc_google_iam_v1-0.14.5.tar.gz", hash = "sha256:07fd3a9fafb586588e771831fbfc8f6597050181d0c3b45e039d18b8fdc1aab5", size = 23736, upload-time = "2026-08-06T06:24:54.489Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/ab/be3ad0d46cffe35fd1e7cc3f9947edd6cb3c552229de3be2742f15f7ea47/grpc_google_iam_v1-0.14.5-py3-none-any.whl", hash = "sha256:0f5e680b20aa0a9441e68c769da04d94d70fca4e43751a82d8abb8aa6a7181ca", size = 32674, upload-time = "2026-08-06T06:23:49.467Z" }, +] + [[package]] name = "grpcio" version = "1.83.0" @@ -1245,6 +1607,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a1/00/b1b26431c9d54eee11724fd6e5585473a2ed47fbc1fb95e5204906a642ce/grpcio-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:2bb48cb5e6dd005ca12b89ce4b6ac0b48ff3112c747542ee7986ef611a8ca6d9", size = 5298932, upload-time = "2026-07-23T15:20:35.48Z" }, ] +[[package]] +name = "grpcio-health-checking" +version = "1.81.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/65/46/6b6678b5a922765ae7637205bb6d0618a4da8b35f2ce6116f8bcff262370/grpcio_health_checking-1.81.1.tar.gz", hash = "sha256:ecc61480e25058a4a04e11e4ab6900ad7439b32e60a8ce4ece7d9f219221c85d", size = 17107, upload-time = "2026-06-11T12:58:49.632Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/87/174bbfb5613794862c45b5cdf567fcfb478ad5a3a7b594e11d998656b2f9/grpcio_health_checking-1.81.1-py3-none-any.whl", hash = "sha256:cbc6a4171825ec64389de2f062d296ba129a5c27eefd0dd55fa837909184bdf9", size = 19120, upload-time = "2026-06-11T12:58:38.008Z" }, +] + +[[package]] +name = "grpcio-status" +version = "1.83.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/fd/848dd7e009de85f8ca59999d1cc618ff8ebf7ea5636d083a47455d212d24/grpcio_status-1.83.0.tar.gz", hash = "sha256:837219c6de9afdccb6f6f72b34bc71e151a2011ef04040e3faaca746a57e54ae", size = 13965, upload-time = "2026-07-23T15:24:26.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/00/73204406228cf989bea6b0fd9fe4702fab49a8a152a0c6f90856dadb6ac7/grpcio_status-1.83.0-py3-none-any.whl", hash = "sha256:f6a838a7c5fb84ae98833ec0ef81ed438c26e11e54b2ddb8e92ad328c861de69", size = 14636, upload-time = "2026-07-23T15:23:49.044Z" }, +] + [[package]] name = "gunicorn" version = "23.0.0" @@ -1306,6 +1695,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "humanfriendly" +version = "10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyreadline3", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794, upload-time = "2021-09-17T21:40:39.897Z" }, +] + +[[package]] +name = "humanize" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0a/ea/13a1ef3c12d12662905801495283530251918b70d62d368f1d2e0272c70d/humanize-4.16.0.tar.gz", hash = "sha256:7dc2244a2f84a4bfb1d36c37bac80cd78e35cdc5c119206d87b018e1445f3a3f", size = 89515, upload-time = "2026-06-30T16:17:29.859Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/aa/0b7365d30fed43e7a3449aba1fe20a0a7174d9cf13e282af4e69ac825441/humanize-4.16.0-py3-none-any.whl", hash = "sha256:353eb2f34c09d098b2880eee8bef21832eae6d174f48c5762fff7e5fcb74d01d", size = 137209, upload-time = "2026-06-30T16:17:28.36Z" }, +] + [[package]] name = "identify" version = "2.6.12" @@ -1366,6 +1776,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "joserfc" version = "1.7.1" @@ -1378,6 +1797,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/00/fa62404c3e347f946faa13aa21085205f9cc06ad17671e37f81a51662ae8/joserfc-1.7.1-py3-none-any.whl", hash = "sha256:b3e3d655612e2e1ef67b2600f2f420e12e537b020208fab1761fad647319c164", size = 70423, upload-time = "2026-06-08T07:21:32.001Z" }, ] +[[package]] +name = "jsonpath-ng" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/32/58/250751940d75c8019659e15482d548a4aa3b6ce122c515102a4bfdac50e3/jsonpath_ng-1.8.0.tar.gz", hash = "sha256:54252968134b5e549ea5b872f1df1168bd7defe1a52fed5a358c194e1943ddc3", size = 74513, upload-time = "2026-02-24T14:42:06.182Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/99/33c7d78a3fb70d545fd5411ac67a651c81602cc09c9cf0df383733f068c5/jsonpath_ng-1.8.0-py3-none-any.whl", hash = "sha256:b8dde192f8af58d646fc031fac9c99fe4d00326afc4148f1f043c601a8cfe138", size = 67844, upload-time = "2026-02-28T00:53:19.637Z" }, +] + [[package]] name = "jsonschema" version = "4.26.0" @@ -1669,6 +2097,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/07/458c344f0f0c178f4481dad5cca790626ffe4c34eabf9467069d06ee4999/numpy-2.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:5f8e00be2ec6f45f4e8a41a527f68d44a7d96fee92a650e4d8b1326f77f61e6e", size = 10748103, upload-time = "2026-08-09T13:48:24.21Z" }, ] +[[package]] +name = "oauthlib" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, +] + [[package]] name = "ocotilloapi" version = "1.2.0" @@ -1788,6 +2225,13 @@ dev = [ { name = "python-dotenv" }, { name = "requests" }, ] +ingestion = [ + { name = "dagster" }, + { name = "dagster-cloud" }, + { name = "dlt", extra = ["filesystem", "gs"] }, + { name = "gcsfs" }, + { name = "pyarrow" }, +] [package.metadata] requires-dist = [ @@ -1905,6 +2349,13 @@ dev = [ { name = "python-dotenv", specifier = ">=1.1.1" }, { name = "requests", specifier = ">=2.34.2" }, ] +ingestion = [ + { name = "dagster", specifier = ">=1.13.18" }, + { name = "dagster-cloud", specifier = ">=1.13.18" }, + { name = "dlt", extras = ["filesystem", "gs"], specifier = ">=1.30.0" }, + { name = "gcsfs", specifier = ">=2026.8.0" }, + { name = "pyarrow", specifier = ">=25.0.1" }, +] [[package]] name = "openpyxl" @@ -1958,6 +2409,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7a/5e/5958555e09635d09b75de3c4f8b9cae7335ca545d77392ffe7331534c402/opentelemetry_semantic_conventions-0.60b1-py3-none-any.whl", hash = "sha256:9fa8c8b0c110da289809292b0591220d3a7b53c1526a23021e977d68597893fb", size = 219982, upload-time = "2025-12-11T13:32:36.955Z" }, ] +[[package]] +name = "orjson" +version = "3.12.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/f3/742fb1f62b825f2c010697eaf4e828004bc2a81e7e806666989c132c7c42/orjson-3.12.0.tar.gz", hash = "sha256:d14203fb1aae2ad9b3d52f8a0e82aeb10197ef1c9bc61da7f358bd70b00123d5", size = 4142915, upload-time = "2026-08-14T16:13:30.607Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/cb/d7b78218a987eb8a8ce4eeae0286b1bb679333eb631ea0eeaf6371680bfc/orjson-3.12.0-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9a36ec60f1796f9a3f13e3b98390295e17a1c7c10155b448d264098bf9ee5900", size = 223397, upload-time = "2026-08-14T16:12:44.003Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4a/bc87c45e7ec639d35ebefd62618e01939531ac8e171426606a01bda05914/orjson-3.12.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ad0422b92d5195443a39f80c3bcf731cc2e00f153bd32063a47b73b057bd0f03", size = 123662, upload-time = "2026-08-14T16:12:45.433Z" }, + { url = "https://files.pythonhosted.org/packages/94/ee/c9a4ff3f2dbedbbe9e635d0fa72c8866adede09b6335ef9644f53752f0d8/orjson-3.12.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:5a0fdbc216388f653d3752ff310e710f59253bd4ed6a2bfb3f4f06b84714bbd8", size = 113374, upload-time = "2026-08-14T16:12:46.755Z" }, + { url = "https://files.pythonhosted.org/packages/75/09/3f330a026a796c8b4c97a6f429652a5e912e7065039bf96ed25e42aa7b25/orjson-3.12.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2eb5c56e534127b2b8fa38d2363c8b1b8190367ee0d1d16c041517d880843b94", size = 130029, upload-time = "2026-08-14T16:12:48.06Z" }, + { url = "https://files.pythonhosted.org/packages/7d/40/094cc53126a3d22f76cdf83b6ea67338bed01d774037621a785aa8e6e5ea/orjson-3.12.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:784106539f4b9d4b930e0b4eb8d45168507dae001945e71b4675a367f1e5e806", size = 130528, upload-time = "2026-08-14T16:12:49.362Z" }, + { url = "https://files.pythonhosted.org/packages/bc/74/89bb236deb9565f99434b13052bb40ddfcce4adf3afbfa3132ee7e421468/orjson-3.12.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c680706fc8396d95e7c4c1f9482563f552137aef91b57237a3ad5aaf64629df", size = 131075, upload-time = "2026-08-14T16:12:50.692Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ac/1176360d762c01b5bd34acd56fc098e936c491363d8b6b397ad4aa475547/orjson-3.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:83445adc40cba26d6d621185a45128ce455b766af368cad2ab64b970603a7978", size = 135321, upload-time = "2026-08-14T16:12:52.114Z" }, + { url = "https://files.pythonhosted.org/packages/7a/02/bbd881c8b9276d50b998de38b4e97de8ace1aac940b0ee545aedbf65ed00/orjson-3.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:644d005bc82f917337a95ce270c9f6f92f9834c2bed7b1477572f8db00784222", size = 127472, upload-time = "2026-08-14T16:12:53.517Z" }, + { url = "https://files.pythonhosted.org/packages/8e/02/a0934d7503e6dcbedd6afac3e7f3f8597fd09389949ad94d0f7540e9dbca/orjson-3.12.0-cp313-cp313-win32.whl", hash = "sha256:d8e78d3d93705e3d27cc17cdb209e44d7a8ea203010cac6ce9c7ffc1ae1996f1", size = 128000, upload-time = "2026-08-14T16:12:55.14Z" }, + { url = "https://files.pythonhosted.org/packages/52/87/69f98f8d40faff103a965a5fbb83f08241b01beaf92badb5413fbc9358cc/orjson-3.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b85931be5b6763c31283805c9bdaae1ca03ad9f6f12a15f1cbf6745b907932c2", size = 121841, upload-time = "2026-08-14T16:12:56.507Z" }, + { url = "https://files.pythonhosted.org/packages/e6/07/b83046a4e3cadcc0987d0f160696107c4af706a619b56e4ad01940cadadf/orjson-3.12.0-cp313-cp313-win_arm64.whl", hash = "sha256:6a31348d7dfa64cd9c78bd1f510ff44c48fe64d71094e6b90e364dba3b55949e", size = 126765, upload-time = "2026-08-14T16:12:57.806Z" }, + { url = "https://files.pythonhosted.org/packages/12/9d/3931253e6f3148abf2cbe14830367042a4806b362ea520df2303db188fb9/orjson-3.12.0-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9e6fee342a48760e854d743e7a81534d8e2925a6f46e09f750cf56b50fd1de5d", size = 223391, upload-time = "2026-08-14T16:12:59.184Z" }, + { url = "https://files.pythonhosted.org/packages/8a/0e/b4a4f1e305367245877b967a0bad70fcf001d77c54ac4339a120b66fdae4/orjson-3.12.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:8c3bb86dd10f39b3fbf434b7d5dc7cac77d6fc8ac572ae30a10731ede2c4b647", size = 123659, upload-time = "2026-08-14T16:13:00.548Z" }, + { url = "https://files.pythonhosted.org/packages/96/f3/6782c6fa85e2702bc66be183c3b421486167dcf266ee4dc1403fe3824870/orjson-3.12.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2bb3ce43203936072dd8b4917b01d3aecfc02329bfb42510cb7cfb24708adc9c", size = 113337, upload-time = "2026-08-14T16:13:02.009Z" }, + { url = "https://files.pythonhosted.org/packages/bf/79/b32ab64bacda9d0fa4942ef483bd03cabf0eaf2be819ca9fb7ff610c559d/orjson-3.12.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:6a2a79c89984dc719817d388c8709e0efc2a2795a934eaa746b4882eb6045adc", size = 130112, upload-time = "2026-08-14T16:13:03.404Z" }, + { url = "https://files.pythonhosted.org/packages/ee/49/6e6142999ca01509219be5e5a9c338a3e5ea011f63e91ff473fbbf3734ed/orjson-3.12.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f06dd838d1e07d9b1de0932ec0485ec92c4d5f5d1ad4817a656268c3e88be1e1", size = 130520, upload-time = "2026-08-14T16:13:04.798Z" }, + { url = "https://files.pythonhosted.org/packages/49/d0/3745af0a4cc9867784f29722929cec4d10bd1c877cd754b01ba6d96eb21a/orjson-3.12.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6b11be792c3d2c6a4be2af4ebf97a68d0bf5f580aca6e86a418a354f6cc846a", size = 131053, upload-time = "2026-08-14T16:13:06.14Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f4/6fe5a22fa478fffb190e65c338c84df5c311ef597b363150a17cc57063c0/orjson-3.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:477ecaf6b9f88f873341b91fcc736119ca81b5e002a9f7f308ff5b4f2ce2a70e", size = 135321, upload-time = "2026-08-14T16:13:07.544Z" }, + { url = "https://files.pythonhosted.org/packages/ff/41/b1b0ec30289646a81a76e2dbaae2686b96fcccb7cb0323dc1dd78cbc7875/orjson-3.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f3c0683136acdc29afdf88a5bc2f7d3d0e34087788d1d63c0144b805a87a196f", size = 127485, upload-time = "2026-08-14T16:13:08.88Z" }, + { url = "https://files.pythonhosted.org/packages/bf/2b/277404bdcc21c93b112b963655b76443ebfe828f8a3ff1de7d90f8850eb3/orjson-3.12.0-cp314-cp314-win32.whl", hash = "sha256:d39f3f5c3927e2dc0913fe5bbc1a2f6b1b9d1bba1de6358340d0ad0d0c00ca92", size = 128048, upload-time = "2026-08-14T16:13:10.305Z" }, + { url = "https://files.pythonhosted.org/packages/41/2b/395b36fa2b4ce7af70b651d715e88f80d884b2c2b14a6b53e84d554fb5f0/orjson-3.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:0b1ac5bf6609b2716c7954011c5fef6254922df029f45d032ee4ebf5d363cbed", size = 121858, upload-time = "2026-08-14T16:13:11.634Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a3/833e895ff452859eebe75093d26691fe9108f1a7a6a08435d7a5780ea652/orjson-3.12.0-cp314-cp314-win_arm64.whl", hash = "sha256:50fae885cb073eac7556353ff3df93312b0d5137b0a5056b2bb63f97ed9a93c7", size = 126749, upload-time = "2026-08-14T16:13:13.117Z" }, + { url = "https://files.pythonhosted.org/packages/58/64/99c8947ece10c17176af9aae85c4948f1d109da77440ec14d87239efaf73/orjson-3.12.0-cp315-cp315-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:01efac2074fffb4cb1ea3fab7861e9d0f2a26913854a972f5ac760525dbdaf6e", size = 223398, upload-time = "2026-08-14T16:13:14.694Z" }, + { url = "https://files.pythonhosted.org/packages/3e/30/cf983fe09f2731420fda097a9f7ef4343f47fa216c228961ad8f6da44f3d/orjson-3.12.0-cp315-cp315-macosx_15_0_arm64.whl", hash = "sha256:ed4ca42bd55955aa34deedcfdfd0e0c31abf51143aae158ae2bc3520b626e517", size = 123655, upload-time = "2026-08-14T16:13:16.221Z" }, + { url = "https://files.pythonhosted.org/packages/11/50/9cb8ae73fa4749dbbc20f617004213b5ff01c20aaeec34c3f31124f2c1d8/orjson-3.12.0-cp315-cp315-manylinux_2_39_aarch64.whl", hash = "sha256:40f92192227505acca4e2533ce565f8e6b9535f7d0d09b0968452f18b7376b38", size = 130515, upload-time = "2026-08-14T16:13:17.601Z" }, + { url = "https://files.pythonhosted.org/packages/9f/0a/adb6ce1a5b5fbf9cb1790f9961bb668a0dd5429aadaf6cee044724681795/orjson-3.12.0-cp315-cp315-manylinux_2_39_armv7l.whl", hash = "sha256:33efefcf5d88eaf400b47e2eba02f91f319bb9951be61ca500b7d536d3f2079d", size = 113327, upload-time = "2026-08-14T16:13:18.927Z" }, + { url = "https://files.pythonhosted.org/packages/51/5c/d17f61581d8dbdde7048f87a330fa24915edec38db4d72b381fec14fbb56/orjson-3.12.0-cp315-cp315-manylinux_2_39_i686.whl", hash = "sha256:8e386b0bc0ddd7cd2056f884b5a0af33592bd01ac66a7ca4b42a65a7e7774a13", size = 130105, upload-time = "2026-08-14T16:13:20.317Z" }, + { url = "https://files.pythonhosted.org/packages/9f/b7/938befcf33bee4704a92ecec6a2731224c539d939bf9429fd39396d28931/orjson-3.12.0-cp315-cp315-manylinux_2_39_x86_64.whl", hash = "sha256:58c58e1de0006ffb580368d6793c36c7b0b021db066479cf281bf5061e732328", size = 131049, upload-time = "2026-08-14T16:13:21.719Z" }, + { url = "https://files.pythonhosted.org/packages/b0/15/cfa2021d64d5aa8bb5c9f604ef375e00ec8b657651b5dd650b1b7ad13df1/orjson-3.12.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:08231552159be266a7269555bd9f7c016aee7d9ad6dab06eb58796c5ccb7101c", size = 135320, upload-time = "2026-08-14T16:13:23.415Z" }, + { url = "https://files.pythonhosted.org/packages/1a/50/3e75dfe357c1e8f9e287c7a5740260ef15bd23a5299eae8d0835dcad5375/orjson-3.12.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:a15f9a891bce5f5cc5d210e3ad8614d4d1b489a56448c099d6d2a7168b2d954a", size = 127488, upload-time = "2026-08-14T16:13:24.791Z" }, + { url = "https://files.pythonhosted.org/packages/11/a6/79aed402eb3ab284dc5b4791a7ad62c5875127de01b8e3f04bd92d551298/orjson-3.12.0-cp315-cp315-win32.whl", hash = "sha256:03091c8a64db4be38746597ceea68f33c238e27acd9bfe99fb59420224ae7a55", size = 128048, upload-time = "2026-08-14T16:13:26.217Z" }, + { url = "https://files.pythonhosted.org/packages/64/f7/2723e264aab7248c1ed6ecaad8e5d0cb866c0cffde75442102ffa7491aba/orjson-3.12.0-cp315-cp315-win_amd64.whl", hash = "sha256:2b7bcefb9f40fa242fa6b06377232c048e655747790829609168c01162f60578", size = 121860, upload-time = "2026-08-14T16:13:27.577Z" }, + { url = "https://files.pythonhosted.org/packages/82/56/630c9113ec8996778f1f0304b364b091b9a9db5fef5fdc17cca622f5ea24/orjson-3.12.0-cp315-cp315-win_arm64.whl", hash = "sha256:859fc4196855890150bb08e649b30d2c93b249b3e3edd0d3bb2231abf8aa8adc", size = 126754, upload-time = "2026-08-14T16:13:28.962Z" }, +] + [[package]] name = "packaging" version = "26.3" @@ -2029,6 +2521,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/85/8d/eef3d8cdccc32abdd91b1286884c99b8c3a6d3b135affcc2a7a0f383bb32/parse_type-0.6.6-py2.py3-none-any.whl", hash = "sha256:3ca79bbe71e170dfccc8ec6c341edfd1c2a0fc1e5cfd18330f93af938de2348c", size = 27085, upload-time = "2025-08-11T22:53:46.396Z" }, ] +[[package]] +name = "pathlib-abc" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/cb/448649d7f25d228bf0be3a04590ab7afa77f15e056f8fa976ed05ec9a78f/pathlib_abc-0.5.2.tar.gz", hash = "sha256:fcd56f147234645e2c59c7ae22808b34c364bb231f685ddd9f96885aed78a94c", size = 33342, upload-time = "2025-10-10T18:37:20.524Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/29/c028a0731e202035f0e2e0bfbf1a3e46ad6c628cbb17f6f1cc9eea5d9ff1/pathlib_abc-0.5.2-py3-none-any.whl", hash = "sha256:4c9d94cf1b23af417ce7c0417b43333b06a106c01000b286c99de230d95eefbb", size = 19070, upload-time = "2025-10-10T18:37:19.437Z" }, +] + [[package]] name = "pathspec" version = "1.0.4" @@ -2038,6 +2539,57 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, ] +[[package]] +name = "pathvalidate" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/2a/52a8da6fe965dea6192eb716b357558e103aea0a1e9a8352ad575a8406ca/pathvalidate-3.3.1.tar.gz", hash = "sha256:b18c07212bfead624345bb8e1d6141cdcf15a39736994ea0b94035ad2b1ba177", size = 63262, upload-time = "2025-06-15T09:07:20.736Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/70/875f4a23bfc4731703a5835487d0d2fb999031bd415e7d17c0ae615c18b7/pathvalidate-3.3.1-py3-none-any.whl", hash = "sha256:5263baab691f8e1af96092fa5137ee17df5bdfbd6cff1fcac4d6ef4bc2e1735f", size = 24305, upload-time = "2025-06-15T09:07:19.117Z" }, +] + +[[package]] +name = "pendulum" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/72/9a51afa0a822b09e286c4cb827ed7b00bc818dac7bd11a5f161e493a217d/pendulum-3.2.0.tar.gz", hash = "sha256:e80feda2d10fa3ff8b1526715f7d33dcb7e08494b3088f2c8a3ac92d4a4331ce", size = 86912, upload-time = "2026-01-30T11:22:24.093Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/8c/400c8b8dbd7524424f3d9902ded64741e82e5e321d1aabbd68ade89e71cf/pendulum-3.2.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:addb0512f919fe5b70c8ee534ee71c775630d3efe567ea5763d92acff857cfc3", size = 337820, upload-time = "2026-01-30T11:21:24.305Z" }, + { url = "https://files.pythonhosted.org/packages/59/38/7c16f26cc55d9206d71da294ce6857d0da381e26bc9e0c2a069424c2b173/pendulum-3.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3aaa50342dc174acebdc21089315012e63789353957b39ac83cac9f9fc8d1075", size = 327551, upload-time = "2026-01-30T11:21:25.747Z" }, + { url = "https://files.pythonhosted.org/packages/0b/cd/f36ec5d56d55104232380fdbf84ff53cc05607574af3cbdc8a43991ac8a7/pendulum-3.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:927e9c9ab52ff68e71b76dd410e5f1cd78f5ea6e7f0a9f5eb549aea16a4d5354", size = 339894, upload-time = "2026-01-30T11:21:27.229Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/b9a1e546519c3a92d5bc17787cea925e06a20def2ae344fa136d2fc40338/pendulum-3.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:249d18f5543c9f43aba3bd77b34864ec8cf6f64edbead405f442e23c94fce63d", size = 373766, upload-time = "2026-01-30T11:21:28.642Z" }, + { url = "https://files.pythonhosted.org/packages/ea/a6/6471ab87ae2260594501f071586a765fc894817043b7d2d4b04e2eff4f31/pendulum-3.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c644cc15eec5fb02291f0f193195156780fd5a0affd7a349592403826d1a35e", size = 379837, upload-time = "2026-01-30T11:21:30.637Z" }, + { url = "https://files.pythonhosted.org/packages/0d/79/0ba0c14e862388f7b822626e6e989163c23bebe7f96de5ec4b207cbe7c3d/pendulum-3.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:063ab61af953bb56ad5bc8e131fd0431c915ed766d90ccecd7549c8090b51004", size = 348904, upload-time = "2026-01-30T11:21:32.436Z" }, + { url = "https://files.pythonhosted.org/packages/17/34/df922c7c0b12719589d4954bfa5bdca9e02bcde220f5c5c1838a87118960/pendulum-3.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:26a3ae26c9dd70a4256f1c2f51addc43641813574c0db6ce5664f9861cd93621", size = 517173, upload-time = "2026-01-30T11:21:34.428Z" }, + { url = "https://files.pythonhosted.org/packages/87/ec/3b9e061eeee97b72a47c1434ee03f6d85f0284d9285d92b12b0fff2d19ac/pendulum-3.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:2b10d91dc00f424444a42f47c69e6b3bfd79376f330179dc06bc342184b35f9a", size = 561744, upload-time = "2026-01-30T11:21:35.861Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7e/f12fdb6070b7975c1fcfa5685dbe4ab73c788878a71f4d1d7e3c87979e37/pendulum-3.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:63070ff03e30a57b16c8e793ee27da8dac4123c1d6e0cf74c460ce9ee8a64aa4", size = 258746, upload-time = "2026-01-30T11:21:37.782Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b8/5abd872056357f069ae34a9b24a75ac58e79092d16201d779a8dd31386bb/pendulum-3.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:c8dde63e2796b62070a49ce813ce200aba9186130307f04ec78affcf6c2e8122", size = 253028, upload-time = "2026-01-30T11:21:39.381Z" }, + { url = "https://files.pythonhosted.org/packages/82/99/5b9cc823862450910bcb2c7cdc6884c0939b268639146d30e4a4f55eb1f1/pendulum-3.2.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c17ac069e88c5a1e930a5ae0ef17357a14b9cc5a28abadda74eaa8106d241c8e", size = 338281, upload-time = "2026-01-30T11:21:40.812Z" }, + { url = "https://files.pythonhosted.org/packages/cd/3a/64a35260f6ac36c0ad50eeb5f1a465b98b0d7603f79a5c2077c41326d639/pendulum-3.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e1fbb540edecb21f8244aebfb05a1f2333ddc6c7819378c099d4a61cc91ae93c", size = 328030, upload-time = "2026-01-30T11:21:42.778Z" }, + { url = "https://files.pythonhosted.org/packages/da/6b/1140e09310035a2afb05bb90a2b8fbda9d3222e03b92de9533123afe6b65/pendulum-3.2.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8c67fb9a1fe8fc1adae2cc01b0c292b268c12475b4609ff4aed71c9dd367b4d", size = 340206, upload-time = "2026-01-30T11:21:44.148Z" }, + { url = "https://files.pythonhosted.org/packages/52/4a/a493de56cbc24a64b21ac6ba98513a9ec5c67daa3dba325e39a8e53f30d8/pendulum-3.2.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:baa9a66c980defda6cfe1275103a94b22e90d83ebd7a84cc961cee6cbd25a244", size = 373976, upload-time = "2026-01-30T11:21:45.56Z" }, + { url = "https://files.pythonhosted.org/packages/3c/4c/f083c4fd1a161d4ab218680cc906338c541497b3098373f2241f58c429cb/pendulum-3.2.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef8f783fa7a14973b0596d8af2a5b2d90858a55030e9b4c6885eb4284b88314f", size = 380075, upload-time = "2026-01-30T11:21:46.959Z" }, + { url = "https://files.pythonhosted.org/packages/57/b6/333a0fcb33bf15eb879a46a11ce6300c1698a141e689665fe430783ff8d6/pendulum-3.2.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7d2e9bfb065727d8676e7ada3793b47a24349500a5e9637404355e482c822be", size = 349026, upload-time = "2026-01-30T11:21:48.271Z" }, + { url = "https://files.pythonhosted.org/packages/43/1a/dfb526ec0cba1e7cd6a5e4f4dd64a6ada7428d1449c54b15f7b295f6e122/pendulum-3.2.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:55d7ba6bb74171c3ee409bf30076ee3a259a3c2bb147ac87ebb76aaa3cf5d3a2", size = 517395, upload-time = "2026-01-30T11:21:49.643Z" }, + { url = "https://files.pythonhosted.org/packages/c9/37/b4f2b5f1200351c4869b8b46ad5c21019e3dbe0417f5867ae969fad7b5fe/pendulum-3.2.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:a50d8cf42f06d3d8c3f8bb2a7ac47fa93b5145e69de6a7209be6a47afdd9cf76", size = 561926, upload-time = "2026-01-30T11:21:51.698Z" }, + { url = "https://files.pythonhosted.org/packages/a0/9e/567376582da58f5fe8e4f579db2bcfbf243cf619a5825bdf1023ad1436b3/pendulum-3.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:e5bbb92b155cd5018b3cf70ee49ed3b9c94398caaaa7ed97fe41e5bb5a968418", size = 258817, upload-time = "2026-01-30T11:21:53.074Z" }, + { url = "https://files.pythonhosted.org/packages/95/67/dfffd7eb50d67fa821cd4d92cf71575ead6162930202bc40dfcedf78c38c/pendulum-3.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:d53134418e04335c3029a32e9341cccc9b085a28744fb5ee4e6a8f5039363b1a", size = 253292, upload-time = "2026-01-30T11:21:54.484Z" }, + { url = "https://files.pythonhosted.org/packages/02/fb/d65db067a67df7252f18b0cb7420dda84078b9e8bfb375215469c14a50be/pendulum-3.2.0-py3-none-any.whl", hash = "sha256:f3a9c18a89b4d9ef39c5fa6a78722aaff8d5be2597c129a3b16b9f40a561acf3", size = 114111, upload-time = "2026-01-30T11:22:22.361Z" }, +] + +[[package]] +name = "pex" +version = "2.59.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/35/6bfd6677428fa0c3aec43d05760854ec9b0a054f0e456665903b1a08098f/pex-2.59.5.tar.gz", hash = "sha256:3bcad71f7dd2df47f9b4dd0dea5d837fe4d9d2792b79fd34726c00bfd5f05923", size = 5136890, upload-time = "2025-10-09T04:19:06.255Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/eb/80e9012146f6dbba4e1b3dae8b45c771b459fc1615e17e80b1f322794d7c/pex-2.59.5-py2.py3-none-any.whl", hash = "sha256:b9dba4f05b6be08da89b0c54f4bc75ed01c32dfde8ee06f87e82b3a6c69ee388", size = 3873314, upload-time = "2025-10-09T04:19:04.123Z" }, +] + [[package]] name = "pg8000" version = "1.31.5" @@ -2156,6 +2708,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/45/e2/bbb7129c9e7999a6b8ee9cca3b66486c25c423ab5a75f34071798b74ce94/pre_commit-4.6.2-py2.py3-none-any.whl", hash = "sha256:e2dde9a75d3bce11bd3831c26d134df00a2803c1d818be6a0383c3dcda25dc4e", size = 226202, upload-time = "2026-08-10T22:07:16.942Z" }, ] +[[package]] +name = "prompt-toolkit" +version = "3.0.53" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/ea/39b988c938f75cb75d7045b5c69f8bfed47ee2152c8837fb403de29d6fb8/prompt_toolkit-3.0.53.tar.gz", hash = "sha256:9ec8a0ad96d5c56148b3f914aa79c1564c3fde5d2e6b876e7bc327e353cf8fa6", size = 435492, upload-time = "2026-07-26T20:56:14.758Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/6f/84908cad2d6aa5144abcf7b42709fe4fdb459bc640ec7ac5786e7693dabc/prompt_toolkit-3.0.53-py3-none-any.whl", hash = "sha256:01c0891d7f9237d5e339f7d3e42cdae80b7534abb1c7c0e3352efba6231492f2", size = 392288, upload-time = "2026-07-26T20:56:12.512Z" }, +] + [[package]] name = "propcache" version = "0.5.2" @@ -2318,6 +2882,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/be/b732c8418ffa5bcfda002890f5dc4c869fc17db66ff11f53b17cfe44afc0/psycopg2_binary-2.9.12-cp314-cp314-win_amd64.whl", hash = "sha256:f12ae41fcafadb39b2785e64a40f9db05d6de2ac114077457e0e7c597f3af980", size = 2848762, upload-time = "2026-04-20T23:35:46.421Z" }, ] +[[package]] +name = "pyarrow" +version = "25.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/e3/27f57f80141379d60defe6703eb50a707325706f07fedfd1312c7a751995/pyarrow-25.0.1.tar.gz", hash = "sha256:9150a83248bfed9813ea3c3af74c3856c1984d444aa28e58bf7733b9750ddf6a", size = 1201653, upload-time = "2026-08-10T12:40:53.904Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/8d/8f271a7a034c834910ec925d56fa4b29733b1380f5289419f5aaa3b02777/pyarrow-25.0.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:c7c534ec03c358a76ea3e505e74c1b6aef290af90c444dfd092dbfe23e755b85", size = 35855328, upload-time = "2026-08-10T12:38:45.489Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cd/5bac242f4e841b9971d5eb94fdfe2577e2b70be983e27401e72055786037/pyarrow-25.0.1-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:dda9470024204d7bbf2042b47c6e8a0e47a3eeb8e34405882dfaea6577e0c153", size = 37622415, upload-time = "2026-08-10T12:38:51.107Z" }, + { url = "https://files.pythonhosted.org/packages/63/1f/96d03b4e1506524f7087adb0fd6b2f69f0c9c7aaff1ec36d8030082e15a5/pyarrow-25.0.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:44a9120ce5bd81936b8ab9a88076e3fd47c2c6838e0e43630fed83626aca81d9", size = 46813813, upload-time = "2026-08-10T12:38:57.773Z" }, + { url = "https://files.pythonhosted.org/packages/98/d6/33a411115b61dbfc16ad6ad73e71730f6fea654ee3667673bc53ab0e2fe7/pyarrow-25.0.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:0befcf816e45a1af33ac775a9970b749e4868a230c7372f0ae5e932bee27039f", size = 50104452, upload-time = "2026-08-10T12:39:04.579Z" }, + { url = "https://files.pythonhosted.org/packages/33/ae/b1b97c9ca87f9f9ddbb5230c798df94eccce61bd79b9b45458c69a478588/pyarrow-25.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3f89685964f46e4216103c75483aac0c0692a5f72212d7ca835adba5ede56ce3", size = 49951343, upload-time = "2026-08-10T12:39:11.8Z" }, + { url = "https://files.pythonhosted.org/packages/98/9e/a112df5cfd5a68cb1d9fc31cfe38c28d5aec9f10865ce37ecef2e4450873/pyarrow-25.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6943e2fe7954d29d84de45d29d34c8dc36ce96570e67d89aa9976e650a4a9138", size = 53144784, upload-time = "2026-08-10T12:39:20.503Z" }, + { url = "https://files.pythonhosted.org/packages/31/24/97e8bd98f1e3b07e2ba08bcdff690674fbe16d69a7d2712cc3884665e615/pyarrow-25.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:31e49a7888fcdf3a835da33ae777f6bb9a866334e5a789282fc26dcf426f7f15", size = 27870159, upload-time = "2026-08-10T12:39:26.161Z" }, + { url = "https://files.pythonhosted.org/packages/36/4c/b525824ad3094076919273cd97db61fb3d78252dee76fa3b8dc8f76774aa/pyarrow-25.0.1-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:bf0b672390cdcb640d7288f96b826d71ff4e9abb254a86c89890baf51a29cee6", size = 35885255, upload-time = "2026-08-10T12:39:32.366Z" }, + { url = "https://files.pythonhosted.org/packages/08/62/448bb0e940de41aec31d1a956e63ad9c54afdf122a103cc3ab20c2a3ce33/pyarrow-25.0.1-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:38a9a4b4b9613380e200641891495a56c3d5a98a092db4a870af9975e220471d", size = 37644461, upload-time = "2026-08-10T12:39:38.142Z" }, + { url = "https://files.pythonhosted.org/packages/6e/9a/13587e38bd4806fd218f50fd13b8903fab60588a699ff0c406372e5b4043/pyarrow-25.0.1-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0b726ad7e7b669be982b0c71c07fe4b037d654354130da79a7902a669e93a66b", size = 46877146, upload-time = "2026-08-10T12:39:43.722Z" }, + { url = "https://files.pythonhosted.org/packages/8d/61/1c5d1229fa21da4cff5365e41e57177aaac57c563c727f35419b8513d1c1/pyarrow-25.0.1-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:9171748cdf796972d85a4b60157c279913e242992e350c90c7450182a9838b2a", size = 50131616, upload-time = "2026-08-10T12:39:49.304Z" }, + { url = "https://files.pythonhosted.org/packages/43/20/291e1d65cc0b09aa19f03cf25cf51a2f5fa94b5db315178f2d254ed5cad4/pyarrow-25.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b7a296aac7a71fa0886c08e155ddb6c636a50013f801f6178daafa0f9e726188", size = 50008879, upload-time = "2026-08-10T12:39:56.891Z" }, + { url = "https://files.pythonhosted.org/packages/8b/7c/1b7c9ec28e76576337e4f97b31141c9a181b89b6d1d6221e9d8205621a58/pyarrow-25.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0fe7c8b6c03969b49c8c66182e4a18e3819ab92d07cfab5d8370c531b9369ef0", size = 53170864, upload-time = "2026-08-10T12:40:04.918Z" }, + { url = "https://files.pythonhosted.org/packages/b7/75/f3d789dc06011a765d14d86bda799cf72ac1d715b6a6edecaa0d73d95062/pyarrow-25.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:f729cfdbd36fd99d543b67a914d2de044c84ebe45be8b34902b299b608c15c8f", size = 28620729, upload-time = "2026-08-10T12:40:51.41Z" }, + { url = "https://files.pythonhosted.org/packages/fc/05/647a8ee6f7c2662feb6921315617bc04dcd6034763fb61b1199720bf6162/pyarrow-25.0.1-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:59a2de54c0cbd954da861eee4d1d330f8e909c45b53455baef696380f2c55033", size = 36130288, upload-time = "2026-08-10T12:40:11.014Z" }, + { url = "https://files.pythonhosted.org/packages/93/f8/c9ee997554d7bea94520667dd1933f109ac1da3ee3556d2b49381e023484/pyarrow-25.0.1-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:35935cd5de130aa5cf4dea052a63e6bf2e17006c35c3a468194242b9b2bf5956", size = 37762187, upload-time = "2026-08-10T12:40:16.592Z" }, + { url = "https://files.pythonhosted.org/packages/a2/08/a28c01c7fe9e96e8233ce2d13df1d402f4f999f848f51d2daacd6bb4c036/pyarrow-25.0.1-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f3831aaa25c67a99f99dc8b05873cb9d64560390372e2aa197ce9dd4a3f06a44", size = 46888003, upload-time = "2026-08-10T12:40:23.242Z" }, + { url = "https://files.pythonhosted.org/packages/1b/b9/58612e977d28dc58c878448866838369ee8da2f1e7cc8ed2c84b952aafee/pyarrow-25.0.1-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6a1fdfc6659b6b19022f2e50627fb5cf7156a66c46bf4299379955cbe742382a", size = 50079036, upload-time = "2026-08-10T12:40:29.169Z" }, + { url = "https://files.pythonhosted.org/packages/72/13/66e1402dcc860e1dc2760b1e0292c9a569b62b3bccab69def1b3e907d006/pyarrow-25.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:169d3429d5be7c752125890620f75a60776d38b0035eddae939651640822332e", size = 50040226, upload-time = "2026-08-10T12:40:35.186Z" }, + { url = "https://files.pythonhosted.org/packages/78/10/3f1a5497a7ef732ab0f03ecca3e66d89d9c0f57fdc61b4794c456b781f01/pyarrow-25.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:119297a6dc197e45d9c6d4415f7814a67ffa36c180d26f68c154c58067ae782d", size = 53149035, upload-time = "2026-08-10T12:40:41.454Z" }, + { url = "https://files.pythonhosted.org/packages/93/c0/37d4a7e8e2f7a6076283673d5298018ca26478b934c6ee369e10505ab32c/pyarrow-25.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4288f27577352d608ca08553b0865e4a9b3aa14820c5d95b53337218d609835b", size = 28753071, upload-time = "2026-08-10T12:40:46.623Z" }, +] + [[package]] name = "pyasn1" version = "0.6.4" @@ -2517,6 +3110,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, ] +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pymssql" version = "2.3.13" @@ -2595,6 +3193,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/73/a7141a1a0559bf1a7aa42a11c879ceb19f02f5c6c371c6d57fd86cefd4d1/pyproj-3.7.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d9d25bae416a24397e0d85739f84d323b55f6511e45a522dd7d7eae70d10c7e4", size = 6391844, upload-time = "2025-08-14T12:05:40.745Z" }, ] +[[package]] +name = "pyreadline3" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/6d/f94028646d7bbe6d9d873c47ee7c246f2d29129d253f0d96cb6fcab70733/pyreadline3-3.5.6.tar.gz", hash = "sha256:61e53218b99656091ddb077df9e71f25850e72e030b6183b39c9b7e6e4f4a9bf", size = 100368, upload-time = "2026-05-14T17:55:04.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/5e/35c856e186b74678c24927847ad9895a51f1bc02a0c6126477a6c6040064/pyreadline3-3.5.6-py3-none-any.whl", hash = "sha256:8449b734232e42a5dcd74048e39b60db2839a4c38cf3ae2bf7707d58b5389c0d", size = 85243, upload-time = "2026-05-14T17:55:03.262Z" }, +] + [[package]] name = "pyshp" version = "2.3.1" @@ -2711,6 +3318,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0f/7b/39c34ca613b0b198cb866466651b26b045e2009864c5183c979a3b83f383/pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815", size = 508283, upload-time = "2026-07-25T15:12:05.782Z" }, ] +[[package]] +name = "pywin32" +version = "312" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/41/12fbfd7f36ed2146d8bc9de96c2741296bf0d490b98508496cff322e274c/pywin32-312-cp313-cp313-win32.whl", hash = "sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c", size = 6370184, upload-time = "2026-06-04T07:49:36.253Z" }, + { url = "https://files.pythonhosted.org/packages/ba/db/36a78e3403099d31d9746d13fdcde5accc43c1155f375a34d15983a479a7/pywin32-312-cp313-cp313-win_amd64.whl", hash = "sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9", size = 6914298, upload-time = "2026-06-04T07:49:38.876Z" }, + { url = "https://files.pythonhosted.org/packages/84/37/c1697194092b76de9ed47ca124323f02c57ffc8a45c06f88a3d5acaf01eb/pywin32-312-cp313-cp313-win_arm64.whl", hash = "sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831", size = 6727640, upload-time = "2026-06-04T07:49:41.083Z" }, + { url = "https://files.pythonhosted.org/packages/fc/2b/1f3cded5822fd49c02f40544cbb5f58c7cfd6b1694869fd476cb6170ee97/pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b", size = 6468928, upload-time = "2026-06-04T07:49:43.188Z" }, + { url = "https://files.pythonhosted.org/packages/21/82/3bf86d2e2808902013132e1ce905a7da0da53790f3836c64bf44d55e24f3/pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e", size = 7024157, upload-time = "2026-06-04T07:49:45.34Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0e/73f6d6800b4f27655abd9e9f6aaeaefcddb2b946e4674efa2bab184a7f7b/pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa", size = 6839598, upload-time = "2026-06-04T07:49:47.613Z" }, + { url = "https://files.pythonhosted.org/packages/eb/61/caa39686032d2ebdd04ff0ab5cbe163126c0066d98e00c9018646e42393b/pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed", size = 6471159, upload-time = "2026-06-04T07:49:50.035Z" }, + { url = "https://files.pythonhosted.org/packages/0f/cd/7e1de64a4a6f69c04214169657ccab0d93a670ea50e35eb8f489d7378249/pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5", size = 7025293, upload-time = "2026-06-04T07:49:54.857Z" }, + { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", size = 6840337, upload-time = "2026-06-04T07:49:57.531Z" }, +] + [[package]] name = "pyyaml" version = "6.0.2" @@ -2728,6 +3351,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446, upload-time = "2024-08-06T20:33:04.33Z" }, ] +[[package]] +name = "questionary" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "prompt-toolkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/45/eafb0bba0f9988f6a2520f9ca2df2c82ddfa8d67c95d6625452e97b204a5/questionary-2.1.1.tar.gz", hash = "sha256:3d7e980292bb0107abaa79c68dd3eee3c561b83a0f89ae482860b181c8bd412d", size = 25845, upload-time = "2025-08-28T19:00:20.851Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/26/1062c7ec1b053db9e499b4d2d5bc231743201b74051c973dadeac80a8f43/questionary-2.1.1-py3-none-any.whl", hash = "sha256:a51af13f345f1cdea62347589fbb6df3b290306ab8930713bfae4d475a7d4a59", size = 36753, upload-time = "2025-08-28T19:00:19.56Z" }, +] + [[package]] name = "rasterio" version = "1.5.0" @@ -2869,6 +3504,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "requests-oauthlib" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "oauthlib" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, +] + +[[package]] +name = "requirements-parser" +version = "0.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/89/1a/5f3c22d38bf1d87d1f4a961489d9eba35c4370a21395562d94410cdd0e73/requirements_parser-0.13.1.tar.gz", hash = "sha256:78811383b2089b6c5197a1431bc2c12ff950245edca39a23eea3460782038dd3", size = 22783, upload-time = "2026-06-18T07:52:25.291Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/f9/15b44d5e4401b0013bbcefe3c09d7bfddcce28cc3d41b1d3077bcedf5b1f/requirements_parser-0.13.1-py3-none-any.whl", hash = "sha256:6e385663eb32589d16e5b22bb6e5251a57908e73803ffff438b53cd6ea2056e0", size = 14926, upload-time = "2026-06-18T07:52:24.171Z" }, +] + [[package]] name = "rich" version = "14.3.2" @@ -2882,6 +3542,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/45/615f5babd880b4bd7d405cc0dc348234c5ffb6ed1ea33e152ede08b2072d/rich-14.3.2-py3-none-any.whl", hash = "sha256:08e67c3e90884651da3239ea668222d19bea7b589149d8014a21c633420dbb69", size = 309963, upload-time = "2026-02-01T16:20:46.078Z" }, ] +[[package]] +name = "rich-argparse" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/e5/1064c43203a357d668cd42435f7a15fe6af51512d85b2104fecb937aa861/rich_argparse-1.8.0.tar.gz", hash = "sha256:679df3d832fa94ad6e4bdb07ded088cd7ea2dddc58ae9b2b46346a40b06cbc0c", size = 38940, upload-time = "2026-05-01T15:18:43.604Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/35/1cceccc5fcb50fa2ed53e2aa278cd032f3902682a73e763fb1ac3be8e6fa/rich_argparse-1.8.0-py3-none-any.whl", hash = "sha256:d2a3ce7854654e2253c578763ab0a32f05016f23a55fadba7b9a91b6c0e92142", size = 25616, upload-time = "2026-05-01T15:18:42.395Z" }, +] + [[package]] name = "rpds-py" version = "0.30.0" @@ -2960,6 +3632,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/8d/0133e4eb4beed9e425d9a98ed6e081a55d195481b7632472be1af08d2f6b/rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762", size = 34696, upload-time = "2025-04-16T09:51:17.142Z" }, ] +[[package]] +name = "s3fs" +version = "2026.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiobotocore" }, + { name = "aiohttp" }, + { name = "fsspec" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/12/60/69fc080b72a32971b2fb5acbc80802b0e876b606f6e27b1689caac4bb57b/s3fs-2026.7.0.tar.gz", hash = "sha256:76b062d1b2bc7bf4bcd9e7d8f1eb2b5dd9d5cee96ce888664c4ddb5f563146bf", size = 87595, upload-time = "2026-07-28T17:14:10.595Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/cc/bcde19a37952ecc58e7d9d67ecaa048e1e21b17d014ce0863a6a6101e606/s3fs-2026.7.0-py3-none-any.whl", hash = "sha256:64edf3c01ebffab1eec38ff9c09eefbf86a3db14c87d248f795da0e7b801d698", size = 32659, upload-time = "2026-07-28T17:14:09.497Z" }, +] + [[package]] name = "scramp" version = "1.4.17" @@ -2972,6 +3658,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/99/0e372781210cd36b2f2727e5be3ea93066edad7edd6fa2dfdec3b3e28845/scramp-1.4.17-py3-none-any.whl", hash = "sha256:a4e3fd2e8169461a28a13777a166d3da94274454f0714a7d3023fee124474ac8", size = 16131, upload-time = "2026-08-07T17:19:39.591Z" }, ] +[[package]] +name = "semver" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/d1/d3159231aec234a59dd7d601e9dd9fe96f3afff15efd33c1070019b26132/semver-3.0.4.tar.gz", hash = "sha256:afc7d8c584a5ed0a11033af086e8af226a9c0b206f313e0301f8dd7b6b589602", size = 269730, upload-time = "2025-01-24T13:19:27.617Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/24/4d91e05817e92e3a61c8a21e08fd0f390f5301f1c448b137c57c4bc6e543/semver-3.0.4-py3-none-any.whl", hash = "sha256:9c824d87ba7f7ab4a1890799cec8596f15c1241cb473404ea1cb0c55e4b04746", size = 17912, upload-time = "2025-01-24T13:19:24.949Z" }, +] + [[package]] name = "sentry-sdk" version = "2.68.0" @@ -2990,6 +3685,15 @@ fastapi = [ { name = "fastapi" }, ] +[[package]] +name = "setuptools" +version = "84.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/44/f5da03a8ef95d369145c5bb53050e7877c9f3d312e128605fd9504829143/setuptools-84.0.0.tar.gz", hash = "sha256:f4695c21257f0d9b537ec2692c941d02ee143b7cc1276941349a546573b2ef73", size = 1168449, upload-time = "2026-08-08T18:27:58.365Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/9c/c510029fc6ef33a6275cd2c5d3cecd6613dfd6aa401d57c54f1c18852ccf/setuptools-84.0.0-py3-none-any.whl", hash = "sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670", size = 818216, upload-time = "2026-08-08T18:27:56.719Z" }, +] + [[package]] name = "shapely" version = "2.1.2" @@ -3042,6 +3746,48 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "simplejson" +version = "4.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0e/2a/54837395a3487c725669428d513293612a48d82b95a0642c936932e5d898/simplejson-4.1.1.tar.gz", hash = "sha256:c08eb9f7a90f77ae470e19a07472e9a79ebc0d1c2315d86a72767665bd5ba79f", size = 118860, upload-time = "2026-04-24T19:24:59.819Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/a9/47b445eeb559c9593453a0648e0fd6d08e8adff64dd5e5ced66726da8a09/simplejson-4.1.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dff52fc7af272e84fc21cc5a06c927c823ca6ae00af14f3b0d7707b42775ed98", size = 113160, upload-time = "2026-04-24T19:23:26.033Z" }, + { url = "https://files.pythonhosted.org/packages/4c/65/cb72db31523c164dea5dc55b02dad065a40c478856bc7534b279d2b51906/simplejson-4.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:971aed0647ad6e840a3943bec812fcda5f2d26a5497a4981d1fb49aa4f9a396c", size = 91521, upload-time = "2026-04-24T19:23:27.572Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e5/54cb7c50ad5fdc1e0a86b7df4b135c2cbd5c4623605aa94466659098e8da/simplejson-4.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:249e2e220aa6d9b9d936bde84eb7bf79d5b6c5a8273c6e411f8b1635a9073f2d", size = 91407, upload-time = "2026-04-24T19:23:28.991Z" }, + { url = "https://files.pythonhosted.org/packages/38/2e/21a3ede87f0bf82d6c7bcb90480d50a6490eb974c6ab20881188e440957c/simplejson-4.1.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8e5cdd6a5d52299f345c15ab5678cc4249e24f383f361d986afbc3c7072a6b6b", size = 192451, upload-time = "2026-04-24T19:23:30.56Z" }, + { url = "https://files.pythonhosted.org/packages/59/df/9903edd3102bf0b5984edfcb90c88612330996efa3b4fbf8a971d6e17839/simplejson-4.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:642cec364e0676e2d5a73fa4d31d0c7c55886997caa2fde24e8292ca44d32728", size = 189015, upload-time = "2026-04-24T19:23:32.647Z" }, + { url = "https://files.pythonhosted.org/packages/98/cd/33230927a780e1398b857e3944abb914556994d252b1d765ae40d112cb25/simplejson-4.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:76fe296ca1df23d290033f10aaacf534fd1b3e3007e7f9ff8aa68b21413aaa78", size = 196658, upload-time = "2026-04-24T19:23:34.563Z" }, + { url = "https://files.pythonhosted.org/packages/cd/84/2c5a7444eb53e9a86d3738299bffddd9f53aeed799ded2f45368221fdb19/simplejson-4.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f0ad25b7dc4e0fb23858355819f2e994f1a5badcdcde8737eac7921c2f1ed2a", size = 185967, upload-time = "2026-04-24T19:23:36.191Z" }, + { url = "https://files.pythonhosted.org/packages/d3/68/454378e06d059cd412a7ed5d87fb6d29fd5b60f13a4d89fc1f764ff434df/simplejson-4.1.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a59ebd0533f03fd06ff0c42ba0f02d93cbcdd7944922bf3b93911327a95b901f", size = 193940, upload-time = "2026-04-24T19:23:38.151Z" }, + { url = "https://files.pythonhosted.org/packages/d5/d5/a15bf915f623a2c5a079d6e3be8256fdb8ef06f110669493a09b9d6933e0/simplejson-4.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bccbf4419676b517939852e5aeff2af6aee4dc046881c67a1581fa6f1cb01abd", size = 189795, upload-time = "2026-04-24T19:23:40.139Z" }, + { url = "https://files.pythonhosted.org/packages/d2/c9/37212ae7dc4b607f0978c408e8633f05c810884e054c33113184c6c2c8a2/simplejson-4.1.1-cp313-cp313-win32.whl", hash = "sha256:6c845363eb5fd166fb7c72243da38f4fcfde666ede7fdf2cc6fd7762894626f7", size = 88773, upload-time = "2026-04-24T19:23:41.754Z" }, + { url = "https://files.pythonhosted.org/packages/fe/a5/c7a0a47883a9015b54c9d8a4b62f2aba17bd4335b1787b9b8a0fc2fa6d52/simplejson-4.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:104d8324c34f25b4b90800bc5fa363780cbc3d8496aef061cba7ce1af9162270", size = 90888, upload-time = "2026-04-24T19:23:43.11Z" }, + { url = "https://files.pythonhosted.org/packages/d3/18/4a118a6a92eb33bb08c8e2fe7ec85cb96f0673491bb2b829930831ee4fbe/simplejson-4.1.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:ed7473602b6625de793b6acba49aa949f144a475f538792067e4cf2fda2071f5", size = 110492, upload-time = "2026-04-24T19:23:44.957Z" }, + { url = "https://files.pythonhosted.org/packages/07/f4/84d160e9fa8cada1e0a9381cae4fa81eecd573577a5b34366d8ced59bdf7/simplejson-4.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:225c9caa324c5b554d009fb9cac22aee7711e71bd96f487938c659af467e828e", size = 90152, upload-time = "2026-04-24T19:23:46.355Z" }, + { url = "https://files.pythonhosted.org/packages/68/31/9a5432c433a7671107182cdc9a20ea78a70f99c4e5334aa54b6d4d0d79ed/simplejson-4.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:95407269340c7f22f09776ea7b717a52cf56cfcf119b5e45f66faa4a26445bea", size = 90115, upload-time = "2026-04-24T19:23:47.743Z" }, + { url = "https://files.pythonhosted.org/packages/78/91/3635cdb13318cb0a328abaa69e2b91251caad39d6779aa308098f341f6cb/simplejson-4.1.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3851658d642c1184d2023f0e6c9ce44a21eb1629e74e7c84ef956b128841fe12", size = 184036, upload-time = "2026-04-24T19:23:49.472Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/149b6ec5393f6849d98c59cadba888b710a8ef4b805ab91e11a566960d40/simplejson-4.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:95a3bb0f78e85f4937f99092239f2011ce06f0f2d803df5c299cc05abbeae008", size = 180543, upload-time = "2026-04-24T19:23:51.023Z" }, + { url = "https://files.pythonhosted.org/packages/df/7c/a5d968d0b527a748b667e62bea94309ccbcb1e2b108e8f0cf8547efaa12b/simplejson-4.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bbfdaa7c0603f75b7b14b211b7f2be44696d4e26833ad2d91d5c87bf5fb9a920", size = 188725, upload-time = "2026-04-24T19:23:52.995Z" }, + { url = "https://files.pythonhosted.org/packages/db/e3/6a8d11181d587ef00e2db9112357e6832111e56dd56b01b5c11758a1965d/simplejson-4.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:39e3c584071dced8c21b4689f0254303521daeb9b5bc1f4289755d71fa3cb0d3", size = 177492, upload-time = "2026-04-24T19:23:54.581Z" }, + { url = "https://files.pythonhosted.org/packages/67/e3/8b0eb8b06e8198cfbd1270487da163d0093df05cc4f557350cd65e2f7e79/simplejson-4.1.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:036a27bd0469b9d79557cbddb392969f876cd7f278cfbd0fba81534927a06575", size = 185281, upload-time = "2026-04-24T19:23:56.13Z" }, + { url = "https://files.pythonhosted.org/packages/dc/5f/64990f07ec9e2cb1a814c674e2e21b5693207f74ac70eb72151b847ea4e6/simplejson-4.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b70bfd2f67f3351baba08aa3ae9233c83f21fd95ae5e6b3d0ecb8c647929112f", size = 181848, upload-time = "2026-04-24T19:23:57.92Z" }, + { url = "https://files.pythonhosted.org/packages/61/a5/bbc1bc0447f339f79f99ab8c37f7f037cb2f1f93af75d6a4d553096bb0c3/simplejson-4.1.1-cp314-cp314-win32.whl", hash = "sha256:37233c72ce88d06acb92747347742b3c07871eba6789f060c179c9302dde8efe", size = 88761, upload-time = "2026-04-24T19:23:59.397Z" }, + { url = "https://files.pythonhosted.org/packages/18/72/ec1b5cbdcb140c132e6c7bdf99bd73e4f675439e77126c88f472fcffa09c/simplejson-4.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:cc0442dea71cd9cbf30a0b8b9929ab5aa6c02c0443a3d977351e6ec5bada4388", size = 91018, upload-time = "2026-04-24T19:24:00.85Z" }, + { url = "https://files.pythonhosted.org/packages/3d/97/4fa437f68ff72219bac3bf3d050de9c6265691f3a170e16954bd69d7cddd/simplejson-4.1.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:c996a4d38290c515af347740659ce095b425449c164a5c9fa3977caa6eff5dbe", size = 113919, upload-time = "2026-04-24T19:24:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/c2/83/59de041d09eb4a9577f7015d7263c32095dfb7fde49717dff62145d89809/simplejson-4.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c65c763fb20d7ca113c1c14dce2fc04a0fc3a57aceff533d6fdac707c7bffb40", size = 91904, upload-time = "2026-04-24T19:24:03.812Z" }, + { url = "https://files.pythonhosted.org/packages/03/8e/46bb345d540f6eb31427d984a4e518cdb182d0621814fee4fee045e8815b/simplejson-4.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0da5c9f57206ee7ef280ff7f1d924937b0a64f9a271a5ef371a2ecdbebba7421", size = 91752, upload-time = "2026-04-24T19:24:05.622Z" }, + { url = "https://files.pythonhosted.org/packages/83/e2/1b2ce97f068835eb3d253c116a4df7a3f436b7bf2fb5ff1ba29287e8b0ec/simplejson-4.1.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ea3426e786425d10e9e82f8a6eda74a7d6eb10d99165ac3d0d3bbcb65c0ea343", size = 214021, upload-time = "2026-04-24T19:24:07.447Z" }, + { url = "https://files.pythonhosted.org/packages/48/70/d93e556df6a0786298644a7c08304fcbeddc248325f23f38acbebeb21165/simplejson-4.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d75cea7a1025edd7e439b2966b3d977c45b5b899e2adaf422811b3ac702ed9fb", size = 213530, upload-time = "2026-04-24T19:24:09.289Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a5/c93bf305b9f00d7259e09e713d60e75bd0f7f53da970f716ab90491770e7/simplejson-4.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:63c2ada8e58f266491f19eed2eeeb7c25c6141e52f8f9e820f6bb94156cf8dbc", size = 218282, upload-time = "2026-04-24T19:24:10.991Z" }, + { url = "https://files.pythonhosted.org/packages/0c/20/a9b5d2e27ec44b069ee251bd55544fc76929a067107b1050001566ba86f3/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d1fffb56305c5b475ee746cf9e04f97423ba5aaacd292dc1255bd75b1d3b124b", size = 209249, upload-time = "2026-04-24T19:24:12.662Z" }, + { url = "https://files.pythonhosted.org/packages/97/e4/e06ee682ed5df67592181f5ecb062e35878967e27f5b6e087237d4548d95/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:a6525ec733f43d0541206cffa64fd2aad5a7ae3eb76566aff49cd4db6382209a", size = 213963, upload-time = "2026-04-24T19:24:14.302Z" }, + { url = "https://files.pythonhosted.org/packages/9c/9f/1e160e4cd8cdbf062bf6a454cdf814dc7a48eb47e566fdb8f80ccb202605/simplejson-4.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:861e393260508efa64d8805a8e49c416c3484907e3f146ce966c69552b49b9a3", size = 210474, upload-time = "2026-04-24T19:24:15.917Z" }, + { url = "https://files.pythonhosted.org/packages/7a/e6/cecd913df322df5bbe7ebb8ba39e0708e505a165553900da8a7761026d6f/simplejson-4.1.1-cp314-cp314t-win32.whl", hash = "sha256:d083b89d30948a751d3d97476c2ed91e4caaa24a1a1459bdbadb8876242c71fe", size = 91134, upload-time = "2026-04-24T19:24:17.635Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/f540dde99cc1d393bd062ab3b5735b777561a5d8f8a5f2e241164444d77a/simplejson-4.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:4cbb299d0528ec0447fe366d8c9641860e28f997a62730690fef905f1f41046e", size = 94467, upload-time = "2026-04-24T19:24:19.109Z" }, + { url = "https://files.pythonhosted.org/packages/ce/6a/8b74c52ffd33dbbde00fe7251fee6a0acdc8cea33f7a43805aed258fb79b/simplejson-4.1.1-py3-none-any.whl", hash = "sha256:2ce92b3748f02423e26d2bfb636fb9d7a8f67c8f5854dcae69d350d123b2eee2", size = 69195, upload-time = "2026-04-24T19:24:57.962Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -3051,6 +3797,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "smmap" +version = "5.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1f/ea/49c993d6dfdd7338c9b1000a0f36817ed7ec84577ae2e52f890d1a4ff909/smmap-5.0.3.tar.gz", hash = "sha256:4d9debb8b99007ae47165abc08670bd74cb74b5227dda7f643eccc4e9eb5642c", size = 22506, upload-time = "2026-03-09T03:43:26.1Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -3125,6 +3880,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/25/7400c18c3ee97914cc99c90007795c00a4ec5b60c853b49db7ba24d11179/sqlalchemy_utils-0.42.1-py3-none-any.whl", hash = "sha256:243cfe1b3a1dae3c74118ae633f1d1e0ed8c787387bc33e556e37c990594ac80", size = 91761, upload-time = "2025-12-13T03:14:15.014Z" }, ] +[[package]] +name = "sqlglot" +version = "30.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/56/d4/da49abcc81beebbb25f29ddf87f2980c63c949569d7f2da40c06d95fa415/sqlglot-30.17.0.tar.gz", hash = "sha256:2d6b8def93304fa300f4d20f48e3909e7f436fda56ca1fafd8975f6c561ef62c", size = 5999019, upload-time = "2026-08-12T19:36:50.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/bc/2a07cef49046e6cf5d1a8b1de553aaef8491b4170dbfe254c8687c764408/sqlglot-30.17.0-py3-none-any.whl", hash = "sha256:84435ac283a60173da31b5fd7d11a725037a1c3fd6ed1e21fb065de74ddb579f", size = 741795, upload-time = "2026-08-12T19:36:48.699Z" }, +] + [[package]] name = "sqlparse" version = "0.6.0" @@ -3146,6 +3910,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/cb/6a6a47d5b464bd08695d254f3da6e7986cc70c9fa5d778eda57538edfe56/starlette-1.6.0-py3-none-any.whl", hash = "sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c", size = 75969, upload-time = "2026-08-08T18:27:56.196Z" }, ] +[[package]] +name = "structlog" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/89/b4a0bcfdf4f71a3dea31379f095929613d7e4528a0996bca6aa964cd0dca/structlog-26.1.0.tar.gz", hash = "sha256:f63a716cbd1b1291cf7661de7794b455acfa4c43c5bcf1630e6ad5ddc1adb3b7", size = 1459881, upload-time = "2026-06-06T07:33:39.348Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/18/489c97b834dfff9cf2fc2507cede4bcd4b11e67f84bc462acd1992496f86/structlog-26.1.0-py3-none-any.whl", hash = "sha256:e081a26d6c373e6d201eca24eede26d8ffab07f88f477822e679183428d3d91e", size = 73764, upload-time = "2026-06-06T07:33:38.046Z" }, +] + +[[package]] +name = "tabulate" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, +] + +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + [[package]] name = "tinydb" version = "4.8.2" @@ -3155,6 +3946,72 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/17/853354204e1ca022d6b7d011ca7f3206c4f8faa3cc743e92609b49c1d83f/tinydb-4.8.2-py3-none-any.whl", hash = "sha256:f97030ee5cbc91eeadd1d7af07ab0e48ceb04aa63d4a983adbaca4cba16e86c3", size = 24888, upload-time = "2024-10-12T15:23:59.833Z" }, ] +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "tomlkit" +version = "0.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/96/e07752635b98536177fa1f37671c8f3cdde2e724c6bcf6034b2cfb571565/tomlkit-0.15.1.tar.gz", hash = "sha256:e25bbf38843005246210a12982776f27f99cb9be67160e14434d0c0d21ee1e97", size = 180129, upload-time = "2026-07-17T01:48:04.562Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/bc/8c13eb66537dce1d2bd3a57132902f38d0e7f5bb46fa9f4daed9fe9d76ee/tomlkit-0.15.1-py3-none-any.whl", hash = "sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304", size = 49449, upload-time = "2026-07-17T01:48:05.728Z" }, +] + +[[package]] +name = "toposort" +version = "1.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/19/8e955d90985ecbd3b9adb2a759753a6840da2dff3c569d412b2c9217678b/toposort-1.10.tar.gz", hash = "sha256:bfbb479c53d0a696ea7402601f4e693c97b0367837c8898bc6471adfca37a6bd", size = 11132, upload-time = "2023-02-27T13:59:51.834Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f6/17/57b444fd314d5e1593350b9a31d000e7411ba8e17ce12dc7ad54ca76b810/toposort-1.10-py3-none-any.whl", hash = "sha256:cbdbc0d0bee4d2695ab2ceec97fe0679e9c10eab4b2a87a9372b929e70563a87", size = 8500, upload-time = "2023-02-25T20:07:06.538Z" }, +] + +[[package]] +name = "tqdm" +version = "4.70.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/21/3b/6c24bec5be5e743ffd99576daa5cc077722fc7d5bbc00bd133fa0c698dc6/tqdm-4.70.0.tar.gz", hash = "sha256:55b0b0dbd97462d06ebee91e4dac24ed4d4702be82b24f07e6c1d27e08cea220", size = 795438, upload-time = "2026-07-27T11:33:15.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/1c/01bfd571a64e7f270e6bab5e33777debe0edc56759233ce84f27dec92d14/tqdm-4.70.0-py3-none-any.whl", hash = "sha256:7f585706bfddbdebf89daac705b2dfcc16890130727d3197ca62c732b4310953", size = 80184, upload-time = "2026-07-27T11:33:13.167Z" }, +] + [[package]] name = "typer" version = "0.27.1" @@ -3221,6 +4078,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, ] +[[package]] +name = "universal-pathlib" +version = "0.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fsspec" }, + { name = "pathlib-abc" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/6e/d997a70ee8f4c61f9a7e2f4f8af721cf072a3326848fc881b05187e52558/universal_pathlib-0.3.10.tar.gz", hash = "sha256:4487cbc90730a48cfb64f811d99e14b6faed6d738420cd5f93f59f48e6930bfb", size = 261110, upload-time = "2026-02-22T14:40:58.87Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/1a/5d9a402b39ec892d856bbdd9db502ff73ce28cdf4aff72eb1ce1d6843506/universal_pathlib-0.3.10-py3-none-any.whl", hash = "sha256:dfaf2fb35683d2eb1287a3ed7b215e4d6016aa6eaf339c607023d22f90821c66", size = 83528, upload-time = "2026-02-22T14:40:57.316Z" }, +] + [[package]] name = "uritemplate" version = "4.2.0" @@ -3261,6 +4131,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/2b/ebd108734a8204c6b4b93c681c9a38c5273b3ccd5d129fee4ffc1d97772c/uvicorn-0.52.3-py3-none-any.whl", hash = "sha256:116af2710dbf47c80f463cd20ee4884b6662f4c9f227d797ddc7279d2fcc2c7c", size = 79859, upload-time = "2026-08-13T16:50:01.323Z" }, ] +[[package]] +name = "validators" +version = "0.35.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/66/a435d9ae49850b2f071f7ebd8119dd4e84872b01630d6736761e6e7fd847/validators-0.35.0.tar.gz", hash = "sha256:992d6c48a4e77c81f1b4daba10d16c3a9bb0dbb79b3a19ea847ff0928e70497a", size = 73399, upload-time = "2025-05-01T05:42:06.7Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/6e/3e955517e22cbdd565f2f8b2e73d52528b14b8bcfdb04f62466b071de847/validators-0.35.0-py3-none-any.whl", hash = "sha256:e8c947097eae7892cb3d26868d637f79f47b4a0554bc6b80065dfe5aac3705dd", size = 44712, upload-time = "2025-05-01T05:42:04.203Z" }, +] + [[package]] name = "virtualenv" version = "20.32.0" @@ -3275,6 +4154,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5c/c6/f8f28009920a736d0df434b52e9feebfb4d702ba942f15338cb4a83eafc1/virtualenv-20.32.0-py3-none-any.whl", hash = "sha256:2c310aecb62e5aa1b06103ed7c2977b81e042695de2697d01017ff0f1034af56", size = 6057761, upload-time = "2025-07-21T04:09:48.059Z" }, ] +[[package]] +name = "watchdog" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/98/b0345cabdce2041a01293ba483333582891a3bd5769b08eceb0d406056ef/watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c", size = 96480, upload-time = "2024-11-01T14:06:42.952Z" }, + { url = "https://files.pythonhosted.org/packages/85/83/cdf13902c626b28eedef7ec4f10745c52aad8a8fe7eb04ed7b1f111ca20e/watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134", size = 88451, upload-time = "2024-11-01T14:06:45.084Z" }, + { url = "https://files.pythonhosted.org/packages/fe/c4/225c87bae08c8b9ec99030cd48ae9c4eca050a59bf5c2255853e18c87b50/watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b", size = 89057, upload-time = "2024-11-01T14:06:47.324Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, + { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, + { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, + { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, +] + +[[package]] +name = "wcwidth" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/34/74/c6428f875774288bec1396f5bfcbc2d925700a4dad61727fd5f2b12f249d/wcwidth-0.8.2.tar.gz", hash = "sha256:91fbef97204b96a3d4d421609b80340b760cf33e26da123ff243d76b1fda8dda", size = 1466253, upload-time = "2026-06-29T18:11:11.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/42/3e5985a0a7e57de470b320c6d6a1a67c844f6737a587f3d44dd13d1819e7/wcwidth-0.8.2-py3-none-any.whl", hash = "sha256:d63947694a0539a1d51e01eda7caf800c291020e6cdd7e28ad7b14dd33ad4f85", size = 323166, upload-time = "2026-06-29T18:11:09.888Z" }, +] + [[package]] name = "werkzeug" version = "3.1.6" @@ -3287,6 +4196,59 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4d/ec/d58832f89ede95652fd01f4f24236af7d32b70cab2196dfcc2d2fd13c5c2/werkzeug-3.1.6-py3-none-any.whl", hash = "sha256:7ddf3357bb9564e407607f988f683d72038551200c704012bb9a4c523d42f131", size = 225166, upload-time = "2026-02-19T15:17:17.475Z" }, ] +[[package]] +name = "wrapt" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2b/b0/c1f5a970721f06b85c0cd5142e0ff8fe067708abd779b0c4f4be7d61d09f/wrapt-2.3.0.tar.gz", hash = "sha256:681a2d0eefd721998f90642762b8e75c2159ec531b20ad5e437245ea7b06a107", size = 131509, upload-time = "2026-07-28T06:06:14.895Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/6e/0f88a072483e76b881e3fdcd6b6ffb4a5791002514fe541e72b1b73c859a/wrapt-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0d3fb71e65b001adfc42684522eeccd9c21d8ba679945abc993439567b66e59f", size = 81960, upload-time = "2026-07-28T06:04:49.622Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ff/b7e2776e7c294075eb712cc9ef573d1b818f393006d09787262b8fc871c4/wrapt-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:51a7a4181c1295774812271fbcd7c909df372bc25579d4ed9eb875caaf0ae86f", size = 82435, upload-time = "2026-07-28T06:04:50.9Z" }, + { url = "https://files.pythonhosted.org/packages/d8/90/343bb5d0f1f9669bc252a6073f085b4abf862511bd5c9c9eaec754341f1d/wrapt-2.3.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9045917809c63fdf7abe3a2ceaed3d670b8ee4500ddd9291192d30aeb34467c5", size = 170350, upload-time = "2026-07-28T06:04:52.187Z" }, + { url = "https://files.pythonhosted.org/packages/59/f8/13b79a392930bd0dd6b86cbfbfe1c40944110456e1dc6d809e5c46ece904/wrapt-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:54ca1d5573f69b5fe1d74f1f65799c68015e82f685efec9fd8cfa40a094c44d0", size = 170022, upload-time = "2026-07-28T06:04:53.599Z" }, + { url = "https://files.pythonhosted.org/packages/b2/fc/4f1b6918f5290db959d6e0c07f77385d87cede29c39c9cf8f145e9c82954/wrapt-2.3.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:242b60c21e30866e6a2fa606c612b47c553fa60c0eaeeeb7797fb842ac0ce609", size = 161043, upload-time = "2026-07-28T06:04:54.936Z" }, + { url = "https://files.pythonhosted.org/packages/01/e1/45d3cf74414780bdff6d0380467e003f6eb0f028b6c9403db868dbc7209c/wrapt-2.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3f3d7ec0a51fbfe00d3aef047641ff2c58b25565b4717fc1f90e050be01cba8", size = 168576, upload-time = "2026-07-28T06:04:56.261Z" }, + { url = "https://files.pythonhosted.org/packages/f3/73/2fa58dd97f191c997755e2c6d569a68f0c433db4e4b36099bdd7227b6cac/wrapt-2.3.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:261f53870cd4fb2bf38f9f972c56c728fd224cb7c65721307de59d9e7e6741ae", size = 159140, upload-time = "2026-07-28T06:04:57.754Z" }, + { url = "https://files.pythonhosted.org/packages/29/a8/08a56e2000a8816d449dcbad8c8b081697acbbd490821ceca0f9d8e8d20c/wrapt-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8159ec0b0cb7608175eb150de94c19e34f4d47ac655f5ca9baf45df6b688ffd3", size = 169263, upload-time = "2026-07-28T06:04:59.161Z" }, + { url = "https://files.pythonhosted.org/packages/9e/d4/354e1725e35a73b2af4fa70a3e024c7a5d1bf1802dfb862dcb668aae0253/wrapt-2.3.0-cp313-cp313-win32.whl", hash = "sha256:10461884b3014fbfc8eb7d09a93c5f246363e6711d9d881f95eb8c27fdef049f", size = 78241, upload-time = "2026-07-28T06:05:00.507Z" }, + { url = "https://files.pythonhosted.org/packages/6c/7e/34c87fa2174848dfee820322aaa318bab08913998ccecc8d2f57b4ad4639/wrapt-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:ac870cc97b73bb00ac353329e9559a4bebc47c4c86792ed9b23b58c15b6ad838", size = 81113, upload-time = "2026-07-28T06:05:01.839Z" }, + { url = "https://files.pythonhosted.org/packages/11/86/fcc9a530579e008c9478bb565a6cdfbfd33536660f069c8b91a6607c5050/wrapt-2.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:a65e8db2b4e90c2e7ade931086351c98ef420bf7a94ee08c95ac8a3cbbc43579", size = 80182, upload-time = "2026-07-28T06:05:03.152Z" }, + { url = "https://files.pythonhosted.org/packages/96/50/3864848b95b28ef73e17551fc8dccbff2628a834f52cf26a57f9c419fb83/wrapt-2.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:fd1f2f557dd3491fe75905e578f4db967393d40d1a8f468edc4d40ac7f2d5944", size = 83921, upload-time = "2026-07-28T06:05:04.476Z" }, + { url = "https://files.pythonhosted.org/packages/3b/4c/3d1921a60c3e8c71c540ff136e6a47a1fbccf7f671e818394889f7871d9c/wrapt-2.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9f5d2aec29dfc76c37e23897dee92766a3fd4f3bff3ae7fc9c6b4bf37d8c1360", size = 84412, upload-time = "2026-07-28T06:05:05.921Z" }, + { url = "https://files.pythonhosted.org/packages/fa/1a/4a796ff7adb26ada6d4b758c94d47a38320b085e7099afc088efbbcdb006/wrapt-2.3.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:646d20d413ffcd1b0a2f700076e2d0252d872dcb7754860a73e45a59ea883614", size = 207168, upload-time = "2026-07-28T06:05:07.256Z" }, + { url = "https://files.pythonhosted.org/packages/1d/3e/d7777776806c579b761bac2f91721dda9f04c7a1b380213c5935cc750ae6/wrapt-2.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:379f670f45b7bb8993edd9f6fc36c6cc65edb81cffa0b504be34acb0303fff0a", size = 214351, upload-time = "2026-07-28T06:05:08.945Z" }, + { url = "https://files.pythonhosted.org/packages/63/27/2d64d394df7bf181955b3bb562bf33c4492fb4be113f53071106d43ad8b5/wrapt-2.3.0-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6208f302f110295d64b22a7ac96500c791bf492dce4366e622e4912b077c9687", size = 199020, upload-time = "2026-07-28T06:05:10.418Z" }, + { url = "https://files.pythonhosted.org/packages/3e/3d/fb31d3db7d9834d265fb1a27a2adf0ddf51557c67458c97b22439ad6ae3d/wrapt-2.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ed635a9ca4f3a5a2b900c10c69e823373bc00ebc114b459383596d3487da3570", size = 209969, upload-time = "2026-07-28T06:05:11.983Z" }, + { url = "https://files.pythonhosted.org/packages/1f/d1/8724b5da582e62070dc9bf4d8bf1972f317297eefd7ba1f2b5c6393ccf6c/wrapt-2.3.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:e3b9eaa742ae7a0aaaaad4ca4b69469d757af2d6e6663ef1dadc47adec0aeb41", size = 196324, upload-time = "2026-07-28T06:05:13.557Z" }, + { url = "https://files.pythonhosted.org/packages/0d/5c/3d9ef411149543016ee6bcf3af707f787cebd946527452b94bf122e9b7b4/wrapt-2.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:d0f7284f88f4833705132d06d3b425a43095c2cbd07c58166aac3ab646ba12a4", size = 202610, upload-time = "2026-07-28T06:05:15.048Z" }, + { url = "https://files.pythonhosted.org/packages/13/9b/4fc042ceb757866dd4a5fc057b3b736f2b360d3703ce9f830d83dc9226e0/wrapt-2.3.0-cp313-cp313t-win32.whl", hash = "sha256:7ebb274aba688b043429eb1500ff8a76ce0cb8ac0812ca3e301f06247b8722b3", size = 79178, upload-time = "2026-07-28T06:05:16.469Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ff/b94878f8eed809ca042685276bcea9f24e8c2ca7c9653bb80bbb920a68a5/wrapt-2.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:c4bded758ad6f03b965830944a2f0bc5b2eb3767fe5a7310134315d1a6610e98", size = 82634, upload-time = "2026-07-28T06:05:18.026Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/663e1de5332a71685a729754312d327d4cada767c36e1c5a2db4c8de49e6/wrapt-2.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:d2cc64539da63e39ffb9c7ede849b6e8ddaaf7b3876b5cfb04efd85a5f3f4eb6", size = 81387, upload-time = "2026-07-28T06:05:19.417Z" }, + { url = "https://files.pythonhosted.org/packages/58/10/b073beaea89bc0d3670a75ff51139430a54b6af7ba7796507730634536dd/wrapt-2.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ea52a0d0f08c584943d5764be0e84efa912c8da23c23e1e285ff2f5641c18fcc", size = 81978, upload-time = "2026-07-28T06:05:21.133Z" }, + { url = "https://files.pythonhosted.org/packages/b3/31/0916d9cebf848ed3f1a0c1888faee421747df77331e4db2bc527a9a85988/wrapt-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:fd85b0aa88efdb189d6ae2f35f4526943a8f091c38599c9c31478241c819e6a1", size = 82518, upload-time = "2026-07-28T06:05:22.562Z" }, + { url = "https://files.pythonhosted.org/packages/f5/73/31c1bf0f3384062751c2094dadb314916d70aa9b6bfd26d994b4a7b393fa/wrapt-2.3.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:141ed6211286a9660d8d6702de598b43f0934b4f0eda16393f100a80f501d945", size = 170187, upload-time = "2026-07-28T06:05:23.904Z" }, + { url = "https://files.pythonhosted.org/packages/ed/25/fce087d54b79b8905f3c3c9dd5f454bbd8d8acb80b960c4a6aee5b4659b3/wrapt-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e49885a62ec4ee854d1b9e6371fda6afd219917225752abf729a3f36d4df9a5", size = 169288, upload-time = "2026-07-28T06:05:25.378Z" }, + { url = "https://files.pythonhosted.org/packages/c7/30/0d09e6dddc6b7a7230ac77f50254b5980ab4fcd22976f72f8cc8a0404458/wrapt-2.3.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d6159c9b2fefec02314e1332dbbbfaf960e369dfd26bcf7f8b258b5732065b3", size = 160932, upload-time = "2026-07-28T06:05:27.022Z" }, + { url = "https://files.pythonhosted.org/packages/2c/ca/0913af0d2ec0c43865d32d615f518fea66c13c5c930e489e9b0de248e9a8/wrapt-2.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:24da48596326ef8e448cfa837b454f638713d3531262375f00e5a9681682fc07", size = 169017, upload-time = "2026-07-28T06:05:28.501Z" }, + { url = "https://files.pythonhosted.org/packages/c3/f2/3d1e47ea81b822210f5df1bf942fd90780a75c055243d569b664529dea88/wrapt-2.3.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:cd3a2edf0427013736b8127955cec62608c56e53ea47e82812ea32059cda407f", size = 159065, upload-time = "2026-07-28T06:05:30.01Z" }, + { url = "https://files.pythonhosted.org/packages/43/a5/ef2066ced8e5fca204e2b361e9708e36555b40949c583d997ea3b590817d/wrapt-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4fa0df3bff4e7ce45759f33fd39335fe2f60477bb9ecf7b8aa41e7d07ee36a23", size = 168821, upload-time = "2026-07-28T06:05:31.649Z" }, + { url = "https://files.pythonhosted.org/packages/d5/e1/016104650d4e572fa91506eb396b3dd8efbccc9284fdc1c9479c3d21db28/wrapt-2.3.0-cp314-cp314-win32.whl", hash = "sha256:2935d5454b3f179a29b12cf390ee47246740ba2c3a7545b1b46ba31a5f2a4a0b", size = 78700, upload-time = "2026-07-28T06:05:33.391Z" }, + { url = "https://files.pythonhosted.org/packages/3d/97/6fdc20a9f2ca304748b3f0819cbf377d55260562777bf0b615431bc3c181/wrapt-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:cc2cea812e5cb179a796b766747e7d3b21088760d8deb95676d482b8c8e6fa7d", size = 81422, upload-time = "2026-07-28T06:05:34.774Z" }, + { url = "https://files.pythonhosted.org/packages/5e/a4/9cbd53bf05746bea2c392af39cb052427a8ec95cbd494d930733d8f44681/wrapt-2.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:22cc5c0a717bd4da87018ae0bffd4c19c6fb679d3ff357216ba566ab26c76cab", size = 80639, upload-time = "2026-07-28T06:05:36.228Z" }, + { url = "https://files.pythonhosted.org/packages/43/bb/6c5e4a0f66ea0d2b2dd267e8dd05a0014eea56840b3c8595d40b0a5d1f91/wrapt-2.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a6b5984cd65dd639546f0eb4b8eacf1c31cb2fe9fb5c27bffe240987cdb2cf84", size = 84030, upload-time = "2026-07-28T06:05:37.714Z" }, + { url = "https://files.pythonhosted.org/packages/6a/eb/a1aedf03283bc9cbf8a1783995ddc54e3c5a86878f19002d2c428494f4c5/wrapt-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c88abcf53daef80e01a75c7530e727fa6e2c1888fe83e3dcdba4c96216a1f5c7", size = 84419, upload-time = "2026-07-28T06:05:39.131Z" }, + { url = "https://files.pythonhosted.org/packages/63/61/50d511c0dc5105563849e86daa3e16ac7feef699f79fb05af45ea70107d5/wrapt-2.3.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:85de890ff968196e92dd1ae73a9fb8970495e7650a457b1c9ef0ac3dd550bce2", size = 207171, upload-time = "2026-07-28T06:05:40.69Z" }, + { url = "https://files.pythonhosted.org/packages/3f/59/9b538cf7795217e810699d16bc88b96a830d9b5c403eb2ec2db6b5f2ae81/wrapt-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50f416b74d092bb9f41b424e90dd457f365f7ba4b11de62a23679769a21bd85c", size = 214329, upload-time = "2026-07-28T06:05:42.287Z" }, + { url = "https://files.pythonhosted.org/packages/b3/28/9935d62b1499e5c8b3d191e99ba4eb31ca237a0b699142011a837e9dc7ea/wrapt-2.3.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:39febbee6d77301d31da6996b152ce52452da7c7ef72aba10c2fa976dff9c295", size = 199079, upload-time = "2026-07-28T06:05:43.958Z" }, + { url = "https://files.pythonhosted.org/packages/2b/01/4446b80fa2ffa47a3449b250d004ba1c1937f07f64a179608fec735df866/wrapt-2.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:93513bec052c6cd987f9f580c3df068c8bc4ebae6543736be3ca7ec5959cafcd", size = 209992, upload-time = "2026-07-28T06:05:45.677Z" }, + { url = "https://files.pythonhosted.org/packages/d4/07/56f26c9f9979586a021e8148747004aba4498f49458c90b0502969b904e1/wrapt-2.3.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:729126e667da34d251b8ebf8a45ef0c5ddadc21542b3d6e1abf4259ece6508df", size = 196334, upload-time = "2026-07-28T06:05:47.608Z" }, + { url = "https://files.pythonhosted.org/packages/8b/41/6d7bcc895b0f28b2250e10908f060687b9165429dcd7f22ddb3d4c031b74/wrapt-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:626b69db2021aa01671ec7bbc9740e558522bd44c18cf2ce69bf3d666a014109", size = 202644, upload-time = "2026-07-28T06:05:49.183Z" }, + { url = "https://files.pythonhosted.org/packages/cd/25/7860927edba06b758b8852a6f02e832be715563c67a6795d94350bc81099/wrapt-2.3.0-cp314-cp314t-win32.whl", hash = "sha256:629d73378082c00a8173031f9fb30a3ac6abbc894a5bfdfae71fabc60642d501", size = 79685, upload-time = "2026-07-28T06:05:50.976Z" }, + { url = "https://files.pythonhosted.org/packages/c4/0f/270bafe92fde3b069a39bc01e39ee79340895b335640df861d43d2a51885/wrapt-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:42869085687f0aefd57c0f636c3f9354f8ffb321a8ba9cb52d19beb796e561c5", size = 83104, upload-time = "2026-07-28T06:05:52.405Z" }, + { url = "https://files.pythonhosted.org/packages/55/b3/af176d79a8515a8a720eccdad9a96f6e31a30abf2865430c8c42adf2fd13/wrapt-2.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b1e5aa486e269b00ed35e64771c7d0ab8096cfd2643405ca8cd60ebedc099a51", size = 81774, upload-time = "2026-07-28T06:05:53.902Z" }, + { url = "https://files.pythonhosted.org/packages/00/39/3daf9f47be208606586de4568ba6713db53ebc8fd7a575aea1fe57983b69/wrapt-2.3.0-py3-none-any.whl", hash = "sha256:d8c7ed08477429752b8c44991f40ad7838b18332a160698740a6bfbc10d998a2", size = 61866, upload-time = "2026-07-28T06:06:12.9Z" }, +] + [[package]] name = "yarl" version = "1.24.5"