From 5f619cf13747f5cc00db18f71d6954ff16a4c91b Mon Sep 17 00:00:00 2001 From: Henry Balen Date: Fri, 27 Feb 2026 10:34:23 -0500 Subject: [PATCH 1/3] Start of refactoring project to ease dev --- .gitignore | 193 +++++++++++++++++- .python-version | 1 + LICENSE | 21 ++ Montreal McGill.log => README.md | 0 .../McGill_metadata.json | 0 data_post_processing/Montreal McGill.log | 0 .../create_meta_tables.py | 0 .../data_validation.py | 0 .../db_connection.py | 0 .../get_metadata.py | 0 .../iso_mapping.py | 0 lmrlib.py => data_post_processing/lmrlib.py | 0 .../post-process_all.py | 0 .../setup_raw_data_table.py | 0 .../time_utils.py | 0 .../transcription_data_processing.py | 0 justfile | 0 pyproject.toml | 78 +++++++ 18 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 .python-version create mode 100644 LICENSE rename Montreal McGill.log => README.md (100%) rename McGill_metadata.json => data_post_processing/McGill_metadata.json (100%) create mode 100644 data_post_processing/Montreal McGill.log rename create_meta_tables.py => data_post_processing/create_meta_tables.py (100%) rename data_validation.py => data_post_processing/data_validation.py (100%) rename db_connection.py => data_post_processing/db_connection.py (100%) rename get_metadata.py => data_post_processing/get_metadata.py (100%) rename iso_mapping.py => data_post_processing/iso_mapping.py (100%) rename lmrlib.py => data_post_processing/lmrlib.py (100%) rename post-process_all.py => data_post_processing/post-process_all.py (100%) rename setup_raw_data_table.py => data_post_processing/setup_raw_data_table.py (100%) rename time_utils.py => data_post_processing/time_utils.py (100%) rename transcription_data_processing.py => data_post_processing/transcription_data_processing.py (100%) create mode 100644 justfile create mode 100644 pyproject.toml diff --git a/.gitignore b/.gitignore index 500678f..69c0a7f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,196 @@ __pycache__/ # Distribution / packaging .Python -myy-awjz-hfw .Rproj.user +myy-awjz-hfw +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# UV +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +#uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock +#poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +#pdm.lock +#pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +#pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore +# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, +# you could uncomment the following to ignore the entire vscode folder +# .vscode/ + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Cursor +# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to +# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data +# refer to https://docs.cursor.com/context/ignore-files +.cursorignore +.cursorindexingignore + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# local temp files +webhook_logs/ +instance/ + +# Ignore OS X artifacts +.DS_Store diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..3767b4b --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.14 \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8ba0046 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 open-data-rescue + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Montreal McGill.log b/README.md similarity index 100% rename from Montreal McGill.log rename to README.md diff --git a/McGill_metadata.json b/data_post_processing/McGill_metadata.json similarity index 100% rename from McGill_metadata.json rename to data_post_processing/McGill_metadata.json diff --git a/data_post_processing/Montreal McGill.log b/data_post_processing/Montreal McGill.log new file mode 100644 index 0000000..e69de29 diff --git a/create_meta_tables.py b/data_post_processing/create_meta_tables.py similarity index 100% rename from create_meta_tables.py rename to data_post_processing/create_meta_tables.py diff --git a/data_validation.py b/data_post_processing/data_validation.py similarity index 100% rename from data_validation.py rename to data_post_processing/data_validation.py diff --git a/db_connection.py b/data_post_processing/db_connection.py similarity index 100% rename from db_connection.py rename to data_post_processing/db_connection.py diff --git a/get_metadata.py b/data_post_processing/get_metadata.py similarity index 100% rename from get_metadata.py rename to data_post_processing/get_metadata.py diff --git a/iso_mapping.py b/data_post_processing/iso_mapping.py similarity index 100% rename from iso_mapping.py rename to data_post_processing/iso_mapping.py diff --git a/lmrlib.py b/data_post_processing/lmrlib.py similarity index 100% rename from lmrlib.py rename to data_post_processing/lmrlib.py diff --git a/post-process_all.py b/data_post_processing/post-process_all.py similarity index 100% rename from post-process_all.py rename to data_post_processing/post-process_all.py diff --git a/setup_raw_data_table.py b/data_post_processing/setup_raw_data_table.py similarity index 100% rename from setup_raw_data_table.py rename to data_post_processing/setup_raw_data_table.py diff --git a/time_utils.py b/data_post_processing/time_utils.py similarity index 100% rename from time_utils.py rename to data_post_processing/time_utils.py diff --git a/transcription_data_processing.py b/data_post_processing/transcription_data_processing.py similarity index 100% rename from transcription_data_processing.py rename to data_post_processing/transcription_data_processing.py diff --git a/justfile b/justfile new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d55c2d7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,78 @@ +[project] +name = "data-post-processing-v2" +version = "1.0.0" +description = "DRAW post processing code" +authors = [] +dependencies = [ + "pydantic>=2.10.0", + "pydantic-settings>=2.7.0", + "sqlalchemy>=2.0.0", +] + +requires-python = ">=3.14" +readme = "README.md" +license = { text = "MIT" } + +[dependency-groups] +dev = [ + "pytest>=8.3.0", + "pytest-asyncio>=0.25.0", + "pytest-mock>=3.14.0", + "ruff>=0.8.0", + "mypy>=1.13.0", + "coverage[toml]>=7.12.0", + "icecream>=2.1.8", + "pytest-clarity>=1.0.1", +] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] +filterwarnings = ["error", "ignore::DeprecationWarning"] +markers = [ + "templates: when referring to template code, bypass any template mocks" +] + +[tool.ruff] +line-length = 88 +target-version = "py314" +exclude = ["migrations"] + +[tool.ruff.lint] +ignore = [ + "E501", # line too long, handled by formatter +] + +[tool.ruff.lint.isort] +known-first-party = ["regfox_bridge"] + +[tool.mypy] +python_version = "3.14" + +[[tool.mypy.overrides]] +module = ["tests.*"] +disallow_untyped_defs = false + +[tool.coverage.run] +source = ["regfox_bridge"] +omit = ["*/tests/*", "*/__pycache__/*"] + +[tool.coverage.report] +exclude_lines = [ + "pragma: no cover", + "def __repr__", + "raise AssertionError", + "raise NotImplementedError", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.coverage.html] +directory = "htmlcov" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["data-post-processing"] From 6e209fc80a92bf386e91b2f44736edbed386cdd3 Mon Sep 17 00:00:00 2001 From: Henry Balen Date: Fri, 27 Feb 2026 10:38:40 -0500 Subject: [PATCH 2/3] remove non-project files --- .DS_Store | Bin 6148 -> 0 bytes .spyproject/config/codestyle.ini | 8 -------- .../config/defaults/defaults-codestyle-0.2.0.ini | 5 ----- .../config/defaults/defaults-encoding-0.2.0.ini | 3 --- .../config/defaults/defaults-vcs-0.2.0.ini | 4 ---- .../config/defaults/defaults-workspace-0.2.0.ini | 6 ------ .spyproject/config/encoding.ini | 6 ------ .spyproject/config/vcs.ini | 7 ------- .spyproject/config/workspace.ini | 12 ------------ 9 files changed, 51 deletions(-) delete mode 100644 .DS_Store delete mode 100644 .spyproject/config/codestyle.ini delete mode 100644 .spyproject/config/defaults/defaults-codestyle-0.2.0.ini delete mode 100644 .spyproject/config/defaults/defaults-encoding-0.2.0.ini delete mode 100644 .spyproject/config/defaults/defaults-vcs-0.2.0.ini delete mode 100644 .spyproject/config/defaults/defaults-workspace-0.2.0.ini delete mode 100644 .spyproject/config/encoding.ini delete mode 100644 .spyproject/config/vcs.ini delete mode 100644 .spyproject/config/workspace.ini diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 273b3e206b75894932182de356692c2eef339820..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKJ5Iwu5S=wCfqOQw7_3gfS+BBwrNBsbP+zk&rWe{nx$!flJya*@0@-dbnc#a!|~ydT%PX`)*1OV0$_J)3wJHS{mQOpG~0&`Ldlv1Zh3@7D?$C{TY=7LgA zP7fbWS9W?raej5oANzE2iJ-NnfGH3vuqKyH-v9Ti&;MbOU6}%=z`s(!)zWU-#x3dI yTDm#jYh(B!oQ?Bx!FdTfek(>UZ^c`1Z-~d-0VayMAR;jPBj9DQ#uWHf1-<~MLwWZA diff --git a/.spyproject/config/codestyle.ini b/.spyproject/config/codestyle.ini deleted file mode 100644 index 0f54b4c..0000000 --- a/.spyproject/config/codestyle.ini +++ /dev/null @@ -1,8 +0,0 @@ -[codestyle] -indentation = True -edge_line = True -edge_line_columns = 79 - -[main] -version = 0.2.0 - diff --git a/.spyproject/config/defaults/defaults-codestyle-0.2.0.ini b/.spyproject/config/defaults/defaults-codestyle-0.2.0.ini deleted file mode 100644 index 0b95e5c..0000000 --- a/.spyproject/config/defaults/defaults-codestyle-0.2.0.ini +++ /dev/null @@ -1,5 +0,0 @@ -[codestyle] -indentation = True -edge_line = True -edge_line_columns = 79 - diff --git a/.spyproject/config/defaults/defaults-encoding-0.2.0.ini b/.spyproject/config/defaults/defaults-encoding-0.2.0.ini deleted file mode 100644 index 0ce193c..0000000 --- a/.spyproject/config/defaults/defaults-encoding-0.2.0.ini +++ /dev/null @@ -1,3 +0,0 @@ -[encoding] -text_encoding = utf-8 - diff --git a/.spyproject/config/defaults/defaults-vcs-0.2.0.ini b/.spyproject/config/defaults/defaults-vcs-0.2.0.ini deleted file mode 100644 index ee25483..0000000 --- a/.spyproject/config/defaults/defaults-vcs-0.2.0.ini +++ /dev/null @@ -1,4 +0,0 @@ -[vcs] -use_version_control = False -version_control_system = - diff --git a/.spyproject/config/defaults/defaults-workspace-0.2.0.ini b/.spyproject/config/defaults/defaults-workspace-0.2.0.ini deleted file mode 100644 index 2a73ab7..0000000 --- a/.spyproject/config/defaults/defaults-workspace-0.2.0.ini +++ /dev/null @@ -1,6 +0,0 @@ -[workspace] -restore_data_on_startup = True -save_data_on_exit = True -save_history = True -save_non_project_files = False - diff --git a/.spyproject/config/encoding.ini b/.spyproject/config/encoding.ini deleted file mode 100644 index a17aced..0000000 --- a/.spyproject/config/encoding.ini +++ /dev/null @@ -1,6 +0,0 @@ -[encoding] -text_encoding = utf-8 - -[main] -version = 0.2.0 - diff --git a/.spyproject/config/vcs.ini b/.spyproject/config/vcs.ini deleted file mode 100644 index fd66eae..0000000 --- a/.spyproject/config/vcs.ini +++ /dev/null @@ -1,7 +0,0 @@ -[vcs] -use_version_control = False -version_control_system = - -[main] -version = 0.2.0 - diff --git a/.spyproject/config/workspace.ini b/.spyproject/config/workspace.ini deleted file mode 100644 index 4d9540a..0000000 --- a/.spyproject/config/workspace.ini +++ /dev/null @@ -1,12 +0,0 @@ -[workspace] -restore_data_on_startup = True -save_data_on_exit = True -save_history = True -save_non_project_files = False -project_type = 'empty-project-type' -recent_files = [] - -[main] -version = 0.2.0 -recent_files = [] - From 489acd762e597aecfb134f6809eeb89ef810e4d2 Mon Sep 17 00:00:00 2001 From: Henry Balen Date: Wed, 8 Apr 2026 21:17:29 -0400 Subject: [PATCH 3/3] Refactor phase 1: Clean the code Make sure params are typed Use proper logging Remove redundancy Reduce memory usage by working on one variable at a time --- README.md | 26 + alembic.ini | 149 ++ .../McGill_metadata.json | 2 +- data_post_processing/Montreal McGill.log | 0 data_post_processing/config.py | 30 + data_post_processing/create_meta_tables.py | 63 - data_post_processing/data_utils.py | 213 ++ data_post_processing/data_validation.py | 211 +- data_post_processing/database.py | 27 + data_post_processing/db_connection.py | 55 - data_post_processing/get_metadata.py | 29 - data_post_processing/iso_mapping.py | 322 +-- data_post_processing/lmrlib.py | 624 +++-- data_post_processing/logging.py | 67 + data_post_processing/models.py | 213 ++ data_post_processing/post-process_all.py | 1008 ++++++-- data_post_processing/setup_raw_data_table.py | 16 - data_post_processing/time_utils.py | 133 +- .../transcription_data_processing.py | 2304 +++++++++++++---- justfile | 45 + logs/debug.log | 39 + main.py | 189 ++ migrations/README | 1 + migrations/env.py | 85 + migrations/script.py.mako | 28 + .../180739ab4fa2_create_iso_results_table.py | 44 + .../6b85f9c3707d_create_metadata_global.py | 79 + .../b6539120d112_create_station_metadata.py | 55 + pyproject.toml | 22 +- uv.lock | 703 +++++ 30 files changed, 5253 insertions(+), 1529 deletions(-) create mode 100644 alembic.ini rename {data_post_processing => data}/McGill_metadata.json (87%) delete mode 100644 data_post_processing/Montreal McGill.log create mode 100644 data_post_processing/config.py delete mode 100644 data_post_processing/create_meta_tables.py create mode 100644 data_post_processing/data_utils.py create mode 100644 data_post_processing/database.py delete mode 100644 data_post_processing/db_connection.py delete mode 100644 data_post_processing/get_metadata.py create mode 100644 data_post_processing/logging.py create mode 100644 data_post_processing/models.py delete mode 100644 data_post_processing/setup_raw_data_table.py create mode 100644 logs/debug.log create mode 100755 main.py create mode 100644 migrations/README create mode 100644 migrations/env.py create mode 100644 migrations/script.py.mako create mode 100644 migrations/versions/180739ab4fa2_create_iso_results_table.py create mode 100644 migrations/versions/6b85f9c3707d_create_metadata_global.py create mode 100644 migrations/versions/b6539120d112_create_station_metadata.py create mode 100644 uv.lock diff --git a/README.md b/README.md index e69de29..b579542 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,26 @@ +# data-post-processing + +Data post processor for DRAW data + +## Quick Start + +### Prerequisites + +- Python 3.14+ +- [UV](https://docs.astral.sh/uv/) package manager +- [Just](https://github.com/casey/just) task runner (optional) + +### Installation + + +### Development + +uv run alembic revision -m "create test table" + +### Configuration + +### Testing + +## License + +See `./LICENSE` diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..08f1d9f --- /dev/null +++ b/alembic.ini @@ -0,0 +1,149 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/migrations + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s +# Or organize into date-based subdirectories (requires recursive_version_locations = true) +# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the tzdata library which can be installed by adding +# `alembic[tz]` to the pip requirements. +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL. This is consumed by the user-maintained env.py script only. +# other means of configuring database URLs may be customized within the env.py +# file. +sqlalchemy.url = driver://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module +# hooks = ruff +# ruff.type = module +# ruff.module = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Alternatively, use the exec runner to execute a binary found on your PATH +# hooks = ruff +# ruff.type = exec +# ruff.executable = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/data_post_processing/McGill_metadata.json b/data/McGill_metadata.json similarity index 87% rename from data_post_processing/McGill_metadata.json rename to data/McGill_metadata.json index bd2e166..8eeab06 100644 --- a/data_post_processing/McGill_metadata.json +++ b/data/McGill_metadata.json @@ -10,7 +10,7 @@ "alt":"60", "prefix":"MontrealMcGill", "utcOffset":5, - "timezone":"America/Montreal". + "timezone":"America/Montreal", "project":"DRAW", "organization":"DataRescueArchivesWeather", "link":"https://draw.geog.mcgill.ca/" diff --git a/data_post_processing/Montreal McGill.log b/data_post_processing/Montreal McGill.log deleted file mode 100644 index e69de29..0000000 diff --git a/data_post_processing/config.py b/data_post_processing/config.py new file mode 100644 index 0000000..761f688 --- /dev/null +++ b/data_post_processing/config.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +"""Application configuration.""" + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + """Application settings.""" + + # Database configuration + db_host: str = "localhost" + db_port: int = 3306 + db_database: str = "climate_data_rescue" + db_user: str + db_password: str + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + case_sensitive=False, + extra="ignore", + ) + + # + log_level: str = "DEBUG" + # log_format: str = "json" + log_format: str = "text" + + +settings = Settings() diff --git a/data_post_processing/create_meta_tables.py b/data_post_processing/create_meta_tables.py deleted file mode 100644 index 6f50be1..0000000 --- a/data_post_processing/create_meta_tables.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Wed Jan 14 10:42:40 2026 - -@author: vicky -""" -import mysql_indexes -import os -import pymysql -import sqlalchemy -from sqlalchemy import URL -import tables - -import db_connection as db -import setup_raw_data_table as setup - -#%% ###### connect to database #### - -# call db_connection -import mysql.connector - -mydb = mysql.connector.connect( - host="localhost", - user="yourusername", - password="yourpassword", - database="mydatabase" -) - -## check if table exists - -mycursor = mydb.cursor() -mycursor.execute("SHOW TABLES") - -## create any missing tables - -for x in mycursor: - if 'station_metadata' and 'organization' in x: - continue_flag = 0 - elif 'organization' in x: - continue_flag = 1 - elif 'station_metadata' in x: - continue_flag = 2 - -# get metadata from json file - - -def set_up_metadata_table(continue_flag): - if continue_flag == 1: - create_organiztion_table() - elif continue_flag == 2: - create_stationmetadata_table() - -def create_organiztion_table(): - - -def create_stationmetadata_table(): - - - tables.add__table() - - - diff --git a/data_post_processing/data_utils.py b/data_post_processing/data_utils.py new file mode 100644 index 0000000..a99708f --- /dev/null +++ b/data_post_processing/data_utils.py @@ -0,0 +1,213 @@ +"""Database utls.""" + +import array + +import pandas +import sqlalchemy as sa +from sqlalchemy.orm import Session + +from data_post_processing.database import db +from data_post_processing.models import ( + Annotation, + DataEntry, + Field, + MetaDataGlobal, + Page, + PageInfo, + StationMetaData, + Transcription, + User, +) + + +# +# Get the meta data for the project, a project also includes the +# +def get_project_meta_data() -> StationMetaData | None: + sess = Session(db) + md = sess.query(MetaDataGlobal).first() + + return md + + +# +# Get the meta data for the Stations +# +def get_station_meta_data() -> StationMetaData | None: + sess = Session(db) + # Get the first station + md = sess.query(StationMetaData).all() + + return md + + +# +# Get the Field meta data as a Panda DataFrame +# +def get_field_meta_panda(field_ids: array) -> pandas.DataFrame | None: + sql = sa.select(Field).where(Field.id.in_(field_ids)) + + df = pandas.read_sql_query(sql=sql, con=db) + + return df + + +# +# Get the count of data entries per user +# +def get_entries_per_user(): + sess = Session(db) + + res = ( + sess.query( + User.id, + User.display_name, + sa.func.count(DataEntry.id).label("count"), + ) + .join(User, User.id == DataEntry.user_id) + .group_by(DataEntry.user_id) + .all() + ) + + return res + + +# +def get_station_field_ids(station: str, multiple: bool = False): + sql = sa.select(DataEntry.field_id).order_by(DataEntry.field_id.asc()) + + if multiple: + sql = ( + sql.join( + Annotation, + Annotation.id == DataEntry.annotation_id, + ) + .join( + Page, + Page.id == Annotation.page_id, + ) + .join(PageInfo, PageInfo.page_id == Page.id) + .where(PageInfo.location == station) + ) + + sql = sql.distinct() + + # sess = Session(db) + # res = sess.execute(sql) + + # return res + + df = pandas.read_sql_query(sql=sql, con=db) + + return df + + +# +# +# +def get_station_field_types(station: str, multiple: bool = False): + sql = ( + sa.select(Field.odr_type) + .join( + DataEntry, + DataEntry.field_id == Field.id, + ) + .filter(Field.odr_type != None, Field.odr_type != "") + .order_by(Field.odr_type.asc()) + ) + + if multiple: + sql = ( + sql.join( + Annotation, + Annotation.id == DataEntry.annotation_id, + ) + .join( + Page, + Page.id == Annotation.page_id, + ) + .join(PageInfo, PageInfo.page_id == Page.id) + .where(PageInfo.location == station) + ) + + sql = sql.distinct() + + # sess = Session(db) + # res = sess.execute(sql) + + # return res + + df = pandas.read_sql_query(sql=sql, con=db) + + return df + + +# +# Get the data for the Station for the given field +# order by the observation and updated at dates +# +# TODO: if there are multiple transcriptions for the same page take the newest one +# +def get_station_data_panda( + station: str, odr_type: str, field_id: int, multiple: bool = False +) -> pandas.DataFrame | None: + + sql = ( + sa.select( + sa.func.concat(station).label("station"), + Annotation.observation_date.label("observation_date"), + DataEntry.value, # lower + Field.odr_type, + Field.measurement_unit_original, + Field.field_key, + DataEntry.field_id, + DataEntry.id, + DataEntry.annotation_id, + Annotation.transcription_id, + DataEntry.page_id, + Page.image_file_name, + Transcription.user_id, + Transcription.created_at.label("created_at"), + Transcription.updated_at.label("updated_at"), + ) + .join( + DataEntry, + DataEntry.annotation_id == Annotation.id, + ) + .join( + Page, + Page.id == Annotation.page_id, + ) + .join( + Field, + Field.id == DataEntry.field_id, + ) + .join( + Transcription, + Transcription.id == Annotation.transcription_id, + ) + .filter(Field.odr_type == odr_type, DataEntry.field_id == field_id) + .filter( + # Exclude values that are "empty" or just have "-" + sa.func.lower(DataEntry.value) != "empty", + DataEntry.value != "-", + ) + .order_by(Annotation.observation_date.asc()) + .order_by(Annotation.updated_at.desc()) + ) + + # if we have multiple stations then we select just the pages + # for the desired station + if multiple: + sql = sql.join(PageInfo, PageInfo.page_id == Page.id).where( + PageInfo.location == station + ) + + df = pandas.read_sql_query(sql=sql, con=db) + + return df + + +def store_panda(pd: pandas.DataFrame, name: str): + # iso_list.to_sql("odr_iso_data_iso", conn, None, "append", chunksize=1000) + pd.to_sql(name=name, con=db, if_exists="append", index=False, chunksize=1000) diff --git a/data_post_processing/data_validation.py b/data_post_processing/data_validation.py index 671cb02..b0168d6 100644 --- a/data_post_processing/data_validation.py +++ b/data_post_processing/data_validation.py @@ -15,115 +15,178 @@ import pymysql -plt.rcParams['backend'] = "MacOSX" +plt.rcParams["backend"] = "MacOSX" + +# sefField="p" +# sefCfg="config_Kingston.json" +sampleSize = 100 -#sefField="p" -#sefCfg="config_Kingston.json" -sampleSize=100 # Load SEF File into a dataframe def parseSef(cfg): # Find SEF file and load it into a dataframe - sef_pattern=cfg["site"]["source"]+"_"+cfg["site"]["org"]+"_"+cfg["site"]["name"]+"_*-"+args.f[0]+".tsv" + sef_pattern = ( + cfg["site"]["source"] + + "_" + + cfg["site"]["org"] + + "_" + + cfg["site"]["name"] + + "_*-" + + args.f[0] + + ".tsv" + ) files = glob.glob(sef_pattern) if files: - return pd.read_csv(files[0],sep='\t',header=12) + return pd.read_csv(files[0], sep="\t", header=12) return None + # Plotting outliers -def plotOutliers(outliers,ans,x,y,std_count,standard_deviation,outlierFile): - ans_max=ans+std_count*standard_deviation - ans_min=ans-std_count*standard_deviation - fig, ax =plt.subplots(1, figsize=(20, 8)) +def plotOutliers(outliers, ans, x, y, std_count, standard_deviation, outlierFile): + ans_max = ans + std_count * standard_deviation + ans_min = ans - std_count * standard_deviation + fig, ax = plt.subplots(1, figsize=(20, 8)) fig.autofmt_xdate() - ax.plot( x, y, '.', color='black', label="data") - ax.plot(x, ans, '--', color="blue", label="rolling mean") - ax.plot(x,ans_min, '-', color='green', label=str(std_count)+" * sigma") - ax.plot(x,ans_max, '-', color='green') - ax.plot(outliers.observation_date, outliers.Value, 'o',color='red', label='Outliers') - ax.set_title(" Field: "+args.f[0]) + ax.plot(x, y, ".", color="black", label="data") + ax.plot(x, ans, "--", color="blue", label="rolling mean") + ax.plot(x, ans_min, "-", color="green", label=str(std_count) + " * sigma") + ax.plot(x, ans_max, "-", color="green") + ax.plot( + outliers.observation_date, outliers.Value, "o", color="red", label="Outliers" + ) + ax.set_title(" Field: " + args.f[0]) ax.legend() - #plt.show() - figname=outlierFile.name[:-4]+"_"+x.array[0].strftime("%Y-%m-%d")+"_"+x.array[-1].strftime("%Y-%m-%d")+".jpg" + # plt.show() + figname = ( + outlierFile.name[:-4] + + "_" + + x.array[0].strftime("%Y-%m-%d") + + "_" + + x.array[-1].strftime("%Y-%m-%d") + + ".jpg" + ) fig.savefig(figname) + # Print link to outlier fixing -def saveOutliers(outliers, outlierFile, myCursor,cfg): - url="https://eccc.opendatarescue.org/en/transcriptions/" - for index,outlier in outliers.iterrows(): - orig_value=outlier.Meta.split()[0][5:] - sql_query=" SELECT a.transcription_id from data_entries de join annotations a on a.id=de.annotation_id where de.value='"+ orig_value +\ - "' and a.observation_date LIKE '"+outlier.observation_date.strftime("%Y-%m-%d %")+"' "+\ - " and a.page_id in (select id from pages where title like '"+cfg["site"]["prefix"]+"%') " - #print(sql_query) +def saveOutliers(outliers, outlierFile, myCursor, cfg): + url = "https://eccc.opendatarescue.org/en/transcriptions/" + for index, outlier in outliers.iterrows(): + orig_value = outlier.Meta.split()[0][5:] + sql_query = ( + " SELECT a.transcription_id from data_entries de join annotations a on a.id=de.annotation_id where de.value='" + + orig_value + + "' and a.observation_date LIKE '" + + outlier.observation_date.strftime("%Y-%m-%d %") + + "' " + + " and a.page_id in (select id from pages where title like '" + + cfg["site"]["prefix"] + + "%') " + ) + # print(sql_query) myCursor.execute(sql_query) - results=myCursor.fetchall() + results = myCursor.fetchall() for result in results: if result is not None: - trans=result[0] - outlierFile.write(url+str(trans)+"/edit,"+orig_value+","+str(outlier.observation_date)+"\n") + trans = result[0] + outlierFile.write( + url + + str(trans) + + "/edit," + + orig_value + + "," + + str(outlier.observation_date) + + "\n" + ) # Identify outliers -def findOutliers(df,outlierFile,myCursor,cfg): - list_partial=[] - obs_date=None - days_gap_allowed=5 - std_count=5 - for index,row in df.iterrows(): - new_observation_date=row['observation_date'] - #print(new_observation_date) +def findOutliers(df, outlierFile, myCursor, cfg): + list_partial = [] + obs_date = None + days_gap_allowed = 5 + std_count = 5 + for index, row in df.iterrows(): + new_observation_date = row["observation_date"] + # print(new_observation_date) if row.Value == -999 or row.Value == -888: pass - elif (obs_date==None or (new_observation_date-obs_date).days < days_gap_allowed) and len(list_partial)<1000: + elif ( + obs_date == None + or (new_observation_date - obs_date).days < days_gap_allowed + ) and len(list_partial) < 1000: list_partial.append(row) - obs_date=new_observation_date - print(".",end='') + obs_date = new_observation_date + print(".", end="") else: - print("\nEvaluating list_partial because gap is " + str((new_observation_date-obs_date).days)+" and size is "+str(len(list_partial))) - df_partial=pd.DataFrame(list_partial) - x=df_partial.observation_date - y=df_partial.Value - if y.size>6: - ans=y.rolling(5,center=True).mean() - delta=(y-ans).abs() - standard_deviation=delta.std() - outliers=df[df.index.isin(delta[delta.gt(std_count*standard_deviation)].index)] - #print(outliers) - if outliers.size >0: - plotOutliers(outliers,ans,x,y,std_count,standard_deviation,outlierFile) - saveOutliers(outliers, outlierFile, myCursor,cfg) - - list_partial=[] - obs_date=new_observation_date + print( + "\nEvaluating list_partial because gap is " + + str((new_observation_date - obs_date).days) + + " and size is " + + str(len(list_partial)) + ) + df_partial = pd.DataFrame(list_partial) + x = df_partial.observation_date + y = df_partial.Value + if y.size > 6: + ans = y.rolling(5, center=True).mean() + delta = (y - ans).abs() + standard_deviation = delta.std() + outliers = df[ + df.index.isin(delta[delta.gt(std_count * standard_deviation)].index) + ] + # print(outliers) + if outliers.size > 0: + plotOutliers( + outliers, ans, x, y, std_count, standard_deviation, outlierFile + ) + saveOutliers(outliers, outlierFile, myCursor, cfg) + + list_partial = [] + obs_date = new_observation_date list_partial.append(row) - -parser = argparse.ArgumentParser(description='Validates SEF data.') -parser.add_argument('config', metavar='config', type=str, nargs=1, - help='json config file for each station') -parser.add_argument('-f', metavar='field', type=str, nargs=1, - help='SEF field') +parser = argparse.ArgumentParser(description="Validates SEF data.") +parser.add_argument( + "config", + metavar="config", + type=str, + nargs=1, + help="json config file for each station", +) +parser.add_argument("-f", metavar="field", type=str, nargs=1, help="SEF field") args = parser.parse_args() # open config file with open(args.config[0]) as json_data_file: -#json_data_file = sefCfg - cfg = json.load(json_data_file) # cfg is the json config file + # json_data_file = sefCfg + cfg = json.load(json_data_file) # cfg is the json config file -mycursor=db.connect(cfg) +mycursor = db.connect(cfg) -df=parseSef(cfg) +df = parseSef(cfg) if df is None: print("No SEF file has been found.") else: - #Add an observation date column that concatenates the Year Month Day Hour Minute fields for easier manipulation - df['observation_date']= pd.to_datetime(df[['Day','Month','Year', 'Hour', 'Minute']] - .astype(str).apply(' '.join, 1), format='%d %m %Y %H %M') - - outlierFile = open(cfg["site"]["source"]+"_"+cfg["site"]["org"]+"_"+cfg["site"]["name"]+"-"+args.f[0]+"-outliers.csv","w",buffering=1) - - - findOutliers(df,outlierFile,mycursor,cfg) + # Add an observation date column that concatenates the Year Month Day Hour Minute fields for easier manipulation + df["observation_date"] = pd.to_datetime( + df[["Day", "Month", "Year", "Hour", "Minute"]].astype(str).apply(" ".join, 1), + format="%d %m %Y %H %M", + ) + + outlierFile = open( + cfg["site"]["source"] + + "_" + + cfg["site"]["org"] + + "_" + + cfg["site"]["name"] + + "-" + + args.f[0] + + "-outliers.csv", + "w", + buffering=1, + ) + + findOutliers(df, outlierFile, mycursor, cfg) outlierFile.close() diff --git a/data_post_processing/database.py b/data_post_processing/database.py new file mode 100644 index 0000000..795e72b --- /dev/null +++ b/data_post_processing/database.py @@ -0,0 +1,27 @@ +"""Database initialization.""" + +# from sqlalchemy import SQLAlchemy +from sqlalchemy import create_engine + +# from sqlalchemy import URL +from data_post_processing.config import settings + + +def db_url(): + return "mysql+pymysql://{0}:{1}@{2}:{3}/{4}".format( + settings.db_user, + settings.db_password, + settings.db_host, + settings.db_port, + settings.db_database, + ) + + +def connect(): + engine = create_engine(db_url()) + + return engine + # return (mydb, mydb.cursor(), engine) + + +db = connect() diff --git a/data_post_processing/db_connection.py b/data_post_processing/db_connection.py deleted file mode 100644 index 72c6ad1..0000000 --- a/data_post_processing/db_connection.py +++ /dev/null @@ -1,55 +0,0 @@ -# -*- coding: utf-8 -*- -import pymysql -import sqlalchemy -from sqlalchemy import URL -import os - - -def connect(): - - ## ECCC server database - # dbHost = os.getenv("ECCC_db_host") - # dbPort = int(os.getenv("ECCC_db_port")) - # dbUser = os.getenv("ECCC_db_user") - # dbPasswd = os.getenv("ECCC_db_passwd") - # dbDB = os.getenv("ECCC_db_database") - - # localhost - # dbHost = os.getenv("localhost") - # dbPort = 3306 - # dbUser = os.getenv("db_root") - # dbPasswd = os.getenv("db_password_root") - # dbDB = "Canada_wx" - - #dbHost = "54.39.21.6" - #dbPort = 3306 - #dbUser = "mysql" - #dbPasswd = "3589a8dea043af14" - #dbDB = "eccc_db" - - dbHost = os.getenv("localhost") - dbPort = 3306 - dbUser = os.getenv("db_root") - dbPasswd = os.getenv("db_password_root") - dbDB = "DRAW" - - mydb = pymysql.connect( - host=dbHost, - port=dbPort, - user=dbUser, - password=dbPasswd, - database=dbDB - ) - - url = URL.create( - "mysql+mysqlconnector", - username=dbUser, - host=dbHost, - port=dbPort, - password=dbPasswd, - database=dbDB - ) - - engine = sqlalchemy.create_engine(url) - - return (mydb, mydb.cursor(), engine) diff --git a/data_post_processing/get_metadata.py b/data_post_processing/get_metadata.py deleted file mode 100644 index fb41922..0000000 --- a/data_post_processing/get_metadata.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Created on Wed Jul 24 10:11:53 2024 - -@author: vicky -""" - - -def get_metadata(station, mydb, mycursor): - - query = ("select mg.source, mg.project, ms.stationName, mg.country, ms.latitude, ms.longitude, ms.elevation, mg.link,ms.timeZone, ms.UTCoffset from metadata_global mgjoin MetadataStations ms on ms.project = mg.project where stationName like '{station}';") - - mycursor.execute(query) - result = mycursor.fetchall() - source = result[0] - project = result[1] - stationName = result[2] - country = result[3] - latitude = result[4] - longitude = result[5] - elevation = result[6] - link = result[7] - timeZone = result[8] - UTCoffset = result[9] - - stationID = stationName+country - - return (source, project, stationName, stationID, latitude, longitude, elevation, link, timeZone, UTCoffset) diff --git a/data_post_processing/iso_mapping.py b/data_post_processing/iso_mapping.py index c6594fe..78e3248 100644 --- a/data_post_processing/iso_mapping.py +++ b/data_post_processing/iso_mapping.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- + def convertUpperCloud(cloud): value = cloud.lower() if "cumulo" in value and "nimbus" in value: @@ -20,10 +21,23 @@ def convertUpperCloud(cloud): return "Ch9" elif "scud" in value or "passing" in value: return "Cm4" - elif ("obscured" in value or "hidden" in value or "fog" in value or "haz" in value or "smoke" in value - or "mist" in value or "overcast" in value): + elif ( + "obscured" in value + or "hidden" in value + or "fog" in value + or "haz" in value + or "smoke" in value + or "mist" in value + or "overcast" in value + ): return "Ch/" - elif ("-" in value or "clear" in value or "none apparent" in value or "0" in value or "imp" in value): + elif ( + "-" in value + or "clear" in value + or "none apparent" in value + or "0" in value + or "imp" in value + ): return "Ch0" return "-999" @@ -46,10 +60,24 @@ def convertLowerCloud(cloud): return "Cl2" elif "scud" in value or "passing" in value: return "Cl5" - elif ("obscured" in value or "hidden" in value or "fog" in value or "haz" in value or "smoke" in value - or "mist" in value or "vapour" in value or "overcast" in value): + elif ( + "obscured" in value + or "hidden" in value + or "fog" in value + or "haz" in value + or "smoke" in value + or "mist" in value + or "vapour" in value + or "overcast" in value + ): return "Cl/" - elif ("-" in value or "clear" in value or "none apparent" in value or "0" in value or "imp" in value): + elif ( + "-" in value + or "clear" in value + or "none apparent" in value + or "0" in value + or "imp" in value + ): return "Cl0" return "-999" @@ -69,14 +97,14 @@ def convertBeauforttext(wf): else: return 26.8 elif "hurricane" in value: - return 35. + return 35.0 elif "calm" in value: return 0 elif "air" in value: return 1 elif "light" in value: return 2.6 - elif ("gentle" in value or "modest" in value): + elif "gentle" in value or "modest" in value: return 4.6 elif "moderate" in value: return 6.7 @@ -117,7 +145,7 @@ def convertBeaufort(wf): elif value == 12: return 35 else: - return (-999) + return -999 def convertSmithsonian(wf): @@ -127,25 +155,25 @@ def convertSmithsonian(wf): elif value == 1: return 1 # 2mph /2.237 elif value == 2: - return (float(4/2.237)) + return float(4 / 2.237) elif value == 3: - return (float(12/2.237)) + return float(12 / 2.237) elif value == 4: - return (float(25/2.237)) + return float(25 / 2.237) elif value == 5: - return (float(35/2.237)) + return float(35 / 2.237) elif value == 6: - return (float(45/2.237)) + return float(45 / 2.237) elif value == 7: - return (float(60/2.237)) + return float(60 / 2.237) elif value == 8: - return (float(75/2.237)) + return float(75 / 2.237) elif value == 9: - return (float(90/2.237)) + return float(90 / 2.237) elif value == 10: - return (float(100/2.237)) + return float(100 / 2.237) else: - return (-999) + return -999 # map to convert directions to iso - key should be lower case @@ -182,7 +210,8 @@ def convertSmithsonian(wf): "northwest by north": 326.25, "north-north-west": 337.5, "north by west": 348.75, - "hidden": -999} + "hidden": -999, +} # map to convert directions to abbreviation - key should be lower case @@ -221,139 +250,144 @@ def convertSmithsonian(wf): "north by west": "NbyW", "calm": "C", "0": "C", - "hidden": "N/A" + "hidden": "N/A", } -convert_clouds_upper = {"cirrus": "Ch6", - "cirrostratus": "Ch8", - "cirro-stratus": "Ch8", - "cirro-cumulus": "Cm8", - "nimbostratus": "Cm2", - "cumulonimbus": "Cl9", - "stratus": "Ch8", - "nimbus": "Cm2", - "cumulostratus": "Cm7", - "cumulonimbus": "Cl9", - "cumulus": "Ch9", - "passing": "Cm4", - "scud": "Cm4", - "obscured": "Ch/", - "hidden": "Ch/", - "mist": "Ch/", - "fog": "Ch/", - "haze": "Ch/", - "hazy": "Ch/", - "smoke": "Ch/", - "?": "Ch/", - "clear": "Ch0", - "none apparent": "C10", - "0": "Ch0" - } +convert_clouds_upper = { + "cirrus": "Ch6", + "cirrostratus": "Ch8", + "cirro-stratus": "Ch8", + "cirro-cumulus": "Cm8", + "nimbostratus": "Cm2", + "cumulonimbus": "Cl9", + "stratus": "Ch8", + "nimbus": "Cm2", + "cumulostratus": "Cm7", + "cumulonimbus": "Cl9", + "cumulus": "Ch9", + "passing": "Cm4", + "scud": "Cm4", + "obscured": "Ch/", + "hidden": "Ch/", + "mist": "Ch/", + "fog": "Ch/", + "haze": "Ch/", + "hazy": "Ch/", + "smoke": "Ch/", + "?": "Ch/", + "clear": "Ch0", + "none apparent": "C10", + "0": "Ch0", +} -convert_cloud_lower = {"cirrus": "Ch6", - "stratus cirro-stratus": "Ch8 Cl6", - "stratus cirrus": "Ch8 Cl6", - "stratus cumulonimbus": "Ch6 Ch8 Cl4", - "cirro-stratus": "Ch8", - "cirrostratus": "Ch8", - "cirro-cumulus": "Ch9", - "cirro-cumulus illegible": "Ch9", - "cirrocumulus": "Ch9", - "nimbostratus": "Cm2", - "stratus": "Cl6", - "nimbus": "Cl7", - "cumulo-stratus": "Cl4", - "cumulostratus": "Cl4", - "cumulonimbus": "Cl9", - "cumulus cirro-stratus": "C12, Ch8", - "cumulus": "Cl2", - "passing": "Cl2", - "scud": "Cl5", - "hidden": "Cl/", - "obscured": "Cl/", - "mist": "Cl/", - "fog": "Cl/", - "haze": "Cl/", - "hazy": "Cl/", - "smoke": "Cl/", - "overcast": "Cl7", - "?": "Cl/", - "0": "Cl0", - "clear": "Cl0", - "none apparent": "C10", - "imp": "Cl0" - } +convert_cloud_lower = { + "cirrus": "Ch6", + "stratus cirro-stratus": "Ch8 Cl6", + "stratus cirrus": "Ch8 Cl6", + "stratus cumulonimbus": "Ch6 Ch8 Cl4", + "cirro-stratus": "Ch8", + "cirrostratus": "Ch8", + "cirro-cumulus": "Ch9", + "cirro-cumulus illegible": "Ch9", + "cirrocumulus": "Ch9", + "nimbostratus": "Cm2", + "stratus": "Cl6", + "nimbus": "Cl7", + "cumulo-stratus": "Cl4", + "cumulostratus": "Cl4", + "cumulonimbus": "Cl9", + "cumulus cirro-stratus": "C12, Ch8", + "cumulus": "Cl2", + "passing": "Cl2", + "scud": "Cl5", + "hidden": "Cl/", + "obscured": "Cl/", + "mist": "Cl/", + "fog": "Cl/", + "haze": "Cl/", + "hazy": "Cl/", + "smoke": "Cl/", + "overcast": "Cl7", + "?": "Cl/", + "0": "Cl0", + "clear": "Cl0", + "none apparent": "C10", + "imp": "Cl0", +} -abbr_cloud = {"cirrus": "Ci", - "cirro-stratus": "CiSt", - "cirro-cumulus": "CiCu", - "stratus": "St", - "nimbus": "Ni", - "nimbostratus": "NiSt", - "cumulo-stratus": "CuSt", - "cirrostratus": "CiSt", - "cirrocumulus": "CiCu", - "cirro-cumulus": "CiCu", - "cumulonimbus": "CuNi", - "cumulus": "Cu", - "overcast": "NiSt", - "passing": "P", - "scud": "Sc", - "hidden": "Hi", - "obscured": "Hi", - "mist": "Ms", - "fog": "Fg", - "haze": "Hz", - "hazy": "Hz", - "smoke": "Sm", - "clear": "Cl", - "0": "Cl" - } +abbr_cloud = { + "cirrus": "Ci", + "cirro-stratus": "CiSt", + "cirro-cumulus": "CiCu", + "stratus": "St", + "nimbus": "Ni", + "nimbostratus": "NiSt", + "cumulo-stratus": "CuSt", + "cirrostratus": "CiSt", + "cirrocumulus": "CiCu", + "cirro-cumulus": "CiCu", + "cumulonimbus": "CuNi", + "cumulus": "Cu", + "overcast": "NiSt", + "passing": "P", + "scud": "Sc", + "hidden": "Hi", + "obscured": "Hi", + "mist": "Ms", + "fog": "Fg", + "haze": "Hz", + "hazy": "Hz", + "smoke": "Sm", + "clear": "Cl", + "0": "Cl", +} # weather: R =rain, D =drizzle, F = foggy, Gloomy = cloudy -weather = {"dull": "cloudy", - "gloomy": "cloudy", - "fair": "clear", - "r": "rain", - "d": "drizzle", - "f": "fog", - "s": "snow", - "sh": "shower", - "sn sh": "snow shower" - } +weather = { + "dull": "cloudy", + "gloomy": "cloudy", + "fair": "clear", + "r": "rain", + "d": "drizzle", + "f": "fog", + "s": "snow", + "sh": "shower", + "sn sh": "snow shower", +} # define unit translation -map_unit = {"F": "C", - "inHg": "hPa", - "in": "mm", - "okta": "okta", - "dir": "deg", - "direction": "deg", - "lct": "cloudatlas", - "uct": "cloudatlas", - "Sm": "mps", - "Bf": "mps", - "p": "%", - "%": "%", - "cloudtext": "none", - "mno": "manual observation", - "cloudvel": "0-10", - "hcv": "0-10", - "cv": "0-10", - "hr": "hour", - "tenths": "tenths", - "tenth": "tenths", - "abb": "1-10", - "hh:mm": "none", - "text": "none", - "du": "du", - "RE_scale": "Beaufort", - "mph": "mps", - "lbsft": "mps", - "oz": "scale", - "scale": "scale" - } +map_unit = { + "F": "C", + "inHg": "hPa", + "in": "mm", + "okta": "okta", + "dir": "deg", + "direction": "deg", + "lct": "cloudatlas", + "uct": "cloudatlas", + "Sm": "mps", + "Bf": "mps", + "p": "%", + "%": "%", + "cloudtext": "none", + "mno": "manual observation", + "cloudvel": "0-10", + "hcv": "0-10", + "cv": "0-10", + "hr": "hour", + "tenths": "tenths", + "tenth": "tenths", + "abb": "1-10", + "hh:mm": "none", + "text": "none", + "du": "du", + "RE_scale": "Beaufort", + "mph": "mps", + "lbsft": "mps", + "oz": "scale", + "scale": "scale", +} diff --git a/data_post_processing/lmrlib.py b/data_post_processing/lmrlib.py index 1452b6a..75537be 100644 --- a/data_post_processing/lmrlib.py +++ b/data_post_processing/lmrlib.py @@ -1,20 +1,20 @@ """ -version 0.2 - - 4/9/2019 corrected time_date2julianday +version 0.2 + - 4/9/2019 corrected time_date2julianday ============================================================================= - Comprehensive Ocean-Atmosphere Data Set (COADS): Python Code + Comprehensive Ocean-Atmosphere Data Set (COADS): Python Code Python translation of Scott Woodruff and Sandy Lubker' lmrlib Fortran library See http://icoads.noaa.gov/software/lmrlib - Comments are preserved from the original. + Comments are preserved from the original. - Function: Tools to assist conversions into LMR6 + Function: Tools to assist conversions into LMR6 ============================================================================= Functionality: This is a library of tools to assist conversions from other formats into LMR6, whose functions are individually described in the comments - at the beginning of each function. - + at the beginning of each function. + Contents: Following are the routines included, and their broader groupings: barometric conversions: {baro_mm2mb} millimeters Hg to millibars @@ -61,18 +61,20 @@ Machine dependencies: None known. ----------------------------------------------------------------------- """ + import sys import math from datetime import date, timedelta import calendar import numpy as np -#============================================================================= -# WARNING: Code beyond this point should not require any modification. -#============================================================================= -#--Barometric conversions----------------------------------------------------- -#============================================================================= -#-----Convert barometric pressure in (standard) millimeters of mercury (mm) + +# ============================================================================= +# WARNING: Code beyond this point should not require any modification. +# ============================================================================= +# --Barometric conversions----------------------------------------------------- +# ============================================================================= +# -----Convert barometric pressure in (standard) millimeters of mercury (mm) # to millibars (hPa), e.g., baro_mm2mb(760.) = 1013.25 (one atmosphere) # (List, 1966, p. 13). # References: @@ -80,13 +82,14 @@ # Smithsonian Institution, Washington, DC, 527 pp. # WMO (World Meteorological Organization), 1966: International # Meteorological Tables, WMO-No.188.TP.94. -#-----factor from List (1966), p. 13 and Table 11; also in WMO (1966). +# -----factor from List (1966), p. 13 and Table 11; also in WMO (1966). def baro_mm2mb(mm): baro_mm2mb = mm * 1.333224 return baro_mm2mb -#============================================================================= -#-----Convert barometric pressure in millibars (hPa; mb) to (standard) + +# ============================================================================= +# -----Convert barometric pressure in millibars (hPa; mb) to (standard) # millimeters of mercury. Numerical inverse of {baro_mm2mb} (see for # background). Note: This method yields better numerical agreement # in cross-testing against that routine than the factor 0.750062. @@ -94,8 +97,9 @@ def baro_mb2mm(mb): baro_mb2mm = mb / 1.333224 return baro_mb2mm -#============================================================================= -#-----Convert barometric pressure in (standard) inches (English) of + +# ============================================================================= +# -----Convert barometric pressure in (standard) inches (English) of # mercury (in) to millibars (hPa), e.g., baro_Eng_in2mb(29.9213) = 1013.25 # (one atmosphere) (List, 1966, p. 13). # References: @@ -103,8 +107,8 @@ def baro_mb2mm(mb): # Smithsonian Institution, Washington, DC, 527 pp. # WMO (World Meteorological Organization), 1966: International # Meteorological Tables, WMO-No.188.TP.94. -#-----factor from List (1966), Table 9. Note: a slightly different factor -# 33.8639 appears also on p. 13 of List (1966), and in WMO (1966). Tests +# -----factor from List (1966), Table 9. Note: a slightly different factor +# 33.8639 appears also on p. 13 of List (1966), and in WMO (1966). Tests # (32-bit Sun f77) over a wide range of pressure values (25.69"-31.73", # approximately equivalent to ~870-1074.6 mb) indicated that the choice # of constant made no numeric difference when data were converted to mb @@ -115,71 +119,82 @@ def baro_Eng_in2mb(ei): baro_Eng_in2mb = ei * 33.86389 return baro_Eng_in2mb -#============================================================================= -#-----Convert barometric pressure in millibars (hPa; mb) to (standard) + +# ============================================================================= +# -----Convert barometric pressure in millibars (hPa; mb) to (standard) # inches (English) of mercury. Numerical inverse of {baro_Eng_in2mb} (see for # background). Note: This method yields better numerical agreement # in cross-testing against that routine than the factor 0.0295300. def baro_mb2Eng_in(mb): baro_mb2Eng_in = mb / 33.86389 return baro_mb2Eng_in -#============================================================================= -#-----Convert barometric pressure in inches (French) of mercury (fi) to + + +# ============================================================================= +# -----Convert barometric pressure in inches (French) of mercury (fi) to # millibars (hPa). Paris, instead of French, inches are referred # to in Lamb (1986), but these appear to be equivalent units. Note: # data in lines (twelve lines per inch) or in inches and lines need -# to be converted to inches (plus any decimal fraction). +# to be converted to inches (plus any decimal fraction). # References: # IMC (International Meteorological Committee), 1890: International # Meteorological Tables, published in Conformity with a Resolution # of the Congress of Rome, 1879. Gauthier-Villars et Fils, Paris. # Lamb, H.H., 1986: Ancient units used by the pioneers of meteorological # instruments. Weather, 41, 230-233. -#-----factor for conversion of French inches to mm (IMC, 1890, p. B.2); +# -----factor for conversion of French inches to mm (IMC, 1890, p. B.2); # mm are then converted to mb via {baro_mm2mb} def baro_Fr_in2mb(fi): baro_Fr_in2mb = baro_mm2mb(fi * 27.069953) return baro_Fr_in2mb -#============================================================================= -#-----Convert barometric pressure in millibars (hPa; mb) to inches (French) + +# ============================================================================= +# -----Convert barometric pressure in millibars (hPa; mb) to inches (French) # of mercury. Numerical inverse of {baro_Fr_in2mb} (see for background). def baro_mb2Fr_in(mb): baro_mb2Fr_in = baro_mb2mm(mb) / 27.069953 return baro_mb2Fr_in -#============================================================================= -#-----Correction value of barometric pressure (in mm or mb; standard + +# ============================================================================= +# -----Correction value of barometric pressure (in mm or mb; standard # temperature of scale 0C) (bp) for temperature in Celsius (tc) # Reference: # List, R.J., 1966: Smithsonian Meteorological Tables. # Smithsonian Institution, Washington, DC, 527 pp. -#-----constants m and l from List (1966), p. 136. -def baro_tempC_correction(bp,tc): - m=0.0001818 - l=0.0000184 - baro_tempC_correction = -bp * ( ((m-l)*tc) / (1.+(m*tc)) ) +# -----constants m and l from List (1966), p. 136. +def baro_tempC_correction(bp, tc): + m = 0.0001818 + l = 0.0000184 + baro_tempC_correction = -bp * (((m - l) * tc) / (1.0 + (m * tc))) return baro_tempC_correction -#============================================================================= -#-----Correction value of barometric pressure (in inches; standard + + +# ============================================================================= +# -----Correction value of barometric pressure (in inches; standard # temperature of scale 62F) (bp) for temperature in Fahrenheit (tf) # Reference: # List, R.J., 1966: Smithsonian Meteorological Tables. # Smithsonian Institution, Washington, DC, 527 pp. -#-----constants m and l from List (1966), p. 137. -def baro_tempF_correction(bp,tf): - m=0.000101 - l=0.0000102 - baro_tempF_correction = -bp * ( ((m*(tf-32.))-(l*(tf-62.))) / (1.+m*(tf-32.)) ) +# -----constants m and l from List (1966), p. 137. +def baro_tempF_correction(bp, tf): + m = 0.000101 + l = 0.0000102 + baro_tempF_correction = -bp * ( + ((m * (tf - 32.0)) - (l * (tf - 62.0))) / (1.0 + m * (tf - 32.0)) + ) return baro_tempF_correction -#============================================================================= -#-----Correction value (generalized) of barometric pressure (bp) for + + +# ============================================================================= +# -----Correction value (generalized) of barometric pressure (bp) for # temperature (t), depending on units (u): # standard temperature: # u bp t of scale (ts) of mercury (th) # - ---------- ---------- ------------- ------------------- # 0 mm or mb Celsius 0C 0C -# 1 Eng. in. Fahrenheit 62F (16.667C) 32F (0C) (pre-1955) +# 1 Eng. in. Fahrenheit 62F (16.667C) 32F (0C) (pre-1955) # 2 Eng. in. Fahrenheit 32F (0C) 32F (0C) (1955-) # 3 French in. Reaumur 13R (16.25C) 0R (0C) # The returned {baro_temp_correction} value is in the same units as, and is to be @@ -211,25 +226,29 @@ def baro_tempF_correction(bp,tf): # (WBAN), Volume 1 (1st ed.). US GPO, Washington, DC. # WMO (World Meteorological Organization), 1966: International # Meteorological Tables, WMO-No.188.TP.94. -#-----constants ts and th are from List (1966), pp. 136-137 (u=1-2); WBAN +# -----constants ts and th are from List (1966), pp. 136-137 (u=1-2); WBAN # 12 App.1.4.1--3 (u=3); and IMC (1890), p. B.24 (u=4). -def baro_temp_correction(bp,t,u): - tsList=[0.0,62.,32.,13.] - thList=[0.0,32.,32.,0.0] -#-----constants m and l are from List (1966), pp. 136-137 (u=1-3) and WBAN, -# pp. 5-4 and 5-5 (for metric and English units). For u=4, the u=1 -# constants were multiplied by 5/4 (after List, 1966, p. 137). - mList=[0.0001818, 0.000101, 0.000101, 0.000227] - lList=[0.0000184,0.0000102,0.0000102,0.0000230] -#-----test u for valid range +def baro_temp_correction(bp, t, u): + tsList = [0.0, 62.0, 32.0, 13.0] + thList = [0.0, 32.0, 32.0, 0.0] + # -----constants m and l are from List (1966), pp. 136-137 (u=1-3) and WBAN, + # pp. 5-4 and 5-5 (for metric and English units). For u=4, the u=1 + # constants were multiplied by 5/4 (after List, 1966, p. 137). + mList = [0.0001818, 0.000101, 0.000101, 0.000227] + lList = [0.0000184, 0.0000102, 0.0000102, 0.0000230] + # -----test u for valid range if u < 0 or u > 3: - sys.exit("baro_temp_correction error: invalid u") + sys.exit("baro_temp_correction error: invalid u") - baro_temp_correction = -bp * ( ((mList[u]*(t-thList[u]))-(lList[u]*(t-tsList[u]))) / - (1.+(mList[u]*(t-thList[u]))) ) + baro_temp_correction = -bp * ( + ((mList[u] * (t - thList[u])) - (lList[u] * (t - tsList[u]))) + / (1.0 + (mList[u] * (t - thList[u]))) + ) return baro_temp_correction -#============================================================================= -#-----Correction value of barometric pressure (bp) for gravity depending on + + +# ============================================================================= +# -----Correction value of barometric pressure (bp) for gravity depending on # latitude (rlat), with constants set depending on gmode (for COADS, we # adopt gmode=1 for 1955-forward, and gmode=2 for data prior to 1955): # g1 (equation 1) g2 (equation 2) Comment @@ -248,7 +267,7 @@ def baro_temp_correction(bp,t,u): # 45 deg, at sea level" (WBAN, 12 App.1.4.1--2; see also List, 1966, pp. # 3-4, and WMO, 1966). For example, UK Met. Office MK I (MK II) barometers # issued before (starting) 1 January 1955 were graduated to read correctly -# when the value of gravity was g45 (g0) (UKMO, 1969). +# when the value of gravity was g45 (g0) (UKMO, 1969). # References: # List, R.J., 1966: Smithsonian Meteorological Tables. # Smithsonian Institution, Washington, DC, 527 pp. @@ -259,42 +278,44 @@ def baro_temp_correction(bp,t,u): # (WBAN), Volume 1 (1st ed.). US GPO, Washington, DC. # WMO (World Meteorological Organization), 1966: International # Meteorological Tables, WMO-No.188.TP.94. -def baro_G_correction(bp,rlat,gmode): - pi=3.14159265358979323846264338327950288 -#-----g45 from List (1966), p. 488 ("best" sea-level gravity at latitude 45) - g45=980.616 -#-----g0 from List (1966), p. 200 ("standard" acceleration of gravity) - g0=980.665 -#-----check latitude - if rlat < -90. or rlat > 90.: - sys.exit("baro_G_correction error: invalid rlat") - -#-----check gmode, and set g1 and g2 +def baro_G_correction(bp, rlat, gmode): + pi = 3.14159265358979323846264338327950288 + # -----g45 from List (1966), p. 488 ("best" sea-level gravity at latitude 45) + g45 = 980.616 + # -----g0 from List (1966), p. 200 ("standard" acceleration of gravity) + g0 = 980.665 + # -----check latitude + if rlat < -90.0 or rlat > 90.0: + sys.exit("baro_G_correction error: invalid rlat") + + # -----check gmode, and set g1 and g2 if gmode == 1: - g1 = g45 - g2 = g0 + g1 = g45 + g2 = g0 elif gmode == 2: - g1 = g0 - g2 = g0 + g1 = g0 + g2 = g0 elif gmode == 3: - g1 = g45 - g2 = g45 + g1 = g45 + g2 = g45 else: - sys.exit("baro_G_correction error: invalid gmode") -#-----convert degrees to radians - rlatr = rlat * (pi/180.) -#-----List (1966), p. 488, equation 1 (c is the local acceleration of gravity) - a = 0.0000059 * (math.cos(2.0*rlatr)**2) - b = 1. - 0.0026373 * math.cos(2.0*rlatr) - c = g1 * (a + b) -#-----List (1966), p. 202, equation 2 - baro_G_correction = ((c - g2)/g2) * bp + sys.exit("baro_G_correction error: invalid gmode") + # -----convert degrees to radians + rlatr = rlat * (pi / 180.0) + # -----List (1966), p. 488, equation 1 (c is the local acceleration of gravity) + a = 0.0000059 * (math.cos(2.0 * rlatr) ** 2) + b = 1.0 - 0.0026373 * math.cos(2.0 * rlatr) + c = g1 * (a + b) + # -----List (1966), p. 202, equation 2 + baro_G_correction = ((c - g2) / g2) * bp return baro_G_correction -#============================================================================= -#=======================================================================------- -#-----cloud conversions-------------------------------------------------------- -#=======================================================================------- -#-----Convert "proportion of sky clear" in tenths (t0), to oktas (eighths + + +# ============================================================================= +# =======================================================================------- +# -----cloud conversions-------------------------------------------------------- +# =======================================================================------- +# -----Convert "proportion of sky clear" in tenths (t0), to oktas (eighths # of sky covered; WMO code 2700). The t0 code, specified in Maury # (1854), was documented for use, e.g., for US Marine Meteorological # Journals (1878-1893). The dates of transition to instead reporting @@ -321,19 +342,21 @@ def baro_G_correction(bp,rlat,gmode): # Accompany the Wind and Current Charts, 6th Ed., Washington, DC, # pp. 54-88. + def cloud_tenthsclear2oktas(t0): -#-----check validity of t0 + # -----check validity of t0 if t0 < 0 or t0 > 10: - sys.exit('cloud_tenthsclear2oktas error: illegal t0=',t0) -#-----convert tenths of "sky clear" (t0) to tenths of "sky covered" (t1) -# (Note: assumption: no known basis in documentation) - t1 = 10 - t0 -#-----convert tenths of "sky covered" to oktas + sys.exit("cloud_tenthsclear2oktas error: illegal t0=", t0) + # -----convert tenths of "sky clear" (t0) to tenths of "sky covered" (t1) + # (Note: assumption: no known basis in documentation) + t1 = 10 - t0 + # -----convert tenths of "sky covered" to oktas cloud_tenthsclear2oktas = cloud_tenthscovered2oktas(t1) return cloud_tenthsclear2oktas -#=======================================================================------- -#-----Convert tenths (of sky covered) (t1), to oktas (eighths of sky + +# =======================================================================------- +# -----Convert tenths (of sky covered) (t1), to oktas (eighths of sky # covered; WMO code 2700). This implements the mapping of tenths # to oktas shown below (left-hand columns) from NCDC (1968), section # 4.5, scale 7. In contrast, the right-hand columns show a reverse @@ -349,7 +372,7 @@ def cloud_tenthsclear2oktas(t0): # 4 5 | 4 5 # 5 6 | 5 7.5 # 6 7 or 8 | 6 9 -# 7 9 | 7 10 +# 7 9 | 7 10 # 8 10 | 8 10 # 9 obscured # Input t1 values must be limited to 0-10; "obscured" is not handled. @@ -359,140 +382,163 @@ def cloud_tenthsclear2oktas(t0): # Riehl, 1947: Diurnal variation of cloudiness over the subtropical # Atlantic Ocean. Bull. Amer. Meteor. Soc., 28, 37-40. def cloud_tenthscovered2oktas(t1): - okList = [0,1,2,2,3,4,5,6,6,7,8] -#-----check validity of t1 + okList = [0, 1, 2, 2, 3, 4, 5, 6, 6, 7, 8] + # -----check validity of t1 if t1 < 0 or t1 > 10: - sys.exit("cloud_tenthscovered2oktas error: illegal t1=",t1) -#-----convert from tenths to oktas + sys.exit("cloud_tenthscovered2oktas error: illegal t1=", t1) + # -----convert from tenths to oktas cloud_tenthscovered2oktas = okList[t1] return cloud_tenthscovered2oktas -#=======================================================================------- -#-----temperature conversions-------------------------------------------------- -#=======================================================================------- -#-----Convert temperature in degrees Fahrenheit (tc) to degrees Celsius. + + +# =======================================================================------- +# -----temperature conversions-------------------------------------------------- +# =======================================================================------- +# -----Convert temperature in degrees Fahrenheit (tc) to degrees Celsius. # Reference: # List, R.J., 1966: Smithsonian Meteorological Tables. # Smithsonian Institution, Washington, DC, 527 pp. -#-----equation from List (1966), Table 2 (p. 17). +# -----equation from List (1966), Table 2 (p. 17). def temp_f2c(tf): - temp_f2c = (5.0/9.0) * (tf - 32.0) + temp_f2c = (5.0 / 9.0) * (tf - 32.0) return temp_f2c -#============================================================================= -#-----Convert temperature in degrees Celsius (tc) to degrees Fahrenheit. + + +# ============================================================================= +# -----Convert temperature in degrees Celsius (tc) to degrees Fahrenheit. # Reference: # List, R.J., 1966: Smithsonian Meteorological Tables. # Smithsonian Institution, Washington, DC, 527 pp. -#-----equation from List (1966), Table 2 (p. 17). +# -----equation from List (1966), Table 2 (p. 17). def temp_c2f(tc): - temp_c2f = ((9.0/5.0) * tc) + 32.0 + temp_c2f = ((9.0 / 5.0) * tc) + 32.0 return temp_c2f -#============================================================================= -#-----Convert temperature in Kelvins (tk) to degrees Celsius. -#-----Adapted from colib5s.01J function {cvtkc} (1984); + + +# ============================================================================= +# -----Convert temperature in Kelvins (tk) to degrees Celsius. +# -----Adapted from colib5s.01J function {cvtkc} (1984); def temp_k2c(tk): if tk < 0.0: - sys.exit("temp_k2c error: negative input tk=",tk) + sys.exit("temp_k2c error: negative input tk=", tk) temp_k2c = tk - 273.15 return temp_k2c -#============================================================================= -#-----Convert temperature in degrees Celsius (tc) to Kelvins. -#-----Adapted from colib5s.01J function {cvtck} (1984); + + +# ============================================================================= +# -----Convert temperature in degrees Celsius (tc) to Kelvins. +# -----Adapted from colib5s.01J function {cvtck} (1984); def temp_c2k(tc): temp_c2k = tc + 273.15 if temp_c2k < 0.0: - sys.exit("temp_c2k error: negative output=",temp_c2k) + sys.exit("temp_c2k error: negative output=", temp_c2k) return temp_c2k -#============================================================================= -#-----Convert temperature in degrees Reaumur (tc) to degrees Celsius. + + +# ============================================================================= +# -----Convert temperature in degrees Reaumur (tc) to degrees Celsius. # Reference: # List, R.J., 1966: Smithsonian Meteorological Tables. # Smithsonian Institution, Washington, DC, 527 pp. -#-----equation from List (1966), Table 2 (p. 17). +# -----equation from List (1966), Table 2 (p. 17). def temp_r2c(tr): - temp_r2c = (5.0/4.0) * tr + temp_r2c = (5.0 / 4.0) * tr return temp_r2c -#============================================================================= -#-----Convert temperature in degrees Celsius (tc) to degrees Reaumur. + + +# ============================================================================= +# -----Convert temperature in degrees Celsius (tc) to degrees Reaumur. # Reference: # List, R.J., 1966: Smithsonian Meteorological Tables. # Smithsonian Institution, Washington, DC, 527 pp. -#-----equation from List (1966), Table 2 (p. 17). +# -----equation from List (1966), Table 2 (p. 17). def temp_c2r(tc): - temp_c2r = (4.0/5.0) * tc + temp_c2r = (4.0 / 5.0) * tc return temp_c2r -#============================================================================= -#=======================================================================------- -#-----wind conversions--------------------------------------------------------- -#=======================================================================------- -#-----Convert wind vector eastward and northward components (u,v) to + + +# ============================================================================= +# =======================================================================------- +# -----wind conversions--------------------------------------------------------- +# =======================================================================------- +# -----Convert wind vector eastward and northward components (u,v) to # direction (from) in degrees (clockwise from 0 degrees North). -#-----Adapted from colib5s.01J function {dduv} (1984); -def wind_uv2dir(u,v): +# -----Adapted from colib5s.01J function {dduv} (1984); +def wind_uv2dir(u, v): if u == 0.0 and v == 0.0: - a = 0.0 + a = 0.0 else: - a = math.atan2(v,u)*(180.0/3.14159265358979323846264338327950288) + a = math.atan2(v, u) * (180.0 / 3.14159265358979323846264338327950288) wind_uv2dir = 270.0 - a if wind_uv2dir > 360.0: - wind_uv2dir -= 360.0 + wind_uv2dir -= 360.0 return wind_uv2dir -#============================================================================= -def wind_uv2vel(u,v): -#-----Convert wind vector eastward and northward components (u,v) to -# velocity. -#-----Adapted from colib5s.01J function {vvuv} (1984); + + +# ============================================================================= +def wind_uv2vel(u, v): + # -----Convert wind vector eastward and northward components (u,v) to + # velocity. + # -----Adapted from colib5s.01J function {vvuv} (1984); wind_uv2vel = math.sqrt(u**2 + v**2) return wind_uv2vel -#============================================================================= + +# ============================================================================= def wind_kts2mps(kt): -#-----Convert from knots (kt; with respect to the international nautical -# mile) to meters per second (see {tpktms} for details). -#-----Adapted from colib5s.01J function {cvskm} (1984); + # -----Convert from knots (kt; with respect to the international nautical + # mile) to meters per second (see {tpktms} for details). + # -----Adapted from colib5s.01J function {cvskm} (1984); wind_kts2mps = kt * 0.51444444444444444444 return wind_kts2mps -#============================================================================= + +# ============================================================================= def wind_mps2kts(ms): -#-----Convert from meters per second (ms) to knots (with respect to the -# international nautical mile) (see {tpktms} for details). -#-----Adapted from colib5s.01J function {cvsmk} (1984); - wind_mps2kts = ms * 1.9438444924406047516 + # -----Convert from meters per second (ms) to knots (with respect to the + # international nautical mile) (see {tpktms} for details). + # -----Adapted from colib5s.01J function {cvsmk} (1984); + wind_mps2kts = ms * 1.9438444924406047516 return wind_mps2kts -#============================================================================= -#-----Convert from knots (k0; with respect to the U.S. nautical mile) to + +# ============================================================================= +# -----Convert from knots (k0; with respect to the U.S. nautical mile) to # meters per second (see {wind_kts2mps} for details). def wind_us_kts2mps(k0): wind_us_kts2mps = k0 * 0.51479111111111111111 return wind_us_kts2mps -#============================================================================= -#-----Convert from meters per second (ms) to knots (with respect to the + +# ============================================================================= +# -----Convert from meters per second (ms) to knots (with respect to the # U.S. nautical mile) (see {wind_kts2mps} for details). def wind_mps2us_kts(ms): - wind_mps2us_kts = ms * 1.9425354836481679732 + wind_mps2us_kts = ms * 1.9425354836481679732 return wind_mps2us_kts -#============================================================================= -#-----Convert from knots (k1; with respect to the Admiralty nautical mile) + +# ============================================================================= +# -----Convert from knots (k1; with respect to the Admiralty nautical mile) # to meters per second (see {wind_kts2mps} for details). def wind_a_kts2mps(k1): wind_a_kts2mps = k1 * 0.51477333333333333333 return wind_a_kts2mps -#============================================================================= -#-----Convert from meters per second (ms) to knots (with respect to the + +# ============================================================================= +# -----Convert from meters per second (ms) to knots (with respect to the # Admiralty nautical mile) (see {wind_kts2mps} for details). def wind_mps2a_kts(ms): - wind_mps2a_kts = ms * 1.9426025694156651471 + wind_mps2a_kts = ms * 1.9426025694156651471 return wind_mps2a_kts -#============================================================================= -#-----Convert from Beaufort force 0-12 (bf) to "old" (WMO code 1100) + +# ============================================================================= +# -----Convert from Beaufort force 0-12 (bf) to "old" (WMO code 1100) # midpoint in knots. From NCDC (1968), conversion scale 5 (sec. # 4.4). Note: Midpoint value 18 looks questionable, but appeared # originally in UKMO (1948). @@ -505,14 +551,16 @@ def wind_mps2a_kts(ms): # from 1st January, 1949). Air Ministry, Meteorological Office, # HM Stationary Office, London, 39 pp. def wind_Beaufort2kts(bf): - ktList = [0,2,5,9,13,18,24,30,37,44,52,60,68] + ktList = [0, 2, 5, 9, 13, 18, 24, 30, 37, 44, 52, 60, 68] if bf < 0 or bf > 12: - sys.exit("wind_Beaufort2kts error: bf=",bf) + sys.exit("wind_Beaufort2kts error: bf=", bf) wind_Beaufort2kts = ktList[bf] return wind_Beaufort2kts -#============================================================================= -#-----Convert from Beaufort force 0-12 (bf) to "old" (WMO code 1100) + + +# ============================================================================= +# -----Convert from Beaufort force 0-12 (bf) to "old" (WMO code 1100) # midpoint in meters per second. From Slutz et al. (1985) supp. # K, Table K5-5 (p. K29). See {wind_Beaufort2kts} for additional background. # Reference: @@ -522,52 +570,118 @@ def wind_Beaufort2kts(bf): # Environmental Research Laboratories, Climate Research # Program, Boulder, Colo., 268 pp. (NTIS PB86-105723). def wind_Beaufort2mps(bf): - msList = [0.,1.,2.6,4.6,6.7,9.3,12.3,15.4,19.,22.6,26.8,30.9,35.] + msList = [0.0, 1.0, 2.6, 4.6, 6.7, 9.3, 12.3, 15.4, 19.0, 22.6, 26.8, 30.9, 35.0] if bf < 0 or bf > 12: - sys.exit("wind_Beaufort2mps error: bf=",bf) + sys.exit("wind_Beaufort2mps error: bf=", bf) wind_Beaufort2mps = msList[bf] return wind_Beaufort2mps -#============================================================================= -#-----Convert 4-character 32-point wind direction abbreviation c32 into + +# ============================================================================= +# -----Convert 4-character 32-point wind direction abbreviation c32 into # degrees, or return imiss if unrecognized; also return numeric code # 1-32 (or imiss) in dc (see {wind_dircode2deg} for background). Recognized # abbreviations are in cwd, with these characteristics: left-justified, # upper-case, with trailing blank fill, and where "X" stands for "by". # NOTE: No constraint is placed on imiss (it could overlap with data). -def wind_4chardir2deg(c32,dc,imiss): - cwdList = ['NXE ','NNE ','NEXN','NE ','NEXE','ENE ','EXN ','E ', - 'EXS ','ESE ','SEXE','SE ','SEXS','SSE ','SXE ','S ', - 'SXW ','SSW ','SWXS','SW ','SWXW','WSW ','WXS ','W ', - 'WXN ','WNW ','NWXW','NW ','NWXN','NNW ','NXW ','N '] +def wind_4chardir2deg(c32, dc, imiss): + cwdList = [ + "NXE ", + "NNE ", + "NEXN", + "NE ", + "NEXE", + "ENE ", + "EXN ", + "E ", + "EXS ", + "ESE ", + "SEXE", + "SE ", + "SEXS", + "SSE ", + "SXE ", + "S ", + "SXW ", + "SSW ", + "SWXS", + "SW ", + "SWXW", + "WSW ", + "WXS ", + "W ", + "WXN ", + "WNW ", + "NWXW", + "NW ", + "NWXN", + "NNW ", + "NXW ", + "N ", + ] wind_4chardir2deg = imiss - for j in range(1,32): - if c32 == cwdList[j-1]: - wind_4chardir2deg = wind_dircode2deg(j,imiss) - dc = j - return wind_4chardir2deg + for j in range(1, 32): + if c32 == cwdList[j - 1]: + wind_4chardir2deg = wind_dircode2deg(j, imiss) + dc = j + return wind_4chardir2deg return wind_4chardir2deg -#============================================================================= -#-----Convert 32-point wind direction numeric code dc into degrees, or + + +# ============================================================================= +# -----Convert 32-point wind direction numeric code dc into degrees, or # return imiss if dc is out of range 1-32. Release 1, Table F2-1 # defines the mapping of code dc to degrees in dwd. # NOTE: No constraint is placed on imiss (it could overlap with data). -def wind_dircode2deg(dc,imiss): - dwdList = [ 11, 23, 34, 45, 56, 68, 79, 90, - 101, 113, 124, 135, 146, 158, 169, 180, - 191, 203, 214, 225, 236, 248, 259, 270, - 281, 293, 304, 315, 326, 338, 349, 360] - if dc >= 1 and dc<= 32: - wind_dircode2deg = dwdList[dc-1] +def wind_dircode2deg(dc, imiss): + dwdList = [ + 11, + 23, + 34, + 45, + 56, + 68, + 79, + 90, + 101, + 113, + 124, + 135, + 146, + 158, + 169, + 180, + 191, + 203, + 214, + 225, + 236, + 248, + 259, + 270, + 281, + 293, + 304, + 315, + 326, + 338, + 349, + 360, + ] + if dc >= 1 and dc <= 32: + wind_dircode2deg = dwdList[dc - 1] else: - wind_dircode2deg = imiss + wind_dircode2deg = imiss return wind_dircode2deg -#============================================================================= -#=======================================================================------- -#-----time conversions--------------------------------------------------------- -#=======================================================================------- -#-----Convert local standard hour (ihr; in hundredths 0-2399) and "Julian" + +# ============================================================================= +# =======================================================================------- +# -----time conversions--------------------------------------------------------- +# =======================================================================------- + + +# -----Convert local standard hour (ihr; in hundredths 0-2399) and "Julian" # day (i.e., any incrementable integer date) (idy) into coordinated # universal time (UTC) hour (uhr) and day (udy; decremented if the # dateline is crossed), using longitude (elon; in hundredths of degrees @@ -575,93 +689,105 @@ def wind_dircode2deg(dc,imiss): # including the International Date Line, are not employed. b) In all # cases the western (eastern) boundary of each time zone is inclusive # (exclusive), i.e., 7.50W-7.49E, 7.50E-22.49E, ..., 172.50E-172.51W. -def time_local_hour_julianday2UTC(ihr,idy,elon):#,uhr,udy): +def time_local_hour_julianday2UTC(ihr, idy, elon): # ,uhr,udy): if ihr < 0 or ihr > 2399: - sys.exit("error time_local_hour_julianday2UTC: ihr=",ihr) + sys.exit("error time_local_hour_julianday2UTC: ihr=", ihr) elif elon < 0 or elon > 35999: - sys.exit("error time_local_hour_julianday2UTC: elon=",elon) + sys.exit("error time_local_hour_julianday2UTC: elon=", elon) wlon = 36000 - elon udy = idy - dhr = int((wlon + 749)/1500) - uhr = ihr + dhr*100 + dhr = int((wlon + 749) / 1500) + uhr = ihr + dhr * 100 if uhr >= 2400: - udy = udy + 1 - uhr = uhr - 2400 + udy = udy + 1 + uhr = uhr - 2400 if wlon >= 18000: - udy = udy - 1 - - return uhr,udy - -#============================================================================= -#-----Convert from date (iday,imonth,iyear) to number of days since -# 1 Jan 1770. -def time_date2julianday(iday,imonth,iyear): - daysList = [31,28,31,30,31,30,31,31,30,31,30,31] + udy = udy - 1 - if (iyear < 1770 or imonth < 1 or imonth > 12 - or iday < 1 - or (iday > daysList[imonth-1] - and (imonth != 2 or not(calendar.isleap(iyear)) or iday != 29))): + return uhr, udy - sys.exit("time_date2julianday: invalid day,month,year") - start = date(1770,1,1) - end = date(iyear,imonth,iday) +# ============================================================================= +# -----Convert from date (iday,imonth,iyear) to number of days since +# 1 Jan 1770. +def time_date2julianday(iday, imonth, iyear): + daysList = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + + if ( + iyear < 1770 + or imonth < 1 + or imonth > 12 + or iday < 1 + or ( + iday > daysList[imonth - 1] + and (imonth != 2 or not (calendar.isleap(iyear)) or iday != 29) + ) + ): + sys.exit("time_date2julianday: invalid day,month,year") + + start = date(1770, 1, 1) + end = date(iyear, imonth, iday) delta_time = end - start time_date2julianday = delta_time.days - return time_date2julianday + return time_date2julianday + -#============================================================================= -#-----Convert from number of days (ndays) since 1 Jan 1770 to +# ============================================================================= +# -----Convert from number of days (ndays) since 1 Jan 1770 to # date (iday,imonth,iyear). # iday=-1, imonth=-1, and iyear=-1 if ndays is invalid. def time_julianday2date(ndays): -#,iday,imonth,iyear): + # ,iday,imonth,iyear): - start = date(1770,1,1) + start = date(1770, 1, 1) delta = timedelta(ndays) offset = start + delta - iday = '{0.day:02d}'.format(offset) - imonth = '{0.month:02d}'.format(offset) - iyear = '{0.year:04d}'.format(offset) + iday = "{0.day:02d}".format(offset) + imonth = "{0.month:02d}".format(offset) + iyear = "{0.year:04d}".format(offset) - return iday,imonth,iyear + return iday, imonth, iyear -#============================================================================= -#-----miscellaneous------------------------------------------------------------ -#============================================================================= + +# ============================================================================= +# -----miscellaneous------------------------------------------------------------ +# ============================================================================= def print_epsilon(): -#-----Print calculated machine epsilon, i.e., the smallest power of 2, -# 2**no = ep, such that 1+2**no>1 - print("print_epsilon output:",np.finfo(float32).eps) + # -----Print calculated machine epsilon, i.e., the smallest power of 2, + # 2**no = ep, such that 1+2**no>1 + print("print_epsilon output:", np.finfo(float32).eps) return -#============================================================================= + +# ============================================================================= def print_dblepsilon(): -#-----Double precision version of {print_epsilon} (see for background). - print("print_dblepsilon output:",np.finfo(np.float).eps) + # -----Double precision version of {print_epsilon} (see for background). + print("print_dblepsilon output:", np.finfo(np.float).eps) return -#============================================================================= + +# ============================================================================= def round(x): -#-----Round real x into integer round such that a fractional part of x -# of exactly 0.5 results in rounding to the nearest even integer. -#-----Adapted from colib5s.01J function {iround} (1984); - nextHighInt = math.ceil(x / 2.) * 2 - x + # -----Round real x into integer round such that a fractional part of x + # of exactly 0.5 results in rounding to the nearest even integer. + # -----Adapted from colib5s.01J function {iround} (1984); + nextHighInt = math.ceil(x / 2.0) * 2 - x deltaHigh = nextHighInt - x deltaLow = deltaHigh - 2 if deltaHigh < deltaLow: - round = nextHighInt + round = nextHighInt else: - round = nextHighInt - 2 + round = nextHighInt - 2 return round -#============================================================================= + + +# ============================================================================= diff --git a/data_post_processing/logging.py b/data_post_processing/logging.py new file mode 100644 index 0000000..26af918 --- /dev/null +++ b/data_post_processing/logging.py @@ -0,0 +1,67 @@ +import sys +from pathlib import Path +from typing import Any + +import structlog +from data_post_processing.config import settings +# from data_post_processing.log_capture import capture_processing_logs + + +def configure_logging() -> None: + """Configure structlog for the application.""" + if settings.log_format == "json": + processors = [ + structlog.contextvars.merge_contextvars, + structlog.processors.add_log_level, + structlog.processors.TimeStamper(fmt="iso"), + structlog.processors.StackInfoRenderer(), + structlog.processors.format_exc_info, + structlog.processors.JSONRenderer(), + ] + else: + processors = [ + structlog.contextvars.merge_contextvars, + structlog.processors.add_log_level, + structlog.processors.TimeStamper(fmt="iso"), + structlog.processors.StackInfoRenderer(), + structlog.processors.format_exc_info, + structlog.dev.ConsoleRenderer(colors=True) + ] + + structlog.configure( + processors=processors, # type: ignore[arg-type] + wrapper_class=structlog.make_filtering_bound_logger( + settings.log_level + ), + context_class=dict, + logger_factory=structlog.PrintLoggerFactory(file=sys.stdout), + cache_logger_on_first_use=True, + ) + + +def get_file_logger(filename: str = "debug") -> structlog.BoundLogger: + """" Provide a bound logger to output to a file """ + logger = structlog.wrap_logger( + structlog.WriteLogger( + file=Path("logs/"+filename).with_suffix(".log").open("wt") + ), + # wrapper_class=structlog.BoundLogger, + wrapper_class=structlog.make_filtering_bound_logger( + settings.log_level + ), + processors=[ + structlog.contextvars.merge_contextvars, + structlog.processors.add_log_level, + structlog.processors.TimeStamper(fmt="iso"), + structlog.processors.StackInfoRenderer(), + structlog.processors.format_exc_info, + # structlog.processors.LogfmtRenderer(key_order=["timestamp"]), + structlog.dev.ConsoleRenderer(colors=False), + ] + ) + return logger + + +def get_logger() -> structlog.BoundLogger: + """Get a structlog logger.""" + return structlog.get_logger() diff --git a/data_post_processing/models.py b/data_post_processing/models.py new file mode 100644 index 0000000..573eb3c --- /dev/null +++ b/data_post_processing/models.py @@ -0,0 +1,213 @@ +""" + +Database models. + +Used for reading the data from the DB, these define the tables +along with the fields that they contain that we are interested in. + +""" + +from datetime import date, datetime +from decimal import Decimal +from typing import List + +from sqlalchemy import Date, DateTime, Integer, Numeric, String, Text +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + + +# Base for all the models, allow for +# declarative definitions for the model +# database table +class Base(DeclarativeBase): + pass + + +# Question - as this can have multiple rows what makes each distinct (no duplicates) +class MetaDataGlobal(Base): + """The project Metadata.""" + + __tablename__ = "odr_global_metadata" + + source: Mapped[str] = mapped_column(String(100)) + project: Mapped[str] = mapped_column(String(100)) + # TODO: can we use ISO values for this + country: Mapped[str] = mapped_column(String(100)) + link: Mapped[str] = mapped_column(String(100)) + # Dummy primary key so we can use the ORM + pkey: Mapped[int] = mapped_column(Integer, primary_key=True) + + +class StationMetaData(Base): + """The Station Metadata.""" + + __tablename__ = "odr_station_metadata" + + station_id: Mapped[int] = mapped_column(Integer, primary_key=True) + station_name: Mapped[str] = mapped_column(String(255)) + start_date: Mapped[date] = mapped_column(Date) + end_date: Mapped[date] = mapped_column(Date) + source_id: Mapped[str] = mapped_column(String(100)) + WMOID: Mapped[str] = mapped_column(String(50)) + MSCID: Mapped[str] = mapped_column(String(50)) + other_station_name: Mapped[str] = mapped_column(String(100)) + region: Mapped[str] = mapped_column(String(100)) + latitude: Mapped[Decimal] = mapped_column(Numeric(precision=7, scale=3)) + document_latitude: Mapped[str] = mapped_column(String(100)) + longitude: Mapped[Decimal] = mapped_column(Numeric(precision=7, scale=3)) + document_longitude: Mapped[str] = mapped_column(String(100)) + elevation: Mapped[Decimal] = mapped_column(Numeric(precision=10, scale=3)) + document_elevation: Mapped[str] = mapped_column(String(100)) + timezone: Mapped[str] = mapped_column(String(100)) + UTC_offset: Mapped[Decimal] = mapped_column(Numeric(precision=6, scale=3)) + document_country: Mapped[str] = mapped_column(String(255)) + additional_territory_designators: Mapped[str] = mapped_column(Text) + observation_times_local: Mapped[str] = mapped_column(String(255)) + originator_organization: Mapped[str] = mapped_column(String(255)) + originator_name: Mapped[str] = mapped_column(String(255)) + originator_position: Mapped[str] = mapped_column(String(255)) + originator_address: Mapped[str] = mapped_column(Text) + project: Mapped[str] = mapped_column(String(100)) + # relationship to global meta-data + # meta_data: Mapped[MetaDataGlobal] = relationship( + # foreign_keys="MetaDataGlobal.project", + # primaryjoin="StationMetaData.project == MetaDataGlobal.project", + # viewonly=True + # ) + + +class Field(Base): + """The Fields table.""" + + __tablename__ = "fields" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + field_key: Mapped[str] = mapped_column(String(255)) + internal_name: Mapped[str] = mapped_column(String(255)) + odr_type: Mapped[str] = mapped_column(String(255)) + period: Mapped[str] = mapped_column(String(255)) + time_of_day: Mapped[str] = mapped_column(String(255)) + measurement_unit_original: Mapped[str] = mapped_column(String(255)) + measurement_unit_si: Mapped[str] = mapped_column(String(255)) + data_type: Mapped[str] = mapped_column(String(255)) + + +class Annotation(Base): + """The Annotations table.""" + + __tablename__ = "annotations" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + transcription_id: Mapped[int] = mapped_column(Integer, nullable=False) + page_id: Mapped[int] = mapped_column(Integer, nullable=False) + observation_date: Mapped[datetime | None] = mapped_column(DateTime, nullable=False) + created_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=False) + updated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=False) + + transcription: Mapped[Transcription] = relationship( + foreign_keys="Annotation.transcription_id", + primaryjoin="Annotation.transcription_id == Transcription.id", + viewonly=True, + ) + data_entries: Mapped[List["DataEntry"]] = relationship( + foreign_keys="DataEntry.annotation_id", + primaryjoin="Annotation.id == DataEntry.annotation_id", + viewonly=True, + ) + + +class Transcription(Base): + """The Transcriptions table.""" + + __tablename__ = "transcriptions" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + user_id: Mapped[int] = mapped_column(Integer, nullable=False) + page_id: Mapped[int] = mapped_column(Integer, nullable=False) + created_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=False) + updated_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=False) + + user: Mapped[User] = relationship( + foreign_keys="Transcription.user_id", + primaryjoin="Transcription.user_id == User.id", + viewonly=True, + ) + page: Mapped[Page] = relationship( + foreign_keys="Transcription.page_id", + primaryjoin="Transcription.page_id == Page.id", + viewonly=True, + ) + annotations: Mapped[List["Annotation"]] = relationship( + foreign_keys="Transcription.id", + primaryjoin="Transcription.id == Annotation.transcription_id", + viewonly=True, + ) + + +class DataEntry(Base): + """The Data Entries table.""" + + __tablename__ = "data_entries" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + value: Mapped[str] = mapped_column(Text, nullable=True) + annotation_id: Mapped[int] = mapped_column(Integer, nullable=False) + user_id: Mapped[int] = mapped_column(Integer, nullable=False) + # TODO FK + page_id: Mapped[int] = mapped_column(Integer, nullable=False) + # TODO FK + field_id: Mapped[int] = mapped_column(Integer, nullable=False) + + user: Mapped["User"] = relationship( + foreign_keys="User.id", + primaryjoin="DataEntry.user_id == User.id", + ) + annotation: Mapped["Annotation"] = relationship( + foreign_keys="Annotation.id", + primaryjoin="DataEntry.annotation_id == Annotation.id", + viewonly=True, + ) + + +class PageInfo(Base): + __tablename__ = "page_infos" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + page_id: Mapped[int] = mapped_column(Integer, primary_key=True) + user_id: Mapped[int] = mapped_column(Integer, primary_key=True) + location: Mapped[str] = mapped_column(String(255)) + + +class Page(Base): + """The Pages table.""" + + __tablename__ = "pages" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + image_file_name: Mapped[str] = mapped_column(String(255), nullable=True) + + transcriptions: Mapped[List["Transcription"]] = relationship( + foreign_keys="Page.id", + primaryjoin="Page.id == Transcription.page_id", + viewonly=True, + ) + + +class User(Base): + """The Users table.""" + + __tablename__ = "users" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + display_name: Mapped[str] = mapped_column(String(255), nullable=True) + full_name: Mapped[str] = mapped_column(String(255), nullable=True) + + transcriptions: Mapped[List["Transcription"]] = relationship( + foreign_keys="User.id", + primaryjoin="User.id == Transcription.user_id", + viewonly=True, + ) + date_entries: Mapped[List["DataEntry"]] = relationship( + foreign_keys="User.id", + primaryjoin="User.id == DataEntry.user_id", + viewonly=True, + ) diff --git a/data_post_processing/post-process_all.py b/data_post_processing/post-process_all.py index 8e31592..aa17b8f 100644 --- a/data_post_processing/post-process_all.py +++ b/data_post_processing/post-process_all.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -#%% packages +# %% packages import argparse import datetime -#import json + +# import json import numpy as np import os import pandas as pd @@ -15,94 +16,179 @@ from sqlalchemy import URL import statistics -#%% internal scripts +# %% internal scripts import db_connection as db # subroutine to connect to the database import get_metadata as gm import iso_mapping as im # subroutine with maps for cloud and weather codes import time_utils as tu # subroutine to get times import transcription_data_processing as td # subroutine to clean, validate and convert data import warnings -warnings.simplefilter(action='ignore', category=FutureWarning) -from dateutil.parser import parse +warnings.simplefilter(action="ignore", category=FutureWarning) +from dateutil.parser import parse # %% define functions here def isEmptyValue(value): if value is None: return False - v=value.lower() + v = value.lower() if "empty" in v or v == "-": return True return False + def getDatesWithData(engine, fields_id, station): - dates=[] + dates = [] for field in fields_id.id: - querytext = "select a.observation_date, de1.value from annotations a join data_entries de1 on (de1.annotation_id=a.id and de1.field_id=" + str(field) + ") join pages p on (p.id=a.page_id) join page_infos pi2 on pi2.page_id = de1.page_id where a.page_id in (select pp.id from pages pp where location = '" + str(station) + "') order by a.observation_date asc, a.updated_at desc ;" - with engine.connect() as conn: + querytext = ( + "select a.observation_date, de1.value from annotations a join data_entries de1 on (de1.annotation_id=a.id and de1.field_id=" + + str(field) + + ") join pages p on (p.id=a.page_id) join page_infos pi2 on pi2.page_id = de1.page_id where a.page_id in (select pp.id from pages pp where location = '" + + str(station) + + "') order by a.observation_date asc, a.updated_at desc ;" + ) + with engine.connect() as conn: query = conn.execute(text(querytext)) results = query.fetchall() for result in results: value = result[1] - date = result[0].replace(hour=0,minute=0,second=0) + date = result[0].replace(hour=0, minute=0, second=0) if date not in dates and isEmptyValue(value) == False: dates.append(date) conn.close() return dates -#%% basic cleanup -remove symbols from number fields, check decimals... + + +# %% basic cleanup -remove symbols from number fields, check decimals... # ## ADD /CHECK FIELD TABLE FOR SUPPOSED VARIABLE TYPE - STRING, INTEGER, DECIMAL, ETC # if field numeric, check for characters, decimals -def preProcess_num(value, debugLog, result_day, fields, odr_type, transcription_id, unit, error_code): + +def preProcess_num( + value, debugLog, result_day, fields, odr_type, transcription_id, unit, error_code +): if value is not None: value = value.casefold() correctedValue = value -## add in remove extra characters + ## add in remove extra characters # vxariables where empty is not missing but no value or zero value, e.g. precipitation, wind # odr_type: nl_mean, pr,sd, ss, swe,w_anem,w_anem24,wf, - if (odr_type == "pr" or odr_type == "rr" or odr_type == "sd" or odr_type == "swe" or odr_type == "ss" - or odr_type == "w" or odr_type == "w_anom" or odr_type == "wf" or odr_type == "nl" or odr_type == "nl_mean" - or odr_type == "cd" or odr_type == "wf") and (value == "empty" or value == "dot"): + if ( + odr_type == "pr" + or odr_type == "rr" + or odr_type == "sd" + or odr_type == "swe" + or odr_type == "ss" + or odr_type == "w" + or odr_type == "w_anom" + or odr_type == "wf" + or odr_type == "nl" + or odr_type == "nl_mean" + or odr_type == "cd" + or odr_type == "wf" + ) and (value == "empty" or value == "dot"): error_code = 102 return ("0", "empty", error_code) elif unit == "in" and value == "None": error_code = 102 return ("0", "empty", error_code) -# Handle the case where the data is considered as missing - missing_terms = ["not.taken", "not taken", "unknown symbol", "retracted", "-999", "none", "no grass", - "no place", "suspended", "abt on duty"] - instrument_error_terms = ["out of order", "out.of.order", "broken", "unserviceable", "-888", "incorrect", - "not reliable", "covered in snow", "covered with snow", "observer", "no error"] - - if (value is None or value == "-" or value == " " or value == "" or - any([x.lower() in value for x in missing_terms])): - correctedValue = '-999' + # Handle the case where the data is considered as missing + missing_terms = [ + "not.taken", + "not taken", + "unknown symbol", + "retracted", + "-999", + "none", + "no grass", + "no place", + "suspended", + "abt on duty", + ] + instrument_error_terms = [ + "out of order", + "out.of.order", + "broken", + "unserviceable", + "-888", + "incorrect", + "not reliable", + "covered in snow", + "covered with snow", + "observer", + "no error", + ] + + if ( + value is None + or value == "-" + or value == " " + or value == "" + or any([x.lower() in value for x in missing_terms]) + ): + correctedValue = "-999" flag = "missing" error_code = 10 - elif (any([x.lower() in value for x in instrument_error_terms])): - correctedValue = '-999' + elif any([x.lower() in value for x in instrument_error_terms]): + correctedValue = "-999" flag = "instrument error" error_code = 31 - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n") + debugLog.write( + "\tTranscriptionID: " + + str(transcription_id) + + " " + + "Invalid data: " + + value + + " on " + + result_day.strftime(dateFormat) + + "\tField: " + + fields["id"] + + " - " + + fields["name"] + + "\n" + ) - elif ("illegible" in value and "cirrus" not in value): - correctedValue = '-999' + elif "illegible" in value and "cirrus" not in value: + correctedValue = "-999" error_code = 30 - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n") + debugLog.write( + "\tTranscriptionID: " + + str(transcription_id) + + " " + + "Invalid data: " + + value + + " on " + + result_day.strftime(dateFormat) + + "\tField: " + + fields["id"] + + " - " + + fields["name"] + + "\n" + ) - elif ("illegible" in value): - correctedValue = '-999' + elif "illegible" in value: + correctedValue = "-999" flag = "illegible" error_code = 30 - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n") + debugLog.write( + "\tTranscriptionID: " + + str(transcription_id) + + " " + + "Invalid data: " + + value + + " on " + + result_day.strftime(dateFormat) + + "\tField: " + + fields["id"] + + " - " + + fields["name"] + + "\n" + ) else: correctedValue = value @@ -111,53 +197,123 @@ def preProcess_num(value, debugLog, result_day, fields, odr_type, transcription_ return (correctedValue, flag) -def preProcess_int(value, debugLog, result_day, fields, odr_type, transcription_id, unit, error_code): +def preProcess_int( + value, debugLog, result_day, fields, odr_type, transcription_id, unit, error_code +): if value is not None: value = value.casefold() correctedValue = value # variables where empty is not missing but no value or zero value, e.g. precipitation, wind # odr_type: n, nl,cv,hv,skc, - if (odr_type == "n" or odr_type == "nl" or odr_type == "cv" or odr_type == "hv" or odr_type == "skc" - or odr_type == "wf") and (value == "empty" or value == "dot"): + if ( + odr_type == "n" + or odr_type == "nl" + or odr_type == "cv" + or odr_type == "hv" + or odr_type == "skc" + or odr_type == "wf" + ) and (value == "empty" or value == "dot"): error_code = 102 return ("0", "empty", error_code) elif unit == "in" and value == "None": error_code = 102 return ("0", "empty", error_code) -# Handle the case where the data is considered as missing - missing_terms = ["not.taken", "not taken", "unknown symbol", "retracted", "-999", "none", "no grass", - "no place", "suspended", "abt on duty"] - instrument_error_terms = ["out of order", "out.of.order", "broken", "unserviceable", "-888", "incorrect", - "not reliable", "covered in snow", "covered with snow", "observer", "no error"] - - if (value is None or value == "-" or value == " " or value == "" or - any([x.lower() in value for x in missing_terms])): - correctedValue = '-999' + # Handle the case where the data is considered as missing + missing_terms = [ + "not.taken", + "not taken", + "unknown symbol", + "retracted", + "-999", + "none", + "no grass", + "no place", + "suspended", + "abt on duty", + ] + instrument_error_terms = [ + "out of order", + "out.of.order", + "broken", + "unserviceable", + "-888", + "incorrect", + "not reliable", + "covered in snow", + "covered with snow", + "observer", + "no error", + ] + + if ( + value is None + or value == "-" + or value == " " + or value == "" + or any([x.lower() in value for x in missing_terms]) + ): + correctedValue = "-999" flag = "missing" error_code = 10 - elif (any([x.lower() in value for x in instrument_error_terms])): - correctedValue = '-999' + elif any([x.lower() in value for x in instrument_error_terms]): + correctedValue = "-999" flag = "instrument error" error_code = 31 - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n") + debugLog.write( + "\tTranscriptionID: " + + str(transcription_id) + + " " + + "Invalid data: " + + value + + " on " + + result_day.strftime(dateFormat) + + "\tField: " + + fields["id"] + + " - " + + fields["name"] + + "\n" + ) - elif ("illegible" in value and "cirrus" not in value): - correctedValue = '-999' + elif "illegible" in value and "cirrus" not in value: + correctedValue = "-999" flag = "illegible" error_code = 30 - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n") + debugLog.write( + "\tTranscriptionID: " + + str(transcription_id) + + " " + + "Invalid data: " + + value + + " on " + + result_day.strftime(dateFormat) + + "\tField: " + + fields["id"] + + " - " + + fields["name"] + + "\n" + ) - elif ("illegible" in value): - correctedValue = '-999' + elif "illegible" in value: + correctedValue = "-999" flag = "illegible" error_code = 30 - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n") + debugLog.write( + "\tTranscriptionID: " + + str(transcription_id) + + " " + + "Invalid data: " + + value + + " on " + + result_day.strftime(dateFormat) + + "\tField: " + + fields["id"] + + " - " + + fields["name"] + + "\n" + ) else: correctedValue = value @@ -165,7 +321,9 @@ def preProcess_int(value, debugLog, result_day, fields, odr_type, transcription_ return (correctedValue, flag, error_code) -def preProcess_char(value, debugLog, result_day, fields, odr_type, transcription_id, unit, error_code): +def preProcess_char( + value, debugLog, result_day, fields, odr_type, transcription_id, unit, error_code +): if value is not None: value = value.casefold() correctedValue = value @@ -173,150 +331,246 @@ def preProcess_char(value, debugLog, result_day, fields, odr_type, transcription return (value, None) # variables where empty is not missing but no value or zero value, e.g. precipitation, wind # au,c,ch,cl,cd,dd,hd,pr_Type - if (odr_type == 'au', odr_type == 'c' or odr_type == 'cd' or odr_type == 'ch' or odr_type == 'ste' - or odr_type == 'cl' or odr_type == 'dd' or odr_type == 'hd' or odr_type == 'hv' or odr_type == 'pr_type' - or odr_type == 'stb' or odr_type == 'rtb' or odr_type == 'rte' or odr_type == 'ptb' or odr_type == 'pte') and \ - (value == "empty" or value == "dot"): + if ( + odr_type == "au", + odr_type == "c" + or odr_type == "cd" + or odr_type == "ch" + or odr_type == "ste" + or odr_type == "cl" + or odr_type == "dd" + or odr_type == "hd" + or odr_type == "hv" + or odr_type == "pr_type" + or odr_type == "stb" + or odr_type == "rtb" + or odr_type == "rte" + or odr_type == "ptb" + or odr_type == "pte", + ) and (value == "empty" or value == "dot"): error_code = 102 return ("0", "empty", error_code) elif unit == "direction" and value == "empty": error_code = 102 return ("calm", "empty", error_code) -# Handle the case where the data is considered as missing - missing_terms = ["not.taken", "not taken", "unknown symbol", "retracted", "-999", "none", "no grass", - "no place", "suspended", "abt on duty"] - instrument_error_terms = ["out of order", "out.of.order", "broken", "unserviceable", "-888", "incorrect", - "not reliable", "covered in snow", "covered with snow", "observer", "no error"] - - if (value is None or value == "-" or value == " " or value == "" or - any([x.lower() in value for x in missing_terms])): - correctedValue = '-999' + # Handle the case where the data is considered as missing + missing_terms = [ + "not.taken", + "not taken", + "unknown symbol", + "retracted", + "-999", + "none", + "no grass", + "no place", + "suspended", + "abt on duty", + ] + instrument_error_terms = [ + "out of order", + "out.of.order", + "broken", + "unserviceable", + "-888", + "incorrect", + "not reliable", + "covered in snow", + "covered with snow", + "observer", + "no error", + ] + + if ( + value is None + or value == "-" + or value == " " + or value == "" + or any([x.lower() in value for x in missing_terms]) + ): + correctedValue = "-999" error_code = 10 flag = "missing" - elif (any([x.lower() in value for x in instrument_error_terms])): - correctedValue = '-888' + elif any([x.lower() in value for x in instrument_error_terms]): + correctedValue = "-888" flag = "instrument error" error_code = 31 - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n") + debugLog.write( + "\tTranscriptionID: " + + str(transcription_id) + + " " + + "Invalid data: " + + value + + " on " + + result_day.strftime(dateFormat) + + "\tField: " + + fields["id"] + + " - " + + fields["name"] + + "\n" + ) - elif ("illegible" in value and "cirrus" not in value): - correctedValue = '-888' + elif "illegible" in value and "cirrus" not in value: + correctedValue = "-888" flag = "illegible" error_code = 30 - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n") + debugLog.write( + "\tTranscriptionID: " + + str(transcription_id) + + " " + + "Invalid data: " + + value + + " on " + + result_day.strftime(dateFormat) + + "\tField: " + + fields["id"] + + " - " + + fields["name"] + + "\n" + ) - elif ("illegible" in value): - correctedValue = '-888' + elif "illegible" in value: + correctedValue = "-888" flag = "illegible" error_code = 30 - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n") + debugLog.write( + "\tTranscriptionID: " + + str(transcription_id) + + " " + + "Invalid data: " + + value + + " on " + + result_day.strftime(dateFormat) + + "\tField: " + + fields["id"] + + " - " + + fields["name"] + + "\n" + ) else: correctedValue = value flag = "None" return (correctedValue, flag, error_code) -def removeDuplicates(data, project): -# +def removeDuplicates(data, project): + # - print ("Removing Duplicates") + print("Removing Duplicates") counter = 0 checked_values = {} # error_code = - #power user IDs should be in the config + # power user IDs should be in the config if project.lower == "draw": - power_users=(7,393) + power_users = (7, 393) else: - power_users=(3,5,26,32,36) + power_users = (3, 5, 26, 32, 36) ## find transcription id of duplicate data - duplicates = data[data.duplicated(subset=['field_id','observation_date'], keep=False)] - - for index, row in duplicates .iterrows(): - print(row['observation_date'], row['page_id'], row['field_id'],row['user_id'], row['updated_at'] ) - date_dup = row['observation_date'] - #pid_dup = row['page_id'] - fid_dup = row['field_id'] + duplicates = data[ + data.duplicated(subset=["field_id", "observation_date"], keep=False) + ] + + for index, row in duplicates.iterrows(): + print( + row["observation_date"], + row["page_id"], + row["field_id"], + row["user_id"], + row["updated_at"], + ) + date_dup = row["observation_date"] + # pid_dup = row['page_id'] + fid_dup = row["field_id"] # merge into one field date+field_id - dup_data = pd.DataFrame(data.loc[(data['observation_date'] == date_dup) & (data['field_id'] == fid_dup)]) - #print(date_dup, fid_dup,dup_data) - dup_data_counts=dup_data['value'].value_counts() - dup_data_counts=dup_data_counts[dup_data_counts > 1] + dup_data = pd.DataFrame( + data.loc[ + (data["observation_date"] == date_dup) & (data["field_id"] == fid_dup) + ] + ) + # print(date_dup, fid_dup,dup_data) + dup_data_counts = dup_data["value"].value_counts() + dup_data_counts = dup_data_counts[dup_data_counts > 1] # take the mode value if at least there are 2 duplicates with the same value - v=None + v = None - if len(dup_data_counts > 0) : + if len(dup_data_counts > 0): # get modal value - if (len(dup_data['value'].mode()) == 1): - v = dup_data['value'].mode().iloc[0] + if len(dup_data["value"].mode()) == 1: + v = dup_data["value"].mode().iloc[0] else: - print (dup_data['value'].mode()) + print(dup_data["value"].mode()) else: # take value from user X first for user_id in power_users: - if v is None and len(dup_data[dup_data.user_id==user_id].sort_values(by=['updated_at'], ascending=False).value) >0: - v = dup_data[dup_data.user_id==user_id].sort_values(by=['updated_at'], ascending=False).value.iloc[0] + if ( + v is None + and len( + dup_data[dup_data.user_id == user_id] + .sort_values(by=["updated_at"], ascending=False) + .value + ) + > 0 + ): + v = ( + dup_data[dup_data.user_id == user_id] + .sort_values(by=["updated_at"], ascending=False) + .value.iloc[0] + ) if v is None: # this means that all values were different, and no power user - take latest data - v=dup_data.sort_values(by=['updated_at']).value.iloc[0] + v = dup_data.sort_values(by=["updated_at"]).value.iloc[0] counter += 1 if (counter % 1000) == 0: - print('.', end="") + print(".", end="") if (counter % 50000) == 0: print("") - #ensure that all values in the dataframe are set to the value we wanted to have - data.loc[(data.observation_date == date_dup) & (data.field_id == fid_dup), 'value'] = v - print ("") - return data.drop_duplicates(subset=['field_id','observation_date']) + # ensure that all values in the dataframe are set to the value we wanted to have + data.loc[ + (data.observation_date == date_dup) & (data.field_id == fid_dup), "value" + ] = v + print("") + return data.drop_duplicates(subset=["field_id", "observation_date"]) -#%% testing script with one station +# %% testing script with one station # stuff to decide (set up in argparse) -dataset='DRAW' -useCsv=False -keep_wind_cloud_type=True +dataset = "DRAW" +useCsv = False +keep_wind_cloud_type = True useUtcOffset = True -#%% connect to db -dbHost = os.getenv("localhost") +# %% connect to db +dbHost = os.getenv("localhost") dbPort = 3306 -dbUser = os.getenv("db_root") +dbUser = os.getenv("db_root") dbPasswd = os.getenv("db_password_root") dbDB = "DRAW" mydb = pymysql.connect( - host=dbHost, - port=dbPort, - user=dbUser, - password=dbPasswd, - database=dbDB - ) + host=dbHost, port=dbPort, user=dbUser, password=dbPasswd, database=dbDB +) url = URL.create( - "mysql+mysqlconnector", - username=dbUser, - host=dbHost, - port=dbPort, - password=dbPasswd, - database=dbDB - ) + "mysql+mysqlconnector", + username=dbUser, + host=dbHost, + port=dbPort, + password=dbPasswd, + database=dbDB, +) engine = sqlalchemy.create_engine(url) - -#%% +# %% # First, get station metadata information (turn in function get_metadata) # query = "select mg.source, mg.project, ms.stationName, mg.country, ms.latitude, ms.longitude, ms.elevation, mg.link, \ # ms.timeZone, ms.UTCoffset from metadata_global mg join MetadataStations ms on ms.project = mg.project ;" @@ -327,22 +581,33 @@ def removeDuplicates(data, project): metadata = pd.read_sql(query, conn) -project = metadata.at[0, 'project'] +project = metadata.at[0, "project"] stations = metadata.station_name.unique() -#%% +# %% if project == "NORTHERN": - stationOverrides=pd.DataFrame(columns=['stationName','overrideType', 'odr_type', 'fromDate', \ - 'toDate', 'initialValue', 'overrideValue']) - query = "select stationName, overrideType, odr_type, fromDate, toDate, initialValue, overrideValue \ + stationOverrides = pd.DataFrame( + columns=[ + "stationName", + "overrideType", + "odr_type", + "fromDate", + "toDate", + "initialValue", + "overrideValue", + ] + ) + query = ( + "select stationName, overrideType, odr_type, fromDate, toDate, initialValue, overrideValue \ from station_override" + ) with engine.connect() as conn: - stationOverrides = pd.read_sql(query, conn) - conn.close() + stationOverrides = pd.read_sql(query, conn) + conn.close() else: conn.close() if project == "DRAW": - metadata = metadata.rename(columns={'station_name': 'stationName'}) -#%% then get fields metadata from database + metadata = metadata.rename(columns={"station_name": "stationName"}) +# %% then get fields metadata from database # fetch all field types from database query = "select id, field_key, internal_name, odr_type, period, time_of_day, measurement_unit_original, measurement_unit_si, data_type from fields f ;" @@ -351,50 +616,56 @@ def removeDuplicates(data, project): fields_meta = pd.read_sql(query, conn) conn.close() -odr_types = pd.DataFrame(fields_meta.loc[:, 'odr_type']) +odr_types = pd.DataFrame(fields_meta.loc[:, "odr_type"]) odr_types = odr_types.dropna() odr_types = np.unique(odr_types) -field_ids = pd.DataFrame(fields_meta.loc[:, 'id']) +field_ids = pd.DataFrame(fields_meta.loc[:, "id"]) field_ids = field_ids.dropna() field_ids = np.unique(field_ids) - -#%% get user statistics - number of entries per user (turn into function) -querytext = "select user_id,count(value) from data_entries de join users u on de.user_id = u.id \ +# %% get user statistics - number of entries per user (turn into function) +querytext = ( + "select user_id,count(value) from data_entries de join users u on de.user_id = u.id \ group by user_id order by count(value) desc ;" +) with engine.connect() as conn: users = pd.read_sql(querytext, conn) conn.close() -#%% run if getting data from database +# %% run if getting data from database ## for testing only if project != "DRAW": for station in stations: print(station) - querytext = "select pi2.location, a.observation_date, de1.value, f.odr_type, f.measurement_unit_original,f.field_key, field_id, de1.id, de1.annotation_id, a.transcription_id, de1.page_id, p.image_file_name,t.user_id, t.created_at, t.updated_at from annotations a join data_entries de1 on (de1.annotation_id=a.id) join pages p on (p.id=a.page_id) join fields f on de1.field_id = f.id join page_infos pi2 on (pi2.page_id = de1.page_id) join transcriptions t on (t.id = a.transcription_id) where a.page_id in (select id from pages where location='"+station+"') order by a.observation_date asc, a.updated_at desc;" + querytext = ( + "select pi2.location, a.observation_date, de1.value, f.odr_type, f.measurement_unit_original,f.field_key, field_id, de1.id, de1.annotation_id, a.transcription_id, de1.page_id, p.image_file_name,t.user_id, t.created_at, t.updated_at from annotations a join data_entries de1 on (de1.annotation_id=a.id) join pages p on (p.id=a.page_id) join fields f on de1.field_id = f.id join page_infos pi2 on (pi2.page_id = de1.page_id) join transcriptions t on (t.id = a.transcription_id) where a.page_id in (select id from pages where location='" + + station + + "') order by a.observation_date asc, a.updated_at desc;" + ) # debugLog.write(query+"\n") + # TODO: THIS THROWS AWAY ALL BUT THE LAST QUERY ... WHY with engine.connect() as conn: data = pd.read_sql(querytext, conn) conn.close() else: - querytext = "select a.observation_date, de1.value, f.odr_type, f.measurement_unit_original,f.field_key, field_id, de1.id, de1.annotation_id, a.transcription_id, de1.page_id, p.image_file_name,t.user_id, t.created_at, t.updated_at from annotations a join data_entries de1 on (de1.annotation_id=a.id) join pages p on (p.id=a.page_id) join fields f on de1.field_id = f.id join transcriptions t on (t.id = a.transcription_id) order by a.observation_date asc, a.updated_at desc;" - # debugLog.write(query+"\n") - with engine.connect() as conn: - data = pd.read_sql(querytext, conn) - conn.close() - -data.observation_date = data.observation_date.astype(str) -#%% + querytext = "select a.observation_date, de1.value, f.odr_type, f.measurement_unit_original,f.field_key, field_id, de1.id, de1.annotation_id, a.transcription_id, de1.page_id, p.image_file_name,t.user_id, t.created_at, t.updated_at from annotations a join data_entries de1 on (de1.annotation_id=a.id) join pages p on (p.id=a.page_id) join fields f on de1.field_id = f.id join transcriptions t on (t.id = a.transcription_id) order by a.observation_date asc, a.updated_at desc;" + # debugLog.write(query+"\n") + with engine.connect() as conn: + data = pd.read_sql(querytext, conn) + conn.close() + +data.observation_date = data.observation_date.astype(str) +# %% if project != "DRAW": - data = data.rename(columns={'location': 'stationName'}) + data = data.rename(columns={"location": "stationName"}) else: # add in location column - data['stationName'] = stations[0] + data["stationName"] = stations[0] station = stations[0] - - # %% - #if useCsv: # use if reading in from off-line csv files + + # %% + # if useCsv: # use if reading in from off-line csv files # path = '/Users/vicky/WorkGreen/ACRE-Canada/ECCC_work/data_output/transform_manual_csv/' # file = 'fortsimpson_1849-1850_in.csv' # data = pd.read_csv(path+file) @@ -405,13 +676,13 @@ def removeDuplicates(data, project): fields_list = pd.DataFrame(data.field_id.unique()) fields_list.columns = ["field_id"] - #station = data.loc[1, 'StationName'] - #stations = [(station)] #??? + # station = data.loc[1, 'StationName'] + # stations = [(station)] #??? - utcOffset=0 + utcOffset = 0 if useUtcOffset: - utcOffset=float(metadata[metadata.stationName ==station].UTCoffset.iloc[0]) - timezone=metadata[metadata.stationName == station].timeZone.iloc[0] + utcOffset = float(metadata[metadata.stationName == station].UTCoffset.iloc[0]) + timezone = metadata[metadata.stationName == station].timeZone.iloc[0] odr_list = pd.DataFrame(data.odr_type.unique()) odr_list.columns = ["odr_type"] @@ -420,56 +691,103 @@ def removeDuplicates(data, project): debugLog = open(station + ".log", "w") # log with error notes value = None # inititalize values as None - #data=removeDuplicates(data, project) + # data=removeDuplicates(data, project) # %% go through values by field type stn_fields = fields_meta[fields_meta.id.isin(data_fields_list)] dates = getDatesWithData(engine, stn_fields, station) - #Go through each odr type of the station + # Go through each odr type of the station # %% set up dataframe and file names for output # set up dataframe for cleaned data, original units - df_clean = pd.DataFrame(columns=['StationName', 'Timezone', 'UTCoffset', 'ObservationDate', 'UTCDate', 'value', - 'unit', 'field_key', 'fieldID', 'annotationID', 'transcriptionID', 'pageID']) + df_clean = pd.DataFrame( + columns=[ + "StationName", + "Timezone", + "UTCoffset", + "ObservationDate", + "UTCDate", + "value", + "unit", + "field_key", + "fieldID", + "annotationID", + "transcriptionID", + "pageID", + ] + ) # set up dataframe for cleaned data, iso units - df_iso = pd.DataFrame(columns=['StationName', 'Timezone', 'UTCoffset', 'ObservationDate', 'UTCDate', 'origValue', - 'value', 'unit', 'field_key', 'fieldID', 'annotationID', 'transcriptionID', 'pageID']) + df_iso = pd.DataFrame( + columns=[ + "StationName", + "Timezone", + "UTCoffset", + "ObservationDate", + "UTCDate", + "origValue", + "value", + "unit", + "field_key", + "fieldID", + "annotationID", + "transcriptionID", + "pageID", + ] + ) # filenames for csv file - filename_clean = metadata.at[0, 'source'] + "_" + metadata.at[0, 'project'] + "_" + station + \ - "_" + "clean" + ".csv" - #df_clean.to_csv(filename_clean) - - filename_iso = metadata.at[0, 'source'] + "_" + metadata.at[0, 'project'] + "_" + station + \ - "_" "iso" + ".csv" - #df_iso.to_csv(filename_iso) + filename_clean = ( + metadata.at[0, "source"] + + "_" + + metadata.at[0, "project"] + + "_" + + station + + "_" + + "clean" + + ".csv" + ) + # df_clean.to_csv(filename_clean) + + filename_iso = ( + metadata.at[0, "source"] + "_" + metadata.at[0, "project"] + "_" + station + "_" + "iso" + ".csv" + ) + # df_iso.to_csv(filename_iso) clean_list = [] iso_list = [] -#%% + # %% for type in stn_fields.odr_type.unique(): print(type) # name of tsv (sef) file starts to be composed here # set "2", etc to match station - filename = metadata.at[0, "project"] + "_" + metadata.at[0, "source"] + "_" + station + "_" + filename = ( + metadata.at[0, "project"] + + "_" + + metadata.at[0, "source"] + + "_" + + station + + "_" + ) t = type type_result_set = [] type_error_set = [] - field_id = stn_fields.loc[stn_fields['odr_type'] == t, 'id'] + field_id = stn_fields.loc[stn_fields["odr_type"] == t, "id"] - # we want to go through the list of fields that are assoiated to the type - field_ids=stn_fields[stn_fields.odr_type == t].id + # we want to go through the list of fields that are assoiated to the type + field_ids = stn_fields[stn_fields.odr_type == t].id for field in field_ids: - timeOfDay = stn_fields[stn_fields.id==field].time_of_day.iloc[0] - unit = stn_fields[stn_fields.id==field].measurement_unit_original.iloc[0] - data_type = stn_fields[stn_fields.id==field].data_type.iloc[0] + timeOfDay = stn_fields[stn_fields.id == field].time_of_day.iloc[0] + unit = stn_fields[stn_fields.id == field].measurement_unit_original.iloc[0] + data_type = stn_fields[stn_fields.id == field].data_type.iloc[0] print(field, timeOfDay, unit) - results=data[(data.odr_type == t) & (data.field_id ==field)].sort_values(by='observation_date') + results = data[(data.odr_type == t) & (data.field_id == field)].sort_values( + by="observation_date" + ) previous_result_day = None for index, result in results.iterrows(): - flag = "passed" result_day = parse(result.observation_date) value = result.value @@ -482,73 +800,166 @@ def removeDuplicates(data, project): odr_type = t annotation_id = result.annotation_id page_id = result.page_id - fields={"id": str(field), "name": field_name} + fields = {"id": str(field), "name": field_name} # print(value) if value == "": value = None error_code = 0 - # if more than one transcription take the most recent - if (previous_result_day is None or - (result_day != previous_result_day and value is not None)): + # if more than one transcription take the most recent + if previous_result_day is None or ( + result_day != previous_result_day and value is not None + ): error = None result_day = result_day.replace(hour=0) result_day = result_day.replace(minute=0) - (period, non_utc_result_day, hour, minute) = tu.getUTCResultDay (0, stationOverrides, t, timeOfDay, result_day, station_name) - (period, utc_result_day, hour, minute) = \ - tu.getUTCResultDay(utcOffset, stationOverrides, t, - timeOfDay, result_day, station_name) - unit = tu.getUnit(stationOverrides, unit, t, result_day, station_name) - result_datetime = tu.getDateTimeResult(stationOverrides, t, timeOfDay, result_day, station_name) - + (period, non_utc_result_day, hour, minute) = tu.getUTCResultDay( + 0, stationOverrides, t, timeOfDay, result_day, station_name + ) + (period, utc_result_day, hour, minute) = tu.getUTCResultDay( + utcOffset, + stationOverrides, + t, + timeOfDay, + result_day, + station_name, + ) + unit = tu.getUnit( + stationOverrides, unit, t, result_day, station_name + ) + result_datetime = tu.getDateTimeResult( + stationOverrides, t, timeOfDay, result_day, station_name + ) # Transform and validate data into SI - error_prefix = ("Field: " + str(field_id) + " - " + field_name + "\t" + - result_day.strftime('%Y-%m-%d') + " " + str(hour) + ":00:00\t " + - str(value) + "\t") - - #if t=="pr" and value!="Empty": + error_prefix = ( + "Field: " + + str(field_id) + + " - " + + field_name + + "\t" + + result_day.strftime("%Y-%m-%d") + + " " + + str(hour) + + ":00:00\t " + + str(value) + + "\t" + ) + + # if t=="pr" and value!="Empty": # print ("what is going on I wonder") # Pre-processing value entry if value is not None: - (correctedValue, flag) = td.preProcess(value, debugLog, result_day, dates, fields, - transcription_id, unit) + (correctedValue, flag) = td.preProcess( + value, + debugLog, + result_day, + dates, + fields, + transcription_id, + unit, + ) pre_iso_value = correctedValue # write cleaned value to dataframe - new_row_clean = (station_name, timezone, str(utcOffset), non_utc_result_day, utc_result_day, - str(pre_iso_value), unit, field_name, str(field_id), str(annotation_id), - str(transcription_id), str(page_id)) - #df_clean.loc[i] = new_row_clean + new_row_clean = ( + station_name, + timezone, + str(utcOffset), + non_utc_result_day, + utc_result_day, + str(pre_iso_value), + unit, + field_name, + str(field_id), + str(annotation_id), + str(transcription_id), + str(page_id), + ) + # df_clean.loc[i] = new_row_clean clean_list.append(new_row_clean) - debugLog.write(station_name + "," + str(utc_result_day) + "," + str(pre_iso_value) + - "," + field_name + ", " + str(field_id) + "," + str(annotation_id) + - "," + str(transcription_id) + "," + str(page_id) + "\n") + debugLog.write( + station_name + + "," + + str(utc_result_day) + + "," + + str(pre_iso_value) + + "," + + field_name + + ", " + + str(field_id) + + "," + + str(annotation_id) + + "," + + str(transcription_id) + + "," + + str(page_id) + + "\n" + ) else: pre_iso_value = None if pre_iso_value is not None: - (flag, correctedValue, error) = td.getProcessedDataValue(flag, unit, pre_iso_value, - error_prefix, data_entry_id, - debugLog, t, transcription_id, - result_day, fields, hour, - keep_wind_cloud_type) + (flag, correctedValue, error) = td.getProcessedDataValue( + flag, + unit, + pre_iso_value, + error_prefix, + data_entry_id, + debugLog, + t, + transcription_id, + result_day, + fields, + hour, + keep_wind_cloud_type, + ) # #write ISO value to table in database if correctedValue is not None: - - new_row_iso = (station_name, timezone, str(utcOffset), non_utc_result_day, utc_result_day, - str(pre_iso_value), str(correctedValue), unit, str(field_name), str(field_id), - str(annotation_id), str(transcription_id), str(page_id)) + new_row_iso = ( + station_name, + timezone, + str(utcOffset), + non_utc_result_day, + utc_result_day, + str(pre_iso_value), + str(correctedValue), + unit, + str(field_name), + str(field_id), + str(annotation_id), + str(transcription_id), + str(page_id), + ) # df_iso.loc[j] = new_row_iso iso_list.append(new_row_iso) # build line to be written to SEF file - resultAsString = (utc_result_day.strftime('%Y') + "\t" + str(utc_result_day.strftime('%m')) + - "\t" + str(utc_result_day.strftime('%d')) + "\t" + - str(utc_result_day.strftime('%H')) + "\t" + str(minute) + "\t" + str(period) + - "\t" + str(correctedValue) + "\t|" + "\torig=" + str(value) + - " " + str(unit) + "|Local time: " + str(timeOfDay) + "|QC flag: " + - str(flag) + "|Image File: ") + resultAsString = ( + utc_result_day.strftime("%Y") + + "\t" + + str(utc_result_day.strftime("%m")) + + "\t" + + str(utc_result_day.strftime("%d")) + + "\t" + + str(utc_result_day.strftime("%H")) + + "\t" + + str(minute) + + "\t" + + str(period) + + "\t" + + str(correctedValue) + + "\t|" + + "\torig=" + + str(value) + + " " + + str(unit) + + "|Local time: " + + str(timeOfDay) + + "|QC flag: " + + str(flag) + + "|Image File: " + ) if image_file_name is None: resultAsString += "None" else: @@ -562,7 +973,6 @@ def removeDuplicates(data, project): error = None previous_result_day = result_day - if len(type_result_set) > 0: sorted_type_results = sorted(type_result_set) # sort results by date # need the start and end dates for naming the SEF file @@ -571,7 +981,9 @@ def removeDuplicates(data, project): index_start = 0 while start_found == 0 and index_start < len(type_result_set): entry_value = sorted_type_results[index_start].split("\t")[6] - if entry_value == "-999": # we want to remove the leading -999 ato make the SEF file more compact + if ( + entry_value == "-999" + ): # we want to remove the leading -999 ato make the SEF file more compact index_start += 1 else: start_found = 1 @@ -580,35 +992,63 @@ def removeDuplicates(data, project): index_end = -1 while end_found == 0 and index_end > -len(type_result_set): entry_value = sorted_type_results[index_end].split("\t")[6] - if entry_value == "-999": # we want to remove the trailing -999 ato make the SEF file more compact + if ( + entry_value == "-999" + ): # we want to remove the trailing -999 ato make the SEF file more compact index_end -= 1 else: end_found = 1 if index_start < len(type_result_set): startStr = sorted_type_results[index_start].split("\t") - filename = filename+startStr[0] + "-"+startStr[1] + "_" + filename = filename + startStr[0] + "-" + startStr[1] + "_" endStr = sorted_type_results[index_end].split("\t") # complete the building of the SEF filename - filename = filename+endStr[0] + "-"+endStr[1] + "-" + t + ".tsv" + filename = filename + endStr[0] + "-" + endStr[1] + "-" + t + ".tsv" print(filename) f = open(filename, "w", encoding="utf-8") # write the metadata for the SEF file f.write("SEF\t1.0.0\n") - f.write("ID\t"+ station +"\n") - f.write("Name\t" + station +"\n") - f.write("Lat\t" + str(metadata[metadata.stationName == station].latitude) +"\n") - f.write("Lon\t" + str(metadata[metadata.stationName == station].longitude) +"\n") - f.write("Alt\t" + str(metadata[metadata.stationName == station].elevation) +"\n") - f.write("Source\t" + str(metadata[metadata.stationName == station].source) + "\n") - f.write("Link\t" + str(metadata[metadata.stationName == station].link)+"\n") - f.write("Vbl\t" + t.split("_")[0]+"\n") + f.write("ID\t" + station + "\n") + f.write("Name\t" + station + "\n") + f.write( + "Lat\t" + + str(metadata[metadata.stationName == station].latitude) + + "\n" + ) + f.write( + "Lon\t" + + str(metadata[metadata.stationName == station].longitude) + + "\n" + ) + f.write( + "Alt\t" + + str(metadata[metadata.stationName == station].elevation) + + "\n" + ) + f.write( + "Source\t" + + str(metadata[metadata.stationName == station].source) + + "\n" + ) + f.write( + "Link\t" + + str(metadata[metadata.stationName == station].link) + + "\n" + ) + f.write("Vbl\t" + t.split("_")[0] + "\n") f.write("Stat\t") if "mean" in t: f.write("mean\n") else: f.write("point\n") - f.write("Unit\t" + str(stn_fields[stn_fields.id==field].measurement_unit_si.iloc[0]) + "\n") + f.write( + "Unit\t" + + str( + stn_fields[stn_fields.id == field].measurement_unit_si.iloc[0] + ) + + "\n" + ) f.write("Meta\t") # indicate whether the pressure has been temperature gravity corrected # if (sub_type=='PGC_PTC'): @@ -621,7 +1061,7 @@ def removeDuplicates(data, project): # f.write("NO") # f.write("Observer Name="+str(observer_name)) f.write("\tUTCOffset=") # indicate if the time is in UTC or local - if (utcOffset != 0): + if utcOffset != 0: f.write("Applied\tUTCOffset=" + str(utcOffset)) # UTC time else: f.write("NO") # local time @@ -629,28 +1069,68 @@ def removeDuplicates(data, project): f.write("Year\tMonth\tDay\tHour\tMinute\tPeriod\tValue\t|\tMeta\n") index = 0 for res in sorted_type_results: - if index >= index_start and index <= index_end + len(sorted_type_results): + if index >= index_start and index <= index_end + len( + sorted_type_results + ): f.write(res) # write the main body of the SEF file index += 1 if len(type_error_set) > 0: - error_filename = str(metadata[metadata.stationName == station].source) + "_" + \ - str(metadata[metadata.stationName == station].project)+"_"+station +"_"+t+".err" + error_filename = ( + str(metadata[metadata.stationName == station].source) + + "_" + + str(metadata[metadata.stationName == station].project) + + "_" + + station + + "_" + + t + + ".err" + ) error_file = open(error_filename, "w") for err in type_error_set: error_file.write(err) - df_clean = pd.DataFrame(clean_list, columns=["StationName", "Timezone", "UTCoffset", "ObservationDate", "UTCDate", "value", "unit", - "field_key", "fieldID", "annotationID", "transcriptionID", "pageID"]).astype("string") - df_iso = pd.DataFrame(iso_list, columns=["StationName", "Timezone", "UTCoffset", "ObservationDate", "UTCDate", "origValue", - "value", "unit", "field_key", "fieldID", "annotationID", "transcriptionID", "pageID"]).astype("string") - - - df_clean.to_csv(filename_clean, encoding='utf-8', index = False, mode='a') - df_iso.to_csv(filename_iso, encoding='utf-8', index = False, mode='a') + df_clean = pd.DataFrame( + clean_list, + columns=[ + "StationName", + "Timezone", + "UTCoffset", + "ObservationDate", + "UTCDate", + "value", + "unit", + "field_key", + "fieldID", + "annotationID", + "transcriptionID", + "pageID", + ], + ).astype("string") + df_iso = pd.DataFrame( + iso_list, + columns=[ + "StationName", + "Timezone", + "UTCoffset", + "ObservationDate", + "UTCDate", + "origValue", + "value", + "unit", + "field_key", + "fieldID", + "annotationID", + "transcriptionID", + "pageID", + ], + ).astype("string") + + df_clean.to_csv(filename_clean, encoding="utf-8", index=False, mode="a") + df_iso.to_csv(filename_iso, encoding="utf-8", index=False, mode="a") print("write csv") with engine.connect() as conn: - df_clean.to_sql('data_clean', conn, None, 'append', chunksize=1000) - df_iso.to_sql('data_iso', conn, None, 'append', chunksize=1000) + df_clean.to_sql("data_clean", conn, None, "append", chunksize=1000) + df_iso.to_sql("data_iso", conn, None, "append", chunksize=1000) conn.close() diff --git a/data_post_processing/setup_raw_data_table.py b/data_post_processing/setup_raw_data_table.py deleted file mode 100644 index b0fb811..0000000 --- a/data_post_processing/setup_raw_data_table.py +++ /dev/null @@ -1,16 +0,0 @@ -import tables -import config - - -def set_up_raw_data_table(continue_flag): - tables.add_ppid_column_fields_table() - tables.update_fields_ppid(1, config.ppid_to_field_id[1]) - tables.update_fields_ppid(2, config.ppid_to_field_id[2]) - tables.update_fields_ppid(3, config.ppid_to_field_id[3]) - tables.update_fields_ppid(4, config.ppid_to_field_id[4]) - tables.update_fields_ppid(5, config.ppid_to_field_id[5]) - tables.update_fields_ppid(6, config.ppid_to_field_id[6]) - tables.update_fields_ppid(7, config.ppid_to_field_id[7]) - # TODO : update other field id's with their respective pp_id - - tables.create_raw_data_table(continue_flag) diff --git a/data_post_processing/time_utils.py b/data_post_processing/time_utils.py index 6a227f1..a0885ea 100644 --- a/data_post_processing/time_utils.py +++ b/data_post_processing/time_utils.py @@ -1,73 +1,78 @@ -# -*- coding: utf-8 -*- -import datetime +# # -*- coding: utf-8 -*- +# import datetime -def getTimeOfDay(cfg, sefType, time, date): - if "changetimeOfDay" in cfg and cfg["changetimeOfDay"]["active"] is True: - for timeChange in cfg["timeChanges"]: - if sefType in timeChange["sefTypes"]: - for change in timeChange["changes"]: - if (change["fromTime"] == time and - date >= datetime.datetime.strptime(change["fromDate"], "%Y-%m-%d") and - date <= datetime.datetime.strptime(change["toDate"], "%Y-%m-%d")): - return change["toTime"] - return time +# def getTimeOfDay(cfg, sefType, time, date): +# if "changetimeOfDay" in cfg and cfg["changetimeOfDay"]["active"] is True: +# for timeChange in cfg["timeChanges"]: +# if sefType in timeChange["sefTypes"]: +# for change in timeChange["changes"]: +# if ( +# change["fromTime"] == time +# and date +# >= datetime.datetime.strptime(change["fromDate"], "%Y-%m-%d") +# and date +# <= datetime.datetime.strptime(change["toDate"], "%Y-%m-%d") +# ): +# return change["toTime"] +# return time -def getDateTimeResult(cfg, t, timeOfDay, result_day): - utcOffset = 0 - if (timeOfDay == "mean" or timeOfDay == "total" or timeOfDay == "daily"): - period = '24' - hour = 0 - minute = '00' - dateResult = '' - elif timeOfDay == "sunrise": - period = '0' - hour = 6 - minute = "{0:02d}".format(int(0+60*(utcOffset-int(utcOffset)))) - else: - period = '0' - overridenTimeOfDay = getTimeOfDay(cfg, t, timeOfDay, result_day) - hour = int(overridenTimeOfDay[:2]) - minute = int(overridenTimeOfDay[2:]) - # hour=hour + utcOffset - minute = "{0:02d}".format(int(minute+60*(utcOffset-int(utcOffset)))) - result_day = result_day + datetime.timedelta(hours=hour, minutes=int(minute)) - return (result_day) +# def getDateTimeResult(cfg, t, timeOfDay, result_day): +# utcOffset = 0 +# if timeOfDay == "mean" or timeOfDay == "total" or timeOfDay == "daily": +# period = "24" +# hour = 0 +# minute = "00" +# dateResult = "" +# elif timeOfDay == "sunrise": +# period = "0" +# hour = 6 +# minute = "{0:02d}".format(int(0 + 60 * (utcOffset - int(utcOffset)))) +# else: +# period = "0" +# overridenTimeOfDay = getTimeOfDay(cfg, t, timeOfDay, result_day) +# hour = int(overridenTimeOfDay[:2]) +# minute = int(overridenTimeOfDay[2:]) +# # hour=hour + utcOffset +# minute = "{0:02d}".format(int(minute + 60 * (utcOffset - int(utcOffset)))) +# result_day = result_day + datetime.timedelta(hours=hour, minutes=int(minute)) +# return result_day -def getUTCResultDay(utcOffset, cfg, t, timeOfDay, result_day): - if (timeOfDay == "mean" or timeOfDay == "total" or timeOfDay == "daily"): - period = '24' - hour = 0 - minute = '00' - elif timeOfDay == "sunrise": - period = '0' - hour = 6 + utcOffset - minute = "{0:02d}".format(int(0+60*(utcOffset-int(utcOffset)))) - else: - period = '0' - overridenTimeOfDay = getTimeOfDay(cfg, t, timeOfDay, result_day) - hour = int(overridenTimeOfDay[:2]) - minute = int(overridenTimeOfDay[2:]) - hour = hour + utcOffset - minute = "{0:02d}".format(int(minute+60*(utcOffset-int(utcOffset)))) - utc_result_day = result_day + datetime.timedelta(hours=hour, minutes=int(minute)) - return (period, utc_result_day, hour, minute) +# def getUTCResultDay(utcOffset, cfg, t, timeOfDay, result_day): +# if timeOfDay == "mean" or timeOfDay == "total" or timeOfDay == "daily": +# period = "24" +# hour = 0 +# minute = "00" +# elif timeOfDay == "sunrise": +# period = "0" +# hour = 6 + utcOffset +# minute = "{0:02d}".format(int(0 + 60 * (utcOffset - int(utcOffset)))) +# else: +# period = "0" +# overridenTimeOfDay = getTimeOfDay(cfg, t, timeOfDay, result_day) +# hour = int(overridenTimeOfDay[:2]) +# minute = int(overridenTimeOfDay[2:]) +# hour = hour + utcOffset +# minute = "{0:02d}".format(int(minute + 60 * (utcOffset - int(utcOffset)))) +# utc_result_day = result_day + datetime.timedelta(hours=hour, minutes=int(minute)) +# return (period, utc_result_day, hour, minute) -def getUnit(cfg, unit,sefType, date): +# def getUnit(cfg, unit, sefType, date): - if "unitOverride" in cfg: - for override in cfg["unitOverride"]: - if override["type"] == sefType: - fromDate = "1000-01-01" - toDate = "2024-01-01" - if "fromDate" in override and override["fromDate"] != "": - fromDate = override["fromDate"] - if "toDate" in override and override["toDate"] != "": - toDate = override["toDate"] - if (date >= datetime.datetime.strptime(fromDate, "%Y-%m-%d") and - date <= datetime.datetime.strptime(toDate, "%Y-%m-%d")): - unit = override["unit"] - return unit +# if "unitOverride" in cfg: +# for override in cfg["unitOverride"]: +# if override["type"] == sefType: +# fromDate = "1000-01-01" +# toDate = "2024-01-01" +# if "fromDate" in override and override["fromDate"] != "": +# fromDate = override["fromDate"] +# if "toDate" in override and override["toDate"] != "": +# toDate = override["toDate"] +# if date >= datetime.datetime.strptime( +# fromDate, "%Y-%m-%d" +# ) and date <= datetime.datetime.strptime(toDate, "%Y-%m-%d"): +# unit = override["unit"] +# return unit diff --git a/data_post_processing/transcription_data_processing.py b/data_post_processing/transcription_data_processing.py index bf04ce7..0fa7929 100644 --- a/data_post_processing/transcription_data_processing.py +++ b/data_post_processing/transcription_data_processing.py @@ -1,135 +1,88 @@ # -*- coding: utf-8 -*- -import lmrlib as lmr # code from NOAA to convert historical units to SI -import iso_mapping as im # subroutines written as part of post-processing -import numpy as np +import structlog from numpy import sqrt -import pandas as pd -dateFormat = "%Y-%m-%d " - - -# get metadata for station -# def get_metadata(mydb, mycursor): - -# query = ("select mg.source, mg.project, ms.stationName, mg.country, ms.latitude, ms.longitude, ms.elevation, mg.link,ms.timeZone, ms.UTCoffset from metadata_global mgjoin MetadataStations ms on ms.project = mg.project where stationName like '{station}';") - -# mycursor.execute(query) -# result = mycursor.fetchall() -# source = result[0] -# project = result[1] -# stationName = result[2] -# country = result[3] -# latitude = result[4] -# longitude = result[5] -# elevation = result[6] -# link = result[7] -# timeZone = result[8] -# UTCoffset = result[9] - -# stationID = stationName+country - -# return (source, project, stationName, stationID, latitude, longitude, elevation, link, timeZone, UTCoffset) - - -def get_UserStats(engine): - - querytext = "select user_id,count(value) from data_entries de join users u on de.user_id = u.id group by user_id order by count(value) desc ;" - - with engine.connect() as conn: - users = pd.read_sql(querytext, conn) - - conn.close() - return (users) - - -def get_Fields(engine): - query = "select id, field_key, internal_name, odr_type, measurement_unit_original, measurement_unit_si, data_type from fields f ;" - - with engine.connect() as conn: - fields_meta = pd.read_sql(query, conn) - conn.close() - - types = fields_meta.loc[:, 'odr_type'] - types = types.dropna() - types = np.unique(types) -# units = fields_meta.loc[:, 'measurement_unit_original'] -# units = units.dropna() -# units = np.unique(units) - - return (fields_meta, types) - - -# get dates without data -def isEmptyValue(value): - - v = value.lower() - if "empty" in v or v == "-" or v == " " or v == "" or v == "none" or v == "na": - return True - return False - - -def getDatesWithData(engine, fields_id, station): - - dates=[] - for field in fields_id: - query = "select a.observation_date, de1.value from annotations a join data_entries de1 on (de1.annotation_id=a.id and de1.field_id=" + str(field) + ") join pages p on (p.id=a.page_id) join page_infos pi2 on pi2.page_id = de1.page_id where a.page_id in (select id from pages where title like '%" + str(station) + "%') order by a.observation_date asc, a.updated_at desc ;" - - with engine.connect() as conn: - results = pd.read_sql(query, conn) - conn.close() - for result in results: - value = result[1] - date = result[0].replace(hour=0, minute=0, second=0) - if date not in dates and isEmptyValue(value) == False: - dates.append(date) - return dates +import data_post_processing.iso_mapping as im # subroutines written as part of post-processing +import data_post_processing.lmrlib as lmr # code from NOAA to convert historical units to SI # pre-processing # removes extraneous characters -def preProcess(value, debugLog, result_day, dates, fields, transcription_id, unit): +def preProcess(value: str, unit: str, logger: structlog.BoundLogger): if value is not None: value = value.casefold() correctedValue = value if unit == "mno" and value is not None: return (value, None) -# variables where empty is not missing but no value or zero value, e.g. precipitation, wind - if result_day in dates: - if (unit == "Bf" or unit == "Sm" or unit == "mph" or unit == "lbsft" or unit == "lct" or - unit == "uct" or unit == "cloudvel" or unit == "okta" or unit == "in") and (value == "empty" or value == "dot"): - return ("0", "empty") - elif unit == "in" and value == "None": - return ("0", "empty") - elif unit == "dir" and value == "empty": - return ("calm", "empty") - -# Handle the case where the data is considered as missing - missing_terms = ["not.taken", "not taken", "unknown symbol", "retracted", "-999", "none", "no grass", - "no place", "suspended", "abt on duty"] - instrument_error_terms = ["out of order", "out.of.order", "broken", "unserviceable", "-888", "incorrect", - "not reliable", "covered in snow", "covered with snow", "observer", "no error"] - - if (value is None or value == "-" or value == " " or value == "" or - any([x.lower() in value for x in missing_terms])): - correctedValue = '-999' + # variables where empty is not missing but no value or zero value, e.g. precipitation, wind + # if result_day in dates: + if ( + unit == "Bf" + or unit == "Sm" + or unit == "mph" + or unit == "lbsft" + or unit == "lct" + or unit == "uct" + or unit == "cloudvel" + or unit == "okta" + or unit == "in" + ) and (value == "empty" or value == "dot"): + return ("0", "empty") + elif unit == "in" and value == "None": + return ("0", "empty") + elif unit == "dir" and value == "empty": + return ("calm", "empty") + + # Handle the case where the data is considered as missing + missing_terms = [ + "not.taken", + "not taken", + "unknown symbol", + "retracted", + "-999", + "none", + "no grass", + "no place", + "suspended", + "abt on duty", + ] + instrument_error_terms = [ + "out of order", + "out.of.order", + "broken", + "unserviceable", + "-888", + "incorrect", + "not reliable", + "covered in snow", + "covered with snow", + "observer", + "no error", + ] + + if ( + value is None + or value == "-" + or value == " " + or value == "" + or any([x.lower() in value for x in missing_terms]) + ): + correctedValue = "-999" flag = "missing" - elif (any([x.lower() in value for x in instrument_error_terms])): - correctedValue = '-888' + elif any([x.lower() in value for x in instrument_error_terms]): + correctedValue = "-888" flag = "instrument error" - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n") + logger.debug("Invalid data", value=value) - elif ("illegible" in value and "cirrus" not in value): - correctedValue = '-888' + elif "illegible" in value and "cirrus" not in value: + correctedValue = "-888" flag = "illegible" - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n") + logger.debug("Invalid data", value=value) - elif ("illegible" in value): - correctedValue = '-888' + elif "illegible" in value: + correctedValue = "-888" flag = "illegible" - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n") + logger.debug("Invalid data", value=value) else: correctedValue = value @@ -139,125 +92,486 @@ def preProcess(value, debugLog, result_day, dates, fields, transcription_id, uni # get processed (cleaned and SI unit) value depending on unit. Including errors and debug. Keep wind and cloud type # returns values in cloud type (Ci, Cu, etc) and compass rose directions -def getProcessedDataValue(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type): +def getProcessedDataValue( + flag: str, + value: str, + data_entry_id: int, + field_id: int, + field_name: str, + odr_type: str, + unit: str, + keep_wind_cloud_type: bool, + logger: structlog.BoundLogger, + # flag, + # # unit, + # value, + # # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # # transcription_id, + # result_day, + fields, + # hour, + # keep_wind_cloud_type, +): value = value.casefold() match unit: case "inHg": # inches of mercury e.g. barometer, vapour pressure - return getInHg(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getInHg( + flag=flag, + value=value, + data_entry_id=data_entry_id, + odr_type=odr_type, + logger=logger, + ) case "mmHg": # mm of mercury e.g. barometer, vapour pressure - return getmmHg(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getmmHg( + flag=flag, + value=value, + data_entry_id=data_entry_id, + odr_type=odr_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "F": # all thermometer values - return getF(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, - hour, keep_wind_cloud_type) - case "ºF": # all thermometer values - return getF(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getF( + flag=flag, + value=value, + data_entry_id=data_entry_id, + odr_type=odr_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) + # case "ºF": # all thermometer values + # return getF( + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + # ) case "p": # percentage values - return getP(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getP( + flag=flag, + value=value, + data_entry_id=data_entry_id, + odr_type=odr_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "rh": # relative humidty - return getRH(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getRH( + flag=flag, + value=value, + data_entry_id=data_entry_id, + odr_type=odr_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "Sm": # Smithsonian wind scale (0-12) - return getSM(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getSM( + flag=flag, + value=value, + data_entry_id=data_entry_id, + field_id=field_id, + odr_type=odr_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "Bf": # Beaufort wind scale (0-10), also for cloud velocity - return getBf(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getBf( + flag=flag, + value=value, + data_entry_id=data_entry_id, + field_id=field_id, + odr_type=odr_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "Bf_text": # Beaufort wind scale in text format (light, strong, etc) - return getBf_text(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getBf_text( + flag=flag, + value=value, + data_entry_id=data_entry_id, + field_id=field_id, + odr_type=odr_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "mph": # miles per hour e.g. wind - return getMPH(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getMPH( + flag=flag, + value=value, + data_entry_id=data_entry_id, + field_id=field_id, + odr_type=odr_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "lbsft": # pounds per square inch, e.g. wind force - return getLbsFt(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getLbsFt( + flag=flag, + value=value, + data_entry_id=data_entry_id, + field_id=field_id, + odr_type=odr_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "in": # inches, e.g. precipitation - return getIn(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getIn( + flag=flag, + value=value, + data_entry_id=data_entry_id, + field_id=field_id, + odr_type=odr_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "dir": # direction - return getDir(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getDir( + flag=flag, + value=value, + data_entry_id=data_entry_id, + field_id=field_id, + odr_type=odr_type, + keep_wind_cloud_type=keep_wind_cloud_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + ) case "uct": # upper cloud type - return getUCT(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getUCT( + flag=flag, + value=value, + data_entry_id=data_entry_id, + field_id=field_id, + odr_type=odr_type, + keep_wind_cloud_type=keep_wind_cloud_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "lct": # lower cloud type - return getLCT(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getLCT( + flag=flag, + value=value, + data_entry_id=data_entry_id, + field_id=field_id, + odr_type=odr_type, + keep_wind_cloud_type=keep_wind_cloud_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "mno": # manual observation, e.g. weather - return getMNO(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getMNO( + flag=flag, + value=value, + data_entry_id=data_entry_id, + field_id=field_id, + odr_type=odr_type, + keep_wind_cloud_type=keep_wind_cloud_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "oz": # ozone. No conversion yet - return getOz(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getOz( + flag=flag, + value=value, + data_entry_id=data_entry_id, + field_id=field_id, + odr_type=odr_type, + keep_wind_cloud_type=keep_wind_cloud_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "cloudvel": # cloud velocity - return getCloudVel(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getCloudVel( + flag=flag, + value=value, + data_entry_id=data_entry_id, + field_id=field_id, + odr_type=odr_type, + keep_wind_cloud_type=keep_wind_cloud_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "okta": # cloud cover - return getOkta(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getOkta( + flag=flag, + value=value, + data_entry_id=data_entry_id, + field_id=field_id, + field_name=field_name, + odr_type=odr_type, + keep_wind_cloud_type=keep_wind_cloud_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case "tenths": # cloud cover in tenths /clearness of sky - return getOkta(flag, unit, value, error_prefix, data_entry_id, - debugLog, t, transcription_id, result_day, fields, hour, - keep_wind_cloud_type) + return getOkta( + flag=flag, + value=value, + data_entry_id=data_entry_id, + field_id=field_id, + field_name=field_name, + odr_type=odr_type, + keep_wind_cloud_type=keep_wind_cloud_type, + logger=logger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, + ) case _: return (None, value, None) # Cleaning up value -replace spaces, commas or double points with decimal point -def cleanupValue(error_prefix, value, data_entry_id, flag): +def cleanupValue(value, data_entry_id, flag): error = None correctedValue = value - error_suffix = "UPDATE data_entries set value='" + str(value) + "' WHERE id=" + str(data_entry_id) + ";\n" -# common transcription errors - if (" " in value): + # error_suffix = ( + # "UPDATE data_entries set value='" + # + str(value) + # + "' WHERE id=" + # + str(data_entry_id) + # + ";\n" + # ) + # common transcription errors + if " " in value: value = value.replace(" ", ".") - error = error_prefix + "SYNTAX\tSpace in value\t" + error_suffix - if ("," in value): + error = "SYNTAX\tSpace in value\t" + if "," in value: value = value.replace(",", ".") - error = error_prefix + "SYNTAX\tComma in value\t" + error_suffix - if (".." in value): + error = "SYNTAX\tComma in value\t" + if ".." in value: value = value.replace("..", ".") - error = error_prefix + "SYNTAX\t\"..\" in value\t" + error_suffix - if ("- " in value): # negative sign has space between it and value + error = 'SYNTAX\t".." in value\t' + if "- " in value: # negative sign has space between it and value value = value.replace("- ", "-") - error = error_prefix + "SYNTAX\tSpace in negative sign\t" + error_suffix - if ("+ " in value): # value has unnecessary positive sign + error = "SYNTAX\tSpace in negative sign\t" + if "+ " in value: # value has unnecessary positive sign value = value.replace("+ ", "") - if ("[" in value): # value has unnecessary bracket + if "[" in value: # value has unnecessary bracket value = value.replace("[", "") - if ("]" in value): # value has unnecessary bracket + if "]" in value: # value has unnecessary bracket value = value.replace("]", "") - error = error_prefix + "SYNTAX\tSpace in postive sign\t" + error_suffix - if ("." in value and "/" in value): # remove decimal if value is fraction + error = "SYNTAX\tSpace in postive sign\t" + if "." in value and "/" in value: # remove decimal if value is fraction value = value.replace(".", " ") -# values indicating the observation could not be recorded - if ("\"" in value or "'" in value or "~" in value or "-" in value or "\'\'" in value or "\"\"" in value - or ":" in value or "*" in value or "”" in value): + # values indicating the observation could not be recorded + if ( + '"' in value + or "'" in value + or "~" in value + or "-" in value + or "''" in value + or '""' in value + or ":" in value + or "*" in value + or "”" in value + ): correctedValue = -222 flag = "value unable to be recorded" @@ -266,49 +580,94 @@ def cleanupValue(error_prefix, value, data_entry_id, flag): # convert values based on unit + # Beaufort wind scale conversion -def getBf(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getBf( + flag: str, + value: str, + field_id: int, + data_entry_id: int, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None - value=value.lower() - field_id = fields["id"] - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) + value = value.lower() + # field_id = fields["id"] + (value, error, flag, correctedValue) = cleanupValue(value, data_entry_id, flag) # Some fields contain two different data entries: wind force and direction. Here we select wind force if field_id == "177" or field_id == "178" or field_id == "179" or field_id == "180": value = value.replace(",", " ") # if value != "0": - if ("." in value): + if "." in value: values = value.split(".") else: values = value.split(" ") - debugLog.write(result_day.strftime(dateFormat)+" Wind force. Values:" - + value + " - looking if isdigit for: " + values[-1] + "\n") - if values[-1].isdigit(): # if both direction and force in same entry, see which value is numeric + # debugLog.write( + # result_day.strftime(dateFormat) + # + " Wind force. Values:" + # + value + # + " - looking if isdigit for: " + # + values[-1] + # + "\n" + # ) + if ( + values[-1].isdigit() + ): # if both direction and force in same entry, see which value is numeric value = values[-1] else: value = values[0] - if ("|" in value): + if "|" in value: values = value.split("|") value = values[-1] - debugLog.write("Selected wind force: " + value + "\n") + # debugLog.write("Selected wind force: " + value + "\n") try: - if (int(value) == -222 or int(value) == -888): - value = '-999' # remove -888 and -222 - if (value == "empty" or flag == "empty"): + if int(value) == -222 or int(value) == -888: + value = "-999" # remove -888 and -222 + if value == "empty" or flag == "empty": value = "0" correctedValue = 0 flag = "no entry: calm" - elif (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): + elif ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): from fractions import Fraction + res = float(sum(Fraction(s) for s in value.split())) res = int(res) - if (res < 0 or res > 12): # if value is out of range, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Out of range: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: "+fields["id"] + - " - " + fields["name"] + "\n") + if ( + res < 0 or res > 12 + ): # if value is out of range, write debug log and error type + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Out of range: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) flag = "out of range" correctedValue = -999 else: @@ -318,58 +677,130 @@ def getBf(flag, unit, value, error_prefix, data_entry_id, debugLog, t, correctedValue = "{0:.0f}".format(res) else: correctedValue = -999 - flag = 'missing' + flag = "missing" except ValueError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + " Hour:" + str(hour) + "\tField: " + fields["id"] + - " - " + fields["name"] + "\n") + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) correctedValue = -999 - flag = 'missing' + flag = "missing" return (flag, correctedValue, error) # Beaufort text wind scale conversion -def getBf_text(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getBf_text( + flag: str, + value: str, + field_id: int, + data_entry_id: int, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None value = value.lower() - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) + (value, error, flag, correctedValue) = cleanupValue(value, data_entry_id, flag) try: - if (int(value) == -222 or int(value) == -888): - value = '-999' # remove -888 and -222 - if (value == "empty" or flag == "empty"): + if int(value) == -222 or int(value) == -888: + value = "-999" # remove -888 and -222 + if value == "empty" or flag == "empty": value = "0" correctedValue = 0 flag = "no entry: calm" - elif (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): + elif ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): res = im.convertBeauforttext(value) correctedValue = "{0:.0f}".format(res) else: correctedValue = -999 - flag = 'missing' + flag = "missing" except ValueError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + " Hour:" + str(hour) + "\tField: " + fields["id"] + - " - " + fields["name"] + "\n") + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) correctedValue = -999 - flag = 'missing' + flag = "missing" return (flag, correctedValue, error) # CLoudVel cloud velocity -def getCloudVel(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getCloudVel( + flag: str, + value: str, + field_id: int, + data_entry_id: int, + keep_wind_cloud_type: bool, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) + (value, error, flag, correctedValue) = cleanupValue(value, data_entry_id, flag) try: - if (int(value) == -222 or int(value) == -888): - value = '-999' # remove -888 and -222 - if (value == "empty" or flag == "empty"): + if int(value) == -222 or int(value) == -888: + value = "-999" # remove -888 and -222 + if value == "empty" or flag == "empty": value = "0" correctedValue = "0" flag = "no entry: calm" - if (flag != "missing" and flag != "instrument error" and flag != "illegible" and flag != "empty"): + if ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): # if more than one value in entry, choose first value if len(value) > 1: values = value.split(" ") @@ -381,53 +812,111 @@ def getCloudVel(flag, unit, value, error_prefix, data_entry_id, debugLog, t, res = float(value) correctedValue = res # Check range - if (res < 0 or res > 10): - debugLog.write("\tTranscriptionID: " + str(transcription_id) + "Out of range: " + value + - " on " + result_day.strftime(dateFormat) + "\tField: " + - fields["id"] + " - " + fields["name"]+"\n") - flag = 'out of range' + if res < 0 or res > 10: + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + "Out of range: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + flag = "out of range" correctedValue = -999 correctedValue = "{0:.0f}".format(res) else: correctedValue = -999 - flag = 'missing' + flag = "missing" except ValueError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + - value + " on " + result_day.strftime(dateFormat) + " Hour:" + - str(str(hour) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n")) + correctedValue = -999 + flag = "missing" + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str( + # str(hour) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n" + # ) + # ) return (flag, correctedValue, error) # DIR direction -def getDir(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getDir( + flag: str, + value: str, + field_id: int, + data_entry_id: int, + keep_wind_cloud_type: bool, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) - field_id = fields["id"] + (value, error, flag, correctedValue) = cleanupValue(value, data_entry_id, flag) # Some fields contains both the direction and the speed of the wind, so here we extract the direction - if (field_id == "177" or field_id == "178" or field_id == "179" or - field_id == "180" or field_id == "181" or field_id == "182" or - field_id == "183" or field_id == "184"): + if ( + field_id == "177" + or field_id == "178" + or field_id == "179" + or field_id == "180" + or field_id == "181" + or field_id == "182" + or field_id == "183" + or field_id == "184" + ): value = value.replace(",", " ") if value != "0": values = value.split(" ") # if more than one value in entry split by space - if values[-1].isdigit(): # if the value is a number than treat it as wind velocity + if values[ + -1 + ].isdigit(): # if the value is a number than treat it as wind velocity value = values[0] else: value = values[-1] # else it is wind direction else: value = "Calm" # if value = 0 descriptor is "Calm" - correctedValue = 'Calm' + correctedValue = "Calm" try: - if (value == "empty" or flag == 'empty'): + if value == "empty" or flag == "empty": value = "0" - correctedValue = 'Calm' + correctedValue = "Calm" flag = "no entry: calm" - if (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): - if (value == "Calm" or value == "perceptible" or value == "none" or - value == "imp" or value == "not" in value.lower() or - value == "0" in value.lower()): + if ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): + if ( + value == "Calm" + or value == "perceptible" + or value == "none" + or value == "imp" + or value == "not" in value.lower() + or value == "0" in value.lower() + ): correctedValue = "Calm" elif "variable" in value.lower(): correctedValue = "Variable" @@ -435,99 +924,201 @@ def getDir(flag, unit, value, error_prefix, data_entry_id, debugLog, t, correctedValue = "Scud" elif "Hidden" in value.lower(): correctedValue = "Hidden" - # if no extraneous text found, we will try to map the value and convert + # if no extraneous text found, we will try to map the value and convert if keep_wind_cloud_type: correctedValue = im.abbr_directions[value.split()[0].lower()] else: correctedValue = im.convert_directions[value.split()[0].lower()] else: correctedValue = -999 - flag = 'missing' + flag = "missing" except KeyError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + - "Invalid data: " + value + " on " + result_day.strftime(dateFormat) + - " Hour:" + str(hour) + "\tField: " + fields["id"] + " - " + - fields["name"]+"\n") + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) correctedValue = -999 - flag = 'missing' + flag = "missing" return (flag, correctedValue, error) # Fahrenheit -def getF(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getF( + flag: str, + value: str, + data_entry_id: int, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None try: - if (int(value) == -222 or int(value) == -888): - value = '-999' # remove -888 and -222 + if int(value) == -222 or int(value) == -888: + value = "-999" # remove -888 and -222 # find error = None common errors in temperature data # common signs for values not able to be recorded (instrument error, too cold, etc in cleanup) - if ("tdb" in t or "tb" in t or "td" in t): - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) - if (value.lower() == "empty"): + if "tdb" in odr_type or "tb" in odr_type or "td" in odr_type: + (value, error, flag, correctedValue) = cleanupValue( + value, data_entry_id, flag + ) + if value.lower() == "empty": value = "-999" correctedValue = "-999" flag = "missing" - elif (flag != "missing" and flag != "instrument error" - and flag != "illegible" and flag != "empty" and value != "-999" and value != "-888" and value != "-222"): + elif ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + and value != "-999" + and value != "-888" + and value != "-222" + ): from fractions import Fraction # if value contaisn fraction + temperature = float(sum(Fraction(s) for s in value.split())) - if (temperature < -50 or temperature > 120): # out of range - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Out of range: " + value + - " on " + result_day.strftime(dateFormat) + "\tField: " + fields["id"] + - " - " + fields["name"]+"\n") + if temperature < -50 or temperature > 120: # out of range + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Out of range: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) flag = "out of range" - if (temperature < -45): # low values + if temperature < -45: # low values flag = "low value" - elif (temperature > 100): # high values + elif temperature > 100: # high values flag = "high value" # convert - res = (float(temperature)-32)*5/9 + res = (float(temperature) - 32) * 5 / 9 correctedValue = "{0:.2f}".format(res) else: correctedValue = -999 - flag = 'missing' + flag = "missing" except ValueError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + - " on " + result_day.strftime(dateFormat) + " Hour:" + str(hour) + "\tField: " + fields["id"] - + " - " + fields["name"]+"\n") - correctedValue = '-999' + correctedValue = -999 + flag = "missing" + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + correctedValue = "-999" flag = "missing" return (flag, correctedValue, error) # Inches e.g. precipitation -def getIn(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getIn( + flag: str, + value: str, + field_id: int, + data_entry_id: int, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None try: - if (value == '-222' or value == '-888'): - value = '-999' # remove -888 and -222 - if (value == "empty" or flag == "empty"): + if value == "-222" or value == "-888": + value = "-999" # remove -888 and -222 + if value == "empty" or flag == "empty": value = "0" correctedValue = 0 flag = "no entry" - if (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): + if ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): ress = value - if ("in" in value.lower() or "slight" in value.lower() - or "meas" in value.lower() or "trace" in value.lower()): + if ( + "in" in value.lower() + or "slight" in value.lower() + or "meas" in value.lower() + or "trace" in value.lower() + ): ress = 0.009 - flag = 'trace' + flag = "trace" elif "R" in ress: ress = ress.replace("R ", "0.009") - flag = 'trace' + flag = "trace" elif "S" in ress: ress = ress.replace("S ", "0.009") - flag = 'trace' - elif (", " in ress or "|" in ress or " / " in ress): + flag = "trace" + elif ", " in ress or "|" in ress or " / " in ress: values = ress.split(",") res = 0 for v in values: res += float(v) - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Two values added: " + value + - " on " + result_day.strftime(dateFormat) + "\tField: " + fields["id"] + - " - " + fields["name"] + "\n") + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Two values added: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) flag = "Two values added" elif "," in ress: ress = ress.replace(",", ".") @@ -535,48 +1126,109 @@ def getIn(flag, unit, value, error_prefix, data_entry_id, debugLog, t, try: ress = ress.replace(".", " ") from fractions import Fraction + res = float(sum(Fraction(s) for s in ress.split())) except ZeroDivisionError: - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + - " on " + result_day.strftime(dateFormat) + " Hour:" + str(hour) + - "\tField: " + fields["id"] + " - " + fields["name"] + "\n") - correctedValue = '-999' - flag = 'missing' + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + correctedValue = "-999" + flag = "missing" res = float(ress) # convert value - res = res*25.4 + res = res * 25.4 correctedValue = "{0:.2f}".format(res) - if (res < 0 or res > 30): - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Out of range: " + value + - " on " + result_day.strftime(dateFormat) + "\tField: " + fields["id"] + - " - " + fields["name"]+"\n") - flag = 'high value' + if res < 0 or res > 30: + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Out of range: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + flag = "high value" else: correctedValue = -999 - flag = 'missing' + flag = "missing" except ValueError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + - " on " + result_day.strftime(dateFormat) + " Hour:" + str(hour) + "\tField: " + fields["id"] - + " - " + fields["name"] + "\n") + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) correctedValue = -999 - flag = 'missing' + flag = "missing" return (flag, correctedValue, error) # LCT lower cloud type. Conversions in iso_mapping (im). Option keep_cloud_wind_type allows for either letter codes # (Ci, St) or Cloud Atlas types -def getLCT(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getLCT( + flag: str, + value: str, + field_id: int, + data_entry_id: int, + keep_wind_cloud_type: bool, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) + (value, error, flag, correctedValue) = cleanupValue(value, data_entry_id, flag) try: - if (value == "empty" or flag == "empty"): + if value == "empty" or flag == "empty": value = "0" correctedValue = "Cl0" flag = "no entry: clear" - if (flag != "missing" and flag != "instrument error" - and flag != "illegible" and flag != "empty"): + if ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): if keep_wind_cloud_type: correctedValue = str(im.abbr_cloud[value.lower()]) flag = "None" @@ -585,165 +1237,326 @@ def getLCT(flag, unit, value, error_prefix, data_entry_id, debugLog, t, flag = "None" else: correctedValue = "-999" - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + - "Missing data: " + value + " on " + result_day.strftime(dateFormat) + - " Hour:" + str(hour) + "\tField: " + fields["id"] + " - " + fields["name"]+"\n") - flag = 'missing' + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Missing data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + flag = "missing" except KeyError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + - value.lower() + " on " + result_day.strftime(dateFormat) + - " Hour:" + str(hour) + "\tField: " + fields["id"] + " - " + fields["name"]+"\n") + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + value.lower() + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) correctedValue = "-999" - flag = 'missing' + flag = "missing" return (flag, correctedValue, error) # LBSFT pounds per square foot (e.g. wind force) -def getLbsFt(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getLbsFt( + flag, + unit, + value, + error_prefix, + data_entry_id, + # debugLog, + t, + transcription_id, + result_day, + fields, + hour, + keep_wind_cloud_type, +): error = None - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) + (value, error, flag, correctedValue) = cleanupValue(value, data_entry_id, flag) try: - if (int(value) == -222 or int(value) == -888): - value = '-999' # remove -888 and -222 - if (value == "empty" or flag == "empty"): + if int(value) == -222 or int(value) == -888: + value = "-999" # remove -888 and -222 + if value == "empty" or flag == "empty": value = "0" correctedValue = 0 flag = "no entry: calm" - elif (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): + elif ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): v = value - if (" " in value): # if fraction in value + if " " in value: # if fraction in value (main, remainder) = value.split() - v = float(main) + float(remainder)/16 # 16 ounces = 1 pound + v = float(main) + float(remainder) / 16 # 16 ounces = 1 pound v = float(v) - if (v >= 0): - vel = sqrt(float(v)/0.00256) - res = vel*0.44704 + if v >= 0: + vel = sqrt(float(v) / 0.00256) + res = vel * 0.44704 correctedValue = "{0:.2f}".format(res) else: correctedValue = -999 - flag = 'missing' + flag = "missing" except ValueError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat)+" Hour:" + str(hour) + "\tField: "+fields["id"] + - " - " + fields["name"]+"\n") + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) correctedValue = -999 - flag = 'missing' + flag = "missing" return (flag, correctedValue, error) # Inches Mercury pressure -def getInHg(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getInHg( + flag: str, + value: str, + data_entry_id: int, + odr_type: str, + logger: structlog.BoundLogger, +): error = None try: - if (int(value) == -222 or int(value) == -888): - value = '-999' # remove -888 and -222 - if (value == "empty" or flag == "empty"): + if int(value) == -222 or int(value) == -888: + value = "-999" # remove -888 and -222 + if value == "empty" or flag == "empty": value = "-999" correctedValue = "-999" flag = "missing" - if (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) + if ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): + (value, error, flag, correctedValue) = cleanupValue( + value, data_entry_id, flag + ) v = float(value) # Handling case of pressure correction, where sometimes the dot is missing in the transcription - if t == "e" and "." not in value and v > 1: - v = v/1000 + if odr_type == "e" and "." not in value and v > 1: + v = v / 1000 res = lmr.baro_Eng_in2mb(v) # use value from iCoads lmrlib.py 2021.11.16 correctedValue = "{0:.2f}".format(res) # Some pressure values are correction data so range is different - if (t != "e" and (v < 27 or v > 32)) or (t == "e" and (v < 0 or v > 2)): # check range - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Out of range: " + str(value) + - " on " + result_day.strftime(dateFormat) + "\tField: " + str(fields["id"]) + - " - " + fields["name"] + "\n") + if (odr_type != "e" and (v < 27 or v > 32)) or ( + odr_type == "e" and (v < 0 or v > 2) + ): # check range + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Out of range: " + # + str(value) + # + " on " + # + result_day.strftime(dateFormat) + # + "\tField: " + # + str(fields["id"]) + # + " - " + # + fields["name"] + # + "\n" + # ) flag = "out of range" # Flag values which are not out of range for Canada but which are still high or low - if (t != "e" and (v < 28.5 and v >= -100)): + if odr_type != "e" and (v < 28.5 and v >= -100): flag = "low value" - elif (t != "e" and (v > 28.5 and v <= 30.6)): + elif odr_type != "e" and (v > 28.5 and v <= 30.6): flag = "none" - elif (t != "e" and (v > 30.6)): + elif odr_type != "e" and (v > 30.6): flag = "high value" - if (t == "e" and (v < 0.2 and v >= -100)): + if odr_type == "e" and (v < 0.2 and v >= -100): flag = "low value" - elif (t == "e" and (v > 0.2 and v <= 1.5)): + elif odr_type == "e" and (v > 0.2 and v <= 1.5): flag = "None" - elif (t == "e" and (v > 1.5)): + elif odr_type == "e" and (v > 1.5): flag = "high value" else: correctedValue = -999 - flag = 'missing' + flag = "missing" except ValueError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + - " - " + fields["name"] + "\n") - correctedValue = '-999' + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + correctedValue = "-999" flag = "missing" return (flag, correctedValue, error) # mm Mercury e.g. pressure -def getmmHg(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getmmHg( + flag: str, + value: str, + data_entry_id: int, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None try: - if (int(value) == -222 or int(value) == -888): - value = '-999' # remove -888 and -222 - if (value == "empty"): + if int(value) == -222 or int(value) == -888: + value = "-999" # remove -888 and -222 + if value == "empty": value = "-999" correctedValue = -999 flag = "missing" - if (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) + if ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): + (value, error, flag, correctedValue) = cleanupValue( + value, data_entry_id, flag + ) v = float(value) # Handling case of pressure correction, where sometimes the dot is missing in the transcription - if t == "e" and "." not in value and v > 1: # checking for vapour pressure. If decimal place not recorded, divide to get decimal value - v = v/1000 + if ( + odr_type == "e" and "." not in value and v > 1 + ): # checking for vapour pressure. If decimal place not recorded, divide to get decimal value + v = v / 1000 res = lmr.baro_mm2mb(v) # use value from iCoads lmrlib.py 2021.11.16 correctedValue = "{0:.2f}".format(res) # Some pressure values are correction data so range is different - if (t != "e" and (v < 700 or v > 780)) or (t == "e" and (v < 0 or v > 5)): # check range - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Out of range: " + value + - " on " + result_day.strftime(dateFormat) + "\tField: " + fields["id"] + - " - " + fields["name"] + "\n") + if (odr_type != "e" and (v < 700 or v > 780)) or ( + odr_type == "e" and (v < 0 or v > 5) + ): # check range + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Out of range: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) flag = "out of range" # Flag values which are not out of range for Canada but which are still high or low - if (t != "e" and (v < 700 and v >= -100)): + if odr_type != "e" and (v < 700 and v >= -100): flag = "low value" - elif (t != "e" and (v > 740 and v <= 780)): + elif odr_type != "e" and (v > 740 and v <= 780): flag = "none" - elif (t != "e" and (v > 780)): + elif odr_type != "e" and (v > 780): flag = "high value" - if (t == "e" and (v < 0.5 and v >= -100)): + if odr_type == "e" and (v < 0.5 and v >= -100): flag = "low value" - elif (t == "e" and (v > 0.5 and v <= 5)): + elif odr_type == "e" and (v > 0.5 and v <= 5): flag = "None" - elif (t == "e" and (v > 5)): + elif odr_type == "e" and (v > 5): flag = "high value" else: correctedValue = -999 - flag = 'missing' + flag = "missing" except ValueError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + " - " + fields["name"] + "\n") - correctedValue = '-999' + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + correctedValue = "-999" flag = "missing" return (flag, correctedValue, error) # MNO -def getMNO(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getMNO( + flag: str, + value: str, + field_id: int, + data_entry_id: int, + keep_wind_cloud_type: bool, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None - if (value == "empty" or flag == "empty"): + if value == "empty" or flag == "empty": value = "no data" correctedValue = "no data" flag = "no entry" - if (flag != "missing" and flag != "instrument error" and flag != "illegible" - and flag != "empty"): + if ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): v = value.lower() # synoptic codes for weather types if "snow" in v: @@ -849,13 +1662,24 @@ def getMNO(flag, unit, value, error_prefix, data_entry_id, debugLog, t, elif "clear" in v: correctedValue = "SKC" else: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + - "Invalid data: " + str(value) + " on " + - result_day.strftime(dateFormat) + " Hour:" - + str(hour) + "\tField: " + fields["id"] + " - " + - fields["name"] + "\n") - correctedValue = '-999' - flag = 'missing' + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + str(value) + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + correctedValue = "-999" + flag = "missing" # if len(correctedValue) > 90: # correctedValue = correctedValue[0:90] @@ -863,260 +1687,541 @@ def getMNO(flag, unit, value, error_prefix, data_entry_id, debugLog, t, # MPH miles per hour -def getMPH(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getMPH( + flag: str, + value: str, + field_id: int, + data_entry_id: int, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) + (value, error, flag, correctedValue) = cleanupValue(value, data_entry_id, flag) try: - if (int(value) == -222 or int(value) == -888): - value = '-999' # remove -888 and -222 - if (value == "empty" or flag == "empty"): + if int(value) == -222 or int(value) == -888: + value = "-999" # remove -888 and -222 + if value == "empty" or flag == "empty": value = "0" correctedValue = 0 flag = "no entry: calm" - elif (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): - res = float(value)/2.237 + elif ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): + res = float(value) / 2.237 correctedValue = "{0:.2f}".format(res) else: correctedValue = -999 - flag = 'missing' + flag = "missing" except ValueError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + " Hour:" + str(hour) + "\tField: " + fields["id"] + - " - " + fields["name"]+"\n") + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) correctedValue = -999 - flag = 'missing' + flag = "missing" return (flag, correctedValue, error) # Okta cloud cover in eighths. Most historical cloud is in tenths -def getOkta(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getOkta( + flag: str, + value: str, + field_id: int, + field_name: str, + data_entry_id: int, + keep_wind_cloud_type: bool, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None - field_id = fields["id"] - field_name = fields["name"] - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) + (value, error, flag, correctedValue) = cleanupValue(value, data_entry_id, flag) if "clearness" in field_name: # test for clearness of sky - # if field_id == "173" or field_id == "174" or field_id == "175" or field_id == "176": - # + # if field_id == "173" or field_id == "174" or field_id == "175" or field_id == "176": + # try: - if (int(value) == -222 or int(value) == -888): - value = '-999' # remove -888 and -222 - if (value == "empty" or flag == "empty"): + if int(value) == -222 or int(value) == -888: + value = "-999" # remove -888 and -222 + if value == "empty" or flag == "empty": value = "0" correctedValue = "0" flag = "no entry: clear" - if (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): - if ("fog" in value.lower() or "smoke" in value.lower() or "haze" in value.lower() - or "scud" in value.lower() or "hidden" in value.lower()): + if ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): + if ( + "fog" in value.lower() + or "smoke" in value.lower() + or "haze" in value.lower() + or "scud" in value.lower() + or "hidden" in value.lower() + ): value = "9" elif value == "Zero" or "clear" in value.lower(): value = "0" res = int(value) # out of range test - if (res < 0 or res > 10): - debugLog.write("\tTranscriptionID: " + str(transcription_id) + - " " + "Out of range: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + - " - " + fields["name"] + "\n") - flag = 'out of range' + if res < 0 or res > 10: + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Out of range: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + flag = "out of range" else: res = lmr.cloud_tenthsclear2oktas(res) else: correctedValue = -999 - flag = 'missing' + flag = "missing" except ValueError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + - " " + "Invalid data: " + value + " on " + result_day.strftime(dateFormat) + - " Hour:" + str(hour) + "\tField: " + - fields["id"] + " - " + fields["name"] + "\n") - correctedValue = -999 - flag = 'missing' + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + correctedValue = -999 + flag = "missing" else: - try: - if (value == "empty" or flag == "empty"): + if value == "empty" or flag == "empty": value = "0" correctedValue = "0" flag = "no entry: clear" - if (int(value) == -222 or int(value) == -888): - value = '-999' # remove -888 and -222 - if (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): - if ("fog" in value.lower() or "smoke" in value.lower() or "haze" in value.lower() - or "scud" in value.lower() or "hidden" in value.lower()): + if int(value) == -222 or int(value) == -888: + value = "-999" # remove -888 and -222 + if ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): + if ( + "fog" in value.lower() + or "smoke" in value.lower() + or "haze" in value.lower() + or "scud" in value.lower() + or "hidden" in value.lower() + ): value = "9" elif value == "Zero" or "clear" in value.lower(): value = "0" res = float(value) # out of range - if (res < 0 or res > 10): - debugLog.write("\tTranscriptionID: " + str(transcription_id) + - " " + "Out of range: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + - " - " + fields["name"] + "\n") - flag = 'out of range' + if res < 0 or res > 10: + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Out of range: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + flag = "out of range" else: - res = res*0.8 + res = res * 0.8 correctedValue = "{0:.2f}".format(res) else: correctedValue = -999 - flag = 'missing' + flag = "missing" except ValueError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + - " " + "Invalid data: " + value + " on " + result_day.strftime(dateFormat) + - " Hour:" + str(hour) + "\tField: " + - fields["id"] + " - " + fields["name"] + "\n") + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) correctedValue = -999 - flag = 'missing' + flag = "missing" return (flag, correctedValue, error) # Ozone (Oz) No known conversion -def getOz(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getOz( + flag: str, + value: str, + field_id: int, + data_entry_id: int, + keep_wind_cloud_type: bool, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) - if (value == "empty" or flag == "empty"): + (value, error, flag, correctedValue) = cleanupValue(value, data_entry_id, flag) + if value == "empty" or flag == "empty": value = "-999" correctedValue = "-999" flag = "missing" - if (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): + if ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): correctedValue = value flag = "None" else: - debugLog.write("\tTranscriptionID: " + str(transcription_id) + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + "\tField: " + fields["id"] + - " - " + fields["name"] + "\n") + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) correctedValue = -999 flag = "missing" return (flag, correctedValue, error) # Percentage -def getP(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getP( + flag: str, + value: str, + data_entry_id: int, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) + (value, error, flag, correctedValue) = cleanupValue(value, data_entry_id, flag) try: - if (int(value) == -222 or int(value) == -888): - value = '-999' # remove -888 and -222 - if (value == "empty"): + if int(value) == -222 or int(value) == -888: + value = "-999" # remove -888 and -222 + if value == "empty": value = "-999" correctedValue = "-999" flag = "missing" - if (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): + if ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): res = float(value) - if (res < 0 or res > 100): # check range - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Out of range: " + value + - " on " + result_day.strftime(dateFormat) + "\tField: " + fields["id"] + - " - " + fields["name"]+"\n") - elif (res > 0 and res <= 1): # check if decimal instead of prercentage + # if res < 0 or res > 100: # check range + # # debugLog.write( + # # "\tTranscriptionID: " + # # + str(transcription_id) + # # + " " + # # + "Out of range: " + # # + value + # # + " on " + # # + result_day.strftime(dateFormat) + # # + "\tField: " + # # + fields["id"] + # # + " - " + # # + fields["name"] + # # + "\n" + # # ) + if res > 0 and res <= 1: # check if decimal instead of prercentage # debugLog.write("TranscriptionID: "+ str(transcription_id) + "\less than 1: " + value +" on " + # result_day.strftime(dateFormat)+"\tField: "+fields["id"]+ " - " + fields["name"]+"\n" ) - res = res*100 + res = res * 100 flag = "multiply by 100" correctedValue = "{0:.2f}".format(res) else: correctedValue = -999 - flag = 'missing' + flag = "missing" except ValueError: - debugLog.write("\tTranscriptionID: " + str(transcription_id) + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + " Hour:" + str(hour) + "\tField: " + fields["id"] + - " - " + fields["name"] + "\n") - correctedValue = '-999' + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + correctedValue = "-999" return (flag, correctedValue, error) # Relative humidity check if in fraction pr precentage -def getRH(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getRH( + flag: str, + value: str, + data_entry_id: int, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) + (value, error, flag, correctedValue) = cleanupValue(value, data_entry_id, flag) try: - if (int(value) == -222 or int(value) == -888): - value = '-999' # remove -888 and -222 - if (value == "empty" or flag == "empty"): + if int(value) == -222 or int(value) == -888: + value = "-999" # remove -888 and -222 + if value == "empty" or flag == "empty": value = "-999" correctedValue = -999 flag = "missing" - if (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): + if ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): res = float(value) - if (res < 0 or res > 100): # range for precentage data - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Out of range: " + value + - " on " + result_day.strftime(dateFormat) + "\tField: " + fields["id"] + - " - " + fields["name"] + "\n") - elif (res > 0 and res <= 1): # if value recorded as fraction instead of percentage + # if res < 0 or res > 100: # range for precentage data + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Out of range: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + if ( + res > 0 and res <= 1 + ): # if value recorded as fraction instead of percentage # lines below removed as too many values were transformed # debugLog.write("TranscriptionID: "+ str(transcription_id) + "\less than 1: " + value +" on " + # result_day.strftime(dateFormat)+"\tField: "+fields["id"]+ " - " + fields["name"]+"\n" ) - res = res*100 + res = res * 100 flag = "multiply by 100" correctedValue = "{0:.2f}".format(res) else: correctedValue = -999 - flag = 'missing' + flag = "missing" except ValueError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + " Hour:" + str(hour) + "\tField: " + fields["id"] + - " - " + fields["name"] + "\n") + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) correctedValue = -999 flag = "missing" return (flag, correctedValue, error) # SM Smithsonian wind force scale (0-10) -def getSM(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getSM( + flag: str, + value: str, + field_id: int, + data_entry_id: int, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None - field_id = fields["id"] - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) + # field_id = fields["id"] + (value, error, flag, correctedValue) = cleanupValue(value, data_entry_id, flag) # Some fields contain two different data entries: wind force and direction. Here we select wind force if field_id == "177" or field_id == "178" or field_id == "179" or field_id == "180": value = value.replace(",", " ") # if value != "0": - if ("." in value): + if "." in value: values = value.split(".") else: values = value.split(" ") - debugLog.write(result_day.strftime(dateFormat)+" Wind force. Values:" - + value + " - looking if isdigit for: " + values[-1] + "\n") - if values[-1].isdigit(): # if both direction and force in same entry, see which value is numeric + # debugLog.write( + # result_day.strftime(dateFormat) + # + " Wind force. Values:" + # + value + # + " - looking if isdigit for: " + # + values[-1] + # + "\n" + # ) + if ( + values[-1].isdigit() + ): # if both direction and force in same entry, see which value is numeric value = values[-1] else: value = values[0] - if ("|" in value): + if "|" in value: values = value.split("|") value = values[-1] - flag = 'None' + flag = "None" - debugLog.write("Selected wind force: " + value + "\n") + # debugLog.write("Selected wind force: " + value + "\n") # print("split dir, force", "force = ", value) try: - if (int(value) == -222 or int(value) == -888): - value = '-999' # remove -888 and -222 - if (value == "empty" or flag == "empty"): + if int(value) == -222 or int(value) == -888: + value = "-999" # remove -888 and -222 + if value == "empty" or flag == "empty": value = "0" correctedValue = 0 flag = "no entry: calm" - elif (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): + elif ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): from fractions import Fraction + res = float(sum(Fraction(s) for s in value.split())) res = int(res) - if (res < 0 or res > 10): # if value out of range write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Out of range: " + value + - " on " + result_day.strftime(dateFormat) + "\tField: " + fields["id"] + - " - " + fields["name"] + "\n") + if ( + res < 0 or res > 10 + ): # if value out of range write debug log and error type + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Out of range: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) flag = "out of range" correctedValue = -999 else: @@ -1124,41 +2229,102 @@ def getSM(flag, unit, value, error_prefix, data_entry_id, debugLog, t, correctedValue = "{0:.0f}".format(res) else: correctedValue = -999 - flag = 'missing' + flag = "missing" except ValueError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + "Invalid data: " + value + " on " + - result_day.strftime(dateFormat) + " Hour:" + str(hour) + "\tField: " + fields["id"] + - " - " + fields["name"]+"\n") + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) correctedValue = -999 - flag = 'missing' + flag = "missing" return (flag, correctedValue, error) # UCT upper cloud type -def getUCT(flag, unit, value, error_prefix, data_entry_id, debugLog, t, - transcription_id, result_day, fields, hour, keep_wind_cloud_type): +def getUCT( + flag: str, + value: str, + field_id: int, + data_entry_id: int, + keep_wind_cloud_type: bool, + odr_type: str, + logger: structlog.BoundLogger, + # flag, + # unit, + # value, + # error_prefix, + # data_entry_id, + # # debugLog, + # t, + # transcription_id, + # result_day, + # fields, + # hour, + # keep_wind_cloud_type, +): error = None - (value, error, flag, correctedValue) = cleanupValue(error_prefix, value, data_entry_id, flag) + (value, error, flag, correctedValue) = cleanupValue(value, data_entry_id, flag) try: - if (value == "empty" or flag == "empty"): + if value == "empty" or flag == "empty": value = "0" correctedValue = "Ch0" flag = "no entry: clear" - if (flag != "missing" and flag != "instrument error" and - flag != "illegible" and flag != "empty"): + if ( + flag != "missing" + and flag != "instrument error" + and flag != "illegible" + and flag != "empty" + ): if keep_wind_cloud_type: correctedValue = im.abbr_cloud[value.lower()] else: correctedValue = "-999" - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + - "Missing data: " + value + " on " + result_day.strftime(dateFormat) + - " Hour:" + str(hour) + "\tField: " + fields["id"] + " - " + fields["name"]+"\n") - flag = 'missing' + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Missing data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + flag = "missing" except KeyError: # if value does not conform, write debug log and error type - debugLog.write("\tTranscriptionID: " + str(transcription_id) + " " + - "Invalid data: " + value + " on " + result_day.strftime(dateFormat) + - " Hour:" + str(hour) + "\tField: " + fields["id"] + " - " + - fields["name"] + "\n") - correctedValue = '-999' - flag = 'missing' + # debugLog.write( + # "\tTranscriptionID: " + # + str(transcription_id) + # + " " + # + "Invalid data: " + # + value + # + " on " + # + result_day.strftime(dateFormat) + # + " Hour:" + # + str(hour) + # + "\tField: " + # + fields["id"] + # + " - " + # + fields["name"] + # + "\n" + # ) + correctedValue = "-999" + flag = "missing" return (flag, correctedValue, error) diff --git a/justfile b/justfile index e69de29..29740ca 100644 --- a/justfile +++ b/justfile @@ -0,0 +1,45 @@ +venv_path := justfile_directory() / ".venv" +set dotenv-load := true +os := os() +devcontainer := if env_var_or_default("USER", "nobody") == "vscode" {"true"} else {"false"} +# serve_host := if env_var_or_default("CODESPACES", "false") == "true" { "0.0.0.0" } else { "127.0.0.1" } +# serve_port := env_var_or_default("PORT", "12010") + +@default: + just --list + +# Install dependencies +install: + uv sync --dev + +# Format code with ruff +format: + uv run ruff format . + +# Check code formatting with ruff +format-check: + uv run ruff format --check . + +# Fix linting issues automatically +lint-fix: + uv run ruff check --fix . + +# Type check with mypy +typecheck: + uv run mypy data_post_processing/ + +# Update dependencies +update: + uv sync --upgrade + +clean: + rm -rf .pytest_cache + rm -rf htmlcov + rm -rf .coverage + rm -rf __pycache__ + find . -type d -name "__pycache__" -exec rm -rf {} + + find . -type f -name "*.pyc" -delete + +# Development shell with all dependencies available +shell: + uv run python diff --git a/logs/debug.log b/logs/debug.log new file mode 100644 index 0000000..5ebac77 --- /dev/null +++ b/logs/debug.log @@ -0,0 +1,39 @@ +2026-04-08T20:36:45.566739Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=23:00:00 observation_date=Timestamp('1874-12-27 23:13:00') result_day=datetime.date(1874, 12, 27) transcription_id=2752 value=illegible +2026-04-08T20:36:46.137477Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=01:00:00 observation_date=Timestamp('1875-02-12 01:48:00') result_day=datetime.date(1875, 2, 12) transcription_id=524 value=illegible +2026-04-08T20:36:48.785392Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=22:00:00 observation_date=Timestamp('1876-04-07 22:48:00') result_day=datetime.date(1876, 4, 7) transcription_id=2771 value=illegible +2026-04-08T20:36:52.416911Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=10:00:00 observation_date=Timestamp('1877-07-06 10:48:00') result_day=datetime.date(1877, 7, 6) transcription_id=2292 value=illegible +2026-04-08T20:36:53.569396Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=01:00:00 observation_date=Timestamp('1877-11-28 01:48:00') result_day=datetime.date(1877, 11, 28) transcription_id=2561 value=illegible +2026-04-08T20:36:53.981765Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=01:00:00 observation_date=Timestamp('1877-12-06 01:48:00') result_day=datetime.date(1877, 12, 6) transcription_id=2562 value=illegible +2026-04-08T20:36:54.236452Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=13:00:00 observation_date=Timestamp('1877-12-10 13:00:00') result_day=datetime.date(1877, 12, 10) transcription_id=2696 value=illegible +2026-04-08T20:36:54.344170Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=13:00:00 observation_date=Timestamp('1877-12-12 13:00:00') result_day=datetime.date(1877, 12, 12) transcription_id=2696 value=illegible +2026-04-08T20:36:54.576723Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=07:00:00 observation_date=Timestamp('1877-12-18 07:00:00') result_day=datetime.date(1877, 12, 18) transcription_id=2734 value=illegible +2026-04-08T20:36:54.580684Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=07:00:00 observation_date=Timestamp('1877-12-18 07:48:00') result_day=datetime.date(1877, 12, 18) transcription_id=2433 value=illegible +2026-04-08T20:36:54.645016Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=19:00:00 observation_date=Timestamp('1877-12-22 19:00:00') result_day=datetime.date(1877, 12, 22) transcription_id=2772 value=illegible +2026-04-08T20:36:54.703252Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=13:00:00 observation_date=Timestamp('1877-12-24 13:00:00') result_day=datetime.date(1877, 12, 24) transcription_id=2436 value=illegible +2026-04-08T20:36:54.786591Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=22:00:00 observation_date=Timestamp('1877-12-26 22:00:00') result_day=datetime.date(1877, 12, 26) transcription_id=2834 value=illegible +2026-04-08T20:36:54.871371Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=13:00:00 observation_date=Timestamp('1877-12-28 13:00:00') result_day=datetime.date(1877, 12, 28) transcription_id=2845 value=illegible +2026-04-08T20:36:57.236397Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=19:00:00 observation_date=Timestamp('1878-04-23 19:48:00') result_day=datetime.date(1878, 4, 23) transcription_id=2716 value=illegible +2026-04-08T20:36:57.237534Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=19:00:00 observation_date=Timestamp('1878-04-23 19:48:00') result_day=datetime.date(1878, 4, 23) transcription_id=2485 value=illegible +2026-04-08T20:36:58.665257Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=01:00:00 observation_date=Timestamp('1878-07-23 01:48:00') result_day=datetime.date(1878, 7, 23) transcription_id=1074 value=illegible +2026-04-08T20:37:01.491014Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=22:00:00 observation_date=Timestamp('1879-03-04 22:48:00') result_day=datetime.date(1879, 3, 4) transcription_id=2948 value=illegible +2026-04-08T20:37:01.493430Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=23:00:00 observation_date=Timestamp('1879-03-04 23:13:00') result_day=datetime.date(1879, 3, 4) transcription_id=2948 value=illegible +2026-04-08T20:37:02.213121Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=16:00:00 observation_date=Timestamp('1879-04-23 16:48:00') result_day=datetime.date(1879, 4, 23) transcription_id=2984 value=illegible +2026-04-08T20:37:02.222674Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=13:00:00 observation_date=Timestamp('1879-04-24 13:48:00') result_day=datetime.date(1879, 4, 24) transcription_id=2984 value=illegible +2026-04-08T20:37:02.223904Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=16:00:00 observation_date=Timestamp('1879-04-24 16:48:00') result_day=datetime.date(1879, 4, 24) transcription_id=2984 value=illegible +2026-04-08T20:37:02.247159Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=22:00:00 observation_date=Timestamp('1879-04-26 22:48:00') result_day=datetime.date(1879, 4, 26) transcription_id=2981 value=illegible +2026-04-08T20:37:02.267375Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=10:00:00 observation_date=Timestamp('1879-04-29 10:48:00') result_day=datetime.date(1879, 4, 29) transcription_id=2096 value=illegible +2026-04-08T20:37:06.784899Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=01:00:00 observation_date=Timestamp('1879-09-14 01:48:00') result_day=datetime.date(1879, 9, 14) transcription_id=1214 value=illegible +2026-04-08T20:37:07.956133Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=01:00:00 observation_date=Timestamp('1879-10-11 01:48:00') result_day=datetime.date(1879, 10, 11) transcription_id=537 value=illegible +2026-04-08T20:37:23.735579Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=03:00:00 observation_date=Timestamp('1883-11-24 03:08:00') result_day=datetime.date(1883, 11, 24) transcription_id=1983 value=illegible +2026-04-08T20:37:27.826751Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=11:00:00 observation_date=Timestamp('1885-07-03 11:00:00') result_day=datetime.date(1885, 7, 3) transcription_id=4025 value=illegible +2026-04-08T20:37:28.682177Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=11:00:00 observation_date=Timestamp('1885-10-09 11:00:00') result_day=datetime.date(1885, 10, 9) transcription_id=4099 value=illegible +2026-04-08T20:37:34.745742Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=03:00:00 observation_date=Timestamp('1886-09-08 03:00:00') result_day=datetime.date(1886, 9, 8) transcription_id=2878 value=illegible +2026-04-08T20:37:34.909752Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=15:00:00 observation_date=Timestamp('1886-11-04 15:00:00') result_day=datetime.date(1886, 11, 4) transcription_id=2416 value=illegible +2026-04-08T20:37:43.620530Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=23:00:00 observation_date=Timestamp('1895-10-11 23:00:00') result_day=datetime.date(1895, 10, 11) transcription_id=1781 value=illegible +2026-04-08T20:37:44.100409Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=23:00:00 observation_date=Timestamp('1896-02-08 23:00:00') result_day=datetime.date(1896, 2, 8) transcription_id=1028 value=illegible +2026-04-08T20:37:44.205716Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=03:00:00 observation_date=Timestamp('1896-02-28 03:00:00') result_day=datetime.date(1896, 2, 28) transcription_id=2703 value=illegible +2026-04-08T20:37:44.252375Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=03:00:00 observation_date=Timestamp('1896-03-20 03:00:00') result_day=datetime.date(1896, 3, 20) transcription_id=2385 value=illegible +2026-04-08T20:37:45.342401Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=01:00:00 observation_date=Timestamp('1897-06-18 01:48:00') result_day=datetime.date(1897, 6, 18) transcription_id=533 value=illegible +2026-04-08T20:37:48.653344Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=23:00:00 observation_date=Timestamp('1903-02-14 23:13:00') result_day=datetime.date(1903, 2, 14) transcription_id=2012 value=illegible +2026-04-08T20:37:51.245854Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=19:00:00 observation_date=Timestamp('1906-07-03 19:50:00') result_day=datetime.date(1906, 7, 3) transcription_id=3043 value=illegible +2026-04-08T20:37:51.248583Z [debug ] Invalid data field_id=4 field_name=Barometer_Observed hour=19:00:00 observation_date=Timestamp('1906-07-03 19:50:00') result_day=datetime.date(1906, 7, 3) transcription_id=1327 value=illegible diff --git a/main.py b/main.py new file mode 100755 index 0000000..9b2fe66 --- /dev/null +++ b/main.py @@ -0,0 +1,189 @@ +#!/usr/bin/env -S uv run python +# + +import datetime + +import pandas as pd +import pytz + +from data_post_processing.data_utils import ( + get_field_meta_panda, + get_station_data_panda, + get_station_field_ids, + get_station_meta_data, + store_panda, +) +from data_post_processing.iso_mapping import map_unit +from data_post_processing.logging import configure_logging, get_file_logger, get_logger +from data_post_processing.transcription_data_processing import ( + getProcessedDataValue, + preProcess, +) + +configure_logging() + +# TODO: what is the table station_override ?? + + +def main(): + # NOTE: We can use bind to put in the information re the application + # such as the day, type, transcription etc + logger = get_logger() + logger.debug("Application started") + + # Logs messages to a file in logs directory + file_log = get_file_logger() + + # First, get station metadata information + station_meta_data = get_station_meta_data() + logger.debug("We have station meta data", station_meta_data=station_meta_data) + + # For each station in the database we will process the data entries + for station in station_meta_data: + logger.debug("Process Station", station=station.station_name) + # Get the field ids used for this station + data_fields_list = get_station_field_ids(station=station.station_name) + # Get the field information for the data for the station + field_meta_data = get_field_meta_panda( + field_ids=data_fields_list["field_id"].to_numpy() + ) + + # Get the types with data for this station + # data_types = get_station_field_types(station=station.stationName) + logger.debug( + "Data Fields & Types", + data_fields_list=data_fields_list["field_id"].to_numpy(), + field_meta_data=field_meta_data, + ) + + # dates = getDatesWithData(engine, stn_fields, station) + + # Process each field type that has the data + for field_id in data_fields_list.field_id: + # Dataframe for the cleaned up imperial and ISO values + iso_list = pd.DataFrame( + columns=[ + "station_name", + "timezone", + "utc_offset", + "observation_datetime", + "utc_datetime", + "original_value", + "unit", + "value", + "iso_unit", + "field_key", + "field_id", + "annotation_id", + "transcription_id", + "page_id", + ] + ) + + # Get the unit for the field + unit = field_meta_data[ + field_meta_data.id == field_id + ].measurement_unit_original.iloc[0] + iso_unit = map_unit[unit] + # Extract the ODR type for the measurement + odr_type = field_meta_data[field_meta_data.id == field_id].odr_type.iloc[0] + + # file_log.debug( + # "Processing", timeOfDay=timeOfDay, field_id=field_id, unit=unit + # ) + + # Get the data for the Station given the ORD type and Field id + station_data = get_station_data_panda( + station=station.station_name, odr_type=odr_type, field_id=field_id + ) + + # For each station process the current field + for index, result in station_data.iterrows(): + # + pre_iso_value = None + if result.value is not None: + # Bind the file logger with the details for this result + res_log = file_log.bind( + transcription_id=result.transcription_id, + observation_date=result.observation_date, + result_day=result.observation_date.date(), + hour=f"{result.observation_date.hour:02d}:00:00", + field_id=result.field_id, + field_name=result.field_key, + value=result.value, + ) + + # Get the timezone based on the station offset + tz = datetime.timezone( + datetime.timedelta(hours=int(station.UTC_offset)) + ) + # Set the station TZ for the observation datetime + observation_date = result.observation_date.replace(tzinfo=tz) + + # Determine the corrected value for the transcription + (pre_iso_value, flag) = preProcess( + value=result.value, unit=unit, logger=res_log + ) + + if pre_iso_value is not None: + # Get the cleaned up ISO measurement + (flag, correctedValue, error) = getProcessedDataValue( + flag=flag, + value=pre_iso_value, + data_entry_id=result.id, + field_id=field_id, + field_name=result.field_key, + odr_type=odr_type, + unit=unit, + keep_wind_cloud_type=True, + logger=res_log, + ) + + # Add to ISO list if there is a corrected value + if correctedValue is not None: + iso_list = pd.concat( + [ + iso_list, + pd.DataFrame( + [ + { + "station_id": station.station_id, + "station_name": station.station_name, + "timezone": station.timezone, + "utc_offset": station.UTC_offset, + "observation_datetime": result.observation_date.replace( + tzinfo=tz + ) + .replace(tzinfo=None) + .replace(microsecond=0) + .isoformat(), + "utc_datetime": observation_date.astimezone( + pytz.utc + ) + .replace(tzinfo=None) + .replace(microsecond=0) + .isoformat(), + "original_value": pre_iso_value, + "unit": unit, + "value": correctedValue, + "iso_unit": iso_unit, + "field_key": result.field_key, + "field_id": result.field_id, + "annotation_id": result.annotation_id, + "transcription_id": result.transcription_id, + "page_id": result.page_id, + } + ] + ), + ] + ).reset_index(drop=True) + + # Store the ISO / Clean lists in the database + store_panda(pd=iso_list, name="odr_iso_data") + # Also generate the SEF files? + + # break + + +if __name__ == "__main__": + main() diff --git a/migrations/README b/migrations/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..b112f2f --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,85 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +from data_post_processing.database import db_url + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# +# setup the database URL for sql alchemy +# +config.set_main_option('sqlalchemy.url', db_url()) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..1101630 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/180739ab4fa2_create_iso_results_table.py b/migrations/versions/180739ab4fa2_create_iso_results_table.py new file mode 100644 index 0000000..fca10f7 --- /dev/null +++ b/migrations/versions/180739ab4fa2_create_iso_results_table.py @@ -0,0 +1,44 @@ +"""Create ISO results table + +Revision ID: 180739ab4fa2 +Revises: b6539120d112 +Create Date: 2026-04-08 14:44:26.751703 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = '180739ab4fa2' +down_revision: Union[str, Sequence[str], None] = 'b6539120d112' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table('odr_iso_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.Column('station_id', sa.Integer(), nullable=False), + sa.Column('station_name', sa.String(length=255), nullable=True), + sa.Column('timezone', sa.String(length=100), nullable=False), + sa.Column('UTC_offset', sa.Numeric(precision=6,scale=3), nullable=False), + sa.Column('observation_datetime', sa.DateTime(), nullable=False), + sa.Column('utc_datetime', sa.DateTime(), nullable=False), + sa.Column('original_value', sa.Text(), nullable=True), + sa.Column('unit', sa.String(length=255), nullable=True), + sa.Column('value', sa.Numeric(precision=15,scale=5), nullable=True), + sa.Column('iso_unit', sa.String(length=255), nullable=True), + sa.Column('field_key', sa.String(length=255), nullable=False), + sa.Column('field_id', sa.Integer(), nullable=False), + sa.Column('annotation_id', sa.Integer(), nullable=False), + sa.Column('transcription_id', sa.Integer(), nullable=False), + sa.Column('page_id', sa.Integer(), nullable=False), + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_table('odr_iso_data') diff --git a/migrations/versions/6b85f9c3707d_create_metadata_global.py b/migrations/versions/6b85f9c3707d_create_metadata_global.py new file mode 100644 index 0000000..672ae2c --- /dev/null +++ b/migrations/versions/6b85f9c3707d_create_metadata_global.py @@ -0,0 +1,79 @@ +"""create odr_global_metadata + +Revision ID: 6b85f9c3707d +Revises: d7929bc5d967 +Create Date: 2026-03-05 17:35:43.896276 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = '6b85f9c3707d' +down_revision: Union[str, Sequence[str], None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table('odr_global_metadata', + # We put in a primary key so that we can user relationships etc + sa.Column('pkey', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('pkey'), + sa.Column('source', sa.String(length=100), nullable=True), + sa.Column('project', sa.String(length=100), nullable=False), + sa.UniqueConstraint('project', name='uq_meta_project'), + sa.Column('country', sa.String(length=100), nullable=True), + # ISO3 should only be 3 characters ... + sa.Column('territoryOfOriginOfData_ISO3CountryCode', sa.String(length=6), nullable=True), + # FIPS code is the US geographic identifier, which is a short (usually 2 char) string + sa.Column('fipsCode', sa.String(length=6), nullable=True), + sa.Column('continent', sa.String(length=100), nullable=True), + sa.Column('regionOfOriginOfData', sa.String(length=100), nullable=True), + # sa.Column('descriptionDataset', sa.String(length=100), nullable=True), + # sa.Column('dataRepository', sa.String(length=100), nullable=True), + sa.Column('stationDataPolicy', sa.String(length=100), nullable=True), + # sa.Column('useConstraints', sa.String(length=100), nullable=True), + # sa.Column('dataUpdateStatus', sa.String(length=100), nullable=True), + sa.Column('version', sa.String(length=10), nullable=True), + sa.Column('metadataStandard', sa.String(length=30), nullable=True), + # sa.Column('stationMetadataLink', sa.String(length=100), nullable=True), + sa.Column('applicationArea', sa.String(length=40), nullable=True), + sa.Column('pointOfContact', sa.String(length=100), nullable=True), + sa.Column('pointOfContactName', sa.String(length=100), nullable=True), + sa.Column('pointOfContactPosition', sa.String(length=100), nullable=True), + sa.Column('pointOfContactAddress', sa.String(length=100), nullable=True), + sa.Column('prinicpalInvestigatorName', sa.String(length=100), nullable=True), + sa.Column('principlaInvestigatorAddress', sa.String(length=100), nullable=True), + # sa.Column('principalInvestigatorOnlineResource', sa.String(length=100), nullable=True), + sa.Column('timestep', sa.String(length=30), nullable=True), + sa.Column('observationFrequency', sa.String(length=50), nullable=True), + sa.Column('spatialExtent', sa.String(length=20), nullable=True), + # sa.Column('observedVariableMeasurandDomain', sa.String(length=100), nullable=True), + # sa.Column('observedVariableMeasurandSubdomain', sa.String(length=100), nullable=True), + sa.Column('representativenessOfObservation', sa.String(20), nullable=True), + sa.Column('stationStatus', sa.String(length=15), nullable=True), + # sa.Column('measurementObservingMethod', sa.String(length=100), nullable=True), + # sa.Column('instrumentSpecifications', sa.String(length=100), nullable=True), + # sa.Column('instrumentOperatingStatus', sa.String(length=100), nullable=True), + # Datetime should be typed + sa.Column('creationDate', sa.Date(), nullable=True), + sa.Column('revisionDate', sa.Date(), nullable=True), + # Language of the metadata composed of an ISO639-2/T three letter language code + # and an ISO3166-1 three letter country code?? i.e en for english + sa.Column('languageCode', sa.String(length=7), nullable=True), + # MD_CharacterSetCode + sa.Column('characterEncoding', sa.String(length=15), nullable=True), + # sa.Column('statusOfObservation', sa.String(length=100), nullable=True), + sa.Column('qualityFlaggingSystem', sa.String(length=20), nullable=True), + sa.Column('traceability', sa.String(length=20), nullable=True), + # Is this the link to more metadata as per spec or something else? + sa.Column('link', sa.String(length=255), nullable=True), + ) + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_table('odr_global_metadata') diff --git a/migrations/versions/b6539120d112_create_station_metadata.py b/migrations/versions/b6539120d112_create_station_metadata.py new file mode 100644 index 0000000..cc1ab87 --- /dev/null +++ b/migrations/versions/b6539120d112_create_station_metadata.py @@ -0,0 +1,55 @@ +"""create odr_station_metadata + +Revision ID: b6539120d112 +Revises: 6b85f9c3707d +Create Date: 2026-03-10 11:49:53.265079 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = 'b6539120d112' +down_revision: Union[str, Sequence[str], None] = '6b85f9c3707d' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +def upgrade() -> None: + """Upgrade schema - create odr_station_metadata table.""" + op.create_table('odr_station_metadata', + # ECCC db has the stationID as the PKey and it is an integer + sa.Column('station_id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('station_id'), + sa.Column('station_name', sa.String(length=255), nullable=True), + sa.Column('start_date', sa.Date(), nullable=True), + sa.Column('end_date', sa.Date(), nullable=True), + sa.Column('source_id', sa.String(length=100), nullable=True), + sa.Column('WMOID', sa.String(length=50), nullable=True), + sa.Column('MSCID', sa.String(length=50), nullable=True), + sa.Column('other_station_name', sa.String(length=100), nullable=True), + sa.Column('region', sa.String(length=100), nullable=True), + sa.Column('latitude', sa.Numeric(precision=7,scale=3), nullable=True), + sa.Column('document_latitude', sa.String(length=100), nullable=True), + sa.Column('longitude', sa.Numeric(precision=7,scale=3), nullable=True), + sa.Column('document_longitude', sa.String(length=100), nullable=True), + sa.Column('elevation', sa.Numeric(precision=10,scale=3), nullable=True), + sa.Column('document_elevation', sa.String(length=100), nullable=True), + sa.Column('timezone', sa.String(length=100), nullable=False), + sa.Column('UTC_offset', sa.Numeric(precision=6,scale=3), nullable=False), + sa.Column('document_country', sa.String(length=255), nullable=True), + sa.Column('additional_territory_designators', sa.Text(), nullable=True), + sa.Column('observation_times_local', sa.String(255), nullable=True), + sa.Column('originator_organization', sa.String(500), nullable=True), + sa.Column('originator_name', sa.String(255), nullable=True), + sa.Column('originator_position', sa.String(255), nullable=True), + sa.Column('originator_address', sa.Text(), nullable=True), + sa.Column('project', sa.String(100), + sa.ForeignKey("odr_global_metadata.project")), + ) + + +def downgrade() -> None: + """Downgrade schema - drop odr_station_metadata table.""" + op.drop_table('odr_station_metadata') diff --git a/pyproject.toml b/pyproject.toml index d55c2d7..22ccc58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,13 @@ authors = [] dependencies = [ "pydantic>=2.10.0", "pydantic-settings>=2.7.0", - "sqlalchemy>=2.0.0", + "pymysql>=1.1.2", + "sqlalchemy>=2.0.48", + "pandas>=3.0.1", + "logging>=0.4.9.6", + "structlog>=25.5.0", + "alembic>=1.18.4", + "pytz>=2026.1.post1", ] requires-python = ">=3.14" @@ -76,3 +82,17 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["data-post-processing"] + +[tool.alembic] +script_location = "%(here)s/migrations" + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s +# Or organize into date-based subdirectories (requires recursive_version_locations = true) +# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s + +# additional paths to be prepended to sys.path. defaults to the current working directory. +prepend_sys_path = [ + "." +] diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..b9f9a23 --- /dev/null +++ b/uv.lock @@ -0,0 +1,703 @@ +version = 1 +revision = 3 +requires-python = ">=3.14" +resolution-markers = [ + "sys_platform == 'win32'", + "sys_platform == 'emscripten'", + "sys_platform != 'emscripten' and sys_platform != 'win32'", +] + +[[package]] +name = "alembic" +version = "1.18.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/94/13/8b084e0f2efb0275a1d534838844926f798bd766566b1375174e2448cd31/alembic-1.18.4.tar.gz", hash = "sha256:cb6e1fd84b6174ab8dbb2329f86d631ba9559dd78df550b57804d607672cedbc", size = 2056725, upload-time = "2026-02-10T16:00:47.195Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/29/6533c317b74f707ea28f8d633734dbda2119bbadfc61b2f3640ba835d0f7/alembic-1.18.4-py3-none-any.whl", hash = "sha256:a5ed4adcf6d8a4cb575f3d759f071b03cd6e5c7618eb796cb52497be25bfe19a", size = 263893, upload-time = "2026-02-10T16:00:49.997Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + +[[package]] +name = "asttokens" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/a5/8e3f9b6771b0b408517c82d97aed8f2036509bc247d46114925e32fe33f0/asttokens-3.0.1.tar.gz", hash = "sha256:71a4ee5de0bde6a31d64f6b13f2293ac190344478f081c3d1bccfcf5eacb0cb7", size = 62308, upload-time = "2025-11-15T16:43:48.578Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/39/e7eaf1799466a4aef85b6a4fe7bd175ad2b1c6345066aa33f1f58d4b18d0/asttokens-3.0.1-py3-none-any.whl", hash = "sha256:15a3ebc0f43c2d0a50eeafea25e19046c68398e487b9f1f5b517f7c0f40f976a", size = 27047, upload-time = "2025-11-15T16:43:16.109Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.13.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/56/95b7e30fa389756cb56630faa728da46a27b8c6eb46f9d557c68fff12b65/coverage-7.13.4.tar.gz", hash = "sha256:e5c8f6ed1e61a8b2dcdf31eb0b9bbf0130750ca79c1c49eb898e2ad86f5ccc91", size = 827239, upload-time = "2026-02-09T12:59:03.86Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/11/a9cf762bb83386467737d32187756a42094927150c3e107df4cb078e8590/coverage-7.13.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:300deaee342f90696ed186e3a00c71b5b3d27bffe9e827677954f4ee56969601", size = 219522, upload-time = "2026-02-09T12:58:08.623Z" }, + { url = "https://files.pythonhosted.org/packages/d3/28/56e6d892b7b052236d67c95f1936b6a7cf7c3e2634bf27610b8cbd7f9c60/coverage-7.13.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:29e3220258d682b6226a9b0925bc563ed9a1ebcff3cad30f043eceea7eaf2689", size = 219855, upload-time = "2026-02-09T12:58:10.176Z" }, + { url = "https://files.pythonhosted.org/packages/e5/69/233459ee9eb0c0d10fcc2fe425a029b3fa5ce0f040c966ebce851d030c70/coverage-7.13.4-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:391ee8f19bef69210978363ca930f7328081c6a0152f1166c91f0b5fdd2a773c", size = 250887, upload-time = "2026-02-09T12:58:12.503Z" }, + { url = "https://files.pythonhosted.org/packages/06/90/2cdab0974b9b5bbc1623f7876b73603aecac11b8d95b85b5b86b32de5eab/coverage-7.13.4-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0dd7ab8278f0d58a0128ba2fca25824321f05d059c1441800e934ff2efa52129", size = 253396, upload-time = "2026-02-09T12:58:14.615Z" }, + { url = "https://files.pythonhosted.org/packages/ac/15/ea4da0f85bf7d7b27635039e649e99deb8173fe551096ea15017f7053537/coverage-7.13.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78cdf0d578b15148b009ccf18c686aa4f719d887e76e6b40c38ffb61d264a552", size = 254745, upload-time = "2026-02-09T12:58:16.162Z" }, + { url = "https://files.pythonhosted.org/packages/99/11/bb356e86920c655ca4d61daee4e2bbc7258f0a37de0be32d233b561134ff/coverage-7.13.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:48685fee12c2eb3b27c62f2658e7ea21e9c3239cba5a8a242801a0a3f6a8c62a", size = 257055, upload-time = "2026-02-09T12:58:17.892Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0f/9ae1f8cb17029e09da06ca4e28c9e1d5c1c0a511c7074592e37e0836c915/coverage-7.13.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4e83efc079eb39480e6346a15a1bcb3e9b04759c5202d157e1dd4303cd619356", size = 250911, upload-time = "2026-02-09T12:58:19.495Z" }, + { url = "https://files.pythonhosted.org/packages/89/3a/adfb68558fa815cbc29747b553bc833d2150228f251b127f1ce97e48547c/coverage-7.13.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ecae9737b72408d6a950f7e525f30aca12d4bd8dd95e37342e5beb3a2a8c4f71", size = 252754, upload-time = "2026-02-09T12:58:21.064Z" }, + { url = "https://files.pythonhosted.org/packages/32/b1/540d0c27c4e748bd3cd0bd001076ee416eda993c2bae47a73b7cc9357931/coverage-7.13.4-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ae4578f8528569d3cf303fef2ea569c7f4c4059a38c8667ccef15c6e1f118aa5", size = 250720, upload-time = "2026-02-09T12:58:22.622Z" }, + { url = "https://files.pythonhosted.org/packages/c7/95/383609462b3ffb1fe133014a7c84fc0dd01ed55ac6140fa1093b5af7ebb1/coverage-7.13.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:6fdef321fdfbb30a197efa02d48fcd9981f0d8ad2ae8903ac318adc653f5df98", size = 254994, upload-time = "2026-02-09T12:58:24.548Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ba/1761138e86c81680bfc3c49579d66312865457f9fe405b033184e5793cb3/coverage-7.13.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:2b0f6ccf3dbe577170bebfce1318707d0e8c3650003cb4b3a9dd744575daa8b5", size = 250531, upload-time = "2026-02-09T12:58:26.271Z" }, + { url = "https://files.pythonhosted.org/packages/f8/8e/05900df797a9c11837ab59c4d6fe94094e029582aab75c3309a93e6fb4e3/coverage-7.13.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:75fcd519f2a5765db3f0e391eb3b7d150cce1a771bf4c9f861aeab86c767a3c0", size = 252189, upload-time = "2026-02-09T12:58:27.807Z" }, + { url = "https://files.pythonhosted.org/packages/00/bd/29c9f2db9ea4ed2738b8a9508c35626eb205d51af4ab7bf56a21a2e49926/coverage-7.13.4-cp314-cp314-win32.whl", hash = "sha256:8e798c266c378da2bd819b0677df41ab46d78065fb2a399558f3f6cae78b2fbb", size = 222258, upload-time = "2026-02-09T12:58:29.441Z" }, + { url = "https://files.pythonhosted.org/packages/a7/4d/1f8e723f6829977410efeb88f73673d794075091c8c7c18848d273dc9d73/coverage-7.13.4-cp314-cp314-win_amd64.whl", hash = "sha256:245e37f664d89861cf2329c9afa2c1fe9e6d4e1a09d872c947e70718aeeac505", size = 223073, upload-time = "2026-02-09T12:58:31.026Z" }, + { url = "https://files.pythonhosted.org/packages/51/5b/84100025be913b44e082ea32abcf1afbf4e872f5120b7a1cab1d331b1e13/coverage-7.13.4-cp314-cp314-win_arm64.whl", hash = "sha256:ad27098a189e5838900ce4c2a99f2fe42a0bf0c2093c17c69b45a71579e8d4a2", size = 221638, upload-time = "2026-02-09T12:58:32.599Z" }, + { url = "https://files.pythonhosted.org/packages/a7/e4/c884a405d6ead1370433dad1e3720216b4f9fd8ef5b64bfd984a2a60a11a/coverage-7.13.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:85480adfb35ffc32d40918aad81b89c69c9cc5661a9b8a81476d3e645321a056", size = 220246, upload-time = "2026-02-09T12:58:34.181Z" }, + { url = "https://files.pythonhosted.org/packages/81/5c/4d7ed8b23b233b0fffbc9dfec53c232be2e695468523242ea9fd30f97ad2/coverage-7.13.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:79be69cf7f3bf9b0deeeb062eab7ac7f36cd4cc4c4dd694bd28921ba4d8596cc", size = 220514, upload-time = "2026-02-09T12:58:35.704Z" }, + { url = "https://files.pythonhosted.org/packages/2f/6f/3284d4203fd2f28edd73034968398cd2d4cb04ab192abc8cff007ea35679/coverage-7.13.4-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:caa421e2684e382c5d8973ac55e4f36bed6821a9bad5c953494de960c74595c9", size = 261877, upload-time = "2026-02-09T12:58:37.864Z" }, + { url = "https://files.pythonhosted.org/packages/09/aa/b672a647bbe1556a85337dc95bfd40d146e9965ead9cc2fe81bde1e5cbce/coverage-7.13.4-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:14375934243ee05f56c45393fe2ce81fe5cc503c07cee2bdf1725fb8bef3ffaf", size = 264004, upload-time = "2026-02-09T12:58:39.492Z" }, + { url = "https://files.pythonhosted.org/packages/79/a1/aa384dbe9181f98bba87dd23dda436f0c6cf2e148aecbb4e50fc51c1a656/coverage-7.13.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25a41c3104d08edb094d9db0d905ca54d0cd41c928bb6be3c4c799a54753af55", size = 266408, upload-time = "2026-02-09T12:58:41.852Z" }, + { url = "https://files.pythonhosted.org/packages/53/5e/5150bf17b4019bc600799f376bb9606941e55bd5a775dc1e096b6ffea952/coverage-7.13.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f01afcff62bf9a08fb32b2c1d6e924236c0383c02c790732b6537269e466a72", size = 267544, upload-time = "2026-02-09T12:58:44.093Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ed/f1de5c675987a4a7a672250d2c5c9d73d289dbf13410f00ed7181d8017dd/coverage-7.13.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:eb9078108fbf0bcdde37c3f4779303673c2fa1fe8f7956e68d447d0dd426d38a", size = 260980, upload-time = "2026-02-09T12:58:45.721Z" }, + { url = "https://files.pythonhosted.org/packages/b3/e3/fe758d01850aa172419a6743fe76ba8b92c29d181d4f676ffe2dae2ba631/coverage-7.13.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0e086334e8537ddd17e5f16a344777c1ab8194986ec533711cbe6c41cde841b6", size = 263871, upload-time = "2026-02-09T12:58:47.334Z" }, + { url = "https://files.pythonhosted.org/packages/b6/76/b829869d464115e22499541def9796b25312b8cf235d3bb00b39f1675395/coverage-7.13.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:725d985c5ab621268b2edb8e50dfe57633dc69bda071abc470fed55a14935fd3", size = 261472, upload-time = "2026-02-09T12:58:48.995Z" }, + { url = "https://files.pythonhosted.org/packages/14/9e/caedb1679e73e2f6ad240173f55218488bfe043e38da577c4ec977489915/coverage-7.13.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:3c06f0f1337c667b971ca2f975523347e63ec5e500b9aa5882d91931cd3ef750", size = 265210, upload-time = "2026-02-09T12:58:51.178Z" }, + { url = "https://files.pythonhosted.org/packages/3a/10/0dd02cb009b16ede425b49ec344aba13a6ae1dc39600840ea6abcb085ac4/coverage-7.13.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:590c0ed4bf8e85f745e6b805b2e1c457b2e33d5255dd9729743165253bc9ad39", size = 260319, upload-time = "2026-02-09T12:58:53.081Z" }, + { url = "https://files.pythonhosted.org/packages/92/8e/234d2c927af27c6d7a5ffad5bd2cf31634c46a477b4c7adfbfa66baf7ebb/coverage-7.13.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:eb30bf180de3f632cd043322dad5751390e5385108b2807368997d1a92a509d0", size = 262638, upload-time = "2026-02-09T12:58:55.258Z" }, + { url = "https://files.pythonhosted.org/packages/2f/64/e5547c8ff6964e5965c35a480855911b61509cce544f4d442caa759a0702/coverage-7.13.4-cp314-cp314t-win32.whl", hash = "sha256:c4240e7eded42d131a2d2c4dec70374b781b043ddc79a9de4d55ca71f8e98aea", size = 223040, upload-time = "2026-02-09T12:58:56.936Z" }, + { url = "https://files.pythonhosted.org/packages/c7/96/38086d58a181aac86d503dfa9c47eb20715a79c3e3acbdf786e92e5c09a8/coverage-7.13.4-cp314-cp314t-win_amd64.whl", hash = "sha256:4c7d3cc01e7350f2f0f6f7036caaf5673fb56b6998889ccfe9e1c1fe75a9c932", size = 224148, upload-time = "2026-02-09T12:58:58.645Z" }, + { url = "https://files.pythonhosted.org/packages/ce/72/8d10abd3740a0beb98c305e0c3faf454366221c0f37a8bcf8f60020bb65a/coverage-7.13.4-cp314-cp314t-win_arm64.whl", hash = "sha256:23e3f687cf945070d1c90f85db66d11e3025665d8dafa831301a0e0038f3db9b", size = 222172, upload-time = "2026-02-09T12:59:00.396Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4a/331fe2caf6799d591109bb9c08083080f6de90a823695d412a935622abb2/coverage-7.13.4-py3-none-any.whl", hash = "sha256:1af1641e57cf7ba1bd67d677c9abdbcd6cc2ab7da3bca7fa1e2b7e50e65f2ad0", size = 211242, upload-time = "2026-02-09T12:59:02.032Z" }, +] + +[[package]] +name = "data-post-processing-v2" +version = "1.0.0" +source = { editable = "." } +dependencies = [ + { name = "alembic" }, + { name = "logging" }, + { name = "pandas" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pymysql" }, + { name = "pytz" }, + { name = "sqlalchemy" }, + { name = "structlog" }, +] + +[package.dev-dependencies] +dev = [ + { name = "coverage" }, + { name = "icecream" }, + { name = "mypy" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-clarity" }, + { name = "pytest-mock" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", specifier = ">=1.18.4" }, + { name = "logging", specifier = ">=0.4.9.6" }, + { name = "pandas", specifier = ">=3.0.1" }, + { name = "pydantic", specifier = ">=2.10.0" }, + { name = "pydantic-settings", specifier = ">=2.7.0" }, + { name = "pymysql", specifier = ">=1.1.2" }, + { name = "pytz", specifier = ">=2026.1.post1" }, + { name = "sqlalchemy", specifier = ">=2.0.48" }, + { name = "structlog", specifier = ">=25.5.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "coverage", extras = ["toml"], specifier = ">=7.12.0" }, + { name = "icecream", specifier = ">=2.1.8" }, + { name = "mypy", specifier = ">=1.13.0" }, + { name = "pytest", specifier = ">=8.3.0" }, + { name = "pytest-asyncio", specifier = ">=0.25.0" }, + { name = "pytest-clarity", specifier = ">=1.0.1" }, + { name = "pytest-mock", specifier = ">=3.14.0" }, + { name = "ruff", specifier = ">=0.8.0" }, +] + +[[package]] +name = "executing" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/28/c14e053b6762b1044f34a13aab6859bbf40456d37d23aa286ac24cfd9a5d/executing-2.2.1.tar.gz", hash = "sha256:3632cc370565f6648cc328b32435bd120a1e4ebb20c77e3fdde9a13cd1e533c4", size = 1129488, upload-time = "2025-09-01T09:48:10.866Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/ea/53f2148663b321f21b5a606bd5f191517cf40b7072c0497d3c92c4a13b1e/executing-2.2.1-py2.py3-none-any.whl", hash = "sha256:760643d3452b4d777d295bb167ccc74c64a81df23fb5e08eff250c425a4b2017", size = 28317, upload-time = "2025-09-01T09:48:08.5Z" }, +] + +[[package]] +name = "greenlet" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/51/1664f6b78fc6ebbd98019a1fd730e83fa78f2db7058f72b1463d3612b8db/greenlet-3.3.2.tar.gz", hash = "sha256:2eaf067fc6d886931c7962e8c6bede15d2f01965560f3359b27c80bde2d151f2", size = 188267, upload-time = "2026-02-20T20:54:15.531Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/ae/8bffcbd373b57a5992cd077cbe8858fff39110480a9d50697091faea6f39/greenlet-3.3.2-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8d1658d7291f9859beed69a776c10822a0a799bc4bfe1bd4272bb60e62507dab", size = 279650, upload-time = "2026-02-20T20:18:00.783Z" }, + { url = "https://files.pythonhosted.org/packages/d1/c0/45f93f348fa49abf32ac8439938726c480bd96b2a3c6f4d949ec0124b69f/greenlet-3.3.2-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18cb1b7337bca281915b3c5d5ae19f4e76d35e1df80f4ad3c1a7be91fadf1082", size = 650295, upload-time = "2026-02-20T20:47:34.036Z" }, + { url = "https://files.pythonhosted.org/packages/b3/de/dd7589b3f2b8372069ab3e4763ea5329940fc7ad9dcd3e272a37516d7c9b/greenlet-3.3.2-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2e47408e8ce1c6f1ceea0dffcdf6ebb85cc09e55c7af407c99f1112016e45e9", size = 662163, upload-time = "2026-02-20T20:56:01.295Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d8/09bfa816572a4d83bccd6750df1926f79158b1c36c5f73786e26dbe4ee38/greenlet-3.3.2-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63d10328839d1973e5ba35e98cccbca71b232b14051fd957b6f8b6e8e80d0506", size = 664160, upload-time = "2026-02-20T20:21:04.015Z" }, + { url = "https://files.pythonhosted.org/packages/48/cf/56832f0c8255d27f6c35d41b5ec91168d74ec721d85f01a12131eec6b93c/greenlet-3.3.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8e4ab3cfb02993c8cc248ea73d7dae6cec0253e9afa311c9b37e603ca9fad2ce", size = 1619181, upload-time = "2026-02-20T20:49:36.052Z" }, + { url = "https://files.pythonhosted.org/packages/0a/23/b90b60a4aabb4cec0796e55f25ffbfb579a907c3898cd2905c8918acaa16/greenlet-3.3.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:94ad81f0fd3c0c0681a018a976e5c2bd2ca2d9d94895f23e7bb1af4e8af4e2d5", size = 1687713, upload-time = "2026-02-20T20:21:11.684Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ca/2101ca3d9223a1dc125140dbc063644dca76df6ff356531eb27bc267b446/greenlet-3.3.2-cp314-cp314-win_amd64.whl", hash = "sha256:8c4dd0f3997cf2512f7601563cc90dfb8957c0cff1e3a1b23991d4ea1776c492", size = 232034, upload-time = "2026-02-20T20:20:08.186Z" }, + { url = "https://files.pythonhosted.org/packages/f6/4a/ecf894e962a59dea60f04877eea0fd5724618da89f1867b28ee8b91e811f/greenlet-3.3.2-cp314-cp314-win_arm64.whl", hash = "sha256:cd6f9e2bbd46321ba3bbb4c8a15794d32960e3b0ae2cc4d49a1a53d314805d71", size = 231437, upload-time = "2026-02-20T20:18:59.722Z" }, + { url = "https://files.pythonhosted.org/packages/98/6d/8f2ef704e614bcf58ed43cfb8d87afa1c285e98194ab2cfad351bf04f81e/greenlet-3.3.2-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:e26e72bec7ab387ac80caa7496e0f908ff954f31065b0ffc1f8ecb1338b11b54", size = 286617, upload-time = "2026-02-20T20:19:29.856Z" }, + { url = "https://files.pythonhosted.org/packages/5e/0d/93894161d307c6ea237a43988f27eba0947b360b99ac5239ad3fe09f0b47/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b466dff7a4ffda6ca975979bab80bdadde979e29fc947ac3be4451428d8b0e4", size = 655189, upload-time = "2026-02-20T20:47:35.742Z" }, + { url = "https://files.pythonhosted.org/packages/f5/2c/d2d506ebd8abcb57386ec4f7ba20f4030cbe56eae541bc6fd6ef399c0b41/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b8bddc5b73c9720bea487b3bffdb1840fe4e3656fba3bd40aa1489e9f37877ff", size = 658225, upload-time = "2026-02-20T20:56:02.527Z" }, + { url = "https://files.pythonhosted.org/packages/8e/30/3a09155fbf728673a1dea713572d2d31159f824a37c22da82127056c44e4/greenlet-3.3.2-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b26b0f4428b871a751968285a1ac9648944cea09807177ac639b030bddebcea4", size = 657907, upload-time = "2026-02-20T20:21:05.259Z" }, + { url = "https://files.pythonhosted.org/packages/f3/fd/d05a4b7acd0154ed758797f0a43b4c0962a843bedfe980115e842c5b2d08/greenlet-3.3.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1fb39a11ee2e4d94be9a76671482be9398560955c9e568550de0224e41104727", size = 1618857, upload-time = "2026-02-20T20:49:37.309Z" }, + { url = "https://files.pythonhosted.org/packages/6f/e1/50ee92a5db521de8f35075b5eff060dd43d39ebd46c2181a2042f7070385/greenlet-3.3.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:20154044d9085151bc309e7689d6f7ba10027f8f5a8c0676ad398b951913d89e", size = 1680010, upload-time = "2026-02-20T20:21:13.427Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/45d90626aef8e65336bed690106d1382f7a43665e2249017e9527df8823b/greenlet-3.3.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c04c5e06ec3e022cbfe2cd4a846e1d4e50087444f875ff6d2c2ad8445495cf1a", size = 237086, upload-time = "2026-02-20T20:20:45.786Z" }, +] + +[[package]] +name = "icecream" +version = "2.1.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asttokens" }, + { name = "colorama" }, + { name = "executing" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fe/e2/3d064dfedbbc16687e0f56cd9b1d55e4e6dfd13d61b9435b61c250aaef3c/icecream-2.1.10.tar.gz", hash = "sha256:15900126ba7dbe1f83819583cbe5ff79a2943224600878d89307e4633b32e528", size = 13924, upload-time = "2026-01-21T07:34:17.652Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/4f/91c4ee1af60bb4b2519540fd46fb56dfc70ede0cf3d97f57aff62d61190b/icecream-2.1.10-py3-none-any.whl", hash = "sha256:6b0ae3e899de12954cd26d8611dcff86518ff19f40deef333427da2ccf4036b2", size = 16373, upload-time = "2026-01-21T07:34:15.801Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "librt" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/56/9c/b4b0c54d84da4a94b37bd44151e46d5e583c9534c7e02250b961b1b6d8a8/librt-0.8.1.tar.gz", hash = "sha256:be46a14693955b3bd96014ccbdb8339ee8c9346fbe11c1b78901b55125f14c73", size = 177471, upload-time = "2026-02-17T16:13:06.101Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/6a/907ef6800f7bca71b525a05f1839b21f708c09043b1c6aa77b6b827b3996/librt-0.8.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:6cfa7fe54fd4d1f47130017351a959fe5804bda7a0bc7e07a2cdbc3fdd28d34f", size = 66081, upload-time = "2026-02-17T16:12:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/1b/18/25e991cd5640c9fb0f8d91b18797b29066b792f17bf8493da183bf5caabe/librt-0.8.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:228c2409c079f8c11fb2e5d7b277077f694cb93443eb760e00b3b83cb8b3176c", size = 68309, upload-time = "2026-02-17T16:12:13.756Z" }, + { url = "https://files.pythonhosted.org/packages/a4/36/46820d03f058cfb5a9de5940640ba03165ed8aded69e0733c417bb04df34/librt-0.8.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7aae78ab5e3206181780e56912d1b9bb9f90a7249ce12f0e8bf531d0462dd0fc", size = 196804, upload-time = "2026-02-17T16:12:14.818Z" }, + { url = "https://files.pythonhosted.org/packages/59/18/5dd0d3b87b8ff9c061849fbdb347758d1f724b9a82241aa908e0ec54ccd0/librt-0.8.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:172d57ec04346b047ca6af181e1ea4858086c80bdf455f61994c4aa6fc3f866c", size = 206907, upload-time = "2026-02-17T16:12:16.513Z" }, + { url = "https://files.pythonhosted.org/packages/d1/96/ef04902aad1424fd7299b62d1890e803e6ab4018c3044dca5922319c4b97/librt-0.8.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b1977c4ea97ce5eb7755a78fae68d87e4102e4aaf54985e8b56806849cc06a3", size = 221217, upload-time = "2026-02-17T16:12:17.906Z" }, + { url = "https://files.pythonhosted.org/packages/6d/ff/7e01f2dda84a8f5d280637a2e5827210a8acca9a567a54507ef1c75b342d/librt-0.8.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:10c42e1f6fd06733ef65ae7bebce2872bcafd8d6e6b0a08fe0a05a23b044fb14", size = 214622, upload-time = "2026-02-17T16:12:19.108Z" }, + { url = "https://files.pythonhosted.org/packages/1e/8c/5b093d08a13946034fed57619742f790faf77058558b14ca36a6e331161e/librt-0.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4c8dfa264b9193c4ee19113c985c95f876fae5e51f731494fc4e0cf594990ba7", size = 221987, upload-time = "2026-02-17T16:12:20.331Z" }, + { url = "https://files.pythonhosted.org/packages/d3/cc/86b0b3b151d40920ad45a94ce0171dec1aebba8a9d72bb3fa00c73ab25dd/librt-0.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:01170b6729a438f0dedc4a26ed342e3dc4f02d1000b4b19f980e1877f0c297e6", size = 215132, upload-time = "2026-02-17T16:12:21.54Z" }, + { url = "https://files.pythonhosted.org/packages/fc/be/8588164a46edf1e69858d952654e216a9a91174688eeefb9efbb38a9c799/librt-0.8.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7b02679a0d783bdae30d443025b94465d8c3dc512f32f5b5031f93f57ac32071", size = 215195, upload-time = "2026-02-17T16:12:23.073Z" }, + { url = "https://files.pythonhosted.org/packages/f5/f2/0b9279bea735c734d69344ecfe056c1ba211694a72df10f568745c899c76/librt-0.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:190b109bb69592a3401fe1ffdea41a2e73370ace2ffdc4a0e8e2b39cdea81b78", size = 237946, upload-time = "2026-02-17T16:12:24.275Z" }, + { url = "https://files.pythonhosted.org/packages/e9/cc/5f2a34fbc8aeb35314a3641f9956fa9051a947424652fad9882be7a97949/librt-0.8.1-cp314-cp314-win32.whl", hash = "sha256:e70a57ecf89a0f64c24e37f38d3fe217a58169d2fe6ed6d70554964042474023", size = 50689, upload-time = "2026-02-17T16:12:25.766Z" }, + { url = "https://files.pythonhosted.org/packages/a0/76/cd4d010ab2147339ca2b93e959c3686e964edc6de66ddacc935c325883d7/librt-0.8.1-cp314-cp314-win_amd64.whl", hash = "sha256:7e2f3edca35664499fbb36e4770650c4bd4a08abc1f4458eab9df4ec56389730", size = 57875, upload-time = "2026-02-17T16:12:27.465Z" }, + { url = "https://files.pythonhosted.org/packages/84/0f/2143cb3c3ca48bd3379dcd11817163ca50781927c4537345d608b5045998/librt-0.8.1-cp314-cp314-win_arm64.whl", hash = "sha256:0d2f82168e55ddefd27c01c654ce52379c0750ddc31ee86b4b266bcf4d65f2a3", size = 48058, upload-time = "2026-02-17T16:12:28.556Z" }, + { url = "https://files.pythonhosted.org/packages/d2/0e/9b23a87e37baf00311c3efe6b48d6b6c168c29902dfc3f04c338372fd7db/librt-0.8.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2c74a2da57a094bd48d03fa5d196da83d2815678385d2978657499063709abe1", size = 68313, upload-time = "2026-02-17T16:12:29.659Z" }, + { url = "https://files.pythonhosted.org/packages/db/9a/859c41e5a4f1c84200a7d2b92f586aa27133c8243b6cac9926f6e54d01b9/librt-0.8.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a355d99c4c0d8e5b770313b8b247411ed40949ca44e33e46a4789b9293a907ee", size = 70994, upload-time = "2026-02-17T16:12:31.516Z" }, + { url = "https://files.pythonhosted.org/packages/4c/28/10605366ee599ed34223ac2bf66404c6fb59399f47108215d16d5ad751a8/librt-0.8.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2eb345e8b33fb748227409c9f1233d4df354d6e54091f0e8fc53acdb2ffedeb7", size = 220770, upload-time = "2026-02-17T16:12:33.294Z" }, + { url = "https://files.pythonhosted.org/packages/af/8d/16ed8fd452dafae9c48d17a6bc1ee3e818fd40ef718d149a8eff2c9f4ea2/librt-0.8.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9be2f15e53ce4e83cc08adc29b26fb5978db62ef2a366fbdf716c8a6c8901040", size = 235409, upload-time = "2026-02-17T16:12:35.443Z" }, + { url = "https://files.pythonhosted.org/packages/89/1b/7bdf3e49349c134b25db816e4a3db6b94a47ac69d7d46b1e682c2c4949be/librt-0.8.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:785ae29c1f5c6e7c2cde2c7c0e148147f4503da3abc5d44d482068da5322fd9e", size = 246473, upload-time = "2026-02-17T16:12:36.656Z" }, + { url = "https://files.pythonhosted.org/packages/4e/8a/91fab8e4fd2a24930a17188c7af5380eb27b203d72101c9cc000dbdfd95a/librt-0.8.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1d3a7da44baf692f0c6aeb5b2a09c5e6fc7a703bca9ffa337ddd2e2da53f7732", size = 238866, upload-time = "2026-02-17T16:12:37.849Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e0/c45a098843fc7c07e18a7f8a24ca8496aecbf7bdcd54980c6ca1aaa79a8e/librt-0.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5fc48998000cbc39ec0d5311312dda93ecf92b39aaf184c5e817d5d440b29624", size = 250248, upload-time = "2026-02-17T16:12:39.445Z" }, + { url = "https://files.pythonhosted.org/packages/82/30/07627de23036640c952cce0c1fe78972e77d7d2f8fd54fa5ef4554ff4a56/librt-0.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:e96baa6820280077a78244b2e06e416480ed859bbd8e5d641cf5742919d8beb4", size = 240629, upload-time = "2026-02-17T16:12:40.889Z" }, + { url = "https://files.pythonhosted.org/packages/fb/c1/55bfe1ee3542eba055616f9098eaf6eddb966efb0ca0f44eaa4aba327307/librt-0.8.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31362dbfe297b23590530007062c32c6f6176f6099646bb2c95ab1b00a57c382", size = 239615, upload-time = "2026-02-17T16:12:42.446Z" }, + { url = "https://files.pythonhosted.org/packages/2b/39/191d3d28abc26c9099b19852e6c99f7f6d400b82fa5a4e80291bd3803e19/librt-0.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc3656283d11540ab0ea01978378e73e10002145117055e03722417aeab30994", size = 263001, upload-time = "2026-02-17T16:12:43.627Z" }, + { url = "https://files.pythonhosted.org/packages/b9/eb/7697f60fbe7042ab4e88f4ee6af496b7f222fffb0a4e3593ef1f29f81652/librt-0.8.1-cp314-cp314t-win32.whl", hash = "sha256:738f08021b3142c2918c03692608baed43bc51144c29e35807682f8070ee2a3a", size = 51328, upload-time = "2026-02-17T16:12:45.148Z" }, + { url = "https://files.pythonhosted.org/packages/7c/72/34bf2eb7a15414a23e5e70ecb9440c1d3179f393d9349338a91e2781c0fb/librt-0.8.1-cp314-cp314t-win_amd64.whl", hash = "sha256:89815a22daf9c51884fb5dbe4f1ef65ee6a146e0b6a8df05f753e2e4a9359bf4", size = 58722, upload-time = "2026-02-17T16:12:46.85Z" }, + { url = "https://files.pythonhosted.org/packages/b2/c8/d148e041732d631fc76036f8b30fae4e77b027a1e95b7a84bb522481a940/librt-0.8.1-cp314-cp314t-win_arm64.whl", hash = "sha256:bf512a71a23504ed08103a13c941f763db13fb11177beb3d9244c98c29fb4a61", size = 48755, upload-time = "2026-02-17T16:12:47.943Z" }, +] + +[[package]] +name = "logging" +version = "0.4.9.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/93/4b/979db9e44be09f71e85c9c8cfc42f258adfb7d93ce01deed2788b2948919/logging-0.4.9.6.tar.gz", hash = "sha256:26f6b50773f085042d301085bd1bf5d9f3735704db9f37c1ce6d8b85c38f2417", size = 96029, upload-time = "2013-06-04T23:43:22.086Z" } + +[[package]] +name = "mako" +version = "1.3.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/38/bd5b78a920a64d708fe6bc8e0a2c075e1389d53bef8413725c63ba041535/mako-1.3.10.tar.gz", hash = "sha256:99579a6f39583fa7e5630a28c3c1f440e4e97a414b80372649c0ce338da2ea28", size = 392474, upload-time = "2025-04-10T12:44:31.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" }, +] + +[[package]] +name = "markdown-it-py" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + +[[package]] +name = "mypy" +version = "1.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "librt", marker = "platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/db/4efed9504bc01309ab9c2da7e352cc223569f05478012b5d9ece38fd44d2/mypy-1.19.1.tar.gz", hash = "sha256:19d88bb05303fe63f71dd2c6270daca27cb9401c4ca8255fe50d1d920e0eb9ba", size = 3582404, upload-time = "2025-12-15T05:03:48.42Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/eb/b83e75f4c820c4247a58580ef86fcd35165028f191e7e1ba57128c52782d/mypy-1.19.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06e6170bd5836770e8104c8fdd58e5e725cfeb309f0a6c681a811f557e97eac1", size = 13199744, upload-time = "2025-12-15T05:03:30.823Z" }, + { url = "https://files.pythonhosted.org/packages/94/28/52785ab7bfa165f87fcbb61547a93f98bb20e7f82f90f165a1f69bce7b3d/mypy-1.19.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:804bd67b8054a85447c8954215a906d6eff9cabeabe493fb6334b24f4bfff718", size = 12215815, upload-time = "2025-12-15T05:02:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/0a/c6/bdd60774a0dbfb05122e3e925f2e9e846c009e479dcec4821dad881f5b52/mypy-1.19.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21761006a7f497cb0d4de3d8ef4ca70532256688b0523eee02baf9eec895e27b", size = 12740047, upload-time = "2025-12-15T05:03:33.168Z" }, + { url = "https://files.pythonhosted.org/packages/32/2a/66ba933fe6c76bd40d1fe916a83f04fed253152f451a877520b3c4a5e41e/mypy-1.19.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28902ee51f12e0f19e1e16fbe2f8f06b6637f482c459dd393efddd0ec7f82045", size = 13601998, upload-time = "2025-12-15T05:03:13.056Z" }, + { url = "https://files.pythonhosted.org/packages/e3/da/5055c63e377c5c2418760411fd6a63ee2b96cf95397259038756c042574f/mypy-1.19.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:481daf36a4c443332e2ae9c137dfee878fcea781a2e3f895d54bd3002a900957", size = 13807476, upload-time = "2025-12-15T05:03:17.977Z" }, + { url = "https://files.pythonhosted.org/packages/cd/09/4ebd873390a063176f06b0dbf1f7783dd87bd120eae7727fa4ae4179b685/mypy-1.19.1-cp314-cp314-win_amd64.whl", hash = "sha256:8bb5c6f6d043655e055be9b542aa5f3bdd30e4f3589163e85f93f3640060509f", size = 10281872, upload-time = "2025-12-15T05:03:05.549Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f4/4ce9a05ce5ded1de3ec1c1d96cf9f9504a04e54ce0ed55cfa38619a32b8d/mypy-1.19.1-py3-none-any.whl", hash = "sha256:f1235f5ea01b7db5468d53ece6aaddf1ad0b88d9e7462b86ef96fe04995d7247", size = 2471239, upload-time = "2025-12-15T05:03:07.248Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "numpy" +version = "2.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/10/8b/c265f4823726ab832de836cdd184d0986dcf94480f81e8739692a7ac7af2/numpy-2.4.3.tar.gz", hash = "sha256:483a201202b73495f00dbc83796c6ae63137a9bdade074f7648b3e32613412dd", size = 20727743, upload-time = "2026-03-09T07:58:53.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/ae/3936f79adebf8caf81bd7a599b90a561334a658be4dcc7b6329ebf4ee8de/numpy-2.4.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:5884ce5c7acfae1e4e1b6fde43797d10aa506074d25b531b4f54bde33c0c31d4", size = 16664563, upload-time = "2026-03-09T07:57:43.817Z" }, + { url = "https://files.pythonhosted.org/packages/9b/62/760f2b55866b496bb1fa7da2a6db076bef908110e568b02fcfc1422e2a3a/numpy-2.4.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:297837823f5bc572c5f9379b0c9f3a3365f08492cbdc33bcc3af174372ebb168", size = 14702161, upload-time = "2026-03-09T07:57:46.169Z" }, + { url = "https://files.pythonhosted.org/packages/32/af/a7a39464e2c0a21526fb4fb76e346fb172ebc92f6d1c7a07c2c139cc17b1/numpy-2.4.3-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:a111698b4a3f8dcbe54c64a7708f049355abd603e619013c346553c1fd4ca90b", size = 5208738, upload-time = "2026-03-09T07:57:48.506Z" }, + { url = "https://files.pythonhosted.org/packages/29/8c/2a0cf86a59558fa078d83805589c2de490f29ed4fb336c14313a161d358a/numpy-2.4.3-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:4bd4741a6a676770e0e97fe9ab2e51de01183df3dcbcec591d26d331a40de950", size = 6543618, upload-time = "2026-03-09T07:57:50.591Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b8/612ce010c0728b1c363fa4ea3aa4c22fe1c5da1de008486f8c2f5cb92fae/numpy-2.4.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:54f29b877279d51e210e0c80709ee14ccbbad647810e8f3d375561c45ef613dd", size = 15680676, upload-time = "2026-03-09T07:57:52.34Z" }, + { url = "https://files.pythonhosted.org/packages/a9/7e/4f120ecc54ba26ddf3dc348eeb9eb063f421de65c05fc961941798feea18/numpy-2.4.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:679f2a834bae9020f81534671c56fd0cc76dd7e5182f57131478e23d0dc59e24", size = 16613492, upload-time = "2026-03-09T07:57:54.91Z" }, + { url = "https://files.pythonhosted.org/packages/2c/86/1b6020db73be330c4b45d5c6ee4295d59cfeef0e3ea323959d053e5a6909/numpy-2.4.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d84f0f881cb2225c2dfd7f78a10a5645d487a496c6668d6cc39f0f114164f3d0", size = 17031789, upload-time = "2026-03-09T07:57:57.641Z" }, + { url = "https://files.pythonhosted.org/packages/07/3a/3b90463bf41ebc21d1b7e06079f03070334374208c0f9a1f05e4ae8455e7/numpy-2.4.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d213c7e6e8d211888cc359bab7199670a00f5b82c0978b9d1c75baf1eddbeac0", size = 18339941, upload-time = "2026-03-09T07:58:00.577Z" }, + { url = "https://files.pythonhosted.org/packages/a8/74/6d736c4cd962259fd8bae9be27363eb4883a2f9069763747347544c2a487/numpy-2.4.3-cp314-cp314-win32.whl", hash = "sha256:52077feedeff7c76ed7c9f1a0428558e50825347b7545bbb8523da2cd55c547a", size = 6007503, upload-time = "2026-03-09T07:58:03.331Z" }, + { url = "https://files.pythonhosted.org/packages/48/39/c56ef87af669364356bb011922ef0734fc49dad51964568634c72a009488/numpy-2.4.3-cp314-cp314-win_amd64.whl", hash = "sha256:0448e7f9caefb34b4b7dd2b77f21e8906e5d6f0365ad525f9f4f530b13df2afc", size = 12444915, upload-time = "2026-03-09T07:58:06.353Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1f/ab8528e38d295fd349310807496fabb7cf9fe2e1f70b97bc20a483ea9d4a/numpy-2.4.3-cp314-cp314-win_arm64.whl", hash = "sha256:b44fd60341c4d9783039598efadd03617fa28d041fc37d22b62d08f2027fa0e7", size = 10494875, upload-time = "2026-03-09T07:58:08.734Z" }, + { url = "https://files.pythonhosted.org/packages/e6/ef/b7c35e4d5ef141b836658ab21a66d1a573e15b335b1d111d31f26c8ef80f/numpy-2.4.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0a195f4216be9305a73c0e91c9b026a35f2161237cf1c6de9b681637772ea657", size = 14822225, upload-time = "2026-03-09T07:58:11.034Z" }, + { url = "https://files.pythonhosted.org/packages/cd/8d/7730fa9278cf6648639946cc816e7cc89f0d891602584697923375f801ed/numpy-2.4.3-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:cd32fbacb9fd1bf041bf8e89e4576b6f00b895f06d00914820ae06a616bdfef7", size = 5328769, upload-time = "2026-03-09T07:58:13.67Z" }, + { url = "https://files.pythonhosted.org/packages/47/01/d2a137317c958b074d338807c1b6a383406cdf8b8e53b075d804cc3d211d/numpy-2.4.3-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:2e03c05abaee1f672e9d67bc858f300b5ccba1c21397211e8d77d98350972093", size = 6649461, upload-time = "2026-03-09T07:58:15.912Z" }, + { url = "https://files.pythonhosted.org/packages/5c/34/812ce12bc0f00272a4b0ec0d713cd237cb390666eb6206323d1cc9cedbb2/numpy-2.4.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7d1ce23cce91fcea443320a9d0ece9b9305d4368875bab09538f7a5b4131938a", size = 15725809, upload-time = "2026-03-09T07:58:17.787Z" }, + { url = "https://files.pythonhosted.org/packages/25/c0/2aed473a4823e905e765fee3dc2cbf504bd3e68ccb1150fbdabd5c39f527/numpy-2.4.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c59020932feb24ed49ffd03704fbab89f22aa9c0d4b180ff45542fe8918f5611", size = 16655242, upload-time = "2026-03-09T07:58:20.476Z" }, + { url = "https://files.pythonhosted.org/packages/f2/c8/7e052b2fc87aa0e86de23f20e2c42bd261c624748aa8efd2c78f7bb8d8c6/numpy-2.4.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9684823a78a6cd6ad7511fc5e25b07947d1d5b5e2812c93fe99d7d4195130720", size = 17080660, upload-time = "2026-03-09T07:58:23.067Z" }, + { url = "https://files.pythonhosted.org/packages/f3/3d/0876746044db2adcb11549f214d104f2e1be00f07a67edbb4e2812094847/numpy-2.4.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0200b25c687033316fb39f0ff4e3e690e8957a2c3c8d22499891ec58c37a3eb5", size = 18380384, upload-time = "2026-03-09T07:58:25.839Z" }, + { url = "https://files.pythonhosted.org/packages/07/12/8160bea39da3335737b10308df4f484235fd297f556745f13092aa039d3b/numpy-2.4.3-cp314-cp314t-win32.whl", hash = "sha256:5e10da9e93247e554bb1d22f8edc51847ddd7dde52d85ce31024c1b4312bfba0", size = 6154547, upload-time = "2026-03-09T07:58:28.289Z" }, + { url = "https://files.pythonhosted.org/packages/42/f3/76534f61f80d74cc9cdf2e570d3d4eeb92c2280a27c39b0aaf471eda7b48/numpy-2.4.3-cp314-cp314t-win_amd64.whl", hash = "sha256:45f003dbdffb997a03da2d1d0cb41fbd24a87507fb41605c0420a3db5bd4667b", size = 12633645, upload-time = "2026-03-09T07:58:30.384Z" }, + { url = "https://files.pythonhosted.org/packages/1f/b6/7c0d4334c15983cec7f92a69e8ce9b1e6f31857e5ee3a413ac424e6bd63d/numpy-2.4.3-cp314-cp314t-win_arm64.whl", hash = "sha256:4d382735cecd7bcf090172489a525cd7d4087bc331f7df9f60ddc9a296cf208e", size = 10565454, upload-time = "2026-03-09T07:58:33.031Z" }, +] + +[[package]] +name = "packaging" +version = "26.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, +] + +[[package]] +name = "pandas" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/0c/b28ed414f080ee0ad153f848586d61d1878f91689950f037f976ce15f6c8/pandas-3.0.1.tar.gz", hash = "sha256:4186a699674af418f655dbd420ed87f50d56b4cd6603784279d9eef6627823c8", size = 4641901, upload-time = "2026-02-17T22:20:16.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/8b/4bb774a998b97e6c2fd62a9e6cfdaae133b636fd1c468f92afb4ae9a447a/pandas-3.0.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:99d0f92ed92d3083d140bf6b97774f9f13863924cf3f52a70711f4e7588f9d0a", size = 10322465, upload-time = "2026-02-17T22:19:36.803Z" }, + { url = "https://files.pythonhosted.org/packages/72/3a/5b39b51c64159f470f1ca3b1c2a87da290657ca022f7cd11442606f607d1/pandas-3.0.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3b66857e983208654294bb6477b8a63dee26b37bdd0eb34d010556e91261784f", size = 9910632, upload-time = "2026-02-17T22:19:39.001Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f7/b449ffb3f68c11da12fc06fbf6d2fa3a41c41e17d0284d23a79e1c13a7e4/pandas-3.0.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56cf59638bf24dc9bdf2154c81e248b3289f9a09a6d04e63608c159022352749", size = 10440535, upload-time = "2026-02-17T22:19:41.157Z" }, + { url = "https://files.pythonhosted.org/packages/55/77/6ea82043db22cb0f2bbfe7198da3544000ddaadb12d26be36e19b03a2dc5/pandas-3.0.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1a9f55e0f46951874b863d1f3906dcb57df2d9be5c5847ba4dfb55b2c815249", size = 10893940, upload-time = "2026-02-17T22:19:43.493Z" }, + { url = "https://files.pythonhosted.org/packages/03/30/f1b502a72468c89412c1b882a08f6eed8a4ee9dc033f35f65d0663df6081/pandas-3.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1849f0bba9c8a2fb0f691d492b834cc8dadf617e29015c66e989448d58d011ee", size = 11442711, upload-time = "2026-02-17T22:19:46.074Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f0/ebb6ddd8fc049e98cabac5c2924d14d1dda26a20adb70d41ea2e428d3ec4/pandas-3.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c3d288439e11b5325b02ae6e9cc83e6805a62c40c5a6220bea9beb899c073b1c", size = 11963918, upload-time = "2026-02-17T22:19:48.838Z" }, + { url = "https://files.pythonhosted.org/packages/09/f8/8ce132104074f977f907442790eaae24e27bce3b3b454e82faa3237ff098/pandas-3.0.1-cp314-cp314-win_amd64.whl", hash = "sha256:93325b0fe372d192965f4cca88d97667f49557398bbf94abdda3bf1b591dbe66", size = 9862099, upload-time = "2026-02-17T22:19:51.081Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b7/6af9aac41ef2456b768ef0ae60acf8abcebb450a52043d030a65b4b7c9bd/pandas-3.0.1-cp314-cp314-win_arm64.whl", hash = "sha256:97ca08674e3287c7148f4858b01136f8bdfe7202ad25ad04fec602dd1d29d132", size = 9185333, upload-time = "2026-02-17T22:19:53.266Z" }, + { url = "https://files.pythonhosted.org/packages/66/fc/848bb6710bc6061cb0c5badd65b92ff75c81302e0e31e496d00029fe4953/pandas-3.0.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:58eeb1b2e0fb322befcf2bbc9ba0af41e616abadb3d3414a6bc7167f6cbfce32", size = 10772664, upload-time = "2026-02-17T22:19:55.806Z" }, + { url = "https://files.pythonhosted.org/packages/69/5c/866a9bbd0f79263b4b0db6ec1a341be13a1473323f05c122388e0f15b21d/pandas-3.0.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:cd9af1276b5ca9e298bd79a26bda32fa9cc87ed095b2a9a60978d2ca058eaf87", size = 10421286, upload-time = "2026-02-17T22:19:58.091Z" }, + { url = "https://files.pythonhosted.org/packages/51/a4/2058fb84fb1cfbfb2d4a6d485e1940bb4ad5716e539d779852494479c580/pandas-3.0.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94f87a04984d6b63788327cd9f79dda62b7f9043909d2440ceccf709249ca988", size = 10342050, upload-time = "2026-02-17T22:20:01.376Z" }, + { url = "https://files.pythonhosted.org/packages/22/1b/674e89996cc4be74db3c4eb09240c4bb549865c9c3f5d9b086ff8fcfbf00/pandas-3.0.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85fe4c4df62e1e20f9db6ebfb88c844b092c22cd5324bdcf94bfa2fc1b391221", size = 10740055, upload-time = "2026-02-17T22:20:04.328Z" }, + { url = "https://files.pythonhosted.org/packages/d0/f8/e954b750764298c22fa4614376531fe63c521ef517e7059a51f062b87dca/pandas-3.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:331ca75a2f8672c365ae25c0b29e46f5ac0c6551fdace8eec4cd65e4fac271ff", size = 11357632, upload-time = "2026-02-17T22:20:06.647Z" }, + { url = "https://files.pythonhosted.org/packages/6d/02/c6e04b694ffd68568297abd03588b6d30295265176a5c01b7459d3bc35a3/pandas-3.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:15860b1fdb1973fffade772fdb931ccf9b2f400a3f5665aef94a00445d7d8dd5", size = 11810974, upload-time = "2026-02-17T22:20:08.946Z" }, + { url = "https://files.pythonhosted.org/packages/89/41/d7dfb63d2407f12055215070c42fc6ac41b66e90a2946cdc5e759058398b/pandas-3.0.1-cp314-cp314t-win_amd64.whl", hash = "sha256:44f1364411d5670efa692b146c748f4ed013df91ee91e9bec5677fb1fd58b937", size = 10884622, upload-time = "2026-02-17T22:20:11.711Z" }, + { url = "https://files.pythonhosted.org/packages/68/b0/34937815889fa982613775e4b97fddd13250f11012d769949c5465af2150/pandas-3.0.1-cp314-cp314t-win_arm64.whl", hash = "sha256:108dd1790337a494aa80e38def654ca3f0968cf4f362c85f44c15e471667102d", size = 9452085, upload-time = "2026-02-17T22:20:14.331Z" }, +] + +[[package]] +name = "pathspec" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/36/e27608899f9b8d4dff0617b2d9ab17ca5608956ca44461ac14ac48b44015/pathspec-1.0.4.tar.gz", hash = "sha256:0210e2ae8a21a9137c0d470578cb0e595af87edaa6ebf12ff176f14a02e0e645", size = 131200, upload-time = "2026-01-27T03:59:46.938Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/3c/2c197d226f9ea224a9ab8d197933f9da0ae0aac5b6e0f884e2b8d9c8e9f7/pathspec-1.0.4-py3-none-any.whl", hash = "sha256:fb6ae2fd4e7c921a165808a552060e722767cfa526f99ca5156ed2ce45a5c723", size = 55206, upload-time = "2026-01-27T03:59:45.137Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pprintpp" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/1a/7737e7a0774da3c3824d654993cf57adc915cb04660212f03406334d8c0b/pprintpp-0.4.0.tar.gz", hash = "sha256:ea826108e2c7f49dc6d66c752973c3fc9749142a798d6b254e1e301cfdbc6403", size = 17995, upload-time = "2018-07-01T01:42:34.87Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/d1/e4ed95fdd3ef13b78630280d9e9e240aeb65cc7c544ec57106149c3942fb/pprintpp-0.4.0-py2.py3-none-any.whl", hash = "sha256:b6b4dcdd0c0c0d75e4d7b2f21a9e933e5b2ce62b26e1a54537f9651ae5a5c01d", size = 16952, upload-time = "2018-07-01T01:42:36.496Z" }, +] + +[[package]] +name = "pydantic" +version = "2.12.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.41.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/28/46b7c5c9635ae96ea0fbb779e271a38129df2550f763937659ee6c5dbc65/pydantic_core-2.41.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:3f37a19d7ebcdd20b96485056ba9e8b304e27d9904d233d7b1015db320e51f0a", size = 2119622, upload-time = "2025-11-04T13:40:56.68Z" }, + { url = "https://files.pythonhosted.org/packages/74/1a/145646e5687e8d9a1e8d09acb278c8535ebe9e972e1f162ed338a622f193/pydantic_core-2.41.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1d1d9764366c73f996edd17abb6d9d7649a7eb690006ab6adbda117717099b14", size = 1891725, upload-time = "2025-11-04T13:40:58.807Z" }, + { url = "https://files.pythonhosted.org/packages/23/04/e89c29e267b8060b40dca97bfc64a19b2a3cf99018167ea1677d96368273/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25e1c2af0fce638d5f1988b686f3b3ea8cd7de5f244ca147c777769e798a9cd1", size = 1915040, upload-time = "2025-11-04T13:41:00.853Z" }, + { url = "https://files.pythonhosted.org/packages/84/a3/15a82ac7bd97992a82257f777b3583d3e84bdb06ba6858f745daa2ec8a85/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:506d766a8727beef16b7adaeb8ee6217c64fc813646b424d0804d67c16eddb66", size = 2063691, upload-time = "2025-11-04T13:41:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/74/9b/0046701313c6ef08c0c1cf0e028c67c770a4e1275ca73131563c5f2a310a/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4819fa52133c9aa3c387b3328f25c1facc356491e6135b459f1de698ff64d869", size = 2213897, upload-time = "2025-11-04T13:41:05.804Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cd/6bac76ecd1b27e75a95ca3a9a559c643b3afcd2dd62086d4b7a32a18b169/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2b761d210c9ea91feda40d25b4efe82a1707da2ef62901466a42492c028553a2", size = 2333302, upload-time = "2025-11-04T13:41:07.809Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d2/ef2074dc020dd6e109611a8be4449b98cd25e1b9b8a303c2f0fca2f2bcf7/pydantic_core-2.41.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22f0fb8c1c583a3b6f24df2470833b40207e907b90c928cc8d3594b76f874375", size = 2064877, upload-time = "2025-11-04T13:41:09.827Z" }, + { url = "https://files.pythonhosted.org/packages/18/66/e9db17a9a763d72f03de903883c057b2592c09509ccfe468187f2a2eef29/pydantic_core-2.41.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c870e99878c634505236d81e5443092fba820f0373997ff75f90f68cd553", size = 2180680, upload-time = "2025-11-04T13:41:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/d3/9e/3ce66cebb929f3ced22be85d4c2399b8e85b622db77dad36b73c5387f8f8/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0177272f88ab8312479336e1d777f6b124537d47f2123f89cb37e0accea97f90", size = 2138960, upload-time = "2025-11-04T13:41:14.627Z" }, + { url = "https://files.pythonhosted.org/packages/a6/62/205a998f4327d2079326b01abee48e502ea739d174f0a89295c481a2272e/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:63510af5e38f8955b8ee5687740d6ebf7c2a0886d15a6d65c32814613681bc07", size = 2339102, upload-time = "2025-11-04T13:41:16.868Z" }, + { url = "https://files.pythonhosted.org/packages/3c/0d/f05e79471e889d74d3d88f5bd20d0ed189ad94c2423d81ff8d0000aab4ff/pydantic_core-2.41.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:e56ba91f47764cc14f1daacd723e3e82d1a89d783f0f5afe9c364b8bb491ccdb", size = 2326039, upload-time = "2025-11-04T13:41:18.934Z" }, + { url = "https://files.pythonhosted.org/packages/ec/e1/e08a6208bb100da7e0c4b288eed624a703f4d129bde2da475721a80cab32/pydantic_core-2.41.5-cp314-cp314-win32.whl", hash = "sha256:aec5cf2fd867b4ff45b9959f8b20ea3993fc93e63c7363fe6851424c8a7e7c23", size = 1995126, upload-time = "2025-11-04T13:41:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/48/5d/56ba7b24e9557f99c9237e29f5c09913c81eeb2f3217e40e922353668092/pydantic_core-2.41.5-cp314-cp314-win_amd64.whl", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf", size = 2015489, upload-time = "2025-11-04T13:41:24.076Z" }, + { url = "https://files.pythonhosted.org/packages/4e/bb/f7a190991ec9e3e0ba22e4993d8755bbc4a32925c0b5b42775c03e8148f9/pydantic_core-2.41.5-cp314-cp314-win_arm64.whl", hash = "sha256:e672ba74fbc2dc8eea59fb6d4aed6845e6905fc2a8afe93175d94a83ba2a01a0", size = 1977288, upload-time = "2025-11-04T13:41:26.33Z" }, + { url = "https://files.pythonhosted.org/packages/92/ed/77542d0c51538e32e15afe7899d79efce4b81eee631d99850edc2f5e9349/pydantic_core-2.41.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8566def80554c3faa0e65ac30ab0932b9e3a5cd7f8323764303d468e5c37595a", size = 2120255, upload-time = "2025-11-04T13:41:28.569Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3d/6913dde84d5be21e284439676168b28d8bbba5600d838b9dca99de0fad71/pydantic_core-2.41.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:b80aa5095cd3109962a298ce14110ae16b8c1aece8b72f9dafe81cf597ad80b3", size = 1863760, upload-time = "2025-11-04T13:41:31.055Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f0/e5e6b99d4191da102f2b0eb9687aaa7f5bea5d9964071a84effc3e40f997/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3006c3dd9ba34b0c094c544c6006cc79e87d8612999f1a5d43b769b89181f23c", size = 1878092, upload-time = "2025-11-04T13:41:33.21Z" }, + { url = "https://files.pythonhosted.org/packages/71/48/36fb760642d568925953bcc8116455513d6e34c4beaa37544118c36aba6d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72f6c8b11857a856bcfa48c86f5368439f74453563f951e473514579d44aa612", size = 2053385, upload-time = "2025-11-04T13:41:35.508Z" }, + { url = "https://files.pythonhosted.org/packages/20/25/92dc684dd8eb75a234bc1c764b4210cf2646479d54b47bf46061657292a8/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cb1b2f9742240e4bb26b652a5aeb840aa4b417c7748b6f8387927bc6e45e40d", size = 2218832, upload-time = "2025-11-04T13:41:37.732Z" }, + { url = "https://files.pythonhosted.org/packages/e2/09/f53e0b05023d3e30357d82eb35835d0f6340ca344720a4599cd663dca599/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd3d54f38609ff308209bd43acea66061494157703364ae40c951f83ba99a1a9", size = 2327585, upload-time = "2025-11-04T13:41:40Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4e/2ae1aa85d6af35a39b236b1b1641de73f5a6ac4d5a7509f77b814885760c/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ff4321e56e879ee8d2a879501c8e469414d948f4aba74a2d4593184eb326660", size = 2041078, upload-time = "2025-11-04T13:41:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/cd/13/2e215f17f0ef326fc72afe94776edb77525142c693767fc347ed6288728d/pydantic_core-2.41.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d0d2568a8c11bf8225044aa94409e21da0cb09dcdafe9ecd10250b2baad531a9", size = 2173914, upload-time = "2025-11-04T13:41:45.221Z" }, + { url = "https://files.pythonhosted.org/packages/02/7a/f999a6dcbcd0e5660bc348a3991c8915ce6599f4f2c6ac22f01d7a10816c/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:a39455728aabd58ceabb03c90e12f71fd30fa69615760a075b9fec596456ccc3", size = 2129560, upload-time = "2025-11-04T13:41:47.474Z" }, + { url = "https://files.pythonhosted.org/packages/3a/b1/6c990ac65e3b4c079a4fb9f5b05f5b013afa0f4ed6780a3dd236d2cbdc64/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:239edca560d05757817c13dc17c50766136d21f7cd0fac50295499ae24f90fdf", size = 2329244, upload-time = "2025-11-04T13:41:49.992Z" }, + { url = "https://files.pythonhosted.org/packages/d9/02/3c562f3a51afd4d88fff8dffb1771b30cfdfd79befd9883ee094f5b6c0d8/pydantic_core-2.41.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:2a5e06546e19f24c6a96a129142a75cee553cc018ffee48a460059b1185f4470", size = 2331955, upload-time = "2025-11-04T13:41:54.079Z" }, + { url = "https://files.pythonhosted.org/packages/5c/96/5fb7d8c3c17bc8c62fdb031c47d77a1af698f1d7a406b0f79aaa1338f9ad/pydantic_core-2.41.5-cp314-cp314t-win32.whl", hash = "sha256:b4ececa40ac28afa90871c2cc2b9ffd2ff0bf749380fbdf57d165fd23da353aa", size = 1988906, upload-time = "2025-11-04T13:41:56.606Z" }, + { url = "https://files.pythonhosted.org/packages/22/ed/182129d83032702912c2e2d8bbe33c036f342cc735737064668585dac28f/pydantic_core-2.41.5-cp314-cp314t-win_amd64.whl", hash = "sha256:80aa89cad80b32a912a65332f64a4450ed00966111b6615ca6816153d3585a8c", size = 1981607, upload-time = "2025-11-04T13:41:58.889Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ed/068e41660b832bb0b1aa5b58011dea2a3fe0ba7861ff38c4d4904c1c1a99/pydantic_core-2.41.5-cp314-cp314t-win_arm64.whl", hash = "sha256:35b44f37a3199f771c3eaa53051bc8a70cd7b54f333531c59e29fd4db5d15008", size = 1974769, upload-time = "2025-11-04T13:42:01.186Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/6d/fffca34caecc4a3f97bda81b2098da5e8ab7efc9a66e819074a11955d87e/pydantic_settings-2.13.1.tar.gz", hash = "sha256:b4c11847b15237fb0171e1462bf540e294affb9b86db4d9aa5c01730bdbe4025", size = 223826, upload-time = "2026-02-19T13:45:08.055Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/4b/ccc026168948fec4f7555b9164c724cf4125eac006e176541483d2c959be/pydantic_settings-2.13.1-py3-none-any.whl", hash = "sha256:d56fd801823dbeae7f0975e1f8c8e25c258eb75d278ea7abb5d9cebb01b56237", size = 58929, upload-time = "2026-02-19T13:45:06.034Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pymysql" +version = "1.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/ae/1fe3fcd9f959efa0ebe200b8de88b5a5ce3e767e38c7ac32fb179f16a388/pymysql-1.1.2.tar.gz", hash = "sha256:4961d3e165614ae65014e361811a724e2044ad3ea3739de9903ae7c21f539f03", size = 48258, upload-time = "2025-08-24T12:55:55.146Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/4c/ad33b92b9864cbde84f259d5df035a6447f91891f5be77788e2a3892bce3/pymysql-1.1.2-py3-none-any.whl", hash = "sha256:e6b1d89711dd51f8f74b1631fe08f039e7d76cf67a42a323d3178f0f25762ed9", size = 45300, upload-time = "2025-08-24T12:55:53.394Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901, upload-time = "2025-12-06T21:30:51.014Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801, upload-time = "2025-12-06T21:30:49.154Z" }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087, upload-time = "2025-11-10T16:07:47.256Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, +] + +[[package]] +name = "pytest-clarity" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pprintpp" }, + { name = "pytest" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/5c/cafa97944de55738a6a2c5a7cee00d073cb80495032d2b112c4546525eca/pytest-clarity-1.0.1.tar.gz", hash = "sha256:505fe345fad4fe11c6a4187fe683f2c7c52c077caa1e135f3e483fe112db7772", size = 4891, upload-time = "2021-06-11T18:16:18.372Z" } + +[[package]] +name = "pytest-mock" +version = "3.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/14/eb014d26be205d38ad5ad20d9a80f7d201472e08167f0bb4361e251084a9/pytest_mock-3.15.1.tar.gz", hash = "sha256:1849a238f6f396da19762269de72cb1814ab44416fa73a8686deac10b0d87a0f", size = 34036, upload-time = "2025-09-16T16:37:27.081Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/cc/06253936f4a7fa2e0f48dfe6d851d9c56df896a9ab09ac019d70b760619c/pytest_mock-3.15.1-py3-none-any.whl", hash = "sha256:0a25e2eb88fe5168d535041d09a4529a188176ae608a6d249ee65abc0949630d", size = 10095, upload-time = "2025-09-16T16:37:25.734Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, +] + +[[package]] +name = "pytz" +version = "2026.1.post1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/56/db/b8721d71d945e6a8ac63c0fc900b2067181dbb50805958d4d4661cf7d277/pytz-2026.1.post1.tar.gz", hash = "sha256:3378dde6a0c3d26719182142c56e60c7f9af7e968076f31aae569d72a0358ee1", size = 321088, upload-time = "2026-03-03T07:47:50.683Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/99/781fe0c827be2742bcc775efefccb3b048a3a9c6ce9aec0cbf4a101677e5/pytz-2026.1.post1-py2.py3-none-any.whl", hash = "sha256:f2fd16142fda348286a75e1a524be810bb05d444e5a081f37f7affc635035f7a", size = 510489, upload-time = "2026-03-03T07:47:49.167Z" }, +] + +[[package]] +name = "rich" +version = "14.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/c6/f3b320c27991c46f43ee9d856302c70dc2d0fb2dba4842ff739d5f46b393/rich-14.3.3.tar.gz", hash = "sha256:b8daa0b9e4eef54dd8cf7c86c03713f53241884e814f4e2f5fb342fe520f639b", size = 230582, upload-time = "2026-02-19T17:23:12.474Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/25/b208c5683343959b670dc001595f2f3737e051da617f66c31f7c4fa93abc/rich-14.3.3-py3-none-any.whl", hash = "sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d", size = 310458, upload-time = "2026-02-19T17:23:13.732Z" }, +] + +[[package]] +name = "ruff" +version = "0.15.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/77/9b/840e0039e65fcf12758adf684d2289024d6140cde9268cc59887dc55189c/ruff-0.15.5.tar.gz", hash = "sha256:7c3601d3b6d76dce18c5c824fc8d06f4eef33d6df0c21ec7799510cde0f159a2", size = 4574214, upload-time = "2026-03-05T20:06:34.946Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/20/5369c3ce21588c708bcbe517a8fbe1a8dfdb5dfd5137e14790b1da71612c/ruff-0.15.5-py3-none-linux_armv6l.whl", hash = "sha256:4ae44c42281f42e3b06b988e442d344a5b9b72450ff3c892e30d11b29a96a57c", size = 10478185, upload-time = "2026-03-05T20:06:29.093Z" }, + { url = "https://files.pythonhosted.org/packages/44/ed/e81dd668547da281e5dce710cf0bc60193f8d3d43833e8241d006720e42b/ruff-0.15.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:6edd3792d408ebcf61adabc01822da687579a1a023f297618ac27a5b51ef0080", size = 10859201, upload-time = "2026-03-05T20:06:32.632Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8f/533075f00aaf19b07c5cd6aa6e5d89424b06b3b3f4583bfa9c640a079059/ruff-0.15.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:89f463f7c8205a9f8dea9d658d59eff49db05f88f89cc3047fb1a02d9f344010", size = 10184752, upload-time = "2026-03-05T20:06:40.312Z" }, + { url = "https://files.pythonhosted.org/packages/66/0e/ba49e2c3fa0395b3152bad634c7432f7edfc509c133b8f4529053ff024fb/ruff-0.15.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba786a8295c6574c1116704cf0b9e6563de3432ac888d8f83685654fe528fd65", size = 10534857, upload-time = "2026-03-05T20:06:19.581Z" }, + { url = "https://files.pythonhosted.org/packages/59/71/39234440f27a226475a0659561adb0d784b4d247dfe7f43ffc12dd02e288/ruff-0.15.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fd4b801e57955fe9f02b31d20375ab3a5c4415f2e5105b79fb94cf2642c91440", size = 10309120, upload-time = "2026-03-05T20:06:00.435Z" }, + { url = "https://files.pythonhosted.org/packages/f5/87/4140aa86a93df032156982b726f4952aaec4a883bb98cb6ef73c347da253/ruff-0.15.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:391f7c73388f3d8c11b794dbbc2959a5b5afe66642c142a6effa90b45f6f5204", size = 11047428, upload-time = "2026-03-05T20:05:51.867Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f7/4953e7e3287676f78fbe85e3a0ca414c5ca81237b7575bdadc00229ac240/ruff-0.15.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8dc18f30302e379fe1e998548b0f5e9f4dff907f52f73ad6da419ea9c19d66c8", size = 11914251, upload-time = "2026-03-05T20:06:22.887Z" }, + { url = "https://files.pythonhosted.org/packages/77/46/0f7c865c10cf896ccf5a939c3e84e1cfaeed608ff5249584799a74d33835/ruff-0.15.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1cc6e7f90087e2d27f98dc34ed1b3ab7c8f0d273cc5431415454e22c0bd2a681", size = 11333801, upload-time = "2026-03-05T20:05:57.168Z" }, + { url = "https://files.pythonhosted.org/packages/d3/01/a10fe54b653061585e655f5286c2662ebddb68831ed3eaebfb0eb08c0a16/ruff-0.15.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1cb7169f53c1ddb06e71a9aebd7e98fc0fea936b39afb36d8e86d36ecc2636a", size = 11206821, upload-time = "2026-03-05T20:06:03.441Z" }, + { url = "https://files.pythonhosted.org/packages/7a/0d/2132ceaf20c5e8699aa83da2706ecb5c5dcdf78b453f77edca7fb70f8a93/ruff-0.15.5-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9b037924500a31ee17389b5c8c4d88874cc6ea8e42f12e9c61a3d754ff72f1ca", size = 11133326, upload-time = "2026-03-05T20:06:25.655Z" }, + { url = "https://files.pythonhosted.org/packages/72/cb/2e5259a7eb2a0f87c08c0fe5bf5825a1e4b90883a52685524596bfc93072/ruff-0.15.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:65bb414e5b4eadd95a8c1e4804f6772bbe8995889f203a01f77ddf2d790929dd", size = 10510820, upload-time = "2026-03-05T20:06:37.79Z" }, + { url = "https://files.pythonhosted.org/packages/ff/20/b67ce78f9e6c59ffbdb5b4503d0090e749b5f2d31b599b554698a80d861c/ruff-0.15.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:d20aa469ae3b57033519c559e9bc9cd9e782842e39be05b50e852c7c981fa01d", size = 10302395, upload-time = "2026-03-05T20:05:54.504Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e5/719f1acccd31b720d477751558ed74e9c88134adcc377e5e886af89d3072/ruff-0.15.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:15388dd28c9161cdb8eda68993533acc870aa4e646a0a277aa166de9ad5a8752", size = 10754069, upload-time = "2026-03-05T20:06:06.422Z" }, + { url = "https://files.pythonhosted.org/packages/c3/9c/d1db14469e32d98f3ca27079dbd30b7b44dbb5317d06ab36718dee3baf03/ruff-0.15.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b30da330cbd03bed0c21420b6b953158f60c74c54c5f4c1dabbdf3a57bf355d2", size = 11304315, upload-time = "2026-03-05T20:06:10.867Z" }, + { url = "https://files.pythonhosted.org/packages/28/3a/950367aee7c69027f4f422059227b290ed780366b6aecee5de5039d50fa8/ruff-0.15.5-py3-none-win32.whl", hash = "sha256:732e5ee1f98ba5b3679029989a06ca39a950cced52143a0ea82a2102cb592b74", size = 10551676, upload-time = "2026-03-05T20:06:13.705Z" }, + { url = "https://files.pythonhosted.org/packages/b8/00/bf077a505b4e649bdd3c47ff8ec967735ce2544c8e4a43aba42ee9bf935d/ruff-0.15.5-py3-none-win_amd64.whl", hash = "sha256:821d41c5fa9e19117616c35eaa3f4b75046ec76c65e7ae20a333e9a8696bc7fe", size = 11678972, upload-time = "2026-03-05T20:06:45.379Z" }, + { url = "https://files.pythonhosted.org/packages/fe/4e/cd76eca6db6115604b7626668e891c9dd03330384082e33662fb0f113614/ruff-0.15.5-py3-none-win_arm64.whl", hash = "sha256:b498d1c60d2fe5c10c45ec3f698901065772730b411f164ae270bb6bfcc4740b", size = 10965572, upload-time = "2026-03-05T20:06:16.984Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.0.48" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/73/b4a9737255583b5fa858e0bb8e116eb94b88c910164ed2ed719147bde3de/sqlalchemy-2.0.48.tar.gz", hash = "sha256:5ca74f37f3369b45e1f6b7b06afb182af1fd5dde009e4ffd831830d98cbe5fe7", size = 9886075, upload-time = "2026-03-02T15:28:51.474Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/b3/f437eaa1cf028bb3c927172c7272366393e73ccd104dcf5b6963f4ab5318/sqlalchemy-2.0.48-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e2d0d88686e3d35a76f3e15a34e8c12d73fc94c1dea1cd55782e695cc14086dd", size = 2154401, upload-time = "2026-03-02T15:49:17.24Z" }, + { url = "https://files.pythonhosted.org/packages/6c/1c/b3abdf0f402aa3f60f0df6ea53d92a162b458fca2321d8f1f00278506402/sqlalchemy-2.0.48-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49b7bddc1eebf011ea5ab722fdbe67a401caa34a350d278cc7733c0e88fecb1f", size = 3274528, upload-time = "2026-03-02T15:50:41.489Z" }, + { url = "https://files.pythonhosted.org/packages/f2/5e/327428a034407651a048f5e624361adf3f9fbac9d0fa98e981e9c6ff2f5e/sqlalchemy-2.0.48-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:426c5ca86415d9b8945c7073597e10de9644802e2ff502b8e1f11a7a2642856b", size = 3279523, upload-time = "2026-03-02T15:53:32.962Z" }, + { url = "https://files.pythonhosted.org/packages/2a/ca/ece73c81a918add0965b76b868b7b5359e068380b90ef1656ee995940c02/sqlalchemy-2.0.48-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:288937433bd44e3990e7da2402fabc44a3c6c25d3704da066b85b89a85474ae0", size = 3224312, upload-time = "2026-03-02T15:50:42.996Z" }, + { url = "https://files.pythonhosted.org/packages/88/11/fbaf1ae91fa4ee43f4fe79661cead6358644824419c26adb004941bdce7c/sqlalchemy-2.0.48-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8183dc57ae7d9edc1346e007e840a9f3d6aa7b7f165203a99e16f447150140d2", size = 3246304, upload-time = "2026-03-02T15:53:34.937Z" }, + { url = "https://files.pythonhosted.org/packages/fa/a8/5fb0deb13930b4f2f698c5541ae076c18981173e27dd00376dbaea7a9c82/sqlalchemy-2.0.48-cp314-cp314-win32.whl", hash = "sha256:1182437cb2d97988cfea04cf6cdc0b0bb9c74f4d56ec3d08b81e23d621a28cc6", size = 2116565, upload-time = "2026-03-02T15:54:38.321Z" }, + { url = "https://files.pythonhosted.org/packages/95/7e/e83615cb63f80047f18e61e31e8e32257d39458426c23006deeaf48f463b/sqlalchemy-2.0.48-cp314-cp314-win_amd64.whl", hash = "sha256:144921da96c08feb9e2b052c5c5c1d0d151a292c6135623c6b2c041f2a45f9e0", size = 2142205, upload-time = "2026-03-02T15:54:39.831Z" }, + { url = "https://files.pythonhosted.org/packages/83/e3/69d8711b3f2c5135e9cde5f063bc1605860f0b2c53086d40c04017eb1f77/sqlalchemy-2.0.48-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5aee45fd2c6c0f2b9cdddf48c48535e7471e42d6fb81adfde801da0bd5b93241", size = 3563519, upload-time = "2026-03-02T15:57:52.387Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4f/a7cce98facca73c149ea4578981594aaa5fd841e956834931de503359336/sqlalchemy-2.0.48-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7cddca31edf8b0653090cbb54562ca027c421c58ddde2c0685f49ff56a1690e0", size = 3528611, upload-time = "2026-03-02T16:04:42.097Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7d/5936c7a03a0b0cb0fa0cc425998821c6029756b0855a8f7ee70fba1de955/sqlalchemy-2.0.48-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7a936f1bb23d370b7c8cc079d5fce4c7d18da87a33c6744e51a93b0f9e97e9b3", size = 3472326, upload-time = "2026-03-02T15:57:54.423Z" }, + { url = "https://files.pythonhosted.org/packages/f4/33/cea7dfc31b52904efe3dcdc169eb4514078887dff1f5ae28a7f4c5d54b3c/sqlalchemy-2.0.48-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e004aa9248e8cb0a5f9b96d003ca7c1c0a5da8decd1066e7b53f59eb8ce7c62b", size = 3478453, upload-time = "2026-03-02T16:04:44.584Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/32107c4d13be077a9cae61e9ae49966a35dc4bf442a8852dd871db31f62e/sqlalchemy-2.0.48-cp314-cp314t-win32.whl", hash = "sha256:b8438ec5594980d405251451c5b7ea9aa58dda38eb7ac35fb7e4c696712ee24f", size = 2147209, upload-time = "2026-03-02T15:52:54.274Z" }, + { url = "https://files.pythonhosted.org/packages/d2/d7/1e073da7a4bc645eb83c76067284a0374e643bc4be57f14cc6414656f92c/sqlalchemy-2.0.48-cp314-cp314t-win_amd64.whl", hash = "sha256:d854b3970067297f3a7fbd7a4683587134aa9b3877ee15aa29eea478dc68f933", size = 2182198, upload-time = "2026-03-02T15:52:55.606Z" }, + { url = "https://files.pythonhosted.org/packages/46/2c/9664130905f03db57961b8980b05cab624afd114bf2be2576628a9f22da4/sqlalchemy-2.0.48-py3-none-any.whl", hash = "sha256:a66fe406437dd65cacd96a72689a3aaaecaebbcd62d81c5ac1c0fdbeac835096", size = 1940202, upload-time = "2026-03-02T15:52:43.285Z" }, +] + +[[package]] +name = "structlog" +version = "25.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ef/52/9ba0f43b686e7f3ddfeaa78ac3af750292662284b3661e91ad5494f21dbc/structlog-25.5.0.tar.gz", hash = "sha256:098522a3bebed9153d4570c6d0288abf80a031dfdb2048d59a49e9dc2190fc98", size = 1460830, upload-time = "2025-10-27T08:28:23.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/45/a132b9074aa18e799b891b91ad72133c98d8042c70f6240e4c5f9dabee2f/structlog-25.5.0-py3-none-any.whl", hash = "sha256:a8453e9b9e636ec59bd9e79bbd4a72f025981b3ba0f5837aebf48f02f37a7f9f", size = 72510, upload-time = "2025-10-27T08:28:21.535Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + +[[package]] +name = "tzdata" +version = "2025.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/a7/c202b344c5ca7daf398f3b8a477eeb205cf3b6f32e7ec3a6bac0629ca975/tzdata-2025.3.tar.gz", hash = "sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7", size = 196772, upload-time = "2025-12-13T17:45:35.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1", size = 348521, upload-time = "2025-12-13T17:45:33.889Z" }, +]