From b8787285cece697e11a0a5b19af4838452c0af00 Mon Sep 17 00:00:00 2001 From: mcarans Date: Wed, 7 May 2025 14:22:53 +1200 Subject: [PATCH 1/2] Update structure --- .config/coveragerc => .coveragerc | 0 ...mit-config.yaml => .pre-commit-config.yaml | 3 +- documentation/mkdocs.yaml | 4 +- hatch.toml | 34 ++++++++++++++ pyproject.toml | 46 ------------------- .config/pytest.ini => pytest.ini | 2 +- .config/ruff.toml => ruff.toml | 1 - 7 files changed, 39 insertions(+), 51 deletions(-) rename .config/coveragerc => .coveragerc (100%) rename .config/pre-commit-config.yaml => .pre-commit-config.yaml (88%) create mode 100644 hatch.toml rename .config/pytest.ini => pytest.ini (69%) rename .config/ruff.toml => ruff.toml (93%) diff --git a/.config/coveragerc b/.coveragerc similarity index 100% rename from .config/coveragerc rename to .coveragerc diff --git a/.config/pre-commit-config.yaml b/.pre-commit-config.yaml similarity index 88% rename from .config/pre-commit-config.yaml rename to .pre-commit-config.yaml index 0824642..e66d3ee 100644 --- a/.config/pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,10 +12,9 @@ repos: hooks: # Run the linter. - id: ruff - args: [--config, .config/ruff.toml, --fix] + args: [ --fix ] # Run the formatter. - id: ruff-format - args: [--config, .config/ruff.toml] - repo: https://github.com/astral-sh/uv-pre-commit rev: 0.6.5 hooks: diff --git a/documentation/mkdocs.yaml b/documentation/mkdocs.yaml index 48a5435..da3e516 100644 --- a/documentation/mkdocs.yaml +++ b/documentation/mkdocs.yaml @@ -12,5 +12,7 @@ plugins: nav: - Home: index.md - API Documentation: - - Database: $src/hdx.database.database.* + - Database: + - $src/hdx.database.Database + - $src/hdx.database.DatabaseError - PostgreSQL specific: $src/hdx.database.postgresql.* diff --git a/hatch.toml b/hatch.toml new file mode 100644 index 0000000..a2125af --- /dev/null +++ b/hatch.toml @@ -0,0 +1,34 @@ +# Build + +[build.targets.wheel] +packages = ["src/hdx"] + +[build.hooks.vcs] +version-file = "src/hdx/database/_version.py" + +# Versioning + +[version] +source = "vcs" + +[version.raw-options] +local_scheme = "no-local-version" +version_scheme = "python-simplified-semver" + +# Tests + +[envs.hatch-test] +features = ["test"] + +[[envs.hatch-test.matrix]] +python = ["3.12"] + +[envs.hatch-test.scripts] +run = """ + pytest --rootdir=. --junitxml=test-results.xml --cov --no-cov-on-fail \ + --cov-report=lcov --cov-report=term-missing + """ + +[envs.hatch-static-analysis] +config-path = "none" +dependencies = ["ruff==0.9.10"] diff --git a/pyproject.toml b/pyproject.toml index fe9ed5f..55746c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,49 +51,3 @@ postgresql = ["psycopg[binary]"] test = ["pytest", "pytest-cov"] dev = ["pre-commit"] docs = ["mkapi"] - - -######### -# Hatch # -######### - -# Build - -[tool.hatch.build.targets.wheel] -packages = ["src/hdx"] - -[tool.hatch.build.hooks.vcs] -version-file = "src/hdx/database/_version.py" - -# Versioning - -[tool.hatch.version] -source = "vcs" - -[tool.hatch.version.raw-options] -local_scheme = "no-local-version" -version_scheme = "python-simplified-semver" - -# Tests - -[tool.hatch.envs.hatch-test] -features = ["sshtunnel", "postgresql", "test"] - -[[tool.hatch.envs.hatch-test.matrix]] -python = ["3.12"] - -[tool.hatch.envs.hatch-test.scripts] -run = """ - pytest -c .config/pytest.ini --rootdir=. --junitxml=test-results.xml \ - --cov --cov-config=.config/coveragerc --no-cov-on-fail \ - --cov-report=lcov --cov-report=term-missing - """ - -[tool.hatch.envs.hatch-static-analysis] -dependencies = ["ruff==0.9.10"] - -[tool.hatch.envs.hatch-static-analysis.scripts] -format-check = ["ruff format --config .config/ruff.toml --check --diff {args:.}",] -format-fix = ["ruff format --config .config/ruff.toml {args:.}",] -lint-check = ["ruff check --config .config/ruff.toml {args:.}",] -lint-fix = ["ruff check --config .config/ruff.toml --fix {args:.}",] diff --git a/.config/pytest.ini b/pytest.ini similarity index 69% rename from .config/pytest.ini rename to pytest.ini index bb28fd3..3009ece 100644 --- a/.config/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] -pythonpath = ../src +pythonpath = src addopts = "--color=yes" log_cli = 1 diff --git a/.config/ruff.toml b/ruff.toml similarity index 93% rename from .config/ruff.toml rename to ruff.toml index 4625a34..cf4db09 100644 --- a/.config/ruff.toml +++ b/ruff.toml @@ -1,4 +1,3 @@ -line-length = 79 exclude = ["_version.py"] [lint] From a6e73223522fc3b40ca8655a7f24db9aee3e8873 Mon Sep 17 00:00:00 2001 From: mcarans Date: Wed, 7 May 2025 14:25:49 +1200 Subject: [PATCH 2/2] Update structure --- pyproject.toml | 2 +- requirements.txt | 55 ++++++++++++----------------- src/hdx/database/database.py | 8 ++--- src/hdx/database/utils.py | 4 +-- tests/hdx/database/test_database.py | 4 +-- tests/hdx/database/test_dburi.py | 16 +++------ tests/hdx/database/test_ssh.py | 4 +-- 7 files changed, 33 insertions(+), 60 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 55746c8..08b9a7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,6 @@ Homepage = "https://github.com/OCHA-DAP/hdx-python-database" [project.optional-dependencies] sshtunnel = ["sshtunnel"] postgresql = ["psycopg[binary]"] -test = ["pytest", "pytest-cov"] +test = ["sshtunnel", "psycopg[binary]", "pytest", "pytest-cov"] dev = ["pre-commit"] docs = ["mkapi"] diff --git a/requirements.txt b/requirements.txt index 6e383b0..bb2e5c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # This file was autogenerated by uv via the following command: # uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt -astdoc==1.1.5 +astdoc==1.2.1 # via mkapi babel==2.17.0 # via mkdocs-material @@ -8,7 +8,7 @@ backrefs==5.8 # via mkdocs-material bcrypt==4.3.0 # via paramiko -certifi==2025.1.31 +certifi==2025.4.26 # via requests cffi==1.17.1 # via @@ -16,15 +16,15 @@ cffi==1.17.1 # pynacl cfgv==3.4.0 # via pre-commit -charset-normalizer==3.4.1 +charset-normalizer==3.4.2 # via requests click==8.1.8 # via mkdocs colorama==0.4.6 # via mkdocs-material -coverage==7.7.0 +coverage==7.8.0 # via pytest-cov -cryptography==44.0.2 +cryptography==44.0.3 # via paramiko distlib==0.3.9 # via virtualenv @@ -32,38 +32,33 @@ filelock==3.18.0 # via virtualenv ghp-import==2.1.0 # via mkdocs -greenlet==3.1.1 +greenlet==3.2.1 # via sqlalchemy -identify==2.6.9 +identify==2.6.10 # via pre-commit idna==3.10 # via requests -iniconfig==2.0.0 +iniconfig==2.1.0 # via pytest jinja2==3.1.6 # via # mkapi # mkdocs # mkdocs-material -markdown==3.7 +markdown==3.8 # via - # astdoc # mkdocs # mkdocs-material # pymdown-extensions -markdown-it-py==3.0.0 - # via rich markupsafe==3.0.2 # via # jinja2 # mkdocs -mdurl==0.1.2 - # via markdown-it-py mergedeep==1.3.4 # via # mkdocs # mkdocs-get-deps -mkapi==4.1.3 +mkapi==4.3.2 # via hdx-python-database (pyproject.toml) mkdocs==1.6.1 # via @@ -71,13 +66,13 @@ mkdocs==1.6.1 # mkdocs-material mkdocs-get-deps==0.2.0 # via mkdocs -mkdocs-material==9.6.8 +mkdocs-material==9.6.12 # via mkapi mkdocs-material-extensions==1.3.1 # via mkdocs-material nodeenv==1.9.1 # via pre-commit -packaging==24.2 +packaging==25.0 # via # mkdocs # pytest @@ -87,25 +82,23 @@ paramiko==3.5.1 # via sshtunnel pathspec==0.12.1 # via mkdocs -platformdirs==4.3.6 +platformdirs==4.3.7 # via # mkdocs-get-deps # virtualenv pluggy==1.5.0 # via pytest -pre-commit==4.1.0 +pre-commit==4.2.0 # via hdx-python-database (pyproject.toml) -psycopg==3.2.6 +psycopg==3.2.7 # via hdx-python-database (pyproject.toml) -psycopg-binary==3.2.6 +psycopg-binary==3.2.7 # via psycopg pycparser==2.22 # via cffi pygments==2.19.1 - # via - # mkdocs-material - # rich -pymdown-extensions==10.14.3 + # via mkdocs-material +pymdown-extensions==10.15 # via mkdocs-material pynacl==1.5.0 # via paramiko @@ -113,7 +106,7 @@ pytest==8.3.5 # via # hdx-python-database (pyproject.toml) # pytest-cov -pytest-cov==6.0.0 +pytest-cov==6.1.1 # via hdx-python-database (pyproject.toml) python-dateutil==2.9.0.post0 # via ghp-import @@ -128,21 +121,19 @@ pyyaml-env-tag==0.1 # via mkdocs requests==2.32.3 # via mkdocs-material -rich==13.9.4 - # via mkapi six==1.17.0 # via python-dateutil -sqlalchemy==2.0.39 +sqlalchemy==2.0.40 # via hdx-python-database (pyproject.toml) sshtunnel==0.4.0 # via hdx-python-database (pyproject.toml) -typing-extensions==4.12.2 +typing-extensions==4.13.2 # via # psycopg # sqlalchemy -urllib3==2.3.0 +urllib3==2.4.0 # via requests -virtualenv==20.29.3 +virtualenv==20.31.1 # via pre-commit watchdog==6.0.0 # via mkdocs diff --git a/src/hdx/database/database.py b/src/hdx/database/database.py index 767030f..73edf97 100644 --- a/src/hdx/database/database.py +++ b/src/hdx/database/database.py @@ -281,9 +281,7 @@ def create_session( engine = create_engine(db_uri, poolclass=NullPool, echo=False) if reflect: Base = automap_base(declarative_base=table_base) - Base.prepare( - autoload_with=engine, reflection_options={"views": True} - ) + Base.prepare(autoload_with=engine, reflection_options={"views": True}) table_base = Base else: table_base.metadata.create_all(engine) @@ -307,9 +305,7 @@ def recreate_schema(engine: Engine, schema_name: str = "public") -> bool: DropSchema(schema_name, cascade=True, if_exists=True) ) connection.commit() - connection.execute( - CreateSchema(schema_name, if_not_exists=True) - ) + connection.execute(CreateSchema(schema_name, if_not_exists=True)) connection.commit() return True except SQLAlchemyError: diff --git a/src/hdx/database/utils.py b/src/hdx/database/utils.py index 246cce7..719dd75 100644 --- a/src/hdx/database/utils.py +++ b/src/hdx/database/utils.py @@ -13,7 +13,5 @@ def camel_to_snake_case(string: str) -> str: Returns: str: String in snake case """ - string = re.sub( - r"((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))", r"_\1", string - ) + string = re.sub(r"((?<=[a-z0-9])[A-Z]|(?!^)[A-Z](?=[a-z]))", r"_\1", string) return string.lower().lstrip("_") diff --git a/tests/hdx/database/test_database.py b/tests/hdx/database/test_database.py index 3c9b3ac..b3eefc5 100755 --- a/tests/hdx/database/test_database.py +++ b/tests/hdx/database/test_database.py @@ -45,9 +45,7 @@ def prepare_fn(): now = datetime(2023, 10, 20, 22, 35, 55, tzinfo=timezone.utc) rows = [{"test_date": now}] dbdatabase.batch_populate(rows, DBTestDate) - dbtestdate = ( - dbsession.execute(select(DBTestDate)).all()[1][0].test_date - ) + dbtestdate = dbsession.execute(select(DBTestDate)).all()[1][0].test_date assert dbtestdate == now remove(dbpath) diff --git a/tests/hdx/database/test_dburi.py b/tests/hdx/database/test_dburi.py index b39e1b7..4266bab 100644 --- a/tests/hdx/database/test_dburi.py +++ b/tests/hdx/database/test_dburi.py @@ -17,9 +17,7 @@ class TestDBURI: "dialect": "postgresql", "driver": "psycopg", } - connection_uri_pg = ( - "postgresql+psycopg://myuser:mypass@myserver:1234/mydatabase" - ) + connection_uri_pg = "postgresql+psycopg://myuser:mypass@myserver:1234/mydatabase" params_pg_no_driver = { "database": "mydatabase", "host": "myserver", @@ -37,9 +35,7 @@ class TestDBURI: "dialect": "postgresql", "driver": None, } - connection_uri_pg_no_driver = ( - "postgresql://myuser:mypass@myserver:1234/mydatabase" - ) + connection_uri_pg_no_driver = "postgresql://myuser:mypass@myserver:1234/mydatabase" params_sq_no_driver = { "database": "mydatabase", "host": "myserver", @@ -67,9 +63,7 @@ def test_get_params_from_connection_uri(self): include_driver=False, ) assert result == TestDBURI.params_pg_no_driver - result = get_params_from_connection_uri( - TestDBURI.connection_uri_pg_no_driver - ) + result = get_params_from_connection_uri(TestDBURI.connection_uri_pg_no_driver) assert result == TestDBURI.params_pg_driver_none result = get_params_from_connection_uri(TestDBURI.connection_uri_sq) assert result == TestDBURI.params_sq_driver_none @@ -77,9 +71,7 @@ def test_get_params_from_connection_uri(self): def test_get_connection_uri(self): result = get_connection_uri(**TestDBURI.params_pg) assert result == TestDBURI.connection_uri_pg - result = get_connection_uri( - **TestDBURI.params_pg, include_driver=False - ) + result = get_connection_uri(**TestDBURI.params_pg, include_driver=False) assert result == TestDBURI.connection_uri_pg_no_driver result = get_connection_uri(**TestDBURI.params_pg_no_driver) assert result == TestDBURI.connection_uri_pg diff --git a/tests/hdx/database/test_ssh.py b/tests/hdx/database/test_ssh.py index 78edddc..ba42367 100755 --- a/tests/hdx/database/test_ssh.py +++ b/tests/hdx/database/test_ssh.py @@ -53,9 +53,7 @@ def test_get_session_ssh( ) assert dbdatabase._base == NoTZBase - with Database( - ssh_host="mysshhost", ssh_port=25, **params - ) as dbdatabase: + with Database(ssh_host="mysshhost", ssh_port=25, **params) as dbdatabase: dbsession = dbdatabase.get_session() assert ( str(dbsession.bind.engine.url)