From a823233e329c2b8eafc583a8d17c23a21e65a75a Mon Sep 17 00:00:00 2001 From: mcarans Date: Thu, 24 Jul 2025 10:36:19 +1200 Subject: [PATCH 1/8] Allow xz compressed restore file --- .pre-commit-config.yaml | 8 ++--- hatch.toml | 4 +-- pyproject.toml | 2 +- requirements.txt | 64 ++++++++++++++++++---------------- src/hdx/database/postgresql.py | 9 +++-- 5 files changed, 47 insertions(+), 40 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e66d3ee..cb45cfd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,5 @@ default_language_version: - python: python3.12 + python: python3.13 repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 @@ -8,15 +8,15 @@ repos: - id: end-of-file-fixer - id: check-ast - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.10 + rev: v0.12.0 hooks: # Run the linter. - - id: ruff + - id: ruff-check args: [ --fix ] # Run the formatter. - id: ruff-format - repo: https://github.com/astral-sh/uv-pre-commit - rev: 0.6.5 + rev: 0.7.14 hooks: # Run the pip compile - id: pip-compile diff --git a/hatch.toml b/hatch.toml index a2125af..7a33144 100644 --- a/hatch.toml +++ b/hatch.toml @@ -21,7 +21,7 @@ version_scheme = "python-simplified-semver" features = ["test"] [[envs.hatch-test.matrix]] -python = ["3.12"] +python = ["3.13"] [envs.hatch-test.scripts] run = """ @@ -31,4 +31,4 @@ run = """ [envs.hatch-static-analysis] config-path = "none" -dependencies = ["ruff==0.9.10"] +dependencies = ["ruff==0.12.0"] diff --git a/pyproject.toml b/pyproject.toml index 08b9a7e..e745fc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ "Operating System :: MacOS", "Operating System :: Microsoft :: Windows", ] -requires-python = ">=3.8" +requires-python = ">=3.10" dependencies = [ "sqlalchemy>=2", diff --git a/requirements.txt b/requirements.txt index bb2e5c0..faeb4ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ # This file was autogenerated by uv via the following command: # uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt -astdoc==1.2.1 +astdoc==1.3.0 # via mkapi babel==2.17.0 # via mkdocs-material -backrefs==5.8 +backrefs==5.9 # via mkdocs-material bcrypt==4.3.0 # via paramiko -certifi==2025.4.26 +certifi==2025.7.14 # via requests cffi==1.17.1 # via @@ -18,23 +18,23 @@ cfgv==3.4.0 # via pre-commit charset-normalizer==3.4.2 # via requests -click==8.1.8 +click==8.2.1 # via mkdocs colorama==0.4.6 # via mkdocs-material -coverage==7.8.0 +coverage==7.9.2 # via pytest-cov -cryptography==44.0.3 +cryptography==45.0.5 # via paramiko -distlib==0.3.9 +distlib==0.4.0 # via virtualenv filelock==3.18.0 # via virtualenv ghp-import==2.1.0 # via mkdocs -greenlet==3.2.1 +greenlet==3.2.3 # via sqlalchemy -identify==2.6.10 +identify==2.6.12 # via pre-commit idna==3.10 # via requests @@ -45,7 +45,7 @@ jinja2==3.1.6 # mkapi # mkdocs # mkdocs-material -markdown==3.8 +markdown==3.8.2 # via # mkdocs # mkdocs-material @@ -58,7 +58,7 @@ mergedeep==1.3.4 # via # mkdocs # mkdocs-get-deps -mkapi==4.3.2 +mkapi==4.4.4 # via hdx-python-database (pyproject.toml) mkdocs==1.6.1 # via @@ -66,7 +66,7 @@ mkdocs==1.6.1 # mkdocs-material mkdocs-get-deps==0.2.0 # via mkdocs -mkdocs-material==9.6.12 +mkdocs-material==9.6.15 # via mkapi mkdocs-material-extensions==1.3.1 # via mkdocs-material @@ -82,31 +82,35 @@ paramiko==3.5.1 # via sshtunnel pathspec==0.12.1 # via mkdocs -platformdirs==4.3.7 +platformdirs==4.3.8 # via # mkdocs-get-deps # virtualenv -pluggy==1.5.0 - # via pytest +pluggy==1.6.0 + # via + # pytest + # pytest-cov pre-commit==4.2.0 # via hdx-python-database (pyproject.toml) -psycopg==3.2.7 +psycopg==3.2.9 # via hdx-python-database (pyproject.toml) -psycopg-binary==3.2.7 +psycopg-binary==3.2.9 # via psycopg pycparser==2.22 # via cffi -pygments==2.19.1 - # via mkdocs-material -pymdown-extensions==10.15 +pygments==2.19.2 + # via + # mkdocs-material + # pytest +pymdown-extensions==10.16 # via mkdocs-material pynacl==1.5.0 # via paramiko -pytest==8.3.5 +pytest==8.4.1 # via # hdx-python-database (pyproject.toml) # pytest-cov -pytest-cov==6.1.1 +pytest-cov==6.2.1 # via hdx-python-database (pyproject.toml) python-dateutil==2.9.0.post0 # via ghp-import @@ -117,23 +121,21 @@ pyyaml==6.0.2 # pre-commit # pymdown-extensions # pyyaml-env-tag -pyyaml-env-tag==0.1 +pyyaml-env-tag==1.1 # via mkdocs -requests==2.32.3 +requests==2.32.4 # via mkdocs-material six==1.17.0 # via python-dateutil -sqlalchemy==2.0.40 +sqlalchemy==2.0.41 # via hdx-python-database (pyproject.toml) sshtunnel==0.4.0 # via hdx-python-database (pyproject.toml) -typing-extensions==4.13.2 - # via - # psycopg - # sqlalchemy -urllib3==2.4.0 +typing-extensions==4.14.1 + # via sqlalchemy +urllib3==2.5.0 # via requests -virtualenv==20.31.1 +virtualenv==20.32.0 # via pre-commit watchdog==6.0.0 # via mkdocs diff --git a/src/hdx/database/postgresql.py b/src/hdx/database/postgresql.py index f3e3e05..9be56f1 100644 --- a/src/hdx/database/postgresql.py +++ b/src/hdx/database/postgresql.py @@ -59,7 +59,11 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str: str: Output from the pg_restore command """ db_params = get_params_from_connection_uri(db_uri) - subprocess_params = ["pg_restore", "-c"] + if pg_restore_file[:-3] == ".xz": + subprocess_params = ["unxz", "-d", pg_restore_file, "|", "pg_restore", "-c"] + pg_restore_file = None + else: + subprocess_params = ["pg_restore", "-c"] for key, value in db_params.items(): match key: case "database": @@ -74,7 +78,8 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str: continue subprocess_params.append(f"{value}") - subprocess_params.append(f"{pg_restore_file}") + if pg_restore_file: + subprocess_params.append(pg_restore_file) env = environ.copy() password = db_params.get("password") if password: From c424d93299d15506618243f56235dfa8034548fb Mon Sep 17 00:00:00 2001 From: mcarans Date: Thu, 24 Jul 2025 10:50:32 +1200 Subject: [PATCH 2/8] Allow xz compressed restore file --- src/hdx/database/postgresql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdx/database/postgresql.py b/src/hdx/database/postgresql.py index 9be56f1..8621742 100644 --- a/src/hdx/database/postgresql.py +++ b/src/hdx/database/postgresql.py @@ -59,7 +59,7 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str: str: Output from the pg_restore command """ db_params = get_params_from_connection_uri(db_uri) - if pg_restore_file[:-3] == ".xz": + if pg_restore_file[-3:] == ".xz": subprocess_params = ["unxz", "-d", pg_restore_file, "|", "pg_restore", "-c"] pg_restore_file = None else: From b3f13e2092da53190624e4b42b2109b01b39e72c Mon Sep 17 00:00:00 2001 From: mcarans Date: Thu, 24 Jul 2025 10:55:22 +1200 Subject: [PATCH 3/8] Allow xz compressed restore file --- src/hdx/database/postgresql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdx/database/postgresql.py b/src/hdx/database/postgresql.py index 8621742..d03e0ff 100644 --- a/src/hdx/database/postgresql.py +++ b/src/hdx/database/postgresql.py @@ -60,7 +60,7 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str: """ db_params = get_params_from_connection_uri(db_uri) if pg_restore_file[-3:] == ".xz": - subprocess_params = ["unxz", "-d", pg_restore_file, "|", "pg_restore", "-c"] + subprocess_params = ["unxz", "-d", pg_restore_file, ">", "pg_restore", "-c"] pg_restore_file = None else: subprocess_params = ["pg_restore", "-c"] From 4509c19524ca2d7075736a2aa34ad06615c26114 Mon Sep 17 00:00:00 2001 From: mcarans Date: Thu, 24 Jul 2025 11:15:44 +1200 Subject: [PATCH 4/8] Allow xz compressed restore file --- src/hdx/database/postgresql.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/hdx/database/postgresql.py b/src/hdx/database/postgresql.py index d03e0ff..690ceae 100644 --- a/src/hdx/database/postgresql.py +++ b/src/hdx/database/postgresql.py @@ -60,7 +60,15 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str: """ db_params = get_params_from_connection_uri(db_uri) if pg_restore_file[-3:] == ".xz": - subprocess_params = ["unxz", "-d", pg_restore_file, ">", "pg_restore", "-c"] + subprocess_params = [ + "unxz", + "-c", + "-d", + pg_restore_file, + "|", + "pg_restore", + "-c", + ] pg_restore_file = None else: subprocess_params = ["pg_restore", "-c"] From d2c5d40fd9e04bc2ed8a9e695aafd9b772424368 Mon Sep 17 00:00:00 2001 From: mcarans Date: Thu, 24 Jul 2025 11:28:06 +1200 Subject: [PATCH 5/8] Allow xz compressed restore file --- src/hdx/database/postgresql.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/hdx/database/postgresql.py b/src/hdx/database/postgresql.py index 690ceae..ce14f43 100644 --- a/src/hdx/database/postgresql.py +++ b/src/hdx/database/postgresql.py @@ -60,18 +60,12 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str: """ db_params = get_params_from_connection_uri(db_uri) if pg_restore_file[-3:] == ".xz": - subprocess_params = [ - "unxz", - "-c", - "-d", - pg_restore_file, - "|", - "pg_restore", - "-c", - ] - pg_restore_file = None + decompress = subprocess.Popen( + ("unxz", "-c", "-d", pg_restore_file), stdout=subprocess.PIPE + ) else: - subprocess_params = ["pg_restore", "-c"] + decompress = None + subprocess_params = ["pg_restore", "-c"] for key, value in db_params.items(): match key: case "database": @@ -86,14 +80,18 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str: continue subprocess_params.append(f"{value}") - if pg_restore_file: + if not decompress: subprocess_params.append(pg_restore_file) env = environ.copy() password = db_params.get("password") if password: env["PGPASSWORD"] = password process = subprocess.run( - subprocess_params, env=env, capture_output=True, encoding="utf-8" + subprocess_params, + env=env, + capture_output=True, + encoding="utf-8", + stdin=decompress, ) try: process.check_returncode() From 23c2f297e22d67dd6330619b0508c9b398f59820 Mon Sep 17 00:00:00 2001 From: mcarans Date: Thu, 24 Jul 2025 11:30:43 +1200 Subject: [PATCH 6/8] Allow xz compressed restore file --- src/hdx/database/postgresql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdx/database/postgresql.py b/src/hdx/database/postgresql.py index ce14f43..acfaccb 100644 --- a/src/hdx/database/postgresql.py +++ b/src/hdx/database/postgresql.py @@ -62,7 +62,7 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str: if pg_restore_file[-3:] == ".xz": decompress = subprocess.Popen( ("unxz", "-c", "-d", pg_restore_file), stdout=subprocess.PIPE - ) + ).stdout else: decompress = None subprocess_params = ["pg_restore", "-c"] From 9d033150703fb8672d8bc97543fb0ea9e1f88ed5 Mon Sep 17 00:00:00 2001 From: mcarans Date: Thu, 24 Jul 2025 11:38:39 +1200 Subject: [PATCH 7/8] Add drop schema --- src/hdx/database/database.py | 22 +++++++++++++++++++ ...test_recreate.py => test_drop_recreate.py} | 6 +++++ 2 files changed, 28 insertions(+) rename tests/hdx/database/{test_recreate.py => test_drop_recreate.py} (54%) diff --git a/src/hdx/database/database.py b/src/hdx/database/database.py index 73edf97..4d9392a 100644 --- a/src/hdx/database/database.py +++ b/src/hdx/database/database.py @@ -287,6 +287,28 @@ def create_session( table_base.metadata.create_all(engine) return Session(engine), table_base + @staticmethod + def drop_schema(engine: Engine, schema_name: str = "public") -> bool: + """Wipe schema in database using SQLAlchemy. + + Args: + engine (Engine): SQLAlchemy engine to use. + schema_name (str): Schema name. Defaults to "public". + + Returns: + bool: True if all successful, False if not + """ + # Wipe and create an empty schema + try: + with engine.connect() as connection: + connection.execute( + DropSchema(schema_name, cascade=True, if_exists=True) + ) + connection.commit() + return True + except SQLAlchemyError: + return False + @staticmethod def recreate_schema(engine: Engine, schema_name: str = "public") -> bool: """Wipe and create empty schema in database using SQLAlchemy. diff --git a/tests/hdx/database/test_recreate.py b/tests/hdx/database/test_drop_recreate.py similarity index 54% rename from tests/hdx/database/test_recreate.py rename to tests/hdx/database/test_drop_recreate.py index 46c5d1b..5352308 100644 --- a/tests/hdx/database/test_recreate.py +++ b/tests/hdx/database/test_drop_recreate.py @@ -1,6 +1,12 @@ from hdx.database import Database +def test_drop_schema(mock_engine): + db_uri = "postgresql+psycopg://myuser:mypass@0.0.0.0:12345/mydatabase" + assert Database.drop_schema(mock_engine, db_uri) is True + db_uri = "Error" + assert Database.drop_schema(mock_engine, db_uri) is False + def test_recreate_schema(mock_engine): db_uri = "postgresql+psycopg://myuser:mypass@0.0.0.0:12345/mydatabase" assert Database.recreate_schema(mock_engine, db_uri) is True From 81ea25b03a8bcf6cc0de8a4a58741377bdebe171 Mon Sep 17 00:00:00 2001 From: mcarans Date: Thu, 24 Jul 2025 13:32:07 +1200 Subject: [PATCH 8/8] Fix styling --- tests/hdx/database/test_drop_recreate.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/hdx/database/test_drop_recreate.py b/tests/hdx/database/test_drop_recreate.py index 5352308..1b18ddc 100644 --- a/tests/hdx/database/test_drop_recreate.py +++ b/tests/hdx/database/test_drop_recreate.py @@ -7,6 +7,7 @@ def test_drop_schema(mock_engine): db_uri = "Error" assert Database.drop_schema(mock_engine, db_uri) is False + def test_recreate_schema(mock_engine): db_uri = "postgresql+psycopg://myuser:mypass@0.0.0.0:12345/mydatabase" assert Database.recreate_schema(mock_engine, db_uri) is True