HDXDSYS-2336 Add xz support to HDX Python Database

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 default_language_version:
-    python: python3.12
+    python: python3.13
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
@@ -8,15 +8,15 @@ repos:
       - id: end-of-file-fixer
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.10
+    rev: v0.12.0
     hooks:
       # Run the linter.
-      - id: ruff
+      - id: ruff-check
         args: [ --fix ]
       # Run the formatter.
       - id: ruff-format
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.6.5
+    rev: 0.7.14
     hooks:
       # Run the pip compile
       - id: pip-compile

hatch.toml

@@ -21,7 +21,7 @@ version_scheme = "python-simplified-semver"
 features = ["test"]
 
 [[envs.hatch-test.matrix]]
-python = ["3.12"]
+python = ["3.13"]
 
 [envs.hatch-test.scripts]
 run = """
@@ -31,4 +31,4 @@ run = """
 
 [envs.hatch-static-analysis]
 config-path = "none"
-dependencies = ["ruff==0.9.10"]
+dependencies = ["ruff==0.12.0"]

pyproject.toml

@@ -31,7 +31,7 @@ classifiers = [
     "Operating System :: MacOS",
     "Operating System :: Microsoft :: Windows",
 ]
-requires-python = ">=3.8"
+requires-python = ">=3.10"
 
 dependencies = [
     "sqlalchemy>=2",

requirements.txt

@@ -1,14 +1,14 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
-astdoc==1.2.1
+astdoc==1.3.0
     # via mkapi
 babel==2.17.0
     # via mkdocs-material
-backrefs==5.8
+backrefs==5.9
     # via mkdocs-material
 bcrypt==4.3.0
     # via paramiko
-certifi==2025.4.26
+certifi==2025.7.14
     # via requests
 cffi==1.17.1
     # via
@@ -18,23 +18,23 @@ cfgv==3.4.0
     # via pre-commit
 charset-normalizer==3.4.2
     # via requests
-click==8.1.8
+click==8.2.1
     # via mkdocs
 colorama==0.4.6
     # via mkdocs-material
-coverage==7.8.0
+coverage==7.9.2
     # via pytest-cov
-cryptography==44.0.3
+cryptography==45.0.5
     # via paramiko
-distlib==0.3.9
+distlib==0.4.0
     # via virtualenv
 filelock==3.18.0
     # via virtualenv
 ghp-import==2.1.0
     # via mkdocs
-greenlet==3.2.1
+greenlet==3.2.3
     # via sqlalchemy
-identify==2.6.10
+identify==2.6.12
     # via pre-commit
 idna==3.10
     # via requests
@@ -45,7 +45,7 @@ jinja2==3.1.6
     #   mkapi
     #   mkdocs
     #   mkdocs-material
-markdown==3.8
+markdown==3.8.2
     # via
     #   mkdocs
     #   mkdocs-material
@@ -58,15 +58,15 @@ mergedeep==1.3.4
     # via
     #   mkdocs
     #   mkdocs-get-deps
-mkapi==4.3.2
+mkapi==4.4.4
     # via hdx-python-database (pyproject.toml)
 mkdocs==1.6.1
     # via
     #   mkapi
     #   mkdocs-material
 mkdocs-get-deps==0.2.0
     # via mkdocs
-mkdocs-material==9.6.12
+mkdocs-material==9.6.15
     # via mkapi
 mkdocs-material-extensions==1.3.1
     # via mkdocs-material
@@ -82,31 +82,35 @@ paramiko==3.5.1
     # via sshtunnel
 pathspec==0.12.1
     # via mkdocs
-platformdirs==4.3.7
+platformdirs==4.3.8
     # via
     #   mkdocs-get-deps
     #   virtualenv
-pluggy==1.5.0
-    # via pytest
+pluggy==1.6.0
+    # via
+    #   pytest
+    #   pytest-cov
 pre-commit==4.2.0
     # via hdx-python-database (pyproject.toml)
-psycopg==3.2.7
+psycopg==3.2.9
     # via hdx-python-database (pyproject.toml)
-psycopg-binary==3.2.7
+psycopg-binary==3.2.9
     # via psycopg
 pycparser==2.22
     # via cffi
-pygments==2.19.1
-    # via mkdocs-material
-pymdown-extensions==10.15
+pygments==2.19.2
+    # via
+    #   mkdocs-material
+    #   pytest
+pymdown-extensions==10.16
     # via mkdocs-material
 pynacl==1.5.0
     # via paramiko
-pytest==8.3.5
+pytest==8.4.1
     # via
     #   hdx-python-database (pyproject.toml)
     #   pytest-cov
-pytest-cov==6.1.1
+pytest-cov==6.2.1
     # via hdx-python-database (pyproject.toml)
 python-dateutil==2.9.0.post0
     # via ghp-import
@@ -117,23 +121,21 @@ pyyaml==6.0.2
     #   pre-commit
     #   pymdown-extensions
     #   pyyaml-env-tag
-pyyaml-env-tag==0.1
+pyyaml-env-tag==1.1
     # via mkdocs
-requests==2.32.3
+requests==2.32.4
     # via mkdocs-material
 six==1.17.0
     # via python-dateutil
-sqlalchemy==2.0.40
+sqlalchemy==2.0.41
     # via hdx-python-database (pyproject.toml)
 sshtunnel==0.4.0
     # via hdx-python-database (pyproject.toml)
-typing-extensions==4.13.2
-    # via
-    #   psycopg
-    #   sqlalchemy
-urllib3==2.4.0
+typing-extensions==4.14.1
+    # via sqlalchemy
+urllib3==2.5.0
     # via requests
-virtualenv==20.31.1
+virtualenv==20.32.0
     # via pre-commit
 watchdog==6.0.0
     # via mkdocs

src/hdx/database/database.py

@@ -287,6 +287,28 @@ def create_session(
             table_base.metadata.create_all(engine)
         return Session(engine), table_base
 
+    @staticmethod
+    def drop_schema(engine: Engine, schema_name: str = "public") -> bool:
+        """Wipe schema in database using SQLAlchemy.
+
+        Args:
+            engine (Engine): SQLAlchemy engine to use.
+            schema_name (str): Schema name. Defaults to "public".
+
+        Returns:
+            bool: True if all successful, False if not
+        """
+        # Wipe and create an empty schema
+        try:
+            with engine.connect() as connection:
+                connection.execute(
+                    DropSchema(schema_name, cascade=True, if_exists=True)
+                )
+                connection.commit()
+                return True
+        except SQLAlchemyError:
+            return False
+
     @staticmethod
     def recreate_schema(engine: Engine, schema_name: str = "public") -> bool:
         """Wipe and create empty schema in database using SQLAlchemy.

src/hdx/database/postgresql.py

@@ -59,6 +59,12 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str:
         str: Output from the pg_restore command
     """
     db_params = get_params_from_connection_uri(db_uri)
+    if pg_restore_file[-3:] == ".xz":
+        decompress = subprocess.Popen(
+            ("unxz", "-c", "-d", pg_restore_file), stdout=subprocess.PIPE
+        ).stdout
+    else:
+        decompress = None
     subprocess_params = ["pg_restore", "-c"]
     for key, value in db_params.items():
         match key:
@@ -74,13 +80,18 @@ def restore_from_pgfile(db_uri: str, pg_restore_file: str) -> str:
                 continue
         subprocess_params.append(f"{value}")
 
-    subprocess_params.append(f"{pg_restore_file}")
+    if not decompress:
+        subprocess_params.append(pg_restore_file)
     env = environ.copy()
     password = db_params.get("password")
     if password:
         env["PGPASSWORD"] = password
     process = subprocess.run(
-        subprocess_params, env=env, capture_output=True, encoding="utf-8"
+        subprocess_params,
+        env=env,
+        capture_output=True,
+        encoding="utf-8",
+        stdin=decompress,
     )
     try:
         process.check_returncode()

tests/hdx/database/test_recreate.py → tests/hdx/database/test_drop_recreate.py

@@ -1,6 +1,13 @@
 from hdx.database import Database
 
 
+def test_drop_schema(mock_engine):
+    db_uri = "postgresql+psycopg://myuser:mypass@0.0.0.0:12345/mydatabase"
+    assert Database.drop_schema(mock_engine, db_uri) is True
+    db_uri = "Error"
+    assert Database.drop_schema(mock_engine, db_uri) is False
+
+
 def test_recreate_schema(mock_engine):
     db_uri = "postgresql+psycopg://myuser:mypass@0.0.0.0:12345/mydatabase"
     assert Database.recreate_schema(mock_engine, db_uri) is True