HDXDSYS-2489 Investigate HAPI Pipelines food prices failure - fix batch_populate

requirements.txt

@@ -1,72 +1,74 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
-astdoc==1.3.0
+astdoc==1.3.2
     # via mkapi
 babel==2.17.0
     # via mkdocs-material
 backrefs==5.9
     # via mkdocs-material
-bcrypt==4.3.0
+bcrypt==5.0.0
     # via paramiko
-certifi==2025.7.14
+certifi==2025.10.5
     # via requests
-cffi==1.17.1
+cffi==2.0.0
     # via
     #   cryptography
     #   pynacl
 cfgv==3.4.0
     # via pre-commit
-charset-normalizer==3.4.2
+charset-normalizer==3.4.4
     # via requests
-click==8.2.1
+click==8.3.0
     # via mkdocs
 colorama==0.4.6
     # via mkdocs-material
-coverage==7.9.2
+coverage==7.11.0
     # via pytest-cov
-cryptography==45.0.5
+cryptography==46.0.3
     # via paramiko
 distlib==0.4.0
     # via virtualenv
-filelock==3.18.0
+filelock==3.20.0
     # via virtualenv
 ghp-import==2.1.0
     # via mkdocs
-greenlet==3.2.3
+greenlet==3.2.4
     # via sqlalchemy
-identify==2.6.12
+identify==2.6.15
     # via pre-commit
-idna==3.10
+idna==3.11
     # via requests
-iniconfig==2.1.0
+iniconfig==2.3.0
     # via pytest
+invoke==2.2.1
+    # via paramiko
 jinja2==3.1.6
     # via
     #   mkapi
     #   mkdocs
     #   mkdocs-material
-markdown==3.8.2
+markdown==3.9
     # via
     #   mkdocs
     #   mkdocs-material
     #   pymdown-extensions
-markupsafe==3.0.2
+markupsafe==3.0.3
     # via
     #   jinja2
     #   mkdocs
 mergedeep==1.3.4
     # via
     #   mkdocs
     #   mkdocs-get-deps
-mkapi==4.4.4
+mkapi==4.5.0
     # via hdx-python-database (pyproject.toml)
 mkdocs==1.6.1
     # via
     #   mkapi
     #   mkdocs-material
 mkdocs-get-deps==0.2.0
     # via mkdocs
-mkdocs-material==9.6.15
+mkdocs-material==9.6.23
     # via mkapi
 mkdocs-material-extensions==1.3.1
     # via mkdocs-material
@@ -78,43 +80,43 @@ packaging==25.0
     #   pytest
 paginate==0.5.7
     # via mkdocs-material
-paramiko==3.5.1
+paramiko==4.0.0
     # via sshtunnel
 pathspec==0.12.1
     # via mkdocs
-platformdirs==4.3.8
+platformdirs==4.5.0
     # via
     #   mkdocs-get-deps
     #   virtualenv
 pluggy==1.6.0
     # via
     #   pytest
     #   pytest-cov
-pre-commit==4.2.0
+pre-commit==4.3.0
     # via hdx-python-database (pyproject.toml)
-psycopg==3.2.9
+psycopg==3.2.12
     # via hdx-python-database (pyproject.toml)
-psycopg-binary==3.2.9
+psycopg-binary==3.2.12
     # via psycopg
-pycparser==2.22
+pycparser==2.23
     # via cffi
 pygments==2.19.2
     # via
     #   mkdocs-material
     #   pytest
-pymdown-extensions==10.16
+pymdown-extensions==10.16.1
     # via mkdocs-material
-pynacl==1.5.0
+pynacl==1.6.0
     # via paramiko
-pytest==8.4.1
+pytest==8.4.2
     # via
     #   hdx-python-database (pyproject.toml)
     #   pytest-cov
-pytest-cov==6.2.1
+pytest-cov==7.0.0
     # via hdx-python-database (pyproject.toml)
 python-dateutil==2.9.0.post0
     # via ghp-import
-pyyaml==6.0.2
+pyyaml==6.0.3
     # via
     #   mkdocs
     #   mkdocs-get-deps
@@ -123,19 +125,19 @@ pyyaml==6.0.2
     #   pyyaml-env-tag
 pyyaml-env-tag==1.1
     # via mkdocs
-requests==2.32.4
+requests==2.32.5
     # via mkdocs-material
 six==1.17.0
     # via python-dateutil
-sqlalchemy==2.0.41
+sqlalchemy==2.0.44
     # via hdx-python-database (pyproject.toml)
 sshtunnel==0.4.0
     # via hdx-python-database (pyproject.toml)
-typing-extensions==4.14.1
+typing-extensions==4.15.0
     # via sqlalchemy
 urllib3==2.5.0
     # via requests
-virtualenv==20.32.0
+virtualenv==20.35.4
     # via pre-commit
 watchdog==6.0.0
     # via mkdocs

src/hdx/database/database.py

@@ -244,12 +244,10 @@ def batch_populate(
         Returns:
             Tuple[Session, Any]: (SQLAlchemy session, base)
         """
-        batches = range(len(rows) // batch_size + 1)
-        for batch in batches:
-            start_row = batch * batch_size
-            end_row = start_row + batch_size
-            batch_rows = rows[start_row:end_row]
-            self._session.execute(insert(dbtable), batch_rows)
+
+        for i in range(0, len(rows), batch_size):
+            batch = rows[i : i + batch_size]
+            self._session.execute(insert(dbtable), batch)
         self._session.commit()
 
     @staticmethod