update conda deps

.github/workflows/main.yml

@@ -17,13 +17,13 @@ jobs:
         with:
           update-conda: true
           python-version: ${{ matrix.python-version }}
-          conda-channels: defaults, conda-forge
-      - name: Conda env configuration with mamba
+          conda-channels: conda-forge
+      - name: Install locked Ansible environment
         run: |
-            conda install mamba
-            mamba env create -f environment.yml
+            conda create --yes --name duck --file linux-64.spec
             source activate duck
             pip install -e ".[dev]"
+            python -m pip check
       - name: Test with conda
         run: |
             source activate duck
@@ -33,4 +33,4 @@ jobs:
         run: |
             source activate duck
             flake8 duck
-        if: matrix.python-version == 3.10
+        if: matrix.python-version == '3.10'

MANIFEST.in

@@ -1,5 +1,7 @@
 include Makefile
 include *.txt
+include *.spec
+include environment.yml
 include *.rst
 include *.md
 include tox.ini

docs/source/dev_guide.rst

@@ -35,7 +35,7 @@ First activate the ``duck`` Conda environment and install ``pytest``.
 
 .. code-block:: console
 
-   $ source activate duck
+   $ conda activate duck
    $ pip install -r requirements_dev.txt  # if not already installed
    OR
    $ make develop
@@ -79,10 +79,17 @@ Update the Conda specification file to build identical environments_ on a specif
 .. code-block:: console
 
   $ conda env create -f environment.yml
-  $ source activate duck
+  $ conda activate duck
   $ make clean
   $ make install
-  $ conda list -n duck --explicit > spec-file.txt
+  $ conda install -c conda-forge gunicorn gevent psycopg2=2.9.12 drmaa=0.7.9 dill pytest-cov
+  $ conda list -n duck --explicit --md5 > linux-64.spec
+
+Include the current ``wps_conda_packages`` from the Ansible playbook before
+exporting; the command above reflects the deployment packages at this update.
+Commit ``linux-64.spec`` together with changes to ``environment.yml`` and
+``requirements.txt``. The spec contains Conda packages only; PyWPS, CRAI and its models
+are installed separately by pip.
 
 .. _environments: https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#building-identical-conda-environments
 

docs/source/installation.rst

@@ -29,7 +29,7 @@ Create Conda environment named `duck`:
 .. code-block:: console
 
    $ conda env create -f environment.yml
-   $ source activate duck
+   $ conda activate duck
 
 Install Duck app:
 
@@ -108,6 +108,39 @@ Use Ansible to deploy Duck on your System
 -----------------------------------------
 
 Use the `Ansible playbook`_ for PyWPS to deploy Duck on your system.
+The Linux x86_64 deployment environment is locked in ``linux-64.spec``, replacing
+``spec-list.txt``. It includes the Conda runtime dependencies and the playbook's
+Gunicorn, gevent, psycopg2 2.9.12, DRMAA 0.7.9, dill and test packages.
 
+Set these inventory variables, including any service-specific overrides:
+
+.. code-block:: yaml
+
+   conda_env_use_spec: true
+   conda_env_spec_file: linux-64.spec
+
+Run a full environment deployment so the new packages are installed, rather than
+only updating application code. To install manually on Linux x86_64:
+
+.. code-block:: console
+
+   $ conda create -n duck --file linux-64.spec
+   $ conda activate duck
+   $ python -m pip install .
+
+PyWPS is installed by pip, along with the CRAI application and pretrained models,
+through ``requirements.txt``. These packages are not in the Conda spec.
+``environment.yml`` also installs PyWPS through its pip section. This avoids a
+Conda GDAL/PROJ conflict between Python 3.10's geospatial builds and the playbook's
+new PostgreSQL libraries; PyWPS's pip dependencies provide binary wheels instead.
+
+The CRAI Git dependencies' upstream branches are not frozen by the spec. Both
+currently require setuptools 59.5.0, so that pin is retained. Its Conda builds support Python
+up to 3.10, which determines the Python version for this environment.
+
+The environment uses Python 3.10, PyWPS 4.7 and CPU-only PyTorch 2.5 with
+Torchvision 0.20. Duck explicitly selects CPU inference. The PyTorch upper bound
+preserves the checkpoint-loading behavior used by CRAI. NumPy stays on 1.x.
+For other platforms, use ``environment.yml`` before installing Duck with pip.
 
 .. _Ansible playbook: http://ansible-wps-playbook.readthedocs.io/en/latest/index.html

duck/clintai.py

@@ -52,7 +52,7 @@ def run(dataset, dataset_name, variable_name, outdir, update_status):
         eval_name=dataset.stem,
         data_name=dataset.name,
         data_type=variable_name,
-        eval_parameters=info_models[dataset_name]["eval-parameters"])
+        eval_parameters=info_models[dataset_name]["eval_parameters"])
     # print(f"written cfg {cfg_file}")
     try:
         evaluate(arg_file=cfg_file.as_posix(), prog_func=update_status)

duck/processes/wps_clintai.py

@@ -33,7 +33,7 @@ def __init__(self):
                          abstract="Enter a URL pointing to a NetCDF file with missing values.",
                          min_occurs=1,
                          max_occurs=1,
-                         default="https://www.metoffice.gov.uk/hadobs/hadcrut5/data/current/non-infilled/HadCRUT.5.0.1.0.anomalies.ensemble_mean.nc", # noqa
+                         default="https://www.metoffice.gov.uk/hadobs/hadcrut5/data/HadCRUT.5.0.2.0/non-infilled/HadCRUT.5.0.2.0.anomalies.ensemble_mean.nc", # noqa
                          supported_formats=[FORMATS.NETCDF, FORMATS.ZIP]),
             LiteralInput('variable_name', 'Variable name', data_type='string',
                          abstract='Enter here the variable name to be infilled.',

environment-docs.yml

@@ -2,9 +2,10 @@
 name: duck-docs
 channels:
 - conda-forge
-- defaults
+- nodefaults
 dependencies:
-- pywps>=4.4
+- python=3.10
+- pywps>=4.7,<4.8
 - sphinx
 - nbsphinx
 - ipython

environment.yml

@@ -1,31 +1,38 @@
 name: duck
 channels:
-- conda-forge
-- pytorch
+  - conda-forge
+  - nodefaults
 dependencies:
-- pip
-- python>=3.7
-- pywps>=4.5.1,<4.6
-- jinja2
-- click
-- psutil
-# tests
-- pytest
-- requests
-- flake8
-# clint ai
-- pytorch>=1.11.0
-- tqdm>=4.64.0
-- torchvision>=0.12.0
-- numpy>=1.21.6
-- matplotlib>=3.5.1
-- tensorboardX>=2.5
-- tensorboard>=2.9.0
-- xarray>=2022.3.0
-- dask>=2022.7.0
-- netcdf4>=1.5.8
-- setuptools==59.5.0
-- xesmf>=0.6.2
-# plot
-- cartopy>=0.20.2
-- numba>=0.55.1
+  # CRAI requires setuptools 59.5; its Conda builds support Python <=3.10.
+  - python=3.10
+  - pip
+  - jinja2
+  - click
+  - psutil
+  - requests
+  # CRAI runs inference on the CPU; retain legacy checkpoint loading behavior.
+  - pytorch>=2.5,<2.6
+  - torchvision>=0.20,<0.21
+  - pytorch-cpu
+  # Match PyTorch 2.5.1 wheel metadata when pip installs CRAI.
+  - sympy=1.13.1
+  - torchmetrics>=0.11.2,<0.12
+  - tqdm>=4.64
+  - numpy>=1.26,<2
+  - matplotlib-base>=3.8,<3.9
+  - tensorboardx>=2.5
+  - tensorboard>=2.9
+  - xarray>=2023.1,<2025
+  - dask>=2023.1,<2025
+  - netcdf4>=1.6
+  # Required by both upstream CRAI packages' build metadata.
+  - setuptools=59.5.0
+  - xesmf>=0.8,<0.9
+  - cartopy>=0.22,<0.24
+  - numba>=0.59,<0.61
+  # PyWPS uses pip wheels to avoid the Conda GDAL/PROJ conflict on Python 3.10.
+  - pip:
+      - pywps>=4.7,<4.8
+  # tests
+  - pytest>=6,<9
+  - flake8