Add rocm CI

.github/workflows/docker_build_tpls.yml

@@ -159,6 +159,20 @@ jobs:
           #   DOCKER_REPOSITORY: geosx/rockylinux9-gcc15-cuda13.2.1
           #   SPEC: "+cuda cuda_arch=86,120 ~openmp ~pygeosx ~docs %gcc-15 ^cuda@13.2.1+allow-unsupported-compilers"
 
+          # ROCm build. Unlike the rows above it does not layer on a
+          # geosx/<os>:<tag>-<sha> base image: AMD's own ROCm image already
+          # ships the amdclang toolchain and the ROCm math libraries, so the
+          # base image is given literally via DOCKER_BASE_IMAGE_OVERRIDE.
+          - name: Ubuntu 24.04 - amdclang 19 + ROCm 6.4.3
+            RUNS_ON: streak2
+            TPL_DOCKERFILE: docker/tpl-ubuntu-hip.Dockerfile
+            DOCKER_BASE_IMAGE_OVERRIDE: rocm/dev-ubuntu-24.04:6.4.3
+            DOCKER_REPOSITORY: geosx/ubuntu24.04-amdclang19.0.0-rocm6.4.3
+            ROCM_VERSION: 6.4.3
+            AMDGPU_TARGET: gfx942
+            SPACK_BUILD_JOBS: 8
+            SPEC: "+rocm ~uncrustify ~openmp ~pygeosx ~docs ~trilinos ~petsc amdgpu_target=gfx942 %amdclang-19 ^caliper~papi~gotcha~sampler~libunwind~libdw"
+
     steps:
     - name: Checkout
       uses: actions/checkout@v6
@@ -175,13 +189,30 @@ jobs:
     - name: Compose DOCKER_BASE_IMAGE
       id: base
       run: |
-        echo "DOCKER_BASE_IMAGE=${{ matrix.DOCKER_BASE_IMAGE_REPO }}:${{ matrix.DOCKER_BASE_IMAGE_TAG }}-${DOCKER_BASE_IMAGE_SHA}" >> "$GITHUB_OUTPUT"
+        # Rows that do not build on a docker_base_images image (e.g. ROCm)
+        # pin their base image directly.
+        if [ -n "${{ matrix.DOCKER_BASE_IMAGE_OVERRIDE }}" ]; then
+          echo "DOCKER_BASE_IMAGE=${{ matrix.DOCKER_BASE_IMAGE_OVERRIDE }}" >> "$GITHUB_OUTPUT"
+        else
+          echo "DOCKER_BASE_IMAGE=${{ matrix.DOCKER_BASE_IMAGE_REPO }}:${{ matrix.DOCKER_BASE_IMAGE_TAG }}-${DOCKER_BASE_IMAGE_SHA}" >> "$GITHUB_OUTPUT"
+        fi
 
     - name: Inject CA certificate into Docker build
       if: matrix.RUNS_ON == 'streak2'
       run: |
         cp /etc/pki/tls/certs/ca-bundle.crt ./ca-bundle.crt
-        sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' ${{ matrix.TPL_DOCKERFILE }}
+
+        # Inject distro-specific trust-store setup into the Dockerfile. Rocky
+        # images use dnf/update-ca-trust, the Ubuntu and ROCm ones use
+        # apt/update-ca-certificates.
+        if grep -q 'RUN dnf clean all' "${{ matrix.TPL_DOCKERFILE }}"; then
+          sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' "${{ matrix.TPL_DOCKERFILE }}"
+        elif grep -q 'apt-get update' "${{ matrix.TPL_DOCKERFILE }}"; then
+          perl -0pi -e 's/(RUN .*?apt-get update)/COPY ca-bundle.crt \/usr\/local\/share\/ca-certificates\/host-ca.crt\nRUN update-ca-certificates\n\n$1/s' "${{ matrix.TPL_DOCKERFILE }}"
+        else
+          echo "Could not find a CA-injection hook in ${{ matrix.TPL_DOCKERFILE }}"
+          exit 1
+        fi
 
     - name: Run the docker build docker script
       env:
@@ -190,6 +221,9 @@ jobs:
         DOCKER_BASE_IMAGE: ${{ steps.base.outputs.DOCKER_BASE_IMAGE }}
         GCC_VERSION: ${{ matrix.GCC_VERSION }}
         CLANG_VERSION: ${{ matrix.CLANG_VERSION }}
+        ROCM_VERSION: ${{ matrix.ROCM_VERSION }}
+        AMDGPU_TARGET: ${{ matrix.AMDGPU_TARGET }}
+        SPACK_BUILD_JOBS: ${{ matrix.SPACK_BUILD_JOBS }}
         INSTALL_DIR_ROOT: ${{ matrix.INSTALL_DIR_ROOT || '/opt/GEOS' }}
         HOST_CONFIG: ${{ matrix.HOST_CONFIG || 'host-configs/environment.cmake' }}
         SPEC: ${{ matrix.SPEC || 'undefined' }}

.gitignore

@@ -4,6 +4,7 @@
 .project
 .pydevproject
 .vscode/
+spack/
 spack-*.txt
 __pycache__
 uberenv_*

.uberenv_config.json

@@ -6,8 +6,9 @@
 "spack_configs_path": "scripts/spack_configs",
 "spack_packages_path": "scripts/spack_packages/packages",
 "spack_url": "https://github.com/spack/spack",
-"spack_commit": "2e2169d5282d166f63e3ee4db8d4446c43cefa8a",
-"spack_commit_note": "v1.1.1 (Jan 14th 2026)",
-"spack_packages_commit": "3dd98680871078353a28ee508fa76c7554f918fa",
-"spack_packages_note": "Feb 25th 2026"
+"spack_commit": "3e19345b6e12f5ff1b874f4059622fc6a1fd804a",
+"spack_commit_note": "v1.2.2 (Jul 20th 2026)",
+"spack_packages_commit": "bd9b4838fbf9e4ca01ad7b6c6df633ddcc750d71",
+"spack_packages_note": "Jul 20th 2026",
+"spack_setup_clingo": 0
 }

CMakeLists.txt

@@ -47,10 +47,12 @@ option( BUILD_SHARED_LIBS "" OFF )
 
 option( ENABLE_CALIPER "" ON )
 option( ENABLE_HYPRE "" ON )
-option( ENABLE_TRILINOS "" ON )
+option( ENABLE_TRILINOS "" OFF )
 option( ENABLE_PETSC "" OFF )
 option( ENABLE_SCOTCH "" ON )
 option( ENABLE_SUITESPARSE "" ON )
+option( ENABLE_SUPERLU_DIST "" ON )
+option( ENABLE_HYPRE_GPU_PROFILING "" OFF )
 option( ENABLE_VTK "" ON )
 option( ENABLE_VTK_WRAP_PYTHON "" OFF )
 
@@ -125,6 +127,50 @@ else( TRUE )
 endif()
 
 
+################################
+# HIP / ROCm
+################################
+if( ENABLE_HIP )
+    if( NOT DEFINED ROCM_PATH )
+        if( DEFINED HIP_ROOT )
+            set( ROCM_PATH ${HIP_ROOT} )
+        elseif( DEFINED ENV{ROCM_PATH} )
+            set( ROCM_PATH $ENV{ROCM_PATH} )
+        else()
+            message( FATAL_ERROR "ENABLE_HIP is ON but neither ROCM_PATH nor HIP_ROOT is defined." )
+        endif()
+    endif()
+
+    if( NOT DEFINED HIP_ROOT )
+        set( HIP_ROOT ${ROCM_PATH} )
+    endif()
+
+    if( NOT DEFINED CMAKE_HIP_ARCHITECTURES )
+        message( FATAL_ERROR "ENABLE_HIP is ON but CMAKE_HIP_ARCHITECTURES is not defined (e.g. gfx90a, gfx942, gfx1100)." )
+    endif()
+
+    # hypre is configured outside of cmake, so it needs the MPI include directory explicitly
+    # when its sources are compiled for the device.
+    if( ENABLE_MPI AND NOT DEFINED MPI_INCLUDE_DIR )
+        if( DEFINED ENV{MPICH_DIR} )               # set by the cray-mpich module on LC
+            set( MPI_INCLUDE_DIR "$ENV{MPICH_DIR}/include" )
+        elseif( DEFINED ENV{MPI_ROOT} )
+            set( MPI_INCLUDE_DIR "$ENV{MPI_ROOT}/include" )
+        elseif( MPI_C_INCLUDE_DIRS )               # set by find_package( MPI ) through BLT
+            list( GET MPI_C_INCLUDE_DIRS 0 MPI_INCLUDE_DIR )
+        elseif( MPI_C_COMPILER )                   # <prefix>/bin/mpicc -> <prefix>/include
+            get_filename_component( MPI_BIN_DIR ${MPI_C_COMPILER} DIRECTORY )
+            get_filename_component( MPI_PREFIX_DIR ${MPI_BIN_DIR} DIRECTORY )
+            set( MPI_INCLUDE_DIR "${MPI_PREFIX_DIR}/include" )
+        else()
+            message( FATAL_ERROR "ENABLE_HIP is ON but MPI_INCLUDE_DIR is not defined and cannot be deduced." )
+        endif()
+        message( STATUS "MPI_INCLUDE_DIR = ${MPI_INCLUDE_DIR}" )
+    endif()
+
+    message( STATUS "ROCM_PATH = ${ROCM_PATH}" )
+    message( STATUS "CMAKE_HIP_ARCHITECTURES = ${CMAKE_HIP_ARCHITECTURES}" )
+endif()
 
 
 ################################
@@ -340,6 +386,13 @@ set(RAJA_URL "${TPL_MIRROR_DIR}/RAJA-v2025.12.0.tar.gz")
 
 message(STATUS "Building RAJA found at ${RAJA_URL}")
 
+# RAJA's vectorization layer does not compile with the ROCm toolchain.
+if( ENABLE_HIP )
+    set( RAJA_ENABLE_VECTORIZATION OFF )
+else()
+    set( RAJA_ENABLE_VECTORIZATION ON )
+endif()
+
 ExternalProject_Add( raja
                      URL ${RAJA_URL}
                      PREFIX ${PROJECT_BINARY_DIR}/raja
@@ -360,10 +413,18 @@ ExternalProject_Add( raja
                                 -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}
                                 -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES}
                                 -D CMAKE_CUDA_FLAGS:STRING=${RAJA_CUDA_FLAGS}
+                                -D ENABLE_HIP:BOOL=${ENABLE_HIP}
+                                -D RAJA_ENABLE_HIP:BOOL=${ENABLE_HIP}
+                                -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER}
+                                -D RAJA_ENABLE_VECTORIZATION:BOOL=${RAJA_ENABLE_VECTORIZATION}
+                                -D hip_DIR=${HIP_ROOT}/lib/cmake/hip
+                                -D ROCPRIM_DIR=${HIP_ROOT}
+                                -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES}
                                 -D ENABLE_TESTS:BOOL=OFF
                                 -D ENABLE_EXAMPLES:BOOL=OFF
                                 -D ENABLE_OPENMP:BOOL=${ENABLE_OPENMP}
                                 -D CMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+                                -D CMAKE_INSTALL_LIBDIR:PATH=lib
                                 -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}
                                 -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE}
                     )
@@ -398,8 +459,14 @@ ExternalProject_Add( chai
                                 -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}
                                 -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES}
                                 -D CMAKE_CUDA_FLAGS:STRING=${CHAI_CUDA_FLAGS}
+                                -D ENABLE_HIP:BOOL=${ENABLE_HIP}
+                                -D CHAI_ENABLE_HIP:BOOL=${ENABLE_HIP}
+                                -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER}
+                                -D hip_DIR=${HIP_ROOT}/lib/cmake/hip
+                                -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES}
                                 -D ENABLE_OPENMP=${ENABLE_OPENMP}
                                 -D CMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+                                -D CMAKE_INSTALL_LIBDIR:PATH=lib
                                 -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS}
                                 -D ENABLE_TESTS=OFF
                                 -D ENABLE_BENCHMARKS=OFF
@@ -444,6 +511,7 @@ if (ENABLE_CALIPER)
                                     -D MPI_CXX_COMPILER:STRING=${MPI_CXX_COMPILER}
                                     -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE}
                                     -D CMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+                                    -D CMAKE_INSTALL_LIBDIR:PATH=lib
                                     -D BLT_SOURCE_DIR=${BLT_SOURCE_DIR}
                        )
 
@@ -480,14 +548,23 @@ if (ENABLE_CALIPER)
                                     -D CUDA_TOOLKIT_ROOT_DIR:PATH=${CUDA_TOOLKIT_ROOT_DIR}
                                     -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE}
                                     -D CMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+                                    -D CMAKE_INSTALL_LIBDIR:PATH=lib
                                     -D WITH_PAPI:BOOL=${ENABLE_PAPI}
                                     -D PAPI_PREFIX:STRING=${PAPI_PREFIX}
                                     -D WITH_ADIAK:BOOL=ON
                                     -D adiak_DIR:STRING=${ADIAK_DIR}/lib/cmake/adiak/
                                     -D WITH_CUPTI=${CALIPER_WITH_CUPTI}
                                     -D WITH_NVPROF=${ENABLE_CUDA}
+                                    -D WITH_ROCM=${ENABLE_ROCTX}
+                                    -D ROCM_PREFIX:PATH=${ROCM_PATH}
                        )
 
+    if( BUILD_SHARED_LIBS )
+        set( CALIPER_LIB "libcaliper.so" )
+    else()
+        set( CALIPER_LIB "libcaliper.a" )
+    endif()
+
     list(APPEND build_list caliper adiak)
 else()
     set( ENABLE_CALIPER_HYPRE OFF CACHE BOOL "" FORCE )
@@ -589,11 +666,6 @@ if (ENABLE_TRILINOS)
                                  -D TPL_MKL_LIBRARIES:STRING=${MKL_LIBRARIES})
     endif()
 
-    if ( ENABLE_ESSL )
-        set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS}
-                                 -D HAVE_dggsvd3:BOOL=ON)
-    endif()
-
     if ( DEFINED OpenMP_Fortran_FLAGS )
         set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS}
                                  -D OpenMP_Fortran_FLAGS:STRING=${OpenMP_Fortran_FLAGS})
@@ -737,7 +809,7 @@ list(APPEND build_list metis )
 ################################
 # SUPERLU --
 ################################
-if( ENABLE_MPI )
+if( ENABLE_MPI AND ENABLE_SUPERLU_DIST )
   set(SUPERLU_DIR "${CMAKE_INSTALL_PREFIX}/superlu_dist")
   set(SUPERLU_URL "${TPL_MIRROR_DIR}/superlu_dist-0f6efc3.tar.gz")
 
@@ -784,6 +856,7 @@ if( ENABLE_MPI )
                                   -D CMAKE_Fortran_FLAGS_RELEASE=${CMAKE_Fortran_FLAGS_RELEASE}
                                   -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE}
                                   -D CMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
+                                  -D CMAKE_INSTALL_LIBDIR:PATH=lib
                                   -D enable_openmp=${ENABLE_OPENMP}
           )
 
@@ -814,6 +887,9 @@ if( ENABLE_HYPRE )
     set( HYPRE_C_COMPILER ${MPI_C_COMPILER} )
     set( HYPRE_CXX_COMPILER ${MPI_CXX_COMPILER} )
     set( HYPRE_Fortran_COMPILER ${MPI_Fortran_COMPILER} )
+  endif()
+
+  if( ENABLE_MPI AND ENABLE_SUPERLU_DIST )
     set( HYPRE_SUPERLU_LIBS "${SUPERLU_DIR}/lib/libsuperlu_dist.a ${PARMETIS_DIR}/lib/libparmetis.a ${METIS_DIR}/lib/libmetis.a -lstdc++" )
     if( ENABLE_OPENMP )
       set( HYPRE_SUPERLU_LIBS "${HYPRE_SUPERLU_LIBS} ${OpenMP_C_FLAGS}" ) # Sergey: why is this needed?
@@ -844,9 +920,41 @@ if( ENABLE_HYPRE )
 --with-umpire-include=${CHAI_DIR}/include \
 --with-umpire-lib-dirs=${CHAI_DIR}/lib \
 --with-umpire-libs=umpire " )
+
+      if( ENABLE_HYPRE_GPU_AWARE_MPI )
+        set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-aware-mpi" )
+      endif()
+
+      if( ENABLE_HYPRE_GPU_PROFILING )
+        set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-profiling" )
+      endif()
     endif()
   elseif( ENABLE_HYPRE_DEVICE STREQUAL "HIP" )
-    message( FATAL_ERROR "hip tpls build only supported through spack" )
+    if( NOT ENABLE_HIP )
+      message( FATAL_ERROR "Non-compatible options: HYPRE device cannot be set to HIP when ENABLE_HIP is OFF" )
+    else()
+      list( APPEND HYPRE_DEPENDS chai )
+      set( HYPRE_HIP_FLAGS "\
+ROCM_PATH=${ROCM_PATH} \
+--with-hip \
+--enable-rocblas \
+--enable-rocsparse \
+--enable-rocsolver \
+--with-gpu-arch=${CMAKE_HIP_ARCHITECTURES} \
+--with-umpire-device \
+--with-umpire-include=${CHAI_DIR}/include \
+--with-umpire-lib-dirs=${CHAI_DIR}/lib \
+--with-umpire-libs=\"umpire camp\" \
+--with-MPI-include=${MPI_INCLUDE_DIR} " )
+
+      if( ENABLE_HYPRE_GPU_AWARE_MPI )
+        set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-aware-mpi" )
+      endif()
+
+      if( ENABLE_HYPRE_GPU_PROFILING )
+        set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-profiling" )
+      endif()
+    endif()
   endif()
 
   if( CMAKE_HOST_APPLE )
@@ -856,7 +964,7 @@ if( ENABLE_HYPRE )
   endif()
 
   if( ENABLE_CALIPER_HYPRE )
-    set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/libs/caliper.a\"" )
+    set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/lib/${CALIPER_LIB}\"" )
     list( APPEND HYPRE_DEPENDS caliper )
   endif()
 
@@ -881,6 +989,7 @@ if( ENABLE_HYPRE )
   ${HYPRE_SUPERLU_FLAGS} \
   ${HYPRE_OPENMP_FLAG} \
   ${HYPRE_CUDA_FLAGS} \
+  ${HYPRE_HIP_FLAGS} \
   ${HYPRE_CALIPER_FLAGS}\n")
 
   ExternalProject_Add( hypre

cmake/thirdparty/FindMathLibraries.cmake

@@ -12,15 +12,6 @@ if( ENABLE_MKL )
     set( BLAS_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE )
     set( LAPACK_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE) 
 
-elseif( ENABLE_ESSL )
-    if ( NOT DEFINED ESSL_INCLUDE_DIRS )
-        message( FATAL_ERROR "ESSL is enabled but ESSL_INCLUDE_DIRS is not defined." )
-    elseif ( NOT DEFINED ESSL_LIBRARIES )
-        message( FATAL_ERROR "ESSL is enabled but ESSL_LIBRARIES is not defined." )
-    endif()
-
-    set( BLAS_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE )
-    set( LAPACK_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE )
 endif()
 
 if( NOT DEFINED BLAS_LIBRARIES )

docker/pecan-spack.yaml

@@ -111,6 +111,8 @@ spack:
       externals:
       - spec: py-sphinx@4.3.2
         prefix: /usr
+        dependencies:
+        - spec: python@3.10.12
     python:
       buildable: false
       externals: