From 8b4e0a510140eaa2bf9c12ed597d7863a41f59dc Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 19:29:30 -0700 Subject: [PATCH 1/8] Update spack to v1.2.2 --- .gitignore | 1 + .uberenv_config.json | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 704d7cdd..488ea52a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .project .pydevproject .vscode/ +spack/ spack-*.txt __pycache__ uberenv_* diff --git a/.uberenv_config.json b/.uberenv_config.json index b541fb7c..11d6a5ee 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -6,8 +6,8 @@ "spack_configs_path": "scripts/spack_configs", "spack_packages_path": "scripts/spack_packages/packages", "spack_url": "https://github.com/spack/spack", -"spack_commit": "2e2169d5282d166f63e3ee4db8d4446c43cefa8a", -"spack_commit_note": "v1.1.1 (Jan 14th 2026)", -"spack_packages_commit": "3dd98680871078353a28ee508fa76c7554f918fa", -"spack_packages_note": "Feb 25th 2026" +"spack_commit": "3e19345b6e12f5ff1b874f4059622fc6a1fd804a", +"spack_commit_note": "v1.2.2 (Jul 20th 2026)", +"spack_packages_commit": "bd9b4838fbf9e4ca01ad7b6c6df633ddcc750d71", +"spack_packages_note": "Jul 20th 2026" } From ff13119379c69f7f7d6db4c8bff51d30c6dc78e6 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 22:39:01 -0400 Subject: [PATCH 2/8] Add unified build.sh and HIP support to the TPL superbuild - build.sh: single driver for z5tux (cpu/cuda/hip) and LC (matrix, tioga, tuo-cpu, tuo-gpu), cmake+ninja by default, --spack for uberenv builds - CMakeLists: HIP plumbing for raja, chai, caliper and hypre - Make python explicit in the py-sphinx externals (spack v1.2 deprecation) --- .gitignore | 1 + CMakeLists.txt | 120 ++- build.sh | 813 ++++++++++++++++++ docker/pecan-spack.yaml | 2 + docker/ubuntu-spack.yaml | 2 + .../spack_configs/toss_4_x86_64_ib/spack.yaml | 2 + 6 files changed, 937 insertions(+), 3 deletions(-) create mode 100755 build.sh diff --git a/.gitignore b/.gitignore index 488ea52a..36d0b675 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /build* +!/build.sh /install* .cproject .project diff --git a/CMakeLists.txt b/CMakeLists.txt index 321f3ca0..dc910be1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,8 @@ option( ENABLE_TRILINOS "" ON ) option( ENABLE_PETSC "" OFF ) option( ENABLE_SCOTCH "" ON ) option( ENABLE_SUITESPARSE "" ON ) +option( ENABLE_SUPERLU_DIST "" ON ) +option( ENABLE_HYPRE_GPU_PROFILING "" OFF ) option( ENABLE_VTK "" ON ) option( ENABLE_VTK_WRAP_PYTHON "" OFF ) @@ -125,6 +127,50 @@ else( TRUE ) endif() +################################ +# HIP / ROCm +################################ +if( ENABLE_HIP ) + if( NOT DEFINED ROCM_PATH ) + if( DEFINED HIP_ROOT ) + set( ROCM_PATH ${HIP_ROOT} ) + elseif( DEFINED ENV{ROCM_PATH} ) + set( ROCM_PATH $ENV{ROCM_PATH} ) + else() + message( FATAL_ERROR "ENABLE_HIP is ON but neither ROCM_PATH nor HIP_ROOT is defined." ) + endif() + endif() + + if( NOT DEFINED HIP_ROOT ) + set( HIP_ROOT ${ROCM_PATH} ) + endif() + + if( NOT DEFINED CMAKE_HIP_ARCHITECTURES ) + message( FATAL_ERROR "ENABLE_HIP is ON but CMAKE_HIP_ARCHITECTURES is not defined (e.g. gfx90a, gfx942, gfx1100)." ) + endif() + + # hypre is configured outside of cmake, so it needs the MPI include directory explicitly + # when its sources are compiled for the device. + if( ENABLE_MPI AND NOT DEFINED MPI_INCLUDE_DIR ) + if( DEFINED ENV{MPICH_DIR} ) # set by the cray-mpich module on LC + set( MPI_INCLUDE_DIR "$ENV{MPICH_DIR}/include" ) + elseif( DEFINED ENV{MPI_ROOT} ) + set( MPI_INCLUDE_DIR "$ENV{MPI_ROOT}/include" ) + elseif( MPI_C_INCLUDE_DIRS ) # set by find_package( MPI ) through BLT + list( GET MPI_C_INCLUDE_DIRS 0 MPI_INCLUDE_DIR ) + elseif( MPI_C_COMPILER ) # /bin/mpicc -> /include + get_filename_component( MPI_BIN_DIR ${MPI_C_COMPILER} DIRECTORY ) + get_filename_component( MPI_PREFIX_DIR ${MPI_BIN_DIR} DIRECTORY ) + set( MPI_INCLUDE_DIR "${MPI_PREFIX_DIR}/include" ) + else() + message( FATAL_ERROR "ENABLE_HIP is ON but MPI_INCLUDE_DIR is not defined and cannot be deduced." ) + endif() + message( STATUS "MPI_INCLUDE_DIR = ${MPI_INCLUDE_DIR}" ) + endif() + + message( STATUS "ROCM_PATH = ${ROCM_PATH}" ) + message( STATUS "CMAKE_HIP_ARCHITECTURES = ${CMAKE_HIP_ARCHITECTURES}" ) +endif() ################################ @@ -340,6 +386,13 @@ set(RAJA_URL "${TPL_MIRROR_DIR}/RAJA-v2025.12.0.tar.gz") message(STATUS "Building RAJA found at ${RAJA_URL}") +# RAJA's vectorization layer does not compile with the ROCm toolchain. +if( ENABLE_HIP ) + set( RAJA_ENABLE_VECTORIZATION OFF ) +else() + set( RAJA_ENABLE_VECTORIZATION ON ) +endif() + ExternalProject_Add( raja URL ${RAJA_URL} PREFIX ${PROJECT_BINARY_DIR}/raja @@ -360,10 +413,18 @@ ExternalProject_Add( raja -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CMAKE_CUDA_FLAGS:STRING=${RAJA_CUDA_FLAGS} + -D ENABLE_HIP:BOOL=${ENABLE_HIP} + -D RAJA_ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER} + -D RAJA_ENABLE_VECTORIZATION:BOOL=${RAJA_ENABLE_VECTORIZATION} + -D hip_DIR=${HIP_ROOT}/lib/cmake/hip + -D ROCPRIM_DIR=${HIP_ROOT} + -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} -D ENABLE_TESTS:BOOL=OFF -D ENABLE_EXAMPLES:BOOL=OFF -D ENABLE_OPENMP:BOOL=${ENABLE_OPENMP} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} ) @@ -398,8 +459,14 @@ ExternalProject_Add( chai -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CMAKE_CUDA_FLAGS:STRING=${CHAI_CUDA_FLAGS} + -D ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CHAI_ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER} + -D hip_DIR=${HIP_ROOT}/lib/cmake/hip + -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} -D ENABLE_OPENMP=${ENABLE_OPENMP} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -D ENABLE_TESTS=OFF -D ENABLE_BENCHMARKS=OFF @@ -444,6 +511,7 @@ if (ENABLE_CALIPER) -D MPI_CXX_COMPILER:STRING=${MPI_CXX_COMPILER} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BLT_SOURCE_DIR=${BLT_SOURCE_DIR} ) @@ -480,14 +548,23 @@ if (ENABLE_CALIPER) -D CUDA_TOOLKIT_ROOT_DIR:PATH=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D WITH_PAPI:BOOL=${ENABLE_PAPI} -D PAPI_PREFIX:STRING=${PAPI_PREFIX} -D WITH_ADIAK:BOOL=ON -D adiak_DIR:STRING=${ADIAK_DIR}/lib/cmake/adiak/ -D WITH_CUPTI=${CALIPER_WITH_CUPTI} -D WITH_NVPROF=${ENABLE_CUDA} + -D WITH_ROCM=${ENABLE_ROCTX} + -D ROCM_PREFIX:PATH=${ROCM_PATH} ) + if( BUILD_SHARED_LIBS ) + set( CALIPER_LIB "libcaliper.so" ) + else() + set( CALIPER_LIB "libcaliper.a" ) + endif() + list(APPEND build_list caliper adiak) else() set( ENABLE_CALIPER_HYPRE OFF CACHE BOOL "" FORCE ) @@ -737,7 +814,7 @@ list(APPEND build_list metis ) ################################ # SUPERLU -- ################################ -if( ENABLE_MPI ) +if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) set(SUPERLU_DIR "${CMAKE_INSTALL_PREFIX}/superlu_dist") set(SUPERLU_URL "${TPL_MIRROR_DIR}/superlu_dist-0f6efc3.tar.gz") @@ -784,6 +861,7 @@ if( ENABLE_MPI ) -D CMAKE_Fortran_FLAGS_RELEASE=${CMAKE_Fortran_FLAGS_RELEASE} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D enable_openmp=${ENABLE_OPENMP} ) @@ -814,6 +892,9 @@ if( ENABLE_HYPRE ) set( HYPRE_C_COMPILER ${MPI_C_COMPILER} ) set( HYPRE_CXX_COMPILER ${MPI_CXX_COMPILER} ) set( HYPRE_Fortran_COMPILER ${MPI_Fortran_COMPILER} ) + endif() + + if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) set( HYPRE_SUPERLU_LIBS "${SUPERLU_DIR}/lib/libsuperlu_dist.a ${PARMETIS_DIR}/lib/libparmetis.a ${METIS_DIR}/lib/libmetis.a -lstdc++" ) if( ENABLE_OPENMP ) set( HYPRE_SUPERLU_LIBS "${HYPRE_SUPERLU_LIBS} ${OpenMP_C_FLAGS}" ) # Sergey: why is this needed? @@ -844,9 +925,41 @@ if( ENABLE_HYPRE ) --with-umpire-include=${CHAI_DIR}/include \ --with-umpire-lib-dirs=${CHAI_DIR}/lib \ --with-umpire-libs=umpire " ) + + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-aware-mpi" ) + endif() + + if( ENABLE_HYPRE_GPU_PROFILING ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-profiling" ) + endif() endif() elseif( ENABLE_HYPRE_DEVICE STREQUAL "HIP" ) - message( FATAL_ERROR "hip tpls build only supported through spack" ) + if( NOT ENABLE_HIP ) + message( FATAL_ERROR "Non-compatible options: HYPRE device cannot be set to HIP when ENABLE_HIP is OFF" ) + else() + list( APPEND HYPRE_DEPENDS chai ) + set( HYPRE_HIP_FLAGS "\ +ROCM_PATH=${ROCM_PATH} \ +--with-hip \ +--enable-rocblas \ +--enable-rocsparse \ +--enable-rocsolver \ +--with-gpu-arch=${CMAKE_HIP_ARCHITECTURES} \ +--with-umpire-device \ +--with-umpire-include=${CHAI_DIR}/include \ +--with-umpire-lib-dirs=${CHAI_DIR}/lib \ +--with-umpire-libs=\"umpire camp\" \ +--with-MPI-include=${MPI_INCLUDE_DIR} " ) + + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-aware-mpi" ) + endif() + + if( ENABLE_HYPRE_GPU_PROFILING ) + set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-profiling" ) + endif() + endif() endif() if( CMAKE_HOST_APPLE ) @@ -856,7 +969,7 @@ if( ENABLE_HYPRE ) endif() if( ENABLE_CALIPER_HYPRE ) - set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/libs/caliper.a\"" ) + set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/lib/${CALIPER_LIB}\"" ) list( APPEND HYPRE_DEPENDS caliper ) endif() @@ -881,6 +994,7 @@ if( ENABLE_HYPRE ) ${HYPRE_SUPERLU_FLAGS} \ ${HYPRE_OPENMP_FLAG} \ ${HYPRE_CUDA_FLAGS} \ + ${HYPRE_HIP_FLAGS} \ ${HYPRE_CALIPER_FLAGS}\n") ExternalProject_Add( hypre diff --git a/build.sh b/build.sh new file mode 100755 index 00000000..723ab3f3 --- /dev/null +++ b/build.sh @@ -0,0 +1,813 @@ +#!/usr/bin/env bash +# +# Unified driver to build the GEOS third-party libraries. +# +# Supported configurations: +# z5tux gcc + mpich, host only +# z5tux-cuda gcc + mpich + CUDA (sm_120) +# z5tux-hip amdclang + mpich + ROCm (gfx1100) +# z5tux-sycl not supported yet (see NOTES below) +# matrix gcc + mvapich2 + CUDA (sm_90) +# tioga amdclang + cray-mpich + ROCm (gfx90a) +# tuo-cpu amdclang + cray-mpich, host only +# tuo-gpu amdclang + cray-mpich + ROCm (gfx942) +# +# By default the TPLs are built with the CMake superbuild of this repository +# (CMakeLists.txt), configured with the Ninja generator both for the superbuild +# itself and for every sub-project (-DENABLE_NINJA=ON). With --spack the same +# configurations are built through uberenv/spack instead. +# +# The script is meant to be run *on* the target machine: on the LC systems, +# clone/copy this repository there and run it from a login node (optionally +# with --alloc to push the compilation onto a compute node). +# +# NOTES +# * z5tux-sycl is declared but disabled: neither the superbuild nor the spack +# packages have SYCL plumbing, and no SYCL toolchain is installed on z5tux. +# * Trilinos is off by default (GEOS defaults to the hypre interface); enable +# it with --trilinos. +# * The ROCm configurations compile Fortran with gfortran, not amdflang: the +# flang runtime is not pulled in when clang++ links Fortran objects, which +# breaks superlu_dist. Override with --fc if needed. +# * Known issue: ROCm 7.2 (AMD clang 22) miscompiles hypre's spgemm kernels +# for gfx942/gfx1100 ("Illegal instruction detected: Operand has incorrect +# register class"); gfx90a is fine. LC uses ROCm 6.4.3, which is unaffected. +# +# Usage: ./build.sh [config] [options] [-D= ...] +# Run ./build.sh --help for the full option list. + +set -euo pipefail + +SCRIPT_NAME="$(basename "$0")" +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" + +# ------------------------------------------------------------------------------ +# Defaults +# ------------------------------------------------------------------------------ +CONFIG="" +BUILD_TYPE="Release" +BUILD_DIR="" +INSTALL_DIR="" +JOBS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 8)" +TOP_JOBS=1 +GENERATOR="auto" # auto | ninja | make +GPU_ARCH="" # overrides the per-config cuda arch / amdgpu target +ROCM_DIR_OPT="" +CUDA_DIR_OPT="" +MPI_DIR_OPT="" +FC_OPT="" +USE_ALLOC=0 +ALLOC_CMD="" +ACCOUNT="${GEOS_TPL_ACCOUNT:-vortex}" +LOAD_MODULES=1 +MODULES_OPT="" +CLEAN=0 +DO_CONFIGURE=1 +DO_BUILD=1 +USE_SPACK=0 +SPACK_SPEC_OPT="" +ENABLE_TRILINOS="OFF" +ENABLE_OPENMP="" # per-config default unless set here +ENABLE_ROCTX="OFF" +ENABLE_SUPERLU_DIST="ON" +ENABLE_HYPRE_GPU_PROFILING="OFF" +DRY_RUN=0 +LOG_FILE="" +EXTRA_CMAKE_ARGS=() +FORWARDED_ARGS=() + +# Populated by select_config() +CFG_DESC="" +CFG_SUPPORTED=1 +CFG_UNSUPPORTED_REASON="" +CMAKE_ARGS=() +MPI_PREFIX="" # set by the setup_*_mpi helpers, used by setup_hip + +# ------------------------------------------------------------------------------ +# Helpers +# ------------------------------------------------------------------------------ +log() { printf '[%s] %s\n' "${SCRIPT_NAME}" "$*"; } +warn() { printf '[%s] WARNING: %s\n' "${SCRIPT_NAME}" "$*" >&2; } +die() { printf '[%s] ERROR: %s\n' "${SCRIPT_NAME}" "$*" >&2; exit 1; } + +run() { + printf '[%s] + %s\n' "${SCRIPT_NAME}" "$*" + if [[ ${DRY_RUN} -eq 0 ]]; then + "$@" + fi +} + +# Newest (version-sorted) existing path matching the given glob, empty if none. +newest_path() { + local match + # shellcheck disable=SC2086 # deliberate globbing + match="$(ls -d $1 2>/dev/null | sort -V | tail -1 || true)" + printf '%s' "${match}" +} + +require_file() { + [[ -e "$1" ]] || die "${2:-required file} not found: $1" +} + +require_exec() { + [[ -x "$1" ]] || die "${2:-required executable} not found or not executable: $1" +} + +add_arg() { CMAKE_ARGS+=( "$@" ); } + +usage() { + cat <= ...] [-- ] + +Configurations (auto-detected from the hostname when omitted): + z5tux gcc + mpich, host only + z5tux-cuda gcc + mpich + CUDA, sm_120 + z5tux-hip amdclang + mpich + ROCm, gfx1100 + z5tux-sycl not supported yet + matrix gcc + mvapich2 + CUDA, sm_90 (LC) + tioga amdclang + cray-mpich + ROCm, gfx90a (LC) + tuo-cpu amdclang + cray-mpich, host only (LC, tuolumne) + tuo-gpu amdclang + cray-mpich + ROCm, gfx942 (LC, tuolumne) + +Options: + -bt, --build-type Release|RelWithDebInfo|Debug|MinSizeRel (default: ${BUILD_TYPE}) + -b, --build-dir build directory (default: /build--) + -i, --install-dir install directory (default: /install--) + -j, --jobs parallelism inside each TPL (default: ${JOBS}) + --top-jobs TPLs built concurrently (default: ${TOP_JOBS}) + --generator auto|ninja|make (default: auto, prefers ninja) + --gpu-arch override the cuda arch (e.g. 90) or amdgpu target (e.g. gfx942) + --rocm ROCm installation to use + --cuda CUDA toolkit to use + --mpi MPI installation to use (expects /bin/mpicc) + --fc Fortran compiler to use + --trilinos build Trilinos as well (off by default) + --openmp override the per-configuration OpenMP setting + --roctx enable ROCTx/rocm profiling support (raja, caliper, hypre) + --superlu build superlu_dist and link it into hypre (default: ${ENABLE_SUPERLU_DIST}) + --gpu-profiling configure hypre with --enable-gpu-profiling (nvtx/roctx) + --alloc run the build step through the machine's scheduler + --alloc-cmd scheduler command to use with --alloc + --account bank used by --alloc (default: ${ACCOUNT}) + --modules "" replace the default module list + --no-modules do not touch modules at all + --spack build through uberenv/spack instead of the cmake superbuild + --spec "" spack spec to build (implies --spack) + --clean remove the build and install directories first + --configure-only stop after the cmake configure step + --build-only skip the configure step (reuse an existing build directory) + --log also write the output to + -n, --dry-run print the commands instead of running them + -l, --list list the configurations and exit + -h, --help show this help and exit + + Any -D= argument is forwarded to cmake (or, with --spack, any + argument after -- is forwarded to uberenv.py). + +Examples: + ./${SCRIPT_NAME} z5tux + ./${SCRIPT_NAME} z5tux-cuda -bt RelWithDebInfo -j 24 + ./${SCRIPT_NAME} z5tux-hip --gpu-arch gfx1036 + ./${SCRIPT_NAME} tuo-gpu --alloc # on a tuolumne login node + ./${SCRIPT_NAME} matrix --spack # same config, through spack + ./${SCRIPT_NAME} tioga -DENABLE_VTK=OFF +EOF +} + +# Static description of each configuration: used by --list and by the run banner, +# so that neither needs the toolchain to be resolved first. +describe_config() { + case "$1" in + z5tux) printf 'z5tux: gcc + mpich, host only' ;; + z5tux-cuda) printf 'z5tux: gcc + mpich + CUDA, sm_%s' "${GPU_ARCH:-120}" ;; + z5tux-hip) printf 'z5tux: amdclang + mpich + ROCm, %s' "${GPU_ARCH:-gfx1100}" ;; + z5tux-sycl) printf 'z5tux: SYCL (Intel Battlemage)' ;; + matrix) printf 'matrix: gcc + mvapich2 + CUDA, sm_%s' "${GPU_ARCH:-90}" ;; + tioga|tioga-gpu) printf 'tioga: amdclang + cray-mpich + ROCm, %s' "${GPU_ARCH:-gfx90a}" ;; + tuo-cpu|tuolumne-cpu) printf 'tuolumne: amdclang + cray-mpich, host only' ;; + tuo-gpu|tuolumne-gpu|tuolumne) printf 'tuolumne: amdclang + cray-mpich + ROCm, %s' "${GPU_ARCH:-gfx942}" ;; + *) printf '' ;; + esac +} + +# Modules to load on the LC machines. Kept separate from select_config() so that +# they are loaded *before* the compilers and cmake/ninja are looked up. +modules_for_config() { + case "$1" in + matrix) + printf '%s\n' "gcc/13.3.1-magic" "mvapich2/2.3.7" "cuda/12.9.1" "cmake/3.30.5" "ninja" ;; + tioga|tioga-gpu|tuo-cpu|tuo-gpu|tuolumne|tuolumne-cpu|tuolumne-gpu) + printf '%s\n' "rocm" "cray-mpich" "cmake/3.29" "ninja" ;; + *) + ;; + esac +} + +# Machine a configuration runs on (used for the scheduler defaults). +machine_for_config() { + case "$1" in + z5tux|z5tux-cuda|z5tux-hip|z5tux-sycl) printf 'z5tux' ;; + matrix) printf 'matrix' ;; + tioga|tioga-gpu) printf 'tioga' ;; + tuo-cpu|tuo-gpu|tuolumne*) printf 'tuolumne' ;; + *) printf '' ;; + esac +} + +# Spack spec used by --spack, mirroring scripts/setupLC-TPL-uberenv.bash. +# '%%' propagates the compiler choice to the dependencies (spack v1 syntax). +spack_spec_for_config() { + case "$1" in + z5tux) + printf '~docs %%gcc ^vtk generator=ninja' ;; + z5tux-cuda) + printf '+cuda ~uncrustify ~docs cuda_arch=%s %%gcc ^vtk generator=ninja' "${GPU_ARCH:-120}" ;; + z5tux-hip) + printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%llvm-amdgpu ^vtk generator=ninja' "${GPU_ARCH:-gfx1100}" ;; + matrix) + printf '+cuda ~uncrustify cuda_arch=%s %%%%gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ^vtk generator=ninja' "${GPU_ARCH:-90}" ;; + tioga|tioga-gpu) + printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%%%cce-20 ^vtk generator=ninja' "${GPU_ARCH:-gfx90a}" ;; + tuo-cpu|tuolumne-cpu) + printf '~pygeosx ~trilinos ~petsc ~docs %%%%cce-20 ^vtk generator=ninja' ;; + tuo-gpu|tuolumne-gpu|tuolumne) + printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%%%cce-20 ^vtk generator=ninja' "${GPU_ARCH:-gfx942}" ;; + *) + printf '' ;; + esac +} + +# Configurations that cannot be built (yet); prints the reason. +unsupported_reason() { + case "$1" in + z5tux-sycl) printf 'no SYCL support in the superbuild nor in the spack packages, and no SYCL toolchain installed on z5tux' ;; + *) printf '' ;; + esac +} + +list_configs() { + local cfg + printf '%-12s %s\n' "CONFIG" "DESCRIPTION" + for cfg in z5tux z5tux-cuda z5tux-hip matrix tioga tuo-cpu tuo-gpu; do + printf '%-12s %s\n' "${cfg}" "$(describe_config "${cfg}")" + done + printf '%-12s %s [UNSUPPORTED: %s]\n' "z5tux-sycl" "$(describe_config z5tux-sycl)" \ + "no SYCL support in the superbuild nor in the spack packages" +} + +# ------------------------------------------------------------------------------ +# Argument parsing +# ------------------------------------------------------------------------------ +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) usage; exit 0 ;; + -l|--list) list_configs; exit 0 ;; + -bt|--build-type) BUILD_TYPE="${2:?"missing value for $1"}"; shift 2 ;; + -b|--build-dir) BUILD_DIR="${2:?"missing value for $1"}"; shift 2 ;; + -i|--install-dir) INSTALL_DIR="${2:?"missing value for $1"}"; shift 2 ;; + -j|--jobs) JOBS="${2:?"missing value for $1"}"; shift 2 ;; + --top-jobs) TOP_JOBS="${2:?"missing value for $1"}"; shift 2 ;; + --generator) GENERATOR="${2:?"missing value for $1"}"; shift 2 ;; + --gpu-arch) GPU_ARCH="${2:?"missing value for $1"}"; shift 2 ;; + --rocm) ROCM_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --cuda) CUDA_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --mpi) MPI_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --fc) FC_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --trilinos) ENABLE_TRILINOS="ON"; shift ;; + --openmp) ENABLE_OPENMP="${2:?"missing value for $1"}"; shift 2 ;; + --roctx) ENABLE_ROCTX="ON"; shift ;; + --superlu) ENABLE_SUPERLU_DIST="${2:?"missing value for $1"}"; shift 2 ;; + --gpu-profiling) ENABLE_HYPRE_GPU_PROFILING="ON"; shift ;; + --alloc) USE_ALLOC=1; shift ;; + --alloc-cmd) ALLOC_CMD="$2"; USE_ALLOC=1; shift 2 ;; + --account) ACCOUNT="${2:?"missing value for $1"}"; shift 2 ;; + --modules) MODULES_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --no-modules) LOAD_MODULES=0; shift ;; + --spack) USE_SPACK=1; shift ;; + --spec) SPACK_SPEC_OPT="${2:?"missing value for $1"}"; USE_SPACK=1; shift 2 ;; + --clean) CLEAN=1; shift ;; + --configure-only) DO_BUILD=0; shift ;; + --build-only) DO_CONFIGURE=0; shift ;; + --log) LOG_FILE="${2:?"missing value for $1"}"; shift 2 ;; + -n|--dry-run) DRY_RUN=1; shift ;; + -D*) EXTRA_CMAKE_ARGS+=( "$1" ); shift ;; + --) shift; FORWARDED_ARGS+=( "$@" ); break ;; + -*) die "unknown option '$1' (see --help)" ;; + *) + [[ -z "${CONFIG}" ]] || die "more than one configuration given ('${CONFIG}' and '$1')" + CONFIG="$1"; shift ;; + esac +done + +# ------------------------------------------------------------------------------ +# Configurations +# ------------------------------------------------------------------------------ + +# Compilers + MPI wrappers of the mpich installation used on z5tux. +setup_z5tux_mpi() { + local mpi_dir="${MPI_DIR_OPT:-${MPICH_HOME:-/home/victor/projects/mpich-4.2.2/install/release}}" + require_exec "${mpi_dir}/bin/mpicc" "mpicc (use --mpi )" + add_arg -DENABLE_MPI=ON \ + -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ + -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ + -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ + -DMPIEXEC_EXECUTABLE="${mpi_dir}/bin/mpirun" \ + -DMPIEXEC_NUMPROC_FLAG="-np" + MPI_PREFIX="${mpi_dir}" +} + +# The mpich/cray-mpich wrappers pick their back-end compiler from these. +export_mpich_backend() { + export MPICH_CC="$1" + export MPICH_CXX="$2" + export MPICH_FC="$3" + export MPICH_F90="$3" +} + +# ROCm installation: --rocm, then $ROCM_PATH, then the newest /opt/rocm-*. +resolve_rocm() { + local rocm="${ROCM_DIR_OPT:-${ROCM_PATH:-}}" + if [[ -z "${rocm}" ]]; then + rocm="$(newest_path '/opt/rocm-*')" + [[ -n "${rocm}" ]] || rocm="/opt/rocm" + fi + [[ -d "${rocm}" ]] || die "ROCm installation not found (tried '${rocm}', use --rocm )" + printf '%s' "${rocm}" +} + +# CUDA toolkit: --cuda, then $CUDA_HOME, then the LC/system locations. +resolve_cuda() { + local cuda="${CUDA_DIR_OPT:-${CUDA_HOME:-}}" + if [[ -z "${cuda}" ]]; then + cuda="$(newest_path '/usr/tce/packages/cuda/cuda-*')" + [[ -n "${cuda}" ]] || cuda="$(newest_path '/usr/local/cuda-*')" + [[ -n "${cuda}" ]] || cuda="/usr/local/cuda" + fi + [[ -d "${cuda}" ]] || die "CUDA toolkit not found (tried '${cuda}', use --cuda )" + printf '%s' "${cuda}" +} + +# Fortran compiler paired with the ROCm toolchain. gfortran is preferred over +# amdflang: clang++ links Fortran objects without the flang runtime, which +# breaks the mixed-language TPLs (superlu_dist). +resolve_fortran() { + local rocm="$1" fc + if [[ -n "${FC_OPT}" ]]; then + printf '%s' "${FC_OPT}"; return 0 + fi + for fc in /usr/bin/gfortran \ + "$(newest_path '/usr/tce/packages/gcc-tce/gcc-*/bin/gfortran')" \ + "$(newest_path '/usr/tce/packages/gcc/gcc-*-magic/bin/gfortran')"; do + if [[ -n "${fc}" && -x "${fc}" ]]; then + printf '%s' "${fc}"; return 0 + fi + done + warn "no gfortran found, falling back to amdflang (mixed C++/Fortran TPLs may fail to link)" + printf '%s' "${rocm}/llvm/bin/amdflang" +} + +# amdclang/amdclang++ out of a ROCm installation, plus a Fortran compiler. +setup_rocm_compilers() { + local rocm="$1" + local cc="${rocm}/llvm/bin/amdclang" + local cxx="${rocm}/llvm/bin/amdclang++" + local fc + fc="$(resolve_fortran "${rocm}")" + require_exec "${cc}" "amdclang" + require_exec "${cxx}" "amdclang++" + require_exec "${fc}" "Fortran compiler (use --fc )" + add_arg -DCMAKE_C_COMPILER="${cc}" \ + -DCMAKE_CXX_COMPILER="${cxx}" \ + -DCMAKE_Fortran_COMPILER="${fc}" + export_mpich_backend "${cc}" "${cxx}" "${fc}" +} + +# HIP settings shared by z5tux-hip, tioga and tuo-gpu. +setup_hip() { + local rocm="$1" arch="$2" + # cmake's HIP language requires clang itself, the hipcc wrapper is rejected. + require_exec "${rocm}/llvm/bin/amdclang++" "amdclang++" + add_arg -DENABLE_HIP=ON \ + -DROCM_PATH="${rocm}" \ + -DHIP_ROOT="${rocm}" \ + -DCMAKE_HIP_COMPILER="${rocm}/llvm/bin/amdclang++" \ + -DCMAKE_HIP_ARCHITECTURES="${arch}" \ + -DCMAKE_HIP_STANDARD=17 \ + -DENABLE_ROCTX="${ENABLE_ROCTX}" \ + -DENABLE_HYPRE=ON \ + -DENABLE_HYPRE_DEVICE=HIP + # hypre compiles its device sources outside of cmake and needs the MPI headers + [[ -z "${MPI_PREFIX}" ]] || add_arg -DMPI_INCLUDE_DIR="${MPI_PREFIX}/include" +} + +# CUDA settings shared by z5tux-cuda and matrix. +setup_cuda() { + local cuda="$1" arch="$2" + require_exec "${cuda}/bin/nvcc" "nvcc" + add_arg -DENABLE_CUDA=ON \ + -DENABLE_CUDA_NVTOOLSEXT=ON \ + -DCUDA_TOOLKIT_ROOT_DIR="${cuda}" \ + -DCMAKE_CUDA_COMPILER="${cuda}/bin/nvcc" \ + -DCMAKE_CUDA_ARCHITECTURES="${arch}" \ + -DCUDA_ARCH="sm_${arch}" \ + -DCMAKE_CUDA_STANDARD=17 \ + -DENABLE_HYPRE=ON \ + -DENABLE_HYPRE_DEVICE=CUDA +} + +# cray-mpich as installed on tioga/tuolumne. Prefers the flavor matching the +# compiler family ('amd' for amdclang, 'cray' for cce). +setup_cray_mpi() { + local flavor="${1:-amd}" + local mpi_dir="${MPI_DIR_OPT:-${MPICH_DIR:-}}" + if [[ -z "${mpi_dir}" ]]; then + mpi_dir="$(newest_path "/opt/cray/pe/mpich/*/ofi/${flavor}/*")" + fi + [[ -n "${mpi_dir}" && -x "${mpi_dir}/bin/mpicc" ]] \ + || die "cray-mpich not found (tried '${mpi_dir:-/opt/cray/pe/mpich/*/ofi/${flavor}/*}'); load the cray-mpich module or pass --mpi " + add_arg -DENABLE_MPI=ON \ + -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ + -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ + -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ + -DMPIEXEC_EXECUTABLE="srun" \ + -DMPIEXEC_NUMPROC_FLAG="-n" + MPI_PREFIX="${mpi_dir}" +} + +# BLAS/LAPACK: openblas when available, reference blas/lapack otherwise. +setup_blas_lapack() { + local candidates=( "$@" ) lib + for lib in "${candidates[@]}"; do + if [[ -e "${lib}" ]]; then + add_arg -DBLAS_LIBRARIES="${lib}" -DLAPACK_LIBRARIES="${lib}" + return 0 + fi + done + if [[ -e /usr/lib64/libblas.so && -e /usr/lib64/liblapack.so ]]; then + add_arg -DBLAS_LIBRARIES="/usr/lib64/libblas.so" -DLAPACK_LIBRARIES="/usr/lib64/liblapack.so" + return 0 + fi + warn "no BLAS/LAPACK found in the usual locations, letting cmake search for it" +} + +# Resolves the toolchain of a configuration into cmake cache arguments. +select_config() { + local cfg="$1" + CMAKE_ARGS=() + + case "${cfg}" in + # --------------------------------------------------------------- z5tux --- + z5tux) + add_arg -DCMAKE_C_COMPILER=/usr/bin/gcc \ + -DCMAKE_CXX_COMPILER=/usr/bin/g++ \ + -DCMAKE_Fortran_COMPILER="${FC_OPT:-/usr/bin/gfortran}" \ + -DENABLE_OPENMP="${ENABLE_OPENMP:-ON}" + setup_z5tux_mpi + setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so + ;; + + z5tux-cuda) + add_arg -DCMAKE_C_COMPILER=/usr/bin/gcc \ + -DCMAKE_CXX_COMPILER=/usr/bin/g++ \ + -DCMAKE_Fortran_COMPILER="${FC_OPT:-/usr/bin/gfortran}" \ + -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + setup_z5tux_mpi + setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so + setup_cuda "$(resolve_cuda)" "${GPU_ARCH:-120}" + ;; + + z5tux-hip) + local rocm; rocm="$(resolve_rocm)" + setup_rocm_compilers "${rocm}" + add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + setup_z5tux_mpi + setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so + setup_hip "${rocm}" "${GPU_ARCH:-gfx1100}" + ;; + + # ------------------------------------------------------------------ LC --- + matrix) + + local gcc_dir mpi_dir + gcc_dir="$(newest_path '/usr/tce/packages/gcc/gcc-13*-magic')" + [[ -n "${gcc_dir}" ]] || gcc_dir="$(newest_path '/usr/tce/packages/gcc/gcc-*-magic')" + [[ -n "${gcc_dir}" ]] || die "no gcc found under /usr/tce/packages/gcc" + add_arg -DCMAKE_C_COMPILER="${gcc_dir}/bin/gcc" \ + -DCMAKE_CXX_COMPILER="${gcc_dir}/bin/g++" \ + -DCMAKE_Fortran_COMPILER="${FC_OPT:-${gcc_dir}/bin/gfortran}" \ + -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + + local gcc_ver + gcc_ver="$(basename "${gcc_dir}")"; gcc_ver="${gcc_ver#gcc-}"; gcc_ver="${gcc_ver%-magic}" + mpi_dir="${MPI_DIR_OPT:-$(newest_path "/usr/tce/packages/mvapich2/mvapich2-*-gcc-${gcc_ver}-magic")}" + [[ -n "${mpi_dir}" ]] || mpi_dir="$(newest_path '/usr/tce/packages/mvapich2/mvapich2-*-gcc-*-magic')" + [[ -n "${mpi_dir}" && -x "${mpi_dir}/bin/mpicc" ]] \ + || die "mvapich2 not found under /usr/tce/packages/mvapich2 (use --mpi )" + add_arg -DENABLE_MPI=ON \ + -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ + -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ + -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ + -DMPIEXEC_EXECUTABLE="srun" \ + -DMPIEXEC_NUMPROC_FLAG="-n" + MPI_PREFIX="${mpi_dir}" + + setup_blas_lapack /usr/lib64/libopenblas.so + setup_cuda "$(resolve_cuda)" "${GPU_ARCH:-90}" + ;; + + tioga|tioga-gpu) + local rocm; rocm="$(resolve_rocm)" + setup_rocm_compilers "${rocm}" + add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + setup_cray_mpi amd + setup_blas_lapack /usr/lib64/libopenblas.so + setup_hip "${rocm}" "${GPU_ARCH:-gfx90a}" + ;; + + tuo-cpu|tuolumne-cpu) + local rocm; rocm="$(resolve_rocm)" + setup_rocm_compilers "${rocm}" + add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-ON}" + setup_cray_mpi amd + setup_blas_lapack /usr/lib64/libopenblas.so + ;; + + tuo-gpu|tuolumne-gpu|tuolumne) + local rocm; rocm="$(resolve_rocm)" + setup_rocm_compilers "${rocm}" + add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + setup_cray_mpi amd + setup_blas_lapack /usr/lib64/libopenblas.so + setup_hip "${rocm}" "${GPU_ARCH:-gfx942}" + ;; + + *) + die "unknown configuration '${cfg}' (run --list to see the supported ones)" + ;; + esac +} + +# Scheduler command used by --alloc, per machine. +default_alloc_cmd() { + case "${CFG_MACHINE}" in + matrix) printf 'srun -N 1 --exclusive -t 180 -A %s' "${ACCOUNT}" ;; + tuolumne) printf 'srun -N 1 --exclusive -t 180 -A %s' "${ACCOUNT}" ;; + tioga) printf 'flux run -N 1 -t 180m' ;; + *) printf '' ;; + esac +} + +detect_config() { + local host="${HOSTNAME:-$(hostname)}" + case "${host}" in + z5tux*) printf 'z5tux' ;; + matrix*|rzmatrix*) printf 'matrix' ;; + tioga*|rzvernal*) printf 'tioga' ;; + tuolumne*|tuo[0-9]*) die "on tuolumne the configuration is ambiguous, pass 'tuo-cpu' or 'tuo-gpu'" ;; + *) die "cannot guess the configuration for host '${host}', pass one explicitly (--list)" ;; + esac +} + +# ------------------------------------------------------------------------------ +# Modules +# ------------------------------------------------------------------------------ +init_modules() { + if ! declare -F module >/dev/null 2>&1; then + local init + for init in /usr/share/lmod/lmod/init/bash /etc/profile.d/modules.sh /usr/share/Modules/init/bash; do + if [[ -r "${init}" ]]; then + # shellcheck disable=SC1090 + source "${init}" + break + fi + done + fi + declare -F module >/dev/null 2>&1 || type module >/dev/null 2>&1 +} + +load_modules() { + local -a mods=() + local m + if [[ -n "${MODULES_OPT}" ]]; then + read -r -a mods <<< "${MODULES_OPT}" + else + while IFS= read -r m; do + [[ -n "${m}" ]] && mods+=( "${m}" ) + done < <(modules_for_config "${CONFIG}") + fi + [[ ${#mods[@]} -gt 0 ]] || return 0 + + if ! init_modules; then + warn "no module command available, skipping module loading" + return 0 + fi + + local m + for m in "${mods[@]}"; do + if [[ ${DRY_RUN} -eq 1 ]]; then + log "+ module load ${m}" + continue + fi + if module load "${m}" >/dev/null 2>&1; then + log "module load ${m}" + else + # Retry without the version: LC renames those more often than the toolchains move. + local base="${m%%/*}" + if [[ "${base}" != "${m}" ]] && module load "${base}" >/dev/null 2>&1; then + log "module load ${base} (fallback, '${m}' unavailable)" + else + warn "could not load module '${m}' (continuing; run 'module avail ${base}' to check)" + fi + fi + done +} + +# ------------------------------------------------------------------------------ +# Build back-ends +# ------------------------------------------------------------------------------ +resolve_generator() { + case "${GENERATOR}" in + ninja) + command -v ninja >/dev/null 2>&1 || die "ninja not found in PATH (try --generator make)" + ;; + make) + ;; + auto) + if command -v ninja >/dev/null 2>&1; then + GENERATOR="ninja" + else + warn "ninja not found in PATH, falling back to Unix Makefiles" + GENERATOR="make" + fi + ;; + *) + die "unknown generator '${GENERATOR}' (auto|ninja|make)" + ;; + esac +} + +configure_cmake() { + local -a cmd=( cmake -S "${REPO_DIR}" -B "${BUILD_DIR}" ) + + if [[ "${GENERATOR}" == "ninja" ]]; then + cmd+=( -G Ninja -DENABLE_NINJA=ON ) + else + cmd+=( -G "Unix Makefiles" -DENABLE_NINJA=OFF ) + fi + + cmd+=( -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -DNUM_PROC="${JOBS}" + -DENABLE_TRILINOS="${ENABLE_TRILINOS}" + -DENABLE_SUPERLU_DIST="${ENABLE_SUPERLU_DIST}" + -DENABLE_HYPRE_GPU_PROFILING="${ENABLE_HYPRE_GPU_PROFILING}" + -DBLT_CXX_STD=c++17 + -DCMAKE_CXX_STANDARD=17 ) + cmd+=( "${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}" ) + cmd+=( "${EXTRA_CMAKE_ARGS[@]+"${EXTRA_CMAKE_ARGS[@]}"}" ) + cmd+=( "${FORWARDED_ARGS[@]+"${FORWARDED_ARGS[@]}"}" ) + + run "${cmd[@]}" + write_build_env +} + +# The mpi wrappers pick their back-end compiler from the MPICH_* variables, and +# the TPLs configured outside of cmake (hypre) inherit them from this script. +# Dump them next to the build so that running ninja by hand stays equivalent. +write_build_env() { + [[ ${DRY_RUN} -eq 0 && -n "${MPICH_CC:-}" ]] || return 0 + local env_file="${BUILD_DIR}/build-env.sh" + cat > "${env_file}" </dev/null 2>&1 || die "cmake not found in PATH" + log "generator : ${GENERATOR}" + log "jobs : ${JOBS} per TPL, ${TOP_JOBS} TPL(s) at a time" + + if [[ ${DO_CONFIGURE} -eq 1 ]]; then + configure_cmake + else + [[ -d "${BUILD_DIR}" ]] || die "--build-only given but '${BUILD_DIR}' does not exist" + fi + + if [[ ${DO_BUILD} -eq 1 ]]; then + build_cmake + fi + fi + + log "done." + if [[ ${DO_BUILD} -eq 1 && ${DRY_RUN} -eq 0 ]]; then + log "TPLs installed in ${INSTALL_DIR}" + fi +} + +if [[ -n "${LOG_FILE}" ]]; then + main 2>&1 | tee "${LOG_FILE}" + exit "${PIPESTATUS[0]}" +else + main +fi diff --git a/docker/pecan-spack.yaml b/docker/pecan-spack.yaml index 89ca8678..300cef98 100644 --- a/docker/pecan-spack.yaml +++ b/docker/pecan-spack.yaml @@ -111,6 +111,8 @@ spack: externals: - spec: py-sphinx@4.3.2 prefix: /usr + dependencies: + - spec: python@3.10.12 python: buildable: false externals: diff --git a/docker/ubuntu-spack.yaml b/docker/ubuntu-spack.yaml index e27514bf..40915d04 100644 --- a/docker/ubuntu-spack.yaml +++ b/docker/ubuntu-spack.yaml @@ -275,6 +275,8 @@ spack: externals: - spec: py-sphinx@7.4.7 prefix: /usr + dependencies: + - spec: python@3.12 python: buildable: false externals: diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index d0ffeed9..8b29be18 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -231,6 +231,8 @@ spack: externals: - spec: py-sphinx@1.6.3 prefix: /usr/gapps/GEOSX/thirdPartyLibs/python/quartz-gcc-python/python/ + dependencies: + - spec: python@3.10.8 autoconf: buildable: False externals: From 39cc571c43ab666366870aa0d2b4a4cd6021558f Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 22:40:30 -0400 Subject: [PATCH 3/8] Remove build.sh from versioning --- .gitignore | 1 - build.sh | 813 ----------------------------------------------------- 2 files changed, 814 deletions(-) delete mode 100755 build.sh diff --git a/.gitignore b/.gitignore index 36d0b675..488ea52a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ /build* -!/build.sh /install* .cproject .project diff --git a/build.sh b/build.sh deleted file mode 100755 index 723ab3f3..00000000 --- a/build.sh +++ /dev/null @@ -1,813 +0,0 @@ -#!/usr/bin/env bash -# -# Unified driver to build the GEOS third-party libraries. -# -# Supported configurations: -# z5tux gcc + mpich, host only -# z5tux-cuda gcc + mpich + CUDA (sm_120) -# z5tux-hip amdclang + mpich + ROCm (gfx1100) -# z5tux-sycl not supported yet (see NOTES below) -# matrix gcc + mvapich2 + CUDA (sm_90) -# tioga amdclang + cray-mpich + ROCm (gfx90a) -# tuo-cpu amdclang + cray-mpich, host only -# tuo-gpu amdclang + cray-mpich + ROCm (gfx942) -# -# By default the TPLs are built with the CMake superbuild of this repository -# (CMakeLists.txt), configured with the Ninja generator both for the superbuild -# itself and for every sub-project (-DENABLE_NINJA=ON). With --spack the same -# configurations are built through uberenv/spack instead. -# -# The script is meant to be run *on* the target machine: on the LC systems, -# clone/copy this repository there and run it from a login node (optionally -# with --alloc to push the compilation onto a compute node). -# -# NOTES -# * z5tux-sycl is declared but disabled: neither the superbuild nor the spack -# packages have SYCL plumbing, and no SYCL toolchain is installed on z5tux. -# * Trilinos is off by default (GEOS defaults to the hypre interface); enable -# it with --trilinos. -# * The ROCm configurations compile Fortran with gfortran, not amdflang: the -# flang runtime is not pulled in when clang++ links Fortran objects, which -# breaks superlu_dist. Override with --fc if needed. -# * Known issue: ROCm 7.2 (AMD clang 22) miscompiles hypre's spgemm kernels -# for gfx942/gfx1100 ("Illegal instruction detected: Operand has incorrect -# register class"); gfx90a is fine. LC uses ROCm 6.4.3, which is unaffected. -# -# Usage: ./build.sh [config] [options] [-D= ...] -# Run ./build.sh --help for the full option list. - -set -euo pipefail - -SCRIPT_NAME="$(basename "$0")" -REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" - -# ------------------------------------------------------------------------------ -# Defaults -# ------------------------------------------------------------------------------ -CONFIG="" -BUILD_TYPE="Release" -BUILD_DIR="" -INSTALL_DIR="" -JOBS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 8)" -TOP_JOBS=1 -GENERATOR="auto" # auto | ninja | make -GPU_ARCH="" # overrides the per-config cuda arch / amdgpu target -ROCM_DIR_OPT="" -CUDA_DIR_OPT="" -MPI_DIR_OPT="" -FC_OPT="" -USE_ALLOC=0 -ALLOC_CMD="" -ACCOUNT="${GEOS_TPL_ACCOUNT:-vortex}" -LOAD_MODULES=1 -MODULES_OPT="" -CLEAN=0 -DO_CONFIGURE=1 -DO_BUILD=1 -USE_SPACK=0 -SPACK_SPEC_OPT="" -ENABLE_TRILINOS="OFF" -ENABLE_OPENMP="" # per-config default unless set here -ENABLE_ROCTX="OFF" -ENABLE_SUPERLU_DIST="ON" -ENABLE_HYPRE_GPU_PROFILING="OFF" -DRY_RUN=0 -LOG_FILE="" -EXTRA_CMAKE_ARGS=() -FORWARDED_ARGS=() - -# Populated by select_config() -CFG_DESC="" -CFG_SUPPORTED=1 -CFG_UNSUPPORTED_REASON="" -CMAKE_ARGS=() -MPI_PREFIX="" # set by the setup_*_mpi helpers, used by setup_hip - -# ------------------------------------------------------------------------------ -# Helpers -# ------------------------------------------------------------------------------ -log() { printf '[%s] %s\n' "${SCRIPT_NAME}" "$*"; } -warn() { printf '[%s] WARNING: %s\n' "${SCRIPT_NAME}" "$*" >&2; } -die() { printf '[%s] ERROR: %s\n' "${SCRIPT_NAME}" "$*" >&2; exit 1; } - -run() { - printf '[%s] + %s\n' "${SCRIPT_NAME}" "$*" - if [[ ${DRY_RUN} -eq 0 ]]; then - "$@" - fi -} - -# Newest (version-sorted) existing path matching the given glob, empty if none. -newest_path() { - local match - # shellcheck disable=SC2086 # deliberate globbing - match="$(ls -d $1 2>/dev/null | sort -V | tail -1 || true)" - printf '%s' "${match}" -} - -require_file() { - [[ -e "$1" ]] || die "${2:-required file} not found: $1" -} - -require_exec() { - [[ -x "$1" ]] || die "${2:-required executable} not found or not executable: $1" -} - -add_arg() { CMAKE_ARGS+=( "$@" ); } - -usage() { - cat <= ...] [-- ] - -Configurations (auto-detected from the hostname when omitted): - z5tux gcc + mpich, host only - z5tux-cuda gcc + mpich + CUDA, sm_120 - z5tux-hip amdclang + mpich + ROCm, gfx1100 - z5tux-sycl not supported yet - matrix gcc + mvapich2 + CUDA, sm_90 (LC) - tioga amdclang + cray-mpich + ROCm, gfx90a (LC) - tuo-cpu amdclang + cray-mpich, host only (LC, tuolumne) - tuo-gpu amdclang + cray-mpich + ROCm, gfx942 (LC, tuolumne) - -Options: - -bt, --build-type Release|RelWithDebInfo|Debug|MinSizeRel (default: ${BUILD_TYPE}) - -b, --build-dir build directory (default: /build--) - -i, --install-dir install directory (default: /install--) - -j, --jobs parallelism inside each TPL (default: ${JOBS}) - --top-jobs TPLs built concurrently (default: ${TOP_JOBS}) - --generator auto|ninja|make (default: auto, prefers ninja) - --gpu-arch override the cuda arch (e.g. 90) or amdgpu target (e.g. gfx942) - --rocm ROCm installation to use - --cuda CUDA toolkit to use - --mpi MPI installation to use (expects /bin/mpicc) - --fc Fortran compiler to use - --trilinos build Trilinos as well (off by default) - --openmp override the per-configuration OpenMP setting - --roctx enable ROCTx/rocm profiling support (raja, caliper, hypre) - --superlu build superlu_dist and link it into hypre (default: ${ENABLE_SUPERLU_DIST}) - --gpu-profiling configure hypre with --enable-gpu-profiling (nvtx/roctx) - --alloc run the build step through the machine's scheduler - --alloc-cmd scheduler command to use with --alloc - --account bank used by --alloc (default: ${ACCOUNT}) - --modules "" replace the default module list - --no-modules do not touch modules at all - --spack build through uberenv/spack instead of the cmake superbuild - --spec "" spack spec to build (implies --spack) - --clean remove the build and install directories first - --configure-only stop after the cmake configure step - --build-only skip the configure step (reuse an existing build directory) - --log also write the output to - -n, --dry-run print the commands instead of running them - -l, --list list the configurations and exit - -h, --help show this help and exit - - Any -D= argument is forwarded to cmake (or, with --spack, any - argument after -- is forwarded to uberenv.py). - -Examples: - ./${SCRIPT_NAME} z5tux - ./${SCRIPT_NAME} z5tux-cuda -bt RelWithDebInfo -j 24 - ./${SCRIPT_NAME} z5tux-hip --gpu-arch gfx1036 - ./${SCRIPT_NAME} tuo-gpu --alloc # on a tuolumne login node - ./${SCRIPT_NAME} matrix --spack # same config, through spack - ./${SCRIPT_NAME} tioga -DENABLE_VTK=OFF -EOF -} - -# Static description of each configuration: used by --list and by the run banner, -# so that neither needs the toolchain to be resolved first. -describe_config() { - case "$1" in - z5tux) printf 'z5tux: gcc + mpich, host only' ;; - z5tux-cuda) printf 'z5tux: gcc + mpich + CUDA, sm_%s' "${GPU_ARCH:-120}" ;; - z5tux-hip) printf 'z5tux: amdclang + mpich + ROCm, %s' "${GPU_ARCH:-gfx1100}" ;; - z5tux-sycl) printf 'z5tux: SYCL (Intel Battlemage)' ;; - matrix) printf 'matrix: gcc + mvapich2 + CUDA, sm_%s' "${GPU_ARCH:-90}" ;; - tioga|tioga-gpu) printf 'tioga: amdclang + cray-mpich + ROCm, %s' "${GPU_ARCH:-gfx90a}" ;; - tuo-cpu|tuolumne-cpu) printf 'tuolumne: amdclang + cray-mpich, host only' ;; - tuo-gpu|tuolumne-gpu|tuolumne) printf 'tuolumne: amdclang + cray-mpich + ROCm, %s' "${GPU_ARCH:-gfx942}" ;; - *) printf '' ;; - esac -} - -# Modules to load on the LC machines. Kept separate from select_config() so that -# they are loaded *before* the compilers and cmake/ninja are looked up. -modules_for_config() { - case "$1" in - matrix) - printf '%s\n' "gcc/13.3.1-magic" "mvapich2/2.3.7" "cuda/12.9.1" "cmake/3.30.5" "ninja" ;; - tioga|tioga-gpu|tuo-cpu|tuo-gpu|tuolumne|tuolumne-cpu|tuolumne-gpu) - printf '%s\n' "rocm" "cray-mpich" "cmake/3.29" "ninja" ;; - *) - ;; - esac -} - -# Machine a configuration runs on (used for the scheduler defaults). -machine_for_config() { - case "$1" in - z5tux|z5tux-cuda|z5tux-hip|z5tux-sycl) printf 'z5tux' ;; - matrix) printf 'matrix' ;; - tioga|tioga-gpu) printf 'tioga' ;; - tuo-cpu|tuo-gpu|tuolumne*) printf 'tuolumne' ;; - *) printf '' ;; - esac -} - -# Spack spec used by --spack, mirroring scripts/setupLC-TPL-uberenv.bash. -# '%%' propagates the compiler choice to the dependencies (spack v1 syntax). -spack_spec_for_config() { - case "$1" in - z5tux) - printf '~docs %%gcc ^vtk generator=ninja' ;; - z5tux-cuda) - printf '+cuda ~uncrustify ~docs cuda_arch=%s %%gcc ^vtk generator=ninja' "${GPU_ARCH:-120}" ;; - z5tux-hip) - printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%llvm-amdgpu ^vtk generator=ninja' "${GPU_ARCH:-gfx1100}" ;; - matrix) - printf '+cuda ~uncrustify cuda_arch=%s %%%%gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ^vtk generator=ninja' "${GPU_ARCH:-90}" ;; - tioga|tioga-gpu) - printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%%%cce-20 ^vtk generator=ninja' "${GPU_ARCH:-gfx90a}" ;; - tuo-cpu|tuolumne-cpu) - printf '~pygeosx ~trilinos ~petsc ~docs %%%%cce-20 ^vtk generator=ninja' ;; - tuo-gpu|tuolumne-gpu|tuolumne) - printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%%%cce-20 ^vtk generator=ninja' "${GPU_ARCH:-gfx942}" ;; - *) - printf '' ;; - esac -} - -# Configurations that cannot be built (yet); prints the reason. -unsupported_reason() { - case "$1" in - z5tux-sycl) printf 'no SYCL support in the superbuild nor in the spack packages, and no SYCL toolchain installed on z5tux' ;; - *) printf '' ;; - esac -} - -list_configs() { - local cfg - printf '%-12s %s\n' "CONFIG" "DESCRIPTION" - for cfg in z5tux z5tux-cuda z5tux-hip matrix tioga tuo-cpu tuo-gpu; do - printf '%-12s %s\n' "${cfg}" "$(describe_config "${cfg}")" - done - printf '%-12s %s [UNSUPPORTED: %s]\n' "z5tux-sycl" "$(describe_config z5tux-sycl)" \ - "no SYCL support in the superbuild nor in the spack packages" -} - -# ------------------------------------------------------------------------------ -# Argument parsing -# ------------------------------------------------------------------------------ -while [[ $# -gt 0 ]]; do - case "$1" in - -h|--help) usage; exit 0 ;; - -l|--list) list_configs; exit 0 ;; - -bt|--build-type) BUILD_TYPE="${2:?"missing value for $1"}"; shift 2 ;; - -b|--build-dir) BUILD_DIR="${2:?"missing value for $1"}"; shift 2 ;; - -i|--install-dir) INSTALL_DIR="${2:?"missing value for $1"}"; shift 2 ;; - -j|--jobs) JOBS="${2:?"missing value for $1"}"; shift 2 ;; - --top-jobs) TOP_JOBS="${2:?"missing value for $1"}"; shift 2 ;; - --generator) GENERATOR="${2:?"missing value for $1"}"; shift 2 ;; - --gpu-arch) GPU_ARCH="${2:?"missing value for $1"}"; shift 2 ;; - --rocm) ROCM_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --cuda) CUDA_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --mpi) MPI_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --fc) FC_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --trilinos) ENABLE_TRILINOS="ON"; shift ;; - --openmp) ENABLE_OPENMP="${2:?"missing value for $1"}"; shift 2 ;; - --roctx) ENABLE_ROCTX="ON"; shift ;; - --superlu) ENABLE_SUPERLU_DIST="${2:?"missing value for $1"}"; shift 2 ;; - --gpu-profiling) ENABLE_HYPRE_GPU_PROFILING="ON"; shift ;; - --alloc) USE_ALLOC=1; shift ;; - --alloc-cmd) ALLOC_CMD="$2"; USE_ALLOC=1; shift 2 ;; - --account) ACCOUNT="${2:?"missing value for $1"}"; shift 2 ;; - --modules) MODULES_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --no-modules) LOAD_MODULES=0; shift ;; - --spack) USE_SPACK=1; shift ;; - --spec) SPACK_SPEC_OPT="${2:?"missing value for $1"}"; USE_SPACK=1; shift 2 ;; - --clean) CLEAN=1; shift ;; - --configure-only) DO_BUILD=0; shift ;; - --build-only) DO_CONFIGURE=0; shift ;; - --log) LOG_FILE="${2:?"missing value for $1"}"; shift 2 ;; - -n|--dry-run) DRY_RUN=1; shift ;; - -D*) EXTRA_CMAKE_ARGS+=( "$1" ); shift ;; - --) shift; FORWARDED_ARGS+=( "$@" ); break ;; - -*) die "unknown option '$1' (see --help)" ;; - *) - [[ -z "${CONFIG}" ]] || die "more than one configuration given ('${CONFIG}' and '$1')" - CONFIG="$1"; shift ;; - esac -done - -# ------------------------------------------------------------------------------ -# Configurations -# ------------------------------------------------------------------------------ - -# Compilers + MPI wrappers of the mpich installation used on z5tux. -setup_z5tux_mpi() { - local mpi_dir="${MPI_DIR_OPT:-${MPICH_HOME:-/home/victor/projects/mpich-4.2.2/install/release}}" - require_exec "${mpi_dir}/bin/mpicc" "mpicc (use --mpi )" - add_arg -DENABLE_MPI=ON \ - -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ - -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ - -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ - -DMPIEXEC_EXECUTABLE="${mpi_dir}/bin/mpirun" \ - -DMPIEXEC_NUMPROC_FLAG="-np" - MPI_PREFIX="${mpi_dir}" -} - -# The mpich/cray-mpich wrappers pick their back-end compiler from these. -export_mpich_backend() { - export MPICH_CC="$1" - export MPICH_CXX="$2" - export MPICH_FC="$3" - export MPICH_F90="$3" -} - -# ROCm installation: --rocm, then $ROCM_PATH, then the newest /opt/rocm-*. -resolve_rocm() { - local rocm="${ROCM_DIR_OPT:-${ROCM_PATH:-}}" - if [[ -z "${rocm}" ]]; then - rocm="$(newest_path '/opt/rocm-*')" - [[ -n "${rocm}" ]] || rocm="/opt/rocm" - fi - [[ -d "${rocm}" ]] || die "ROCm installation not found (tried '${rocm}', use --rocm )" - printf '%s' "${rocm}" -} - -# CUDA toolkit: --cuda, then $CUDA_HOME, then the LC/system locations. -resolve_cuda() { - local cuda="${CUDA_DIR_OPT:-${CUDA_HOME:-}}" - if [[ -z "${cuda}" ]]; then - cuda="$(newest_path '/usr/tce/packages/cuda/cuda-*')" - [[ -n "${cuda}" ]] || cuda="$(newest_path '/usr/local/cuda-*')" - [[ -n "${cuda}" ]] || cuda="/usr/local/cuda" - fi - [[ -d "${cuda}" ]] || die "CUDA toolkit not found (tried '${cuda}', use --cuda )" - printf '%s' "${cuda}" -} - -# Fortran compiler paired with the ROCm toolchain. gfortran is preferred over -# amdflang: clang++ links Fortran objects without the flang runtime, which -# breaks the mixed-language TPLs (superlu_dist). -resolve_fortran() { - local rocm="$1" fc - if [[ -n "${FC_OPT}" ]]; then - printf '%s' "${FC_OPT}"; return 0 - fi - for fc in /usr/bin/gfortran \ - "$(newest_path '/usr/tce/packages/gcc-tce/gcc-*/bin/gfortran')" \ - "$(newest_path '/usr/tce/packages/gcc/gcc-*-magic/bin/gfortran')"; do - if [[ -n "${fc}" && -x "${fc}" ]]; then - printf '%s' "${fc}"; return 0 - fi - done - warn "no gfortran found, falling back to amdflang (mixed C++/Fortran TPLs may fail to link)" - printf '%s' "${rocm}/llvm/bin/amdflang" -} - -# amdclang/amdclang++ out of a ROCm installation, plus a Fortran compiler. -setup_rocm_compilers() { - local rocm="$1" - local cc="${rocm}/llvm/bin/amdclang" - local cxx="${rocm}/llvm/bin/amdclang++" - local fc - fc="$(resolve_fortran "${rocm}")" - require_exec "${cc}" "amdclang" - require_exec "${cxx}" "amdclang++" - require_exec "${fc}" "Fortran compiler (use --fc )" - add_arg -DCMAKE_C_COMPILER="${cc}" \ - -DCMAKE_CXX_COMPILER="${cxx}" \ - -DCMAKE_Fortran_COMPILER="${fc}" - export_mpich_backend "${cc}" "${cxx}" "${fc}" -} - -# HIP settings shared by z5tux-hip, tioga and tuo-gpu. -setup_hip() { - local rocm="$1" arch="$2" - # cmake's HIP language requires clang itself, the hipcc wrapper is rejected. - require_exec "${rocm}/llvm/bin/amdclang++" "amdclang++" - add_arg -DENABLE_HIP=ON \ - -DROCM_PATH="${rocm}" \ - -DHIP_ROOT="${rocm}" \ - -DCMAKE_HIP_COMPILER="${rocm}/llvm/bin/amdclang++" \ - -DCMAKE_HIP_ARCHITECTURES="${arch}" \ - -DCMAKE_HIP_STANDARD=17 \ - -DENABLE_ROCTX="${ENABLE_ROCTX}" \ - -DENABLE_HYPRE=ON \ - -DENABLE_HYPRE_DEVICE=HIP - # hypre compiles its device sources outside of cmake and needs the MPI headers - [[ -z "${MPI_PREFIX}" ]] || add_arg -DMPI_INCLUDE_DIR="${MPI_PREFIX}/include" -} - -# CUDA settings shared by z5tux-cuda and matrix. -setup_cuda() { - local cuda="$1" arch="$2" - require_exec "${cuda}/bin/nvcc" "nvcc" - add_arg -DENABLE_CUDA=ON \ - -DENABLE_CUDA_NVTOOLSEXT=ON \ - -DCUDA_TOOLKIT_ROOT_DIR="${cuda}" \ - -DCMAKE_CUDA_COMPILER="${cuda}/bin/nvcc" \ - -DCMAKE_CUDA_ARCHITECTURES="${arch}" \ - -DCUDA_ARCH="sm_${arch}" \ - -DCMAKE_CUDA_STANDARD=17 \ - -DENABLE_HYPRE=ON \ - -DENABLE_HYPRE_DEVICE=CUDA -} - -# cray-mpich as installed on tioga/tuolumne. Prefers the flavor matching the -# compiler family ('amd' for amdclang, 'cray' for cce). -setup_cray_mpi() { - local flavor="${1:-amd}" - local mpi_dir="${MPI_DIR_OPT:-${MPICH_DIR:-}}" - if [[ -z "${mpi_dir}" ]]; then - mpi_dir="$(newest_path "/opt/cray/pe/mpich/*/ofi/${flavor}/*")" - fi - [[ -n "${mpi_dir}" && -x "${mpi_dir}/bin/mpicc" ]] \ - || die "cray-mpich not found (tried '${mpi_dir:-/opt/cray/pe/mpich/*/ofi/${flavor}/*}'); load the cray-mpich module or pass --mpi " - add_arg -DENABLE_MPI=ON \ - -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ - -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ - -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ - -DMPIEXEC_EXECUTABLE="srun" \ - -DMPIEXEC_NUMPROC_FLAG="-n" - MPI_PREFIX="${mpi_dir}" -} - -# BLAS/LAPACK: openblas when available, reference blas/lapack otherwise. -setup_blas_lapack() { - local candidates=( "$@" ) lib - for lib in "${candidates[@]}"; do - if [[ -e "${lib}" ]]; then - add_arg -DBLAS_LIBRARIES="${lib}" -DLAPACK_LIBRARIES="${lib}" - return 0 - fi - done - if [[ -e /usr/lib64/libblas.so && -e /usr/lib64/liblapack.so ]]; then - add_arg -DBLAS_LIBRARIES="/usr/lib64/libblas.so" -DLAPACK_LIBRARIES="/usr/lib64/liblapack.so" - return 0 - fi - warn "no BLAS/LAPACK found in the usual locations, letting cmake search for it" -} - -# Resolves the toolchain of a configuration into cmake cache arguments. -select_config() { - local cfg="$1" - CMAKE_ARGS=() - - case "${cfg}" in - # --------------------------------------------------------------- z5tux --- - z5tux) - add_arg -DCMAKE_C_COMPILER=/usr/bin/gcc \ - -DCMAKE_CXX_COMPILER=/usr/bin/g++ \ - -DCMAKE_Fortran_COMPILER="${FC_OPT:-/usr/bin/gfortran}" \ - -DENABLE_OPENMP="${ENABLE_OPENMP:-ON}" - setup_z5tux_mpi - setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so - ;; - - z5tux-cuda) - add_arg -DCMAKE_C_COMPILER=/usr/bin/gcc \ - -DCMAKE_CXX_COMPILER=/usr/bin/g++ \ - -DCMAKE_Fortran_COMPILER="${FC_OPT:-/usr/bin/gfortran}" \ - -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - setup_z5tux_mpi - setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so - setup_cuda "$(resolve_cuda)" "${GPU_ARCH:-120}" - ;; - - z5tux-hip) - local rocm; rocm="$(resolve_rocm)" - setup_rocm_compilers "${rocm}" - add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - setup_z5tux_mpi - setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so - setup_hip "${rocm}" "${GPU_ARCH:-gfx1100}" - ;; - - # ------------------------------------------------------------------ LC --- - matrix) - - local gcc_dir mpi_dir - gcc_dir="$(newest_path '/usr/tce/packages/gcc/gcc-13*-magic')" - [[ -n "${gcc_dir}" ]] || gcc_dir="$(newest_path '/usr/tce/packages/gcc/gcc-*-magic')" - [[ -n "${gcc_dir}" ]] || die "no gcc found under /usr/tce/packages/gcc" - add_arg -DCMAKE_C_COMPILER="${gcc_dir}/bin/gcc" \ - -DCMAKE_CXX_COMPILER="${gcc_dir}/bin/g++" \ - -DCMAKE_Fortran_COMPILER="${FC_OPT:-${gcc_dir}/bin/gfortran}" \ - -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - - local gcc_ver - gcc_ver="$(basename "${gcc_dir}")"; gcc_ver="${gcc_ver#gcc-}"; gcc_ver="${gcc_ver%-magic}" - mpi_dir="${MPI_DIR_OPT:-$(newest_path "/usr/tce/packages/mvapich2/mvapich2-*-gcc-${gcc_ver}-magic")}" - [[ -n "${mpi_dir}" ]] || mpi_dir="$(newest_path '/usr/tce/packages/mvapich2/mvapich2-*-gcc-*-magic')" - [[ -n "${mpi_dir}" && -x "${mpi_dir}/bin/mpicc" ]] \ - || die "mvapich2 not found under /usr/tce/packages/mvapich2 (use --mpi )" - add_arg -DENABLE_MPI=ON \ - -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ - -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ - -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ - -DMPIEXEC_EXECUTABLE="srun" \ - -DMPIEXEC_NUMPROC_FLAG="-n" - MPI_PREFIX="${mpi_dir}" - - setup_blas_lapack /usr/lib64/libopenblas.so - setup_cuda "$(resolve_cuda)" "${GPU_ARCH:-90}" - ;; - - tioga|tioga-gpu) - local rocm; rocm="$(resolve_rocm)" - setup_rocm_compilers "${rocm}" - add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - setup_cray_mpi amd - setup_blas_lapack /usr/lib64/libopenblas.so - setup_hip "${rocm}" "${GPU_ARCH:-gfx90a}" - ;; - - tuo-cpu|tuolumne-cpu) - local rocm; rocm="$(resolve_rocm)" - setup_rocm_compilers "${rocm}" - add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-ON}" - setup_cray_mpi amd - setup_blas_lapack /usr/lib64/libopenblas.so - ;; - - tuo-gpu|tuolumne-gpu|tuolumne) - local rocm; rocm="$(resolve_rocm)" - setup_rocm_compilers "${rocm}" - add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - setup_cray_mpi amd - setup_blas_lapack /usr/lib64/libopenblas.so - setup_hip "${rocm}" "${GPU_ARCH:-gfx942}" - ;; - - *) - die "unknown configuration '${cfg}' (run --list to see the supported ones)" - ;; - esac -} - -# Scheduler command used by --alloc, per machine. -default_alloc_cmd() { - case "${CFG_MACHINE}" in - matrix) printf 'srun -N 1 --exclusive -t 180 -A %s' "${ACCOUNT}" ;; - tuolumne) printf 'srun -N 1 --exclusive -t 180 -A %s' "${ACCOUNT}" ;; - tioga) printf 'flux run -N 1 -t 180m' ;; - *) printf '' ;; - esac -} - -detect_config() { - local host="${HOSTNAME:-$(hostname)}" - case "${host}" in - z5tux*) printf 'z5tux' ;; - matrix*|rzmatrix*) printf 'matrix' ;; - tioga*|rzvernal*) printf 'tioga' ;; - tuolumne*|tuo[0-9]*) die "on tuolumne the configuration is ambiguous, pass 'tuo-cpu' or 'tuo-gpu'" ;; - *) die "cannot guess the configuration for host '${host}', pass one explicitly (--list)" ;; - esac -} - -# ------------------------------------------------------------------------------ -# Modules -# ------------------------------------------------------------------------------ -init_modules() { - if ! declare -F module >/dev/null 2>&1; then - local init - for init in /usr/share/lmod/lmod/init/bash /etc/profile.d/modules.sh /usr/share/Modules/init/bash; do - if [[ -r "${init}" ]]; then - # shellcheck disable=SC1090 - source "${init}" - break - fi - done - fi - declare -F module >/dev/null 2>&1 || type module >/dev/null 2>&1 -} - -load_modules() { - local -a mods=() - local m - if [[ -n "${MODULES_OPT}" ]]; then - read -r -a mods <<< "${MODULES_OPT}" - else - while IFS= read -r m; do - [[ -n "${m}" ]] && mods+=( "${m}" ) - done < <(modules_for_config "${CONFIG}") - fi - [[ ${#mods[@]} -gt 0 ]] || return 0 - - if ! init_modules; then - warn "no module command available, skipping module loading" - return 0 - fi - - local m - for m in "${mods[@]}"; do - if [[ ${DRY_RUN} -eq 1 ]]; then - log "+ module load ${m}" - continue - fi - if module load "${m}" >/dev/null 2>&1; then - log "module load ${m}" - else - # Retry without the version: LC renames those more often than the toolchains move. - local base="${m%%/*}" - if [[ "${base}" != "${m}" ]] && module load "${base}" >/dev/null 2>&1; then - log "module load ${base} (fallback, '${m}' unavailable)" - else - warn "could not load module '${m}' (continuing; run 'module avail ${base}' to check)" - fi - fi - done -} - -# ------------------------------------------------------------------------------ -# Build back-ends -# ------------------------------------------------------------------------------ -resolve_generator() { - case "${GENERATOR}" in - ninja) - command -v ninja >/dev/null 2>&1 || die "ninja not found in PATH (try --generator make)" - ;; - make) - ;; - auto) - if command -v ninja >/dev/null 2>&1; then - GENERATOR="ninja" - else - warn "ninja not found in PATH, falling back to Unix Makefiles" - GENERATOR="make" - fi - ;; - *) - die "unknown generator '${GENERATOR}' (auto|ninja|make)" - ;; - esac -} - -configure_cmake() { - local -a cmd=( cmake -S "${REPO_DIR}" -B "${BUILD_DIR}" ) - - if [[ "${GENERATOR}" == "ninja" ]]; then - cmd+=( -G Ninja -DENABLE_NINJA=ON ) - else - cmd+=( -G "Unix Makefiles" -DENABLE_NINJA=OFF ) - fi - - cmd+=( -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" - -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - -DNUM_PROC="${JOBS}" - -DENABLE_TRILINOS="${ENABLE_TRILINOS}" - -DENABLE_SUPERLU_DIST="${ENABLE_SUPERLU_DIST}" - -DENABLE_HYPRE_GPU_PROFILING="${ENABLE_HYPRE_GPU_PROFILING}" - -DBLT_CXX_STD=c++17 - -DCMAKE_CXX_STANDARD=17 ) - cmd+=( "${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}" ) - cmd+=( "${EXTRA_CMAKE_ARGS[@]+"${EXTRA_CMAKE_ARGS[@]}"}" ) - cmd+=( "${FORWARDED_ARGS[@]+"${FORWARDED_ARGS[@]}"}" ) - - run "${cmd[@]}" - write_build_env -} - -# The mpi wrappers pick their back-end compiler from the MPICH_* variables, and -# the TPLs configured outside of cmake (hypre) inherit them from this script. -# Dump them next to the build so that running ninja by hand stays equivalent. -write_build_env() { - [[ ${DRY_RUN} -eq 0 && -n "${MPICH_CC:-}" ]] || return 0 - local env_file="${BUILD_DIR}/build-env.sh" - cat > "${env_file}" </dev/null 2>&1 || die "cmake not found in PATH" - log "generator : ${GENERATOR}" - log "jobs : ${JOBS} per TPL, ${TOP_JOBS} TPL(s) at a time" - - if [[ ${DO_CONFIGURE} -eq 1 ]]; then - configure_cmake - else - [[ -d "${BUILD_DIR}" ]] || die "--build-only given but '${BUILD_DIR}' does not exist" - fi - - if [[ ${DO_BUILD} -eq 1 ]]; then - build_cmake - fi - fi - - log "done." - if [[ ${DO_BUILD} -eq 1 && ${DRY_RUN} -eq 0 ]]; then - log "TPLs installed in ${INSTALL_DIR}" - fi -} - -if [[ -n "${LOG_FILE}" ]]; then - main 2>&1 | tee "${LOG_FILE}" - exit "${PIPESTATUS[0]}" -else - main -fi From 497f78ef30d2dabdb44d7a2e7e6f3152b026174d Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 22:56:03 -0400 Subject: [PATCH 4/8] Remove blueos spack config --- .../blueos_3_ppc64le_ib_p9/spack.yaml | 233 ------------------ 1 file changed, 233 deletions(-) delete mode 100644 scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml diff --git a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml b/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml deleted file mode 100644 index 1c720f3e..00000000 --- a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml +++ /dev/null @@ -1,233 +0,0 @@ -#------------------------------------------------------------------------------------------------------------ -# SPDX-License-Identifier: LGPL-2.1-only -# -# Copyright (c) 2018-2020 Lawrence Livermore National Security LLC -# Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University -# Copyright (c) 2018-2020 TotalEnergies -# Copyright (c) 2019- GEOSX Contributors -# All rights reserved -# -# See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. -#------------------------------------------------------------------------------------------------------------ - -# geosx@develop%clang@10.0.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%gcc@8.3.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%clang@13.0.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%clang@13.0.1+cuda cuda_arch=70 ^cuda@12.2.2+allow-unsupported-compilers -# -# Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-10 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %gcc-8 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-13 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~openmp~uncrustify cuda_arch=70 %clang-13 ^cuda@12.2.2+allow-unsupported-compilers" - - -spack: - config: - install_tree: - root: $spack/.. - projections: - all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - misc_cache: $spack/../misc_cache - test_stage: $spack/../test_stage - build_stage:: - - $spack/../build_stage - - # Regular TPLs do not need views - view: false - - # Include shared variants and versions - include: - - ../defaults.yaml - - ../versions.yaml - - toolchains: - clang-13: - - spec: '%[virtuals=c]llvm@13.0.1+clang~flang~lld~lldb' - when: '%c' - - spec: '%[virtuals=cxx]llvm@13.0.1+clang~flang~lld~lldb' - when: '%cxx' - - spec: '%[virtuals=fortran]gcc@8.3.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.clang.13' - when: '%mpi' - clang-10: - - spec: '%[virtuals=c]llvm@10.0.1+clang~flang~lld~lldb' - when: '%c' - - spec: '%[virtuals=cxx]llvm@10.0.1+clang~flang~lld~lldb' - when: '%cxx' - - spec: '%[virtuals=fortran]gcc@8.3.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.clang.10' - when: '%mpi' - gcc-12: - - spec: '%c=gcc@12.1.1' - when: '%c' - - spec: '%cxx=gcc@12.1.1' - when: '%cxx' - - spec: '%fortran=gcc@12.1.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.gcc.12' - when: '%mpi' - gcc-8: - - spec: '%c=gcc@8.3.1 ' - when: '%c' - - spec: '%cxx=gcc@8.3.1 ' - when: '%cxx' - - spec: '%fortran=gcc@8.3.1 ' - when: '%fortran' - - spec: '%spectrum-mpi@release.gcc.8' - when: '%mpi' - - packages: - mpi: - require: - - spectrum-mpi - - zlib-api: - require: - - zlib - - blas: - require: - - essl - lapack: - require: - - essl - - llvm: - externals: - - spec: llvm@10.0.1+clang~flang~lld~lldb - prefix: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang++ - flags: {} - extra_rpaths: [] - - spec: llvm@13.0.1+clang~flang~lld~lldb - prefix: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang++ - flags: {} - extra_rpaths: [] - gcc: - externals: - - spec: gcc@8.3.1 languages:='c,c++,fortran' - prefix: /usr/tce/packages/gcc/gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ - fortran: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - extra_rpaths: [] - - spec: gcc@12.2.1 languages:='c,c++,fortran' - prefix: /usr/tce/packages/gcc/gcc-12.2.1 - extra_attributes: - compilers: - c: /usr/tce/packages/gcc/gcc-12.2.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-12.2.1/bin/g++ - fortran: /usr/tce/packages/gcc/gcc-12.2.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - extra_rpaths: [] - - spectrum-mpi: - buildable: False - externals: - - spec: spectrum-mpi@release.gcc.8 %gcc@8.3.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-8.3.1 - - spec: spectrum-mpi@release.gcc.12 %gcc@12.2.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-12.2.1 - # Previously clang@upstream - clingo doesn't like "upstream" version - - spec: spectrum-mpi@release.clang.10 %llvm@10.0.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-10.0.1-gcc-8.3.1 - - spec: spectrum-mpi@release.clang.13 %llvm@13.0.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-13.0.1-gcc-8.3.1 - - essl: - buildable: False - externals: - - spec: essl@6.3.0.2 - prefix: /usr/tcetmp/packages/essl/essl-6.3.0.2/ - - cuda: - buildable: False - externals: - - spec: cuda@11.8.0 +allow-unsupported-compilers - prefix: /usr/tce/packages/cuda/cuda-11.8.0 - - spec: cuda@12.2.2 +allow-unsupported-compilers - prefix: /usr/tce/packages/cuda/cuda-12.2.2 - - # System level packages to not build - cmake: - buildable: False - externals: - - spec: cmake@3.29.2 - prefix: /usr/tce/packages/cmake/cmake-3.29.2 - readline: - buildable: False - externals: - - spec: readline@7.0 - prefix: /collab/usr/gapps/python/build/spack-coralea.3/opt/spack/linux-rhel7-ppc64le/gcc-4.9.3/readline-7.0-owal6z2zh5zotgvdmwow6sgdn6cqfn43/ - m4: - buildable: False - externals: - - spec: m4@1.4.16 - prefix: /usr - perl: - buildable: false - externals: - - spec: perl@5.16.3 - prefix: /usr - pkg-config: - buildable: false - externals: - - spec: pkg-config@0.27.1 - prefix: /usr - diffutils: - buildable: False - externals: - - spec: diffutils@3.3 - prefix: /usr/bin/ - # This needs to be the prefix to the python3 LC installation - # or pygeosx package built by spack/uberenv - python: - buildable: False - externals: - - spec: python@3.8.2 - prefix: /usr/gapps/GEOSX/thirdPartyLibs/python/lassen-gcc-python/python/ - autoconf: - buildable: False - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - buildable: False - externals: - - spec: automake@1.13.4 - prefix: /usr - libtool: - buildable: False - externals: - - spec: libtool@2.4.2 - prefix: /usr - gettext: - buildable: False - externals: - - spec: gettext@0.19.8.1 - prefix: /usr/bin/ - zlib: - buildable: false - externals: - - spec: zlib@1.2.7 - prefix: /usr From ee8eafdf169c1d3c8e975755d0015bde0ced3ec1 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 23:04:09 -0400 Subject: [PATCH 5/8] Remove Lassen and ESSL support --- CMakeLists.txt | 5 --- cmake/thirdparty/FindMathLibraries.cmake | 9 ---- scripts/createLapackForESSL.sh | 33 --------------- .../spack_packages/packages/essl/package.py | 41 ------------------- .../spack_packages/packages/geosx/package.py | 30 +------------- 5 files changed, 1 insertion(+), 117 deletions(-) delete mode 100755 scripts/createLapackForESSL.sh delete mode 100644 scripts/spack_packages/packages/essl/package.py diff --git a/CMakeLists.txt b/CMakeLists.txt index dc910be1..379a583f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -666,11 +666,6 @@ if (ENABLE_TRILINOS) -D TPL_MKL_LIBRARIES:STRING=${MKL_LIBRARIES}) endif() - if ( ENABLE_ESSL ) - set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS} - -D HAVE_dggsvd3:BOOL=ON) - endif() - if ( DEFINED OpenMP_Fortran_FLAGS ) set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS} -D OpenMP_Fortran_FLAGS:STRING=${OpenMP_Fortran_FLAGS}) diff --git a/cmake/thirdparty/FindMathLibraries.cmake b/cmake/thirdparty/FindMathLibraries.cmake index 340c3e1a..3c6e13c9 100644 --- a/cmake/thirdparty/FindMathLibraries.cmake +++ b/cmake/thirdparty/FindMathLibraries.cmake @@ -12,15 +12,6 @@ if( ENABLE_MKL ) set( BLAS_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE ) set( LAPACK_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE) -elseif( ENABLE_ESSL ) - if ( NOT DEFINED ESSL_INCLUDE_DIRS ) - message( FATAL_ERROR "ESSL is enabled but ESSL_INCLUDE_DIRS is not defined." ) - elseif ( NOT DEFINED ESSL_LIBRARIES ) - message( FATAL_ERROR "ESSL is enabled but ESSL_LIBRARIES is not defined." ) - endif() - - set( BLAS_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE ) - set( LAPACK_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE ) endif() if( NOT DEFINED BLAS_LIBRARIES ) diff --git a/scripts/createLapackForESSL.sh b/scripts/createLapackForESSL.sh deleted file mode 100755 index 59b09ca9..00000000 --- a/scripts/createLapackForESSL.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# Adapted from IBM Engineering and Scientific Subroutine Library for Linux on POWER Version 6.2 ESSL Guide and Reference -# -# First Pass - get the symbols in ESSL and LAPACK -echo "The essl library is: " $1 -echo "The lapack library is: " $2 -echo "The output library is: liblapackforessl.a" -nm $2 | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.lapack -nm $1 | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.essl -diff -y symbols.in.lapack symbols.in.essl | grep -v -e '>' -e '|' -e '<' | awk '{ print $1 }' > duplicate.symbols -wc -l duplicate.symbols - -# Second Pass - remove the ESSL symbols from the LAPACK library -cp -p $2 liblapackforessl.a - -for SYMBOL in $(cat duplicate.symbols | awk '{ print $1 }') -do - fn=$SYMBOL - unders="no" - echo $SYMBOL | grep "_" > /dev/null && unders="yes" - if [ "$unders" = "yes" ] - then - fn=`echo $SYMBOL | awk -F'_' '{print $1}'` - fi - - echo $SYMBOL - ar d liblapackforessl.a $fn.o -done - -# Check it worked -nm liblapackforessl.a | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.updatedlapack -diff -y symbols.in.updatedlapack symbols.in.essl | grep -v -e '>' -e '|' -e '<' | awk '{ print $1 }' > duplicate.symbols.inupdatedlapack -wc -l duplicate.symbols.inupdatedlapack diff --git a/scripts/spack_packages/packages/essl/package.py b/scripts/spack_packages/packages/essl/package.py deleted file mode 100644 index fa2a2cb7..00000000 --- a/scripts/spack_packages/packages/essl/package.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack.package import * -from spack_repo.builtin.build_systems.bundle import BundlePackage - -# Recipe for pre-built essl library on blueos machines. -# Defines additonal flags for blueos: -# https://lc.llnl.gov/confluence/display/SIERRA/Math+Libraries -class Essl(BundlePackage): - """IBM's Engineering and Scientific Subroutine Library (ESSL).""" - - homepage = "https://www.ibm.com/systems/power/software/essl/" - - version("6.3.0.2") - - provides("blas") - provides("lapack") - - @property - def blas_libs(self): - spec = self.spec - prefix = self.prefix - - essl_root = prefix.lib64 - essl_libs = ["libessl", "liblapackforessl", "liblapackforessl_"] - all_libs = find_libraries(essl_libs, root=essl_root, shared=True) - - return all_libs - - @property - def lapack_libs(self): - spec = self.spec - prefix = self.prefix - - essl_root = prefix.lib64 - essl_libs = ["libessl", "liblapackforessl", "liblapackforessl_"] - all_libs = find_libraries(essl_libs, root=essl_root, shared=True) - - return all_libs diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index eaa469de..d75a4ccc 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -356,9 +356,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): debug_flags = "-g" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) - if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) - cfg.write("#{0}\n".format("-" * 80)) cfg.write("# CMake Standard\n") cfg.write("#{0}\n\n".format("-" * 80)) @@ -375,16 +372,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): hostname = socket.gethostname().rstrip('1234567890') - if sys_type in ('linux-rhel7-ppc64le', 'linux-rhel8-ppc64le', 'blueos_3_ppc64le_ib_p9') \ - and hostname != 'p3dev': - cfg.write(cmake_cache_option('ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC', True)) - if hostname in ('lassen', 'rzansel'): - cfg.write(cmake_cache_entry('MPIEXEC', 'lrun')) - cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) - else: - cfg.write(cmake_cache_entry('MPIEXEC', 'jsrun')) - cfg.write(cmake_cache_list('MPIEXEC_NUMPROC_FLAG', ['-g1', '--bind', 'rs', '-n'])) - elif sys_type in ('toss_4_x86_64_ib_cray'): + if sys_type in ('toss_4_x86_64_ib_cray'): cfg.write(cmake_cache_entry('MPIEXEC', 'srun')) cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) else: @@ -470,10 +458,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS', cmake_cuda_flags)) - # System specific flags - if sys_type in ('linux-rhel7-ppc64le', 'linux-rhel8-ppc64le', 'blueos_3_ppc64le_ib_p9'): - cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELEASE', '-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -Xcompiler -mcpu=powerpc64le -Xcompiler -mtune=powerpc64le')) - cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELWITHDEBINFO', '-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}')) cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_DEBUG', '-g -G -O0 -Xcompiler -O0')) @@ -568,12 +552,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('ENABLE_MKL', True)) cfg.write(cmake_cache_entry('MKL_INCLUDE_DIRS', spec['intel-mkl'].prefix.include)) cfg.write(cmake_cache_list('MKL_LIBRARIES', spec['intel-mkl'].libs)) - elif spec["blas"].name == "essl": - cfg.write(cmake_cache_option('ENABLE_ESSL', True)) - cfg.write(cmake_cache_entry('ESSL_INCLUDE_DIRS', spec['essl'].prefix.include)) - cfg.write(cmake_cache_list('ESSL_LIBRARIES', spec['blas'].libs)) - - cfg.write(cmake_cache_option('FORTRAN_MANGLE_NO_UNDERSCORE', True)) else: cfg.write(cmake_cache_list('BLAS_LIBRARIES', spec['blas'].libs)) cfg.write(cmake_cache_list('LAPACK_LIBRARIES', spec['lapack'].libs)) @@ -689,9 +667,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('GEOS_BUILD_SHARED_LIBS', False)) # ATS - # Lassen - if sys_type in ('blueos_3_ppc64le_ib_p9'): - cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--ats jsrun_omp --ats jsrun_bind=packed')) # Dane/Matrix if sys_type in ('toss_4_x86_64_ib'): cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--machine slurm112')) @@ -769,9 +744,6 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): debug_flags = "-g" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) - if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) - cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Cuda\n') cfg.write('#{0}\n\n'.format('-' * 80)) From 8bb038e041012f4163562791028fbf126fd426fd Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 23:04:25 -0400 Subject: [PATCH 6/8] Use the new spack installer on the LC machines --- scripts/spack_configs/toss_4_x86_64_ib/spack.yaml | 2 ++ scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index 8b29be18..e6b12357 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -39,6 +39,8 @@ spack: test_stage: $spack/../test_stage build_stage: $spack/../build_stage build_jobs: 112 + # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. + installer: new # Regular TPLs do not need views view: false diff --git a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml index 76404e3a..48af6ee7 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml @@ -31,6 +31,8 @@ spack: test_stage: $spack/../test_stage build_stage: $spack/../build_stage build_jobs: 96 + # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. + installer: new # Regular TPLs do not need views view: false From bc978545b488f7f30dd967433c4566ba3e7a74c0 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 23:12:47 -0400 Subject: [PATCH 7/8] Disable Trilinos by default --- CMakeLists.txt | 2 +- scripts/spack_packages/packages/geosx/package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 379a583f..dbbe214d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,7 +47,7 @@ option( BUILD_SHARED_LIBS "" OFF ) option( ENABLE_CALIPER "" ON ) option( ENABLE_HYPRE "" ON ) -option( ENABLE_TRILINOS "" ON ) +option( ENABLE_TRILINOS "" OFF ) option( ENABLE_PETSC "" OFF ) option( ENABLE_SCOTCH "" ON ) option( ENABLE_SUITESPARSE "" ON ) diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index d75a4ccc..5bd2a7ae 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -64,7 +64,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): variant('shared', default=True, description='Build Shared Libs.') variant('caliper', default=True, description='Build Caliper support.') variant('vtk', default=True, description='Build VTK support.') - variant('trilinos', default=True, description='Build Trilinos support.') + variant('trilinos', default=False, description='Build Trilinos support.') variant('hypre', default=True, description='Build HYPRE support.') variant('hypredrive', default=True, description='Build hypredrive support.') variant('petsc', default=False, description='Build PETSc support.') From 7ac30d4b5f8453db87410455826de0b6c30e8d6b Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 5 Aug 2026 03:01:45 -0400 Subject: [PATCH 8/8] Install xz-devel in the Rocky TPL image --- docker/tpl-rockylinux.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/tpl-rockylinux.Dockerfile b/docker/tpl-rockylinux.Dockerfile index 27cbe2bb..ac39daf7 100644 --- a/docker/tpl-rockylinux.Dockerfile +++ b/docker/tpl-rockylinux.Dockerfile @@ -41,7 +41,8 @@ RUN dnf clean all && \ bzip2 \ gnupg2 \ perl \ - xz && \ + xz \ + xz-devel && \ (dnf -y install python3-virtualenv || \ /usr/bin/python3 -m pip install --no-cache-dir virtualenv) && \ dnf clean all && rm -rf /var/cache/dnf /var/lib/dnf