diff --git a/.gitignore b/.gitignore index 704d7cdd..488ea52a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .project .pydevproject .vscode/ +spack/ spack-*.txt __pycache__ uberenv_* diff --git a/.uberenv_config.json b/.uberenv_config.json index b541fb7c..11d6a5ee 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -6,8 +6,8 @@ "spack_configs_path": "scripts/spack_configs", "spack_packages_path": "scripts/spack_packages/packages", "spack_url": "https://github.com/spack/spack", -"spack_commit": "2e2169d5282d166f63e3ee4db8d4446c43cefa8a", -"spack_commit_note": "v1.1.1 (Jan 14th 2026)", -"spack_packages_commit": "3dd98680871078353a28ee508fa76c7554f918fa", -"spack_packages_note": "Feb 25th 2026" +"spack_commit": "3e19345b6e12f5ff1b874f4059622fc6a1fd804a", +"spack_commit_note": "v1.2.2 (Jul 20th 2026)", +"spack_packages_commit": "bd9b4838fbf9e4ca01ad7b6c6df633ddcc750d71", +"spack_packages_note": "Jul 20th 2026" } diff --git a/CMakeLists.txt b/CMakeLists.txt index 321f3ca0..dbbe214d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,10 +47,12 @@ option( BUILD_SHARED_LIBS "" OFF ) option( ENABLE_CALIPER "" ON ) option( ENABLE_HYPRE "" ON ) -option( ENABLE_TRILINOS "" ON ) +option( ENABLE_TRILINOS "" OFF ) option( ENABLE_PETSC "" OFF ) option( ENABLE_SCOTCH "" ON ) option( ENABLE_SUITESPARSE "" ON ) +option( ENABLE_SUPERLU_DIST "" ON ) +option( ENABLE_HYPRE_GPU_PROFILING "" OFF ) option( ENABLE_VTK "" ON ) option( ENABLE_VTK_WRAP_PYTHON "" OFF ) @@ -125,6 +127,50 @@ else( TRUE ) endif() +################################ +# HIP / ROCm +################################ +if( ENABLE_HIP ) + if( NOT DEFINED ROCM_PATH ) + if( DEFINED HIP_ROOT ) + set( ROCM_PATH ${HIP_ROOT} ) + elseif( DEFINED ENV{ROCM_PATH} ) + set( ROCM_PATH $ENV{ROCM_PATH} ) + else() + message( FATAL_ERROR "ENABLE_HIP is ON but neither ROCM_PATH nor HIP_ROOT is defined." ) + endif() + endif() + + if( NOT DEFINED HIP_ROOT ) + set( HIP_ROOT ${ROCM_PATH} ) + endif() + + if( NOT DEFINED CMAKE_HIP_ARCHITECTURES ) + message( FATAL_ERROR "ENABLE_HIP is ON but CMAKE_HIP_ARCHITECTURES is not defined (e.g. gfx90a, gfx942, gfx1100)." ) + endif() + + # hypre is configured outside of cmake, so it needs the MPI include directory explicitly + # when its sources are compiled for the device. + if( ENABLE_MPI AND NOT DEFINED MPI_INCLUDE_DIR ) + if( DEFINED ENV{MPICH_DIR} ) # set by the cray-mpich module on LC + set( MPI_INCLUDE_DIR "$ENV{MPICH_DIR}/include" ) + elseif( DEFINED ENV{MPI_ROOT} ) + set( MPI_INCLUDE_DIR "$ENV{MPI_ROOT}/include" ) + elseif( MPI_C_INCLUDE_DIRS ) # set by find_package( MPI ) through BLT + list( GET MPI_C_INCLUDE_DIRS 0 MPI_INCLUDE_DIR ) + elseif( MPI_C_COMPILER ) # /bin/mpicc -> /include + get_filename_component( MPI_BIN_DIR ${MPI_C_COMPILER} DIRECTORY ) + get_filename_component( MPI_PREFIX_DIR ${MPI_BIN_DIR} DIRECTORY ) + set( MPI_INCLUDE_DIR "${MPI_PREFIX_DIR}/include" ) + else() + message( FATAL_ERROR "ENABLE_HIP is ON but MPI_INCLUDE_DIR is not defined and cannot be deduced." ) + endif() + message( STATUS "MPI_INCLUDE_DIR = ${MPI_INCLUDE_DIR}" ) + endif() + + message( STATUS "ROCM_PATH = ${ROCM_PATH}" ) + message( STATUS "CMAKE_HIP_ARCHITECTURES = ${CMAKE_HIP_ARCHITECTURES}" ) +endif() ################################ @@ -340,6 +386,13 @@ set(RAJA_URL "${TPL_MIRROR_DIR}/RAJA-v2025.12.0.tar.gz") message(STATUS "Building RAJA found at ${RAJA_URL}") +# RAJA's vectorization layer does not compile with the ROCm toolchain. +if( ENABLE_HIP ) + set( RAJA_ENABLE_VECTORIZATION OFF ) +else() + set( RAJA_ENABLE_VECTORIZATION ON ) +endif() + ExternalProject_Add( raja URL ${RAJA_URL} PREFIX ${PROJECT_BINARY_DIR}/raja @@ -360,10 +413,18 @@ ExternalProject_Add( raja -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CMAKE_CUDA_FLAGS:STRING=${RAJA_CUDA_FLAGS} + -D ENABLE_HIP:BOOL=${ENABLE_HIP} + -D RAJA_ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER} + -D RAJA_ENABLE_VECTORIZATION:BOOL=${RAJA_ENABLE_VECTORIZATION} + -D hip_DIR=${HIP_ROOT}/lib/cmake/hip + -D ROCPRIM_DIR=${HIP_ROOT} + -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} -D ENABLE_TESTS:BOOL=OFF -D ENABLE_EXAMPLES:BOOL=OFF -D ENABLE_OPENMP:BOOL=${ENABLE_OPENMP} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} ) @@ -398,8 +459,14 @@ ExternalProject_Add( chai -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CMAKE_CUDA_FLAGS:STRING=${CHAI_CUDA_FLAGS} + -D ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CHAI_ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER} + -D hip_DIR=${HIP_ROOT}/lib/cmake/hip + -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} -D ENABLE_OPENMP=${ENABLE_OPENMP} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -D ENABLE_TESTS=OFF -D ENABLE_BENCHMARKS=OFF @@ -444,6 +511,7 @@ if (ENABLE_CALIPER) -D MPI_CXX_COMPILER:STRING=${MPI_CXX_COMPILER} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BLT_SOURCE_DIR=${BLT_SOURCE_DIR} ) @@ -480,14 +548,23 @@ if (ENABLE_CALIPER) -D CUDA_TOOLKIT_ROOT_DIR:PATH=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D WITH_PAPI:BOOL=${ENABLE_PAPI} -D PAPI_PREFIX:STRING=${PAPI_PREFIX} -D WITH_ADIAK:BOOL=ON -D adiak_DIR:STRING=${ADIAK_DIR}/lib/cmake/adiak/ -D WITH_CUPTI=${CALIPER_WITH_CUPTI} -D WITH_NVPROF=${ENABLE_CUDA} + -D WITH_ROCM=${ENABLE_ROCTX} + -D ROCM_PREFIX:PATH=${ROCM_PATH} ) + if( BUILD_SHARED_LIBS ) + set( CALIPER_LIB "libcaliper.so" ) + else() + set( CALIPER_LIB "libcaliper.a" ) + endif() + list(APPEND build_list caliper adiak) else() set( ENABLE_CALIPER_HYPRE OFF CACHE BOOL "" FORCE ) @@ -589,11 +666,6 @@ if (ENABLE_TRILINOS) -D TPL_MKL_LIBRARIES:STRING=${MKL_LIBRARIES}) endif() - if ( ENABLE_ESSL ) - set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS} - -D HAVE_dggsvd3:BOOL=ON) - endif() - if ( DEFINED OpenMP_Fortran_FLAGS ) set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS} -D OpenMP_Fortran_FLAGS:STRING=${OpenMP_Fortran_FLAGS}) @@ -737,7 +809,7 @@ list(APPEND build_list metis ) ################################ # SUPERLU -- ################################ -if( ENABLE_MPI ) +if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) set(SUPERLU_DIR "${CMAKE_INSTALL_PREFIX}/superlu_dist") set(SUPERLU_URL "${TPL_MIRROR_DIR}/superlu_dist-0f6efc3.tar.gz") @@ -784,6 +856,7 @@ if( ENABLE_MPI ) -D CMAKE_Fortran_FLAGS_RELEASE=${CMAKE_Fortran_FLAGS_RELEASE} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D enable_openmp=${ENABLE_OPENMP} ) @@ -814,6 +887,9 @@ if( ENABLE_HYPRE ) set( HYPRE_C_COMPILER ${MPI_C_COMPILER} ) set( HYPRE_CXX_COMPILER ${MPI_CXX_COMPILER} ) set( HYPRE_Fortran_COMPILER ${MPI_Fortran_COMPILER} ) + endif() + + if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) set( HYPRE_SUPERLU_LIBS "${SUPERLU_DIR}/lib/libsuperlu_dist.a ${PARMETIS_DIR}/lib/libparmetis.a ${METIS_DIR}/lib/libmetis.a -lstdc++" ) if( ENABLE_OPENMP ) set( HYPRE_SUPERLU_LIBS "${HYPRE_SUPERLU_LIBS} ${OpenMP_C_FLAGS}" ) # Sergey: why is this needed? @@ -844,9 +920,41 @@ if( ENABLE_HYPRE ) --with-umpire-include=${CHAI_DIR}/include \ --with-umpire-lib-dirs=${CHAI_DIR}/lib \ --with-umpire-libs=umpire " ) + + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-aware-mpi" ) + endif() + + if( ENABLE_HYPRE_GPU_PROFILING ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-profiling" ) + endif() endif() elseif( ENABLE_HYPRE_DEVICE STREQUAL "HIP" ) - message( FATAL_ERROR "hip tpls build only supported through spack" ) + if( NOT ENABLE_HIP ) + message( FATAL_ERROR "Non-compatible options: HYPRE device cannot be set to HIP when ENABLE_HIP is OFF" ) + else() + list( APPEND HYPRE_DEPENDS chai ) + set( HYPRE_HIP_FLAGS "\ +ROCM_PATH=${ROCM_PATH} \ +--with-hip \ +--enable-rocblas \ +--enable-rocsparse \ +--enable-rocsolver \ +--with-gpu-arch=${CMAKE_HIP_ARCHITECTURES} \ +--with-umpire-device \ +--with-umpire-include=${CHAI_DIR}/include \ +--with-umpire-lib-dirs=${CHAI_DIR}/lib \ +--with-umpire-libs=\"umpire camp\" \ +--with-MPI-include=${MPI_INCLUDE_DIR} " ) + + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-aware-mpi" ) + endif() + + if( ENABLE_HYPRE_GPU_PROFILING ) + set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-profiling" ) + endif() + endif() endif() if( CMAKE_HOST_APPLE ) @@ -856,7 +964,7 @@ if( ENABLE_HYPRE ) endif() if( ENABLE_CALIPER_HYPRE ) - set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/libs/caliper.a\"" ) + set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/lib/${CALIPER_LIB}\"" ) list( APPEND HYPRE_DEPENDS caliper ) endif() @@ -881,6 +989,7 @@ if( ENABLE_HYPRE ) ${HYPRE_SUPERLU_FLAGS} \ ${HYPRE_OPENMP_FLAG} \ ${HYPRE_CUDA_FLAGS} \ + ${HYPRE_HIP_FLAGS} \ ${HYPRE_CALIPER_FLAGS}\n") ExternalProject_Add( hypre diff --git a/cmake/thirdparty/FindMathLibraries.cmake b/cmake/thirdparty/FindMathLibraries.cmake index 340c3e1a..3c6e13c9 100644 --- a/cmake/thirdparty/FindMathLibraries.cmake +++ b/cmake/thirdparty/FindMathLibraries.cmake @@ -12,15 +12,6 @@ if( ENABLE_MKL ) set( BLAS_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE ) set( LAPACK_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE) -elseif( ENABLE_ESSL ) - if ( NOT DEFINED ESSL_INCLUDE_DIRS ) - message( FATAL_ERROR "ESSL is enabled but ESSL_INCLUDE_DIRS is not defined." ) - elseif ( NOT DEFINED ESSL_LIBRARIES ) - message( FATAL_ERROR "ESSL is enabled but ESSL_LIBRARIES is not defined." ) - endif() - - set( BLAS_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE ) - set( LAPACK_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE ) endif() if( NOT DEFINED BLAS_LIBRARIES ) diff --git a/docker/pecan-spack.yaml b/docker/pecan-spack.yaml index 89ca8678..300cef98 100644 --- a/docker/pecan-spack.yaml +++ b/docker/pecan-spack.yaml @@ -111,6 +111,8 @@ spack: externals: - spec: py-sphinx@4.3.2 prefix: /usr + dependencies: + - spec: python@3.10.12 python: buildable: false externals: diff --git a/docker/tpl-rockylinux.Dockerfile b/docker/tpl-rockylinux.Dockerfile index 27cbe2bb..ac39daf7 100644 --- a/docker/tpl-rockylinux.Dockerfile +++ b/docker/tpl-rockylinux.Dockerfile @@ -41,7 +41,8 @@ RUN dnf clean all && \ bzip2 \ gnupg2 \ perl \ - xz && \ + xz \ + xz-devel && \ (dnf -y install python3-virtualenv || \ /usr/bin/python3 -m pip install --no-cache-dir virtualenv) && \ dnf clean all && rm -rf /var/cache/dnf /var/lib/dnf diff --git a/docker/ubuntu-spack.yaml b/docker/ubuntu-spack.yaml index e27514bf..40915d04 100644 --- a/docker/ubuntu-spack.yaml +++ b/docker/ubuntu-spack.yaml @@ -275,6 +275,8 @@ spack: externals: - spec: py-sphinx@7.4.7 prefix: /usr + dependencies: + - spec: python@3.12 python: buildable: false externals: diff --git a/scripts/createLapackForESSL.sh b/scripts/createLapackForESSL.sh deleted file mode 100755 index 59b09ca9..00000000 --- a/scripts/createLapackForESSL.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# Adapted from IBM Engineering and Scientific Subroutine Library for Linux on POWER Version 6.2 ESSL Guide and Reference -# -# First Pass - get the symbols in ESSL and LAPACK -echo "The essl library is: " $1 -echo "The lapack library is: " $2 -echo "The output library is: liblapackforessl.a" -nm $2 | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.lapack -nm $1 | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.essl -diff -y symbols.in.lapack symbols.in.essl | grep -v -e '>' -e '|' -e '<' | awk '{ print $1 }' > duplicate.symbols -wc -l duplicate.symbols - -# Second Pass - remove the ESSL symbols from the LAPACK library -cp -p $2 liblapackforessl.a - -for SYMBOL in $(cat duplicate.symbols | awk '{ print $1 }') -do - fn=$SYMBOL - unders="no" - echo $SYMBOL | grep "_" > /dev/null && unders="yes" - if [ "$unders" = "yes" ] - then - fn=`echo $SYMBOL | awk -F'_' '{print $1}'` - fi - - echo $SYMBOL - ar d liblapackforessl.a $fn.o -done - -# Check it worked -nm liblapackforessl.a | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.updatedlapack -diff -y symbols.in.updatedlapack symbols.in.essl | grep -v -e '>' -e '|' -e '<' | awk '{ print $1 }' > duplicate.symbols.inupdatedlapack -wc -l duplicate.symbols.inupdatedlapack diff --git a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml b/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml deleted file mode 100644 index 1c720f3e..00000000 --- a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml +++ /dev/null @@ -1,233 +0,0 @@ -#------------------------------------------------------------------------------------------------------------ -# SPDX-License-Identifier: LGPL-2.1-only -# -# Copyright (c) 2018-2020 Lawrence Livermore National Security LLC -# Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University -# Copyright (c) 2018-2020 TotalEnergies -# Copyright (c) 2019- GEOSX Contributors -# All rights reserved -# -# See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. -#------------------------------------------------------------------------------------------------------------ - -# geosx@develop%clang@10.0.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%gcc@8.3.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%clang@13.0.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%clang@13.0.1+cuda cuda_arch=70 ^cuda@12.2.2+allow-unsupported-compilers -# -# Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-10 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %gcc-8 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-13 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~openmp~uncrustify cuda_arch=70 %clang-13 ^cuda@12.2.2+allow-unsupported-compilers" - - -spack: - config: - install_tree: - root: $spack/.. - projections: - all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - misc_cache: $spack/../misc_cache - test_stage: $spack/../test_stage - build_stage:: - - $spack/../build_stage - - # Regular TPLs do not need views - view: false - - # Include shared variants and versions - include: - - ../defaults.yaml - - ../versions.yaml - - toolchains: - clang-13: - - spec: '%[virtuals=c]llvm@13.0.1+clang~flang~lld~lldb' - when: '%c' - - spec: '%[virtuals=cxx]llvm@13.0.1+clang~flang~lld~lldb' - when: '%cxx' - - spec: '%[virtuals=fortran]gcc@8.3.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.clang.13' - when: '%mpi' - clang-10: - - spec: '%[virtuals=c]llvm@10.0.1+clang~flang~lld~lldb' - when: '%c' - - spec: '%[virtuals=cxx]llvm@10.0.1+clang~flang~lld~lldb' - when: '%cxx' - - spec: '%[virtuals=fortran]gcc@8.3.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.clang.10' - when: '%mpi' - gcc-12: - - spec: '%c=gcc@12.1.1' - when: '%c' - - spec: '%cxx=gcc@12.1.1' - when: '%cxx' - - spec: '%fortran=gcc@12.1.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.gcc.12' - when: '%mpi' - gcc-8: - - spec: '%c=gcc@8.3.1 ' - when: '%c' - - spec: '%cxx=gcc@8.3.1 ' - when: '%cxx' - - spec: '%fortran=gcc@8.3.1 ' - when: '%fortran' - - spec: '%spectrum-mpi@release.gcc.8' - when: '%mpi' - - packages: - mpi: - require: - - spectrum-mpi - - zlib-api: - require: - - zlib - - blas: - require: - - essl - lapack: - require: - - essl - - llvm: - externals: - - spec: llvm@10.0.1+clang~flang~lld~lldb - prefix: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang++ - flags: {} - extra_rpaths: [] - - spec: llvm@13.0.1+clang~flang~lld~lldb - prefix: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang++ - flags: {} - extra_rpaths: [] - gcc: - externals: - - spec: gcc@8.3.1 languages:='c,c++,fortran' - prefix: /usr/tce/packages/gcc/gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ - fortran: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - extra_rpaths: [] - - spec: gcc@12.2.1 languages:='c,c++,fortran' - prefix: /usr/tce/packages/gcc/gcc-12.2.1 - extra_attributes: - compilers: - c: /usr/tce/packages/gcc/gcc-12.2.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-12.2.1/bin/g++ - fortran: /usr/tce/packages/gcc/gcc-12.2.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - extra_rpaths: [] - - spectrum-mpi: - buildable: False - externals: - - spec: spectrum-mpi@release.gcc.8 %gcc@8.3.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-8.3.1 - - spec: spectrum-mpi@release.gcc.12 %gcc@12.2.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-12.2.1 - # Previously clang@upstream - clingo doesn't like "upstream" version - - spec: spectrum-mpi@release.clang.10 %llvm@10.0.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-10.0.1-gcc-8.3.1 - - spec: spectrum-mpi@release.clang.13 %llvm@13.0.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-13.0.1-gcc-8.3.1 - - essl: - buildable: False - externals: - - spec: essl@6.3.0.2 - prefix: /usr/tcetmp/packages/essl/essl-6.3.0.2/ - - cuda: - buildable: False - externals: - - spec: cuda@11.8.0 +allow-unsupported-compilers - prefix: /usr/tce/packages/cuda/cuda-11.8.0 - - spec: cuda@12.2.2 +allow-unsupported-compilers - prefix: /usr/tce/packages/cuda/cuda-12.2.2 - - # System level packages to not build - cmake: - buildable: False - externals: - - spec: cmake@3.29.2 - prefix: /usr/tce/packages/cmake/cmake-3.29.2 - readline: - buildable: False - externals: - - spec: readline@7.0 - prefix: /collab/usr/gapps/python/build/spack-coralea.3/opt/spack/linux-rhel7-ppc64le/gcc-4.9.3/readline-7.0-owal6z2zh5zotgvdmwow6sgdn6cqfn43/ - m4: - buildable: False - externals: - - spec: m4@1.4.16 - prefix: /usr - perl: - buildable: false - externals: - - spec: perl@5.16.3 - prefix: /usr - pkg-config: - buildable: false - externals: - - spec: pkg-config@0.27.1 - prefix: /usr - diffutils: - buildable: False - externals: - - spec: diffutils@3.3 - prefix: /usr/bin/ - # This needs to be the prefix to the python3 LC installation - # or pygeosx package built by spack/uberenv - python: - buildable: False - externals: - - spec: python@3.8.2 - prefix: /usr/gapps/GEOSX/thirdPartyLibs/python/lassen-gcc-python/python/ - autoconf: - buildable: False - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - buildable: False - externals: - - spec: automake@1.13.4 - prefix: /usr - libtool: - buildable: False - externals: - - spec: libtool@2.4.2 - prefix: /usr - gettext: - buildable: False - externals: - - spec: gettext@0.19.8.1 - prefix: /usr/bin/ - zlib: - buildable: false - externals: - - spec: zlib@1.2.7 - prefix: /usr diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index d0ffeed9..e6b12357 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -39,6 +39,8 @@ spack: test_stage: $spack/../test_stage build_stage: $spack/../build_stage build_jobs: 112 + # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. + installer: new # Regular TPLs do not need views view: false @@ -231,6 +233,8 @@ spack: externals: - spec: py-sphinx@1.6.3 prefix: /usr/gapps/GEOSX/thirdPartyLibs/python/quartz-gcc-python/python/ + dependencies: + - spec: python@3.10.8 autoconf: buildable: False externals: diff --git a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml index 76404e3a..48af6ee7 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml @@ -31,6 +31,8 @@ spack: test_stage: $spack/../test_stage build_stage: $spack/../build_stage build_jobs: 96 + # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. + installer: new # Regular TPLs do not need views view: false diff --git a/scripts/spack_packages/packages/essl/package.py b/scripts/spack_packages/packages/essl/package.py deleted file mode 100644 index fa2a2cb7..00000000 --- a/scripts/spack_packages/packages/essl/package.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack.package import * -from spack_repo.builtin.build_systems.bundle import BundlePackage - -# Recipe for pre-built essl library on blueos machines. -# Defines additonal flags for blueos: -# https://lc.llnl.gov/confluence/display/SIERRA/Math+Libraries -class Essl(BundlePackage): - """IBM's Engineering and Scientific Subroutine Library (ESSL).""" - - homepage = "https://www.ibm.com/systems/power/software/essl/" - - version("6.3.0.2") - - provides("blas") - provides("lapack") - - @property - def blas_libs(self): - spec = self.spec - prefix = self.prefix - - essl_root = prefix.lib64 - essl_libs = ["libessl", "liblapackforessl", "liblapackforessl_"] - all_libs = find_libraries(essl_libs, root=essl_root, shared=True) - - return all_libs - - @property - def lapack_libs(self): - spec = self.spec - prefix = self.prefix - - essl_root = prefix.lib64 - essl_libs = ["libessl", "liblapackforessl", "liblapackforessl_"] - all_libs = find_libraries(essl_libs, root=essl_root, shared=True) - - return all_libs diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index eaa469de..5bd2a7ae 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -64,7 +64,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): variant('shared', default=True, description='Build Shared Libs.') variant('caliper', default=True, description='Build Caliper support.') variant('vtk', default=True, description='Build VTK support.') - variant('trilinos', default=True, description='Build Trilinos support.') + variant('trilinos', default=False, description='Build Trilinos support.') variant('hypre', default=True, description='Build HYPRE support.') variant('hypredrive', default=True, description='Build hypredrive support.') variant('petsc', default=False, description='Build PETSc support.') @@ -356,9 +356,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): debug_flags = "-g" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) - if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) - cfg.write("#{0}\n".format("-" * 80)) cfg.write("# CMake Standard\n") cfg.write("#{0}\n\n".format("-" * 80)) @@ -375,16 +372,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): hostname = socket.gethostname().rstrip('1234567890') - if sys_type in ('linux-rhel7-ppc64le', 'linux-rhel8-ppc64le', 'blueos_3_ppc64le_ib_p9') \ - and hostname != 'p3dev': - cfg.write(cmake_cache_option('ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC', True)) - if hostname in ('lassen', 'rzansel'): - cfg.write(cmake_cache_entry('MPIEXEC', 'lrun')) - cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) - else: - cfg.write(cmake_cache_entry('MPIEXEC', 'jsrun')) - cfg.write(cmake_cache_list('MPIEXEC_NUMPROC_FLAG', ['-g1', '--bind', 'rs', '-n'])) - elif sys_type in ('toss_4_x86_64_ib_cray'): + if sys_type in ('toss_4_x86_64_ib_cray'): cfg.write(cmake_cache_entry('MPIEXEC', 'srun')) cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) else: @@ -470,10 +458,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS', cmake_cuda_flags)) - # System specific flags - if sys_type in ('linux-rhel7-ppc64le', 'linux-rhel8-ppc64le', 'blueos_3_ppc64le_ib_p9'): - cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELEASE', '-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -Xcompiler -mcpu=powerpc64le -Xcompiler -mtune=powerpc64le')) - cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELWITHDEBINFO', '-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}')) cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_DEBUG', '-g -G -O0 -Xcompiler -O0')) @@ -568,12 +552,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('ENABLE_MKL', True)) cfg.write(cmake_cache_entry('MKL_INCLUDE_DIRS', spec['intel-mkl'].prefix.include)) cfg.write(cmake_cache_list('MKL_LIBRARIES', spec['intel-mkl'].libs)) - elif spec["blas"].name == "essl": - cfg.write(cmake_cache_option('ENABLE_ESSL', True)) - cfg.write(cmake_cache_entry('ESSL_INCLUDE_DIRS', spec['essl'].prefix.include)) - cfg.write(cmake_cache_list('ESSL_LIBRARIES', spec['blas'].libs)) - - cfg.write(cmake_cache_option('FORTRAN_MANGLE_NO_UNDERSCORE', True)) else: cfg.write(cmake_cache_list('BLAS_LIBRARIES', spec['blas'].libs)) cfg.write(cmake_cache_list('LAPACK_LIBRARIES', spec['lapack'].libs)) @@ -689,9 +667,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('GEOS_BUILD_SHARED_LIBS', False)) # ATS - # Lassen - if sys_type in ('blueos_3_ppc64le_ib_p9'): - cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--ats jsrun_omp --ats jsrun_bind=packed')) # Dane/Matrix if sys_type in ('toss_4_x86_64_ib'): cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--machine slurm112')) @@ -769,9 +744,6 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): debug_flags = "-g" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) - if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) - cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Cuda\n') cfg.write('#{0}\n\n'.format('-' * 80))