diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 22333d22..93ac19d3 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -159,6 +159,20 @@ jobs: # DOCKER_REPOSITORY: geosx/rockylinux9-gcc15-cuda13.2.1 # SPEC: "+cuda cuda_arch=86,120 ~openmp ~pygeosx ~docs %gcc-15 ^cuda@13.2.1+allow-unsupported-compilers" + # ROCm build. Unlike the rows above it does not layer on a + # geosx/:- base image: AMD's own ROCm image already + # ships the amdclang toolchain and the ROCm math libraries, so the + # base image is given literally via DOCKER_BASE_IMAGE_OVERRIDE. + - name: Ubuntu 24.04 - amdclang 19 + ROCm 6.4.3 + RUNS_ON: streak2 + TPL_DOCKERFILE: docker/tpl-ubuntu-hip.Dockerfile + DOCKER_BASE_IMAGE_OVERRIDE: rocm/dev-ubuntu-24.04:6.4.3 + DOCKER_REPOSITORY: geosx/ubuntu24.04-amdclang19.0.0-rocm6.4.3 + ROCM_VERSION: 6.4.3 + AMDGPU_TARGET: gfx942 + SPACK_BUILD_JOBS: 8 + SPEC: "+rocm ~uncrustify ~openmp ~pygeosx ~docs ~trilinos ~petsc amdgpu_target=gfx942 %amdclang-19 ^caliper~papi~gotcha~sampler~libunwind~libdw" + steps: - name: Checkout uses: actions/checkout@v6 @@ -175,13 +189,30 @@ jobs: - name: Compose DOCKER_BASE_IMAGE id: base run: | - echo "DOCKER_BASE_IMAGE=${{ matrix.DOCKER_BASE_IMAGE_REPO }}:${{ matrix.DOCKER_BASE_IMAGE_TAG }}-${DOCKER_BASE_IMAGE_SHA}" >> "$GITHUB_OUTPUT" + # Rows that do not build on a docker_base_images image (e.g. ROCm) + # pin their base image directly. + if [ -n "${{ matrix.DOCKER_BASE_IMAGE_OVERRIDE }}" ]; then + echo "DOCKER_BASE_IMAGE=${{ matrix.DOCKER_BASE_IMAGE_OVERRIDE }}" >> "$GITHUB_OUTPUT" + else + echo "DOCKER_BASE_IMAGE=${{ matrix.DOCKER_BASE_IMAGE_REPO }}:${{ matrix.DOCKER_BASE_IMAGE_TAG }}-${DOCKER_BASE_IMAGE_SHA}" >> "$GITHUB_OUTPUT" + fi - name: Inject CA certificate into Docker build if: matrix.RUNS_ON == 'streak2' run: | cp /etc/pki/tls/certs/ca-bundle.crt ./ca-bundle.crt - sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' ${{ matrix.TPL_DOCKERFILE }} + + # Inject distro-specific trust-store setup into the Dockerfile. Rocky + # images use dnf/update-ca-trust, the Ubuntu and ROCm ones use + # apt/update-ca-certificates. + if grep -q 'RUN dnf clean all' "${{ matrix.TPL_DOCKERFILE }}"; then + sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' "${{ matrix.TPL_DOCKERFILE }}" + elif grep -q 'apt-get update' "${{ matrix.TPL_DOCKERFILE }}"; then + perl -0pi -e 's/(RUN .*?apt-get update)/COPY ca-bundle.crt \/usr\/local\/share\/ca-certificates\/host-ca.crt\nRUN update-ca-certificates\n\n$1/s' "${{ matrix.TPL_DOCKERFILE }}" + else + echo "Could not find a CA-injection hook in ${{ matrix.TPL_DOCKERFILE }}" + exit 1 + fi - name: Run the docker build docker script env: @@ -190,6 +221,9 @@ jobs: DOCKER_BASE_IMAGE: ${{ steps.base.outputs.DOCKER_BASE_IMAGE }} GCC_VERSION: ${{ matrix.GCC_VERSION }} CLANG_VERSION: ${{ matrix.CLANG_VERSION }} + ROCM_VERSION: ${{ matrix.ROCM_VERSION }} + AMDGPU_TARGET: ${{ matrix.AMDGPU_TARGET }} + SPACK_BUILD_JOBS: ${{ matrix.SPACK_BUILD_JOBS }} INSTALL_DIR_ROOT: ${{ matrix.INSTALL_DIR_ROOT || '/opt/GEOS' }} HOST_CONFIG: ${{ matrix.HOST_CONFIG || 'host-configs/environment.cmake' }} SPEC: ${{ matrix.SPEC || 'undefined' }} diff --git a/.gitignore b/.gitignore index 704d7cdd..488ea52a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .project .pydevproject .vscode/ +spack/ spack-*.txt __pycache__ uberenv_* diff --git a/.uberenv_config.json b/.uberenv_config.json index b541fb7c..746d0c0f 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -6,8 +6,9 @@ "spack_configs_path": "scripts/spack_configs", "spack_packages_path": "scripts/spack_packages/packages", "spack_url": "https://github.com/spack/spack", -"spack_commit": "2e2169d5282d166f63e3ee4db8d4446c43cefa8a", -"spack_commit_note": "v1.1.1 (Jan 14th 2026)", -"spack_packages_commit": "3dd98680871078353a28ee508fa76c7554f918fa", -"spack_packages_note": "Feb 25th 2026" +"spack_commit": "3e19345b6e12f5ff1b874f4059622fc6a1fd804a", +"spack_commit_note": "v1.2.2 (Jul 20th 2026)", +"spack_packages_commit": "bd9b4838fbf9e4ca01ad7b6c6df633ddcc750d71", +"spack_packages_note": "Jul 20th 2026", +"spack_setup_clingo": 0 } diff --git a/CMakeLists.txt b/CMakeLists.txt index 321f3ca0..dbbe214d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,10 +47,12 @@ option( BUILD_SHARED_LIBS "" OFF ) option( ENABLE_CALIPER "" ON ) option( ENABLE_HYPRE "" ON ) -option( ENABLE_TRILINOS "" ON ) +option( ENABLE_TRILINOS "" OFF ) option( ENABLE_PETSC "" OFF ) option( ENABLE_SCOTCH "" ON ) option( ENABLE_SUITESPARSE "" ON ) +option( ENABLE_SUPERLU_DIST "" ON ) +option( ENABLE_HYPRE_GPU_PROFILING "" OFF ) option( ENABLE_VTK "" ON ) option( ENABLE_VTK_WRAP_PYTHON "" OFF ) @@ -125,6 +127,50 @@ else( TRUE ) endif() +################################ +# HIP / ROCm +################################ +if( ENABLE_HIP ) + if( NOT DEFINED ROCM_PATH ) + if( DEFINED HIP_ROOT ) + set( ROCM_PATH ${HIP_ROOT} ) + elseif( DEFINED ENV{ROCM_PATH} ) + set( ROCM_PATH $ENV{ROCM_PATH} ) + else() + message( FATAL_ERROR "ENABLE_HIP is ON but neither ROCM_PATH nor HIP_ROOT is defined." ) + endif() + endif() + + if( NOT DEFINED HIP_ROOT ) + set( HIP_ROOT ${ROCM_PATH} ) + endif() + + if( NOT DEFINED CMAKE_HIP_ARCHITECTURES ) + message( FATAL_ERROR "ENABLE_HIP is ON but CMAKE_HIP_ARCHITECTURES is not defined (e.g. gfx90a, gfx942, gfx1100)." ) + endif() + + # hypre is configured outside of cmake, so it needs the MPI include directory explicitly + # when its sources are compiled for the device. + if( ENABLE_MPI AND NOT DEFINED MPI_INCLUDE_DIR ) + if( DEFINED ENV{MPICH_DIR} ) # set by the cray-mpich module on LC + set( MPI_INCLUDE_DIR "$ENV{MPICH_DIR}/include" ) + elseif( DEFINED ENV{MPI_ROOT} ) + set( MPI_INCLUDE_DIR "$ENV{MPI_ROOT}/include" ) + elseif( MPI_C_INCLUDE_DIRS ) # set by find_package( MPI ) through BLT + list( GET MPI_C_INCLUDE_DIRS 0 MPI_INCLUDE_DIR ) + elseif( MPI_C_COMPILER ) # /bin/mpicc -> /include + get_filename_component( MPI_BIN_DIR ${MPI_C_COMPILER} DIRECTORY ) + get_filename_component( MPI_PREFIX_DIR ${MPI_BIN_DIR} DIRECTORY ) + set( MPI_INCLUDE_DIR "${MPI_PREFIX_DIR}/include" ) + else() + message( FATAL_ERROR "ENABLE_HIP is ON but MPI_INCLUDE_DIR is not defined and cannot be deduced." ) + endif() + message( STATUS "MPI_INCLUDE_DIR = ${MPI_INCLUDE_DIR}" ) + endif() + + message( STATUS "ROCM_PATH = ${ROCM_PATH}" ) + message( STATUS "CMAKE_HIP_ARCHITECTURES = ${CMAKE_HIP_ARCHITECTURES}" ) +endif() ################################ @@ -340,6 +386,13 @@ set(RAJA_URL "${TPL_MIRROR_DIR}/RAJA-v2025.12.0.tar.gz") message(STATUS "Building RAJA found at ${RAJA_URL}") +# RAJA's vectorization layer does not compile with the ROCm toolchain. +if( ENABLE_HIP ) + set( RAJA_ENABLE_VECTORIZATION OFF ) +else() + set( RAJA_ENABLE_VECTORIZATION ON ) +endif() + ExternalProject_Add( raja URL ${RAJA_URL} PREFIX ${PROJECT_BINARY_DIR}/raja @@ -360,10 +413,18 @@ ExternalProject_Add( raja -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CMAKE_CUDA_FLAGS:STRING=${RAJA_CUDA_FLAGS} + -D ENABLE_HIP:BOOL=${ENABLE_HIP} + -D RAJA_ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER} + -D RAJA_ENABLE_VECTORIZATION:BOOL=${RAJA_ENABLE_VECTORIZATION} + -D hip_DIR=${HIP_ROOT}/lib/cmake/hip + -D ROCPRIM_DIR=${HIP_ROOT} + -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} -D ENABLE_TESTS:BOOL=OFF -D ENABLE_EXAMPLES:BOOL=OFF -D ENABLE_OPENMP:BOOL=${ENABLE_OPENMP} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} ) @@ -398,8 +459,14 @@ ExternalProject_Add( chai -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CMAKE_CUDA_FLAGS:STRING=${CHAI_CUDA_FLAGS} + -D ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CHAI_ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER} + -D hip_DIR=${HIP_ROOT}/lib/cmake/hip + -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} -D ENABLE_OPENMP=${ENABLE_OPENMP} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -D ENABLE_TESTS=OFF -D ENABLE_BENCHMARKS=OFF @@ -444,6 +511,7 @@ if (ENABLE_CALIPER) -D MPI_CXX_COMPILER:STRING=${MPI_CXX_COMPILER} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BLT_SOURCE_DIR=${BLT_SOURCE_DIR} ) @@ -480,14 +548,23 @@ if (ENABLE_CALIPER) -D CUDA_TOOLKIT_ROOT_DIR:PATH=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D WITH_PAPI:BOOL=${ENABLE_PAPI} -D PAPI_PREFIX:STRING=${PAPI_PREFIX} -D WITH_ADIAK:BOOL=ON -D adiak_DIR:STRING=${ADIAK_DIR}/lib/cmake/adiak/ -D WITH_CUPTI=${CALIPER_WITH_CUPTI} -D WITH_NVPROF=${ENABLE_CUDA} + -D WITH_ROCM=${ENABLE_ROCTX} + -D ROCM_PREFIX:PATH=${ROCM_PATH} ) + if( BUILD_SHARED_LIBS ) + set( CALIPER_LIB "libcaliper.so" ) + else() + set( CALIPER_LIB "libcaliper.a" ) + endif() + list(APPEND build_list caliper adiak) else() set( ENABLE_CALIPER_HYPRE OFF CACHE BOOL "" FORCE ) @@ -589,11 +666,6 @@ if (ENABLE_TRILINOS) -D TPL_MKL_LIBRARIES:STRING=${MKL_LIBRARIES}) endif() - if ( ENABLE_ESSL ) - set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS} - -D HAVE_dggsvd3:BOOL=ON) - endif() - if ( DEFINED OpenMP_Fortran_FLAGS ) set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS} -D OpenMP_Fortran_FLAGS:STRING=${OpenMP_Fortran_FLAGS}) @@ -737,7 +809,7 @@ list(APPEND build_list metis ) ################################ # SUPERLU -- ################################ -if( ENABLE_MPI ) +if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) set(SUPERLU_DIR "${CMAKE_INSTALL_PREFIX}/superlu_dist") set(SUPERLU_URL "${TPL_MIRROR_DIR}/superlu_dist-0f6efc3.tar.gz") @@ -784,6 +856,7 @@ if( ENABLE_MPI ) -D CMAKE_Fortran_FLAGS_RELEASE=${CMAKE_Fortran_FLAGS_RELEASE} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D enable_openmp=${ENABLE_OPENMP} ) @@ -814,6 +887,9 @@ if( ENABLE_HYPRE ) set( HYPRE_C_COMPILER ${MPI_C_COMPILER} ) set( HYPRE_CXX_COMPILER ${MPI_CXX_COMPILER} ) set( HYPRE_Fortran_COMPILER ${MPI_Fortran_COMPILER} ) + endif() + + if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) set( HYPRE_SUPERLU_LIBS "${SUPERLU_DIR}/lib/libsuperlu_dist.a ${PARMETIS_DIR}/lib/libparmetis.a ${METIS_DIR}/lib/libmetis.a -lstdc++" ) if( ENABLE_OPENMP ) set( HYPRE_SUPERLU_LIBS "${HYPRE_SUPERLU_LIBS} ${OpenMP_C_FLAGS}" ) # Sergey: why is this needed? @@ -844,9 +920,41 @@ if( ENABLE_HYPRE ) --with-umpire-include=${CHAI_DIR}/include \ --with-umpire-lib-dirs=${CHAI_DIR}/lib \ --with-umpire-libs=umpire " ) + + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-aware-mpi" ) + endif() + + if( ENABLE_HYPRE_GPU_PROFILING ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-profiling" ) + endif() endif() elseif( ENABLE_HYPRE_DEVICE STREQUAL "HIP" ) - message( FATAL_ERROR "hip tpls build only supported through spack" ) + if( NOT ENABLE_HIP ) + message( FATAL_ERROR "Non-compatible options: HYPRE device cannot be set to HIP when ENABLE_HIP is OFF" ) + else() + list( APPEND HYPRE_DEPENDS chai ) + set( HYPRE_HIP_FLAGS "\ +ROCM_PATH=${ROCM_PATH} \ +--with-hip \ +--enable-rocblas \ +--enable-rocsparse \ +--enable-rocsolver \ +--with-gpu-arch=${CMAKE_HIP_ARCHITECTURES} \ +--with-umpire-device \ +--with-umpire-include=${CHAI_DIR}/include \ +--with-umpire-lib-dirs=${CHAI_DIR}/lib \ +--with-umpire-libs=\"umpire camp\" \ +--with-MPI-include=${MPI_INCLUDE_DIR} " ) + + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-aware-mpi" ) + endif() + + if( ENABLE_HYPRE_GPU_PROFILING ) + set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-profiling" ) + endif() + endif() endif() if( CMAKE_HOST_APPLE ) @@ -856,7 +964,7 @@ if( ENABLE_HYPRE ) endif() if( ENABLE_CALIPER_HYPRE ) - set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/libs/caliper.a\"" ) + set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/lib/${CALIPER_LIB}\"" ) list( APPEND HYPRE_DEPENDS caliper ) endif() @@ -881,6 +989,7 @@ if( ENABLE_HYPRE ) ${HYPRE_SUPERLU_FLAGS} \ ${HYPRE_OPENMP_FLAG} \ ${HYPRE_CUDA_FLAGS} \ + ${HYPRE_HIP_FLAGS} \ ${HYPRE_CALIPER_FLAGS}\n") ExternalProject_Add( hypre diff --git a/cmake/thirdparty/FindMathLibraries.cmake b/cmake/thirdparty/FindMathLibraries.cmake index 340c3e1a..3c6e13c9 100644 --- a/cmake/thirdparty/FindMathLibraries.cmake +++ b/cmake/thirdparty/FindMathLibraries.cmake @@ -12,15 +12,6 @@ if( ENABLE_MKL ) set( BLAS_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE ) set( LAPACK_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE) -elseif( ENABLE_ESSL ) - if ( NOT DEFINED ESSL_INCLUDE_DIRS ) - message( FATAL_ERROR "ESSL is enabled but ESSL_INCLUDE_DIRS is not defined." ) - elseif ( NOT DEFINED ESSL_LIBRARIES ) - message( FATAL_ERROR "ESSL is enabled but ESSL_LIBRARIES is not defined." ) - endif() - - set( BLAS_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE ) - set( LAPACK_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE ) endif() if( NOT DEFINED BLAS_LIBRARIES ) diff --git a/docker/pecan-spack.yaml b/docker/pecan-spack.yaml index 89ca8678..300cef98 100644 --- a/docker/pecan-spack.yaml +++ b/docker/pecan-spack.yaml @@ -111,6 +111,8 @@ spack: externals: - spec: py-sphinx@4.3.2 prefix: /usr + dependencies: + - spec: python@3.10.12 python: buildable: false externals: diff --git a/docker/spack-rocm.yaml b/docker/spack-rocm.yaml new file mode 100644 index 00000000..90a8d257 --- /dev/null +++ b/docker/spack-rocm.yaml @@ -0,0 +1,472 @@ +spack: + config: + install_tree: + root: $spack/.. + projections: + all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + misc_cache: $spack/../misc_cache + test_stage: $spack/../test_stage + build_stage: $spack/../build_stage + + # Regular TPLs do not need views + view: false + + # Include shared variants and versions + include: + - ../defaults.yaml + - ../versions.yaml + + toolchains: + clang-15: + - spec: '%c=llvm@15.0.7+clang~flang~lld~lldb' + when: '%c' + - spec: '%cxx=llvm@15.0.7+clang~flang~lld~lldb' + when: '%cxx' + - spec: '%fortran=gcc@11.4.0' + when: '%fortran' + - spec: '%openmpi@4.1.6 %llvm@15.0.7' + when: '%mpi' + clang-10: + - spec: '%c=llvm@10.0.0+clang~flang~lld~lldb openmp=project' + when: '%c' + - spec: '%cxx=llvm@10.0.0+clang~flang~lld~lldb openmp=project' + when: '%cxx' + - spec: '%fortran=gcc@9.4.0' + when: '%fortran' + - spec: '%openmpi@4.1.6 %llvm@10.0.0' + when: '%mpi' + gcc-9: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@9.4.0' + when: '%c' + - spec: '%cxx=gcc@9.4.0' + when: '%cxx' + - spec: '%fortran=gcc@9.4.0' + when: '%fortran' + - spec: '%openmpi %gcc@9.4.0' + when: '%mpi' + gcc-10: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@10.5.0' + when: '%c' + - spec: '%cxx=gcc@10.5.0' + when: '%cxx' + - spec: '%fortran=gcc@10.5.0' + when: '%fortran' + - spec: '%openmpi %gcc@10.5.0' + when: '%mpi' + gcc-11: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@11.4.0' + when: '%c' + - spec: '%cxx=gcc@11.4.0' + when: '%cxx' + - spec: '%fortran=gcc@11.4.0' + when: '%fortran' + - spec: '%openmpi %gcc@11.4.0' + when: '%mpi' + gcc-12: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@12.3.0' + when: '%c' + - spec: '%cxx=gcc@12.3.0' + when: '%cxx' + - spec: '%fortran=gcc@12.3.0' + when: '%fortran' + - spec: '%openmpi %gcc@12.3.0' + when: '%mpi' + gcc-13: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@13.3.0' + when: '%c' + - spec: '%cxx=gcc@13.3.0' + when: '%cxx' + - spec: '%fortran=gcc@13.3.0' + when: '%fortran' + - spec: '%openmpi@4.1.6 %gcc@13.3.0' + when: '%mpi' + amdclang-19: + - spec: '%c=llvm-amdgpu@6.4.3' + when: '%c' + - spec: '%cxx=llvm-amdgpu@6.4.3' + when: '%cxx' + - spec: '%fortran=gcc@13.3.0' + when: '%fortran' + - spec: '%openmpi@4.1.6 %llvm-amdgpu@6.4.3' + when: '%mpi' + + packages: + all: + target: [x86_64] + providers: + mpi: [openmpi] + blas: [netlib-lapack] + lapack: [netlib-lapack] + c: [llvm, gcc] + cxx: [llvm, gcc] + fortran: [gcc] + + mpi: + require: + - openmpi@4.1.6 + + llvm: + externals: + - spec: llvm@10.0.0+clang~flang~lld~lldb openmp=project + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang + cxx: /usr/bin/clang++ + - spec: llvm@15.0.7+clang~flang~lld~lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang-15 + cxx: /usr/bin/clang++-15 + llvm-amdgpu: + buildable: false + externals: + - spec: llvm-amdgpu@6.4.3 + prefix: /opt/rocm-6.4.3/lib/llvm + extra_attributes: + compilers: + c: /usr/local/bin/amdclang-gcc13 + cxx: /usr/local/bin/amdclang++-gcc13 + environment: + set: + BISON: bison + FLEX: flex + CC: /usr/local/bin/amdclang-gcc13 + CXX: /usr/local/bin/amdclang++-gcc13 + FC: /usr/bin/gfortran-13 + F77: /usr/bin/gfortran-13 + MPICC: /usr/bin/mpicc + MPICXX: /usr/bin/mpicxx + MPIEXEC: /usr/bin/mpirun + OMPI_CC: /usr/local/bin/amdclang-gcc13 + OMPI_CXX: /usr/local/bin/amdclang++-gcc13 + HIPCXX: /usr/local/bin/amdclang++-gcc13 + CMAKE_HIP_COMPILER: /usr/local/bin/amdclang++-gcc13 + gcc: + externals: + - spec: gcc@9.4.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-9 + cxx: /usr/bin/g++-9 + fortran: /usr/bin/gfortran-9 + - spec: gcc@10.5.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-10 + cxx: /usr/bin/g++-10 + fortran: /usr/bin/gfortran-10 + - spec: gcc@11.4.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-11 + cxx: /usr/bin/g++-11 + fortran: /usr/bin/gfortran-11 + - spec: gcc@12.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-12 + cxx: /usr/bin/g++-12 + fortran: /usr/bin/gfortran-12 + - spec: gcc@13.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-13 + cxx: /usr/bin/g++-13 + fortran: /usr/bin/gfortran-13 + cuda: + buildable: False + externals: + - spec: cuda@11.8.0 +allow-unsupported-compilers + prefix: /usr/local/cuda + hip: + buildable: false + externals: + - spec: hip@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + rocprim: + buildable: false + externals: + - spec: rocprim@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + rocsparse: + buildable: false + externals: + - spec: rocsparse@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + roctracer: + buildable: false + externals: + - spec: roctracer@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + rocblas: + buildable: false + externals: + - spec: rocblas@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + rocrand: + buildable: false + externals: + - spec: rocrand@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + rocsolver: + buildable: false + externals: + - spec: rocsolver@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + rocthrust: + buildable: false + externals: + - spec: rocthrust@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + hipblas: + buildable: false + externals: + - spec: hipblas@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + hipsparse: + buildable: false + externals: + - spec: hipsparse@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + hipfft: + buildable: false + externals: + - spec: hipfft@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + hipsolver: + buildable: false + externals: + - spec: hipsolver@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + hiprand: + buildable: false + externals: + - spec: hiprand@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + rocm-device-libs: + buildable: false + externals: + - spec: rocm-device-libs@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + hsa-rocr-dev: + buildable: false + externals: + - spec: hsa-rocr-dev@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + + autoconf: + version: [2.71] + buildable: false + externals: + - spec: autoconf@2.71 + prefix: /usr + automake: + version: [1.16.5] + buildable: false + externals: + - spec: automake@1.16.5 + prefix: /usr + cmake: + version: [3.28.6] + buildable: false + externals: + - spec: cmake@3.28.6 + prefix: /usr/local + findutils: + version: [4.9.0] + buildable: false + externals: + - spec: findutils@4.9.0 + prefix: /usr + m4: + buildable: false + externals: + - spec: m4@1.4.18 + prefix: /usr + mpfr: + buildable: false + externals: + - spec: mpfr@4.2.1 + prefix: /usr + netlib-lapack: + buildable: false + externals: + - spec: netlib-lapack@3.12.0 + prefix: /usr + netlib-blas: + buildable: false + externals: + - spec: netlib-blas@3.12.0 + prefix: /usr + openmpi: + buildable: false + externals: + - spec: openmpi@4.1.6 %llvm@10.0.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + modules: [mpi] + - spec: openmpi@4.1.6 %llvm@15.0.7 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + modules: [mpi] + - spec: openmpi@4.1.6 %llvm-amdgpu@6.4.3 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.6 %gcc@9.4.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.6 %gcc@10.5.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.6 %gcc@11.4.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.6 %gcc@12.3.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.6 %gcc@13.3.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + perl: + buildable: false + externals: + - spec: perl@5.38.2 + prefix: /usr + pkg-config: + buildable: false + externals: + - spec: pkg-config@0.29.2 + prefix: /usr + py-sphinx: + buildable: false + externals: + - spec: py-sphinx@7.2.6 + prefix: /usr + dependencies: + - spec: python@3.12.3 + python: + buildable: false + externals: + - spec: python@3.12.3 + prefix: /usr + readline: + buildable: false + externals: + - spec: readline@8.2 + prefix: /usr + tar: + buildable: false + externals: + - spec: tar@1.35 + prefix: /usr + unzip: + buildable: false + externals: + - spec: unzip@6.0 + prefix: /usr + xz: + buildable: false + externals: + - spec: xz@5.6.1 + prefix: /usr + zlib: + buildable: false + externals: + - spec: zlib@1.3 + prefix: /usr + z3: + buildable: false + externals: + - spec: z3@4.8.12 + prefix: /usr + gettext: + buildable: false + externals: + - spec: gettext@0.21 + prefix: /usr + bzip2: + buildable: false + externals: + - spec: bzip2@1.0.8 + prefix: /usr + help2man: + buildable: false + externals: + - spec: help2man@1.49.3 + prefix: /usr + libtool: + buildable: false + externals: + - spec: libtool@2.4.7 + prefix: /usr + gmp: + buildable: false + externals: + - spec: gmp@6.3.0 + prefix: /usr + pugixml: + buildable: true + gmake: + buildable: false + externals: + - spec: gmake@4.3 + prefix: /usr + ninja: + buildable: false + externals: + - spec: ninja@1.11.1 + prefix: /usr diff --git a/docker/tpl-rockylinux.Dockerfile b/docker/tpl-rockylinux.Dockerfile index 27cbe2bb..ac39daf7 100644 --- a/docker/tpl-rockylinux.Dockerfile +++ b/docker/tpl-rockylinux.Dockerfile @@ -41,7 +41,8 @@ RUN dnf clean all && \ bzip2 \ gnupg2 \ perl \ - xz && \ + xz \ + xz-devel && \ (dnf -y install python3-virtualenv || \ /usr/bin/python3 -m pip install --no-cache-dir virtualenv) && \ dnf clean all && rm -rf /var/cache/dnf /var/lib/dnf diff --git a/docker/tpl-ubuntu-hip.Dockerfile b/docker/tpl-ubuntu-hip.Dockerfile new file mode 100644 index 00000000..ae3a25d9 --- /dev/null +++ b/docker/tpl-ubuntu-hip.Dockerfile @@ -0,0 +1,232 @@ +# TPL build Dockerfile for ROCm/HIP images. +# +# Unlike docker/tpl-ubuntu.Dockerfile, this file does NOT layer on one of the +# geosx/ubuntu:* images from https://github.com/GEOS-DEV/docker_base_images. +# AMD's own rocm/dev-ubuntu-* image already ships the amdclang toolchain and the +# ROCm math libraries, so DOCKER_BASE_IMAGE points straight at it and the +# toolchain bits the geosx base images would normally provide (compiler, cmake) +# are installed here instead. +# +# The matrix in .github/workflows/docker_build_tpls.yml selects the base image, +# the spack toolchain (SPEC) and the AMD GPU target. + +# Temporary local variables dedicated to the TPL build +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/thirdPartyLibs +ARG BLD_DIR=$TMP_DIR/build + +ARG DOCKER_BASE_IMAGE=rocm/dev-ubuntu-24.04:6.4.3 +FROM ${DOCKER_BASE_IMAGE} AS tpl_toolchain_intersect_geosx_toolchain +ARG SRC_DIR + +# Install directory provided as a docker build argument; forwarded via ENV +# (GEOSX_TPL_DIR is part of the image contract consumed by GEOS). +ARG INSTALL_DIR +ENV GEOSX_TPL_DIR=$INSTALL_DIR + +# ROCm parameters +ARG AMDGPU_TARGET=gfx942 +ARG ROCM_VERSION=6.4.3 + +# Allow changing the number of cores used for building code via spack +ARG SPACK_BUILD_JOBS=4 +ENV SPACK_BUILD_JOBS=${SPACK_BUILD_JOBS} + +RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime && \ + apt-get update + +# Packages needed both for the TPL build and for the downstream GEOS build, +# plus the ROCm math libraries GEOS links against. +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + wget \ + gnupg \ + ca-certificates \ + gcc-13 \ + g++-13 \ + gfortran-13 \ + libtbb12 \ + libtbbmalloc2 \ + libblas-dev \ + liblapack-dev \ + libz3-dev \ + zlib1g-dev \ + openmpi-bin \ + libopenmpi-dev \ + python3 \ + python3-dev \ + python3-pip \ + python3-sphinx \ + doxygen \ + pkg-config \ + xz-utils \ + gettext \ + bzip2 \ + help2man \ + libtool \ + libgmp-dev \ + unzip \ + libmpfr-dev \ + lbzip2 \ + bzip2 \ + gnupg \ + virtualenv \ + libpugixml-dev \ + roctracer-dev \ + rocsparse-dev \ + rocsolver-dev \ + rocblas-dev \ + hipblas-dev \ + hipsparse-dev \ + hipfft-dev \ + hipsolver-dev \ + hiprand-dev \ + rocprim-dev \ + rocrand-dev \ + rocthrust-dev \ + git && \ + rm -rf /var/lib/apt/lists/* + +# Install clingo for Spack. Do not upgrade Ubuntu's Debian-managed pip in +# place; Ubuntu 24.04's pip package cannot be uninstalled by pip. +RUN python3 -m pip install clingo --break-system-packages + +# The ROCm base image does not ship cmake, so install it here (the geosx base +# images used by the other TPL Dockerfiles provide it already). +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh + +# OpenMPI hack for Ubuntu and provide amdclang wrappers with the GCC 13 system toolchain. +# clang++ inside the ROCm LLVM bin is also wrapped so that spack's cached_cmake.py, +# which hardcodes CMAKE_HIP_COMPILER to that path, gets the correct --gcc-toolchain. +RUN ln -s /usr/bin /usr/lib/x86_64-linux-gnu/openmpi && \ + printf '%s\n' '#!/bin/sh' "exec /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/amdclang --gcc-toolchain=/usr \"\$@\"" > /usr/local/bin/amdclang-gcc13 && \ + chmod +x /usr/local/bin/amdclang-gcc13 && \ + printf '%s\n' '#!/bin/sh' "exec /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/amdclang++ --gcc-toolchain=/usr \"\$@\"" > /usr/local/bin/amdclang++-gcc13 && \ + chmod +x /usr/local/bin/amdclang++-gcc13 && \ + rm -f /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/clang++ && \ + printf '%s\n' '#!/bin/sh' "exec /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/amdclang++ --gcc-toolchain=/usr \"\$@\"" > /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/clang++ && \ + chmod +x /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/clang++ && \ + ln -s /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/clang /usr/bin/clang && \ + ln -s /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/clang++ /usr/bin/clang++ + +# ----- TPL build stage ----- +FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain +ARG SRC_DIR +ARG BLD_DIR +ARG AMDGPU_TARGET +ARG SPEC + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libtbb-dev \ + make \ + autopoint \ + autotools-dev \ + automake \ + ninja-build \ + bc \ + file \ + patch \ + ca-certificates \ + git && \ + rm -rf /var/lib/apt/lists/* + +# Run uberenv. The SPEC is supplied by the matrix, as for the other TPL images. +# Have to create install directory first for uberenv. +# -k flag is to ignore SSL errors. +# +# A background heartbeat prints progress every minute: the ROCm TPL build runs +# for hours and spack is otherwise silent, which makes a stalled build +# indistinguishable from a slow one in CI logs. +RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ + mkdir -p ${GEOSX_TPL_DIR} && \ + GEOSX_SPEC="${SPEC}" && \ + if [ -z "${GEOSX_SPEC}" ] || [ "${GEOSX_SPEC}" = "undefined" ]; then \ + echo "ERROR: SPEC build-arg must be supplied" >&2 ; \ + exit 1 ; \ + fi && \ + { \ + ( while true; do \ + sleep 60; \ + echo "[heartbeat] $(date -Iseconds) uberenv/spack still running"; \ + find ${GEOSX_TPL_DIR}/build_stage -maxdepth 2 -mindepth 2 -type d -printf '%T@ %p\n' 2>/dev/null | \ + sort -nr | head -n 3 | \ + while read -r _ path; do \ + echo "[heartbeat] recent stage dir: ${path}"; \ + done; \ + find ${GEOSX_TPL_DIR}/build_stage -maxdepth 4 \( -name spack-build-out.txt -o -name spack-build-env.txt -o -name spack-configure-args.txt \) -printf '%T@ %p\n' 2>/dev/null | \ + sort -nr | head -n 3 | \ + while read -r _ path; do \ + echo "[heartbeat] recent stage file: ${path}"; \ + tail -n 5 "${path}" 2>/dev/null || true; \ + printf '\n'; \ + done; \ + ps -eo pid,ppid,etime,%cpu,%mem,cmd 2>/dev/null | \ + grep -E 'spack|python3|curl|wget|git|cmake|ninja|make|amdclang|gfortran|gcc|g\\+\\+' | \ + grep -v grep | \ + tail -n 20 | \ + cut -c1-240 || true; \ + done ) & \ + hb=$!; \ + ./scripts/uberenv/uberenv.py \ + --spec "${GEOSX_SPEC}" \ + --spack-env-file=${SRC_DIR}/docker/spack-rocm.yaml \ + --project-json=${SRC_DIR}/.uberenv_config.json \ + --prefix ${GEOSX_TPL_DIR} \ + -j ${SPACK_BUILD_JOBS} \ + -k; \ + rc=$?; \ + kill ${hb} 2>/dev/null || true; \ + wait ${hb} 2>/dev/null || true; \ + test ${rc} -eq 0; \ + } && \ + rm -f lvarray* && \ + cp *.cmake /spack-generated.cmake && \ + cd ${GEOSX_TPL_DIR} && \ + rm -rf bin/ build_stage/ builtin_spack_packages_repo/ misc_cache/ spack/ spack_env/ .spack-db/ + +# ----- Final GEOS-build image ----- +FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain +ARG SRC_DIR +# ARGs are scoped per stage, so the ROCm ones must be redeclared here for the +# ENV defaults at the bottom of this stage to interpolate. +ARG AMDGPU_TARGET +ARG ROCM_VERSION + +COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR + +# Extract the generated host-config +COPY --from=tpl_toolchain /spack-generated.cmake / + +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get install -y --no-install-recommends \ + openssh-client \ + ca-certificates \ + curl \ + python3 \ + texlive \ + texlive-latex-extra \ + graphviz \ + libxml2-utils \ + git \ + ghostscript \ + ninja-build \ + python3-dev \ + python3-sphinx \ + python3-mpi4py \ + python3-scipy \ + python3-virtualenv \ + python3-matplotlib \ + python3-venv \ + python3-pytest && \ + rm -rf /var/lib/apt/lists/* + +# Install sccache to speed up downstream GEOS builds +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh +ENV SCCACHE=/opt/sccache/bin/sccache + +# Helpful environment defaults for HIP +ENV ROCM_PATH=/opt/rocm-${ROCM_VERSION} +ENV HIP_PATH=${ROCM_PATH}/hip +ENV PATH=${ROCM_PATH}/bin:${ROCM_PATH}/llvm/bin:${PATH} +ENV LD_LIBRARY_PATH=${ROCM_PATH}/lib:${ROCM_PATH}/lib64:${ROCM_PATH}/llvm/lib +ENV CMAKE_HIP_ARCHITECTURES=${AMDGPU_TARGET} diff --git a/docker/ubuntu-spack.yaml b/docker/ubuntu-spack.yaml index e27514bf..40915d04 100644 --- a/docker/ubuntu-spack.yaml +++ b/docker/ubuntu-spack.yaml @@ -275,6 +275,8 @@ spack: externals: - spec: py-sphinx@7.4.7 prefix: /usr + dependencies: + - spec: python@3.12 python: buildable: false externals: diff --git a/scripts/createLapackForESSL.sh b/scripts/createLapackForESSL.sh deleted file mode 100755 index 59b09ca9..00000000 --- a/scripts/createLapackForESSL.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# Adapted from IBM Engineering and Scientific Subroutine Library for Linux on POWER Version 6.2 ESSL Guide and Reference -# -# First Pass - get the symbols in ESSL and LAPACK -echo "The essl library is: " $1 -echo "The lapack library is: " $2 -echo "The output library is: liblapackforessl.a" -nm $2 | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.lapack -nm $1 | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.essl -diff -y symbols.in.lapack symbols.in.essl | grep -v -e '>' -e '|' -e '<' | awk '{ print $1 }' > duplicate.symbols -wc -l duplicate.symbols - -# Second Pass - remove the ESSL symbols from the LAPACK library -cp -p $2 liblapackforessl.a - -for SYMBOL in $(cat duplicate.symbols | awk '{ print $1 }') -do - fn=$SYMBOL - unders="no" - echo $SYMBOL | grep "_" > /dev/null && unders="yes" - if [ "$unders" = "yes" ] - then - fn=`echo $SYMBOL | awk -F'_' '{print $1}'` - fi - - echo $SYMBOL - ar d liblapackforessl.a $fn.o -done - -# Check it worked -nm liblapackforessl.a | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.updatedlapack -diff -y symbols.in.updatedlapack symbols.in.essl | grep -v -e '>' -e '|' -e '<' | awk '{ print $1 }' > duplicate.symbols.inupdatedlapack -wc -l duplicate.symbols.inupdatedlapack diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 59218646..4eb0b526 100644 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -1,12 +1,33 @@ #!/bin/bash +set -e env -# We save memory for the docker context -echo .git > .dockerignore +# Keep the docker context to the sources the TPL build actually needs. +# +# In CI this is a fresh checkout and only .git matters. Run locally via +# scripts/reproduce_ci.sh, however, the working tree usually also holds build +# and install trees, which added ~27GB to every job's context, and stray +# host-configs left at the repo root by earlier local builds. Those last ones +# are not merely wasteful: the TPL Dockerfiles finish with +# `cp *.cmake /spack-generated.cmake`, so any root-level *.cmake beyond the one +# uberenv just generated makes cp treat the destination as a directory and the +# build fails after the whole TPL stack has been compiled. No tracked file at +# the repo root matches these patterns. +cat > .dockerignore <<'DOCKERIGNORE' +.git +build +build-* +install +install-* +*_tpls +.rocm-ci-local.* +*.cmake +*.log +__pycache__ +DOCKERIGNORE # Get uberenv submodule -git submodule update --init scripts/uberenv - +git submodule update --init --force scripts/uberenv # This script will build an image from TPL_DOCKERFILE. # The new TPL Dockerfiles (docker/tpl-ubuntu.Dockerfile, @@ -30,6 +51,10 @@ echo "Docker base image is ${DOCKER_BASE_IMAGE}" EXTRA_BUILD_ARGS=() if [ -n "${GCC_VERSION}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "GCC_VERSION=${GCC_VERSION}"); fi if [ -n "${CLANG_VERSION}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "CLANG_VERSION=${CLANG_VERSION}"); fi +# ROCm-only knobs, forwarded the same way (see docker/tpl-ubuntu-hip.Dockerfile). +if [ -n "${AMDGPU_TARGET}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "AMDGPU_TARGET=${AMDGPU_TARGET}"); fi +if [ -n "${ROCM_VERSION}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "ROCM_VERSION=${ROCM_VERSION}"); fi +if [ -n "${SPACK_BUILD_JOBS}" ]; then EXTRA_BUILD_ARGS+=(--build-arg "SPACK_BUILD_JOBS=${SPACK_BUILD_JOBS}"); fi BUILDER_ARGS=() if [ -n "${DOCKER_BUILDER:-}" ]; then BUILDER_ARGS+=(--builder "${DOCKER_BUILDER}"); fi diff --git a/scripts/reproduce_ci.sh b/scripts/reproduce_ci.sh index 35ca6c48..75ddf01c 100755 --- a/scripts/reproduce_ci.sh +++ b/scripts/reproduce_ci.sh @@ -7,7 +7,12 @@ repo_root=$(cd -- "${script_dir}/.." && pwd) cd -- "${repo_root}" # Active build_images entries from .github/workflows/docker_build_tpls.yml. -# Fields: name|dockerfile|base_tag|base_repository|repository|gcc|clang|spec +# Fields: name|dockerfile|base_tag|base_repository|repository|gcc|clang|spec|base_image|extra_env +# +# The last two fields are optional and may simply be omitted: +# base_image overrides the composed :- for +# rows that do not build on a docker_base_images image (ROCm). +# extra_env space-separated KEY=VALUE pairs passed to scripts/docker-build.sh. MATRIX=( 'Ubuntu 24.04 - gcc 12|docker/tpl-ubuntu.Dockerfile|24.04-gcc12|geosx/ubuntu|geosx/ubuntu24.04-gcc12|12||~pygeosx ~docs %gcc-12' 'Ubuntu 24.04 - gcc 13 (docs)|docker/tpl-ubuntu.Dockerfile|24.04-gcc13|geosx/ubuntu|geosx/ubuntu24.04-gcc13|13||~pygeosx +docs %gcc-13' @@ -20,6 +25,7 @@ MATRIX=( 'Ubuntu 24.04 - gcc 13 + CUDA 12.9.1|docker/tpl-ubuntu.Dockerfile|24.04-gcc13-cuda12.9.1|geosx/ubuntu|geosx/ubuntu24.04-gcc13-cuda12.9.1|13||+cuda cuda_arch=86,120 ~openmp ~pygeosx ~docs %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers' 'Ubuntu 24.04 - clang 19 + CUDA 12.9.1|docker/tpl-ubuntu.Dockerfile|24.04-clang19-cuda12.9.1|geosx/ubuntu|geosx/ubuntu24.04-clang19-cuda12.9.1||19|+cuda cuda_arch=86,120 ~openmp ~pygeosx ~docs %clang-19 ^cuda@12.9.1+allow-unsupported-compilers' 'Rocky Linux 8 - gcc 13 + CUDA 12.9.1|docker/tpl-rockylinux.Dockerfile|8-gcc13-cuda12.9.1|geosx/rockylinux|geosx/rockylinux8-gcc13-cuda12.9.1|||+cuda cuda_arch=86,120 ~openmp ~pygeosx ~docs %gcc-13 ^cuda@12.9.1+allow-unsupported-compilers' +'Ubuntu 24.04 - amdclang 19 + ROCm 6.4.3|docker/tpl-ubuntu-hip.Dockerfile|||geosx/ubuntu24.04-amdclang19.0.0-rocm6.4.3|||+rocm ~uncrustify ~openmp ~pygeosx ~docs ~trilinos ~petsc amdgpu_target=gfx942 %amdclang-19 ^caliper~papi~gotcha~sampler~libunwind~libdw|rocm/dev-ubuntu-24.04:6.4.3|ROCM_VERSION=6.4.3 AMDGPU_TARGET=gfx942 SPACK_BUILD_JOBS=8' ) DOCKER_BASE_IMAGE_SHA=${DOCKER_BASE_IMAGE_SHA:-1c3c049b3f629d9d44838656fd306b2a0c04c9e8} @@ -171,20 +177,24 @@ failed_builders=() build() { local index=$1 name=$2 dockerfile=$3 base_tag=$4 base_repository=$5 repository=$6 - local gcc=$7 clang=$8 spec=$9 + local gcc=$7 clang=$8 spec=$9 base_image=${10} extra_env=${11} local builder="reproduce-ci-$$-${index}" image="${repository}:${DOCKER_TAG}" local cleanup_failed=0 + # Rows without an explicit base_image layer on a docker_base_images image. + [[ ${base_image} ]] || base_image="${base_repository}:${base_tag}-${DOCKER_BASE_IMAGE_SHA}" + printf '\n==> %s\n' "${name}" if (( dry_run )); then printf ' dockerfile %s\n' "${dockerfile}" - printf ' base image %s:%s-%s\n' "${base_repository}" "${base_tag}" "${DOCKER_BASE_IMAGE_SHA}" + printf ' base image %s\n' "${base_image}" printf ' repository %s\n' "${repository}" printf ' tag %s\n' "${DOCKER_TAG}" printf ' spec %s\n' "${spec}" [[ ${gcc} ]] && printf ' gcc %s\n' "${gcc}" [[ ${clang} ]] && printf ' clang %s\n' "${clang}" + [[ ${extra_env} ]] && printf ' extra env %s\n' "${extra_env}" printf ' commit %s\n' "${COMMIT}" printf ' install dir %s\n' "${INSTALL_DIR_ROOT}" printf ' host config %s\n' "${HOST_CONFIG}" @@ -197,9 +207,11 @@ build() { return fi + # extra_env is expanded, and bash only honours literal VAR=value words as + # assignments, so it has to go through env rather than the prefix list. if ! TPL_DOCKERFILE=${dockerfile} \ DOCKER_REPOSITORY=${repository} \ - DOCKER_BASE_IMAGE="${base_repository}:${base_tag}-${DOCKER_BASE_IMAGE_SHA}" \ + DOCKER_BASE_IMAGE="${base_image}" \ GCC_VERSION=${gcc} \ CLANG_VERSION=${clang} \ INSTALL_DIR_ROOT=${INSTALL_DIR_ROOT} \ @@ -210,6 +222,7 @@ build() { DOCKER_TAG=${DOCKER_TAG} \ DOCKER_BUILDER=${builder} \ DOCKER_LOAD=1 \ + env ${extra_env} \ bash -x scripts/docker-build.sh; then printf 'FAILED: %s (builder retained: %s)\n' "${name}" "${builder}" >&2 failed=1 @@ -247,9 +260,9 @@ row=0 for idx in "${indices[@]}"; do row=$((row + 1)) IFS='|' read -r name dockerfile base_tag base_repository repository gcc clang spec \ - <<<"${MATRIX[${idx}]}" + base_image extra_env <<<"${MATRIX[${idx}]}" build "${row}" "${name}" "${dockerfile}" "${base_tag}" "${base_repository}" \ - "${repository}" "${gcc}" "${clang}" "${spec}" + "${repository}" "${gcc}" "${clang}" "${spec}" "${base_image}" "${extra_env}" done if (( dry_run )); then diff --git a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml b/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml deleted file mode 100644 index 1c720f3e..00000000 --- a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml +++ /dev/null @@ -1,233 +0,0 @@ -#------------------------------------------------------------------------------------------------------------ -# SPDX-License-Identifier: LGPL-2.1-only -# -# Copyright (c) 2018-2020 Lawrence Livermore National Security LLC -# Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University -# Copyright (c) 2018-2020 TotalEnergies -# Copyright (c) 2019- GEOSX Contributors -# All rights reserved -# -# See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. -#------------------------------------------------------------------------------------------------------------ - -# geosx@develop%clang@10.0.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%gcc@8.3.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%clang@13.0.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%clang@13.0.1+cuda cuda_arch=70 ^cuda@12.2.2+allow-unsupported-compilers -# -# Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-10 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %gcc-8 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-13 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~openmp~uncrustify cuda_arch=70 %clang-13 ^cuda@12.2.2+allow-unsupported-compilers" - - -spack: - config: - install_tree: - root: $spack/.. - projections: - all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - misc_cache: $spack/../misc_cache - test_stage: $spack/../test_stage - build_stage:: - - $spack/../build_stage - - # Regular TPLs do not need views - view: false - - # Include shared variants and versions - include: - - ../defaults.yaml - - ../versions.yaml - - toolchains: - clang-13: - - spec: '%[virtuals=c]llvm@13.0.1+clang~flang~lld~lldb' - when: '%c' - - spec: '%[virtuals=cxx]llvm@13.0.1+clang~flang~lld~lldb' - when: '%cxx' - - spec: '%[virtuals=fortran]gcc@8.3.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.clang.13' - when: '%mpi' - clang-10: - - spec: '%[virtuals=c]llvm@10.0.1+clang~flang~lld~lldb' - when: '%c' - - spec: '%[virtuals=cxx]llvm@10.0.1+clang~flang~lld~lldb' - when: '%cxx' - - spec: '%[virtuals=fortran]gcc@8.3.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.clang.10' - when: '%mpi' - gcc-12: - - spec: '%c=gcc@12.1.1' - when: '%c' - - spec: '%cxx=gcc@12.1.1' - when: '%cxx' - - spec: '%fortran=gcc@12.1.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.gcc.12' - when: '%mpi' - gcc-8: - - spec: '%c=gcc@8.3.1 ' - when: '%c' - - spec: '%cxx=gcc@8.3.1 ' - when: '%cxx' - - spec: '%fortran=gcc@8.3.1 ' - when: '%fortran' - - spec: '%spectrum-mpi@release.gcc.8' - when: '%mpi' - - packages: - mpi: - require: - - spectrum-mpi - - zlib-api: - require: - - zlib - - blas: - require: - - essl - lapack: - require: - - essl - - llvm: - externals: - - spec: llvm@10.0.1+clang~flang~lld~lldb - prefix: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang++ - flags: {} - extra_rpaths: [] - - spec: llvm@13.0.1+clang~flang~lld~lldb - prefix: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang++ - flags: {} - extra_rpaths: [] - gcc: - externals: - - spec: gcc@8.3.1 languages:='c,c++,fortran' - prefix: /usr/tce/packages/gcc/gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ - fortran: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - extra_rpaths: [] - - spec: gcc@12.2.1 languages:='c,c++,fortran' - prefix: /usr/tce/packages/gcc/gcc-12.2.1 - extra_attributes: - compilers: - c: /usr/tce/packages/gcc/gcc-12.2.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-12.2.1/bin/g++ - fortran: /usr/tce/packages/gcc/gcc-12.2.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - extra_rpaths: [] - - spectrum-mpi: - buildable: False - externals: - - spec: spectrum-mpi@release.gcc.8 %gcc@8.3.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-8.3.1 - - spec: spectrum-mpi@release.gcc.12 %gcc@12.2.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-12.2.1 - # Previously clang@upstream - clingo doesn't like "upstream" version - - spec: spectrum-mpi@release.clang.10 %llvm@10.0.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-10.0.1-gcc-8.3.1 - - spec: spectrum-mpi@release.clang.13 %llvm@13.0.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-13.0.1-gcc-8.3.1 - - essl: - buildable: False - externals: - - spec: essl@6.3.0.2 - prefix: /usr/tcetmp/packages/essl/essl-6.3.0.2/ - - cuda: - buildable: False - externals: - - spec: cuda@11.8.0 +allow-unsupported-compilers - prefix: /usr/tce/packages/cuda/cuda-11.8.0 - - spec: cuda@12.2.2 +allow-unsupported-compilers - prefix: /usr/tce/packages/cuda/cuda-12.2.2 - - # System level packages to not build - cmake: - buildable: False - externals: - - spec: cmake@3.29.2 - prefix: /usr/tce/packages/cmake/cmake-3.29.2 - readline: - buildable: False - externals: - - spec: readline@7.0 - prefix: /collab/usr/gapps/python/build/spack-coralea.3/opt/spack/linux-rhel7-ppc64le/gcc-4.9.3/readline-7.0-owal6z2zh5zotgvdmwow6sgdn6cqfn43/ - m4: - buildable: False - externals: - - spec: m4@1.4.16 - prefix: /usr - perl: - buildable: false - externals: - - spec: perl@5.16.3 - prefix: /usr - pkg-config: - buildable: false - externals: - - spec: pkg-config@0.27.1 - prefix: /usr - diffutils: - buildable: False - externals: - - spec: diffutils@3.3 - prefix: /usr/bin/ - # This needs to be the prefix to the python3 LC installation - # or pygeosx package built by spack/uberenv - python: - buildable: False - externals: - - spec: python@3.8.2 - prefix: /usr/gapps/GEOSX/thirdPartyLibs/python/lassen-gcc-python/python/ - autoconf: - buildable: False - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - buildable: False - externals: - - spec: automake@1.13.4 - prefix: /usr - libtool: - buildable: False - externals: - - spec: libtool@2.4.2 - prefix: /usr - gettext: - buildable: False - externals: - - spec: gettext@0.19.8.1 - prefix: /usr/bin/ - zlib: - buildable: false - externals: - - spec: zlib@1.2.7 - prefix: /usr diff --git a/scripts/spack_configs/pangea-4/spack.yaml b/scripts/spack_configs/pangea-4/spack.yaml index ff2d4c36..22994511 100644 --- a/scripts/spack_configs/pangea-4/spack.yaml +++ b/scripts/spack_configs/pangea-4/spack.yaml @@ -73,7 +73,6 @@ spack: require: "@26.1" grpc: require: "@1.65.5-full-clone" - #### # external compiler gcc: diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index d0ffeed9..e6b12357 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -39,6 +39,8 @@ spack: test_stage: $spack/../test_stage build_stage: $spack/../build_stage build_jobs: 112 + # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. + installer: new # Regular TPLs do not need views view: false @@ -231,6 +233,8 @@ spack: externals: - spec: py-sphinx@1.6.3 prefix: /usr/gapps/GEOSX/thirdPartyLibs/python/quartz-gcc-python/python/ + dependencies: + - spec: python@3.10.8 autoconf: buildable: False externals: diff --git a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml index 76404e3a..48af6ee7 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml @@ -31,6 +31,8 @@ spack: test_stage: $spack/../test_stage build_stage: $spack/../build_stage build_jobs: 96 + # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. + installer: new # Regular TPLs do not need views view: false diff --git a/scripts/spack_packages/packages/chai/package.py b/scripts/spack_packages/packages/chai/package.py index 16b68a56..bdc3f94b 100644 --- a/scripts/spack_packages/packages/chai/package.py +++ b/scripts/spack_packages/packages/chai/package.py @@ -1,3 +1,7 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + import os from spack.package import * @@ -6,4 +10,12 @@ class Chai(BuiltinChai): # Bypass llnl_link_helpers failure depends_on("fortran") + depends_on("c", type="build") + depends_on("cxx", type="build") + # From radiuss-packages (PR #143) + depends_on("blt@0.7.1:", type="build", when="@2025.09.0:") + depends_on("blt@0.7.0:", type="build", when="@2025.03.0:") + depends_on("umpire@2025.09.0:", when="@2025.09.0:") + depends_on("umpire@2025.03.0:", when="@2025.03.0:") + depends_on("umpire@2024.07.0", when="@2024.07.0") diff --git a/scripts/spack_packages/packages/essl/package.py b/scripts/spack_packages/packages/essl/package.py deleted file mode 100644 index fa2a2cb7..00000000 --- a/scripts/spack_packages/packages/essl/package.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack.package import * -from spack_repo.builtin.build_systems.bundle import BundlePackage - -# Recipe for pre-built essl library on blueos machines. -# Defines additonal flags for blueos: -# https://lc.llnl.gov/confluence/display/SIERRA/Math+Libraries -class Essl(BundlePackage): - """IBM's Engineering and Scientific Subroutine Library (ESSL).""" - - homepage = "https://www.ibm.com/systems/power/software/essl/" - - version("6.3.0.2") - - provides("blas") - provides("lapack") - - @property - def blas_libs(self): - spec = self.spec - prefix = self.prefix - - essl_root = prefix.lib64 - essl_libs = ["libessl", "liblapackforessl", "liblapackforessl_"] - all_libs = find_libraries(essl_libs, root=essl_root, shared=True) - - return all_libs - - @property - def lapack_libs(self): - spec = self.spec - prefix = self.prefix - - essl_root = prefix.lib64 - essl_libs = ["libessl", "liblapackforessl", "liblapackforessl_"] - all_libs = find_libraries(essl_libs, root=essl_root, shared=True) - - return all_libs diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index eaa469de..b701132d 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -9,21 +9,25 @@ import socket import os +import spack.llnl.util.tty as tty + from os import environ as env from os.path import join as pjoin -from spack_repo.builtin.build_systems.cached_cmake import cmake_cache_path - from spack_repo.builtin.build_systems.cmake import CMakePackage from spack_repo.builtin.build_systems.cuda import CudaPackage +from spack_repo.builtin.build_systems.cached_cmake import ( + cmake_cache_option, + cmake_cache_path, + cmake_cache_string, +) # Tested specs are located at scripts/spack_configs/<$SYS_TYPE>/spack.yaml (e.g. %clang@10.0.1) - # WARNING: +petsc variant is yet to be tested. def cmake_cache_entry(name, value, comment=""): - """Generate a string for a cmake cache variable""" + """Generate a string for a cmake cache variable.""" return 'set(%s "%s" CACHE PATH "%s")\n\n' % (name, value, comment) @@ -36,19 +40,6 @@ def cmake_cache_list(name, value, comment=""): return 'set(%s %s CACHE STRING "%s")\n\n' % (name, join_str.join(value), comment) -def cmake_cache_string(name, string, comment=""): - """Generate a string for a cmake cache variable""" - - return 'set(%s "%s" CACHE STRING "%s")\n\n' % (name, string, comment) - - -def cmake_cache_option(name, boolean_value, comment=""): - """Generate a string for a cmake configuration option""" - - value = "ON" if boolean_value else "OFF" - return 'set(%s %s CACHE BOOL "%s")\n\n' % (name, value, comment) - - class Geosx(CMakePackage, CudaPackage, ROCmPackage): """GEOSX simulation framework.""" @@ -64,7 +55,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): variant('shared', default=True, description='Build Shared Libs.') variant('caliper', default=True, description='Build Caliper support.') variant('vtk', default=True, description='Build VTK support.') - variant('trilinos', default=True, description='Build Trilinos support.') + variant('trilinos', default=False, description='Build Trilinos support.') variant('hypre', default=True, description='Build HYPRE support.') variant('hypredrive', default=True, description='Build hypredrive support.') variant('petsc', default=False, description='Build PETSc support.') @@ -187,6 +178,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): depends_on("hypre +cuda+superlu-dist+mixedint+mpi+umpire~unified-memory", when='+cuda') depends_on("hypre +rocm+superlu-dist+mixedint+mpi+umpire~unified-memory", when='+rocm') depends_on("hypre ~openmp", when="~openmp") + depends_on("hypre +caliper", when="+caliper") depends_on("hypre +pic", when="~shared") depends_on("hypre +shared", when="+shared") @@ -260,22 +252,22 @@ def _get_sys_type(self, spec): return sys_type def _get_host_config_path(self, spec, lvarray=False): - var = '' - - if '+cuda' in spec: - var = '-'.join([var, 'cuda']) - var += "@" + str(spec['cuda'].version) - elif '+rocm' in spec: - var = '-'.join([var, 'rocm']) - var += "@" + str(spec['hip'].version) - + gpu_backend = "" + if "+cuda" in spec: + gpu_backend = f"cuda@{spec['cuda'].version}" + elif "+rocm" in spec: + gpu_backend = f"rocm@{spec['hip'].version}" hostname = socket.gethostname().rstrip('1234567890') if lvarray: hostname = "lvarray-" + hostname - host_config_path = "%s-%s-%s@%s%s.cmake" % (hostname, self._get_sys_type(spec), (str(spec.compiler.name)), str(spec.compiler.version),var) + host_config_path = "%s-%s-%s@%s%s.cmake" % (hostname, + self._get_sys_type(spec), + str(spec.compiler.name), + str(spec.compiler.version), + gpu_backend) dest_dir = self.stage.source_path host_config_path = os.path.abspath(pjoin(dest_dir, host_config_path)) @@ -337,15 +329,15 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write("#{0}\n".format("-" * 80)) cfg.write("# Compilers\n") cfg.write("#{0}\n\n".format("-" * 80)) - cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) + cfg.write(cmake_cache_path("CMAKE_C_COMPILER", c_compiler)) cflags = ' '.join(spec.compiler_flags['cflags']) if cflags: - cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) + cfg.write(cmake_cache_string("CMAKE_C_FLAGS", cflags)) - cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) + cfg.write(cmake_cache_path("CMAKE_CXX_COMPILER", cpp_compiler)) cxxflags = ' '.join(spec.compiler_flags['cxxflags']) if cxxflags: - cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) + cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS", cxxflags)) release_flags = "-O3 -DNDEBUG" if "clang" in self.compiler.cxx: @@ -356,9 +348,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): debug_flags = "-g" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) - if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) - cfg.write("#{0}\n".format("-" * 80)) cfg.write("# CMake Standard\n") cfg.write("#{0}\n\n".format("-" * 80)) @@ -370,23 +359,14 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write("#{0}\n\n".format("-" * 80)) cfg.write(cmake_cache_option('ENABLE_MPI', True)) - cfg.write(cmake_cache_entry('MPI_C_COMPILER', spec['mpi'].mpicc)) - cfg.write(cmake_cache_entry('MPI_CXX_COMPILER', spec['mpi'].mpicxx)) + cfg.write(cmake_cache_path('MPI_C_COMPILER', spec['mpi'].mpicc)) + cfg.write(cmake_cache_path('MPI_CXX_COMPILER', spec['mpi'].mpicxx)) hostname = socket.gethostname().rstrip('1234567890') - if sys_type in ('linux-rhel7-ppc64le', 'linux-rhel8-ppc64le', 'blueos_3_ppc64le_ib_p9') \ - and hostname != 'p3dev': - cfg.write(cmake_cache_option('ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC', True)) - if hostname in ('lassen', 'rzansel'): - cfg.write(cmake_cache_entry('MPIEXEC', 'lrun')) - cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) - else: - cfg.write(cmake_cache_entry('MPIEXEC', 'jsrun')) - cfg.write(cmake_cache_list('MPIEXEC_NUMPROC_FLAG', ['-g1', '--bind', 'rs', '-n'])) - elif sys_type in ('toss_4_x86_64_ib_cray'): - cfg.write(cmake_cache_entry('MPIEXEC', 'srun')) - cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) + if sys_type in ('toss_4_x86_64_ib_cray'): + cfg.write(cmake_cache_string('MPIEXEC', 'srun')) + cfg.write(cmake_cache_string('MPIEXEC_NUMPROC_FLAG', '-n')) else: # Taken from cached_cmake class: # https://github.com/spack/spack/blob/develop/lib/spack/spack/build_systems/cached_cmake.py#L180-234 @@ -450,9 +430,9 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string('CMAKE_CUDA_STANDARD', spec.variants['cxxstd'].value)) cudatoolkitdir = spec['cuda'].prefix - cfg.write(cmake_cache_entry('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) + cfg.write(cmake_cache_path('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) cudacompiler = '${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc' - cfg.write(cmake_cache_entry('CMAKE_CUDA_COMPILER', cudacompiler)) + cfg.write(cmake_cache_path('CMAKE_CUDA_COMPILER', cudacompiler)) cmake_cuda_flags = ('-restrict --expt-extended-lambda -Werror ' 'cross-execution-space-call,reorder,' @@ -470,10 +450,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS', cmake_cuda_flags)) - # System specific flags - if sys_type in ('linux-rhel7-ppc64le', 'linux-rhel8-ppc64le', 'blueos_3_ppc64le_ib_p9'): - cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELEASE', '-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -Xcompiler -mcpu=powerpc64le -Xcompiler -mtune=powerpc64le')) - cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELWITHDEBINFO', '-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}')) cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_DEBUG', '-g -G -O0 -Xcompiler -O0')) @@ -481,7 +457,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cuda_stack_size = int(spec.variants['cuda_stack_size'].value) if 0 < cuda_stack_size: cfg.write(cmake_cache_option('ENABLE_CUDA_STACK_SIZE', True, "Adjust the CUDA stack size limit")) - cfg.write(cmake_cache_entry('CUDA_STACK_SIZE', cuda_stack_size, "CUDA stack size in KB")) + cfg.write(cmake_cache_string('CUDA_STACK_SIZE', cuda_stack_size, "CUDA stack size in KB")) else: cfg.write(cmake_cache_option('ENABLE_CUDA', False)) @@ -492,13 +468,13 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+rocm' in spec: cfg.write(cmake_cache_option('ENABLE_HIP', True)) cfg.write(cmake_cache_string('CMAKE_HIP_STANDARD', spec.variants['cxxstd'].value)) - cfg.write(cmake_cache_entry('CMAKE_HIP_COMPILER', spec['hip'].prefix.bin.hipcc)) + cfg.write(cmake_cache_path('CMAKE_HIP_COMPILER', spec['hip'].prefix.bin.hipcc)) if not spec.satisfies('amdgpu_target=none'): cmake_hip_archs = ";".join(spec.variants["amdgpu_target"].value) cfg.write(cmake_cache_string('CMAKE_HIP_ARCHITECTURES', cmake_hip_archs)) - cfg.write(cmake_cache_entry('ROCM_PATH', spec['hip'].prefix)) + cfg.write(cmake_cache_path('ROCM_PATH', spec['hip'].prefix)) else: cfg.write(cmake_cache_option('ENABLE_HIP', False)) @@ -507,14 +483,14 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) cfg.write(cmake_cache_option('ENABLE_CHAI', True)) - cfg.write(cmake_cache_entry('CHAI_DIR', spec['chai'].prefix)) + cfg.write(cmake_cache_path('CHAI_DIR', spec['chai'].prefix)) - cfg.write(cmake_cache_entry('RAJA_DIR', spec['raja'].prefix)) + cfg.write(cmake_cache_path('RAJA_DIR', spec['raja'].prefix)) cfg.write(cmake_cache_option('ENABLE_UMPIRE', True)) - cfg.write(cmake_cache_entry('UMPIRE_DIR', spec['umpire'].prefix)) + cfg.write(cmake_cache_path('UMPIRE_DIR', spec['umpire'].prefix)) - cfg.write(cmake_cache_entry('CAMP_DIR', spec['camp'].prefix)) + cfg.write(cmake_cache_path('CAMP_DIR', spec['camp'].prefix)) # yapf: disable io_tpls = ( @@ -534,8 +510,8 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+caliper' in spec: cfg.write(cmake_cache_option('ENABLE_CALIPER', True)) - cfg.write(cmake_cache_entry('CALIPER_DIR', spec['caliper'].prefix)) - cfg.write(cmake_cache_entry('ADIAK_DIR', spec['adiak'].prefix)) + cfg.write(cmake_cache_path('CALIPER_DIR', spec['caliper'].prefix)) + cfg.write(cmake_cache_path('ADIAK_DIR', spec['adiak'].prefix)) for tpl, cmake_name, enable in io_tpls: if enable: @@ -552,7 +528,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): raise KeyError("No zlib provider (zlib/zlib-ng/zlib-api) found in {0}".format(spec)) else: dep_spec = spec[tpl] - cfg.write(cmake_cache_entry('{}_DIR'.format(cmake_name), dep_spec.prefix)) + cfg.write(cmake_cache_path('{}_DIR'.format(cmake_name), dep_spec.prefix)) else: cfg.write(cmake_cache_option('ENABLE_{}'.format(cmake_name), False)) @@ -562,18 +538,12 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if spec["blas"].name == "intel-oneapi-mkl": cfg.write(cmake_cache_option('ENABLE_MKL', True)) - cfg.write(cmake_cache_entry('MKL_INCLUDE_DIRS', spec['intel-oneapi-mkl'].prefix.include)) + cfg.write(cmake_cache_path('MKL_INCLUDE_DIRS', spec['intel-oneapi-mkl'].prefix.include)) cfg.write(cmake_cache_list('MKL_LIBRARIES', spec['intel-oneapi-mkl'].libs)) elif spec["blas"].name == "mkl": cfg.write(cmake_cache_option('ENABLE_MKL', True)) - cfg.write(cmake_cache_entry('MKL_INCLUDE_DIRS', spec['intel-mkl'].prefix.include)) + cfg.write(cmake_cache_path('MKL_INCLUDE_DIRS', spec['intel-mkl'].prefix.include)) cfg.write(cmake_cache_list('MKL_LIBRARIES', spec['intel-mkl'].libs)) - elif spec["blas"].name == "essl": - cfg.write(cmake_cache_option('ENABLE_ESSL', True)) - cfg.write(cmake_cache_entry('ESSL_INCLUDE_DIRS', spec['essl'].prefix.include)) - cfg.write(cmake_cache_list('ESSL_LIBRARIES', spec['blas'].libs)) - - cfg.write(cmake_cache_option('FORTRAN_MANGLE_NO_UNDERSCORE', True)) else: cfg.write(cmake_cache_list('BLAS_LIBRARIES', spec['blas'].libs)) cfg.write(cmake_cache_list('LAPACK_LIBRARIES', spec['lapack'].libs)) @@ -599,7 +569,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if enable: if tpl == 'hypredrive': cfg.write(cmake_cache_option('ENABLE_HYPREDRV', True)) - cfg.write(cmake_cache_entry('{}_DIR'.format(cmake_name), spec[tpl].prefix)) + cfg.write(cmake_cache_path('{}_DIR'.format(cmake_name), spec[tpl].prefix)) if tpl == 'hypre' and '+cuda' in spec: cfg.write(cmake_cache_string('ENABLE_HYPRE_DEVICE', "CUDA")) @@ -622,8 +592,8 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('# Python\n') cfg.write('#{0}\n\n'.format('-' * 80)) - cfg.write(cmake_cache_entry('Python3_ROOT_DIR', os.path.join(spec['python'].prefix))) - cfg.write(cmake_cache_entry('Python3_EXECUTABLE', os.path.join(spec['python'].prefix.bin, 'python3'))) + cfg.write(cmake_cache_path('Python3_ROOT_DIR', os.path.join(spec['python'].prefix))) + cfg.write(cmake_cache_path('Python3_EXECUTABLE', os.path.join(spec['python'].prefix.bin, 'python3'))) if '+pygeosx' in spec: cfg.write(cmake_cache_option('ENABLE_PYGEOSX', True)) @@ -635,10 +605,10 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) if '+docs' in spec: sphinx_bin_dir = spec['py-sphinx'].prefix.bin - cfg.write(cmake_cache_entry('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) + cfg.write(cmake_cache_path('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) doxygen_bin_dir = spec['doxygen'].prefix.bin - cfg.write(cmake_cache_entry('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) + cfg.write(cmake_cache_path('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) else: cfg.write(cmake_cache_option('ENABLE_DOCS', False)) cfg.write(cmake_cache_option('ENABLE_DOXYGEN', False)) @@ -651,14 +621,14 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('ENABLE_UNCRUSTIFY', '+uncrustify' in spec)) if '+uncrustify' in spec: cfg.write( - cmake_cache_entry('UNCRUSTIFY_EXECUTABLE', os.path.join(spec['uncrustify'].prefix.bin, 'uncrustify'))) + cmake_cache_path('UNCRUSTIFY_EXECUTABLE', os.path.join(spec['uncrustify'].prefix.bin, 'uncrustify'))) if '+addr2line' in spec: cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# addr2line\n') cfg.write('#{0}\n\n'.format('-' * 80)) cfg.write(cmake_cache_option('ENABLE_ADDR2LINE', True)) - cfg.write(cmake_cache_entry('ADDR2LINE_EXEC', os.path.join(spec['addr2line'].prefix.bin, 'addr2line'))) + cfg.write(cmake_cache_path('ADDR2LINE_EXEC', os.path.join(spec['addr2line'].prefix.bin, 'addr2line'))) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Other\n') @@ -666,7 +636,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+mathpresso' in spec: cfg.write(cmake_cache_option('ENABLE_MATHPRESSO', True)) - cfg.write(cmake_cache_entry('MATHPRESSO_DIR', spec['mathpresso'].prefix)) + cfg.write(cmake_cache_path('MATHPRESSO_DIR', spec['mathpresso'].prefix)) cfg.write(cmake_cache_option('ENABLE_XML_UPDATES', True)) else: cfg.write(cmake_cache_option('ENABLE_MATHPRESSO', False)) @@ -689,9 +659,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('GEOS_BUILD_SHARED_LIBS', False)) # ATS - # Lassen - if sys_type in ('blueos_3_ppc64le_ib_p9'): - cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--ats jsrun_omp --ats jsrun_bind=packed')) # Dane/Matrix if sys_type in ('toss_4_x86_64_ib'): cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--machine slurm112')) @@ -752,15 +719,15 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write("#{0}\n".format("-" * 80)) cfg.write("# Compilers\n") cfg.write("#{0}\n\n".format("-" * 80)) - cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) + cfg.write(cmake_cache_path("CMAKE_C_COMPILER", c_compiler)) cflags = ' '.join(spec.compiler_flags['cflags']) if cflags: - cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) + cfg.write(cmake_cache_string("CMAKE_C_FLAGS", cflags)) - cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) + cfg.write(cmake_cache_path("CMAKE_CXX_COMPILER", cpp_compiler)) cxxflags = ' '.join(spec.compiler_flags['cxxflags']) if cxxflags: - cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) + cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS", cxxflags)) release_flags = "-O3 -DNDEBUG" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_RELEASE", release_flags)) @@ -769,9 +736,6 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): debug_flags = "-g" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) - if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) - cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Cuda\n') cfg.write('#{0}\n\n'.format('-' * 80)) @@ -780,9 +744,9 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string('CMAKE_CUDA_STANDARD', spec.variants['cxxstd'].value)) cudatoolkitdir = spec['cuda'].prefix - cfg.write(cmake_cache_entry('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) + cfg.write(cmake_cache_path('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) cudacompiler = '${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc' - cfg.write(cmake_cache_entry('CMAKE_CUDA_COMPILER', cudacompiler)) + cfg.write(cmake_cache_path('CMAKE_CUDA_COMPILER', cudacompiler)) cmake_cuda_flags = ('-restrict --expt-extended-lambda -Werror ' 'cross-execution-space-call,reorder,' @@ -812,14 +776,14 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) cfg.write(cmake_cache_option('ENABLE_CHAI', True)) - cfg.write(cmake_cache_entry('CHAI_DIR', spec['chai'].prefix)) + cfg.write(cmake_cache_path('CHAI_DIR', spec['chai'].prefix)) - cfg.write(cmake_cache_entry('RAJA_DIR', spec['raja'].prefix)) + cfg.write(cmake_cache_path('RAJA_DIR', spec['raja'].prefix)) cfg.write(cmake_cache_option('ENABLE_UMPIRE', True)) - cfg.write(cmake_cache_entry('UMPIRE_DIR', spec['umpire'].prefix)) + cfg.write(cmake_cache_path('UMPIRE_DIR', spec['umpire'].prefix)) - cfg.write(cmake_cache_entry('CAMP_DIR', spec['camp'].prefix)) + cfg.write(cmake_cache_path('CAMP_DIR', spec['camp'].prefix)) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# IO TPLs\n') @@ -827,18 +791,18 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+caliper' in spec: cfg.write(cmake_cache_option('ENABLE_CALIPER', True)) - cfg.write(cmake_cache_entry('CALIPER_DIR', spec['caliper'].prefix)) - cfg.write(cmake_cache_entry('adiak_DIR', spec['adiak'].prefix + '/lib/cmake/adiak')) + cfg.write(cmake_cache_path('CALIPER_DIR', spec['caliper'].prefix)) + cfg.write(cmake_cache_path('adiak_DIR', spec['adiak'].prefix + '/lib/cmake/adiak')) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Documentation\n') cfg.write('#{0}\n\n'.format('-' * 80)) if '+docs' in spec: sphinx_bin_dir = spec['py-sphinx'].prefix.bin - cfg.write(cmake_cache_entry('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) + cfg.write(cmake_cache_path('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) doxygen_bin_dir = spec['doxygen'].prefix.bin - cfg.write(cmake_cache_entry('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) + cfg.write(cmake_cache_path('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) else: cfg.write(cmake_cache_option('ENABLE_DOXYGEN', False)) cfg.write(cmake_cache_option('ENABLE_SPHINX', False)) @@ -852,7 +816,7 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('# addr2line\n') cfg.write('#{0}\n\n'.format('-' * 80)) cfg.write(cmake_cache_option('ENABLE_ADDR2LINE', True)) - cfg.write(cmake_cache_entry('ADDR2LINE_EXEC', os.path.join(spec['addr2line'].prefix.bin, 'addr2line'))) + cfg.write(cmake_cache_path('ADDR2LINE_EXEC', os.path.join(spec['addr2line'].prefix.bin, 'addr2line'))) def cmake_args(self): pass