From 25978987aadc916b73363d66362e84bceb7738bf Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 16 Mar 2026 09:57:10 -0400 Subject: [PATCH 01/27] Add rocm CI --- .github/workflows/docker_build_tpls.yml | 20 +- docker/spack-rocm.yaml | 481 ++++++++++++++++++ docker/spack.yaml | 140 +++-- docker/tpl-centos-gcc-cuda.Dockerfile | 14 +- .../tpl-rockylinux-clang-cuda-12.Dockerfile | 4 + docker/tpl-rockylinux-gcc-cuda-12.Dockerfile | 4 + docker/tpl-ubuntu-clang-cuda.Dockerfile | 20 + docker/tpl-ubuntu-clang.Dockerfile | 25 +- docker/tpl-ubuntu-gcc.Dockerfile | 20 +- docker/tpl-ubuntu-hip.Dockerfile | 178 +++++++ scripts/spack_configs/pangea-4/spack.yaml | 1 - scripts/spack_configs/versions.yaml | 4 +- .../spack_packages/packages/chai/package.py | 12 + .../spack_packages/packages/geosx/package.py | 151 +++--- 14 files changed, 938 insertions(+), 136 deletions(-) create mode 100644 docker/spack-rocm.yaml create mode 100644 docker/tpl-ubuntu-hip.Dockerfile diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index c69ecb5c..d03b6935 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -79,6 +79,13 @@ jobs: # SPEC: "~pygeosx~openmp cuda_arch=86 %gcc@10.1.0 ^cuda@12.4.0+allow-unsupported-compilers" # INSTALL_DIR_ROOT: /oak/stanford/groups/tchelepi/geos-sherlock/GPU # RUNS_ON: ubuntu-latest + - name: Ubuntu (24.04, amdclang 19 + rocm 6.4.3) + DOCKER_REPOSITORY: geosx/ubuntu24.04-amdclang19.0.0-rocm6.4.3 + TPL_DOCKERFILE: docker/tpl-ubuntu-hip.Dockerfile + DOCKER_ROOT_IMAGE: rocm/dev-ubuntu-24.04:6.4.3 + RUNS_ON: streak2 + NPROC: 8 + DOCKER_RUN_ARGS: "--cpus=8 --memory=128g -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.crt:/certs/ca-bundle.crt:ro" steps: - name: Checkout @@ -88,7 +95,7 @@ jobs: lfs: true - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Print environment run: printenv @@ -99,8 +106,15 @@ jobs: # 1. Copy the host's CA bundle into the Docker build context cp /etc/pki/tls/certs/ca-bundle.crt ./ca-bundle.crt - # 2. Inject the COPY and update-ca-trust commands right before dnf runs - sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' ${{ matrix.TPL_DOCKERFILE }} + # 2. Inject distro-specific trust-store setup into the Dockerfile. + if grep -q 'RUN dnf clean all' "${{ matrix.TPL_DOCKERFILE }}"; then + sed -i '/RUN dnf clean all/i COPY ca-bundle.crt /etc/pki/ca-trust/source/anchors/ca-bundle.crt\nRUN update-ca-trust extract' "${{ matrix.TPL_DOCKERFILE }}" + elif grep -q 'apt-get update' "${{ matrix.TPL_DOCKERFILE }}"; then + perl -0pi -e 's/(RUN .*?apt-get update)/COPY ca-bundle.crt \/usr\/local\/share\/ca-certificates\/host-ca.crt\nRUN update-ca-certificates\n\n$1/s' "${{ matrix.TPL_DOCKERFILE }}" + else + echo "Could not find a CA-injection hook in ${{ matrix.TPL_DOCKERFILE }}" + exit 1 + fi - name: Run the docker build docker script env: diff --git a/docker/spack-rocm.yaml b/docker/spack-rocm.yaml new file mode 100644 index 00000000..1abd4e46 --- /dev/null +++ b/docker/spack-rocm.yaml @@ -0,0 +1,481 @@ +spack: + config: + install_tree: + root: $spack/.. + projections: + all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' + misc_cache: $spack/../misc_cache + test_stage: $spack/../test_stage + build_stage: $spack/../build_stage + + # Regular TPLs do not need views + view: false + + # Include shared variants and versions + include: + - ../defaults.yaml + - ../versions.yaml + + toolchains: + clang-15: + - spec: '%c=llvm@15.0.7+clang~flang~lld~lldb' + when: '%c' + - spec: '%cxx=llvm@15.0.7+clang~flang~lld~lldb' + when: '%cxx' + - spec: '%fortran=gcc@11.4.0' + when: '%fortran' + - spec: '%openmpi@4.1.2 %llvm@15.0.7' + when: '%mpi' + clang-10: + - spec: '%c=llvm@10.0.0+clang~flang~lld~lldb openmp=project' + when: '%c' + - spec: '%cxx=llvm@10.0.0+clang~flang~lld~lldb openmp=project' + when: '%cxx' + - spec: '%fortran=gcc@9.4.0' + when: '%fortran' + - spec: '%openmpi@4.1.2 %llvm@10.0.0' + when: '%mpi' + gcc-9: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@9.4.0' + when: '%c' + - spec: '%cxx=gcc@9.4.0' + when: '%cxx' + - spec: '%fortran=gcc@9.4.0' + when: '%fortran' + - spec: '%openmpi %gcc@9.4.0' + when: '%mpi' + gcc-10: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@10.5.0' + when: '%c' + - spec: '%cxx=gcc@10.5.0' + when: '%cxx' + - spec: '%fortran=gcc@10.5.0' + when: '%fortran' + - spec: '%openmpi %gcc@10.5.0' + when: '%mpi' + gcc-11: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@11.4.0' + when: '%c' + - spec: '%cxx=gcc@11.4.0' + when: '%cxx' + - spec: '%fortran=gcc@11.4.0' + when: '%fortran' + - spec: '%openmpi %gcc@11.4.0' + when: '%mpi' + gcc-12: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@12.3.0' + when: '%c' + - spec: '%cxx=gcc@12.3.0' + when: '%cxx' + - spec: '%fortran=gcc@12.3.0' + when: '%fortran' + - spec: '%openmpi %gcc@12.3.0' + when: '%mpi' + gcc-13: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@13.3.0' + when: '%c' + - spec: '%cxx=gcc@13.3.0' + when: '%cxx' + - spec: '%fortran=gcc@13.3.0' + when: '%fortran' + - spec: '%openmpi@4.1.2 %gcc@13.3.0' + when: '%mpi' + amdclang-19: + - spec: '%c=llvm-amdgpu@6.4.3' + when: '%c' + - spec: '%cxx=llvm-amdgpu@6.4.3' + when: '%cxx' + - spec: '%fortran=gcc@13.3.0' + when: '%fortran' + - spec: '%openmpi@4.1.2 %llvm-amdgpu@6.4.3' + when: '%mpi' + + packages: + all: + target: [x86_64] + providers: + mpi: [openmpi] + blas: [netlib-lapack] + lapack: [netlib-lapack] + c: [llvm, gcc] + cxx: [llvm, gcc] + fortran: [gcc] + + mpi: + require: + - openmpi@4.1.2 + + llvm: + externals: + - spec: llvm@10.0.0+clang~flang~lld~lldb openmp=project + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang + cxx: /usr/bin/clang++ + - spec: llvm@15.0.7+clang~flang~lld~lldb + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/clang-15 + cxx: /usr/bin/clang++-15 + llvm-amdgpu: + buildable: false + externals: + - spec: llvm-amdgpu@6.4.3 + prefix: /opt/rocm-6.4.3/lib/llvm + extra_attributes: + compilers: + c: /usr/local/bin/amdclang-gcc13 + cxx: /usr/local/bin/amdclang++-gcc13 + environment: + set: + BISON: bison + FLEX: flex + CC: /usr/local/bin/amdclang-gcc13 + CXX: /usr/local/bin/amdclang++-gcc13 + FC: /usr/bin/gfortran-13 + F77: /usr/bin/gfortran-13 + MPICC: /usr/bin/mpicc + MPICXX: /usr/bin/mpicxx + MPIEXEC: /usr/bin/mpirun + OMPI_CC: /usr/local/bin/amdclang-gcc13 + OMPI_CXX: /usr/local/bin/amdclang++-gcc13 + HIPCXX: /usr/local/bin/amdclang++-gcc13 + CMAKE_HIP_COMPILER: /usr/local/bin/amdclang++-gcc13 + gcc: + externals: + - spec: gcc@9.4.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-9 + cxx: /usr/bin/g++-9 + fortran: /usr/bin/gfortran-9 + - spec: gcc@10.5.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-10 + cxx: /usr/bin/g++-10 + fortran: /usr/bin/gfortran-10 + - spec: gcc@11.4.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-11 + cxx: /usr/bin/g++-11 + fortran: /usr/bin/gfortran-11 + - spec: gcc@12.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-12 + cxx: /usr/bin/g++-12 + fortran: /usr/bin/gfortran-12 + - spec: gcc@13.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-13 + cxx: /usr/bin/g++-13 + fortran: /usr/bin/gfortran-13 + cuda: + buildable: False + externals: + - spec: cuda@11.8.0 +allow-unsupported-compilers + prefix: /usr/local/cuda + hip: + buildable: false + externals: + - spec: hip@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + rocprim: + buildable: false + externals: + - spec: rocprim@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + rocsparse: + buildable: false + externals: + - spec: rocsparse@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + roctracer: + buildable: false + externals: + - spec: roctracer@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + rocblas: + buildable: false + externals: + - spec: rocblas@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + rocrand: + buildable: false + externals: + - spec: rocrand@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + rocsolver: + buildable: false + externals: + - spec: rocsolver@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + rocthrust: + buildable: false + externals: + - spec: rocthrust@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + hipblas: + buildable: false + externals: + - spec: hipblas@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + hipsparse: + buildable: false + externals: + - spec: hipsparse@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + hipfft: + buildable: false + externals: + - spec: hipfft@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + hipsolver: + buildable: false + externals: + - spec: hipsolver@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + hiprand: + buildable: false + externals: + - spec: hiprand@6.4.3 +rocm + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + rocm-device-libs: + buildable: false + externals: + - spec: rocm-device-libs@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + hsa-rocr-dev: + buildable: false + externals: + - spec: hsa-rocr-dev@6.4.3 + prefix: /opt/rocm-6.4.3 + require: + - when: "%llvm-amdgpu@6.4.3" + spec: "@6.4.3" + + autoconf: + version: [2.71] + buildable: false + externals: + - spec: autoconf@2.71 + prefix: /usr + automake: + version: [1.16.5] + buildable: false + externals: + - spec: automake@1.16.5 + prefix: /usr + cmake: + version: [3.28.3] + buildable: false + externals: + - spec: cmake@3.28.3 + prefix: /usr/local + findutils: + version: [4.7.0] + buildable: false + externals: + - spec: findutils@4.7.0 + prefix: /usr + m4: + buildable: false + externals: + - spec: m4@1.4.18 + prefix: /usr + mpfr: + buildable: false + externals: + - spec: mpfr@6.0.2 + prefix: /usr + netlib-lapack: + buildable: false + externals: + - spec: netlib-lapack@3.10.0 + prefix: /usr + netlib-blas: + buildable: false + externals: + - spec: netlib-blas@3.10.0 + prefix: /usr + openmpi: + buildable: false + externals: + - spec: openmpi@4.1.2 %llvm@10.0.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + modules: [mpi] + - spec: openmpi@4.1.2 %llvm@15.0.7 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + modules: [mpi] + - spec: openmpi@4.1.2 %llvm-amdgpu@6.4.3 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.2 %gcc@9.4.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.2 %gcc@10.5.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.2 %gcc@11.4.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.2 %gcc@12.3.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.2 %gcc@13.3.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi + perl: + buildable: false + externals: + - spec: perl@5.34.0 + prefix: /usr + pkg-config: + buildable: false + externals: + - spec: pkg-config@0.29.2 + prefix: /usr + py-sphinx: + buildable: false + externals: + - spec: py-sphinx@4.3.2 + prefix: /usr + python: + buildable: false + externals: + - spec: python@3.10.12 + prefix: /usr + readline: + buildable: false + externals: + - spec: readline@8.0 + prefix: /usr + tar: + buildable: false + externals: + - spec: tar@1.34 + prefix: /usr + unzip: + buildable: false + externals: + - spec: unzip@6.0 + prefix: /usr + xz: + buildable: false + externals: + - spec: xz@5.2.5 + prefix: /usr + zlib: + buildable: false + externals: + - spec: zlib@1.2.11 + prefix: /usr + z3: + buildable: false + externals: + - spec: z3@4.8.12 + prefix: /usr + gettext: + buildable: false + externals: + - spec: gettext@0.21 + prefix: /usr + bison: + buildable: false + externals: + - spec: bison@3.8.2 + prefix: /usr + # Rely on spack to build a newer flex for scotch + # flex: + # buildable: false + # externals: + # - spec: flex@2.6.4 + # prefix: /usr + bzip2: + buildable: false + externals: + - spec: bzip2@1.0.8 + prefix: /usr + help2man: + buildable: false + externals: + - spec: help2man@1.49.3 + prefix: /usr + libtool: + buildable: false + externals: + - spec: libtool@2.4.7 + prefix: /usr + gmp: + buildable: false + externals: + - spec: gmp@6.3.0 + prefix: /usr + pugixml: + buildable: true + gmake: + buildable: false + externals: + - spec: gmake@4.4.1 + prefix: /usr + ninja: + buildable: false + externals: + - spec: ninja@1.11.1 + prefix: /usr diff --git a/docker/spack.yaml b/docker/spack.yaml index d59364f2..77a134a6 100644 --- a/docker/spack.yaml +++ b/docker/spack.yaml @@ -6,8 +6,7 @@ spack: all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' misc_cache: $spack/../misc_cache test_stage: $spack/../test_stage - build_stage:: - - $spack/../build_stage + build_stage: $spack/../build_stage # Regular TPLs do not need views view: false @@ -19,22 +18,22 @@ spack: toolchains: clang-15: - - spec: '%[virtuals=c]llvm@15.0.7+clang~flang~lld~lldb' + - spec: '%c=llvm@15.0.7+clang~flang~lld~lldb' when: '%c' - - spec: '%[virtuals=cxx]llvm@15.0.7+clang~flang~lld~lldb' + - spec: '%cxx=llvm@15.0.7+clang~flang~lld~lldb' when: '%cxx' - - spec: '%[virtuals=fortran]gcc@11.4.0' + - spec: '%fortran=gcc@11.4.0' when: '%fortran' - - spec: '%openmpi' + - spec: '%openmpi@4.1.2 %llvm@15.0.7' when: '%mpi' clang-10: - - spec: '%[virtuals=c]llvm@10.0.0+clang~flang~lld~lldb openmp=project' + - spec: '%c=llvm@10.0.0+clang~flang~lld~lldb openmp=project' when: '%c' - - spec: '%[virtuals=cxx]llvm@10.0.0+clang~flang~lld~lldb openmp=project' + - spec: '%cxx=llvm@10.0.0+clang~flang~lld~lldb openmp=project' when: '%cxx' - - spec: '%[virtuals=fortran]gcc@9.4.0' + - spec: '%fortran=gcc@9.4.0' when: '%fortran' - - spec: '%openmpi' + - spec: '%openmpi@4.1.2 %llvm@10.0.0' when: '%mpi' gcc-9: - spec: cxxflags='-pthread' @@ -80,22 +79,35 @@ spack: when: '%fortran' - spec: '%openmpi %gcc@12.3.0' when: '%mpi' + gcc-13: + - spec: cxxflags='-pthread' + - spec: cflags='-pthread' + - spec: '%c=gcc@13.3.0' + when: '%c' + - spec: '%cxx=gcc@13.3.0' + when: '%cxx' + - spec: '%fortran=gcc@13.3.0' + when: '%fortran' + - spec: '%openmpi@4.1.2 %gcc@13.3.0' + when: '%mpi' packages: all: target: [x86_64] + providers: + mpi: [openmpi] + blas: [netlib-lapack] + lapack: [netlib-lapack] + c: [llvm, gcc] + cxx: [llvm, gcc] + fortran: [gcc] + # Require mpi: require: - openmpi@4.1.2 - blas: - require: - - "netlib-lapack" - lapack: - require: - - "netlib-lapack" - + # Compiler packages llvm: externals: - spec: llvm@10.0.0+clang~flang~lld~lldb openmp=project @@ -110,7 +122,6 @@ spack: compilers: c: /usr/bin/clang-15 cxx: /usr/bin/clang++-15 - gcc: externals: - spec: gcc@9.4.0 languages:='c,c++,fortran' @@ -141,7 +152,21 @@ spack: c: /usr/bin/gcc-12 cxx: /usr/bin/g++-12 fortran: /usr/bin/gfortran-12 + - spec: gcc@13.3.0 languages:='c,c++,fortran' + prefix: /usr + extra_attributes: + compilers: + c: /usr/bin/gcc-13 + cxx: /usr/bin/g++-13 + fortran: /usr/bin/gfortran-13 + # CUDA packages + cuda: + buildable: False + externals: + - spec: cuda@11.8.0 +allow-unsupported-compilers + prefix: /usr/local/cuda + # System packages autoconf: version: [2.71] buildable: false @@ -160,11 +185,6 @@ spack: externals: - spec: cmake@3.28.3 prefix: /usr/local - cuda: - buildable: False - externals: - - spec: cuda@11.8.0 +allow-unsupported-compilers - prefix: /usr/local/cuda findutils: version: [4.7.0] buildable: false @@ -181,7 +201,6 @@ spack: externals: - spec: mpfr@6.0.2 prefix: /usr - netlib-lapack: buildable: false externals: @@ -195,22 +214,22 @@ spack: openmpi: buildable: false externals: - # Ubuntu 22.04 (apt) provides OpenMPI 4.1.2. We treat it as external for all toolchains - # to prevent Spack from building OpenMPI (which may otherwise drift to v5). - - spec: openmpi@4.1.2 %clang@15.0.7 - prefix: /usr + - spec: openmpi@4.1.2 %llvm@10.0.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi modules: [mpi] - - spec: openmpi@4.1.2 %clang@10.0.0 - prefix: /usr + - spec: openmpi@4.1.2 %llvm@15.0.7 + prefix: /usr/lib/x86_64-linux-gnu/openmpi modules: [mpi] - spec: openmpi@4.1.2 %gcc@9.4.0 - prefix: /usr + prefix: /usr/lib/x86_64-linux-gnu/openmpi - spec: openmpi@4.1.2 %gcc@10.5.0 - prefix: /usr + prefix: /usr/lib/x86_64-linux-gnu/openmpi - spec: openmpi@4.1.2 %gcc@11.4.0 - prefix: /usr + prefix: /usr/lib/x86_64-linux-gnu/openmpi - spec: openmpi@4.1.2 %gcc@12.3.0 - prefix: /usr + prefix: /usr/lib/x86_64-linux-gnu/openmpi + - spec: openmpi@4.1.2 %gcc@13.3.0 + prefix: /usr/lib/x86_64-linux-gnu/openmpi perl: buildable: false externals: @@ -256,3 +275,56 @@ spack: externals: - spec: zlib@1.2.11 prefix: /usr + z3: + buildable: false + externals: + - spec: z3@4.8.12 + prefix: /usr + gettext: + buildable: false + externals: + - spec: gettext@0.21 + prefix: /usr + bison: + buildable: false + externals: + - spec: bison@3.8.2 + prefix: /usr + # Rely on spack to build a newer flex for scotch + # flex: + # buildable: false + # externals: + # - spec: flex@2.6.4 + # prefix: /usr + bzip2: + buildable: false + externals: + - spec: bzip2@1.0.8 + prefix: /usr + help2man: + buildable: false + externals: + - spec: help2man@1.49.3 + prefix: /usr + libtool: + buildable: false + externals: + - spec: libtool@2.4.7 + prefix: /usr + gmp: + buildable: false + externals: + - spec: gmp@6.3.0 + prefix: /usr + pugixml: + buildable: true + gmake: + buildable: false + externals: + - spec: gmake@4.4.1 + prefix: /usr + ninja: + buildable: false + externals: + - spec: ninja@1.11.1 + prefix: /usr diff --git a/docker/tpl-centos-gcc-cuda.Dockerfile b/docker/tpl-centos-gcc-cuda.Dockerfile index dbea5b7b..fe16f984 100644 --- a/docker/tpl-centos-gcc-cuda.Dockerfile +++ b/docker/tpl-centos-gcc-cuda.Dockerfile @@ -1,5 +1,5 @@ # NOTE: see docker/tpl-ubuntu-gcc.Dockerfile for detailed comments -ARG TMP_DIR=/tmp +ARG TMP_DIR=/tmp ARG SRC_DIR=$TMP_DIR/thirdPartyLibs ARG BLD_DIR=$TMP_DIR/build @@ -11,12 +11,12 @@ ENV GEOSX_TPL_DIR=$INSTALL_DIR RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \ sed -i s/^#.*baseurl=http/baseurl=https/g /etc/yum.repos.d/*.repo && \ - sed -i s/^mirrorlist=http/#mirrorlist=https/g /etc/yum.repos.d/*.repo + sed -i s/^mirrorlist=http/#mirrorlist=https/g /etc/yum.repos.d/*.repo # Using gcc 8.3.1 provided by the Software Collections (SCL). RUN yum install -y \ centos-release-scl - + # Modify the SCLo repository configuration RUN sed -i 's|^mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-SCLo-scl.repo && \ sed -i 's|^baseurl=http://mirror.centos.org/centos/\$releasever/sclo/\$basearch/rh|baseurl=http://vault.centos.org/7.9.2009/sclo/x86_64/rh|g' /etc/yum.repos.d/CentOS-SCLo-scl.repo && \ @@ -25,7 +25,7 @@ RUN sed -i 's|^mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-SCLo-scl.repo # Install necessary tools and update the system RUN yum -y update && \ - yum -y install yum-utils + yum -y install yum-utils RUN yum install -y \ devtoolset-8-gcc \ @@ -48,6 +48,10 @@ RUN yum -y install \ # xz \ unzip \ bzip2 \ + flex \ + bison \ + gettext \ + help2man \ gnupg \ && pip3 install virtualenv @@ -65,6 +69,8 @@ ARG BLD_DIR RUN yum install -y \ tbb-devel \ make \ + autopoint \ + autotools-dev \ bc \ file \ patch \ diff --git a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile index 85b0007f..bec7c84c 100644 --- a/docker/tpl-rockylinux-clang-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-clang-cuda-12.Dockerfile @@ -28,7 +28,11 @@ RUN dnf clean all && \ unzip \ mpfr-devel \ bzip2 \ + flex \ + bison \ + gettext \ gnupg \ + libtool \ xz \ python3-virtualenv diff --git a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile index de0b95a9..13227a47 100644 --- a/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile +++ b/docker/tpl-rockylinux-gcc-cuda-12.Dockerfile @@ -26,7 +26,11 @@ RUN dnf clean all && \ unzip \ mpfr-devel \ bzip2 \ + flex \ + bison \ + gettext \ gnupg \ + libtool \ xz \ python3-virtualenv diff --git a/docker/tpl-ubuntu-clang-cuda.Dockerfile b/docker/tpl-ubuntu-clang-cuda.Dockerfile index 1663462b..812f1557 100644 --- a/docker/tpl-ubuntu-clang-cuda.Dockerfile +++ b/docker/tpl-ubuntu-clang-cuda.Dockerfile @@ -34,6 +34,11 @@ RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime && \ libmpfr-dev \ lbzip2 \ bzip2 \ + flex \ + bison \ + gettext \ + help2man \ + libtool \ gnupg \ virtualenv @@ -52,10 +57,25 @@ RUN apt-get install -y --no-install-recommends \ libtbb-dev \ bc \ file \ + autopoint \ + autotools-dev \ + automake \ patch \ ca-certificates \ git +# OpenMPI hack for Ubuntu +RUN ln -s /usr/bin /usr/lib/x86_64-linux-gnu/openmpi + +# MPI environment variables +ENV CC=/usr/bin/clang \ + CXX=/usr/bin/clang++ \ + MPICC=/usr/bin/mpicc \ + MPICXX=/usr/bin/mpicxx \ + MPIEXEC=/usr/bin/mpirun \ + OMPI_CC=/usr/bin/clang \ + OMPI_CXX=/usr/bin/clang++ + # Run uberenv # Have to create install directory first for uberenv # -k flag is to ignore SSL errors diff --git a/docker/tpl-ubuntu-clang.Dockerfile b/docker/tpl-ubuntu-clang.Dockerfile index 4e0058ad..574353ea 100644 --- a/docker/tpl-ubuntu-clang.Dockerfile +++ b/docker/tpl-ubuntu-clang.Dockerfile @@ -39,6 +39,14 @@ RUN DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ libmpfr-dev \ lbzip2 \ bzip2 \ + flex \ + bison \ + gettext \ + help2man \ + libtool \ + autopoint \ + autotools-dev \ + automake \ gnupg \ virtualenv @@ -69,16 +77,17 @@ RUN apt-get install -y --no-install-recommends \ ca-certificates \ git -# Add MPI environment path info -ENV CC=/usr/bin/gcc-$GCC_MAJOR_VERSION \ - CXX=/usr/bin/g++-$GCC_MAJOR_VERSION \ +# OpenMPI hack for Ubuntu +RUN ln -s /usr/bin /usr/lib/x86_64-linux-gnu/openmpi + +# MPI environment variables +ENV CC=/usr/bin/clang-$CLANG_MAJOR_VERSION \ + CXX=/usr/bin/clang++-$CLANG_MAJOR_VERSION \ MPICC=/usr/bin/mpicc \ MPICXX=/usr/bin/mpicxx \ - MPIEXEC=/usr/bin/mpirun -# The multi-line definition of arguments does not seem happy -# when a variable uses the value of another variable previously defined on the same line. -ENV OMPI_CC=$CC \ - OMPI_CXX=$CXX + MPIEXEC=/usr/bin/mpirun \ + OMPI_CC=/usr/bin/clang-$CLANG_MAJOR_VERSION \ + OMPI_CXX=/usr/bin/clang++-$CLANG_MAJOR_VERSION # Run uberenv # Have to create install directory first for uberenv diff --git a/docker/tpl-ubuntu-gcc.Dockerfile b/docker/tpl-ubuntu-gcc.Dockerfile index 2f74d46c..68fbd86e 100644 --- a/docker/tpl-ubuntu-gcc.Dockerfile +++ b/docker/tpl-ubuntu-gcc.Dockerfile @@ -64,6 +64,11 @@ RUN DEBIAN_FRONTEND=noninteractive TZ=America/Los_Angeles \ libmpfr-dev \ lbzip2 \ bzip2 \ + gettext \ + bison \ + flex \ + help2man \ + libtool \ gnupg \ virtualenv @@ -73,15 +78,17 @@ RUN python3 -m pip install --upgrade pip && \ RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh +# OpenMPI hack for Ubuntu +RUN ln -s /usr/bin /usr/lib/x86_64-linux-gnu/openmpi + +# MPI environment variables ENV CC=/usr/bin/gcc-$GCC_MAJOR_VERSION \ CXX=/usr/bin/g++-$GCC_MAJOR_VERSION \ MPICC=/usr/bin/mpicc \ MPICXX=/usr/bin/mpicxx \ - MPIEXEC=/usr/bin/mpirun -# The multi-line definition of arguments does not seem happy -# when a variable uses the value of another variable previously defined on the same line. -ENV OMPI_CC=$CC \ - OMPI_CXX=$CXX + MPIEXEC=/usr/bin/mpirun \ + OMPI_CC=/usr/bin/gcc-$GCC_MAJOR_VERSION \ + OMPI_CXX=/usr/bin/g++-$GCC_MAJOR_VERSION # This stage is dedicated to TPLs uniquely. # A multi-stage build patern will allow to extract what we need for the GEOSX build. @@ -96,6 +103,9 @@ ARG GCC_MAJOR_VERSION RUN apt-get install -y --no-install-recommends \ libtbb-dev \ make \ + autopoint \ + autotools-dev \ + automake \ bc \ file \ # GEOS patches some tpl. Remove when it's not the case anymore. diff --git a/docker/tpl-ubuntu-hip.Dockerfile b/docker/tpl-ubuntu-hip.Dockerfile new file mode 100644 index 00000000..5ab3edde --- /dev/null +++ b/docker/tpl-ubuntu-hip.Dockerfile @@ -0,0 +1,178 @@ +# NOTE: see docker/tpl-ubuntu-gcc.Dockerfile for detailed comments +ARG TMP_DIR=/tmp +ARG SRC_DIR=$TMP_DIR/thirdPartyLibs +ARG BLD_DIR=$TMP_DIR/build + +# Base image is set by workflow via DOCKER_ROOT_IMAGE +ARG DOCKER_ROOT_IMAGE=rocm/dev-ubuntu-24.04:6.4.3 + +FROM ${DOCKER_ROOT_IMAGE} AS tpl_toolchain_intersect_geosx_toolchain +ARG SRC_DIR + +ARG INSTALL_DIR +ENV GEOSX_TPL_DIR=$INSTALL_DIR + +# Parameters +ARG AMDGPU_TARGET=gfx942 +ARG ROCM_VERSION=6.4.3 + +# Allow changing the number of cores used for building code via spack +ARG SPACK_BUILD_JOBS=4 +ENV SPACK_BUILD_JOBS=${SPACK_BUILD_JOBS} + +RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime && \ + apt-get update + +# Install system packages +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + wget \ + gnupg \ + ca-certificates \ + gcc-13 \ + g++-13 \ + gfortran-13 \ + libtbb12 \ + libtbbmalloc2 \ + libblas-dev \ + liblapack-dev \ + libz3-dev \ + zlib1g-dev \ + openmpi-bin \ + libopenmpi-dev \ + python3 \ + python3-dev \ + python3-pip \ + python3-sphinx \ + doxygen \ + pkg-config \ + xz-utils \ + gettext \ + bison \ + flex \ + bzip2 \ + help2man \ + libtool \ + libgmp-dev \ + unzip \ + libmpfr-dev \ + lbzip2 \ + bzip2 \ + gnupg \ + virtualenv \ + libpugixml-dev \ + roctracer-dev \ + rocsparse-dev \ + rocsolver-dev \ + rocblas-dev \ + hipblas-dev \ + hipsparse-dev \ + hipfft-dev \ + hipsolver-dev \ + hiprand-dev \ + rocprim-dev \ + rocrand-dev \ + rocthrust-dev \ + git && \ + rm -rf /var/lib/apt/lists/* + +# Install clingo for Spack (use pip without upgrading pip to avoid Debian conflict) +RUN python3 -m pip install clingo --break-system-packages + +# Install CMake +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh + +# OpenMPI hack for Ubuntu and provide amdclang wrappers with the GCC 13 system toolchain. +# clang++ inside the ROCm LLVM bin is also wrapped so that spack's cached_cmake.py, +# which hardcodes CMAKE_HIP_COMPILER to that path, gets the correct --gcc-toolchain. +RUN ln -s /usr/bin /usr/lib/x86_64-linux-gnu/openmpi && \ + printf '%s\n' '#!/bin/sh' "exec /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/amdclang --gcc-toolchain=/usr \"\$@\"" > /usr/local/bin/amdclang-gcc13 && \ + chmod +x /usr/local/bin/amdclang-gcc13 && \ + printf '%s\n' '#!/bin/sh' "exec /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/amdclang++ --gcc-toolchain=/usr \"\$@\"" > /usr/local/bin/amdclang++-gcc13 && \ + chmod +x /usr/local/bin/amdclang++-gcc13 && \ + rm -f /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/clang++ && \ + printf '%s\n' '#!/bin/sh' "exec /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/amdclang++ --gcc-toolchain=/usr \"\$@\"" > /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/clang++ && \ + chmod +x /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/clang++ && \ + ln -s /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/clang /usr/bin/clang && \ + ln -s /opt/rocm-${ROCM_VERSION}/lib/llvm/bin/clang++ /usr/bin/clang++ + +# Installing TPLs +FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain +ARG SRC_DIR +ARG BLD_DIR + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libtbb-dev \ + make \ + autopoint \ + autotools-dev \ + automake \ + ninja-build \ + bc \ + file \ + patch \ + ca-certificates \ + git && \ + rm -rf /var/lib/apt/lists/* + +# Run uberenv +# Have to create install directory first for uberenv +# -k flag is to ignore SSL errors +# --spack-debug to debug build +RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ + mkdir -p ${GEOSX_TPL_DIR} && \ + ./scripts/uberenv/uberenv.py \ + --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} %amdclang-19 ^caliper~papi~gotcha~sampler~libunwind~libdw" \ + --spack-env-file=${SRC_DIR}/docker/spack-rocm.yaml \ + --project-json=.uberenv_config.json \ + --prefix ${GEOSX_TPL_DIR} \ + -k && \ +# Remove host-config generated for LvArray + rm lvarray* && \ +# Rename and copy spack-generated host-config to root directory + cp *.cmake /spack-generated.cmake && \ +# Remove extraneous spack files + cd ${GEOSX_TPL_DIR} && \ + rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/ + +# Extract only TPLs from previous stage +FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain +ARG SRC_DIR + +COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR + +# Extract the generated host-config +COPY --from=tpl_toolchain /spack-generated.cmake / + +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get install -y --no-install-recommends \ + openssh-client \ + ca-certificates \ + curl \ + python3 \ + texlive \ + texlive-latex-extra \ + graphviz \ + libxml2-utils \ + git \ + ghostscript \ + ninja-build \ + python3-dev \ + python3-sphinx \ + python3-mpi4py \ + python3-scipy \ + python3-virtualenv \ + python3-matplotlib \ + python3-venv \ + python3-pytest && \ + rm -rf /var/lib/apt/lists/* + +RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-sccache.sh +ENV SCCACHE=/opt/sccache/bin/sccache + +# Helpful environment defaults for HIP +ENV ROCM_PATH=/opt/rocm-${ROCM_VERSION} +ENV HIP_PATH=${ROCM_PATH}/hip +ENV PATH=${ROCM_PATH}/bin:${ROCM_PATH}/llvm/bin:${PATH} +ENV LD_LIBRARY_PATH=${ROCM_PATH}/lib:${ROCM_PATH}/lib64:${ROCM_PATH}/llvm/lib +ENV CMAKE_HIP_ARCHITECTURES=${AMDGPU_TARGET} diff --git a/scripts/spack_configs/pangea-4/spack.yaml b/scripts/spack_configs/pangea-4/spack.yaml index 7070bb4d..47511c0c 100644 --- a/scripts/spack_configs/pangea-4/spack.yaml +++ b/scripts/spack_configs/pangea-4/spack.yaml @@ -68,7 +68,6 @@ spack: require: "@26.1" grpc: require: "@1.65.5-full-clone" - #### # external compiler gcc: diff --git a/scripts/spack_configs/versions.yaml b/scripts/spack_configs/versions.yaml index 293a75ba..26fc5811 100644 --- a/scripts/spack_configs/versions.yaml +++ b/scripts/spack_configs/versions.yaml @@ -14,9 +14,9 @@ # This file lists the package versions for geos's dependencies # packages: - # master - 01/23/26 + # master - 02/23/26 hypre: - require: "@git.8b0093306228fef1b92384d9face7fbe5a63b460=master" + require: "@git.7e403d172640473c07e8391f6a9fb69cd824d48a=master" # master - 04/12/20 uncrustify: diff --git a/scripts/spack_packages/packages/chai/package.py b/scripts/spack_packages/packages/chai/package.py index 16b68a56..bdc3f94b 100644 --- a/scripts/spack_packages/packages/chai/package.py +++ b/scripts/spack_packages/packages/chai/package.py @@ -1,3 +1,7 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + import os from spack.package import * @@ -6,4 +10,12 @@ class Chai(BuiltinChai): # Bypass llnl_link_helpers failure depends_on("fortran") + depends_on("c", type="build") + depends_on("cxx", type="build") + # From radiuss-packages (PR #143) + depends_on("blt@0.7.1:", type="build", when="@2025.09.0:") + depends_on("blt@0.7.0:", type="build", when="@2025.03.0:") + depends_on("umpire@2025.09.0:", when="@2025.09.0:") + depends_on("umpire@2025.03.0:", when="@2025.03.0:") + depends_on("umpire@2024.07.0", when="@2024.07.0") diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 91b7b6f0..4704eda8 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -9,21 +9,25 @@ import socket import os +import llnl.util.tty as tty + from os import environ as env from os.path import join as pjoin -from spack_repo.builtin.build_systems.cached_cmake import cmake_cache_path - from spack_repo.builtin.build_systems.cmake import CMakePackage from spack_repo.builtin.build_systems.cuda import CudaPackage +from spack_repo.builtin.build_systems.cached_cmake import ( + cmake_cache_option, + cmake_cache_path, + cmake_cache_string, +) # Tested specs are located at scripts/spack_configs/<$SYS_TYPE>/spack.yaml (e.g. %clang@10.0.1) - # WARNING: +petsc variant is yet to be tested. def cmake_cache_entry(name, value, comment=""): - """Generate a string for a cmake cache variable""" + """Generate a string for a cmake cache variable.""" return 'set(%s "%s" CACHE PATH "%s")\n\n' % (name, value, comment) @@ -36,19 +40,6 @@ def cmake_cache_list(name, value, comment=""): return 'set(%s %s CACHE STRING "%s")\n\n' % (name, join_str.join(value), comment) -def cmake_cache_string(name, string, comment=""): - """Generate a string for a cmake cache variable""" - - return 'set(%s "%s" CACHE STRING "%s")\n\n' % (name, string, comment) - - -def cmake_cache_option(name, boolean_value, comment=""): - """Generate a string for a cmake configuration option""" - - value = "ON" if boolean_value else "OFF" - return 'set(%s %s CACHE BOOL "%s")\n\n' % (name, value, comment) - - class Geosx(CMakePackage, CudaPackage, ROCmPackage): """GEOSX simulation framework.""" @@ -159,8 +150,8 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): # # Math # - depends_on("parmetis@4.0.3+int64~shared cflags='-fPIC' cxxflags='-fPIC'") - depends_on("metis +int64~shared cflags='-fPIC' cxxflags='-fPIC'") + depends_on("parmetis@4.0.3+int64+shared cflags='-fPIC' cxxflags='-fPIC'") + depends_on("metis +int64+shared cflags='-fPIC' cxxflags='-fPIC'") depends_on("superlu-dist@9.2.1 +int64 fflags='-fPIC'") depends_on("superlu-dist@9.2.1 +int64 fflags='-fPIC -ef'", when="%cce") @@ -185,6 +176,8 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): depends_on("hypre +cuda+superlu-dist+mixedint+mpi+umpire+unified-memory", when='+cuda') depends_on("hypre +rocm+superlu-dist+mixedint+mpi+umpire+unified-memory", when='+rocm') depends_on("hypre ~openmp", when="~openmp") + depends_on("hypre +openmp", when="+openmp") + depends_on("hypre +caliper", when="+caliper") depends_on("hypre +pic", when="~shared") depends_on("hypre +shared", when="+shared") @@ -251,22 +244,22 @@ def _get_sys_type(self, spec): return sys_type def _get_host_config_path(self, spec, lvarray=False): - var = '' - - if '+cuda' in spec: - var = '-'.join([var, 'cuda']) - var += "@" + str(spec['cuda'].version) - elif '+rocm' in spec: - var = '-'.join([var, 'rocm']) - var += "@" + str(spec['hip'].version) - + gpu_backend = "" + if "+cuda" in spec: + gpu_backend = f"cuda@{spec['cuda'].version}" + elif "+rocm" in spec: + gpu_backend = f"rocm@{spec['hip'].version}" hostname = socket.gethostname().rstrip('1234567890') if lvarray: hostname = "lvarray-" + hostname - host_config_path = "%s-%s-%s@%s%s.cmake" % (hostname, self._get_sys_type(spec), (str(spec.compiler.name)), str(spec.compiler.version),var) + host_config_path = "%s-%s-%s@%s%s.cmake" % (hostname, + self._get_sys_type(spec), + str(spec.compiler.name), + str(spec.compiler.version), + gpu_backend) dest_dir = self.stage.source_path host_config_path = os.path.abspath(pjoin(dest_dir, host_config_path)) @@ -328,15 +321,15 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write("#{0}\n".format("-" * 80)) cfg.write("# Compilers\n") cfg.write("#{0}\n\n".format("-" * 80)) - cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) + cfg.write(cmake_cache_path("CMAKE_C_COMPILER", c_compiler)) cflags = ' '.join(spec.compiler_flags['cflags']) if cflags: - cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) + cfg.write(cmake_cache_string("CMAKE_C_FLAGS", cflags)) - cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) + cfg.write(cmake_cache_path("CMAKE_CXX_COMPILER", cpp_compiler)) cxxflags = ' '.join(spec.compiler_flags['cxxflags']) if cxxflags: - cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) + cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS", cxxflags)) release_flags = "-O3 -DNDEBUG" if "clang" in self.compiler.cxx: @@ -348,7 +341,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) + cfg.write(cmake_cache_string("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) cfg.write("#{0}\n".format("-" * 80)) cfg.write("# CMake Standard\n") @@ -361,8 +354,8 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write("#{0}\n\n".format("-" * 80)) cfg.write(cmake_cache_option('ENABLE_MPI', True)) - cfg.write(cmake_cache_entry('MPI_C_COMPILER', spec['mpi'].mpicc)) - cfg.write(cmake_cache_entry('MPI_CXX_COMPILER', spec['mpi'].mpicxx)) + cfg.write(cmake_cache_path('MPI_C_COMPILER', spec['mpi'].mpicc)) + cfg.write(cmake_cache_path('MPI_CXX_COMPILER', spec['mpi'].mpicxx)) hostname = socket.gethostname().rstrip('1234567890') @@ -370,14 +363,14 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): and hostname != 'p3dev': cfg.write(cmake_cache_option('ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC', True)) if hostname in ('lassen', 'rzansel'): - cfg.write(cmake_cache_entry('MPIEXEC', 'lrun')) - cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) + cfg.write(cmake_cache_string('MPIEXEC', 'lrun')) + cfg.write(cmake_cache_string('MPIEXEC_NUMPROC_FLAG', '-n')) else: - cfg.write(cmake_cache_entry('MPIEXEC', 'jsrun')) + cfg.write(cmake_cache_string('MPIEXEC', 'jsrun')) cfg.write(cmake_cache_list('MPIEXEC_NUMPROC_FLAG', ['-g1', '--bind', 'rs', '-n'])) elif sys_type in ('toss_4_x86_64_ib_cray'): - cfg.write(cmake_cache_entry('MPIEXEC', 'srun')) - cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) + cfg.write(cmake_cache_string('MPIEXEC', 'srun')) + cfg.write(cmake_cache_string('MPIEXEC_NUMPROC_FLAG', '-n')) else: # Taken from cached_cmake class: # https://github.com/spack/spack/blob/develop/lib/spack/spack/build_systems/cached_cmake.py#L180-234 @@ -441,9 +434,9 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string('CMAKE_CUDA_STANDARD', spec.variants['cxxstd'].value)) cudatoolkitdir = spec['cuda'].prefix - cfg.write(cmake_cache_entry('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) + cfg.write(cmake_cache_path('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) cudacompiler = '${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc' - cfg.write(cmake_cache_entry('CMAKE_CUDA_COMPILER', cudacompiler)) + cfg.write(cmake_cache_path('CMAKE_CUDA_COMPILER', cudacompiler)) cmake_cuda_flags = ('-restrict --expt-extended-lambda -Werror ' 'cross-execution-space-call,reorder,' @@ -473,7 +466,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cuda_stack_size = int(spec.variants['cuda_stack_size'].value) if 0 < cuda_stack_size: cfg.write(cmake_cache_option('ENABLE_CUDA_STACK_SIZE', True, "Adjust the CUDA stack size limit")) - cfg.write(cmake_cache_entry('CUDA_STACK_SIZE', cuda_stack_size, "CUDA stack size in KB")) + cfg.write(cmake_cache_string('CUDA_STACK_SIZE', cuda_stack_size, "CUDA stack size in KB")) else: cfg.write(cmake_cache_option('ENABLE_CUDA', False)) @@ -484,13 +477,13 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+rocm' in spec: cfg.write(cmake_cache_option('ENABLE_HIP', True)) cfg.write(cmake_cache_string('CMAKE_HIP_STANDARD', spec.variants['cxxstd'].value)) - cfg.write(cmake_cache_entry('CMAKE_HIP_COMPILER', spec['hip'].prefix.bin.hipcc)) + cfg.write(cmake_cache_path('CMAKE_HIP_COMPILER', spec['hip'].prefix.bin.hipcc)) if not spec.satisfies('amdgpu_target=none'): cmake_hip_archs = ";".join(spec.variants["amdgpu_target"].value) cfg.write(cmake_cache_string('CMAKE_HIP_ARCHITECTURES', cmake_hip_archs)) - cfg.write(cmake_cache_entry('ROCM_PATH', spec['hip'].prefix)) + cfg.write(cmake_cache_path('ROCM_PATH', spec['hip'].prefix)) else: cfg.write(cmake_cache_option('ENABLE_HIP', False)) @@ -499,14 +492,14 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) cfg.write(cmake_cache_option('ENABLE_CHAI', True)) - cfg.write(cmake_cache_entry('CHAI_DIR', spec['chai'].prefix)) + cfg.write(cmake_cache_path('CHAI_DIR', spec['chai'].prefix)) - cfg.write(cmake_cache_entry('RAJA_DIR', spec['raja'].prefix)) + cfg.write(cmake_cache_path('RAJA_DIR', spec['raja'].prefix)) cfg.write(cmake_cache_option('ENABLE_UMPIRE', True)) - cfg.write(cmake_cache_entry('UMPIRE_DIR', spec['umpire'].prefix)) + cfg.write(cmake_cache_path('UMPIRE_DIR', spec['umpire'].prefix)) - cfg.write(cmake_cache_entry('CAMP_DIR', spec['camp'].prefix)) + cfg.write(cmake_cache_path('CAMP_DIR', spec['camp'].prefix)) # yapf: disable io_tpls = ( @@ -526,8 +519,8 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+caliper' in spec: cfg.write(cmake_cache_option('ENABLE_CALIPER', True)) - cfg.write(cmake_cache_entry('CALIPER_DIR', spec['caliper'].prefix)) - cfg.write(cmake_cache_entry('ADIAK_DIR', spec['adiak'].prefix)) + cfg.write(cmake_cache_path('CALIPER_DIR', spec['caliper'].prefix)) + cfg.write(cmake_cache_path('ADIAK_DIR', spec['adiak'].prefix)) for tpl, cmake_name, enable in io_tpls: if enable: @@ -544,7 +537,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): raise KeyError("No zlib provider (zlib/zlib-ng/zlib-api) found in {0}".format(spec)) else: dep_spec = spec[tpl] - cfg.write(cmake_cache_entry('{}_DIR'.format(cmake_name), dep_spec.prefix)) + cfg.write(cmake_cache_path('{}_DIR'.format(cmake_name), dep_spec.prefix)) else: cfg.write(cmake_cache_option('ENABLE_{}'.format(cmake_name), False)) @@ -554,15 +547,15 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if spec["blas"].name == "intel-oneapi-mkl": cfg.write(cmake_cache_option('ENABLE_MKL', True)) - cfg.write(cmake_cache_entry('MKL_INCLUDE_DIRS', spec['intel-oneapi-mkl'].prefix.include)) + cfg.write(cmake_cache_path('MKL_INCLUDE_DIRS', spec['intel-oneapi-mkl'].prefix.include)) cfg.write(cmake_cache_list('MKL_LIBRARIES', spec['intel-oneapi-mkl'].libs)) elif spec["blas"].name == "mkl": cfg.write(cmake_cache_option('ENABLE_MKL', True)) - cfg.write(cmake_cache_entry('MKL_INCLUDE_DIRS', spec['intel-mkl'].prefix.include)) + cfg.write(cmake_cache_path('MKL_INCLUDE_DIRS', spec['intel-mkl'].prefix.include)) cfg.write(cmake_cache_list('MKL_LIBRARIES', spec['intel-mkl'].libs)) elif spec["blas"].name == "essl": cfg.write(cmake_cache_option('ENABLE_ESSL', True)) - cfg.write(cmake_cache_entry('ESSL_INCLUDE_DIRS', spec['essl'].prefix.include)) + cfg.write(cmake_cache_path('ESSL_INCLUDE_DIRS', spec['essl'].prefix.include)) cfg.write(cmake_cache_list('ESSL_LIBRARIES', spec['blas'].libs)) cfg.write(cmake_cache_option('FORTRAN_MANGLE_NO_UNDERSCORE', True)) @@ -588,7 +581,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) for tpl, cmake_name, enable in math_tpls: if enable: - cfg.write(cmake_cache_entry('{}_DIR'.format(cmake_name), spec[tpl].prefix)) + cfg.write(cmake_cache_path('{}_DIR'.format(cmake_name), spec[tpl].prefix)) if tpl == 'hypre' and '+cuda' in spec: cfg.write(cmake_cache_string('ENABLE_HYPRE_DEVICE', "CUDA")) @@ -611,8 +604,8 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('# Python\n') cfg.write('#{0}\n\n'.format('-' * 80)) - cfg.write(cmake_cache_entry('Python3_ROOT_DIR', os.path.join(spec['python'].prefix))) - cfg.write(cmake_cache_entry('Python3_EXECUTABLE', os.path.join(spec['python'].prefix.bin, 'python3'))) + cfg.write(cmake_cache_path('Python3_ROOT_DIR', os.path.join(spec['python'].prefix))) + cfg.write(cmake_cache_path('Python3_EXECUTABLE', os.path.join(spec['python'].prefix.bin, 'python3'))) if '+pygeosx' in spec: cfg.write(cmake_cache_option('ENABLE_PYGEOSX', True)) @@ -624,10 +617,10 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) if '+docs' in spec: sphinx_bin_dir = spec['py-sphinx'].prefix.bin - cfg.write(cmake_cache_entry('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) + cfg.write(cmake_cache_path('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) doxygen_bin_dir = spec['doxygen'].prefix.bin - cfg.write(cmake_cache_entry('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) + cfg.write(cmake_cache_path('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) else: cfg.write(cmake_cache_option('ENABLE_DOCS', False)) cfg.write(cmake_cache_option('ENABLE_DOXYGEN', False)) @@ -640,14 +633,14 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('ENABLE_UNCRUSTIFY', '+uncrustify' in spec)) if '+uncrustify' in spec: cfg.write( - cmake_cache_entry('UNCRUSTIFY_EXECUTABLE', os.path.join(spec['uncrustify'].prefix.bin, 'uncrustify'))) + cmake_cache_path('UNCRUSTIFY_EXECUTABLE', os.path.join(spec['uncrustify'].prefix.bin, 'uncrustify'))) if '+addr2line' in spec: cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# addr2line\n') cfg.write('#{0}\n\n'.format('-' * 80)) cfg.write(cmake_cache_option('ENABLE_ADDR2LINE', True)) - cfg.write(cmake_cache_entry('ADDR2LINE_EXEC ', '/usr/bin/addr2line')) + cfg.write(cmake_cache_path('ADDR2LINE_EXEC ', '/usr/bin/addr2line')) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Other\n') @@ -655,7 +648,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+mathpresso' in spec: cfg.write(cmake_cache_option('ENABLE_MATHPRESSO', True)) - cfg.write(cmake_cache_entry('MATHPRESSO_DIR', spec['mathpresso'].prefix)) + cfg.write(cmake_cache_path('MATHPRESSO_DIR', spec['mathpresso'].prefix)) cfg.write(cmake_cache_option('ENABLE_XML_UPDATES', True)) else: cfg.write(cmake_cache_option('ENABLE_MATHPRESSO', False)) @@ -741,15 +734,15 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write("#{0}\n".format("-" * 80)) cfg.write("# Compilers\n") cfg.write("#{0}\n\n".format("-" * 80)) - cfg.write(cmake_cache_entry("CMAKE_C_COMPILER", c_compiler)) + cfg.write(cmake_cache_path("CMAKE_C_COMPILER", c_compiler)) cflags = ' '.join(spec.compiler_flags['cflags']) if cflags: - cfg.write(cmake_cache_entry("CMAKE_C_FLAGS", cflags)) + cfg.write(cmake_cache_string("CMAKE_C_FLAGS", cflags)) - cfg.write(cmake_cache_entry("CMAKE_CXX_COMPILER", cpp_compiler)) + cfg.write(cmake_cache_path("CMAKE_CXX_COMPILER", cpp_compiler)) cxxflags = ' '.join(spec.compiler_flags['cxxflags']) if cxxflags: - cfg.write(cmake_cache_entry("CMAKE_CXX_FLAGS", cxxflags)) + cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS", cxxflags)) release_flags = "-O3 -DNDEBUG" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_RELEASE", release_flags)) @@ -759,7 +752,7 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) + cfg.write(cmake_cache_string("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Cuda\n') @@ -769,9 +762,9 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string('CMAKE_CUDA_STANDARD', spec.variants['cxxstd'].value)) cudatoolkitdir = spec['cuda'].prefix - cfg.write(cmake_cache_entry('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) + cfg.write(cmake_cache_path('CUDA_TOOLKIT_ROOT_DIR', cudatoolkitdir)) cudacompiler = '${CUDA_TOOLKIT_ROOT_DIR}/bin/nvcc' - cfg.write(cmake_cache_entry('CMAKE_CUDA_COMPILER', cudacompiler)) + cfg.write(cmake_cache_path('CMAKE_CUDA_COMPILER', cudacompiler)) cmake_cuda_flags = ('-restrict --expt-extended-lambda -Werror ' 'cross-execution-space-call,reorder,' @@ -802,14 +795,14 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write('#{0}\n\n'.format('-' * 80)) cfg.write(cmake_cache_option('ENABLE_CHAI', True)) - cfg.write(cmake_cache_entry('CHAI_DIR', spec['chai'].prefix)) + cfg.write(cmake_cache_path('CHAI_DIR', spec['chai'].prefix)) - cfg.write(cmake_cache_entry('RAJA_DIR', spec['raja'].prefix)) + cfg.write(cmake_cache_path('RAJA_DIR', spec['raja'].prefix)) cfg.write(cmake_cache_option('ENABLE_UMPIRE', True)) - cfg.write(cmake_cache_entry('UMPIRE_DIR', spec['umpire'].prefix)) + cfg.write(cmake_cache_path('UMPIRE_DIR', spec['umpire'].prefix)) - cfg.write(cmake_cache_entry('CAMP_DIR', spec['camp'].prefix)) + cfg.write(cmake_cache_path('CAMP_DIR', spec['camp'].prefix)) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# IO TPLs\n') @@ -817,18 +810,18 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): if '+caliper' in spec: cfg.write(cmake_cache_option('ENABLE_CALIPER', True)) - cfg.write(cmake_cache_entry('CALIPER_DIR', spec['caliper'].prefix)) - cfg.write(cmake_cache_entry('adiak_DIR', spec['adiak'].prefix + '/lib/cmake/adiak')) + cfg.write(cmake_cache_path('CALIPER_DIR', spec['caliper'].prefix)) + cfg.write(cmake_cache_path('adiak_DIR', spec['adiak'].prefix + '/lib/cmake/adiak')) cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Documentation\n') cfg.write('#{0}\n\n'.format('-' * 80)) if '+docs' in spec: sphinx_bin_dir = spec['py-sphinx'].prefix.bin - cfg.write(cmake_cache_entry('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) + cfg.write(cmake_cache_path('SPHINX_EXECUTABLE', os.path.join(sphinx_bin_dir, 'sphinx-build'))) doxygen_bin_dir = spec['doxygen'].prefix.bin - cfg.write(cmake_cache_entry('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) + cfg.write(cmake_cache_path('DOXYGEN_EXECUTABLE', os.path.join(doxygen_bin_dir, 'doxygen'))) else: cfg.write(cmake_cache_option('ENABLE_DOXYGEN', False)) cfg.write(cmake_cache_option('ENABLE_SPHINX', False)) From bc7397d7bbf11680214b37806597e01492f7af40 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 16 Mar 2026 10:29:28 -0400 Subject: [PATCH 02/27] Remove flex and bison overrides from ROCm CI branch --- docker/spack-rocm.yaml | 11 ----------- docker/spack.yaml | 11 ----------- docker/tpl-ubuntu-hip.Dockerfile | 2 -- 3 files changed, 24 deletions(-) diff --git a/docker/spack-rocm.yaml b/docker/spack-rocm.yaml index 1abd4e46..8bd43182 100644 --- a/docker/spack-rocm.yaml +++ b/docker/spack-rocm.yaml @@ -436,17 +436,6 @@ spack: externals: - spec: gettext@0.21 prefix: /usr - bison: - buildable: false - externals: - - spec: bison@3.8.2 - prefix: /usr - # Rely on spack to build a newer flex for scotch - # flex: - # buildable: false - # externals: - # - spec: flex@2.6.4 - # prefix: /usr bzip2: buildable: false externals: diff --git a/docker/spack.yaml b/docker/spack.yaml index 77a134a6..42e3c4d9 100644 --- a/docker/spack.yaml +++ b/docker/spack.yaml @@ -285,17 +285,6 @@ spack: externals: - spec: gettext@0.21 prefix: /usr - bison: - buildable: false - externals: - - spec: bison@3.8.2 - prefix: /usr - # Rely on spack to build a newer flex for scotch - # flex: - # buildable: false - # externals: - # - spec: flex@2.6.4 - # prefix: /usr bzip2: buildable: false externals: diff --git a/docker/tpl-ubuntu-hip.Dockerfile b/docker/tpl-ubuntu-hip.Dockerfile index 5ab3edde..02a46da2 100644 --- a/docker/tpl-ubuntu-hip.Dockerfile +++ b/docker/tpl-ubuntu-hip.Dockerfile @@ -47,8 +47,6 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ pkg-config \ xz-utils \ gettext \ - bison \ - flex \ bzip2 \ help2man \ libtool \ From b281ae50e11cad20b021b3e5bb4a6f974ac5cec4 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 16 Mar 2026 10:29:28 -0400 Subject: [PATCH 03/27] Remove flex and bison overrides from ROCm CI branch --- docker/spack-rocm.yaml | 11 ----------- docker/spack.yaml | 11 ----------- docker/tpl-ubuntu-hip.Dockerfile | 2 -- scripts/spack_packages/packages/geosx/package.py | 5 ++--- 4 files changed, 2 insertions(+), 27 deletions(-) diff --git a/docker/spack-rocm.yaml b/docker/spack-rocm.yaml index 1abd4e46..8bd43182 100644 --- a/docker/spack-rocm.yaml +++ b/docker/spack-rocm.yaml @@ -436,17 +436,6 @@ spack: externals: - spec: gettext@0.21 prefix: /usr - bison: - buildable: false - externals: - - spec: bison@3.8.2 - prefix: /usr - # Rely on spack to build a newer flex for scotch - # flex: - # buildable: false - # externals: - # - spec: flex@2.6.4 - # prefix: /usr bzip2: buildable: false externals: diff --git a/docker/spack.yaml b/docker/spack.yaml index 77a134a6..42e3c4d9 100644 --- a/docker/spack.yaml +++ b/docker/spack.yaml @@ -285,17 +285,6 @@ spack: externals: - spec: gettext@0.21 prefix: /usr - bison: - buildable: false - externals: - - spec: bison@3.8.2 - prefix: /usr - # Rely on spack to build a newer flex for scotch - # flex: - # buildable: false - # externals: - # - spec: flex@2.6.4 - # prefix: /usr bzip2: buildable: false externals: diff --git a/docker/tpl-ubuntu-hip.Dockerfile b/docker/tpl-ubuntu-hip.Dockerfile index 5ab3edde..02a46da2 100644 --- a/docker/tpl-ubuntu-hip.Dockerfile +++ b/docker/tpl-ubuntu-hip.Dockerfile @@ -47,8 +47,6 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ pkg-config \ xz-utils \ gettext \ - bison \ - flex \ bzip2 \ help2man \ libtool \ diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 4704eda8..96d993d5 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -150,8 +150,8 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): # # Math # - depends_on("parmetis@4.0.3+int64+shared cflags='-fPIC' cxxflags='-fPIC'") - depends_on("metis +int64+shared cflags='-fPIC' cxxflags='-fPIC'") + depends_on("parmetis@4.0.3+int64~shared cflags='-fPIC' cxxflags='-fPIC'") + depends_on("metis +int64~shared cflags='-fPIC' cxxflags='-fPIC'") depends_on("superlu-dist@9.2.1 +int64 fflags='-fPIC'") depends_on("superlu-dist@9.2.1 +int64 fflags='-fPIC -ef'", when="%cce") @@ -176,7 +176,6 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): depends_on("hypre +cuda+superlu-dist+mixedint+mpi+umpire+unified-memory", when='+cuda') depends_on("hypre +rocm+superlu-dist+mixedint+mpi+umpire+unified-memory", when='+rocm') depends_on("hypre ~openmp", when="~openmp") - depends_on("hypre +openmp", when="+openmp") depends_on("hypre +caliper", when="+caliper") depends_on("hypre +pic", when="~shared") depends_on("hypre +shared", when="+shared") From 0b9777445b14e6b4df49426f640a3c88e24cf87a Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 16 Mar 2026 11:49:42 -0400 Subject: [PATCH 04/27] Add script for running docker rocm CI locally --- scripts/run-rocm-ci-local.sh | 193 +++++++++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100755 scripts/run-rocm-ci-local.sh diff --git a/scripts/run-rocm-ci-local.sh b/scripts/run-rocm-ci-local.sh new file mode 100755 index 00000000..6744d02d --- /dev/null +++ b/scripts/run-rocm-ci-local.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: scripts/run-rocm-ci-local.sh [options] + +Launch the ROCm CI Docker build locally, using the same Dockerfile and +roughly the same build arguments as the GitHub Actions ROCm job. + +Options: + --tag TAG Docker tag to use. Default: local + --repository REPO Docker repository name. + Default: geosx/ubuntu24.04-amdclang19.0.0-rocm6.4.3 + --install-dir-root DIR Install root inside the image. Default: /opt/GEOS + --docker-root-image IMG Base image. Default: rocm/dev-ubuntu-24.04:6.4.3 + --spack-build-jobs N Pass SPACK_BUILD_JOBS to the Docker build. + --ca-bundle PATH Inject this CA bundle into the Docker build. + --no-ca-inject Disable CA injection even if a host CA bundle is found. + --no-cache Build with --no-cache. + --dry-run Print the docker build command and exit. + -h, --help Show this help. +EOF +} + +repo_root="$(git -C "$(dirname "$0")/.." rev-parse --show-toplevel)" +cd "$repo_root" + +docker_repository="geosx/ubuntu24.04-amdclang19.0.0-rocm6.4.3" +docker_tag="local" +install_dir_root="/opt/GEOS" +docker_root_image="rocm/dev-ubuntu-24.04:6.4.3" +spack_build_jobs="" +ca_bundle="" +inject_ca="auto" +no_cache=0 +dry_run=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --tag) + docker_tag="$2" + shift 2 + ;; + --repository) + docker_repository="$2" + shift 2 + ;; + --install-dir-root) + install_dir_root="$2" + shift 2 + ;; + --docker-root-image) + docker_root_image="$2" + shift 2 + ;; + --spack-build-jobs) + spack_build_jobs="$2" + shift 2 + ;; + --ca-bundle) + ca_bundle="$2" + inject_ca="yes" + shift 2 + ;; + --no-ca-inject) + inject_ca="no" + shift + ;; + --no-cache) + no_cache=1 + shift + ;; + --dry-run) + dry_run=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ "$inject_ca" == "auto" ]]; then + for candidate in \ + /etc/pki/tls/certs/ca-bundle.crt \ + /etc/ssl/certs/ca-certificates.crt + do + if [[ -f "$candidate" ]]; then + ca_bundle="$candidate" + inject_ca="yes" + break + fi + done +fi + +tmp_dir="$(mktemp -d "$repo_root/.rocm-ci-local.XXXXXX")" +cleanup() { + rm -rf "$tmp_dir" +} +trap cleanup EXIT + +tmp_dockerfile="$tmp_dir/tpl-ubuntu-hip.local.Dockerfile" +cp docker/tpl-ubuntu-hip.Dockerfile "$tmp_dockerfile" + +if [[ "$inject_ca" == "yes" ]]; then + if [[ -z "$ca_bundle" || ! -f "$ca_bundle" ]]; then + echo "CA injection requested but bundle was not found: ${ca_bundle:-}" >&2 + exit 1 + fi + + cp "$ca_bundle" "$tmp_dir/host-ca.crt" + python3 - "$tmp_dockerfile" "$(basename "$tmp_dir")/host-ca.crt" <<'PY' +import pathlib +import sys + +dockerfile = pathlib.Path(sys.argv[1]) +bundle_rel = sys.argv[2] +text = dockerfile.read_text() +needle = "RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime && \\\n apt-get update" +replacement = ( + f"COPY {bundle_rel} /usr/local/share/ca-certificates/host-ca.crt\n" + "RUN update-ca-certificates\n\n" + + needle +) +if needle not in text: + raise SystemExit("Could not find apt-get hook for CA injection") +dockerfile.write_text(text.replace(needle, replacement, 1)) +PY +fi + +if [[ ! -f scripts/uberenv/uberenv.py ]]; then + if [[ $dry_run -eq 1 ]]; then + echo "uberenv submodule missing; dry-run skipping submodule update" + else + echo "Updating uberenv submodule" + git submodule update --init scripts/uberenv + fi +fi + +commit_sha="$(git rev-parse HEAD)" +install_dir="${install_dir_root}/GEOS_TPL-${docker_tag}-${commit_sha:0:7}" + +docker_args=( + build + --progress=plain + --build-arg "DOCKER_ROOT_IMAGE=${docker_root_image}" + --build-arg "INSTALL_DIR=${install_dir}" + --tag "${docker_repository}:${docker_tag}" + --file "$tmp_dockerfile" + --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" + --label "org.opencontainers.image.source=https://github.com/GEOS-DEV/thirdPartyLibs" + --label "org.opencontainers.image.revision=${commit_sha}" + --label "org.opencontainers.image.title=Building environment for GEOS" +) + +if [[ -n "$spack_build_jobs" ]]; then + docker_args+=(--build-arg "SPACK_BUILD_JOBS=${spack_build_jobs}") +fi + +if [[ $no_cache -eq 1 ]]; then + docker_args+=(--no-cache) +fi + +docker_args+=("$repo_root") + +echo "Repository: ${docker_repository}" +echo "Tag: ${docker_tag}" +echo "Install dir: ${install_dir}" +echo "Dockerfile: ${tmp_dockerfile}" +if [[ "$inject_ca" == "yes" ]]; then + echo "Injected CA bundle: ${ca_bundle}" +else + echo "Injected CA bundle: none" +fi + +printf 'Running: docker' +for arg in "${docker_args[@]}"; do + printf ' %q' "$arg" +done +printf '\n' + +if [[ $dry_run -eq 1 ]]; then + exit 0 +fi + +DOCKER_BUILDKIT=1 docker "${docker_args[@]}" From 2b858438b1fac59c1a9646a887f9b555954bf9db Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 16 Mar 2026 13:19:42 -0400 Subject: [PATCH 05/27] Experimenting with clingo --- .uberenv_config.json | 3 ++- scripts/run-rocm-ci-local.sh | 28 ++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.uberenv_config.json b/.uberenv_config.json index b541fb7c..db0448d0 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -9,5 +9,6 @@ "spack_commit": "2e2169d5282d166f63e3ee4db8d4446c43cefa8a", "spack_commit_note": "v1.1.1 (Jan 14th 2026)", "spack_packages_commit": "3dd98680871078353a28ee508fa76c7554f918fa", -"spack_packages_note": "Feb 25th 2026" +"spack_packages_note": "Feb 25th 2026", +"spack_setup_clingo": 0 } diff --git a/scripts/run-rocm-ci-local.sh b/scripts/run-rocm-ci-local.sh index 6744d02d..8d1be09e 100755 --- a/scripts/run-rocm-ci-local.sh +++ b/scripts/run-rocm-ci-local.sh @@ -14,7 +14,9 @@ Options: Default: geosx/ubuntu24.04-amdclang19.0.0-rocm6.4.3 --install-dir-root DIR Install root inside the image. Default: /opt/GEOS --docker-root-image IMG Base image. Default: rocm/dev-ubuntu-24.04:6.4.3 - --spack-build-jobs N Pass SPACK_BUILD_JOBS to the Docker build. + --amdgpu-target TARGET AMD GPU target passed into the build. Default: gfx942 + --spack-build-jobs N Pass SPACK_BUILD_JOBS to the Docker build. Default: 2 + --target STAGE Stop the build at a specific Docker stage. --ca-bundle PATH Inject this CA bundle into the Docker build. --no-ca-inject Disable CA injection even if a host CA bundle is found. --no-cache Build with --no-cache. @@ -30,7 +32,9 @@ docker_repository="geosx/ubuntu24.04-amdclang19.0.0-rocm6.4.3" docker_tag="local" install_dir_root="/opt/GEOS" docker_root_image="rocm/dev-ubuntu-24.04:6.4.3" -spack_build_jobs="" +amdgpu_target="gfx942" +spack_build_jobs="2" +docker_target="" ca_bundle="" inject_ca="auto" no_cache=0 @@ -54,10 +58,18 @@ while [[ $# -gt 0 ]]; do docker_root_image="$2" shift 2 ;; + --amdgpu-target) + amdgpu_target="$2" + shift 2 + ;; --spack-build-jobs) spack_build_jobs="$2" shift 2 ;; + --target) + docker_target="$2" + shift 2 + ;; --ca-bundle) ca_bundle="$2" inject_ca="yes" @@ -151,6 +163,7 @@ docker_args=( build --progress=plain --build-arg "DOCKER_ROOT_IMAGE=${docker_root_image}" + --build-arg "AMDGPU_TARGET=${amdgpu_target}" --build-arg "INSTALL_DIR=${install_dir}" --tag "${docker_repository}:${docker_tag}" --file "$tmp_dockerfile" @@ -164,6 +177,10 @@ if [[ -n "$spack_build_jobs" ]]; then docker_args+=(--build-arg "SPACK_BUILD_JOBS=${spack_build_jobs}") fi +if [[ -n "$docker_target" ]]; then + docker_args+=(--target "${docker_target}") +fi + if [[ $no_cache -eq 1 ]]; then docker_args+=(--no-cache) fi @@ -174,6 +191,13 @@ echo "Repository: ${docker_repository}" echo "Tag: ${docker_tag}" echo "Install dir: ${install_dir}" echo "Dockerfile: ${tmp_dockerfile}" +echo "AMDGPU target: ${amdgpu_target}" +echo "Spack build jobs: ${spack_build_jobs}" +if [[ -n "$docker_target" ]]; then + echo "Build target: ${docker_target}" +else + echo "Build target: final image" +fi if [[ "$inject_ca" == "yes" ]]; then echo "Injected CA bundle: ${ca_bundle}" else From c448c58779f075bd1e09b370206890522241e0ff Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 16 Mar 2026 21:37:50 -0400 Subject: [PATCH 06/27] Debugging rocm build --- docker/tpl-ubuntu-hip.Dockerfile | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docker/tpl-ubuntu-hip.Dockerfile b/docker/tpl-ubuntu-hip.Dockerfile index 02a46da2..cdf7c79f 100644 --- a/docker/tpl-ubuntu-hip.Dockerfile +++ b/docker/tpl-ubuntu-hip.Dockerfile @@ -119,12 +119,27 @@ RUN apt-get update && \ # --spack-debug to debug build RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ mkdir -p ${GEOSX_TPL_DIR} && \ + ( while true; do \ + sleep 60; \ + echo "[heartbeat] $(date -Iseconds) uberenv/spack still running"; \ + find ${GEOSX_TPL_DIR}/build_stage -maxdepth 3 -name spack-build-out.txt -printf '%T@ %p\n' 2>/dev/null | \ + sort -nr | head -n 1 | \ + while read -r _ path; do \ + echo "[heartbeat] recent build log: ${path}"; \ + tail -n 5 "${path}" 2>/dev/null || true; \ + done; \ + done ) & \ + hb=$!; \ ./scripts/uberenv/uberenv.py \ --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} %amdclang-19 ^caliper~papi~gotcha~sampler~libunwind~libdw" \ --spack-env-file=${SRC_DIR}/docker/spack-rocm.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ - -k && \ + -k; \ + rc=$?; \ + kill ${hb} 2>/dev/null || true; \ + wait ${hb} 2>/dev/null || true; \ + test ${rc} -eq 0 && \ # Remove host-config generated for LvArray rm lvarray* && \ # Rename and copy spack-generated host-config to root directory From f68dcb5b40d0b72a25c728cc03a8dcf988eb150c Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 16 Mar 2026 21:38:04 -0400 Subject: [PATCH 07/27] Fix spack warning with v1.1 --- scripts/spack_packages/packages/geosx/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index 96d993d5..76b6e4bd 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -9,7 +9,7 @@ import socket import os -import llnl.util.tty as tty +import spack.llnl.util.tty as tty from os import environ as env from os.path import join as pjoin From 82d893566765092e949c65a87912ca79ec4b3b5d Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Mon, 16 Mar 2026 21:38:17 -0400 Subject: [PATCH 08/27] Fix spack warning with v1.1 --- docker/spack-rocm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/spack-rocm.yaml b/docker/spack-rocm.yaml index 8bd43182..880ce7ce 100644 --- a/docker/spack-rocm.yaml +++ b/docker/spack-rocm.yaml @@ -394,7 +394,7 @@ spack: py-sphinx: buildable: false externals: - - spec: py-sphinx@4.3.2 + - spec: py-sphinx@4.3.2 ^python@3.10.12 prefix: /usr python: buildable: false From 1b3f872faed2437607e978a8215950f66eecb4ed Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 17 Mar 2026 10:27:33 -0400 Subject: [PATCH 09/27] Fix docker-build.sh: fail on submodule init error and force uberenv checkout --- scripts/docker-build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 8cd56c39..93d72eeb 100644 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -1,11 +1,12 @@ #!/bin/bash +set -e env # We save memory for the docker context echo .git > .dockerignore # Get uberenv submodule -git submodule update --init scripts/uberenv +git submodule update --init --force scripts/uberenv # This script will build an image from TPL_DOCKERFILE From 2b2995815909afcd6045167e92e5ded03e740c83 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 17 Mar 2026 12:23:07 -0400 Subject: [PATCH 10/27] Fix uberenv not found: work around Docker 29.x readwrite mount bug --- docker/tpl-ubuntu-hip.Dockerfile | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docker/tpl-ubuntu-hip.Dockerfile b/docker/tpl-ubuntu-hip.Dockerfile index cdf7c79f..9ea91b38 100644 --- a/docker/tpl-ubuntu-hip.Dockerfile +++ b/docker/tpl-ubuntu-hip.Dockerfile @@ -117,8 +117,21 @@ RUN apt-get update && \ # Have to create install directory first for uberenv # -k flag is to ignore SSL errors # --spack-debug to debug build -RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \ - mkdir -p ${GEOSX_TPL_DIR} && \ +# +# NOTE: We use a read-only mount (no `readwrite`) because Docker 29.x's +# built-in BuildKit has a bug where `--mount=src=.,readwrite` mounts an +# empty directory instead of exposing the build context. We work around +# it by mounting the context read-only at /tmp/src and copying only the +# small set of files uberenv actually needs into the writable container +# filesystem before running. +RUN --mount=src=.,dst=/tmp/src \ + mkdir -p ${SRC_DIR}/scripts ${GEOSX_TPL_DIR} && \ + cp -r /tmp/src/scripts/uberenv ${SRC_DIR}/scripts/ && \ + cp -r /tmp/src/scripts/spack_packages ${SRC_DIR}/scripts/ && \ + cp -r /tmp/src/scripts/spack_configs ${SRC_DIR}/scripts/ && \ + cp /tmp/src/.uberenv_config.json ${SRC_DIR}/ && \ + cp -r /tmp/src/docker ${SRC_DIR}/ && \ + cd ${SRC_DIR} && \ ( while true; do \ sleep 60; \ echo "[heartbeat] $(date -Iseconds) uberenv/spack still running"; \ From c48a9b0f5523bb00f8dbd3150144cc4ffa41d375 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 17 Mar 2026 13:37:28 -0400 Subject: [PATCH 11/27] Remove uberenv .git pointer so BuildKit includes submodule files in build context --- scripts/docker-build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 93d72eeb..2269a04a 100644 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -7,6 +7,8 @@ echo .git > .dockerignore # Get uberenv submodule git submodule update --init --force scripts/uberenv +# Remove the .git gitdir pointer so BuildKit treats uberenv files as regular files in the build context +rm -f scripts/uberenv/.git # This script will build an image from TPL_DOCKERFILE From 3a3be3f3b4c46a6e6ccae5e7d1dde30d9324fed5 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 17 Mar 2026 16:09:00 -0400 Subject: [PATCH 12/27] Fix BuildKit gitlink exclusion: re-add uberenv as regular tracked files before docker build --- scripts/docker-build.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 2269a04a..94c70506 100644 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -7,8 +7,12 @@ echo .git > .dockerignore # Get uberenv submodule git submodule update --init --force scripts/uberenv -# Remove the .git gitdir pointer so BuildKit treats uberenv files as regular files in the build context +# BuildKit's git-aware context builder treats submodules as gitlinks (empty dirs). +# Fix: remove the .git pointer, strip the gitlink from the index, and re-add the +# files as regular tracked entries so BuildKit includes them in the build context. rm -f scripts/uberenv/.git +git rm --cached scripts/uberenv 2>/dev/null || true +git add -f scripts/uberenv/ # This script will build an image from TPL_DOCKERFILE From d2bf7930ae23727d1f566ca964e1c38161819b8f Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 17 Mar 2026 16:42:40 -0400 Subject: [PATCH 13/27] Copy uberenv to plain docker/_uberenv_bundle to bypass BuildKit gitlink exclusion --- docker/tpl-ubuntu-hip.Dockerfile | 2 +- scripts/docker-build.sh | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docker/tpl-ubuntu-hip.Dockerfile b/docker/tpl-ubuntu-hip.Dockerfile index 9ea91b38..7f13a7c6 100644 --- a/docker/tpl-ubuntu-hip.Dockerfile +++ b/docker/tpl-ubuntu-hip.Dockerfile @@ -126,7 +126,7 @@ RUN apt-get update && \ # filesystem before running. RUN --mount=src=.,dst=/tmp/src \ mkdir -p ${SRC_DIR}/scripts ${GEOSX_TPL_DIR} && \ - cp -r /tmp/src/scripts/uberenv ${SRC_DIR}/scripts/ && \ + cp -r /tmp/src/docker/_uberenv_bundle ${SRC_DIR}/scripts/uberenv && \ cp -r /tmp/src/scripts/spack_packages ${SRC_DIR}/scripts/ && \ cp -r /tmp/src/scripts/spack_configs ${SRC_DIR}/scripts/ && \ cp /tmp/src/.uberenv_config.json ${SRC_DIR}/ && \ diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 94c70506..26b94122 100644 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -7,12 +7,11 @@ echo .git > .dockerignore # Get uberenv submodule git submodule update --init --force scripts/uberenv -# BuildKit's git-aware context builder treats submodules as gitlinks (empty dirs). -# Fix: remove the .git pointer, strip the gitlink from the index, and re-add the -# files as regular tracked entries so BuildKit includes them in the build context. -rm -f scripts/uberenv/.git -git rm --cached scripts/uberenv 2>/dev/null || true -git add -f scripts/uberenv/ +# BuildKit's git-aware context builder treats submodules as gitlinks and sends +# only an empty directory placeholder for them. Work around this by copying +# uberenv into a plain (non-submodule) path that BuildKit will include in full. +rm -rf docker/_uberenv_bundle +cp -r scripts/uberenv docker/_uberenv_bundle # This script will build an image from TPL_DOCKERFILE From d5501732f824ccdf64d080fb2b3eaeadff0775ce Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 18 Mar 2026 09:52:34 -0400 Subject: [PATCH 14/27] Pass scripts/uberenv as a separate named BuildKit context and mount it explicitly in the Dockerfile --- docker/tpl-ubuntu-hip.Dockerfile | 12 ++++++------ scripts/docker-build.sh | 6 +----- scripts/run-rocm-ci-local.sh | 1 + 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/docker/tpl-ubuntu-hip.Dockerfile b/docker/tpl-ubuntu-hip.Dockerfile index 7f13a7c6..2b4fe3c9 100644 --- a/docker/tpl-ubuntu-hip.Dockerfile +++ b/docker/tpl-ubuntu-hip.Dockerfile @@ -120,13 +120,13 @@ RUN apt-get update && \ # # NOTE: We use a read-only mount (no `readwrite`) because Docker 29.x's # built-in BuildKit has a bug where `--mount=src=.,readwrite` mounts an -# empty directory instead of exposing the build context. We work around -# it by mounting the context read-only at /tmp/src and copying only the -# small set of files uberenv actually needs into the writable container -# filesystem before running. +# empty directory instead of exposing the build context. We also mount the +# uberenv submodule as a separate named context, because BuildKit's git-aware +# handling of the main repo context drops submodule contents. RUN --mount=src=.,dst=/tmp/src \ - mkdir -p ${SRC_DIR}/scripts ${GEOSX_TPL_DIR} && \ - cp -r /tmp/src/docker/_uberenv_bundle ${SRC_DIR}/scripts/uberenv && \ + --mount=from=uberenv,target=/tmp/uberenv \ + mkdir -p ${SRC_DIR}/scripts/uberenv ${GEOSX_TPL_DIR} && \ + cp -r /tmp/uberenv/. ${SRC_DIR}/scripts/uberenv/ && \ cp -r /tmp/src/scripts/spack_packages ${SRC_DIR}/scripts/ && \ cp -r /tmp/src/scripts/spack_configs ${SRC_DIR}/scripts/ && \ cp /tmp/src/.uberenv_config.json ${SRC_DIR}/ && \ diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index 26b94122..eb73f0ab 100644 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -7,11 +7,6 @@ echo .git > .dockerignore # Get uberenv submodule git submodule update --init --force scripts/uberenv -# BuildKit's git-aware context builder treats submodules as gitlinks and sends -# only an empty directory placeholder for them. Work around this by copying -# uberenv into a plain (non-submodule) path that BuildKit will include in full. -rm -rf docker/_uberenv_bundle -cp -r scripts/uberenv docker/_uberenv_bundle # This script will build an image from TPL_DOCKERFILE @@ -27,6 +22,7 @@ INSTALL_DIR=${INSTALL_DIR_ROOT}/GEOS_TPL-${DOCKER_TAG}-${COMMIT:0:7} echo "Installation directory is ${INSTALL_DIR}" docker build --progress=plain ${DOCKER_COMPILER_BUILD_ARG} \ +--build-context uberenv=./scripts/uberenv \ --build-arg HOST_CONFIG=${HOST_CONFIG} \ --build-arg DOCKER_ROOT_IMAGE=${DOCKER_ROOT_IMAGE} \ --build-arg INSTALL_DIR=${INSTALL_DIR} \ diff --git a/scripts/run-rocm-ci-local.sh b/scripts/run-rocm-ci-local.sh index 8d1be09e..ea4de5b8 100755 --- a/scripts/run-rocm-ci-local.sh +++ b/scripts/run-rocm-ci-local.sh @@ -162,6 +162,7 @@ install_dir="${install_dir_root}/GEOS_TPL-${docker_tag}-${commit_sha:0:7}" docker_args=( build --progress=plain + --build-context "uberenv=${repo_root}/scripts/uberenv" --build-arg "DOCKER_ROOT_IMAGE=${docker_root_image}" --build-arg "AMDGPU_TARGET=${amdgpu_target}" --build-arg "INSTALL_DIR=${install_dir}" From 4d67831088c3f2b3b594cd60d2032d558ee1c263 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 18 Mar 2026 11:07:35 -0400 Subject: [PATCH 15/27] More fixes --- docker/tpl-ubuntu-hip.Dockerfile | 69 ++++++++++++++++---------------- scripts/docker-build.sh | 10 ++++- scripts/run-rocm-ci-local.sh | 7 +++- 3 files changed, 50 insertions(+), 36 deletions(-) diff --git a/docker/tpl-ubuntu-hip.Dockerfile b/docker/tpl-ubuntu-hip.Dockerfile index 2b4fe3c9..aab0ac73 100644 --- a/docker/tpl-ubuntu-hip.Dockerfile +++ b/docker/tpl-ubuntu-hip.Dockerfile @@ -118,41 +118,42 @@ RUN apt-get update && \ # -k flag is to ignore SSL errors # --spack-debug to debug build # -# NOTE: We use a read-only mount (no `readwrite`) because Docker 29.x's -# built-in BuildKit has a bug where `--mount=src=.,readwrite` mounts an -# empty directory instead of exposing the build context. We also mount the -# uberenv submodule as a separate named context, because BuildKit's git-aware -# handling of the main repo context drops submodule contents. -RUN --mount=src=.,dst=/tmp/src \ - --mount=from=uberenv,target=/tmp/uberenv \ - mkdir -p ${SRC_DIR}/scripts/uberenv ${GEOSX_TPL_DIR} && \ - cp -r /tmp/uberenv/. ${SRC_DIR}/scripts/uberenv/ && \ - cp -r /tmp/src/scripts/spack_packages ${SRC_DIR}/scripts/ && \ - cp -r /tmp/src/scripts/spack_configs ${SRC_DIR}/scripts/ && \ - cp /tmp/src/.uberenv_config.json ${SRC_DIR}/ && \ - cp -r /tmp/src/docker ${SRC_DIR}/ && \ +# NOTE: We copy files into the image instead of relying on read-write +# BuildKit mounts because Docker 29.x's built-in BuildKit has two issues on +# our runners: `--mount=src=.,readwrite` can expose an empty main context, and +# `--mount=from=` can expose an empty uberenv context at exec +# time. Using COPY keeps both contexts stable. +COPY --from=uberenv . ${SRC_DIR}/scripts/uberenv/ +COPY scripts/spack_packages ${SRC_DIR}/scripts/spack_packages +COPY scripts/spack_configs ${SRC_DIR}/scripts/spack_configs +COPY .uberenv_config.json ${SRC_DIR}/ +COPY docker ${SRC_DIR}/docker + +RUN mkdir -p ${GEOSX_TPL_DIR} && \ cd ${SRC_DIR} && \ - ( while true; do \ - sleep 60; \ - echo "[heartbeat] $(date -Iseconds) uberenv/spack still running"; \ - find ${GEOSX_TPL_DIR}/build_stage -maxdepth 3 -name spack-build-out.txt -printf '%T@ %p\n' 2>/dev/null | \ - sort -nr | head -n 1 | \ - while read -r _ path; do \ - echo "[heartbeat] recent build log: ${path}"; \ - tail -n 5 "${path}" 2>/dev/null || true; \ - done; \ - done ) & \ - hb=$!; \ - ./scripts/uberenv/uberenv.py \ - --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} %amdclang-19 ^caliper~papi~gotcha~sampler~libunwind~libdw" \ - --spack-env-file=${SRC_DIR}/docker/spack-rocm.yaml \ - --project-json=.uberenv_config.json \ - --prefix ${GEOSX_TPL_DIR} \ - -k; \ - rc=$?; \ - kill ${hb} 2>/dev/null || true; \ - wait ${hb} 2>/dev/null || true; \ - test ${rc} -eq 0 && \ + { \ + ( while true; do \ + sleep 60; \ + echo "[heartbeat] $(date -Iseconds) uberenv/spack still running"; \ + find ${GEOSX_TPL_DIR}/build_stage -maxdepth 3 -name spack-build-out.txt -printf '%T@ %p\n' 2>/dev/null | \ + sort -nr | head -n 1 | \ + while read -r _ path; do \ + echo "[heartbeat] recent build log: ${path}"; \ + tail -n 5 "${path}" 2>/dev/null || true; \ + done; \ + done ) & \ + hb=$!; \ + sh ./scripts/uberenv/uberenv.py \ + --spec "+rocm~uncrustify~openmp~pygeosx~trilinos~petsc amdgpu_target=${AMDGPU_TARGET} %amdclang-19 ^caliper~papi~gotcha~sampler~libunwind~libdw" \ + --spack-env-file=${SRC_DIR}/docker/spack-rocm.yaml \ + --project-json=.uberenv_config.json \ + --prefix ${GEOSX_TPL_DIR} \ + -k; \ + rc=$?; \ + kill ${hb} 2>/dev/null || true; \ + wait ${hb} 2>/dev/null || true; \ + test ${rc} -eq 0; \ + } && \ # Remove host-config generated for LvArray rm lvarray* && \ # Rename and copy spack-generated host-config to root directory diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index eb73f0ab..a0fc182c 100644 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -8,6 +8,14 @@ echo .git > .dockerignore # Get uberenv submodule git submodule update --init --force scripts/uberenv +uberenv_context_dir="$(mktemp -d "${TMPDIR:-/tmp}/uberenv-context.XXXXXX")" +cleanup() { + rm -rf "${uberenv_context_dir}" +} +trap cleanup EXIT + +cp -r scripts/uberenv/. "${uberenv_context_dir}/" +rm -f "${uberenv_context_dir}/.git" # This script will build an image from TPL_DOCKERFILE # with (optional) DOCKER_COMPILER_BUILD_ARG build arguments. @@ -22,7 +30,7 @@ INSTALL_DIR=${INSTALL_DIR_ROOT}/GEOS_TPL-${DOCKER_TAG}-${COMMIT:0:7} echo "Installation directory is ${INSTALL_DIR}" docker build --progress=plain ${DOCKER_COMPILER_BUILD_ARG} \ ---build-context uberenv=./scripts/uberenv \ +--build-context uberenv="${uberenv_context_dir}" \ --build-arg HOST_CONFIG=${HOST_CONFIG} \ --build-arg DOCKER_ROOT_IMAGE=${DOCKER_ROOT_IMAGE} \ --build-arg INSTALL_DIR=${INSTALL_DIR} \ diff --git a/scripts/run-rocm-ci-local.sh b/scripts/run-rocm-ci-local.sh index ea4de5b8..8af0e47f 100755 --- a/scripts/run-rocm-ci-local.sh +++ b/scripts/run-rocm-ci-local.sh @@ -156,13 +156,18 @@ if [[ ! -f scripts/uberenv/uberenv.py ]]; then fi fi +uberenv_context_dir="$tmp_dir/uberenv-context" +mkdir -p "$uberenv_context_dir" +cp -r scripts/uberenv/. "$uberenv_context_dir/" +rm -f "$uberenv_context_dir/.git" + commit_sha="$(git rev-parse HEAD)" install_dir="${install_dir_root}/GEOS_TPL-${docker_tag}-${commit_sha:0:7}" docker_args=( build --progress=plain - --build-context "uberenv=${repo_root}/scripts/uberenv" + --build-context "uberenv=${uberenv_context_dir}" --build-arg "DOCKER_ROOT_IMAGE=${docker_root_image}" --build-arg "AMDGPU_TARGET=${amdgpu_target}" --build-arg "INSTALL_DIR=${install_dir}" From 96c4de3f95009c02f24d4cd68e756b1964fc6fb6 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 18 Mar 2026 11:36:30 -0400 Subject: [PATCH 16/27] Better logging --- docker/tpl-ubuntu-hip.Dockerfile | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/docker/tpl-ubuntu-hip.Dockerfile b/docker/tpl-ubuntu-hip.Dockerfile index aab0ac73..7eebcd3b 100644 --- a/docker/tpl-ubuntu-hip.Dockerfile +++ b/docker/tpl-ubuntu-hip.Dockerfile @@ -135,12 +135,23 @@ RUN mkdir -p ${GEOSX_TPL_DIR} && \ ( while true; do \ sleep 60; \ echo "[heartbeat] $(date -Iseconds) uberenv/spack still running"; \ - find ${GEOSX_TPL_DIR}/build_stage -maxdepth 3 -name spack-build-out.txt -printf '%T@ %p\n' 2>/dev/null | \ - sort -nr | head -n 1 | \ + find ${GEOSX_TPL_DIR}/build_stage -maxdepth 2 -mindepth 2 -type d -printf '%T@ %p\n' 2>/dev/null | \ + sort -nr | head -n 3 | \ while read -r _ path; do \ - echo "[heartbeat] recent build log: ${path}"; \ + echo "[heartbeat] recent stage dir: ${path}"; \ + done; \ + find ${GEOSX_TPL_DIR}/build_stage -maxdepth 4 \( -name spack-build-out.txt -o -name spack-build-env.txt -o -name spack-configure-args.txt \) -printf '%T@ %p\n' 2>/dev/null | \ + sort -nr | head -n 3 | \ + while read -r _ path; do \ + echo "[heartbeat] recent stage file: ${path}"; \ tail -n 5 "${path}" 2>/dev/null || true; \ + printf '\n'; \ done; \ + ps -eo pid,ppid,etime,%cpu,%mem,cmd 2>/dev/null | \ + grep -E 'spack|python3|curl|wget|git|cmake|ninja|make|amdclang|gfortran|gcc|g\\+\\+' | \ + grep -v grep | \ + tail -n 20 | \ + cut -c1-240 || true; \ done ) & \ hb=$!; \ sh ./scripts/uberenv/uberenv.py \ @@ -148,6 +159,7 @@ RUN mkdir -p ${GEOSX_TPL_DIR} && \ --spack-env-file=${SRC_DIR}/docker/spack-rocm.yaml \ --project-json=.uberenv_config.json \ --prefix ${GEOSX_TPL_DIR} \ + -j ${SPACK_BUILD_JOBS} \ -k; \ rc=$?; \ kill ${hb} 2>/dev/null || true; \ From 8b4e0a510140eaa2bf9c12ed597d7863a41f59dc Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 19:29:30 -0700 Subject: [PATCH 17/27] Update spack to v1.2.2 --- .gitignore | 1 + .uberenv_config.json | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 704d7cdd..488ea52a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .project .pydevproject .vscode/ +spack/ spack-*.txt __pycache__ uberenv_* diff --git a/.uberenv_config.json b/.uberenv_config.json index b541fb7c..11d6a5ee 100644 --- a/.uberenv_config.json +++ b/.uberenv_config.json @@ -6,8 +6,8 @@ "spack_configs_path": "scripts/spack_configs", "spack_packages_path": "scripts/spack_packages/packages", "spack_url": "https://github.com/spack/spack", -"spack_commit": "2e2169d5282d166f63e3ee4db8d4446c43cefa8a", -"spack_commit_note": "v1.1.1 (Jan 14th 2026)", -"spack_packages_commit": "3dd98680871078353a28ee508fa76c7554f918fa", -"spack_packages_note": "Feb 25th 2026" +"spack_commit": "3e19345b6e12f5ff1b874f4059622fc6a1fd804a", +"spack_commit_note": "v1.2.2 (Jul 20th 2026)", +"spack_packages_commit": "bd9b4838fbf9e4ca01ad7b6c6df633ddcc750d71", +"spack_packages_note": "Jul 20th 2026" } From ff13119379c69f7f7d6db4c8bff51d30c6dc78e6 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 22:39:01 -0400 Subject: [PATCH 18/27] Add unified build.sh and HIP support to the TPL superbuild - build.sh: single driver for z5tux (cpu/cuda/hip) and LC (matrix, tioga, tuo-cpu, tuo-gpu), cmake+ninja by default, --spack for uberenv builds - CMakeLists: HIP plumbing for raja, chai, caliper and hypre - Make python explicit in the py-sphinx externals (spack v1.2 deprecation) --- .gitignore | 1 + CMakeLists.txt | 120 ++- build.sh | 813 ++++++++++++++++++ docker/pecan-spack.yaml | 2 + docker/ubuntu-spack.yaml | 2 + .../spack_configs/toss_4_x86_64_ib/spack.yaml | 2 + 6 files changed, 937 insertions(+), 3 deletions(-) create mode 100755 build.sh diff --git a/.gitignore b/.gitignore index 488ea52a..36d0b675 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /build* +!/build.sh /install* .cproject .project diff --git a/CMakeLists.txt b/CMakeLists.txt index 321f3ca0..dc910be1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,6 +51,8 @@ option( ENABLE_TRILINOS "" ON ) option( ENABLE_PETSC "" OFF ) option( ENABLE_SCOTCH "" ON ) option( ENABLE_SUITESPARSE "" ON ) +option( ENABLE_SUPERLU_DIST "" ON ) +option( ENABLE_HYPRE_GPU_PROFILING "" OFF ) option( ENABLE_VTK "" ON ) option( ENABLE_VTK_WRAP_PYTHON "" OFF ) @@ -125,6 +127,50 @@ else( TRUE ) endif() +################################ +# HIP / ROCm +################################ +if( ENABLE_HIP ) + if( NOT DEFINED ROCM_PATH ) + if( DEFINED HIP_ROOT ) + set( ROCM_PATH ${HIP_ROOT} ) + elseif( DEFINED ENV{ROCM_PATH} ) + set( ROCM_PATH $ENV{ROCM_PATH} ) + else() + message( FATAL_ERROR "ENABLE_HIP is ON but neither ROCM_PATH nor HIP_ROOT is defined." ) + endif() + endif() + + if( NOT DEFINED HIP_ROOT ) + set( HIP_ROOT ${ROCM_PATH} ) + endif() + + if( NOT DEFINED CMAKE_HIP_ARCHITECTURES ) + message( FATAL_ERROR "ENABLE_HIP is ON but CMAKE_HIP_ARCHITECTURES is not defined (e.g. gfx90a, gfx942, gfx1100)." ) + endif() + + # hypre is configured outside of cmake, so it needs the MPI include directory explicitly + # when its sources are compiled for the device. + if( ENABLE_MPI AND NOT DEFINED MPI_INCLUDE_DIR ) + if( DEFINED ENV{MPICH_DIR} ) # set by the cray-mpich module on LC + set( MPI_INCLUDE_DIR "$ENV{MPICH_DIR}/include" ) + elseif( DEFINED ENV{MPI_ROOT} ) + set( MPI_INCLUDE_DIR "$ENV{MPI_ROOT}/include" ) + elseif( MPI_C_INCLUDE_DIRS ) # set by find_package( MPI ) through BLT + list( GET MPI_C_INCLUDE_DIRS 0 MPI_INCLUDE_DIR ) + elseif( MPI_C_COMPILER ) # /bin/mpicc -> /include + get_filename_component( MPI_BIN_DIR ${MPI_C_COMPILER} DIRECTORY ) + get_filename_component( MPI_PREFIX_DIR ${MPI_BIN_DIR} DIRECTORY ) + set( MPI_INCLUDE_DIR "${MPI_PREFIX_DIR}/include" ) + else() + message( FATAL_ERROR "ENABLE_HIP is ON but MPI_INCLUDE_DIR is not defined and cannot be deduced." ) + endif() + message( STATUS "MPI_INCLUDE_DIR = ${MPI_INCLUDE_DIR}" ) + endif() + + message( STATUS "ROCM_PATH = ${ROCM_PATH}" ) + message( STATUS "CMAKE_HIP_ARCHITECTURES = ${CMAKE_HIP_ARCHITECTURES}" ) +endif() ################################ @@ -340,6 +386,13 @@ set(RAJA_URL "${TPL_MIRROR_DIR}/RAJA-v2025.12.0.tar.gz") message(STATUS "Building RAJA found at ${RAJA_URL}") +# RAJA's vectorization layer does not compile with the ROCm toolchain. +if( ENABLE_HIP ) + set( RAJA_ENABLE_VECTORIZATION OFF ) +else() + set( RAJA_ENABLE_VECTORIZATION ON ) +endif() + ExternalProject_Add( raja URL ${RAJA_URL} PREFIX ${PROJECT_BINARY_DIR}/raja @@ -360,10 +413,18 @@ ExternalProject_Add( raja -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CMAKE_CUDA_FLAGS:STRING=${RAJA_CUDA_FLAGS} + -D ENABLE_HIP:BOOL=${ENABLE_HIP} + -D RAJA_ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER} + -D RAJA_ENABLE_VECTORIZATION:BOOL=${RAJA_ENABLE_VECTORIZATION} + -D hip_DIR=${HIP_ROOT}/lib/cmake/hip + -D ROCPRIM_DIR=${HIP_ROOT} + -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} -D ENABLE_TESTS:BOOL=OFF -D ENABLE_EXAMPLES:BOOL=OFF -D ENABLE_OPENMP:BOOL=${ENABLE_OPENMP} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} ) @@ -398,8 +459,14 @@ ExternalProject_Add( chai -D CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_CUDA_ARCHITECTURES:STRING=${CMAKE_CUDA_ARCHITECTURES} -D CMAKE_CUDA_FLAGS:STRING=${CHAI_CUDA_FLAGS} + -D ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CHAI_ENABLE_HIP:BOOL=${ENABLE_HIP} + -D CMAKE_HIP_COMPILER=${CMAKE_HIP_COMPILER} + -D hip_DIR=${HIP_ROOT}/lib/cmake/hip + -D CMAKE_HIP_ARCHITECTURES:STRING=${CMAKE_HIP_ARCHITECTURES} -D ENABLE_OPENMP=${ENABLE_OPENMP} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} -D ENABLE_TESTS=OFF -D ENABLE_BENCHMARKS=OFF @@ -444,6 +511,7 @@ if (ENABLE_CALIPER) -D MPI_CXX_COMPILER:STRING=${MPI_CXX_COMPILER} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D BLT_SOURCE_DIR=${BLT_SOURCE_DIR} ) @@ -480,14 +548,23 @@ if (ENABLE_CALIPER) -D CUDA_TOOLKIT_ROOT_DIR:PATH=${CUDA_TOOLKIT_ROOT_DIR} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D WITH_PAPI:BOOL=${ENABLE_PAPI} -D PAPI_PREFIX:STRING=${PAPI_PREFIX} -D WITH_ADIAK:BOOL=ON -D adiak_DIR:STRING=${ADIAK_DIR}/lib/cmake/adiak/ -D WITH_CUPTI=${CALIPER_WITH_CUPTI} -D WITH_NVPROF=${ENABLE_CUDA} + -D WITH_ROCM=${ENABLE_ROCTX} + -D ROCM_PREFIX:PATH=${ROCM_PATH} ) + if( BUILD_SHARED_LIBS ) + set( CALIPER_LIB "libcaliper.so" ) + else() + set( CALIPER_LIB "libcaliper.a" ) + endif() + list(APPEND build_list caliper adiak) else() set( ENABLE_CALIPER_HYPRE OFF CACHE BOOL "" FORCE ) @@ -737,7 +814,7 @@ list(APPEND build_list metis ) ################################ # SUPERLU -- ################################ -if( ENABLE_MPI ) +if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) set(SUPERLU_DIR "${CMAKE_INSTALL_PREFIX}/superlu_dist") set(SUPERLU_URL "${TPL_MIRROR_DIR}/superlu_dist-0f6efc3.tar.gz") @@ -784,6 +861,7 @@ if( ENABLE_MPI ) -D CMAKE_Fortran_FLAGS_RELEASE=${CMAKE_Fortran_FLAGS_RELEASE} -D CMAKE_POSITION_INDEPENDENT_CODE=${CMAKE_POSITION_INDEPENDENT_CODE} -D CMAKE_INSTALL_PREFIX:PATH= + -D CMAKE_INSTALL_LIBDIR:PATH=lib -D enable_openmp=${ENABLE_OPENMP} ) @@ -814,6 +892,9 @@ if( ENABLE_HYPRE ) set( HYPRE_C_COMPILER ${MPI_C_COMPILER} ) set( HYPRE_CXX_COMPILER ${MPI_CXX_COMPILER} ) set( HYPRE_Fortran_COMPILER ${MPI_Fortran_COMPILER} ) + endif() + + if( ENABLE_MPI AND ENABLE_SUPERLU_DIST ) set( HYPRE_SUPERLU_LIBS "${SUPERLU_DIR}/lib/libsuperlu_dist.a ${PARMETIS_DIR}/lib/libparmetis.a ${METIS_DIR}/lib/libmetis.a -lstdc++" ) if( ENABLE_OPENMP ) set( HYPRE_SUPERLU_LIBS "${HYPRE_SUPERLU_LIBS} ${OpenMP_C_FLAGS}" ) # Sergey: why is this needed? @@ -844,9 +925,41 @@ if( ENABLE_HYPRE ) --with-umpire-include=${CHAI_DIR}/include \ --with-umpire-lib-dirs=${CHAI_DIR}/lib \ --with-umpire-libs=umpire " ) + + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-aware-mpi" ) + endif() + + if( ENABLE_HYPRE_GPU_PROFILING ) + set( HYPRE_CUDA_FLAGS "${HYPRE_CUDA_FLAGS} --enable-gpu-profiling" ) + endif() endif() elseif( ENABLE_HYPRE_DEVICE STREQUAL "HIP" ) - message( FATAL_ERROR "hip tpls build only supported through spack" ) + if( NOT ENABLE_HIP ) + message( FATAL_ERROR "Non-compatible options: HYPRE device cannot be set to HIP when ENABLE_HIP is OFF" ) + else() + list( APPEND HYPRE_DEPENDS chai ) + set( HYPRE_HIP_FLAGS "\ +ROCM_PATH=${ROCM_PATH} \ +--with-hip \ +--enable-rocblas \ +--enable-rocsparse \ +--enable-rocsolver \ +--with-gpu-arch=${CMAKE_HIP_ARCHITECTURES} \ +--with-umpire-device \ +--with-umpire-include=${CHAI_DIR}/include \ +--with-umpire-lib-dirs=${CHAI_DIR}/lib \ +--with-umpire-libs=\"umpire camp\" \ +--with-MPI-include=${MPI_INCLUDE_DIR} " ) + + if( ENABLE_HYPRE_GPU_AWARE_MPI ) + set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-aware-mpi" ) + endif() + + if( ENABLE_HYPRE_GPU_PROFILING ) + set( HYPRE_HIP_FLAGS "${HYPRE_HIP_FLAGS} --enable-gpu-profiling" ) + endif() + endif() endif() if( CMAKE_HOST_APPLE ) @@ -856,7 +969,7 @@ if( ENABLE_HYPRE ) endif() if( ENABLE_CALIPER_HYPRE ) - set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/libs/caliper.a\"" ) + set( HYPRE_CALIPER_FLAGS "--with-caliper --with-caliper-include=\"${CALIPER_DIR}/include\" --with-caliper-lib=\"${CALIPER_DIR}/lib/${CALIPER_LIB}\"" ) list( APPEND HYPRE_DEPENDS caliper ) endif() @@ -881,6 +994,7 @@ if( ENABLE_HYPRE ) ${HYPRE_SUPERLU_FLAGS} \ ${HYPRE_OPENMP_FLAG} \ ${HYPRE_CUDA_FLAGS} \ + ${HYPRE_HIP_FLAGS} \ ${HYPRE_CALIPER_FLAGS}\n") ExternalProject_Add( hypre diff --git a/build.sh b/build.sh new file mode 100755 index 00000000..723ab3f3 --- /dev/null +++ b/build.sh @@ -0,0 +1,813 @@ +#!/usr/bin/env bash +# +# Unified driver to build the GEOS third-party libraries. +# +# Supported configurations: +# z5tux gcc + mpich, host only +# z5tux-cuda gcc + mpich + CUDA (sm_120) +# z5tux-hip amdclang + mpich + ROCm (gfx1100) +# z5tux-sycl not supported yet (see NOTES below) +# matrix gcc + mvapich2 + CUDA (sm_90) +# tioga amdclang + cray-mpich + ROCm (gfx90a) +# tuo-cpu amdclang + cray-mpich, host only +# tuo-gpu amdclang + cray-mpich + ROCm (gfx942) +# +# By default the TPLs are built with the CMake superbuild of this repository +# (CMakeLists.txt), configured with the Ninja generator both for the superbuild +# itself and for every sub-project (-DENABLE_NINJA=ON). With --spack the same +# configurations are built through uberenv/spack instead. +# +# The script is meant to be run *on* the target machine: on the LC systems, +# clone/copy this repository there and run it from a login node (optionally +# with --alloc to push the compilation onto a compute node). +# +# NOTES +# * z5tux-sycl is declared but disabled: neither the superbuild nor the spack +# packages have SYCL plumbing, and no SYCL toolchain is installed on z5tux. +# * Trilinos is off by default (GEOS defaults to the hypre interface); enable +# it with --trilinos. +# * The ROCm configurations compile Fortran with gfortran, not amdflang: the +# flang runtime is not pulled in when clang++ links Fortran objects, which +# breaks superlu_dist. Override with --fc if needed. +# * Known issue: ROCm 7.2 (AMD clang 22) miscompiles hypre's spgemm kernels +# for gfx942/gfx1100 ("Illegal instruction detected: Operand has incorrect +# register class"); gfx90a is fine. LC uses ROCm 6.4.3, which is unaffected. +# +# Usage: ./build.sh [config] [options] [-D= ...] +# Run ./build.sh --help for the full option list. + +set -euo pipefail + +SCRIPT_NAME="$(basename "$0")" +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" + +# ------------------------------------------------------------------------------ +# Defaults +# ------------------------------------------------------------------------------ +CONFIG="" +BUILD_TYPE="Release" +BUILD_DIR="" +INSTALL_DIR="" +JOBS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 8)" +TOP_JOBS=1 +GENERATOR="auto" # auto | ninja | make +GPU_ARCH="" # overrides the per-config cuda arch / amdgpu target +ROCM_DIR_OPT="" +CUDA_DIR_OPT="" +MPI_DIR_OPT="" +FC_OPT="" +USE_ALLOC=0 +ALLOC_CMD="" +ACCOUNT="${GEOS_TPL_ACCOUNT:-vortex}" +LOAD_MODULES=1 +MODULES_OPT="" +CLEAN=0 +DO_CONFIGURE=1 +DO_BUILD=1 +USE_SPACK=0 +SPACK_SPEC_OPT="" +ENABLE_TRILINOS="OFF" +ENABLE_OPENMP="" # per-config default unless set here +ENABLE_ROCTX="OFF" +ENABLE_SUPERLU_DIST="ON" +ENABLE_HYPRE_GPU_PROFILING="OFF" +DRY_RUN=0 +LOG_FILE="" +EXTRA_CMAKE_ARGS=() +FORWARDED_ARGS=() + +# Populated by select_config() +CFG_DESC="" +CFG_SUPPORTED=1 +CFG_UNSUPPORTED_REASON="" +CMAKE_ARGS=() +MPI_PREFIX="" # set by the setup_*_mpi helpers, used by setup_hip + +# ------------------------------------------------------------------------------ +# Helpers +# ------------------------------------------------------------------------------ +log() { printf '[%s] %s\n' "${SCRIPT_NAME}" "$*"; } +warn() { printf '[%s] WARNING: %s\n' "${SCRIPT_NAME}" "$*" >&2; } +die() { printf '[%s] ERROR: %s\n' "${SCRIPT_NAME}" "$*" >&2; exit 1; } + +run() { + printf '[%s] + %s\n' "${SCRIPT_NAME}" "$*" + if [[ ${DRY_RUN} -eq 0 ]]; then + "$@" + fi +} + +# Newest (version-sorted) existing path matching the given glob, empty if none. +newest_path() { + local match + # shellcheck disable=SC2086 # deliberate globbing + match="$(ls -d $1 2>/dev/null | sort -V | tail -1 || true)" + printf '%s' "${match}" +} + +require_file() { + [[ -e "$1" ]] || die "${2:-required file} not found: $1" +} + +require_exec() { + [[ -x "$1" ]] || die "${2:-required executable} not found or not executable: $1" +} + +add_arg() { CMAKE_ARGS+=( "$@" ); } + +usage() { + cat <= ...] [-- ] + +Configurations (auto-detected from the hostname when omitted): + z5tux gcc + mpich, host only + z5tux-cuda gcc + mpich + CUDA, sm_120 + z5tux-hip amdclang + mpich + ROCm, gfx1100 + z5tux-sycl not supported yet + matrix gcc + mvapich2 + CUDA, sm_90 (LC) + tioga amdclang + cray-mpich + ROCm, gfx90a (LC) + tuo-cpu amdclang + cray-mpich, host only (LC, tuolumne) + tuo-gpu amdclang + cray-mpich + ROCm, gfx942 (LC, tuolumne) + +Options: + -bt, --build-type Release|RelWithDebInfo|Debug|MinSizeRel (default: ${BUILD_TYPE}) + -b, --build-dir build directory (default: /build--) + -i, --install-dir install directory (default: /install--) + -j, --jobs parallelism inside each TPL (default: ${JOBS}) + --top-jobs TPLs built concurrently (default: ${TOP_JOBS}) + --generator auto|ninja|make (default: auto, prefers ninja) + --gpu-arch override the cuda arch (e.g. 90) or amdgpu target (e.g. gfx942) + --rocm ROCm installation to use + --cuda CUDA toolkit to use + --mpi MPI installation to use (expects /bin/mpicc) + --fc Fortran compiler to use + --trilinos build Trilinos as well (off by default) + --openmp override the per-configuration OpenMP setting + --roctx enable ROCTx/rocm profiling support (raja, caliper, hypre) + --superlu build superlu_dist and link it into hypre (default: ${ENABLE_SUPERLU_DIST}) + --gpu-profiling configure hypre with --enable-gpu-profiling (nvtx/roctx) + --alloc run the build step through the machine's scheduler + --alloc-cmd scheduler command to use with --alloc + --account bank used by --alloc (default: ${ACCOUNT}) + --modules "" replace the default module list + --no-modules do not touch modules at all + --spack build through uberenv/spack instead of the cmake superbuild + --spec "" spack spec to build (implies --spack) + --clean remove the build and install directories first + --configure-only stop after the cmake configure step + --build-only skip the configure step (reuse an existing build directory) + --log also write the output to + -n, --dry-run print the commands instead of running them + -l, --list list the configurations and exit + -h, --help show this help and exit + + Any -D= argument is forwarded to cmake (or, with --spack, any + argument after -- is forwarded to uberenv.py). + +Examples: + ./${SCRIPT_NAME} z5tux + ./${SCRIPT_NAME} z5tux-cuda -bt RelWithDebInfo -j 24 + ./${SCRIPT_NAME} z5tux-hip --gpu-arch gfx1036 + ./${SCRIPT_NAME} tuo-gpu --alloc # on a tuolumne login node + ./${SCRIPT_NAME} matrix --spack # same config, through spack + ./${SCRIPT_NAME} tioga -DENABLE_VTK=OFF +EOF +} + +# Static description of each configuration: used by --list and by the run banner, +# so that neither needs the toolchain to be resolved first. +describe_config() { + case "$1" in + z5tux) printf 'z5tux: gcc + mpich, host only' ;; + z5tux-cuda) printf 'z5tux: gcc + mpich + CUDA, sm_%s' "${GPU_ARCH:-120}" ;; + z5tux-hip) printf 'z5tux: amdclang + mpich + ROCm, %s' "${GPU_ARCH:-gfx1100}" ;; + z5tux-sycl) printf 'z5tux: SYCL (Intel Battlemage)' ;; + matrix) printf 'matrix: gcc + mvapich2 + CUDA, sm_%s' "${GPU_ARCH:-90}" ;; + tioga|tioga-gpu) printf 'tioga: amdclang + cray-mpich + ROCm, %s' "${GPU_ARCH:-gfx90a}" ;; + tuo-cpu|tuolumne-cpu) printf 'tuolumne: amdclang + cray-mpich, host only' ;; + tuo-gpu|tuolumne-gpu|tuolumne) printf 'tuolumne: amdclang + cray-mpich + ROCm, %s' "${GPU_ARCH:-gfx942}" ;; + *) printf '' ;; + esac +} + +# Modules to load on the LC machines. Kept separate from select_config() so that +# they are loaded *before* the compilers and cmake/ninja are looked up. +modules_for_config() { + case "$1" in + matrix) + printf '%s\n' "gcc/13.3.1-magic" "mvapich2/2.3.7" "cuda/12.9.1" "cmake/3.30.5" "ninja" ;; + tioga|tioga-gpu|tuo-cpu|tuo-gpu|tuolumne|tuolumne-cpu|tuolumne-gpu) + printf '%s\n' "rocm" "cray-mpich" "cmake/3.29" "ninja" ;; + *) + ;; + esac +} + +# Machine a configuration runs on (used for the scheduler defaults). +machine_for_config() { + case "$1" in + z5tux|z5tux-cuda|z5tux-hip|z5tux-sycl) printf 'z5tux' ;; + matrix) printf 'matrix' ;; + tioga|tioga-gpu) printf 'tioga' ;; + tuo-cpu|tuo-gpu|tuolumne*) printf 'tuolumne' ;; + *) printf '' ;; + esac +} + +# Spack spec used by --spack, mirroring scripts/setupLC-TPL-uberenv.bash. +# '%%' propagates the compiler choice to the dependencies (spack v1 syntax). +spack_spec_for_config() { + case "$1" in + z5tux) + printf '~docs %%gcc ^vtk generator=ninja' ;; + z5tux-cuda) + printf '+cuda ~uncrustify ~docs cuda_arch=%s %%gcc ^vtk generator=ninja' "${GPU_ARCH:-120}" ;; + z5tux-hip) + printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%llvm-amdgpu ^vtk generator=ninja' "${GPU_ARCH:-gfx1100}" ;; + matrix) + printf '+cuda ~uncrustify cuda_arch=%s %%%%gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ^vtk generator=ninja' "${GPU_ARCH:-90}" ;; + tioga|tioga-gpu) + printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%%%cce-20 ^vtk generator=ninja' "${GPU_ARCH:-gfx90a}" ;; + tuo-cpu|tuolumne-cpu) + printf '~pygeosx ~trilinos ~petsc ~docs %%%%cce-20 ^vtk generator=ninja' ;; + tuo-gpu|tuolumne-gpu|tuolumne) + printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%%%cce-20 ^vtk generator=ninja' "${GPU_ARCH:-gfx942}" ;; + *) + printf '' ;; + esac +} + +# Configurations that cannot be built (yet); prints the reason. +unsupported_reason() { + case "$1" in + z5tux-sycl) printf 'no SYCL support in the superbuild nor in the spack packages, and no SYCL toolchain installed on z5tux' ;; + *) printf '' ;; + esac +} + +list_configs() { + local cfg + printf '%-12s %s\n' "CONFIG" "DESCRIPTION" + for cfg in z5tux z5tux-cuda z5tux-hip matrix tioga tuo-cpu tuo-gpu; do + printf '%-12s %s\n' "${cfg}" "$(describe_config "${cfg}")" + done + printf '%-12s %s [UNSUPPORTED: %s]\n' "z5tux-sycl" "$(describe_config z5tux-sycl)" \ + "no SYCL support in the superbuild nor in the spack packages" +} + +# ------------------------------------------------------------------------------ +# Argument parsing +# ------------------------------------------------------------------------------ +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) usage; exit 0 ;; + -l|--list) list_configs; exit 0 ;; + -bt|--build-type) BUILD_TYPE="${2:?"missing value for $1"}"; shift 2 ;; + -b|--build-dir) BUILD_DIR="${2:?"missing value for $1"}"; shift 2 ;; + -i|--install-dir) INSTALL_DIR="${2:?"missing value for $1"}"; shift 2 ;; + -j|--jobs) JOBS="${2:?"missing value for $1"}"; shift 2 ;; + --top-jobs) TOP_JOBS="${2:?"missing value for $1"}"; shift 2 ;; + --generator) GENERATOR="${2:?"missing value for $1"}"; shift 2 ;; + --gpu-arch) GPU_ARCH="${2:?"missing value for $1"}"; shift 2 ;; + --rocm) ROCM_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --cuda) CUDA_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --mpi) MPI_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --fc) FC_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --trilinos) ENABLE_TRILINOS="ON"; shift ;; + --openmp) ENABLE_OPENMP="${2:?"missing value for $1"}"; shift 2 ;; + --roctx) ENABLE_ROCTX="ON"; shift ;; + --superlu) ENABLE_SUPERLU_DIST="${2:?"missing value for $1"}"; shift 2 ;; + --gpu-profiling) ENABLE_HYPRE_GPU_PROFILING="ON"; shift ;; + --alloc) USE_ALLOC=1; shift ;; + --alloc-cmd) ALLOC_CMD="$2"; USE_ALLOC=1; shift 2 ;; + --account) ACCOUNT="${2:?"missing value for $1"}"; shift 2 ;; + --modules) MODULES_OPT="${2:?"missing value for $1"}"; shift 2 ;; + --no-modules) LOAD_MODULES=0; shift ;; + --spack) USE_SPACK=1; shift ;; + --spec) SPACK_SPEC_OPT="${2:?"missing value for $1"}"; USE_SPACK=1; shift 2 ;; + --clean) CLEAN=1; shift ;; + --configure-only) DO_BUILD=0; shift ;; + --build-only) DO_CONFIGURE=0; shift ;; + --log) LOG_FILE="${2:?"missing value for $1"}"; shift 2 ;; + -n|--dry-run) DRY_RUN=1; shift ;; + -D*) EXTRA_CMAKE_ARGS+=( "$1" ); shift ;; + --) shift; FORWARDED_ARGS+=( "$@" ); break ;; + -*) die "unknown option '$1' (see --help)" ;; + *) + [[ -z "${CONFIG}" ]] || die "more than one configuration given ('${CONFIG}' and '$1')" + CONFIG="$1"; shift ;; + esac +done + +# ------------------------------------------------------------------------------ +# Configurations +# ------------------------------------------------------------------------------ + +# Compilers + MPI wrappers of the mpich installation used on z5tux. +setup_z5tux_mpi() { + local mpi_dir="${MPI_DIR_OPT:-${MPICH_HOME:-/home/victor/projects/mpich-4.2.2/install/release}}" + require_exec "${mpi_dir}/bin/mpicc" "mpicc (use --mpi )" + add_arg -DENABLE_MPI=ON \ + -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ + -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ + -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ + -DMPIEXEC_EXECUTABLE="${mpi_dir}/bin/mpirun" \ + -DMPIEXEC_NUMPROC_FLAG="-np" + MPI_PREFIX="${mpi_dir}" +} + +# The mpich/cray-mpich wrappers pick their back-end compiler from these. +export_mpich_backend() { + export MPICH_CC="$1" + export MPICH_CXX="$2" + export MPICH_FC="$3" + export MPICH_F90="$3" +} + +# ROCm installation: --rocm, then $ROCM_PATH, then the newest /opt/rocm-*. +resolve_rocm() { + local rocm="${ROCM_DIR_OPT:-${ROCM_PATH:-}}" + if [[ -z "${rocm}" ]]; then + rocm="$(newest_path '/opt/rocm-*')" + [[ -n "${rocm}" ]] || rocm="/opt/rocm" + fi + [[ -d "${rocm}" ]] || die "ROCm installation not found (tried '${rocm}', use --rocm )" + printf '%s' "${rocm}" +} + +# CUDA toolkit: --cuda, then $CUDA_HOME, then the LC/system locations. +resolve_cuda() { + local cuda="${CUDA_DIR_OPT:-${CUDA_HOME:-}}" + if [[ -z "${cuda}" ]]; then + cuda="$(newest_path '/usr/tce/packages/cuda/cuda-*')" + [[ -n "${cuda}" ]] || cuda="$(newest_path '/usr/local/cuda-*')" + [[ -n "${cuda}" ]] || cuda="/usr/local/cuda" + fi + [[ -d "${cuda}" ]] || die "CUDA toolkit not found (tried '${cuda}', use --cuda )" + printf '%s' "${cuda}" +} + +# Fortran compiler paired with the ROCm toolchain. gfortran is preferred over +# amdflang: clang++ links Fortran objects without the flang runtime, which +# breaks the mixed-language TPLs (superlu_dist). +resolve_fortran() { + local rocm="$1" fc + if [[ -n "${FC_OPT}" ]]; then + printf '%s' "${FC_OPT}"; return 0 + fi + for fc in /usr/bin/gfortran \ + "$(newest_path '/usr/tce/packages/gcc-tce/gcc-*/bin/gfortran')" \ + "$(newest_path '/usr/tce/packages/gcc/gcc-*-magic/bin/gfortran')"; do + if [[ -n "${fc}" && -x "${fc}" ]]; then + printf '%s' "${fc}"; return 0 + fi + done + warn "no gfortran found, falling back to amdflang (mixed C++/Fortran TPLs may fail to link)" + printf '%s' "${rocm}/llvm/bin/amdflang" +} + +# amdclang/amdclang++ out of a ROCm installation, plus a Fortran compiler. +setup_rocm_compilers() { + local rocm="$1" + local cc="${rocm}/llvm/bin/amdclang" + local cxx="${rocm}/llvm/bin/amdclang++" + local fc + fc="$(resolve_fortran "${rocm}")" + require_exec "${cc}" "amdclang" + require_exec "${cxx}" "amdclang++" + require_exec "${fc}" "Fortran compiler (use --fc )" + add_arg -DCMAKE_C_COMPILER="${cc}" \ + -DCMAKE_CXX_COMPILER="${cxx}" \ + -DCMAKE_Fortran_COMPILER="${fc}" + export_mpich_backend "${cc}" "${cxx}" "${fc}" +} + +# HIP settings shared by z5tux-hip, tioga and tuo-gpu. +setup_hip() { + local rocm="$1" arch="$2" + # cmake's HIP language requires clang itself, the hipcc wrapper is rejected. + require_exec "${rocm}/llvm/bin/amdclang++" "amdclang++" + add_arg -DENABLE_HIP=ON \ + -DROCM_PATH="${rocm}" \ + -DHIP_ROOT="${rocm}" \ + -DCMAKE_HIP_COMPILER="${rocm}/llvm/bin/amdclang++" \ + -DCMAKE_HIP_ARCHITECTURES="${arch}" \ + -DCMAKE_HIP_STANDARD=17 \ + -DENABLE_ROCTX="${ENABLE_ROCTX}" \ + -DENABLE_HYPRE=ON \ + -DENABLE_HYPRE_DEVICE=HIP + # hypre compiles its device sources outside of cmake and needs the MPI headers + [[ -z "${MPI_PREFIX}" ]] || add_arg -DMPI_INCLUDE_DIR="${MPI_PREFIX}/include" +} + +# CUDA settings shared by z5tux-cuda and matrix. +setup_cuda() { + local cuda="$1" arch="$2" + require_exec "${cuda}/bin/nvcc" "nvcc" + add_arg -DENABLE_CUDA=ON \ + -DENABLE_CUDA_NVTOOLSEXT=ON \ + -DCUDA_TOOLKIT_ROOT_DIR="${cuda}" \ + -DCMAKE_CUDA_COMPILER="${cuda}/bin/nvcc" \ + -DCMAKE_CUDA_ARCHITECTURES="${arch}" \ + -DCUDA_ARCH="sm_${arch}" \ + -DCMAKE_CUDA_STANDARD=17 \ + -DENABLE_HYPRE=ON \ + -DENABLE_HYPRE_DEVICE=CUDA +} + +# cray-mpich as installed on tioga/tuolumne. Prefers the flavor matching the +# compiler family ('amd' for amdclang, 'cray' for cce). +setup_cray_mpi() { + local flavor="${1:-amd}" + local mpi_dir="${MPI_DIR_OPT:-${MPICH_DIR:-}}" + if [[ -z "${mpi_dir}" ]]; then + mpi_dir="$(newest_path "/opt/cray/pe/mpich/*/ofi/${flavor}/*")" + fi + [[ -n "${mpi_dir}" && -x "${mpi_dir}/bin/mpicc" ]] \ + || die "cray-mpich not found (tried '${mpi_dir:-/opt/cray/pe/mpich/*/ofi/${flavor}/*}'); load the cray-mpich module or pass --mpi " + add_arg -DENABLE_MPI=ON \ + -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ + -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ + -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ + -DMPIEXEC_EXECUTABLE="srun" \ + -DMPIEXEC_NUMPROC_FLAG="-n" + MPI_PREFIX="${mpi_dir}" +} + +# BLAS/LAPACK: openblas when available, reference blas/lapack otherwise. +setup_blas_lapack() { + local candidates=( "$@" ) lib + for lib in "${candidates[@]}"; do + if [[ -e "${lib}" ]]; then + add_arg -DBLAS_LIBRARIES="${lib}" -DLAPACK_LIBRARIES="${lib}" + return 0 + fi + done + if [[ -e /usr/lib64/libblas.so && -e /usr/lib64/liblapack.so ]]; then + add_arg -DBLAS_LIBRARIES="/usr/lib64/libblas.so" -DLAPACK_LIBRARIES="/usr/lib64/liblapack.so" + return 0 + fi + warn "no BLAS/LAPACK found in the usual locations, letting cmake search for it" +} + +# Resolves the toolchain of a configuration into cmake cache arguments. +select_config() { + local cfg="$1" + CMAKE_ARGS=() + + case "${cfg}" in + # --------------------------------------------------------------- z5tux --- + z5tux) + add_arg -DCMAKE_C_COMPILER=/usr/bin/gcc \ + -DCMAKE_CXX_COMPILER=/usr/bin/g++ \ + -DCMAKE_Fortran_COMPILER="${FC_OPT:-/usr/bin/gfortran}" \ + -DENABLE_OPENMP="${ENABLE_OPENMP:-ON}" + setup_z5tux_mpi + setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so + ;; + + z5tux-cuda) + add_arg -DCMAKE_C_COMPILER=/usr/bin/gcc \ + -DCMAKE_CXX_COMPILER=/usr/bin/g++ \ + -DCMAKE_Fortran_COMPILER="${FC_OPT:-/usr/bin/gfortran}" \ + -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + setup_z5tux_mpi + setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so + setup_cuda "$(resolve_cuda)" "${GPU_ARCH:-120}" + ;; + + z5tux-hip) + local rocm; rocm="$(resolve_rocm)" + setup_rocm_compilers "${rocm}" + add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + setup_z5tux_mpi + setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so + setup_hip "${rocm}" "${GPU_ARCH:-gfx1100}" + ;; + + # ------------------------------------------------------------------ LC --- + matrix) + + local gcc_dir mpi_dir + gcc_dir="$(newest_path '/usr/tce/packages/gcc/gcc-13*-magic')" + [[ -n "${gcc_dir}" ]] || gcc_dir="$(newest_path '/usr/tce/packages/gcc/gcc-*-magic')" + [[ -n "${gcc_dir}" ]] || die "no gcc found under /usr/tce/packages/gcc" + add_arg -DCMAKE_C_COMPILER="${gcc_dir}/bin/gcc" \ + -DCMAKE_CXX_COMPILER="${gcc_dir}/bin/g++" \ + -DCMAKE_Fortran_COMPILER="${FC_OPT:-${gcc_dir}/bin/gfortran}" \ + -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + + local gcc_ver + gcc_ver="$(basename "${gcc_dir}")"; gcc_ver="${gcc_ver#gcc-}"; gcc_ver="${gcc_ver%-magic}" + mpi_dir="${MPI_DIR_OPT:-$(newest_path "/usr/tce/packages/mvapich2/mvapich2-*-gcc-${gcc_ver}-magic")}" + [[ -n "${mpi_dir}" ]] || mpi_dir="$(newest_path '/usr/tce/packages/mvapich2/mvapich2-*-gcc-*-magic')" + [[ -n "${mpi_dir}" && -x "${mpi_dir}/bin/mpicc" ]] \ + || die "mvapich2 not found under /usr/tce/packages/mvapich2 (use --mpi )" + add_arg -DENABLE_MPI=ON \ + -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ + -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ + -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ + -DMPIEXEC_EXECUTABLE="srun" \ + -DMPIEXEC_NUMPROC_FLAG="-n" + MPI_PREFIX="${mpi_dir}" + + setup_blas_lapack /usr/lib64/libopenblas.so + setup_cuda "$(resolve_cuda)" "${GPU_ARCH:-90}" + ;; + + tioga|tioga-gpu) + local rocm; rocm="$(resolve_rocm)" + setup_rocm_compilers "${rocm}" + add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + setup_cray_mpi amd + setup_blas_lapack /usr/lib64/libopenblas.so + setup_hip "${rocm}" "${GPU_ARCH:-gfx90a}" + ;; + + tuo-cpu|tuolumne-cpu) + local rocm; rocm="$(resolve_rocm)" + setup_rocm_compilers "${rocm}" + add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-ON}" + setup_cray_mpi amd + setup_blas_lapack /usr/lib64/libopenblas.so + ;; + + tuo-gpu|tuolumne-gpu|tuolumne) + local rocm; rocm="$(resolve_rocm)" + setup_rocm_compilers "${rocm}" + add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" + setup_cray_mpi amd + setup_blas_lapack /usr/lib64/libopenblas.so + setup_hip "${rocm}" "${GPU_ARCH:-gfx942}" + ;; + + *) + die "unknown configuration '${cfg}' (run --list to see the supported ones)" + ;; + esac +} + +# Scheduler command used by --alloc, per machine. +default_alloc_cmd() { + case "${CFG_MACHINE}" in + matrix) printf 'srun -N 1 --exclusive -t 180 -A %s' "${ACCOUNT}" ;; + tuolumne) printf 'srun -N 1 --exclusive -t 180 -A %s' "${ACCOUNT}" ;; + tioga) printf 'flux run -N 1 -t 180m' ;; + *) printf '' ;; + esac +} + +detect_config() { + local host="${HOSTNAME:-$(hostname)}" + case "${host}" in + z5tux*) printf 'z5tux' ;; + matrix*|rzmatrix*) printf 'matrix' ;; + tioga*|rzvernal*) printf 'tioga' ;; + tuolumne*|tuo[0-9]*) die "on tuolumne the configuration is ambiguous, pass 'tuo-cpu' or 'tuo-gpu'" ;; + *) die "cannot guess the configuration for host '${host}', pass one explicitly (--list)" ;; + esac +} + +# ------------------------------------------------------------------------------ +# Modules +# ------------------------------------------------------------------------------ +init_modules() { + if ! declare -F module >/dev/null 2>&1; then + local init + for init in /usr/share/lmod/lmod/init/bash /etc/profile.d/modules.sh /usr/share/Modules/init/bash; do + if [[ -r "${init}" ]]; then + # shellcheck disable=SC1090 + source "${init}" + break + fi + done + fi + declare -F module >/dev/null 2>&1 || type module >/dev/null 2>&1 +} + +load_modules() { + local -a mods=() + local m + if [[ -n "${MODULES_OPT}" ]]; then + read -r -a mods <<< "${MODULES_OPT}" + else + while IFS= read -r m; do + [[ -n "${m}" ]] && mods+=( "${m}" ) + done < <(modules_for_config "${CONFIG}") + fi + [[ ${#mods[@]} -gt 0 ]] || return 0 + + if ! init_modules; then + warn "no module command available, skipping module loading" + return 0 + fi + + local m + for m in "${mods[@]}"; do + if [[ ${DRY_RUN} -eq 1 ]]; then + log "+ module load ${m}" + continue + fi + if module load "${m}" >/dev/null 2>&1; then + log "module load ${m}" + else + # Retry without the version: LC renames those more often than the toolchains move. + local base="${m%%/*}" + if [[ "${base}" != "${m}" ]] && module load "${base}" >/dev/null 2>&1; then + log "module load ${base} (fallback, '${m}' unavailable)" + else + warn "could not load module '${m}' (continuing; run 'module avail ${base}' to check)" + fi + fi + done +} + +# ------------------------------------------------------------------------------ +# Build back-ends +# ------------------------------------------------------------------------------ +resolve_generator() { + case "${GENERATOR}" in + ninja) + command -v ninja >/dev/null 2>&1 || die "ninja not found in PATH (try --generator make)" + ;; + make) + ;; + auto) + if command -v ninja >/dev/null 2>&1; then + GENERATOR="ninja" + else + warn "ninja not found in PATH, falling back to Unix Makefiles" + GENERATOR="make" + fi + ;; + *) + die "unknown generator '${GENERATOR}' (auto|ninja|make)" + ;; + esac +} + +configure_cmake() { + local -a cmd=( cmake -S "${REPO_DIR}" -B "${BUILD_DIR}" ) + + if [[ "${GENERATOR}" == "ninja" ]]; then + cmd+=( -G Ninja -DENABLE_NINJA=ON ) + else + cmd+=( -G "Unix Makefiles" -DENABLE_NINJA=OFF ) + fi + + cmd+=( -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" + -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -DNUM_PROC="${JOBS}" + -DENABLE_TRILINOS="${ENABLE_TRILINOS}" + -DENABLE_SUPERLU_DIST="${ENABLE_SUPERLU_DIST}" + -DENABLE_HYPRE_GPU_PROFILING="${ENABLE_HYPRE_GPU_PROFILING}" + -DBLT_CXX_STD=c++17 + -DCMAKE_CXX_STANDARD=17 ) + cmd+=( "${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}" ) + cmd+=( "${EXTRA_CMAKE_ARGS[@]+"${EXTRA_CMAKE_ARGS[@]}"}" ) + cmd+=( "${FORWARDED_ARGS[@]+"${FORWARDED_ARGS[@]}"}" ) + + run "${cmd[@]}" + write_build_env +} + +# The mpi wrappers pick their back-end compiler from the MPICH_* variables, and +# the TPLs configured outside of cmake (hypre) inherit them from this script. +# Dump them next to the build so that running ninja by hand stays equivalent. +write_build_env() { + [[ ${DRY_RUN} -eq 0 && -n "${MPICH_CC:-}" ]] || return 0 + local env_file="${BUILD_DIR}/build-env.sh" + cat > "${env_file}" </dev/null 2>&1 || die "cmake not found in PATH" + log "generator : ${GENERATOR}" + log "jobs : ${JOBS} per TPL, ${TOP_JOBS} TPL(s) at a time" + + if [[ ${DO_CONFIGURE} -eq 1 ]]; then + configure_cmake + else + [[ -d "${BUILD_DIR}" ]] || die "--build-only given but '${BUILD_DIR}' does not exist" + fi + + if [[ ${DO_BUILD} -eq 1 ]]; then + build_cmake + fi + fi + + log "done." + if [[ ${DO_BUILD} -eq 1 && ${DRY_RUN} -eq 0 ]]; then + log "TPLs installed in ${INSTALL_DIR}" + fi +} + +if [[ -n "${LOG_FILE}" ]]; then + main 2>&1 | tee "${LOG_FILE}" + exit "${PIPESTATUS[0]}" +else + main +fi diff --git a/docker/pecan-spack.yaml b/docker/pecan-spack.yaml index 89ca8678..300cef98 100644 --- a/docker/pecan-spack.yaml +++ b/docker/pecan-spack.yaml @@ -111,6 +111,8 @@ spack: externals: - spec: py-sphinx@4.3.2 prefix: /usr + dependencies: + - spec: python@3.10.12 python: buildable: false externals: diff --git a/docker/ubuntu-spack.yaml b/docker/ubuntu-spack.yaml index e27514bf..40915d04 100644 --- a/docker/ubuntu-spack.yaml +++ b/docker/ubuntu-spack.yaml @@ -275,6 +275,8 @@ spack: externals: - spec: py-sphinx@7.4.7 prefix: /usr + dependencies: + - spec: python@3.12 python: buildable: false externals: diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index d0ffeed9..8b29be18 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -231,6 +231,8 @@ spack: externals: - spec: py-sphinx@1.6.3 prefix: /usr/gapps/GEOSX/thirdPartyLibs/python/quartz-gcc-python/python/ + dependencies: + - spec: python@3.10.8 autoconf: buildable: False externals: From 39cc571c43ab666366870aa0d2b4a4cd6021558f Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 22:40:30 -0400 Subject: [PATCH 19/27] Remove build.sh from versioning --- .gitignore | 1 - build.sh | 813 ----------------------------------------------------- 2 files changed, 814 deletions(-) delete mode 100755 build.sh diff --git a/.gitignore b/.gitignore index 36d0b675..488ea52a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ /build* -!/build.sh /install* .cproject .project diff --git a/build.sh b/build.sh deleted file mode 100755 index 723ab3f3..00000000 --- a/build.sh +++ /dev/null @@ -1,813 +0,0 @@ -#!/usr/bin/env bash -# -# Unified driver to build the GEOS third-party libraries. -# -# Supported configurations: -# z5tux gcc + mpich, host only -# z5tux-cuda gcc + mpich + CUDA (sm_120) -# z5tux-hip amdclang + mpich + ROCm (gfx1100) -# z5tux-sycl not supported yet (see NOTES below) -# matrix gcc + mvapich2 + CUDA (sm_90) -# tioga amdclang + cray-mpich + ROCm (gfx90a) -# tuo-cpu amdclang + cray-mpich, host only -# tuo-gpu amdclang + cray-mpich + ROCm (gfx942) -# -# By default the TPLs are built with the CMake superbuild of this repository -# (CMakeLists.txt), configured with the Ninja generator both for the superbuild -# itself and for every sub-project (-DENABLE_NINJA=ON). With --spack the same -# configurations are built through uberenv/spack instead. -# -# The script is meant to be run *on* the target machine: on the LC systems, -# clone/copy this repository there and run it from a login node (optionally -# with --alloc to push the compilation onto a compute node). -# -# NOTES -# * z5tux-sycl is declared but disabled: neither the superbuild nor the spack -# packages have SYCL plumbing, and no SYCL toolchain is installed on z5tux. -# * Trilinos is off by default (GEOS defaults to the hypre interface); enable -# it with --trilinos. -# * The ROCm configurations compile Fortran with gfortran, not amdflang: the -# flang runtime is not pulled in when clang++ links Fortran objects, which -# breaks superlu_dist. Override with --fc if needed. -# * Known issue: ROCm 7.2 (AMD clang 22) miscompiles hypre's spgemm kernels -# for gfx942/gfx1100 ("Illegal instruction detected: Operand has incorrect -# register class"); gfx90a is fine. LC uses ROCm 6.4.3, which is unaffected. -# -# Usage: ./build.sh [config] [options] [-D= ...] -# Run ./build.sh --help for the full option list. - -set -euo pipefail - -SCRIPT_NAME="$(basename "$0")" -REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" - -# ------------------------------------------------------------------------------ -# Defaults -# ------------------------------------------------------------------------------ -CONFIG="" -BUILD_TYPE="Release" -BUILD_DIR="" -INSTALL_DIR="" -JOBS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 8)" -TOP_JOBS=1 -GENERATOR="auto" # auto | ninja | make -GPU_ARCH="" # overrides the per-config cuda arch / amdgpu target -ROCM_DIR_OPT="" -CUDA_DIR_OPT="" -MPI_DIR_OPT="" -FC_OPT="" -USE_ALLOC=0 -ALLOC_CMD="" -ACCOUNT="${GEOS_TPL_ACCOUNT:-vortex}" -LOAD_MODULES=1 -MODULES_OPT="" -CLEAN=0 -DO_CONFIGURE=1 -DO_BUILD=1 -USE_SPACK=0 -SPACK_SPEC_OPT="" -ENABLE_TRILINOS="OFF" -ENABLE_OPENMP="" # per-config default unless set here -ENABLE_ROCTX="OFF" -ENABLE_SUPERLU_DIST="ON" -ENABLE_HYPRE_GPU_PROFILING="OFF" -DRY_RUN=0 -LOG_FILE="" -EXTRA_CMAKE_ARGS=() -FORWARDED_ARGS=() - -# Populated by select_config() -CFG_DESC="" -CFG_SUPPORTED=1 -CFG_UNSUPPORTED_REASON="" -CMAKE_ARGS=() -MPI_PREFIX="" # set by the setup_*_mpi helpers, used by setup_hip - -# ------------------------------------------------------------------------------ -# Helpers -# ------------------------------------------------------------------------------ -log() { printf '[%s] %s\n' "${SCRIPT_NAME}" "$*"; } -warn() { printf '[%s] WARNING: %s\n' "${SCRIPT_NAME}" "$*" >&2; } -die() { printf '[%s] ERROR: %s\n' "${SCRIPT_NAME}" "$*" >&2; exit 1; } - -run() { - printf '[%s] + %s\n' "${SCRIPT_NAME}" "$*" - if [[ ${DRY_RUN} -eq 0 ]]; then - "$@" - fi -} - -# Newest (version-sorted) existing path matching the given glob, empty if none. -newest_path() { - local match - # shellcheck disable=SC2086 # deliberate globbing - match="$(ls -d $1 2>/dev/null | sort -V | tail -1 || true)" - printf '%s' "${match}" -} - -require_file() { - [[ -e "$1" ]] || die "${2:-required file} not found: $1" -} - -require_exec() { - [[ -x "$1" ]] || die "${2:-required executable} not found or not executable: $1" -} - -add_arg() { CMAKE_ARGS+=( "$@" ); } - -usage() { - cat <= ...] [-- ] - -Configurations (auto-detected from the hostname when omitted): - z5tux gcc + mpich, host only - z5tux-cuda gcc + mpich + CUDA, sm_120 - z5tux-hip amdclang + mpich + ROCm, gfx1100 - z5tux-sycl not supported yet - matrix gcc + mvapich2 + CUDA, sm_90 (LC) - tioga amdclang + cray-mpich + ROCm, gfx90a (LC) - tuo-cpu amdclang + cray-mpich, host only (LC, tuolumne) - tuo-gpu amdclang + cray-mpich + ROCm, gfx942 (LC, tuolumne) - -Options: - -bt, --build-type Release|RelWithDebInfo|Debug|MinSizeRel (default: ${BUILD_TYPE}) - -b, --build-dir build directory (default: /build--) - -i, --install-dir install directory (default: /install--) - -j, --jobs parallelism inside each TPL (default: ${JOBS}) - --top-jobs TPLs built concurrently (default: ${TOP_JOBS}) - --generator auto|ninja|make (default: auto, prefers ninja) - --gpu-arch override the cuda arch (e.g. 90) or amdgpu target (e.g. gfx942) - --rocm ROCm installation to use - --cuda CUDA toolkit to use - --mpi MPI installation to use (expects /bin/mpicc) - --fc Fortran compiler to use - --trilinos build Trilinos as well (off by default) - --openmp override the per-configuration OpenMP setting - --roctx enable ROCTx/rocm profiling support (raja, caliper, hypre) - --superlu build superlu_dist and link it into hypre (default: ${ENABLE_SUPERLU_DIST}) - --gpu-profiling configure hypre with --enable-gpu-profiling (nvtx/roctx) - --alloc run the build step through the machine's scheduler - --alloc-cmd scheduler command to use with --alloc - --account bank used by --alloc (default: ${ACCOUNT}) - --modules "" replace the default module list - --no-modules do not touch modules at all - --spack build through uberenv/spack instead of the cmake superbuild - --spec "" spack spec to build (implies --spack) - --clean remove the build and install directories first - --configure-only stop after the cmake configure step - --build-only skip the configure step (reuse an existing build directory) - --log also write the output to - -n, --dry-run print the commands instead of running them - -l, --list list the configurations and exit - -h, --help show this help and exit - - Any -D= argument is forwarded to cmake (or, with --spack, any - argument after -- is forwarded to uberenv.py). - -Examples: - ./${SCRIPT_NAME} z5tux - ./${SCRIPT_NAME} z5tux-cuda -bt RelWithDebInfo -j 24 - ./${SCRIPT_NAME} z5tux-hip --gpu-arch gfx1036 - ./${SCRIPT_NAME} tuo-gpu --alloc # on a tuolumne login node - ./${SCRIPT_NAME} matrix --spack # same config, through spack - ./${SCRIPT_NAME} tioga -DENABLE_VTK=OFF -EOF -} - -# Static description of each configuration: used by --list and by the run banner, -# so that neither needs the toolchain to be resolved first. -describe_config() { - case "$1" in - z5tux) printf 'z5tux: gcc + mpich, host only' ;; - z5tux-cuda) printf 'z5tux: gcc + mpich + CUDA, sm_%s' "${GPU_ARCH:-120}" ;; - z5tux-hip) printf 'z5tux: amdclang + mpich + ROCm, %s' "${GPU_ARCH:-gfx1100}" ;; - z5tux-sycl) printf 'z5tux: SYCL (Intel Battlemage)' ;; - matrix) printf 'matrix: gcc + mvapich2 + CUDA, sm_%s' "${GPU_ARCH:-90}" ;; - tioga|tioga-gpu) printf 'tioga: amdclang + cray-mpich + ROCm, %s' "${GPU_ARCH:-gfx90a}" ;; - tuo-cpu|tuolumne-cpu) printf 'tuolumne: amdclang + cray-mpich, host only' ;; - tuo-gpu|tuolumne-gpu|tuolumne) printf 'tuolumne: amdclang + cray-mpich + ROCm, %s' "${GPU_ARCH:-gfx942}" ;; - *) printf '' ;; - esac -} - -# Modules to load on the LC machines. Kept separate from select_config() so that -# they are loaded *before* the compilers and cmake/ninja are looked up. -modules_for_config() { - case "$1" in - matrix) - printf '%s\n' "gcc/13.3.1-magic" "mvapich2/2.3.7" "cuda/12.9.1" "cmake/3.30.5" "ninja" ;; - tioga|tioga-gpu|tuo-cpu|tuo-gpu|tuolumne|tuolumne-cpu|tuolumne-gpu) - printf '%s\n' "rocm" "cray-mpich" "cmake/3.29" "ninja" ;; - *) - ;; - esac -} - -# Machine a configuration runs on (used for the scheduler defaults). -machine_for_config() { - case "$1" in - z5tux|z5tux-cuda|z5tux-hip|z5tux-sycl) printf 'z5tux' ;; - matrix) printf 'matrix' ;; - tioga|tioga-gpu) printf 'tioga' ;; - tuo-cpu|tuo-gpu|tuolumne*) printf 'tuolumne' ;; - *) printf '' ;; - esac -} - -# Spack spec used by --spack, mirroring scripts/setupLC-TPL-uberenv.bash. -# '%%' propagates the compiler choice to the dependencies (spack v1 syntax). -spack_spec_for_config() { - case "$1" in - z5tux) - printf '~docs %%gcc ^vtk generator=ninja' ;; - z5tux-cuda) - printf '+cuda ~uncrustify ~docs cuda_arch=%s %%gcc ^vtk generator=ninja' "${GPU_ARCH:-120}" ;; - z5tux-hip) - printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%llvm-amdgpu ^vtk generator=ninja' "${GPU_ARCH:-gfx1100}" ;; - matrix) - printf '+cuda ~uncrustify cuda_arch=%s %%%%gcc-13 ^cuda@12.9.1+allow-unsupported-compilers ^vtk generator=ninja' "${GPU_ARCH:-90}" ;; - tioga|tioga-gpu) - printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%%%cce-20 ^vtk generator=ninja' "${GPU_ARCH:-gfx90a}" ;; - tuo-cpu|tuolumne-cpu) - printf '~pygeosx ~trilinos ~petsc ~docs %%%%cce-20 ^vtk generator=ninja' ;; - tuo-gpu|tuolumne-gpu|tuolumne) - printf '+rocm ~pygeosx ~trilinos ~petsc ~docs amdgpu_target=%s %%%%cce-20 ^vtk generator=ninja' "${GPU_ARCH:-gfx942}" ;; - *) - printf '' ;; - esac -} - -# Configurations that cannot be built (yet); prints the reason. -unsupported_reason() { - case "$1" in - z5tux-sycl) printf 'no SYCL support in the superbuild nor in the spack packages, and no SYCL toolchain installed on z5tux' ;; - *) printf '' ;; - esac -} - -list_configs() { - local cfg - printf '%-12s %s\n' "CONFIG" "DESCRIPTION" - for cfg in z5tux z5tux-cuda z5tux-hip matrix tioga tuo-cpu tuo-gpu; do - printf '%-12s %s\n' "${cfg}" "$(describe_config "${cfg}")" - done - printf '%-12s %s [UNSUPPORTED: %s]\n' "z5tux-sycl" "$(describe_config z5tux-sycl)" \ - "no SYCL support in the superbuild nor in the spack packages" -} - -# ------------------------------------------------------------------------------ -# Argument parsing -# ------------------------------------------------------------------------------ -while [[ $# -gt 0 ]]; do - case "$1" in - -h|--help) usage; exit 0 ;; - -l|--list) list_configs; exit 0 ;; - -bt|--build-type) BUILD_TYPE="${2:?"missing value for $1"}"; shift 2 ;; - -b|--build-dir) BUILD_DIR="${2:?"missing value for $1"}"; shift 2 ;; - -i|--install-dir) INSTALL_DIR="${2:?"missing value for $1"}"; shift 2 ;; - -j|--jobs) JOBS="${2:?"missing value for $1"}"; shift 2 ;; - --top-jobs) TOP_JOBS="${2:?"missing value for $1"}"; shift 2 ;; - --generator) GENERATOR="${2:?"missing value for $1"}"; shift 2 ;; - --gpu-arch) GPU_ARCH="${2:?"missing value for $1"}"; shift 2 ;; - --rocm) ROCM_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --cuda) CUDA_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --mpi) MPI_DIR_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --fc) FC_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --trilinos) ENABLE_TRILINOS="ON"; shift ;; - --openmp) ENABLE_OPENMP="${2:?"missing value for $1"}"; shift 2 ;; - --roctx) ENABLE_ROCTX="ON"; shift ;; - --superlu) ENABLE_SUPERLU_DIST="${2:?"missing value for $1"}"; shift 2 ;; - --gpu-profiling) ENABLE_HYPRE_GPU_PROFILING="ON"; shift ;; - --alloc) USE_ALLOC=1; shift ;; - --alloc-cmd) ALLOC_CMD="$2"; USE_ALLOC=1; shift 2 ;; - --account) ACCOUNT="${2:?"missing value for $1"}"; shift 2 ;; - --modules) MODULES_OPT="${2:?"missing value for $1"}"; shift 2 ;; - --no-modules) LOAD_MODULES=0; shift ;; - --spack) USE_SPACK=1; shift ;; - --spec) SPACK_SPEC_OPT="${2:?"missing value for $1"}"; USE_SPACK=1; shift 2 ;; - --clean) CLEAN=1; shift ;; - --configure-only) DO_BUILD=0; shift ;; - --build-only) DO_CONFIGURE=0; shift ;; - --log) LOG_FILE="${2:?"missing value for $1"}"; shift 2 ;; - -n|--dry-run) DRY_RUN=1; shift ;; - -D*) EXTRA_CMAKE_ARGS+=( "$1" ); shift ;; - --) shift; FORWARDED_ARGS+=( "$@" ); break ;; - -*) die "unknown option '$1' (see --help)" ;; - *) - [[ -z "${CONFIG}" ]] || die "more than one configuration given ('${CONFIG}' and '$1')" - CONFIG="$1"; shift ;; - esac -done - -# ------------------------------------------------------------------------------ -# Configurations -# ------------------------------------------------------------------------------ - -# Compilers + MPI wrappers of the mpich installation used on z5tux. -setup_z5tux_mpi() { - local mpi_dir="${MPI_DIR_OPT:-${MPICH_HOME:-/home/victor/projects/mpich-4.2.2/install/release}}" - require_exec "${mpi_dir}/bin/mpicc" "mpicc (use --mpi )" - add_arg -DENABLE_MPI=ON \ - -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ - -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ - -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ - -DMPIEXEC_EXECUTABLE="${mpi_dir}/bin/mpirun" \ - -DMPIEXEC_NUMPROC_FLAG="-np" - MPI_PREFIX="${mpi_dir}" -} - -# The mpich/cray-mpich wrappers pick their back-end compiler from these. -export_mpich_backend() { - export MPICH_CC="$1" - export MPICH_CXX="$2" - export MPICH_FC="$3" - export MPICH_F90="$3" -} - -# ROCm installation: --rocm, then $ROCM_PATH, then the newest /opt/rocm-*. -resolve_rocm() { - local rocm="${ROCM_DIR_OPT:-${ROCM_PATH:-}}" - if [[ -z "${rocm}" ]]; then - rocm="$(newest_path '/opt/rocm-*')" - [[ -n "${rocm}" ]] || rocm="/opt/rocm" - fi - [[ -d "${rocm}" ]] || die "ROCm installation not found (tried '${rocm}', use --rocm )" - printf '%s' "${rocm}" -} - -# CUDA toolkit: --cuda, then $CUDA_HOME, then the LC/system locations. -resolve_cuda() { - local cuda="${CUDA_DIR_OPT:-${CUDA_HOME:-}}" - if [[ -z "${cuda}" ]]; then - cuda="$(newest_path '/usr/tce/packages/cuda/cuda-*')" - [[ -n "${cuda}" ]] || cuda="$(newest_path '/usr/local/cuda-*')" - [[ -n "${cuda}" ]] || cuda="/usr/local/cuda" - fi - [[ -d "${cuda}" ]] || die "CUDA toolkit not found (tried '${cuda}', use --cuda )" - printf '%s' "${cuda}" -} - -# Fortran compiler paired with the ROCm toolchain. gfortran is preferred over -# amdflang: clang++ links Fortran objects without the flang runtime, which -# breaks the mixed-language TPLs (superlu_dist). -resolve_fortran() { - local rocm="$1" fc - if [[ -n "${FC_OPT}" ]]; then - printf '%s' "${FC_OPT}"; return 0 - fi - for fc in /usr/bin/gfortran \ - "$(newest_path '/usr/tce/packages/gcc-tce/gcc-*/bin/gfortran')" \ - "$(newest_path '/usr/tce/packages/gcc/gcc-*-magic/bin/gfortran')"; do - if [[ -n "${fc}" && -x "${fc}" ]]; then - printf '%s' "${fc}"; return 0 - fi - done - warn "no gfortran found, falling back to amdflang (mixed C++/Fortran TPLs may fail to link)" - printf '%s' "${rocm}/llvm/bin/amdflang" -} - -# amdclang/amdclang++ out of a ROCm installation, plus a Fortran compiler. -setup_rocm_compilers() { - local rocm="$1" - local cc="${rocm}/llvm/bin/amdclang" - local cxx="${rocm}/llvm/bin/amdclang++" - local fc - fc="$(resolve_fortran "${rocm}")" - require_exec "${cc}" "amdclang" - require_exec "${cxx}" "amdclang++" - require_exec "${fc}" "Fortran compiler (use --fc )" - add_arg -DCMAKE_C_COMPILER="${cc}" \ - -DCMAKE_CXX_COMPILER="${cxx}" \ - -DCMAKE_Fortran_COMPILER="${fc}" - export_mpich_backend "${cc}" "${cxx}" "${fc}" -} - -# HIP settings shared by z5tux-hip, tioga and tuo-gpu. -setup_hip() { - local rocm="$1" arch="$2" - # cmake's HIP language requires clang itself, the hipcc wrapper is rejected. - require_exec "${rocm}/llvm/bin/amdclang++" "amdclang++" - add_arg -DENABLE_HIP=ON \ - -DROCM_PATH="${rocm}" \ - -DHIP_ROOT="${rocm}" \ - -DCMAKE_HIP_COMPILER="${rocm}/llvm/bin/amdclang++" \ - -DCMAKE_HIP_ARCHITECTURES="${arch}" \ - -DCMAKE_HIP_STANDARD=17 \ - -DENABLE_ROCTX="${ENABLE_ROCTX}" \ - -DENABLE_HYPRE=ON \ - -DENABLE_HYPRE_DEVICE=HIP - # hypre compiles its device sources outside of cmake and needs the MPI headers - [[ -z "${MPI_PREFIX}" ]] || add_arg -DMPI_INCLUDE_DIR="${MPI_PREFIX}/include" -} - -# CUDA settings shared by z5tux-cuda and matrix. -setup_cuda() { - local cuda="$1" arch="$2" - require_exec "${cuda}/bin/nvcc" "nvcc" - add_arg -DENABLE_CUDA=ON \ - -DENABLE_CUDA_NVTOOLSEXT=ON \ - -DCUDA_TOOLKIT_ROOT_DIR="${cuda}" \ - -DCMAKE_CUDA_COMPILER="${cuda}/bin/nvcc" \ - -DCMAKE_CUDA_ARCHITECTURES="${arch}" \ - -DCUDA_ARCH="sm_${arch}" \ - -DCMAKE_CUDA_STANDARD=17 \ - -DENABLE_HYPRE=ON \ - -DENABLE_HYPRE_DEVICE=CUDA -} - -# cray-mpich as installed on tioga/tuolumne. Prefers the flavor matching the -# compiler family ('amd' for amdclang, 'cray' for cce). -setup_cray_mpi() { - local flavor="${1:-amd}" - local mpi_dir="${MPI_DIR_OPT:-${MPICH_DIR:-}}" - if [[ -z "${mpi_dir}" ]]; then - mpi_dir="$(newest_path "/opt/cray/pe/mpich/*/ofi/${flavor}/*")" - fi - [[ -n "${mpi_dir}" && -x "${mpi_dir}/bin/mpicc" ]] \ - || die "cray-mpich not found (tried '${mpi_dir:-/opt/cray/pe/mpich/*/ofi/${flavor}/*}'); load the cray-mpich module or pass --mpi " - add_arg -DENABLE_MPI=ON \ - -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ - -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ - -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ - -DMPIEXEC_EXECUTABLE="srun" \ - -DMPIEXEC_NUMPROC_FLAG="-n" - MPI_PREFIX="${mpi_dir}" -} - -# BLAS/LAPACK: openblas when available, reference blas/lapack otherwise. -setup_blas_lapack() { - local candidates=( "$@" ) lib - for lib in "${candidates[@]}"; do - if [[ -e "${lib}" ]]; then - add_arg -DBLAS_LIBRARIES="${lib}" -DLAPACK_LIBRARIES="${lib}" - return 0 - fi - done - if [[ -e /usr/lib64/libblas.so && -e /usr/lib64/liblapack.so ]]; then - add_arg -DBLAS_LIBRARIES="/usr/lib64/libblas.so" -DLAPACK_LIBRARIES="/usr/lib64/liblapack.so" - return 0 - fi - warn "no BLAS/LAPACK found in the usual locations, letting cmake search for it" -} - -# Resolves the toolchain of a configuration into cmake cache arguments. -select_config() { - local cfg="$1" - CMAKE_ARGS=() - - case "${cfg}" in - # --------------------------------------------------------------- z5tux --- - z5tux) - add_arg -DCMAKE_C_COMPILER=/usr/bin/gcc \ - -DCMAKE_CXX_COMPILER=/usr/bin/g++ \ - -DCMAKE_Fortran_COMPILER="${FC_OPT:-/usr/bin/gfortran}" \ - -DENABLE_OPENMP="${ENABLE_OPENMP:-ON}" - setup_z5tux_mpi - setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so - ;; - - z5tux-cuda) - add_arg -DCMAKE_C_COMPILER=/usr/bin/gcc \ - -DCMAKE_CXX_COMPILER=/usr/bin/g++ \ - -DCMAKE_Fortran_COMPILER="${FC_OPT:-/usr/bin/gfortran}" \ - -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - setup_z5tux_mpi - setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so - setup_cuda "$(resolve_cuda)" "${GPU_ARCH:-120}" - ;; - - z5tux-hip) - local rocm; rocm="$(resolve_rocm)" - setup_rocm_compilers "${rocm}" - add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - setup_z5tux_mpi - setup_blas_lapack /usr/lib/x86_64-linux-gnu/libopenblas.so - setup_hip "${rocm}" "${GPU_ARCH:-gfx1100}" - ;; - - # ------------------------------------------------------------------ LC --- - matrix) - - local gcc_dir mpi_dir - gcc_dir="$(newest_path '/usr/tce/packages/gcc/gcc-13*-magic')" - [[ -n "${gcc_dir}" ]] || gcc_dir="$(newest_path '/usr/tce/packages/gcc/gcc-*-magic')" - [[ -n "${gcc_dir}" ]] || die "no gcc found under /usr/tce/packages/gcc" - add_arg -DCMAKE_C_COMPILER="${gcc_dir}/bin/gcc" \ - -DCMAKE_CXX_COMPILER="${gcc_dir}/bin/g++" \ - -DCMAKE_Fortran_COMPILER="${FC_OPT:-${gcc_dir}/bin/gfortran}" \ - -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - - local gcc_ver - gcc_ver="$(basename "${gcc_dir}")"; gcc_ver="${gcc_ver#gcc-}"; gcc_ver="${gcc_ver%-magic}" - mpi_dir="${MPI_DIR_OPT:-$(newest_path "/usr/tce/packages/mvapich2/mvapich2-*-gcc-${gcc_ver}-magic")}" - [[ -n "${mpi_dir}" ]] || mpi_dir="$(newest_path '/usr/tce/packages/mvapich2/mvapich2-*-gcc-*-magic')" - [[ -n "${mpi_dir}" && -x "${mpi_dir}/bin/mpicc" ]] \ - || die "mvapich2 not found under /usr/tce/packages/mvapich2 (use --mpi )" - add_arg -DENABLE_MPI=ON \ - -DMPI_C_COMPILER="${mpi_dir}/bin/mpicc" \ - -DMPI_CXX_COMPILER="${mpi_dir}/bin/mpicxx" \ - -DMPI_Fortran_COMPILER="${mpi_dir}/bin/mpifort" \ - -DMPIEXEC_EXECUTABLE="srun" \ - -DMPIEXEC_NUMPROC_FLAG="-n" - MPI_PREFIX="${mpi_dir}" - - setup_blas_lapack /usr/lib64/libopenblas.so - setup_cuda "$(resolve_cuda)" "${GPU_ARCH:-90}" - ;; - - tioga|tioga-gpu) - local rocm; rocm="$(resolve_rocm)" - setup_rocm_compilers "${rocm}" - add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - setup_cray_mpi amd - setup_blas_lapack /usr/lib64/libopenblas.so - setup_hip "${rocm}" "${GPU_ARCH:-gfx90a}" - ;; - - tuo-cpu|tuolumne-cpu) - local rocm; rocm="$(resolve_rocm)" - setup_rocm_compilers "${rocm}" - add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-ON}" - setup_cray_mpi amd - setup_blas_lapack /usr/lib64/libopenblas.so - ;; - - tuo-gpu|tuolumne-gpu|tuolumne) - local rocm; rocm="$(resolve_rocm)" - setup_rocm_compilers "${rocm}" - add_arg -DENABLE_OPENMP="${ENABLE_OPENMP:-OFF}" - setup_cray_mpi amd - setup_blas_lapack /usr/lib64/libopenblas.so - setup_hip "${rocm}" "${GPU_ARCH:-gfx942}" - ;; - - *) - die "unknown configuration '${cfg}' (run --list to see the supported ones)" - ;; - esac -} - -# Scheduler command used by --alloc, per machine. -default_alloc_cmd() { - case "${CFG_MACHINE}" in - matrix) printf 'srun -N 1 --exclusive -t 180 -A %s' "${ACCOUNT}" ;; - tuolumne) printf 'srun -N 1 --exclusive -t 180 -A %s' "${ACCOUNT}" ;; - tioga) printf 'flux run -N 1 -t 180m' ;; - *) printf '' ;; - esac -} - -detect_config() { - local host="${HOSTNAME:-$(hostname)}" - case "${host}" in - z5tux*) printf 'z5tux' ;; - matrix*|rzmatrix*) printf 'matrix' ;; - tioga*|rzvernal*) printf 'tioga' ;; - tuolumne*|tuo[0-9]*) die "on tuolumne the configuration is ambiguous, pass 'tuo-cpu' or 'tuo-gpu'" ;; - *) die "cannot guess the configuration for host '${host}', pass one explicitly (--list)" ;; - esac -} - -# ------------------------------------------------------------------------------ -# Modules -# ------------------------------------------------------------------------------ -init_modules() { - if ! declare -F module >/dev/null 2>&1; then - local init - for init in /usr/share/lmod/lmod/init/bash /etc/profile.d/modules.sh /usr/share/Modules/init/bash; do - if [[ -r "${init}" ]]; then - # shellcheck disable=SC1090 - source "${init}" - break - fi - done - fi - declare -F module >/dev/null 2>&1 || type module >/dev/null 2>&1 -} - -load_modules() { - local -a mods=() - local m - if [[ -n "${MODULES_OPT}" ]]; then - read -r -a mods <<< "${MODULES_OPT}" - else - while IFS= read -r m; do - [[ -n "${m}" ]] && mods+=( "${m}" ) - done < <(modules_for_config "${CONFIG}") - fi - [[ ${#mods[@]} -gt 0 ]] || return 0 - - if ! init_modules; then - warn "no module command available, skipping module loading" - return 0 - fi - - local m - for m in "${mods[@]}"; do - if [[ ${DRY_RUN} -eq 1 ]]; then - log "+ module load ${m}" - continue - fi - if module load "${m}" >/dev/null 2>&1; then - log "module load ${m}" - else - # Retry without the version: LC renames those more often than the toolchains move. - local base="${m%%/*}" - if [[ "${base}" != "${m}" ]] && module load "${base}" >/dev/null 2>&1; then - log "module load ${base} (fallback, '${m}' unavailable)" - else - warn "could not load module '${m}' (continuing; run 'module avail ${base}' to check)" - fi - fi - done -} - -# ------------------------------------------------------------------------------ -# Build back-ends -# ------------------------------------------------------------------------------ -resolve_generator() { - case "${GENERATOR}" in - ninja) - command -v ninja >/dev/null 2>&1 || die "ninja not found in PATH (try --generator make)" - ;; - make) - ;; - auto) - if command -v ninja >/dev/null 2>&1; then - GENERATOR="ninja" - else - warn "ninja not found in PATH, falling back to Unix Makefiles" - GENERATOR="make" - fi - ;; - *) - die "unknown generator '${GENERATOR}' (auto|ninja|make)" - ;; - esac -} - -configure_cmake() { - local -a cmd=( cmake -S "${REPO_DIR}" -B "${BUILD_DIR}" ) - - if [[ "${GENERATOR}" == "ninja" ]]; then - cmd+=( -G Ninja -DENABLE_NINJA=ON ) - else - cmd+=( -G "Unix Makefiles" -DENABLE_NINJA=OFF ) - fi - - cmd+=( -DCMAKE_BUILD_TYPE="${BUILD_TYPE}" - -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" - -DNUM_PROC="${JOBS}" - -DENABLE_TRILINOS="${ENABLE_TRILINOS}" - -DENABLE_SUPERLU_DIST="${ENABLE_SUPERLU_DIST}" - -DENABLE_HYPRE_GPU_PROFILING="${ENABLE_HYPRE_GPU_PROFILING}" - -DBLT_CXX_STD=c++17 - -DCMAKE_CXX_STANDARD=17 ) - cmd+=( "${CMAKE_ARGS[@]+"${CMAKE_ARGS[@]}"}" ) - cmd+=( "${EXTRA_CMAKE_ARGS[@]+"${EXTRA_CMAKE_ARGS[@]}"}" ) - cmd+=( "${FORWARDED_ARGS[@]+"${FORWARDED_ARGS[@]}"}" ) - - run "${cmd[@]}" - write_build_env -} - -# The mpi wrappers pick their back-end compiler from the MPICH_* variables, and -# the TPLs configured outside of cmake (hypre) inherit them from this script. -# Dump them next to the build so that running ninja by hand stays equivalent. -write_build_env() { - [[ ${DRY_RUN} -eq 0 && -n "${MPICH_CC:-}" ]] || return 0 - local env_file="${BUILD_DIR}/build-env.sh" - cat > "${env_file}" </dev/null 2>&1 || die "cmake not found in PATH" - log "generator : ${GENERATOR}" - log "jobs : ${JOBS} per TPL, ${TOP_JOBS} TPL(s) at a time" - - if [[ ${DO_CONFIGURE} -eq 1 ]]; then - configure_cmake - else - [[ -d "${BUILD_DIR}" ]] || die "--build-only given but '${BUILD_DIR}' does not exist" - fi - - if [[ ${DO_BUILD} -eq 1 ]]; then - build_cmake - fi - fi - - log "done." - if [[ ${DO_BUILD} -eq 1 && ${DRY_RUN} -eq 0 ]]; then - log "TPLs installed in ${INSTALL_DIR}" - fi -} - -if [[ -n "${LOG_FILE}" ]]; then - main 2>&1 | tee "${LOG_FILE}" - exit "${PIPESTATUS[0]}" -else - main -fi From 497f78ef30d2dabdb44d7a2e7e6f3152b026174d Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 22:56:03 -0400 Subject: [PATCH 20/27] Remove blueos spack config --- .../blueos_3_ppc64le_ib_p9/spack.yaml | 233 ------------------ 1 file changed, 233 deletions(-) delete mode 100644 scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml diff --git a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml b/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml deleted file mode 100644 index 1c720f3e..00000000 --- a/scripts/spack_configs/blueos_3_ppc64le_ib_p9/spack.yaml +++ /dev/null @@ -1,233 +0,0 @@ -#------------------------------------------------------------------------------------------------------------ -# SPDX-License-Identifier: LGPL-2.1-only -# -# Copyright (c) 2018-2020 Lawrence Livermore National Security LLC -# Copyright (c) 2018-2020 The Board of Trustees of the Leland Stanford Junior University -# Copyright (c) 2018-2020 TotalEnergies -# Copyright (c) 2019- GEOSX Contributors -# All rights reserved -# -# See top level LICENSE, COPYRIGHT, CONTRIBUTORS, NOTICE, and ACKNOWLEDGEMENTS files for details. -#------------------------------------------------------------------------------------------------------------ - -# geosx@develop%clang@10.0.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%gcc@8.3.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%clang@13.0.1+cuda cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers -# geosx@develop%clang@13.0.1+cuda cuda_arch=70 ^cuda@12.2.2+allow-unsupported-compilers -# -# Uberenv command to build geos dependencies: -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-10 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %gcc-8 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~uncrustify cuda_arch=70 %clang-13 ^cuda@11.8.0+allow-unsupported-compilers" -# -# python3 ./scripts/uberenv/uberenv.py --spec="+cuda~openmp~uncrustify cuda_arch=70 %clang-13 ^cuda@12.2.2+allow-unsupported-compilers" - - -spack: - config: - install_tree: - root: $spack/.. - projections: - all: '{compiler.name}-{compiler.version}/{name}-{version}-{hash}' - misc_cache: $spack/../misc_cache - test_stage: $spack/../test_stage - build_stage:: - - $spack/../build_stage - - # Regular TPLs do not need views - view: false - - # Include shared variants and versions - include: - - ../defaults.yaml - - ../versions.yaml - - toolchains: - clang-13: - - spec: '%[virtuals=c]llvm@13.0.1+clang~flang~lld~lldb' - when: '%c' - - spec: '%[virtuals=cxx]llvm@13.0.1+clang~flang~lld~lldb' - when: '%cxx' - - spec: '%[virtuals=fortran]gcc@8.3.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.clang.13' - when: '%mpi' - clang-10: - - spec: '%[virtuals=c]llvm@10.0.1+clang~flang~lld~lldb' - when: '%c' - - spec: '%[virtuals=cxx]llvm@10.0.1+clang~flang~lld~lldb' - when: '%cxx' - - spec: '%[virtuals=fortran]gcc@8.3.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.clang.10' - when: '%mpi' - gcc-12: - - spec: '%c=gcc@12.1.1' - when: '%c' - - spec: '%cxx=gcc@12.1.1' - when: '%cxx' - - spec: '%fortran=gcc@12.1.1' - when: '%fortran' - - spec: '%spectrum-mpi@release.gcc.12' - when: '%mpi' - gcc-8: - - spec: '%c=gcc@8.3.1 ' - when: '%c' - - spec: '%cxx=gcc@8.3.1 ' - when: '%cxx' - - spec: '%fortran=gcc@8.3.1 ' - when: '%fortran' - - spec: '%spectrum-mpi@release.gcc.8' - when: '%mpi' - - packages: - mpi: - require: - - spectrum-mpi - - zlib-api: - require: - - zlib - - blas: - require: - - essl - lapack: - require: - - essl - - llvm: - externals: - - spec: llvm@10.0.1+clang~flang~lld~lldb - prefix: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang++ - flags: {} - extra_rpaths: [] - - spec: llvm@13.0.1+clang~flang~lld~lldb - prefix: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang - cxx: /usr/tce/packages/clang/clang-13.0.1-gcc-8.3.1/bin/clang++ - flags: {} - extra_rpaths: [] - gcc: - externals: - - spec: gcc@8.3.1 languages:='c,c++,fortran' - prefix: /usr/tce/packages/gcc/gcc-8.3.1 - extra_attributes: - compilers: - c: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++ - fortran: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - extra_rpaths: [] - - spec: gcc@12.2.1 languages:='c,c++,fortran' - prefix: /usr/tce/packages/gcc/gcc-12.2.1 - extra_attributes: - compilers: - c: /usr/tce/packages/gcc/gcc-12.2.1/bin/gcc - cxx: /usr/tce/packages/gcc/gcc-12.2.1/bin/g++ - fortran: /usr/tce/packages/gcc/gcc-12.2.1/bin/gfortran - flags: - # Fix for "undefined reference to `_gfortran_transfer_integer_write'" - ldlibs: -lgfortran - extra_rpaths: [] - - spectrum-mpi: - buildable: False - externals: - - spec: spectrum-mpi@release.gcc.8 %gcc@8.3.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-8.3.1 - - spec: spectrum-mpi@release.gcc.12 %gcc@12.2.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-gcc-12.2.1 - # Previously clang@upstream - clingo doesn't like "upstream" version - - spec: spectrum-mpi@release.clang.10 %llvm@10.0.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-10.0.1-gcc-8.3.1 - - spec: spectrum-mpi@release.clang.13 %llvm@13.0.1 - prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-13.0.1-gcc-8.3.1 - - essl: - buildable: False - externals: - - spec: essl@6.3.0.2 - prefix: /usr/tcetmp/packages/essl/essl-6.3.0.2/ - - cuda: - buildable: False - externals: - - spec: cuda@11.8.0 +allow-unsupported-compilers - prefix: /usr/tce/packages/cuda/cuda-11.8.0 - - spec: cuda@12.2.2 +allow-unsupported-compilers - prefix: /usr/tce/packages/cuda/cuda-12.2.2 - - # System level packages to not build - cmake: - buildable: False - externals: - - spec: cmake@3.29.2 - prefix: /usr/tce/packages/cmake/cmake-3.29.2 - readline: - buildable: False - externals: - - spec: readline@7.0 - prefix: /collab/usr/gapps/python/build/spack-coralea.3/opt/spack/linux-rhel7-ppc64le/gcc-4.9.3/readline-7.0-owal6z2zh5zotgvdmwow6sgdn6cqfn43/ - m4: - buildable: False - externals: - - spec: m4@1.4.16 - prefix: /usr - perl: - buildable: false - externals: - - spec: perl@5.16.3 - prefix: /usr - pkg-config: - buildable: false - externals: - - spec: pkg-config@0.27.1 - prefix: /usr - diffutils: - buildable: False - externals: - - spec: diffutils@3.3 - prefix: /usr/bin/ - # This needs to be the prefix to the python3 LC installation - # or pygeosx package built by spack/uberenv - python: - buildable: False - externals: - - spec: python@3.8.2 - prefix: /usr/gapps/GEOSX/thirdPartyLibs/python/lassen-gcc-python/python/ - autoconf: - buildable: False - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - buildable: False - externals: - - spec: automake@1.13.4 - prefix: /usr - libtool: - buildable: False - externals: - - spec: libtool@2.4.2 - prefix: /usr - gettext: - buildable: False - externals: - - spec: gettext@0.19.8.1 - prefix: /usr/bin/ - zlib: - buildable: false - externals: - - spec: zlib@1.2.7 - prefix: /usr From ee8eafdf169c1d3c8e975755d0015bde0ced3ec1 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 23:04:09 -0400 Subject: [PATCH 21/27] Remove Lassen and ESSL support --- CMakeLists.txt | 5 --- cmake/thirdparty/FindMathLibraries.cmake | 9 ---- scripts/createLapackForESSL.sh | 33 --------------- .../spack_packages/packages/essl/package.py | 41 ------------------- .../spack_packages/packages/geosx/package.py | 30 +------------- 5 files changed, 1 insertion(+), 117 deletions(-) delete mode 100755 scripts/createLapackForESSL.sh delete mode 100644 scripts/spack_packages/packages/essl/package.py diff --git a/CMakeLists.txt b/CMakeLists.txt index dc910be1..379a583f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -666,11 +666,6 @@ if (ENABLE_TRILINOS) -D TPL_MKL_LIBRARIES:STRING=${MKL_LIBRARIES}) endif() - if ( ENABLE_ESSL ) - set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS} - -D HAVE_dggsvd3:BOOL=ON) - endif() - if ( DEFINED OpenMP_Fortran_FLAGS ) set( TRILINOS_EXTRA_ARGS ${TRILINOS_EXTRA_ARGS} -D OpenMP_Fortran_FLAGS:STRING=${OpenMP_Fortran_FLAGS}) diff --git a/cmake/thirdparty/FindMathLibraries.cmake b/cmake/thirdparty/FindMathLibraries.cmake index 340c3e1a..3c6e13c9 100644 --- a/cmake/thirdparty/FindMathLibraries.cmake +++ b/cmake/thirdparty/FindMathLibraries.cmake @@ -12,15 +12,6 @@ if( ENABLE_MKL ) set( BLAS_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE ) set( LAPACK_LIBRARIES "${MKL_LIBRARIES}" CACHE STRING "" FORCE) -elseif( ENABLE_ESSL ) - if ( NOT DEFINED ESSL_INCLUDE_DIRS ) - message( FATAL_ERROR "ESSL is enabled but ESSL_INCLUDE_DIRS is not defined." ) - elseif ( NOT DEFINED ESSL_LIBRARIES ) - message( FATAL_ERROR "ESSL is enabled but ESSL_LIBRARIES is not defined." ) - endif() - - set( BLAS_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE ) - set( LAPACK_LIBRARIES "${ESSL_LIBRARIES}" CACHE STRING "" FORCE ) endif() if( NOT DEFINED BLAS_LIBRARIES ) diff --git a/scripts/createLapackForESSL.sh b/scripts/createLapackForESSL.sh deleted file mode 100755 index 59b09ca9..00000000 --- a/scripts/createLapackForESSL.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# Adapted from IBM Engineering and Scientific Subroutine Library for Linux on POWER Version 6.2 ESSL Guide and Reference -# -# First Pass - get the symbols in ESSL and LAPACK -echo "The essl library is: " $1 -echo "The lapack library is: " $2 -echo "The output library is: liblapackforessl.a" -nm $2 | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.lapack -nm $1 | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.essl -diff -y symbols.in.lapack symbols.in.essl | grep -v -e '>' -e '|' -e '<' | awk '{ print $1 }' > duplicate.symbols -wc -l duplicate.symbols - -# Second Pass - remove the ESSL symbols from the LAPACK library -cp -p $2 liblapackforessl.a - -for SYMBOL in $(cat duplicate.symbols | awk '{ print $1 }') -do - fn=$SYMBOL - unders="no" - echo $SYMBOL | grep "_" > /dev/null && unders="yes" - if [ "$unders" = "yes" ] - then - fn=`echo $SYMBOL | awk -F'_' '{print $1}'` - fi - - echo $SYMBOL - ar d liblapackforessl.a $fn.o -done - -# Check it worked -nm liblapackforessl.a | grep " T " | awk '{ print $3 }' | sort -n > symbols.in.updatedlapack -diff -y symbols.in.updatedlapack symbols.in.essl | grep -v -e '>' -e '|' -e '<' | awk '{ print $1 }' > duplicate.symbols.inupdatedlapack -wc -l duplicate.symbols.inupdatedlapack diff --git a/scripts/spack_packages/packages/essl/package.py b/scripts/spack_packages/packages/essl/package.py deleted file mode 100644 index fa2a2cb7..00000000 --- a/scripts/spack_packages/packages/essl/package.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack.package import * -from spack_repo.builtin.build_systems.bundle import BundlePackage - -# Recipe for pre-built essl library on blueos machines. -# Defines additonal flags for blueos: -# https://lc.llnl.gov/confluence/display/SIERRA/Math+Libraries -class Essl(BundlePackage): - """IBM's Engineering and Scientific Subroutine Library (ESSL).""" - - homepage = "https://www.ibm.com/systems/power/software/essl/" - - version("6.3.0.2") - - provides("blas") - provides("lapack") - - @property - def blas_libs(self): - spec = self.spec - prefix = self.prefix - - essl_root = prefix.lib64 - essl_libs = ["libessl", "liblapackforessl", "liblapackforessl_"] - all_libs = find_libraries(essl_libs, root=essl_root, shared=True) - - return all_libs - - @property - def lapack_libs(self): - spec = self.spec - prefix = self.prefix - - essl_root = prefix.lib64 - essl_libs = ["libessl", "liblapackforessl", "liblapackforessl_"] - all_libs = find_libraries(essl_libs, root=essl_root, shared=True) - - return all_libs diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index eaa469de..d75a4ccc 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -356,9 +356,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): debug_flags = "-g" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) - if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) - cfg.write("#{0}\n".format("-" * 80)) cfg.write("# CMake Standard\n") cfg.write("#{0}\n\n".format("-" * 80)) @@ -375,16 +372,7 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): hostname = socket.gethostname().rstrip('1234567890') - if sys_type in ('linux-rhel7-ppc64le', 'linux-rhel8-ppc64le', 'blueos_3_ppc64le_ib_p9') \ - and hostname != 'p3dev': - cfg.write(cmake_cache_option('ENABLE_WRAP_ALL_TESTS_WITH_MPIEXEC', True)) - if hostname in ('lassen', 'rzansel'): - cfg.write(cmake_cache_entry('MPIEXEC', 'lrun')) - cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) - else: - cfg.write(cmake_cache_entry('MPIEXEC', 'jsrun')) - cfg.write(cmake_cache_list('MPIEXEC_NUMPROC_FLAG', ['-g1', '--bind', 'rs', '-n'])) - elif sys_type in ('toss_4_x86_64_ib_cray'): + if sys_type in ('toss_4_x86_64_ib_cray'): cfg.write(cmake_cache_entry('MPIEXEC', 'srun')) cfg.write(cmake_cache_entry('MPIEXEC_NUMPROC_FLAG', '-n')) else: @@ -470,10 +458,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS', cmake_cuda_flags)) - # System specific flags - if sys_type in ('linux-rhel7-ppc64le', 'linux-rhel8-ppc64le', 'blueos_3_ppc64le_ib_p9'): - cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELEASE', '-O3 -DNDEBUG -Xcompiler -DNDEBUG -Xcompiler -O3 -Xcompiler -mcpu=powerpc64le -Xcompiler -mtune=powerpc64le')) - cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_RELWITHDEBINFO', '-g -lineinfo ${CMAKE_CUDA_FLAGS_RELEASE}')) cfg.write(cmake_cache_string('CMAKE_CUDA_FLAGS_DEBUG', '-g -G -O0 -Xcompiler -O0')) @@ -568,12 +552,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('ENABLE_MKL', True)) cfg.write(cmake_cache_entry('MKL_INCLUDE_DIRS', spec['intel-mkl'].prefix.include)) cfg.write(cmake_cache_list('MKL_LIBRARIES', spec['intel-mkl'].libs)) - elif spec["blas"].name == "essl": - cfg.write(cmake_cache_option('ENABLE_ESSL', True)) - cfg.write(cmake_cache_entry('ESSL_INCLUDE_DIRS', spec['essl'].prefix.include)) - cfg.write(cmake_cache_list('ESSL_LIBRARIES', spec['blas'].libs)) - - cfg.write(cmake_cache_option('FORTRAN_MANGLE_NO_UNDERSCORE', True)) else: cfg.write(cmake_cache_list('BLAS_LIBRARIES', spec['blas'].libs)) cfg.write(cmake_cache_list('LAPACK_LIBRARIES', spec['lapack'].libs)) @@ -689,9 +667,6 @@ def geos_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): cfg.write(cmake_cache_option('GEOS_BUILD_SHARED_LIBS', False)) # ATS - # Lassen - if sys_type in ('blueos_3_ppc64le_ib_p9'): - cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--ats jsrun_omp --ats jsrun_bind=packed')) # Dane/Matrix if sys_type in ('toss_4_x86_64_ib'): cfg.write(cmake_cache_string('ATS_ARGUMENTS', '--machine slurm112')) @@ -769,9 +744,6 @@ def lvarray_hostconfig(self, spec, prefix, py_site_pkgs_dir=None): debug_flags = "-g" cfg.write(cmake_cache_string("CMAKE_CXX_FLAGS_DEBUG", debug_flags)) - if "%clang arch=linux-rhel7-ppc64le" in spec: - cfg.write(cmake_cache_entry("CMAKE_EXE_LINKER_FLAGS", "-Wl,--no-toc-optimize")) - cfg.write('#{0}\n'.format('-' * 80)) cfg.write('# Cuda\n') cfg.write('#{0}\n\n'.format('-' * 80)) From 8bb038e041012f4163562791028fbf126fd426fd Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 23:04:25 -0400 Subject: [PATCH 22/27] Use the new spack installer on the LC machines --- scripts/spack_configs/toss_4_x86_64_ib/spack.yaml | 2 ++ scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml index 8b29be18..e6b12357 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib/spack.yaml @@ -39,6 +39,8 @@ spack: test_stage: $spack/../test_stage build_stage: $spack/../build_stage build_jobs: 112 + # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. + installer: new # Regular TPLs do not need views view: false diff --git a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml index 76404e3a..48af6ee7 100644 --- a/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml +++ b/scripts/spack_configs/toss_4_x86_64_ib_cray/spack.yaml @@ -31,6 +31,8 @@ spack: test_stage: $spack/../test_stage build_stage: $spack/../build_stage build_jobs: 96 + # Rewritten installer, default since spack v1.2.0. Set to 'old' to fall back. + installer: new # Regular TPLs do not need views view: false From bc978545b488f7f30dd967433c4566ba3e7a74c0 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Tue, 4 Aug 2026 23:12:47 -0400 Subject: [PATCH 23/27] Disable Trilinos by default --- CMakeLists.txt | 2 +- scripts/spack_packages/packages/geosx/package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 379a583f..dbbe214d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -47,7 +47,7 @@ option( BUILD_SHARED_LIBS "" OFF ) option( ENABLE_CALIPER "" ON ) option( ENABLE_HYPRE "" ON ) -option( ENABLE_TRILINOS "" ON ) +option( ENABLE_TRILINOS "" OFF ) option( ENABLE_PETSC "" OFF ) option( ENABLE_SCOTCH "" ON ) option( ENABLE_SUITESPARSE "" ON ) diff --git a/scripts/spack_packages/packages/geosx/package.py b/scripts/spack_packages/packages/geosx/package.py index d75a4ccc..5bd2a7ae 100644 --- a/scripts/spack_packages/packages/geosx/package.py +++ b/scripts/spack_packages/packages/geosx/package.py @@ -64,7 +64,7 @@ class Geosx(CMakePackage, CudaPackage, ROCmPackage): variant('shared', default=True, description='Build Shared Libs.') variant('caliper', default=True, description='Build Caliper support.') variant('vtk', default=True, description='Build VTK support.') - variant('trilinos', default=True, description='Build Trilinos support.') + variant('trilinos', default=False, description='Build Trilinos support.') variant('hypre', default=True, description='Build HYPRE support.') variant('hypredrive', default=True, description='Build hypredrive support.') variant('petsc', default=False, description='Build PETSc support.') From c6dfa5796abae6fe46f5f54d0533dd339e96befb Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 5 Aug 2026 01:06:35 -0400 Subject: [PATCH 24/27] Restrict the docker build context to what the TPL build needs --- scripts/docker-build.sh | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index b86bcc0c..4eb0b526 100644 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -2,8 +2,29 @@ set -e env -# We save memory for the docker context -echo .git > .dockerignore +# Keep the docker context to the sources the TPL build actually needs. +# +# In CI this is a fresh checkout and only .git matters. Run locally via +# scripts/reproduce_ci.sh, however, the working tree usually also holds build +# and install trees, which added ~27GB to every job's context, and stray +# host-configs left at the repo root by earlier local builds. Those last ones +# are not merely wasteful: the TPL Dockerfiles finish with +# `cp *.cmake /spack-generated.cmake`, so any root-level *.cmake beyond the one +# uberenv just generated makes cp treat the destination as a directory and the +# build fails after the whole TPL stack has been compiled. No tracked file at +# the repo root matches these patterns. +cat > .dockerignore <<'DOCKERIGNORE' +.git +build +build-* +install +install-* +*_tpls +.rocm-ci-local.* +*.cmake +*.log +__pycache__ +DOCKERIGNORE # Get uberenv submodule git submodule update --init --force scripts/uberenv From 14fe67f3650455c242ea21842eaca353e964c91d Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 5 Aug 2026 01:47:34 -0400 Subject: [PATCH 25/27] Install xz-devel in the Rocky TPL image docker/rocky-spack.yaml declares xz as a non-buildable external at /usr, but the image only installed the xz command, not its headers, so /usr/include/lzma.h was absent. Nothing needed it until the spack v1.2.2 package repo pulled in file@5.46, whose configure then failed the Rocky builds with: configure: error: xzlib support requested but not found Ubuntu is unaffected: file never enters its dependency graph. Note the same file still declares xz@5.2.5 while Rocky 8 actually ships 5.2.4; that entry is shared with the Rocky 9 rows, which do ship 5.2.5, so it is left alone here rather than split per-distro. --- docker/tpl-rockylinux.Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/tpl-rockylinux.Dockerfile b/docker/tpl-rockylinux.Dockerfile index 27cbe2bb..ac39daf7 100644 --- a/docker/tpl-rockylinux.Dockerfile +++ b/docker/tpl-rockylinux.Dockerfile @@ -41,7 +41,8 @@ RUN dnf clean all && \ bzip2 \ gnupg2 \ perl \ - xz && \ + xz \ + xz-devel && \ (dnf -y install python3-virtualenv || \ /usr/bin/python3 -m pip install --no-cache-dir virtualenv) && \ dnf clean all && rm -rf /var/cache/dnf /var/lib/dnf From e6b15f5249cbf4393c7a9199bf1995c63332fd6c Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 5 Aug 2026 03:00:48 -0400 Subject: [PATCH 26/27] Pass reproduce_ci extra_env through env Bash only treats literal VAR=value words as assignments, so the expanded ${extra_env} was parsed as the command name and the ROCm row died with 'ROCM_VERSION=6.4.3: command not found' before docker build ran. --- scripts/reproduce_ci.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/reproduce_ci.sh b/scripts/reproduce_ci.sh index 8c6b5a5e..75ddf01c 100755 --- a/scripts/reproduce_ci.sh +++ b/scripts/reproduce_ci.sh @@ -207,6 +207,8 @@ build() { return fi + # extra_env is expanded, and bash only honours literal VAR=value words as + # assignments, so it has to go through env rather than the prefix list. if ! TPL_DOCKERFILE=${dockerfile} \ DOCKER_REPOSITORY=${repository} \ DOCKER_BASE_IMAGE="${base_image}" \ @@ -220,7 +222,7 @@ build() { DOCKER_TAG=${DOCKER_TAG} \ DOCKER_BUILDER=${builder} \ DOCKER_LOAD=1 \ - ${extra_env} \ + env ${extra_env} \ bash -x scripts/docker-build.sh; then printf 'FAILED: %s (builder retained: %s)\n' "${name}" "${builder}" >&2 failed=1 From 8cf106d26d4380dfc55657c80730a8911f5dc7b6 Mon Sep 17 00:00:00 2001 From: "Victor A. P. Magri" Date: Wed, 5 Aug 2026 03:05:11 -0400 Subject: [PATCH 27/27] Realign cmake and findutils version preferences in the ROCm spack env The earlier external-version corrections updated the external specs but not the matching 'version:' preference for cmake and findutils. Both are buildable:false, so the stale preference matched no known version and spack refused to concretize: ==> Error: Preference for version 3.28.3 does not match any known version of cmake --- docker/spack-rocm.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/spack-rocm.yaml b/docker/spack-rocm.yaml index 5231d9c7..90a8d257 100644 --- a/docker/spack-rocm.yaml +++ b/docker/spack-rocm.yaml @@ -329,13 +329,13 @@ spack: - spec: automake@1.16.5 prefix: /usr cmake: - version: [3.28.3] + version: [3.28.6] buildable: false externals: - spec: cmake@3.28.6 prefix: /usr/local findutils: - version: [4.7.0] + version: [4.9.0] buildable: false externals: - spec: findutils@4.9.0