Accelerator substrate, CPU vertical slice, and a working 1D Euler HIP contract

CMakeLists.txt

@@ -72,6 +72,11 @@ set_property ( GLOBAL PROPERTY THIRD_PROPERTY::DEFINE_SYMBOL )
 
 enable_testing()
 
+option(
+    ONEFLOW_ENABLE_HIP_TESTS
+    "Register hardware-dependent HIP/DCU tests in the root CTest tree"
+    OFF )
+
 add_subdirectory ( ThirdParty ${THIRD_PARTY_DIR} )
 add_subdirectory ( codes      ${CMAKE_PROJECT_NAME} )
 add_subdirectory ( tests )

ci/kunshan/README.md

@@ -46,8 +46,18 @@ For the isolated Kunshan Euler backend contract, use
 explicitly enables `ONEFLOW_1D_ENABLE_HIP` and `ONEFLOW_1D_ENABLE_GTEST`. The
 job executes the GoogleTest binary and CTest, and records CMake, compiler, HIP
 architecture, `rocminfo`, and Slurm evidence. CPU/HIP tests use `CPU.`/`HIP.`
-CTest prefixes; an empty discovery result or a zero exit code without the expected
-summary is not successful validation.
+CTest prefixes and the shared helper attaches
+`hardware;hip;dcu` metadata. The cluster CTest 2.8 compatibility path exposes
+`hardware` as the selectable label, so the DCU runner uses `-L hardware -R HIP`;
+an empty discovery result or a zero exit code without the expected summary is
+not successful validation.
+
+The root project exposes the same HIP contract through
+`-DONEFLOW_ENABLE_HIP_TESTS=ON`. It is deliberately opt-in: a normal CPU build
+keeps the root CTest suite hardware-independent, while a target-node build must
+also provide `CMAKE_HIP_COMPILER` and `CMAKE_HIP_ARCHITECTURES` (currently
+`gfx906` on Kunshan Z100). The standalone project and the root project share the
+same CMake registration helper.
 
 ## Execution
 
@@ -109,6 +119,7 @@ Dependency wiring uses the variables the top-level CMake reads:
 - `strict` profile (`1e-15`, `ONEFLOW_RESIDUAL_TEST_OUTPUT=1`): 5/5 cases
   passed, max absolute difference 1.1e-17.
 - Kunshan port CPU contract test (`-DONEFLOW_1D_ENABLE_GTEST=ON`): 5/5 passed.
+- Kunshan standalone HIP contract (`dcu:1`, `gfx906`): 9/9 GoogleTest and 9/9 CTest passed on DTK 26.04.
 
 ## Verified DCU/HIP environment (kshdnormal)
 
@@ -123,7 +134,7 @@ CPU solver with the DTK toolchain:
 
 ### Correctness
 
-- HIP contract test: 6/6 passed (GoogleTest and CTest, `HIP.` prefix).
+- HIP contract test: 9/9 passed (GoogleTest and CTest, `HIP.` prefix), including WENO5 CPU-oracle comparison.
 - Stateful benchmark, all four sizes: `final_max_abs_error = 0.000000` and
   identical CPU/HIP checksums.
 
@@ -171,7 +182,7 @@ the cluster-side `README.md`; the structure is:
 | Suite | Partition | Resources | Content | Pass criteria |
 |---|---|---|---|---|
 | `cpu-regression` | `kshcnormal` | 16 CPU, 54G | five-case normal+strict; port CPU contract test | 5/5 normal, 5/5 strict, 5/5 contract |
-| `dcu-single` | `kshdnormal` | 8 CPU, 27G, `dcu:1` | HIP contract test; stateful benchmark, four sizes | 6/6; max abs error 0 and matching checksums |
+| `dcu-single` | `kshdnormal` | 8 CPU, 27G, `dcu:1` | HIP contract test; stateful benchmark, four sizes | 9/9 contract; max abs error 0 and matching checksums for benchmark |
 | `cpu-mpi` | `kshcnormal` | 32 ranks × 1 CPU | 32-rank CPU MPI benchmark, four sizes | exit 0, hashes valid |
 | `dcu-mpi` | `kshdnormal` | 4 ranks × 8 CPU, `dcu:4` | 1-rank and 4-rank DCU MPI benchmark, four sizes | exit 0, hashes match `cpu-mpi`, `visible_devices=4` |
 

ci/kunshan/e3-cpu-oracle.slurm

@@ -0,0 +1,105 @@
+#!/bin/bash -l
+# Compare the legacy and CPU-batch 3D inviscid flux paths on one identical case.
+# The source tree and already-built solver are supplied by environment variables.
+#SBATCH --job-name=oneflow_e3_cpu_oracle
+#SBATCH --partition=kshcnormal
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=16
+#SBATCH --mem=54G
+#SBATCH --time=00:20:00
+#SBATCH --output=%x-%j.out
+#SBATCH --error=%x-%j.err
+
+set -euo pipefail
+
+: "${ONEFLOW_E3_ORACLE_SOURCE_DIR:?Set ONEFLOW_E3_ORACLE_SOURCE_DIR}"
+: "${ONEFLOW_E3_ORACLE_BINARY:?Set ONEFLOW_E3_ORACLE_BINARY}"
+
+module purge
+module load compiler/gcc/9.3.0
+module load mpi/openmpi/gcc-9.3.0/4.1.5
+module load python/3.8.10
+
+source_case="$ONEFLOW_E3_ORACLE_SOURCE_DIR/test/m6wingroe_sa"
+oracle_root="${ONEFLOW_E3_ORACLE_RUN_DIR:-$PWD/e3-cpu-oracle-${SLURM_JOB_ID:-manual}}"
+legacy_case="$oracle_root/legacy/m6wingroe_sa"
+batch_case="$oracle_root/batch/m6wingroe_sa"
+
+rm -rf "$oracle_root"
+mkdir -p "$oracle_root/legacy" "$oracle_root/batch"
+cp -a "$source_case" "$legacy_case"
+cp -a "$source_case" "$batch_case"
+
+for case_dir in "$legacy_case" "$batch_case"; do
+    rm -rf "$case_dir/results" "$case_dir/restart" "$case_dir/log"
+    mkdir -p "$case_dir/results" "$case_dir/restart" "$case_dir/log"
+    sed -i -E 's/^int[[:space:]]+ischeme[[:space:]]*=.*/int ischeme   = 5; \/\/ Lax-Friedrichs oracle/' \
+        "$case_dir/script/cfd.txt"
+done
+
+run_case() {
+    local mode=$1
+    local case_dir=$2
+    (
+        cd "$(dirname "$case_dir")"
+        export ONEFLOW_ACCEL_BACKEND=CPU
+        if [ "$mode" = batch ]; then
+            export ONEFLOW_ENABLE_UNS_CPU_BATCH=1
+        else
+            unset ONEFLOW_ENABLE_UNS_CPU_BATCH || true
+        fi
+        timeout --signal=TERM --kill-after=10s 180 \
+            mpirun -np 1 "$ONEFLOW_E3_ORACLE_BINARY" 0 "$(basename "$case_dir")"
+    ) > "$oracle_root/$mode.log" 2>&1
+}
+
+run_case legacy "$legacy_case"
+run_case batch "$batch_case"
+
+python3 - "$legacy_case" "$batch_case" <<'PY'
+import math
+import pathlib
+import sys
+
+legacy = pathlib.Path(sys.argv[1])
+batch = pathlib.Path(sys.argv[2])
+files = [
+    "results/aero.dat",
+    "results/wallaero.dat",
+    "results/res.dat",
+    "results/turbres.dat",
+]
+absolute_limit = 2.0e-12
+relative_limit = 1.0e-10
+overall_abs = 0.0
+overall_rel = 0.0
+
+def numbers(path):
+    for line in path.read_text(encoding="utf-8-sig").splitlines():
+        for token in line.split():
+            try:
+                yield float(token)
+            except ValueError:
+                continue
+
+for rel in files:
+    left = list(numbers(legacy / rel))
+    right = list(numbers(batch / rel))
+    if len(left) != len(right):
+        raise SystemExit(f"ORACLE_FAIL file={rel} count={len(left)} vs {len(right)}")
+    file_abs = 0.0
+    file_rel = 0.0
+    for a, b in zip(left, right):
+        delta = abs(a - b)
+        scale = max(abs(a), abs(b), 1.0e-300)
+        file_abs = max(file_abs, delta)
+        file_rel = max(file_rel, delta / scale)
+    overall_abs = max(overall_abs, file_abs)
+    overall_rel = max(overall_rel, file_rel)
+    print(f"ORACLE file={rel} max_absolute={file_abs:.17g} max_relative={file_rel:.17g}")
+    if file_abs > absolute_limit and file_rel > relative_limit:
+        raise SystemExit(f"ORACLE_FAIL file={rel}")
+
+print(f"ORACLE_PASS max_absolute={overall_abs:.17g} max_relative={overall_rel:.17g}")
+PY

ci/kunshan/e6-cpu-trace-verify.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""Compare OneFLOW main-solver qf1/qf2/invflux CPU traces."""
+import argparse
+import math
+import pathlib
+import struct
+
+MAGIC = bytes((79, 70, 84, 82, 67, 48, 49, 0))
+HEADER_SIZE = 24
+
+def read_trace(path):
+    path = pathlib.Path(path)
+    data = path.read_bytes()
+    magic, n_faces, n_equations, n_arrays = struct.unpack_from(
+        "<8sQII", data, 0)
+    if magic != MAGIC or n_arrays != 3:
+        raise SystemExit("TRACE_HEADER_FAIL path={}".format(path))
+    count = n_faces * n_equations
+    offset = HEADER_SIZE
+    arrays = []
+    for _ in range(3):
+        end = offset + count * 8
+        if end > len(data):
+            raise SystemExit("TRACE_SIZE_FAIL path={}".format(path))
+        arrays.append(struct.unpack_from("<{}d".format(count), data, offset))
+        offset = end
+    if offset != len(data):
+        raise SystemExit("TRACE_SIZE_FAIL path={}".format(path))
+    return int(n_faces), int(n_equations), arrays
+
+def compare(legacy_path, batch_path):
+    legacy = read_trace(legacy_path)
+    batch = read_trace(batch_path)
+    if legacy[:2] != batch[:2]:
+        raise SystemExit(
+            "TRACE_SHAPE_FAIL legacy={} batch={}".format(
+                legacy[:2], batch[:2]))
+    n_faces, n_equations, legacy_arrays = legacy
+    _, _, batch_arrays = batch
+    overall_absolute = 0.0
+    overall_relative = 0.0
+    for name, left, right in zip(
+            ("qf1", "qf2", "invflux"), legacy_arrays, batch_arrays):
+        if not all(math.isfinite(value) for value in left + right):
+            raise SystemExit("TRACE_FINITE_FAIL array={}".format(name))
+        absolute = max(abs(a - b) for a, b in zip(left, right))
+        relative = max(
+            abs(a - b) / max(abs(a), abs(b), 1.0e-300)
+            for a, b in zip(left, right))
+        overall_absolute = max(overall_absolute, absolute)
+        overall_relative = max(overall_relative, relative)
+        print(
+            "TRACE array={} n={} max_absolute={:.17g} "
+            "max_relative={:.17g}".format(
+                name, len(left), absolute, relative))
+    for name, values in (
+            ("qf1", legacy_arrays[0]), ("qf2", legacy_arrays[1])):
+        density = values[:n_faces]
+        pressure = values[4 * n_faces:5 * n_faces]
+        if min(density) <= 0.0 or min(pressure) <= 0.0:
+            raise SystemExit("TRACE_PHYSICAL_FAIL array={}".format(name))
+        print(
+            "PHYSICAL array={} finite=true min_density={:.17g} "
+            "min_pressure={:.17g}".format(
+                name, min(density), min(pressure)))
+    print(
+        "TRACE_PASS faces={} equations={} max_absolute={:.17g} "
+        "max_relative={:.17g}".format(
+            n_faces, n_equations, overall_absolute, overall_relative))
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("legacy_trace")
+    parser.add_argument("batch_trace")
+    args = parser.parse_args()
+    compare(args.legacy_trace, args.batch_trace)
+
+if __name__ == "__main__":
+    main()

ci/kunshan/e6-cpu-trace.slurm

@@ -0,0 +1,65 @@
+#!/bin/bash -l
+# Compare complete main-solver qf1/qf2/invflux traces for legacy and CPU batch.
+#SBATCH --job-name=oneflow_e6_cpu_trace
+#SBATCH --partition=kshcnormal
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=16
+#SBATCH --mem=54G
+#SBATCH --time=00:20:00
+#SBATCH --output=%x-%j.out
+#SBATCH --error=%x-%j.err
+
+set -euo pipefail
+
+: "${ONEFLOW_E6_TRACE_SOURCE_DIR:?Set ONEFLOW_E6_TRACE_SOURCE_DIR}"
+: "${ONEFLOW_E6_TRACE_BINARY:?Set ONEFLOW_E6_TRACE_BINARY}"
+
+module purge
+module load compiler/gcc/9.3.0
+module load mpi/openmpi/gcc-9.3.0/4.1.5
+module load python/3.8.10
+
+source_case="$ONEFLOW_E6_TRACE_SOURCE_DIR/test/m6wingroe_sa"
+trace_root="${ONEFLOW_E6_TRACE_RUN_DIR:-$PWD/e6-cpu-trace-${SLURM_JOB_ID:-manual}}"
+legacy_case="$trace_root/legacy"
+batch_case="$trace_root/batch"
+
+rm -rf "$trace_root"
+mkdir -p "$trace_root"
+cp -a "$source_case" "$legacy_case"
+cp -a "$source_case" "$batch_case"
+
+binary_dir=$(dirname "$ONEFLOW_E6_TRACE_BINARY")
+if [ ! -d "$binary_dir/system" ]; then
+    mkdir -p "$binary_dir"
+    cp -a "$ONEFLOW_E6_TRACE_SOURCE_DIR/system" "$binary_dir/system"
+fi
+
+for case_dir in "$legacy_case" "$batch_case"; do
+    rm -rf "$case_dir/results" "$case_dir/restart" "$case_dir/log"
+    mkdir -p "$case_dir/results" "$case_dir/restart" "$case_dir/log"
+    sed -i -E 's/^int[[:space:]]+maxSteps[[:space:]]*=.*/int maxSteps = 1;/'         "$case_dir/script/cfd.txt"
+    sed -i -E 's/^int[[:space:]]+ischeme[[:space:]]*=.*/int ischeme = 5; \/\/ Lax-Friedrichs trace/'         "$case_dir/script/cfd.txt"
+done
+
+run_case() {
+    local mode=$1
+    local case_dir=$2
+    (
+        cd "$trace_root"
+        export ONEFLOW_ACCEL_BACKEND=CPU
+        export ONEFLOW_UNS_TRACE_FILE="$trace_root/$mode.trace"
+        if [ "$mode" = batch ]; then
+            export ONEFLOW_ENABLE_UNS_CPU_BATCH=1
+        else
+            unset ONEFLOW_ENABLE_UNS_CPU_BATCH || true
+        fi
+        timeout --signal=TERM --kill-after=10s 180             mpirun -np 1 "$ONEFLOW_E6_TRACE_BINARY" 0 "$mode"
+    ) > "$trace_root/$mode.log" 2>&1
+}
+
+run_case legacy "$legacy_case"
+run_case batch "$batch_case"
+
+python3 "$ONEFLOW_E6_TRACE_SOURCE_DIR/ci/kunshan/e6-cpu-trace-verify.py"     "$trace_root/legacy.trace" "$trace_root/batch.trace"

ci/kunshan/euler-dcu-gtest.slurm

@@ -24,19 +24,30 @@ fi
 
 build_dir="${ONEFLOW_BUILD_DIR:-$project/.oneflow-build/${SLURM_JOB_NAME:-oneflow-euler-gtest}}"
 artifact_dir="${ONEFLOW_ARTIFACT_DIR:-$project/.oneflow-artifacts/${SLURM_JOB_ID:-local}}"
-# Resolve host tools before `module purge`; the DTK module may put an old
-# system CMake first on PATH.
-cmake_bin="${CMAKE_BIN:-$(command -v cmake)}"
-ctest_bin="${CTEST_BIN:-$(dirname "$cmake_bin")/ctest}"
 dtk_module="${DTK_MODULE:-compiler/dtk/26.04}"
+gcc_module="${GCC_MODULE:-compiler/gcc/9.3.0}"
+cmake_module="${CMAKE_MODULE:-compiler/cmake/3.25.0}"
 
 module purge
+module load "$cmake_module"
+module load "$gcc_module"
 module load "$dtk_module"
+# Resolve CMake after module setup. The login environment exposes CMake 2.8,
+# which does not support the -S/-B configure form used below.
+cmake_bin="${CMAKE_BIN:-$(command -v cmake)}"
+ctest_bin="${CTEST_BIN:-$(dirname "$cmake_bin")/ctest}"
 clang_bin="$(command -v clang)"
+gcc_root="${GCC_ROOT:-$(dirname "$(dirname "$(command -v gcc)")")}"
+gcc_toolchain_flag="--gcc-toolchain=$gcc_root"
+stdlib_flag="-stdlib=libstdc++"
+compiler_flags="$gcc_toolchain_flag $stdlib_flag"
 llvm_root="${LLVM_ROOT:-$(dirname "$(dirname "$clang_bin")")}"
 dtk_root="${DTK_ROOT:-$(dirname "$llvm_root")}"
+amd_comgr_dir="${AMD_COMGR_DIR:-$dtk_root/dcc/comgr/lib64/cmake/amd_comgr}"
+amd_device_libs_dir="${AMD_DEVICE_LIBS_DIR:-$dtk_root/dcc/lib64/cmake/AMDDeviceLibs}"
 export PATH="$dtk_root/bin:$llvm_root/bin:$PATH"
-export LD_LIBRARY_PATH="$dtk_root/hip/lib:$llvm_root/lib:$dtk_root/lib:$dtk_root/lib64:${LD_LIBRARY_PATH:-}"
+export CMAKE_PREFIX_PATH="$dtk_root/dcc/lib64/cmake/AMDDeviceLibs:$dtk_root/dcc/comgr/lib64/cmake:$dtk_root:$dtk_root/.hyhal:${CMAKE_PREFIX_PATH:-}"
+export LD_LIBRARY_PATH="$dtk_root/hip/lib:$llvm_root/lib:$dtk_root/lib:$dtk_root/lib64:$dtk_root/dcc/lib64:$dtk_root/dcc/comgr/lib64:${LD_LIBRARY_PATH:-}"
 export CC="$llvm_root/bin/clang"
 export CXX="$llvm_root/bin/clang++"
 export HIP_VISIBLE_DEVICES="${HIP_VISIBLE_DEVICES:-0}"
@@ -74,10 +85,16 @@ rc_test=1
   -DCMAKE_BUILD_TYPE=Release \
   -DCMAKE_C_COMPILER="$CC" \
   -DCMAKE_CXX_COMPILER="$CXX" \
+  -DCMAKE_CXX_FLAGS="$compiler_flags" \
+  -DCMAKE_HIP_FLAGS="$compiler_flags" \
   -DCMAKE_HIP_COMPILER="$CXX" \
-  -DCMAKE_PREFIX_PATH="$dtk_root;$dtk_root/.hyhal" \
+  -DCMAKE_PREFIX_PATH="$dtk_root/dcc/lib64/cmake/AMDDeviceLibs;$dtk_root/dcc/comgr/lib64/cmake;$dtk_root;$dtk_root/.hyhal" \
+  -DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON \
+  -DCMAKE_TRY_COMPILE_PLATFORM_VARIABLES="CMAKE_PREFIX_PATH;amd_comgr_DIR;AMDDeviceLibs_DIR" \
   -Dhsa-runtime64_DIR="$dtk_root/.hyhal/lib/cmake/hsa-runtime64" \
   -DHSA_HEADER="$dtk_root/.hyhal/include" \
+  -Damd_comgr_DIR="$amd_comgr_dir" \
+  -DAMDDeviceLibs_DIR="$amd_device_libs_dir" \
   -DCMAKE_HIP_ARCHITECTURES="${ONEFLOW_HIP_ARCHITECTURES:-gfx906}" \
   -DONEFLOW_1D_ENABLE_HIP=ON \
   -DONEFLOW_1D_ENABLE_GTEST=ON \
@@ -99,17 +116,18 @@ if [ "$rc_build" -eq 0 ]; then
   test_binary="$build_dir/oneflow_1d_euler_hip_backend_test"
   "$test_binary" > "$artifact_dir/gtest.log" 2>&1
   rc_test=$?
-  if [ "$rc_test" -eq 0 ] && ! grep -Eq "\[  PASSED  \] 6 tests" "$artifact_dir/gtest.log"; then
-    echo "HIP GoogleTest did not report 6 passed tests" >&2
+  if [ "$rc_test" -eq 0 ] && ! grep -Eq "\[  PASSED  \] [1-9][0-9]* tests?\.?$" "$artifact_dir/gtest.log"; then
+    echo "HIP GoogleTest did not report a passing test suite" >&2
     rc_test=1
   fi
   if [ "$rc_test" -eq 0 ]; then
     "$ctest_bin" --test-dir "$build_dir" \
-      -R '^HIP\.' \
+      -L 'hardware' \
+      -R 'HIP' \
       --output-on-failure > "$artifact_dir/ctest.log" 2>&1
     rc_ctest=$?
-    if [ "$rc_ctest" -ne 0 ] || ! grep -Eq "100% tests passed, 0 tests failed out of 6" "$artifact_dir/ctest.log"; then
-      echo "HIP CTest did not report 6 passed tests" >&2
+    if [ "$rc_ctest" -ne 0 ] || ! grep -Eq "100% tests passed, 0 tests failed out of [1-9][0-9]*" "$artifact_dir/ctest.log"; then
+      echo "HIP CTest did not report a fully passing test suite" >&2
       rc_test=1
     fi
   fi