From 2eac10f6a85df84a749582d61939e7d52bd67872 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Mon, 12 Oct 2026 13:07:28 +0200 Subject: [PATCH 1/3] Run dpnp.tensor tests in coverage workflow Run dpnp.tensor tests in a separate pytest session that appends to the coverage data of the main dpnp session, and merge the LLVM profiles of both sessions. Add --skip-tensor-tests option to keep the previous behavior, and --tensor-cpp-coverage option to build dpnp.tensor pybind11 extensions with coverage instrumentation and include them into C++ coverage. Raise coverage workflow timeouts to account for the extra session. --- .github/workflows/generate_coverage.yaml | 5 +- scripts/gen_coverage.py | 68 ++++++++++++++++++++---- 2 files changed, 60 insertions(+), 13 deletions(-) diff --git a/.github/workflows/generate_coverage.yaml b/.github/workflows/generate_coverage.yaml index 6a52fb8bde0a..662840c695ee 100644 --- a/.github/workflows/generate_coverage.yaml +++ b/.github/workflows/generate_coverage.yaml @@ -11,7 +11,7 @@ jobs: name: Generate coverage and push to Coveralls.io runs-on: ubuntu-latest - timeout-minutes: 150 + timeout-minutes: 180 permissions: # Needed to cancel any previous runs that are not completed for a given workflow @@ -123,7 +123,7 @@ jobs: uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: shell: bash - timeout_minutes: 120 + timeout_minutes: 150 max_attempts: 5 retry_on: error command: | @@ -131,7 +131,6 @@ jobs: conda activate coverage [ -f /opt/intel/oneapi/setvars.sh ] && source /opt/intel/oneapi/setvars.sh git clean -fxd - export SKIP_TENSOR_TESTS=1 python scripts/gen_coverage.py - name: Total number of coverage attempts diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index 545fd888c1ba..1f0807d78026 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -112,6 +112,19 @@ def parse_args(): default="", type=str, ) + p.add_argument( + "--skip-tensor-tests", + dest="run_tensor_tests", + action="store_false", + help="Skip running dpnp.tensor tests (run in a separate pytest session)", + ) + p.add_argument( + "--tensor-cpp-coverage", + dest="tensor_cpp_coverage", + action="store_true", + help="Build dpnp.tensor pybind11 extensions with coverage " + "instrumentation and include them into C++ coverage report", + ) # build relating options p.add_argument( @@ -175,6 +188,10 @@ def main(): verbose=args.verbose, ) cmake_args.append("-DDPNP_GENERATE_COVERAGE=ON") + if args.tensor_cpp_coverage: + cmake_args.append( + "-DDPNP_TENSOR_GENERATE_COVERAGE_FOR_PYBIND11_EXTENSIONS=ON" + ) env = os.environ.copy() @@ -203,7 +220,6 @@ def main(): install_editable(setup_dir, env) if args.run_pytest: - env["LLVM_PROFILE_FILE"] = "dpnp_pytest.profraw" pytest_cmd = [ "pytest", "-q", @@ -213,13 +229,38 @@ def main(): "pyproject.toml", "--cov", "dpnp", + ] + cov_report = [ "--cov-report=term-missing", "--cov-report=lcov:coverage-python.lcov", - "--pyargs", - "dpnp", - *args.pytest_opts.split(), ] - run(pytest_cmd, env=env, cwd=setup_dir) + + # dpnp tests w/o tensor ones, those are run in a separate session + env["SKIP_TENSOR_TESTS"] = "1" + env["LLVM_PROFILE_FILE"] = "dpnp_pytest.profraw" + profraw_files = [env["LLVM_PROFILE_FILE"]] + run( + pytest_cmd + # report once, on the combined data of the last session + + (["--cov-report="] if args.run_tensor_tests else cov_report) + + ["--pyargs", "dpnp", *args.pytest_opts.split()], + env=env, + cwd=setup_dir, + ) + + if args.run_tensor_tests: + # separate session to limit peak memory, appends to coverage data + env["SKIP_TENSOR_TESTS"] = "0" + env["LLVM_PROFILE_FILE"] = "dpnp_tensor_pytest.profraw" + profraw_files.append(env["LLVM_PROFILE_FILE"]) + run( + pytest_cmd + + ["--cov-append"] + + cov_report + + ["--pyargs", "dpnp.tests.tensor", *args.pytest_opts.split()], + env=env, + cwd=setup_dir, + ) def find_objects(): objects = [] @@ -227,11 +268,18 @@ def find_objects(): search_path = os.path.join(dpnp_path, "dpnp") for root, _, files in os.walk(search_path): for file in files: - if ( - file.endswith("_c.so") - or root.find("extensions") != -1 + is_backend_obj = file.endswith("_c.so") or ( + root.find("extensions") != -1 + and file.find("_impl.cpython") != -1 + ) + # tensor pybind11 extensions, e.g. _tensor_impl + is_tensor_obj = ( + args.tensor_cpp_coverage + and os.path.basename(root) == "tensor" + and file.startswith("_tensor_") and file.find("_impl.cpython") != -1 - ): + ) + if is_backend_obj or is_tensor_obj: objects.extend(["-object", os.path.join(root, file)]) return objects @@ -244,7 +292,7 @@ def find_objects(): os.path.join(bin_llvm, "llvm-profdata"), "merge", "-sparse", - env["LLVM_PROFILE_FILE"], + *profraw_files, "-o", instr_profile_fn, ] From 8c4fe3a33caafe24a62b4cd414731dc821f34c10 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 13 Oct 2026 15:20:17 +0200 Subject: [PATCH 2/3] Split coverage workflow into build, test and report steps Add --stage option to gen_coverage.py to run build, dpnp tests, dpnp.tensor tests and report generation separately. Coverage reports are now generated by the report stage from the combined data of all test sessions. Run each stage as a separate workflow step, so dpnp.tensor tests have their own timeout and are retried (also on timeout) without rebuilding dpnp and re-running dpnp tests. --- .github/workflows/generate_coverage.yaml | 37 ++- scripts/gen_coverage.py | 307 ++++++++++++++--------- 2 files changed, 219 insertions(+), 125 deletions(-) diff --git a/.github/workflows/generate_coverage.yaml b/.github/workflows/generate_coverage.yaml index 662840c695ee..a5a8b4114337 100644 --- a/.github/workflows/generate_coverage.yaml +++ b/.github/workflows/generate_coverage.yaml @@ -119,23 +119,48 @@ jobs: mamba list - name: Build dpnp with coverage - id: build_coverage + timeout-minutes: 60 + run: | + python scripts/gen_coverage.py --stage build + + - name: Run dpnp tests with coverage + id: dpnp_tests uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: shell: bash - timeout_minutes: 150 - max_attempts: 5 + timeout_minutes: 90 + max_attempts: 2 retry_on: error command: | . $CONDA/etc/profile.d/conda.sh conda activate coverage [ -f /opt/intel/oneapi/setvars.sh ] && source /opt/intel/oneapi/setvars.sh - git clean -fxd - python scripts/gen_coverage.py + python scripts/gen_coverage.py --stage dpnp-tests + + - name: Run dpnp.tensor tests with coverage + id: tensor_tests + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 + with: + shell: bash + timeout_minutes: 30 + max_attempts: 3 + # also retry on timeout, the tests may hang + retry_on: any + command: | + . $CONDA/etc/profile.d/conda.sh + conda activate coverage + [ -f /opt/intel/oneapi/setvars.sh ] && source /opt/intel/oneapi/setvars.sh + python scripts/gen_coverage.py --stage tensor-tests - name: Total number of coverage attempts + if: ${{ !cancelled() }} + run: | + echo "dpnp tests attempts: ${{ steps.dpnp_tests.outputs.total_attempts }}" + echo "dpnp.tensor tests attempts: ${{ steps.tensor_tests.outputs.total_attempts }}" + + - name: Generate coverage reports run: | - echo "Total number of coverage attempts: ${{ steps.build_coverage.outputs.total_attempts }}" + python scripts/gen_coverage.py --stage report - name: Upload coverage data to coveralls.io uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2.3.8 diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index 1f0807d78026..7fc2cc89186f 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -44,6 +44,12 @@ run, ) +# in the order they are run, see --stage option +_STAGES = ("build", "dpnp-tests", "tensor-tests", "report") + +_DPNP_PROFRAW = "dpnp_pytest.profraw" +_TENSOR_PROFRAW = "dpnp_tensor_pytest.profraw" + def parse_args(): p = argparse.ArgumentParser(description="Build dpnp and generate coverage") @@ -116,14 +122,23 @@ def parse_args(): "--skip-tensor-tests", dest="run_tensor_tests", action="store_false", - help="Skip running dpnp.tensor tests (run in a separate pytest session)", + help="Skip running dpnp.tensor tests (tensor-tests stage)", ) p.add_argument( "--tensor-cpp-coverage", dest="tensor_cpp_coverage", action="store_true", help="Build dpnp.tensor pybind11 extensions with coverage " - "instrumentation and include them into C++ coverage report", + "instrumentation and include them into C++ coverage report " + "(pass to both build and report stages)", + ) + p.add_argument( + "--stage", + dest="stages", + action="append", + choices=_STAGES, + help="Stage to run, can be repeated (default: all stages). " + "Allows to run and retry each stage in a separate CI step", ) # build relating options @@ -160,21 +175,7 @@ def find_bin_llvm(compiler): return bin_llvm -def main(): - is_linux = "linux" in sys.platform - if not is_linux: - err(f"{sys.platform} not supported", "gen_coverage") - - args = parse_args() - setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - c_compiler, cxx_compiler = resolve_compilers( - args.oneapi, - args.c_compiler, - args.cxx_compiler, - args.compiler_root, - ) - +def build_dpnp(args, setup_dir, env, c_compiler, cxx_compiler): dpctl_cmake_dir = get_dpctl_cmake_dir() print(f"[gen_coverage] Found DPCTL CMake dir: {dpctl_cmake_dir}") @@ -193,20 +194,6 @@ def main(): "-DDPNP_TENSOR_GENERATE_COVERAGE_FOR_PYBIND11_EXTENSIONS=ON" ) - env = os.environ.copy() - - if args.bin_llvm: - bin_llvm = args.bin_llvm - else: - bin_llvm = find_bin_llvm(c_compiler) - print( - f"[gen_coverage] Path to folder with llvm-cov/llvm-profdata: {bin_llvm}" - ) - - if bin_llvm: - env["PATH"] = ":".join((env.get("PATH", ""), bin_llvm)) - env["LLVM_TOOLS_HOME"] = bin_llvm - log_cmake_args(cmake_args, "gen_coverage") build_extension( @@ -219,8 +206,10 @@ def main(): ) install_editable(setup_dir, env) - if args.run_pytest: - pytest_cmd = [ + +def run_pytest(args, setup_dir, env, tensor_tests): + run( + [ "pytest", "-q", "-ra", @@ -229,98 +218,178 @@ def main(): "pyproject.toml", "--cov", "dpnp", - ] - cov_report = [ - "--cov-report=term-missing", - "--cov-report=lcov:coverage-python.lcov", - ] - - # dpnp tests w/o tensor ones, those are run in a separate session - env["SKIP_TENSOR_TESTS"] = "1" - env["LLVM_PROFILE_FILE"] = "dpnp_pytest.profraw" - profraw_files = [env["LLVM_PROFILE_FILE"]] - run( - pytest_cmd - # report once, on the combined data of the last session - + (["--cov-report="] if args.run_tensor_tests else cov_report) - + ["--pyargs", "dpnp", *args.pytest_opts.split()], - env=env, - cwd=setup_dir, - ) + # reported by report stage, on the data of all sessions + "--cov-report=", + # tensor session appends to the data of dpnp one + *(["--cov-append"] if tensor_tests else []), + "--pyargs", + "dpnp.tests.tensor" if tensor_tests else "dpnp", + *args.pytest_opts.split(), + ], + env={ + **env, + "SKIP_TENSOR_TESTS": "0" if tensor_tests else "1", + "LLVM_PROFILE_FILE": ( + _TENSOR_PROFRAW if tensor_tests else _DPNP_PROFRAW + ), + }, + cwd=setup_dir, + ) + - if args.run_tensor_tests: - # separate session to limit peak memory, appends to coverage data - env["SKIP_TENSOR_TESTS"] = "0" - env["LLVM_PROFILE_FILE"] = "dpnp_tensor_pytest.profraw" - profraw_files.append(env["LLVM_PROFILE_FILE"]) - run( - pytest_cmd - + ["--cov-append"] - + cov_report - + ["--pyargs", "dpnp.tests.tensor", *args.pytest_opts.split()], - env=env, - cwd=setup_dir, - ) - - def find_objects(): - objects = [] - dpnp_path = os.getcwd() - search_path = os.path.join(dpnp_path, "dpnp") - for root, _, files in os.walk(search_path): - for file in files: - is_backend_obj = file.endswith("_c.so") or ( - root.find("extensions") != -1 - and file.find("_impl.cpython") != -1 - ) - # tensor pybind11 extensions, e.g. _tensor_impl - is_tensor_obj = ( - args.tensor_cpp_coverage - and os.path.basename(root) == "tensor" - and file.startswith("_tensor_") - and file.find("_impl.cpython") != -1 - ) - if is_backend_obj or is_tensor_obj: - objects.extend(["-object", os.path.join(root, file)]) - return objects - - objects = find_objects() - instr_profile_fn = "dpnp_pytest.profdata" - - # generate instrumentation profile data - run( +def export_coverage(args, setup_dir, env, bin_llvm): + # python coverage of all test sessions + run( + ["coverage", "report", "-m", "--rcfile", "pyproject.toml"], + env=env, + cwd=setup_dir, + ) + run( + [ + "coverage", + "lcov", + "--rcfile", + "pyproject.toml", + "-o", + "coverage-python.lcov", + ], + env=env, + cwd=setup_dir, + ) + + expected_profraw_files = [_DPNP_PROFRAW] + if args.run_tensor_tests: + expected_profraw_files.append(_TENSOR_PROFRAW) + + profraw_files = [] + for fn in expected_profraw_files: + if os.path.exists(os.path.join(setup_dir, fn)): + profraw_files.append(fn) + else: + # e.g. the stage failed, report on the rest of data + print(f"[gen_coverage] Warning: {fn} not found, skipping it") + if not profraw_files: + err("no LLVM profile data found", "gen_coverage") + + def find_objects(): + objects = [] + search_path = os.path.join(setup_dir, "dpnp") + for root, _, files in os.walk(search_path): + for file in files: + is_backend_obj = file.endswith("_c.so") or ( + root.find("extensions") != -1 + and file.find("_impl.cpython") != -1 + ) + # tensor pybind11 extensions, e.g. _tensor_impl + is_tensor_obj = ( + args.tensor_cpp_coverage + and os.path.basename(root) == "tensor" + and file.startswith("_tensor_") + and file.find("_impl.cpython") != -1 + ) + if is_backend_obj or is_tensor_obj: + objects.extend(["-object", os.path.join(root, file)]) + return objects + + objects = find_objects() + instr_profile_fn = "dpnp_pytest.profdata" + + # generate instrumentation profile data + run( + [ + os.path.join(bin_llvm, "llvm-profdata"), + "merge", + "-sparse", + *profraw_files, + "-o", + instr_profile_fn, + ], + cwd=setup_dir, + ) + + # export lcov + with open(os.path.join(setup_dir, "coverage-cpp.lcov"), "w") as fh: + subprocess.check_call( [ - os.path.join(bin_llvm, "llvm-profdata"), - "merge", - "-sparse", - *profraw_files, - "-o", - instr_profile_fn, + os.path.join(bin_llvm, "llvm-cov"), + "export", + "-format=lcov", + "-ignore-filename-regex=/tmp/icpx*", + r"-ignore-filename-regex=.*/backend/kernels/elementwise_functions/.*\.hpp$", + r"-ignore-filename-regex=.*/backend/kernels/indexing/.*\.hpp$", + r"-ignore-filename-regex=.*/backend/kernels/statistics/.*\.hpp$", + r"-ignore-filename-regex=.*/backend/kernels/window/.*\.hpp$", + "-instr-profile=" + instr_profile_fn, ] + + objects + + ["-sources", "dpnp"], + cwd=setup_dir, + env=env, + stdout=fh, ) - # export lcov - with open("coverage-cpp.lcov", "w") as fh: - subprocess.check_call( - [ - os.path.join(bin_llvm, "llvm-cov"), - "export", - "-format=lcov", - "-ignore-filename-regex=/tmp/icpx*", - r"-ignore-filename-regex=.*/backend/kernels/elementwise_functions/.*\.hpp$", - r"-ignore-filename-regex=.*/backend/kernels/indexing/.*\.hpp$", - r"-ignore-filename-regex=.*/backend/kernels/statistics/.*\.hpp$", - r"-ignore-filename-regex=.*/backend/kernels/window/.*\.hpp$", - "-instr-profile=" + instr_profile_fn, - ] - + objects - + ["-sources", "dpnp"], - cwd=setup_dir, - env=env, - stdout=fh, - ) - - print("[gen_coverage] Coverage export is completed") + print("[gen_coverage] Coverage export is completed") + + +def main(): + is_linux = "linux" in sys.platform + if not is_linux: + err(f"{sys.platform} not supported", "gen_coverage") + + args = parse_args() + setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + # keep the order of _STAGES, whatever the order of --stage options + stages = [ + s + for s in _STAGES + if s in (args.stages or _STAGES) + and (args.run_pytest or s == "build") + and (args.run_tensor_tests or s != "tensor-tests") + ] + + c_compiler, cxx_compiler = resolve_compilers( + args.oneapi, + args.c_compiler, + args.cxx_compiler, + args.compiler_root, + ) + + env = os.environ.copy() + + if args.bin_llvm: + bin_llvm = args.bin_llvm else: + bin_llvm = find_bin_llvm(c_compiler) + print( + f"[gen_coverage] Path to folder with llvm-cov/llvm-profdata: {bin_llvm}" + ) + + if bin_llvm: + env["PATH"] = ":".join((env.get("PATH", ""), bin_llvm)) + env["LLVM_TOOLS_HOME"] = bin_llvm + + if "build" in stages: + build_dpnp(args, setup_dir, env, c_compiler, cxx_compiler) + + if "dpnp-tests" in stages: + # start from scratch, as pytest-cov erases .coverage w/o --cov-append + for fn in (_DPNP_PROFRAW, _TENSOR_PROFRAW): + fn = os.path.join(setup_dir, fn) + if os.path.exists(fn): + os.remove(fn) + + # dpnp tests w/o tensor ones, those are run by tensor-tests stage + run_pytest(args, setup_dir, env, tensor_tests=False) + + if "tensor-tests" in stages: + # separate session to limit peak memory + run_pytest(args, setup_dir, env, tensor_tests=True) + + if "report" in stages: + export_coverage(args, setup_dir, env, bin_llvm) + + if not args.run_pytest: print( "[gen_coverage] Skipping pytest and coverage collection " "(--skip-pytest)" From 95ce0eba0591ac6ee755059baa0b98054372c7cc Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Tue, 13 Oct 2026 16:52:14 +0200 Subject: [PATCH 3/3] Source oneAPI environment in coverage build and report steps The oneAPI compiler is installed with apt and is not in PATH unless setvars.sh is sourced, which the build and report stages require. --- .github/workflows/generate_coverage.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/generate_coverage.yaml b/.github/workflows/generate_coverage.yaml index a5a8b4114337..b95262b5d896 100644 --- a/.github/workflows/generate_coverage.yaml +++ b/.github/workflows/generate_coverage.yaml @@ -121,6 +121,7 @@ jobs: - name: Build dpnp with coverage timeout-minutes: 60 run: | + [ -f /opt/intel/oneapi/setvars.sh ] && source /opt/intel/oneapi/setvars.sh python scripts/gen_coverage.py --stage build - name: Run dpnp tests with coverage @@ -160,6 +161,7 @@ jobs: - name: Generate coverage reports run: | + [ -f /opt/intel/oneapi/setvars.sh ] && source /opt/intel/oneapi/setvars.sh python scripts/gen_coverage.py --stage report - name: Upload coverage data to coveralls.io