diff --git a/.github/workflows/generate_coverage.yaml b/.github/workflows/generate_coverage.yaml index 6a52fb8bde0a..b95262b5d896 100644 --- a/.github/workflows/generate_coverage.yaml +++ b/.github/workflows/generate_coverage.yaml @@ -11,7 +11,7 @@ jobs: name: Generate coverage and push to Coveralls.io runs-on: ubuntu-latest - timeout-minutes: 150 + timeout-minutes: 180 permissions: # Needed to cancel any previous runs that are not completed for a given workflow @@ -119,24 +119,50 @@ jobs: mamba list - name: Build dpnp with coverage - id: build_coverage + timeout-minutes: 60 + run: | + [ -f /opt/intel/oneapi/setvars.sh ] && source /opt/intel/oneapi/setvars.sh + python scripts/gen_coverage.py --stage build + + - name: Run dpnp tests with coverage + id: dpnp_tests uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 with: shell: bash - timeout_minutes: 120 - max_attempts: 5 + timeout_minutes: 90 + max_attempts: 2 retry_on: error command: | . $CONDA/etc/profile.d/conda.sh conda activate coverage [ -f /opt/intel/oneapi/setvars.sh ] && source /opt/intel/oneapi/setvars.sh - git clean -fxd - export SKIP_TENSOR_TESTS=1 - python scripts/gen_coverage.py + python scripts/gen_coverage.py --stage dpnp-tests + + - name: Run dpnp.tensor tests with coverage + id: tensor_tests + uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60 # v4.0.0 + with: + shell: bash + timeout_minutes: 30 + max_attempts: 3 + # also retry on timeout, the tests may hang + retry_on: any + command: | + . $CONDA/etc/profile.d/conda.sh + conda activate coverage + [ -f /opt/intel/oneapi/setvars.sh ] && source /opt/intel/oneapi/setvars.sh + python scripts/gen_coverage.py --stage tensor-tests - name: Total number of coverage attempts + if: ${{ !cancelled() }} + run: | + echo "dpnp tests attempts: ${{ steps.dpnp_tests.outputs.total_attempts }}" + echo "dpnp.tensor tests attempts: ${{ steps.tensor_tests.outputs.total_attempts }}" + + - name: Generate coverage reports run: | - echo "Total number of coverage attempts: ${{ steps.build_coverage.outputs.total_attempts }}" + [ -f /opt/intel/oneapi/setvars.sh ] && source /opt/intel/oneapi/setvars.sh + python scripts/gen_coverage.py --stage report - name: Upload coverage data to coveralls.io uses: coverallsapp/github-action@8d6379e14d29928660c4ba802d8e85393440b329 # v2.3.8 diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index 545fd888c1ba..7fc2cc89186f 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -44,6 +44,12 @@ run, ) +# in the order they are run, see --stage option +_STAGES = ("build", "dpnp-tests", "tensor-tests", "report") + +_DPNP_PROFRAW = "dpnp_pytest.profraw" +_TENSOR_PROFRAW = "dpnp_tensor_pytest.profraw" + def parse_args(): p = argparse.ArgumentParser(description="Build dpnp and generate coverage") @@ -112,6 +118,28 @@ def parse_args(): default="", type=str, ) + p.add_argument( + "--skip-tensor-tests", + dest="run_tensor_tests", + action="store_false", + help="Skip running dpnp.tensor tests (tensor-tests stage)", + ) + p.add_argument( + "--tensor-cpp-coverage", + dest="tensor_cpp_coverage", + action="store_true", + help="Build dpnp.tensor pybind11 extensions with coverage " + "instrumentation and include them into C++ coverage report " + "(pass to both build and report stages)", + ) + p.add_argument( + "--stage", + dest="stages", + action="append", + choices=_STAGES, + help="Stage to run, can be repeated (default: all stages). " + "Allows to run and retry each stage in a separate CI step", + ) # build relating options p.add_argument( @@ -147,21 +175,7 @@ def find_bin_llvm(compiler): return bin_llvm -def main(): - is_linux = "linux" in sys.platform - if not is_linux: - err(f"{sys.platform} not supported", "gen_coverage") - - args = parse_args() - setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - c_compiler, cxx_compiler = resolve_compilers( - args.oneapi, - args.c_compiler, - args.cxx_compiler, - args.compiler_root, - ) - +def build_dpnp(args, setup_dir, env, c_compiler, cxx_compiler): dpctl_cmake_dir = get_dpctl_cmake_dir() print(f"[gen_coverage] Found DPCTL CMake dir: {dpctl_cmake_dir}") @@ -175,20 +189,10 @@ def main(): verbose=args.verbose, ) cmake_args.append("-DDPNP_GENERATE_COVERAGE=ON") - - env = os.environ.copy() - - if args.bin_llvm: - bin_llvm = args.bin_llvm - else: - bin_llvm = find_bin_llvm(c_compiler) - print( - f"[gen_coverage] Path to folder with llvm-cov/llvm-profdata: {bin_llvm}" - ) - - if bin_llvm: - env["PATH"] = ":".join((env.get("PATH", ""), bin_llvm)) - env["LLVM_TOOLS_HOME"] = bin_llvm + if args.tensor_cpp_coverage: + cmake_args.append( + "-DDPNP_TENSOR_GENERATE_COVERAGE_FOR_PYBIND11_EXTENSIONS=ON" + ) log_cmake_args(cmake_args, "gen_coverage") @@ -202,9 +206,10 @@ def main(): ) install_editable(setup_dir, env) - if args.run_pytest: - env["LLVM_PROFILE_FILE"] = "dpnp_pytest.profraw" - pytest_cmd = [ + +def run_pytest(args, setup_dir, env, tensor_tests): + run( + [ "pytest", "-q", "-ra", @@ -213,66 +218,178 @@ def main(): "pyproject.toml", "--cov", "dpnp", - "--cov-report=term-missing", - "--cov-report=lcov:coverage-python.lcov", + # reported by report stage, on the data of all sessions + "--cov-report=", + # tensor session appends to the data of dpnp one + *(["--cov-append"] if tensor_tests else []), "--pyargs", - "dpnp", + "dpnp.tests.tensor" if tensor_tests else "dpnp", *args.pytest_opts.split(), - ] - run(pytest_cmd, env=env, cwd=setup_dir) - - def find_objects(): - objects = [] - dpnp_path = os.getcwd() - search_path = os.path.join(dpnp_path, "dpnp") - for root, _, files in os.walk(search_path): - for file in files: - if ( - file.endswith("_c.so") - or root.find("extensions") != -1 - and file.find("_impl.cpython") != -1 - ): - objects.extend(["-object", os.path.join(root, file)]) - return objects - - objects = find_objects() - instr_profile_fn = "dpnp_pytest.profdata" - - # generate instrumentation profile data - run( + ], + env={ + **env, + "SKIP_TENSOR_TESTS": "0" if tensor_tests else "1", + "LLVM_PROFILE_FILE": ( + _TENSOR_PROFRAW if tensor_tests else _DPNP_PROFRAW + ), + }, + cwd=setup_dir, + ) + + +def export_coverage(args, setup_dir, env, bin_llvm): + # python coverage of all test sessions + run( + ["coverage", "report", "-m", "--rcfile", "pyproject.toml"], + env=env, + cwd=setup_dir, + ) + run( + [ + "coverage", + "lcov", + "--rcfile", + "pyproject.toml", + "-o", + "coverage-python.lcov", + ], + env=env, + cwd=setup_dir, + ) + + expected_profraw_files = [_DPNP_PROFRAW] + if args.run_tensor_tests: + expected_profraw_files.append(_TENSOR_PROFRAW) + + profraw_files = [] + for fn in expected_profraw_files: + if os.path.exists(os.path.join(setup_dir, fn)): + profraw_files.append(fn) + else: + # e.g. the stage failed, report on the rest of data + print(f"[gen_coverage] Warning: {fn} not found, skipping it") + if not profraw_files: + err("no LLVM profile data found", "gen_coverage") + + def find_objects(): + objects = [] + search_path = os.path.join(setup_dir, "dpnp") + for root, _, files in os.walk(search_path): + for file in files: + is_backend_obj = file.endswith("_c.so") or ( + root.find("extensions") != -1 + and file.find("_impl.cpython") != -1 + ) + # tensor pybind11 extensions, e.g. _tensor_impl + is_tensor_obj = ( + args.tensor_cpp_coverage + and os.path.basename(root) == "tensor" + and file.startswith("_tensor_") + and file.find("_impl.cpython") != -1 + ) + if is_backend_obj or is_tensor_obj: + objects.extend(["-object", os.path.join(root, file)]) + return objects + + objects = find_objects() + instr_profile_fn = "dpnp_pytest.profdata" + + # generate instrumentation profile data + run( + [ + os.path.join(bin_llvm, "llvm-profdata"), + "merge", + "-sparse", + *profraw_files, + "-o", + instr_profile_fn, + ], + cwd=setup_dir, + ) + + # export lcov + with open(os.path.join(setup_dir, "coverage-cpp.lcov"), "w") as fh: + subprocess.check_call( [ - os.path.join(bin_llvm, "llvm-profdata"), - "merge", - "-sparse", - env["LLVM_PROFILE_FILE"], - "-o", - instr_profile_fn, + os.path.join(bin_llvm, "llvm-cov"), + "export", + "-format=lcov", + "-ignore-filename-regex=/tmp/icpx*", + r"-ignore-filename-regex=.*/backend/kernels/elementwise_functions/.*\.hpp$", + r"-ignore-filename-regex=.*/backend/kernels/indexing/.*\.hpp$", + r"-ignore-filename-regex=.*/backend/kernels/statistics/.*\.hpp$", + r"-ignore-filename-regex=.*/backend/kernels/window/.*\.hpp$", + "-instr-profile=" + instr_profile_fn, ] + + objects + + ["-sources", "dpnp"], + cwd=setup_dir, + env=env, + stdout=fh, ) - # export lcov - with open("coverage-cpp.lcov", "w") as fh: - subprocess.check_call( - [ - os.path.join(bin_llvm, "llvm-cov"), - "export", - "-format=lcov", - "-ignore-filename-regex=/tmp/icpx*", - r"-ignore-filename-regex=.*/backend/kernels/elementwise_functions/.*\.hpp$", - r"-ignore-filename-regex=.*/backend/kernels/indexing/.*\.hpp$", - r"-ignore-filename-regex=.*/backend/kernels/statistics/.*\.hpp$", - r"-ignore-filename-regex=.*/backend/kernels/window/.*\.hpp$", - "-instr-profile=" + instr_profile_fn, - ] - + objects - + ["-sources", "dpnp"], - cwd=setup_dir, - env=env, - stdout=fh, - ) - - print("[gen_coverage] Coverage export is completed") + print("[gen_coverage] Coverage export is completed") + + +def main(): + is_linux = "linux" in sys.platform + if not is_linux: + err(f"{sys.platform} not supported", "gen_coverage") + + args = parse_args() + setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + # keep the order of _STAGES, whatever the order of --stage options + stages = [ + s + for s in _STAGES + if s in (args.stages or _STAGES) + and (args.run_pytest or s == "build") + and (args.run_tensor_tests or s != "tensor-tests") + ] + + c_compiler, cxx_compiler = resolve_compilers( + args.oneapi, + args.c_compiler, + args.cxx_compiler, + args.compiler_root, + ) + + env = os.environ.copy() + + if args.bin_llvm: + bin_llvm = args.bin_llvm else: + bin_llvm = find_bin_llvm(c_compiler) + print( + f"[gen_coverage] Path to folder with llvm-cov/llvm-profdata: {bin_llvm}" + ) + + if bin_llvm: + env["PATH"] = ":".join((env.get("PATH", ""), bin_llvm)) + env["LLVM_TOOLS_HOME"] = bin_llvm + + if "build" in stages: + build_dpnp(args, setup_dir, env, c_compiler, cxx_compiler) + + if "dpnp-tests" in stages: + # start from scratch, as pytest-cov erases .coverage w/o --cov-append + for fn in (_DPNP_PROFRAW, _TENSOR_PROFRAW): + fn = os.path.join(setup_dir, fn) + if os.path.exists(fn): + os.remove(fn) + + # dpnp tests w/o tensor ones, those are run by tensor-tests stage + run_pytest(args, setup_dir, env, tensor_tests=False) + + if "tensor-tests" in stages: + # separate session to limit peak memory + run_pytest(args, setup_dir, env, tensor_tests=True) + + if "report" in stages: + export_coverage(args, setup_dir, env, bin_llvm) + + if not args.run_pytest: print( "[gen_coverage] Skipping pytest and coverage collection " "(--skip-pytest)"