diff --git a/.ci/requirements.txt b/.ci/requirements.txt index 45eb253548496..6c6abcada3eaf 100644 --- a/.ci/requirements.txt +++ b/.ci/requirements.txt @@ -1,3 +1,4 @@ junitparser==3.2.0 google-cloud-storage==3.3.0 PyGithub==2.8.1 +defusedxml==0.7.1 diff --git a/.github/workflows/ur-build-hw.yml b/.github/workflows/ur-build-hw.yml index b34761780b288..192874089e5db 100644 --- a/.github/workflows/ur-build-hw.yml +++ b/.github/workflows/ur-build-hw.yml @@ -113,10 +113,6 @@ jobs: run: | sudo -E bash devops/scripts/install_drivers.sh devops/dependencies.json --igfx - - name: Check NVML version compatibility - if: ${{ inputs.adapter_name == 'CUDA' }} - run: python3 devops/scripts/check_nvml_version.py - - name: Configure Unified Runtime project # ">" is used to avoid adding "\" at the end of each line; this command is quite long run: > @@ -145,21 +141,46 @@ jobs: # This is to check that install command does not fail run: cmake --install build - - name: Test adapter specific + - name: Build artifact name suffix + id: artifact_suffix + shell: bash env: - ZE_ENABLE_LOADER_DEBUG_TRACE: 1 - LIT_OPTS: "--timeout 120 -j 50" - # These tests cause timeouts on CI - LIT_FILTER_OUT: "(adapters/level_zero/memcheck.test|adapters/level_zero/v2/deferred_kernel_memcheck.test)" - run: cmake --build build -j $(($(nproc)/3)) -- check-unified-runtime-adapter + RUNNER_NAME: ${{ inputs.runner_name }} + STATIC_LOADER: ${{ matrix.adapter.static_Loader }} + OTHER_NAME: ${{ matrix.adapter.other_name }} + run: | + RUNNER_SUFFIX=$(echo "${RUNNER_NAME}" | sed 's/^UR_//') + SUFFIX="${RUNNER_SUFFIX}" + + if [ "${STATIC_LOADER}" = "ON" ]; then + SUFFIX="${SUFFIX}-static" + fi + + if [ -n "${OTHER_NAME}" ]; then + SUFFIX="${SUFFIX}-${OTHER_NAME}" + fi + + echo "suffix=${SUFFIX}" >> $GITHUB_OUTPUT + + - name: Check NVML version compatibility + if: ${{ inputs.adapter_name == 'CUDA' }} + run: python3 devops/scripts/check_nvml_version.py + + - name: Test adapter specific + uses: ./devops/actions/run-tests/ur + with: + test_type: 'adapter-specific' + build_dir: build + artifact_name: 'ur-${{matrix.adapter.name}}-${{steps.artifact_suffix.outputs.suffix}}-adapter-specific' # Don't run adapter specific tests when building multiple adapters if: ${{ matrix.adapter.other_name == '' }} - name: Test adapters - env: - ZE_ENABLE_LOADER_DEBUG_TRACE: 1 - LIT_OPTS: "--timeout 120 -j 50" - run: cmake --build build -j $(($(nproc)/3)) -- check-unified-runtime-conformance + uses: ./devops/actions/run-tests/ur + with: + test_type: 'conformance' + build_dir: build + artifact_name: 'ur-${{matrix.adapter.name}}-${{steps.artifact_suffix.outputs.suffix}}-conformance' - name: Debug CI platform information if: ${{ always() }} diff --git a/devops/actions/run-tests/ur/action.yml b/devops/actions/run-tests/ur/action.yml new file mode 100644 index 0000000000000..5a03e0d6c6482 --- /dev/null +++ b/devops/actions/run-tests/ur/action.yml @@ -0,0 +1,127 @@ +name: 'Run UR Tests' + +inputs: + test_type: + description: 'Type of tests to run (adapter-specific or conformance)' + required: true + build_dir: + required: false + default: 'build' + artifact_name: + description: 'Name for uploaded test artifacts (logs, XML)' + required: false + default: '' + +runs: + using: "composite" + steps: + - name: Run ${{ inputs.test_type }} tests + id: run_tests + continue-on-error: true + shell: bash + env: + BUILD_DIR: ${{ inputs.build_dir }} + TEST_TYPE: ${{ inputs.test_type }} + run: | + # Run tests via unified CLI (handles validation, config, cmake execution) + # Use temp file instead of command substitution to preserve outputs even on error + OUTPUTS_FILE=$(mktemp) + set +e # Don't exit on error + devops/scripts/ur-test run "$TEST_TYPE" "$BUILD_DIR" "$GITHUB_WORKSPACE" > "$OUTPUTS_FILE" 2>&1 + EXIT_CODE=$? + set -e # Re-enable exit on error + + # Parse Python output and set GitHub Actions outputs (not env for security) + while IFS='=' read -r key value; do + if [ -n "$key" ] && [ -n "$value" ]; then + echo "$key=$value" >> $GITHUB_OUTPUT + echo " Set output: $key=$value" + fi + done < <(grep -E "^(log-file|xml-file|skip-artifacts)=" "$OUTPUTS_FILE") + + rm -f "$OUTPUTS_FILE" + exit $EXIT_CODE + + - name: Filter test log + if: ${{ always() && steps.run_tests.outputs['skip-artifacts'] != '1' }} + shell: bash + env: + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} + run: | + if [ ! -f "$LOG_FILE" ]; then + echo "::warning::Test log not found" + exit 0 + fi + + echo "::group::Test Log" + devops/scripts/ur-test filter-log "$LOG_FILE" || { echo "::error::Filter failed"; head -1000 "$LOG_FILE"; } + echo "::endgroup::" + + - name: Report test failures + if: steps.run_tests.outcome != 'success' + shell: bash + env: + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} + run: | + devops/scripts/ur-test extract-errors "$LOG_FILE" + exit 1 + + - name: Test summary + if: ${{ always() && steps.run_tests.outputs['skip-artifacts'] != '1' }} + shell: bash + env: + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} + XML_FILE: ${{ steps.run_tests.outputs['xml-file'] || 'adapter_tests_results.xml' }} + ARTIFACT_NAME: ${{ inputs.artifact_name }} + WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + if [ ! -f "$LOG_FILE" ]; then + echo "::warning::Test log not found" + exit 0 + fi + + if [ ! -s "$LOG_FILE" ]; then + echo "::warning::Test log is empty" + exit 0 + fi + + devops/scripts/ur-test summary "$LOG_FILE" "${XML_FILE:-}" + + # Add link to artifacts if they will be uploaded + if [ -n "$ARTIFACT_NAME" ]; then + echo "" + echo "---" + echo "" + echo "Test artifacts (full logs, XML) will be available at:" + echo "$WORKFLOW_URL" + fi + + - name: Copy artifacts + if: ${{ always() && inputs.artifact_name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }} + shell: bash + env: + LOG_FILE: ${{ steps.run_tests.outputs['log-file'] || 'adapter_tests.log' }} + XML_FILE: ${{ steps.run_tests.outputs['xml-file'] || 'adapter_tests_results.xml' }} + run: | + set -f # Disable glob expansion + + # Clean artifacts directory to avoid mixing files from different test runs + rm -rf test_artifacts + mkdir -p test_artifacts + + if [ -f "$LOG_FILE" ]; then + cp "$LOG_FILE" test_artifacts/ + fi + + if [ -n "$XML_FILE" ] && [ -f "$XML_FILE" ]; then + cp "$XML_FILE" test_artifacts/ + fi + + - name: Upload test artifacts + if: ${{ always() && inputs.artifact_name != '' && (inputs.test_type != 'adapter-specific' || steps.run_tests.outputs['skip-artifacts'] != '1') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.artifact_name }} + path: test_artifacts/ + retention-days: 7 + if-no-files-found: warn diff --git a/devops/scripts/ur-test b/devops/scripts/ur-test new file mode 100755 index 0000000000000..fcb4aaf4af43a --- /dev/null +++ b/devops/scripts/ur-test @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +"""Unified Runtime test management CLI.""" + +import sys +from ur_test_tools.cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/devops/scripts/ur_test_tools/__init__.py b/devops/scripts/ur_test_tools/__init__.py new file mode 100644 index 0000000000000..c8d7cdb6a98ac --- /dev/null +++ b/devops/scripts/ur_test_tools/__init__.py @@ -0,0 +1,36 @@ +"""UR Test Tools - Modular test orchestration and summary generation.""" + +__version__ = "1.0.0" +__author__ = "Unified Runtime Team" + +from .models import ( + TestLists, + TestCounts, + TimingSummary, + TestConfig, + SummaryConfigFromLines, +) +from .test_runner import TestRunner +from .summary_generator import SummaryReporter +from .validation import ( + PathValidator, +) +from .outputs import ( + ConsoleOutput, + GitHubActionsOutput, +) + +__all__ = [ + "__version__", + "__author__", + "TestLists", + "TestCounts", + "TimingSummary", + "TestConfig", + "SummaryConfigFromLines", + "TestRunner", + "SummaryReporter", + "PathValidator", + "ConsoleOutput", + "GitHubActionsOutput", +] diff --git a/devops/scripts/ur_test_tools/cli.py b/devops/scripts/ur_test_tools/cli.py new file mode 100644 index 0000000000000..1275bf46f1a6a --- /dev/null +++ b/devops/scripts/ur_test_tools/cli.py @@ -0,0 +1,151 @@ +"""CLI entry points for UR test tools.""" + +import sys +import os +from pathlib import Path + +from .models.config import SummaryConfigFromLines, TestConfig, TestExecutionContext +from .validation.path_validator import PathValidator +from .parsers.log_parser import ( + LITLogParser, + read_log_file, +) +from .outputs.console import ConsoleOutput +from .summary_generator import SummaryReporter +from .test_runner import ( + TestRunner, + get_test_config, +) +from .outputs.github_actions import GitHubActionsOutput + + +def main() -> int: + """Unified CLI entry point.""" + if len(sys.argv) < 2: + print("Error: Missing command", file=sys.stderr) + return 1 + + command = sys.argv[1] + + if command == "run": + return main_ci_utils("run-tests") + + elif command in ("summary", "extract-errors", "filter-log"): + internal_cmd = "show-summary" if command == "summary" else command + return main_test_summary(internal_cmd) + + else: + print(f"Error: Unknown command '{command}'", file=sys.stderr) + return 1 + + +def main_test_summary(command: str) -> int: + """Entry point for ur_test_summary CLI.""" + try: + if len(sys.argv) < 3: + print( + f"Error: {sys.argv[0]} [xml_file]", + file=sys.stderr, + ) + return 1 + + log_file = sys.argv[2] + PathValidator.validate_log_path(log_file) + lines = read_log_file(log_file) + parser = LITLogParser(lines) + + if command == "extract-errors": + for line in parser.extract_error_details(): + print(line, end="") + + elif command == "filter-log": + for line in ConsoleOutput.filter_log_for_display(lines): + print(line, end="") + + elif command == "show-summary": + xml_file = PathValidator.validate_optional_path( + sys.argv[3] if len(sys.argv) > 3 else "", "XML", allow_absolute=True + ) + config = SummaryConfigFromLines( + log_lines=lines, xml_file=xml_file or None + ) + SummaryReporter(config).generate() + + else: + print(f"Error: Unknown command '{command}'", file=sys.stderr) + return 1 + + return 0 + + except (OSError, ValueError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + +def main_ci_utils(command: str) -> int: + """Entry point for ur_ci_utils CLI.""" + if command == "run-tests": + return _run_tests_command() + + else: + print(f"Error: Unknown command '{command}'", file=sys.stderr) + return 1 + + +def _run_tests_command() -> int: + """Execute run-tests command.""" + if len(sys.argv) < 5: + print( + f"Error: run-tests ", + file=sys.stderr, + ) + return 1 + + test_type = sys.argv[2] + build_dir = sys.argv[3] + workspace = sys.argv[4] + + # Validate inputs + gha = GitHubActionsOutput() + if not PathValidator.validate_build_dir(build_dir, workspace): + gha.print_error("Invalid build_dir") + return 1 + + try: + config = get_test_config(test_type) + except ValueError as e: + gha.print_error(str(e)) + return 1 + + # Convert to paths and create context + workspace_path = Path(workspace).resolve() + build_dir_path = workspace_path / build_dir + + xml_output_name = f"{test_type.replace('-', '_')}_results.xml" + xml_output_path = (build_dir_path / xml_output_name).absolute() + xml_output_path.parent.mkdir(parents=True, exist_ok=True) + + log_file_path = workspace_path / config.log_file + + env = os.environ.copy() + + context = TestExecutionContext( + test_type=test_type, + build_dir=build_dir_path, + workspace=workspace_path, + xml_output_path=xml_output_path, + log_file_path=log_file_path, + config=config, + env=env, + ) + + # Validate context + try: + context.validate() + except ValueError as e: + gha.print_error(str(e)) + return 1 + + # Run tests + runner = TestRunner(context) + return runner.run() diff --git a/devops/scripts/ur_test_tools/constants.py b/devops/scripts/ur_test_tools/constants.py new file mode 100644 index 0000000000000..11610e6d5e894 --- /dev/null +++ b/devops/scripts/ur_test_tools/constants.py @@ -0,0 +1,36 @@ +"""Constants for UR test tools.""" + +import re + +# File I/O +MAX_LINES_TO_SCAN = 1000 + +SEPARATOR_WIDTH = 70 + +# Job Calculation +MAX_JOBS = 16 + +# LIT Configuration +DEFAULT_LIT_TIMEOUT = 120 +DEFAULT_LIT_JOBS = 50 + +# Test Type Identifiers +TEST_TYPE_ADAPTER_SPECIFIC = "adapter-specific" + +# Constants +TEST_NOT_SELECTED_MSG = "Test not selected" +SLOWEST_TESTS_HEADER = "Slowest Tests:" +TEST_TIMES_HEADERS = ("Tests Times:", "Test Times:") + +# LIT Output Patterns +FAIL_TIMEOUT_PATTERN = re.compile(r"^(FAIL|TIMEOUT):") +TEST_LIST_HEADER_PATTERN = re.compile( + r"^(Passed|Unsupported|Failed|Expectedly Failed|" + r"Timed Out|Unexpectedly Passed|Unresolved) Tests \(" +) +STATS_PATTERN = re.compile( + r"^\s*(Total Discovered|Expected Passes|Expectedly Failed|" + r"Excluded|Unsupported|Skipped|Passed|Passed With Retry|" + r"Failed|Timed Out|Unexpectedly Passed|Unresolved)(\s+Tests)?\s*:" +) +TEST_CATEGORY_PATTERN = re.compile(r"^([A-Za-z]+(?: [A-Za-z]+)*) Tests \((\d+)\):") diff --git a/devops/scripts/ur_test_tools/models/__init__.py b/devops/scripts/ur_test_tools/models/__init__.py new file mode 100644 index 0000000000000..f242b3bc63400 --- /dev/null +++ b/devops/scripts/ur_test_tools/models/__init__.py @@ -0,0 +1,25 @@ +"""Models package - Data structures for UR test tools.""" + +from .test_data import ( + TestLists, + TestCounts, + TimingSummary, + SkippedTestsResult, + ExcludedTestsResult, +) +from .config import ( + TestConfig, + TestExecutionContext, + SummaryConfigFromLines, +) + +__all__ = [ + "TestLists", + "TestCounts", + "TimingSummary", + "SkippedTestsResult", + "ExcludedTestsResult", + "TestConfig", + "TestExecutionContext", + "SummaryConfigFromLines", +] diff --git a/devops/scripts/ur_test_tools/models/config.py b/devops/scripts/ur_test_tools/models/config.py new file mode 100644 index 0000000000000..dbdcfb97df9e4 --- /dev/null +++ b/devops/scripts/ur_test_tools/models/config.py @@ -0,0 +1,48 @@ +"""Configuration dataclasses for UR test tools.""" + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional, Dict, List + + +@dataclass +class TestConfig: + """Test execution configuration.""" + + target: str + log_file: str + lit_filter_out: Optional[str] = None + + def __post_init__(self): + """Validate configuration on creation.""" + if not all([self.target, self.log_file]): + raise ValueError("target and log_file are required") + + +@dataclass(frozen=True) +class TestExecutionContext: + """Context for test execution (immutable).""" + + test_type: str + build_dir: Path + workspace: Path + xml_output_path: Path + log_file_path: Path + config: TestConfig + env: Dict[str, str] = field(default_factory=dict) + + def validate(self) -> None: + try: + workspace_resolved = self.workspace.resolve() + for path in [self.log_file_path, self.xml_output_path, self.build_dir]: + path.resolve().relative_to(workspace_resolved) + except ValueError as e: + raise ValueError(f"Path outside workspace: {e}") from e + + +@dataclass +class SummaryConfigFromLines: + """Configuration for summary generation from parsed log lines.""" + + log_lines: List[str] + xml_file: Optional[str] = None diff --git a/devops/scripts/ur_test_tools/models/test_data.py b/devops/scripts/ur_test_tools/models/test_data.py new file mode 100644 index 0000000000000..782c475609dc8 --- /dev/null +++ b/devops/scripts/ur_test_tools/models/test_data.py @@ -0,0 +1,50 @@ +"""Type definitions for test data structures.""" + +from typing import TypedDict, List + + +class TestLists(TypedDict, total=False): + """Type definition for test list dictionary.""" + + Passed: List[str] + Failed: List[str] + Skipped: List[str] + Unsupported: List[str] + Excluded: List[str] + Unresolved: List[str] + + +class TestCounts(TypedDict, total=False): + """Type definition for test count dictionary.""" + + Passed: int + Failed: int + Skipped: int + Unsupported: int + Excluded: int + Unresolved: int + + +class TimingSummary(TypedDict): + """Type definition for test timing summary.""" + + slowest: List[str] + histogram: List[str] + + +class SkippedTestsResult(TypedDict): + """Result of skipped tests analysis.""" + + tests: List[str] + count: int + source: str + note: str + + +class ExcludedTestsResult(TypedDict): + """Result of excluded tests analysis.""" + + tests: List[str] + count: int + source: str + note: str diff --git a/devops/scripts/ur_test_tools/outputs/__init__.py b/devops/scripts/ur_test_tools/outputs/__init__.py new file mode 100644 index 0000000000000..9273918957af5 --- /dev/null +++ b/devops/scripts/ur_test_tools/outputs/__init__.py @@ -0,0 +1,9 @@ +"""Outputs package - Console and GitHub Actions output generation.""" + +from .console import ConsoleOutput +from .github_actions import GitHubActionsOutput + +__all__ = [ + "ConsoleOutput", + "GitHubActionsOutput", +] diff --git a/devops/scripts/ur_test_tools/outputs/console.py b/devops/scripts/ur_test_tools/outputs/console.py new file mode 100644 index 0000000000000..a0b5f98fb2d86 --- /dev/null +++ b/devops/scripts/ur_test_tools/outputs/console.py @@ -0,0 +1,119 @@ +"""Console output formatting for test results.""" + +from typing import List, Optional + +from ..constants import ( + SEPARATOR_WIDTH, + SLOWEST_TESTS_HEADER, + STATS_PATTERN, + TEST_CATEGORY_PATTERN, + TEST_TIMES_HEADERS, +) +from ..models.test_data import TimingSummary +from ..parsers.log_parser import LITLogParser + + +class ConsoleOutput: + """Format test results for console output.""" + + @staticmethod + def print_test_group( + title: str, tests: List[str], note: str = "", count: Optional[int] = None + ) -> None: + """Print GitHub Actions collapsible group with test list.""" + test_count = count if count is not None else len(tests) + print(f"::group::{title} ({test_count})") + if note: + print(note) + print() + for test in tests: + print(test) + print("::endgroup::") + + @staticmethod + def print_statistics(stats: List[str]) -> None: + """Print statistics section.""" + if stats: + print("=== Test Statistics ===") + for stat in stats: + print(stat.rstrip()) + print() + + @staticmethod + def print_timing_summary(lines: List[str]) -> None: + """Print timing information section.""" + parser = LITLogParser(lines) + time_info = parser.extract_time_summary() + + testing_time = None + for line in lines: + if line.strip().startswith("Testing Time:"): + testing_time = line.strip() + break + + if not (time_info["slowest"] or time_info["histogram"] or testing_time): + return + + print("::group::Test Timing Summary") + + if testing_time: + print(testing_time) + print() + + if time_info["slowest"]: + print(SLOWEST_TESTS_HEADER) + print("-" * SEPARATOR_WIDTH) + for line in time_info["slowest"]: + print(line) + print() + + if time_info["histogram"]: + print("Test Times Distribution:") + print("-" * SEPARATOR_WIDTH) + for line in time_info["histogram"]: + print(line) + + print("::endgroup::") + + @staticmethod + def filter_log_for_display(lines: List[str]) -> List[str]: + """Remove statistics, test lists, and timing from log.""" + result = [] + skip_until_empty = False + in_timing = False + + for line in lines: + stripped = line.strip() + + # Skip statistics lines + if STATS_PATTERN.match(line): + continue + + # Skip test category sections + if TEST_CATEGORY_PATTERN.match(line): + skip_until_empty = True + continue + + # Skip timing sections + if stripped == SLOWEST_TESTS_HEADER or stripped in TEST_TIMES_HEADERS: + in_timing = True + continue + + if in_timing and stripped.replace("*", "") == "": + in_timing = False + continue + + if in_timing: + continue + + if stripped.startswith("Testing Time:"): + continue + + if skip_until_empty: + if not stripped: + skip_until_empty = False + continue + + result.append(line) + + return result diff --git a/devops/scripts/ur_test_tools/outputs/github_actions.py b/devops/scripts/ur_test_tools/outputs/github_actions.py new file mode 100644 index 0000000000000..e892ad3f2f036 --- /dev/null +++ b/devops/scripts/ur_test_tools/outputs/github_actions.py @@ -0,0 +1,20 @@ +"""GitHub Actions-specific output formatting.""" + +import sys + + +class GitHubActionsOutput: + """Format output for GitHub Actions.""" + + @staticmethod + def print_error(message: str) -> None: + print(f"::error::{message}", file=sys.stderr) + + @staticmethod + def print_warning(message: str) -> None: + print(f"::warning::{message}", file=sys.stderr) + + @staticmethod + def set_output(name: str, value: str) -> None: + print(f"{name}={value}", flush=True) + sys.stdout.flush() diff --git a/devops/scripts/ur_test_tools/parsers/__init__.py b/devops/scripts/ur_test_tools/parsers/__init__.py new file mode 100644 index 0000000000000..55721e06f10c6 --- /dev/null +++ b/devops/scripts/ur_test_tools/parsers/__init__.py @@ -0,0 +1,19 @@ +"""Parsers package - Data extraction from logs and XML.""" + +from .log_parser import ( + LITLogParser, + read_log_file, +) +from .xml_parser import ( + JUnitXMLParser, + ParsedXMLTests, +) +from .stats_parser import get_count_from_stats + +__all__ = [ + "LITLogParser", + "read_log_file", + "JUnitXMLParser", + "ParsedXMLTests", + "get_count_from_stats", +] diff --git a/devops/scripts/ur_test_tools/parsers/log_parser.py b/devops/scripts/ur_test_tools/parsers/log_parser.py new file mode 100644 index 0000000000000..2291973b0ebea --- /dev/null +++ b/devops/scripts/ur_test_tools/parsers/log_parser.py @@ -0,0 +1,151 @@ +"""Parse LIT text output for test information.""" + +import sys +from pathlib import Path +from typing import Iterator, List, Tuple + +from ..constants import ( + FAIL_TIMEOUT_PATTERN, + TEST_LIST_HEADER_PATTERN, + STATS_PATTERN, + TEST_CATEGORY_PATTERN, + SLOWEST_TESTS_HEADER, + TEST_TIMES_HEADERS, +) +from ..models.test_data import TestLists, TestCounts, TimingSummary + + +def _read_with_utf8_fallback(path: str, read_func): + try: + with open(path, "r", encoding="utf-8", errors="strict") as f: + return read_func(f) + except UnicodeDecodeError: + print( + f"Warning: File contains non-UTF-8 characters, replacing with U+FFFD", + file=sys.stderr, + ) + with open(path, "r", encoding="utf-8", errors="replace") as f: + return read_func(f) + + +def read_log_file(log_path: str) -> List[str]: + path = Path(log_path) + file_size = path.stat().st_size + + if file_size > 10 * 1024 * 1024: # 10 MB + print( + f"Large log file: {file_size / (1024 * 1024):.1f} MB. " + f"This may indicate a test problem.", + file=sys.stderr, + ) + + try: + return _read_with_utf8_fallback(log_path, lambda f: f.readlines()) + except OSError as e: + raise OSError(f"Cannot read log file: {e}") from e + + +class LITLogParser: + """Parse LIT (llvm-lit) text output for test information.""" + + def __init__(self, lines: List[str]): + self.lines = lines + + def extract_error_details(self) -> List[str]: + result = [] + in_error = False + + for line in self.lines: + if FAIL_TIMEOUT_PATTERN.match(line): + in_error = True + + # Stop at test list headers or timing summaries + if in_error and ( + TEST_LIST_HEADER_PATTERN.match(line) + or line.strip() == SLOWEST_TESTS_HEADER + or line.strip() in TEST_TIMES_HEADERS + ): + break + + if in_error: + result.append(line) + + return result + + def extract_statistics(self) -> List[str]: + return [line for line in self.lines if STATS_PATTERN.match(line)] + + def extract_time_summary(self) -> TimingSummary: + result: TimingSummary = {"slowest": [], "histogram": []} + current_section = None + skip_next_hr = False + + for line in self.lines: + stripped = line.strip() + + if stripped == SLOWEST_TESTS_HEADER: + current_section = "slowest" + skip_next_hr = True + continue + elif stripped in TEST_TIMES_HEADERS: + current_section = "histogram" + skip_next_hr = True + continue + + if skip_next_hr and stripped.startswith("---"): + skip_next_hr = False + continue + + if current_section == "slowest": + if not stripped: + current_section = None + elif not stripped.startswith("---"): + result["slowest"].append(line.rstrip()) + elif current_section == "histogram": + if not stripped: + current_section = None + elif stripped.replace("*", "") == "": + current_section = None + elif stripped.startswith("[") or stripped.replace("-", "") == "": + result["histogram"].append(line.rstrip()) + else: + current_section = None + + return result + + def extract_test_lists(self) -> Tuple[TestLists, TestCounts]: + categories: TestLists = {} + declared_counts: TestCounts = {} + current_category = None + current_tests = [] + current_declared_count = 0 + + for line in self.lines: + match = TEST_CATEGORY_PATTERN.match(line) + if match: + if current_category: + categories[current_category] = current_tests + declared_counts[current_category] = current_declared_count + + current_category = match.group(1) + current_declared_count = int(match.group(2)) + current_tests = [] + continue + + if current_category: + if not line.strip(): + categories[current_category] = current_tests + declared_counts[current_category] = current_declared_count + current_category = None + current_tests = [] + current_declared_count = 0 + else: + test_name = line.strip() + if test_name: + current_tests.append(test_name) + + if current_category: + categories[current_category] = current_tests + declared_counts[current_category] = current_declared_count + + return categories, declared_counts diff --git a/devops/scripts/ur_test_tools/parsers/stats_parser.py b/devops/scripts/ur_test_tools/parsers/stats_parser.py new file mode 100644 index 0000000000000..5424e6c084405 --- /dev/null +++ b/devops/scripts/ur_test_tools/parsers/stats_parser.py @@ -0,0 +1,16 @@ +"""Parse statistics from LIT output.""" + +import re +from typing import List + + +def get_count_from_stats(stats: List[str], keywords: List[str]) -> int: + # Build regex pattern from keywords (compile once per call) + pattern = re.compile("|".join(re.escape(kw) for kw in keywords)) + + for stat in stats: + if pattern.search(stat): + match = re.search(r"(\d+)", stat) + if match: + return int(match.group(1)) + return 0 diff --git a/devops/scripts/ur_test_tools/parsers/xml_parser.py b/devops/scripts/ur_test_tools/parsers/xml_parser.py new file mode 100644 index 0000000000000..33295c911d223 --- /dev/null +++ b/devops/scripts/ur_test_tools/parsers/xml_parser.py @@ -0,0 +1,90 @@ +"""Parse JUnit XML test results.""" + +import sys +from pathlib import Path +from typing import List, NamedTuple, Optional + +import defusedxml.ElementTree as ET + +from ..constants import TEST_NOT_SELECTED_MSG + + +class ParsedXMLTests(NamedTuple): + """Skipped and excluded tests from XML parsing.""" + skipped: List[str] + excluded: List[str] + + +def _format_test_name(classname: str, name: str) -> str: + if classname and name: + return f"{classname}.{name}" + return name + + +class JUnitXMLParser: + """Parse JUnit XML from LIT --xunit-xml-output.""" + + def __init__(self, xml_path: Optional[str]): + self.xml_path = xml_path + self._root = None + + def parse(self) -> bool: + if not self.xml_path: + return False + + xml_file = Path(self.xml_path) + if not xml_file.exists(): + return False + + try: + tree = ET.parse(self.xml_path) + self._root = tree.getroot() + return True + except ET.ParseError as e: + print( + f"Warning: Failed to parse XML file {self.xml_path}: {e}", + file=sys.stderr, + ) + return False + except (OSError, ValueError) as e: + print( + f"Warning: Error reading XML file {self.xml_path}: {e}", + file=sys.stderr + ) + return False + + def extract_tests_from_xml(self) -> ParsedXMLTests: + if not self.parse(): + return ParsedXMLTests([], []) + + skipped = [] + excluded = [] + + for testcase in self._root.findall(".//testcase"): + skipped_elem = testcase.find("skipped") + if skipped_elem is None: + continue + + message = skipped_elem.get("message", "") + test_name = _format_test_name( + testcase.get("classname", ""), testcase.get("name", "") + ) + + if not test_name: + continue + + # Separate by message type + if TEST_NOT_SELECTED_MSG in message: + excluded.append(test_name) + else: + skipped.append(test_name) + + return ParsedXMLTests(skipped=skipped, excluded=excluded) + + def extract_skipped_tests(self) -> List[str]: + skipped, _ = self.extract_tests_from_xml() + return skipped + + def extract_excluded_tests(self) -> List[str]: + _, excluded = self.extract_tests_from_xml() + return excluded diff --git a/devops/scripts/ur_test_tools/summary_generator.py b/devops/scripts/ur_test_tools/summary_generator.py new file mode 100644 index 0000000000000..e4ccac6c8f8fa --- /dev/null +++ b/devops/scripts/ur_test_tools/summary_generator.py @@ -0,0 +1,224 @@ +"""Generate test summary reports.""" + +import sys +from typing import List + +from .models.config import SummaryConfigFromLines +from .models.test_data import ( + TestLists, + TestCounts, + SkippedTestsResult, + ExcludedTestsResult, +) +from .parsers.log_parser import LITLogParser +from .parsers.xml_parser import JUnitXMLParser +from .parsers.stats_parser import get_count_from_stats +from .outputs.console import ConsoleOutput +from .validation.data_validator import validate_test_counts + + +class SummaryReporter: + """Generate comprehensive test summary.""" + + def __init__(self, config: SummaryConfigFromLines): + self.config = config + + def generate(self) -> None: + parser = LITLogParser(self.config.log_lines) + stats = parser.extract_statistics() + test_lists, declared_counts = parser.extract_test_lists() + total_discovered = get_count_from_stats(stats, ["Total Discovered"]) + + xml_parser = JUnitXMLParser(self.config.xml_file) + parsed_xml = xml_parser.extract_tests_from_xml() + skipped_xml = parsed_xml.skipped + excluded_xml = parsed_xml.excluded + + ConsoleOutput.print_statistics(stats) + + skipped_result = self._analyze_skipped_tests(test_lists, stats, skipped_xml) + self._validate_skipped_counts(skipped_result, declared_counts, stats) + self._display_skipped_tests(skipped_result) + if skipped_result["count"] > 0: + self._cleanup_skipped_from_test_lists(test_lists) + + excluded_result = self._analyze_excluded_tests(test_lists, stats, excluded_xml) + self._validate_excluded_counts(excluded_result, declared_counts, stats) + self._display_excluded_tests(excluded_result) + if excluded_result["count"] > 0: + self._cleanup_excluded_from_test_lists(test_lists) + + self._display_remaining_categories(test_lists) + + validate_test_counts( + total_discovered, + test_lists, + skipped_result["count"], + excluded_result["count"], + ) + + ConsoleOutput.print_timing_summary(self.config.log_lines) + + def _analyze_skipped_tests( + self, test_lists: TestLists, stats: List[str], skipped_xml: List[str] + ) -> SkippedTestsResult: + """Analyze skipped tests (priority: XML > Log > Stats).""" + skipped_from_log = test_lists.get("Skipped", test_lists.get("Unsupported", [])) + stats_count = get_count_from_stats(stats, ["Skipped", "Unsupported"]) + + # Priority 1: XML data (most reliable - structured output) + if skipped_xml: + return SkippedTestsResult( + tests=skipped_xml, + count=len(skipped_xml), + source="xml", + note="", + ) + + # Priority 2: Log data + if skipped_from_log: + return SkippedTestsResult( + tests=skipped_from_log, + count=len(skipped_from_log), + source="log", + note="", + ) + + # Priority 3: Stats only (no individual test names) + if stats_count: + return SkippedTestsResult( + tests=[], + count=stats_count, + source="stats", + note="Warning: Test names not available", + ) + + # No data available + return SkippedTestsResult(tests=[], count=0, source="none", note="") + + def _validate_skipped_counts( + self, result: SkippedTestsResult, declared_counts: TestCounts, stats: List[str] + ) -> None: + """Validate skipped counts (warns on mismatch).""" + actual_count = result["count"] + if actual_count == 0: + return # Nothing to validate + + declared_count = declared_counts.get( + "Skipped", declared_counts.get("Unsupported", 0) + ) + stats_count = get_count_from_stats(stats, ["Skipped", "Unsupported"]) + + # Build list of mismatches + mismatches = [] + + if declared_count and declared_count != actual_count: + mismatches.append(f"log header: {declared_count}") + + if stats_count and stats_count != actual_count: + mismatches.append(f"statistics: {stats_count}") + + # Display warning only if mismatches found + if mismatches: + sources_str = ", ".join(mismatches) + print( + f"Warning: Skipped test count mismatch. " + f"Using {actual_count} from {result['source']}, " + f"but found {sources_str}", + file=sys.stderr, + ) + + def _display_skipped_tests(self, result: SkippedTestsResult) -> None: + if result["count"] > 0: + ConsoleOutput.print_test_group( + "Skipped Tests", + result["tests"], + note=result["note"], + count=result["count"] if not result["tests"] else None, + ) + + def _cleanup_skipped_from_test_lists(self, test_lists: TestLists) -> None: + test_lists.pop("Skipped", None) + test_lists.pop("Unsupported", None) + + def _analyze_excluded_tests( + self, test_lists: TestLists, stats: List[str], excluded_xml: List[str] + ) -> ExcludedTestsResult: + """Analyze excluded tests (priority: Log > XML > Stats).""" + excluded_from_log = test_lists.get("Excluded", []) + stats_count = get_count_from_stats(stats, ["Excluded"]) + + # Priority 1: Log data + if excluded_from_log: + return ExcludedTestsResult( + tests=excluded_from_log, + count=len(excluded_from_log), + source="log", + note="", + ) + + # Priority 2: XML data + if excluded_xml: + return ExcludedTestsResult( + tests=excluded_xml, count=len(excluded_xml), source="xml", note="" + ) + + # Priority 3: Stats only (no individual test names) + if stats_count: + return ExcludedTestsResult( + tests=[], + count=stats_count, + source="stats", + note="Warning: Test names not available", + ) + + # No data available + return ExcludedTestsResult(tests=[], count=0, source="none", note="") + + def _validate_excluded_counts( + self, result: ExcludedTestsResult, declared_counts: TestCounts, stats: List[str] + ) -> None: + """Validate excluded counts (warns on mismatch).""" + actual_count = result["count"] + if actual_count == 0: + return # Nothing to validate + + declared_count = declared_counts.get("Excluded", 0) + stats_count = get_count_from_stats(stats, ["Excluded"]) + + # Build list of mismatches + mismatches = [] + + if declared_count and declared_count != actual_count: + mismatches.append(f"log header: {declared_count}") + + if stats_count and stats_count != actual_count: + mismatches.append(f"statistics: {stats_count}") + + # Display warning only if mismatches found + if mismatches: + sources_str = ", ".join(mismatches) + print( + f"Warning: Excluded test count mismatch. " + f"Using {actual_count} from {result['source']}, " + f"but found {sources_str}", + file=sys.stderr, + ) + + def _display_excluded_tests(self, result: ExcludedTestsResult) -> None: + if result["count"] > 0: + ConsoleOutput.print_test_group( + "Excluded Tests", + result["tests"], + note=result["note"], + count=result["count"] if not result["tests"] else None, + ) + + def _cleanup_excluded_from_test_lists(self, test_lists: TestLists) -> None: + test_lists.pop("Excluded", None) + + def _display_remaining_categories(self, test_lists: TestLists) -> None: + for category, tests in test_lists.items(): + if tests: + ConsoleOutput.print_test_group(f"{category} Tests", tests) + diff --git a/devops/scripts/ur_test_tools/test_runner.py b/devops/scripts/ur_test_tools/test_runner.py new file mode 100644 index 0000000000000..60185a35e7191 --- /dev/null +++ b/devops/scripts/ur_test_tools/test_runner.py @@ -0,0 +1,164 @@ +"""Test execution.""" + +import os +import sys +import subprocess # nosec B404 - Used safely with list args, no shell=True +from pathlib import Path +from typing import List, Optional + +from .constants import ( + DEFAULT_LIT_TIMEOUT, + DEFAULT_LIT_JOBS, + TEST_TYPE_ADAPTER_SPECIFIC, + MAX_LINES_TO_SCAN, + MAX_JOBS, +) +from .models.config import TestConfig, TestExecutionContext +from .outputs.github_actions import GitHubActionsOutput +from .parsers.log_parser import _read_with_utf8_fallback + + +def get_test_config(test_type: str) -> TestConfig: + """Get test configuration for test type.""" + if test_type == "adapter-specific": + return TestConfig( + target="check-unified-runtime-adapter", + log_file="adapter_tests.log", + lit_filter_out=( + "(adapters/level_zero/memcheck.test|" + "adapters/level_zero/v2/deferred_kernel_memcheck.test)" + ), + ) + elif test_type == "conformance": + return TestConfig( + target="check-unified-runtime-conformance", + log_file="conformance_tests.log", + ) + else: + raise ValueError(f"Invalid test_type: {test_type}") + + +def calculate_jobs() -> int: + """Calculate parallel jobs (nproc/3 capped at MAX_JOBS).""" + try: + nproc = os.cpu_count() or 4 + return min(nproc // 3, MAX_JOBS) + except (OSError, AttributeError): + return 4 + + +def check_log_has_tests(log_file: str) -> bool: + """Check if log contains test results.""" + + def _scan_for_testing(f): + for _ in range(MAX_LINES_TO_SCAN): + line = f.readline() + if not line: + break + if "Testing:" in line: + return True + return False + + try: + return _read_with_utf8_fallback(log_file, _scan_for_testing) + except OSError: + return False + + +class TestRunner: + """Execute UR tests.""" + + def __init__(self, context: TestExecutionContext): + self.context = context + self.github_output = GitHubActionsOutput() + self.jobs = calculate_jobs() + + def run(self) -> int: + """Run tests and return exit code.""" + self._setup_environment() + + result = self._execute_tests() + if result is None: + return 1 + + if not self._validate_output(): + return 1 + + self._publish_outputs(result) + return result.returncode + + def _setup_environment(self) -> None: + lit_opts = ( + f"--show-unsupported --show-pass --show-xfail --no-progress-bar " + f"-v --timeout {DEFAULT_LIT_TIMEOUT} -j {DEFAULT_LIT_JOBS} " + f"--time-tests --show-flakypass --show-skipped " + f"--xunit-xml-output {self.context.xml_output_path}" + ) + self.context.env["LIT_OPTS"] = lit_opts + + if self.context.config.lit_filter_out: + self.context.env["LIT_FILTER_OUT"] = self.context.config.lit_filter_out + + self.context.env["ZE_ENABLE_LOADER_DEBUG_TRACE"] = "1" + + def _build_cmake_command(self) -> List[str]: + return [ + "cmake", + "--build", + str(self.context.build_dir), + "-j", + str(self.jobs), + "--", + self.context.config.target, + ] + + def _execute_tests(self) -> Optional[subprocess.CompletedProcess]: + cmd = self._build_cmake_command() + + print(f"Running: {' '.join(cmd)}", file=sys.stderr) + print( + f"Log: {self.context.log_file_path}, Jobs: {self.jobs}", file=sys.stderr + ) + print( + f"Expected XML: {self.context.xml_output_path}", file=sys.stderr + ) + + try: + with open(self.context.log_file_path, "w", encoding="utf-8") as log: + return subprocess.run( # nosec B603 B607 + cmd, + stdout=log, + stderr=subprocess.STDOUT, + env=self.context.env, + cwd=self.context.workspace, + ) + except (OSError, PermissionError) as e: + self.github_output.print_error(f"Test execution failed: {e}") + return None + + def _validate_output(self) -> bool: + log_path = self.context.log_file_path + + if not log_path.exists() or log_path.stat().st_size == 0: + self.github_output.print_error("No log generated") + return False + + return True + + def _publish_outputs(self, result: subprocess.CompletedProcess) -> None: + self.github_output.set_output("log-file", str(self.context.log_file_path)) + + if ( + self.context.test_type == TEST_TYPE_ADAPTER_SPECIFIC + and not check_log_has_tests(str(self.context.log_file_path)) + ): + print("No adapter-specific tests found", file=sys.stderr) + self.github_output.set_output("skip-artifacts", "1") + return + + if self.context.xml_output_path.exists(): + self.github_output.set_output("xml-file", str(self.context.xml_output_path)) + else: + self.github_output.print_warning( + f"Expected XML file not found at {self.context.xml_output_path}" + ) diff --git a/devops/scripts/ur_test_tools/validation/__init__.py b/devops/scripts/ur_test_tools/validation/__init__.py new file mode 100644 index 0000000000000..081e0c05ae8f7 --- /dev/null +++ b/devops/scripts/ur_test_tools/validation/__init__.py @@ -0,0 +1,9 @@ +"""Validation package - Security and data validation.""" + +from .path_validator import PathValidator +from .data_validator import validate_test_counts + +__all__ = [ + "PathValidator", + "validate_test_counts", +] diff --git a/devops/scripts/ur_test_tools/validation/data_validator.py b/devops/scripts/ur_test_tools/validation/data_validator.py new file mode 100644 index 0000000000000..b63cc715f685c --- /dev/null +++ b/devops/scripts/ur_test_tools/validation/data_validator.py @@ -0,0 +1,36 @@ +"""Test data validation.""" + +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..models.test_data import TestLists + + +def validate_test_counts( + total_discovered: int, + test_lists: "TestLists", + displayed_skipped: int, + displayed_excluded: int, +) -> None: + """Validate test counts match (prints warnings on mismatch).""" + if not total_discovered: + return + + sum_categories = sum(len(tests) for tests in test_lists.values()) + + # Add skipped tests if they were displayed but not in test_lists + if displayed_skipped > 0 and "Skipped" not in test_lists: + sum_categories += displayed_skipped + + # Add excluded tests if they were displayed but not in test_lists + if displayed_excluded > 0 and "Excluded" not in test_lists: + sum_categories += displayed_excluded + + if total_discovered != sum_categories: + print() + print( + f"::warning::Test count mismatch: Total Discovered = " + f"{total_discovered}, but sum of all categories = {sum_categories}" + ) + print() diff --git a/devops/scripts/ur_test_tools/validation/path_validator.py b/devops/scripts/ur_test_tools/validation/path_validator.py new file mode 100644 index 0000000000000..c2094e142ca62 --- /dev/null +++ b/devops/scripts/ur_test_tools/validation/path_validator.py @@ -0,0 +1,96 @@ +"""Path validation for security.""" + +from pathlib import Path +from typing import Optional + + +class PathValidator: + """Validate paths for security and correctness.""" + + @staticmethod + def validate_build_dir(build_dir: str, workspace: Optional[str] = None) -> bool: + """Validate build directory is safe and within workspace.""" + if not build_dir or ".." in build_dir or build_dir.startswith("/"): + return False + + # Block shell metacharacters, quotes, and control characters + # to prevent injection in f-strings, env vars, and logs + dangerous_chars = {";", "&", "#", "$", "|", "`", "\\", "'", '"', "\n", "\r"} + if any(c in build_dir for c in dangerous_chars): + return False + + if workspace: + try: + build_path = Path(build_dir).resolve(strict=False) + workspace_path = Path(workspace).resolve(strict=False) + build_path.relative_to(workspace_path) + return True + except (ValueError, OSError): + return False + return True + + @staticmethod + def validate_log_path(path: str) -> None: + """Validate log file path (detects path traversal).""" + try: + # Resolve path to detect encoded forms of path traversal (e.g., %2e%2e) + resolved = Path(path).resolve(strict=False) + + # Check for path traversal in original string (simple check) + if ".." in path: + raise ValueError( + f"Invalid log file path (path traversal not allowed): {path}" + ) + + # Verify file exists + if not resolved.exists(): + raise ValueError(f"Log file not found: {path}") + except (OSError, ValueError) as e: + if isinstance(e, ValueError): + raise + raise OSError(f"Invalid log file path: {path} ({e})") from e + + @staticmethod + def validate_optional_path( + path: str, path_type: str, allow_absolute: bool = False + ) -> str: + """Validate optional file path.""" + if not path: + return "" + + try: + # Resolve path to detect encoded forms of path traversal + Path(path).resolve(strict=False) + + # Check for path traversal in original string + if ".." in path: + raise ValueError( + f"Invalid {path_type} file path (path traversal): {path}" + ) + + # Check absolute path restriction + if not allow_absolute and path.startswith("/"): + raise ValueError( + f"Invalid {path_type} file path " + f"(absolute paths not allowed): {path}" + ) + except (OSError, ValueError) as e: + if isinstance(e, ValueError): + raise + raise OSError(f"Invalid {path_type} file path: {path} ({e})") from e + + return path + + @staticmethod + def ensure_within_workspace(path: Path, workspace: Path) -> Path: + """Ensure path is within workspace.""" + resolved = path.resolve() + workspace_resolved = workspace.resolve() + + try: + resolved.relative_to(workspace_resolved) + return resolved + except ValueError as e: + raise ValueError( + f"Path outside workspace: {path} not in {workspace}" + ) from e diff --git a/unified-runtime/test/CMakeLists.txt b/unified-runtime/test/CMakeLists.txt index 2d3e36efbc12f..30f700ca4355f 100644 --- a/unified-runtime/test/CMakeLists.txt +++ b/unified-runtime/test/CMakeLists.txt @@ -86,7 +86,8 @@ function(add_ur_lit_testsuite suite) if(UR_STANDALONE_BUILD) add_custom_target(${TARGET} - COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" -sv + COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" + --show-unsupported --show-pass --show-xfail --no-progress-bar --succinct --timeout 120 -j 50 --time-tests --show-flakypass --show-skipped USES_TERMINAL ) else() diff --git a/unified-runtime/test/adapters/CMakeLists.txt b/unified-runtime/test/adapters/CMakeLists.txt index 14dc2e7175e6a..97bde72f80b99 100644 --- a/unified-runtime/test/adapters/CMakeLists.txt +++ b/unified-runtime/test/adapters/CMakeLists.txt @@ -4,6 +4,9 @@ add_custom_target(check-unified-runtime-adapter) +# CI validation test suite (intentional failures/timeouts for testing CI logging) +add_subdirectory(ci-validation) + if(UR_BUILD_ADAPTER_CUDA OR UR_BUILD_ADAPTER_ALL) add_subdirectory(cuda) endif() diff --git a/unified-runtime/test/adapters/ci-validation/CMakeLists.txt b/unified-runtime/test/adapters/ci-validation/CMakeLists.txt new file mode 100644 index 0000000000000..1566c77b940cb --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright (C) 2025 Intel Corporation +# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# CI validation test suite - intentionally generates different test outcomes +# to validate CI logging and categorization + +add_ur_lit_testsuite(adapter-ci-validation) + +# NOTE: CI validation tests are now integrated into adapter-specific test suites +# (e.g., cuda/ci-validation/) to appear together in test results. +# This standalone suite is kept for compatibility but not added as dependency. diff --git a/unified-runtime/test/adapters/ci-validation/README.md b/unified-runtime/test/adapters/ci-validation/README.md new file mode 100644 index 0000000000000..043e0dda70779 --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/README.md @@ -0,0 +1,53 @@ +# CI Validation Test Suite + +This test suite is designed to validate CI logging and test categorization in GitHub Actions workflows. + +## Purpose + +These tests intentionally generate different test outcomes to verify that our CI workflow correctly: +- Extracts and displays test statistics +- Categorizes tests into appropriate groups (Passed, Failed, Skipped, etc.) +- Properly displays collapsed sections in GitHub Actions Step Summary + +## Test Scenarios + +| Test File | Expected Outcome | CI Category | +|-----------|------------------|-------------| +| `test_pass.test` | Pass | Passed Tests | +| `test_fail.test` | Fail | Failed Tests | +| `test_unsupported.test` | Skip | Unsupported Tests | +| `test_xfail.test` | Expected Fail | Expectedly Failed Tests | +| `test_unexpected_pass.test` | Unexpected Pass | Unexpectedly Passed Tests | +| `test_timeout.test` | Timeout | Timed Out Tests | + +## Running Locally + +```bash +# Run all validation tests +cd build +cmake --build . --target check-unified-runtime-adapter-ci-validation + +# Or use LIT directly with timeout +python3 llvm/utils/lit/lit.py \ + --show-pass --show-unsupported --show-xfail --succinct \ + --timeout 5 \ + unified-runtime/test/adapters/ci-validation +``` + +## Expected Statistics + +When run on a Linux system, you should see approximately: +- Total Discovered Tests: 6 +- Passed: 1 +- Failed: 1 +- Unsupported: 1 +- Expectedly Failed: 1 +- Unexpectedly Passed: 1 +- Timed Out: 1 + +## Notes + +- `test_timeout.test` requires LIT timeout to be set (e.g., `--timeout 5`) +- `test_unsupported.test` is marked UNSUPPORTED on linux/windows (i.e., all platforms) +- `test_xfail.test` uses XFAIL marker - shows as "Expectedly Failed Tests" in LIT output +- These tests should NOT be run in production CI on every commit (too slow due to timeout test) diff --git a/unified-runtime/test/adapters/ci-validation/lit.local.cfg.py b/unified-runtime/test/adapters/ci-validation/lit.local.cfg.py new file mode 100644 index 0000000000000..91ac6fd6d0ff3 --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/lit.local.cfg.py @@ -0,0 +1,10 @@ +""" +Copyright (C) 2025 Intel Corporation + +Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +See LICENSE.TXT +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +""" + +config.suffixes = [".test"] diff --git a/unified-runtime/test/adapters/ci-validation/test_fail.test b/unified-runtime/test/adapters/ci-validation/test_fail.test new file mode 100644 index 0000000000000..f5824a12b3bd5 --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/test_fail.test @@ -0,0 +1,9 @@ +// Copyright (C) 2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// RUN: not python3 %S/test_helper.py fail | FileCheck --check-prefix=CHECK-FAIL %s +// CHECK-FAIL: Test failed: expected 42, got 41 + +// This test demonstrates a failing test scenario. +// It should appear in "Failed Tests" category in CI logs. diff --git a/unified-runtime/test/adapters/ci-validation/test_helper.py b/unified-runtime/test/adapters/ci-validation/test_helper.py new file mode 100755 index 0000000000000..8d5f7740615da --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/test_helper.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +""" +Copyright (C) 2025 Intel Corporation +Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +Helper script for CI test validation - generates different test outcomes +""" + +import sys +import time + + +def main(): + if len(sys.argv) < 2: + print("Usage: test_helper.py ") + sys.exit(1) + + test_type = sys.argv[1] + + if test_type == "pass": + print("Test passed: result = 42") + sys.exit(0) + elif test_type == "fail": + print("Test failed: expected 42, got 41") + sys.exit(1) + elif test_type == "timeout": + print("Test starting infinite loop...") + while True: + time.sleep(1) + elif test_type == "crash": + print("Test about to crash...") + sys.exit(137) # SIGKILL + else: + print(f"Unknown test type: {test_type}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/unified-runtime/test/adapters/ci-validation/test_pass.test b/unified-runtime/test/adapters/ci-validation/test_pass.test new file mode 100644 index 0000000000000..889f6af45b0b4 --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/test_pass.test @@ -0,0 +1,9 @@ +// Copyright (C) 2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// RUN: python3 %S/test_helper.py pass | FileCheck --check-prefix=CHECK-PASS %s +// CHECK-PASS: Test passed: result = 42 + +// This test demonstrates a passing test scenario. +// It should appear in "Passed Tests" category in CI logs. diff --git a/unified-runtime/test/adapters/ci-validation/test_timeout.test b/unified-runtime/test/adapters/ci-validation/test_timeout.test new file mode 100644 index 0000000000000..3d0e8a85a39c7 --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/test_timeout.test @@ -0,0 +1,10 @@ +// Copyright (C) 2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// RUN: python3 %S/test_helper.py timeout | FileCheck --check-prefix=CHECK-TIMEOUT %s +// CHECK-TIMEOUT: Test starting infinite loop + +// This test intentionally times out (infinite loop). +// It should appear in "Timed Out Tests" category in CI logs. +// Note: LIT timeout must be configured (e.g., --timeout 5) to catch this. diff --git a/unified-runtime/test/adapters/ci-validation/test_unexpected_pass.test b/unified-runtime/test/adapters/ci-validation/test_unexpected_pass.test new file mode 100644 index 0000000000000..eede12ab0b014 --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/test_unexpected_pass.test @@ -0,0 +1,10 @@ +// Copyright (C) 2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// XFAIL: * +// RUN: python3 %S/test_helper.py pass | FileCheck --check-prefix=CHECK-PASS %s +// CHECK-PASS: Test passed: result = 42 + +// This test is expected to fail (XFAIL) but actually passes. +// It should appear in "Unexpectedly Passed Tests" category in CI logs. diff --git a/unified-runtime/test/adapters/ci-validation/test_unsupported.test b/unified-runtime/test/adapters/ci-validation/test_unsupported.test new file mode 100644 index 0000000000000..0a02802fef2dd --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/test_unsupported.test @@ -0,0 +1,10 @@ +// Copyright (C) 2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// UNSUPPORTED: linux, windows +// RUN: python3 %S/test_helper.py pass | FileCheck --check-prefix=CHECK-PASS %s +// CHECK-PASS: Test passed: result = 42 + +// This test is unsupported on all common platforms. +// It should appear in "Skipped" or "Unsupported Tests" category in CI logs. diff --git a/unified-runtime/test/adapters/ci-validation/test_xfail.test b/unified-runtime/test/adapters/ci-validation/test_xfail.test new file mode 100644 index 0000000000000..0f12645c8b84e --- /dev/null +++ b/unified-runtime/test/adapters/ci-validation/test_xfail.test @@ -0,0 +1,10 @@ +// Copyright (C) 2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// XFAIL: * +// RUN: python3 %S/test_helper.py fail | FileCheck --check-prefix=CHECK-FAIL %s +// CHECK-FAIL: Test failed: expected 42, got 41 + +// This test is expected to fail (XFAIL). +// It should appear in "Expected Failures" category in CI logs. diff --git a/unified-runtime/test/adapters/cuda/CMakeLists.txt b/unified-runtime/test/adapters/cuda/CMakeLists.txt index 0a6c9d63f67e3..a7b85c233814f 100644 --- a/unified-runtime/test/adapters/cuda/CMakeLists.txt +++ b/unified-runtime/test/adapters/cuda/CMakeLists.txt @@ -23,6 +23,7 @@ add_conformance_devices_test(adapter-cuda kernel_tests.cpp memory_tests.cpp event_tests.cpp + ci_validation_tests.cpp #FIXME: make this cleaner ${CMAKE_CURRENT_SOURCE_DIR}/../../../source/adapters/cuda/queue.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../../source/adapters/cuda/common.cpp diff --git a/unified-runtime/test/adapters/cuda/ci_validation_tests.cpp b/unified-runtime/test/adapters/cuda/ci_validation_tests.cpp new file mode 100644 index 0000000000000..74aa8b46d276c --- /dev/null +++ b/unified-runtime/test/adapters/cuda/ci_validation_tests.cpp @@ -0,0 +1,46 @@ +// Copyright (C) 2025 Intel Corporation +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// CI Validation Tests +// These tests intentionally generate different test outcomes +// to validate CI logging and categorization + +#include + +// Test that always passes +TEST(CIValidation, test_pass) { EXPECT_EQ(42, 42); } + +// Test that always fails with detailed error information +TEST(CIValidation, test_fail) { + int expected = 42; + int actual = 0; // Changed to 0 to make the mismatch more obvious + + EXPECT_EQ(expected, actual) + << "Memory allocation test failed!" << std::endl + << "Expected value: " << expected << std::endl + << "Actual value: " << actual << std::endl + << "This simulates a real test failure with detailed diagnostics"; +} + +// Test expected to fail (XFAIL equivalent in Google Test) +// Disabled tests are skipped, which is close to XFAIL behavior +TEST(CIValidation, DISABLED_test_xfail) { + EXPECT_EQ(42, 41) << "Test failed: expected 42, got 41"; +} + +// Test expected to fail but actually passes (XPASS) +// This will show as disabled/skipped, but if someone runs it manually it passes +TEST(CIValidation, DISABLED_test_unexpected_pass) { EXPECT_EQ(42, 42); } + +// Test unsupported on common platforms +TEST(CIValidation, DISABLED_test_unsupported) { EXPECT_EQ(42, 42); } + +// Test that times out (infinite loop) +TEST(CIValidation, test_timeout) { + while (true) { + // Infinite loop to trigger timeout + } +} diff --git a/unified-runtime/test/conformance/CMakeLists.txt b/unified-runtime/test/conformance/CMakeLists.txt index 57b39ddbe7c4e..444805b22045a 100644 --- a/unified-runtime/test/conformance/CMakeLists.txt +++ b/unified-runtime/test/conformance/CMakeLists.txt @@ -41,8 +41,10 @@ foreach(adapter ${UR_ADAPTERS_LIST}) if(NOT "${adapter}" STREQUAL "mock") if(UR_STANDALONE_BUILD) add_custom_target(check-unified-runtime-conformance-${adapter} - COMMAND "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" - -v -Dselector=${adapter}:* + COMMAND ${CMAKE_COMMAND} -E env "LIT_OPTS=$ENV{LIT_OPTS}" + "${URLIT_LIT_BINARY}" "${CMAKE_CURRENT_BINARY_DIR}" + --show-unsupported --show-pass --show-xfail --no-progress-bar --succinct --timeout 120 -j 50 --time-tests --show-flakypass --show-skipped + -Dselector=${adapter}:* DEPENDS deps_check-unified-runtime-conformance ) else() diff --git a/unified-runtime/third_party/requirements_testing.txt b/unified-runtime/third_party/requirements_testing.txt index c41dbabde3fe3..2ac825e67cfbd 100644 --- a/unified-runtime/third_party/requirements_testing.txt +++ b/unified-runtime/third_party/requirements_testing.txt @@ -4,3 +4,6 @@ lit==18.1.8 # For timeouts in lit psutil==7.0.0 + +# For secure XML parsing in test result processing +defusedxml==0.7.1