[HS3] [RF] Add HS3 conformance test suite integration

.github/workflows/root-ci-config/buildconfig/global.txt

@@ -68,6 +68,7 @@ roofit_multiprocess=OFF
 root7=ON
 rootbench=OFF
 roottest=ON
+test_roofit_hs3testsuite=ON
 runtime_cxxmodules=ON
 shadowpw=OFF
 shared=ON

cmake/modules/RootBuildOptions.cmake

@@ -189,6 +189,7 @@ option(gminimal "Enable only required options by default, but include X11/Cocoa"
 option(minimal "Enable only required options by default" OFF)
 option(rootbench "Build rootbench if rootbench exists in root or if it is a sibling directory (implies testing=ON)" OFF)
 option(roottest "Build roottest (implies testing=ON)" OFF)
+option(test_roofit_hs3testsuite "Setup and use the HS3 conformance test suite (requires network)" OFF)
 option(testing "Enable testing with CTest" OFF)
 option(asan "Build ROOT with address sanitizer instrumentation" OFF)
 
@@ -338,6 +339,10 @@ if(testing)
   set(testsupport ON CACHE BOOL "" FORCE)
 endif()
 
+#---running HS3 test suite requires both testing and pyroot
+if(test_roofit_hs3testsuite AND (NOT testing OR NOT pyroot))
+    message(FATAL_ERROR "-Dtest_roofit_hs3testsuite=ON requires both -Dtesting=ON and -Dpyroot=ON)")
+endif()
 
 if(unfold AND NOT xml)
   message(STATUS "Cannot enable unfold without enabling xml: unfold is disabled.")

roofit/hs3/test/CMakeLists.txt

@@ -1,2 +1,19 @@
 ROOT_ADD_GTEST(testRooFitHS3 testRooFitHS3.cxx LIBRARIES RooFitCore RooFit RooFitHS3)
 ROOT_ADD_GTEST(testHS3SimultaneousFit testHS3SimultaneousFit.cxx LIBRARIES RooFitCore RooFit RooFitHS3 RooStats)
+
+if(test_roofit_hs3testsuite)
+    include(FetchContent)
+    FetchContent_Declare(
+        hs3testsuite
+        GIT_REPOSITORY https://github.com/hep-statistics-serialization-standard/HS3TestSuite.git
+        GIT_TAG 9d04e321ae6fddd283a35507f14ecf852eb7df61
+    )
+    # suppress git interactive prompt so fetch fails fast if the repo is gone or unreachable
+    set(ENV{GIT_TERMINAL_PROMPT} "0")
+    FetchContent_MakeAvailable(hs3testsuite)
+    ROOT_ADD_PYUNITTEST(hs3-suite test_hs3_suite.py 
+        PYTHON_DEPS jsonschema 
+        ENVIRONMENT HS3TESTSUITE_ROOT=${hs3testsuite_SOURCE_DIR}
+        COPY_TO_BUILDDIR hs3testsuite_roofit_backend.py
+        )
+endif()

roofit/hs3/test/hs3testsuite_roofit_backend.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+import os
+import sys
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Any
+
+
+class RooFitBackend:
+    name = "roofit"
+
+    def __init__(self) -> None:
+        import ROOT  # type: ignore
+
+        self.ROOT = ROOT
+        ROOT.gROOT.SetBatch(True)
+        ROOT.gErrorIgnoreLevel = ROOT.kFatal
+        ROOT.RooMsgService.instance().setGlobalKillBelow(ROOT.RooFit.FATAL)
+
+    def load_workspace(self, path: Path):
+        ws = self.ROOT.RooWorkspace("hs3suite_ws")
+        tool = self.ROOT.RooJSONFactoryWSTool(ws)
+        with suppress_root_output():
+            tool.importJSON(str(path))
+        return ws
+
+    def structure(self, workspace) -> dict[str, list[str]]:
+        return {
+            "pdfs": sorted(obj.GetName() for obj in workspace.allPdfs()),
+            "functions": sorted(obj.GetName() for obj in workspace.allFunctions()),
+            "data": sorted(obj.GetName() for obj in workspace.allData()),
+        }
+
+    def run_structure_check(self, workspace, check: dict[str, Any]) -> None:
+        actual = self.structure(workspace)
+        target = check.get("target", {})
+        for key in ("pdfs", "functions", "data"):
+            required = set(target.get(key, []))
+            missing = required.difference(actual[key])
+            if missing:
+                raise AssertionError(f"missing {key}: {sorted(missing)}")
+
+    def run_twice_delta_nll_scan(self, workspace, check: dict[str, Any]) -> list[float]:
+        target = check["target"]
+        pdf = workspace.pdf(target["pdf"])
+        if pdf is None:
+            raise AssertionError(f"PDF {target['pdf']!r} not found")
+        data = workspace.data(target["data"])
+        if data is None:
+            raise AssertionError(f"data {target['data']!r} not found")
+
+        self._apply_parameter_point(workspace, check["reference_point"])
+        with suppress_root_output():
+            nll = pdf.createNLL(
+                data,
+                self.ROOT.RooFit.NumCPU(1),
+                self.ROOT.RooFit.EvalBackend("legacy"),
+            )
+            reference = float(nll.getVal())
+        values = []
+        scan_parameter = check["scan_parameter"]
+        for point in check["scan_points"]:
+            self._apply_parameter_point(workspace, check["reference_point"])
+            var = workspace.var(scan_parameter)
+            if var is None:
+                raise AssertionError(f"scan parameter {scan_parameter!r} not found")
+            var.setVal(float(point))
+            with suppress_root_output():
+                values.append(2.0 * (float(nll.getVal()) - reference))
+        return values
+
+    def _apply_parameter_point(self, workspace, values: dict[str, float]) -> None:
+        for name, value in values.items():
+            var = workspace.var(name)
+            if var is not None:
+                var.setVal(float(value))
+
+
+@contextmanager
+def suppress_root_output():
+    """Suppress noisy C++ diagnostics that bypass RooMsgService."""
+
+    sys.stdout.flush()
+    sys.stderr.flush()
+    devnull_fd = os.open(os.devnull, os.O_WRONLY)
+    stdout_fd = os.dup(1)
+    stderr_fd = os.dup(2)
+    try:
+        os.dup2(devnull_fd, 1)
+        os.dup2(devnull_fd, 2)
+        yield
+    finally:
+        os.dup2(stdout_fd, 1)
+        os.dup2(stderr_fd, 2)
+        os.close(stdout_fd)
+        os.close(stderr_fd)
+        os.close(devnull_fd)

roofit/hs3/test/test_hs3_suite.py

@@ -0,0 +1,52 @@
+import os
+import sys
+import unittest
+from pathlib import Path
+
+hs3_root_dir = os.environ.get("HS3TESTSUITE_ROOT")
+if hs3_root_dir:
+    sys.path.insert(0, hs3_root_dir)
+
+try:
+    from hs3suite.backends import _BACKENDS
+    from hs3suite.runner import run_suite
+
+    _BACKENDS["roofit"] = ("hs3testsuite_roofit_backend", "RooFitBackend")
+
+except ImportError as e:
+    TestHS3Suite = type(
+        "TestHS3Suite",
+        (unittest.TestCase,),
+        dict(test_dependencies=lambda self, e=e: self.fail(f"Missing dependency: {e}")),
+    )
+else:
+    hs3_root_path = Path(hs3_root_dir)
+    try:
+        results = run_suite(hs3_root_path, "roofit")
+    except Exception as e:
+
+        def test(self, e=e):
+            self.fail(f"HS3TestSuite failed to run: {e}")
+
+        test.__name__ = "test_hs3testsuite_execution"
+        TestHS3Suite = type("TestHS3Suite", (unittest.TestCase,), {test.__name__: test})
+    else:
+        namespace = dict(__module__=__name__)
+        for r in results:
+
+            def _make(result):
+                def test(self):
+                    if result.status == "failed":
+                        self.fail(result.message)
+                    return
+
+                test.__name__ = f"test_{result.test_id}__{result.check_id}"
+                test.__doc__ = f"{result.test_id}::{result.check_id}"
+                return test
+
+            func = _make(r)
+            namespace[func.__name__] = func
+        TestHS3Suite = type("TestHS3Suite", (unittest.TestCase,), namespace)
+
+if __name__ == "__main__":
+    unittest.main()