Rift o4d junior calmarg in loop: AFTER main 'distance' merge

.gitlab-ci.yml

@@ -87,6 +87,10 @@ import_check:
   stage: system tests
   script:
     - python .travis/test-all-mod.py
+    - python -m pytest -q MonteCarloMarginalizeCode/Code/test/test_lisa_response_import.py
+    - python -m pytest -q MonteCarloMarginalizeCode/Code/test/test_lisa_lalsimutils_compat.py
+    - python -m pytest -q MonteCarloMarginalizeCode/Code/test/test_lisa_operational_synthetic.py
+    - python -m pytest -q MonteCarloMarginalizeCode/Code/test/test_lisa_helper_contract.py
 
 sim_manager_check:
   stage: unit tests

.travis/test-all-bin.sh

@@ -17,6 +17,10 @@ set -e
 
 # loop over all bin/ scripts
 for EXE in MonteCarloMarginalizeCode/Code/bin/*; do
+   if [[ ! -f ${EXE} ]]; then
+       echo " Not file : " ${EXE}
+       continue
+   fi
    # skip scripts with explicit bilby dependence
    if [[ ${EXE} == *'calibration_reweighting.py' ]]; then
        continue
@@ -70,6 +74,16 @@ for EXE in MonteCarloMarginalizeCode/Code/bin/*; do
          echo "Hyperpipe  " ${EXE}
          continue
    fi   
+   # skip optional JAX driver; base CI does not install the jax extras
+   if [[ ${EXE} == *"integrate_likelihood_extrinsic_jax" ]]; then
+         echo "JAX optional  " ${EXE}
+         continue
+   fi
+   # skip optional HTCondor2 driver; base CI does not install htcondor2
+   if [[ ${EXE} == *"cepp_basic_htcondor" ]]; then
+         echo "HTCondor2 optional  " ${EXE}
+         continue
+   fi
    # skip tests that require condor environment
    if [[ ${EXE} == *"check_CIP_complete_work.py" ]]; then
          continue

.travis/test-all-mod.py

@@ -57,6 +57,20 @@
     r"\ANo module named 'cupy'\Z",
     r"\ANo module named asimov\Z",
     r"\ANo module named 'asimov'\Z",
+    r"\ANo module named jax\Z",
+    r"\ANo module named 'jax'\Z",
+    r"\ANo module named optax\Z",
+    r"\ANo module named 'optax'\Z",
+    r"\ANo module named equinox\Z",
+    r"\ANo module named 'equinox'\Z",
+    r"\ANo module named tinygp\Z",
+    r"\ANo module named 'tinygp'\Z",
+    r"\ANo module named numpyro\Z",
+    r"\ANo module named 'numpyro'\Z",
+    r"\ANo module named flowMC\Z",
+    r"\ANo module named 'flowMC'\Z",
+    r"\ANo module named htcondor2\Z",
+    r"\ANo module named 'htcondor2'\Z",
 ])))
 
 

CHANGES.rst

@@ -2,6 +2,11 @@
 0.0.18.0
 ------------
 development tree is rift_O4d.
+  - (rc0) O4d base refresh, from rift_O4c to rift_O4d: Python/numpy CI modernization (py3.10-py3.13,
+    numpy 2.x checks), Asimov/RIFT smoke tests, docs deployment, pluggable workflow backends and
+    simulation-manager prototypes, distance-grid/distance-slice likelihood export, container-family and pixi/SWIG
+    canaries, hyperpipeline ASCII workflow support, parsimonious-placement preview, EFPE and standalone NR-frame
+    utilities, plot_RIFT diagnostics, and GPU/CuPy portability fixes.
 
 ** (rc0)
   - parsimonious-placement (preview): new RIFT.misc.tracer_placement engine (SMC+MALA, birth-death,
@@ -37,6 +42,24 @@ development tree is rift_O4d.
   - ascii data format; waveforms (epfe);  container auto-selection framework (first draft); cepp_alternate now feature-parity;
     mcsamplerEnsemble full GPU path
   - multiple demos
+  - (rc1 pending) In-loop calibration marginalization: move calibration draws into ILE, add loop and fused
+    GPU kernels (including distance-marginalized and phase-marginalized paths), export cal/time diagnostics,
+    account for calibration Monte-Carlo error in reported uncertainty, add adaptive/pilot calibration proposal
+    scaffolding, and include a runnable calmarg demo with OSG/container transfer fixes.
+  - (rc1 pending) Extrinsic proposal handoff and sampler robustness: save and consolidate extrinsic GMM
+    breadcrumbs across iterations, seed later ILE jobs from those proposals, add GMM proposal/adaptation
+    controls and diagnostics, improve log-domain covariance/ESS handling, and thread the cal/extrinsic seed
+    barriers through CEPP/puffball workflows.
+  - (rc1 pending) Differentiable JAX likelihood/export work: optional jax_gp interpolators, differentiable
+    export artifacts, quad-GP and RF/GP validation tooling, jax_cip experiments, and a JAX ILE driver with
+    AD-compatible extrinsic likelihood, distance/phase marginalization, network coordinates, and gradient-aware
+    samplers. These remain optional extras and are skipped by base CI when JAX is not installed.
+  - (rc1 pending) LISA support: add a LISA compatibility layer, response import path, standalone helper
+    contract, LISA ILE scaffold, and synthetic operational/import tests.
+  - (rc1 pending) EOS/coordinate and workflow updates: add NMB/PCA/tabular EOS sequence dispatch and
+    single-EOS-index evidence support, coordinate-plugin hooks for posterior plotting and puffing, Morisaki-frame
+    and aligned-spin coordinate helpers for GP/JAX studies, container-universe/OSDF scitokens support, and
+    per-machine container image selection.
 
 0.0.17.9
 ------------

MonteCarloMarginalizeCode/Code/RIFT/LISA/__init__.py

@@ -0,0 +1,2 @@
+"""LISA-specific response and analysis helpers for RIFT."""
+

MonteCarloMarginalizeCode/Code/RIFT/LISA/lalsimutils_compat.py

@@ -0,0 +1,283 @@
+"""Compatibility helpers for the standalone LISA ILE fork.
+
+These routines are adapted from Aasim Z. Jan's LISA-RIFT
+``lisa_rift_paper`` branch.  Keep LISA-only shims here while the normal
+``RIFT.lalsimutils`` and ILE driver continue to evolve independently.
+"""
+
+import os
+import sys
+
+import h5py
+import lal
+import lalsimulation as lalsim
+import numpy as np
+
+import RIFT.lalsimutils as lalsimutils
+
+
+def _filter_modes(hlms, modes):
+    if modes is None:
+        return hlms
+
+    requested_modes = {tuple(map(int, mode)) for mode in modes}
+    return {mode: hlm for mode, hlm in hlms.items() if mode in requested_modes}
+
+
+def hlmoff_for_LISA(
+    P,
+    Lmax=4,
+    modes=None,
+    fd_standoff_factor=0.964,
+    fd_alignment_postevent_time=None,
+    path_to_NR_hdf5=None,
+    **kwargs
+):
+    """Generate frequency-domain modes with the sizing expected by LISA ILE."""
+    assert Lmax >= 2
+    assert (not np.isnan(P.m1)) and (not np.isnan(P.m2)), "Masses are NaN."
+
+    extra_waveform_args = {}
+    if "extra_waveform_args" in kwargs:
+        extra_waveform_args.update(kwargs["extra_waveform_args"])
+    extra_params = P.to_lal_dict_extended(extra_args_dict=extra_waveform_args)
+
+    fNyq = 0.5 / P.deltaT
+    TDlen = int(1.0 / (P.deltaT * P.deltaF))
+    if fd_alignment_postevent_time:
+        if fd_alignment_postevent_time >= TDlen * P.deltaT / 2:
+            print(
+                " Warning: fd alignment postevent time requested incompatible with short duration ",
+                file=sys.stderr,
+            )
+
+    if path_to_NR_hdf5 is not None:
+        hlms_struct = lalsimutils.hlmoft_from_NRhdf5(
+            path_to_NR_hdf5,
+            P,
+            Lmax,
+            only_mode=modes,
+            taper_percent=10,
+            beta=8,
+            verbose=True,
+        )
+        return {mode: lalsimutils.DataFourier(hlms_struct[mode]) for mode in hlms_struct}
+
+    fd_mode_approximants = {
+        lalsimutils.lalIMRPhenomHM,
+        lalsimutils.lalIMRPhenomXPHM,
+        lalsimutils.lalIMRPhenomXHM,
+    }
+    if P.approx in fd_mode_approximants:
+        hlms_struct = lalsim.SimInspiralChooseFDModes(
+            P.m1,
+            P.m2,
+            P.s1x,
+            P.s1y,
+            P.s1z,
+            P.s2x,
+            P.s2y,
+            P.s2z,
+            P.deltaF,
+            P.fmin * fd_standoff_factor,
+            fNyq,
+            P.fref,
+            P.phiref,
+            P.dist,
+            P.incl,
+            extra_params,
+            P.approx,
+        )
+        hlmsdict = lalsimutils.SphHarmFrequencySeries_to_dict(hlms_struct, Lmax)
+        hlmsdict = _filter_modes(hlmsdict, modes)
+        return {
+            mode: lal.ResizeCOMPLEX16FrequencySeries(hlm, 0, TDlen)
+            for mode, hlm in hlmsdict.items()
+        }
+
+    if P.approx in {lalsimutils.lalNRHybSur3dq8, lalsimutils.lalIMRPhenomD}:
+        hlms_struct = lalsimutils.hlmoff(P, Lmax=Lmax)
+        hlmsdict = lalsimutils.SphHarmFrequencySeries_to_dict(hlms_struct, Lmax)
+        hlmsdict = _filter_modes(hlmsdict, modes)
+        for mode, hlm in list(hlmsdict.items()):
+            if not (1 / hlm.deltaF == P.deltaT * TDlen):
+                print(
+                    "WARNING: RESIZING IN FD DOMAIN "
+                    f"(1/deltaF = {1 / hlm.deltaF}, deltaT*TDlen = {P.deltaT * TDlen}), "
+                    "THIS SHOULD NOT BE HAPPENING."
+                )
+            hlmsdict[mode] = lal.ResizeCOMPLEX16FrequencySeries(hlm, 0, TDlen)
+        return hlmsdict
+
+    raise ValueError(f"Unsupported LISA FD mode approximant: {P.approx}")
+
+
+def _cache_path_for_channel(fname, channel):
+    cache_data = np.loadtxt(fname, dtype=str)
+    if cache_data.ndim == 1:
+        cache_data = cache_data.reshape(1, len(cache_data))
+
+    channel_prefix = channel[0]
+    for row in cache_data:
+        if row[0] == channel_prefix:
+            raw_path = row[-1]
+            if "localhost" in raw_path:
+                raw_path = raw_path.split("localhost", 1)[1]
+            elif raw_path.startswith("file://"):
+                raw_path = raw_path[len("file://"):]
+            return os.path.expanduser(raw_path)
+
+    raise ValueError(f"Could not find channel {channel!r} in cache {fname!r}")
+
+
+def frame_h5_to_hoff(fname, channel, start=None, stop=None, verbose=True):
+    """Read LISA frequency-domain HDF5 data from a cache entry."""
+    if verbose:
+        print(" ++ Loading from cache ", fname, channel)
+
+    path_to_h5 = _cache_path_for_channel(fname, channel)
+    if verbose:
+        print(f"Reading h5 file {path_to_h5}")
+
+    with h5py.File(path_to_h5, "r") as data:
+        hoff = lal.CreateCOMPLEX16FrequencySeries(
+            "hoff",
+            data.attrs["epoch"],
+            data.attrs["f0"],
+            data.attrs["deltaF"],
+            lalsimutils.lsu_HertzUnit,
+            int(data.attrs["length"]),
+        )
+        hoff.data.data = data["data"][()]
+
+    return hoff
+
+
+def frame_h5_to_hoft(fname, channel, start=None, stop=None, verbose=True):
+    """Read LISA time-domain HDF5 data from a cache entry."""
+    if verbose:
+        print(" ++ Loading from cache ", fname, channel)
+
+    path_to_h5 = _cache_path_for_channel(fname, channel)
+    if verbose:
+        print(f"Reading h5 file {path_to_h5}")
+
+    with h5py.File(path_to_h5, "r") as data:
+        hoft = lal.CreateREAL8TimeSeries(
+            "hoft",
+            data.attrs["epoch"],
+            data.attrs["f0"],
+            data.attrs["deltaT"],
+            lal.DimensionlessUnit,
+            int(data.attrs["length"]),
+        )
+        hoft.data.data = data["data"][()]
+
+    return hoft
+
+
+def frame_data_to_non_herm_hoff(
+    fname,
+    channel,
+    start=None,
+    stop=None,
+    TDlen=0,
+    window_shape=0.0,
+    verbose=True,
+    deltaT=None,
+    h5_frame=False,
+):
+    """Read frame/HDF5 time-domain data and FFT to two-sided frequency data."""
+    if not h5_frame:
+        return lalsimutils.frame_data_to_non_herm_hoff(
+            fname,
+            channel,
+            start=start,
+            stop=stop,
+            TDlen=TDlen,
+            window_shape=window_shape,
+            verbose=verbose,
+            deltaT=deltaT,
+        )
+
+    ht = frame_h5_to_hoft(fname, channel, start, stop, verbose)
+
+    tmplen = ht.data.length
+    if TDlen == -1:
+        TDlen = tmplen
+    elif TDlen == 0:
+        TDlen = lalsimutils.nextPow2(tmplen)
+    else:
+        assert TDlen >= tmplen
+
+    ht = lal.ResizeREAL8TimeSeries(ht, 0, TDlen)
+    hoftC = lal.CreateCOMPLEX16TimeSeries(
+        "h(t)", ht.epoch, ht.f0, ht.deltaT, ht.sampleUnits, TDlen
+    )
+    hoftC.data.data = ht.data.data + 0j
+    fwdplan = lal.CreateForwardCOMPLEX16FFTPlan(TDlen, 0)
+    hf = lal.CreateCOMPLEX16FrequencySeries(
+        "Template h(f)",
+        ht.epoch,
+        ht.f0,
+        1.0 / ht.deltaT / TDlen,
+        lalsimutils.lsu_HertzUnit,
+        TDlen,
+    )
+    lal.COMPLEX16TimeFreqFFT(hf, hoftC, fwdplan)
+    if verbose:
+        print(
+            " ++ Loaded data h(f) of length n= ",
+            hf.data.length,
+            " (= ",
+            len(hf.data.data) * ht.deltaT,
+            "s) at sampling rate ",
+            1.0 / ht.deltaT,
+        )
+    return hf
+
+
+def print_params_lisa(self, show_system_frame=False):
+    """LISA-flavored parameter printer used by the standalone ILE fork."""
+    print("This ChooseWaveformParams has the following parameter values:")
+    print(f"m1 = {self.m1 / lalsimutils.lsu_MSUN / 1e3:0.3f} x 1e3 (Msun)")
+    print(f"m2 = {self.m2 / lalsimutils.lsu_MSUN / 1e3:0.3f} x 1e3 (Msun)")
+    print("s1x =", self.s1x)
+    print("s1y =", self.s1y)
+    print("s1z =", self.s1z)
+    print("s2x =", self.s2x)
+    print("s2y =", self.s2y)
+    print("s2z =", self.s2z)
+    if show_system_frame:
+        self.print_params(show_system_frame=show_system_frame)
+    print("lambda1 =", self.lambda1)
+    print("lambda2 =", self.lambda2)
+    print("inclination =", self.incl)
+    print("distance =", self.dist / 1.0e9 / lalsimutils.lsu_PC, "(Gpc)")
+    print("reference orbital phase =", self.phiref)
+    print("polarization angle =", self.psi)
+    print("eccentricity = ", self.eccentricity)
+    print("reference time = ", float(self.tref), "(s)")
+    print("detector is: LISA")
+    print("ecliptic latitude (beta):", self.theta, "(radians)")
+    print("ecliptic longitude (lambda):", self.phi, "(radians)")
+    print("starting frequency is =", self.fmin, "(Hz)")
+    print("reference frequency is =", self.fref, "(Hz)")
+    print("Max frequency is =", self.fmax, "(Hz)")
+    print("time step =", self.deltaT, "(s) <==>", 1.0 / self.deltaT, "(Hz) sample rate")
+    print("freq. bin size is =", self.deltaF, "(Hz)")
+
+
+def install_choose_waveform_print_params_lisa():
+    """Install the LISA printer on ChooseWaveformParams for the LISA fork."""
+    lalsimutils.ChooseWaveformParams.print_params_lisa = print_params_lisa
+
+
+__all__ = [
+    "frame_data_to_non_herm_hoff",
+    "frame_h5_to_hoff",
+    "frame_h5_to_hoft",
+    "hlmoff_for_LISA",
+    "install_choose_waveform_print_params_lisa",
+    "print_params_lisa",
+]