Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,14 @@

## [2.0.0] - Unreleased

### Resource-handle pass (2026-08-17)

- Added `saxsabs.io.detector_images` as the common copy-and-close detector
loader. Shared-reference, strict 2D main/resume, integrate1d mask/EDF, and
Workbench FabIO readers now copy owned pixels/headers and close handles on
both the success and failure paths. Workbench optional `.npy` mask/flat loads
refuse pickled arrays.

### JOSS reviewer polish (2026-08-16)

- Make `saxsabs estimate-k --ref` optional so the CLI can use the same built-in
Expand Down Expand Up @@ -107,8 +115,8 @@
alpha uncertainty, propagation model, and uncertainty type in portable
provenance. Frame reports include `BufferAlphaUncertainty`.
- Closed FabIO handles in the strict 1D readers and strict 2D resume verifier.
Shared-reference, strict-2D main-read, and Workbench readers still require a
common close-safe loader.
A later resource-handle pass (2026-08-17) added the common close-safe loader
for the remaining shared-reference, strict-2D main-read, and Workbench paths.

### Fixed
- Deferred the TkAgg backend selection until the desktop Workbench is actually
Expand Down
112 changes: 81 additions & 31 deletions SASAbs.py
Original file line number Diff line number Diff line change
Expand Up @@ -1383,6 +1383,54 @@ def _profile_uncertainty(profile):
make_sample_id = None
write_calibrated2d_package = None

try:
from saxsabs.io.detector_images import (
load_detector_header as _core_load_detector_header,
load_detector_image as _core_load_detector_image,
)
except Exception:
_core_load_detector_header = None
_core_load_detector_image = None


def _close_detector_handle(image):
close = getattr(image, "close", None)
if callable(close):
close()


def _workbench_load_detector_image(path, *, dtype=np.float64):
"""Copy pixels/header through the shared loader, keeping fabio.open patchable."""
if _core_load_detector_image is not None:
return _core_load_detector_image(path, dtype=dtype, open_image_fn=fabio.open)
image = fabio.open(path)
try:
raw = getattr(image, "data", None)
if raw is None:
raise ValueError(f"detector image has no pixel data: {path}")
if dtype is None:
data = np.array(raw, copy=True)
else:
data = np.array(raw, dtype=dtype, copy=True, order="C")
header = dict(getattr(image, "header", None) or {})
return SimpleNamespace(data=data, header=header)
finally:
_close_detector_handle(image)


def _workbench_load_detector_header(path):
if _core_load_detector_header is not None:
return _core_load_detector_header(path, open_image_fn=fabio.open)
image = fabio.open(path)
try:
return dict(getattr(image, "header", None) or {})
finally:
_close_detector_handle(image)


def _workbench_load_detector_pixels(path, *, dtype=np.float64):
return _workbench_load_detector_image(path, dtype=dtype).data


def _calibrated2d_package_paths(root_dir, sample_id):
root = Path(root_dir)
Expand Down Expand Up @@ -1468,7 +1516,7 @@ def _validate_existing_calibrated2d_package(
)

try:
image_data = np.asarray(fabio.open(str(paths["image"])).data)
image_data = np.asarray(_workbench_load_detector_pixels(paths["image"], dtype=None))
except Exception as exc:
raise ValueError(f"unreadable calibrated 2D EDF image: {paths['image']}: {exc}") from exc
if image_data.ndim != 2 or image_data.size == 0 or not np.any(np.isfinite(image_data)):
Expand All @@ -1479,7 +1527,7 @@ def _validate_existing_calibrated2d_package(
except Exception as exc:
raise ValueError(f"unreadable calibrated 2D NPY mask: {paths['mask_npy']}: {exc}") from exc
try:
mask_edf = np.asarray(fabio.open(str(paths["mask_edf"])).data)
mask_edf = np.asarray(_workbench_load_detector_pixels(paths["mask_edf"], dtype=None))
except Exception as exc:
raise ValueError(f"unreadable calibrated 2D EDF mask: {paths['mask_edf']}: {exc}") from exc
if mask_npy.ndim != 2 or mask_edf.ndim != 2:
Expand Down Expand Up @@ -2263,13 +2311,13 @@ def build_composite_bg_net(
dark = np.asarray(d_dark, dtype=np.float64)

for bg_path in bg_paths:
img = fabio.open(bg_path)
d_bg = np.asarray(img.data, dtype=np.float64)
loaded = _workbench_load_detector_image(bg_path)
d_bg = np.asarray(loaded.data, dtype=np.float64)
self._assert_same_shape(d_bg, dark, "bg", "dark")
if ref_shape is not None and tuple(d_bg.shape) != tuple(ref_shape):
raise ValueError(f"BG 尺寸不匹配: {d_bg.shape} vs {ref_shape}")

exp, mon, trans = self.parse_header(bg_path, header_dict=getattr(img, "header", {}))
exp, mon, trans = self.parse_header(bg_path, header_dict=loaded.header)
exp_use = exp if exp is not None else fallback_triplet[0]
mon_use = mon if mon is not None else fallback_triplet[1]
trans_use = trans if trans is not None else fallback_triplet[2]
Expand Down Expand Up @@ -2998,8 +3046,8 @@ def has_keys():
need_text_fallback = not has_keys()
else:
try:
img = fabio.open(filepath)
for k, v in getattr(img, "header", {}).items():
header = _workbench_load_detector_header(filepath)
for k, v in header.items():
add_meta(k, v)
need_text_fallback = not has_keys()
except Exception:
Expand Down Expand Up @@ -3142,10 +3190,10 @@ def extract_instrument_signature(self, filepath, header_dict=None, shape=None):
meta = self.normalize_header_dict(header_dict)
if not meta:
try:
img = fabio.open(filepath)
meta = self.normalize_header_dict(getattr(img, "header", {}))
loaded = _workbench_load_detector_image(filepath, dtype=None)
meta = self.normalize_header_dict(loaded.header)
if shape is None:
shape = tuple(img.data.shape)
shape = tuple(loaded.data.shape)
except Exception:
pass

Expand Down Expand Up @@ -3378,9 +3426,11 @@ def check_instrument_consistency(self, file_paths, poni_path=None, tol_pct=0.5):
sigs = []
for fp in file_paths:
try:
img = fabio.open(fp)
d = img.data
sig = self.extract_instrument_signature(fp, header_dict=getattr(img, "header", {}), shape=d.shape)
loaded = _workbench_load_detector_image(fp, dtype=None)
d = loaded.data
sig = self.extract_instrument_signature(
fp, header_dict=loaded.header, shape=d.shape
)
sigs.append(sig)
except Exception as e:
sigs.append({"path": str(fp), "shape": None, "error": str(e)})
Expand Down Expand Up @@ -3716,9 +3766,9 @@ def load_optional_array(self, path, name):
return None
p = Path(path)
if p.suffix.lower() == ".npy":
arr = np.load(path)
arr = np.load(path, allow_pickle=False)
else:
arr = fabio.open(path).data
arr = _workbench_load_detector_pixels(path, dtype=None)
if arr is None:
raise ValueError(f"{name} 文件无法读取: {path}")
return np.asarray(arr)
Expand Down Expand Up @@ -3753,9 +3803,9 @@ def build_reference_library(self, paths, *, return_rejections=False):
rejected = []
for p in list(dict.fromkeys(paths or [])):
try:
img = fabio.open(p)
data = np.asarray(img.data)
exp, mon, trans = self.parse_header(p, header_dict=getattr(img, "header", {}))
loaded = _workbench_load_detector_image(p, dtype=None)
data = np.asarray(loaded.data)
exp, mon, trans = self.parse_header(p, header_dict=loaded.header)
refs.append({
"path": str(p),
"shape": tuple(data.shape),
Expand Down Expand Up @@ -7356,8 +7406,8 @@ def run_calibration(self):
self.report(self.tr("rpt_solid_angle").format(state='ON' if apply_solid_angle else 'OFF'))

ai = pyFAI.load(files["poni"])
d_std = fabio.open(files["std"]).data.astype(np.float64)
d_dark = fabio.open(files["dark"]).data.astype(np.float64)
d_std = _workbench_load_detector_pixels(files["std"])
d_dark = _workbench_load_detector_pixels(files["dark"])
self._assert_same_shape(d_std, d_dark, "std", "dark")
dark_exposure_s = self.read_required_dark_exposure(files["dark"])
mask_path = self.global_vars["mask_path"].get().strip()
Expand Down Expand Up @@ -7559,9 +7609,9 @@ def run_calibration(self):
bg_transmissions = []
bg_exposures = []
for bg_path in bg_used_paths:
bg_image = fabio.open(bg_path)
bg_header = _workbench_load_detector_header(bg_path)
bg_exp, bg_mon, bg_trans = self.parse_header(
bg_path, header_dict=getattr(bg_image, "header", {})
bg_path, header_dict=bg_header
)
bg_exp_use = bg_exp if bg_exp is not None else p["bg_exp"]
bg_mon_use = bg_mon if bg_mon is not None else p["bg_i0"]
Expand Down Expand Up @@ -8165,11 +8215,11 @@ def log_line(msg):

def load_data(path):
if context["parallel"]:
return fabio.open(path).data.astype(np.float64)
return _workbench_load_detector_pixels(path)
with context["cache_lock"]:
if path in context["image_cache"]:
return context["image_cache"][path]
d = fabio.open(path).data.astype(np.float64)
d = _workbench_load_detector_pixels(path)
with context["cache_lock"]:
context["image_cache"][path] = d
return d
Expand Down Expand Up @@ -8270,9 +8320,9 @@ def load_data(path):
return {"row": row, "logs": logs, "mode_stats": mode_stats}

ai = context["ai_shared"] if not context["parallel"] else pyFAI.load(context["poni_path"])
sample = fabio.open(fpath)
d_s = sample.data.astype(np.float64)
sample_header = getattr(sample, "header", {})
loaded_sample = _workbench_load_detector_image(fpath)
d_s = np.asarray(loaded_sample.data, dtype=np.float64)
sample_header = loaded_sample.header

exp, mon, trans = self.parse_header(fpath, header_dict=sample_header)
monitor_mode = context["monitor_mode"]
Expand Down Expand Up @@ -8893,7 +8943,7 @@ def prepare_batch_references(self, *, ref_mode, bg_path, dark_path, monitor_mode
dark_text = str(dark_path or "").strip()
if not bg_text or not dark_text:
raise ValueError("固定参考模式缺少背景或暗场文件。")
fixed_dark_data = fabio.open(dark_text).data.astype(np.float64)
fixed_dark_data = _workbench_load_detector_pixels(dark_text)
fixed_dark_exposure_s = self.read_required_dark_exposure(dark_text)
bg_paths = self.split_path_list(bg_text)
if not bg_paths:
Expand Down Expand Up @@ -9949,13 +9999,13 @@ def dry_run(self):

if self.t2_ref_mode.get() == "auto":
try:
img = fabio.open(fp)
loaded = _workbench_load_detector_image(fp, dtype=None)
smeta = {
"exp": e if (e is not None and np.isfinite(e)) else None,
"mon": m if m is not None else None,
"trans": t if t is not None else None,
"mtime": Path(fp).stat().st_mtime if Path(fp).exists() else None,
"shape": tuple(img.data.shape),
"shape": tuple(loaded.data.shape),
}
bg_ref, _, bg_rejected = self.select_best_reference(
smeta,
Expand Down Expand Up @@ -10114,7 +10164,7 @@ def _get_t2_preview_context(self):
raise ValueError("请先在 Tab1/Tab2 设置 poni 文件。")

ai = pyFAI.load(poni_path)
data = fabio.open(sample_path).data.astype(np.float64)
data = _workbench_load_detector_pixels(sample_path)
if data.ndim != 2:
raise ValueError(f"样品图像维度错误: {data.shape}")

Expand Down
15 changes: 8 additions & 7 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -113,10 +113,11 @@

## Resource and publication boundaries

- FabIO handles are closed in the strict 1D readers and the strict 2D resume
verification reader. The shared reference loader, the strict 2D main image
loader, and multiple Workbench paths still require one common copy-and-close
loader.
- FabIO handles go through one common copy-and-close helper
(`saxsabs.io.detector_images`). The shared reference loader, the strict 2D
main and resume readers, the integrate1d mask/EDF loaders, and Workbench
detector-image paths copy pixels/header out and close the handle, including
after a read failure. Callers can inject `open_image_fn` at the I/O edge.
- The reusable calibrated-2D package has transactional multi-file publication
and signature-aware resume. Workbench existence-only resume is now disabled
and hard-blocked, but no equivalent content-signature resume exists. The strict
Expand Down Expand Up @@ -153,6 +154,6 @@
cancellable. Users should prefer headless workflows for unattended or large
campaigns. `CAUTION` remains visible but is not separately persisted as an
acknowledgement.
- **Input resources**: not every Workbench FabIO path uses one common
copy-and-close helper. The strict readers exercised by the headless workflows
close their handles, and reviewers should use the documented portable fixtures.
- **Input resources**: Workbench and headless detector readers share
`saxsabs.io.detector_images`. Reviewers should still use the documented
portable fixtures rather than private beamline files.
4 changes: 3 additions & 1 deletion examples/manual-verification.md
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,9 @@ applicable:
content-signature resume (existence-only resume must remain disabled);
- Workbench preflight binds critical file content hashes and persists explicit
CAUTION acceptance;
- all FabIO readers pass normal and exceptional close tests on Windows;
- all FabIO readers pass OS-level handle audits on every Windows workstation
(unit tests now cover the shared copy-and-close helper; a full desktop
handle audit remains a local check);
- a cancellable background JobController keeps long jobs off the Tk UI thread.

## Notes
Expand Down
23 changes: 5 additions & 18 deletions src/saxsabs/core/reference_matching.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@

import numpy as np

from saxsabs.io.detector_images import load_detector_image


NO_USABLE_REFERENCE_SCORE = 1e9
DEFAULT_MAX_SCORE_THRESHOLD = 0.5
Expand Down Expand Up @@ -116,24 +118,13 @@ def build_reference_library(
def parse_header_fn(p, header_dict=None): # type: ignore
return None, None, None

if open_image_fn is None:
def _lazy_fabio_open(p):
import fabio

return fabio.open(p)

open_image_fn = _lazy_fabio_open

refs: list[dict[str, Any]] = []
rejected: list[dict[str, Any]] = []
for p in unique_paths:
img = None
try:
img = open_image_fn(p)
raw_data = getattr(img, "data", None)
shape = tuple(np.asarray(raw_data).shape) if raw_data is not None else None
hdr = getattr(img, "header", {}) or {}
exp, mon, trans = parse_header_fn(p, header_dict=hdr)
loaded = load_detector_image(p, dtype=None, open_image_fn=open_image_fn)
shape = tuple(np.asarray(loaded.data).shape)
exp, mon, trans = parse_header_fn(p, header_dict=loaded.header)
mtime = Path(p).stat().st_mtime if Path(p).exists() else None
refs.append(
{
Expand All @@ -153,10 +144,6 @@ def _lazy_fabio_open(p):
}
)
continue
finally:
close = getattr(img, "close", None) if img is not None else None
if callable(close):
close()
return (refs, rejected) if return_rejections else refs


Expand Down
12 changes: 12 additions & 0 deletions src/saxsabs/io/__init__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,10 @@
from .detector_images import (
DetectorImageLoad,
close_image_handle,
load_detector_header,
load_detector_image,
load_detector_pixels,
)
from .parsers import (
extract_acquisition_timestamp,
parse_header_values,
Expand All @@ -18,6 +25,11 @@
)

__all__ = [
"DetectorImageLoad",
"close_image_handle",
"load_detector_header",
"load_detector_image",
"load_detector_pixels",
"parse_header_values",
"parse_header_values_with_meta",
"extract_acquisition_timestamp",
Expand Down
Loading