diff --git a/src/murfey/client/contexts/sxt.py b/src/murfey/client/contexts/sxt.py index 23acc111c..48a7d6a58 100644 --- a/src/murfey/client/contexts/sxt.py +++ b/src/murfey/client/contexts/sxt.py @@ -139,7 +139,7 @@ def register_sxt_data_collection( "source": str(self._basepath), "tag": tilt_series, "pixel_size_on_image": str( - round(data_collection_parameters.get("pixel_size", 100), 2) * 1e-10 + data_collection_parameters.get("pixel_size", 100) * 1e-10 ), # expected in metres "image_size_x": data_collection_parameters.get("image_size_x", 0), "image_size_y": data_collection_parameters.get("image_size_y", 0), @@ -210,6 +210,7 @@ def post_transfer( if xrm_ole.exists("ImageInfo/XPosition") and xrm_ole.exists( "ImageInfo/YPosition" ): + # Get stage locations in microns x_tiles = _get_ole_header_value( xrm_ole, "ImageInfo/XPosition", np.float32 ).tolist() @@ -220,9 +221,13 @@ def post_transfer( metadata["y_position"] = y_tiles[int(len(y_tiles) / 2)] if xrm_ole.exists("ImageInfo/PixelSize"): - metadata["pixel_size"] = _get_ole_header_value( - xrm_ole, "ImageInfo/PixelSize", np.float32 - ).tolist()[0] + # Pixel size in microns, convert to metres + metadata["pixel_size"] = ( + _get_ole_header_value( + xrm_ole, "ImageInfo/PixelSize", np.float32 + ).tolist()[0] + / 1e6 + ) if xrm_ole.exists("ImageInfo/ImageHeight"): metadata["height"] = _get_ole_header_value( @@ -276,14 +281,20 @@ def post_transfer( if ( metadata.get("mosaic_size", 1) > 0 - and metadata.get("pixel_size", 0) > 0.1 + and metadata.get("pixel_size", 0) > 1e-7 ): # Large pixel size, this is an atlas + thumbnail_pixel_size = ( + metadata["pixel_size"] + * metadata.get("height", 0) + * metadata["mosaic_rows"] + / 1024 + ) dcg_data = { "experiment_type_id": 44, # Atlas "tag": dcg_tag, "atlas": str(thumbnail_path), - "atlas_pixel_size": round(metadata.get("pixel_size", 0), 2), + "atlas_pixel_size": float(thumbnail_pixel_size), "atlas_x_stage_position": metadata.get("x_position", None), "atlas_y_stage_position": metadata.get("y_position", None), "atlas_height": int( @@ -317,7 +328,7 @@ def post_transfer( "tag": dcg_tag, "x_stage_position": metadata.get("x_position", None), "y_stage_position": metadata.get("y_position", None), - "pixel_size": round(metadata.get("pixel_size", 0), 2), + "pixel_size": metadata.get("pixel_size", 0), "height": int( metadata.get("height", 0) * metadata["mosaic_rows"] ), @@ -342,6 +353,19 @@ def post_transfer( if txrm_ole.exists("ReferenceData/Image"): metadata["has_reference"] = True + if txrm_ole.exists("ImageInfo/XPosition") and txrm_ole.exists( + "ImageInfo/YPosition" + ): + # Get stage locations in microns + x_tiles = _get_ole_header_value( + txrm_ole, "ImageInfo/XPosition", np.float32 + ).tolist() + y_tiles = _get_ole_header_value( + txrm_ole, "ImageInfo/YPosition", np.float32 + ).tolist() + metadata["x_position"] = x_tiles[int(len(x_tiles) / 2)] + metadata["y_position"] = y_tiles[int(len(y_tiles) / 2)] + if txrm_ole.exists("ImageInfo/Angles"): angles = _get_ole_header_value( txrm_ole, "ImageInfo/Angles", np.float32 @@ -350,6 +374,7 @@ def post_transfer( metadata["maximum_angle"] = max(angles) if txrm_ole.exists("ImageInfo/PixelSize"): + # Pixel size in microns, converted to angstroms pixel_size_txrm = _get_ole_header_value( txrm_ole, "ImageInfo/PixelSize", np.float32 ).tolist() @@ -492,9 +517,7 @@ def post_transfer( data={ "tag": tilt_series_tag, "source": destination_search_dir, - "pixel_size": round( - metadata.get("pixel_size", 100), 2 - ), # angstroms + "pixel_size": metadata.get("pixel_size", 100), # angstroms "tilt_offset": midpoint(angles), "tilt_series_length": metadata.get( "tilt_series_length", len(angles) @@ -503,6 +526,8 @@ def post_transfer( "xrm_reference": str(reference_file_transferred_to) if reference_file_transferred_to else None, + "x_stage_position": metadata.get("x_position", None), + "y_stage_position": metadata.get("y_position", None), }, ) return True diff --git a/src/murfey/workflows/sxt/process_sxt_tilt_series.py b/src/murfey/workflows/sxt/process_sxt_tilt_series.py index a3165460b..7b55c4b8c 100644 --- a/src/murfey/workflows/sxt/process_sxt_tilt_series.py +++ b/src/murfey/workflows/sxt/process_sxt_tilt_series.py @@ -1,6 +1,7 @@ import logging from pathlib import Path +import numpy as np from pydantic import BaseModel from sqlmodel import select from sqlmodel.orm.session import Session as SQLModelSession @@ -15,6 +16,7 @@ DataCollection, DataCollectionGroup, ProcessingJob, + SearchMap, Session, TiltSeries, ) @@ -30,6 +32,8 @@ class SXTTiltSeriesInfo(BaseModel): pixel_size: float tilt_offset: int xrm_reference: str | None + x_stage_position: float | None = None + y_stage_position: float | None = None def process_sxt_tilt_series( @@ -82,6 +86,63 @@ def process_sxt_tilt_series( instrument_name ] + # Determine pixel location on a roi for display + min_distance: float | None = None + matching_roi: SearchMap | None = None + x_pixel_location: int | None = None + y_pixel_location: int | None = None + if tilt_series_info.x_stage_position and tilt_series_info.y_stage_position: + # Find all rois for this grid + dcg_rois = murfey_db.exec( + select(SearchMap) + .where(SearchMap.session_id == session_id) + .where(SearchMap.tag == tilt_series.rsync_source) + ).all() + for roi in dcg_rois: + if roi.x_stage_position is not None and roi.y_stage_position is not None: + # Determine the roi which is closest to this tomogram + roi_distance = np.sqrt( + (tilt_series_info.x_stage_position - roi.x_stage_position) ** 2 + + (tilt_series_info.y_stage_position - roi.y_stage_position) ** 2 + ) + if min_distance is None or roi_distance < min_distance: + min_distance = roi_distance + matching_roi = roi + + # Calculate the position on the roi + if ( + matching_roi is not None + and matching_roi.x_stage_position is not None + and matching_roi.y_stage_position is not None + and matching_roi.height + and matching_roi.width + and matching_roi.pixel_size + ): + # Convert from stage position to pixel locations + # Stage position in microns, pixel size in metres + x_location_centered = ( + (tilt_series_info.x_stage_position - matching_roi.x_stage_position) + / matching_roi.pixel_size + / 1e6 + ) + y_location_centered = ( + (tilt_series_info.y_stage_position - matching_roi.y_stage_position) + / matching_roi.pixel_size + / 1e6 + ) + + # Scaling from different pixel size of atlas and roi, and atlas thumbnail size + x_pixel_location = int( + x_location_centered * 1024 / matching_roi.width + 512 + ) + y_pixel_location = int( + 512 - y_location_centered * 1024 / matching_roi.height + ) + else: + logger.warning( + f"Cannot match ROI {matching_roi.id if matching_roi else None} for {tilt_series_info.tag}" + ) + # Find the visit folder and any subfolders needed parts = [secure_filename(p) for p in Path(tilt_series_info.txrm).parts] visit_idx = parts.index(visit_name) @@ -119,6 +180,9 @@ def process_sxt_tilt_series( "pixel_size": tilt_series_info.pixel_size, "manual_tilt_offset": -tilt_series_info.tilt_offset, "node_creator_queue": machine_config.node_creator_queue, + "search_map_id": matching_roi.id if matching_roi else None, + "x_location": x_pixel_location, + "y_location": y_pixel_location, }, } if _transport_object: diff --git a/src/murfey/workflows/sxt/sxt_metadata.py b/src/murfey/workflows/sxt/sxt_metadata.py index 57c9e95d7..25fba7ae6 100644 --- a/src/murfey/workflows/sxt/sxt_metadata.py +++ b/src/murfey/workflows/sxt/sxt_metadata.py @@ -80,24 +80,34 @@ def register_sxt_roi( atlas.image_pixels_y, ] ): + # Pixel size of full-size mosaic in metres + original_atlas_pixel_size = atlas.image_pixel_size * 1024 / atlas.image_pixels_x + # Convert from stage position to pixel locations - roi.x_location = (roi.x_stage_position - atlas.pos_x) / atlas.image_pixel_size - roi.y_location = (roi.y_stage_position - atlas.pos_y) / atlas.image_pixel_size + # Stage position in microns, pixel size in metres + roi.x_location = ( + (roi.x_stage_position - atlas.pos_x) / original_atlas_pixel_size / 1e6 + ) + roi.y_location = ( + (roi.y_stage_position - atlas.pos_y) / original_atlas_pixel_size / 1e6 + ) # Scaling from different pixel size of atlas and roi, and atlas thumbnail size roi_parameters.x_location = roi.x_location * (512 / atlas.image_pixels_x) + 256 roi_parameters.y_location = 256 - roi.y_location * (512 / atlas.image_pixels_y) + # Find size in terms of atlas pixels + # Factor of 512 to account for thumbnailing assumption in pato roi_parameters.width_on_atlas = int( round( roi.width - * (roi.pixel_size / atlas.image_pixel_size) + * (roi.pixel_size / original_atlas_pixel_size) * (512 / atlas.image_pixels_x) ) ) roi_parameters.height_on_atlas = int( round( roi.height - * (roi.pixel_size / atlas.image_pixel_size) + * (roi.pixel_size / original_atlas_pixel_size) * (512 / atlas.image_pixels_y) ) ) diff --git a/tests/client/contexts/test_sxt.py b/tests/client/contexts/test_sxt.py index fe1442a2c..e7a4c42f7 100644 --- a/tests/client/contexts/test_sxt.py +++ b/tests/client/contexts/test_sxt.py @@ -83,9 +83,9 @@ def test_sxt_context_xrm_atlas(mock_ole_file, mock_post, tmp_path): "experiment_type_id": 44, "tag": f"{tmp_path}/cm12345-6/grid1", "atlas": "/path/to/dest/cm12345-6/processed/grid1/example_atlas_Annotated_thumbnail.jpg", - "atlas_pixel_size": 0.3, - "atlas_x_stage_position": 1, - "atlas_y_stage_position": -1, + "atlas_pixel_size": float(float(np.float32(0.3)) / 1e6 * 1000 * 6 / 1024), + "atlas_x_stage_position": 1.0, + "atlas_y_stage_position": -1.0, "atlas_height": 6000, "atlas_width": 4500, }, @@ -161,9 +161,9 @@ def test_sxt_context_xrm_roi(mock_ole_file, mock_post, tmp_path): "http://localhost:8000/workflow/sxt/sessions/1/sxt_roi/example_roi", json={ "tag": f"{tmp_path}/cm12345-6/grid1", - "x_stage_position": 1, - "y_stage_position": -1, - "pixel_size": 0.03, + "x_stage_position": 1.0, + "y_stage_position": -1.0, + "pixel_size": float(np.float32(0.03)) * 1e-6, "height": 6000, "width": 4500, "image": "/path/to/dest/cm12345-6/processed/grid1/example_roi_Annotated_thumbnail.jpg", @@ -183,6 +183,8 @@ def test_sxt_context_txrm(mock_ole_file, mock_post, tmp_path): ) # Metadata encoded arrays mock_ole_file().__enter__().openstream().getvalue.side_effect = [ + np.array([0.01], dtype=np.float32).tobytes(), # X position + np.array([0.02], dtype=np.float32).tobytes(), # Y position np.array([-55, -25, 5, 35, 65], dtype=np.float32).tobytes(), # Angles np.array([0.01001], dtype=np.float32).tobytes(), # Pixel size np.array([1024], dtype=np.int32).tobytes(), # Image Width @@ -218,8 +220,8 @@ def test_sxt_context_txrm(mock_ole_file, mock_post, tmp_path): ) mock_ole_file.assert_any_call(str(tmp_path / "cm12345-6/grid1/example.txrm")) - assert mock_ole_file().__enter__().exists.call_count == 10 - assert mock_ole_file().__enter__().openstream.call_count == 11 # 9 + 2 above + assert mock_ole_file().__enter__().exists.call_count == 12 + assert mock_ole_file().__enter__().openstream.call_count == 13 # 11 + 2 above mock_ole_file().__enter__().exists.assert_any_call("ReferenceData/Image") for field_name in [ "ImageInfo/Angles", @@ -254,7 +256,7 @@ def test_sxt_context_txrm(mock_ole_file, mock_post, tmp_path): "data_collection_tag": "example", "source": f"{tmp_path}/cm12345-6/grid1", "tag": "example", - "pixel_size_on_image": str(100.1 * 1e-10), + "pixel_size_on_image": str(float(np.float32(0.01001)) * 1e-6), "image_size_x": 1024, "image_size_y": 2048, "magnification": 1000, @@ -281,11 +283,13 @@ def test_sxt_context_txrm(mock_ole_file, mock_post, tmp_path): json={ "tag": "example", "source": f"{tmp_path}/cm12345-6/grid1", - "pixel_size": 100.1, + "pixel_size": float(np.float32(0.01001)) * 1e4, "tilt_offset": 5, "tilt_series_length": 200, "txrm": str(tmp_path / "destination/cm12345-6/grid1/example.txrm"), "xrm_reference": None, + "x_stage_position": float(np.float32(0.01)), + "y_stage_position": float(np.float32(0.02)), }, headers={"Authorization": "Bearer "}, ) @@ -304,6 +308,8 @@ def test_sxt_context_txrm_external_ref(mock_ole_file, mock_post, tmp_path): ) # Metadata encoded arrays mock_ole_file().__enter__().openstream().getvalue.side_effect = [ + np.array([0.01], dtype=np.float32).tobytes(), # X position + np.array([0.02], dtype=np.float32).tobytes(), # Y position np.array([-55, -25, 5, 35, 65], dtype=np.float32).tobytes(), # Angles np.array([0.01001], dtype=np.float32).tobytes(), # Pixel size np.array([1024], dtype=np.int32).tobytes(), # Image Width @@ -368,7 +374,7 @@ def test_sxt_context_txrm_external_ref(mock_ole_file, mock_post, tmp_path): "data_collection_tag": "example", "source": f"{tmp_path}/cm12345-6/grid1", "tag": "example", - "pixel_size_on_image": str(100.1 * 1e-10), + "pixel_size_on_image": str(float(np.float32(0.01001)) * 1e-6), "image_size_x": 1024, "image_size_y": 2048, "magnification": 1000, @@ -405,13 +411,15 @@ def test_sxt_context_txrm_external_ref(mock_ole_file, mock_post, tmp_path): json={ "tag": "example", "source": f"{tmp_path}/cm12345-6/grid1", - "pixel_size": 100.1, + "pixel_size": float(np.float32(0.01001)) * 1e4, "tilt_offset": 5, "tilt_series_length": 200, "txrm": str( tmp_path / "destination/cm12345-6/grid1/example_-60to60@0.5.txrm" ), "xrm_reference": str(tmp_path / "destination/cm12345-6/grid1/ref.xrm"), + "x_stage_position": float(np.float32(0.01)), + "y_stage_position": float(np.float32(0.02)), }, headers={"Authorization": "Bearer "}, ) @@ -430,6 +438,8 @@ def test_sxt_context_txrm_zero_angles(mock_ole_file, mock_post, tmp_path): ) # Metadata encoded arrays mock_ole_file().__enter__().openstream().getvalue.side_effect = [ + np.array([0.01], dtype=np.float32).tobytes(), # X position + np.array([0.02], dtype=np.float32).tobytes(), # Y position np.array([0, 0, 0, 0, 0], dtype=np.float32).tobytes(), # Angles np.array([0.01001], dtype=np.float32).tobytes(), # Pixel size np.array([1024], dtype=np.int32).tobytes(), # Image Width diff --git a/tests/workflows/sxt/test_process_sxt_tilt_series.py b/tests/workflows/sxt/test_process_sxt_tilt_series.py index 53ea80968..414d52c5e 100644 --- a/tests/workflows/sxt/test_process_sxt_tilt_series.py +++ b/tests/workflows/sxt/test_process_sxt_tilt_series.py @@ -7,6 +7,7 @@ DataCollection, DataCollectionGroup, ProcessingJob, + SearchMap, TiltSeries, ) from murfey.workflows.sxt import process_sxt_tilt_series @@ -67,6 +68,33 @@ def set_up_db(murfey_db_session: Session): "pj_id": imod_pj_entry.id, }, ) + get_or_create_db_entry( + murfey_db_session, + SearchMap, + lookup_kwargs={ + "id": 0, + "name": "0", + "session_id": ExampleVisit.murfey_session_id, + "tag": "/path/to/tomogram_source", + "x_stage_position": 30, + "y_stage_position": 40, + }, + ) + get_or_create_db_entry( + murfey_db_session, + SearchMap, + lookup_kwargs={ + "id": 1, + "name": "1", + "session_id": ExampleVisit.murfey_session_id, + "tag": "/path/to/tomogram_source", + "x_stage_position": 10, + "y_stage_position": 20, + "height": 10, + "width": 5, + "pixel_size": 1e-6, + }, + ) return dcg_entry.id, dc_entry.id, aretomo_autoproc_entry.id, imod_autoproc_entry.id @@ -86,6 +114,8 @@ def test_process_new_sxt_tilt_series( tilt_series_length=5, pixel_size=100, tilt_offset=1, + x_stage_position=10.1, + y_stage_position=20.2, ) # Run the registration @@ -111,6 +141,9 @@ def test_process_new_sxt_tilt_series( "pixel_size": 100, "manual_tilt_offset": -1, "node_creator_queue": "node_creator", + "search_map_id": 1, + "x_location": int(0.1 * 1024 / 5 + 512), + "y_location": int(512 - 0.2 * 1024 / 10), }, }, new_connection=True, @@ -129,6 +162,9 @@ def test_process_new_sxt_tilt_series( "pixel_size": 100, "manual_tilt_offset": -1, "node_creator_queue": "node_creator", + "search_map_id": 1, + "x_location": int(0.1 * 1024 / 5 + 512), + "y_location": int(512 - 0.2 * 1024 / 10), }, }, new_connection=True, diff --git a/tests/workflows/sxt/test_sxt_metadata.py b/tests/workflows/sxt/test_sxt_metadata.py index 7519abcc8..09d2746fe 100644 --- a/tests/workflows/sxt/test_sxt_metadata.py +++ b/tests/workflows/sxt/test_sxt_metadata.py @@ -33,7 +33,7 @@ def set_up_db(murfey_db_session: Session): "site_name": "site", "pos_x": 2, "pos_y": 3, - "image_pixel_size": 0.5, + "image_pixel_size": 0.5 * 400 / 1024 / 1e6, "image_pixels_x": 400, "image_pixels_y": 500, }, @@ -111,8 +111,8 @@ def test_update_sxt_roi(mock_transport, murfey_db_session: Session, tmp_path): # Check the second update roi_params.x_location = 16 * 512 / 400 + 256 roi_params.y_location = 256 - 34 * 512 / 500 - roi_params.width_on_atlas = int(round(200 * 0.05 * 512 / 400)) - roi_params.height_on_atlas = int(round(400 * 0.05 * 512 / 500)) + roi_params.width_on_atlas = int(round(200 * 0.05 * 512 / 400 * 1e6)) + roi_params.height_on_atlas = int(round(400 * 0.05 * 512 / 500 * 1e6)) mock_transport.do_update_sxt_roi.assert_any_call(1, roi_params) # Check the database insert