From d3dc53dcdb3715e942f7d0a6d3d526ef7a7b1bdf Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Thu, 13 Aug 2026 09:38:34 +0100 Subject: [PATCH 1/6] when finding the data collection group need to account for that fact that the tag may have changed when moving from just an atlas to a data collection --- src/murfey/server/api/session_control.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index c789b1901..672f4bc14 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -9,7 +9,7 @@ from fastapi.responses import FileResponse from ispyb.sqlalchemy import AutoProcProgram as ISPyBAutoProcProgram from pydantic import BaseModel -from sqlalchemy import func +from sqlalchemy import desc, func from sqlmodel import select try: @@ -396,6 +396,16 @@ def register_atlas( .where(DataCollectionGroup.session_id == session_id) .where(DataCollectionGroup.tag == atlas_registration_data.tag) ).one_or_none() + if dcg is None: + sample = int( + atlas_registration_data.tag.split("Sample")[1].split("/")[0] + ) + dcg = db.exec( + select(DataCollectionGroup) + .where(DataCollectionGroup.session_id == session_id) + .where(DataCollectionGroup.sample == sample) + .order_by(desc(DataCollectionGroup.id)) + ).first() grid_uuid = dcg.smartem_grid_uuid else: possible_grids = smartem_client.get_acquisition_grids( From af2cf52df0f8735c166c37b684dd91ce38ba26df Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Thu, 13 Aug 2026 09:55:59 +0100 Subject: [PATCH 2/6] add smartem atlas tile registration --- src/murfey/server/api/session_control.py | 36 +++++++++++++++++------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index 672f4bc14..f37c943dd 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -13,9 +13,10 @@ from sqlmodel import select try: + from smartem_agent.fs_parser import EpuParser from smartem_backend.api_client import SmartEMAPIClient from smartem_backend.keycloak_client import KeycloakClient, load_keycloak_config - from smartem_common.schemas import AtlasData + from smartem_common.schemas import AtlasTileGridSquarePositionData keycloak_client = KeycloakClient( load_keycloak_config(Path(os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") or "")) @@ -390,6 +391,7 @@ def register_atlas( keycloak_client=keycloak_client, ) grid_uuid = None + atlas_path = None if atlas_registration_data.tag: dcg = db.exec( select(DataCollectionGroup) @@ -414,8 +416,9 @@ def register_atlas( for grid in possible_grids: if grid.name == atlas_registration_data.name.replace("_atlas", ""): grid_uuid = grid.uuid + atlas_path = Path(grid.atlas_dir).parent break - if grid_uuid is not None: + if grid_uuid is not None and atlas_path is not None: existing_atlas = smartem_client.get_grid_atlas(grid_uuid) if ( existing_atlas.name == atlas_registration_data.name @@ -424,16 +427,29 @@ def register_atlas( ): # there is a question here of whether the grid should be registered if specified return - atlas_data = AtlasData( - id=atlas_registration_data.name, - acquisition_date=datetime.now(), - storage_folder=atlas_registration_data.storage_folder, - name=atlas_registration_data.name, - tiles=[], - gridsquare_positions=None, - grid_uuid=grid_uuid, + parser = EpuParser() + atlas_data = parser.parse_atlas_manifest( + str(atlas_path / "Atlas.dm"), grid_uuid ) + atlas_data.acquisition_date = atlas_data.acquisition_date.replace( + tzinfo=None + ) # timezone information is not consistently provided so drop it smartem_client.create_grid_atlas(atlas_data) + registered_squares = smartem_client.get_grid_gridsquares(grid_uuid) + gs_uuid_map = {gs.gridsquare_id: gs.uuid for gs in registered_squares} + for atlastile in atlas_data.tiles: + pos_data_for_tile = [] + for gsid, gs_tile_pos in atlastile.gridsquare_positions.items(): + for pos in gs_tile_pos: + pos_data_for_tile.append( + AtlasTileGridSquarePositionData( + gridsquare_uuid=gs_uuid_map[gsid], + tile_uuid=atlastile.uuid, + position=pos.position, + size=pos.size, + ) + ) + smartem_client.link_atlas_tile_and_gridsquares(pos_data_for_tile) if atlas_registration_data.register_grid: smartem_client.grid_registered(grid_uuid) else: From a40008644946e36fa2f7db4b492477b3be790639 Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Thu, 13 Aug 2026 10:17:30 +0100 Subject: [PATCH 3/6] allow the location of a keycloak configuration file for smartem to be configured from the murfey security configuration for convenience --- src/murfey/server/api/instrument.py | 9 ++++++++- src/murfey/server/api/session_control.py | 9 ++++++++- src/murfey/server/api/workflow.py | 9 ++++++++- src/murfey/util/config.py | 3 +++ src/murfey/workflows/spa/ctf_estimation.py | 9 ++++++++- src/murfey/workflows/spa/flush_spa_preprocess.py | 9 ++++++++- src/murfey/workflows/spa/motion_correction.py | 9 ++++++++- src/murfey/workflows/spa/picking.py | 9 ++++++++- 8 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/murfey/server/api/instrument.py b/src/murfey/server/api/instrument.py index 9606a0f6b..7f5f34dcb 100644 --- a/src/murfey/server/api/instrument.py +++ b/src/murfey/server/api/instrument.py @@ -19,8 +19,15 @@ from smartem_backend.keycloak_client import KeycloakClient, load_keycloak_config from smartem_common.schemas import AcquisitionData, MicroscopeData + from murfey.util.config import get_security_config + keycloak_client = KeycloakClient( - load_keycloak_config(Path(os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") or "")) + load_keycloak_config( + Path( + os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") + or get_security_config().smartem_keycloak_config + ) + ) ) SMARTEM_ACTIVE = True except ImportError: diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index f37c943dd..256a93634 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -18,8 +18,15 @@ from smartem_backend.keycloak_client import KeycloakClient, load_keycloak_config from smartem_common.schemas import AtlasTileGridSquarePositionData + from murfey.util.config import get_security_config + keycloak_client = KeycloakClient( - load_keycloak_config(Path(os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") or "")) + load_keycloak_config( + Path( + os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") + or get_security_config().smartem_keycloak_config + ) + ) ) SMARTEM_ACTIVE = True except ImportError: diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index 195897d3e..4d82f6ec7 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -30,8 +30,15 @@ MicrographManifest as SmartEMMicrographManifest, ) + from murfey.util.config import get_security_config + keycloak_client = KeycloakClient( - load_keycloak_config(Path(os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") or "")) + load_keycloak_config( + Path( + os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") + or get_security_config().smartem_keycloak_config + ) + ) ) SMARTEM_ACTIVE = True except ImportError: diff --git a/src/murfey/util/config.py b/src/murfey/util/config.py index fcf7ca243..e392eb0e8 100644 --- a/src/murfey/util/config.py +++ b/src/murfey/util/config.py @@ -261,6 +261,9 @@ class Security(BaseModel): session_token_timeout: Optional[int] = None allow_origins: list[str] = ["*"] + # SmartEM keycloak config file + smartem_keycloak_config: str = "" + # RabbitMQ settings rabbitmq_credentials: Path feedback_queue: str = "murfey_feedback" diff --git a/src/murfey/workflows/spa/ctf_estimation.py b/src/murfey/workflows/spa/ctf_estimation.py index 5f88abd77..8f95f70e0 100644 --- a/src/murfey/workflows/spa/ctf_estimation.py +++ b/src/murfey/workflows/spa/ctf_estimation.py @@ -19,8 +19,15 @@ from smartem_backend.model.http_response import MicrographResponse from smartem_common.entity_status import MicrographStatus + from murfey.util.config import get_security_config + keycloak_client = KeycloakClient( - load_keycloak_config(Path(os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") or "")) + load_keycloak_config( + Path( + os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") + or get_security_config().smartem_keycloak_config + ) + ) ) SMARTEM_ACTIVE = True except ImportError: diff --git a/src/murfey/workflows/spa/flush_spa_preprocess.py b/src/murfey/workflows/spa/flush_spa_preprocess.py index 39d3547dc..6541d537c 100644 --- a/src/murfey/workflows/spa/flush_spa_preprocess.py +++ b/src/murfey/workflows/spa/flush_spa_preprocess.py @@ -17,8 +17,15 @@ GridSquareMetadata as SmartEMGridSquareMetadata, ) + from murfey.util.config import get_security_config + keycloak_client = KeycloakClient( - load_keycloak_config(Path(os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") or "")) + load_keycloak_config( + Path( + os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") + or get_security_config().smartem_keycloak_config + ) + ) ) SMARTEM_ACTIVE = True except ImportError: diff --git a/src/murfey/workflows/spa/motion_correction.py b/src/murfey/workflows/spa/motion_correction.py index 8aa5c6a2d..a3d855df7 100644 --- a/src/murfey/workflows/spa/motion_correction.py +++ b/src/murfey/workflows/spa/motion_correction.py @@ -19,8 +19,15 @@ from smartem_backend.model.http_response import MicrographResponse from smartem_common.entity_status import MicrographStatus + from murfey.util.config import get_security_config + keycloak_client = KeycloakClient( - load_keycloak_config(Path(os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") or "")) + load_keycloak_config( + Path( + os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") + or get_security_config().smartem_keycloak_config + ) + ) ) SMARTEM_ACTIVE = True except ImportError: diff --git a/src/murfey/workflows/spa/picking.py b/src/murfey/workflows/spa/picking.py index ec229fc30..500738de7 100644 --- a/src/murfey/workflows/spa/picking.py +++ b/src/murfey/workflows/spa/picking.py @@ -38,8 +38,15 @@ from smartem_backend.model.http_response import MicrographResponse from smartem_common.entity_status import MicrographStatus + from murfey.util.config import get_security_config + keycloak_client = KeycloakClient( - load_keycloak_config(Path(os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") or "")) + load_keycloak_config( + Path( + os.getenv("SMARTEM_KEYCLOAK_CONFIGURATION") + or get_security_config().smartem_keycloak_config + ) + ) ) SMARTEM_ACTIVE = True except ImportError: From 20a87c08e0e2a0df46e19055b0740e6b4e6a2bed Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Thu, 13 Aug 2026 10:42:20 +0100 Subject: [PATCH 4/6] add motion correction and ctf results registration for smartem --- src/murfey/workflows/spa/ctf_estimation.py | 19 +++++++++++++++++-- src/murfey/workflows/spa/motion_correction.py | 19 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/murfey/workflows/spa/ctf_estimation.py b/src/murfey/workflows/spa/ctf_estimation.py index 8f95f70e0..799cc0612 100644 --- a/src/murfey/workflows/spa/ctf_estimation.py +++ b/src/murfey/workflows/spa/ctf_estimation.py @@ -15,8 +15,14 @@ try: from smartem_backend.api_client import SmartEMAPIClient from smartem_backend.keycloak_client import KeycloakClient, load_keycloak_config - from smartem_backend.model.http_request import MicrographUpdateRequest - from smartem_backend.model.http_response import MicrographResponse + from smartem_backend.model.http_request import ( + CtfEstimationRegisteredRequest, + MicrographUpdateRequest, + ) + from smartem_backend.model.http_response import ( + MicrographResponse, + ProcessingFeedbackPublishResponse, + ) from smartem_common.entity_status import MicrographStatus from murfey.util.config import get_security_config @@ -62,6 +68,15 @@ def ctf_estimated(message: dict, murfey_db: Session) -> dict[str, bool]: update, MicrographResponse, ) + registered_request = CtfEstimationRegisteredRequest( + quality=True, metric_name="ctfmaxresolution" + ) # True is a placeholder until we figure out the best way to calculate this + smartem_client._request( + "post", + f"micrographs/{movie.smartem_uuid}/ctf_estimation/registered", + registered_request, + ProcessingFeedbackPublishResponse, + ) except Exception: logger.warning( "Failed to emit CTF estimation complete event to smartem", diff --git a/src/murfey/workflows/spa/motion_correction.py b/src/murfey/workflows/spa/motion_correction.py index a3d855df7..218da437b 100644 --- a/src/murfey/workflows/spa/motion_correction.py +++ b/src/murfey/workflows/spa/motion_correction.py @@ -15,8 +15,14 @@ try: from smartem_backend.api_client import SmartEMAPIClient from smartem_backend.keycloak_client import KeycloakClient, load_keycloak_config - from smartem_backend.model.http_request import MicrographUpdateRequest - from smartem_backend.model.http_response import MicrographResponse + from smartem_backend.model.http_request import ( + MicrographUpdateRequest, + MotionCorrectionRegisteredRequest, + ) + from smartem_backend.model.http_response import ( + MicrographResponse, + ProcessingFeedbackPublishResponse, + ) from smartem_common.entity_status import MicrographStatus from murfey.util.config import get_security_config @@ -64,6 +70,15 @@ def motion_corrected(message: dict, murfey_db: Session) -> dict[str, bool]: update, MicrographResponse, ) + registered_request = MotionCorrectionRegisteredRequest( + quality=True, metric_name="motioncorrection" + ) # True is a placeholder until we figure out the best way to calculate this + smartem_client._request( + "post", + f"micrographs/{movie.smartem_uuid}/motion_correction/registered", + registered_request, + ProcessingFeedbackPublishResponse, + ) except Exception: logger.warning( "Failed to emit motion correction complete event to smartem", From 59bc52320976dfacfbabfe22484f4b39605c31d5 Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Tue, 18 Aug 2026 09:17:57 +0100 Subject: [PATCH 5/6] avoid exception if db query returns None --- src/murfey/server/api/session_control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/murfey/server/api/session_control.py b/src/murfey/server/api/session_control.py index 256a93634..efa7e60a3 100644 --- a/src/murfey/server/api/session_control.py +++ b/src/murfey/server/api/session_control.py @@ -415,7 +415,7 @@ def register_atlas( .where(DataCollectionGroup.sample == sample) .order_by(desc(DataCollectionGroup.id)) ).first() - grid_uuid = dcg.smartem_grid_uuid + grid_uuid = dcg.smartem_grid_uuid if dcg is not None else None else: possible_grids = smartem_client.get_acquisition_grids( atlas_registration_data.acquisition_uuid From 0ba970050300c7f5d85f0c43e714d71047ad55eb Mon Sep 17 00:00:00 2001 From: Daniel Hatton Date: Tue, 18 Aug 2026 09:49:42 +0100 Subject: [PATCH 6/6] endpoint being used to update smartem acquisition uuid in the database deosn't exist. just do it directly, you're in the server anyway --- src/murfey/server/api/instrument.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/murfey/server/api/instrument.py b/src/murfey/server/api/instrument.py index 7f5f34dcb..035563081 100644 --- a/src/murfey/server/api/instrument.py +++ b/src/murfey/server/api/instrument.py @@ -201,14 +201,10 @@ async def setup_multigrid_watcher( else: log.info("smartem not configured") if acquisition_uuid is not None: - async with clientsession.post( - f"{machine_config.instrument_server_url}{url_path_for('api.router', 'update_session', session_id=session_id)}", - parameters={"smartem_acquisition_uuid": acquisition_uuid}, - headers={ - "Authorization": f"Bearer {instrument_server_tokens[session_id]['access_token']}" - }, - ) as resp: - await resp.json() + session = db.exec(select(Session).where(Session.id == session_id)).one() + session.smartem_acquisition_uuid = acquisition_uuid + db.add(session) + db.commit() async with clientsession.post( f"{machine_config.instrument_server_url}{url_path_for('api.router', 'setup_multigrid_watcher', session_id=session_id)}",