From 75b742b9474f04cb68ec7cbe8a827eae421b9384 Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Mon, 10 Aug 2026 09:40:14 +0100
Subject: [PATCH 01/15] ci: add system test

---
 .github/workflows/_system_test.yml            | 27 ++++++++++
 .github/workflows/ci.yml                      |  3 ++
 .gitmodules                                   |  4 ++
 tests/system_tests/compose.yaml               | 54 ++++++++++++++++---
 .../system_tests/services/blueapi/config.yaml | 52 ++++++++++++++++++
 tests/system_tests/services/blueapi/start.sh  |  4 ++
 .../services/daq_queuing_service_config.yaml  | 17 ++++++
 tests/system_tests/services/rabbitmq_plugins  |  1 -
 tests/system_tests/test_system.py             |  2 +
 9 files changed, 155 insertions(+), 9 deletions(-)
 create mode 100644 .github/workflows/_system_test.yml
 create mode 100644 .gitmodules
 create mode 100644 tests/system_tests/services/blueapi/config.yaml
 create mode 100644 tests/system_tests/services/blueapi/start.sh
 create mode 100644 tests/system_tests/services/daq_queuing_service_config.yaml
 delete mode 100644 tests/system_tests/services/rabbitmq_plugins
 create mode 100644 tests/system_tests/test_system.py

diff --git a/.github/workflows/_system_test.yml b/.github/workflows/_system_test.yml
new file mode 100644
index 0000000..d9f5cb6
--- /dev/null
+++ b/.github/workflows/_system_test.yml
@@ -0,0 +1,27 @@
+on:
+  workflow_call:
+
+env:
+  # https://github.com/pytest-dev/pytest/issues/2042
+  PY_IGNORE_IMPORTMISMATCH: "1"
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 1
+          # Check out example services
+          submodules: true
+
+      - name: Set up Docker Compose
+        uses: docker/setup-compose-action@v2
+      
+      - name: Compose service up
+        run: docker compose --profile test -f tests/system_tests/compose.yaml up -d
+
+      - name: Run tests
+        run: uv run --locked tox -e tests
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a50bfe5..bad4c24 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -58,3 +58,6 @@ jobs:
     uses: ./.github/workflows/_release.yml
     permissions:
       contents: write
+
+  system-test:
+  uses: ./.github/workflows/_system_test.yml
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..53d7548
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,4 @@
+[submodule "tests/system_tests/dev-environment"]
+	path = tests/system_tests/dev-environment
+	url = https://github.com/zohebshaikh/dev-environment.git
+	branch = 4ee981e64ee96368186fc187499e366ba6787e35
diff --git a/tests/system_tests/compose.yaml b/tests/system_tests/compose.yaml
index a3eb208..5e3f666 100644
--- a/tests/system_tests/compose.yaml
+++ b/tests/system_tests/compose.yaml
@@ -1,10 +1,48 @@
+include: 
+  - dev-environment/daq-services/compose.yaml
+
 services:
-  rabbitmq:
-    image: docker.io/rabbitmq:4.0-management
-    ports:
-      - "1883:1883"
-      - "5672:5672"
-      - "15672:15672"
-      - "61613:61613"
+  blueapi:
+    image: ghcr.io/diamondlightsource/blueapi:1.18.0
+    network_mode: host # 8000
     volumes:
-      - ./services/rabbitmq_plugins:/etc/rabbitmq/enabled_plugins:ro,Z
+      - ./services/blueapi:/blueapi
+    entrypoint:  "sh /blueapi/start.sh"
+    depends_on:
+      keycloak:
+        condition: service_healthy
+    security_opt:
+      - label=disable
+    # Might be not required 
+    # don't know if daq-queing-service make a call to blueapi at startup
+    # healthcheck:
+    #   test: curl https://localhost/healthz
+    #   interval: 10s
+    #   timeout: 5s
+    #   retries: 5
+    #   start_period: 10s
+    profiles:
+      - test
+     
+  daq-queuing-service:
+    build:
+      context: ../../
+      dockerfile: Dockerfile
+    volumes: ./services/daq_queuing_service.yaml:/etc/config/config.yaml
+    security_opt:
+      - label=disable
+    profiles:
+      - test
+  nexus:
+    image: gitlab.diamond.ac.uk:5050/daq/d2acq/services/nexus-file-converter:4.1.6
+    environment:
+      - BEAMLINE=i15-1
+    security_opt:
+      - label=disable
+    profiles:
+      - diamond
+  
+  # A container with just puts bundler data for OPA
+  # and creates any keycloak clients that are required
+
+  #configure_beamline:
diff --git a/tests/system_tests/services/blueapi/config.yaml b/tests/system_tests/services/blueapi/config.yaml
new file mode 100644
index 0000000..929cf10
--- /dev/null
+++ b/tests/system_tests/services/blueapi/config.yaml
@@ -0,0 +1,52 @@
+    api:
+      url: https://localhost:8000
+    env:
+      metadata:
+         instrument: i15-1
+      sources:
+      - kind: deviceManager
+        module: dodal.beamlines.i15_1
+        mock: True
+      - kind: planFunctions
+        module: dodal.plans
+      - kind: planFunctions
+        module: dodal.plan_stubs
+      - kind: planFunctions
+        module: crystallography_bluesky.i15_1.plans
+      - kind: planFunctions
+        module: crystallography_bluesky.common.plans
+      events:
+        broadcast_status_events: false
+
+    stomp:
+      auth:
+        username: guest
+        password: guest
+      url: tcp://localhost:61613/
+      enabled: true
+
+    numtracker:
+      url: http://localhost:8406/graphql
+
+    tiled:
+      enabled: true
+      url: "http://localhost:8407/api/v1"
+      authentication:
+        client_id: "tiled-writer"
+        client_secret: "secret"
+
+    oidc:
+      issuer: "http://localhost:8081/realms/master"
+      client_id: "ixx-cli-blueapi"
+      client_audience: "ixx-blueapi"
+      logout_redirect_endpoint: "oauth2/sign_out"
+
+    scratch:
+      repositories:
+        - name: dodal
+          remote_url: https://github.com/DiamondLightSource/dodal.git
+        - name: ophyd-async
+          remote_url: https://github.com/bluesky/ophyd-async.git
+        - name: crystallography-bluesky
+          remote_url: https://github.com/DiamondLightSource/crystallography-bluesky.git
+          target_revision: main
diff --git a/tests/system_tests/services/blueapi/start.sh b/tests/system_tests/services/blueapi/start.sh
new file mode 100644
index 0000000..14dbc9a
--- /dev/null
+++ b/tests/system_tests/services/blueapi/start.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+mkdir -p /tmp/scratch/blueapi
+blueapi -c /blueapi/config.yaml setup-scratch
+blueapi -c /blueapi/config.yaml serve
diff --git a/tests/system_tests/services/daq_queuing_service_config.yaml b/tests/system_tests/services/daq_queuing_service_config.yaml
new file mode 100644
index 0000000..d5dc83b
--- /dev/null
+++ b/tests/system_tests/services/daq_queuing_service_config.yaml
@@ -0,0 +1,17 @@
+config:
+  converter:
+    path: "daq_queuing_service.plugins.i15_1_converter"
+    name: "I151Converter"
+  blueapi:
+    api: 
+      url: "http://localhost:8000"
+    stomp:
+      enabled: true
+      url: tcp://localhost:61613
+      auth:
+        username: guest
+        password: guest
+udcSecret:
+  enabled: true
+  udc-secret: "secret"
+  clientId: "i15-1udc"
diff --git a/tests/system_tests/services/rabbitmq_plugins b/tests/system_tests/services/rabbitmq_plugins
deleted file mode 100644
index 18ea340..0000000
--- a/tests/system_tests/services/rabbitmq_plugins
+++ /dev/null
@@ -1 +0,0 @@
-[rabbitmq_management,rabbitmq_prometheus,rabbitmq_stomp,rabbitmq_mqtt].
diff --git a/tests/system_tests/test_system.py b/tests/system_tests/test_system.py
new file mode 100644
index 0000000..7624f2b
--- /dev/null
+++ b/tests/system_tests/test_system.py
@@ -0,0 +1,2 @@
+def test_one():
+    assert 1 + 1 == 2

From b87b2aa9ac39aa9c622b0cb9f5963768e6dce1c1 Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Mon, 10 Aug 2026 09:42:29 +0100
Subject: [PATCH 02/15] trigger CI

---
 .github/workflows/ci.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bad4c24..e2c3d3a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2,10 +2,6 @@ name: CI
 
 on:
   push:
-    branches:
-      - main
-    tags:
-      - "*"
   pull_request:
 
 jobs:

From 53ea48742c639149bce0af35fcc88b5887c9a992 Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Mon, 10 Aug 2026 10:01:44 +0100
Subject: [PATCH 03/15] fix indetation

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index e2c3d3a..d4218e5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -56,4 +56,4 @@ jobs:
       contents: write
 
   system-test:
-  uses: ./.github/workflows/_system_test.yml
+    uses: ./.github/workflows/_system_test.yml

From 4a3ca060824034f711cfc30ddebf5f7974dc0572 Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Mon, 10 Aug 2026 10:28:50 +0100
Subject: [PATCH 04/15] change submodule to a branch name

---
 .gitmodules | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitmodules b/.gitmodules
index 53d7548..222d778 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,4 +1,4 @@
 [submodule "tests/system_tests/dev-environment"]
 	path = tests/system_tests/dev-environment
 	url = https://github.com/zohebshaikh/dev-environment.git
-	branch = 4ee981e64ee96368186fc187499e366ba6787e35
+	branch = add-compose

From df736f9214e3115d22682eaadf1603b08c1155c3 Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Mon, 10 Aug 2026 10:34:16 +0100
Subject: [PATCH 05/15] remove path from submodule

---
 .gitmodules | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index 222d778..152646f 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,4 +1,3 @@
-[submodule "tests/system_tests/dev-environment"]
-	path = tests/system_tests/dev-environment
+[submodule "dev-environment"]
 	url = https://github.com/zohebshaikh/dev-environment.git
-	branch = add-compose
+	branch = 4ee981e64ee96368186fc187499e366ba6787e35

From bb2ef4a5270d335907c488d1a34272007046be77 Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Mon, 10 Aug 2026 10:40:18 +0100
Subject: [PATCH 06/15] add submoudle

---
 .gitmodules     | 2 +-
 dev-environment | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 160000 dev-environment

diff --git a/.gitmodules b/.gitmodules
index 152646f..79332f0 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,3 @@
 [submodule "dev-environment"]
+	path = dev-environment
 	url = https://github.com/zohebshaikh/dev-environment.git
-	branch = 4ee981e64ee96368186fc187499e366ba6787e35
diff --git a/dev-environment b/dev-environment
new file mode 160000
index 0000000..4ee981e
--- /dev/null
+++ b/dev-environment
@@ -0,0 +1 @@
+Subproject commit 4ee981e64ee96368186fc187499e366ba6787e35

From 9929cc43a715a2069b2f1898070a989f804f2cab Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Mon, 10 Aug 2026 10:44:06 +0100
Subject: [PATCH 07/15] add path

---
 tests/system_tests/compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/system_tests/compose.yaml b/tests/system_tests/compose.yaml
index 5e3f666..35b8e73 100644
--- a/tests/system_tests/compose.yaml
+++ b/tests/system_tests/compose.yaml
@@ -1,5 +1,5 @@
 include: 
-  - dev-environment/daq-services/compose.yaml
+  - ../../dev-environment/daq-services/compose.yaml
 
 services:
   blueapi:

From 8ad08e987344d874980684bc114ff19a4a0703a6 Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Mon, 10 Aug 2026 11:00:09 +0100
Subject: [PATCH 08/15] update

---
 dev-environment | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev-environment b/dev-environment
index 4ee981e..5526b78 160000
--- a/dev-environment
+++ b/dev-environment
@@ -1 +1 @@
-Subproject commit 4ee981e64ee96368186fc187499e366ba6787e35
+Subproject commit 5526b78c55e9db62d09847d0ba415d917631df78

From 7ba482a7d3dea8a8501ebec1efb795e29436c194 Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Mon, 10 Aug 2026 11:02:28 +0100
Subject: [PATCH 09/15] make volumens arrays

---
 tests/system_tests/compose.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/system_tests/compose.yaml b/tests/system_tests/compose.yaml
index 35b8e73..054ba3b 100644
--- a/tests/system_tests/compose.yaml
+++ b/tests/system_tests/compose.yaml
@@ -28,7 +28,8 @@ services:
     build:
       context: ../../
       dockerfile: Dockerfile
-    volumes: ./services/daq_queuing_service.yaml:/etc/config/config.yaml
+    volumes: 
+      - ./services/daq_queuing_service.yaml:/etc/config/config.yaml
     security_opt:
       - label=disable
     profiles:

From c56154fc14e56eba56ee7ce4a8c5480dcc9922c4 Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Mon, 10 Aug 2026 11:09:57 +0100
Subject: [PATCH 10/15] cache images

---
 .github/workflows/_system_test.yml | 31 ++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/.github/workflows/_system_test.yml b/.github/workflows/_system_test.yml
index d9f5cb6..dc9e117 100644
--- a/.github/workflows/_system_test.yml
+++ b/.github/workflows/_system_test.yml
@@ -19,9 +19,40 @@ jobs:
 
       - name: Set up Docker Compose
         uses: docker/setup-compose-action@v2
+
+      - name: Cache Docker images
+        uses: actions/cache@v4
+        with:
+          path: /tmp/docker-cache
+          key: docker-images-${{ runner.os }}-${{ hashFiles('tests/system_tests/compose.yaml') }}
+          restore-keys: |
+            docker-images-${{ runner.os }}-
+
+      - name: Load Docker images
+        run: |
+          if [ -f /tmp/docker-cache/images.tar ]; then
+            echo "Loading cached Docker images..."
+            docker load -i /tmp/docker-cache/images.tar
+          else
+            echo "No cached Docker images found"
+          fi
       
+
       - name: Compose service up
         run: docker compose --profile test -f tests/system_tests/compose.yaml up -d
 
+
+      - name: Save Docker images
+        if: always()
+        run: |
+          mkdir -p /tmp/docker-cache
+
+          docker save \
+            $(docker images --format '{{.Repository}}:{{.Tag}}') \
+            -o /tmp/docker-cache/images.tar
+      
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+       
       - name: Run tests
         run: uv run --locked tox -e tests

From 71751565e03ed3cfc3277fccfa6f6376e456e483 Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Tue, 11 Aug 2026 15:01:57 +0100
Subject: [PATCH 11/15] change submodule path

---
 .gitmodules | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitmodules b/.gitmodules
index 79332f0..1b261ec 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,3 @@
 [submodule "dev-environment"]
-	path = dev-environment
+	path = tests/system_tests/dev-environment
 	url = https://github.com/zohebshaikh/dev-environment.git

From 90296b4acbf5fff72181d2a77c6459456be804be Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Tue, 11 Aug 2026 15:03:20 +0100
Subject: [PATCH 12/15] delete submodule

---
 .gitmodules     | 3 ---
 dev-environment | 1 -
 2 files changed, 4 deletions(-)
 delete mode 100644 .gitmodules
 delete mode 160000 dev-environment

diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 1b261ec..0000000
--- a/.gitmodules
+++ /dev/null
@@ -1,3 +0,0 @@
-[submodule "dev-environment"]
-	path = tests/system_tests/dev-environment
-	url = https://github.com/zohebshaikh/dev-environment.git
diff --git a/dev-environment b/dev-environment
deleted file mode 160000
index 5526b78..0000000
--- a/dev-environment
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 5526b78c55e9db62d09847d0ba415d917631df78

From f061222499460864c63c7647f2d671f6a969bf78 Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Tue, 11 Aug 2026 15:06:46 +0100
Subject: [PATCH 13/15] add compose files

---
 .gitmodules                        | 3 +++
 tests/system_tests/dev-environment | 1 +
 2 files changed, 4 insertions(+)
 create mode 100644 .gitmodules
 create mode 160000 tests/system_tests/dev-environment

diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..fde8560
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "tests/system_tests/dev-environment"]
+	path = tests/system_tests/dev-environment
+	url = https://github.com/zohebshaikh/dev-environment
diff --git a/tests/system_tests/dev-environment b/tests/system_tests/dev-environment
new file mode 160000
index 0000000..bfe85a9
--- /dev/null
+++ b/tests/system_tests/dev-environment
@@ -0,0 +1 @@
+Subproject commit bfe85a952e252ddc25dc819863197d07cb8377e8

From 57716d3deede352b62cbc52bac4fafdddce5c724 Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Tue, 11 Aug 2026 15:07:59 +0100
Subject: [PATCH 14/15] update the compose files

---
 .github/workflows/_system_test.yml | 28 ----------------------------
 tests/system_tests/compose.yaml    |  2 +-
 2 files changed, 1 insertion(+), 29 deletions(-)

diff --git a/.github/workflows/_system_test.yml b/.github/workflows/_system_test.yml
index dc9e117..d6b7830 100644
--- a/.github/workflows/_system_test.yml
+++ b/.github/workflows/_system_test.yml
@@ -19,37 +19,9 @@ jobs:
 
       - name: Set up Docker Compose
         uses: docker/setup-compose-action@v2
-
-      - name: Cache Docker images
-        uses: actions/cache@v4
-        with:
-          path: /tmp/docker-cache
-          key: docker-images-${{ runner.os }}-${{ hashFiles('tests/system_tests/compose.yaml') }}
-          restore-keys: |
-            docker-images-${{ runner.os }}-
-
-      - name: Load Docker images
-        run: |
-          if [ -f /tmp/docker-cache/images.tar ]; then
-            echo "Loading cached Docker images..."
-            docker load -i /tmp/docker-cache/images.tar
-          else
-            echo "No cached Docker images found"
-          fi
       
-
       - name: Compose service up
         run: docker compose --profile test -f tests/system_tests/compose.yaml up -d
-
-
-      - name: Save Docker images
-        if: always()
-        run: |
-          mkdir -p /tmp/docker-cache
-
-          docker save \
-            $(docker images --format '{{.Repository}}:{{.Tag}}') \
-            -o /tmp/docker-cache/images.tar
       
       - name: Install uv
         uses: astral-sh/setup-uv@v7
diff --git a/tests/system_tests/compose.yaml b/tests/system_tests/compose.yaml
index 054ba3b..61c9094 100644
--- a/tests/system_tests/compose.yaml
+++ b/tests/system_tests/compose.yaml
@@ -1,5 +1,5 @@
 include: 
-  - ../../dev-environment/daq-services/compose.yaml
+  - dev-environment/daq-services/compose.yaml
 
 services:
   blueapi:

From 68cf34b5b5950f75ccbba5bfcbe4224947cde46d Mon Sep 17 00:00:00 2001
From: Zoheb Shaikh <26975142+ZohebShaikh@users.noreply.github.com>
Date: Tue, 11 Aug 2026 15:09:30 +0100
Subject: [PATCH 15/15] update the file path

---
 tests/system_tests/compose.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/system_tests/compose.yaml b/tests/system_tests/compose.yaml
index 61c9094..cff087d 100644
--- a/tests/system_tests/compose.yaml
+++ b/tests/system_tests/compose.yaml
@@ -1,5 +1,5 @@
 include: 
-  - dev-environment/daq-services/compose.yaml
+  - ./dev-environment/daq-services/compose.yaml
 
 services:
   blueapi: